Merge pull request #1689 from projectdiscovery/dev

v2.11.0

Author: dogancanbakir

pkg/passive/passive.go

@@ -65,11 +65,15 @@ func (a *Agent) EnumerateSubdomainsWithCtx(ctx context.Context, domain string, p
 		for _, runner := range a.sources {
 			wg.Add(1)
 			go func(source subscraping.Source) {
+				defer wg.Done()
 				ctxWithValue := context.WithValue(ctx, subscraping.CtxSourceArg, source.Name())
 				for resp := range source.Run(ctxWithValue, domain, session) {
-					results <- resp
+					select {
+					case <-ctx.Done():
+						return
+					case results <- resp:
+					}
 				}
-				wg.Done()
 			}(runner)
 		}
 		wg.Wait()

pkg/passive/sources.go

@@ -25,8 +25,8 @@ import (
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/digitorus"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdb"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdumpster"
-	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/domainsproject"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsrepo"
+	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/domainsproject"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/driftnet"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/facebook"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/fofa"
@@ -36,8 +36,10 @@ import (
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hudsonrock"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/intelx"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/leakix"
+	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/merklemap"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/netlas"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/onyphe"
+	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/profundis"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/pugrecon"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/quake"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/rapiddns"
@@ -47,6 +49,7 @@ import (
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/securitytrails"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/shodan"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/sitedossier"
+	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/thc"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/threatbook"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/threatcrowd"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/virustotal"
@@ -81,6 +84,7 @@ var AllSources = [...]subscraping.Source{
 	&hackertarget.Source{},
 	&intelx.Source{},
 	&netlas.Source{},
+	&merklemap.Source{},
 	&onyphe.Source{},
 	&leakix.Source{},
 	&quake.Source{},
@@ -91,6 +95,7 @@ var AllSources = [...]subscraping.Source{
 	&robtex.Source{},
 	&rsecloud.Source{},
 	&securitytrails.Source{},
+	&profundis.Source{},
 	&shodan.Source{},
 	&sitedossier.Source{},
 	&threatbook.Source{},
@@ -106,6 +111,7 @@ var AllSources = [...]subscraping.Source{
 	&builtwith.Source{},
 	&hudsonrock.Source{},
 	&digitalyama.Source{},
+	&thc.Source{},
 }
 
 var sourceWarnings = mapsutil.NewSyncLockMap[string, string](

pkg/passive/sources_test.go

@@ -43,6 +43,7 @@ var (
 		"robtex",
 		"rsecloud",
 		"securitytrails",
+		"profundis",
 		"shodan",
 		"sitedossier",
 		"threatbook",
@@ -59,6 +60,8 @@ var (
 		"builtwith",
 		"hudsonrock",
 		"digitalyama",
+		"merklemap",
+		"thc",
 	}
 
 	expectedDefaultSources = []string{
@@ -88,6 +91,7 @@ var (
 		// "riddler", // failing due to cloudfront protection
 		"rsecloud",
 		"securitytrails",
+		"profundis",
 		"shodan",
 		"windvane",
 		"virustotal",
@@ -98,6 +102,7 @@ var (
 		// "reconcloud",
 		"builtwith",
 		"digitalyama",
+		"thc",
 	}
 
 	expectedDefaultRecursiveSources = []string{
@@ -113,6 +118,7 @@ var (
 		"virustotal",
 		"leakix",
 		"facebook",
+		"merklemap",
 		// "reconcloud",
 	}
 )

pkg/runner/banners.go

@@ -17,7 +17,7 @@ const banner = `
 const ToolName = `subfinder`
 
 // Version is the current version of subfinder
-const version = `v2.10.1`
+const version = `v2.11.0`
 
 // showBanner is used to show the banner to the user
 func showBanner() {

pkg/subscraping/sources/alienvault/alienvault.go

@@ -73,8 +73,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 		}
 
 		for _, record := range response.PassiveDNS {
-			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Hostname}
-			s.results++
+			select {
+			case <-ctx.Done():
+				return
+			case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Hostname}:
+				s.results++
+			}
 		}
 	}()
 

pkg/subscraping/sources/anubis/anubis.go

@@ -56,8 +56,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 		session.DiscardHTTPResponse(resp)
 
 		for _, record := range subdomains {
-			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record}
-			s.results++
+			select {
+			case <-ctx.Done():
+				return
+			case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record}:
+				s.results++
+			}
 		}
 
 	}()

pkg/subscraping/sources/bevigil/bevigil.go

@@ -70,8 +70,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 		}
 
 		for _, subdomain := range subdomains {
-			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
-			s.results++
+			select {
+			case <-ctx.Done():
+				return
+			case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}:
+				s.results++
+			}
 		}
 
 	}()

pkg/subscraping/sources/bufferover/bufferover.go

@@ -96,8 +96,12 @@ func (s *Source) getData(ctx context.Context, sourceURL string, apiKey string, s
 
 	for _, subdomain := range subdomains {
 		for _, value := range session.Extractor.Extract(subdomain) {
-			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: value}
-			s.results++
+			select {
+			case <-ctx.Done():
+				return
+			case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: value}:
+				s.results++
+			}
 		}
 	}
 }

pkg/subscraping/sources/builtwith/builtwith.go

@@ -74,8 +74,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 		session.DiscardHTTPResponse(resp)
 		for _, result := range data.Results {
 			for _, path := range result.Result.Paths {
-				results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: fmt.Sprintf("%s.%s", path.SubDomain, path.Domain)}
-				s.results++
+				select {
+				case <-ctx.Done():
+					return
+				case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: fmt.Sprintf("%s.%s", path.SubDomain, path.Domain)}:
+					s.results++
+				}
 			}
 		}
 	}()

pkg/subscraping/sources/c99/c99.go

@@ -81,8 +81,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 
 		for _, data := range response.Subdomains {
 			if !strings.HasPrefix(data.Subdomain, ".") {
-				results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: data.Subdomain}
-				s.results++
+				select {
+				case <-ctx.Done():
+					return
+				case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: data.Subdomain}:
+					s.results++
+				}
 			}
 		}
 	}()