Extract IPs and CIDRs from SPF records in extract_targets

extract_targets already pulled hostnames from TXT content; it now also extracts the ip4:/ip6: IPs and CIDR ranges using bbot's existing ip/cidr regexes, reading from the already-resolved TXT (no extra resolution). The mechanism prefix is stripped before matching so it doesn't pollute the IPv6 match, and a CIDR's bare network address is not emitted alongside it.

Closes #2545

Author: thunderstornX

bbot/core/helpers/dns/helpers.py

@@ -1,10 +1,15 @@
 import logging
+import re
 
-from bbot.core.helpers.regexes import dns_name_extraction_regex
+from bbot.core.helpers.regexes import dns_name_extraction_regex, ip_range_regexes, ipv4_regex, ipv6_regex
 from bbot.core.helpers.misc import clean_dns_record
 
 log = logging.getLogger("bbot.core.helpers.dns")
 
+# leading SPF qualifier (+-~?) and ip4:/ip6: mechanism prefix; stripped before IP
+# matching so the "ip6:" in e.g. "ip6:2001:db8::/48" doesn't pollute the match
+_spf_ip_mechanism_prefix = re.compile(r"^[+\-~?]?(?:ip[46]:)?", re.I)
+
 
 # Default rdtypes BBOT cares about during recursive resolution
 all_rdtypes = ["A", "AAAA", "SRV", "MX", "NS", "SOA", "CNAME", "TXT"]
@@ -16,8 +21,9 @@ def extract_targets(record):
     For structured rdata (A/AAAA/CNAME/NS/PTR/MX/SOA/SRV/etc), blastdns has
     already extracted the embedded names in Rust -- we just hand those back.
 
-    For TXT records we additionally apply a hostname regex to the text content,
-    since SPF / DKIM / similar TXT payloads commonly embed hostnames worth
+    For TXT records we additionally apply hostname/IP/CIDR regexes to the text
+    content, since SPF / DKIM / similar TXT payloads commonly embed hostnames, and
+    SPF in particular embeds IPs and CIDR ranges (ip4:/ip6: mechanisms) worth
     pivoting on. That regex extraction is BBOT-specific and stays here.
     """
     results = set()
@@ -26,14 +32,25 @@ def extract_targets(record):
         if cleaned:
             results.add((rdtype, cleaned))
 
-    # TXT: pull additional hostnames out of the free-form text content
+    # TXT: pull additional targets out of the free-form text content, token by token
     is_txt = "TXT" in record.rdata
     if is_txt:
-        text = record.to_text()
-        for match in dns_name_extraction_regex.finditer(text):
-            cleaned = clean_dns_record(text[match.start() : match.end()])
-            if cleaned:
-                results.add(("TXT", cleaned))
+        for token in record.to_text().split():
+            # strip a leading SPF qualifier and ip4:/ip6: mechanism prefix, if any
+            candidate = _spf_ip_mechanism_prefix.sub("", token)
+            # CIDR range, e.g. SPF "ip4:1.2.3.0/24" -> IP_NETWORK
+            if any(regex.fullmatch(candidate) for regex in ip_range_regexes):
+                results.add(("TXT", candidate))
+                continue
+            # individual IP, e.g. SPF "ip4:1.2.3.4" -> IP_ADDRESS
+            if ipv4_regex.fullmatch(candidate) or ipv6_regex.fullmatch(candidate):
+                results.add(("TXT", candidate))
+                continue
+            # hostnames, e.g. SPF "include:cloudprovider.com", DKIM selectors, etc.
+            for match in dns_name_extraction_regex.finditer(token):
+                cleaned = clean_dns_record(token[match.start() : match.end()])
+                if cleaned:
+                    results.add(("TXT", cleaned))
 
     return results
 

bbot/test/test_step_1/test_dns.py

@@ -70,6 +70,30 @@ async def test_dns_engine(bbot_scanner):
     assert not extracted
 
 
+@pytest.mark.asyncio
+async def test_extract_targets_spf_ips(bbot_scanner):
+    """SPF TXT records: extract IPs and CIDRs (ip4:/ip6:) alongside hostnames."""
+    scan = bbot_scanner()
+    await scan._prep()
+    await scan.helpers.dns._mock_dns(
+        {
+            "evilcorp.com": {
+                "TXT": ['"v=spf1 ip4:1.2.3.4 ip4:5.6.7.0/24 ip6:2001:db8::/48 include:cloudprovider.com -all"'],
+            },
+        }
+    )
+    response = await scan.helpers.dns.resolve_full("evilcorp.com", "TXT")
+    extracted = set()
+    for ans in response.response.answers:
+        extracted.update(extract_targets(ans))
+    hosts = {host for _, host in extracted}
+    assert "1.2.3.4" in hosts, "single IPv4 from ip4: mechanism not extracted"
+    assert "5.6.7.0/24" in hosts, "IPv4 CIDR from ip4: mechanism not extracted"
+    assert "2001:db8::/48" in hosts, "IPv6 CIDR from ip6: mechanism not extracted"
+    assert "cloudprovider.com" in hosts, "include: hostname not extracted"
+    assert "5.6.7.0" not in hosts, "bare network address must not be emitted alongside its CIDR"
+
+
 @pytest.mark.asyncio
 async def test_dns_resolution(bbot_scanner):
     """Multi-rdtype resolution + SPF affiliate tagging end-to-end."""