Fix unbounded lightfuzz queue growth from parameter re-discovery loop

bbot/core/config/models.py

@@ -395,6 +395,7 @@ class BBOTConfig(BaseModel):
     # URL handling
     url_querystring_remove: Optional[bool] = None
     url_querystring_collapse: Optional[bool] = None
+    url_max_path_param_repeats: Optional[int] = None
     url_extension_blacklist: Optional[list[str]] = None
     url_extension_special: Optional[list[str]] = None
     url_extension_static: Optional[list[str]] = None

bbot/core/event/base.py

@@ -1636,9 +1636,22 @@ def sanitize_data(self, data):
                 log.verbose(f"Error detecting envelopes for {self}: {e}")
         return data
 
+    def _dedup_url(self, url):
+        """The page the parameter lives on. The query string of whichever request revealed it
+        is context rather than identity, so it is dropped; scans that opt into per-value
+        fuzzing with url_querystring_collapse=False keep it, normalized for ordering."""
+        base, sep, query = url.partition("?")
+        if not sep or self.scan is None:
+            return url
+        if self.scan.config.get("url_querystring_collapse", True):
+            return base
+        query_dict = parse_qs(query)
+        kept = "&".join(f"{k}={','.join(sorted(v))}" for k, v in sorted(query_dict.items()))
+        return f"{base}?{kept}"
+
     def _data_id(self):
         # dedupe by url:name:param_type
-        url = self.data.get("url", "")
+        url = self._dedup_url(self.data.get("url", ""))
         name = self.data.get("name", "")
         param_type = self.data.get("type", "")
         envelopes = getattr(self, "envelopes", "")
@@ -1650,7 +1663,7 @@ def _outgoing_dedup_hash(self, event):
         return hash(
             (
                 str(event.host),
-                event.data["url"],
+                event._dedup_url(event.data["url"]),
                 event.data.get("name", ""),
                 event.data.get("type", ""),
                 event.data.get("envelopes", ""),

bbot/core/helpers/url.py

@@ -1,5 +1,6 @@
 import uuid
 import logging
+from collections import Counter
 from contextlib import suppress
 from urllib.parse import urlparse, parse_qs, urlencode, ParseResult
 
@@ -211,6 +212,46 @@ def hash_url(url):
     return hash(tuple(to_hash))
 
 
+def max_path_param_repeats(url):
+    """
+    Count how many times the most-repeated path parameter appears in a URL's path.
+
+    Path (matrix) parameters are the ";key=value" pieces a server may append to a path
+    segment; some reissue one on every redirect, so the path grows without bound. Keys are
+    compared case-insensitively. Ordinary path segments are deliberately not counted: a deep
+    path that repeats a directory name is finite, and depth is bounded by web_spider_depth.
+
+    Args:
+        url (Union[str, ParseResult]): The URL to inspect.
+
+    Returns:
+        int: The highest number of times any single path parameter appears.
+
+    Examples:
+        >>> max_path_param_repeats('https://www.evilcorp.com/foo/bar')
+        0
+
+        >>> max_path_param_repeats('https://www.evilcorp.com/;JSESSIONID=a;JSESSIONID=b')
+        2
+
+        >>> max_path_param_repeats('https://www.evilcorp.com/app;a=1/b;c=2')
+        1
+    """
+    parsed = url if hasattr(url, "path") else parse_url(url)
+    # urlparse peels the final segment's matrix parameters off into .params
+    path = parsed.path
+    trailing_params = getattr(parsed, "params", "")
+    if trailing_params:
+        path = f"{path};{trailing_params}"
+    counts = Counter()
+    for segment in path.split("/"):
+        # the first ";"-delimited piece is the segment itself, not a parameter
+        for token in segment.split(";")[1:]:
+            if token:
+                counts[token.split("=", 1)[0].lower()] += 1
+    return max(counts.values(), default=0)
+
+
 def url_depth(url):
     """
     Calculate the depth of the given URL based on its path components.

bbot/defaults.yml

@@ -191,6 +191,8 @@ python: True
 url_querystring_remove: True
 # When query string is retained, by default collapse parameter values down to a single value per parameter
 url_querystring_collapse: True
+# Reject URLs whose path repeats the same path parameter this many times (e.g. a session id appended on every redirect)
+url_max_path_param_repeats: 10
 
 # Completely ignore URLs with these extensions
 url_extension_blacklist:

bbot/modules/lightfuzz/lightfuzz.py

@@ -41,6 +41,10 @@ class Config(BaseModuleConfig):
             True,
             description="Emit canonical baseline responses as HTTP_RESPONSE events so excavate can mine them for new params/URLs.",
         )
+        max_baseline_generations: int = Field(
+            10,
+            description="Stop emitting baseline responses once a parameter is this many lightfuzz generations deep, bounding excavate feedback loops.",
+        )
 
     meta = {
         "description": "BBOT's DAST module — lightly fuzz web parameters discovered during recon for common vulnerability classes",
@@ -64,6 +68,8 @@ async def setup(self):
         self.interactsh_disable = self.scan.config.get("interactsh_disable", False)
         self.avoid_wafs = self.scan.config.get("avoid_wafs", True)
         self.emit_baseline_responses = self.config.get("emit_baseline_responses", True)
+        self.max_baseline_generations = self.config.get("max_baseline_generations", 10)
+        self._baseline_generation_cap_hit = False
         self.submodules = {}
         # Per-event baseline cache so submodules with identical request signatures share one HttpCompare.
         self._baseline_cache = {}
@@ -186,12 +192,40 @@ def store_cached_baseline(self, event, signature, http_compare):
         """Store an HttpCompare in the per-event baseline cache."""
         self._baseline_cache.setdefault(event.id, {})[signature] = http_compare
 
+    def baseline_generations(self, event):
+        """How many lightfuzz-emitted events are already in this event's ancestry.
+
+        Every baseline response we emit is mined by excavate, which hands the parameters it
+        finds back to us. Pages that vary their form on each load (rotating CSRF tokens,
+        honeypot fields) make each round look new, so the chain is bounded by counting it.
+        """
+        count = 0
+        e = event
+        while 1:
+            parent = e.parent
+            if parent is None or parent == e:
+                break
+            if getattr(parent.module, "name", "") == self.name:
+                count += 1
+                if count >= self.max_baseline_generations:
+                    break
+            e = parent
+        return count
+
     async def emit_baseline_response(self, response, event, method):
         """Emit a baseline blasthttp Response as an HTTP_RESPONSE event so excavate can mine post-submit pages."""
         if not self.emit_baseline_responses:
             return
         if response is None:
             return
+        if self.baseline_generations(event) >= self.max_baseline_generations:
+            if not self._baseline_generation_cap_hit:
+                self._baseline_generation_cap_hit = True
+                self.verbose(
+                    f"Reached max_baseline_generations ({self.max_baseline_generations}) on {event.host}; "
+                    "no longer emitting baseline responses for parameters nested this deep"
+                )
+            return
         try:
             parsed = urlparse(str(response.url))
             url_input = parsed.netloc or str(response.url)

bbot/scanner/manager.py

@@ -2,6 +2,7 @@
 from contextlib import suppress
 from radixtarget import host_size_key
 
+from bbot.core.helpers.url import max_path_param_repeats
 from bbot.modules.base import BaseInterceptModule
 
 
@@ -104,6 +105,17 @@ async def handle_event(self, event, **kwargs):
                 )
                 event.add_tag("blacklisted")
 
+        # reject degenerate URL paths, which servers generate in unbounded numbers
+        parsed_url = getattr(event, "parsed_url", None)
+        if parsed_url is not None and self.scan.url_max_path_param_repeats > 0:
+            repeats = max_path_param_repeats(parsed_url)
+            if repeats >= self.scan.url_max_path_param_repeats:
+                self.debug(
+                    f"Blacklisting {event} because its path repeats a parameter {repeats:,} times "
+                    f"(url_max_path_param_repeats={self.scan.url_max_path_param_repeats:,})"
+                )
+                event.add_tag("blacklisted")
+
         # main scan blacklist
         host_filterable = getattr(event, "host_filterable", None)
         event_blacklisted = False

bbot/scanner/scanner.py

@@ -285,6 +285,9 @@ def __init__(
         # url querystring behavior
         self.url_querystring_remove = self.config.get("url_querystring_remove", True)
 
+        # reject URLs whose path repeats the same path parameter this many times
+        self.url_max_path_param_repeats = self.config.get("url_max_path_param_repeats", 10)
+
         # blob inclusion
         self._file_blobs = self.config.get("file_blobs", False)
         self._folder_blobs = self.config.get("folder_blobs", False)

bbot/test/test_step_1/test_events.py

@@ -1452,3 +1452,56 @@ async def _instant_finish(ev, _orig=orig_queue):
     assert "original_value" in event.data, "original_value stripped during distribution -- sentinel missing"
 
     await scan._cleanup()
+
+
+@pytest.mark.asyncio
+async def test_web_parameter_querystring_dedup():
+    """A parameter is identified by its name and type on a page, not by the query string of
+    the request that revealed it. url_querystring_collapse=False opts back in to treating
+    sibling parameter values as significant, so those scans fuzz each variant separately."""
+
+    def make_param(scan, url, name="message", param_type="POSTPARAM"):
+        return scan.make_event(
+            {
+                "host": "example.com",
+                "type": param_type,
+                "name": name,
+                "original_value": "",
+                "url": url,
+                "description": f"{param_type} [{name}]",
+            },
+            "WEB_PARAMETER",
+            parent=scan.root_event,
+        )
+
+    base = "https://example.com/contact"
+    rotating_value = f"{base}?csrf=a08157098935259&id=6"
+    rotating_name = f"{base}?csrf=c19ae748d80ed939&id=6&aYBNT794ROfpo=0978126345"
+
+    # collapse=True (the default, and what lightfuzz/lightfuzz-light inherit): pages that
+    # reissue a token or a honeypot field on every load are still one work item
+    scan = Scanner("example.com", config={"url_querystring_remove": False, "url_querystring_collapse": True})
+    await scan._prep()
+    a = make_param(scan, f"{base}?csrf=7f01d97aec3100c7&id=6")
+    for other in (rotating_value, rotating_name, base):
+        e = make_param(scan, other)
+        assert a.data_id == e.data_id, f"query string leaked into dedup key: {other}"
+        assert hash(a) == hash(e)
+        assert a._outgoing_dedup_hash(a) == e._outgoing_dedup_hash(e)
+
+    # a different parameter, page, or parameter type is still its own work item
+    assert make_param(scan, base, name="subject").data_id != a.data_id
+    assert make_param(scan, "https://example.com/other").data_id != a.data_id
+    assert make_param(scan, base, param_type="GETPARAM").data_id != a.data_id
+    await scan._cleanup()
+
+    # collapse=False (lightfuzz-max, lightfuzz-xss): sibling values are significant again
+    scan = Scanner("example.com", config={"url_querystring_remove": False, "url_querystring_collapse": False})
+    await scan._prep()
+    c = make_param(scan, f"{base}?csrf=7f01d97aec3100c7&id=6")
+    assert make_param(scan, rotating_value).data_id != c.data_id
+    assert make_param(scan, rotating_name).data_id != c.data_id
+    assert make_param(scan, base).data_id != c.data_id
+    # ...but the key still does not depend on parameter ordering
+    assert make_param(scan, f"{base}?id=6&csrf=7f01d97aec3100c7").data_id == c.data_id
+    await scan._cleanup()

bbot/test/test_step_1/test_helpers.py

@@ -45,6 +45,18 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_httpserver):
     with pytest.raises(ValueError):
         helpers.validators.clean_url("http://evilcorp,com")
 
+    # path parameters accumulate inside a path segment, and are keyed by name
+    assert helpers.max_path_param_repeats("http://evilcorp.com/;JSESSIONID=a") == 1
+    assert helpers.max_path_param_repeats("http://evilcorp.com/;JSESSIONID=a;JSESSIONID=b;jsessionid=c") == 3
+    assert helpers.max_path_param_repeats("http://evilcorp.com/app;a=1;b=2") == 1
+    assert helpers.max_path_param_repeats("http://evilcorp.com/app;a=1/b;c=2") == 1
+    # ordinary path segments are not parameters, however deep or repetitive the path
+    assert helpers.max_path_param_repeats("http://evilcorp.com/") == 0
+    assert helpers.max_path_param_repeats("http://evilcorp.com/foo/bar") == 0
+    assert helpers.max_path_param_repeats("http://evilcorp.com/" + "/".join(["a"] * 30)) == 0
+    # the query string is not part of the path
+    assert helpers.max_path_param_repeats("http://evilcorp.com/foo?x=1&x=2&x=3") == 0
+
     assert helpers.url_depth("http://evilcorp.com/asdf/user/") == 2
     assert helpers.url_depth("http://evilcorp.com/asdf/user") == 2
     assert helpers.url_depth("http://evilcorp.com/asdf/") == 1

bbot/test/test_step_1/test_scan.py

@@ -520,3 +520,47 @@ def mock_memory_status():
         assert m.num_incoming_events == 0
     scan.modules_status(_log=False)
     assert scan._ingress_delay == 0.0, "throttle must clear in drain mode (no producers, ingress is the drain)"
+
+
+@pytest.mark.asyncio
+async def test_url_max_path_param_repeats():
+    """Servers that append a session id on every redirect, and spider traps, generate an
+    unbounded supply of URLs whose path repeats one key. Ingress drops them."""
+    from bbot.scanner import Scanner
+
+    def jsessionid_url(n):
+        return "http://evilcorp.com/" + "".join(f";JSESSIONID={i:08x}-2ab9-49eb" for i in range(n))
+
+    scan = Scanner("evilcorp.com", config={"url_max_path_param_repeats": 10})
+    await scan._prep()
+    ingress = scan.ingress_module
+
+    # handle_event returns None when it accepts, and (False, reason) when it rejects
+
+    # below the limit the URL is scanned normally
+    shallow = scan.make_event(jsessionid_url(9), "URL_UNVERIFIED", parent=scan.root_event)
+    assert await ingress.handle_event(shallow) is None
+    assert "blacklisted" not in shallow.tags
+
+    # at the limit it is rejected before reaching any module's queue
+    degenerate = scan.make_event(jsessionid_url(10), "URL_UNVERIFIED", parent=scan.root_event)
+    assert await ingress.handle_event(degenerate) == (False, "event is blacklisted")
+    assert "blacklisted" in degenerate.tags
+
+    # ordinary URLs are untouched, however deep or repetitive the path
+    for url in (
+        "http://evilcorp.com/api/v1/users/1",
+        "http://evilcorp.com/" + "/".join(["blacklanternsecurity"] * 20) + ".txt",
+    ):
+        normal = scan.make_event(url, "URL_UNVERIFIED", parent=scan.root_event)
+        assert await ingress.handle_event(normal) is None
+        assert "blacklisted" not in normal.tags
+    await scan._cleanup()
+
+    # the guard can be disabled outright
+    scan = Scanner("evilcorp.com", config={"url_max_path_param_repeats": 0})
+    await scan._prep()
+    off = scan.make_event(jsessionid_url(50), "URL_UNVERIFIED", parent=scan.root_event)
+    await scan.ingress_module.handle_event(off)
+    assert "blacklisted" not in off.tags
+    await scan._cleanup()