Merge pull request #3000 from corest/feature/exclude-proxy-config

Add option to exclude ip/hostname from proxy

Author: liquidsec

README.md

@@ -91,7 +91,7 @@ bbot -t evilcorp.com -p spider
 description: Recursive web spider
 
 modules:
-  - httpx
+  - http
 
 blacklist:
   # Prevent spider from invalidating sessions by logging out
@@ -223,6 +223,13 @@ include:
   - web-screenshots
   - baddns-heavy
 
+config:
+  modules:
+    dnsbrute:
+      recursive_mutations: true
+    dnscommonsrv:
+      recursive_mutations: true
+
 ```
 
 </details>

bbot/core/helpers/web/web.py

@@ -76,6 +76,7 @@ def __init__(self, parent_helper):
         self._http_timeout = self.web_config.get("http_timeout", 20)
         self._http_retries = self.web_config.get("http_retries", 1)
         self._http_proxy = self.web_config.get("http_proxy", None)
+        self._http_proxy_exclude = self.web_config.get("http_proxy_exclude", []) or []
         ua = self.web_config.get("user_agent", "BBOT")
         ua_suffix = self.web_config.get("user_agent_suffix") or ""
         self._user_agent = f"{ua} {ua_suffix}".strip()
@@ -116,6 +117,7 @@ def _build_blasthttp_kwargs(self, url, **kwargs):
         follow_redirects = kwargs.pop("follow_redirects", None)
         max_redirects = kwargs.pop("max_redirects", None)
         proxy = kwargs.pop("proxy", self._http_proxy)
+        no_proxy = kwargs.pop("no_proxy", self._http_proxy_exclude)
         retries = kwargs.pop("retries", self._http_retries)
         params = kwargs.pop("params", None)
         cookies = kwargs.pop("cookies", None)
@@ -211,6 +213,11 @@ def _build_blasthttp_kwargs(self, url, **kwargs):
             blast_kwargs["max_redirects"] = int(max_redirects)
         if proxy:
             blast_kwargs["proxy"] = proxy
+            # no_proxy lists hosts that bypass the proxy; it only has an effect
+            # alongside a proxy (blasthttp errors if it's set without one), so
+            # only forward it when a proxy is actually in play.
+            if no_proxy:
+                blast_kwargs["no_proxy"] = list(no_proxy)
         if max_body_size is not None:
             blast_kwargs["max_body_size"] = int(max_body_size)
         if request_target is not None:

bbot/defaults.yml

@@ -92,6 +92,9 @@ dns:
 web:
   # HTTP proxy
   http_proxy:
+  # Hosts/CIDRs to exclude from HTTP proxy (NO_PROXY equivalent)
+  # Examples: ["localhost", "*.internal.corp", "10.0.0.0/8", "elastic.mycompany.com"]
+  http_proxy_exclude: []
   # Web user-agent
   user_agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.2151.97
   # Suffix to append to user-agent (e.g. for tracking or identification)

bbot/scanner/preset/args.py

@@ -180,6 +180,9 @@ def preset_from_args(self):
         if self.parsed.proxy:
             args_preset.core.merge_custom({"web": {"http_proxy": self.parsed.proxy}})
 
+        if self.parsed.no_proxy:
+            args_preset.core.merge_custom({"web": {"http_proxy_exclude": self.parsed.no_proxy}})
+
         if self.parsed.custom_headers:
             args_preset.core.merge_custom({"web": {"http_headers": self.parsed.custom_headers}})
 
@@ -376,6 +379,13 @@ def create_parser(self, *args, **kwargs):
         misc = p.add_argument_group(title="Misc")
         misc.add_argument("--version", action="store_true", help="show BBOT version and exit")
         misc.add_argument("--proxy", help="Use this proxy for all HTTP requests", metavar="HTTP_PROXY")
+        misc.add_argument(
+            "--no-proxy",
+            nargs="+",
+            default=[],
+            help="Exclude these hosts from proxy (e.g. localhost *.internal.corp 10.0.0.0/8)",
+            metavar="HOST",
+        )
         misc.add_argument(
             "-H",
             "--custom-headers",

bbot/scanner/preset/environ.py

@@ -125,6 +125,13 @@ def prepare(self):
             environ.pop("HTTP_PROXY", None)
             environ.pop("HTTPS_PROXY", None)
 
+        # handle proxy exclusions (NO_PROXY)
+        http_proxy_exclude = self.preset.config.get("web", {}).get("http_proxy_exclude", [])
+        if http_proxy_exclude:
+            environ["NO_PROXY"] = ",".join(str(x) for x in http_proxy_exclude)
+        else:
+            environ.pop("NO_PROXY", None)
+
         # ssl verification
         import urllib3
 

bbot/scanner/scanner.py

@@ -237,6 +237,7 @@ def __init__(
         max_redirects = web_config.get("http_max_redirects", 5)
         self.web_max_redirects = max(max_redirects, self.web_spider_distance)
         self.http_proxy = web_config.get("http_proxy", "")
+        self.http_proxy_exclude = web_config.get("http_proxy_exclude", [])
         self.http_timeout = web_config.get("http_timeout", 10)
         self.blasthttp_timeout = web_config.get("blasthttp_timeout", 5)
         self.http_retries = web_config.get("http_retries", 1)

bbot/test/test_step_1/test_web.py

@@ -450,6 +450,70 @@ async def test_http_proxy(bbot_scanner, bbot_httpserver, proxy_server):
     await scan._cleanup()
 
 
+@pytest.mark.asyncio
+async def test_http_proxy_exclude(bbot_scanner, bbot_httpserver, proxy_server):
+    """Verify that requests to excluded hosts bypass the proxy."""
+    endpoint = "/test_http_proxy_exclude"
+    url = bbot_httpserver.url_for(endpoint)
+    bbot_httpserver.expect_request(uri=endpoint).respond_with_data("proxy_exclude_works")
+
+    proxy_address = f"http://127.0.0.1:{proxy_server.server_address[1]}"
+    # Exclude 127.0.0.1 from proxy
+    scan = bbot_scanner(
+        "127.0.0.1",
+        config={
+            "web": {
+                "http_proxy": proxy_address,
+                "http_proxy_exclude": ["127.0.0.1"],
+            }
+        },
+    )
+
+    await scan._prep()
+
+    proxy_server.RequestHandlerClass.urls.clear()
+    r = await scan.helpers.request(url)
+
+    # Request should NOT go through proxy
+    assert len(proxy_server.RequestHandlerClass.urls) == 0, "Request should have bypassed proxy but went through it"
+    assert r.status_code == 200 and r.text == "proxy_exclude_works"
+
+    await scan._cleanup()
+
+
+@pytest.mark.asyncio
+async def test_http_proxy_exclude_passthrough(bbot_scanner, bbot_httpserver, proxy_server):
+    """Verify that non-excluded hosts still go through the proxy."""
+    endpoint = "/test_proxy_passthrough"
+    url = bbot_httpserver.url_for(endpoint)
+    bbot_httpserver.expect_request(uri=endpoint).respond_with_data("passthrough_works")
+
+    proxy_address = f"http://127.0.0.1:{proxy_server.server_address[1]}"
+    # Exclude a different host, not the one we're requesting
+    scan = bbot_scanner(
+        "127.0.0.1",
+        config={
+            "web": {
+                "http_proxy": proxy_address,
+                "http_proxy_exclude": ["10.0.0.0/8"],
+            }
+        },
+    )
+
+    await scan._prep()
+
+    proxy_server.RequestHandlerClass.urls.clear()
+    r = await scan.helpers.request(url)
+
+    # Request SHOULD go through proxy (127.0.0.1 not in exclusion list)
+    assert len(proxy_server.RequestHandlerClass.urls) == 1, (
+        f"Request to {url} should have gone through proxy but didn't"
+    )
+    assert r.status_code == 200 and r.text == "passthrough_works"
+
+    await scan._cleanup()
+
+
 @pytest.mark.asyncio
 async def test_http_ssl(bbot_scanner, bbot_httpserver_ssl):
     endpoint = "/test_http_ssl"