Add CIDR range expansion support for proxy scanning

Co-authored-by: monosans <76561516+monosans@users.noreply.github.com>

Author: Copilot

Cargo.lock

@@ -16,6 +16,7 @@ foldhash = "=0.2.0"
 futures = { version = "=0.3.31", optional = true }
 hickory-resolver = "=0.25.2"
 httpdate = "=1.0.3"
+ipnetwork = "=0.21.1"
 itertools = "=0.14"
 maxminddb = { version = "=0.26.0", features = ["mmap"] }
 parking_lot = "=0.12.4"

Cargo.toml

@@ -1,5 +1,7 @@
 use std::sync::LazyLock;
 
+use ipnetwork::IpNetwork;
+
 pub static PROXY_REGEX: LazyLock<fancy_regex::Regex> = LazyLock::new(|| {
     let pattern = r"(?:^|[^0-9A-Za-z])(?:(?P<protocol>https?|socks[45]):\/\/)?(?:(?P<username>[0-9A-Za-z]{1,64}):(?P<password>[0-9A-Za-z]{1,64})@)?(?P<host>[A-Za-z][\-\.A-Za-z]{0,251}[A-Za-z]|[A-Za-z]|(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(?:\.(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])){3}):(?P<port>[0-9]|[1-9][0-9]{1,3}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])(?=[^0-9A-Za-z]|$)";
     fancy_regex::RegexBuilder::new(pattern)
@@ -13,10 +15,100 @@ static IPV4_REGEX: LazyLock<fancy_regex::Regex> = LazyLock::new(|| {
     fancy_regex::Regex::new(pattern).unwrap()
 });
 
+static CIDR_REGEX: LazyLock<fancy_regex::Regex> = LazyLock::new(|| {
+    let pattern = r"^\s*(?P<network>(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(?:\.(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])){3})/(?P<prefix>[0-9]|[12][0-9]|3[0-2]):(?P<port>[0-9]|[1-9][0-9]{1,3}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])\s*$";
+    fancy_regex::Regex::new(pattern).unwrap()
+});
+
 pub fn parse_ipv4(s: &str) -> Option<String> {
     if let Ok(Some(captures)) = IPV4_REGEX.captures(s) {
         captures.name("host").map(|capture| capture.as_str().to_owned())
     } else {
         None
     }
 }
+
+/// Expands CIDR ranges in text into individual IP:port entries
+/// Supports format like "192.168.1.0/24:8080" which expands to all IPs in the range
+pub fn expand_cidr_ranges(text: &str) -> String {
+    let mut result = String::new();
+    
+    for line in text.lines() {
+        let line = line.trim();
+        if let Ok(Some(captures)) = CIDR_REGEX.captures(line) {
+            // Extract CIDR range and port
+            if let (Some(network), Some(port)) = (captures.name("network"), captures.name("port")) {
+                let cidr_str = format!("{}/{}", 
+                    network.as_str(), 
+                    captures.name("prefix").unwrap().as_str()
+                );
+                
+                match cidr_str.parse::<IpNetwork>() {
+                    Ok(network) => {
+                        // Expand the network to individual IPs
+                        for ip in network.iter() {
+                            if ip.is_ipv4() {
+                                result.push_str(&format!("{}:{}\n", ip, port.as_str()));
+                            }
+                        }
+                    }
+                    Err(_) => {
+                        // If parsing fails, keep the original line
+                        result.push_str(line);
+                        result.push('\n');
+                    }
+                }
+            } else {
+                // If regex matches but capture groups are missing, keep the original line
+                result.push_str(line);
+                result.push('\n');
+            }
+        } else {
+            // Not a CIDR range, keep the original line
+            result.push_str(line);
+            result.push('\n');
+        }
+    }
+    
+    result
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_cidr_expansion() {
+        // Test basic CIDR expansion
+        let input = "192.168.1.0/30:8080";
+        let result = expand_cidr_ranges(input);
+        let lines: Vec<&str> = result.trim().split('\n').collect();
+        
+        assert_eq!(lines.len(), 4);
+        assert!(lines.contains(&"192.168.1.0:8080"));
+        assert!(lines.contains(&"192.168.1.1:8080"));
+        assert!(lines.contains(&"192.168.1.2:8080"));
+        assert!(lines.contains(&"192.168.1.3:8080"));
+    }
+
+    #[test]
+    fn test_mixed_input() {
+        let input = "192.168.1.0/31:8080\n127.0.0.1:9090\ninvalid-line";
+        let result = expand_cidr_ranges(input);
+        let lines: Vec<&str> = result.trim().split('\n').collect();
+        
+        // Should have 2 CIDR-expanded IPs + 1 regular IP + 1 invalid line
+        assert_eq!(lines.len(), 4);
+        assert!(lines.contains(&"192.168.1.0:8080"));
+        assert!(lines.contains(&"192.168.1.1:8080"));
+        assert!(lines.contains(&"127.0.0.1:9090"));
+        assert!(lines.contains(&"invalid-line"));
+    }
+
+    #[test]
+    fn test_single_ip_cidr() {
+        let input = "10.0.0.1/32:3128";
+        let result = expand_cidr_ranges(input);
+        assert_eq!(result.trim(), "10.0.0.1:3128");
+    }
+}

src/parsers.rs

@@ -9,7 +9,7 @@ use crate::{
     HashSet,
     config::{Config, Source},
     http,
-    parsers::PROXY_REGEX,
+    parsers::{PROXY_REGEX, expand_cidr_ranges},
     proxy::{Proxy, ProxyType},
     utils::pretty_error,
 };
@@ -58,8 +58,11 @@ async fn scrape_one(
         }
     };
 
+    // Expand CIDR ranges to individual IP:port entries
+    let expanded_text = expand_cidr_ranges(&text);
+
     let mut matches = Vec::new();
-    for (i, maybe_capture) in PROXY_REGEX.captures_iter(&text).enumerate() {
+    for (i, maybe_capture) in PROXY_REGEX.captures_iter(&expanded_text).enumerate() {
         if config.scraping.max_proxies_per_source != 0
             && i >= config.scraping.max_proxies_per_source
         {
@@ -116,7 +119,7 @@ async fn scrape_one(
     }
 
     drop(config);
-    drop(text);
+    drop(expanded_text);
 
     #[cfg(feature = "tui")]
     for proto in seen_protocols {

src/scraper.rs

@@ -0,0 +1,144 @@
+# Enable debug logging (shows detailed checking process)
+# Warning: Produces very verbose output
+debug = false
+
+
+[scraping]
+# Maximum proxies to collect per source (0 = unlimited)
+# Helps skip unreliable sources with too many proxies
+max_proxies_per_source = 100000
+
+# Request timeout for fetching proxy sources (seconds)
+# Higher values may find more sources but take longer
+timeout = 60.0
+connect_timeout = 5.0
+
+# HTTP(S),SOCKS4 or SOCKS5 proxy used for fetching sources (e.g., "socks5://user:pass@host:port"). Leave empty to disable.
+proxy = ""
+
+# User-Agent header for scraping requests
+user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36"
+
+
+[checking]
+# URL for checking proxy functionality
+# httpbin-compatible: Returns JSON with IP info for ASN/geo data
+# plain-text: Returns just IP address for basic connectivity
+# Examples:
+#   "https://httpbin.org/ip" - JSON with "origin" key. Full featured checking.
+#   "https://api.ipify.org"  - Simple IP return. Full featured checking.
+#   "https://google.com"     - Basic connect/read check
+#   ""                       - Skip checking (scrape only)
+check_url = "https://api.ipify.org"
+
+# Concurrent proxy checks (adjust based on RAM/network capacity)
+# Start low and increase gradually if system handles it well
+max_concurrent_checks = 1024
+
+# Proxy response timeout (seconds)
+# Higher = more working proxies found, slower checking
+# Lower = faster checking, may miss slower proxies
+timeout = 60.0
+connect_timeout = 5.0
+
+# User-Agent header for proxy check requests
+user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36"
+
+
+[output]
+# Output directory (Docker ignores this setting)
+path = "./out"
+
+# Sort by response time (true) or IP address (false)
+sort_by_speed = true
+
+
+# Plain text output (.txt files)
+[output.txt]
+enabled = false
+
+
+# JSON output with metadata (.json files)
+[output.json]
+enabled = false
+
+# Add ASN (network provider) info to JSON output
+# Uses offline MaxMind database
+include_asn = true
+
+# Add geolocation (country/city) info to JSON output
+# Uses offline MaxMind database
+include_geolocation = true
+
+
+# Proxy sources configuration
+# Add local files: ["./my_proxies.txt"] or URLs
+# Sources are fetched in parallel for speed
+
+enabled = true
+urls = ["file:///tmp/test_cidr.txt"]
+    # Local file examples:
+    # "./my_http_proxies.txt",
+    # "/home/user/my_http_proxies.txt",
+    # "C:/Users/user/Desktop/my_http_proxies.txt",
+    # "file:///home/user/my_http_proxies.txt",
+
+    # Advanced URL configuration examples (with basic auth or custom headers):
+    # HTTP Basic Auth example:
+    # { url = "https://some.api/endpoint", basic_auth = { username = "user", password = "password123" } },
+    # Custom headers example:
+    # { url = "https://some.api/endpoint", headers = { Authorization = "Bearer YOUR_API_KEY" } },
+
+    "https://api.proxyscrape.com/v3/free-proxy-list/get?request=getproxies&protocol=http",
+    "https://api.proxyscrape.com/v3/free-proxy-list/get?request=getproxies&protocol=https",
+    "https://raw.githubusercontent.com/proxifly/free-proxy-list/refs/heads/main/proxies/protocols/http/data.txt",
+    "https://raw.githubusercontent.com/proxifly/free-proxy-list/refs/heads/main/proxies/protocols/https/data.txt",
+    "https://raw.githubusercontent.com/roosterkid/openproxylist/refs/heads/main/HTTPS_RAW.txt",
+    "https://raw.githubusercontent.com/sunny9577/proxy-scraper/refs/heads/master/generated/http_proxies.txt",
+    "https://raw.githubusercontent.com/TheSpeedX/PROXY-List/refs/heads/master/http.txt",
+]
+
+[scraping.socks4]
+enabled = false
+urls = [
+    # Local file examples:
+    # "./my_socks4_proxies.txt",
+    # "/home/user/my_socks4_proxies.txt",
+    # "C:/Users/user/Desktop/my_socks4_proxies.txt",
+    # "file:///home/user/my_socks4_proxies.txt",
+
+    # Advanced URL configuration examples (with basic auth or custom headers):
+    # HTTP Basic Auth example:
+    # { url = "https://some.api/endpoint", basic_auth = { username = "user", password = "password123" } },
+    # Custom headers example:
+    # { url = "https://some.api/endpoint", headers = { Authorization = "Bearer YOUR_API_KEY" } },
+
+    "https://api.proxyscrape.com/v3/free-proxy-list/get?request=getproxies&protocol=socks4",
+    "https://raw.githubusercontent.com/proxifly/free-proxy-list/refs/heads/main/proxies/protocols/socks4/data.txt",
+    "https://raw.githubusercontent.com/roosterkid/openproxylist/refs/heads/main/SOCKS4_RAW.txt",
+    "https://raw.githubusercontent.com/sunny9577/proxy-scraper/refs/heads/master/generated/socks4_proxies.txt",
+    "https://raw.githubusercontent.com/TheSpeedX/PROXY-List/refs/heads/master/socks4.txt",
+]
+
+[scraping.socks5]
+enabled = false
+urls = [
+    # Local file examples:
+    # "./my_socks5_proxies.txt",
+    # "/home/user/my_socks5_proxies.txt",
+    # "C:/Users/user/Desktop/my_socks5_proxies.txt",
+    # "file:///home/user/my_socks5_proxies.txt",
+
+    # Advanced URL configuration examples (with basic auth or custom headers):
+    # HTTP Basic Auth example:
+    # { url = "https://some.api/endpoint", basic_auth = { username = "user", password = "password123" } },
+    # Custom headers example:
+    # { url = "https://some.api/endpoint", headers = { Authorization = "Bearer YOUR_API_KEY" } },
+
+    "https://api.proxyscrape.com/v3/free-proxy-list/get?request=getproxies&protocol=socks5",
+    "https://raw.githubusercontent.com/hookzof/socks5_list/refs/heads/master/proxy.txt",
+    "https://raw.githubusercontent.com/proxifly/free-proxy-list/refs/heads/main/proxies/protocols/socks5/data.txt",
+    "https://raw.githubusercontent.com/roosterkid/openproxylist/refs/heads/main/SOCKS5_RAW.txt",
+    "https://raw.githubusercontent.com/sunny9577/proxy-scraper/refs/heads/master/generated/socks5_proxies.txt",
+    "https://raw.githubusercontent.com/TheSpeedX/PROXY-List/refs/heads/master/socks5.txt",
+]