feat: [generate-openfilters-scanners.py] generate scanner warninglists from OpenFilters/internet-scanners

generate_all.sh

@@ -56,6 +56,7 @@ python3 generate-bunny-net.py
 python3 generate-ovh.py
 python3 generate-microsoft-mdca.py
 python3 generate-palo-alto-networks-cortex-cloud.py
+python3 generate-openfilters-scanners.py
 python3 generate-lots-project.py
 python3 generate-check-host-net.py
 popd

tools/generate-openfilters-scanners.py

@@ -0,0 +1,270 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Generate the warninglists of internet-wide scanning organisations from
+OpenFilters/internet-scanners (https://github.com/OpenFilters/internet-scanners).
+
+That repository tracks the *published* source ranges of scanning organisations --
+the ranges the operators themselves document as their scan origins -- and ships
+them in a machine-readable form under cidr/<org>_v4.txt and cidr/<org>_v6.txt.
+It is released into the public domain (Unlicense).
+
+This is deliberately a single generator writing many lists, in the same way
+generate-microsoft-azure.py and generate-vpn.py do: the upstream repository is
+one snapshot, so one download serves every list and they stay mutually
+consistent.
+
+Note that this data has a different provenance from the *-nt-scanning lists,
+which are derived from CIRCL network-telescope *observations*. Published ranges
+and observed ranges legitimately differ, so the two families are kept separate
+and this generator never writes to a -nt-scanning list.
+"""
+
+import ipaddress
+import json
+import logging
+import tarfile
+
+from generator import (
+    download_to_file,
+    get_abspath_list_file,
+    get_abspath_source_file,
+    get_version,
+    write_to_file,
+    consolidate_networks,
+)
+
+TARBALL_URL = (
+    "https://codeload.github.com/OpenFilters/internet-scanners/tar.gz/refs/heads/main"
+)
+TARBALL_FILE = "openfilters-internet-scanners.tar.gz"
+
+MATCHING_ATTRIBUTES = [
+    "ip-src",
+    "ip-dst",
+    "domain|ip",
+    "ip-src|port",
+    "ip-dst|port",
+]
+
+# Upstream organisation -> MISP warninglist directory.
+#
+# Mapping every organisation explicitly, rather than deriving the directory name
+# from the upstream filename, is what lets this generator notice when upstream
+# adds an organisation: an unmapped name is logged and skipped instead of
+# silently creating an unnamed list.
+ORG_TO_LIST = {
+    # -- organisations that already have a hand-maintained MISP list --
+    "alphastrike": "alphastrike-scanning",
+    "censys": "censys-scanning",
+    "paloaltonetworks": "palo-alto-networks-cortex-xpanse",
+    "shadowforce": "shadowforce-scanning",
+    "shadowserver": "shadowserver",
+    "shodan": "shodan-scanning",
+    # -- organisations that had no published-range list until now --
+    "a10networks": "a10networks-scanning",
+    "academyforinternetresearch": "academyforinternetresearch-scanning",
+    "adscore": "adscore-scanning",
+    "binaryedge": "binaryedge-scanning",
+    "bitsight": "bitsight-scanning",
+    "bufferoverrun": "bufferoverrun-scanning",
+    "criminalip": "criminalip-scanning",
+    "crowdstrikefalconsurface": "crowdstrikefalconsurface-scanning",
+    "cybcube": "cybcube-scanning",
+    "cybergreen": "cybergreen-scanning",
+    "cyberok": "cyberok-scanning",
+    "cyberresilience": "cyberresilience-scanning",
+    "cypex": "cypex-scanning",
+    "dataplane": "dataplane-scanning",
+    "deepfield": "deepfield-scanning",
+    "detectify": "detectify-scanning",
+    "edgewatch": "edgewatch-scanning",
+    "eset": "eset-scanning",
+    "fofa": "fofa-scanning",
+    "groupib": "groupib-scanning",
+    "hackertarget": "hackertarget-scanning",
+    "hadrian": "hadrian-scanning",
+    "hunter": "hunter-scanning",
+    "infrawatch": "infrawatch-scanning",
+    "internetmeasurementresearch": "internetmeasurementresearch-scanning",
+    "internetmetrics": "internetmetrics-scanning",
+    "internetresearchproject": "internetresearchproject-scanning",
+    "internettl": "internettl-scanning",
+    "intrinsec": "intrinsec-scanning",
+    "intruder": "intruder-scanning",
+    "ipinfo": "ipinfo-scanning",
+    "ipip": "ipip-scanning",
+    "ki3": "ki3-scanning",
+    "knoq": "knoq-scanning",
+    "leakix": "leakix-scanning",
+    "mandiantasm": "mandiantasm-scanning",
+    "netlas": "netlas-scanning",
+    "netscout": "netscout-scanning",
+    "odin": "odin-scanning",
+    "onlyscans": "onlyscans-scanning",
+    "pocnroll": "pocnroll-scanning",
+    "probethenet": "probethenet-scanning",
+    "quake": "quake-scanning",
+    "rapid7": "rapid7-scanning",
+    "recordedfuture": "recordedfuture-scanning",
+    "recyber": "recyber-scanning",
+    "researchscanner": "researchscanner-scanning",
+    "rootevidence": "rootevidence-scanning",
+    "securityresearch": "securityresearch-scanning",
+    "strechoid": "stretchoid-scanning",
+    "tenableasm": "tenableasm-scanning",
+    "universities": "university-scanning",
+    "visionheight": "visionheight-scanning",
+    "xmco": "xmco-scanning",
+    "zern": "zern-scanning",
+    "zoomeye": "zoomeye-scanning",
+}
+
+# Organisations deliberately not taken from this source, and why.
+EXCLUDED = {
+    "driftnet": "already generated by tools/generate-driftnet.py",
+    "modat": "already generated by tools/generate-modat-scanner.py",
+    "onyphe": "already generated by tools/generate-onyphe-scanner.py",
+    "other": "unattributed grab-bag; upstream moves entries out of it as they "
+             "are attributed, which would churn two lists per attribution",
+}
+
+
+def read_tarball(file):
+    """Return {member path without the top-level directory: bytes}."""
+    members = {}
+    with tarfile.open(get_abspath_source_file(file), "r:gz") as tar:
+        for member in tar.getmembers():
+            if not member.isfile():
+                continue
+            # Strip the "internet-scanners-main/" prefix GitHub adds.
+            parts = member.name.split("/", 1)
+            if len(parts) != 2:
+                continue
+            handle = tar.extractfile(member)
+            if handle is not None:
+                members[parts[1]] = handle.read()
+    return members
+
+
+def display_name(members, org):
+    """The organisation's human-readable name, from the first line of its .nft."""
+    raw = members.get("{}.nft".format(org))
+    if not raw:
+        return org
+    first = raw.decode("utf-8", "replace").splitlines()[0].strip()
+    return first.lstrip("#").strip() or org
+
+
+def networks_for(members, org):
+    networks = []
+    for suffix in ("_v4", "_v6"):
+        raw = members.get("cidr/{}{}.txt".format(org, suffix))
+        if not raw:
+            continue
+        for line in raw.decode("utf-8", "replace").splitlines():
+            line = line.strip()
+            if not line or line.startswith("#"):
+                continue
+            try:
+                # strict=True on purpose. A CIDR with host bits set is
+                # ambiguous, and the only way to accept one is to widen it to
+                # its enclosing network -- which in a warninglist means
+                # suppressing alerts for addresses the operator never claimed.
+                # Dropping the entry errs the safe way; the warning is there so
+                # the malformed range can be reported upstream.
+                ipaddress.ip_network(line)
+            except ValueError as exc:
+                logging.warning(
+                    "Skipping malformed range %s for organisation %s: %s",
+                    line,
+                    org,
+                    exc,
+                )
+                continue
+            networks.append(line)
+    return networks
+
+
+def existing_list(dst):
+    """The list already committed under dst, or None if it does not exist yet."""
+    try:
+        with open(get_abspath_list_file(dst)) as data_file:
+            return json.load(data_file)
+    except (IOError, OSError, ValueError):
+        return None
+
+
+def process(members, org, dst):
+    networks = networks_for(members, org)
+    if not networks:
+        logging.warning(
+            "No CIDR data upstream for organisation %s, skipping %s", org, dst
+        )
+        return
+
+    existing = existing_list(dst)
+
+    if existing is not None:
+        # Union with the entries already committed, never replace them. Several
+        # of these lists were curated by hand from sources this one does not
+        # cover (BGP name searches, vendor pages), and a range an organisation
+        # scanned from in the past is still worth recognising in a
+        # false-positive warninglist. So the generator only ever adds;
+        # consolidate_networks() then collapses whatever is genuinely
+        # contiguous or redundant.
+        networks.extend(existing.get("list", []))
+
+        # Keep the name and description a pre-existing list already had. MISP
+        # keys warninglists on their name, so renaming one in place would show
+        # up on the server as a second, competing list rather than an update.
+        name = existing["name"]
+        description = existing["description"]
+        matching_attributes = existing.get(
+            "matching_attributes", MATCHING_ATTRIBUTES
+        )
+    else:
+        name = "{} IP ranges used for scanning".format(display_name(members, org))
+        description = (
+            "Published source ranges of the internet-wide scanning organisation "
+            "{}, as tracked by "
+            "https://github.com/OpenFilters/internet-scanners".format(
+                display_name(members, org)
+            )
+        )
+        matching_attributes = MATCHING_ATTRIBUTES
+
+    warninglist = {
+        "name": name,
+        "version": get_version(),
+        "description": description,
+        "type": "cidr",
+        "list": consolidate_networks(networks),
+        "matching_attributes": matching_attributes,
+    }
+    write_to_file(warninglist, dst)
+
+
+def main():
+    download_to_file(TARBALL_URL, TARBALL_FILE)
+    members = read_tarball(TARBALL_FILE)
+
+    # Anything upstream ships that this generator does not know about is a new
+    # organisation. Report it rather than guessing a list name for it.
+    upstream = set()
+    for path in members:
+        if path.startswith("cidr/") and path.endswith(".txt"):
+            upstream.add(path[len("cidr/"):].rsplit("_", 1)[0])
+    for org in sorted(upstream - set(ORG_TO_LIST) - set(EXCLUDED)):
+        logging.warning(
+            "OpenFilters organisation %s has no MISP list mapped to it; "
+            "add it to ORG_TO_LIST to start generating one",
+            org,
+        )
+
+    for org, dst in sorted(ORG_TO_LIST.items()):
+        process(members, org, dst)
+
+
+if __name__ == "__main__":
+    main()