|
5 | 5 | * ``rename_asn_fields`` — the old ``source_asn_name`` / ``source_asn_domain`` |
6 | 6 | fields were renamed upstream to ``source_as_name`` / ``source_as_domain``. |
7 | 7 | Existing documents still use the old names. |
8 | | -* ``refill_enrichment_fields`` — parsedmarc's IP-based enrichment has improved |
9 | | - (GeoIP swap ipdb → ipinfo, better source-name/type classification). Old |
10 | | - documents carry stale values. We call ``parsedmarc.utils.get_ip_address_info`` |
11 | | - inside the parsedmarc container so old docs get the exact same enrichment |
12 | | - new docs get. |
| 8 | +* ``refill_enrichment_fields`` — parsedmarc's IP-based enrichment evolves over |
| 9 | + time (GeoIP data updates, new entries in the reverse-DNS map used for |
| 10 | + source-name/type classification). Old documents carry stale values. We call |
| 11 | + ``parsedmarc.utils.get_ip_address_info`` inside the parsedmarc container |
| 12 | + (with the live PTR lookup and reverse-DNS map enabled) so old docs get the |
| 13 | + same enrichment new docs get. |
13 | 14 |
|
14 | 15 | ``DashboardService.refresh_index_pattern_fields`` handles the third piece |
15 | 16 | (refreshing the cached field list inside each tenant's index-pattern saved |
@@ -86,20 +87,35 @@ def existing = ctx._source.get(f); |
86 | 87 | """.strip() |
87 | 88 |
|
88 | 89 | # Script run inside the parsedmarc container. Reads a JSON list of IPs from |
89 | | -# stdin and writes a JSON dict {ip: info_dict} to stdout. offline=True skips |
90 | | -# DNS/WHOIS so we only hit the local GeoIP DB and name/type classifier. |
| 90 | +# stdin and writes a JSON dict {ip: info_dict} to stdout. |
91 | 91 | # |
92 | | -# No per-IP try/except: with offline=True, get_ip_address_info returns a dict |
93 | | -# with None values for misses rather than raising — the only exceptions that |
94 | | -# can fire here are programming errors (bad kwargs, missing imports) that we |
95 | | -# want to see loudly, not mask as {ip: None}. Let the subprocess fail and |
96 | | -# surface its stderr via the CalledProcessError handler in _lookup_enrichment. |
| 92 | +# offline=False is required: with offline=True, parsedmarc short-circuits the |
| 93 | +# PTR lookup (reverse_dns = None), which in turn skips the reverse-DNS-map |
| 94 | +# lookup entirely (the map is only consulted after a successful PTR). That |
| 95 | +# means name/type/reverse_dns/base_domain always come back None and the |
| 96 | +# migration silently no-ops those fields, which defeats the whole purpose of |
| 97 | +# the command. Running online matches what parsedmarc itself does at ingest. |
| 98 | +# |
| 99 | +# The reverse-DNS map is loaded once per batch and passed by reference to every |
| 100 | +# call. Without this, get_service_from_reverse_dns_base_domain would re-fetch |
| 101 | +# the map from GitHub on every IP. |
| 102 | +# |
| 103 | +# No per-IP try/except: exceptions here are programming errors (bad kwargs, |
| 104 | +# missing imports) that we want to see loudly, not mask as {ip: None}. Let the |
| 105 | +# subprocess fail and surface its stderr via the CalledProcessError handler in |
| 106 | +# _lookup_enrichment. |
97 | 107 | _PARSEDMARC_LOOKUP_SCRIPT = r""" |
98 | 108 | import json, sys |
99 | | -from parsedmarc.utils import get_ip_address_info |
| 109 | +from parsedmarc.utils import get_ip_address_info, load_reverse_dns_map |
| 110 | +
|
| 111 | +reverse_dns_map = {} |
| 112 | +load_reverse_dns_map(reverse_dns_map) |
100 | 113 |
|
101 | 114 | ips = json.load(sys.stdin) |
102 | | -out = {ip: get_ip_address_info(ip, offline=True) for ip in ips} |
| 115 | +out = { |
| 116 | + ip: get_ip_address_info(ip, offline=False, reverse_dns_map=reverse_dns_map) |
| 117 | + for ip in ips |
| 118 | +} |
103 | 119 | json.dump(out, sys.stdout) |
104 | 120 | """ |
105 | 121 |
|
|
0 commit comments