Merge pull request #1 from Geocodio/sync/upstream-10.2.0

chore: sync fork with upstream batch-machine 10.2.0

Author: MiniCodeMonkey

CHANGELOG

@@ -1,3 +1,14 @@
+2026-08-22 v10.2.0
+- Fix lat/lon axis flip for shapefile sources with an explicit `srs` tag https://github.com/openaddresses/batch-machine/pull/113
+- Extract nested zip files regardless of the conform `file` filter, cap zip entry size and nesting depth as a zip bomb guard, and skip recursing into a nested zip once the filter is already satisfied https://github.com/openaddresses/batch-machine/pull/112
+- Add KML support to the conform pipeline https://github.com/openaddresses/batch-machine/pull/111
+- Linearize curve geometries (e.g. `MULTISURFACE`) before exporting to WKT, fixing GDB parcel/building sources that error out https://github.com/openaddresses/batch-machine/pull/110
+- Fix `regexp` function's `replace` mode returning the unmatched field unchanged instead of an empty string https://github.com/openaddresses/batch-machine/pull/109
+- Collect the union of feature property keys before writing `geojson_source_to_csv` output, fixing a crash on GeoJSON sources with non-uniform feature properties https://github.com/openaddresses/batch-machine/pull/108
+
+2026-08-13 v10.1.0
+- Support custom HTTP request headers for a source via `request.headers`, including on the file-extension pre-flight request (needed for downloads gated on Referer/etc.)
+
 2026-03-27 v10.0.0
 - Upgrade base Docker image from GDAL 3.7.1 to 3.11.0 https://github.com/openaddresses/batch-machine/pull/99
 - Update SSL CA certificates to fix download failures https://github.com/openaddresses/batch-machine/pull/98

Dockerfile

@@ -1,4 +1,4 @@
-FROM ghcr.io/osgeo/gdal:alpine-normal-3.11.0
+FROM ghcr.io/osgeo/gdal:alpine-normal-3.11.0@sha256:edf2793e0f1ceb74ab12a1d85bd3404b541a113720fbc1e032613e7df2774f7c
 
 RUN apk add --no-cache nodejs yarn git python3-dev py3-pip \
     make sqlite-dev zlib-dev geos-dev \

README.md

@@ -63,7 +63,7 @@ Supported layer types are `addresses`, `parcels`, `buildings`, and `centerlines`
 
 Review https://github.com/openaddresses/openaddresses/blob/master/CONTRIBUTING.md for input json syntax.
 
-Supported conform formats include `shapefile`, `geojson`, `csv`, `xml`, `gdb`, and `gpkg`.
+Supported conform formats include `shapefile`, `geojson`, `csv`, `xml`, `gdb`, `gpkg`, and `kml` (2D Point placemarks with simple `ExtendedData` attributes are verified; other geometry types and `Schema`-typed attributes go through the same GDAL driver but are untested; altitude/3D coordinates and KMZ are not supported).
 
 ## Geocodio fork notes
 
@@ -77,3 +77,9 @@ diverge from upstream and must be preserved across upstream merges:
   conformed output row, so named-but-unnumbered properties stay individually
   addressable. It is a geocodio-specific addition and is not part of the
   upstream OpenAddresses schema.
+- **Non-finite geometry is never written out** — ESRI services report a null
+  point geometry as the string `"NaN"`, which reached the output as
+  `"coordinates": [NaN, NaN]`. Invalid GeoJSON per RFC 7946, and unparseable by
+  strict decoders. `openaddr/cache.py` now skips those features on download and
+  `openaddr/conform.py` treats any non-finite WKT coordinate as no geometry,
+  with `allow_nan=False` on the output writer as a backstop.

openaddr/VERSION

@@ -1 +1 @@
-10.0.0
+10.2.0

openaddr/__init__.py

@@ -75,8 +75,10 @@ def cache(source_config, destdir, extras):
         source_urls = [source_urls]
 
     protocol_string = source_config.data_source.get('protocol')
+    request_settings = source_config.data_source.get('request') or {}
+    source_headers = request_settings.get('headers') or {}
 
-    task = DownloadTask.from_protocol_string(protocol_string, source_config)
+    task = DownloadTask.from_protocol_string(protocol_string, source_config, headers=source_headers)
     downloaded_files = task.download(source_urls, workdir, source_config)
 
     # FIXME: I wrote the download stuff to assume multiple files because
@@ -128,6 +130,10 @@ def conform(source_config, destdir, extras, disable_centroids=False):
     if not isinstance(source_urls, list):
         source_urls = [source_urls]
 
+    # source_config.data_source['request'] is intentionally not passed here:
+    # this re-downloads from the OA-owned cache artifact (S3), not the
+    # contributor's original host, so contributor-supplied headers don't
+    # apply.
     task1 = URLDownloadTask(source_config.data_source_name)
     downloaded_path = task1.download(source_urls, workdir, source_config)
     _L.info("Downloaded to %s", downloaded_path)

openaddr/cache.py

@@ -47,6 +47,13 @@ def traverse(item):
     else:
         yield item
 
+def coordinate_is_usable(value):
+    "Test a single coordinate for a finite number; ESRI reports null geometry as the string \"NaN\""
+    try:
+        return math.isfinite(float(value))
+    except (TypeError, ValueError):
+        return False
+
 def request(method, url, **kwargs):
     if urlparse(url).scheme == 'ftp':
         if method != 'GET':
@@ -130,22 +137,23 @@ def __init__(self, source_prefix, params={}, headers={}):
 
 
     @classmethod
-    def from_protocol_string(clz, protocol_string, source_prefix=None):
+    def from_protocol_string(clz, protocol_string, source_prefix=None, headers=None):
+        headers = headers or {}
         if protocol_string.lower() == 'http':
-            return URLDownloadTask(source_prefix)
+            return URLDownloadTask(source_prefix, headers=headers)
         elif protocol_string.lower() == 'file':
-            return URLDownloadTask(source_prefix)
+            return URLDownloadTask(source_prefix, headers=headers)
         elif protocol_string.lower() == 'ftp':
-            return URLDownloadTask(source_prefix)
+            return URLDownloadTask(source_prefix, headers=headers)
         elif protocol_string.lower() == 'esri':
-            return EsriRestDownloadTask(source_prefix)
+            return EsriRestDownloadTask(source_prefix, headers=headers)
         else:
             raise KeyError("I don't know how to extract for protocol {}".format(protocol_string))
 
     def download(self, source_urls, workdir, source_config):
         raise NotImplementedError()
 
-def guess_url_file_extension(url):
+def guess_url_file_extension(url, headers=None):
     ''' Get a filename extension for a URL using various hints.
     '''
     scheme, _, path, _, query, _ = urlparse(url)
@@ -172,7 +180,7 @@ def guess_url_file_extension(url):
         # Get a dictionary of headers and a few bytes of content from the URL.
         #
         if scheme in ('http', 'https'):
-            response = request('GET', url, stream=True)
+            response = request('GET', url, headers=headers or {}, stream=True)
             handle, file = mkstemp()
 
             for chunk in response.iter_content(chunk_size=8192):
@@ -256,7 +264,7 @@ def get_file_path(self, url, dir_path):
             hash = sha1((host + path_base).encode('utf-8'))
             name_base = u'{}-{}'.format(self.source_prefix, hash.hexdigest()[:8])
 
-        path_ext = guess_url_file_extension(url)
+        path_ext = guess_url_file_extension(url, self.headers)
         _L.debug(u'Guessed {}{} for {}'.format(name_base, path_ext, url))
 
         return os.path.join(dir_path, name_base + path_ext)
@@ -391,7 +399,7 @@ def download(self, source_urls, workdir, source_config):
                 _L.debug("File exists %s", file_path)
                 continue
 
-            downloader = EsriDumper(source_url, parent_logger=_L, timeout=300)
+            downloader = EsriDumper(source_url, parent_logger=_L, timeout=300, extra_headers=self.headers)
 
             metadata = downloader.get_metadata()
 
@@ -439,8 +447,8 @@ def download(self, source_urls, workdir, source_config):
 
                         if not geom:
                             raise TypeError("No geometry parsed")
-                        if any((isinstance(g, float) and math.isnan(g)) for g in traverse(geom)):
-                            raise TypeError("Geometry has NaN coordinates")
+                        if any(not coordinate_is_usable(c) for c in traverse(geom.get('coordinates'))):
+                            raise TypeError("Geometry has non-finite coordinates")
 
                         shp = shape(geom)
                         row[GEOM_FIELDNAME] = shp.wkt