Merge pull request #373 from sentinel-hub/develop

Release v1.7.15

Author: mlubej

.github/workflows/ci_action.yml

@@ -60,9 +60,9 @@ jobs:
     strategy:
       matrix:
         python-version:
-          - "3.8"
           - "3.10"
           - "3.11"
+          - "3.12"
         include:
           # A flag marks whether full or partial tests should be run
           # We don't run integration tests on pull requests from outside repos, because they don't have secrets
@@ -90,10 +90,8 @@ jobs:
           pip install -e .[DEV,ML]
           pip install gdal==$(gdal-config --version)
 
-      - name: Set up local cluster # we need to install async-timeout until ray 2.9.0 fixes the issue
-        run: |
-          pip install async-timeout
-          ray start --head
+      - name: Set up local cluster
+        run: ray start --head
 
       - name: Run fast tests
         if: ${{ !matrix.full_test_suite }}
@@ -127,7 +125,7 @@ jobs:
       - name: Mirror + trigger CI
         uses: SvanBoxel/gitlab-mirror-and-ci-action@master
         with:
-          args: "https://hello.planet.com/code/eo/code/eo-grow"
+          args: "https://hello.planet.com/code/solution-enablement/eo/code/eo-grow"
         env:
           FOLLOW_TAGS: "true"
           GITLAB_HOSTNAME: "hello.planet.com/code"

CHANGELOG.md

@@ -1,3 +1,9 @@
+## [Version 1.7.15] - 2025-09-17
+
+- Switch to Batch Process API V2 in BatchDownloadPipeline
+- Remove BatchAreaManager
+- Add config validation tests
+
 ## [Version 1.7.14] - 2024-11-08
 
 - Improve batch download retries in case of partial downloads.

docs/source/high-level-overview.md

@@ -108,14 +108,6 @@ The `UtmZoneAreaManager` is probably the most commonly used area manager and mos
 
 For users which have a very specific way of splitting the AOI in mind, we provide the `CustomGridAreaManager`, which accepts a grid file of an already split AOI. The user only needs to provide the grid file folder key and name, along with the `name_column` parameter, which points to the column containing the patch names to be used. The folder key by default points to the `input_data` location, but could be any other location defined by the storage structure. Read the [API docs](reference/eogrow.core.area.custom_grid.html#eogrow.core.area.custom_grid.CustomGridAreaManager.Schema) on the `CustomGridAreaManager` for more info.
 
-#### Batch Area Manager
-
-For users working with [Sentinel Hub Batch API](https://docs.sentinel-hub.com/api/latest/api/batch/), we have prepared the `BatchAreaManager`, which splits the area according to [Sentinel Hub tiling grids](https://docs.sentinel-hub.com/api/latest/api/batch/#tiling-grids). This area manager is meant for larger projects focusing on larger areas.
-
-The interface of the `BatchAreaManager` relies heavily on the predefined configuration options defined for the Batch API, so be sure to provide sensible values for the parameters. For example, the `tiling_grid_id` and `resolution` parameters should correspond to values stated in the [docs](https://docs.sentinel-hub.com/api/latest/api/batch/#tiling-grids).
-
-For existing projects involving Batch API, it is possible to provide the `batch_id` parameter, which will search for existing grids corresponding to the batch request. If the `batch_id` is not provided (this is by default), the `BatchAreaManager` will generate a new batch job with the given parameters. Read the [API docs](reference/eogrow.core.area.batch.html#eogrow.core.area.batch.BatchAreaManager.Schema) on the `BatchAreaManager` for more info.
-
 ### Logging Manager
 
 The logging manager ensures that logging handlers are set up and configured correctly. It allows adjusting which packages to log to files, which to stdout, and which to ignore. It is unlikely you'll ever need to access any of it's methods directly. Use the standard `LOGGER = logging.getLogger(__name__)` for logging.

eogrow/__init__.py

@@ -1,3 +1,3 @@
 """The main module of the eo-grow package."""
 
-__version__ = "1.7.14"
+__version__ = "1.7.15"

eogrow/core/area/__init__.py

@@ -1,5 +1,4 @@
 """A submodule with for implementations of area managers."""
 
-from .batch import BatchAreaManager
 from .custom_grid import CustomGridAreaManager
 from .utm import UtmZoneAreaManager

eogrow/core/area/base.py

@@ -66,10 +66,12 @@ def get_grid(self, filtered: bool = True) -> dict[CRS, gpd.GeoDataFrame]:
         grid_path = fs.path.combine(self.storage.get_cache_folder(), self.get_grid_cache_filename())
 
         if self.storage.filesystem.exists(grid_path):
-            grid = self._load_grid(grid_path)
+            LOGGER.info("Loading grid from %s", grid_path)
+            grid = load_grid(grid_path, self.storage)
         else:
             grid = self._create_grid()
-            self._save_grid(grid, grid_path)
+            LOGGER.info("Saving grid to %s", grid_path)
+            save_grid(grid, grid_path, self.storage)
 
         if filtered and self.config.patch_names is not None:
             folder_path = self.storage.get_folder(self.config.patch_names.input_folder_key)
@@ -97,32 +99,6 @@ def _create_grid(self) -> dict[CRS, gpd.GeoDataFrame]:
         definitions. EOPatch names are stored in a column with identifier `self.NAME_COLUMN`.
         """
 
-    def _load_grid(self, grid_path: str) -> dict[CRS, gpd.GeoDataFrame]:
-        """A method that loads the bounding box grid from the cache folder."""
-        LOGGER.info("Loading grid from %s", grid_path)
-
-        grid = {}
-        with LocalFile(grid_path, mode="r", filesystem=self.storage.filesystem) as local_file:
-            for crs_layer in fiona.listlayers(local_file.path):
-                data = gpd.read_file(local_file.path, layer=crs_layer, engine=self.storage.config.geopandas_backend)
-                grid[CRS(data.crs)] = data
-
-        return grid
-
-    def _save_grid(self, grid: dict[CRS, gpd.GeoDataFrame], grid_path: str) -> None:
-        """A method that saves the bounding box grid to the cache folder."""
-        LOGGER.info("Saving grid to %s", grid_path)
-
-        with LocalFile(grid_path, mode="w", filesystem=self.storage.filesystem) as local_file:
-            for crs_grid in grid.values():
-                crs_grid.to_file(
-                    local_file.path,
-                    driver="GPKG",
-                    encoding="utf-8",
-                    layer=f"Grid EPSG:{crs_grid.crs.to_epsg()}",
-                    engine=self.storage.config.geopandas_backend,
-                )
-
     @abstractmethod
     def get_grid_cache_filename(self) -> str:
         """Provides a filename that is used for caching the grid, including the file extensions (likely .gpkg).
@@ -149,3 +125,27 @@ def get_geometry_from_file(
         area_df = gpd.read_file(local_file.path, engine=geopandas_engine)
         area_shape = shapely.ops.unary_union(area_df.geometry)
         return Geometry(area_shape, CRS(area_df.crs))
+
+
+def load_grid(grid_path: str, storage: StorageManager) -> dict[CRS, gpd.GeoDataFrame]:
+    """A method that loads the bounding box grid from the cache folder."""
+    grid = {}
+    with LocalFile(grid_path, mode="r", filesystem=storage.filesystem) as local_file:
+        for crs_layer in fiona.listlayers(local_file.path):
+            data = gpd.read_file(local_file.path, layer=crs_layer, engine=storage.config.geopandas_backend)
+            grid[CRS(data.crs)] = data
+
+    return grid
+
+
+def save_grid(grid: dict[CRS, gpd.GeoDataFrame], grid_path: str, storage: StorageManager) -> None:
+    """A method that saves the bounding box grid to the cache folder."""
+    with LocalFile(grid_path, mode="w", filesystem=storage.filesystem) as local_file:
+        for crs_grid in grid.values():
+            crs_grid.to_file(
+                local_file.path,
+                driver="GPKG",
+                encoding="utf-8",
+                layer=f"Grid EPSG:{crs_grid.crs.to_epsg()}",
+                engine=storage.config.geopandas_backend,
+            )

eogrow/core/area/batch.py

@@ -12,7 +12,7 @@
 from sentinelhub import CRS, Geometry
 
 from ...utils.vector import concat_gdf
-from .base import BaseAreaManager
+from .base import BaseAreaManager, load_grid
 
 LOGGER = logging.getLogger(__name__)
 
@@ -35,7 +35,7 @@ class Schema(BaseAreaManager.Schema):
 
     def _create_grid(self) -> dict[CRS, gpd.GeoDataFrame]:
         grid_path = fs.path.combine(self.storage.get_folder(self.config.grid_folder_key), self.config.grid_filename)
-        grid = self._load_grid(grid_path)
+        grid = load_grid(grid_path, self.storage)
 
         for crs, crs_gdf in grid.items():
             # Correct name of eopatch-name-column, drop all non-significant ones

eogrow/core/area/custom_grid.py

@@ -51,32 +51,14 @@ def _create_grid(self) -> dict[CRS, GeoDataFrame]:
         """Uses UtmZoneSplitter to create a grid"""
         area_geometry = self.get_area_geometry()
         LOGGER.info("Splitting area geometry into UTM zone grid")
-        splitter = UtmZoneSplitter(
-            [area_geometry.geometry],
-            crs=area_geometry.crs,
-            bbox_size=(self.config.patch.size_x, self.config.patch.size_y),
-            offset=(self.config.offset_x, self.config.offset_y),
-        )
-
-        bbox_list, info_list = splitter.get_bbox_list(), splitter.get_info_list()
-
-        absolute_buffer = self.config.patch.buffer_x, self.config.patch.buffer_y
-        if absolute_buffer != (0, 0):
-            bbox_list = [bbox.buffer(absolute_buffer, relative=False) for bbox in bbox_list]
-
-        crs_to_patches = defaultdict(list)
-        zfill_length = len(str(len(bbox_list) - 1))
-        for i, (bbox, info) in enumerate(zip(bbox_list, info_list)):
-            i_x, i_y = info["index_x"], info["index_y"]
-            name = f"eopatch-id-{i:0{zfill_length}}-col-{i_x}-row-{i_y}"
-            crs_to_patches[bbox.crs].append((name, bbox.geometry))
 
-        grid = {}
-        for crs, named_bbox_geoms in crs_to_patches.items():
-            names, geoms = zip(*named_bbox_geoms)
-            grid[crs] = gpd.GeoDataFrame({self.NAME_COLUMN: names}, geometry=list(geoms), crs=crs.pyproj_crs())
-
-        return grid
+        return create_utm_zone_grid(
+            area_geometry,
+            self.NAME_COLUMN,
+            (self.config.patch.size_x, self.config.patch.size_y),
+            (self.config.offset_x, self.config.offset_y),
+            (self.config.patch.buffer_x, self.config.patch.buffer_y),
+        )
 
     def get_grid_cache_filename(self) -> str:
         input_filename = fs.path.basename(self.config.geometry_filename)
@@ -94,3 +76,28 @@ def get_grid_cache_filename(self) -> str:
         params = [str(param) for param in raw_params]
 
         return f"{self.__class__.__name__}_{'_'.join(params)}.gpkg"
+
+
+def create_utm_zone_grid(
+    geometry: Geometry,
+    name_column: str,
+    bbox_size: tuple[int, int],
+    bbox_offset: tuple[float, float],
+    bbox_buffer: tuple[float, float],
+) -> dict[CRS, GeoDataFrame]:
+    """Creates a grid of bounding boxes covering the given area geometry."""
+    splitter = UtmZoneSplitter([geometry.geometry], crs=geometry.crs, bbox_size=bbox_size, offset=bbox_offset)
+    bbox_list, info_list = splitter.get_bbox_list(), splitter.get_info_list()
+    if bbox_buffer != (0, 0):
+        bbox_list = [bbox.buffer(bbox_buffer, relative=False) for bbox in bbox_list]
+
+    tiles_dict: dict[CRS, list[dict]] = defaultdict(list)
+    zfill_length = len(str(len(bbox_list) - 1))
+    for i, (bbox, info) in enumerate(zip(bbox_list, info_list)):
+        i_x, i_y = info["index_x"], info["index_y"]
+        name = f"eopatch-id-{i:0{zfill_length}}-col-{i_x}-row-{i_y}"
+        tiles_dict[bbox.crs].append({"id": i, name_column: name, "geometry": bbox.geometry})
+
+    return {
+        crs: gpd.GeoDataFrame(tiles, geometry="geometry", crs=crs.pyproj_crs()) for crs, tiles in tiles_dict.items()
+    }

eogrow/core/area/utm.py

@@ -106,7 +106,7 @@ def get_tile_paths(self) -> dict[str, list[str]]:
         """
         tiff_paths = self._get_tiff_paths()
 
-        folder_to_filename_map = defaultdict(list)
+        folder_to_filename_map: dict[str, list[str]] = defaultdict(list)
         for folder, filename in map(fs.path.split, tiff_paths):
             folder_to_filename_map[folder].append(filename)