Merge pull request #911 from MiraGeoscience/GEOPY-2893

GEOPY-2893: h5 compression level seems ignore (from mira-omf)

Author: sebhmg

geoh5py/io/h5_writer.py

@@ -354,7 +354,9 @@ def update_field(
                 "vertices",
                 "z_cell_delimiters",
             ]:
-                H5Writer.write_array_attribute(h5file, entity, attribute, **kwargs)
+                H5Writer.write_array_attribute(
+                    h5file, entity, attribute, compression=compression, **kwargs
+                )
             elif attribute == "property_groups":
                 H5Writer.write_property_groups(h5file, entity)
             elif attribute == "color_map":
@@ -588,14 +590,21 @@ def write_visible(
 
     @staticmethod
     def write_array_attribute(
-        file: str | h5py.File, entity, attribute, values=None, **kwargs
+        file: str | h5py.File,
+        entity,
+        attribute,
+        values=None,
+        compression: int = 9,
+        **kwargs,
     ) -> None:
         """
         Add :obj:`~geoh5py.objects.object_base.ObjectBase.surveys` of an object.
 
         :param file: Name or handle to a geoh5 file.
         :param entity: Target entity.
         :param attribute: Name of the attribute to be written to geoh5
+        :param values: Values to be written to geoh5
+        :param compression: Compression level `compression_opts` of the h5 file.
         """
         with fetch_h5_handle(file, mode="r+") as h5file:
             entity_handle = H5Writer.fetch_handle(h5file, entity)
@@ -624,7 +633,7 @@ def write_array_attribute(
                     KEY_MAP[attribute],
                     data=values,
                     compression="gzip",
-                    compression_opts=9,
+                    compression_opts=compression,
                     **kwargs,
                 )
 

geoh5py/objects/maxwell_plate.py

@@ -256,18 +256,23 @@ def geometry(self, value: PlateGeometry):
 
     @classmethod
     def create(
-        cls, workspace, geometry: PlateGeometry | None = None, **kwargs
+        cls,
+        workspace,
+        compression: int = 5,
+        geometry: PlateGeometry | None = None,
+        **kwargs,
     ) -> MaxwellPlate:
         """
         Function to create an entity.
 
         :param workspace: Workspace to be added to.
+        :param compression: Compression parameter for the Maxwell Plate.
         :param geometry: Geometry parameters for the Maxwell Plate.
         :param kwargs: List of keyword arguments defining the properties of a class.
 
         :return entity: Registered Entity to the workspace.
         """
-        plate = super().create(workspace, **kwargs)
+        plate = super().create(workspace, compression=compression, **kwargs)
 
         if geometry is not None:
             plate.geometry = geometry

geoh5py/shared/entity.py

@@ -227,11 +227,12 @@ def coordinate_reference_system(self, value: dict):
         self.metadata = {"Coordinate Reference System": coordinate_reference_system}
 
     @classmethod
-    def create(cls, workspace, **kwargs):
+    def create(cls, workspace, compression: int = 5, **kwargs):
         """
         Function to create an entity.
 
         :param workspace: Workspace to be added to.
+        :param compression: Compression to be applied to the entity.
         :param kwargs: List of keyword arguments defining the properties of a class.
 
         :return entity: Registered Entity to the workspace.
@@ -242,9 +243,7 @@ def create(cls, workspace, **kwargs):
             else {}
         )
         new_object = workspace.create_entity(
-            cls,
-            entity=kwargs,
-            entity_type=entity_type_kwargs,
+            cls, entity=kwargs, entity_type=entity_type_kwargs, compression=compression
         )
         return new_object
 

geoh5py/shared/utils.py

@@ -46,7 +46,8 @@
     from .entity import Entity
     from .entity_container import EntityContainer
 
-DEFAULT_PAGE_BUF_SIZE = 16777216
+DEFAULT_PAGE_SIZE = 65536  # Default page_size for H5, in bytes
+
 INV_KEY_MAP = {
     "Allow delete": "allow_delete",
     "Allow delete contents": "allow_delete_content",

geoh5py/workspace/workspace.py

@@ -70,7 +70,7 @@
 from geoh5py.shared.entity_type import EntityType
 from geoh5py.shared.exceptions import Geoh5FileClosedError
 from geoh5py.shared.utils import (
-    DEFAULT_PAGE_BUF_SIZE,
+    DEFAULT_PAGE_SIZE,
     ClassIdentifierEnum,
     as_str_if_utf8_bytes,
     clear_array_attributes,
@@ -115,6 +115,7 @@ class Workspace(AbstractContextManager):
     :param name: Name of the project.
     :param repack: Repack the *geoh5* file after closing.
     :param version: Version of the project.
+    :param page_size: Page size of the h5 file, in bytes.
     """
 
     _active_ref: ClassVar[ReferenceType[Workspace]] | type(None) = type(None)  # type: ignore
@@ -136,6 +137,7 @@ def __init__(
         name: str = "GEOSCIENCE",
         repack: bool = False,
         version: float = 2.1,
+        page_size: int = DEFAULT_PAGE_SIZE,
     ):
         self._root: RootGroup
         self._data: dict[uuid.UUID, ReferenceType[data.Data]] = {}
@@ -152,7 +154,7 @@ def __init__(
         self._repack: bool = repack
         self._types: dict[uuid.UUID, ReferenceType[EntityType]] = {}
         self._version: float = version
-
+        self._page_size: int = validate_page_size(page_size)
         self._h5file = self.validate_h5file_input(h5file)
 
         self.open(mode=mode)
@@ -1298,18 +1300,33 @@ def _open_h5(self, mode) -> h5py.File:
     def _create_h5(self) -> h5py.File:
         """
         Generate a new geoh5 file with core structure.
+
+        Default mode for ANALYST uses:
+            - Page size (fs_page_size) of 65536 bytes.
+            - Page buffer (page_buf_size) is able to hold 256 pages.
+            - Library version (libver) fixed with (lower, upper) bound.
         """
         self._geoh5 = h5py.File(
             self.h5file,
             "x",
             fs_strategy="page",
-            page_buf_size=DEFAULT_PAGE_BUF_SIZE,
+            page_buf_size=self._page_size * 256,
+            fs_page_size=self._page_size,
             libver=("v110", "v114"),
         )
         H5Writer.init_geoh5(self._geoh5, self)
 
         return self._geoh5
 
+    @property
+    def page_size(self) -> int:
+        """
+        HDF5 page size.
+
+        Must be a multiple of 2, greater than or equal 512.
+        """
+        return self._page_size
+
     def promote(self, value: Any) -> Any:
         """
         Promote uuid to entity as it is or in a nested structure.
@@ -1605,3 +1622,20 @@ def active_workspace(workspace: Workspace):
     previous_active = previous_active_ref()
     if previous_active is not None:
         previous_active.activate()  # pylint: disable=no-member
+
+
+def validate_page_size(value: int) -> int:
+    """
+    Check if a page size is valid value. Raise an error if not valid,
+    else return the value as-is.
+
+    :param value: A positive integer multiple of 2, >=512.
+    :return: Page size value, same as :param value:
+    """
+    if not isinstance(value, int):
+        raise TypeError("Page size must be an integer.")
+
+    if value < 512 or value % 2 != 0:
+        raise ValueError("Page size must be an integer multiple of 2, and >=512.")
+
+    return value

tests/workspace_test.py

@@ -20,6 +20,8 @@
 from __future__ import annotations
 
 import io
+import operator
+import os
 from pathlib import Path
 
 import numpy as np
@@ -219,3 +221,49 @@ def test_network_drive_warning(tmp_path):
     network_drive.mkdir()
     with pytest.warns(match="Opening workspace with write access in a network drive"):
         _ = Workspace(network_drive / "test.geoh5")
+
+
+def test_page_size(tmp_path):
+
+    with pytest.raises(TypeError, match="Page size must be an integer"):
+        Workspace.create(tmp_path / f"{__name__}.geoh5", page_size="abc")
+
+    with pytest.raises(
+        ValueError, match="Page size must be an integer multiple of 2, and"
+    ):
+        Workspace.create(tmp_path / f"{__name__}.geoh5", page_size=128)
+
+    with pytest.raises(
+        ValueError, match="Page size must be an integer multiple of 2, and"
+    ):
+        Workspace.create(tmp_path / f"{__name__}.geoh5", page_size=601)
+
+    with Workspace.create(tmp_path / f"{__name__}.geoh5", page_size=512) as workspace:
+        assert workspace.page_size == 512
+
+
+@pytest.mark.parametrize(
+    "n_values, compressions, expected_opt",
+    [
+        (100, (1, 5), operator.eq),  # No compression since < page_size
+        (10000, (1, 5), operator.gt),
+    ],
+)
+def test_compression_below_page(
+    tmp_path, n_values: int, compressions: tuple, expected_opt
+):
+    vertices = np.c_[np.arange(n_values), np.arange(n_values), np.arange(n_values)]
+    values = np.arange(n_values)
+
+    with Workspace.create(tmp_path / "test_low.geoh5", page_size=512) as ws:
+        pt = Points.create(ws, vertices=vertices, compression=compressions[0])
+        pt.add_data({"values": {"values": values}}, compression=compressions[0])
+
+    with Workspace.create(tmp_path / "test_high.geoh5", page_size=512) as ws:
+        pt = Points.create(ws, vertices=vertices, compression=compressions[1])
+        pt.add_data({"values": {"values": values}}, compression=compressions[1])
+
+    size_low_comp = os.stat(tmp_path / "test_low.geoh5").st_size
+    size_med_comp = os.stat(tmp_path / "test_high.geoh5").st_size
+
+    assert expected_opt(size_low_comp, size_med_comp)