Skip to content

Commit 7a5d7f1

Browse files
[data] revert(test_gate): remove test gate from test_write_delta.py (#65588)
## Description In the tests "test_write_delta.py", we were doing a round trip assertion on write_delta -> read_delta. These assertions were originally failing in CI with pyarrow version 17 so we added assertions to skip certain tests if pyarrow's available version was < "19.0.0". When attempting to fix the underlying issue, it is not getting reproduced locally using pyarrow v17 or above (supported versions) on both x86_64 and arm64 machines using python 3.10/3.12; Hence I'm raising this pull request to revert the test gates. Script used for local repro - ```python3 import os import sys import tempfile import deltalake # noqa: E402 import pyarrow # noqa: E402 import ray # noqa: E402 # One typed partition value and one null, which Delta stores under the # sentinel directory name "year=__HIVE_DEFAULT_PARTITION__". ROWS = [{"id": 1, "year": 2024}, {"id": 2, "year": None}] def main() -> int: ray.data.DataContext.get_current().enable_progress_bars = False print( f"ray {ray.__version__} | " f"pyarrow {pyarrow.__version__} | " f"deltalake {deltalake.__version__}" ) with tempfile.TemporaryDirectory() as tmp_dir: table_uri = os.path.join(tmp_dir, "delta_table") ray.data.from_items(ROWS).write_delta(table_uri, partition_by=["year"]) print("on disk: ", sorted(os.listdir(table_uri))) # take_all() gives no ordering guarantee across blocks; sort to compare. got = sorted(ray.data.read_delta(table_uri).take_all(), key=lambda r: r["id"]) print("read back:", got) print("expected: ", ROWS) if got != ROWS: print( "FAIL: partition values did not survive the round trip", file=sys.stderr, ) return 1 print("OK") return 0 if __name__ == "__main__": sys.exit(main()) ``` which generates the following output - ```log ray 3.0.0.dev0 | pyarrow 17.0.0 | deltalake 1.6.2 2026-08-20 15:34:17,187 INFO worker.py:2015 -- Started a local Ray instance. View the dashboard at http://127.0.0.1:8265 2026-08-20 15:34:18,148 WARNING __init__.py:28 -- Progress bars disabled. To enable, set `ray.data.DataContext.get_current().enable_progress_bars = True`. on disk: ['_delta_log', 'year=2024', 'year=__HIVE_DEFAULT_PARTITION__'] read back: [{'id': 1, 'year': 2024}, {'id': 2, 'year': None}] expected: [{'id': 1, 'year': 2024}, {'id': 2, 'year': None}] OK ``` ## Related issues Relates to #65143 Signed-off-by: Dhruvil Shah <dhruvil.shah@anyscale.com>
1 parent 6e0abdd commit 7a5d7f1

1 file changed

Lines changed: 0 additions & 37 deletions

File tree

python/ray/data/tests/datasource/test_write_delta.py

Lines changed: 0 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -41,31 +41,6 @@
4141
),
4242
]
4343

44-
# Whether ``read_delta`` gives partition columns back with their declared types.
45-
#
46-
# A Delta reader takes a partitioned column's value from metadata rather than
47-
# from the Parquet file (the writer omits it there), so restoring the declared
48-
# type is the reader's job. ``read_delta`` delegates that to PyArrow -- see the
49-
# comment on ``partition_columns=[]`` in ``ParquetDatasource.from_pyarrow_dataset``
50-
# -- and on pyarrow<19 it doesn't happen: values come back as the raw Hive path
51-
# strings instead, so ``2024`` reads as ``"2024"`` and a null partition as the
52-
# literal ``"__HIVE_DEFAULT_PARTITION__"``.
53-
#
54-
# That's a gap in the *read* path, which this file doesn't own; the writes
55-
# themselves are fine on every version. Rather than weaken the assertions
56-
# everywhere, the round-trip *value* checks below skip on old pyarrow while the
57-
# on-disk layout checks keep running. 19 is the lowest version observed to
58-
# round-trip correctly (17 fails, 19 and 20 pass); 18 is simply untested.
59-
_PARTITION_VALUES_ROUND_TRIP = _pa_version is not None and _pa_version >= parse_version(
60-
"19.0.0"
61-
)
62-
_NO_TYPED_PARTITION_VALUES = (
63-
"read_delta returns partition values as untyped Hive path strings on pyarrow<19"
64-
)
65-
_skip_without_typed_partition_values = pytest.mark.skipif(
66-
not _PARTITION_VALUES_ROUND_TRIP, reason=_NO_TYPED_PARTITION_VALUES
67-
)
68-
6944

7045
@pytest.fixture
7146
def temp_delta_path(tmp_path):
@@ -197,13 +172,10 @@ def test_single_column_partition(temp_delta_path):
197172
ray.data.from_items(rows).write_delta(temp_delta_path, partition_by=["year"])
198173
assert set(os.listdir(temp_delta_path)) >= {"year=2024", "year=2025"}
199174

200-
if not _PARTITION_VALUES_ROUND_TRIP:
201-
pytest.skip(_NO_TYPED_PARTITION_VALUES)
202175
out = sorted(_read_all(temp_delta_path), key=lambda r: r["id"])
203176
assert out == rows
204177

205178

206-
@_skip_without_typed_partition_values
207179
def test_multi_column_partition(temp_delta_path):
208180
rows = [
209181
{"year": 2024, "month": 1, "id": 1},
@@ -226,8 +198,6 @@ def test_null_partition_value_round_trips_as_none(temp_delta_path):
226198
ray.data.from_items(rows).write_delta(temp_delta_path, partition_by=["year"])
227199
assert "year=__HIVE_DEFAULT_PARTITION__" in os.listdir(temp_delta_path)
228200

229-
if not _PARTITION_VALUES_ROUND_TRIP:
230-
pytest.skip(_NO_TYPED_PARTITION_VALUES)
231201
out = sorted(_read_all(temp_delta_path), key=lambda r: r["id"])
232202
assert out == rows
233203

@@ -270,8 +240,6 @@ def test_append_without_partition_by_inherits_existing_partitioning(temp_delta_p
270240

271241
assert "year=2025" in os.listdir(temp_delta_path)
272242

273-
if not _PARTITION_VALUES_ROUND_TRIP:
274-
pytest.skip(_NO_TYPED_PARTITION_VALUES)
275243
out = sorted(_read_all(temp_delta_path), key=lambda r: r["id"])
276244
assert out == [{"year": 2024, "id": 1}, {"year": 2025, "id": 2}]
277245

@@ -330,12 +298,9 @@ def test_partition_by_mismatch_rejected(
330298

331299
# Only the baseline tables that are themselves partitioned depend on
332300
# partition values surviving the read; the unpartitioned case is unaffected.
333-
if create_partition_by and not _PARTITION_VALUES_ROUND_TRIP:
334-
pytest.skip(_NO_TYPED_PARTITION_VALUES)
335301
assert _read_all(temp_delta_path) == first
336302

337303

338-
@_skip_without_typed_partition_values
339304
def test_partition_by_matching_existing_is_allowed(temp_delta_path):
340305
"""Passing exactly the table's own partition columns is fine."""
341306
ray.data.from_items([{"year": 2024, "id": 1}]).write_delta(
@@ -368,8 +333,6 @@ def test_overwrite_without_partition_by_inherits_existing_partitioning(
368333

369334
assert "year=2025" in os.listdir(temp_delta_path)
370335

371-
if not _PARTITION_VALUES_ROUND_TRIP:
372-
pytest.skip(_NO_TYPED_PARTITION_VALUES)
373336
assert _read_all(temp_delta_path) == [{"year": 2025, "id": 2}]
374337

375338

0 commit comments

Comments
 (0)