Skip to content

Commit 2487810

Browse files
committed
adding testcase
Signed-off-by: Gagan Dhakrey <gagandhakrey@gmail.com>
1 parent 50c71aa commit 2487810

1 file changed

Lines changed: 95 additions & 0 deletions

File tree

tests/io/array_test.py

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,11 @@
66

77
# pyre-strict
88

9+
import gc
910
import io
11+
import sys
1012
import unittest
13+
from collections.abc import Callable
1114
from io import BytesIO
1215

1316
import numpy as np
@@ -256,3 +259,95 @@ def test_load_npy_cpp(self) -> None:
256259
buffer = spdl.io.load_npy(data)
257260
hyp = np.array(buffer, copy=False)
258261
np.testing.assert_array_equal(hyp, ref)
262+
263+
264+
def _reuse_freed_memory(size: int, count: int = 2000) -> list[bytearray]:
265+
"""Allocate over recently freed memory.
266+
267+
If the archive was released, a stale pointer into it reads this pattern
268+
instead of the original data.
269+
"""
270+
return [bytearray(b"\xab" * size) for _ in range(count)]
271+
272+
273+
class TestNpzBufferLifetime(unittest.TestCase):
274+
"""`NpzFile` does not copy the archive.
275+
276+
It holds a raw pointer into the source buffer, and the arrays it returns for
277+
stored (uncompressed) entries are views into the same memory. Both read
278+
freed memory unless the source buffer is kept alive.
279+
"""
280+
281+
def test_load_npz_retains_source(self) -> None:
282+
"""`load_npz` keeps a reference to the source buffer."""
283+
ref = np.arange(10)
284+
data = _dump_npz(x=ref)
285+
286+
num_refs = sys.getrefcount(data)
287+
npz = spdl.io.load_npz(data)
288+
289+
self.assertGreater(
290+
sys.getrefcount(data),
291+
num_refs,
292+
"`NpzFile` must keep a reference to the source buffer, "
293+
"as it holds a pointer into it.",
294+
)
295+
np.testing.assert_array_equal(npz["x"], ref)
296+
297+
def test_getitem_retains_source(self) -> None:
298+
"""Arrays of stored entries keep the source buffer alive.
299+
300+
Such an array is a view into the archive, so it can outlive the
301+
`NpzFile` it was retrieved from.
302+
"""
303+
ref = np.arange(10)
304+
data = _dump_npz(x=ref)
305+
306+
num_refs = sys.getrefcount(data)
307+
# The `NpzFile` is released as soon as the entry is retrieved.
308+
arr = spdl.io.load_npz(data)["x"]
309+
gc.collect()
310+
311+
self.assertGreater(
312+
sys.getrefcount(data),
313+
num_refs,
314+
"The array must keep a reference to the source buffer, "
315+
"as it is a view into it.",
316+
)
317+
np.testing.assert_array_equal(arr, ref)
318+
319+
@parameterized.expand(
320+
[
321+
("stored", _dump_npz),
322+
("deflated", _dump_npz_compressed),
323+
]
324+
)
325+
def test_load_npz_source_may_be_temporary(
326+
self, _: str, dump: Callable[..., bytes]
327+
) -> None:
328+
"""Entries are readable when the caller does not hold the source."""
329+
ref = np.arange(1000, dtype=np.int64)
330+
size = len(dump(x=ref))
331+
332+
# The source is a temporary, so it is released when `load_npz` returns
333+
# unless `NpzFile` retains it.
334+
npz = spdl.io.load_npz(dump(x=ref))
335+
gc.collect()
336+
clobber = _reuse_freed_memory(size)
337+
338+
np.testing.assert_array_equal(npz["x"], ref)
339+
340+
del clobber
341+
342+
def test_array_outlives_npz_file(self) -> None:
343+
"""A stored entry stays valid after the `NpzFile` is released."""
344+
ref = np.arange(1000, dtype=np.int64)
345+
size = len(_dump_npz(x=ref))
346+
347+
arr = spdl.io.load_npz(_dump_npz(x=ref))["x"]
348+
gc.collect()
349+
clobber = _reuse_freed_memory(size)
350+
351+
np.testing.assert_array_equal(arr, ref)
352+
353+
del clobber

0 commit comments

Comments
 (0)