|
6 | 6 |
|
7 | 7 | # pyre-strict |
8 | 8 |
|
| 9 | +import gc |
9 | 10 | import io |
| 11 | +import sys |
10 | 12 | import unittest |
| 13 | +from collections.abc import Callable |
11 | 14 | from io import BytesIO |
12 | 15 |
|
13 | 16 | import numpy as np |
@@ -256,3 +259,95 @@ def test_load_npy_cpp(self) -> None: |
256 | 259 | buffer = spdl.io.load_npy(data) |
257 | 260 | hyp = np.array(buffer, copy=False) |
258 | 261 | np.testing.assert_array_equal(hyp, ref) |
| 262 | + |
| 263 | + |
| 264 | +def _reuse_freed_memory(size: int, count: int = 2000) -> list[bytearray]: |
| 265 | + """Allocate over recently freed memory. |
| 266 | +
|
| 267 | + If the archive was released, a stale pointer into it reads this pattern |
| 268 | + instead of the original data. |
| 269 | + """ |
| 270 | + return [bytearray(b"\xab" * size) for _ in range(count)] |
| 271 | + |
| 272 | + |
| 273 | +class TestNpzBufferLifetime(unittest.TestCase): |
| 274 | + """`NpzFile` does not copy the archive. |
| 275 | +
|
| 276 | + It holds a raw pointer into the source buffer, and the arrays it returns for |
| 277 | + stored (uncompressed) entries are views into the same memory. Both read |
| 278 | + freed memory unless the source buffer is kept alive. |
| 279 | + """ |
| 280 | + |
| 281 | + def test_load_npz_retains_source(self) -> None: |
| 282 | + """`load_npz` keeps a reference to the source buffer.""" |
| 283 | + ref = np.arange(10) |
| 284 | + data = _dump_npz(x=ref) |
| 285 | + |
| 286 | + num_refs = sys.getrefcount(data) |
| 287 | + npz = spdl.io.load_npz(data) |
| 288 | + |
| 289 | + self.assertGreater( |
| 290 | + sys.getrefcount(data), |
| 291 | + num_refs, |
| 292 | + "`NpzFile` must keep a reference to the source buffer, " |
| 293 | + "as it holds a pointer into it.", |
| 294 | + ) |
| 295 | + np.testing.assert_array_equal(npz["x"], ref) |
| 296 | + |
| 297 | + def test_getitem_retains_source(self) -> None: |
| 298 | + """Arrays of stored entries keep the source buffer alive. |
| 299 | +
|
| 300 | + Such an array is a view into the archive, so it can outlive the |
| 301 | + `NpzFile` it was retrieved from. |
| 302 | + """ |
| 303 | + ref = np.arange(10) |
| 304 | + data = _dump_npz(x=ref) |
| 305 | + |
| 306 | + num_refs = sys.getrefcount(data) |
| 307 | + # The `NpzFile` is released as soon as the entry is retrieved. |
| 308 | + arr = spdl.io.load_npz(data)["x"] |
| 309 | + gc.collect() |
| 310 | + |
| 311 | + self.assertGreater( |
| 312 | + sys.getrefcount(data), |
| 313 | + num_refs, |
| 314 | + "The array must keep a reference to the source buffer, " |
| 315 | + "as it is a view into it.", |
| 316 | + ) |
| 317 | + np.testing.assert_array_equal(arr, ref) |
| 318 | + |
| 319 | + @parameterized.expand( |
| 320 | + [ |
| 321 | + ("stored", _dump_npz), |
| 322 | + ("deflated", _dump_npz_compressed), |
| 323 | + ] |
| 324 | + ) |
| 325 | + def test_load_npz_source_may_be_temporary( |
| 326 | + self, _: str, dump: Callable[..., bytes] |
| 327 | + ) -> None: |
| 328 | + """Entries are readable when the caller does not hold the source.""" |
| 329 | + ref = np.arange(1000, dtype=np.int64) |
| 330 | + size = len(dump(x=ref)) |
| 331 | + |
| 332 | + # The source is a temporary, so it is released when `load_npz` returns |
| 333 | + # unless `NpzFile` retains it. |
| 334 | + npz = spdl.io.load_npz(dump(x=ref)) |
| 335 | + gc.collect() |
| 336 | + clobber = _reuse_freed_memory(size) |
| 337 | + |
| 338 | + np.testing.assert_array_equal(npz["x"], ref) |
| 339 | + |
| 340 | + del clobber |
| 341 | + |
| 342 | + def test_array_outlives_npz_file(self) -> None: |
| 343 | + """A stored entry stays valid after the `NpzFile` is released.""" |
| 344 | + ref = np.arange(1000, dtype=np.int64) |
| 345 | + size = len(_dump_npz(x=ref)) |
| 346 | + |
| 347 | + arr = spdl.io.load_npz(_dump_npz(x=ref))["x"] |
| 348 | + gc.collect() |
| 349 | + clobber = _reuse_freed_memory(size) |
| 350 | + |
| 351 | + np.testing.assert_array_equal(arr, ref) |
| 352 | + |
| 353 | + del clobber |
0 commit comments