diff --git a/changes/466.bugfix.md b/changes/466.bugfix.md new file mode 100644 index 0000000000..6faffe2d8a --- /dev/null +++ b/changes/466.bugfix.md @@ -0,0 +1 @@ +`LocalStore.get`, `get_sync` and `get_partial_values` now return `None` on Windows when the key names a directory, instead of raising `PermissionError`. A `PermissionError` on a file still raises. `get_partial_values` now also returns `None` for a missing key. diff --git a/src/zarr/storage/_local.py b/src/zarr/storage/_local.py index 20d4b102b1..10465ca989 100644 --- a/src/zarr/storage/_local.py +++ b/src/zarr/storage/_local.py @@ -45,6 +45,25 @@ def _get(path: Path, prototype: BufferPrototype, byte_range: ByteRequest | None) return prototype.buffer.from_bytes(f.read()) +def _get_or_none( + path: Path, prototype: BufferPrototype, byte_range: ByteRequest | None +) -> Buffer | None: + """Read the file at `path`, or return `None` when no file is there. + + Opening a directory raises `IsADirectoryError` on POSIX and `PermissionError` on Windows. + A `PermissionError` counts as a missing key only when `path` is a directory, so a file the + process may not read still raises. + """ + try: + return _get(path, prototype, byte_range) + except (FileNotFoundError, IsADirectoryError, NotADirectoryError): + return None + except PermissionError: + if path.is_dir(): + return None + raise + + if sys.platform == "win32": # Per the os.rename docs: # On Windows, if dst exists a FileExistsError is always raised. @@ -319,11 +338,7 @@ def get_sync( if prototype is None: prototype = default_buffer_prototype() self._ensure_open_sync() - path = self.root / key - try: - return _get(path, prototype, byte_range) - except (FileNotFoundError, IsADirectoryError, NotADirectoryError): - return None + return _get_or_none(self.root / key, prototype, byte_range) def set_sync(self, key: str, value: Buffer) -> None: self._ensure_open_sync() @@ -351,12 +366,7 @@ async def get( if prototype is None: prototype = default_buffer_prototype() await self._ensure_open() - path = self.root / key - - try: - return await asyncio.to_thread(_get, path, prototype, byte_range) - except (FileNotFoundError, IsADirectoryError, NotADirectoryError): - return None + return await asyncio.to_thread(_get_or_none, self.root / key, prototype, byte_range) async def get_partial_values( self, @@ -367,7 +377,7 @@ async def get_partial_values( args = [] for key, byte_range in key_ranges: path = self.root / key - args.append((_get, path, prototype, byte_range)) + args.append((_get_or_none, path, prototype, byte_range)) return await concurrent_map(args, asyncio.to_thread, limit=None) # TODO: fix limit async def set(self, key: str, value: Buffer) -> None: diff --git a/tests/test_store/test_local.py b/tests/test_store/test_local.py index c619177c34..f8f7b1267f 100644 --- a/tests/test_store/test_local.py +++ b/tests/test_store/test_local.py @@ -16,7 +16,7 @@ import zarr import zarr.storage._local from zarr import create_array -from zarr.core.buffer import Buffer, cpu +from zarr.core.buffer import Buffer, cpu, default_buffer_prototype from zarr.storage import LocalStore from zarr.storage._local import _RETRY_DELAYS, _atomic_write, _move_with_retry from zarr.testing.store import StoreTests @@ -25,6 +25,9 @@ if TYPE_CHECKING: from collections.abc import AsyncIterator, Awaitable, Callable, Iterator + from zarr.abc.store import ByteRequest + from zarr.core.buffer import BufferPrototype + _LOCAL_STORE_FILE = zarr.storage._local.__file__ _ASYNC_CODE_FLAGS = inspect.CO_COROUTINE | inspect.CO_ASYNC_GENERATOR @@ -151,8 +154,8 @@ def wrapper(*args: Any, **kwargs: Any) -> Any: [ ("open", "_ensure_root"), ("lazy_open", "_ensure_root"), - ("get", "_get"), - ("get_partial_values", "_get"), + ("get", "_get_or_none"), + ("get_partial_values", "_get_or_none"), ("set", "_put"), ("set_if_not_exists", "_put"), ("exists", "_exists"), @@ -283,6 +286,39 @@ async def test_get_with_prototype_default(self, store: LocalStore) -> None: observed = await store.get(key, prototype=None) assert_bytes_equal(observed, data_buf) + @pytest.mark.parametrize("reader", ["get", "get_sync", "get_partial_values"]) + @pytest.mark.parametrize("windows_error", [False, True], ids=["native", "windows-error"]) + @pytest.mark.parametrize("name", ["d", "missing", "d/k/below"]) + async def test_read_returns_none_for_a_name_without_a_file( + self, + store: LocalStore, + monkeypatch: pytest.MonkeyPatch, + reader: str, + windows_error: bool, + name: str, + ) -> None: + """Reading a directory, a missing key or a name below a key returns `None`. + + Windows raises `PermissionError` when it opens a directory as a file. The + `windows-error` cases make opening a directory raise it, so that branch runs on + every OS. + """ + await self.set(store, "d/k", self.buffer_cls.from_bytes(b"v")) + if windows_error: + monkeypatch.setattr(zarr.storage._local, "_get", _get_as_on_windows) + observed = await _read(store, reader, name) + assert observed is None + + @pytest.mark.parametrize("reader", ["get", "get_sync", "get_partial_values"]) + async def test_read_raises_permission_error_for_a_file( + self, store: LocalStore, monkeypatch: pytest.MonkeyPatch, reader: str + ) -> None: + """A `PermissionError` on a file is a real error, not a missing key.""" + await self.set(store, "k", self.buffer_cls.from_bytes(b"v")) + monkeypatch.setattr(zarr.storage._local, "_get", _raise_permission_error) + with pytest.raises(PermissionError, match="denied"): + await _read(store, reader, "k") + @pytest.mark.parametrize("ndim", [0, 1, 3]) @pytest.mark.parametrize( "destination", ["destination", "foo/bar/destintion", pathlib.Path("foo/bar/destintion")] @@ -319,6 +355,35 @@ async def test_move( await store2.move(destination) +_GET = zarr.storage._local._get + + +def _get_as_on_windows( + path: pathlib.Path, prototype: BufferPrototype, byte_range: ByteRequest | None +) -> Buffer: + """Read like `_get`, but raise what Windows raises when it opens a directory as a file.""" + if path.is_dir(): + raise PermissionError(13, "Permission denied", str(path)) + return _GET(path, prototype, byte_range) + + +def _raise_permission_error( + path: pathlib.Path, prototype: BufferPrototype, byte_range: ByteRequest | None +) -> Buffer: + raise PermissionError(13, "Permission denied", str(path)) + + +async def _read(store: LocalStore, reader: str, key: str) -> Buffer | None: + """Read `key` from `store` with the named read method.""" + prototype = default_buffer_prototype() + if reader == "get": + return await store.get(key, prototype=prototype) + if reader == "get_sync": + return store.get_sync(key, prototype=prototype) + (value,) = await store.get_partial_values(prototype, [(key, None)]) + return value + + @pytest.mark.parametrize("exclusive", [True, False]) def test_atomic_write_successful(tmp_path: pathlib.Path, exclusive: bool) -> None: path = tmp_path / "data" diff --git a/tests/test_store/test_stateful_conformance.py b/tests/test_store/test_stateful_conformance.py index de2aaea7f0..9afb851bba 100644 --- a/tests/test_store/test_stateful_conformance.py +++ b/tests/test_store/test_stateful_conformance.py @@ -604,7 +604,7 @@ def _windows_only(stores: frozenset[str]) -> frozenset[str]: witness=(("set", ("d/k", b"v")), ("get", ("d", None))), ), KnownDivergence( - stores=_windows_only(frozenset({"local", "latency-local", "fsspec-local"})), + stores=_windows_only(frozenset({"fsspec-local"})), summary="get() on a directory raises PermissionError (Windows opens it as a file)", accepts=_raises_for_a_directory("PermissionError"), witness=(("set", ("d/k", b"v")), ("get", ("d", None))),