Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changes/466.bugfix.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
`LocalStore.get`, `get_sync` and `get_partial_values` now return `None` on Windows when the key names a directory, instead of raising `PermissionError`. A `PermissionError` on a file still raises. `get_partial_values` now also returns `None` for a missing key.
34 changes: 22 additions & 12 deletions src/zarr/storage/_local.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,25 @@ def _get(path: Path, prototype: BufferPrototype, byte_range: ByteRequest | None)
return prototype.buffer.from_bytes(f.read())


def _get_or_none(
path: Path, prototype: BufferPrototype, byte_range: ByteRequest | None
) -> Buffer | None:
"""Read the file at `path`, or return `None` when no file is there.

Opening a directory raises `IsADirectoryError` on POSIX and `PermissionError` on Windows.
A `PermissionError` counts as a missing key only when `path` is a directory, so a file the
process may not read still raises.
"""
try:
return _get(path, prototype, byte_range)
except (FileNotFoundError, IsADirectoryError, NotADirectoryError):
return None
except PermissionError:
if path.is_dir():
return None
raise


if sys.platform == "win32":
# Per the os.rename docs:
# On Windows, if dst exists a FileExistsError is always raised.
Expand Down Expand Up @@ -319,11 +338,7 @@ def get_sync(
if prototype is None:
prototype = default_buffer_prototype()
self._ensure_open_sync()
path = self.root / key
try:
return _get(path, prototype, byte_range)
except (FileNotFoundError, IsADirectoryError, NotADirectoryError):
return None
return _get_or_none(self.root / key, prototype, byte_range)

def set_sync(self, key: str, value: Buffer) -> None:
self._ensure_open_sync()
Expand Down Expand Up @@ -351,12 +366,7 @@ async def get(
if prototype is None:
prototype = default_buffer_prototype()
await self._ensure_open()
path = self.root / key

try:
return await asyncio.to_thread(_get, path, prototype, byte_range)
except (FileNotFoundError, IsADirectoryError, NotADirectoryError):
return None
return await asyncio.to_thread(_get_or_none, self.root / key, prototype, byte_range)

async def get_partial_values(
self,
Expand All @@ -367,7 +377,7 @@ async def get_partial_values(
args = []
for key, byte_range in key_ranges:
path = self.root / key
args.append((_get, path, prototype, byte_range))
args.append((_get_or_none, path, prototype, byte_range))
return await concurrent_map(args, asyncio.to_thread, limit=None) # TODO: fix limit

async def set(self, key: str, value: Buffer) -> None:
Expand Down
71 changes: 68 additions & 3 deletions tests/test_store/test_local.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
import zarr
import zarr.storage._local
from zarr import create_array
from zarr.core.buffer import Buffer, cpu
from zarr.core.buffer import Buffer, cpu, default_buffer_prototype
from zarr.storage import LocalStore
from zarr.storage._local import _RETRY_DELAYS, _atomic_write, _move_with_retry
from zarr.testing.store import StoreTests
Expand All @@ -25,6 +25,9 @@
if TYPE_CHECKING:
from collections.abc import AsyncIterator, Awaitable, Callable, Iterator

from zarr.abc.store import ByteRequest
from zarr.core.buffer import BufferPrototype

_LOCAL_STORE_FILE = zarr.storage._local.__file__
_ASYNC_CODE_FLAGS = inspect.CO_COROUTINE | inspect.CO_ASYNC_GENERATOR

Expand Down Expand Up @@ -151,8 +154,8 @@ def wrapper(*args: Any, **kwargs: Any) -> Any:
[
("open", "_ensure_root"),
("lazy_open", "_ensure_root"),
("get", "_get"),
("get_partial_values", "_get"),
("get", "_get_or_none"),
("get_partial_values", "_get_or_none"),
("set", "_put"),
("set_if_not_exists", "_put"),
("exists", "_exists"),
Expand Down Expand Up @@ -283,6 +286,39 @@ async def test_get_with_prototype_default(self, store: LocalStore) -> None:
observed = await store.get(key, prototype=None)
assert_bytes_equal(observed, data_buf)

@pytest.mark.parametrize("reader", ["get", "get_sync", "get_partial_values"])
@pytest.mark.parametrize("windows_error", [False, True], ids=["native", "windows-error"])
@pytest.mark.parametrize("name", ["d", "missing", "d/k/below"])
async def test_read_returns_none_for_a_name_without_a_file(
self,
store: LocalStore,
monkeypatch: pytest.MonkeyPatch,
reader: str,
windows_error: bool,
name: str,
) -> None:
"""Reading a directory, a missing key or a name below a key returns `None`.

Windows raises `PermissionError` when it opens a directory as a file. The
`windows-error` cases make opening a directory raise it, so that branch runs on
every OS.
"""
await self.set(store, "d/k", self.buffer_cls.from_bytes(b"v"))
if windows_error:
monkeypatch.setattr(zarr.storage._local, "_get", _get_as_on_windows)
observed = await _read(store, reader, name)
assert observed is None

@pytest.mark.parametrize("reader", ["get", "get_sync", "get_partial_values"])
async def test_read_raises_permission_error_for_a_file(
self, store: LocalStore, monkeypatch: pytest.MonkeyPatch, reader: str
) -> None:
"""A `PermissionError` on a file is a real error, not a missing key."""
await self.set(store, "k", self.buffer_cls.from_bytes(b"v"))
monkeypatch.setattr(zarr.storage._local, "_get", _raise_permission_error)
with pytest.raises(PermissionError, match="denied"):
await _read(store, reader, "k")

@pytest.mark.parametrize("ndim", [0, 1, 3])
@pytest.mark.parametrize(
"destination", ["destination", "foo/bar/destintion", pathlib.Path("foo/bar/destintion")]
Expand Down Expand Up @@ -319,6 +355,35 @@ async def test_move(
await store2.move(destination)


_GET = zarr.storage._local._get


def _get_as_on_windows(
path: pathlib.Path, prototype: BufferPrototype, byte_range: ByteRequest | None
) -> Buffer:
"""Read like `_get`, but raise what Windows raises when it opens a directory as a file."""
if path.is_dir():
raise PermissionError(13, "Permission denied", str(path))
return _GET(path, prototype, byte_range)


def _raise_permission_error(
path: pathlib.Path, prototype: BufferPrototype, byte_range: ByteRequest | None
) -> Buffer:
raise PermissionError(13, "Permission denied", str(path))


async def _read(store: LocalStore, reader: str, key: str) -> Buffer | None:
"""Read `key` from `store` with the named read method."""
prototype = default_buffer_prototype()
if reader == "get":
return await store.get(key, prototype=prototype)
if reader == "get_sync":
return store.get_sync(key, prototype=prototype)
(value,) = await store.get_partial_values(prototype, [(key, None)])
return value


@pytest.mark.parametrize("exclusive", [True, False])
def test_atomic_write_successful(tmp_path: pathlib.Path, exclusive: bool) -> None:
path = tmp_path / "data"
Expand Down
2 changes: 1 addition & 1 deletion tests/test_store/test_stateful_conformance.py
Original file line number Diff line number Diff line change
Expand Up @@ -604,7 +604,7 @@ def _windows_only(stores: frozenset[str]) -> frozenset[str]:
witness=(("set", ("d/k", b"v")), ("get", ("d", None))),
),
KnownDivergence(
stores=_windows_only(frozenset({"local", "latency-local", "fsspec-local"})),
stores=_windows_only(frozenset({"fsspec-local"})),
summary="get() on a directory raises PermissionError (Windows opens it as a file)",
accepts=_raises_for_a_directory("PermissionError"),
witness=(("set", ("d/k", b"v")), ("get", ("d", None))),
Expand Down
Loading