diff --git a/.panache.toml b/.panache.toml new file mode 100644 index 000000000..fb9ac1428 --- /dev/null +++ b/.panache.toml @@ -0,0 +1,15 @@ +#:schema https://panache.bz/panache.schema.json +flavor = "myst" +extend-exclude = [".claude/"] + +[format] +wrap = "preserve" +smart-punctuation = "unicode" +tab-stops = "preserve" + +# MyST syntax extensions from `scverse_doc.config.MYST_ENABLE_EXTENSIONS` +[extensions] +myst-colon-fence = true # colon_fence +definition-lists = true # deflist +tex-math-dollars = true # dollarmath, amsmath +raw-html = true # html_image, html_admonition diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0debe4dd8..d584d9479 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,6 +13,11 @@ repos: exclude: ^\.cruft\.json$ # inconsistent indentation with cruft - file never to be modified manually. groups: [format] # further file exclusions and other options are configured in biome.jsonc + - repo: https://github.com/jolars/panache-pre-commit + rev: v3.18.0 + hooks: + - id: panache-format + groups: [format] - repo: https://github.com/tox-dev/pyproject-fmt rev: v2.30.3 hooks: diff --git a/.vscode/extensions.json b/.vscode/extensions.json index caaeb4f73..3e5224e31 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -14,5 +14,6 @@ "editorconfig.editorconfig", "charliermarsh.ruff", "biomejs.biome", + "jolars.panache", ], } diff --git a/.vscode/settings.json b/.vscode/settings.json index e034b91f7..015733be8 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,5 @@ { - "[python][json][jsonc]": { + "[python][json][jsonc][yaml][markdown]": { "editor.formatOnSave": true, }, "[python]": { @@ -9,9 +9,12 @@ "source.organizeImports": "always", }, }, - "[json][jsonc]": { + "[json][jsonc][yaml]": { "editor.defaultFormatter": "biomejs.biome", }, + "[markdown]": { + "editor.defaultFormatter": "jolars.panache", + }, "python.analysis.typeCheckingMode": "basic", "python.testing.pytestEnabled": true, "python.testing.pytestArgs": ["-vv", "--color=yes"], diff --git a/AGENTS.md b/AGENTS.md index d79ac1e53..0531a65cd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -26,7 +26,7 @@ assert not isinstance(transformation, dict) assert_type(transformation, BaseTransformation) ``` -When all branches of a union have been handled, use `typing.assert_never` instead of a generic "should not reach here" +When all branches of a union have been handled, use `typing.assert_never` instead of a generic “should not reach here” error, so the type checker tells us if a new type is added to the union and not handled: ```python @@ -42,7 +42,7 @@ else: #### Why -Because of Python's (multiple-)inheritance model, a positive `isinstance` assertion can never be rejected by a type +Because of Python’s (multiple-)inheritance model, a positive `isinstance` assertion can never be rejected by a type checker, even when it contradicts the declared type: ```python @@ -60,7 +60,7 @@ assert not isinstance(transformations, BaseTransformation) assert_type(transformations, dict[str, BaseTransformation]) ``` -Optional values are already narrowed "negatively", so keep using `is not None`: +Optional values are already narrowed “negatively”, so keep using `is not None`: ```python assert target_depth is not None diff --git a/README.md b/README.md index 4590c6e54..6c2a815a9 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ Another useful resource to get started is the source code of the [`spatialdata-i ## Installation -Check out the docs for more complete [installation instructions](https://spatialdata.scverse.org/en/stable/installation.html). To get started with the "batteries included" installation, you can install via pip: +Check out the docs for more complete [installation instructions](https://spatialdata.scverse.org/en/stable/installation.html). To get started with the “batteries included” installation, you can install via pip: ```bash pip install "spatialdata[extra]" diff --git a/benchmarks/README.md b/benchmarks/README.md index 5af8369c5..fb18df7f3 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -14,7 +14,7 @@ pip install -e . --group dev --group test --group doc --group benchmark ## Usage -Running all the benchmarks is usually not needed. You run the benchmark using `asv run`. See the [asv documentation](https://asv.readthedocs.io/en/stable/commands.html#asv-run) for interesting arguments, like selecting the benchmarks you're interested in by providing a regex pattern `-b` or `--bench` that links to a function or class method. You can run the benchmark in your current environment with `--python=same`. Some example benchmarks: +Running all the benchmarks is usually not needed. You run the benchmark using `asv run`. See the [asv documentation](https://asv.readthedocs.io/en/stable/commands.html#asv-run) for interesting arguments, like selecting the benchmarks you’re interested in by providing a regex pattern `-b` or `--bench` that links to a function or class method. You can run the benchmark in your current environment with `--python=same`. Some example benchmarks: ### Import time benchmarks diff --git a/biome.jsonc b/biome.jsonc index 6fa28f0cd..9810c7a0b 100644 --- a/biome.jsonc +++ b/biome.jsonc @@ -2,6 +2,7 @@ "$schema": "https://biomejs.dev/schemas/2.6.4/schema.json", "vcs": { "enabled": true, "clientKind": "git", "useIgnoreFile": true }, "formatter": { "useEditorconfig": true }, + "markdown": { "formatter": { "enabled": false } }, // handled by panache "overrides": [ { "includes": ["./.vscode/*.json", "**/*.jsonc"], diff --git a/docs/api/io.md b/docs/api/io.md index 714f4ab22..39b99461f 100644 --- a/docs/api/io.md +++ b/docs/api/io.md @@ -1,6 +1,6 @@ # Input/Output -To read the data from a specific technology (e.g., Xenium, MERSCOPE, ...), you can +To read the data from a specific technology (e.g., Xenium, MERSCOPE, …), you can use any of the [spatialdata-io readers](https://spatialdata.scverse.org/projects/io/en/stable/api.html). ```{eval-rst} diff --git a/docs/citation.md b/docs/citation.md index 6f96220b4..115631941 100644 --- a/docs/citation.md +++ b/docs/citation.md @@ -2,4 +2,4 @@ If you use SpatialData in your work, please cite {cite:t}`marconatoSpatialDataOpenUniversal2024`: -> Marconato, L., Palla, G., Yamauchi, K.A. et al. SpatialData: an open and universal data framework for spatial omics. _Nature Methods_ (2024). doi: [10.1038/s41592-024-02212-x](https://doi.org/10.1038/s41592-024-02212-x). +> Marconato, L., Palla, G., Yamauchi, K.A. et al. SpatialData: an open and universal data framework for spatial omics. *Nature Methods* (2024). doi: [10.1038/s41592-024-02212-x](https://doi.org/10.1038/s41592-024-02212-x). diff --git a/docs/contributing.md b/docs/contributing.md index 0b1ab29df..708319043 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -10,7 +10,6 @@ For more extensive tutorials, that also cover the absolute basics, please refer [scanpy developer guide]: https://scanpy.scverse.org/page/dev/ :::{tip} The _hatch_ project manager - We highly recommend to familiarize yourself with [`hatch`][hatch]. Hatch is a Python project manager that @@ -20,14 +19,13 @@ Hatch is a Python project manager that - allows to run tasks defined in `pyproject.toml`, e.g. to build documentation. While the project is setup with `hatch` in mind, it is still possible to use different tools to manage dependencies, such as `uv` or `pip`. - ::: [hatch]: https://hatch.pypa.io/latest/ ## Installing dev dependencies -In addition to the packages needed to _use_ this package, you need additional python packages to [run tests](#writing-tests) and [build the documentation](#docs-building). +In addition to the packages needed to *use* this package, you need additional python packages to [run tests](#writing-tests) and [build the documentation](#docs-building). :::::{tab-set} ::::{tab-item} Hatch @@ -46,9 +44,9 @@ hatch run docs:build # defined in the table [tool.hatch.envs.docs] If you are using VS code, install the [hatch-code][] extension. Additionally, make sure that the `vscode-python-environments` extension is installed (should be by default) and `"python.useEnvironmentsExtension": true` is activated in your `settings.json`. -Next, open the "Python Environment Managers" sidebar. +Next, open the “Python Environment Managers” sidebar. You can do so by opening the command palette (Ctrl+Shift+P) and searching for `Python: Focus on Environment Managers View`. -It will show a collapsible list where you can expand "Hatch" and activate an environment by clicking on the checkmark next to it. +It will show a collapsible list where you can expand “Hatch” and activate an environment by clicking on the checkmark next to it. As the main development environment, we recommend to use `hatch-test` with the latest supported Python version. ### Other IDEs @@ -93,8 +91,6 @@ hatch env find hatch-test.py3.14-stable ``` and manually point it to the python binary. - - :::: ::::{tab-item} uv @@ -111,7 +107,6 @@ uv sync --group=test --group=doc --extra=torch ``` The `.venv` directory is typically automatically discovered by IDEs such as VS Code. - :::: ::::{tab-item} Pip @@ -127,7 +122,6 @@ pip install -e . --group dev --group test --group doc ``` The `.venv` directory is typically automatically discovered by IDEs such as VS Code. - :::: ::::: @@ -155,7 +149,7 @@ prek will automatically download all dependencies when it is run for the first t If you didn’t run the checks locally, the `Pre-commit checks` job of the GitHub Actions CI runs them on your pull request and reports any failures. We strongly encourage installing and running the checks locally first to understand their usage. -Finally, most editors have an _autoformat on save_ feature. +Finally, most editors have an *autoformat on save* feature. Consider enabling this option for [ruff][ruff-editors] and [biome][biome-editors]. [pre-commit]: https://pre-commit.com/ @@ -184,7 +178,6 @@ hatch test # test with the highest supported Python version # or hatch test --all # test with all supported Python versions ``` - :::: ::::{tab-item} uv @@ -193,7 +186,6 @@ hatch test --all # test with all supported Python versions ```bash uv run pytest ``` - :::: ::::{tab-item} Pip @@ -203,7 +195,6 @@ uv run pytest source .venv/bin/activate pytest ``` - :::: ::::: @@ -349,7 +340,6 @@ If you are interested in automatically running notebooks as part of the continuo hatch run docs:build hatch run docs:open ``` - :::: ::::{tab-item} uv @@ -360,7 +350,6 @@ cd docs uv run sphinx-build -M html . _build -W (xdg-)open _build/html/index.html ``` - :::: ::::{tab-item} Pip @@ -372,7 +361,6 @@ cd docs sphinx-build -M html . _build -W (xdg-)open _build/html/index.html ``` - :::: ::::: diff --git a/docs/design_doc.md b/docs/design_doc.md index 73357526a..3fe8c6e3a 100644 --- a/docs/design_doc.md +++ b/docs/design_doc.md @@ -8,11 +8,11 @@ This documents defines the specifications and design of SpatialData: an open and ## Motivation and Scope -Recent advances in molecular profiling technologies allow to measure abundance of RNA and proteins in tissue, at high throughput, multiplexing and resolution. The variety of experimental techniques poses unique challenges in data handling and processing, in particular around data types and size. _SpatialData_ aims at implementing a performant in-memory representation in Python and an on-disk representation based on the Zarr and Parquet data formats and following, when applicable, the OME-NGFF specification. By maximing interoperability, performant implementations and efficient (cloud-based) IO operations, _SpatialData_ aims at laying the foundations for new methods and pipelines for the analysis of spatial omics data. +Recent advances in molecular profiling technologies allow to measure abundance of RNA and proteins in tissue, at high throughput, multiplexing and resolution. The variety of experimental techniques poses unique challenges in data handling and processing, in particular around data types and size. *SpatialData* aims at implementing a performant in-memory representation in Python and an on-disk representation based on the Zarr and Parquet data formats and following, when applicable, the OME-NGFF specification. By maximing interoperability, performant implementations and efficient (cloud-based) IO operations, *SpatialData* aims at laying the foundations for new methods and pipelines for the analysis of spatial omics data. ### Goals -The goals define _what_ SpatialData will be able to do (as opposed to _how_). Goals can have the following priority levels: +The goals define *what* SpatialData will be able to do (as opposed to *how*). Goals can have the following priority levels: - P0: highest priority, required for successful implementation (i.e., must have) - P1: high priority, but not required (i.e., nice to have) @@ -65,21 +65,21 @@ The goals define _what_ SpatialData will be able to do (as opposed to _how_). Go ### Non-goals -- _SpatialData_ is not an analysis library. Instead the aim is to provide an infrastructure to analysis libraries for IO and spatial queries. -- _SpatialData_ is not a format converter. We should not support converting to/from too many formats and instead use OME-NGFF as the interchange format. Nevertheless,[spatialdata-io][] offers a place for some common data conversions (external contributions are highly encouraged). -- _SpatialData_ is based on standard on-disk storage formats (Zarr and Parquet) and on existing specifications (NGFF, AnnData) and uses existing solutions when possible. The resulting storage objects which brings together these technologies defines the _SpatialData on-disk format_, which is described in this document and finely characterized in [this online resource](https://github.com/scverse/spatialdata-notebooks/tree/main/notebooks/developers_resources/storage_format). +- *SpatialData* is not an analysis library. Instead the aim is to provide an infrastructure to analysis libraries for IO and spatial queries. +- *SpatialData* is not a format converter. We should not support converting to/from too many formats and instead use OME-NGFF as the interchange format. Nevertheless,[spatialdata-io][] offers a place for some common data conversions (external contributions are highly encouraged). +- *SpatialData* is based on standard on-disk storage formats (Zarr and Parquet) and on existing specifications (NGFF, AnnData) and uses existing solutions when possible. The resulting storage objects which brings together these technologies defines the *SpatialData on-disk format*, which is described in this document and finely characterized in [this online resource](https://github.com/scverse/spatialdata-notebooks/tree/main/notebooks/developers_resources/storage_format). ## Satellite projects We strongly encourage collaborations and community supports in all of these projects. -- [x] P0. _Visualization_: we are developing a napari plugin for interactive visualization of _SpatialData_ objects @ [napari-spatialdata][]. -- [x] P0. _Raw data IO_: we are implementing readers for raw data of common spatial omics technologies @ [spatialdata-io][]. -- [x] P1. _Static plotting_: a static plotting library for _SpatialData_ @ [spatialdata-plot][]. -- [ ] P2. _Image analysis_: Library to perform image analysis, wrapping common analysis library in python such as skimage. +- [x] P0. *Visualization*: we are developing a napari plugin for interactive visualization of *SpatialData* objects @ [napari-spatialdata][]. +- [x] P0. *Raw data IO*: we are implementing readers for raw data of common spatial omics technologies @ [spatialdata-io][]. +- [x] P1. *Static plotting*: a static plotting library for *SpatialData* @ [spatialdata-plot][]. +- [ ] P2. *Image analysis*: Library to perform image analysis, wrapping common analysis library in python such as skimage. Once ready, we will deprecate such functionalities in [squidpy][]. -- [ ] P2. _Spatial and graph analysis_: [squidpy][] will be refactor to accept SpatialData objects as input. -- [ ] P2. _Database_: Some form of update on released datasets with updated specs as development progresses. A temporary sandbox where we store downloader and converter scripts for representative datasets is available @ [spatialdata-sandbox][]. +- [ ] P2. *Spatial and graph analysis*: [squidpy][] will be refactor to accept SpatialData objects as input. +- [ ] P2. *Database*: Some form of update on released datasets with updated specs as development progresses. A temporary sandbox where we store downloader and converter scripts for representative datasets is available @ [spatialdata-sandbox][]. @@ -93,15 +93,15 @@ We strongly encourage collaborations and community supports in all of these proj ### Terminology -_SpatialData_ is both the name of the Python library as well as the name of the framework (including [spatialdata-io][], [napari-spatialdata][], [spatialdata-plot][]) and the name of the in-memory Python object `SpatialData`. To distinguish between the three, we will use the _italics_ formatting for the _SpatialData_ library and the _SpatialData_ framework (the distinction between them will be clear from the context), and we will use the `code` formatting for the `SpatialData` object. +*SpatialData* is both the name of the Python library as well as the name of the framework (including [spatialdata-io][], [napari-spatialdata][], [spatialdata-plot][]) and the name of the in-memory Python object `SpatialData`. To distinguish between the three, we will use the *italics* formatting for the *SpatialData* library and the *SpatialData* framework (the distinction between them will be clear from the context), and we will use the `code` formatting for the `SpatialData` object. -**_SpatialData_** +***SpatialData*** -The _SpatialData_ library provides a set of specifications and in-memory representations for spatial omics datasets with the goal of unifying spatial omics pipelines and making raw and processed datasets interoperable with browser-based viewers. _SpatialData_ also provides basic operations to query and manipulate such data. The _SpatialData_ specs inherit the OME-NGFF specification for the storage of raster types (images and labels) and for storing several types of metadata. Additional storage requirements not covered by OME-NGFF are described in this document and in [this online resource](https://github.com/scverse/spatialdata-notebooks/tree/main/notebooks/developers_resources/storage_format). _SpatialData_ also implements Python objects to load, save, and interact with spatial data. +The *SpatialData* library provides a set of specifications and in-memory representations for spatial omics datasets with the goal of unifying spatial omics pipelines and making raw and processed datasets interoperable with browser-based viewers. *SpatialData* also provides basic operations to query and manipulate such data. The *SpatialData* specs inherit the OME-NGFF specification for the storage of raster types (images and labels) and for storing several types of metadata. Additional storage requirements not covered by OME-NGFF are described in this document and in [this online resource](https://github.com/scverse/spatialdata-notebooks/tree/main/notebooks/developers_resources/storage_format). *SpatialData* also implements Python objects to load, save, and interact with spatial data. **Elements** -_Elements_ are the building blocks of _SpatialData_ datasets. Each element represents a particular datatype (e.g., raster image, label image, expression table). SpatialData elements are not special classes, but are instead standard scientific Python classes (e.g., `xarray.DataArray`, `AnnData`) with specified metadata. The metadata provides the necessary information for displaying and integrating the different elements (e.g., coordinate systems and coordinate transformations). Elements can either be initialized with valid metadata from disk or from in memory objects (e.g., `numpy` arrays) via _SpatialData_ parser functions. See the Elements section below for a detailed description of the different types of elements. +*Elements* are the building blocks of *SpatialData* datasets. Each element represents a particular datatype (e.g., raster image, label image, expression table). SpatialData elements are not special classes, but are instead standard scientific Python classes (e.g., `xarray.DataArray`, `AnnData`) with specified metadata. The metadata provides the necessary information for displaying and integrating the different elements (e.g., coordinate systems and coordinate transformations). Elements can either be initialized with valid metadata from disk or from in memory objects (e.g., `numpy` arrays) via *SpatialData* parser functions. See the Elements section below for a detailed description of the different types of elements. **`SpatialData`** @@ -111,25 +111,25 @@ The `SpatialData` object contains a set of Elements to be used for analysis. Ele We model a spatial dataset as a composition of distinct elements, of any type. The elements correspond to: -- Pixel-based _Images_, 2D or 3D +- Pixel-based *Images*, 2D or 3D - Regions of interest - - _Shapes_ (circles, polygons, multipolygons), 2D - - Pixel masks (such as segmentation masks), aka _Labels_, 2D, or 3D + - *Shapes* (circles, polygons, multipolygons), 2D + - Pixel masks (such as segmentation masks), aka *Labels*, 2D, or 3D - Points (such as transcript locations, point clouds, ...), 2D or 3D -- _Tables_ of annotations +- *Tables* of annotations Each of these elements should be useful by itself, and in combination with other relevant elements. All elements are stored in the Zarr container in hierarchy store that MAY be flat; currently Zarr hierarchies are not supported, [see here](https://github.com/scverse/spatialdata/issues/340)). -There is no explicit link between elements (e.g. we don't save information equivalent to "this Labels element refers to this Image element"), and one is encouranged to use coordinate systems to semantically group elements together, based on spatial overlap. Coordinate systems are explained later in this document. +There is no explicit link between elements (e.g. we don’t save information equivalent to “this Labels element refers to this Image element”), and one is encouranged to use coordinate systems to semantically group elements together, based on spatial overlap. Coordinate systems are explained later in this document. By decomposing the data model into building blocks (i.e. Elements) we support the storage of any arbitrary combinations of elements, which can be added and modified independently at any moment. #### Assumptions -_SpatialData_ follows the OME-NGFF specifications whenever possible and therefore much of its assumptions are inherited from it. Extra assumptions will be discussed with the OME-NGFF community and adapted to the community-agreed design. The key assumptions are the following: +*SpatialData* follows the OME-NGFF specifications whenever possible and therefore much of its assumptions are inherited from it. Extra assumptions will be discussed with the OME-NGFF community and adapted to the community-agreed design. The key assumptions are the following: -- `Images`, `Labels`, `Points` and `Shapes` MUST have one or more _coordinate systems_ and _coordinate transformations_. -- `Tables` CAN NOT have a _coordinate system_ or _coordinate transforms_. Tables should not contain spatial coordinate: the user can decided to store them there, but they will not be processed by the library and needs to placed in a element and a coordinate system to be recognized by the framework. +- `Images`, `Labels`, `Points` and `Shapes` MUST have one or more *coordinate systems* and *coordinate transformations*. +- `Tables` CAN NOT have a *coordinate system* or *coordinate transforms*. Tables should not contain spatial coordinate: the user can decided to store them there, but they will not be processed by the library and needs to placed in a element and a coordinate system to be recognized by the framework. - `Labels` and `Shapes` are both instances of `Regions`, `Regions` are `Elements`. - Any `Element` MAY be annotated by `Tables`; also `Shapes` and `Points` MAY contain annotations within themselves as additional dataframe columns (e.g. intensity of point spread function of a each point, or gene id). - `Tables` CAN NOT be annotated by other `Tables`. @@ -177,7 +177,7 @@ More precisely, we are using the [spatial-image library][] and [multiscale-spati The coordinate systems and transforms are stored in `spatial_image.SpatialImage.attrs` or in `multiscale_spatial_image.MultiscaleSpatialImage.attrs`. -The xarray coordinates are not saved in the NGFF storage. APIs to take into account for the xarray coordinates, such as converting back and forth between NGFF transformations and xarray coordinates, will be implemented ([see this issue](https://github.com/scverse/spatialdata/issues/308)). In particular, the xarray coordinates will be converted to NGFF transformations before saving the images to disk, and will be reconstructed after reading the data from disk. Supporing the representation of xarray coordinates will allow raster data types to be assigned a coordinate systems; otherwise (as of now) they must be defined in the "pixel space" (this is done implicitly). +The xarray coordinates are not saved in the NGFF storage. APIs to take into account for the xarray coordinates, such as converting back and forth between NGFF transformations and xarray coordinates, will be implemented ([see this issue](https://github.com/scverse/spatialdata/issues/308)). In particular, the xarray coordinates will be converted to NGFF transformations before saving the images to disk, and will be reconstructed after reading the data from disk. Supporing the representation of xarray coordinates will allow raster data types to be assigned a coordinate systems; otherwise (as of now) they must be defined in the “pixel space” (this is done implicitly). @@ -228,8 +228,8 @@ We are considering using the [dask-geopandas library](https://dask-geopandas.rea #### Points -_This representation is still under discussion and it might change. -What is described here is the current implementation. See discussion [here](https://github.com/scverse/spatialdata/issues/233) and [here](https://github.com/scverse/spatialdata/issues/46)._ +*This representation is still under discussion and it might change. +What is described here is the current implementation. See discussion [here](https://github.com/scverse/spatialdata/issues/233) and [here](https://github.com/scverse/spatialdata/issues/46).* Coordinates of points for single molecule data. Each observation is a point, and might have additional information (intensity etc.). Current implementation represent points as a Parquet file and a [`dask.dataframe.DataFrame`](https://docs.dask.org/en/stable/dataframe.html) in memory. @@ -256,11 +256,13 @@ Annotations of regions of interest. Each row in this table corresponds to a sing One region table can refer to multiple sets of Regions. But each row can map to only one region in its Regions element. For example, one region table can store annotation for multiple slides, though each slide would have its own label element. - * `region: str | list[str]`: Regions or list of regions this table refers to. - * `region_key: str`: Key in obs which says which Regions container - this obs exists in (e.g. "library_id"). - * `instance_key: str`: Key in obs that says which instance the obs - represents (e.g. "cell_id"). +``` +* `region: str | list[str]`: Regions or list of regions this table refers to. +* `region_key: str`: Key in obs which says which Regions container + this obs exists in (e.g. "library_id"). +* `instance_key: str`: Key in obs that says which instance the obs + represents (e.g. "cell_id"). +``` If any of `region`, `region_key` and `instance_key` are defined, they all MUST be defined. A table not defining them is still allowed, but it will not be mapped to any spatial element. @@ -328,7 +330,7 @@ We also have a constraint (that we will relax in the future, [see here](https:// We define classes that follow the NGFF specifications to represent the coordinate systems (class `NgffCoordinateSystem`) and coordinate transformations (classes inheriting from `NgffBaseTransformations`). Anyway, these classes are used only during input and output. For operations we define new classes (inheriting from `BaseTransformation`). -Classes inheriting from `NgffBaseTransformation` are: `NgffIdentity`, `NgffMapAxis`, `NgffTranslation`, `NgffScale`, `NgffAffine`, `NgffRotation`, `NgffSequence`, `NgffByDimension`. The following are not supported: `NgffMapIndex`, `NgffDisplacements`, `NgffCoordinates`, `NgffInverseOf`, `NgffBijection`. In the future these classes could be moved outside _SpatialData_, for instance in [ome-zarr-py](https://github.com/ome/ome-zarr-py). +Classes inheriting from `NgffBaseTransformation` are: `NgffIdentity`, `NgffMapAxis`, `NgffTranslation`, `NgffScale`, `NgffAffine`, `NgffRotation`, `NgffSequence`, `NgffByDimension`. The following are not supported: `NgffMapIndex`, `NgffDisplacements`, `NgffCoordinates`, `NgffInverseOf`, `NgffBijection`. In the future these classes could be moved outside *SpatialData*, for instance in [ome-zarr-py](https://github.com/ome/ome-zarr-py). Classes inheriting from `BaseTransformation` are: `Identity`, `MapAxis`, `Translation`, `Scale`, `Affine`, `Sequence`. @@ -340,11 +342,11 @@ The `NgffBaseTransformations` require full specification of the input and output On the contrary, each `BaseTransformation` is self-defined and does not require the information on coordinate systems. Almost (see below) any transformation can be applied unambiguously to any element and almost any pair of transformations can be chained together. The result is either uniquely defined, either an exception is raised when there is ambiguity. -Precisely, this is performed by "passing through" (keeping unaltered) those axis that are present in an element but not in a transformation, and by ignoring axes that are present in a transformation but not in an element. +Precisely, this is performed by “passing through” (keeping unaltered) those axis that are present in an element but not in a transformation, and by ignoring axes that are present in a transformation but not in an element. For example one can apply a `Scale([2, 3, 4], axes=('x', 'y', 'z'))` to a `cyx` image (the axes `c` is passed through unaltered, and the scaling on `z` is ignored since there is no `z` axis.) -An example of transformation that cannot be applied is an `Affine xy -> xyz` to `xyz` data, since `z` can't be passed through as it is also the output of the transformation. +An example of transformation that cannot be applied is an `Affine xy -> xyz` to `xyz` data, since `z` can’t be passed through as it is also the output of the transformation. To know more about the separation between the two set of classes see [this closed issue](https://github.com/scverse/spatialdata/issues/39), [this other closed issue](https://github.com/scverse/spatialdata/issues/47) and [this merged pr](https://github.com/scverse/spatialdata/pull/100). @@ -359,7 +361,7 @@ See [this notebook](https://github.com/scverse/spatialdata-notebooks/blob/main/n ### Roadmap -This section describes a more detailed timeline of future developments, including also more technical tasks like code refactoring of existing functionalities for improving stability/performance. Compared to the "goal" section above, here we provide an concrete timeline. +This section describes a more detailed timeline of future developments, including also more technical tasks like code refactoring of existing functionalities for improving stability/performance. Compared to the “goal” section above, here we provide an concrete timeline. #### 2024 @@ -379,7 +381,7 @@ This section describes a more detailed timeline of future developments, includin - [ ] Support Zarr v3 (sharding). - [ ] Remove Dask constraints ([latest `dask-expr` versions are not compatible](https://github.com/dask/dask/issues/11146)). ---- +-------------------------------------------------------------------------------- ### Examples diff --git a/docs/glossary.md b/docs/glossary.md index 81f83aa87..d1e2c6fbd 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -28,7 +28,7 @@ Rasterization is the process of converting data from a vector format (see defini ## ROI -An ROI (_Region of Interest_) is a specific subset of a dataset that highlights an area of particular relevance, such as a niche, cell, or tissue location. For example, an ROI may be defined as a polygon outlining a lymphoid structure. +An ROI (*Region of Interest*) is a specific subset of a dataset that highlights an area of particular relevance, such as a niche, cell, or tissue location. For example, an ROI may be defined as a polygon outlining a lymphoid structure. ## Spatial query @@ -40,7 +40,7 @@ An extension of the OME-Zarr format used to represent SpatialData objects on dis ## SpatialElements -_SpatialElements_ are the building blocks of _SpatialData_ datasets, and are split into multiple categories: `images`, `labels`, `shapes` and `points`. SpatialData SpatialElements are not special classes, but are instead standard scientific Python classes (e.g., `xarray.DataArray`, `geopandas.GeoDataFrame`) with specified metadata. The metadata provides the necessary information for displaying and integrating the different elements (e.g., coordinate systems and coordinate transformations). SpatialElements can be annotated by `tables`, which are represented as `anndata.AnnData` objects. Tables are a building block of SpatialData, but are not considered SpatialElements since they do not contain spatial information. +*SpatialElements* are the building blocks of *SpatialData* datasets, and are split into multiple categories: `images`, `labels`, `shapes` and `points`. SpatialData SpatialElements are not special classes, but are instead standard scientific Python classes (e.g., `xarray.DataArray`, `geopandas.GeoDataFrame`) with specified metadata. The metadata provides the necessary information for displaying and integrating the different elements (e.g., coordinate systems and coordinate transformations). SpatialElements can be annotated by `tables`, which are represented as `anndata.AnnData` objects. Tables are a building block of SpatialData, but are not considered SpatialElements since they do not contain spatial information. ## Tables @@ -50,7 +50,7 @@ A building block of SpatialData, represented as `anndata.AnnData` objeets. Table Vector data represents spatial information using geometric shapes such as points, circles, or polygons, each defined by mathematical coordinates. -Vectorization is the process of converting raster data (i.e. pixel-based images) into vector format. For instance, a "raster" cell boundary can be "vectorized" into a polygon represented by the coordinates of the vertices of the polygon. +Vectorization is the process of converting raster data (i.e. pixel-based images) into vector format. For instance, a “raster” cell boundary can be “vectorized” into a polygon represented by the coordinates of the vertices of the polygon. ## Zarr storage diff --git a/docs/index.md b/docs/index.md index 45ec33d35..58f1bb4e3 100644 --- a/docs/index.md +++ b/docs/index.md @@ -47,7 +47,7 @@ If you like scverse® and want to support our mission, please consider making a +> @@ -56,6 +56,7 @@ If you like scverse® and want to support our mission, please consider making a This library is currently under active development. We may make changes to the API between versions as the community provides feedback. To ensure reproducibility, please make note of the version you are developing against. ``` + ```{eval-rst} .. card:: Installation :link: installation @@ -100,6 +101,7 @@ If you like scverse® and want to support our mission, please consider making a Learn how to contribute to ``spatialdata``. ``` + ```{toctree} :caption: Getting started diff --git a/docs/installation.md b/docs/installation.md index d87f5819d..a75cde83f 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -74,7 +74,7 @@ Update: currently (Feb 2025), due to particular versions being unavailable on `c ## Docker -A `Dockerfile` is available in the repository; the image that can be built from it contains `spatialdata` (with `torch`), `spatialdata-io` and `spatialdata-plot` (not `napari-spatialdata`)'; the libaries are installed from PyPI. +A `Dockerfile` is available in the repository; the image that can be built from it contains `spatialdata` (with `torch`), `spatialdata-io` and `spatialdata-plot` (not `napari-spatialdata`)’; the libaries are installed from PyPI. To build the image, run: diff --git a/docs/user_guide.md b/docs/user_guide.md index 7142e4eb7..be1240c39 100755 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -19,7 +19,6 @@ pip install spatialdata This command installs barebone SpatialData. For a more detailed description of the installation process including all bells and whistles, see the [installation guide](https://spatialdata.scverse.org/en/stable/installation.html). - ::: :::{dropdown} I have a SpatialData object, how do I open it and see what's inside? @@ -28,19 +27,14 @@ sdata = sd.read_zarr("your-dataset.zarr") ``` For more details, see [the tutorial section on reading zarr data](https://spatialdata.scverse.org/en/stable/tutorials/notebooks/notebooks/examples/models1.html#reading-spatialdata-zarr-data). Additionaly, see [the introductory tutorial on SpatialData](https://spatialdata.scverse.org/en/stable/tutorials/notebooks/notebooks/examples/intro.html) to understand how to explore the different element types within a SpatialData object. - ::: ::::{dropdown} How do I create a SpatialData object? - :::{dropdown} If you have an existing Squidpy object. - Checkout [the tutorial on creating a SpatialData object from scratch](https://spatialdata.scverse.org/en/stable/tutorials/notebooks/notebooks/examples/sdata_from_scratch.html). - ::: :::{dropdown} If you want to start from raw files - If you don’t have a SpatialData object or corresponding zarr data on disk, you will have to create a SpatialData object to make use of the SpatialData framework. You can create a SpatialData object directly from your raw files (e.g. CSVs, cell-gene matrix/cell-proteins matrix, images etc.) by using the [spatialdata-io](https://github.com/scverse/spatialdata-io) library that has reader functions for most spatial omics techniques. See the [list of technologies currently supported by spatialdata-io](https://spatialdata.scverse.org/projects/io/en/latest/). @@ -55,24 +49,16 @@ sdata = spatialdata_io.merscope( For more technical details on creating a SpatialData object from raw data, see [the tutorial section on reader functions from spatialdata-io](https://spatialdata.scverse.org/en/stable/tutorials/notebooks/notebooks/examples/models1.html#reader-functions-from-spatialdata-io). If there is no reader implemented for your data type, see [the tutorial section on constructing a SpatialData object from scratch](https://spatialdata.scverse.org/en/stable/tutorials/notebooks/notebooks/examples/models1.html#construct-a-spatialdata-object-from-scratch). - - ::: :::: :::{dropdown} Do you have datasets I can directly open and play around with? - Yes! Browse our [collection of demo datasets](https://spatialdata.scverse.org/en/stable/tutorials/notebooks/datasets/README.html) (already available as .zarr), generated with different spatial omics technologies. - ::: - - - ## Preparing your data :::{dropdown} How do I align my images? - Sometimes, you have multiple images containing information, but they have different pixels size/are rotated/are moved. For instance, you may have a multi-omics dataset with two images, each representing a different modality, but you need to align them within a single object while preserving their respective molecular signals. Here, we explain how you can spatially align(=register) these images. NOTE: SpatialData only allows for affine(=linear) transformations to map images onto each other. Do you know how to align your images (i.e. Do you have a transformation matrix)? @@ -80,22 +66,18 @@ Do you know how to align your images (i.e. Do you have a transformation matrix)? No → Check out our [Landmark annotation tutorial](https://spatialdata.scverse.org/en/stable/tutorials/notebooks/notebooks/examples/alignment_using_landmarks.html) Yes → Check out our [Transformation/coordinate system tutorial](https://spatialdata.scverse.org/en/stable/tutorials/notebooks/notebooks/examples/transformations.html) - ::: ::::{dropdown} How do I annotate my data? - Annotation can have a lot of meanings. You might want to annotate specific regions in your tissue as tumor, add more details about your cell shapes, group together specific transcripts, include celltype annotations or cell sizes for your cells etc. :::{dropdown} How can I interactively annotate regions in my data? :name: annotate-regions-interactively This is possible within the SpatialData framework, making use of napari, as explained in the [tutorial on ROI annotation with Napari](https://spatialdata.scverse.org/en/stable/tutorials/notebooks/notebooks/examples/napari_rois.html). - ::: :::{dropdown} I have annotated regions in an external tool, how do I add them to SpatialData? - If the annotated regions are saved in a geojson, you can add them as follows: ``` @@ -104,49 +86,36 @@ sdata['very_interesting_regions']=ShapesModel.parse('path_to_geojson') ``` For more details, including information on how to add annotations for these regions, see the [tables tutorial](https://spatialdata.scverse.org/en/stable/tutorials/notebooks/notebooks/examples/tables.html). - ::: :::{dropdown} I have cells in my dataset, how do I annotate them? (usage of AnnData) - One of the most obvious things to do for spatial omics data is to annotate cells using the [AnnData](https://anndata.readthedocs.io/en/stable/) format (called tables in SpatialData). These tables can contain count/intensity data, all types of annotations, and make it possible to make use of [scanpy](https://scanpy.readthedocs.io/en/stable/) functionality (normalization/clustering/DE calculation). For more technical details on how to create a table from scratch to annotate your shapes/labels/points, see [tutorial section on how tables are modeled in SpatialData](https://spatialdata.scverse.org/en/stable/tutorials/notebooks/notebooks/examples/models2.html#tables). - ::: - :::: ::::{dropdown} How do I select a spatial subset of my data for further investigation? - -Eventually, you might want to know which cells are present in which region (e.g. 'Are immune cells infiltrating the tumor core?'). +Eventually, you might want to know which cells are present in which region (e.g. ‘Are immune cells infiltrating the tumor core?’). Therefore, subsampling your data based on spatial coordinates can be helpful. Other examples would be for instance if you want to target your analysis to a specific subregion of the tissue (e.g. tumor core), if you have multiple tissues on one slide, if you want to start small, or if you want to only focus on regions with good tissue quality. :::{dropdown} I want to annotate a specific region myself. - -See {ref}`interactively annotating regions ` under 'How do I annotate my data?' above. This will guide you through the process of extracting coordinates, which will be used for spatially subsetting the data. Next, see 'I know the coordinates of the region I want to subsample.' below. - +See {ref}`interactively annotating regions ` under ‘How do I annotate my data?’ above. This will guide you through the process of extracting coordinates, which will be used for spatially subsetting the data. Next, see ‘I know the coordinates of the region I want to subsample.’ below. ::: :::{dropdown} I know the coordinates of the region I want to subsample. - You will need to perform a spatial query, which filters the data based on spatial coordinates. See the [spatial query tutorial](https://spatialdata.scverse.org/en/stable/tutorials/notebooks/notebooks/examples/spatial_query.html). - ::: - :::: :::{dropdown} How do I summarize/aggregate my data? - You have annotated your object with regions or labels and want to determine which cells or transcripts are present in each area. For example, you may be interested in analyzing the composition of cell types within each region or generating a pseudobulk profile per region or label. Or maybe you just want to know your gene counts per cell. In SpatialData terminology, this corresponds to the **aggregate** function. The method of aggregation depends on the type of elements in your SpatialData object. Learn how to do it in our [Integrate/aggregate signals across spatial layers tutorial](https://spatialdata.scverse.org/en/stable/tutorials/notebooks/notebooks/examples/aggregation.html). - ::: ## Visualization and Analysis :::{dropdown} How do I visualize my data? - To make data visualization easier for each layer in your SpatialData object, we offer [spatialdata-plot](https://github.com/scverse/spatialdata-plot), a package that enhances SpatialData’s functionality for intuitive and detailed plotting. The standard format for plotting a specific layer follows this syntax: @@ -173,22 +142,17 @@ You can also chain different plots. For example, if you want to plot regions of For more details and examples on visualizing technology-specific datasets, explore our [technology-specific tutorials](https://spatialdata.scverse.org/en/stable/tutorials/notebooks/notebooks.html#technology-specific), which includes examples for various technologies (e.g., Visium-HD, Xenium, etc.). Keep in mind that the visualization methods demonstrated in these tutorials are broadly applicable and not limited to a specific technology. If you run into scalability issues with your plotting, or everything feels very slow, [the tutorial on speeding up plotting](https://spatialdata.scverse.org/en/latest/tutorials/notebooks/notebooks/examples/speed_up_illustration.html) might help you out! - ::: :::{dropdown} How can I use SpatialData to analyze my data? - SpatialData is **not** intended to analyze data itself, but to provide data infrastructure to efficiently manipulate spatial omics data. If you want to learn more about analyzing spatial omics data, please browse the [documentation section on extenal tools](https://spatialdata.scverse.org/en/stable/tutorials/notebooks/notebooks.html#external-tutorials) for demos of how various analysis packages work with SpatialData. To see how SpatialData and Squidpy interact, see the [Squidpy integration tutorial](https://spatialdata.scverse.org/en/latest/tutorials/notebooks/notebooks/examples/squidpy_integration.html). - ::: - ## Going further :::{dropdown} SpatialData looks amazing, I want to learn more. Give me all the technical details! - Sure, no problem! There is much more to learn about SpatialData. You want to learn how to combine SpatialData and deep learning? Checkout the [tutorial on deep learning with DenseNet](https://spatialdata.scverse.org/en/stable/tutorials/notebooks/notebooks/examples/densenet.html). @@ -196,12 +160,9 @@ You want to learn how to combine SpatialData and deep learning? Checkout the [tu If you want to create a labels layer from your shapes (rasterize), a shapes layer from your labels (vectorize), or convert your spot-based data to a labels layer, see the [labels/shapes interchangeability tutorial](https://spatialdata.scverse.org/en/stable/tutorials/notebooks/notebooks/examples/labels_shapes_interchangeability.html). This is mainly useful to unlock specific functionalities, or speed up calculations. Finally, here is a more [advanced technical tutorial on transformations](https://spatialdata.scverse.org/en/latest/tutorials/notebooks/notebooks/examples/transformations_advanced.html#other-technical-topics). - ::: :::{dropdown} I am sold, how can I get in touch and contribute to SpatialData? - Thank you for your interest in contributing to SpatialData! The [contributing guide](https://spatialdata.scverse.org/en/stable/contributing.html) explains how to contribute to SpatialData as a developer. If you would like to contribute to satellite projects, such as `spatialdata-io`, `spatialdata-plot` etc, please consult the contribution guides also in those repositories. For larger contributions, and especially regarding new APIs or new design choices, please before writing any code get in touch with us! We are happy to talk via GitHub, via chat or with a call. The [contact section of the README](https://github.com/scverse/spatialdata/#contact) lists multiple ways of reaching out. - :::