diff --git a/.codespellrc b/.codespellrc index 03198ea..6e93c6b 100644 --- a/.codespellrc +++ b/.codespellrc @@ -1,5 +1,5 @@ [codespell] -ignore-words-list = ND,nd,bu, dNe, slippy, tye +ignore-words-list = ND,nd,bu, dNe, slippy, tye, raison skip = *.png,*.jpg,*.gif,*.pdf,*.svg quiet-level = 3 diff --git a/.gitignore b/.gitignore index 546c180..ba975f8 100644 --- a/.gitignore +++ b/.gitignore @@ -92,3 +92,7 @@ tests/.hypothesis docs/_site site .mypy_cache + +# Node.js (screenshot capture script in scripts/) +node_modules/ +package-lock.json diff --git a/README.md b/README.md index 688a665..1775200 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,16 @@ -# Guidance for avoiding common pitfalls when producing and using datacubes. +# Datacube Guide -Status: This is a first draft of the guidance. +Practical guidance for producing, using, and visualizing datacubes. + +Read the guide at [developmentseed.org/datacube-guide](https://developmentseed.org/datacube-guide/). + +> Status: First draft, actively iterating. + +## What's inside + +- **Worst practices** — common pitfalls to avoid when producing and using multi-dimensional data products: chunking, metadata, datatypes, and default library configs (FSSpec, GDAL, Xarray). +- **Visualization** — a catalog and comparison of tools for visualizing Zarr-backed datacubes in the browser, covering both server-side dynamic tilers (TiTiler, Xpublish-tiles) and client-side rendering, with libraries for deck.gl, MapLibre/Mapbox, and Cesium, plus standalone viewer apps. Includes side-by-side comparisons and guidance on choosing an approach. +- **Benchmarking** — a small Python library (`datacube_benchmark`) for measuring read patterns and access costs. ## Installation diff --git a/docs/visualization/browzarr.md b/docs/visualization/browzarr.md index 9d02065..11505f8 100644 --- a/docs/visualization/browzarr.md +++ b/docs/visualization/browzarr.md @@ -1,3 +1,44 @@ # Browzarr -Content coming soon. See [source repo](https://github.com/EarthyScience/Browzarr) for now. +A browser-native viewer for multi-dimensional Zarr and NetCDF datasets, with first-class 3D volumetric rendering. Maintained by the Max Planck Institute for Biogeochemistry (EarthyScience) under the Apache 2.0 license. + +## At a glance + +- **Repo** — [EarthyScience/Browzarr](https://github.com/EarthyScience/Browzarr) +- **Live demo** — [browzarr.io](https://browzarr.io) +- **Shape** — Viewer application (Next.js, static export). Not currently published as an embeddable npm package. +- **Render API** — three.js + react-three-fiber, GLSL3 shaders, with WebGPU support enabled via `webgpu-utils` +- **Zarr versions** — v2 and v3 via zarrita; also reads NetCDF (via `netcdf4-wasm`) and Icechunk stores (via `icechunk-js`) +- **Conventions** — CF `_ARRAY_DIMENSIONS` for axis naming + +## What it does + +Browzarr is the most "data-centric" of the client-side options listed here. The user points the hosted app at a remote Zarr URL or uploads a local file, and the dataset is rendered directly in the browser without a backing tile server. The distinguishing capability is genuine 3D volumetric rendering of Zarr cubes, not just slices of them. + +## How it renders + +Three rendering modes share the same data pipeline. + +- **Data Cube** — full 3D volume rendering via fragment-shader raycasting, with both orthographic and perspective cameras. Implemented in `volFragment.glsl`. +- **FlatMap** — 2D orthographic slicing with pan and zoom. Implemented in `flatFrag.glsl`. +- **Sphere** — globe rendering with displacement mapping for elevation, useful for global datasets. Implemented in `sphereVertex.glsl`. + +Additional point cloud and block modes handle sparse data. Animation is driven by GSAP for temporal transitions, and post-processing (shadows, bloom) is provided by react-three-postprocessing. + +The data pipeline is normalize-to-uint8 then sample. Raw float values are scaled to [0, 1] and uploaded as 8-bit textures; the fragment shader looks them up against one of 70+ colormaps from `js-colormaps-es` (magma, viridis, turbo, Spectral, ...). Adjustable `cOffset` and `cScale` uniforms allow live remapping of the value range without re-uploading texture data, and a separate NaN colour and alpha control nodata appearance. + +## Zarr handling + +Reads use zarrita 0.6.x. An LRU cache (`ZarrLoaderLRU.ts`) holds recently accessed chunks. Geographic datasets get latitude and longitude bounds passed as shader uniforms, and the `useCoordBounds()` hook computes UV mapping for the sphere render mode. There is no GeoZarr support, no tile pyramid, no projection beyond geographic lat/lon, and no basemap concept; the Earth, when present, is just a textured sphere. + +## Where it fits + +Choose Browzarr when the goal is exploration of a multi-dimensional cube as a cube, with volumetric rendering and free 3D camera movement, rather than as a 2D web map. It is also the right starting point for users who want a hosted viewer to point at a Zarr URL with no integration work. It is not a substitute for a basemap-anchored 2D map: if the dataset belongs on top of a vector basemap with picking and panning, deck.gl-raster or zarr-layer are the fits. + +## Links + +- Source: [EarthyScience/Browzarr](https://github.com/EarthyScience/Browzarr) +- Hosted: [browzarr.io](https://browzarr.io) +- Authors: Jeran Poehls and Lazaro Alonso (MPI-BGC Jena) +- Funding: EU Horizon Europe (AI4PEX) and ESA SeasFire +- Related: [zarrita](https://github.com/manzt/zarrita.js), [icechunk-js](https://github.com/earth-mover/icechunk), [@earthyscience/netcdf4-wasm](https://github.com/EarthyScience/netcdf4-wasm) diff --git a/docs/visualization/carbonplan-maps.md b/docs/visualization/carbonplan-maps.md index b85b0cf..907dcba 100644 --- a/docs/visualization/carbonplan-maps.md +++ b/docs/visualization/carbonplan-maps.md @@ -1,3 +1,40 @@ -# Carbonplan/maps +# carbonplan/maps -Content coming soon. See [source repo](https://github.com/carbonplan/maps) for now. +A React component library for rendering Zarr-backed raster data on Mapbox or MapLibre maps, built around `regl`. Maintained by CarbonPlan under the MIT license. The original "dynamic client" library that informed CarbonPlan's later [zarr-layer](zarr-layer.md), and still actively maintained. + +## At a glance + +- **Repo** — [carbonplan/maps](https://github.com/carbonplan/maps) +- **Shape** — React component library, single npm package (`@carbonplan/maps`) +- **Map host** — Mapbox GL JS v1.13.1 bundled in the default export, or bring-your-own Mapbox/MapLibre via the `/core` export with `` +- **Render API** — `regl` (WebGL abstraction) with hand-written GLSL +- **Zarr versions** — v2 (primary) and v3 (via `version` prop), read with `zarr-js` (not zarrita) +- **Conventions** — Strict `ndpyramid` Web Mercator pyramids; reads `.zmetadata` with `multiscales` in `.zattrs` + +## What it does + +`@carbonplan/maps` exposes a small set of React components, ``, ``, ``, ``, ``, plus hooks like `useMap()`, `useRegl()`, `useControls()`, `useRegion()`, that let an application compose a Zarr-backed map view declaratively. It is the rendering backbone of CarbonPlan's published visualizations, including the CMIP6 downscaling, seaweed farming, and forest carbon explainers, so its design choices reflect a production track record rather than a research prototype. + +## How it renders + +The vertex shader handles Web Mercator projection math and tile geometry. The fragment shader looks up data values from a per-tile texture, normalizes against the user-supplied `clim` range, and samples a 1D RGB colormap texture (typically supplied via the sibling `@carbonplan/colormaps` package). Custom fragment-shader logic can be injected via the `frag` prop on ``, which is how multi-band math (averaging, differencing, conditional masking) is expressed. + +Multi-band selectors lift naturally into shader uniforms: passing `selector={{ month: [1, 2, 3] }}` loads three arrays simultaneously and exposes them as `month_1`, `month_2`, `month_3` GLSL uniforms. + +## Zarr handling + +Reads use `zarr-js`. Data is required to be in a Web Mercator pyramid built with `ndpyramid`, with consistent `pixels_per_tile` across zoom levels, and the library validates this at load time and throws if the projection is anything else. The `version` prop selects between Zarr v2 and v3 store layouts. There is no GeoZarr support, no untiled mode, and no in-browser CRS reprojection; the assumption is that all preprocessing happens upstream. + +## Where it fits + +`@carbonplan/maps` is documented here for completeness and is appropriate if you are already using it (e.g., extending one of CarbonPlan's existing visualizations) or if you are committed to a CarbonPlan-style workflow that pre-bakes data into `ndpyramid` Web Mercator pyramids. + +For new MapLibre/Mapbox integrations, this guide does not recommend `@carbonplan/maps` as the starting point. The strict `ndpyramid` Web Mercator pyramid requirement carries the same trade-offs as any pre-rendering pipeline: a frozen projection, a frozen pyramid structure, and a regen step on every data update. CarbonPlan's own newer project, [zarr-layer](zarr-layer.md), removes those constraints (untiled mode, arbitrary CRS via proj4, vanilla-JS API) at the cost of being less mature. For the deck.gl ecosystem, [deck.gl-raster](deck.gl-raster.md) is the better fit since it shares a rendering stack with COG and parses GeoZarr metadata directly. + +## Links + +- Source: [carbonplan/maps](https://github.com/carbonplan/maps) +- Companion: [@carbonplan/colormaps](https://github.com/carbonplan/colormaps) +- Pyramid generator: [ndpyramid](https://github.com/carbonplan/ndpyramid) +- Newer sibling: [zarr-layer](zarr-layer.md), CarbonPlan's vanilla-JS MapLibre/Mapbox custom-layer approach +- Production users: CarbonPlan visualizations (CMIP6 downscaling, seaweed farming, forest carbon) diff --git a/docs/visualization/client-side-comparison.md b/docs/visualization/client-side-comparison.md new file mode 100644 index 0000000..886cd24 --- /dev/null +++ b/docs/visualization/client-side-comparison.md @@ -0,0 +1,106 @@ +# Client-side rendering comparison + +Several browser-side libraries and applications now read Zarr directly (via [zarrita](https://github.com/manzt/zarrita.js) or [zarr-js](https://github.com/freeman-lab/zarr-js)) and render on the GPU, no tile server required. They differ in what they sit on top of, what they do with the data, and what they assume about its shape. This page is the cross-cutting view; per-library detail lives in the sibling pages under "Rendering layers" and "Applications". + +A parallel comparison for server-side tooling (TiTiler vs Xpublish) is at [Dynamic tiling ecosystem comparison](ecosystem-comparison.md). + +The four libraries below are ordered roughly by host: deck.gl, MapLibre/Mapbox (with `@carbonplan/maps` and zarr-layer as React-component-vs-custom-layer alternatives on the same host), and Cesium. The two applications are scene-first viewers with no basemap. + +## Rendering layers + +| | [deck.gl-raster](deck.gl-raster.md) | [@carbonplan/maps](carbonplan-maps.md) | [zarr-layer](zarr-layer.md) | [zarr-cesium](zarr-cesium.md) | +|---|---|---|---|---| +| Primary author | Development Seed | CarbonPlan | CarbonPlan | NOC UK (Atlantis) | +| Shape | Library (12 npm packages) | React component library | Library (custom map layer) | Library (3 Cesium provider classes) | +| Map host | deck.gl | Mapbox GL v1 (bundled) or MapLibre/Mapbox via `/core` | MapLibre GL / Mapbox GL v3 | CesiumJS (3D globe) | +| Render API | WebGL2 via luma.gl shader modules | regl, hand-written GLSL | WebGL2, hand-written GLSL | WebGL2, custom GLSL inside Cesium primitives | +| Geographic context | Full deck.gl basemap stack, projection mesh | Mapbox/MapLibre basemaps | MapLibre/Mapbox basemaps, optional adaptive-mesh reprojection | Cesium globe with imagery layers underneath | +| Native projection support | Web Mercator + reprojection mesh | Web Mercator only (validated, throws otherwise) | EPSG:3857/4326 tiled, arbitrary CRS via proj4 in untiled mode | EPSG:4326 and EPSG:3857 only, autodetected | +| Zarr versions | v3 | v2 (primary) and v3 via `version` prop, read with `zarr-js` | v2 and v3, autodetected, via zarrita | v2 and v3, autodetected, via zarrita | +| Conventions | GeoZarr (multiscales, geo-proj, CRS) | Strict `ndpyramid` Web Mercator pyramids, `multiscales` in `.zattrs` | Tiled XYZ, experimental `multiscales`, user-supplied `spatialDimensions` | `ndpyramid` multiscales, CF time decoding, CF dim-name aliasing | +| Dimensionality | N-D, user passes `selection` per non-spatial dim | N-D via selector arrays (e.g. `{ month: [1,2,3] }` → uniforms `month_1`, `month_2`, `month_3`) | N-D via `selector` (multi-band selectors load multiple textures) | 2D imagery, 3D volumetric via draped slices, animated U/V vector fields | +| Custom shader injection | Composable luma.gl shader modules + custom modules | `frag` prop on `` | User-injectable fragment-shader strings, multi-band selectors expose named GLSL variables | Single fixed shader; geometry varies by provider | +| License | MIT | MIT | MIT | MIT | +| Maturity | v0.6.x, Zarr support added early 2026 | v3.6.0, last release Oct 2025, actively maintained | v0.5.0, "active experiment" | v0.1.4, latest commit Dec 2024 | + +## Applications + +| | [Browzarr](browzarr.md) | [GridLook](gridlook.md) | +|---|---|---| +| Primary author | Jeran Poehls (MPI-BGC) | DKRZ + MPI-M | +| Shape | Hosted viewer app (Next.js) | Vue 3 SPA, also embeddable as a Vue component | +| Scene host | three.js + react-three-fiber | three.js | +| Render API | WebGL / WebGPU, GLSL3 shaders | WebGL, custom GLSL | +| Geographic context | None, just a sphere or flat plane with lat/lon UVs | Coastlines and graticules from Natural Earth, no basemap tiles | +| Native projection support | Globe + flat ortho only | 8 projections plus 3D nearside-perspective globe | +| Zarr versions | v2 and v3, plus NetCDF (wasm) and Icechunk | v2 and v3 | +| Conventions | CF `_ARRAY_DIMENSIONS`, no GeoZarr | CF (`grid_mapping`, `_FillValue`, lat/lon coords) | +| Dimensionality | N-D, with 3D volume raycasting as a first-class mode | 2D fields with time slider; 8 grid topologies (regular, rotated, HEALPix, ICON triangular, Gaussian-reduced, curvilinear, irregular Delaunay) | +| License | Apache 2.0 | MIT | +| Maturity | v0.5.1, pre-1.0 | v1.0.0 (April 2026) | + +## Project framing + +**deck.gl-raster** is the most architecturally ambitious: a monorepo of small packages layered so that the raster pipeline is generic over its tile source. Zarr is one backend, COG is the other, and they share the same renderer. It plugs into deck.gl, so users get the rest of the deck.gl ecosystem for free. (Supersedes the earlier `numeric-data-layer`.) + +**`@carbonplan/maps`** is the original "dynamic client" library. A React component library built on `regl` and Mapbox GL v1, it is the rendering backbone of CarbonPlan's published visualizations (CMIP6 downscaling, seaweed farming, forest carbon). It assumes data is in a Web Mercator `ndpyramid` and validates that strictly; the trade-off for that constraint is a small, opinionated `` + `` API and a polished `frag` prop for custom-shader injection. + +**zarr-layer** is the newer CarbonPlan project, designed to relax `@carbonplan/maps`'s assumptions. It implements MapLibre/Mapbox's `CustomLayerInterface` directly (not React-shaped), supports both tiled and untiled modes, handles arbitrary CRS via proj4 with adaptive Delaunay reprojection, and works with both v2 and v3 via zarrita. The trade-off is that it is less mature (v0.5.0, "active experiment") than the production-tested `@carbonplan/maps`. (Also supersedes the earlier `zarr-gl`.) + +**zarr-cesium** is the same library shape (plug into an existing map host) but the host is Cesium. It exposes three Cesium provider classes for three different visualization shapes: a `ZarrLayerProvider` for 2D imagery on the globe, a `ZarrCubeProvider` for 3D volumetric data rendered as draped slices, and a `ZarrCubeVelocityProvider` that hands U/V components to `cesium-wind-layer` for animated particle visualization. Built by NOC for oceanographic and atmospheric use cases. + +**Browzarr** is a hosted viewer rather than a library, built on three.js and react-three-fiber. Its distinguishing feature is genuine 3D volumetric rendering via raycasting. There is no embeddable npm package and no basemap; the Earth is rendered as a textured sphere when geographic, otherwise the data is a free-floating cube. + +**GridLook** is a Vue 3 viewer with a clearly geoscience-shaped scope: visualizing climate-model output on the native grid topology the model used. Its raison d'être is the eight grid-type detectors (HEALPix, ICON triangular, Gaussian-reduced, rotated lat-lon, ...), not raster pyramids. + +## Where the GPU work happens + +All of these normalize values, apply a colormap, and handle nodata on the GPU; CPU only does I/O and bookkeeping. Differences are in composition. + +- **deck.gl-raster** exposes the pipeline as composable luma.gl shader modules. A user passes `renderPipeline: [LinearRescale, Colormap, FilterNoDataVal, ...]` and the framework wires uniforms and texture bindings. `CompositeBands` allows up to four band textures with per-band UV transforms, which is how the published NDVI and AlphaEarth-embeddings examples work. +- **`@carbonplan/maps`** uses hand-written GLSL via regl. Multi-band is expressed via selector arrays that lift to GLSL uniforms (`{ month: [1,2,3] }` becomes `month_1`, `month_2`, `month_3` in the shader), and the `frag` prop on `` lets users inject custom fragment-shader logic for averaging, differencing, and conditional masking. +- **zarr-layer** uses fixed shader templates with user-injectable fragment-shader strings. Multi-band is supported via `selector: { band: ['B08', 'B04'] }`, which loads each band into its own texture and exposes them as named GLSL variables, so NDVI is `(B08 - B04) / (B08 + B04)` in a custom fragment. +- **zarr-cesium** runs a single fragment shader that handles `scale_factor`/`add_offset`, NaN/nodata masking, normalization, and colormap LUT lookup. The interesting variation is in geometry rather than shading: the 2D provider tiles into Cesium's standard imagery scheme, the 3D provider loads a cube into memory as an `ndarray` and renders user-positioned horizontal/vertical slices as Cesium primitives, and the velocity provider feeds U/V slices to the third-party `cesium-wind-layer` particle simulator. +- **Browzarr** has fixed shaders per render mode (volumetric raycasting, sphere displacement, flat orthographic). The pipeline is normalize-to-uint8, sample colormap LUT, apply transfer function with NaN colour and alpha. Live remap via `cOffset`/`cScale` uniforms avoids re-uploading texture data when the user drags the colour-range slider. +- **GridLook** also uses purpose-built shaders, with the twist that the *projection* runs on the GPU. d3-geo computes geometry on the CPU once, then GLSL handles per-pixel reprojection so panning a Mollweide globe stays smooth. Colormap is a 1D LUT after normalization (offset + scale-factor) and optional posterization. + +## Geospatial integration + +The host environment is the cleanest dividing line. + +**deck.gl-raster**, **`@carbonplan/maps`**, and **zarr-layer** are 2D-map-first. They sit inside an existing web-map ecosystem (deck.gl and Mapbox/MapLibre respectively), inherit basemap, attribution, picking, and projection handling, and primarily produce slippy-tile views. `@carbonplan/maps` enforces Web Mercator strictly; deck.gl-raster and zarr-layer have escape hatches for non-Mercator data, but the default mental model is "raster on a web map." + +**zarr-cesium** is globe-first. The host is CesiumJS, with its imagery layers underneath the data, so the integration story matches the 2D-map libraries (plug into the host's standard extension points), but the canvas is a 3D globe rather than a 2D web map. + +**Browzarr** and **GridLook** are scene-first with no basemap. Browzarr renders into a free three.js scene where the geographic case is just one of several render modes; volumetric raycasting has no map analogue. GridLook draws the Earth as a 3D sphere with coastlines and graticules, plus eight 2D projections, but at no point is there a tile basemap underneath. + +## Zarr depth and conventions + +**deck.gl-raster** has the most opinionated metadata stance: a separate `@developmentseed/geozarr` package parses the GeoZarr conventions (multiscales, geo-proj, CRS as EPSG/WKT2/PROJJSON) and produces a generic `TilesetDescriptor`, which `@developmentseed/deck.gl-zarr`'s `ZarrLayer` feeds into the same renderer that COG uses. Zarr v2, OME-NGFF, and CF are listed as future work. + +**`@carbonplan/maps`** is strict in a different way: it requires the data already be an `ndpyramid` Web Mercator pyramid with `multiscales` metadata in `.zattrs`. There is no in-browser CRS reprojection and no untiled mode; the assumption is all preprocessing happens upstream. + +**zarr-layer** is convention-light by comparison: tiled XYZ where the user supplies the spatial-dim mapping, plus an experimental implementation of the in-flight `multiscales` zarr-conventions standard for untiled data. No GeoZarr. + +**zarr-cesium** uses `ndpyramid` for multiscale, decodes CF time, and aliases common CF dimension names. No GeoZarr; only EPSG:4326 and EPSG:3857 are supported, so curvilinear or rotated grids must be reprojected upstream. + +**Browzarr** reads `_ARRAY_DIMENSIONS` (CF) for axis names but isn't trying to be a geospatial tool, so projections, CRS, and tiling conventions don't really apply. + +**GridLook** leans hard on CF (`grid_mapping`, `_FillValue`/`missing_value`, lat/lon variable detection) and on grid-topology detection rather than on GeoZarr. The HEALPix and ICON-triangular paths in particular are well outside what the others handle. + +## Picking the right tool + +- For tiled raster on a web map with the deck.gl ecosystem: **deck.gl-raster** with its new Zarr layer. +- For Zarr on MapLibre/Mapbox: **zarr-layer**. +- For environmental, oceanographic, or atmospheric data on a Cesium 3D globe, especially with 3D slices or animated vector fields: **zarr-cesium**. +- For exploratory science visualization, especially anything volumetric or where you want a hosted viewer for a Zarr URL: **Browzarr**. +- For climate-model output on its native grid (HEALPix, ICON, rotated lat-lon, Gaussian-reduced) with multiple cartographic projections: **GridLook**. + +`@carbonplan/maps` is intentionally omitted from the recommendation list. It remains actively maintained and is the rendering backbone of CarbonPlan's published visualizations, but for new work the requirement that data be pre-baked into `ndpyramid` Web Mercator pyramids carries the same trade-offs as any pre-rendering pipeline: frozen projection, frozen pyramid structure, and a regen step on every data update. zarr-layer is the more flexible CarbonPlan path for new MapLibre/Mapbox integrations. + +Roughly: deck.gl-raster, zarr-layer, and zarr-cesium are libraries you embed in a host map or globe; Browzarr and GridLook are scenes/apps you point at data. Within the libraries, the host (deck.gl, MapLibre/Mapbox, Cesium) is the choice. Within the apps, the choice is exploratory 3D versus ESM-grid-aware 2D/3D. + +## Related + +- [Dynamic tiling ecosystem comparison](ecosystem-comparison.md): server-side tooling (TiTiler vs Xpublish). +- [NASA-IMPACT Zarr Visualization Report](https://nasa-impact.github.io/zarr-visualization-report/): empirical benchmarks for Zarr v3 sharding, pyramid choices, AWS region effects, and pixels-per-tile, from the 2024 study. diff --git a/docs/visualization/deck.gl-raster.md b/docs/visualization/deck.gl-raster.md index cd8d2c8..386ab23 100644 --- a/docs/visualization/deck.gl-raster.md +++ b/docs/visualization/deck.gl-raster.md @@ -1,3 +1,38 @@ -# Deck.gl-raster +# deck.gl-raster -Content coming soon. See [source repo](https://github.com/developmentseed/deck.gl-raster) for now. +GPU-accelerated raster visualization for deck.gl, with backends for both Cloud-Optimized GeoTIFF (COG) and Zarr. Maintained by Development Seed under the MIT license. Supersedes [numeric-data-layer](numeric-data-layer.md). + +## At a glance + +- **Repo** — [developmentseed/deck.gl-raster](https://github.com/developmentseed/deck.gl-raster) +- **Shape** — Library, distributed as 12 npm packages under the `@developmentseed` namespace +- **Map host** — deck.gl (typically with MapLibre GL, Mapbox GL, or Google Maps as the basemap) +- **Render API** — WebGL2 via luma.gl shader modules +- **Zarr versions** — v3 (in the current `@developmentseed/deck.gl-zarr` package) +- **Conventions** — GeoZarr (multiscales, geo-proj, CRS as EPSG, WKT2, or PROJJSON) + +## What it does + +deck.gl-raster provides a generic raster pipeline that is independent of the tile source. The same `RasterTileLayer` and `RasterLayer` render either COG tiles (via `@developmentseed/cogeotiff`) or GeoZarr tiles (via `@developmentseed/deck.gl-zarr`), because the architecture introduces a common `TilesetDescriptor` interface that both backends satisfy. The result is that a project standardising on deck.gl can mix Zarr and COG layers in the same scene without bringing in a second rendering stack. + +## How it renders + +The pipeline is composed from luma.gl shader modules. A user constructs a render pipeline as a list, for example `renderPipeline: [LinearRescale, Colormap, FilterNoDataVal]`, and the framework wires uniforms and texture bindings on the GPU. Built-in modules cover linear rescaling to [0, 1], colormap LUT sampling against a 256-entry RGBA texture, nodata masking, and band compositing of up to four input bands with per-band UV transforms. Users can write custom fragment shaders to express arbitrary band math; the published examples include NDVI from NIR and Red textures and an AlphaEarth Foundation embeddings mosaic. + +Reprojection from the source CRS to Web Mercator is performed once on the CPU as a triangle mesh (using `@developmentseed/raster-reproject`), then rasterized on the GPU. This keeps non-Mercator data renderable inside an otherwise Mercator deck.gl scene. + +## Zarr handling + +Zarr support is split across two new packages. `@developmentseed/geozarr` parses the GeoZarr conventions, including multiscales, axis order, and CRS, and produces a `TilesetDescriptor`. `@developmentseed/deck.gl-zarr` wraps that descriptor in a `ZarrLayer` that delegates to the generic `RasterTileLayer`. Non-spatial dimensions are addressed via an explicit `selection: Record` map keyed by dimension name, so time, band, depth, and similar axes are first-class. + +Reads use zarrita. Zarr v2, OME-NGFF, and CF conventions are listed as future work. + +## Where it fits + +Choose deck.gl-raster when the wider scene is already deck.gl, when you want a single rendering stack across COG and Zarr, or when you need GeoZarr metadata handling out of the box. The cost is the deck.gl dependency and the assumption of a tile-pyramid view of the data; this is not the right tool for in-browser volumetric exploration of a Zarr cube. + +## Links + +- Source: [developmentseed/deck.gl-raster](https://github.com/developmentseed/deck.gl-raster) +- Key packages: `@developmentseed/raster-layer`, `@developmentseed/deck.gl-zarr`, `@developmentseed/geozarr`, `@developmentseed/raster-reproject` +- Related: deck.gl, luma.gl, [zarrita](https://github.com/manzt/zarrita.js) diff --git a/docs/visualization/ecosystem-comparison.md b/docs/visualization/ecosystem-comparison.md new file mode 100644 index 0000000..cb176f7 --- /dev/null +++ b/docs/visualization/ecosystem-comparison.md @@ -0,0 +1,338 @@ +# Dynamic tiling ecosystem comparison + +This page compares the two main FastAPI-based ecosystems for serving Zarr-backed datacubes as tiles: **TiTiler** (Development Seed) and **Xpublish** with its plugins (UCAR, Earth Mover, and the wider xarray community). For per-tool detail see the [Titiler ecosystem overview](titiler/overview.md) and the [Xpublish ecosystem overview](xpublish/overview.md). For the client-side equivalent (visualization libraries that bypass a tile server entirely), see the [client-side rendering comparison](client-side-comparison.md). + +## Overview + +Both ecosystems provide FastAPI-based web services for publishing geospatial and scientific datasets, but with different architectural philosophies and target use cases: + +- **TiTiler**: specialized for dynamic tile generation with a layered architecture built on rio-tiler. Strongest on COG, STAC, and broad raster-format coverage; widely deployed in NASA-style tile-serving stacks. +- **Xpublish**: a plugin-based data publishing platform around xarray, with separate plugins for tiles ([xpublish-tiles](xpublish/xpublish-tiles.md)), WMS ([xpublish-wms](https://github.com/xpublish-community/xpublish-wms)), EDR ([xpublish-edr](https://github.com/xpublish-community/xpublish-edr)), and OPeNDAP ([opendap-protocol](https://github.com/xpublish-community/opendap-protocol)). Strongest on unstructured and curvilinear grids (ROMS, FVCOM, SELFE, HEALPix, cubed-sphere) and on operational geoscience use cases. + +## Detailed Comparison + +| Factor | TiTiler Ecosystem | Xpublish Ecosystem | +|--------|------------------|-------------------| +| **License** | MIT License (Development Seed) | Apache License 2.0 (UCAR) | +| **Organizational Maintainer** | Development Seed | UCAR/Xarray community | +| **Individual Maintainers** | Vincent Sarago, Aimee Barciauskas | Joe Hamman, Alex Kerney, distributed community | + +### Maintenance, Governance, and Development Models + +| Aspect | TiTiler | Xpublish | +|--------|---------|----------| +| **Governance Model** | Open source, built by Development Seed | Open source, community-driven (scientific community) | +| **Development Focus** | Tile server optimization and performance | Protocol compliance and scientific data standards | +| **Release Cadence** | Regular releases with coordinated ecosystem updates | Community-driven releases, plugin-independent versioning | +| **Commercial Support** | Available through Development Seed | Community support through ESIP/scientific networks | +| **Contributor Base** | Concentrated around geospatial tile serving | Distributed across oceanographic and climate science communities | + +### Tested Input Formats + +| Format | TiTiler Support | Xpublish Support | +|--------|----------------|------------------| +| **Native Zarr** | ✅ Full support via titiler.xarray | ✅ Primary format with optimal performance | +| **NetCDF** | ✅ Full support via h5netcdf engine | ✅ Full support via Xarray integration | +| **Virtual Zarr** | ✅ Supported through zarr v3+ interfaces | ✅ Reference-based access to remote datasets | +| **Cloud Optimized GeoTIFF (COG)** | ✅ Primary format for titiler.core | ❌ Not directly supported | +| **STAC Items** | ✅ Native support for asset discovery | ❌ Not directly supported | +| **HDF5/NetCDF via references** | ✅ Via virtual references | ✅ Via Zarr reference spec | +| **Multi-file datasets** | ✅ Via CMR integration (NASA datasets) | ✅ Via catalog systems (Intake) | +| **Icechunk stores** | ⚠️ Limited experimental support | ✅ Full support for versioned Zarr | + +### IO Methods and Data Access + +| Method | TiTiler | Xpublish | +|--------|---------|----------| +| **Core Libraries** | | | +| • Xarray integration | ✅ Via titiler.xarray package | ✅ Native, primary data structure | +| • Rasterio/GDAL | ✅ Via rio-tiler (titiler.core) | ❌ Not used | +| • Zarr direct access | ✅ Via xarray.open_zarr | ✅ Native zarr.open_consolidated | +| • fsspec support | ✅ For remote datasets (S3, HTTP, etc.) | ✅ For all remote access | +| • h5netcdf engine | ✅ For NetCDF files | ✅ Via xarray backends | +| **Data Loading** | | | +| • Lazy loading | ✅ Via Dask arrays | ✅ Via Dask arrays | +| • Async loading | ⚠️ Limited support | ✅ xarray.load_async() support | +| • Caching | ✅ Redis-based dataset caching | ✅ Plugin-configurable | +| • Chunk-aware | ✅ Leverages Zarr/NetCDF chunking | ✅ Optimized for chunked access | +| **Coordinate Handling** | | | +| • CRS transformations | ✅ Via rioxarray + pyproj | ✅ Via pyproj with thread pool | +| • Coordinate renaming | ✅ Auto-detect lat/lon/x/y | ✅ CF-xarray for detection | +| • Antimeridian handling | ✅ Longitude normalization | ✅ LongitudeCellIndex for wrapping | +| **Indexing Methods** | | | +| • Label-based selection | ✅ Xarray sel() with method parameter | ✅ Direct dimension indexing | +| • Spatial indexing | ✅ Bounding box queries | ✅ Custom spatial indexes (CellTreeIndex) | +| • Temporal indexing | ✅ Time dimension selection | ✅ CF time coordinate support | + +### Tested Grid Structures + +| Grid Type | TiTiler Support | Xpublish Support | +|-----------|----------------|------------------| +| **Regular lat/lon grids** | ✅ Optimized for rectangular grids | ✅ Full support via CF conventions | +| **Projected coordinate systems** | ✅ Via morecantile TileMatrixSets | ✅ Via CF-compliant grid mappings | +| **Curvilinear grids** | ⚠️ Basic support with limitations | ✅ Full support (ROMS: CBOFS, DBOFS, TBOFS, etc.) | +| **FVCOM triangular grids** | ❌ No native support | ✅ Full support (LOOFS, LSOFS, LMHOFS, NGOFS2) | +| **SELFE grids** | ❌ No native support | ✅ Full support (CREOFS) | +| **Irregular/unstructured grids** | ❌ Limited support | ✅ Extensible grid system for custom types | +| **2D non-dimensional grids** | ⚠️ Basic support | ✅ Full support (RTOFS, HRRR-Conus) | +| **HEALPix** | ❌ No native support | ✅ Cell-index coordinates with nested indexing (xpublish-tiles, polygons style) | +| **Cubed-sphere** | ❌ No native support | ✅ FacetedGridSystem in xpublish-tiles for cubed-sphere climate-model output | +| **Polar grids** | ❌ No native support | ✅ Polar grid system in xpublish-tiles for radar/polar data | + +### Supported Endpoints + +| Endpoint Category | TiTiler | Xpublish | +|------------------|---------|----------| +| **Tile Generation** | | | +| • XYZ raster tiles | ✅ `/tiles/{z}/{x}/{y}` with multiple TMS | ✅ Via xpublish-tiles (TilesPlugin) and xpublish-wms | +| • Vector tiles (MVT) | ❌ Not native | ✅ Via xpublish-tiles `vector/cells`, `vector/points`, `vector/contours` styles | +| • Vector tiles (GeoJSON) | ❌ Not native | ✅ Via xpublish-tiles vector styles | +| • WMTS | ✅ OGC WMTS compliance | ✅ Via xpublish-wms (GetMap, GetCapabilities) | +| • TileJSON | ✅ Tile layer metadata | ✅ Via WMS plugin | +| • Legend | ⚠️ Via colormap params on existing endpoints | ✅ Dedicated `/tiles/legend` endpoint in xpublish-tiles (rendered image or JSON colour stops) | +| **Data Access** | | | +| • Zarr API | ✅ Via xarray reader | ✅ Native `.zmetadata`, chunk access | +| • OpenDAP | ❌ No direct support | ✅ Via xpublish-opendap plugin | +| • OGC EDR | ❌ No direct support | ✅ Via xpublish-edr (position, area, cube queries) | +| **Metadata** | | | +| • Dataset info | ✅ `/info` and `/info.geojson` | ✅ Built-in dataset information endpoints | +| • Variable listing | ✅ `/variables` (deprecated) | ✅ Automatic metadata exposure | +| **Analysis** | | | +| • Statistics/Histograms | ✅ `/statistics` (POST) with geometry support | ✅ Via EDR plugin with multiple output formats | +| • Timeseries extraction | ✅ Temporal indexing and selection | ✅ Via EDR temporal querying | +| • Spatial querying | ✅ Bbox and feature queries | ✅ EDR position, area, cube queries | + +### Tile Request Parameters + +| Parameter Category | TiTiler | Xpublish-Tiles | +|-------------------|---------|----------------| +| **Dataset Selection** | | | +| • Variable/Asset | `variable` (xarray), `assets` (STAC) | `variables` (list, required) | +| • Band indexes | `bidx` for multi-band selection | Single variable focus | +| • Group/hierarchy | `group` for Zarr/HDF5 groups | Dataset-level only | +| **Dimension Selection** | | | +| • Dimension indexing | `sel={dim}={value}` (list of strings) | Any dimension as query param | +| • Selection method | `sel_method` (nearest, pad, ffill, etc.) | DSL syntax: `dimension={method}::{value}` with `nearest`, `ffill`/`pad`, `bfill`/`backfill`, `exact` | +| • Time selection | Via sel parameter | Direct time parameter with ISO8601 | +| • Decode times | `decode_times` (boolean) | N/A (always decoded) | +| **Styling & Rendering** | | | +| • Colormap | `colormap_name` or `colormap` (JSON) | `style={type}/{colormap}` format | +| • Color range | `rescale` (min,max) | `colorscalerange` (tuple) | +| • Out-of-range colors | Clamping by default | `abovemaxcolor`, `belowmincolor` for explicit out-of-range colours | +| • Custom colormap | JSON with int→hex mapping | JSON with 0-255→hex mapping | +| • Categorical rendering | Limited | Native: `flag_values`, `flag_meanings`, `flag_colors` from CF metadata | +| • Band math | `expression` (e.g., "b1/b2") | Not supported | +| **Image Parameters** | | | +| • Output format | `f` (jpeg, png, webp, tiff, etc.) | `f` (image/png, image/jpeg) | +| • Tile size | `tile_scale` (1=256x256, 2=512x512) | `width`, `height` (multiples of 256) | +| • Quality | Format-specific params | Not exposed | +| **Reprojection** | | | +| • Resampling | `resampling` (RIOResampling enum) | Automatic via datashader | +| • Warp resampling | `reproject` (WarpResampling enum) | Coordinate transform in pipeline | +| • Nodata | `nodata` value override | Automatic from metadata | +| **Processing** | | | +| • Histogram equalization | Via rescale algorithms | Not supported | +| • Hillshade | Via extensions | Not supported | +| • Statistics | POST to `/statistics` endpoint | Via EDR plugin | +| **Error Handling** | | | +| • Render errors | Standard HTTP errors | `render_errors` (boolean) for image tiles | +| • Missing data | Configurable nodata handling | Transparent or nodata fill | + +### Supported Tiling Features + +| Feature Category | TiTiler | Xpublish | +|-----------------|---------|----------| +| **Output Formats** | | | +| • Raster formats | JPEG, PNG, WebP, TIFF, JP2, NumpyTile | PNG, JPEG via xpublish-tiles and xpublish-wms | +| • Vector formats | ❌ | MVT (Mapbox Vector Tile) and GeoJSON via xpublish-tiles `vector/*` styles; GeoJSON, CSV via xpublish-edr | +| • Scientific formats | Limited | NetCDF, Parquet, GeoTIFF via xpublish-edr | +| **Rendering** | | | +| • Rescaling | Linear, histogram-based, custom functions | Basic rescaling via WMS | +| • Colormaps | Built-in + custom JSON colormaps | WMS styling capabilities | +| • Band combinations | Multi-band composites and band math | Single variable visualization focus | +| • Algorithms | NDVI, hillshade via extensions | Limited processing algorithms | +| **Performance** | | | +| • Caching | Redis-based response caching | Plugin-configurable caching | +| • Concurrent access | Multi-threaded tile generation | FastAPI async processing | +| • Chunk optimization | Leverages Zarr/NetCDF chunking | Optimized for chunked scientific data | + +### Resampling Implementations + +| Resampling Aspect | TiTiler | Xpublish-Tiles | +|-------------------|---------|----------------| +| **Raster Resampling (RIOResampling)** | | | +| • Available methods | nearest, bilinear, cubic, cubic_spline, lanczos, average, mode, gauss, max, min, med, q1, q3 | nearest (via datashader), automatic for continuous data | +| • Default method | nearest | Adaptive based on data type | +| • Usage context | Tile generation, preview, bbox queries | Internal rendering pipeline | +| **Warp Resampling (coordinate reprojection)** | | | +| • Available methods | nearest, bilinear, cubic, cubic_spline, lanczos, average, mode, max, min, med, q1, q3, sum, rms | Custom implementation via pyproj | +| • Default method | nearest | Optimized for 4326→3857 (separable transform) | +| • Thread pool | Not used | ✅ Configurable chunk-based parallel transform | +| **Downsampling/Coarsening** | | | +| • Method | Via GDAL overviews or on-the-fly | DataArray.coarsen().mean() with even factors | +| • Automatic trigger | Based on zoom level | When data exceeds max_renderable_size | +| • Boundary handling | GDAL standard | Configurable padding for edge effects | +| **Grid-Specific Optimizations** | | | +| • Rectilinear grids | Standard rasterio resampling | Datashader raster (3-10x faster than quadmesh) | +| • Curvilinear grids | Limited support | Datashader quadmesh with optional rectilinear approximation | +| • Triangular/UGRID | Not supported | Datashader trimesh with Delaunay triangulation | +| • Categorical data | Mode resampling | Numbagg for nearest-neighbor on discrete data | +| **Coordinate Transform Optimizations** | | | +| • 4326→3857 | Via rioxarray/GDAL | Custom separable implementation (preserves grid structure) | +| • General transforms | GDAL warp with selected kernel | Blocked transformation with thread pool | +| • Chunking strategy | N/A | Configurable TRANSFORM_CHUNK_SIZE (NxN chunks) | +| • Approximate rectilinear | Not implemented | Numba-optimized detection (1-pixel threshold) | +| **Rendering Performance** | | | +| • Rendering engine | rio-tiler (GDAL-based) | Datashader (Numba JIT-compiled) | +| • JIT compilation | Not used | First-invocation blocking (unavoidable) | +| • Numba threads | N/A | Configurable NUMBA_NUM_THREADS | + +### Metadata Conventions + +| Convention Category | TiTiler | Xpublish-Tiles | +|---------------------|---------|----------------| +| **CF Conventions** | | | +| • Standard names | ⚠️ Basic support via rioxarray | ✅ Full support via cf-xarray | +| • Coordinate detection | lat/lon/x/y auto-detection | CF axis detection (X, Y, Z, T) | +| • Grid mappings | Via rioxarray | Multiple grid_mapping support | +| • Vertical coordinates | Limited | Z axis detection and handling | +| • Time coordinates | decode_times parameter | CF-compliant time parsing | +| • Bounds/cells | Not used | CF bounds for accurate cell representation | +| **Zarr Metadata** | | | +| • Format version | Zarr v2/v3 support | Zarr v2 primary, v3 compatible | +| • Consolidated metadata | ✅ .zmetadata support | ✅ zarr.open_consolidated() | +| • Chunk encoding | Standard zarr chunks | Standard + compressor/filters | +| • Dimension names | _ARRAY_DIMENSIONS | _ARRAY_DIMENSIONS | +| • Fill values | _FillValue handling | _FillValue + automatic detection | +| **Custom Attributes** | | | +| • valid_min/valid_max | ✅ Used for rescaling | ✅ Used for continuous data colorscale | +| • valid_range | ✅ Converted to valid_min/max | Standard processing | +| • flag_values | Not directly used | ✅ For categorical/discrete rendering | +| • flag_meanings | Not directly used | ✅ Category labels | +| • flag_colors | Not directly used | ✅ Custom categorical colormaps | +| • long_name | Available in metadata | ✅ Used in tile metadata | +| **OGC Standards** | | | +| • OGC WMTS | ✅ Full compliance | Via WMS plugin | +| • OGC Tiles API | ⚠️ Partial | ✅ Full compliance (1.0) | +| • TileJSON | ✅ 3.0.0 support | ✅ 3.0.0 support | +| • WMS | Limited | ✅ 1.1.1/1.3.0 compliance | +| **STAC Metadata** | | | +| • STAC Items | ✅ Native support | ❌ Not used | +| • Asset metadata | ✅ Full integration | N/A | +| • Temporal extent | Via STAC properties | Via CF time coordinates | +| **Coordinate Reference Systems** | | | +| • CRS detection | rioxarray CRS | CF grid_mapping + PyProj | +| • EPSG codes | ✅ Standard support | ✅ Full PyProj CRS support | +| • Custom projections | Via PROJ strings | Via CF grid_mapping_name | +| • Default CRS | Explicit or epsg:4326 | epsg:4326 fallback | +| **Dataset-Level Metadata** | | | +| • Title/description | Via STAC or dataset attrs | Dataset.attrs with fallbacks | +| • Keywords | Via STAC | attrs["keywords"] | +| • License | Via STAC | attrs["license"] | +| • Attribution | Via STAC | attrs["attribution"] | +| • Contact | Via STAC | attrs["contact"] | +| • Version | Not standard | attrs["version"] | +| **Internal Identifiers** | | | +| • Dataset ID | URL-based | attrs["_xpublish_id"] (required for caching) | +| • Cache keys | URL + parameters | Dataset ID + dimension + variable | + +### Plugin/Extension Points + +| Extension Type | TiTiler | Xpublish | +|---------------|---------|----------| +| **Architecture** | Factory-based endpoint creation | Plugin-based router system | +| **Custom I/O** | Pluggable readers (Xarray, Rasterio) | Dataset provider plugins | +| **Protocol Support** | Limited to tile/analysis endpoints | Full protocol plugins (WMS, EDR, OpenDAP) | +| **Authentication** | Extensions for custom auth | Pluggable auth systems | +| **Data Processing** | Algorithm extensions and middleware | Data transformation plugins | +| **Deployment** | Application factory pattern | Configurable server distributions | + +### Architecture and Core Dependencies + +| Component | TiTiler | Xpublish-Tiles | +|-----------|---------|----------------| +| **Web Framework** | | | +| • Framework | FastAPI | FastAPI (via xpublish) | +| • Async support | ✅ ASGI-based | ✅ ASGI-based | +| • OpenAPI docs | ✅ Automatic | ✅ Automatic | +| **Core Data Libraries** | | | +| • Primary reader | rio-tiler (rasterio/GDAL) | Xarray with custom grid systems | +| • Xarray support | Via titiler.xarray addon | Native and required | +| • Rasterio | ✅ Core dependency | ❌ Not used | +| • GDAL | ✅ Via rasterio | ❌ Not used | +| **Rendering Engines** | | | +| • Tile rendering | rio-tiler (GDAL-based) | Datashader (Numba JIT) | +| • Image encoding | rio-tiler + Pillow | Pillow | +| • Resampling | GDAL/rasterio | Datashader + pyproj | +| **Geospatial Libraries** | | | +| • CRS handling | rioxarray + pyproj | pyproj | +| • Coordinate transforms | GDAL warp | Custom pyproj implementation | +| • Tile matrix sets | morecantile | morecantile (via plugin) | +| **Scientific Computing** | | | +| • Xarray | Optional (titiler.xarray) | Required, core | +| • Dask | Via xarray | Via xarray | +| • NumPy | Via dependencies | Direct use | +| • Numba | Not used | ✅ For JIT compilation | +| • numbagg | Not used | ✅ For aggregations | +| • cf-xarray | Not used | ✅ For CF conventions | +| **Grid/Mesh Support** | | | +| • Regular grids | ✅ Native via GDAL | ✅ Via RasterIndex | +| • Curvilinear grids | ⚠️ Limited | ✅ Full support | +| • Unstructured meshes | ❌ No support | ✅ Via scipy.spatial.Delaunay | +| • UGRID | ❌ No support | ✅ Via CellTreeIndex | +| **Caching** | | | +| • Implementation | Redis for datasets | Plugin-configurable | +| • Cache strategy | Dataset-level | Dataset + grid system | +| • Cache invalidation | TTL-based | Internal via _xpublish_id | +| **Configuration** | | | +| • Environment vars | Via Pydantic settings | Via Pydantic settings | +| • Config files | Application-specific | Config system with env vars | +| • Runtime tuning | Limited | Thread pool, async load, chunk size | + +### Deployment Considerations + +| Aspect | TiTiler | Xpublish-Tiles | +|--------|---------|----------------| +| **Container Support** | | | +| • Official images | ✅ Docker Hub + GitHub registry | No official images | +| • Base requirements | Python + GDAL | Python + scientific stack | +| • Image size | Moderate (GDAL overhead) | Larger (scientific libraries) | +| **Cloud Deployment** | | | +| • AWS Lambda | ✅ CDK examples provided | Possible but not documented | +| • AWS ECS | ✅ CDK examples provided | Manual setup | +| • Kubernetes | Community deployments | Manual setup | +| **Scaling Considerations** | | | +| • Stateless | ✅ With external cache | ✅ Dataset loading needed | +| • Horizontal scaling | ✅ Well-suited | ✅ With shared cache | +| • Resource requirements | Moderate CPU + memory | High CPU for JIT, configurable threads | +| **Performance Tuning** | | | +| • Critical settings | Redis config, worker count | NUMBA_NUM_THREADS, thread pool size, async_load | +| • First request | Fast (no JIT) | Slow (JIT compilation) | +| • Warm-up needed | Only for cache | ✅ Required for datashader JIT | +| • Memory footprint | Dataset + tile buffer | Dataset + coordinate cache + grid cache | + +## Use Case Recommendations + +### Choose TiTiler When: +- **Primary need**: High-performance tile serving and web map visualization +- **Data types**: COGs, STAC catalogs, regular gridded scientific data +- **Requirements**: Fast tile generation, web mapping standards compliance, cloud-native architecture +- **Users**: Web mapping applications, tile-based visualizations, GIS workflows + +### Choose Xpublish When: +- **Primary need**: Multi-protocol data access and scientific data sharing +- **Data types**: Complex gridded scientific datasets (ocean models, climate data) +- **Requirements**: Protocol compliance (OpenDAP, EDR), flexible data access patterns, research workflows +- **Users**: Scientific communities, oceanographic/climate data centers, research institutions + +## Hybrid Approaches + +For comprehensive data serving solutions, consider: + +1. **Complementary deployment**: Use TiTiler for tile-based visualization and Xpublish for data access protocols +2. **Protocol bridging**: Leverage both ecosystems' strengths for different access patterns +3. **Staged architecture**: TiTiler for public-facing maps, Xpublish for research data access + +--- + +*Both ecosystems continue to evolve, with increasing interoperability and shared standards adoption across the scientific data serving landscape.* \ No newline at end of file diff --git a/docs/visualization/gridlook.md b/docs/visualization/gridlook.md new file mode 100644 index 0000000..f22afa7 --- /dev/null +++ b/docs/visualization/gridlook.md @@ -0,0 +1,50 @@ +# GridLook + +A WebGL viewer for Earth system model output, with first-class support for the unstructured and reduced grids that climate models actually produce. Maintained by DKRZ and MPI-M under the MIT license. + +## At a glance + +- **Repo** — [dkrz-icon/gridlook](https://github.com/dkrz-icon/gridlook) +- **Hosted** — [gridlook.pages.dev](https://gridlook.pages.dev) +- **Shape** — Vue 3 single-page app, also embeddable as a Vue component +- **Render API** — three.js with custom GLSL shaders +- **Zarr versions** — v2 and v3 via zarrita +- **Conventions** — CF (`grid_mapping`, `_FillValue`, `missing_value`, lat/lon coordinate variables) + +## What it does + +GridLook's defining feature is grid-topology awareness. Where the other viewers in this section assume a regular lat-lon raster (or accept that as input after preprocessing), GridLook detects and renders eight grid types directly from the data: + +1. Regular lat-lon +2. Rotated regular (CORDEX-style, detected via `grid_mapping_name="rotated_latitude_longitude"`) +3. HEALPix (hierarchical, levels 0 to 11+) +4. Triangular meshes (ICON, detected via `vertex_of_cell`) +5. Gaussian reduced (ERA5-style, longitude tapers toward the poles) +6. Curvilinear (2D lat/lon coordinate arrays) +7. Irregular (CMIP6 unstructured) +8. Irregular Delaunay (on-the-fly Delaunay fallback) + +The README is explicit that GridLook is "not a competitor to high-end visualization suites" but a tool for intuitive, interactive exploration of cloud-hosted ESM output. Operational use includes the WCRP Global Hackathon 2025 HEALPix datasets and DKRZ-hosted ICON, IFS DYAMOND3, and EERIE collections. + +## How it renders + +The primary view is a 3D nearside-perspective globe with orbit controls. Eight 2D cartographic projections are also available: Mercator, Robinson, Mollweide, Cylindrical Equal Area, Equirectangular, Azimuthal Equidistant, and a custom Azimuthal Hybrid for seamless polar plus global views. Projection math is computed once on the CPU (via d3-geo and d3-geo-projection) and the per-pixel reprojection runs in GLSL, so panning between projections stays smooth. + +Coastlines and graticules use Natural Earth GeoJSON at 50m resolution and are rendered via WebGL after a recent refactor; both are toggleable. Land/sea masking is generated CPU-side from GeoJSON onto a canvas, then applied as a GPU texture. + +The data pipeline is load chunk, normalize via offset and scale-factor, optionally posterize, look up against one of 58 built-in colormaps (viridis, inferno, coolwarm, RdBu, BrBG, diff, curl, phase, ...), then render. Histogram-based auto-scaling and per-variable default colormaps and ranges are baked into the dataset catalog. + +## Zarr handling + +Reads use zarrita 0.7.x with bitround codec support. Chunk loading targets 1 to 5 MB requests, with byte-range coalescing and a 512-entry LRU byte cache. CF metadata (`grid_mapping`, `_FillValue`, `missing_value`, NaN) drives both grid detection and nodata masking. Catalog files (JSON) list datasets for institutional deployments and are how DKRZ exposes its hosted collections. + +## Where it fits + +Choose GridLook when the data lives on a non-rectilinear grid that the climate community actually uses (HEALPix, ICON triangular, Gaussian-reduced, rotated lat-lon) and you want it visualized on that native grid rather than after a regridding step. The map-hosted libraries above either require regular gridded input or assume web-map tiles, so they are not substitutes for the unstructured-grid case. The trade-off is that GridLook is its own scene, not a layer on top of a basemap; if you need a vector basemap underneath, look elsewhere. + +## Links + +- Source: [dkrz-icon/gridlook](https://github.com/dkrz-icon/gridlook) +- Hosted: [gridlook.pages.dev](https://gridlook.pages.dev) +- Authors: Andrej Fast (DKRZ), Tobi Kölling (MPI-M), Fabian Wachsmann (DKRZ), Lukas Kluft (MPI-M) +- Related: [zarrita](https://github.com/manzt/zarrita.js), d3-geo, d3-geo-projection diff --git a/docs/visualization/images/architecture-stack.dot b/docs/visualization/images/architecture-stack.dot new file mode 100644 index 0000000..0790e6b --- /dev/null +++ b/docs/visualization/images/architecture-stack.dot @@ -0,0 +1,53 @@ +// Architecture stack for dynamic datacube visualization. +// Render with: dot -Tsvg architecture-stack.dot -o architecture-stack.svg + +digraph ArchitectureStack { + rankdir=LR + bgcolor="transparent" + nodesep=0.6 + ranksep=1.0 + splines=true + + node [fontname="Helvetica,Arial,sans-serif" fontsize=11] + edge [fontname="Helvetica,Arial,sans-serif" fontsize=10 color="#546E7A"] + + // Browser-side stack as a single HTML-label node so the rows align. + // The header row labels the whole stack as the browser-side surface; + // PORTs let us anchor edges at the orchestration row specifically. + Stack [shape=plain label=< + + + + + + +
+ runs in the browser +
+ User interaction layer
+ sliders · clicks · hover · time controls +
+ Data orchestration
+ STAC discovery · request routing · state +
+ Framework
+ deck.gl · MapLibre · CesiumJS +
+ Rendering engine
+ WebGL · WebGPU · SVG · canvas +
+ >] + + Backend [label="Backend services\ntilers · resampling\nformat conversion · caches" + shape=box style="rounded,filled" + fillcolor="#BBDEFB" color="#1565C0" fontcolor="#0D47A1"] + Sources [label="Data sources\nZarr · NetCDF · COG · STAC\non cloud object storage" + shape=box style="rounded,filled" + fillcolor="#C8E6C9" color="#2E7D32" fontcolor="#1B5E20"] + + Stack:orch:e -> Backend [label="API calls"] + Backend -> Sources [label="range reads"] + Stack:orch:e -> Sources [label="direct reads\n(client-side rendering)" style=dashed] + + { rank=same; Backend; Sources; } +} diff --git a/docs/visualization/images/architecture-stack.svg b/docs/visualization/images/architecture-stack.svg new file mode 100644 index 0000000..12ce37a --- /dev/null +++ b/docs/visualization/images/architecture-stack.svg @@ -0,0 +1,87 @@ + + + + + + +ArchitectureStack + + +Stack + + +                 +runs in the browser +             + + +                 +User interaction layer +                 +sliders · clicks · hover · time controls +             + + +                 +Data orchestration +                 +STAC discovery · request routing · state +             + + +                 +Framework +                 +deck.gl · MapLibre · CesiumJS +             + + +                 +Rendering engine +                 +WebGL · WebGPU · SVG · canvas +             + + + +Backend + +Backend services +tilers · resampling +format conversion · caches + + + +Stack:e->Backend + + +API calls + + + +Sources + +Data sources +Zarr · NetCDF · COG · STAC +on cloud object storage + + + +Stack:e->Sources + + +direct reads +(client-side rendering) + + + +Backend->Sources + + +range reads + + + diff --git a/docs/visualization/images/decision-tree.dot b/docs/visualization/images/decision-tree.dot new file mode 100644 index 0000000..9013456 --- /dev/null +++ b/docs/visualization/images/decision-tree.dot @@ -0,0 +1,88 @@ +// Decision tree for selecting a Zarr visualization approach. +// Render with: dot -Tsvg decision-tree.dot -o decision-tree.svg + +digraph DecisionTree { + rankdir=TB + bgcolor="transparent" + nodesep=0.4 + ranksep=0.55 + splines=polyline + + node [fontname="Helvetica,Arial,sans-serif" fontsize=11] + edge [fontname="Helvetica,Arial,sans-serif" fontsize=10 color="#546E7A"] + + // Entry point + Start [label="Visualize Zarr\nin a browser" shape=ellipse style=filled + fillcolor="#CFD8DC" color="#37474F" fontcolor="#263238"] + + // Questions (diamonds) + Q2 [label="OK operating\na backend server?" + shape=diamond style=filled fillcolor="#FAFAFA" color="#455A64"] + Q2a [label="Existing ecosystem?" + shape=diamond style=filled fillcolor="#FAFAFA" color="#455A64"] + Q3 [label="Application\nor library?" + shape=diamond style=filled fillcolor="#FAFAFA" color="#455A64"] + Q3a [label="What shape\nof view?" + shape=diamond style=filled fillcolor="#FAFAFA" color="#455A64"] + Q4 [label="Which rendering host?" + shape=diamond style=filled fillcolor="#FAFAFA" color="#455A64"] + + // Server-side (blue) + Xpub [label="xpublish-tiles" shape=box style="rounded,filled" + fillcolor="#BBDEFB" color="#1565C0" fontcolor="#0D47A1"] + Tit [label="TiTiler" shape=box style="rounded,filled" + fillcolor="#BBDEFB" color="#1565C0" fontcolor="#0D47A1"] + + // Client-side libraries (green) + Deck [label="deck.gl-raster" shape=box style="rounded,filled" + fillcolor="#C8E6C9" color="#2E7D32" fontcolor="#1B5E20"] + Ces [label="zarr-cesium" shape=box style="rounded,filled" + fillcolor="#C8E6C9" color="#2E7D32" fontcolor="#1B5E20"] + ZL [label="zarr-layer" shape=box style="rounded,filled" + fillcolor="#C8E6C9" color="#2E7D32" fontcolor="#1B5E20"] + + // Client-side applications (orange) + Brow [label="Browzarr" shape=box style="rounded,filled" + fillcolor="#FFE0B2" color="#E65100" fontcolor="#BF360C"] + Grid [label="GridLook" shape=box style="rounded,filled" + fillcolor="#FFE0B2" color="#E65100" fontcolor="#BF360C"] + + // Edges + Start -> Q2 + + Q2 -> Q2a [label="Yes, server-side"] + Q2 -> Q3 [label="No, client-side"] + + Q2a -> Xpub [label="xarray-first,\nplugin model"] + Q2a -> Tit [label="broad formats,\nSTAC, NASA-style"] + + Q3 -> Q3a [label="Application"] + Q3 -> Q4 [label="Library"] + + Q3a -> Brow [label="3D volumetric\nor general 2D"] + Q3a -> Grid [label="Climate-model\nnative grid"] + + Q4 -> Deck [label="deck.gl"] + Q4 -> Ces [label="CesiumJS"] + Q4 -> ZL [label="MapLibre/Mapbox"] + + // Legend (subgraph cluster) + subgraph cluster_legend { + label="Outcome categories" + labelloc="t" + fontname="Helvetica,Arial,sans-serif" + fontsize=10 + fontcolor="#455A64" + style="rounded,dashed" + color="#B0BEC5" + + L_server [label="Server-side tiler" shape=box style="rounded,filled" + fillcolor="#BBDEFB" color="#1565C0" fontcolor="#0D47A1" fontsize=10] + L_lib [label="Client-side library" shape=box style="rounded,filled" + fillcolor="#C8E6C9" color="#2E7D32" fontcolor="#1B5E20" fontsize=10] + L_app [label="Viewer application" shape=box style="rounded,filled" + fillcolor="#FFE0B2" color="#E65100" fontcolor="#BF360C" fontsize=10] + + L_server -> L_lib -> L_app [style=invis] + } +} diff --git a/docs/visualization/images/decision-tree.svg b/docs/visualization/images/decision-tree.svg new file mode 100644 index 0000000..5eeb2a6 --- /dev/null +++ b/docs/visualization/images/decision-tree.svg @@ -0,0 +1,206 @@ + + + + + + +DecisionTree + +cluster_legend + +Outcome categories + + + +Start + +Visualize Zarr +in a browser + + + +Q2 + +OK operating +a backend server? + + + +Start->Q2 + + + + + +Q2a + +Existing ecosystem? + + + +Q2->Q2a + + +Yes, server-side + + + +Q3 + +Application +or library? + + + +Q2->Q3 + + +No, client-side + + + +Xpub + +xpublish-tiles + + + +Q2a->Xpub + + +xarray-first, +plugin model + + + +Tit + +TiTiler + + + +Q2a->Tit + + +broad formats, +STAC, NASA-style + + + +Q3a + +What shape +of view? + + + +Q3->Q3a + + +Application + + + +Q4 + +Which rendering host? + + + +Q3->Q4 + + +Library + + + +Brow + +Browzarr + + + +Q3a->Brow + + +3D volumetric +or general 2D + + + +Grid + +GridLook + + + +Q3a->Grid + + +Climate-model +native grid + + + +Deck + +deck.gl-raster + + + +Q4->Deck + + +deck.gl + + + +Ces + +zarr-cesium + + + +Q4->Ces + + +CesiumJS + + + +ZL + +zarr-layer + + + +Q4->ZL + + +MapLibre/Mapbox + + + +L_server + +Server-side tiler + + + +L_lib + +Client-side library + + + + +L_app + +Viewer application + + + + diff --git a/docs/visualization/images/dimensionality-fan.dot b/docs/visualization/images/dimensionality-fan.dot new file mode 100644 index 0000000..4e6cc62 --- /dev/null +++ b/docs/visualization/images/dimensionality-fan.dot @@ -0,0 +1,73 @@ +// How a high-dimensional datacube reduces to common view shapes. +// Render with: dot -Tsvg dimensionality-fan.dot -o dimensionality-fan.svg + +digraph DimensionalityFan { + rankdir=LR + bgcolor="transparent" + nodesep=0.35 + ranksep=1.0 + splines=true + + node [fontname="Helvetica,Arial,sans-serif" fontsize=11] + edge [fontname="Helvetica,Arial,sans-serif" fontsize=10 color="#546E7A"] + + Cube [label=< + 4–5D datacube
+ dims: t · z · y · x · band + > shape=box3d style=filled + fillcolor="#CFD8DC" color="#37474F" fontcolor="#263238" + width=2 height=1.6] + + Map [label=< + 2D map
+ slice at one t, z, band + > shape=box style="rounded,filled" + fillcolor="#C8E6C9" color="#2E7D32" fontcolor="#1B5E20"] + + TS [label=< + 1D timeseries
+ point at x, y, z + > shape=box style="rounded,filled" + fillcolor="#C8E6C9" color="#2E7D32" fontcolor="#1B5E20"] + + Profile [label=< + Vertical profile
+ column at x, y, t + > shape=box style="rounded,filled" + fillcolor="#C8E6C9" color="#2E7D32" fontcolor="#1B5E20"] + + Anim [label=< + Animation
+ 2D map swept over t + > shape=box style="rounded,filled" + fillcolor="#FFE0B2" color="#E65100" fontcolor="#BF360C"] + + Vol [label=< + Volumetric 3D
+ full (z, y, x) at fixed t + > shape=box style="rounded,filled" + fillcolor="#FFE0B2" color="#E65100" fontcolor="#BF360C"] + + Cube -> Map [label="reduce to 2D"] + Cube -> TS [label="reduce to 1D"] + Cube -> Profile [label="reduce to 1D"] + Cube -> Anim [label="2D + time"] + Cube -> Vol [label="reduce to 3D"] + + // Legend + subgraph cluster_legend { + label="View shape" + labelloc="t" + fontname="Helvetica,Arial,sans-serif" + fontsize=10 + fontcolor="#455A64" + style="rounded,dashed" + color="#B0BEC5" + + L_static [label="Static slice" shape=box style="rounded,filled" + fillcolor="#C8E6C9" color="#2E7D32" fontcolor="#1B5E20" fontsize=10] + L_dyn [label="Time / 3D rendering" shape=box style="rounded,filled" + fillcolor="#FFE0B2" color="#E65100" fontcolor="#BF360C" fontsize=10] + L_static -> L_dyn [style=invis] + } +} diff --git a/docs/visualization/images/dimensionality-fan.svg b/docs/visualization/images/dimensionality-fan.svg new file mode 100644 index 0000000..2657497 --- /dev/null +++ b/docs/visualization/images/dimensionality-fan.svg @@ -0,0 +1,128 @@ + + + + + + +DimensionalityFan + +cluster_legend + +View shape + + + +Cube + + + + +         +4–5D datacube +         +dims: t · z · y · x · band +     + + + +Map + +         +2D map +         +slice at one t, z, band +     + + + +Cube->Map + + +reduce to 2D + + + +TS + +         +1D timeseries +         +point at x, y, z +     + + + +Cube->TS + + +reduce to 1D + + + +Profile + +         +Vertical profile +         +column at x, y, t +     + + + +Cube->Profile + + +reduce to 1D + + + +Anim + +         +Animation +         +2D map swept over t +     + + + +Cube->Anim + + +2D + time + + + +Vol + +         +Volumetric 3D +         +full (z, y, x) at fixed t +     + + + +Cube->Vol + + +reduce to 3D + + + +L_static + +Static slice + + + +L_dyn + +Time / 3D rendering + + + + diff --git a/docs/visualization/images/scale-funnel.dot b/docs/visualization/images/scale-funnel.dot new file mode 100644 index 0000000..b6fd556 --- /dev/null +++ b/docs/visualization/images/scale-funnel.dot @@ -0,0 +1,25 @@ +// Scale hierarchy for large datacube visualization. +// Render with: dot -Tsvg scale-funnel.dot -o scale-funnel.svg + +digraph ScaleFunnel { + rankdir=TB + bgcolor="transparent" + nodesep=0.2 + ranksep=0.45 + + node [fontname="Helvetica,Arial,sans-serif" fontsize=11 shape=box style="rounded,filled" height=0.55] + edge [fontname="Helvetica,Arial,sans-serif" fontsize=10 color="#546E7A" arrowhead=none] + + Archive [label=<Archive · TB–PB · thousands of files> + fillcolor="#BBDEFB" color="#1565C0" fontcolor="#0D47A1" width=5.0] + Backend [label=<Backend cache · GB · tiles & query results> + fillcolor="#90CAF9" color="#1565C0" fontcolor="#0D47A1" width=3.8] + CDN [label=<CDN / edge · MB · per-tile PNG / MVT> + fillcolor="#C8E6C9" color="#2E7D32" fontcolor="#1B5E20" width=2.8] + Browser [label=<Browser cache · kB · recent tiles> + fillcolor="#A5D6A7" color="#2E7D32" fontcolor="#1B5E20" width=2.0] + Screen [label=<Screen · ~1 M pixels> + fillcolor="#CFD8DC" color="#37474F" fontcolor="#263238" width=1.4] + + Archive -> Backend -> CDN -> Browser -> Screen +} diff --git a/docs/visualization/images/scale-funnel.svg b/docs/visualization/images/scale-funnel.svg new file mode 100644 index 0000000..ff0a346 --- /dev/null +++ b/docs/visualization/images/scale-funnel.svg @@ -0,0 +1,67 @@ + + + + + + +ScaleFunnel + + +Archive + +Archive +  ·  TB–PB  ·  thousands of files + + + +Backend + +Backend cache +  ·  GB  ·  tiles & query results + + + +Archive->Backend + + + + +CDN + +CDN / edge +  ·  MB  ·  per-tile PNG / MVT + + + +Backend->CDN + + + + +Browser + +Browser cache +  ·  kB  ·  recent tiles + + + +CDN->Browser + + + + +Screen + +Screen +  ·  ~1 M pixels + + + +Browser->Screen + + + + diff --git a/docs/visualization/images/static-vs-dynamic.svg b/docs/visualization/images/static-vs-dynamic.svg new file mode 100644 index 0000000..26fe0b7 --- /dev/null +++ b/docs/visualization/images/static-vs-dynamic.svg @@ -0,0 +1,183 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Static + decisions baked in at render time + + + + + + + + + + 28 °C + 14 °C + 0 °C + + + + SST · 2024-01-15 12:00 UTC + + + + + fixed extent · fixed colormap · fixed timestamp + + one PNG/PDF · regenerate to change anything + + no interaction + + + + + + + + Dynamic + decisions deferred to the user + + + + + + + + + + SST + · + 2024-07-12 12:00 UTC + + + + + + + + + + + + + + − + + + + + + + LAYER + Sea surface temp + + + + + COLORMAP + Viridis + + + + + + + lat 34.21, lon -118.45 + SST: 18.5 °C + + + + + + + + + + + + + + + + + + + + + + + + 2024-01-01 + 2024-12-31 + + + + + + + + + + + + + + + max + min + + + + + zoom / pan / hover · swap colormap · clamp range + + scrub through time · toggle layers + + data fetched on demand from the source + + + diff --git a/docs/visualization/numeric-data-layer.md b/docs/visualization/numeric-data-layer.md index c6f99ba..0f1a29d 100644 --- a/docs/visualization/numeric-data-layer.md +++ b/docs/visualization/numeric-data-layer.md @@ -1,3 +1,10 @@ -# Numeric data layer +# numeric-data-layer -Content coming soon. See [source repo](https://github.com/developmentseed/numeric-data-layer) for now. +numeric-data-layer is a Development Seed deck.gl layer for rendering numeric raster data. It has been **superseded by [deck.gl-raster](deck.gl-raster.md)**, which absorbs and extends the same approach with a modular shader-module pipeline, GeoZarr support, and a generic tile-source abstraction shared with COG. + +If you are starting a new project, use [deck.gl-raster](deck.gl-raster.md). numeric-data-layer is documented here for historical context and for users with existing integrations. + +## Links + +- Source: [developmentseed/numeric-data-layer](https://github.com/developmentseed/numeric-data-layer) +- Successor: [deck.gl-raster](deck.gl-raster.md) diff --git a/docs/visualization/overview.md b/docs/visualization/overview.md index 3d9f359..f7a7fab 100644 --- a/docs/visualization/overview.md +++ b/docs/visualization/overview.md @@ -2,6 +2,8 @@ Datacube visualizations can be sub-divided into two categories: static and dynamic. The contents of a static visualization do not change after creation, similar to printing a map to a piece of paper. Dynamic visualizations respond to user input. For example, a user could change the visualization by zooming in, panning to a different location, or changing the color scheme. This explanation focuses on dynamic visualizations. +![Side-by-side comparison: a static map with a fixed extent, colormap, and timestamp baked in, next to a dynamic viewer with zoom controls, a layer toggle, a colormap dropdown, a hover readout, a time slider, and adjustable colorbar range handles.](images/static-vs-dynamic.svg) + ## What does it take to dynamically visualize data? - Rendering engine: A library that displays the data, commonly consisting of a graphics context (e.g., WebGL, SVG, or DOM elements), drawing primitives (points, lines, shapes, textures, meshes), and coordinate systems (e.g., screen space, world space, data space transformations) @@ -11,6 +13,7 @@ Datacube visualizations can be sub-divided into two categories: static and dynam - Data orchestration: The data orchestration layer of the user interface manages the flow of data from sources to the visualization framework, handling API integration (such as connecting to STAC catalogs to discover available datasets) and coordination with backend services (like tiling servers that process and serve datacube slices). For example, when a user selects a specific time range and geographic region, the data orchestration layer translates this selection into the appropriate API calls and ensures the resulting data reaches the framework in the correct format. - User interaction layer: The user interaction layer of the user interface handles direct user interactions and visual feedback. It provides interface controls (such as time sliders, layer toggles, and zoom controls), processes user input events (mouse clicks, touches, keyboard shortcuts), and updates the visualization state accordingly. The interaction layer also manages visual feedback like hover effects, selection highlighting, and loading indicators to keep users informed about the system's response to their actions. +![Architecture stack for dynamic datacube visualization: a four-row in-browser stack (user interaction → data orchestration → framework → rendering engine) talks to external backend services via API calls, and the backend reads source data with range requests. In purely client-side setups the orchestration layer reads directly from the source.](images/architecture-stack.svg) ## What makes datacube visualization different? @@ -22,3 +25,45 @@ Dynamic datacube visualizations require more complex considerations than visuali - Complexity in data sources: Datacubes may be stored in many different file formats (e.g., GeoTIFF, GRIB, COG, NetCDF, Zarr, etc.) which adds complexity to the backend services. The sources can also span cloud providers (e.g., GCS, AWS) and involve protocols like OPeNDAP. - Coordinate reference systems (CRS): Datacubes often involve complex coordinate reference system transformations between data coordinates, geographic projections, and display coordinates. - Temporal considerations: Animation frameworks, temporal interpolation, and playback controls are more complex for datacubes, especially when integrating multiple data sources with different temporal resolutions or misaligned time coordinates. + +![Dimensionality fan: a 4–5D datacube (t · z · y · x · band) reduces to a 2D map, a 1D timeseries, a 1D vertical profile, an animation (2D map swept over t), or a volumetric 3D rendering depending on which dimensions are held fixed and which are displayed.](images/dimensionality-fan.svg) + +![Scale funnel: a TB–PB archive of thousands of files narrows through a backend tile/query cache, a CDN, and a browser cache down to the ~1 M pixels of the screen.](images/scale-funnel.svg) + +## Choosing an approach + +The Zarr-on-the-web tools documented in this section split along three axes: server-side dynamic tiling vs client-side rendering, library vs viewer application, and (within libraries) which rendering host you target. The decision tree below works through those choices in order. + +This section assumes you want a *dynamic* visualization (the data is in Zarr or another cloud-optimized format and you'll render against it on demand). Pre-rendering raster tile pyramids to flat PNGs is technically possible but increasingly an anti-pattern: dynamic tilers behind a CDN match the performance, and pre-rendering forces colormap and styling decisions at generation time, freezes them, and creates a regeneration pipeline to maintain. If you genuinely have a one-shot static visualization that will never change, PMTiles is a reasonable static option, but none of the Zarr-aware tools below apply to that case. + +![Decision tree for selecting a Zarr visualization approach](images/decision-tree.svg) + +### Short-circuits + +A few requirements pin the choice on a single feature, before the tree applies: + +| If you need... | Choose | Why | +|---|---|---| +| True 3D volumetric raycasting (not just 3D slices) | [Browzarr](browzarr.md) | Only one with full raycasting via `volFragment.glsl` | +| Animated vector fields (U/V particles) | [zarr-cesium](zarr-cesium.md) | Only one with `cesium-wind-layer` integration | +| Native rendering of HEALPix, ICON triangular, Gaussian-reduced, or rotated lat-lon grids | [GridLook](gridlook.md) | Only one with grid-topology detectors | +| One renderer for both COG and Zarr in the same scene | [deck.gl-raster](deck.gl-raster.md) | Generic `TilesetDescriptor` shared across COG and Zarr backends | +| GeoZarr metadata parsed out of the box (multiscales, CRS as EPSG/WKT2/PROJJSON) | [deck.gl-raster](deck.gl-raster.md) | Only one with an explicit `@developmentseed/geozarr` parser | + +### Tree, in words + +For users on a non-image-rendering reader: + +1. **Operating a backend server is OK?** If yes, see the [server-side comparison](ecosystem-comparison.md) for [TiTiler](titiler/overview.md) vs [xpublish-tiles](xpublish/xpublish-tiles.md). If no, continue client-side. +2. **Application or library?** A pre-built application means point a hosted viewer at a Zarr URL: choose [Browzarr](browzarr.md) for general exploration or 3D cubes, or [GridLook](gridlook.md) for non-rectilinear climate-model grids. A library means embed a renderer into your own application: continue. +3. **Which rendering host?** [deck.gl](deck.gl-raster.md) for the deck.gl ecosystem, [CesiumJS](zarr-cesium.md) for a 3D globe, or [zarr-layer](zarr-layer.md) for MapLibre/Mapbox. + +[`@carbonplan/maps`](carbonplan-maps.md) is documented for catalog completeness and remains the rendering backbone of CarbonPlan's published visualizations, but for new work it isn't in the recommendation path: it requires data to be pre-baked into Web Mercator `ndpyramid` pyramids, which commits you to a regeneration pipeline and the same frozen-styling trade-offs that pre-rendering normally implies. + +The Graphviz sources for the figures on this page live in `docs/visualization/images/` (`decision-tree.dot`, `architecture-stack.dot`, `dimensionality-fan.dot`, `scale-funnel.dot`). The static-vs-dynamic comparison is hand-authored SVG. Regenerate the four Graphviz SVGs in one go: + +```sh +for f in decision-tree architecture-stack dimensionality-fan scale-funnel; do + dot -Tsvg "docs/visualization/images/$f.dot" -o "docs/visualization/images/$f.svg" +done +``` diff --git a/docs/visualization/titiler/apis/index.md b/docs/visualization/titiler/apis/index.md index b96748f..ca1b99f 100644 --- a/docs/visualization/titiler/apis/index.md +++ b/docs/visualization/titiler/apis/index.md @@ -1,61 +1,74 @@ -# TiTiler API Reference +# TiTiler API reference -This section provides interactive API documentation for the TiTiler ecosystem components. Each application has its own specialized API while sharing common patterns from the core framework. +This section provides interactive API documentation for the four TiTiler applications most relevant to datacube serving. Each application has its own specialized API while sharing common patterns from `titiler.core`. -## Available Applications +## Available applications
-- **TiTiler Core** +- **titiler.core (reference application)** --- - Foundation API for COGs and STAC items. Base functionality that all other applications extend. + Foundation API for COGs, STAC, MosaicJSON, and (since 2.0) Zarr. The base everything else builds on. - [:octicons-arrow-right-24: Core API Reference](titiler-core.md) + [:octicons-arrow-right-24: titiler.core API reference](titiler-core.md) -- **TiTiler CMR** +- **titiler-cmr** --- - NASA CMR-focused application for satellite data collections with time series support. + NASA Common Metadata Repository application with dual `/xarray/` and `/rasterio/` backends, granule filtering, and per-DAAC EDL credentials. - [:octicons-arrow-right-24: CMR API Reference](titiler-cmr.md) + [:octicons-arrow-right-24: titiler-cmr API reference](titiler-cmr.md) -- **TiTiler Multidim** +- **titiler-multidim** --- - Multi-dimensional dataset processing for NetCDF, Zarr, and scientific data formats. + NASA VEDA multidimensional application built on `titiler.xarray`. Adds Redis caching, OpenTelemetry tracing, and Icechunk support. - [:octicons-arrow-right-24: Multidim API Reference](titiler-multidim.md) + [:octicons-arrow-right-24: titiler-multidim API reference](titiler-multidim.md) + +- **titiler-eopf** + + --- + + ESA Copernicus / EOPF application with a GeoZarr DataTree reader. Deploys as either a TiTiler REST API or an OpenEO backend. + + [:octicons-arrow-right-24: titiler-eopf API reference](titiler-eopf.md)
-## Common API Patterns +## Common API patterns -All TiTiler applications follow consistent patterns: +All TiTiler applications share most of these patterns: ### Authentication -- **API Keys**: Some endpoints require authentication via API keys -- **CORS**: Cross-Origin Resource Sharing is configured for web applications -- **Rate Limiting**: Default rate limits may apply -### Response Formats -- **JSON**: Metadata, statistics, and configuration responses -- **Images**: PNG, JPEG, WebP tiles and previews -- **GeoJSON**: Spatial data responses -- **HTML**: Interactive viewers and documentation +- **API keys** — some endpoints require authentication via API keys. +- **Earthdata Login** — `titiler-cmr` uses EDL bearer tokens, scoped per DAAC. +- **CORS** — Cross-Origin Resource Sharing is configured for web applications. +- **Rate limiting** — default rate limits may apply on shared deployments. -### Error Handling -- **HTTP Status Codes**: Standard codes (200, 400, 404, 500, etc.) -- **Error Messages**: Detailed error descriptions in JSON format -- **Validation**: Parameter validation with helpful error messages +### Response formats + +- **JSON** — metadata, statistics, and configuration responses. +- **Images** — PNG, JPEG, WebP tiles and previews. +- **GeoJSON** — spatial responses, including granule footprints in `titiler-cmr`. +- **HTML** — interactive viewers and OpenAPI documentation. + +### Error handling + +- **HTTP status codes** — standard codes (200, 400, 404, 500, etc.). +- **Error messages** — detailed error descriptions in JSON. +- **Validation** — parameter validation with helpful error messages. ### Performance -- **Caching**: Response caching for improved performance -- **Compression**: Automatic response compression -- **Streaming**: Efficient data streaming for large responses + +- **Caching** — response caching via Redis (in `titiler-multidim`) and via standard CDN front-ends. +- **Compression** — automatic response compression. +- **Streaming** — efficient streaming for large responses. !!! tip "Testing APIs" - Use the embedded interactive documentation to test endpoints directly in your browser. Each API reference page includes a full interactive interface for exploring available endpoints and parameters. + Use the embedded interactive documentation on each per-application page to test endpoints directly in your browser. Each reference page includes the deployed Swagger UI for exploring endpoints and parameters. diff --git a/docs/visualization/titiler/apis/titiler-cmr.md b/docs/visualization/titiler/apis/titiler-cmr.md index fe25bc1..27d1472 100644 --- a/docs/visualization/titiler/apis/titiler-cmr.md +++ b/docs/visualization/titiler/apis/titiler-cmr.md @@ -1,17 +1,88 @@ -# TiTiler CMR API Reference +# titiler-cmr API reference -TiTiler CMR is a NASA-focused application that accepts Concept IDs and uses the Common Metadata Repository (CMR) to discover and serve associated granules as tiles. +`titiler-cmr` is a NASA-focused application that accepts CMR Concept IDs and uses the Common Metadata Repository to discover and serve associated granules as tiles. Currently at v1.0.x, MIT-licensed, maintained by Development Seed. -## Key Features +## Key features -- **CMR Integration**: Direct integration with NASA's Common Metadata Repository -- **Earth Science Data**: Specialized for NASA Earth science data collections -- **Time Series Support**: Built-in temporal analysis capabilities -- **Granule Discovery**: Automatic discovery and aggregation of data granules +- **Dual-backend architecture** — `/xarray/*` routes for NetCDF/HDF5 via `titiler.xarray`, `/rasterio/*` routes for COG-style data via `titiler.core` + `titiler.mosaic`. Selected at the URL prefix; users no longer have to pick a backend out of band. +- **CMR integration** — direct integration with NASA's Common Metadata Repository for granule discovery. +- **Granule filtering** — request-time filters including `orbit_number`, `cloud_cover`, `attribute`, `sort_key`, plus rio-tiler-style `skipcovered`, `coverage_tolerance`, and `exitwhenfull`. +- **Cloud-native I/O** — the xarray backend now uses `obstore` and `obspec-utils` for cloud-native data loading, replacing the older fsspec path. +- **Per-DAAC EDL credentials** — Earthdata Login bearer-token credentials cached per DAAC (5-minute refresh), replacing the older AWS IAM role flow. +- **Expression translation** — named-asset and named-variable expressions (e.g. `B04/B03`) are automatically rewritten to `b1/b2` form for `rio-tiler>=9` compatibility. +- **GeoJSON granule output** — `/granules` endpoints accept `f=geojson` to return FeatureCollections of granule footprints. +- **Multi-variable xarray** — `variables` parameter accepts repeated values for multi-band requests; superseded the singular `variable` parameter. -## Interactive API Documentation +## Backend dependency -The complete, interactive API documentation from the OpenVEDA Cloud deployment is below. Please be kind with this API. +`titiler-cmr` v1.0+ depends on **both** `titiler.core` and `titiler.xarray` (plus `titiler.mosaic`): + +```toml +"titiler-core>=2.0,<3.0" +"titiler-mosaic>=2.0,<3.0" +"titiler-xarray>=2.0,<3.0" +``` + +This resolves the older [issue #35](https://github.com/developmentseed/titiler-cmr/issues/35) by adopting a dual-backend pattern rather than picking one or the other. + +## Examples + +The same endpoint families are mirrored across both backends; the path prefix selects which one runs. + +### Tile request + +=== "xarray backend" + + ```http + GET /xarray/tiles/WebMercatorQuad/8/40/95 + ?collection_concept_id=C2036881712-POCLOUD + &variables=analysed_sst + &temporal=2024-06-15T12:00:00Z + &colormap_name=thermal + &rescale=270,305 + ``` + + Backed by `titiler.xarray`. Use for NetCDF, HDF5, and Zarr granules. Pass repeated `variables=` for multi-variable requests. + +=== "rasterio backend" + + ```http + GET /rasterio/tiles/WebMercatorQuad/8/40/95 + ?collection_concept_id=C2021957295-LPCLOUD + &assets=B04,B03,B02 + &temporal=2024-06-15 + &rescale=0,3000 + ``` + + Backed by `titiler.core` and `titiler.mosaic`. Use for COG-style granules. Multi-band selection via `assets=`; band-math via `expression=`. + +### Statistics + +=== "xarray backend" + + ```http + GET /xarray/statistics + ?collection_concept_id=C2036881712-POCLOUD + &variables=analysed_sst + &temporal=2024-06-15T12:00:00Z + &bbox=-130,30,-60,50 + ``` + +=== "rasterio backend" + + ```http + GET /rasterio/statistics + ?collection_concept_id=C2021957295-LPCLOUD + &assets=B04 + &temporal=2024-06-15 + &bbox=-130,30,-60,50 + ``` + +The granule-search routes (`/bbox/.../granules`, `/point/.../granules`, `/tiles/.../granules`) and the `/timeseries/*` family are backend-independent and live at the application root rather than under the `/xarray/` or `/rasterio/` prefixes. + +## Interactive API documentation + +The complete, interactive API documentation from the OpenVEDA Cloud staging deployment is below. Please be kind with this API. -## Quick Links +## Quick links - [Open API docs in new tab](https://staging.openveda.cloud/api/titiler-cmr/api.html){:target="_blank"} - [OpenAPI Schema JSON](https://staging.openveda.cloud/api/titiler-cmr/api){:target="_blank"} +- [Source on GitHub](https://github.com/developmentseed/titiler-cmr){:target="_blank"} + +## Main endpoint categories -## Main Endpoint Categories +- `/xarray/{endpoint}` — xarray-backed routes (`/tiles`, `/statistics`, `/part`, `/preview`, etc.) for NetCDF and HDF5 granules. +- `/rasterio/{endpoint}` — rasterio/rio-tiler-backed routes for COG-style granules. +- `/granules/*` and granule-search routes: + - `/bbox/{minx},{miny},{maxx},{maxy}/granules` + - `/point/{lon},{lat}/granules` + - `/tiles/{tileMatrixSetId}/{z}/{x}/{y}/granules` +- `/timeseries/*` — dedicated time-series endpoints for temporal analysis across granules. +- Standard helper routes: `/algorithms`, `/colorMaps`, `/tileMatrixSets`, `/conformance`. -- **Collections**: `/collections/{collection_id}` - Work with CMR collections -- **Statistics**: `/collections/{collection_id}/statistics` - Extract statistical data -- **Time Series**: `/collections/{collection_id}/timeseries` - Temporal analysis -- **Tiles**: `/collections/{collection_id}/tiles` - Generate map tiles -- **Items**: Individual granule access and processing +!!! tip "Legacy URL compatibility" + Older `/collections/{collection_id}/...` paths still resolve via 301/308 redirects to the new `/xarray/` or `/rasterio/` prefixes, with parameter renames applied automatically (`concept_id` → `collection_concept_id`, `datetime` → `temporal`, `bands` → `assets`, `bands_regex` → `assets_regex`). Existing integrations should keep working but new code should target the canonical routes. !!! tip "Authentication" - Some endpoints may require authentication depending on the data collection's access restrictions. + Some endpoints may require Earthdata Login authentication depending on the data collection's access restrictions. Bearer tokens are scoped per DAAC and cached server-side. diff --git a/docs/visualization/titiler/apis/titiler-core.md b/docs/visualization/titiler/apis/titiler-core.md index 30679f3..850ec59 100644 --- a/docs/visualization/titiler/apis/titiler-core.md +++ b/docs/visualization/titiler/apis/titiler-core.md @@ -1,18 +1,36 @@ -# TiTiler Core API Reference +# titiler.core API reference -TiTiler Core provides the foundational API patterns used across all TiTiler applications. It handles Cloud Optimized GeoTIFFs (COGs) and SpatioTemporal Asset Catalog (STAC) items. +`titiler.core` is the foundation of the TiTiler ecosystem: a FastAPI framework with factory patterns for building dynamic tile servers, plus the reference application that combines it with `titiler.mosaic` and (since 2.0) `/zarr/*` endpoints from `titiler.xarray`. Currently at v2.0.x, requires `rio-tiler>=9,<10` and Python 3.11–3.14. -## Key Features +## Key features -- **COG Support**: Optimized Cloud Optimized GeoTIFF processing -- **STAC Integration**: Full SpatioTemporal Asset Catalog support -- **OGC Compliance**: Standards-compliant tile serving -- **Extensible Architecture**: Foundation for specialized applications -- **High Performance**: Optimized for cloud-native workflows +- **COG support** — Cloud Optimized GeoTIFF processing via `rio-tiler`. +- **STAC integration** — full SpatioTemporal Asset Catalog support, including remote item URLs. +- **Zarr endpoints** — `/zarr/*` is now part of the default reference application via `titiler.xarray`. +- **OGC compliance** — XYZ, WMTS, and partial OGC Maps API support. +- **Extensible architecture** — factory pattern that the other applications (`titiler-cmr`, `titiler-multidim`, `titiler-eopf`) all build on. -## Interactive API Documentation +## Notable changes in 2.0 -The complete, interactive API documentation from the Development demo deployment is below. Please be kind with this API. +- `256x256` is no longer the implicit default tile size; `/tilejson.json` defaults to `tilesize=512` and `/map.html` to `tilesize=256`, and `tilesize` is an optional query parameter on tile and tilejson endpoints. +- `MultiBandTilerFactory` removed. Multi-band selection is handled through the standard factory. +- The `@{scale}x` suffix on tile endpoints removed; use `tilesize` instead. +- Deprecated parsers `AssetsBidxParams`, `BandsParams`, and related dependency helpers removed. +- WMTS endpoints expose tile dependencies in OpenAPI; performance improvements landed in 2.0.1. + +## Installation + +The bare `titiler` PyPI package was dropped in late 2025. Install subpackages directly: + +```bash +pip install titiler.core titiler.mosaic titiler.xarray +# or, for the bundled reference application: +pip install titiler.application +``` + +## Interactive API documentation + +The complete, interactive API documentation from the public demo deployment is below. Please be kind with this API. -## Quick Links +## Quick links - [Open API docs in new tab](https://titiler.xyz/api.html){:target="_blank"} - [OpenAPI Schema JSON](https://titiler.xyz/api){:target="_blank"} -- [TiTiler Demo Landing Page](https://titiler.xyz/){:target="_blank"} +- [TiTiler demo landing page](https://titiler.xyz/){:target="_blank"} +- [Source on GitHub](https://github.com/developmentseed/titiler){:target="_blank"} -## Main Endpoint Categories +## Main endpoint categories -- **COG Endpoints**: `/cog/*` - Cloud Optimized GeoTIFF processing -- **STAC Endpoints**: `/stac/*` - SpatioTemporal Asset Catalog integration -- **Mosaic Endpoints**: `/mosaicjson/*` - Multi-source mosaicking -- **Algorithms**: `/algorithms` - Available processing algorithms -- **Color Maps**: `/colorMaps` - Available visualization color schemes -- **TMS**: `/tileMatrixSets` - Supported tiling schemes +- `/cog/*` — Cloud Optimized GeoTIFF processing. +- `/stac/*` — SpatioTemporal Asset Catalog integration. +- `/mosaicjson/*` — Multi-source mosaicking via `titiler.mosaic`. +- `/zarr/*` — Zarr / xarray reading via `titiler.xarray` (new standard endpoint as of 2.0). +- `/algorithms` — Available processing algorithms. +- `/colorMaps` — Available colormaps. +- `/tileMatrixSets` — Supported tiling schemes. -!!! info "Foundation Layer" - TiTiler Core serves as the foundation that all other TiTiler applications build upon, providing consistent API patterns and core functionality. +!!! info "Foundation layer" + `titiler.core` is the foundation that the other TiTiler applications build on. Anything documented for the reference application here is generally available, possibly under a different route prefix, in the application-specific pages. diff --git a/docs/visualization/titiler/apis/titiler-eopf.md b/docs/visualization/titiler/apis/titiler-eopf.md new file mode 100644 index 0000000..9cab748 --- /dev/null +++ b/docs/visualization/titiler/apis/titiler-eopf.md @@ -0,0 +1,57 @@ +# titiler-eopf API reference + +`titiler-eopf` is a TiTiler application purpose-built for Earth Observation Processing Framework (EOPF) datasets in the ESA Copernicus ecosystem. It serves Sentinel and other EOPF-shaped data from cloud-hosted Zarr stores via either a standard TiTiler REST API or an OpenEO-compatible backend, from the same image. Apache 2.0-licensed; v0.8.x; maintained by the [EOPF-Explorer](https://github.com/EOPF-Explorer) organization with development from Development Seed. + +## Key features + +- **GeoZarr-native reader** — a custom `GeoZarrReader` understands hierarchical Zarr DataTrees (xarray-style groups) and detects the GeoZarr v1 conventions, lifting spatial metadata out of Zarr attributes rather than expecting flat 2D arrays. +- **EOPF data conventions** — endpoints follow `/collections/{collection_id}/items/{item_id}/...` aligned with Copernicus naming; the backing store is configured via `TITILER_EOPF_STORE_URL` (S3, Azure, GCS, ...) using `obstore`. +- **Dual REST + OpenEO deployment** — the same image can run as a TiTiler REST API (`titiler.eopf.main:app`) or as an OpenEO backend (`titiler.eopf.openeo.main:app`) via the upstream `titiler-openeo` package and Helm chart. +- **EOPF-specific factory extensions** — `DatasetMetadataExtension` (DataTree introspection), `EOPFChunkVizExtension` (Zarr chunk visualization), `EOPFViewerExtension` (map viewer), `EOPFwmtsExtension` (WMTS service). +- **STAC API integration** — collection-level mosaic tiling when a STAC API is configured, via `titiler.stacapi`. + +## Backend dependency + +```toml +"titiler-core>=2.0,<3.0" +"titiler-xarray>=2.0,<3.0" +"titiler-extensions>=2.0,<3.0" +"titiler-stacapi>=2.0,<3.0" +"zarr>=3.1.3" +``` + +Optional: `titiler-openeo` for OpenEO-mode deployment. + +## Interactive API documentation + +A live deployment is operated by ESA Copernicus Dataspace; consult the project README for the current endpoint URL. The OpenAPI surface is also available locally when running the app. + +## Quick links + +- [Source on GitHub](https://github.com/EOPF-Explorer/titiler-eopf){:target="_blank"} +- [Copernicus EOPF Explorer](https://api.explorer.eopf.copernicus.eu/stac/){:target="_blank"} — the live STAC API used by this application. +- [OpenEO API spec](https://openeo.org/){:target="_blank"} — for the alternate deployment mode. + +## Main endpoint categories + +Item-level routes: + +- `/collections/{collection_id}/items/{item_id}/info` +- `/collections/{collection_id}/items/{item_id}/tiles/{z}/{x}/{y}` +- `/collections/{collection_id}/items/{item_id}/tilejson.json` +- `/collections/{collection_id}/items/{item_id}/preview` +- `/collections/{collection_id}/items/{item_id}/point` +- `/collections/{collection_id}/items/{item_id}/part` — bounding-box crops. +- `/collections/{collection_id}/items/{item_id}/dataset/`, `/dataset/dict`, `/dataset/groups` — DataTree introspection. + +Collection-level mosaic (when a STAC API is configured): + +- `/collections/{collection_id}/tiles/{z}/{x}/{y}` + +Standard OGC endpoints: `/conformance`, `/tilematrixsets`, `/colormaps`, `/algorithms`. + +Operational: `/_mgmt/ping`, `/_mgmt/health`, `/_mgmt/cache`. + +## Where it fits + +Choose `titiler-eopf` when working with ESA Copernicus / EOPF data, GeoZarr-shaped hierarchical Zarr stores, or when an OpenEO-compatible deployment is part of the requirement. For NASA CMR data, see [titiler-cmr](titiler-cmr.md). For VEDA-shaped multidimensional workflows, see [titiler-multidim](titiler-multidim.md). For generic COG and STAC, the [reference titiler.application](titiler-core.md) is a better fit. diff --git a/docs/visualization/titiler/apis/titiler-multidim.md b/docs/visualization/titiler/apis/titiler-multidim.md index 5d24cfc..acd35b2 100644 --- a/docs/visualization/titiler/apis/titiler-multidim.md +++ b/docs/visualization/titiler/apis/titiler-multidim.md @@ -1,18 +1,28 @@ -# TiTiler Multidim API Reference +# titiler-multidim API reference -TiTiler Multidim is a comprehensive application built on `titiler.xarray` specifically designed for multi-dimensional datasets like NetCDF and Zarr files. +`titiler-multidim` is a TiTiler application built on `titiler.xarray`, designed for multi-dimensional datasets like NetCDF, Zarr, and (since v0.7) Icechunk repositories. Originally framed as a prototype demonstrating advanced multidimensional capabilities, it is now operationally deployed via NASA's VEDA platform and tracks `titiler.xarray` v0.25.x. -## Key Features +## Key features -- **Multi-dimensional Support**: Native handling of 3D, 4D, and 5D datasets -- **Temporal Processing**: Advanced time-series analysis and animation support -- **Performance Optimizations**: Redis caching and optimized chunking strategies -- **Scientific Data Formats**: NetCDF, Zarr, HDF, and other research data formats -- **VEDA Integration**: Optimized for NASA's VEDA platform infrastructure +- **Multi-dimensional support** — native handling of 3D, 4D, and 5D datasets via `titiler.xarray`. +- **Icechunk support** — full `opener_icechunk()` reads icechunk repositories locally or from S3 (added in v0.7.0). Includes an authorization config for virtual chunk access (`authorized_chunk_access`). +- **Zarr v3** — works against Zarr v3 stores via `zarr>=3.1`. +- **Redis caching** — optional response caching via `TITILER_MULTIDIM_ENABLE_CACHE`. Uses `fakeredis` in tests. +- **OpenTelemetry tracing** — X-Ray and OTel tracing enabled for Lambda and other tracing-aware deployments (added in v0.6.0). +- **VEDA integration** — production deployments orchestrated via [NASA-IMPACT/veda-deploy](https://github.com/NASA-IMPACT/veda-deploy); PRs labelled `deploy-dev` trigger ephemeral environments. -## Interactive API Documentation +## Backend dependency -The complete, interactive API documentation from the OpenVEDA Cloud deployment is below. Please be kind with this API. +```toml +"titiler-core>=0.25.0,<0.26" +"titiler-xarray>=0.25.0,<0.26" +``` + +Note the version pinning here is to the older `titiler.xarray` 0.25 line rather than the 2.x line used by `titiler.application` and `titiler-cmr`. Bumping to `titiler.xarray` 2.x is tracked in the repository. + +## Interactive API documentation + +The complete, interactive API documentation from the OpenVEDA Cloud staging deployment is below. Please be kind with this API. -## Quick Links +## Quick links - [Open API docs in new tab](https://staging.openveda.cloud/api/titiler-multidim/api.html){:target="_blank"} - [OpenAPI Schema JSON](https://staging.openveda.cloud/api/titiler-multidim/api){:target="_blank"} +- [Source on GitHub](https://github.com/developmentseed/titiler-multidim){:target="_blank"} + +## Main endpoint categories + +Inherited from `titiler.xarray`'s `BaseTilerFactory`: + +- `/info` — dataset metadata and structure. +- `/statistics` — statistical analysis across selected dimensions. +- `/tiles/{z}/{x}/{y}` and `/tilejson.json` — map tile generation and tile metadata. + +Custom multidim endpoints (in `factory.py`): + +- `/variables` — list of variables in the dataset. +- `/histogram` — value histogram for a selected variable. +- `/{tileMatrixSetId}/map` — interactive map viewer for the dataset. + +Operational: -## Main Endpoint Categories +- `/healthz` — health check. -- **Dataset Info**: `/info` - Dataset metadata and structure -- **Statistics**: `/statistics` - Statistical analysis across dimensions -- **Tiles**: `/tiles/{z}/{x}/{y}` - Map tile generation -- **Temporal Selection**: Time-based data slicing and selection -- **Dimensional Analysis**: Multi-dimensional data exploration -- **Rendering**: Advanced visualization and color mapping +Format support: NetCDF (via `h5netcdf`), Zarr v3 (via `zarr>=3.1`), Icechunk (via the bundled opener), and virtual references (via `obstore` and `virtualizarr` in dev dependencies). -!!! note "Prototype Application" - TiTiler Multidim serves as a prototype application demonstrating advanced multidimensional data processing capabilities with various optimizations for production use. +!!! note "Status" + `titiler-multidim` was historically described as a prototype. It is now deployed in production via VEDA and is the recommended path for VEDA-shaped multidimensional workflows. The "prototype" framing has been retired. diff --git a/docs/visualization/titiler/overview.md b/docs/visualization/titiler/overview.md index b45393b..968e922 100644 --- a/docs/visualization/titiler/overview.md +++ b/docs/visualization/titiler/overview.md @@ -1,44 +1,44 @@ -# Titiler Ecosystem Overview +# Titiler ecosystem overview -The TiTiler ecosystem is a comprehensive suite of Python tools for creating dynamic tile servers from geospatial datasets. The components are organized by their primary function and built upon a layered architecture that provides flexibility and specialization for different use cases. +The TiTiler ecosystem is a layered Python stack for building dynamic tile servers from geospatial datasets. Each component fills a specific role: a foundation library, several focused extensions, and a set of opinionated applications targeting concrete data ecosystems (NASA CMR, NASA VEDA, ESA EOPF, generic COG/STAC). Components are released independently and pinned to compatible major-version ranges. -## Architecture and Relationships +For a side-by-side comparison with the Xpublish ecosystem, see the [Dynamic tiling ecosystem comparison](../ecosystem-comparison.md). -The TiTiler ecosystem follows a layered architecture that promotes code reuse and specialization: +## Foundation -### Foundation Layer -- **[rio-tiler](https://github.com/cogeotiff/rio-tiler)**: Core tile generation engine -- **[titiler.core](https://github.com/developmentseed/titiler/tree/main/src/titiler/core)**: Base FastAPI framework and patterns +- **[rio-tiler](https://github.com/cogeotiff/rio-tiler)** — core tile generation engine. TiTiler 2.x requires `rio-tiler>=9,<10`. +- **[titiler.core](https://github.com/developmentseed/titiler/tree/main/src/titiler/core)** — the base FastAPI framework, factory patterns, and dependency primitives used by every TiTiler application. -### Extension Layer -- **[titiler.xarray](https://github.com/developmentseed/titiler/tree/main/src/titiler/xarray)**: Multidimensional data support extending titiler.core -- **[titiler.extensions](https://github.com/developmentseed/titiler/tree/main/src/titiler/extensions)**: Plugin system for custom functionality -- **[titiler.mosaic](https://github.com/developmentseed/titiler/tree/main/src/titiler/mosaic)**: Multi-source tiling capabilities +## Extensions -### Application Layer -- **[titiler.application](https://github.com/developmentseed/titiler/tree/main/src/titiler/application)**: Reference implementation using titiler.core -- **[titiler-multidim](https://github.com/developmentseed/titiler-multidim)**: Prototype application using titiler.xarray + optimizations -- **[titiler-cmr](https://github.com/developmentseed/titiler-cmr)**: NASA-specific application using titiler.core + CMR integration -- **[titiler-eopf](https://github.com/EOPF-Explorer/titiler-eopf)**: ESA-specific application using titiler.xarray + EOPF integration +- **[titiler.xarray](https://github.com/developmentseed/titiler/tree/main/src/titiler/xarray)** — multidimensional support that extends `titiler.core` with xarray-based readers for NetCDF, Zarr, and similar formats. As of TiTiler 2.0 the application also exposes `/zarr/*` endpoints by default. +- **[titiler.extensions](https://github.com/developmentseed/titiler/tree/main/src/titiler/extensions)** — plugin system for custom factory behaviour (viewers, custom endpoints, dataset metadata, etc.). +- **[titiler.mosaic](https://github.com/developmentseed/titiler/tree/main/src/titiler/mosaic)** — multi-source mosaic tiling on top of MosaicJSON. -### Key Relationships +## Applications -#### **titiler.core → titiler.xarray** +- **[titiler.application](https://github.com/developmentseed/titiler/tree/main/src/titiler/application)** — reference application bundling `titiler.core`, `titiler.mosaic`, and (since 2.0) the `/zarr/*` endpoints from `titiler.xarray`. Public demo at [titiler.xyz](https://titiler.xyz/api.html). +- **[titiler-cmr](apis/titiler-cmr.md)** — NASA Common Metadata Repository application. Now built on both `titiler.core` and `titiler.xarray` with dual `/xarray/` and `/rasterio/` backend prefixes (formerly `/collections/*`, which still redirect). +- **[titiler-multidim](apis/titiler-multidim.md)** — VEDA-deployed multidimensional application built on `titiler.xarray`. Adds Redis caching, OpenTelemetry tracing, and (since v0.7) icechunk support. No longer labelled a prototype. +- **[titiler-eopf](apis/titiler-eopf.md)** — ESA Copernicus / Earth Observation Processing Framework application. Built on `titiler.xarray` plus `titiler.stacapi`, with a custom GeoZarr reader for hierarchical Zarr DataTrees. Can deploy as either a TiTiler REST API or an OpenEO backend from the same image. -`titiler.xarray` extends the core framework with xarray-based readers and multidimensional dataset support, inheriting all core functionality while adding temporal and dimensional processing capabilities. +## Installation note -#### **titiler.xarray → titiler-multidim / titiler-eopf applications** +The bare `titiler` metapackage on PyPI was dropped in late 2025. Install the specific subpackages you need: `pip install titiler.core titiler.xarray titiler.mosaic`, or one of the application packages directly. -Both `titiler-multidim` and `titiler.eopf` are built on `titiler.xarray`, leveraging its multidimensional capabilities while adding their own optimizations: +## Layering at a glance -- `titiler-multidim` adds Redis caching, VEDA platform integration, and prototypes of optimizations -- `titiler-eopf` adds EOPF-specific data structures, collection/item routing, and ESA workflow integrations +``` +rio-tiler + ↑ +titiler.core ─── titiler.extensions ─── titiler.mosaic + ↑ ↑ + └────────── titiler.xarray ───────────────┤ + ↑ │ + ├── titiler.application + ├── titiler-cmr (also uses titiler.core + titiler.mosaic directly) + ├── titiler-multidim + └── titiler-eopf (also uses titiler.stacapi) +``` -#### **titiler.core → titiler-cmr application** - -`titiler-cmr` is built directly on `titiler.core` rather than `titiler.xarray` due to the development timeline of the two projects. In the future, we anticipate -`titiler-cmr` will depend on `titiler.xarray`, with progress tracked by [titiler-cmr issue #35](https://github.com/developmentseed/titiler-cmr/issues/35). - ---- - -*The TiTiler ecosystem provides a complete stack for building scalable, cloud-native tile servers that can handle everything from simple COG serving to complex multi-dimensional scientific datasets.* +Python support across the stack is currently 3.11 through 3.14. diff --git a/docs/visualization/xpublish-tiles.md b/docs/visualization/xpublish-tiles.md deleted file mode 100644 index 018e964..0000000 --- a/docs/visualization/xpublish-tiles.md +++ /dev/null @@ -1,3 +0,0 @@ -# Xpublish-tiles - -Content coming soon. See [source repo](https://github.com/earth-mover/xpublish-tiles) for now. diff --git a/docs/visualization/xpublish/overview.md b/docs/visualization/xpublish/overview.md new file mode 100644 index 0000000..929580d --- /dev/null +++ b/docs/visualization/xpublish/overview.md @@ -0,0 +1,16 @@ +# Xpublish ecosystem overview + +[Xpublish](https://xpublish.readthedocs.io/) is a FastAPI-based framework for serving xarray datasets via HTTP APIs, enabling web-based access to scientific datasets. The ecosystem follows a plugin-oriented architecture with an intentionally small core library, and most user-facing capabilities are added by separate plugins. + +## Foundation + +- **[xpublish](https://github.com/xpublish-community/xpublish)** — core library that defines plugin extension points and an `xpublish` accessor for serving xarray datasets over HTTP APIs. + +## Plugins + +- **[xpublish-tiles](xpublish-tiles.md)** ([source](https://github.com/earth-mover/xpublish-tiles)) — transforms xarray datasets into raster and vector tiles. Provides OGC Tiles and OGC WMS endpoints as separate plugins (TilesPlugin and WMSPlugin). The detail page covers its grid-system support (regular, curvilinear, triangular, SELFE, polar, cubed-sphere, HEALPix), tile API surface, and rendering pipeline. +- **[xpublish-wms](https://github.com/xpublish-community/xpublish-wms)** — a dedicated Web Map Service implementation. Requires CF-compliant datasets. Predates xpublish-tiles' WMS plugin and is the more mature WMS path today. +- **[xpublish-edr](https://github.com/xpublish-community/xpublish-edr)** — routers for the [OGC EDR API](https://ogcapi.ogc.org/edr/), supporting position, area, and cube queries; useful for time-series and profile extraction rather than map tiles. +- **[opendap-protocol](https://github.com/xpublish-community/opendap-protocol)** — bare-minimum implementation of the DAP 2.0 protocol, for clients that expect OPeNDAP semantics. + +For a side-by-side comparison with the TiTiler ecosystem covering input formats, grid support, endpoints, tile-request parameters, resampling, and deployment, see the [Dynamic tiling ecosystem comparison](../ecosystem-comparison.md). diff --git a/docs/visualization/xpublish/xpublish-tiles.md b/docs/visualization/xpublish/xpublish-tiles.md new file mode 100644 index 0000000..bcb02b5 --- /dev/null +++ b/docs/visualization/xpublish/xpublish-tiles.md @@ -0,0 +1,139 @@ +# xpublish-tiles + +A FastAPI-based xpublish plugin that turns xarray datasets into raster and vector tiles, with a focus on the unstructured and curvilinear grids used in operational ocean and atmospheric models. Maintained by Earth Mover under the Apache 2.0 license. + +## At a glance + +- **Repo** — [earth-mover/xpublish-tiles](https://github.com/earth-mover/xpublish-tiles) +- **Shape** — Server-side library, a pair of xpublish plugins (`TilesPlugin`, `WMSPlugin`) +- **Web framework** — FastAPI, mounted via xpublish +- **Render engine** — [Datashader](https://datashader.org/) with Numba JIT compilation, plus a custom pyproj-based reprojection pipeline (no GDAL or rasterio) +- **Zarr versions** — v2 and v3 (`zarr>=3.1.2`); also reads anything xarray can open (NetCDF, Icechunk) +- **Conventions** — CF-xarray for axis detection, CF time decoding, multiple `grid_mapping` types + +## What it does + +xpublish-tiles takes an xarray dataset (regular, curvilinear, triangular, or HEALPix) and serves OGC Tiles, OGC WMS, and (newer) vector tiles directly from it, without a pre-rendering step. Its design centre is the operational geoscience use case: ocean models on ROMS, FVCOM, and SELFE grids; HEALPix and cubed-sphere from climate models; HRRR and RTOFS on 2D non-dimensional grids. These are precisely the grid topologies that GDAL-based tile servers either don't handle or handle awkwardly. + +The plugin lives inside the xpublish ecosystem; see the [Xpublish ecosystem overview](overview.md) for the broader picture, and the [Dynamic tiling ecosystem comparison](../ecosystem-comparison.md) for how it stacks up against TiTiler. + +## Architecture + +Two plugins ship in the same repository and are loaded independently. + +- **`TilesPlugin`** — OGC API Tiles 1.0 endpoints. Returns raster (PNG, JPEG) and, more recently, vector tiles (MVT, GeoJSON). +- **`WMSPlugin`** — OGC WMS 1.1.1 / 1.3.0 endpoints. Marked as still in active development; not yet at parity with the Tiles plugin or with the older [xpublish-wms](https://github.com/xpublish-community/xpublish-wms). + +The rendering pipeline is: + +1. **Coordinate transform.** A custom pyproj-based implementation, with a separable specialization for EPSG:4326→3857 that preserves the input grid structure (no per-pixel warp) and a thread-pool blocked transform for general cases. Tunable via `TRANSFORM_CHUNK_SIZE` and the thread-pool size. +2. **Resampling and aggregation.** Datashader chooses an engine based on grid type: a fast rectilinear path (3–10× faster than the alternatives), `quadmesh` for curvilinear grids, and `trimesh` with Delaunay triangulation for unstructured / UGRID. `numbagg` handles nearest-neighbour aggregation for discrete data. +3. **Styling.** Continuous data uses `colorscalerange` plus a colormap LUT; categorical data uses `flag_values` / `flag_meanings` / `flag_colors`; `abovemaxcolor` and `belowmincolor` parameters colour out-of-range values explicitly rather than clamping. +4. **Encoding.** Pillow for raster output; protocol buffers for MVT vector tiles. + +Async loading uses `xarray.load_async()`, configurable via the `async_load` setting. Caching keys are built from a required `_xpublish_id` dataset attribute plus the requested dimension and variable. + +## Supported grid systems + +xpublish-tiles' grid-system abstraction is its differentiator. The currently supported systems: + +- **Regular lat/lon** — RasterIndex +- **Curvilinear** — ROMS-family models including CBOFS, DBOFS, TBOFS +- **FVCOM triangular** — LOOFS, LSOFS, LMHOFS, NGOFS2 and similar +- **SELFE unstructured** — CREOFS +- **2D non-dimensional rectilinear** — RTOFS, HRRR-CONUS +- **Polar** — radar and satellite products in polar grids +- **Cubed-sphere** — faceted grid system (`FacetedGridSystem`) for cubed-sphere climate-model output +- **HEALPix** — cell-index coordinates with nested indexing (currently restricted to the `polygons` style) + +The grid system is detected from CF metadata where possible; custom grid systems can be registered. + +## Tile API surface + +Selected request parameters; see the in-repo OpenAPI docs for the full surface. + +| Category | Parameter | Notes | +|---|---|---| +| Variable | `variables` | List, required | +| Dimension selection | `dimension={method}::{value}` | DSL with `nearest`, `ffill`/`pad`, `bfill`/`backfill`, `exact` | +| Style | `style={type}/{colormap}` | Continuous, categorical, vector subtypes | +| Color range | `colorscalerange` | Tuple `(min, max)` | +| Out-of-range | `abovemaxcolor`, `belowmincolor` | Explicit colours for clipped values | +| Custom colormap | JSON | `0–255` → hex mapping | +| Image dimensions | `width`, `height` | Multiples of 256 | +| Image format | `f` | `image/png`, `image/jpeg`, MVT, GeoJSON | +| Error handling | `render_errors` | Boolean; render errors as image tiles vs HTTP 5xx | + +Notable endpoints beyond the standard OGC Tiles set: + +- **Legend endpoint** (`/tiles/legend`) — returns a rendered legend image or JSON colour-stop description for the requested style. +- **Vector tiles** — `vector/cells`, `vector/points`, `vector/contours` styles emit MVT or GeoJSON with cell geometries, suitable for use as the data layer of a MapLibre/Mapbox vector style. + +## Examples + +The same endpoint shape covers raster output, vector output, and the legend; the `style` query parameter (and `f` for explicit format negotiation) selects which. + +=== "Raster tile (PNG)" + + ```http + GET /tiles/WebMercatorQuad/4/8/5 + ?variables=temperature + &style=raster/viridis + &colorscalerange=270,310 + &width=512 + &height=512 + ``` + + Returns `image/png`. The `raster/{colormap}` style is the standard continuous-data path; swap the colormap suffix for any of Datashader's named maps. + +=== "Vector tile (MVT)" + + ```http + GET /tiles/WebMercatorQuad/4/8/5 + ?variables=temperature + &style=vector/cells + &f=application/vnd.mapbox-vector-tile + ``` + + Returns Mapbox Vector Tiles with cell-polygon geometries and the variable as a property. Suitable as the data source for a MapLibre or Mapbox vector style. `vector/points` and `vector/contours` are the other vector styles. + +=== "Legend" + + ```http + GET /tiles/legend + ?variables=temperature + &style=raster/viridis + &colorscalerange=270,310 + ``` + + Returns a rendered legend image matching the style above. Add `f=application/json` to get the colour stops as structured data instead, useful for rendering a custom legend in a frontend rather than using the prebaked image. + +A few request shapes worth knowing: + +- `dimension={method}::{value}` is the dimension-selection DSL (e.g. `time=nearest::2024-06-15T12:00:00Z`). +- `abovemaxcolor` and `belowmincolor` colour out-of-range values explicitly rather than clamping them, useful for highlighting outliers. +- For categorical data, use `style=raster/categorical` and let the dataset's `flag_values` / `flag_colors` drive the rendering. + +## Where it fits + +Choose xpublish-tiles when the data lives on a non-rectilinear grid that the climate or operational-oceanography community uses (ROMS curvilinear, FVCOM triangular, SELFE, HEALPix, cubed-sphere) and you need a server that respects that topology rather than regridding upstream. It is also a strong fit when the wider stack is xpublish-based (xpublish-edr for queries, xpublish-opendap, etc.) because everything mounts on the same FastAPI app over the same dataset accessor. + +The trade-off versus TiTiler is breadth versus depth: TiTiler has the broader format ecosystem (COG, STAC, NASA CMR), wider deployment story (Lambda, ECS), Redis caching, and band-math expressions; xpublish-tiles has the unstructured-grid renderers, CF integration, and async chunked loading that operational geoscience needs. The two ecosystems are increasingly complementary rather than overlapping; see the [comparison](../ecosystem-comparison.md) for details. + +## Configuration knobs + +Set via environment variables or a [donfig](https://github.com/pytroll/donfig) config file: + +- `NUMBA_NUM_THREADS` — Datashader/Numba thread count for JIT rendering. +- `TRANSFORM_CHUNK_SIZE` — chunk size (NxN) for blocked coordinate transforms. +- Thread-pool size — for the chunked transform pipeline. +- `async_load` — toggles `xarray.load_async()`. + +Worth knowing: first request after startup is slow due to Numba JIT compilation. Warm up the renderers in production deployments rather than serving the first user request cold. + +## Links + +- Source: [earth-mover/xpublish-tiles](https://github.com/earth-mover/xpublish-tiles) +- Parent project: [xpublish](https://github.com/xpublish-community/xpublish) +- Dependencies: [Datashader](https://datashader.org/), [pyproj](https://pyproj4.github.io/pyproj/), [cf-xarray](https://github.com/xarray-contrib/cf-xarray), [morecantile](https://github.com/developmentseed/morecantile), [numbagg](https://github.com/numbagg/numbagg) +- See also: [xpublish ecosystem overview](overview.md), [TiTiler vs Xpublish comparison](../ecosystem-comparison.md) diff --git a/docs/visualization/zarr-cesium.md b/docs/visualization/zarr-cesium.md index 93af3ce..f7bc0c0 100644 --- a/docs/visualization/zarr-cesium.md +++ b/docs/visualization/zarr-cesium.md @@ -1,3 +1,49 @@ -# Zarr-cesium +# zarr-cesium -Content coming soon. See [source repo](https://github.com/NOC-OI/zarr-cesium) for now. +CesiumJS providers for visualizing Zarr datasets directly on the Cesium 3D globe, with separate paths for 2D imagery, 3D volumetric slices, and animated vector fields. Maintained by the National Oceanography Centre (NOC) UK under the MIT license. + +## At a glance + +- **Repo** — [NOC-OI/zarr-cesium](https://github.com/NOC-OI/zarr-cesium) +- **Shape** — Library (TypeScript) +- **Map host** — CesiumJS (not Resium, not deck.gl-cesium) +- **Render API** — WebGL2 with custom GLSL fragment shaders, integrated as Cesium imagery and primitive APIs +- **Zarr versions** — v2 and v3, autodetected via zarrita +- **Conventions** — CF time decoding (days/hours since reference), CF dimension-name aliasing (`time`/`t`/`Time`/`time_counter`); supports `ndpyramid` multiscales + +## What it does + +zarr-cesium is the only Zarr-native option in this section that targets CesiumJS. It bridges Cesium's existing geometry and imagery APIs by exposing three provider classes, each fitting a different Cesium extension point: + +- **`ZarrLayerProvider`** produces a Cesium `ImageryLayer` for 2D rasters draped on the globe. +- **`ZarrCubeProvider`** produces Cesium `Primitive` geometry for 3D volumetric data, rendered as draped horizontal and vertical slices through the cube. +- **`ZarrCubeVelocityProvider`** feeds U/V components into `cesium-wind-layer` for animated particle visualization of ocean currents and winds. + +Demo datasets focus on oceanography and atmospheric reanalysis (NEMO model output for salinity, temperature, depth profiles; Hurricane Florence ERA5 wind and pressure), which reflects the NOC use case driving development. + +## How it renders + +All three providers share the same colour pipeline. A fragment shader (`src/shaders.ts`) samples a single-channel data texture, applies `scale_factor` and `add_offset`, masks NaN and nodata, normalizes against per-layer min and max, and looks up a 1D colormap texture built from `jsColormaps` (matplotlib-compatible: viridis, plasma, inferno, jet, ...). All colour mapping happens GPU-side; there is no CPU canvas stage. + +Where the providers diverge is in geometry: + +- The 2D `ZarrLayerProvider` plugs into Cesium's `TilingScheme` (Geographic EPSG:4326 or Web Mercator EPSG:3857), automatically selects a multiscale level for the current zoom, and lets Cesium handle tile lifecycle. +- The 3D `ZarrCubeProvider` loads the requested cube into memory as an `ndarray` and renders user-positioned slices as `RectangleGeometry` primitives with textured materials. This is a slice-rendering approach to volumetric data, distinct from Browzarr's full raycasting. +- The vector provider hands U/V slices off to `cesium-wind-layer`, which runs the particle simulation. + +CRS is autodetected (coordinate-range analysis plus metadata inspection); only EPSG:4326 and EPSG:3857 are supported. Curvilinear or rotated-pole grids must be reprojected before storage. + +## Zarr handling + +Reads use zarrita and try v3 first, falling back to v2. Multiscale pyramids in the `ndpyramid` convention are supported, with automatic level selection in the 2D provider and an explicit `multiscaleLevel` argument in the 3D provider. CF time decoding (`src/decodeCFTime.ts`) handles common reference-date encodings, and time and depth dimensions are sliced dynamically via `updateSelectors()`. Sharding and GeoZarr are not explicitly handled. + +## Where it fits + +Choose zarr-cesium when the host is Cesium and the data is environmental, oceanographic, or atmospheric, especially when the value of the visualization comes from sitting on a 3D globe rather than a 2D web map. The 3D slice rendering and the wind-layer integration are unique among the libraries in this section. The trade-off is that you commit to the Cesium runtime, and the 3D rendering loads full cubes into memory rather than streaming them, so dataset size for the cube provider is bounded by browser memory. + +## Links + +- Source: [NOC-OI/zarr-cesium](https://github.com/NOC-OI/zarr-cesium) +- Docs: in-repo under `docs/` +- Author: Tobias Ferreira (NOC), as part of the Atlantis project +- Related: [CesiumJS](https://cesium.com/platform/cesiumjs/), [cesium-wind-layer](https://github.com/hongfaqiu/cesium-wind-layer), [zarrita](https://github.com/manzt/zarrita.js), [ndpyramid](https://github.com/carbonplan/ndpyramid) diff --git a/docs/visualization/zarr-gl.md b/docs/visualization/zarr-gl.md index 18fe941..299fa54 100644 --- a/docs/visualization/zarr-gl.md +++ b/docs/visualization/zarr-gl.md @@ -1,3 +1,10 @@ -# Zarr-gl +# zarr-gl -Content coming soon. See [source repo](https://github.com/carderne/zarr-gl) for now. +zarr-gl is a Zarr-on-MapLibre custom layer by Chris Arderne. It has been **superseded by [zarr-layer](zarr-layer.md)**, which fills the same niche (Zarr in a MapLibre/Mapbox v3 `CustomLayerInterface`) with broader scope, including untiled-mode rendering with arbitrary CRS support and active CarbonPlan maintenance. + +If you are starting a new project, use [zarr-layer](zarr-layer.md). zarr-gl is documented here for historical context and for users with existing integrations. + +## Links + +- Source: [carderne/zarr-gl](https://github.com/carderne/zarr-gl) +- Successor: [zarr-layer](zarr-layer.md) diff --git a/docs/visualization/zarr-layer.md b/docs/visualization/zarr-layer.md index 7848cf5..1fb0814 100644 --- a/docs/visualization/zarr-layer.md +++ b/docs/visualization/zarr-layer.md @@ -1,3 +1,36 @@ -# Zarr-layer +# zarr-layer -Content coming soon. See [source repo](https://github.com/carbonplan/zarr-layer) for now. +A custom map layer that renders Zarr directly inside MapLibre GL or Mapbox GL JS, without going through deck.gl. Maintained by CarbonPlan under the MIT license. Supersedes [zarr-gl](zarr-gl.md). + +## At a glance + +- **Repo** — [carbonplan/zarr-layer](https://github.com/carbonplan/zarr-layer) +- **Shape** — Library, single npm package +- **Map host** — MapLibre GL or Mapbox GL JS v3+, via the native `CustomLayerInterface` +- **Render API** — WebGL2, hand-written GLSL +- **Zarr versions** — v2 and v3, autodetected +- **Conventions** — Tiled XYZ; experimental support for the in-flight `multiscales` zarr-conventions standard for untiled data + +## What it does + +zarr-layer is a focused alternative to deck.gl-raster for users who want Zarr on a web map but do not want to introduce deck.gl. It plugs into MapLibre or Mapbox v3 as a native custom layer, so it composes with vector basemaps, Mapbox-style projections, and existing layer-management code without the deck.gl runtime. The README acknowledges deck.gl-raster as inspiration and reuses `@developmentseed/raster-reproject` for mesh generation, but the surface area and dependency footprint are deliberately smaller. + +## How it renders + +The layer compiles WebGL2 shaders that normalize values via `(value - clim.min) / (clim.max - clim.min)`, clamp, then look up a 1D RGB16F colormap texture. Multi-band selectors load multiple variables into separate textures and expose them as named GLSL variables (sanitized for shader identifiers), so a NDVI fragment can be written as `(B08 - B04) / (B08 + B04)` directly. Custom fragment shader strings are a supported extension point. + +Two rendering modes coexist. **Tiled mode** assumes the data is already in Web Mercator or geographic XYZ tiles and renders against the standard MapLibre/Mapbox tile grid. **Untiled mode**, marked experimental, loads chunks intersecting the viewport at a zoom-appropriate level and supports arbitrary CRS via a proj4 string plus an adaptive Delaunay reprojection mesh. Polar coverage works on MapLibre out of the box; on Mapbox it requires an experimental `renderPoles: true` flag that uses Mapbox internals. + +## Zarr handling + +Reads use zarrita and detect v3 first, falling back to v2. Codecs supported through zarrita include bytes, zlib, gzip, blosc, lz4, zstd, transpose, crc32c, and bitround, with custom codecs registrable. Spatial dimensions are configured via the `spatialDimensions` option; non-spatial dimensions are sliced via a `selector` API that defaults to index 0 for unspecified dims and accepts either positional indices or coordinate-value lookups. CF conventions like `scale_factor`, `add_offset`, and `_FillValue` are honoured during normalization. GeoZarr is not explicitly modelled. + +## Where it fits + +Choose zarr-layer when the host map is MapLibre or Mapbox and you want a small, native custom layer rather than the deck.gl ecosystem. It is the most direct path from "I have a Zarr in S3" to "it shows up on my MapLibre map." The trade-off versus deck.gl-raster is less metadata machinery (no GeoZarr parser) and no shared rendering stack with COG; the trade-off versus the viewer apps below is that zarr-layer is a layer, not a UI, so time sliders, colorbar widgets, and dataset pickers are the integrator's job. + +## Links + +- Source: [carbonplan/zarr-layer](https://github.com/carbonplan/zarr-layer) +- Sibling project: [@carbonplan/maps](carbonplan-maps.md), CarbonPlan's earlier regl-based React component library; still actively maintained, with a stricter `ndpyramid` Web Mercator assumption and a React-component API surface +- Related: [zarrita](https://github.com/manzt/zarrita.js), proj4js, `@developmentseed/raster-reproject`, delaunator diff --git a/docs/worst-practices/dispersed-metadata.ipynb b/docs/worst-practices/dispersed-metadata.ipynb index c2102e2..320685d 100644 --- a/docs/worst-practices/dispersed-metadata.ipynb +++ b/docs/worst-practices/dispersed-metadata.ipynb @@ -379,11 +379,6 @@ "id": "61c46062-6f15-40a2-bc12-936c18a96198", "metadata": {}, "outputs": [ - { - "data": {}, - "metadata": {}, - "output_type": "display_data" - }, { "data": { "application/vnd.holoviews_exec.v0+json": "", diff --git a/docs/worst-practices/massive-chunks.ipynb b/docs/worst-practices/massive-chunks.ipynb index 2bb2649..35bbfc2 100644 --- a/docs/worst-practices/massive-chunks.ipynb +++ b/docs/worst-practices/massive-chunks.ipynb @@ -610,11 +610,6 @@ "id": "72677350-29be-42d1-afd2-39184241b838", "metadata": {}, "outputs": [ - { - "data": {}, - "metadata": {}, - "output_type": "display_data" - }, { "data": { "application/vnd.holoviews_exec.v0+json": "", diff --git a/docs/worst-practices/tiny-chunks.ipynb b/docs/worst-practices/tiny-chunks.ipynb index deccf7e..3cd886d 100644 --- a/docs/worst-practices/tiny-chunks.ipynb +++ b/docs/worst-practices/tiny-chunks.ipynb @@ -703,11 +703,6 @@ "id": "a3823e4a-9bca-46a9-b627-a6ea47025809", "metadata": {}, "outputs": [ - { - "data": {}, - "metadata": {}, - "output_type": "display_data" - }, { "data": { "application/vnd.holoviews_exec.v0+json": "", diff --git a/docs/worst-practices/tiny-coordinate-chunks.ipynb b/docs/worst-practices/tiny-coordinate-chunks.ipynb index be44171..805f469 100644 --- a/docs/worst-practices/tiny-coordinate-chunks.ipynb +++ b/docs/worst-practices/tiny-coordinate-chunks.ipynb @@ -326,11 +326,6 @@ "id": "25d1b0d5-3262-41e6-8a9d-21527743ca8d", "metadata": {}, "outputs": [ - { - "data": {}, - "metadata": {}, - "output_type": "display_data" - }, { "data": { "application/vnd.holoviews_exec.v0+json": "", diff --git a/hooks/mkdocs_jupyter_md_filter.py b/hooks/mkdocs_jupyter_md_filter.py new file mode 100644 index 0000000..9ee2470 --- /dev/null +++ b/hooks/mkdocs_jupyter_md_filter.py @@ -0,0 +1,24 @@ +"""Prevent the ``mkdocs-jupyter`` plugin from probing ``.md`` files. + +The plugin's ``should_include`` runs ``jupytext.read`` on every ``.md`` file +to look for a Python kernel spec. ``jupytext`` detects any markdown file +containing a line that starts with ``:::`` as pandoc-format and shells out to +pandoc — and pandoc then emits "unclosed Div" warnings for the +``mkdocstrings`` autodoc directives on our API reference pages. + +We have no markdown-format notebooks, so stripping ``.md`` from the plugin's +supported extension list short-circuits the probe before jupytext is ever +called. +""" + +from __future__ import annotations + + +def on_startup(*, command, dirty): # noqa: ARG001 - mkdocs hook signature + try: + from mkdocs_jupyter.plugin import Plugin + except ImportError: # pragma: no cover - plugin not installed + return + Plugin._supported_extensions = [ + ext for ext in Plugin._supported_extensions if ext != ".md" + ] diff --git a/mkdocs.yml b/mkdocs.yml index a459cef..e05f471 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -30,6 +30,7 @@ nav: - Datacube Visualization: - Overview: "visualization/overview.md" - Dynamic tilers: + - Comparison: "visualization/ecosystem-comparison.md" - Titiler: - Ecosystem overview: "visualization/titiler/overview.md" - APIs: @@ -37,22 +38,31 @@ nav: - "visualization/titiler/apis/titiler-core.md" - "visualization/titiler/apis/titiler-cmr.md" - "visualization/titiler/apis/titiler-multidim.md" - - Xpublish-tiles: "visualization/xpublish-tiles.md" + - "visualization/titiler/apis/titiler-eopf.md" + - Xpublish: + - Ecosystem overview: "visualization/xpublish/overview.md" + - xpublish-tiles: "visualization/xpublish/xpublish-tiles.md" - Rendering layers: + - Comparison: "visualization/client-side-comparison.md" - Carbonplan-maps: "visualization/carbonplan-maps.md" - Deck.gl-raster: "visualization/deck.gl-raster.md" - - Numeric-data-layer: "visualization/numeric-data-layer.md" - Zarr-cesium: "visualization/zarr-cesium.md" - - Zarr-gl: "visualization/zarr-gl.md" - Zarr-layer: "visualization/zarr-layer.md" + - Superseded: + - Numeric-data-layer: "visualization/numeric-data-layer.md" + - Zarr-gl: "visualization/zarr-gl.md" - Applications: - Browzarr: "visualization/browzarr.md" + - GridLook: "visualization/gridlook.md" - API Reference: - Benchmarking: - "api-reference/datacube-benchmark.md" - Related Guides: "related.md" +hooks: + - hooks/mkdocs_jupyter_md_filter.py + watch: - packages - docs diff --git a/packages/datacube-benchmark/src/datacube_benchmark/config.py b/packages/datacube-benchmark/src/datacube_benchmark/config.py index b2afe84..0b7e60e 100644 --- a/packages/datacube-benchmark/src/datacube_benchmark/config.py +++ b/packages/datacube-benchmark/src/datacube_benchmark/config.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -import zarr +import zarr.codecs @dataclass diff --git a/packages/datacube-benchmark/src/datacube_benchmark/create.py b/packages/datacube-benchmark/src/datacube_benchmark/create.py index 1e9bd11..814f673 100644 --- a/packages/datacube-benchmark/src/datacube_benchmark/create.py +++ b/packages/datacube-benchmark/src/datacube_benchmark/create.py @@ -4,7 +4,7 @@ from dask import array as da import numpy as np import xarray as xr -from pint import UnitRegistry, Quantity +from pint import Quantity from .defaults import ( default_longitude_coords, default_latitude_coords, @@ -24,8 +24,6 @@ from numcodecs.abc import Codec from zarr.abc.codec import BytesBytesCodec -ureg = UnitRegistry() - def _write_using_obstore( ds: xr.Dataset, diff --git a/uv.lock b/uv.lock index d566756..2063c37 100644 --- a/uv.lock +++ b/uv.lock @@ -2,6 +2,10 @@ version = 1 revision = 3 requires-python = ">=3.13" +[options] +exclude-newer = "2026-04-27T09:53:40.621943Z" +exclude-newer-span = "P1W" + [manifest] members = [ "datacube-benchmark",