From 507095122d94ad941fd969ea3f89f30e79d54488 Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Fri, 6 Mar 2026 16:30:50 +1100 Subject: [PATCH 01/15] feat(explorer): consolidate diagnostic content into per-collection YAML Migrate explorer card definitions from 5 hardcoded TSX theme files into 25 per-collection YAML files served by a new backend API. This makes diagnostic content easy to edit without touching frontend code. Backend: - Add 25 collection YAML files + themes.yaml in backend/static/collections/ - Add Pydantic models and loader in core/collections.py - Add explorer API routes (/collections/, /themes/) in api/routes/explorer.py - Add 28 tests (18 loader + 10 API) Frontend: - Regenerate client SDK with new explorer endpoints - Rewrite thematicContent.tsx to fetch cards from API with snake_case to camelCase mapping at the boundary - Fix yMin/yMax passthrough bug in ensembleChartContent.tsx - Delete 5 static theme TSX files (atmosphere, earthSystem, impactAndAdaptation, land, sea) --- backend/src/ref_backend/api/main.py | 3 +- .../src/ref_backend/api/routes/explorer.py | 40 ++ backend/src/ref_backend/core/collections.py | 224 +++++++ backend/static/collections/1.1.yaml | 21 + backend/static/collections/1.2.yaml | 39 ++ backend/static/collections/1.3.yaml | 37 + backend/static/collections/1.4.yaml | 100 +++ backend/static/collections/1.5.yaml | 23 + backend/static/collections/1.6.yaml | 62 ++ backend/static/collections/2.1.yaml | 34 + backend/static/collections/2.2.yaml | 58 ++ backend/static/collections/2.3.yaml | 37 + backend/static/collections/2.4.yaml | 29 + backend/static/collections/2.5.yaml | 30 + backend/static/collections/2.6.yaml | 29 + backend/static/collections/2.7.yaml | 28 + backend/static/collections/3.1.yaml | 16 + backend/static/collections/3.2.yaml | 14 + backend/static/collections/3.3.yaml | 135 ++++ backend/static/collections/3.6.yaml | 28 + backend/static/collections/3.7.yaml | 21 + backend/static/collections/4.1.yaml | 66 ++ backend/static/collections/4.2.yaml | 37 + backend/static/collections/4.3.yaml | 23 + backend/static/collections/4.4.yaml | 23 + backend/static/collections/4.5.yaml | 18 + backend/static/collections/5.3.yaml | 33 + backend/static/collections/5.4.yaml | 21 + backend/static/collections/themes.yaml | 24 + backend/tests/test_api/test_api_explorer.py | 257 +++++++ .../tests/test_core/test_core_collections.py | 531 +++++++++++++++ frontend/biome.json | 11 +- .../src/client/@tanstack/react-query.gen.ts | 72 +- frontend/src/client/schemas.gen.ts | 633 +++++++++++++++--- frontend/src/client/sdk.gen.ts | 42 +- frontend/src/client/types.gen.ts | 205 +++++- frontend/src/components/app/welcomeModal.tsx | 2 +- .../diagnostics/executionsTable.tsx | 12 +- .../explorer/content/ensembleChartContent.tsx | 2 + .../components/explorer/thematicContent.tsx | 128 +++- .../components/explorer/theme/atmosphere.tsx | 215 ------ .../components/explorer/theme/earthSystem.tsx | 126 ---- .../explorer/theme/impactAndAdaptation.tsx | 24 - .../src/components/explorer/theme/land.tsx | 146 ---- .../src/components/explorer/theme/sea.tsx | 162 ----- 45 files changed, 2971 insertions(+), 850 deletions(-) create mode 100644 backend/src/ref_backend/api/routes/explorer.py create mode 100644 backend/src/ref_backend/core/collections.py create mode 100644 backend/static/collections/1.1.yaml create mode 100644 backend/static/collections/1.2.yaml create mode 100644 backend/static/collections/1.3.yaml create mode 100644 backend/static/collections/1.4.yaml create mode 100644 backend/static/collections/1.5.yaml create mode 100644 backend/static/collections/1.6.yaml create mode 100644 backend/static/collections/2.1.yaml create mode 100644 backend/static/collections/2.2.yaml create mode 100644 backend/static/collections/2.3.yaml create mode 100644 backend/static/collections/2.4.yaml create mode 100644 backend/static/collections/2.5.yaml create mode 100644 backend/static/collections/2.6.yaml create mode 100644 backend/static/collections/2.7.yaml create mode 100644 backend/static/collections/3.1.yaml create mode 100644 backend/static/collections/3.2.yaml create mode 100644 backend/static/collections/3.3.yaml create mode 100644 backend/static/collections/3.6.yaml create mode 100644 backend/static/collections/3.7.yaml create mode 100644 backend/static/collections/4.1.yaml create mode 100644 backend/static/collections/4.2.yaml create mode 100644 backend/static/collections/4.3.yaml create mode 100644 backend/static/collections/4.4.yaml create mode 100644 backend/static/collections/4.5.yaml create mode 100644 backend/static/collections/5.3.yaml create mode 100644 backend/static/collections/5.4.yaml create mode 100644 backend/static/collections/themes.yaml create mode 100644 backend/tests/test_api/test_api_explorer.py create mode 100644 backend/tests/test_core/test_core_collections.py delete mode 100644 frontend/src/components/explorer/theme/atmosphere.tsx delete mode 100644 frontend/src/components/explorer/theme/earthSystem.tsx delete mode 100644 frontend/src/components/explorer/theme/impactAndAdaptation.tsx delete mode 100644 frontend/src/components/explorer/theme/land.tsx delete mode 100644 frontend/src/components/explorer/theme/sea.tsx diff --git a/backend/src/ref_backend/api/main.py b/backend/src/ref_backend/api/main.py index 6a59758..23b8c55 100644 --- a/backend/src/ref_backend/api/main.py +++ b/backend/src/ref_backend/api/main.py @@ -1,11 +1,12 @@ from fastapi import APIRouter -from ref_backend.api.routes import aft, datasets, diagnostics, executions, results, utils +from ref_backend.api.routes import aft, datasets, diagnostics, executions, explorer, results, utils api_router = APIRouter() api_router.include_router(aft.router) api_router.include_router(datasets.router) api_router.include_router(diagnostics.router) api_router.include_router(executions.router) +api_router.include_router(explorer.router) api_router.include_router(results.router) api_router.include_router(utils.router) diff --git a/backend/src/ref_backend/api/routes/explorer.py b/backend/src/ref_backend/api/routes/explorer.py new file mode 100644 index 0000000..f49104b --- /dev/null +++ b/backend/src/ref_backend/api/routes/explorer.py @@ -0,0 +1,40 @@ +from fastapi import APIRouter, HTTPException + +from ref_backend.core.collections import ( + AFTCollectionDetail, + AFTCollectionSummary, + ThemeDetail, + ThemeSummary, + get_collection_by_id, + get_collection_summaries, + get_theme_by_slug, + get_theme_summaries, +) + +router = APIRouter(prefix="/explorer", tags=["Explorer"]) + + +@router.get("/collections/", response_model=list[AFTCollectionSummary]) +async def list_collections() -> list[AFTCollectionSummary]: + return get_collection_summaries() + + +@router.get("/collections/{collection_id}", response_model=AFTCollectionDetail) +async def get_collection(collection_id: str) -> AFTCollectionDetail: + result = get_collection_by_id(collection_id) + if result is None: + raise HTTPException(status_code=404, detail=f"Collection '{collection_id}' not found") + return result + + +@router.get("/themes/", response_model=list[ThemeSummary]) +async def list_themes() -> list[ThemeSummary]: + return get_theme_summaries() + + +@router.get("/themes/{theme_slug}", response_model=ThemeDetail) +async def get_theme(theme_slug: str) -> ThemeDetail: + result = get_theme_by_slug(theme_slug) + if result is None: + raise HTTPException(status_code=404, detail=f"Theme '{theme_slug}' not found") + return result diff --git a/backend/src/ref_backend/core/collections.py b/backend/src/ref_backend/core/collections.py new file mode 100644 index 0000000..8f810df --- /dev/null +++ b/backend/src/ref_backend/core/collections.py @@ -0,0 +1,224 @@ +import logging +from functools import lru_cache +from pathlib import Path +from typing import Literal + +import yaml +from pydantic import BaseModel, ValidationError + +logger = logging.getLogger(__name__) + + +class AFTCollectionGroupingConfig(BaseModel): + group_by: str + hue: str + style: str | None = None + + +class AFTCollectionCardContent(BaseModel): + type: Literal["box-whisker-chart", "figure-gallery", "series-chart", "taylor-diagram"] + provider: str + diagnostic: str + title: str + description: str | None = None + interpretation: str | None = None + span: Literal[1, 2] | None = None + placeholder: bool | None = None + metric_units: str | None = None + clip_min: float | None = None + clip_max: float | None = None + y_min: float | None = None + y_max: float | None = None + show_zero_line: bool | None = None + symmetrical_axes: bool | None = None + reference_stddev: float | None = None + label_template: str | None = None + other_filters: dict[str, str] | None = None + grouping_config: AFTCollectionGroupingConfig | None = None + + +class AFTCollectionCard(BaseModel): + title: str + description: str | None = None + placeholder: bool | None = None + content: list[AFTCollectionCardContent] + + +class AFTCollectionContent(BaseModel): + description: str | None = None + short_description: str | None = None + + +class AFTCollectionDiagnosticLink(BaseModel): + provider_slug: str + diagnostic_slug: str + + +class AFTCollectionSummary(BaseModel): + id: str + name: str + theme: str | None = None + endorser: str | None = None + card_count: int + + +class AFTCollectionDetail(BaseModel): + id: str + name: str + theme: str | None = None + endorser: str | None = None + version_control: str | None = None + reference_dataset: str | None = None + provider_link: str | None = None + content: AFTCollectionContent | None = None + diagnostics: list[AFTCollectionDiagnosticLink] + explorer_cards: list[AFTCollectionCard] + + +class ThemeSummary(BaseModel): + slug: str + title: str + description: str | None = None + collection_count: int + card_count: int + + +class ThemeDetail(BaseModel): + slug: str + title: str + description: str | None = None + collections: list[AFTCollectionDetail] + explorer_cards: list[AFTCollectionCard] + + +def get_collections_dir() -> Path: + return Path(__file__).parents[3] / "static" / "collections" + + +def get_themes_path() -> Path: + return get_collections_dir() / "themes.yaml" + + +@lru_cache(maxsize=1) +def load_all_collections() -> dict[str, AFTCollectionDetail]: + collections_dir = get_collections_dir() + + if not collections_dir.exists(): + logger.warning(f"Collections directory not found: {collections_dir}") + return {} + + result: dict[str, AFTCollectionDetail] = {} + + yaml_files = sorted( + [p for p in collections_dir.glob("*.yaml") if p.name != "themes.yaml"], + key=lambda p: p.stem, + ) + + for yaml_file in yaml_files: + try: + with open(yaml_file, encoding="utf-8") as f: + data = yaml.safe_load(f) + + if not isinstance(data, dict): + logger.warning(f"Skipping {yaml_file.name}: expected a YAML mapping at top level") + continue + + collection = AFTCollectionDetail(**data) + + if collection.id in result: + raise ValueError(f"Duplicate collection ID '{collection.id}' found in {yaml_file.name}") + + result[collection.id] = collection + + except ValidationError as e: + logger.warning(f"Skipping {yaml_file.name}: validation error: {e}") + except Exception as e: + logger.warning(f"Skipping {yaml_file.name}: parse error: {e}") + + return result + + +def get_collection_by_id(collection_id: str) -> AFTCollectionDetail | None: + return load_all_collections().get(collection_id) + + +def get_collection_summaries() -> list[AFTCollectionSummary]: + collections = load_all_collections() + return [ + AFTCollectionSummary( + id=c.id, + name=c.name, + theme=c.theme, + endorser=c.endorser, + card_count=len(c.explorer_cards), + ) + for c in sorted(collections.values(), key=lambda c: c.id) + ] + + +@lru_cache(maxsize=1) +def load_theme_mapping() -> dict[str, ThemeDetail]: + themes_path = get_themes_path() + + if not themes_path.exists(): + logger.warning(f"Themes file not found: {themes_path}") + return {} + + try: + with open(themes_path, encoding="utf-8") as f: + data = yaml.safe_load(f) or [] + except Exception as e: + logger.warning(f"Error loading themes.yaml: {e}") + return {} + + all_collections = load_all_collections() + result: dict[str, ThemeDetail] = {} + + for theme_data in data: + try: + slug = theme_data["slug"] + title = theme_data["title"] + description = theme_data.get("description") + collection_ids: list[str] = theme_data.get("collections", []) + + collections: list[AFTCollectionDetail] = [] + explorer_cards: list[AFTCollectionCard] = [] + + for cid in collection_ids: + col = all_collections.get(cid) + if col is None: + logger.warning(f"Theme '{slug}' references unknown collection ID '{cid}', skipping") + continue + collections.append(col) + explorer_cards.extend(col.explorer_cards) + + result[slug] = ThemeDetail( + slug=slug, + title=title, + description=description, + collections=collections, + explorer_cards=explorer_cards, + ) + + except Exception as e: + logger.warning(f"Skipping theme entry: {e}") + + return result + + +def get_theme_summaries() -> list[ThemeSummary]: + themes = load_theme_mapping() + return [ + ThemeSummary( + slug=t.slug, + title=t.title, + description=t.description, + collection_count=len(t.collections), + card_count=len(t.explorer_cards), + ) + for t in themes.values() + ] + + +def get_theme_by_slug(slug: str) -> ThemeDetail | None: + return load_theme_mapping().get(slug) diff --git a/backend/static/collections/1.1.yaml b/backend/static/collections/1.1.yaml new file mode 100644 index 0000000..94a5c0b --- /dev/null +++ b/backend/static/collections/1.1.yaml @@ -0,0 +1,21 @@ +id: "1.1" +name: "Antarctic annual mean, Arctic September rate of sea ice area (SIA) loss per degree warming (dSIA / dGMST)" +theme: "Oceans and sea ice" +endorser: "CMIP Model Benchmarking Task Team" +version_control: "version 1 - 24-11-04 REF launch" +reference_dataset: "OSI SAF/CCI, HadCRUT" +provider_link: "https://docs.esmvaltool.org/en/latest/recipes/recipe_seaice_sensitivity.html" +content: + description: >- + This metric evaluates the rate of sea ice loss per degree of global warming, + following the approach used for sea ice benchmarking within the Sea Ice Model + Intercomparison Project analysis (Notz and SIMIP Community, 2020; Roach et al., + 2020). The metric is calculated by regressing the time-series of sea ice area on + global mean temperature. Sea ice responds strongly to climate forcing and warming + short_description: >- + Rate of sea ice area loss per degree of warming, as in plot 1d of Notz et al. + and figure 3e of Roach et al. +diagnostics: + - provider_slug: esmvaltool + diagnostic_slug: sea-ice-sensitivity +explorer_cards: [] diff --git a/backend/static/collections/1.2.yaml b/backend/static/collections/1.2.yaml new file mode 100644 index 0000000..d272b67 --- /dev/null +++ b/backend/static/collections/1.2.yaml @@ -0,0 +1,39 @@ +id: "1.2" +name: "Atlantic meridional overturning circulation (AMOC)" +theme: "Oceans and sea ice" +endorser: "CMIP Model Benchmarking Task Team" +version_control: "version 1 - 24-11-04 REF launch" +reference_dataset: "RAPID-v2023-1" +provider_link: "https://doi.org/10.1029/2018MS001354" +content: + description: >- + Provides a key indicator of the strength of ocean circulation, which redistributes + freshwater, heat and carbon across the Atlantic Basin (Le Bras et al., 2023). The + AMOC is a key component of the global ocean conveyor belt and plays an important + role in transporting heat poleward and ocean biogeochemical tracers from the surface + into the ocean interior. The strength of the AMOC at 26.5°N is commonly used for + evaluation of model fidelity since it can be compared with the long-term + RAPID-MOCHA (Rapid Climate Change - Meridional Overturning Circulation and Heatflux + Array) observational dataset (Moat et al., 2025) + short_description: >- + Comparison of the model Atlantic meridional ocean circulation (AMOC) circulation + strength with reference data from RAPID-v2023-1 +diagnostics: + - provider_slug: ilamb + diagnostic_slug: amoc-rapid +explorer_cards: + - title: "AMOC" + placeholder: true + content: + - type: box-whisker-chart + provider: ilamb + diagnostic: amoc-rapid + title: "AMOC Strength" + metric_units: "Sv" + grouping_config: + group_by: statistic + hue: statistic + other_filters: + region: "None" + metric: timeseries + statistic: "Period Mean" diff --git a/backend/static/collections/1.3.yaml b/backend/static/collections/1.3.yaml new file mode 100644 index 0000000..7407a46 --- /dev/null +++ b/backend/static/collections/1.3.yaml @@ -0,0 +1,37 @@ +id: "1.3" +name: "El Niño Southern Oscillation (ENSO) diagnostics (lifecycle, seasonality, amplitude, teleconnections)" +theme: "Oceans and sea ice" +endorser: "CMIP Model Benchmarking Task Team" +version_control: "version 1 - 24-11-04 REF launch" +reference_dataset: "CMAP-V1902, TropFlux-1-0, AVISO-1-0, ERA-5, GPCP-SG-2-3, HadISST-1-1" +provider_link: "https://docs.esmvaltool.org/en/latest/recipes/recipe_enso_ref.html" +content: + description: >- + The El Niño Southern Oscillation (ENSO) is the primary mode of the global + interannual climate variability, mainly reflected by the variations in surface wind + stress and ocean temperature in the tropical Pacific Ocean. The ENSO variability can + be calculated from both sea surface temperature and atmospheric pressure differences + 680 between different tropical Pacific areas. The Southern Oscillation Index (SOI) + uses pressure differences between the Tahiti and Darwin regions. The Oceanic Niño + Index (ONI) summarizes SST anomalies in the Niño 3.4 region. Given its implications + for regional climate variability, capturing the observed ENSO spatial and temporal + characteristics would increase the fidelity and robustness in a model's climate + projections. + short_description: >- + ENSO CLIVAR metrics - reproducing background climatology and ENSO characteristics +diagnostics: + - provider_slug: esmvaltool + diagnostic_slug: enso-basic-climatology + - provider_slug: esmvaltool + diagnostic_slug: enso-characteristics + - provider_slug: pmp + diagnostic_slug: enso_proc + - provider_slug: pmp + diagnostic_slug: enso_tel +explorer_cards: + - title: "El Niño-Southern Oscillation (ENSO)" + description: >- + Characteristics of ENSO, the dominant mode of interannual climate variability, + which influences global weather patterns, precipitation, and temperature extremes. + content: [] + placeholder: true diff --git a/backend/static/collections/1.4.yaml b/backend/static/collections/1.4.yaml new file mode 100644 index 0000000..b497f5c --- /dev/null +++ b/backend/static/collections/1.4.yaml @@ -0,0 +1,100 @@ +id: "1.4" +name: "Sea surface temperature (SST) bias, Sea surface salinity (SSS) bias" +theme: "Oceans and sea ice" +endorser: "CMIP Model Benchmarking Task Team" +version_control: "version 1 - 24-11-04 REF launch" +reference_dataset: "WOA2023" +provider_link: "https://doi.org/10.1029/2018MS001354" +content: + description: >- + The Sea surface temperature (SST) bias, Sea surface salinity (SSS) distributions + provide large scale patterns of surface ocean circulation as well as reflecting + dynamical air-sea interactions and ocean-sea ice interactions in the polar regions. + SST and SSS biases have a significant impact on the coupling of ESM's two major + components, the atmosphere and the ocean. Satellite data products and localized + moored sensors are used to produce measurements that are incorporated into reference + data to calculate SST and SSS biases in models + short_description: >- + Apply the ILAMB Methodology* to compare the model sea surface temperature (SST) + and sea surface salinity (SSS) with reference data from WOA2023 +diagnostics: + - provider_slug: ilamb + diagnostic_slug: so-woa2023-surface + - provider_slug: ilamb + diagnostic_slug: thetao-woa2023-surface +explorer_cards: + - title: "SST & SSS" + placeholder: true + content: + - type: box-whisker-chart + provider: ilamb + diagnostic: so-woa2023-surface + title: "Sea Surface Salinity" + metric_units: "psu" + grouping_config: + group_by: statistic + hue: statistic + other_filters: + region: "None" + metric: Bias + statistic: "Period Mean" + placeholder: true + - type: box-whisker-chart + provider: ilamb + diagnostic: thetao-woa2023-surface + title: "Sea Surface Temperature" + metric_units: "K" + grouping_config: + group_by: statistic + hue: statistic + other_filters: + region: "None" + metric: Bias + statistic: "Period Mean" + placeholder: true + - type: taylor-diagram + provider: ilamb + diagnostic: thetao-woa2023-surface + title: "Sea Surface Temperature (Taylor Diagram)" + description: >- + Taylor diagram showing the performance of global sea surface temperatures + against WOA2023 observations. Taylor diagrams summarize how closely models + match observations in terms of correlation, standard deviation, and + root-mean-square difference. + interpretation: >- + Points closer to the reference (black square) indicate better model + performance. Distance from the origin represents RMSE. + other_filters: + region: "None" + metric: "Spatial Distribution" + placeholder: true + - title: "Surface Ocean Salinity" + placeholder: true + content: + - type: box-whisker-chart + provider: ilamb + diagnostic: so-woa2023-surface + title: "Bias" + description: "Bias in Surface Ocean Salinity with respect to WOA2023" + span: 1 + metric_units: "psu" + other_filters: + statistic: Bias + grouping_config: + group_by: source_id + hue: source_id + placeholder: true + - type: taylor-diagram + provider: ilamb + diagnostic: so-woa2023-surface + title: "Taylor Diagram" + description: >- + Taylor diagram showing the performance of global surface ocean salinity + against WOA2023 observations. + interpretation: >- + Points closer to the reference (black square) indicate better model + performance. Distance from the origin represents RMSE. + other_filters: + region: "None" + metric: "Spatial Distribution" + placeholder: true diff --git a/backend/static/collections/1.5.yaml b/backend/static/collections/1.5.yaml new file mode 100644 index 0000000..64660cc --- /dev/null +++ b/backend/static/collections/1.5.yaml @@ -0,0 +1,23 @@ +id: "1.5" +name: "Ocean heat content (OHC)" +theme: "Oceans and sea ice" +endorser: "CMIP Model Benchmarking Task Team" +version_control: "version 1 - 24-11-04 REF launch" +reference_dataset: "MGOHCTA-WOA09" +provider_link: "https://doi.org/10.1029/2018MS001354" +content: + description: >- + The Ocean Heat Content (OHC) may provide one of the most reliable signals about + the long-term climate change and decadal to multidecadal variability, including + their temporal variation and spatial patterns. It is compared, between models and + observations, on a gridded basis (1° × 1°), based on almost all available in situ + ocean observations (e.g., Argo, conductivity–temperature–depth (CTD) profilers, + Mechanical BathyThermographs, bottles, moorings, gliders, and animal-borne ocean + sensors; Cheng et al., 2024). Before use, the data are carefully bias corrected, + vertically and horizontally interpolated and mapped onto a grid for comparison with + models + short_description: >- + Apply the ILAMB Methodology* to compare the model ocean heat content (OHC) with + reference data from MGOHCTA-WOA09 +diagnostics: [] +explorer_cards: [] diff --git a/backend/static/collections/1.6.yaml b/backend/static/collections/1.6.yaml new file mode 100644 index 0000000..0f184b4 --- /dev/null +++ b/backend/static/collections/1.6.yaml @@ -0,0 +1,62 @@ +id: "1.6" +name: "Antarctic & Arctic sea ice area seasonal cycle" +theme: "Oceans and sea ice" +endorser: "CMIP Model Benchmarking Task Team" +version_control: "version 1 - 24-11-04 REF launch" +reference_dataset: "OSISAF-V3" +provider_link: "https://docs.esmvaltool.org/en/latest/recipes/recipe_ref.html" +content: + description: >- + The sea ice area, calculated as the sum over the Northern (Arctic) and Southern + (Antarctic) Hemisphere grid cell areas multiplied by the sea ice fraction within + each cell, exhibits a distinct seasonal cycle. Arctic sea ice area typically has + minimum values in September, while Antarctic sea ice area is lowest in February. + The seasonal cycle is driven by the seasonal cycle of the insolation, sea ice + processes, as well as the exchange with the atmosphere and ocean and can be seen as + an overview metric for the general state of the sea ice in a model. In addition to + the multi-year average seasonal cycle of Arctic and Antarctic sea ice area, the + diagnostic produces time series of the September (Arctic) and February (Antarctic) + sea ice area. + short_description: >- + Seasonal cycle of Arctic (NH) and Antarctic (SH) sea ice area, time series of + Arctic September (NH) and Antarctic February (SH) sea ice area +diagnostics: + - provider_slug: esmvaltool + diagnostic_slug: sea-ice-area-basic +explorer_cards: + - title: "Sea Ice" + description: "Sea Ice Area Seasonal Cycle" + placeholder: true + content: + - type: series-chart + provider: esmvaltool + diagnostic: sea-ice-area-basic + title: "Sea Ice Area Seasonal Cycle (Southern Hemisphere)" + description: >- + 20-year average seasonal cycle of the sea ice area in the Southern Hemisphere + (OSISAF-CCI reference data currently missing). + span: 2 + metric_units: "1e6 km^2" + other_filters: + region: "Southern Hemisphere" + isolate_ids: "568191,568195,568199,568203,568207,568211,568215,568219,568223,568227,568231,568235,568239,568243,568247,568251,568255,568259,568263,568267,568271,568275,568279,568283,568287,568291,568295,568299,568303,568307,568311,568315,568319,568323,568327,568331,568335,568339,568343,568347,568351,568355,568359,568363,568367,568371,568375,568379,568383,568387,568391,568395,568399,568403,568407,568411,568415,568419,568423,568427,568431,568435,568439,568443,568447,568451,568455,568459,568463,568467,568471,568475,568479,568483,568487,568491,568495,568499,568503,568507,568511,568515,568519,568523,568527,568531,568535,568541,568545,568561,568585,568589,568593,568601,568609,568621,568627,568635,568639,568643" + exclude_ids: "568564,568570" + grouping_config: + group_by: source_id + hue: source_id + - type: series-chart + provider: esmvaltool + diagnostic: sea-ice-area-basic + title: "Sea Ice Area Seasonal Cycle (Northern Hemisphere)" + description: >- + 20-year average seasonal cycle of the sea ice area in the Northern Hemisphere + (OSISAF-CCI reference data currently missing). + span: 2 + metric_units: "1e6 km^2" + other_filters: + statistic: "20-year average seasonal cycle of the sea ice area" + exclude_ids: "568564,568570" + region: "Northern Hemisphere" + grouping_config: + group_by: source_id + hue: source_id diff --git a/backend/static/collections/2.1.yaml b/backend/static/collections/2.1.yaml new file mode 100644 index 0000000..9c38aa4 --- /dev/null +++ b/backend/static/collections/2.1.yaml @@ -0,0 +1,34 @@ +id: "2.1" +name: "Soil carbon" +theme: "Land and land ice" +endorser: "CMIP Model Benchmarking Task Team" +version_control: "version 1 - 24-11-04 REF launch" +reference_dataset: "HWSD-2-0, NCSCD-2-2" +provider_link: "https://doi.org/10.1029/2018MS001354" +content: + description: >- + Soil carbon is the organic matter and inorganic carbon in global soils. It is an + important component of the global carbon cycle and affects soil moisture retention + and saturation. Analyzing stored soil carbon helps track and quantify the dynamics + of the terrestrial carbon cycle within models and the movement of carbon through + the Earth system. + short_description: >- + Apply the ILAMB Methodology* to compare the model soil carbon with reference data + from HWSD-2-0 and NCSCD-2-2 +diagnostics: + - provider_slug: ilamb + diagnostic_slug: csoil-hwsd2 +explorer_cards: + - title: "Soil Carbon" + content: + - type: box-whisker-chart + provider: ilamb + diagnostic: csoil-hwsd2 + title: "Soil Carbon" + description: "Bias in Soil Carbon Content compared to HWSDv2" + metric_units: "kg/m^2" + other_filters: + statistic: Bias + grouping_config: + group_by: region + hue: region diff --git a/backend/static/collections/2.2.yaml b/backend/static/collections/2.2.yaml new file mode 100644 index 0000000..6fcc5ee --- /dev/null +++ b/backend/static/collections/2.2.yaml @@ -0,0 +1,58 @@ +id: "2.2" +name: "Gross primary production (GPP)" +theme: "Land and land ice" +endorser: "CMIP Model Benchmarking Task Team" +version_control: "version 1 - 24-11-04 REF launch" +reference_dataset: "FLUXNET2015-1-0, WECANN-1-0, CRU-4-9 (source_id not confirmed)" +provider_link: "https://doi.org/10.1029/2018MS001354" +content: + description: >- + Gross primary production is the process by which plants "fix" atmospheric or + aqueous carbon dioxide through photosynthetic reduction into organic compounds, and + it is affected by increases in atmospheric carbon dioxide (CO2) levels and warming + (Anav et al., 2015). A fraction of gross primary productivity supports plant + respiration and the rest is stored as biomass in stems, leaves, roots, or other + plant parts. Land use change, heat and drought stress due to anthropogenic warming, + and rising atmospheric CO2 will differentially influence gross primary production + in ecosystems and alter the global carbon cycle. Thus, models must be evaluated to + ensure they capture the observed responses to these changes. + short_description: >- + Apply the ILAMB Methodology* to compare the model gross primary production (GPP) + with reference data from FLUXNET2015-1-0 and WECANN-1-0 +diagnostics: + - provider_slug: ilamb + diagnostic_slug: gpp-wecann + - provider_slug: ilamb + diagnostic_slug: gpp-fluxnet2015 +explorer_cards: + - title: "Gross Primary Production" + placeholder: true + content: + - type: series-chart + provider: ilamb + diagnostic: gpp-fluxnet2015 + title: "Gross Primary Production (GPP) Annual Cycle" + description: "Calculated as the mean seasonal cycle over 2001-2010." + span: 2 + metric_units: "kgC/m^2/s" + other_filters: + metric: cycle_global + grouping_config: + group_by: source_id + hue: source_id + placeholder: true + - type: taylor-diagram + provider: ilamb + diagnostic: gpp-fluxnet2015 + title: "GPP Spatial Performance" + description: >- + Model performance in reproducing spatial patterns of Gross Primary Production + (GPP) compared to FLUXNET2015 observations. + interpretation: >- + Points closer to the reference (black square) indicate better model + performance. Distance from the origin represents RMSE. + span: 1 + other_filters: + region: global + reference_stddev: 1.0 + placeholder: true diff --git a/backend/static/collections/2.3.yaml b/backend/static/collections/2.3.yaml new file mode 100644 index 0000000..937d44d --- /dev/null +++ b/backend/static/collections/2.3.yaml @@ -0,0 +1,37 @@ +id: "2.3" +name: "Runoff" +theme: "Land and land ice" +endorser: "CMIP Model Benchmarking Task Team" +version_control: "version 1 - 24-11-04 REF launch" +reference_dataset: "LORA-1-1" +provider_link: "https://doi.org/10.1029/2018MS001354" +content: + description: >- + Surface water runoff plays an important role in the hydrological cycle by returning + excess precipitation to the oceans and controlling how much water flows into water + systems (Trenberth et al., 2007; Trenberth and Caron, 2001). Changes in atmospheric + circulation and distributions of precipitation have a direct effect on changes in + runoff from land. Models must be evaluated to ensure they exhibit the observed + responses to precipitation and soil moisture processes that lead to runoff and + transport of freshwater into rivers and oceans. + short_description: >- + Apply the ILAMB Methodology* to compare the model surface runoff with reference + data from LORA-1-1 +diagnostics: + - provider_slug: ilamb + diagnostic_slug: mrro-lora +explorer_cards: + - title: "Total Runoff" + content: + - type: box-whisker-chart + provider: ilamb + diagnostic: mrro-lora + title: "Total Runoff" + metric_units: "kg/m^2/s" + grouping_config: + group_by: statistic + hue: statistic + other_filters: + region: global + metric: Bias + statistic: Bias diff --git a/backend/static/collections/2.4.yaml b/backend/static/collections/2.4.yaml new file mode 100644 index 0000000..6b699ad --- /dev/null +++ b/backend/static/collections/2.4.yaml @@ -0,0 +1,29 @@ +id: "2.4" +name: "Surface soil moisture" +theme: "Land and land ice" +endorser: "CMIP Model Benchmarking Task Team" +version_control: "version 1 - 24-11-04 REF launch" +reference_dataset: "OLC-ORS-V0" +provider_link: "https://doi.org/10.1029/2018MS001354" +content: + short_description: >- + Apply the ILAMB Methodology* to compare the model surface soil moisture with + reference data from OLC-ORS-V0 +diagnostics: + - provider_slug: ilamb + diagnostic_slug: mrsos-wangmao +explorer_cards: + - title: "Surface Soil Moisture" + content: + - type: box-whisker-chart + provider: ilamb + diagnostic: mrsos-wangmao + title: "Surface Soil Moisture" + metric_units: "kg/m^2" + grouping_config: + group_by: statistic + hue: statistic + other_filters: + region: global + metric: Bias + statistic: Bias diff --git a/backend/static/collections/2.5.yaml b/backend/static/collections/2.5.yaml new file mode 100644 index 0000000..92a4cdd --- /dev/null +++ b/backend/static/collections/2.5.yaml @@ -0,0 +1,30 @@ +id: "2.5" +name: "Net ecosystem carbon balance" +theme: "Land and land ice" +endorser: "CMIP Model Benchmarking Task Team" +version_control: "version 1 - 24-11-04 REF launch" +reference_dataset: "HOFFMAN-1-0" +provider_link: "https://doi.org/10.1029/2018MS001354" +content: + short_description: >- + Comparison of the model integrated land net ecosystem carbon balance with reference + data from HOFFMAN-1-0 +diagnostics: + - provider_slug: ilamb + diagnostic_slug: nbp-hoffman +explorer_cards: + - title: "Net Biome Production" + placeholder: true + content: + - type: box-whisker-chart + provider: ilamb + diagnostic: nbp-hoffman + title: "Net Biome Production" + description: >- + Bias in Net Biome Production (NBP) compared to Hoffman et al. (2020) estimates + metric_units: "PgC/yr" + clip_max: 2000 + grouping_config: + group_by: statistic + hue: statistic + placeholder: true diff --git a/backend/static/collections/2.6.yaml b/backend/static/collections/2.6.yaml new file mode 100644 index 0000000..29ba1cc --- /dev/null +++ b/backend/static/collections/2.6.yaml @@ -0,0 +1,29 @@ +id: "2.6" +name: "Leaf area index (LAI)" +theme: "Land and land ice" +endorser: "CMIP Model Benchmarking Task Team" +version_control: "version 1 - 24-11-04 REF launch" +reference_dataset: "NOAA-NCEI-LAI-5-0, LAI4g-1-2" +provider_link: "https://doi.org/10.1029/2018MS001354" +content: + short_description: >- + Apply the ILAMB Methodology* to compare the model leaf area index (LAI) with + reference data from NOAA-NCEI-LAI-5-0, and LAI4g-1-2 +diagnostics: + - provider_slug: ilamb + diagnostic_slug: lai-avh15c1 +explorer_cards: + - title: "Leaf Area Index" + content: + - type: box-whisker-chart + provider: ilamb + diagnostic: lai-avh15c1 + title: "Leaf Area Index" + metric_units: "1" + grouping_config: + group_by: statistic + hue: statistic + other_filters: + region: global + metric: Bias + statistic: Bias diff --git a/backend/static/collections/2.7.yaml b/backend/static/collections/2.7.yaml new file mode 100644 index 0000000..8a62eba --- /dev/null +++ b/backend/static/collections/2.7.yaml @@ -0,0 +1,28 @@ +id: "2.7" +name: "Snow cover" +theme: "Land and land ice" +endorser: "CMIP Model Benchmarking Task Team" +version_control: "version 1 - 24-11-04 REF launch" +reference_dataset: "CCI-CryoClim-FSC-1" +provider_link: "https://doi.org/10.1029/2018MS001354" +content: + short_description: >- + Apply the ILAMB Methodology* to compare the model snow cover with reference data + from CCI-CryoClim-FSC-1 +diagnostics: + - provider_slug: ilamb + diagnostic_slug: snc-esacci +explorer_cards: + - title: "Snow Cover Extent" + content: + - type: box-whisker-chart + provider: ilamb + diagnostic: snc-esacci + title: "Snow Cover Extent" + metric_units: "%" + grouping_config: + group_by: statistic + hue: statistic + other_filters: + region: global + metric: Bias diff --git a/backend/static/collections/3.1.yaml b/backend/static/collections/3.1.yaml new file mode 100644 index 0000000..ac62f50 --- /dev/null +++ b/backend/static/collections/3.1.yaml @@ -0,0 +1,16 @@ +id: "3.1" +name: "Annual cycle and seasonal mean of multiple variables" +theme: "Atmosphere" +endorser: "CMIP Model Benchmarking Task Team" +version_control: "version 1 - 24-11-04 REF launch" +reference_dataset: "C3S-GTO-ECV-9-0, SAGE-CCI-OMPS-v0008, ERA-5" +provider_link: "http://pcmdi.github.io/pcmdi_metrics/examples/Demo_1b_mean_climate.html" +content: + short_description: >- + Maps of seasonal and annual climatology are generated for the reference datasets + and model output on a target grid (2.5x2.5 deg), then calculate diverse metrics + including bias, RMSE, spatial pattern correlation, and standard deviation. +diagnostics: + - provider_slug: pmp + diagnostic_slug: annual-cycle +explorer_cards: [] diff --git a/backend/static/collections/3.2.yaml b/backend/static/collections/3.2.yaml new file mode 100644 index 0000000..22b2b9e --- /dev/null +++ b/backend/static/collections/3.2.yaml @@ -0,0 +1,14 @@ +id: "3.2" +name: "Radiative and heat fluxes at the surface and top of atmosphere (TOA)" +theme: "Atmosphere" +endorser: "CMIP Model Benchmarking Task Team" +version_control: "version 1 - 24-11-04 REF launch" +reference_dataset: "CERES-EBAF-4-2" +provider_link: "http://pcmdi.github.io/pcmdi_metrics/examples/Demo_1a_compute_climatologies.html" +content: + short_description: >- + Maps of seasonal and annual climatology are generated for the reference dataset + and model output on a target grid (2.5x2.5 deg), then calculate diverse metrics + including bias, RMSE, spatial pattern correlation, and standard deviation. +diagnostics: [] +explorer_cards: [] diff --git a/backend/static/collections/3.3.yaml b/backend/static/collections/3.3.yaml new file mode 100644 index 0000000..7beba0a --- /dev/null +++ b/backend/static/collections/3.3.yaml @@ -0,0 +1,135 @@ +id: "3.3" +name: "Climate variability modes (e.g., ENSO, Madden-Julian Oscillation (MJO), Extratropical modes of variability, monsoon)" +theme: "Atmosphere" +endorser: "CMIP Model Benchmarking Task Team" +version_control: "version 1 - 24-11-04 REF launch" +reference_dataset: "20CR-V2, HadISST-1-1" +provider_link: "http://pcmdi.github.io/pcmdi_metrics/metrics.html" +content: + short_description: >- + For extratropical modes of variability, maps of variability mode pattern and their + principal component time series are generated from the reference dataset and model + output. Then maps are compared to calculate bias, RMSE, and spatial pattern + correlation, and time series are compared to calculate the ratio from their standard + deviations. +diagnostics: + - provider_slug: pmp + diagnostic_slug: extratropical-modes-of-variability-nam + - provider_slug: pmp + diagnostic_slug: extratropical-modes-of-variability-nao + - provider_slug: pmp + diagnostic_slug: extratropical-modes-of-variability-npgo + - provider_slug: pmp + diagnostic_slug: extratropical-modes-of-variability-npo + - provider_slug: pmp + diagnostic_slug: extratropical-modes-of-variability-pdo + - provider_slug: pmp + diagnostic_slug: extratropical-modes-of-variability-pna + - provider_slug: pmp + diagnostic_slug: extratropical-modes-of-variability-sam +explorer_cards: + - title: "Extratropical Modes of Variability" + description: >- + Spatial comparison of simulated vs. observed leading EOF patterns representing + the main modes of low-frequency variability in the extra-tropical atmosphere and + ocean, based on seasonal-mean sea level pressure and monthly-mean sea surface + temperature anomalies in the spatial domains defined in + [Lee et al. 2019](https://doi.org/10.1007/s00382-018-4355-4), Table 1. Simulated + EOFs are obtained with the "Common Basis Function" (CBF) approach described in + the appendix of this article. The considered time-period is 1901-2005 for the NH + modes and 1951-2005 for the SAM. + content: + - type: box-whisker-chart + provider: pmp + diagnostic: extratropical-modes-of-variability-nam + title: "Northern Annular Mode (NAM) RMSE" + description: >- + Northern Annular Mode (NAM) individual-model pattern RMSE, see + [Lee et al. 2019](https://doi.org/10.1007/s00382-018-4355-4) + other_filters: + method: cbf + statistic: rms + experiment_id: historical + grouping_config: + group_by: season + hue: season + y_min: 0 + y_max: 1.5 + - type: box-whisker-chart + provider: pmp + diagnostic: extratropical-modes-of-variability-sam + title: "Southern Annual Mode (SAM) RMSE" + description: >- + Southern Annual Mode (SAM) individual-model pattern RMSE, see + [Lee et al. 2019](https://doi.org/10.1007/s00382-018-4355-4) + other_filters: + method: cbf + statistic: rms + experiment_id: historical + grouping_config: + group_by: season + hue: season + y_min: 0 + y_max: 1.5 + - type: box-whisker-chart + provider: pmp + diagnostic: extratropical-modes-of-variability-pna + title: "Pacific-North American (PNA) RMSE" + description: >- + Pacific-North American (PNA) individual-model pattern RMSE, see + [Lee et al. 2019](https://doi.org/10.1007/s00382-018-4355-4) + other_filters: + method: cbf + statistic: rms + experiment_id: historical + grouping_config: + group_by: season + hue: season + y_min: 0 + y_max: 1.5 + - type: box-whisker-chart + provider: pmp + diagnostic: extratropical-modes-of-variability-nao + title: "North Atlantic Oscillation (NAO) RMSE" + description: >- + North Atlantic Oscillation (NAO) individual-model pattern RMSE, see + [Lee et al. 2019](https://doi.org/10.1007/s00382-018-4355-4) + other_filters: + method: cbf + statistic: rms + experiment_id: historical + grouping_config: + group_by: season + hue: season + y_min: 0 + y_max: 1.5 + - type: box-whisker-chart + provider: pmp + diagnostic: extratropical-modes-of-variability-pdo + title: "Pacific-Decadal Oscillation (PDO) RMSE" + description: >- + Pacific-Decadal Oscillation (PDO) individual-model pattern RMSE, see + [Lee et al. 2019](https://doi.org/10.1007/s00382-018-4355-4) + other_filters: + method: cbf + statistic: rms + experiment_id: historical + grouping_config: + group_by: season + hue: season + y_min: 0 + - type: box-whisker-chart + provider: pmp + diagnostic: extratropical-modes-of-variability-npgo + title: "North Pacific Gyre Oscillation (NPGO) RMSE" + description: >- + North Pacific Gyre Oscillation (NPGO) individual-model pattern RMSE, see + [Lee et al. 2019](https://doi.org/10.1007/s00382-018-4355-4) + other_filters: + method: cbf + statistic: rms + experiment_id: historical + grouping_config: + group_by: season + hue: experiment_id + y_min: 0 diff --git a/backend/static/collections/3.6.yaml b/backend/static/collections/3.6.yaml new file mode 100644 index 0000000..77ada81 --- /dev/null +++ b/backend/static/collections/3.6.yaml @@ -0,0 +1,28 @@ +id: "3.6" +name: "Cloud radiative effects" +theme: "Atmosphere" +endorser: "CMIP Model Benchmarking Task Team" +version_control: "version 1 - 24-11-04 REF launch" +reference_dataset: "CALIPSO-ICECLOUD-1-0, ESACCI-CLOUD-AVHRR-AMPM-3-0, CERES-EBAF-4-2, ERA-5, GPCP-SG-2-3" +provider_link: "https://docs.esmvaltool.org/en/latest/recipes/recipe_ref.html" +content: + short_description: >- + Maps and zonal means of longwave and shortwave cloud radiative effect +diagnostics: + - provider_slug: esmvaltool + diagnostic_slug: cloud-radiative-effects +explorer_cards: + - title: "Cloud & Radiation" + description: "Cloud properties and their effect on the Earth's energy balance." + placeholder: true + content: + - type: box-whisker-chart + provider: esmvaltool + diagnostic: cloud-radiative-effects + title: "Cloud Radiative Effects" + description: "Not sure" + other_filters: + statistic: bias + grouping_config: + group_by: metric + hue: metric diff --git a/backend/static/collections/3.7.yaml b/backend/static/collections/3.7.yaml new file mode 100644 index 0000000..bc16eca --- /dev/null +++ b/backend/static/collections/3.7.yaml @@ -0,0 +1,21 @@ +id: "3.7" +name: "Scatterplots of two cloud-relevant variables (for specific regions of the globe and specific cloud regimes)" +theme: "Atmosphere" +endorser: "CMIP Model Benchmarking Task Team" +version_control: "version 1 - 24-11-04 REF launch" +reference_dataset: "CALIPSO-ICECLOUD-1-0, ESACCI-CLOUD-AVHRR-AMPM-3-0, CERES-EBAF-4-2, ERA-5, GPCP-SG-2-3" +provider_link: "https://docs.esmvaltool.org/en/latest/recipes/recipe_ref.html" +content: + short_description: "2D histograms with focus on clouds" +diagnostics: + - provider_slug: esmvaltool + diagnostic_slug: cloud-scatterplots-cli-ta + - provider_slug: esmvaltool + diagnostic_slug: cloud-scatterplots-clivi-lwcre + - provider_slug: esmvaltool + diagnostic_slug: cloud-scatterplots-clt-swcre + - provider_slug: esmvaltool + diagnostic_slug: cloud-scatterplots-clwvi-pr + - provider_slug: esmvaltool + diagnostic_slug: cloud-scatterplots-reference +explorer_cards: [] diff --git a/backend/static/collections/4.1.yaml b/backend/static/collections/4.1.yaml new file mode 100644 index 0000000..f55e57e --- /dev/null +++ b/backend/static/collections/4.1.yaml @@ -0,0 +1,66 @@ +id: "4.1" +name: "Equilibrium climate sensitivity (ECS)" +theme: "Earth System" +endorser: "CMIP Model Benchmarking Task Team" +version_control: "version 1 - 24-11-04 REF launch" +reference_dataset: "" +provider_link: "https://docs.esmvaltool.org/en/latest/recipes/recipe_ecs.html" +content: + short_description: >- + Equilibrium climate sensitivity is defined as the change in global mean temperature + as a result of a doubling of the atmospheric CO2 concentration compared to + pre-industrial times after the climate system has reached a new equilibrium. This + diagnostic uses a regression method based on Gregory et al. (2004). +diagnostics: + - provider_slug: esmvaltool + diagnostic_slug: equilibrium-climate-sensitivity +explorer_cards: + - title: "Climate Sensitivity (ECS & Lambda)" + description: "Fundamental metrics of the global climate response to CO2." + content: + - type: box-whisker-chart + provider: esmvaltool + diagnostic: equilibrium-climate-sensitivity + title: "Equilibrium Climate Sensitivity (ECS)" + description: >- + ECS represents the long-term change in global mean surface temperature + following a doubling of atmospheric CO2 concentrations. It is a key metric + for understanding the sensitivity of Earth's climate system to radiative + forcing and is crucial for predicting future climate change and informing + policy decisions. ECS is influenced by various feedback mechanisms, including + water vapor, clouds, and ice-albedo feedbacks, which can amplify or dampen + the initial warming response. + interpretation: >- + Higher ECS values indicate a more sensitive climate system, leading to greater + warming for a given increase in CO2. + metric_units: "" + clip_max: 10 + other_filters: + metric: ecs + grouping_config: + group_by: metric + hue: metric + - type: box-whisker-chart + provider: esmvaltool + diagnostic: equilibrium-climate-sensitivity + title: "Lambda" + description: >- + Climate feedback parameter (λ) quantifies the sensitivity of Earth's climate + system to radiative forcing, representing the change in global mean surface + temperature per unit of radiative forcing (W/m²). It is a key metric for + understanding the balance between incoming solar radiation and outgoing + terrestrial radiation, influencing how the climate responds to factors such + as greenhouse gas concentrations and aerosols. + interpretation: >- + A more negative λ value indicates that the climate system has stronger + stabilizing feedbacks, which help to counteract warming and maintain + equilibrium. Conversely, a less negative or positive λ suggests that the + climate system is more sensitive to perturbations, potentially leading to + amplified warming in response to increased greenhouse gas concentrations. + metric_units: "W/m²/K" + clip_max: 10 + other_filters: + metric: lambda + grouping_config: + group_by: metric + hue: metric diff --git a/backend/static/collections/4.2.yaml b/backend/static/collections/4.2.yaml new file mode 100644 index 0000000..503e185 --- /dev/null +++ b/backend/static/collections/4.2.yaml @@ -0,0 +1,37 @@ +id: "4.2" +name: "Transient climate response (TCR)" +theme: "Earth System" +endorser: "CMIP Model Benchmarking Task Team" +version_control: "version 1 - 24-11-04 REF launch" +reference_dataset: "" +provider_link: "https://docs.esmvaltool.org/en/latest/recipes/recipe_tcr.html" +content: + short_description: >- + The transient climate response (TCR) is defined as the global and annual mean + surface air temperature anomaly in the 1pctCO2 scenario (1% CO2 increase per year) + for a 20 year period centered at the time of CO2 doubling, i.e. using the years 61 + to 80 after the start of the simulation. We calculate the temperature anomaly by + subtracting a linear fit of the piControl run for all 140 years of the 1pctCO2 + experiment prior to the TCR calculation (see Gregory and Forster, 2008). +diagnostics: + - provider_slug: esmvaltool + diagnostic_slug: transient-climate-response +explorer_cards: + - title: "Transient Climate Response" + content: + - type: box-whisker-chart + provider: esmvaltool + diagnostic: transient-climate-response + title: "Transient Climate Response (TCR)" + description: >- + TCR measures the immediate warming response of the climate system to a + sustained increase in CO2 concentrations. It is a key metric for understanding + the short-term impacts of greenhouse gas emissions on global temperatures and + is critical for informing climate policy and adaptation strategies. + interpretation: >- + Higher TCR values indicate a more sensitive climate system, leading to greater + warming in the near term for a given increase in CO2. + metric_units: "K" + grouping_config: + group_by: metric + hue: metric diff --git a/backend/static/collections/4.3.yaml b/backend/static/collections/4.3.yaml new file mode 100644 index 0000000..763e99e --- /dev/null +++ b/backend/static/collections/4.3.yaml @@ -0,0 +1,23 @@ +id: "4.3" +name: "Transient climate response to cumulative emissions of carbon dioxide (TCRE)" +theme: "Earth System" +endorser: "CMIP Model Benchmarking Task Team" +version_control: "version 1 - 24-11-04 REF launch" +reference_dataset: "" +provider_link: "https://docs.esmvaltool.org/en/latest/recipes/recipe_tcre.html" +content: + short_description: >- + The idea that global temperature rise is directly proportional to the total amount + of carbon dioxide (CO2) released into the atmosphere is fundamental to climate + policy. The concept stems from research showing a clear linear relationship between + cumulative CO2 emissions and global temperature change in climate models (Allen et + al. 2009; Matthews et al. 2009; Zickfeld et al. 2009). This relationship is called + the Transient Climate Response to Cumulative CO2 Emissions (TCRE), which represents + the amount of global warming caused by each trillion tonnes of carbon emitted. This + simple yet powerful tool allows policymakers to directly link emission budgets to + specific temperature targets and compare the long-term effects of different + emissions scenarios. +diagnostics: + - provider_slug: esmvaltool + diagnostic_slug: transient-climate-response-emissions +explorer_cards: [] diff --git a/backend/static/collections/4.4.yaml b/backend/static/collections/4.4.yaml new file mode 100644 index 0000000..585d681 --- /dev/null +++ b/backend/static/collections/4.4.yaml @@ -0,0 +1,23 @@ +id: "4.4" +name: "Zero emissions commitment (ZEC)" +theme: "Earth System" +endorser: "CMIP Model Benchmarking Task Team" +version_control: "version 1 - 24-11-04 REF launch" +reference_dataset: "" +provider_link: "https://docs.esmvaltool.org/en/latest/recipes/recipe_zec.html" +content: + short_description: >- + The Zero Emissions Commitment (ZEC) quantifies the change in global mean temperature + expected to occur after net carbon dioxide (CO2) emissions cease. ZEC is therefore + important to consider when estimating the remaining carbon budget. Calculation of + ZEC requires dedicated simulations with emissions set to zero, branching off a base + simulation with emissions. In CMIP6 the simulations were part of ZECMIP, with the + simulations called esm-1pct-brch-xPgC branching off the 1pctCO2 simulation when + emissions reach x PgC. The default x was 1000PgC, with additional simulations for + 750PgC and 2000PgC. In CMIP7, ZEC simulations (esm-flat10-zec) are part of the + fast track and branch off (esm-flat10) with constant emissions of 10GtC/yr at year + 100 (Sanderson 2024). +diagnostics: + - provider_slug: esmvaltool + diagnostic_slug: zero-emission-commitment +explorer_cards: [] diff --git a/backend/static/collections/4.5.yaml b/backend/static/collections/4.5.yaml new file mode 100644 index 0000000..9ce7aba --- /dev/null +++ b/backend/static/collections/4.5.yaml @@ -0,0 +1,18 @@ +id: "4.5" +name: "Historical changes in climate variables (time series, trends)" +theme: "Earth System" +endorser: "CMIP Model Benchmarking Task Team" +version_control: "version 1 - 24-11-04 REF launch" +reference_dataset: "HadCRUT5-0-2-0, CERES-EBAF-4-2, ERA-5, GPCP-SG-2-3, HadISST-1-1" +provider_link: "https://docs.esmvaltool.org/en/latest/recipes/recipe_ref.html" +content: + short_description: >- + Time series, linear trend, and annual cycle for IPCC regions +diagnostics: + - provider_slug: esmvaltool + diagnostic_slug: regional-historical-annual-cycle + - provider_slug: esmvaltool + diagnostic_slug: regional-historical-timeseries + - provider_slug: esmvaltool + diagnostic_slug: regional-historical-trend +explorer_cards: [] diff --git a/backend/static/collections/5.3.yaml b/backend/static/collections/5.3.yaml new file mode 100644 index 0000000..c3bddae --- /dev/null +++ b/backend/static/collections/5.3.yaml @@ -0,0 +1,33 @@ +id: "5.3" +name: "Evaluation of key climate variables at global warming levels" +theme: "Impacts and Adaptation" +endorser: "CMIP Model Benchmarking Task Team" +version_control: "version 1 - 24-11-04 REF launch" +reference_dataset: "GPCP-SG-2-3, HadCRUT5-0-2-0" +provider_link: "https://docs.esmvaltool.org/en/latest/recipes/recipe_calculate_gwl_exceedance_stats.html" +content: + short_description: >- + This diagnostic calculates years of Global Warming Level (GWL) exceedances in CMIP + models as described in Swaminathan et al (2022). Time series of the anomalies in + annual global mean surface air temperature (GSAT) are calculated with respect to + the 1850-1900 time-mean of each individual time series. To limit the influence of + short-term variability, a 21-year centered running mean is applied to the time + series. The year at which the time series exceeds warming levels or temperatures + such as 1.5C is then recorded for the specific model ensemble member and future + scenario. Once the years of exceedance are calculated, the time averaged global + mean and standard deviation for the multimodel ensemble over the 21-year period + around the year of exceedance are plotted. +diagnostics: + - provider_slug: esmvaltool + diagnostic_slug: climate-at-global-warming-levels +explorer_cards: + - title: "Warming Levels" + description: >- + Climate conditions at different global warming levels, relevant to policy targets. + content: + - type: figure-gallery + provider: esmvaltool + diagnostic: climate-at-global-warming-levels + title: "Global Mean Temperature Change at Warming Levels" + span: 2 + placeholder: true diff --git a/backend/static/collections/5.4.yaml b/backend/static/collections/5.4.yaml new file mode 100644 index 0000000..2bdffa8 --- /dev/null +++ b/backend/static/collections/5.4.yaml @@ -0,0 +1,21 @@ +id: "5.4" +name: "Climate drivers for fire (fire burnt area, fire weather and fuel continuity)" +theme: "Impacts and Adaptation" +endorser: "CMIP Model Benchmarking Task Team" +version_control: "version 1 - 24-11-04 REF launch" +reference_dataset: "GFED-5 (source_id not confirmed)" +provider_link: "https://docs.esmvaltool.org/en/latest/recipes/recipe_ref_fire.html" +content: + short_description: >- + The diagnostic relies on the processing of fire climate drivers through the ConFire + model and is based on Jones et al. (2024). The diagnostic computes the burnt + fraction for each grid cell based on a number of drivers. Additionally, the + respective controls due to fire weather and fuel load/continuity are computed. The + stochastic control corresponds to the unmodelled processed influencing to fire + occurrence. +diagnostics: + - provider_slug: esmvaltool + diagnostic_slug: climate-drivers-for-fire + - provider_slug: ilamb + diagnostic_slug: burntfractionall-gfed +explorer_cards: [] diff --git a/backend/static/collections/themes.yaml b/backend/static/collections/themes.yaml new file mode 100644 index 0000000..6128d5d --- /dev/null +++ b/backend/static/collections/themes.yaml @@ -0,0 +1,24 @@ +atmosphere: + title: "Atmosphere" + description: "Atmospheric diagnostics including variability modes, clouds, and radiation." + collections: ["3.1", "3.2", "3.3", "3.6", "3.7"] + +earth-system: + title: "Earth System" + description: "Earth system diagnostics including climate sensitivity and ENSO." + collections: ["1.3", "4.1", "4.2", "4.3", "4.4", "4.5"] + +impact-and-adaptation: + title: "Impact and Adaptation" + description: "Diagnostics related to climate impacts, adaptation, and warming levels." + collections: ["5.3", "5.4"] + +land: + title: "Land and Land Ice" + description: "Land surface diagnostics including carbon cycle, hydrology, and vegetation." + collections: ["2.1", "2.2", "2.3", "2.4", "2.5", "2.6", "2.7"] + +ocean: + title: "Ocean and Sea Ice" + description: "Ocean and sea ice diagnostics including circulation, temperature, and ice area." + collections: ["1.1", "1.2", "1.3", "1.4", "1.5", "1.6"] diff --git a/backend/tests/test_api/test_api_explorer.py b/backend/tests/test_api/test_api_explorer.py new file mode 100644 index 0000000..cbd0b7e --- /dev/null +++ b/backend/tests/test_api/test_api_explorer.py @@ -0,0 +1,257 @@ +"""Tests for the /api/v1/explorer/* endpoints.""" + +from pathlib import Path + +import pytest +import yaml +from fastapi.testclient import TestClient + +from ref_backend.core.collections import load_all_collections, load_theme_mapping + + +@pytest.fixture(autouse=True) +def clear_collection_caches(): + """Clear collection lru_caches before and after each test.""" + load_all_collections.cache_clear() + load_theme_mapping.cache_clear() + yield + load_all_collections.cache_clear() + load_theme_mapping.cache_clear() + + +def _write_yaml(path: Path, data) -> None: + with open(path, "w") as f: + yaml.dump(data, f) + + +@pytest.fixture +def collections_dir(tmp_path: Path, monkeypatch): + """Create a temp collections directory with fixture YAML files.""" + cols_dir = tmp_path / "collections" + cols_dir.mkdir() + + # Collection 1.2 — atmosphere collection with y_min/y_max + _write_yaml( + cols_dir / "1.2.yaml", + { + "id": "1.2", + "name": "Atmosphere Collection", + "theme": "Atmosphere", + "endorser": "WCRP", + "diagnostics": [{"provider_slug": "pmp", "diagnostic_slug": "mean-climate"}], + "explorer_cards": [ + { + "title": "Temperature Bias", + "content": [ + { + "type": "box-whisker-chart", + "provider": "pmp", + "diagnostic": "mean-climate", + "title": "Temperature Bias Chart", + "y_min": -5.0, + "y_max": 5.0, + "grouping_config": {"group_by": "model", "hue": "experiment"}, + } + ], + } + ], + }, + ) + + # Collection for sea ice with other_filters + _write_yaml( + cols_dir / "2.1.yaml", + { + "id": "2.1", + "name": "Sea Ice Collection", + "theme": "Ocean", + "diagnostics": [], + "explorer_cards": [ + { + "title": "Sea Ice Extent", + "content": [ + { + "type": "series-chart", + "provider": "pmp", + "diagnostic": "sea-ice", + "title": "Sea Ice Series", + "other_filters": { + "isolate_ids": "CMIP6.ssp585.NH", + "exclude_ids": "CMIP6.piControl", + }, + "grouping_config": {"group_by": "model", "hue": "experiment"}, + } + ], + } + ], + }, + ) + + # ENSO collection with placeholder card (empty content) + _write_yaml( + cols_dir / "3.1.yaml", + { + "id": "3.1", + "name": "ENSO Collection", + "theme": "Earth System", + "diagnostics": [], + "explorer_cards": [ + { + "title": "ENSO Placeholder", + "placeholder": True, + "content": [], + } + ], + }, + ) + + # themes.yaml + _write_yaml( + cols_dir / "themes.yaml", + [ + { + "slug": "ocean", + "title": "Ocean Theme", + "description": "Ocean diagnostics", + "collections": ["2.1"], + }, + { + "slug": "earth-system", + "title": "Earth System Theme", + "collections": ["1.2", "3.1"], + }, + ], + ) + + monkeypatch.setattr( + "ref_backend.core.collections.get_collections_dir", + lambda: cols_dir, + ) + monkeypatch.setattr( + "ref_backend.core.collections.get_themes_path", + lambda: cols_dir / "themes.yaml", + ) + + return cols_dir + + +def test_list_collections_returns_all_ids(client: TestClient, settings, collections_dir): + """GET /explorer/collections/ returns all collection IDs.""" + r = client.get(f"{settings.API_V1_STR}/explorer/collections/") + assert r.status_code == 200 + data = r.json() + ids = [c["id"] for c in data] + assert "1.2" in ids + assert "2.1" in ids + assert "3.1" in ids + + +def test_get_collection_12_returns_correct_cards(client: TestClient, settings, collections_dir): + """GET /explorer/collections/1.2 returns the atmosphere collection with y_min/y_max.""" + r = client.get(f"{settings.API_V1_STR}/explorer/collections/1.2") + assert r.status_code == 200 + data = r.json() + assert data["id"] == "1.2" + assert data["name"] == "Atmosphere Collection" + assert len(data["explorer_cards"]) == 1 + content = data["explorer_cards"][0]["content"][0] + assert content["y_min"] == -5.0 + assert content["y_max"] == 5.0 + + +def test_get_collection_nonexistent_returns_404(client: TestClient, settings, collections_dir): + """GET /explorer/collections/nonexistent returns 404.""" + r = client.get(f"{settings.API_V1_STR}/explorer/collections/nonexistent") + assert r.status_code == 404 + + +def test_list_themes_returns_expected_slugs(client: TestClient, settings, collections_dir): + """GET /explorer/themes/ returns expected theme slugs.""" + r = client.get(f"{settings.API_V1_STR}/explorer/themes/") + assert r.status_code == 200 + data = r.json() + slugs = [t["slug"] for t in data] + assert "ocean" in slugs + assert "earth-system" in slugs + + +def test_get_theme_ocean_returns_aggregated_cards(client: TestClient, settings, collections_dir): + """GET /explorer/themes/ocean returns theme with aggregated sea-ice cards.""" + r = client.get(f"{settings.API_V1_STR}/explorer/themes/ocean") + assert r.status_code == 200 + data = r.json() + assert data["slug"] == "ocean" + assert len(data["explorer_cards"]) == 1 + card = data["explorer_cards"][0] + assert card["title"] == "Sea Ice Extent" + + +def test_get_theme_nonexistent_returns_404(client: TestClient, settings, collections_dir): + """GET /explorer/themes/nonexistent returns 404.""" + r = client.get(f"{settings.API_V1_STR}/explorer/themes/nonexistent") + assert r.status_code == 404 + + +def test_response_field_names_are_snake_case(client: TestClient, settings, collections_dir): + """Response fields use snake_case, not camelCase.""" + r = client.get(f"{settings.API_V1_STR}/explorer/collections/1.2") + assert r.status_code == 200 + data = r.json() + + card_content = data["explorer_cards"][0]["content"][0] + + # These snake_case keys must be present + assert "y_min" in card_content + assert "y_max" in card_content + assert "grouping_config" in card_content + assert "group_by" in card_content["grouping_config"] + + # No camelCase variants allowed + for key in ["yMin", "yMax", "groupingConfig", "groupBy", "showZeroLine"]: + assert key not in card_content + assert key not in card_content.get("grouping_config", {}) + + # Top-level collection fields + assert "explorer_cards" in data + assert "explorerCards" not in data + + +def test_enso_placeholder_card_in_earth_system_theme(client: TestClient, settings, collections_dir): + """ENSO placeholder card appears in earth-system theme with empty content list.""" + r = client.get(f"{settings.API_V1_STR}/explorer/themes/earth-system") + assert r.status_code == 200 + data = r.json() + + all_cards = data["explorer_cards"] + enso_cards = [c for c in all_cards if c.get("placeholder") is True] + assert len(enso_cards) >= 1 + + enso_card = enso_cards[0] + assert enso_card["title"] == "ENSO Placeholder" + assert enso_card["content"] == [] + + +def test_sea_ice_cards_include_other_filters(client: TestClient, settings, collections_dir): + """Sea ice card content includes isolate_ids and exclude_ids in other_filters.""" + r = client.get(f"{settings.API_V1_STR}/explorer/collections/2.1") + assert r.status_code == 200 + data = r.json() + + content = data["explorer_cards"][0]["content"][0] + assert "other_filters" in content + filters = content["other_filters"] + assert "isolate_ids" in filters + assert "exclude_ids" in filters + assert filters["isolate_ids"] == "CMIP6.ssp585.NH" + assert filters["exclude_ids"] == "CMIP6.piControl" + + +def test_atmosphere_cards_include_y_min_y_max(client: TestClient, settings, collections_dir): + """Atmosphere collection cards include y_min and y_max values.""" + r = client.get(f"{settings.API_V1_STR}/explorer/collections/1.2") + assert r.status_code == 200 + data = r.json() + + content = data["explorer_cards"][0]["content"][0] + assert content["y_min"] == -5.0 + assert content["y_max"] == 5.0 diff --git a/backend/tests/test_core/test_core_collections.py b/backend/tests/test_core/test_core_collections.py new file mode 100644 index 0000000..efb02ea --- /dev/null +++ b/backend/tests/test_core/test_core_collections.py @@ -0,0 +1,531 @@ +"""Tests for collection loading and mapping functionality.""" + +import json +import logging +from pathlib import Path +from unittest.mock import patch + +import pytest +import yaml + +from ref_backend.core.collections import ( + AFTCollectionCard, + AFTCollectionCardContent, + AFTCollectionDetail, + AFTCollectionGroupingConfig, + AFTCollectionSummary, + ThemeDetail, + get_collection_by_id, + get_collection_summaries, + get_theme_by_slug, + get_theme_summaries, + load_all_collections, + load_theme_mapping, +) + + +@pytest.fixture(autouse=True) +def clear_collection_caches(): + """Clear all lru_cache'd functions before and after each test.""" + load_all_collections.cache_clear() + load_theme_mapping.cache_clear() + yield + load_all_collections.cache_clear() + load_theme_mapping.cache_clear() + + +def _write_collection(tmp_path: Path, filename: str, data: dict) -> Path: + """Write a collection YAML file and return its path.""" + p = tmp_path / filename + with open(p, "w") as f: + yaml.dump(data, f) + return p + + +def _write_themes(tmp_path: Path, data: list) -> Path: + """Write a themes.yaml file and return its path.""" + p = tmp_path / "themes.yaml" + with open(p, "w") as f: + yaml.dump(data, f) + return p + + +_MINIMAL_COLLECTION = { + "id": "1.1", + "name": "Test Collection", + "theme": "Ocean", + "endorser": "WCRP", + "version_control": "1.0", + "reference_dataset": "ERA5", + "provider_link": "https://example.com", + "content": { + "description": "Full description", + "short_description": "Short", + }, + "diagnostics": [ + {"provider_slug": "pmp", "diagnostic_slug": "annual-cycle"}, + ], + "explorer_cards": [ + { + "title": "Card One", + "description": "A card", + "content": [ + { + "type": "box-whisker-chart", + "provider": "pmp", + "diagnostic": "annual-cycle", + "title": "Annual Cycle Chart", + "grouping_config": { + "group_by": "model", + "hue": "experiment", + }, + } + ], + } + ], +} + + +class TestLoadAllCollections: + def test_valid_collection_loading(self, tmp_path: Path): + """Valid YAML loads into AFTCollectionDetail model correctly.""" + _write_collection(tmp_path, "1.1.yaml", _MINIMAL_COLLECTION) + + with patch("ref_backend.core.collections.get_collections_dir", return_value=tmp_path): + result = load_all_collections() + + assert "1.1" in result + col = result["1.1"] + assert isinstance(col, AFTCollectionDetail) + assert col.id == "1.1" + assert col.name == "Test Collection" + assert col.theme == "Ocean" + assert col.endorser == "WCRP" + assert col.version_control == "1.0" + assert col.reference_dataset == "ERA5" + assert col.provider_link == "https://example.com" + assert col.content is not None + assert col.content.description == "Full description" + assert col.content.short_description == "Short" + assert len(col.diagnostics) == 1 + assert col.diagnostics[0].provider_slug == "pmp" + assert col.diagnostics[0].diagnostic_slug == "annual-cycle" + assert len(col.explorer_cards) == 1 + card = col.explorer_cards[0] + assert isinstance(card, AFTCollectionCard) + assert card.title == "Card One" + assert len(card.content) == 1 + content = card.content[0] + assert isinstance(content, AFTCollectionCardContent) + assert content.type == "box-whisker-chart" + assert content.grouping_config is not None + assert isinstance(content.grouping_config, AFTCollectionGroupingConfig) + assert content.grouping_config.group_by == "model" + assert content.grouping_config.hue == "experiment" + + def test_all_optional_card_content_fields(self, tmp_path: Path): + """All optional fields on AFTCollectionCardContent deserialize correctly.""" + data = { + "id": "2.1", + "name": "Full Fields Collection", + "diagnostics": [], + "explorer_cards": [ + { + "title": "Detailed Card", + "content": [ + { + "type": "series-chart", + "provider": "ilamb", + "diagnostic": "biomass", + "title": "Biomass Series", + "description": "Desc", + "interpretation": "Some interpretation", + "span": 2, + "metric_units": "kg/m2", + "clip_min": -10.5, + "clip_max": 10.5, + "y_min": -5.0, + "y_max": 5.0, + "show_zero_line": True, + "symmetrical_axes": False, + "reference_stddev": 1.5, + "label_template": "{model} ({experiment})", + "other_filters": { + "isolate_ids": "CMIP6.ssp585", + "exclude_ids": "CMIP6.piControl", + }, + "grouping_config": { + "group_by": "source_id", + "hue": "variant_label", + "style": "experiment_id", + }, + } + ], + } + ], + } + _write_collection(tmp_path, "2.1.yaml", data) + + with patch("ref_backend.core.collections.get_collections_dir", return_value=tmp_path): + result = load_all_collections() + + col = result["2.1"] + content = col.explorer_cards[0].content[0] + assert content.label_template == "{model} ({experiment})" + assert content.grouping_config is not None + assert content.grouping_config.style == "experiment_id" + assert content.other_filters == { + "isolate_ids": "CMIP6.ssp585", + "exclude_ids": "CMIP6.piControl", + } + assert content.y_min == -5.0 + assert content.y_max == 5.0 + assert content.clip_min == -10.5 + assert content.clip_max == 10.5 + assert content.span == 2 + assert content.show_zero_line is True + assert content.symmetrical_axes is False + assert content.reference_stddev == 1.5 + + def test_missing_collections_directory_returns_empty(self, tmp_path: Path): + """Missing collections directory returns empty dict without crashing.""" + nonexistent = tmp_path / "no_such_dir" + with patch("ref_backend.core.collections.get_collections_dir", return_value=nonexistent): + result = load_all_collections() + + assert result == {} + + def test_invalid_yaml_file_is_skipped_with_warning(self, tmp_path: Path, caplog): + """A file with invalid YAML content is skipped and a warning is logged.""" + bad_file = tmp_path / "bad.yaml" + bad_file.write_text("key: [unclosed bracket") + + good_data = {**_MINIMAL_COLLECTION, "id": "1.1", "name": "Good"} + _write_collection(tmp_path, "1.1.yaml", good_data) + + with patch("ref_backend.core.collections.get_collections_dir", return_value=tmp_path): + with caplog.at_level(logging.WARNING, logger="ref_backend.core.collections"): + result = load_all_collections() + + assert "1.1" in result + assert "bad" not in str(result) + assert any("bad.yaml" in msg for msg in caplog.messages) + + def test_invalid_content_type_rejected(self, tmp_path: Path, caplog): + """Invalid content type is rejected by Pydantic and file is skipped.""" + data = { + "id": "3.1", + "name": "Bad Type Collection", + "diagnostics": [], + "explorer_cards": [ + { + "title": "Bad Card", + "content": [ + { + "type": "not-a-valid-chart-type", + "provider": "pmp", + "diagnostic": "test", + "title": "Chart", + } + ], + } + ], + } + _write_collection(tmp_path, "3.1.yaml", data) + + with patch("ref_backend.core.collections.get_collections_dir", return_value=tmp_path): + with caplog.at_level(logging.WARNING, logger="ref_backend.core.collections"): + result = load_all_collections() + + assert "3.1" not in result + assert any("3.1.yaml" in msg for msg in caplog.messages) + + def test_json_serialization_uses_snake_case(self, tmp_path: Path): + """JSON serialization must use snake_case keys, not camelCase.""" + data = { + "id": "4.1", + "name": "Snake Case Test", + "diagnostics": [], + "explorer_cards": [ + { + "title": "Chart Card", + "content": [ + { + "type": "series-chart", + "provider": "pmp", + "diagnostic": "mean-climate", + "title": "Mean Climate", + "y_min": -1.0, + "y_max": 1.0, + "show_zero_line": True, + "grouping_config": { + "group_by": "model", + "hue": "experiment", + }, + } + ], + } + ], + } + _write_collection(tmp_path, "4.1.yaml", data) + + with patch("ref_backend.core.collections.get_collections_dir", return_value=tmp_path): + result = load_all_collections() + + col = result["4.1"] + serialized = json.loads(col.model_dump_json()) + + # Assert snake_case keys + card_content = serialized["explorer_cards"][0]["content"][0] + assert "y_min" in card_content + assert "y_max" in card_content + assert "show_zero_line" in card_content + assert "grouping_config" in card_content + assert "group_by" in card_content["grouping_config"] + + # Assert no camelCase keys + camel_case_keys = ["yMin", "yMax", "showZeroLine", "groupingConfig", "groupBy"] + for key in camel_case_keys: + assert key not in card_content, f"Found camelCase key: {key}" + assert key not in card_content.get("grouping_config", {}) + + def test_summary_card_count(self, tmp_path: Path): + """AFTCollectionSummary.card_count matches actual number of explorer_cards.""" + data = {**_MINIMAL_COLLECTION, "id": "5.1"} + # has 1 card in _MINIMAL_COLLECTION + _write_collection(tmp_path, "5.1.yaml", data) + + with patch("ref_backend.core.collections.get_collections_dir", return_value=tmp_path): + summaries = get_collection_summaries() + + assert len(summaries) == 1 + s = summaries[0] + assert isinstance(s, AFTCollectionSummary) + assert s.card_count == 1 + + def test_collection_with_empty_explorer_cards(self, tmp_path: Path): + """Collection with explorer_cards: [] loads without errors.""" + data = { + "id": "6.1", + "name": "Empty Cards", + "diagnostics": [], + "explorer_cards": [], + } + _write_collection(tmp_path, "6.1.yaml", data) + + with patch("ref_backend.core.collections.get_collections_dir", return_value=tmp_path): + result = load_all_collections() + + assert "6.1" in result + assert result["6.1"].explorer_cards == [] + + def test_placeholder_card_with_empty_content(self, tmp_path: Path): + """Card with content: [] and placeholder: true loads (ENSO case).""" + data = { + "id": "7.1", + "name": "ENSO Collection", + "diagnostics": [], + "explorer_cards": [ + { + "title": "ENSO Placeholder", + "placeholder": True, + "content": [], + } + ], + } + _write_collection(tmp_path, "7.1.yaml", data) + + with patch("ref_backend.core.collections.get_collections_dir", return_value=tmp_path): + result = load_all_collections() + + col = result["7.1"] + assert len(col.explorer_cards) == 1 + card = col.explorer_cards[0] + assert card.placeholder is True + assert card.content == [] + + def test_duplicate_collection_ids_logs_warning(self, tmp_path: Path, caplog): + """Duplicate IDs across YAML files results in a warning for the second file.""" + data1 = {**_MINIMAL_COLLECTION, "id": "dup-1", "name": "First"} + data2 = {**_MINIMAL_COLLECTION, "id": "dup-1", "name": "Second"} + _write_collection(tmp_path, "a.yaml", data1) + _write_collection(tmp_path, "b.yaml", data2) + + with patch("ref_backend.core.collections.get_collections_dir", return_value=tmp_path): + with caplog.at_level(logging.WARNING, logger="ref_backend.core.collections"): + result = load_all_collections() + + # One file was skipped, so only one collection with dup-1 + assert len([c for c in result.values() if c.id == "dup-1"]) == 1 + + def test_collections_sorted_by_id(self, tmp_path: Path): + """Collections are returned sorted by ID for stable ordering.""" + for cid in ["3.1", "1.1", "2.1"]: + data = {**_MINIMAL_COLLECTION, "id": cid, "name": f"Col {cid}"} + _write_collection(tmp_path, f"{cid}.yaml", data) + + with patch("ref_backend.core.collections.get_collections_dir", return_value=tmp_path): + summaries = get_collection_summaries() + + ids = [s.id for s in summaries] + assert ids == sorted(ids) + + +class TestLoadThemeMapping: + def test_theme_mapping_loads_and_resolves_collections(self, tmp_path: Path): + """Theme mapping loads correctly and resolves collection references.""" + _write_collection(tmp_path, "1.1.yaml", _MINIMAL_COLLECTION) + col2 = {**_MINIMAL_COLLECTION, "id": "1.2", "name": "Second Collection"} + _write_collection(tmp_path, "1.2.yaml", col2) + + themes = [ + { + "slug": "ocean", + "title": "Ocean Theme", + "description": "Ocean diagnostics", + "collections": ["1.1", "1.2"], + } + ] + _write_themes(tmp_path, themes) + + with ( + patch("ref_backend.core.collections.get_collections_dir", return_value=tmp_path), + patch("ref_backend.core.collections.get_themes_path", return_value=tmp_path / "themes.yaml"), + ): + mapping = load_theme_mapping() + + assert "ocean" in mapping + theme = mapping["ocean"] + assert isinstance(theme, ThemeDetail) + assert theme.slug == "ocean" + assert theme.title == "Ocean Theme" + assert theme.description == "Ocean diagnostics" + assert len(theme.collections) == 2 + assert theme.collections[0].id == "1.1" + assert theme.collections[1].id == "1.2" + + def test_theme_aggregates_explorer_cards_in_order(self, tmp_path: Path): + """Theme.explorer_cards aggregates all cards from collections in order.""" + col1 = { + **_MINIMAL_COLLECTION, + "id": "c1", + "explorer_cards": [ + {"title": "Card A", "content": []}, + {"title": "Card B", "content": []}, + ], + } + col2 = { + **_MINIMAL_COLLECTION, + "id": "c2", + "explorer_cards": [ + {"title": "Card C", "content": []}, + ], + } + _write_collection(tmp_path, "c1.yaml", col1) + _write_collection(tmp_path, "c2.yaml", col2) + + themes = [{"slug": "climate", "title": "Climate", "collections": ["c1", "c2"]}] + _write_themes(tmp_path, themes) + + with ( + patch("ref_backend.core.collections.get_collections_dir", return_value=tmp_path), + patch("ref_backend.core.collections.get_themes_path", return_value=tmp_path / "themes.yaml"), + ): + theme = get_theme_by_slug("climate") + + assert theme is not None + assert len(theme.explorer_cards) == 3 + assert theme.explorer_cards[0].title == "Card A" + assert theme.explorer_cards[1].title == "Card B" + assert theme.explorer_cards[2].title == "Card C" + + def test_unknown_collection_id_in_themes_logs_warning(self, tmp_path: Path, caplog): + """Unknown collection ID in themes.yaml is skipped with a warning.""" + _write_collection(tmp_path, "1.1.yaml", _MINIMAL_COLLECTION) + themes = [ + { + "slug": "ocean", + "title": "Ocean", + "collections": ["1.1", "nonexistent-id"], + } + ] + _write_themes(tmp_path, themes) + + with ( + patch("ref_backend.core.collections.get_collections_dir", return_value=tmp_path), + patch("ref_backend.core.collections.get_themes_path", return_value=tmp_path / "themes.yaml"), + ): + with caplog.at_level(logging.WARNING, logger="ref_backend.core.collections"): + theme = get_theme_by_slug("ocean") + + assert theme is not None + assert len(theme.collections) == 1 # nonexistent was skipped + assert any("nonexistent-id" in msg for msg in caplog.messages) + + def test_collection_shared_across_themes(self, tmp_path: Path): + """A collection referenced in two themes appears in both theme details.""" + _write_collection(tmp_path, "shared.yaml", {**_MINIMAL_COLLECTION, "id": "shared"}) + + themes = [ + {"slug": "theme-a", "title": "Theme A", "collections": ["shared"]}, + {"slug": "theme-b", "title": "Theme B", "collections": ["shared"]}, + ] + _write_themes(tmp_path, themes) + + with ( + patch("ref_backend.core.collections.get_collections_dir", return_value=tmp_path), + patch("ref_backend.core.collections.get_themes_path", return_value=tmp_path / "themes.yaml"), + ): + theme_a = get_theme_by_slug("theme-a") + theme_b = get_theme_by_slug("theme-b") + + assert theme_a is not None + assert theme_b is not None + assert any(c.id == "shared" for c in theme_a.collections) + assert any(c.id == "shared" for c in theme_b.collections) + + def test_get_theme_summaries(self, tmp_path: Path): + """get_theme_summaries returns ThemeSummary with correct counts.""" + col1 = {**_MINIMAL_COLLECTION, "id": "t1"} # 1 card + col2 = { + **_MINIMAL_COLLECTION, + "id": "t2", + "explorer_cards": [ + {"title": "C1", "content": []}, + {"title": "C2", "content": []}, + ], + } + _write_collection(tmp_path, "t1.yaml", col1) + _write_collection(tmp_path, "t2.yaml", col2) + + themes = [{"slug": "big-theme", "title": "Big Theme", "collections": ["t1", "t2"]}] + _write_themes(tmp_path, themes) + + with ( + patch("ref_backend.core.collections.get_collections_dir", return_value=tmp_path), + patch("ref_backend.core.collections.get_themes_path", return_value=tmp_path / "themes.yaml"), + ): + summaries = get_theme_summaries() + + assert len(summaries) == 1 + s = summaries[0] + assert s.slug == "big-theme" + assert s.collection_count == 2 + assert s.card_count == 3 # 1 from t1 + 2 from t2 + + def test_get_collection_by_id_returns_none_for_missing(self, tmp_path: Path): + """get_collection_by_id returns None for nonexistent ID.""" + with patch("ref_backend.core.collections.get_collections_dir", return_value=tmp_path): + result = get_collection_by_id("does-not-exist") + assert result is None + + def test_missing_themes_file_returns_empty(self, tmp_path: Path): + """Missing themes.yaml returns empty dict without crashing.""" + nonexistent = tmp_path / "themes.yaml" + with ( + patch("ref_backend.core.collections.get_collections_dir", return_value=tmp_path), + patch("ref_backend.core.collections.get_themes_path", return_value=nonexistent), + ): + result = load_theme_mapping() + + assert result == {} diff --git a/frontend/biome.json b/frontend/biome.json index 085e5c7..e9e9b8d 100644 --- a/frontend/biome.json +++ b/frontend/biome.json @@ -12,14 +12,21 @@ "!**/dist", "!**/src/client", "!**/*.gen.ts", - "!**/openapi.json" + "!**/openapi.json", + "!**/.omc" ] }, "formatter": { "enabled": true, "indentStyle": "space" }, - "assist": { "actions": { "source": { "organizeImports": "on" } } }, + "assist": { + "actions": { + "source": { + "organizeImports": "on" + } + } + }, "linter": { "enabled": true, "rules": { diff --git a/frontend/src/client/@tanstack/react-query.gen.ts b/frontend/src/client/@tanstack/react-query.gen.ts index 1601de3..2b69194 100644 --- a/frontend/src/client/@tanstack/react-query.gen.ts +++ b/frontend/src/client/@tanstack/react-query.gen.ts @@ -1,8 +1,8 @@ // This file is auto-generated by @hey-api/openapi-ts -import { type Options, cmip7AssessmentFastTrackAftListAftDiagnostics, cmip7AssessmentFastTrackAftGetAftDiagnostic, datasetsList, datasetsGet, datasetsExecutions, diagnosticsList, diagnosticsFacets, diagnosticsGet, diagnosticsListExecutionGroups, diagnosticsListExecutions, diagnosticsListMetricValues, executionsGetExecutionStatistics, executionsListRecentExecutionGroups, executionsGet, executionsExecution, executionsExecutionDatasets, executionsExecutionLogs, executionsMetricBundle, executionsListMetricValues, executionsExecutionArchive, resultsGetResult, utilsHealthCheck } from '../sdk.gen'; +import { type Options, cmip7AssessmentFastTrackAftListAftDiagnostics, cmip7AssessmentFastTrackAftGetAftDiagnostic, datasetsList, datasetsGet, datasetsExecutions, diagnosticsList, diagnosticsFacets, diagnosticsGet, diagnosticsListExecutionGroups, diagnosticsListExecutions, diagnosticsListMetricValues, executionsGetExecutionStatistics, executionsListRecentExecutionGroups, executionsGet, executionsExecution, executionsExecutionDatasets, executionsExecutionLogs, executionsMetricBundle, executionsListMetricValues, executionsExecutionArchive, explorerListCollections, explorerGetCollection, explorerListThemes, explorerGetTheme, resultsGetResult, utilsHealthCheck } from '../sdk.gen'; import { queryOptions, infiniteQueryOptions, type InfiniteData } from '@tanstack/react-query'; -import type { Cmip7AssessmentFastTrackAftListAftDiagnosticsData, Cmip7AssessmentFastTrackAftGetAftDiagnosticData, DatasetsListData, DatasetsListError, DatasetsListResponse, DatasetsGetData, DatasetsExecutionsData, DatasetsExecutionsError, DatasetsExecutionsResponse, DiagnosticsListData, DiagnosticsFacetsData, DiagnosticsGetData, DiagnosticsListExecutionGroupsData, DiagnosticsListExecutionsData, DiagnosticsListMetricValuesData, ExecutionsGetExecutionStatisticsData, ExecutionsListRecentExecutionGroupsData, ExecutionsListRecentExecutionGroupsError, ExecutionsListRecentExecutionGroupsResponse, ExecutionsGetData, ExecutionsExecutionData, ExecutionsExecutionDatasetsData, ExecutionsExecutionLogsData, ExecutionsMetricBundleData, ExecutionsListMetricValuesData, ExecutionsExecutionArchiveData, ResultsGetResultData, UtilsHealthCheckData } from '../types.gen'; +import type { Cmip7AssessmentFastTrackAftListAftDiagnosticsData, Cmip7AssessmentFastTrackAftGetAftDiagnosticData, DatasetsListData, DatasetsListError, DatasetsListResponse, DatasetsGetData, DatasetsExecutionsData, DatasetsExecutionsError, DatasetsExecutionsResponse, DiagnosticsListData, DiagnosticsFacetsData, DiagnosticsGetData, DiagnosticsListExecutionGroupsData, DiagnosticsListExecutionsData, DiagnosticsListMetricValuesData, ExecutionsGetExecutionStatisticsData, ExecutionsListRecentExecutionGroupsData, ExecutionsListRecentExecutionGroupsError, ExecutionsListRecentExecutionGroupsResponse, ExecutionsGetData, ExecutionsExecutionData, ExecutionsExecutionDatasetsData, ExecutionsExecutionLogsData, ExecutionsMetricBundleData, ExecutionsListMetricValuesData, ExecutionsExecutionArchiveData, ExplorerListCollectionsData, ExplorerGetCollectionData, ExplorerListThemesData, ExplorerGetThemeData, ResultsGetResultData, UtilsHealthCheckData } from '../types.gen'; import { client as _heyApiClient } from '../client.gen'; export type QueryKey = [ @@ -483,6 +483,74 @@ export const executionsExecutionArchiveOptions = (options: Options) => createQueryKey('explorerListCollections', options); + +export const explorerListCollectionsOptions = (options?: Options) => { + return queryOptions({ + queryFn: async ({ queryKey, signal }) => { + const { data } = await explorerListCollections({ + ...options, + ...queryKey[0], + signal, + throwOnError: true + }); + return data; + }, + queryKey: explorerListCollectionsQueryKey(options) + }); +}; + +export const explorerGetCollectionQueryKey = (options: Options) => createQueryKey('explorerGetCollection', options); + +export const explorerGetCollectionOptions = (options: Options) => { + return queryOptions({ + queryFn: async ({ queryKey, signal }) => { + const { data } = await explorerGetCollection({ + ...options, + ...queryKey[0], + signal, + throwOnError: true + }); + return data; + }, + queryKey: explorerGetCollectionQueryKey(options) + }); +}; + +export const explorerListThemesQueryKey = (options?: Options) => createQueryKey('explorerListThemes', options); + +export const explorerListThemesOptions = (options?: Options) => { + return queryOptions({ + queryFn: async ({ queryKey, signal }) => { + const { data } = await explorerListThemes({ + ...options, + ...queryKey[0], + signal, + throwOnError: true + }); + return data; + }, + queryKey: explorerListThemesQueryKey(options) + }); +}; + +export const explorerGetThemeQueryKey = (options: Options) => createQueryKey('explorerGetTheme', options); + +export const explorerGetThemeOptions = (options: Options) => { + return queryOptions({ + queryFn: async ({ queryKey, signal }) => { + const { data } = await explorerGetTheme({ + ...options, + ...queryKey[0], + signal, + throwOnError: true + }); + return data; + }, + queryKey: explorerGetThemeQueryKey(options) + }); +}; + export const resultsGetResultQueryKey = (options: Options) => createQueryKey('resultsGetResult', options); export const resultsGetResultOptions = (options: Options) => { diff --git a/frontend/src/client/schemas.gen.ts b/frontend/src/client/schemas.gen.ts index 14178c4..1c19d2b 100644 --- a/frontend/src/client/schemas.gen.ts +++ b/frontend/src/client/schemas.gen.ts @@ -1,5 +1,448 @@ // This file is auto-generated by @hey-api/openapi-ts +export const AFTCollectionCardSchema = { + properties: { + title: { + type: 'string', + title: 'Title' + }, + description: { + anyOf: [ + { + type: 'string' + }, + { + type: 'null' + } + ], + title: 'Description' + }, + placeholder: { + anyOf: [ + { + type: 'boolean' + }, + { + type: 'null' + } + ], + title: 'Placeholder' + }, + content: { + items: { + '$ref': '#/components/schemas/AFTCollectionCardContent' + }, + type: 'array', + title: 'Content' + } + }, + type: 'object', + required: ['title', 'content'], + title: 'AFTCollectionCard' +} as const; + +export const AFTCollectionCardContentSchema = { + properties: { + type: { + type: 'string', + enum: ['box-whisker-chart', 'figure-gallery', 'series-chart', 'taylor-diagram'], + title: 'Type' + }, + provider: { + type: 'string', + title: 'Provider' + }, + diagnostic: { + type: 'string', + title: 'Diagnostic' + }, + title: { + type: 'string', + title: 'Title' + }, + description: { + anyOf: [ + { + type: 'string' + }, + { + type: 'null' + } + ], + title: 'Description' + }, + interpretation: { + anyOf: [ + { + type: 'string' + }, + { + type: 'null' + } + ], + title: 'Interpretation' + }, + span: { + anyOf: [ + { + type: 'integer', + enum: [1, 2] + }, + { + type: 'null' + } + ], + title: 'Span' + }, + placeholder: { + anyOf: [ + { + type: 'boolean' + }, + { + type: 'null' + } + ], + title: 'Placeholder' + }, + metric_units: { + anyOf: [ + { + type: 'string' + }, + { + type: 'null' + } + ], + title: 'Metric Units' + }, + clip_min: { + anyOf: [ + { + type: 'number' + }, + { + type: 'null' + } + ], + title: 'Clip Min' + }, + clip_max: { + anyOf: [ + { + type: 'number' + }, + { + type: 'null' + } + ], + title: 'Clip Max' + }, + y_min: { + anyOf: [ + { + type: 'number' + }, + { + type: 'null' + } + ], + title: 'Y Min' + }, + y_max: { + anyOf: [ + { + type: 'number' + }, + { + type: 'null' + } + ], + title: 'Y Max' + }, + show_zero_line: { + anyOf: [ + { + type: 'boolean' + }, + { + type: 'null' + } + ], + title: 'Show Zero Line' + }, + symmetrical_axes: { + anyOf: [ + { + type: 'boolean' + }, + { + type: 'null' + } + ], + title: 'Symmetrical Axes' + }, + reference_stddev: { + anyOf: [ + { + type: 'number' + }, + { + type: 'null' + } + ], + title: 'Reference Stddev' + }, + label_template: { + anyOf: [ + { + type: 'string' + }, + { + type: 'null' + } + ], + title: 'Label Template' + }, + other_filters: { + anyOf: [ + { + additionalProperties: { + type: 'string' + }, + type: 'object' + }, + { + type: 'null' + } + ], + title: 'Other Filters' + }, + grouping_config: { + anyOf: [ + { + '$ref': '#/components/schemas/AFTCollectionGroupingConfig' + }, + { + type: 'null' + } + ] + } + }, + type: 'object', + required: ['type', 'provider', 'diagnostic', 'title'], + title: 'AFTCollectionCardContent' +} as const; + +export const AFTCollectionContentSchema = { + properties: { + description: { + anyOf: [ + { + type: 'string' + }, + { + type: 'null' + } + ], + title: 'Description' + }, + short_description: { + anyOf: [ + { + type: 'string' + }, + { + type: 'null' + } + ], + title: 'Short Description' + } + }, + type: 'object', + title: 'AFTCollectionContent' +} as const; + +export const AFTCollectionDetailSchema = { + properties: { + id: { + type: 'string', + title: 'Id' + }, + name: { + type: 'string', + title: 'Name' + }, + theme: { + anyOf: [ + { + type: 'string' + }, + { + type: 'null' + } + ], + title: 'Theme' + }, + endorser: { + anyOf: [ + { + type: 'string' + }, + { + type: 'null' + } + ], + title: 'Endorser' + }, + version_control: { + anyOf: [ + { + type: 'string' + }, + { + type: 'null' + } + ], + title: 'Version Control' + }, + reference_dataset: { + anyOf: [ + { + type: 'string' + }, + { + type: 'null' + } + ], + title: 'Reference Dataset' + }, + provider_link: { + anyOf: [ + { + type: 'string' + }, + { + type: 'null' + } + ], + title: 'Provider Link' + }, + content: { + anyOf: [ + { + '$ref': '#/components/schemas/AFTCollectionContent' + }, + { + type: 'null' + } + ] + }, + diagnostics: { + items: { + '$ref': '#/components/schemas/AFTCollectionDiagnosticLink' + }, + type: 'array', + title: 'Diagnostics' + }, + explorer_cards: { + items: { + '$ref': '#/components/schemas/AFTCollectionCard' + }, + type: 'array', + title: 'Explorer Cards' + } + }, + type: 'object', + required: ['id', 'name', 'diagnostics', 'explorer_cards'], + title: 'AFTCollectionDetail' +} as const; + +export const AFTCollectionDiagnosticLinkSchema = { + properties: { + provider_slug: { + type: 'string', + title: 'Provider Slug' + }, + diagnostic_slug: { + type: 'string', + title: 'Diagnostic Slug' + } + }, + type: 'object', + required: ['provider_slug', 'diagnostic_slug'], + title: 'AFTCollectionDiagnosticLink' +} as const; + +export const AFTCollectionGroupingConfigSchema = { + properties: { + group_by: { + type: 'string', + title: 'Group By' + }, + hue: { + type: 'string', + title: 'Hue' + }, + style: { + anyOf: [ + { + type: 'string' + }, + { + type: 'null' + } + ], + title: 'Style' + } + }, + type: 'object', + required: ['group_by', 'hue'], + title: 'AFTCollectionGroupingConfig' +} as const; + +export const AFTCollectionSummarySchema = { + properties: { + id: { + type: 'string', + title: 'Id' + }, + name: { + type: 'string', + title: 'Name' + }, + theme: { + anyOf: [ + { + type: 'string' + }, + { + type: 'null' + } + ], + title: 'Theme' + }, + endorser: { + anyOf: [ + { + type: 'string' + }, + { + type: 'null' + } + ], + title: 'Endorser' + }, + card_count: { + type: 'integer', + title: 'Card Count' + } + }, + type: 'object', + required: ['id', 'name', 'card_count'], + title: 'AFTCollectionSummary' +} as const; + export const AFTDiagnosticDetailSchema = { properties: { id: { @@ -598,13 +1041,10 @@ export const ExecutionSchema = { }, type: 'array', title: 'Outputs' - }, - execution_group: { - '$ref': '#/components/schemas/ExecutionGroupSummary' } }, type: 'object', - required: ['id', 'dataset_hash', 'dataset_count', 'successful', 'retracted', 'created_at', 'updated_at', 'outputs', 'execution_group'], + required: ['id', 'dataset_hash', 'dataset_count', 'successful', 'retracted', 'created_at', 'updated_at', 'outputs'], title: 'Execution' } as const; @@ -624,7 +1064,7 @@ export const ExecutionGroupSchema = { }, executions: { items: { - '$ref': '#/components/schemas/ExecutionSummary' + '$ref': '#/components/schemas/Execution' }, type: 'array', title: 'Executions' @@ -632,7 +1072,7 @@ export const ExecutionGroupSchema = { latest_execution: { anyOf: [ { - '$ref': '#/components/schemas/ExecutionSummary' + '$ref': '#/components/schemas/Execution' }, { type: 'null' @@ -678,59 +1118,6 @@ export const ExecutionGroupSchema = { title: 'ExecutionGroup' } as const; -export const ExecutionGroupSummarySchema = { - properties: { - id: { - type: 'integer', - title: 'Id' - }, - key: { - type: 'string', - title: 'Key' - }, - dirty: { - type: 'boolean', - title: 'Dirty' - }, - selectors: { - additionalProperties: { - items: { - prefixItems: [ - { - type: 'string' - }, - { - type: 'string' - } - ], - type: 'array', - maxItems: 2, - minItems: 2 - }, - type: 'array' - }, - type: 'object', - title: 'Selectors' - }, - diagnostic: { - '$ref': '#/components/schemas/DiagnosticSummary' - }, - created_at: { - type: 'string', - format: 'date-time', - title: 'Created At' - }, - updated_at: { - type: 'string', - format: 'date-time', - title: 'Updated At' - } - }, - type: 'object', - required: ['id', 'key', 'dirty', 'selectors', 'diagnostic', 'created_at', 'updated_at'], - title: 'ExecutionGroupSummary' -} as const; - export const ExecutionOutputSchema = { properties: { id: { @@ -823,51 +1210,6 @@ export const ExecutionStatsSchema = { description: 'Statistics for execution groups and their success rates.' } as const; -export const ExecutionSummarySchema = { - properties: { - id: { - type: 'integer', - title: 'Id' - }, - dataset_hash: { - type: 'string', - title: 'Dataset Hash' - }, - dataset_count: { - type: 'integer', - title: 'Dataset Count' - }, - successful: { - type: 'boolean', - title: 'Successful' - }, - retracted: { - type: 'boolean', - title: 'Retracted' - }, - created_at: { - type: 'string', - format: 'date-time', - title: 'Created At' - }, - updated_at: { - type: 'string', - format: 'date-time', - title: 'Updated At' - }, - outputs: { - items: { - '$ref': '#/components/schemas/ExecutionOutput' - }, - type: 'array', - title: 'Outputs' - } - }, - type: 'object', - required: ['id', 'dataset_hash', 'dataset_count', 'successful', 'retracted', 'created_at', 'updated_at', 'outputs'], - title: 'ExecutionSummary' -} as const; - export const FacetSchema = { properties: { key: { @@ -1300,6 +1642,82 @@ export const SeriesValueSchema = { This includes the dimensions, values array, index array, and index name` } as const; +export const ThemeDetailSchema = { + properties: { + slug: { + type: 'string', + title: 'Slug' + }, + title: { + type: 'string', + title: 'Title' + }, + description: { + anyOf: [ + { + type: 'string' + }, + { + type: 'null' + } + ], + title: 'Description' + }, + collections: { + items: { + '$ref': '#/components/schemas/AFTCollectionDetail' + }, + type: 'array', + title: 'Collections' + }, + explorer_cards: { + items: { + '$ref': '#/components/schemas/AFTCollectionCard' + }, + type: 'array', + title: 'Explorer Cards' + } + }, + type: 'object', + required: ['slug', 'title', 'collections', 'explorer_cards'], + title: 'ThemeDetail' +} as const; + +export const ThemeSummarySchema = { + properties: { + slug: { + type: 'string', + title: 'Slug' + }, + title: { + type: 'string', + title: 'Title' + }, + description: { + anyOf: [ + { + type: 'string' + }, + { + type: 'null' + } + ], + title: 'Description' + }, + collection_count: { + type: 'integer', + title: 'Collection Count' + }, + card_count: { + type: 'integer', + title: 'Card Count' + } + }, + type: 'object', + required: ['slug', 'title', 'collection_count', 'card_count'], + title: 'ThemeSummary' +} as const; + export const ValidationErrorSchema = { properties: { loc: { @@ -1323,6 +1741,13 @@ export const ValidationErrorSchema = { type: { type: 'string', title: 'Error Type' + }, + input: { + title: 'Input' + }, + ctx: { + type: 'object', + title: 'Context' } }, type: 'object', diff --git a/frontend/src/client/sdk.gen.ts b/frontend/src/client/sdk.gen.ts index 53f1358..f5d11b6 100644 --- a/frontend/src/client/sdk.gen.ts +++ b/frontend/src/client/sdk.gen.ts @@ -1,7 +1,7 @@ // This file is auto-generated by @hey-api/openapi-ts import type { Options as ClientOptions, TDataShape, Client } from '@hey-api/client-fetch'; -import type { Cmip7AssessmentFastTrackAftListAftDiagnosticsData, Cmip7AssessmentFastTrackAftListAftDiagnosticsResponse, Cmip7AssessmentFastTrackAftGetAftDiagnosticData, Cmip7AssessmentFastTrackAftGetAftDiagnosticResponse, Cmip7AssessmentFastTrackAftGetAftDiagnosticError, DatasetsListData, DatasetsListResponse, DatasetsListError, DatasetsGetData, DatasetsGetResponse, DatasetsGetError, DatasetsExecutionsData, DatasetsExecutionsResponse, DatasetsExecutionsError, DiagnosticsListData, DiagnosticsListResponse, DiagnosticsFacetsData, DiagnosticsFacetsResponse, DiagnosticsGetData, DiagnosticsGetResponse, DiagnosticsGetError, DiagnosticsListExecutionGroupsData, DiagnosticsListExecutionGroupsResponse, DiagnosticsListExecutionGroupsError, DiagnosticsListExecutionsData, DiagnosticsListExecutionsResponse, DiagnosticsListExecutionsError, DiagnosticsListMetricValuesData, DiagnosticsListMetricValuesResponse, DiagnosticsListMetricValuesError, ExecutionsGetExecutionStatisticsData, ExecutionsGetExecutionStatisticsResponse, ExecutionsListRecentExecutionGroupsData, ExecutionsListRecentExecutionGroupsResponse, ExecutionsListRecentExecutionGroupsError, ExecutionsGetData, ExecutionsGetResponse, ExecutionsGetError, ExecutionsExecutionData, ExecutionsExecutionResponse, ExecutionsExecutionError, ExecutionsExecutionDatasetsData, ExecutionsExecutionDatasetsResponse, ExecutionsExecutionDatasetsError, ExecutionsExecutionLogsData, ExecutionsExecutionLogsError, ExecutionsMetricBundleData, ExecutionsMetricBundleResponse, ExecutionsMetricBundleError, ExecutionsListMetricValuesData, ExecutionsListMetricValuesResponse, ExecutionsListMetricValuesError, ExecutionsExecutionArchiveData, ExecutionsExecutionArchiveError, ResultsGetResultData, ResultsGetResultError, UtilsHealthCheckData, UtilsHealthCheckResponse } from './types.gen'; +import type { Cmip7AssessmentFastTrackAftListAftDiagnosticsData, Cmip7AssessmentFastTrackAftListAftDiagnosticsResponse, Cmip7AssessmentFastTrackAftGetAftDiagnosticData, Cmip7AssessmentFastTrackAftGetAftDiagnosticResponse, Cmip7AssessmentFastTrackAftGetAftDiagnosticError, DatasetsListData, DatasetsListResponse, DatasetsListError, DatasetsGetData, DatasetsGetResponse, DatasetsGetError, DatasetsExecutionsData, DatasetsExecutionsResponse, DatasetsExecutionsError, DiagnosticsListData, DiagnosticsListResponse, DiagnosticsFacetsData, DiagnosticsFacetsResponse, DiagnosticsGetData, DiagnosticsGetResponse, DiagnosticsGetError, DiagnosticsListExecutionGroupsData, DiagnosticsListExecutionGroupsResponse, DiagnosticsListExecutionGroupsError, DiagnosticsListExecutionsData, DiagnosticsListExecutionsResponse, DiagnosticsListExecutionsError, DiagnosticsListMetricValuesData, DiagnosticsListMetricValuesResponse, DiagnosticsListMetricValuesError, ExecutionsGetExecutionStatisticsData, ExecutionsGetExecutionStatisticsResponse, ExecutionsListRecentExecutionGroupsData, ExecutionsListRecentExecutionGroupsResponse, ExecutionsListRecentExecutionGroupsError, ExecutionsGetData, ExecutionsGetResponse, ExecutionsGetError, ExecutionsExecutionData, ExecutionsExecutionResponse, ExecutionsExecutionError, ExecutionsExecutionDatasetsData, ExecutionsExecutionDatasetsResponse, ExecutionsExecutionDatasetsError, ExecutionsExecutionLogsData, ExecutionsExecutionLogsError, ExecutionsMetricBundleData, ExecutionsMetricBundleResponse, ExecutionsMetricBundleError, ExecutionsListMetricValuesData, ExecutionsListMetricValuesResponse, ExecutionsListMetricValuesError, ExecutionsExecutionArchiveData, ExecutionsExecutionArchiveError, ExplorerListCollectionsData, ExplorerListCollectionsResponse, ExplorerGetCollectionData, ExplorerGetCollectionResponse, ExplorerGetCollectionError, ExplorerListThemesData, ExplorerListThemesResponse, ExplorerGetThemeData, ExplorerGetThemeResponse, ExplorerGetThemeError, ResultsGetResultData, ResultsGetResultError, UtilsHealthCheckData, UtilsHealthCheckResponse } from './types.gen'; import { client as _heyApiClient } from './client.gen'; export type Options = ClientOptions & { @@ -261,6 +261,46 @@ export const executionsExecutionArchive = }); }; +/** + * List Collections + */ +export const explorerListCollections = (options?: Options) => { + return (options?.client ?? _heyApiClient).get({ + url: '/api/v1/explorer/collections/', + ...options + }); +}; + +/** + * Get Collection + */ +export const explorerGetCollection = (options: Options) => { + return (options.client ?? _heyApiClient).get({ + url: '/api/v1/explorer/collections/{collection_id}', + ...options + }); +}; + +/** + * List Themes + */ +export const explorerListThemes = (options?: Options) => { + return (options?.client ?? _heyApiClient).get({ + url: '/api/v1/explorer/themes/', + ...options + }); +}; + +/** + * Get Theme + */ +export const explorerGetTheme = (options: Options) => { + return (options.client ?? _heyApiClient).get({ + url: '/api/v1/explorer/themes/{theme_slug}', + ...options + }); +}; + /** * Get Result * Fetch a result diff --git a/frontend/src/client/types.gen.ts b/frontend/src/client/types.gen.ts index 14c228a..1fe69d6 100644 --- a/frontend/src/client/types.gen.ts +++ b/frontend/src/client/types.gen.ts @@ -1,5 +1,73 @@ // This file is auto-generated by @hey-api/openapi-ts +export type AftCollectionCard = { + title: string; + description?: string | null; + placeholder?: boolean | null; + content: Array; +}; + +export type AftCollectionCardContent = { + type: 'box-whisker-chart' | 'figure-gallery' | 'series-chart' | 'taylor-diagram'; + provider: string; + diagnostic: string; + title: string; + description?: string | null; + interpretation?: string | null; + span?: (1 | 2) | null; + placeholder?: boolean | null; + metric_units?: string | null; + clip_min?: number | null; + clip_max?: number | null; + y_min?: number | null; + y_max?: number | null; + show_zero_line?: boolean | null; + symmetrical_axes?: boolean | null; + reference_stddev?: number | null; + label_template?: string | null; + other_filters?: { + [key: string]: string; + } | null; + grouping_config?: AftCollectionGroupingConfig | null; +}; + +export type AftCollectionContent = { + description?: string | null; + short_description?: string | null; +}; + +export type AftCollectionDetail = { + id: string; + name: string; + theme?: string | null; + endorser?: string | null; + version_control?: string | null; + reference_dataset?: string | null; + provider_link?: string | null; + content?: AftCollectionContent | null; + diagnostics: Array; + explorer_cards: Array; +}; + +export type AftCollectionDiagnosticLink = { + provider_slug: string; + diagnostic_slug: string; +}; + +export type AftCollectionGroupingConfig = { + group_by: string; + hue: string; + style?: string | null; +}; + +export type AftCollectionSummary = { + id: string; + name: string; + theme?: string | null; + endorser?: string | null; + card_count: number; +}; + export type AftDiagnosticDetail = { id: string; name: string; @@ -161,30 +229,14 @@ export type Execution = { created_at: string; updated_at: string; outputs: Array; - execution_group: ExecutionGroupSummary; }; export type ExecutionGroup = { id: number; key: string; dirty: boolean; - executions: Array; - latest_execution: ExecutionSummary | null; - selectors: { - [key: string]: Array<[ - string, - string - ]>; - }; - diagnostic: DiagnosticSummary; - created_at: string; - updated_at: string; -}; - -export type ExecutionGroupSummary = { - id: number; - key: string; - dirty: boolean; + executions: Array; + latest_execution: Execution | null; selectors: { [key: string]: Array<[ string, @@ -239,17 +291,6 @@ export type ExecutionStatsWritable = { total_files: number; }; -export type ExecutionSummary = { - id: number; - dataset_hash: string; - dataset_count: number; - successful: boolean; - retracted: boolean; - created_at: string; - updated_at: string; - outputs: Array; -}; - export type Facet = { key: string; values: Array; @@ -387,10 +428,30 @@ export type SeriesValue = { execution_id: number; }; +export type ThemeDetail = { + slug: string; + title: string; + description?: string | null; + collections: Array; + explorer_cards: Array; +}; + +export type ThemeSummary = { + slug: string; + title: string; + description?: string | null; + collection_count: number; + card_count: number; +}; + export type ValidationError = { loc: Array; msg: string; type: string; + input?: unknown; + ctx?: { + [key: string]: unknown; + }; }; export type Cmip7AssessmentFastTrackAftListAftDiagnosticsData = { @@ -966,6 +1027,92 @@ export type ExecutionsExecutionArchiveResponses = { 200: unknown; }; +export type ExplorerListCollectionsData = { + body?: never; + path?: never; + query?: never; + url: '/api/v1/explorer/collections/'; +}; + +export type ExplorerListCollectionsResponses = { + /** + * Successful Response + */ + 200: Array; +}; + +export type ExplorerListCollectionsResponse = ExplorerListCollectionsResponses[keyof ExplorerListCollectionsResponses]; + +export type ExplorerGetCollectionData = { + body?: never; + path: { + collection_id: string; + }; + query?: never; + url: '/api/v1/explorer/collections/{collection_id}'; +}; + +export type ExplorerGetCollectionErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type ExplorerGetCollectionError = ExplorerGetCollectionErrors[keyof ExplorerGetCollectionErrors]; + +export type ExplorerGetCollectionResponses = { + /** + * Successful Response + */ + 200: AftCollectionDetail; +}; + +export type ExplorerGetCollectionResponse = ExplorerGetCollectionResponses[keyof ExplorerGetCollectionResponses]; + +export type ExplorerListThemesData = { + body?: never; + path?: never; + query?: never; + url: '/api/v1/explorer/themes/'; +}; + +export type ExplorerListThemesResponses = { + /** + * Successful Response + */ + 200: Array; +}; + +export type ExplorerListThemesResponse = ExplorerListThemesResponses[keyof ExplorerListThemesResponses]; + +export type ExplorerGetThemeData = { + body?: never; + path: { + theme_slug: string; + }; + query?: never; + url: '/api/v1/explorer/themes/{theme_slug}'; +}; + +export type ExplorerGetThemeErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type ExplorerGetThemeError = ExplorerGetThemeErrors[keyof ExplorerGetThemeErrors]; + +export type ExplorerGetThemeResponses = { + /** + * Successful Response + */ + 200: ThemeDetail; +}; + +export type ExplorerGetThemeResponse = ExplorerGetThemeResponses[keyof ExplorerGetThemeResponses]; + export type ResultsGetResultData = { body?: never; path: { diff --git a/frontend/src/components/app/welcomeModal.tsx b/frontend/src/components/app/welcomeModal.tsx index 64a2d72..9872502 100644 --- a/frontend/src/components/app/welcomeModal.tsx +++ b/frontend/src/components/app/welcomeModal.tsx @@ -75,7 +75,7 @@ export function WelcomeModal() {

The results presented here focus on CMIP6 datasets, but this will be - updated as new CMIP7 datasets become available We have some example + updated as new CMIP7 datasets become available. We have some example figures available in the Data Explorer{" "} or you can browse the full{" "} Diagnostic Catalog. diff --git a/frontend/src/components/diagnostics/executionsTable.tsx b/frontend/src/components/diagnostics/executionsTable.tsx index 1d47267..1691e18 100644 --- a/frontend/src/components/diagnostics/executionsTable.tsx +++ b/frontend/src/components/diagnostics/executionsTable.tsx @@ -6,7 +6,7 @@ import { } from "@tanstack/react-table"; import { format } from "date-fns"; import { SquareArrowOutUpRight } from "lucide-react"; -import type { ExecutionSummary } from "@/client"; +import type { Execution } from "@/client"; import { DataTable } from "@/components/dataTable/dataTable"; import { Badge } from "@/components/ui/badge"; import { @@ -18,13 +18,13 @@ import { } from "@/components/ui/card"; import { Route } from "@/routes/_app/executions.$groupId/index"; -const columnHelper = createColumnHelper(); +const columnHelper = createColumnHelper(); function OpenCell({ row: { original: { id }, }, -}: CellContext) { +}: CellContext) { const navigate = useNavigate({ from: Route.fullPath }); return ( -

+
{card.description && ( {card.description} )} diff --git a/frontend/src/components/explorer/explorerCardContent.tsx b/frontend/src/components/explorer/explorerCardContent.tsx index 25aed57..43410c9 100644 --- a/frontend/src/components/explorer/explorerCardContent.tsx +++ b/frontend/src/components/explorer/explorerCardContent.tsx @@ -73,7 +73,7 @@ export function ExplorerCardContent({ contentItem }: ExplorerCardContentProps) { {contentItem.placeholder ? "PLACEHOLDER: " : ""}{" "} {contentItem.title} -
+
{contentItem.description && ( {contentItem.description} )} @@ -95,7 +95,7 @@ export function ExplorerCardContent({ contentItem }: ExplorerCardContentProps) { - + {contentItem.interpretation && (
diff --git a/frontend/src/components/explorer/explorerThemeLayout.tsx b/frontend/src/components/explorer/explorerThemeLayout.tsx index cc0e23f..206bc7e 100644 --- a/frontend/src/components/explorer/explorerThemeLayout.tsx +++ b/frontend/src/components/explorer/explorerThemeLayout.tsx @@ -1,4 +1,6 @@ import { useEffect } from "react"; +import type { AftCollectionDetail } from "@/client/types.gen"; +import { CollectionHeader } from "@/components/explorer/collectionHeader"; import { ExplorerCard } from "@/components/explorer/explorerCard"; import type { ExplorerCard as ExplorerCardType } from "@/components/explorer/types"; import { TooltipProvider } from "@/components/ui/tooltip.tsx"; @@ -9,16 +11,24 @@ export type { ExplorerCardContent, } from "@/components/explorer/types"; -interface ExplorerThemeLayoutProps { +interface CollectionGroup { + collection: AftCollectionDetail; cards: ExplorerCardType[]; } -export const ExplorerThemeLayout = ({ cards }: ExplorerThemeLayoutProps) => { +interface ExplorerThemeLayoutProps { + collectionGroups: CollectionGroup[]; + plainLanguage: boolean; +} + +export const ExplorerThemeLayout = ({ + collectionGroups, + plainLanguage, +}: ExplorerThemeLayoutProps) => { // Scroll to card if hash is present in URL useEffect(() => { - const hash = window.location.hash.slice(1); // Remove the # character + const hash = window.location.hash.slice(1); if (hash) { - // Use setTimeout to ensure the DOM is fully rendered setTimeout(() => { const element = document.getElementById(hash); if (element) { @@ -30,11 +40,23 @@ export const ExplorerThemeLayout = ({ cards }: ExplorerThemeLayoutProps) => { return ( -
- {cards.map((card) => ( -
- -
+
+ {collectionGroups.map((group) => ( +
+
+ +
+
+ {group.cards.map((card) => ( +
+ +
+ ))} +
+
))}
diff --git a/frontend/src/components/explorer/thematicContent.tsx b/frontend/src/components/explorer/thematicContent.tsx index 31ce043..70e456f 100644 --- a/frontend/src/components/explorer/thematicContent.tsx +++ b/frontend/src/components/explorer/thematicContent.tsx @@ -1,13 +1,17 @@ import { useSuspenseQuery } from "@tanstack/react-query"; -import { Link } from "@tanstack/react-router"; +import { useNavigate } from "@tanstack/react-router"; +import { BookOpen, FlaskConical } from "lucide-react"; +import { useState } from "react"; import { explorerGetThemeOptions } from "@/client/@tanstack/react-query.gen"; import type { AftCollectionCard, AftCollectionCardContent, + AftCollectionDetail, AftCollectionGroupingConfig, ThemeDetail, } from "@/client/types.gen"; import { Button } from "@/components/ui/button.tsx"; +import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs.tsx"; import { Route } from "@/routes/_app/explorer/themes.tsx"; import { ExplorerThemeLayout } from "./explorerThemeLayout"; import type { ChartGroupingConfig } from "./grouping/types"; @@ -16,10 +20,12 @@ import type { ExplorerCard, ExplorerCardContent } from "./types"; const themes = [ { name: "atmosphere", title: "Atmosphere" }, { name: "earth-system", title: "Earth System" }, - { name: "impact-and-adaptation", title: "Impact and Adaptation" }, - { name: "land", title: "Land and Land Ice" }, - { name: "ocean", title: "Ocean and Sea Ice" }, -]; + { name: "impact-and-adaptation", title: "Impact & Adaptation" }, + { name: "land", title: "Land & Land Ice" }, + { name: "ocean", title: "Ocean & Sea Ice" }, +] as const; + +type ThemeName = (typeof themes)[number]["name"]; function toChartGroupingConfig( apiConfig: AftCollectionGroupingConfig, @@ -84,8 +90,10 @@ function toExplorerCardContent( } } -function themeToExplorerCards(theme: ThemeDetail): ExplorerCard[] { - return theme.explorer_cards.map((card: AftCollectionCard) => ({ +function collectionToExplorerCards( + collection: AftCollectionDetail, +): ExplorerCard[] { + return collection.explorer_cards.map((card: AftCollectionCard) => ({ title: card.title, description: card.description ?? undefined, placeholder: card.placeholder ?? undefined, @@ -93,38 +101,100 @@ function themeToExplorerCards(theme: ThemeDetail): ExplorerCard[] { })); } -function ThemeContent({ slug }: { slug: string }) { +function buildCollectionGroups(theme: ThemeDetail) { + return theme.collections + .map((collection) => ({ + collection, + cards: collectionToExplorerCards(collection), + })) + .filter((group) => group.cards.length > 0); +} + +function hasPlainLanguageContent(theme: ThemeDetail): boolean { + return theme.collections.some((c) => { + const pl = c.content?.plain_language; + return pl?.description || pl?.why_it_matters || pl?.takeaway; + }); +} + +function ThemeContent({ + slug, + plainLanguage, +}: { + slug: string; + plainLanguage: boolean; +}) { const { data: theme } = useSuspenseQuery( explorerGetThemeOptions({ path: { theme_slug: slug } }), ); - const cards = themeToExplorerCards(theme); - return ; + const collectionGroups = buildCollectionGroups(theme); + return ( + + ); } export function ThematicContent() { const { theme } = Route.useSearch(); + const navigate = useNavigate(); const themeObj = themes.find((t) => t.name === theme); + const [plainLanguage, setPlainLanguage] = useState(false); + + const { data: themeData } = useSuspenseQuery( + explorerGetThemeOptions({ path: { theme_slug: theme } }), + ); + const showPlainLanguageToggle = hasPlainLanguageContent(themeData); + return ( <> {`${themeObj?.title} Explorer - Climate REF`} -
- {themes.map((item) => ( - +
+ + value={theme} + onValueChange={(value) => { + navigate({ + to: Route.fullPath, + search: { theme: value }, + }); + }} > + + {themes.map((item) => ( + key={item.name} value={item.name}> + {item.title} + + ))} + + + {showPlainLanguageToggle && ( - - ))} + )} +
+ {showPlainLanguageToggle && ( +

+ Toggle between technical descriptions and plain language summaries + using the button above. +

+ )} +
+
+ {theme && }
-
{theme && }
); } diff --git a/frontend/src/routes/_app/explorer/themes.tsx b/frontend/src/routes/_app/explorer/themes.tsx index d371b44..cf8665c 100644 --- a/frontend/src/routes/_app/explorer/themes.tsx +++ b/frontend/src/routes/_app/explorer/themes.tsx @@ -2,7 +2,6 @@ import { createFileRoute } from "@tanstack/react-router"; import { zodValidator } from "@tanstack/zod-adapter"; import { z } from "zod"; import { ThematicContent } from "@/components/explorer/thematicContent.tsx"; -import { Card, CardContent } from "@/components/ui/card.tsx"; const themesSchema = z.object({ theme: z @@ -18,11 +17,15 @@ const themesSchema = z.object({ const Themes = () => { return ( - - - - - +
+
+

Theme Explorer

+

+ Browse climate model evaluation results organized by scientific theme. +

+
+ +
); }; From d4e5c34a5739b464406bee53fb9e25288bb30ae7 Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Sat, 7 Mar 2026 10:47:51 +1100 Subject: [PATCH 06/15] chore: add a envrc file to automatically set the .env --- .envrc | 1 + 1 file changed, 1 insertion(+) create mode 100644 .envrc diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..76f03ce --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +dotenv_if_exists .env From 63d0d5c9dd708615e3de915c17c9c252a59b6cff Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Sat, 7 Mar 2026 11:00:13 +1100 Subject: [PATCH 07/15] feat: enhance SeriesMetadata with dimensions and simplify slug formatting in CollectionHeader --- frontend/src/components/execution/values/types.ts | 3 ++- frontend/src/components/explorer/collectionHeader.tsx | 10 +--------- frontend/src/routes/_app/explorer/themes.tsx | 2 +- 3 files changed, 4 insertions(+), 11 deletions(-) diff --git a/frontend/src/components/execution/values/types.ts b/frontend/src/components/execution/values/types.ts index e9de891..d885e68 100644 --- a/frontend/src/components/execution/values/types.ts +++ b/frontend/src/components/execution/values/types.ts @@ -44,5 +44,6 @@ export interface SeriesMetadata { seriesIndex: number; label: string; color: string; - isReference: boolean; // NEW - track reference series + isReference: boolean; + dimensions: Record; } diff --git a/frontend/src/components/explorer/collectionHeader.tsx b/frontend/src/components/explorer/collectionHeader.tsx index 01381fc..730bebc 100644 --- a/frontend/src/components/explorer/collectionHeader.tsx +++ b/frontend/src/components/explorer/collectionHeader.tsx @@ -43,14 +43,6 @@ function getTakeaway( return content.takeaway; } -/** Format a slug like "enso" into "Enso" or "annual-cycle" into "Annual Cycle" */ -function formatSlug(slug: string): string { - return slug - .split(/[-_]/) - .map((word) => word.charAt(0).toUpperCase() + word.slice(1)) - .join(" "); -} - export function CollectionHeader({ collection, plainLanguage, @@ -118,7 +110,7 @@ export function CollectionHeader({ variant="outline" className="text-xs font-normal hover:bg-accent cursor-pointer" > - {formatSlug(d.provider_slug)} / {formatSlug(d.diagnostic_slug)} + {d.provider_slug} / {d.diagnostic_slug} ))} diff --git a/frontend/src/routes/_app/explorer/themes.tsx b/frontend/src/routes/_app/explorer/themes.tsx index cf8665c..1c1626e 100644 --- a/frontend/src/routes/_app/explorer/themes.tsx +++ b/frontend/src/routes/_app/explorer/themes.tsx @@ -19,7 +19,7 @@ const Themes = () => { return (
-

Theme Explorer

+

Theme Explorer

Browse climate model evaluation results organized by scientific theme.

From 4da37d162bbbcd243ed68272be53db1cf32861cb Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Sat, 7 Mar 2026 12:34:13 +1100 Subject: [PATCH 08/15] feat(series): rewrite series chart with canvas rendering and dark mode Replace Recharts SVG-based rendering with HTML5 Canvas for dramatically better performance with 500+ series. Key changes: - Canvas-based line rendering with single draw pass instead of per-series SVG - Grid-based spatial index (30px cells) for O(1) nearest-point lookup - Ref-based hover updates to avoid React re-renders of canvas - Rich tooltip with dimensions, reference delta, rank, and units - Dark mode support for canvas axes, grid, and tooltip - Crosshair with dashed vertical line and colored dot - Smart axis formatting (integer ticks for year/month/day axes) - Y axis label shows metric name and units - React.memo on SeriesCanvas and SeriesLegend to prevent re-render cascade - Shared ChartDataPoint type and isIntegerAxis utility to eliminate duplication - Canvas DPR optimization to avoid GPU buffer reallocation on hover redraws --- .../values/series/canvasTooltip.test.ts | 114 +++++ .../execution/values/series/canvasTooltip.tsx | 210 ++++++++ .../execution/values/series/seriesCanvas.tsx | 389 +++++++++++++++ .../execution/values/series/seriesLegend.tsx | 365 ++++++++++++-- .../execution/values/series/seriesToolip.tsx | 88 ---- .../values/series/seriesVisualization.tsx | 472 +++++++++++------- .../values/series/useChartScales.test.ts | 212 ++++++++ .../execution/values/series/useChartScales.ts | 107 ++++ .../values/series/useSpatialIndex.test.ts | 293 +++++++++++ .../values/series/useSpatialIndex.ts | 201 ++++++++ .../execution/values/series/utils.test.ts | 116 ++++- .../execution/values/series/utils.ts | 87 +++- .../explorer/content/seriesChartContent.tsx | 2 + 13 files changed, 2313 insertions(+), 343 deletions(-) create mode 100644 frontend/src/components/execution/values/series/canvasTooltip.test.ts create mode 100644 frontend/src/components/execution/values/series/canvasTooltip.tsx create mode 100644 frontend/src/components/execution/values/series/seriesCanvas.tsx delete mode 100644 frontend/src/components/execution/values/series/seriesToolip.tsx create mode 100644 frontend/src/components/execution/values/series/useChartScales.test.ts create mode 100644 frontend/src/components/execution/values/series/useChartScales.ts create mode 100644 frontend/src/components/execution/values/series/useSpatialIndex.test.ts create mode 100644 frontend/src/components/execution/values/series/useSpatialIndex.ts diff --git a/frontend/src/components/execution/values/series/canvasTooltip.test.ts b/frontend/src/components/execution/values/series/canvasTooltip.test.ts new file mode 100644 index 0000000..e477d60 --- /dev/null +++ b/frontend/src/components/execution/values/series/canvasTooltip.test.ts @@ -0,0 +1,114 @@ +import { describe, expect, it } from "vitest"; +import { + formatDelta, + formatXValue, + formatYValue, + ordinal, +} from "./canvasTooltip"; + +describe("formatXValue", () => { + it("shows integer years without decimals", () => { + expect(formatXValue(2021, "year")).toBe("2021"); + expect(formatXValue(1990, "Year")).toBe("1990"); + }); + + it("shows integer months without decimals", () => { + expect(formatXValue(6, "month")).toBe("6"); + }); + + it("shows integer time values without decimals", () => { + expect(formatXValue(42, "time")).toBe("42"); + }); + + it("shows integer index values without decimals", () => { + expect(formatXValue(0, "index")).toBe("0"); + }); + + it("shows non-integer values with precision", () => { + expect(formatXValue(2021.5, "year")).toBe("2021.50"); + }); + + it("shows integer values for unknown axis names without decimals", () => { + expect(formatXValue(100, "pressure_level")).toBe("100"); + }); + + it("shows decimal values for unknown axes with precision", () => { + expect(formatXValue(45.123, "latitude")).toBe("45.1230"); + }); +}); + +describe("formatYValue", () => { + it("formats normal values with 4 significant figures", () => { + expect(formatYValue(1.234)).toBe("1.234"); + expect(formatYValue(42.56)).toBe("42.56"); + expect(formatYValue(0.5678)).toBe("0.5678"); + }); + + it("formats very large values with scientific notation", () => { + expect(formatYValue(1500000)).toBe("1.500e+6"); + }); + + it("formats very small non-zero values with scientific notation", () => { + expect(formatYValue(0.00012)).toBe("1.200e-4"); + }); + + it("formats zero normally", () => { + expect(formatYValue(0)).toBe("0.000"); + }); + + it("formats negative values correctly", () => { + expect(formatYValue(-2.345)).toBe("-2.345"); + expect(formatYValue(-0.00005)).toBe("-5.000e-5"); + }); +}); + +describe("formatDelta", () => { + it("shows positive deltas with + sign", () => { + expect(formatDelta(0.5)).toBe("+0.500"); + expect(formatDelta(1.23)).toBe("+1.23"); + }); + + it("shows negative deltas with - sign", () => { + expect(formatDelta(-0.5)).toBe("-0.500"); + expect(formatDelta(-1.23)).toBe("-1.23"); + }); + + it("shows zero delta with + sign", () => { + expect(formatDelta(0)).toBe("+0.00"); + }); + + it("uses scientific notation for large deltas", () => { + expect(formatDelta(2500000)).toBe("+2.50e+6"); + }); + + it("uses scientific notation for very small deltas", () => { + expect(formatDelta(0.00005)).toBe("+5.00e-5"); + expect(formatDelta(-0.00003)).toBe("-3.00e-5"); + }); +}); + +describe("ordinal", () => { + it("formats 1st, 2nd, 3rd correctly", () => { + expect(ordinal(1)).toBe("1st"); + expect(ordinal(2)).toBe("2nd"); + expect(ordinal(3)).toBe("3rd"); + }); + + it("formats 4th-20th with 'th'", () => { + expect(ordinal(4)).toBe("4th"); + expect(ordinal(11)).toBe("11th"); + expect(ordinal(12)).toBe("12th"); + expect(ordinal(13)).toBe("13th"); + }); + + it("formats 21st, 22nd, 23rd correctly", () => { + expect(ordinal(21)).toBe("21st"); + expect(ordinal(22)).toBe("22nd"); + expect(ordinal(23)).toBe("23rd"); + }); + + it("formats 100th, 101st correctly", () => { + expect(ordinal(100)).toBe("100th"); + expect(ordinal(101)).toBe("101st"); + }); +}); diff --git a/frontend/src/components/execution/values/series/canvasTooltip.tsx b/frontend/src/components/execution/values/series/canvasTooltip.tsx new file mode 100644 index 0000000..a990bbb --- /dev/null +++ b/frontend/src/components/execution/values/series/canvasTooltip.tsx @@ -0,0 +1,210 @@ +import type { NearestResult } from "./useSpatialIndex"; +import { isIntegerAxis } from "./utils"; + +const MAX_TOOLTIP_ENTRIES = 8; + +interface CanvasTooltipProps { + visible: boolean; + x: number; + y: number; + nearest: NearestResult | null; + allAtX: NearestResult[]; + containerWidth: number; + indexName: string; + units?: string; +} + +export function formatXValue(value: number, indexName: string): string { + if (Number.isInteger(value) && isIntegerAxis(indexName)) { + return value.toString(); + } + if (Number.isInteger(value)) { + return value.toString(); + } + if (isIntegerAxis(indexName) && Math.abs(value - Math.round(value)) < 1e-9) { + return Math.round(value).toString(); + } + return value.toPrecision(6); +} + +export function formatYValue(value: number): string { + if (Math.abs(value) >= 1e6 || (Math.abs(value) < 1e-3 && value !== 0)) { + return value.toExponential(3); + } + return value.toPrecision(4); +} + +export function formatDelta(delta: number): string { + const sign = delta >= 0 ? "+" : ""; + if (Math.abs(delta) >= 1e6 || (Math.abs(delta) < 1e-3 && delta !== 0)) { + return `${sign}${delta.toExponential(2)}`; + } + return `${sign}${delta.toPrecision(3)}`; +} + +export function ordinal(n: number): string { + const s = ["th", "st", "nd", "rd"]; + const v = n % 100; + return n + (s[(v - 20) % 10] || s[v] || s[0]); +} + +export function CanvasTooltip({ + visible, + x, + y, + nearest, + allAtX, + containerWidth, + indexName, + units, +}: CanvasTooltipProps) { + if (!visible || !nearest || allAtX.length === 0) { + return null; + } + + const displayed = allAtX.slice(0, MAX_TOOLTIP_ENTRIES); + const remaining = allAtX.length - displayed.length; + + // Find reference value at this X for delta computation + const referenceEntry = allAtX.find((r) => r.metadata.isReference); + const referenceValue = referenceEntry?.point.y ?? null; + + // Compute rank of nearest among all visible series (sorted by value descending) + const sortedByValue = [...allAtX].sort((a, b) => b.point.y - a.point.y); + const nearestRank = + sortedByValue.findIndex( + (r) => r.metadata.seriesIndex === nearest.metadata.seriesIndex, + ) + 1; + + // Position tooltip to avoid going off-screen + const tooltipWidth = 320; + const flipX = x + tooltipWidth + 20 > containerWidth; + const left = flipX ? x - tooltipWidth - 10 : x + 10; + const top = Math.max(10, y - 60); + + return ( +
+ {/* Header: X value with axis name */} +
+ {indexName}: + + {formatXValue(nearest.point.x, indexName)} + +
+ + {/* Nearest series detail block */} +
+
+
+ + {nearest.metadata.label} + +
+ + {/* Value + delta + rank */} +
+ + {formatYValue(nearest.point.y)} + {units && units !== "unitless" && ( + + {units} + + )} + + {referenceValue !== null && !nearest.metadata.isReference && ( + = 0 + ? "text-red-600 dark:text-red-400" + : "text-blue-600 dark:text-blue-400" + }`} + > + {formatDelta(nearest.point.y - referenceValue)} vs ref + + )} + {allAtX.length > 1 && ( + + {ordinal(nearestRank)} of {allAtX.length} + + )} +
+ + {/* Dimension details for nearest series */} + {Object.keys(nearest.metadata.dimensions).length > 0 && ( +
+ {Object.entries(nearest.metadata.dimensions).map(([key, value]) => ( + + {key}: {value} + + ))} +
+ )} +
+ + {/* Other series at this X position */} + {displayed.length > 1 && ( +
+ {displayed + .filter( + (r) => r.metadata.seriesIndex !== nearest.metadata.seriesIndex, + ) + .slice(0, MAX_TOOLTIP_ENTRIES - 1) + .map(({ point, metadata }) => { + const delta = + referenceValue !== null && !metadata.isReference + ? point.y - referenceValue + : null; + return ( +
+
+ + {metadata.label} + + + {formatYValue(point.y)} + + {delta !== null && ( + = 0 + ? "text-red-600/60 dark:text-red-400/60" + : "text-blue-600/60 dark:text-blue-400/60" + }`} + > + {formatDelta(delta)} + + )} +
+ ); + })} +
+ )} + + {remaining > 0 && ( +
+ +{remaining} more series +
+ )} +
+ ); +} diff --git a/frontend/src/components/execution/values/series/seriesCanvas.tsx b/frontend/src/components/execution/values/series/seriesCanvas.tsx new file mode 100644 index 0000000..b9da226 --- /dev/null +++ b/frontend/src/components/execution/values/series/seriesCanvas.tsx @@ -0,0 +1,389 @@ +import type * as d3 from "d3"; +import { memo, useCallback, useEffect, useRef } from "react"; +import type { SeriesMetadata } from "../types"; +import type { ChartMargins } from "./useChartScales"; +import { type ChartDataPoint, isIntegerAxis } from "./utils"; + +export interface CrosshairPosition { + dataPixelX: number; + nearestPixelX: number; + nearestPixelY: number; + nearestColor: string; +} + +interface SeriesCanvasProps { + chartData: ChartDataPoint[]; + seriesMetadata: SeriesMetadata[]; + indexName: string; + hiddenLabels: Set; + hoveredLabelRef: React.MutableRefObject; + crosshairRef: React.MutableRefObject; + xScale: d3.ScaleLinear; + yScale: d3.ScaleLinear; + margins: ChartMargins; + width: number; + height: number; + innerWidth: number; + innerHeight: number; + isDark: boolean; + metricName?: string; + units?: string; + onMouseMove: (e: React.MouseEvent) => void; + onMouseLeave: () => void; + onClick: (e: React.MouseEvent) => void; +} + +export const SeriesCanvas = memo(function SeriesCanvas({ + chartData, + seriesMetadata, + indexName, + hiddenLabels, + hoveredLabelRef, + crosshairRef, + xScale, + yScale, + margins, + width, + height, + innerWidth, + innerHeight, + isDark, + metricName, + units, + onMouseMove, + onMouseLeave, + onClick, +}: SeriesCanvasProps) { + const canvasRef = useRef(null); + const rafRef = useRef(0); + + const draw = useCallback(() => { + const canvas = canvasRef.current; + if (!canvas) return; + const ctx = canvas.getContext("2d"); + if (!ctx) return; + + const dpr = window.devicePixelRatio || 1; + const targetW = width * dpr; + const targetH = height * dpr; + if (canvas.width !== targetW || canvas.height !== targetH) { + canvas.width = targetW; + canvas.height = targetH; + } + ctx.setTransform(dpr, 0, 0, dpr, 0, 0); + ctx.clearRect(0, 0, width, height); + ctx.save(); + ctx.translate(margins.left, margins.top); + + // Clip to chart area + ctx.beginPath(); + ctx.rect(0, 0, innerWidth, innerHeight); + ctx.clip(); + + // Draw grid + drawGrid(ctx, xScale, yScale, innerWidth, innerHeight, isDark); + + // Draw lines: hidden first, then visible, hovered last + const hoveredLabel = hoveredLabelRef.current; + const sortedMeta = [...seriesMetadata].sort((a, b) => { + const aHidden = hiddenLabels.has(a.label); + const bHidden = hiddenLabels.has(b.label); + if (aHidden && !bHidden) return -1; + if (!aHidden && bHidden) return 1; + const aHovered = a.label === hoveredLabel; + const bHovered = b.label === hoveredLabel; + if (aHovered && !bHovered) return 1; + if (!aHovered && bHovered) return -1; + // Reference series on top of regular + if (a.isReference && !b.isReference) return 1; + if (!a.isReference && b.isReference) return -1; + return 0; + }); + + for (const meta of sortedMeta) { + const isHidden = hiddenLabels.has(meta.label); + const isHovered = meta.label === hoveredLabel; + const key = `series_${meta.seriesIndex}`; + + const refColor = isDark ? "#FFFFFF" : "#000000"; + if (isHidden) { + ctx.strokeStyle = isDark ? "#4B5563" : "#D1D5DB"; + ctx.lineWidth = 0.5; + ctx.globalAlpha = 0.15; + } else if (isHovered) { + ctx.strokeStyle = meta.isReference ? refColor : meta.color; + ctx.lineWidth = meta.isReference ? 5 : 3.5; + ctx.globalAlpha = 1; + } else { + ctx.strokeStyle = meta.isReference ? refColor : meta.color; + ctx.lineWidth = meta.isReference ? 3 : 1.5; + ctx.globalAlpha = meta.isReference ? 1 : 0.8; + } + + ctx.beginPath(); + let started = false; + for (let i = 0; i < chartData.length; i++) { + const xVal = chartData[i][indexName]; + const yVal = chartData[i][key]; + if ( + typeof xVal !== "number" || + typeof yVal !== "number" || + !Number.isFinite(yVal) + ) { + started = false; + continue; + } + const px = xScale(xVal); + const py = yScale(yVal); + if (!started) { + ctx.moveTo(px, py); + started = true; + } else { + ctx.lineTo(px, py); + } + } + ctx.stroke(); + } + + ctx.globalAlpha = 1; + + // Draw crosshair + const crosshair = crosshairRef.current; + if (crosshair) { + // Vertical line at snapped X position + ctx.setLineDash([4, 4]); + ctx.strokeStyle = "#94A3B8"; + ctx.lineWidth = 1; + ctx.beginPath(); + ctx.moveTo(crosshair.dataPixelX, 0); + ctx.lineTo(crosshair.dataPixelX, innerHeight); + ctx.stroke(); + ctx.setLineDash([]); + + // Dot on nearest point + ctx.beginPath(); + ctx.arc( + crosshair.nearestPixelX, + crosshair.nearestPixelY, + 5, + 0, + Math.PI * 2, + ); + ctx.fillStyle = crosshair.nearestColor; + ctx.fill(); + ctx.strokeStyle = "#FFFFFF"; + ctx.lineWidth = 2; + ctx.stroke(); + } + + ctx.restore(); + + // Draw axes outside clip region + drawAxes( + ctx, + xScale, + yScale, + margins, + innerWidth, + innerHeight, + indexName, + isDark, + metricName, + units, + ); + }, [ + chartData, + seriesMetadata, + indexName, + hiddenLabels, + hoveredLabelRef, + crosshairRef, + xScale, + yScale, + margins, + width, + height, + innerWidth, + innerHeight, + isDark, + metricName, + units, + ]); + + // Redraw on data/scale changes + useEffect(() => { + cancelAnimationFrame(rafRef.current); + rafRef.current = requestAnimationFrame(draw); + return () => cancelAnimationFrame(rafRef.current); + }, [draw]); + + // Expose redraw for hover updates (called from parent via ref) + useEffect(() => { + const el = canvasRef.current; + if (el) { + (el as HTMLCanvasElement & { redraw: () => void }).redraw = () => { + cancelAnimationFrame(rafRef.current); + rafRef.current = requestAnimationFrame(draw); + }; + } + }, [draw]); + + return ( +
+ +
+ ); +}); + +function drawGrid( + ctx: CanvasRenderingContext2D, + xScale: d3.ScaleLinear, + yScale: d3.ScaleLinear, + innerWidth: number, + innerHeight: number, + isDark: boolean, +) { + ctx.strokeStyle = isDark ? "#374151" : "#E5E7EB"; + ctx.lineWidth = 1; + ctx.setLineDash([3, 3]); + + // Horizontal grid lines + const yTicks = yScale.ticks(8); + for (const tick of yTicks) { + const y = yScale(tick); + ctx.beginPath(); + ctx.moveTo(0, y); + ctx.lineTo(innerWidth, y); + ctx.stroke(); + } + + // Vertical grid lines + const xTicks = xScale.ticks(10); + for (const tick of xTicks) { + const x = xScale(tick); + ctx.beginPath(); + ctx.moveTo(x, 0); + ctx.lineTo(x, innerHeight); + ctx.stroke(); + } + + ctx.setLineDash([]); +} + +function formatTickValue(value: number, range: number): string { + if (Math.abs(value) >= 1e6 || (range > 0 && Math.abs(value) >= 1e4)) { + return value.toExponential(1); + } + if (Math.abs(value) < 1e-3 && value !== 0) { + return value.toExponential(1); + } + if (range < 0.1) return value.toFixed(3); + if (range < 1) return value.toFixed(2); + if (range < 10) return value.toFixed(2); + if (range < 100) return value.toFixed(1); + return value.toFixed(0); +} + +function formatXTickValue(value: number, indexName: string): string { + if (isIntegerAxis(indexName) && Number.isInteger(value)) { + return value.toString(); + } + if (Number.isInteger(value)) { + return value.toString(); + } + // For non-integer values on an integer axis, still show as integer if close + if (isIntegerAxis(indexName) && Math.abs(value - Math.round(value)) < 1e-9) { + return Math.round(value).toString(); + } + return value.toPrecision(6); +} + +function drawAxes( + ctx: CanvasRenderingContext2D, + xScale: d3.ScaleLinear, + yScale: d3.ScaleLinear, + margins: ChartMargins, + innerWidth: number, + innerHeight: number, + indexName: string, + isDark: boolean, + metricName?: string, + units?: string, +) { + ctx.save(); + + const yDomain = yScale.domain(); + const yRange = yDomain[1] - yDomain[0]; + + // Y axis + ctx.strokeStyle = isDark ? "#374151" : "#E5E7EB"; + ctx.fillStyle = isDark ? "#9CA3AF" : "#6B7280"; + ctx.lineWidth = 1; + ctx.font = "13px ui-monospace, monospace"; + ctx.textAlign = "right"; + ctx.textBaseline = "middle"; + + ctx.beginPath(); + ctx.moveTo(margins.left, margins.top); + ctx.lineTo(margins.left, margins.top + innerHeight); + ctx.stroke(); + + const yTicks = yScale.ticks(8); + for (const tick of yTicks) { + const y = margins.top + yScale(tick); + ctx.fillText(formatTickValue(tick, yRange), margins.left - 8, y); + } + + // Y axis label + ctx.save(); + ctx.translate(14, margins.top + innerHeight / 2); + ctx.rotate(-Math.PI / 2); + ctx.textAlign = "center"; + ctx.font = "14px system-ui, sans-serif"; + const baseName = metricName || "Value"; + const yLabel = + units && units !== "unitless" ? `${baseName} (${units})` : baseName; + ctx.fillText(yLabel, 0, 0); + ctx.restore(); + + // X axis + ctx.textAlign = "center"; + ctx.textBaseline = "top"; + ctx.font = "13px ui-monospace, monospace"; + + ctx.beginPath(); + ctx.moveTo(margins.left, margins.top + innerHeight); + ctx.lineTo(margins.left + innerWidth, margins.top + innerHeight); + ctx.stroke(); + + const xTicksRaw = xScale.ticks(10); + const xTicks = isIntegerAxis(indexName) + ? xTicksRaw.filter((t) => Number.isInteger(t)) + : xTicksRaw; + for (const tick of xTicks) { + const x = margins.left + xScale(tick); + ctx.fillText( + formatXTickValue(tick, indexName), + x, + margins.top + innerHeight + 8, + ); + } + + // X axis label + ctx.textAlign = "center"; + ctx.textBaseline = "top"; + ctx.font = "14px system-ui, sans-serif"; + ctx.fillText( + indexName, + margins.left + innerWidth / 2, + margins.top + innerHeight + 30, + ); + + ctx.restore(); +} diff --git a/frontend/src/components/execution/values/series/seriesLegend.tsx b/frontend/src/components/execution/values/series/seriesLegend.tsx index b905f65..1a9e05d 100644 --- a/frontend/src/components/execution/values/series/seriesLegend.tsx +++ b/frontend/src/components/execution/values/series/seriesLegend.tsx @@ -1,70 +1,325 @@ -import { Card, CardContent } from "@/components/ui/card"; +import { memo, useCallback, useMemo, useState } from "react"; +import { Button } from "@/components/ui/button"; +import { + Collapsible, + CollapsibleContent, + CollapsibleTrigger, +} from "@/components/ui/collapsible"; +import { Input } from "@/components/ui/input"; +import { ScrollArea } from "@/components/ui/scroll-area"; -export function SeriesLegend({ +export interface LegendItem { + label: string; + color: string; + count: number; + isReference: boolean; +} + +interface SeriesLegendProps { + uniqueLabels: LegendItem[]; + hiddenLabels: Set; + hoveredLabel: string | null; + soloedLabel: string | null; + onToggleLabel: (label: string) => void; + onHoverLabel: (label: string | null) => void; + onSoloLabel: (label: string) => void; + onShowAll: () => void; + onHideAll: () => void; + groupByDimension: string | null; + onGroupByDimensionChange: (dimension: string | null) => void; + availableDimensions: string[]; + labelDimensionMap: Map>; +} + +export const SeriesLegend = memo(function SeriesLegend({ uniqueLabels, hiddenLabels, hoveredLabel, + soloedLabel, onToggleLabel, onHoverLabel, -}: { - uniqueLabels: Array<{ label: string; color: string; count: number }>; - hiddenLabels: Set; - hoveredLabel: string | null; - onToggleLabel: (label: string) => void; - onHoverLabel: (label: string | null) => void; -}) { + onSoloLabel, + onShowAll, + onHideAll, + groupByDimension, + onGroupByDimensionChange, + availableDimensions, + labelDimensionMap, +}: SeriesLegendProps) { + const [searchQuery, setSearchQuery] = useState(""); + const [collapsedGroups, setCollapsedGroups] = useState>( + new Set(), + ); + + const filteredLabels = useMemo(() => { + if (!searchQuery) return uniqueLabels; + const query = searchQuery.toLowerCase(); + return uniqueLabels.filter((item) => + item.label.toLowerCase().includes(query), + ); + }, [uniqueLabels, searchQuery]); + + // Group labels by dimension value + const groupedLabels = useMemo(() => { + if (!groupByDimension) { + return null; + } + + const groups = new Map(); + for (const item of filteredLabels) { + const dims = labelDimensionMap.get(item.label); + const groupKey = dims?.[groupByDimension] ?? "Unknown"; + const group = groups.get(groupKey); + if (group) { + group.push(item); + } else { + groups.set(groupKey, [item]); + } + } + + return Array.from(groups.entries()).sort(([a], [b]) => a.localeCompare(b)); + }, [filteredLabels, groupByDimension, labelDimensionMap]); + + const toggleGroupCollapse = useCallback((groupKey: string) => { + setCollapsedGroups((prev) => { + const next = new Set(prev); + if (next.has(groupKey)) { + next.delete(groupKey); + } else { + next.add(groupKey); + } + return next; + }); + }, []); + + const toggleGroupVisibility = useCallback( + (items: LegendItem[]) => { + const allHidden = items.every((item) => hiddenLabels.has(item.label)); + for (const item of items) { + const isHidden = hiddenLabels.has(item.label); + // If all are hidden, show all; otherwise hide all + if (allHidden && isHidden) { + onToggleLabel(item.label); + } else if (!allHidden && !isHidden) { + onToggleLabel(item.label); + } + } + }, + [hiddenLabels, onToggleLabel], + ); + + const visibleCount = uniqueLabels.filter( + (item) => !hiddenLabels.has(item.label), + ).length; + return ( -
- - -
- Legend ({uniqueLabels.length} labels) -
-
- {uniqueLabels.map(({ label, color, count }) => { - const isHidden = hiddenLabels.has(label); - const isHovered = hoveredLabel === label; +
+ {/* Search */} + setSearchQuery(e.target.value)} + className="h-8 text-sm" + /> + + {/* Bulk controls */} +
+ + + {soloedLabel && ( + + )} +
+ + {/* Group by dimension selector */} + {availableDimensions.length > 0 && ( +
+ + Group: + + +
+ )} + + {/* Status bar */} +
+ {visibleCount}/{uniqueLabels.length} visible + {searchQuery && ` (${filteredLabels.length} matching "${searchQuery}")`} + {soloedLabel && ( + + Solo: {soloedLabel} + + )} +
+ + {/* Legend list */} + + {groupedLabels ? ( +
+ {groupedLabels.map(([groupKey, items]) => { + const groupHiddenCount = items.filter((item) => + hiddenLabels.has(item.label), + ).length; + const allGroupHidden = groupHiddenCount === items.length; + const isCollapsed = collapsedGroups.has(groupKey); + return ( - +
+ + + {isCollapsed ? "\u25B6" : "\u25BC"} + + + {groupKey} + + + {items.length - groupHiddenCount}/{items.length} + + + +
+ +
+ {items.map((item) => ( + onToggleLabel(item.label)} + onHover={onHoverLabel} + onSolo={() => onSoloLabel(item.label)} + /> + ))} +
+ + ); })}
- - + ) : ( +
+ {filteredLabels.map((item) => ( + onToggleLabel(item.label)} + onHover={onHoverLabel} + onSolo={() => onSoloLabel(item.label)} + /> + ))} +
+ )} +
); +}); + +function LegendEntry({ + item, + isHidden, + isHovered, + isSoloed, + onToggle, + onHover, + onSolo, +}: { + item: LegendItem; + isHidden: boolean; + isHovered: boolean; + isSoloed: boolean; + onToggle: () => void; + onHover: (label: string | null) => void; + onSolo: () => void; +}) { + return ( + + ); } diff --git a/frontend/src/components/execution/values/series/seriesToolip.tsx b/frontend/src/components/execution/values/series/seriesToolip.tsx deleted file mode 100644 index 25873ba..0000000 --- a/frontend/src/components/execution/values/series/seriesToolip.tsx +++ /dev/null @@ -1,88 +0,0 @@ -import type { SeriesMetadata } from "../types"; - -export function SeriesTooltip({ - active, - payload, - label, - hiddenLabels, - hoveredLabel, - seriesMetadata, -}: { - active?: boolean; - payload?: Array<{ - value: number; - dataKey: string; - color: string; - name: string; - }>; - label?: string | number; - hiddenLabels: Set; - hoveredLabel: string | null; - seriesMetadata: SeriesMetadata[]; -}) { - if (!active || !payload || payload.length === 0) { - return null; - } - - // Filter visible entries - const visibleEntries = payload.filter((entry) => { - const seriesIdx = Number.parseInt(entry.dataKey.replace("series_", ""), 10); - const metadata = seriesMetadata[seriesIdx]; - return metadata && !hiddenLabels.has(metadata.label); - }); - - if (visibleEntries.length === 0) { - return null; - } - - // Show the hovered entry or the first visible entry - let displayEntry = visibleEntries[0]; - if (hoveredLabel) { - const hoveredEntry = visibleEntries.find((entry) => { - const seriesIdx = Number.parseInt( - entry.dataKey.replace("series_", ""), - 10, - ); - const metadata = seriesMetadata[seriesIdx]; - return metadata?.label === hoveredLabel; - }); - if (hoveredEntry) { - displayEntry = hoveredEntry; - } - } - - const seriesIdx = Number.parseInt( - displayEntry.dataKey.replace("series_", ""), - 10, - ); - const metadata = seriesMetadata[seriesIdx]; - - return ( -
-
- {typeof label === "number" ? label.toFixed(2) : label} -
-
-
-
- - {metadata?.label || displayEntry.name} - -
-
- {typeof displayEntry.value === "number" - ? displayEntry.value.toFixed(3) - : displayEntry.value} -
-
- {visibleEntries.length > 1 && ( -
- +{visibleEntries.length - 1} other series at this point -
- )} -
- ); -} diff --git a/frontend/src/components/execution/values/series/seriesVisualization.tsx b/frontend/src/components/execution/values/series/seriesVisualization.tsx index 5f8d64a..5eebba2 100644 --- a/frontend/src/components/execution/values/series/seriesVisualization.tsx +++ b/frontend/src/components/execution/values/series/seriesVisualization.tsx @@ -1,60 +1,106 @@ -import { useCallback, useMemo, useState } from "react"; -import { - CartesianGrid, - Line, - LineChart, - ResponsiveContainer, - Tooltip, - XAxis, - YAxis, -} from "recharts"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { Button } from "@/components/ui/button"; import { Card, CardContent } from "@/components/ui/card"; +import { useTheme } from "@/hooks/useTheme"; import type { SeriesValue } from "../types"; +import { CanvasTooltip } from "./canvasTooltip"; +import { type CrosshairPosition, SeriesCanvas } from "./seriesCanvas"; import { SeriesLegend } from "./seriesLegend"; -import { SeriesTooltip } from "./seriesToolip"; -import { createChartData, createScaledTickFormatter } from "./utils"; +import { useChartScales } from "./useChartScales"; +import type { NearestResult } from "./useSpatialIndex"; +import { useSpatialIndex } from "./useSpatialIndex"; +import { createChartData, getDimensionKeys } from "./utils"; interface SimpleSeriesVisualizationProps { seriesValues: SeriesValue[]; referenceSeriesValues?: SeriesValue[]; - labelTemplate?: string; // e.g., "{variable_id} - {source_id}" + labelTemplate?: string; maxSeriesLimit?: number; symmetricalAxes?: boolean; + metricName?: string; + units?: string; } +const CHART_HEIGHT = 700; + export function SeriesVisualization({ seriesValues, referenceSeriesValues = [], labelTemplate, maxSeriesLimit = 500, symmetricalAxes = false, + metricName, + units, }: SimpleSeriesVisualizationProps) { + const { theme } = useTheme(); + const isDark = theme === "dark"; + const hoveredLabelRef = useRef(null); const [hoveredLabel, setHoveredLabel] = useState(null); + const [soloedLabel, setSoloedLabel] = useState(null); + const [legendVisible, setLegendVisible] = useState(true); + const [groupByDimension, setGroupByDimension] = useState(null); + const containerRef = useRef(null); + const crosshairRef = useRef(null); + const [containerWidth, setContainerWidth] = useState(800); + + // Tooltip state + const [tooltipState, setTooltipState] = useState<{ + visible: boolean; + x: number; + y: number; + nearest: NearestResult | null; + allAtX: NearestResult[]; + }>({ visible: false, x: 0, y: 0, nearest: null, allAtX: [] }); + const tooltipRafRef = useRef(0); + + // Observe container width for responsive sizing + useEffect(() => { + const container = containerRef.current; + if (!container) return; + const observer = new ResizeObserver((entries) => { + for (const entry of entries) { + setContainerWidth(entry.contentRect.width); + } + }); + observer.observe(container); + return () => observer.disconnect(); + }, []); - // Create chart data and metadata + // Create chart data and metadata (stable unless data/props change) const { chartData, seriesMetadata, indexName } = useMemo( () => createChartData(seriesValues, referenceSeriesValues, labelTemplate), [seriesValues, referenceSeriesValues, labelTemplate], ); - // State for hidden labels - initialize with all non-reference series hidden - const [hiddenLabels, setHiddenLabels] = useState>(() => { - const hidden = new Set(); - seriesMetadata.forEach((meta) => { - if (!meta.isReference) { - hidden.add(meta.label); + // Available dimension keys for grouping + const availableDimensions = useMemo( + () => getDimensionKeys(seriesMetadata), + [seriesMetadata], + ); + + // Map from label to its dimensions + const labelDimensionMap = useMemo(() => { + const map = new Map>(); + for (const meta of seriesMetadata) { + if (!map.has(meta.label)) { + map.set(meta.label, meta.dimensions); } - }); - return hidden; - }); + } + return map; + }, [seriesMetadata]); + + // State for hidden labels + const [hiddenLabels, setHiddenLabels] = useState>( + () => new Set(), + ); - // Get unique labels with their colors and counts, sorted alphabetically with Reference at top + // Get unique labels with their colors and counts const uniqueLabels = useMemo(() => { const labelMap = new Map< string, { color: string; count: number; isReference: boolean } >(); - seriesMetadata.forEach((meta) => { + for (const meta of seriesMetadata) { const existing = labelMap.get(meta.label); if (existing) { existing.count += 1; @@ -65,7 +111,7 @@ export function SeriesVisualization({ isReference: meta.isReference, }); } - }); + } return Array.from(labelMap.entries()) .map(([label, { color, count, isReference }]) => ({ label, @@ -74,54 +120,189 @@ export function SeriesVisualization({ isReference, })) .sort((a, b) => { - // Reference series always at the top if (a.isReference && !b.isReference) return -1; if (!a.isReference && b.isReference) return 1; - // Otherwise alphabetical return a.label.localeCompare(b.label); }); }, [seriesMetadata]); + // Effective hidden labels accounting for solo mode + const effectiveHiddenLabels = useMemo(() => { + if (!soloedLabel) return hiddenLabels; + const hidden = new Set(); + for (const item of uniqueLabels) { + if (item.label !== soloedLabel && !item.isReference) { + hidden.add(item.label); + } + } + return hidden; + }, [soloedLabel, hiddenLabels, uniqueLabels]); + + // Compute scales + const { xScale, yScale, innerWidth, innerHeight, margins } = useChartScales( + chartData, + seriesMetadata, + effectiveHiddenLabels, + indexName, + containerWidth, + CHART_HEIGHT, + symmetricalAxes, + ); + + // Spatial index for fast nearest-point lookup + const spatialIndex = useSpatialIndex( + chartData, + seriesMetadata, + indexName, + xScale, + yScale, + ); + // Toggle label visibility - const toggleLabel = useCallback((label: string) => { - setHiddenLabels((prev) => { - const newSet = new Set(prev); - if (newSet.has(label)) { - newSet.delete(label); - } else { - newSet.add(label); + const toggleLabel = useCallback( + (label: string) => { + if (soloedLabel) { + setSoloedLabel(null); + return; } - return newSet; - }); + setHiddenLabels((prev) => { + const newSet = new Set(prev); + if (newSet.has(label)) { + newSet.delete(label); + } else { + newSet.add(label); + } + return newSet; + }); + }, + [soloedLabel], + ); + + // Solo mode + const handleSolo = useCallback((label: string) => { + setSoloedLabel((prev) => (prev === label ? null : label)); + }, []); + + // Bulk actions + const showAll = useCallback(() => { + setSoloedLabel(null); + setHiddenLabels(new Set()); }, []); - // Handle label hover + const hideAll = useCallback(() => { + setSoloedLabel(null); + const allNonRef = new Set(); + for (const item of uniqueLabels) { + if (!item.isReference) { + allNonRef.add(item.label); + } + } + setHiddenLabels(allNonRef); + }, [uniqueLabels]); + + // Handle label hover from legend -- triggers canvas redraw via ref, not React re-render of canvas const handleLabelHover = useCallback((label: string | null) => { + hoveredLabelRef.current = label; setHoveredLabel(label); + // Trigger canvas redraw without React re-render + const canvas = containerRef.current?.querySelector("canvas") as + | (HTMLCanvasElement & { redraw?: () => void }) + | null; + canvas?.redraw?.(); }, []); - // Calculate Y domain from all data values for intelligent tick formatting - const yDomain = useMemo(() => { - const allValues = chartData.flatMap((d) => - Object.values(d).filter((v) => typeof v === "number"), - ) as number[]; + // Canvas mouse handlers + const handleCanvasMouseMove = useCallback( + (e: React.MouseEvent) => { + // Capture values synchronously — React pools events, so currentTarget + // and coordinates become null inside requestAnimationFrame. + const canvas = e.currentTarget as HTMLCanvasElement & { + redraw?: () => void; + }; + const rect = canvas.getBoundingClientRect(); + const mouseX = e.clientX - rect.left - margins.left; + const mouseY = e.clientY - rect.top - margins.top; - if (allValues.length === 0) return [0, 1]; + cancelAnimationFrame(tooltipRafRef.current); + tooltipRafRef.current = requestAnimationFrame(() => { + const nearest = spatialIndex.findNearest( + mouseX, + mouseY, + effectiveHiddenLabels, + ); - if (symmetricalAxes) { - const maxAbs = Math.max(...allValues.map(Math.abs)); - return [-maxAbs, maxAbs]; - } + if (nearest) { + const allAtX = spatialIndex.findNearestAtX( + mouseX, + mouseY, + effectiveHiddenLabels, + ); + + hoveredLabelRef.current = nearest.metadata.label; - const min = Math.min(...allValues); - const max = Math.max(...allValues); - return [min, max]; - }, [chartData, symmetricalAxes]); + // Set crosshair at the snapped X data position + crosshairRef.current = { + dataPixelX: nearest.point.pixelX, + nearestPixelX: nearest.point.pixelX, + nearestPixelY: nearest.point.pixelY, + nearestColor: nearest.metadata.isReference + ? "#000000" + : nearest.metadata.color, + }; - // Create intelligent tick formatter based on data range - const tickFormatter = useMemo( - () => createScaledTickFormatter(yDomain), - [yDomain], + canvas.redraw?.(); + + setTooltipState({ + visible: true, + x: margins.left + nearest.point.pixelX, + y: margins.top + nearest.point.pixelY, + nearest, + allAtX, + }); + } else { + crosshairRef.current = null; + if (hoveredLabelRef.current !== null) { + hoveredLabelRef.current = null; + } + canvas.redraw?.(); + setTooltipState((prev) => + prev.visible + ? { visible: false, x: 0, y: 0, nearest: null, allAtX: [] } + : prev, + ); + } + }); + }, + [spatialIndex, effectiveHiddenLabels, margins], + ); + + const handleCanvasMouseLeave = useCallback(() => { + cancelAnimationFrame(tooltipRafRef.current); + hoveredLabelRef.current = null; + crosshairRef.current = null; + setTooltipState({ visible: false, x: 0, y: 0, nearest: null, allAtX: [] }); + const canvas = containerRef.current?.querySelector("canvas") as + | (HTMLCanvasElement & { redraw?: () => void }) + | null; + canvas?.redraw?.(); + }, []); + + const handleCanvasClick = useCallback( + (e: React.MouseEvent) => { + const rect = e.currentTarget.getBoundingClientRect(); + const mouseX = e.clientX - rect.left - margins.left; + const mouseY = e.clientY - rect.top - margins.top; + + const nearest = spatialIndex.findNearest( + mouseX, + mouseY, + effectiveHiddenLabels, + ); + if (nearest) { + handleSolo(nearest.metadata.label); + } + }, + [spatialIndex, effectiveHiddenLabels, handleSolo, margins], ); // Performance safeguard @@ -159,129 +340,70 @@ export function SeriesVisualization({ return (
{/* Chart */} -
- - +
+ +
+
+ + +
{/* Legend Sidebar */} - + {legendVisible && ( + + )}
); } diff --git a/frontend/src/components/execution/values/series/useChartScales.test.ts b/frontend/src/components/execution/values/series/useChartScales.test.ts new file mode 100644 index 0000000..1d0b329 --- /dev/null +++ b/frontend/src/components/execution/values/series/useChartScales.test.ts @@ -0,0 +1,212 @@ +import * as d3 from "d3"; +import { describe, expect, it } from "vitest"; +import type { SeriesMetadata } from "../types"; + +// Test the scale computation logic directly (extracted from the hook) +// since hooks require a React test environment + +interface ChartDataPoint { + [key: string]: number | string | null; +} + +function computeScales( + chartData: ChartDataPoint[], + seriesMetadata: SeriesMetadata[], + hiddenLabels: Set, + indexName: string, + width: number, + height: number, + symmetricalAxes: boolean, + margins = { top: 10, right: 30, bottom: 60, left: 70 }, +) { + const innerWidth = Math.max(0, width - margins.left - margins.right); + const innerHeight = Math.max(0, height - margins.top - margins.bottom); + + const xValues = chartData + .map((d) => d[indexName]) + .filter((v): v is number => typeof v === "number"); + + const xDomain: [number, number] = + xValues.length > 0 + ? [d3.min(xValues) as number, d3.max(xValues) as number] + : [0, 1]; + + const visibleKeys = new Set(); + for (const meta of seriesMetadata) { + if (!hiddenLabels.has(meta.label)) { + visibleKeys.add(`series_${meta.seriesIndex}`); + } + } + + const yValues: number[] = []; + for (const d of chartData) { + for (const key of visibleKeys) { + const v = d[key]; + if (typeof v === "number" && Number.isFinite(v)) { + yValues.push(v); + } + } + } + + let yDomain: [number, number]; + if (yValues.length === 0) { + yDomain = [0, 1]; + } else if (symmetricalAxes) { + const maxAbs = d3.max(yValues.map(Math.abs)) as number; + yDomain = [-maxAbs, maxAbs]; + } else { + const yMin = d3.min(yValues) as number; + const yMax = d3.max(yValues) as number; + const padding = (yMax - yMin) * 0.05 || 0.1; + yDomain = [yMin - padding, yMax + padding]; + } + + const xScale = d3.scaleLinear().domain(xDomain).range([0, innerWidth]); + const yScale = d3.scaleLinear().domain(yDomain).range([innerHeight, 0]); + + return { xScale, yScale, xDomain, yDomain, innerWidth, innerHeight, margins }; +} + +const meta: SeriesMetadata[] = [ + { + seriesIndex: 0, + label: "ModelA", + color: "#4e79a7", + isReference: false, + dimensions: { source_id: "ModelA" }, + }, + { + seriesIndex: 1, + label: "ModelB", + color: "#f28e2b", + isReference: false, + dimensions: { source_id: "ModelB" }, + }, +]; + +const chartData: ChartDataPoint[] = [ + { year: 2020, series_0: 1.0, series_1: -2.0 }, + { year: 2021, series_0: 3.0, series_1: -1.0 }, + { year: 2022, series_0: 2.0, series_1: 0.5 }, +]; + +describe("computeScales", () => { + it("computes correct X domain from index values", () => { + const { xDomain } = computeScales( + chartData, + meta, + new Set(), + "year", + 800, + 600, + false, + ); + expect(xDomain).toEqual([2020, 2022]); + }); + + it("computes Y domain from visible series only", () => { + const hidden = new Set(["ModelB"]); + const { yDomain } = computeScales( + chartData, + meta, + hidden, + "year", + 800, + 600, + false, + ); + // Only ModelA values: [1, 3, 2], min=1, max=3, padding = 0.1 + expect(yDomain[0]).toBeCloseTo(0.9, 1); + expect(yDomain[1]).toBeCloseTo(3.1, 1); + }); + + it("returns fallback Y domain when all series hidden", () => { + const allHidden = new Set(["ModelA", "ModelB"]); + const { yDomain } = computeScales( + chartData, + meta, + allHidden, + "year", + 800, + 600, + false, + ); + expect(yDomain).toEqual([0, 1]); + }); + + it("creates symmetrical Y domain when symmetricalAxes is true", () => { + const { yDomain } = computeScales( + chartData, + meta, + new Set(), + "year", + 800, + 600, + true, + ); + // Max abs value is 3.0, so domain should be [-3, 3] + expect(yDomain[0]).toBe(-3); + expect(yDomain[1]).toBe(3); + }); + + it("maps X values to pixel range correctly", () => { + const margins = { top: 10, right: 30, bottom: 60, left: 70 }; + const { xScale, innerWidth } = computeScales( + chartData, + meta, + new Set(), + "year", + 800, + 600, + false, + margins, + ); + expect(innerWidth).toBe(700); + expect(xScale(2020)).toBe(0); + expect(xScale(2022)).toBe(700); + }); + + it("maps Y values with inverted axis (high value = low pixel)", () => { + const { yScale } = computeScales( + chartData, + meta, + new Set(), + "year", + 800, + 600, + false, + ); + const yDomain = yScale.domain(); + // Higher values should map to lower pixel values + expect(yScale(yDomain[1])).toBe(0); + expect(yScale(yDomain[0])).toBe(530); // 600 - 10 - 60 + }); + + it("handles empty chart data gracefully", () => { + const { xDomain, yDomain } = computeScales( + [], + meta, + new Set(), + "year", + 800, + 600, + false, + ); + expect(xDomain).toEqual([0, 1]); + expect(yDomain).toEqual([0, 1]); + }); + + it("handles zero-size dimensions without crashing", () => { + const result = computeScales( + chartData, + meta, + new Set(), + "year", + 70, + 70, + false, + ); + expect(result.innerWidth).toBe(0); + expect(result.innerHeight).toBe(0); + }); +}); diff --git a/frontend/src/components/execution/values/series/useChartScales.ts b/frontend/src/components/execution/values/series/useChartScales.ts new file mode 100644 index 0000000..fb7e84c --- /dev/null +++ b/frontend/src/components/execution/values/series/useChartScales.ts @@ -0,0 +1,107 @@ +import * as d3 from "d3"; +import { useMemo } from "react"; +import type { SeriesMetadata } from "../types"; +import type { ChartDataPoint } from "./utils"; + +export interface ChartMargins { + top: number; + right: number; + bottom: number; + left: number; +} + +export const DEFAULT_MARGINS: ChartMargins = { + top: 10, + right: 30, + bottom: 50, + left: 80, +}; + +export interface ChartScales { + xScale: d3.ScaleLinear; + yScale: d3.ScaleLinear; + xDomain: [number, number]; + yDomain: [number, number]; + innerWidth: number; + innerHeight: number; + margins: ChartMargins; +} + +export function useChartScales( + chartData: ChartDataPoint[], + seriesMetadata: SeriesMetadata[], + hiddenLabels: Set, + indexName: string, + width: number, + height: number, + symmetricalAxes: boolean, + margins: ChartMargins = DEFAULT_MARGINS, +): ChartScales { + return useMemo(() => { + const innerWidth = Math.max(0, width - margins.left - margins.right); + const innerHeight = Math.max(0, height - margins.top - margins.bottom); + + // X domain from index values + const xValues = chartData + .map((d) => d[indexName]) + .filter((v): v is number => typeof v === "number"); + + const xDomain: [number, number] = + xValues.length > 0 + ? [d3.min(xValues) as number, d3.max(xValues) as number] + : [0, 1]; + + // Y domain from visible series only + const visibleKeys = new Set(); + for (const meta of seriesMetadata) { + if (!hiddenLabels.has(meta.label)) { + visibleKeys.add(`series_${meta.seriesIndex}`); + } + } + + const yValues: number[] = []; + for (const d of chartData) { + for (const key of visibleKeys) { + const v = d[key]; + if (typeof v === "number" && Number.isFinite(v)) { + yValues.push(v); + } + } + } + + let yDomain: [number, number]; + if (yValues.length === 0) { + yDomain = [0, 1]; + } else if (symmetricalAxes) { + const maxAbs = d3.max(yValues.map(Math.abs)) as number; + yDomain = [-maxAbs, maxAbs]; + } else { + const yMin = d3.min(yValues) as number; + const yMax = d3.max(yValues) as number; + const padding = (yMax - yMin) * 0.05 || 0.1; + yDomain = [yMin - padding, yMax + padding]; + } + + const xScale = d3.scaleLinear().domain(xDomain).range([0, innerWidth]); + const yScale = d3.scaleLinear().domain(yDomain).range([innerHeight, 0]); + + return { + xScale, + yScale, + xDomain, + yDomain, + innerWidth, + innerHeight, + margins, + }; + }, [ + chartData, + seriesMetadata, + hiddenLabels, + indexName, + width, + height, + symmetricalAxes, + margins, + ]); +} diff --git a/frontend/src/components/execution/values/series/useSpatialIndex.test.ts b/frontend/src/components/execution/values/series/useSpatialIndex.test.ts new file mode 100644 index 0000000..44c73e1 --- /dev/null +++ b/frontend/src/components/execution/values/series/useSpatialIndex.test.ts @@ -0,0 +1,293 @@ +import * as d3 from "d3"; +import { describe, expect, it } from "vitest"; +import type { SeriesMetadata } from "../types"; + +// Test spatial index logic directly (extracted from the hook) + +interface ChartDataPoint { + [key: string]: number | string | null; +} + +interface IndexedPoint { + seriesIndex: number; + dataIndex: number; + x: number; + y: number; + pixelX: number; + pixelY: number; +} + +interface NearestResult { + point: IndexedPoint; + metadata: SeriesMetadata; + distance: number; +} + +function cellKey(cx: number, cy: number): string { + return `${cx},${cy}`; +} + +function buildSpatialIndex( + chartData: ChartDataPoint[], + seriesMetadata: SeriesMetadata[], + indexName: string, + xScale: d3.ScaleLinear, + yScale: d3.ScaleLinear, +) { + const cellSize = 30; + const cells = new Map(); + + for (const meta of seriesMetadata) { + const key = `series_${meta.seriesIndex}`; + for (let i = 0; i < chartData.length; i++) { + const xVal = chartData[i][indexName]; + const yVal = chartData[i][key]; + if ( + typeof xVal !== "number" || + typeof yVal !== "number" || + !Number.isFinite(yVal) + ) { + continue; + } + + const pixelX = xScale(xVal); + const pixelY = yScale(yVal); + const cx = Math.floor(pixelX / cellSize); + const cy = Math.floor(pixelY / cellSize); + const k = cellKey(cx, cy); + + const point: IndexedPoint = { + seriesIndex: meta.seriesIndex, + dataIndex: i, + x: xVal, + y: yVal, + pixelX, + pixelY, + }; + + const bucket = cells.get(k); + if (bucket) { + bucket.push(point); + } else { + cells.set(k, [point]); + } + } + } + + function findNearest( + pixelX: number, + pixelY: number, + hiddenLabels: Set, + maxDistance = 50, + ): NearestResult | null { + const cx = Math.floor(pixelX / cellSize); + const cy = Math.floor(pixelY / cellSize); + const searchRadius = Math.ceil(maxDistance / cellSize); + + let best: NearestResult | null = null; + let bestDist = maxDistance * maxDistance; + + for (let dx = -searchRadius; dx <= searchRadius; dx++) { + for (let dy = -searchRadius; dy <= searchRadius; dy++) { + const bucket = cells.get(cellKey(cx + dx, cy + dy)); + if (!bucket) continue; + + for (const pt of bucket) { + const meta = seriesMetadata[pt.seriesIndex]; + if (!meta || hiddenLabels.has(meta.label)) continue; + + const distSq = (pt.pixelX - pixelX) ** 2 + (pt.pixelY - pixelY) ** 2; + if (distSq < bestDist) { + bestDist = distSq; + best = { point: pt, metadata: meta, distance: Math.sqrt(distSq) }; + } + } + } + } + + return best; + } + + function findNearestAtX( + pixelX: number, + pixelY: number, + hiddenLabels: Set, + ): NearestResult[] { + const xVal = xScale.invert(pixelX); + let closestDataIdx = 0; + let closestXDist = Number.POSITIVE_INFINITY; + for (let i = 0; i < chartData.length; i++) { + const v = chartData[i][indexName]; + if (typeof v !== "number") continue; + const dist = Math.abs(v - xVal); + if (dist < closestXDist) { + closestXDist = dist; + closestDataIdx = i; + } + } + + const results: NearestResult[] = []; + for (const meta of seriesMetadata) { + if (hiddenLabels.has(meta.label)) continue; + const key = `series_${meta.seriesIndex}`; + const yVal = chartData[closestDataIdx]?.[key]; + if (typeof yVal !== "number" || !Number.isFinite(yVal)) continue; + + const ptPixelX = xScale(chartData[closestDataIdx][indexName] as number); + const ptPixelY = yScale(yVal); + const dist = Math.sqrt( + (ptPixelX - pixelX) ** 2 + (ptPixelY - pixelY) ** 2, + ); + + results.push({ + point: { + seriesIndex: meta.seriesIndex, + dataIndex: closestDataIdx, + x: chartData[closestDataIdx][indexName] as number, + y: yVal, + pixelX: ptPixelX, + pixelY: ptPixelY, + }, + metadata: meta, + distance: dist, + }); + } + + results.sort((a, b) => a.distance - b.distance); + return results; + } + + return { findNearest, findNearestAtX }; +} + +const meta: SeriesMetadata[] = [ + { + seriesIndex: 0, + label: "ModelA", + color: "#4e79a7", + isReference: false, + dimensions: { source_id: "ModelA" }, + }, + { + seriesIndex: 1, + label: "ModelB", + color: "#f28e2b", + isReference: false, + dimensions: { source_id: "ModelB" }, + }, + { + seriesIndex: 2, + label: "Reference", + color: "#000000", + isReference: true, + dimensions: { source_id: "Reference" }, + }, +]; + +const chartData: ChartDataPoint[] = [ + { year: 2020, series_0: 1.0, series_1: 5.0, series_2: 3.0 }, + { year: 2021, series_0: 2.0, series_1: 4.0, series_2: 3.0 }, + { year: 2022, series_0: 3.0, series_1: 3.0, series_2: 3.0 }, +]; + +const xScale = d3.scaleLinear().domain([2020, 2022]).range([0, 600]); +const yScale = d3.scaleLinear().domain([0, 6]).range([400, 0]); + +describe("buildSpatialIndex", () => { + it("findNearest returns the closest point to cursor", () => { + const index = buildSpatialIndex(chartData, meta, "year", xScale, yScale); + // Pixel position near the first data point of ModelA (2020, 1.0) + // pixelX = 0, pixelY = yScale(1.0) = 333.33 + const result = index.findNearest(0, 333, new Set()); + expect(result).not.toBeNull(); + expect(result!.metadata.label).toBe("ModelA"); + expect(result!.point.x).toBe(2020); + expect(result!.point.y).toBe(1.0); + }); + + it("findNearest returns null when cursor is too far from any point", () => { + const index = buildSpatialIndex(chartData, meta, "year", xScale, yScale); + // Way outside the chart area + const result = index.findNearest(-500, -500, new Set(), 10); + expect(result).toBeNull(); + }); + + it("findNearest skips hidden series", () => { + const index = buildSpatialIndex(chartData, meta, "year", xScale, yScale); + // At x=2022 (pixel 600), all three series have y=3.0 (pixelY = yScale(3.0) = 200) + // Hide ModelA, cursor right on that shared point + const hidden = new Set(["ModelA"]); + const sharedPixelY = yScale(3.0); + const result = index.findNearest(600, sharedPixelY, hidden); + expect(result).not.toBeNull(); + expect(result!.metadata.label).not.toBe("ModelA"); + }); + + it("findNearest distinguishes between close series by Y proximity", () => { + const index = buildSpatialIndex(chartData, meta, "year", xScale, yScale); + // At x=2022 (pixel 600), ModelA=3.0 and ModelB=3.0 and Ref=3.0 + // They share the same y, so all are at same distance + // At x=2021 (pixel 300), ModelA=2.0 (pixelY=266.67), ModelB=4.0 (pixelY=133.33) + // Cursor near ModelB at 2021 + const modelBPixelY = yScale(4.0); // ~133.33 + const result = index.findNearest(300, modelBPixelY, new Set()); + expect(result).not.toBeNull(); + expect(result!.metadata.label).toBe("ModelB"); + expect(result!.point.y).toBe(4.0); + }); + + it("findNearestAtX returns all visible series at closest X index", () => { + const index = buildSpatialIndex(chartData, meta, "year", xScale, yScale); + // Near x=2021 (pixel ~300) + const results = index.findNearestAtX(300, 200, new Set()); + expect(results).toHaveLength(3); // ModelA, ModelB, Reference + // Should be sorted by distance to cursor Y + const labels = results.map((r) => r.metadata.label); + expect(labels).toContain("ModelA"); + expect(labels).toContain("ModelB"); + expect(labels).toContain("Reference"); + }); + + it("findNearestAtX excludes hidden series", () => { + const index = buildSpatialIndex(chartData, meta, "year", xScale, yScale); + const hidden = new Set(["ModelB"]); + const results = index.findNearestAtX(300, 200, hidden); + const labels = results.map((r) => r.metadata.label); + expect(labels).not.toContain("ModelB"); + expect(results).toHaveLength(2); + }); + + it("findNearestAtX sorts by distance to cursor", () => { + const index = buildSpatialIndex(chartData, meta, "year", xScale, yScale); + // Cursor Y near ModelA value at 2021 (y=2.0, pixelY=266.67) + const cursorY = yScale(2.0); + const results = index.findNearestAtX(300, cursorY, new Set()); + // ModelA (y=2.0) should be closest, then Reference (y=3.0), then ModelB (y=4.0) + expect(results[0].metadata.label).toBe("ModelA"); + expect(results[0].point.y).toBe(2.0); + }); + + it("handles empty chart data", () => { + const index = buildSpatialIndex([], meta, "year", xScale, yScale); + const result = index.findNearest(100, 100, new Set()); + expect(result).toBeNull(); + }); + + it("handles non-finite values in series data", () => { + const dataWithNaN: ChartDataPoint[] = [ + { year: 2020, series_0: Number.NaN, series_1: 5.0, series_2: 3.0 }, + { + year: 2021, + series_0: 2.0, + series_1: Number.POSITIVE_INFINITY, + series_2: 3.0, + }, + ]; + const index = buildSpatialIndex(dataWithNaN, meta, "year", xScale, yScale); + // Should still find valid points + const result = index.findNearest(300, yScale(2.0), new Set()); + expect(result).not.toBeNull(); + // NaN and Infinity points should be excluded + expect(Number.isFinite(result!.point.y)).toBe(true); + }); +}); diff --git a/frontend/src/components/execution/values/series/useSpatialIndex.ts b/frontend/src/components/execution/values/series/useSpatialIndex.ts new file mode 100644 index 0000000..d51ab86 --- /dev/null +++ b/frontend/src/components/execution/values/series/useSpatialIndex.ts @@ -0,0 +1,201 @@ +import type * as d3 from "d3"; +import { useMemo } from "react"; +import type { SeriesMetadata } from "../types"; +import type { ChartDataPoint } from "./utils"; + +export interface IndexedPoint { + seriesIndex: number; + dataIndex: number; + x: number; + y: number; + pixelX: number; + pixelY: number; +} + +interface SpatialGrid { + cells: Map; + cellSize: number; +} + +function cellKey(cx: number, cy: number): string { + return `${cx},${cy}`; +} + +function buildGrid( + chartData: ChartDataPoint[], + seriesMetadata: SeriesMetadata[], + indexName: string, + xScale: d3.ScaleLinear, + yScale: d3.ScaleLinear, + cellSize: number, +): SpatialGrid { + const cells = new Map(); + + for (const meta of seriesMetadata) { + const key = `series_${meta.seriesIndex}`; + for (let i = 0; i < chartData.length; i++) { + const xVal = chartData[i][indexName]; + const yVal = chartData[i][key]; + if ( + typeof xVal !== "number" || + typeof yVal !== "number" || + !Number.isFinite(yVal) + ) { + continue; + } + + const pixelX = xScale(xVal); + const pixelY = yScale(yVal); + const cx = Math.floor(pixelX / cellSize); + const cy = Math.floor(pixelY / cellSize); + const k = cellKey(cx, cy); + + const point: IndexedPoint = { + seriesIndex: meta.seriesIndex, + dataIndex: i, + x: xVal, + y: yVal, + pixelX, + pixelY, + }; + + const bucket = cells.get(k); + if (bucket) { + bucket.push(point); + } else { + cells.set(k, [point]); + } + } + } + + return { cells, cellSize }; +} + +export interface NearestResult { + point: IndexedPoint; + metadata: SeriesMetadata; + distance: number; +} + +export interface SpatialIndex { + findNearest: ( + pixelX: number, + pixelY: number, + hiddenLabels: Set, + maxDistance?: number, + ) => NearestResult | null; + findNearestAtX: ( + pixelX: number, + pixelY: number, + hiddenLabels: Set, + ) => NearestResult[]; +} + +export function useSpatialIndex( + chartData: ChartDataPoint[], + seriesMetadata: SeriesMetadata[], + indexName: string, + xScale: d3.ScaleLinear, + yScale: d3.ScaleLinear, +): SpatialIndex { + return useMemo(() => { + const cellSize = 30; + const grid = buildGrid( + chartData, + seriesMetadata, + indexName, + xScale, + yScale, + cellSize, + ); + + function findNearest( + pixelX: number, + pixelY: number, + hiddenLabels: Set, + maxDistance = 50, + ): NearestResult | null { + const cx = Math.floor(pixelX / cellSize); + const cy = Math.floor(pixelY / cellSize); + const searchRadius = Math.ceil(maxDistance / cellSize); + + let best: NearestResult | null = null; + let bestDist = maxDistance * maxDistance; + + for (let dx = -searchRadius; dx <= searchRadius; dx++) { + for (let dy = -searchRadius; dy <= searchRadius; dy++) { + const bucket = grid.cells.get(cellKey(cx + dx, cy + dy)); + if (!bucket) continue; + + for (const pt of bucket) { + const meta = seriesMetadata[pt.seriesIndex]; + if (!meta || hiddenLabels.has(meta.label)) continue; + + const distSq = + (pt.pixelX - pixelX) ** 2 + (pt.pixelY - pixelY) ** 2; + if (distSq < bestDist) { + bestDist = distSq; + best = { point: pt, metadata: meta, distance: Math.sqrt(distSq) }; + } + } + } + } + + return best; + } + + function findNearestAtX( + pixelX: number, + pixelY: number, + hiddenLabels: Set, + ): NearestResult[] { + // Find the closest data index by X pixel position + const xVal = xScale.invert(pixelX); + let closestDataIdx = 0; + let closestXDist = Number.POSITIVE_INFINITY; + for (let i = 0; i < chartData.length; i++) { + const v = chartData[i][indexName]; + if (typeof v !== "number") continue; + const dist = Math.abs(v - xVal); + if (dist < closestXDist) { + closestXDist = dist; + closestDataIdx = i; + } + } + + // Collect all visible series values at this data index + const results: NearestResult[] = []; + for (const meta of seriesMetadata) { + if (hiddenLabels.has(meta.label)) continue; + const key = `series_${meta.seriesIndex}`; + const yVal = chartData[closestDataIdx]?.[key]; + if (typeof yVal !== "number" || !Number.isFinite(yVal)) continue; + + const ptPixelX = xScale(chartData[closestDataIdx][indexName] as number); + const ptPixelY = yScale(yVal); + const dist = Math.sqrt( + (ptPixelX - pixelX) ** 2 + (ptPixelY - pixelY) ** 2, + ); + + results.push({ + point: { + seriesIndex: meta.seriesIndex, + dataIndex: closestDataIdx, + x: chartData[closestDataIdx][indexName] as number, + y: yVal, + pixelX: ptPixelX, + pixelY: ptPixelY, + }, + metadata: meta, + distance: dist, + }); + } + + // Sort by distance to cursor Y + results.sort((a, b) => a.distance - b.distance); + return results; + } + + return { findNearest, findNearestAtX }; + }, [chartData, seriesMetadata, indexName, xScale, yScale]); +} diff --git a/frontend/src/components/execution/values/series/utils.test.ts b/frontend/src/components/execution/values/series/utils.test.ts index 9629d68..caace13 100644 --- a/frontend/src/components/execution/values/series/utils.test.ts +++ b/frontend/src/components/execution/values/series/utils.test.ts @@ -1,6 +1,10 @@ import { describe, expect, it } from "vitest"; import type { SeriesValue } from "../types"; -import { createChartData, createScaledTickFormatter } from "./utils"; +import { + createChartData, + createScaledTickFormatter, + getDimensionKeys, +} from "./utils"; const mockSeriesValue: SeriesValue = { id: 1, @@ -100,6 +104,15 @@ describe("createChartData", () => { expect(result.seriesMetadata[0].color).not.toBe("#000000"); }); + it("includes dimensions in series metadata", () => { + const result = createChartData([mockSeriesValue], []); + expect(result.seriesMetadata[0].dimensions).toEqual({ + source_id: "ModelA", + experiment_id: "exp1", + metric: "rmse", + }); + }); + it("creates reference series metadata with black color", () => { const result = createChartData([], [mockReferenceSeriesValue]); expect(result.seriesMetadata).toHaveLength(1); @@ -119,6 +132,57 @@ describe("createChartData", () => { expect(result.seriesMetadata[1].color).toBe("#000000"); }); + it("deduplicates reference series with the same label", () => { + const refSeries2: SeriesValue = { + id: 10, + execution_group_id: 101, + execution_id: 210, + dimensions: { + source_id: "Reference", + experiment_id: "exp1", + metric: "rmse", + }, + values: [1.5, 2.5, 3.5], + index: [2020, 2021, 2022], + index_name: "year", + }; + const result = createChartData( + [mockSeriesValue], + [mockReferenceSeriesValue, refSeries2], + ); + // Should only have 2 series: 1 regular + 1 deduplicated reference + expect(result.seriesMetadata).toHaveLength(2); + expect(result.seriesMetadata[0].isReference).toBe(false); + expect(result.seriesMetadata[1].isReference).toBe(true); + }); + + it("keeps reference series with different labels", () => { + const refSeriesDifferentMetric: SeriesValue = { + id: 11, + execution_group_id: 101, + execution_id: 211, + dimensions: { + source_id: "Reference", + experiment_id: "exp1", + metric: "bias", + }, + values: [0.1, 0.2, 0.3], + index: [2020, 2021, 2022], + index_name: "year", + }; + const result = createChartData( + [mockSeriesValue], + [mockReferenceSeriesValue, refSeriesDifferentMetric], + ); + // Should have 3 series: 1 regular + 2 distinct reference + expect(result.seriesMetadata).toHaveLength(3); + expect(result.seriesMetadata[1].isReference).toBe(true); + expect(result.seriesMetadata[2].isReference).toBe(true); + expect(result.seriesMetadata[1].label).not.toBe( + result.seriesMetadata[2].label, + ); + }); + it("uses index name from first series", () => { const customSeries: SeriesValue = { ...mockSeriesValue, @@ -253,3 +317,53 @@ describe("createChartData", () => { expect(result.seriesMetadata).toHaveLength(2); }); }); + +describe("getDimensionKeys", () => { + it("returns empty array for empty metadata", () => { + expect(getDimensionKeys([])).toEqual([]); + }); + + it("collects unique dimension keys sorted alphabetically", () => { + const metadata = [ + { + seriesIndex: 0, + label: "A", + color: "#000", + isReference: false, + dimensions: { source_id: "ModelA", experiment_id: "exp1" }, + }, + { + seriesIndex: 1, + label: "B", + color: "#000", + isReference: false, + dimensions: { source_id: "ModelB", metric: "rmse" }, + }, + ]; + expect(getDimensionKeys(metadata)).toEqual([ + "experiment_id", + "metric", + "source_id", + ]); + }); + + it("deduplicates keys across series", () => { + const metadata = [ + { + seriesIndex: 0, + label: "A", + color: "#000", + isReference: false, + dimensions: { source_id: "ModelA" }, + }, + { + seriesIndex: 1, + label: "B", + color: "#000", + isReference: false, + dimensions: { source_id: "ModelB" }, + }, + ]; + expect(getDimensionKeys(metadata)).toEqual(["source_id"]); + }); +}); diff --git a/frontend/src/components/execution/values/series/utils.ts b/frontend/src/components/execution/values/series/utils.ts index c40bb50..7e6720d 100644 --- a/frontend/src/components/execution/values/series/utils.ts +++ b/frontend/src/components/execution/values/series/utils.ts @@ -1,9 +1,20 @@ import type { SeriesMetadata, SeriesValue } from "../types"; -interface ChartData { +export interface ChartDataPoint { [key: string]: number | string | null; } +export function isIntegerAxis(indexName: string): boolean { + const lower = indexName.toLowerCase(); + return ( + lower.includes("year") || + lower.includes("month") || + lower.includes("day") || + lower === "time" || + lower === "index" + ); +} + export function createScaledTickFormatter( values: number[], ): (value: number | string) => string { @@ -77,50 +88,64 @@ function applyLabelTemplate(series: SeriesValue, template?: string): string { return result; } +// 20-color perceptually distinct palette const COLORS = [ - "#8884d8", - "#82ca9d", - "#ffc658", - "#ff7300", - "#00ff00", - "#0088fe", - "#00c49f", - "#ffbb28", - "#ff8042", - "#8dd1e1", + "#4e79a7", + "#f28e2b", + "#e15759", + "#76b7b2", + "#59a14f", + "#edc948", + "#b07aa1", + "#ff9da7", + "#9c755f", + "#bab0ac", + "#af7aa1", + "#d37295", + "#1b9e77", + "#d95f02", + "#7570b3", + "#e7298a", + "#66a61e", + "#e6ab02", + "#a6761d", + "#666666", ]; /** * Get consistent color for a label using hash-based assignment - * @param label - The label string - * @returns Hex color code */ function getLabelColor(label: string): string { const hash = label.split("").reduce((acc, char) => { const newAcc = (acc << 5) - acc + char.charCodeAt(0); - return newAcc & newAcc; + return newAcc | 0; }, 0); return COLORS[Math.abs(hash) % COLORS.length]; } /** * Create chart data structure with all series - * @param seriesValues - Array of regular series values - * @param referenceSeriesValues - Array of reference series values - * @param labelTemplate - Optional template for generating labels - * @returns Chart data and series metadata */ export function createChartData( seriesValues: SeriesValue[], referenceSeriesValues: SeriesValue[], labelTemplate?: string, ): { - chartData: ChartData[]; + chartData: ChartDataPoint[]; seriesMetadata: SeriesMetadata[]; indexName: string; } { - // Combine regular and reference series - const allSeries = [...seriesValues, ...referenceSeriesValues]; + // Deduplicate reference series by label (same observational data repeated across executions) + const seenRefLabels = new Set(); + const dedupedReferenceSeries = referenceSeriesValues.filter((series) => { + const label = applyLabelTemplate(series, labelTemplate); + if (seenRefLabels.has(label)) return false; + seenRefLabels.add(label); + return true; + }); + + // Combine regular and deduplicated reference series + const allSeries = [...seriesValues, ...dedupedReferenceSeries]; if (allSeries.length === 0) { return { chartData: [], seriesMetadata: [], indexName: "index" }; @@ -135,20 +160,21 @@ export function createChartData( // Create series metadata const seriesMetadata: SeriesMetadata[] = allSeries.map((series, idx) => { const label = applyLabelTemplate(series, labelTemplate); - const isReference = idx >= seriesValues.length; // Reference series come after regular series + const isReference = idx >= seriesValues.length; // Deduplicated reference series come after regular series const color = isReference ? "#000000" : getLabelColor(label); return { seriesIndex: idx, label, color, isReference, + dimensions: series.dimensions, }; }); // Build chart data - const chartData: ChartData[] = []; + const chartData: ChartDataPoint[] = []; for (let i = 0; i < maxLength; i++) { - const dataPoint: ChartData = { + const dataPoint: ChartDataPoint = { [indexName]: allSeries[0]?.index?.[i] ?? i, }; @@ -163,3 +189,16 @@ export function createChartData( return { chartData, seriesMetadata, indexName }; } + +/** + * Collect all unique dimension keys from series metadata + */ +export function getDimensionKeys(seriesMetadata: SeriesMetadata[]): string[] { + const keys = new Set(); + for (const meta of seriesMetadata) { + for (const key of Object.keys(meta.dimensions)) { + keys.add(key); + } + } + return Array.from(keys).sort(); +} diff --git a/frontend/src/components/explorer/content/seriesChartContent.tsx b/frontend/src/components/explorer/content/seriesChartContent.tsx index b799506..7199c58 100644 --- a/frontend/src/components/explorer/content/seriesChartContent.tsx +++ b/frontend/src/components/explorer/content/seriesChartContent.tsx @@ -68,6 +68,8 @@ export function SeriesChartContent({ contentItem }: SeriesChartContentProps) { maxSeriesLimit={500} // Limit for performance in preview symmetricalAxes={contentItem.symmetricalAxes ?? false} labelTemplate={contentItem.labelTemplate} + metricName={contentItem.title} + units={contentItem.metricUnits} /> ); } From 8cecc75d5eec142a786e1d8379423ee3ea554658 Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Sat, 7 Mar 2026 12:35:48 +1100 Subject: [PATCH 09/15] refactor(explorer): rename ExplorerCard to ExplorerCardGroup and improve layout Rename component to better reflect its role as a group of content items. Add section dividers between collection groups, constrain text width with max-w-prose, and add horizontal margin to the explorer layout. --- .../src/components/explorer/explorerCard.tsx | 98 ------------------- .../components/explorer/explorerCardGroup.tsx | 28 ++++++ .../explorer/explorerThemeLayout.tsx | 19 ++-- frontend/src/routes/_app/explorer/route.tsx | 18 ++-- 4 files changed, 46 insertions(+), 117 deletions(-) delete mode 100644 frontend/src/components/explorer/explorerCard.tsx create mode 100644 frontend/src/components/explorer/explorerCardGroup.tsx diff --git a/frontend/src/components/explorer/explorerCard.tsx b/frontend/src/components/explorer/explorerCard.tsx deleted file mode 100644 index 2bdcdf8..0000000 --- a/frontend/src/components/explorer/explorerCard.tsx +++ /dev/null @@ -1,98 +0,0 @@ -import { Check, Link2 } from "lucide-react"; -import { Suspense, useCallback, useState } from "react"; -import { Button } from "@/components/ui/button"; -import { - Card, - CardContent, - CardDescription, - CardHeader, - CardTitle, -} from "@/components/ui/card"; -import { cn } from "@/lib/utils"; -import { ErrorBoundary, ErrorFallback } from "../app"; -import { - ExplorerCardContent, - ExplorerCardContentSkeleton, -} from "./explorerCardContent"; -import type { ExplorerCard as ExplorerCardType } from "./types"; - -// import { ExplorerTooltip } from "./explorerTooltip"; - -interface ExplorerCardProps { - card: ExplorerCardType; -} - -// Generate a URL-friendly ID from the card title -function generateCardId(title: string): string { - return title - .toLowerCase() - .replace(/[^a-z0-9]+/g, "-") - .replace(/^-|-$/g, ""); -} - -// The ExplorerCard component renders a card with a header and content area. -// Each card may contain multiple content items, which are rendered using the ExplorerCardContent component. -export function ExplorerCard({ card }: ExplorerCardProps) { - const cardId = generateCardId(card.title); - const [copied, setCopied] = useState(false); - - const handleCopyLink = useCallback(() => { - // Update the URL in the browser - const newUrl = `${window.location.pathname}${window.location.search}#${cardId}`; - window.history.pushState(null, "", newUrl); - - // Copy the full URL to clipboard - const fullUrl = `${window.location.origin}${newUrl}`; - navigator.clipboard.writeText(fullUrl).then(() => { - setCopied(true); - setTimeout(() => setCopied(false), 2000); - }); - }, [cardId]); - - return ( - - -
- - {card.placeholder && "PLACEHOLDER:"} {card.title} - - -
-
- {card.description && ( - {card.description} - )} -
- - -
- {card.content.map((contentItem) => ( - } - > - }> - - - - ))} -
- - - ); -} diff --git a/frontend/src/components/explorer/explorerCardGroup.tsx b/frontend/src/components/explorer/explorerCardGroup.tsx new file mode 100644 index 0000000..cb65d8c --- /dev/null +++ b/frontend/src/components/explorer/explorerCardGroup.tsx @@ -0,0 +1,28 @@ +import { Suspense } from "react"; +import { ErrorBoundary, ErrorFallback } from "../app"; +import { + ExplorerCardContent, + ExplorerCardContentSkeleton, +} from "./explorerCardContent"; +import type { ExplorerCard as ExplorerCardType } from "./types"; + +interface ExplorerCardGroupProps { + card: ExplorerCardType; +} + +export function ExplorerCardGroup({ card }: ExplorerCardGroupProps) { + return ( +
+ {card.content.map((contentItem) => ( + } + > + }> + + + + ))} +
+ ); +} diff --git a/frontend/src/components/explorer/explorerThemeLayout.tsx b/frontend/src/components/explorer/explorerThemeLayout.tsx index 206bc7e..13b0952 100644 --- a/frontend/src/components/explorer/explorerThemeLayout.tsx +++ b/frontend/src/components/explorer/explorerThemeLayout.tsx @@ -1,16 +1,10 @@ import { useEffect } from "react"; import type { AftCollectionDetail } from "@/client/types.gen"; import { CollectionHeader } from "@/components/explorer/collectionHeader"; -import { ExplorerCard } from "@/components/explorer/explorerCard"; +import { ExplorerCardGroup } from "@/components/explorer/explorerCardGroup"; import type { ExplorerCard as ExplorerCardType } from "@/components/explorer/types"; import { TooltipProvider } from "@/components/ui/tooltip.tsx"; -// Re-export types for backward compatibility -export type { - ExplorerCard as ExplorerCardType, - ExplorerCardContent, -} from "@/components/explorer/types"; - interface CollectionGroup { collection: AftCollectionDetail; cards: ExplorerCardType[]; @@ -40,9 +34,12 @@ export const ExplorerThemeLayout = ({ return ( -
- {collectionGroups.map((group) => ( -
+
+ {collectionGroups.map((group, index) => ( +
0 ? "border-t pt-12" : ""} + >
{group.cards.map((card) => (
- +
))}
diff --git a/frontend/src/routes/_app/explorer/route.tsx b/frontend/src/routes/_app/explorer/route.tsx index 72b716e..98c3bdb 100644 --- a/frontend/src/routes/_app/explorer/route.tsx +++ b/frontend/src/routes/_app/explorer/route.tsx @@ -5,14 +5,16 @@ import DataHealthWarning from "@/content/data-health-warning.mdx"; const ExplorerLayout = () => { return ( -
+
-

Data Explorer

-

- Explore and visualize climate model evaluation diagnostics across - different Earth system components and scientific themes. -

+
+

Data Explorer

+

+ Explore and visualize climate model evaluation diagnostics across + different Earth system components and scientific themes. +

+
{ return (

); }, }} /> -

+

What's Available From 54355e7899ff53ba39d78e9ff903eec0a2f98779 Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Sat, 7 Mar 2026 13:04:24 +1100 Subject: [PATCH 10/15] feat(explorer): sort box-whisker chart categories by known orderings Auto-detect and apply well-known category orderings (e.g., seasons DJF, MAM, JJA, SON) to preserve meaningful x-axis order in ensemble charts. Also adds an explicit categoryOrder prop for custom orderings. --- .../components/diagnostics/ensembleChart.tsx | 66 +++++++++++++++++-- 1 file changed, 59 insertions(+), 7 deletions(-) diff --git a/frontend/src/components/diagnostics/ensembleChart.tsx b/frontend/src/components/diagnostics/ensembleChart.tsx index 475db20..d0d6f4f 100644 --- a/frontend/src/components/diagnostics/ensembleChart.tsx +++ b/frontend/src/components/diagnostics/ensembleChart.tsx @@ -22,6 +22,48 @@ import { import useMousePositionAndWidth from "@/hooks/useMousePosition"; import { createScaledTickFormatter } from "../execution/values/series/utils"; +// Well-known category orderings for common climate dimensions +const KNOWN_CATEGORY_ORDERS: Record = { + // Meteorological seasons + season: ["DJF", "MAM", "JJA", "SON"], +}; + +/** + * Sort chart categories using a known ordering if one exists, + * otherwise preserve the original order. + */ +function sortCategories( + items: T[], + categoryOrder?: string[], +): T[] { + const order = categoryOrder; + if (!order) { + // Auto-detect: check if all category names match a known ordering + const names = new Set(items.map((item) => item.name)); + for (const knownOrder of Object.values(KNOWN_CATEGORY_ORDERS)) { + if ( + names.size <= knownOrder.length && + [...names].every((n) => knownOrder.includes(n)) + ) { + return [...items].sort( + (a, b) => knownOrder.indexOf(a.name) - knownOrder.indexOf(b.name), + ); + } + } + return items; + } + + return [...items].sort((a, b) => { + const aIdx = order.indexOf(a.name); + const bIdx = order.indexOf(b.name); + // Items not in the order go to the end, preserving relative order + if (aIdx === -1 && bIdx === -1) return 0; + if (aIdx === -1) return 1; + if (bIdx === -1) return -1; + return aIdx - bIdx; + }); +} + // Color palette for different groups const COLORS = [ "#8884d8", @@ -51,6 +93,8 @@ interface EnsembleChartProps { symmetricalAxes?: boolean; yMin?: number; yMax?: number; + /** Explicit category ordering for the x-axis (e.g., ["DJF", "MAM", "JJA", "SON"]) */ + categoryOrder?: string[]; } interface GroupStatistics { @@ -93,6 +137,7 @@ export const EnsembleChart = ({ symmetricalAxes, yMin, yMax, + categoryOrder, }: EnsembleChartProps) => { const { mousePosition, windowSize } = useMousePositionAndWidth(); const [highlightedPoint, setHighlightedPoint] = useState<{ @@ -208,19 +253,22 @@ export const EnsembleChart = ({ }, ); + // Sort categories using explicit order or well-known orderings (e.g., seasons) + const sortedChartData = sortCategories(chartData, categoryOrder); + // Get all unique group names for rendering multiple bars const allGroupNames = useMemo(() => { const names = new Set(); - chartData.forEach((d) => { + sortedChartData.forEach((d) => { Object.keys(d.groups).forEach((groupName) => { names.add(groupName); }); }); return Array.from(names).sort(); - }, [chartData]); + }, [sortedChartData]); const scale = useMemo(() => { - const allFiniteValues = chartData + const allFiniteValues = sortedChartData .flatMap((d) => Object.values(d.groups) .filter((group) => group !== null) @@ -261,7 +309,7 @@ export const EnsembleChart = ({ yMax !== undefined ? yMax : maxVal + padding, ]) .nice(); - }, [chartData, symmetricalAxes, yMin, yMax]); + }, [sortedChartData, symmetricalAxes, yMin, yMax]); const yDomain = scale.domain() as [number, number]; // Get color for a group @@ -272,18 +320,22 @@ export const EnsembleChart = ({ const fmt = valueFormatter ?? createScaledTickFormatter(yDomain); // Hide x-axis when there are too many items (threshold: 6) - const shouldShowXAxis = chartData.length <= 6; + const shouldShowXAxis = sortedChartData.length <= 6; // Adjust spacing based on number of items // make them closer together when there are many const barCategoryGap = - chartData.length > 20 ? "2%" : chartData.length > 10 ? "5%" : "20%"; + sortedChartData.length > 20 + ? "2%" + : sortedChartData.length > 10 + ? "5%" + : "20%"; return (
From 4c0394aa91f70be9a23d957a5678af90bf5ca5c6 Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Sat, 7 Mar 2026 16:46:30 +1100 Subject: [PATCH 11/15] refactor(diagnostics): remove dead code and add memoization to figure gallery - Remove commented-out CardTemplateGenerator blocks and unused imports from scalars.tsx and series.tsx - Remove unused destructured variables (currentFilters, currentGroupingConfig, isolateIds, excludeIds) from both files - Add useMemo for selectorDimensions, allFigures, and filteredFigures to avoid recomputation on unrelated renders - Hoist regex compilation out of per-figure filter loop --- .../components/diagnostics/figureGallery.tsx | 168 ++++++++++++------ .../scalars.tsx | 45 +---- .../series.tsx | 48 +---- 3 files changed, 131 insertions(+), 130 deletions(-) diff --git a/frontend/src/components/diagnostics/figureGallery.tsx b/frontend/src/components/diagnostics/figureGallery.tsx index 80f74c5..427d892 100644 --- a/frontend/src/components/diagnostics/figureGallery.tsx +++ b/frontend/src/components/diagnostics/figureGallery.tsx @@ -2,7 +2,7 @@ import { useQuery } from "@tanstack/react-query"; import { Link } from "@tanstack/react-router"; import { useWindowVirtualizer } from "@tanstack/react-virtual"; import { Download, ExternalLink, MoreHorizontal } from "lucide-react"; -import { useCallback, useEffect, useState } from "react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import type { ExecutionGroup, ExecutionOutput } from "@/client"; import { diagnosticsListExecutionGroupsOptions } from "@/client/@tanstack/react-query.gen.ts"; import { Figure } from "@/components/diagnostics/figure.tsx"; @@ -65,12 +65,39 @@ const FigureDropDown = ({ figure, executionGroup }: FigureWithGroup) => { ); }; +/** + * Extract unique selector dimensions and their values from execution groups. + * Returns only dimensions with more than one unique value (single-value dimensions + * aren't useful as filters). + */ +function extractSelectorDimensions(groups: ExecutionGroup[]) { + const dimensionValues: Record> = {}; + + for (const group of groups) { + for (const pairs of Object.values(group.selectors)) { + for (const [key, value] of pairs) { + if (!dimensionValues[key]) dimensionValues[key] = new Set(); + dimensionValues[key].add(value); + } + } + } + + return Object.entries(dimensionValues) + .filter(([, values]) => values.size > 1) + .map(([key, values]) => ({ + key, + values: [...values].sort(), + })); +} + export function FigureGallery({ providerSlug, diagnosticSlug, }: DiagnosticFigureGalleryProps) { const [filter, setFilter] = useState(""); - const [selectedGroup, setSelectedGroup] = useState("all"); + const [selectorFilters, setSelectorFilters] = useState< + Record + >({}); const [selectedFigureIndex, setSelectedFigureIndex] = useState( null, ); @@ -101,45 +128,72 @@ export function FigureGallery({ }; }, [getColumns]); - const allFigures: FigureWithGroup[] = (executionGroups?.data ?? []).flatMap( - (group) => - (group.executions ?? []).flatMap((execution) => - (execution.outputs ?? []) - .filter((output) => output.output_type === "plot") - .map((figure) => ({ figure, executionGroup: group })), + const groups = executionGroups?.data ?? []; + const selectorDimensions = useMemo( + () => extractSelectorDimensions(groups), + [groups], + ); + + const allFigures = useMemo( + () => + groups.flatMap((group) => + (group.executions ?? []).flatMap((execution) => + (execution.outputs ?? []) + .filter((output) => output.output_type === "plot") + .map((figure) => ({ figure, executionGroup: group })), + ), ), + [groups], ); - const filteredFigures = allFigures.filter(({ figure, executionGroup }) => { - if ( - selectedGroup !== "all" && - selectedGroup !== executionGroup.id.toString() - ) { - return false; - } - if (filter) { - try { - const regex = new RegExp(filter, "i"); - return regex.test(figure.description) || regex.test(figure.filename); - } catch { - // Invalid regex, don't filter - return true; + const filteredFigures = useMemo(() => { + const filterRegex = filter + ? (() => { + try { + return new RegExp(filter, "i"); + } catch { + return null; + } + })() + : null; + + return allFigures.filter(({ figure, executionGroup }) => { + for (const [key, filterValue] of Object.entries(selectorFilters)) { + if (filterValue === "all") continue; + const groupValues = Object.values(executionGroup.selectors).flatMap( + (pairs) => pairs.filter(([k]) => k === key).map(([, v]) => v), + ); + if (groupValues.length > 0 && !groupValues.includes(filterValue)) { + return false; + } } - } - return true; - }); + if (filterRegex) { + return ( + filterRegex.test(figure.description) || + filterRegex.test(figure.filename) + ); + } + return true; + }); + }, [allFigures, selectorFilters, filter]); + + const listRef = useRef(null); + const [scrollMargin, setScrollMargin] = useState(0); - const uniqueGroups = Array.from( - new Set(allFigures.map(({ executionGroup }) => executionGroup.id)), - ) - .map((id) => executionGroups?.data?.find((g) => g.id === id)) - .filter(Boolean); + // biome-ignore lint/correctness/useExhaustiveDependencies: filteredFigures.length triggers re-measurement when filter changes affect layout position + useEffect(() => { + if (listRef.current) { + const rect = listRef.current.getBoundingClientRect(); + setScrollMargin(rect.top + window.scrollY); + } + }, [filteredFigures.length]); const totalRows = Math.ceil(filteredFigures.length / columns); const rowVirtualizer = useWindowVirtualizer({ count: totalRows, estimateSize: () => 400, overscan: 9, + scrollMargin, }); const goToPrevious = useCallback(() => { @@ -161,25 +215,32 @@ export function FigureGallery({ const items = rowVirtualizer.getVirtualItems(); return (
-
-
- - -
+
+ {selectorDimensions.map((dim) => ( +
+ + +
+ ))}
- + 0 ? (
{items.map((virtualRow) => ( @@ -229,13 +291,13 @@ export function FigureGallery({
- - Execution Group: - {" "} + Group:{" "} {executionGroup.key} - Filename:{" "} + + Filename: + {" "} {figure.filename}
diff --git a/frontend/src/routes/_app/diagnostics.$providerSlug.$diagnosticSlug/scalars.tsx b/frontend/src/routes/_app/diagnostics.$providerSlug.$diagnosticSlug/scalars.tsx index 2c56bc6..4ce82ba 100644 --- a/frontend/src/routes/_app/diagnostics.$providerSlug.$diagnosticSlug/scalars.tsx +++ b/frontend/src/routes/_app/diagnostics.$providerSlug.$diagnosticSlug/scalars.tsx @@ -1,7 +1,6 @@ import { createFileRoute, useNavigate } from "@tanstack/react-router"; import { zodValidator } from "@tanstack/zod-adapter"; import { z } from "zod"; -import { CardTemplateGenerator } from "@/components/diagnostics/cardTemplateGenerator"; import { Values } from "@/components/execution/values"; import { useDiagnosticMetricValues } from "@/hooks/useDiagnosticMetricValues"; @@ -22,22 +21,14 @@ export const ScalarsValuesTab = () => { const search = Route.useSearch(); const navigate = useNavigate({ from: Route.fullPath }); - const { - metricValues, - isLoading, - currentFilters, - currentGroupingConfig, - initialFilters, - isolateIds, - excludeIds, - handlers, - } = useDiagnosticMetricValues({ - providerSlug, - diagnosticSlug, - search, - valueType: "scalar", - navigate, - }); + const { metricValues, isLoading, initialFilters, handlers } = + useDiagnosticMetricValues({ + providerSlug, + diagnosticSlug, + search, + valueType: "scalar", + navigate, + }); return (
@@ -57,26 +48,6 @@ export const ScalarsValuesTab = () => { onCurrentGroupingChange={handlers.onCurrentGroupingChange} onDownload={handlers.onDownload} /> - - {/* Card Template Generator - Inline for visibility */} -
-

- Generate Card Template - For MBTT -

-

- Create a template for this diagnostic to include in the data explorer. -

- -
); }; diff --git a/frontend/src/routes/_app/diagnostics.$providerSlug.$diagnosticSlug/series.tsx b/frontend/src/routes/_app/diagnostics.$providerSlug.$diagnosticSlug/series.tsx index e6e3e0c..eb21d6c 100644 --- a/frontend/src/routes/_app/diagnostics.$providerSlug.$diagnosticSlug/series.tsx +++ b/frontend/src/routes/_app/diagnostics.$providerSlug.$diagnosticSlug/series.tsx @@ -1,7 +1,6 @@ import { createFileRoute, useNavigate } from "@tanstack/react-router"; import { zodValidator } from "@tanstack/zod-adapter"; import { z } from "zod"; -import { CardTemplateGenerator } from "@/components/diagnostics/cardTemplateGenerator"; import { Values } from "@/components/execution/values"; import { useDiagnosticMetricValues } from "@/hooks/useDiagnosticMetricValues"; @@ -22,23 +21,14 @@ export const SeriesValuesTab = () => { const search = Route.useSearch(); const navigate = useNavigate({ from: Route.fullPath }); - const { - metricValues, - isLoading, - currentFilters, - currentGroupingConfig, - filteredData, - initialFilters, - isolateIds, - excludeIds, - handlers, - } = useDiagnosticMetricValues({ - providerSlug, - diagnosticSlug, - search, - valueType: "series", - navigate, - }); + const { metricValues, isLoading, initialFilters, handlers } = + useDiagnosticMetricValues({ + providerSlug, + diagnosticSlug, + search, + valueType: "series", + navigate, + }); return (
@@ -59,28 +49,6 @@ export const SeriesValuesTab = () => { onFilteredDataChange={handlers.onFilteredDataChange} onDownload={handlers.onDownload} /> - - {/* Card Template Generator - Inline for visibility */} -
-

- Generate Card Template - For MBTT -

-

- Create a template for this diagnostic to include in the data explorer. -

- 0 ? filteredData : (metricValues?.data ?? []) - } - currentTab="series" - isolateIds={isolateIds} - excludeIds={excludeIds} - /> -
); }; From 5e6a5ace128b53995ebb86f69b1e92cdab89c66a Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Sat, 7 Mar 2026 16:47:52 +1100 Subject: [PATCH 12/15] feat(collections): add explorer card content and remove unused OHC collection Add explorer_cards definitions for annual-cycle, cloud-scatterplots, TCRE, zero-emissions-commitment, historical-changes, and fire-climate-drivers collections. Remove the empty ocean-heat-content (1.5) collection and its reference from themes.yaml. --- .../collections/1-5_ocean-heat-content.yaml | 24 ------ .../static/collections/3-1_annual-cycle.yaml | 70 +++++++++++++++- .../collections/3-7_cloud-scatterplots.yaml | 45 ++++++++++- backend/static/collections/4-3_tcre.yaml | 35 +++++++- .../4-4_zero-emissions-commitment.yaml | 34 +++++++- .../collections/4-5_historical-changes.yaml | 61 +++++++++++++- .../collections/5-4_fire-climate-drivers.yaml | 81 ++++++++++++++++++- backend/static/collections/themes.yaml | 2 +- 8 files changed, 321 insertions(+), 31 deletions(-) delete mode 100644 backend/static/collections/1-5_ocean-heat-content.yaml diff --git a/backend/static/collections/1-5_ocean-heat-content.yaml b/backend/static/collections/1-5_ocean-heat-content.yaml deleted file mode 100644 index dfe112c..0000000 --- a/backend/static/collections/1-5_ocean-heat-content.yaml +++ /dev/null @@ -1,24 +0,0 @@ -id: "1.5" -name: "Ocean heat content (OHC)" -theme: "Oceans and sea ice" -endorser: "CMIP Model Benchmarking Task Team" -version_control: "version 1 - 24-11-04 REF launch" -reference_dataset: "MGOHCTA-WOA09" -provider_link: "https://doi.org/10.1029/2018MS001354" -content: - description: >- - The OHC diagnostic displays a time series of global ocean heat content from 0 to - 2000 m depth from 2005 to 2015. The diagnostic is computed using each model's 3D - ocean potential temperature field (thetao) and grid cell volume (volcello; which - may be static or time evolving). OHC is computed as thetao multiplied by the heat - capacity of seawater in J/(kg K) and the average in-situ density of seawater - (approximated to be 1026 kg/m3). The OHC is then integrated over the upper 2000 m - of the water column by identifying the grid cell closest to 2000 m in each model's - vertical coordinate. The value is then referenced to year 1 to create the time - series. Simulated OHC is compared against MGOHCTA-WOA09 - Monthly Global Ocean - Heat Content and Temperature Anomalies in-situ (basin averages) from NOAA. - short_description: >- - Apply the ILAMB Methodology* to compare the model ocean heat content (OHC) with - reference data from MGOHCTA-WOA09 -diagnostics: [] -explorer_cards: [] diff --git a/backend/static/collections/3-1_annual-cycle.yaml b/backend/static/collections/3-1_annual-cycle.yaml index 0ca7339..4a054ba 100644 --- a/backend/static/collections/3-1_annual-cycle.yaml +++ b/backend/static/collections/3-1_annual-cycle.yaml @@ -25,4 +25,72 @@ content: diagnostics: - provider_slug: pmp diagnostic_slug: annual-cycle -explorer_cards: [] +explorer_cards: + - title: "Mean Climate Performance" + description: >- + Model skill at reproducing observed climatological patterns, assessed via + spatial correlation, RMSE, and bias for key atmospheric variables. + content: + - type: box-whisker-chart + provider: pmp + diagnostic: annual-cycle + title: "Surface Temperature Spatial Correlation" + description: >- + Spatial pattern correlation between simulated and observed annual-mean + surface temperature (ts). Higher values indicate better agreement with + the observed climatological pattern. + metric_units: "" + other_filters: + variable_id: ts + statistic: cor_xy + region: global + grouping_config: + group_by: season + hue: season + - type: box-whisker-chart + provider: pmp + diagnostic: annual-cycle + title: "Precipitation RMSE" + description: >- + Root mean square error of the spatial pattern of annual and seasonal + precipitation (pr) relative to GPCP observations. Lower values indicate + better performance. + metric_units: "mm/day" + other_filters: + variable_id: pr + statistic: rms_xy + region: global + grouping_config: + group_by: season + hue: season + - type: box-whisker-chart + provider: pmp + diagnostic: annual-cycle + title: "Surface Temperature Bias" + description: >- + Global mean bias in surface temperature (ts) relative to ERA-5. Positive + values indicate the model is too warm; negative values indicate too cold. + metric_units: "K" + other_filters: + variable_id: ts + statistic: bias_xy + region: global + show_zero_line: true + grouping_config: + group_by: season + hue: season + - type: box-whisker-chart + provider: pmp + diagnostic: annual-cycle + title: "TOA Outgoing Longwave Radiation RMSE" + description: >- + Spatial RMSE of outgoing longwave radiation at the top of atmosphere + (rlut) compared to CERES-EBAF observations. + metric_units: "W/m^2" + other_filters: + variable_id: rlut + statistic: rms_xy + region: global + grouping_config: + group_by: season + hue: season diff --git a/backend/static/collections/3-7_cloud-scatterplots.yaml b/backend/static/collections/3-7_cloud-scatterplots.yaml index 7b3c8a8..dfaa993 100644 --- a/backend/static/collections/3-7_cloud-scatterplots.yaml +++ b/backend/static/collections/3-7_cloud-scatterplots.yaml @@ -48,4 +48,47 @@ diagnostics: diagnostic_slug: cloud-scatterplots-clwvi-pr - provider_slug: esmvaltool diagnostic_slug: cloud-scatterplots-reference -explorer_cards: [] +explorer_cards: + - title: "Cloud Scatterplots" + description: >- + 2D histograms comparing cloud-relevant variable pairs in models and + observations, revealing whether models capture regime-dependent cloud + behaviour. + content: + - type: figure-gallery + provider: esmvaltool + diagnostic: cloud-scatterplots-reference + title: "Reference Observations" + description: >- + Observed relationships between paired cloud-relevant variables from + satellite data (CERES, CALIPSO, ERA5, GPCP), providing the benchmark + against which model scatterplots are compared. + span: 2 + - type: figure-gallery + provider: esmvaltool + diagnostic: cloud-scatterplots-clt-swcre + title: "Cloud Fraction vs. Shortwave CRE" + description: >- + Scatterplots of total cloud fraction (clt) against shortwave cloud + radiative effect (SW CRE) for individual models, testing whether models + reproduce the observed relationship between cloud amount and its + radiative impact. + span: 2 + - type: figure-gallery + provider: esmvaltool + diagnostic: cloud-scatterplots-clivi-lwcre + title: "Ice Water Path vs. Longwave CRE" + description: >- + Scatterplots of cloud ice water path (clivi) against longwave cloud + radiative effect (LW CRE), assessing how well models capture the + warming effect of high ice clouds. + span: 2 + - type: figure-gallery + provider: esmvaltool + diagnostic: cloud-scatterplots-clwvi-pr + title: "Liquid Water Path vs. Precipitation" + description: >- + Scatterplots of cloud liquid water path (clwvi) against precipitation + (pr), evaluating the modelled relationship between cloud water content + and precipitation efficiency. + span: 2 diff --git a/backend/static/collections/4-3_tcre.yaml b/backend/static/collections/4-3_tcre.yaml index d3b7ecb..532f0d5 100644 --- a/backend/static/collections/4-3_tcre.yaml +++ b/backend/static/collections/4-3_tcre.yaml @@ -38,4 +38,37 @@ content: diagnostics: - provider_slug: esmvaltool diagnostic_slug: transient-climate-response-emissions -explorer_cards: [] +explorer_cards: + - title: "TCRE" + description: >- + Transient Climate Response to Cumulative Emissions -- the warming per + trillion tonnes of carbon emitted, linking emissions budgets to + temperature targets. + content: + - type: box-whisker-chart + provider: esmvaltool + diagnostic: transient-climate-response-emissions + title: "Transient Climate Response to Cumulative Emissions (TCRE)" + description: >- + TCRE quantifies the global mean surface temperature increase per unit + of cumulative CO2 emissions. It combines the physical climate + sensitivity with the carbon cycle response and is central to estimating + remaining carbon budgets consistent with temperature targets such as + the Paris Agreement. + interpretation: >- + Higher TCRE values mean less cumulative CO2 can be emitted before + reaching a given warming threshold. + metric_units: "K/TtC" + other_filters: + metric: tcre + grouping_config: + group_by: metric + hue: metric + - type: figure-gallery + provider: esmvaltool + diagnostic: transient-climate-response-emissions + title: "TCRE Regression Plots" + description: >- + Scatter plots of cumulative CO2 emissions against global mean + temperature change, with regression lines used to derive TCRE. + span: 2 diff --git a/backend/static/collections/4-4_zero-emissions-commitment.yaml b/backend/static/collections/4-4_zero-emissions-commitment.yaml index a71b233..32729b9 100644 --- a/backend/static/collections/4-4_zero-emissions-commitment.yaml +++ b/backend/static/collections/4-4_zero-emissions-commitment.yaml @@ -33,4 +33,36 @@ content: diagnostics: - provider_slug: esmvaltool diagnostic_slug: zero-emission-commitment -explorer_cards: [] +explorer_cards: + - title: "Zero Emissions Commitment" + description: >- + Temperature change expected after CO2 emissions cease, informing carbon + budget estimates and net-zero policy planning. + content: + - type: box-whisker-chart + provider: esmvaltool + diagnostic: zero-emission-commitment + title: "Zero Emissions Commitment (ZEC)" + description: >- + ZEC represents the additional global warming (or cooling) that occurs + after anthropogenic CO2 emissions are set to zero. Positive values + indicate continued warming; negative values indicate the climate system + begins to cool once emissions stop. + interpretation: >- + A positive ZEC means that even after emissions cease, temperatures + continue to rise, implying a stricter carbon budget is needed. + metric_units: "K" + show_zero_line: true + other_filters: + metric: zec + grouping_config: + group_by: source_id + hue: source_id + - type: figure-gallery + provider: esmvaltool + diagnostic: zero-emission-commitment + title: "ZEC Temperature Evolution" + description: >- + Time series and scatter plots showing the temperature response after + emissions cessation for each model. + span: 2 diff --git a/backend/static/collections/4-5_historical-changes.yaml b/backend/static/collections/4-5_historical-changes.yaml index dd42547..19b0ac1 100644 --- a/backend/static/collections/4-5_historical-changes.yaml +++ b/backend/static/collections/4-5_historical-changes.yaml @@ -24,4 +24,63 @@ diagnostics: diagnostic_slug: regional-historical-timeseries - provider_slug: esmvaltool diagnostic_slug: regional-historical-trend -explorer_cards: [] +explorer_cards: + - title: "Historical Changes" + description: >- + Regional trends and annual cycles of key climate variables, comparing + model simulations against observations and reanalysis for IPCC reference + regions. + content: + - type: box-whisker-chart + provider: esmvaltool + diagnostic: regional-historical-trend + title: "Regional Temperature Trends" + description: >- + Linear trends in near-surface air temperature (tas) across IPCC + reference regions. Comparing modelled and observed trends reveals + whether models reproduce the spatial pattern of recent warming. + metric_units: "K/decade" + other_filters: + variable_id: tas + metric: trend + grouping_config: + group_by: region + hue: region + - type: box-whisker-chart + provider: esmvaltool + diagnostic: regional-historical-trend + title: "Regional Precipitation Trends" + description: >- + Linear trends in precipitation (pr) across IPCC reference regions. + Precipitation trends are harder for models to capture due to large + internal variability and complex regional drivers. + metric_units: "mm/day/decade" + other_filters: + variable_id: pr + metric: trend + grouping_config: + group_by: region + hue: region + - type: series-chart + provider: esmvaltool + diagnostic: regional-historical-annual-cycle + title: "Annual Cycle of Temperature (Global)" + description: >- + Monthly climatology of near-surface air temperature for the global + mean, comparing individual models with reanalysis data. + span: 2 + metric_units: "K" + other_filters: + region: Global + statistic: tas regional mean + grouping_config: + group_by: source_id + hue: source_id + - type: figure-gallery + provider: esmvaltool + diagnostic: regional-historical-trend + title: "Regional Trend Maps" + description: >- + Maps of linear trends across IPCC reference regions for temperature, + precipitation, and other variables. + span: 2 diff --git a/backend/static/collections/5-4_fire-climate-drivers.yaml b/backend/static/collections/5-4_fire-climate-drivers.yaml index 5021118..9dcbecb 100644 --- a/backend/static/collections/5-4_fire-climate-drivers.yaml +++ b/backend/static/collections/5-4_fire-climate-drivers.yaml @@ -36,4 +36,83 @@ diagnostics: diagnostic_slug: climate-drivers-for-fire - provider_slug: ilamb diagnostic_slug: burntfractionall-gfed -explorer_cards: [] +explorer_cards: + - title: "Fire & Burnt Area" + description: >- + Model evaluation of burnt area and the climate drivers of fire, comparing + simulated fire extent with GFED observations. + content: + - type: box-whisker-chart + provider: ilamb + diagnostic: burntfractionall-gfed + title: "Burnt Fraction Bias" + description: >- + Spatial bias in modelled burnt fraction relative to GFED observations. + Values near zero indicate good agreement; large positive or negative + values highlight regions where models over- or under-estimate fire + extent. + metric_units: "" + show_zero_line: true + other_filters: + metric: Bias + region: global + grouping_config: + group_by: statistic + hue: statistic + - type: box-whisker-chart + provider: ilamb + diagnostic: burntfractionall-gfed + title: "Burnt Fraction Spatial Distribution" + description: >- + Spatial distribution score for burnt fraction, measuring how well + models reproduce the observed geographic pattern of fire occurrence. + metric_units: "" + other_filters: + metric: Spatial Distribution + region: global + grouping_config: + group_by: statistic + hue: statistic + - type: series-chart + provider: ilamb + diagnostic: burntfractionall-gfed + title: "Global Burnt Fraction Time Series" + description: >- + Time series of global mean burnt fraction for each model compared with + GFED reference data. + span: 2 + other_filters: + metric: trace_global + grouping_config: + group_by: source_id + hue: source_id + - type: series-chart + provider: ilamb + diagnostic: burntfractionall-gfed + title: "Burnt Fraction Annual Cycle (Global)" + description: >- + Annual cycle of global burnt fraction, comparing the seasonal timing + and amplitude of fire activity across models and observations. + span: 2 + other_filters: + metric: cycle_global + grouping_config: + group_by: source_id + hue: source_id + - type: figure-gallery + provider: esmvaltool + diagnostic: climate-drivers-for-fire + title: "Fire Climate Drivers (ConFire)" + description: >- + Maps of modelled burnt area and the relative contribution of fire + weather versus fuel load/continuity, computed using the ConFire + Bayesian model. + span: 2 + - type: figure-gallery + provider: ilamb + diagnostic: burntfractionall-gfed + title: "Burnt Fraction Spatial Maps" + description: >- + Spatial maps of burnt fraction bias, RMSE, and model-observation + comparison for individual models. + span: 2 diff --git a/backend/static/collections/themes.yaml b/backend/static/collections/themes.yaml index 6128d5d..84c2b7a 100644 --- a/backend/static/collections/themes.yaml +++ b/backend/static/collections/themes.yaml @@ -21,4 +21,4 @@ land: ocean: title: "Ocean and Sea Ice" description: "Ocean and sea ice diagnostics including circulation, temperature, and ice area." - collections: ["1.1", "1.2", "1.3", "1.4", "1.5", "1.6"] + collections: ["1.1", "1.2", "1.3", "1.4", "1.6"] From 90405079ad91d54823cd837edc522d24080a9e5d Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Sat, 7 Mar 2026 23:01:52 +0900 Subject: [PATCH 13/15] refactor(diagnostics): split metadata into per-provider files and surface reference datasets - Split monolithic metadata.yaml into per-provider files (pmp.yaml, esmvaltool.yaml, ilamb.yaml) with directory-based loading - Enrich explorer card content with reference dataset badges from diagnostic metadata at collection load time - Replace linear scan in get_aft_for_ref_diagnostic with O(1) reverse index - Remove duplicate interpretation field from BoxWhiskerChartContent type - Eliminate duplicate useSuspenseQuery call in ThematicContent - Update generate_metadata script to output per-provider files --- README.md | 3 +- backend/scripts/generate_metadata.py | 80 +-- backend/src/ref_backend/core/aft.py | 33 +- backend/src/ref_backend/core/collections.py | 30 + backend/src/ref_backend/core/config.py | 15 +- .../ref_backend/core/diagnostic_metadata.py | 92 ++- backend/static/diagnostics/esmvaltool.yaml | 257 +++++++++ backend/static/diagnostics/ilamb.yaml | 134 +++++ backend/static/diagnostics/metadata.yaml | 526 ------------------ backend/static/diagnostics/pmp.yaml | 168 ++++++ .../tests/test_api/test_routes/test_aft.py | 6 +- .../test_core/test_diagnostic_metadata.py | 75 +++ frontend/src/client/schemas.gen.ts | 14 + frontend/src/client/types.gen.ts | 1 + .../explorer/explorerCardContent.tsx | 39 +- .../components/explorer/thematicContent.tsx | 12 +- frontend/src/components/explorer/types.ts | 3 +- 17 files changed, 865 insertions(+), 623 deletions(-) create mode 100644 backend/static/diagnostics/esmvaltool.yaml create mode 100644 backend/static/diagnostics/ilamb.yaml delete mode 100644 backend/static/diagnostics/metadata.yaml create mode 100644 backend/static/diagnostics/pmp.yaml diff --git a/README.md b/README.md index 82965d0..dd66be6 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,8 @@ Display metadata for each AFT diagnostic collection (descriptions, explanations, is maintained in YAML files under [`backend/static/collections/`](backend/static/collections/). See the [collections README](backend/static/collections/README.md) for the full schema and instructions. -Diagnostic-level metadata overrides (display names, reference datasets, tags) are in `backend/static/diagnostics/metadata.yaml`, +Diagnostic-level metadata overrides (display names, reference datasets, tags) are split into per-provider +YAML files under `backend/static/diagnostics/` (e.g. `pmp.yaml`, `esmvaltool.yaml`, `ilamb.yaml`), which can be regenerated from the provider registry with `make generate-metadata`. After changing content fields or adding new collections, regenerate the frontend TypeScript client with `make generate-client`. diff --git a/backend/scripts/generate_metadata.py b/backend/scripts/generate_metadata.py index 4b3cdb2..3842666 100644 --- a/backend/scripts/generate_metadata.py +++ b/backend/scripts/generate_metadata.py @@ -1,16 +1,17 @@ """ Generate diagnostic metadata YAML from the current provider registry. -This script bootstraps or updates the metadata.yaml file by iterating all -registered diagnostics and capturing their current state (display_name, -description, tags, reference_datasets). Existing values in metadata.yaml +This script bootstraps or updates the per-provider metadata files by iterating +all registered diagnostics and capturing their current state (display_name, +description, tags, reference_datasets). Existing values in the metadata files take precedence over auto-generated values. Usage: cd backend && uv run python scripts/generate_metadata.py Options: - --output PATH Write to a specific file (default: static/diagnostics/metadata.yaml) + --output PATH Write to a specific file (default: writes per-provider files + into static/diagnostics/) --dry-run Print to stdout instead of writing to file """ @@ -72,45 +73,35 @@ def _build_entry( def generate_metadata(output_path: Path | None = None, *, dry_run: bool = False) -> None: - """Generate metadata.yaml from the provider registry, merging with existing values.""" + """Generate per-provider metadata YAML files from the provider registry, merging with existing values.""" settings = Settings() ref_config = get_ref_config(settings) database = get_database(ref_config) provider_registry = get_provider_registry(ref_config) - # Load existing metadata (existing values take precedence) - default_metadata_path = backend_dir / "static" / "diagnostics" / "metadata.yaml" - metadata_path = output_path or default_metadata_path - existing_metadata = load_diagnostic_metadata(metadata_path) + # Load existing metadata from the directory (existing values take precedence) + default_metadata_dir = backend_dir / "static" / "diagnostics" + metadata_dir = output_path or default_metadata_dir + existing_metadata = load_diagnostic_metadata(metadata_dir) - # Iterate all registered diagnostics - generated: dict[str, dict[str, Any]] = {} + # Group diagnostics by provider + by_provider: dict[str, dict[str, dict[str, Any]]] = {} with database.session.connection(): for provider_slug, diagnostics in provider_registry.metrics.items(): for diagnostic_slug, concrete_diagnostic in diagnostics.items(): key = f"{provider_slug}/{diagnostic_slug}" - generated[key] = _build_entry(key, diagnostic_slug, concrete_diagnostic, existing_metadata) + entry = _build_entry(key, diagnostic_slug, concrete_diagnostic, existing_metadata) + by_provider.setdefault(provider_slug, {})[key] = entry # Also include any entries from existing metadata that weren't found in the registry for key, metadata in existing_metadata.items(): - if key not in generated: - generated[key] = _metadata_to_dict(metadata) - - # Sort by key for consistent output - sorted_metadata = dict(sorted(generated.items())) - - # Generate YAML output - yaml_content = yaml.dump( - sorted_metadata, - default_flow_style=False, - sort_keys=False, - allow_unicode=True, - width=120, - ) + provider_slug = key.split("/")[0] + if key not in by_provider.get(provider_slug, {}): + by_provider.setdefault(provider_slug, {})[key] = _metadata_to_dict(metadata) header = ( - "# Diagnostic Metadata\n" + "# {provider} Diagnostic Metadata\n" "#\n" "# Auto-generated by: cd backend && uv run python scripts/generate_metadata.py\n" "#\n" @@ -120,15 +111,32 @@ def generate_metadata(output_path: Path | None = None, *, dry_run: bool = False) "#\n\n" ) - output = header + yaml_content + total = 0 + for provider_slug, entries in sorted(by_provider.items()): + sorted_entries = dict(sorted(entries.items())) + total += len(sorted_entries) + + yaml_content = yaml.dump( + sorted_entries, + default_flow_style=False, + sort_keys=False, + allow_unicode=True, + width=120, + ) + + output = header.format(provider=provider_slug) + yaml_content + + if dry_run: + print(f"--- {provider_slug}.yaml ---") + print(output) + else: + metadata_dir.mkdir(parents=True, exist_ok=True) + file_path = metadata_dir / f"{provider_slug}.yaml" + file_path.write_text(output) + print(f"Generated metadata written to {file_path} ({len(sorted_entries)} diagnostics)") - if dry_run: - print(output) - else: - metadata_path.parent.mkdir(parents=True, exist_ok=True) - metadata_path.write_text(output) - print(f"Generated metadata written to {metadata_path}") - print(f"Total diagnostics: {len(sorted_metadata)}") + if not dry_run: + print(f"Total diagnostics across all providers: {total}") def main() -> None: @@ -137,7 +145,7 @@ def main() -> None: "--output", type=Path, default=None, - help="Output file path (default: static/diagnostics/metadata.yaml)", + help="Output directory (default: static/diagnostics/)", ) parser.add_argument( "--dry-run", diff --git a/backend/src/ref_backend/core/aft.py b/backend/src/ref_backend/core/aft.py index f2c508f..2d49cb1 100644 --- a/backend/src/ref_backend/core/aft.py +++ b/backend/src/ref_backend/core/aft.py @@ -75,7 +75,23 @@ def get_aft_diagnostic_by_id(aft_id: str) -> AFTDiagnosticDetail | None: return AFTDiagnosticDetail(**base.model_dump(), diagnostics=refs) -@lru_cache(maxsize=128) +@lru_cache(maxsize=1) +def _build_ref_to_aft_index() -> dict[tuple[str, str], str]: + """Build a reverse index from (provider_slug, diagnostic_slug) to AFT collection ID.""" + index: dict[tuple[str, str], str] = {} + for col_id, col in load_all_collections().items(): + for diag in col.diagnostics: + key = (diag.provider_slug, diag.diagnostic_slug) + if key in index: + logger.warning( + f"Multiple AFT IDs for {diag.provider_slug}/{diag.diagnostic_slug}: " + f"{index[key]} and {col_id}, keeping first" + ) + continue + index[key] = col_id + return index + + def get_aft_for_ref_diagnostic(provider_slug: str, diagnostic_slug: str) -> str | None: """ Get AFT diagnostic ID associated with a REF diagnostic. @@ -88,17 +104,4 @@ def get_aft_for_ref_diagnostic(provider_slug: str, diagnostic_slug: str) -> str ------- The AFT diagnostic ID if found, None otherwise """ - collections = load_all_collections() - aft_ids = [] - - for col_id, col in collections.items(): - for diag in col.diagnostics: - if diag.provider_slug == provider_slug and diag.diagnostic_slug == diagnostic_slug: - aft_ids.append(col_id) - - if len(aft_ids) == 1: - return aft_ids[0] - if len(aft_ids) > 1: - logger.warning(f"Multiple AFT IDs found for {provider_slug}/{diagnostic_slug}: {aft_ids}") - return aft_ids[0] - return None + return _build_ref_to_aft_index().get((provider_slug, diagnostic_slug)) diff --git a/backend/src/ref_backend/core/collections.py b/backend/src/ref_backend/core/collections.py index 1c3f1c0..d8269df 100644 --- a/backend/src/ref_backend/core/collections.py +++ b/backend/src/ref_backend/core/collections.py @@ -6,6 +6,12 @@ import yaml from pydantic import BaseModel, HttpUrl, ValidationError +from ref_backend.core.diagnostic_metadata import ( + DiagnosticMetadata, + ReferenceDatasetLink, + load_diagnostic_metadata, +) + logger = logging.getLogger(__name__) @@ -35,6 +41,7 @@ class AFTCollectionCardContent(BaseModel): label_template: str | None = None other_filters: dict[str, str] | None = None grouping_config: AFTCollectionGroupingConfig | None = None + reference_datasets: list[ReferenceDatasetLink] | None = None class AFTCollectionCard(BaseModel): @@ -101,6 +108,10 @@ class ThemeDetail(BaseModel): explorer_cards: list[AFTCollectionCard] +def get_diagnostics_dir() -> Path: + return Path(__file__).parents[3] / "static" / "diagnostics" + + def get_collections_dir() -> Path: return Path(__file__).parents[3] / "static" / "collections" @@ -109,6 +120,23 @@ def get_themes_path() -> Path: return get_collections_dir() / "themes.yaml" +def _enrich_card_content_with_ref_datasets( + collection: AFTCollectionDetail, + metadata: dict[str, DiagnosticMetadata], +) -> None: + """Populate reference_datasets on each card content item from diagnostic metadata.""" + for card in collection.explorer_cards: + for content in card.content: + key = f"{content.provider}/{content.diagnostic}" + if key in metadata and metadata[key].reference_datasets: + content.reference_datasets = metadata[key].reference_datasets + + +@lru_cache(maxsize=1) +def _load_diagnostic_metadata_cached() -> dict[str, DiagnosticMetadata]: + return load_diagnostic_metadata(get_diagnostics_dir()) + + @lru_cache(maxsize=1) def load_all_collections() -> dict[str, AFTCollectionDetail]: collections_dir = get_collections_dir() @@ -117,6 +145,7 @@ def load_all_collections() -> dict[str, AFTCollectionDetail]: logger.warning(f"Collections directory not found: {collections_dir}") return {} + diagnostic_metadata = _load_diagnostic_metadata_cached() result: dict[str, AFTCollectionDetail] = {} yaml_files = sorted( @@ -134,6 +163,7 @@ def load_all_collections() -> dict[str, AFTCollectionDetail]: continue collection = AFTCollectionDetail(**data) + _enrich_card_content_with_ref_datasets(collection, diagnostic_metadata) if collection.id in result: logger.warning(f"Duplicate collection ID '{collection.id}' in {yaml_file.name}, skipping") diff --git a/backend/src/ref_backend/core/config.py b/backend/src/ref_backend/core/config.py index 0b2aa27..4ad3ca1 100644 --- a/backend/src/ref_backend/core/config.py +++ b/backend/src/ref_backend/core/config.py @@ -63,25 +63,26 @@ def all_cors_origins(self) -> list[str]: DIAGNOSTIC_METADATA_PATH: Path | None = None """ - Path to the diagnostic metadata YAML file. + Path to the diagnostic metadata YAML file or directory. - This file provides additional metadata for diagnostics that can override or supplement - the default values from diagnostic implementations. If not provided, defaults to - 'static/diagnostics/metadata.yaml' relative to the backend directory. + Provides additional metadata for diagnostics that can override or supplement + the default values from diagnostic implementations. Accepts either a single + YAML file or a directory of YAML files (one per provider). If not provided, + defaults to 'static/diagnostics/' relative to the backend directory. """ @computed_field # type: ignore[prop-decorator] @property def diagnostic_metadata_path_resolved(self) -> Path: """ - Get the resolved path to the diagnostic metadata file. + Get the resolved path to the diagnostic metadata file or directory. Returns the configured path or the default location. """ if self.DIAGNOSTIC_METADATA_PATH is not None: return self.DIAGNOSTIC_METADATA_PATH - # Default to static/diagnostics/metadata.yaml relative to backend directory - return Path(__file__).parent.parent.parent.parent / "static" / "diagnostics" / "metadata.yaml" + # Default to static/diagnostics/ directory relative to backend directory + return Path(__file__).parent.parent.parent.parent / "static" / "diagnostics" def _check_default_secret(self, var_name: str, value: str | None) -> None: if value == "changethis": diff --git a/backend/src/ref_backend/core/diagnostic_metadata.py b/backend/src/ref_backend/core/diagnostic_metadata.py index ce542de..c102398 100644 --- a/backend/src/ref_backend/core/diagnostic_metadata.py +++ b/backend/src/ref_backend/core/diagnostic_metadata.py @@ -54,13 +54,9 @@ class DiagnosticMetadata(BaseModel): tags: list[str] | None = Field(None, description="Tags for categorizing the diagnostic") -def load_diagnostic_metadata(yaml_path: Path) -> dict[str, DiagnosticMetadata]: +def _load_metadata_from_file(yaml_path: Path) -> dict[str, DiagnosticMetadata]: """ - Load diagnostic metadata from a YAML file. - - This function loads metadata overrides from a YAML file, which should follow - the structure defined in DiagnosticMetadata. The file uses diagnostic keys - in the format "provider-slug/diagnostic-slug" to map metadata to diagnostics. + Load diagnostic metadata from a single YAML file. Args: yaml_path: Path to the YAML metadata file @@ -68,27 +64,8 @@ def load_diagnostic_metadata(yaml_path: Path) -> dict[str, DiagnosticMetadata]: Returns ------- Dictionary mapping diagnostic keys (provider/diagnostic) to their metadata. - Returns an empty dict if the file doesn't exist or cannot be parsed. - - Example YAML structure: - ```yaml - pmp/annual-cycle: - reference_datasets: - - slug: "obs4mips.CERES-EBAF.v4.2" - description: "CERES Energy Balanced and Filled" - type: "primary" - display_name: "Annual Cycle Analysis" - tags: ["atmosphere", "seasonal-cycle"] - ``` + Returns an empty dict if the file cannot be parsed. """ - if not yaml_path.exists(): - logger.warning( - f"Diagnostic metadata file not found at {yaml_path}. " - "No metadata overrides will be applied. This is expected if you haven't " - "created the metadata file yet." - ) - return {} - try: with open(yaml_path) as f: raw_data = yaml.safe_load(f) @@ -97,7 +74,6 @@ def load_diagnostic_metadata(yaml_path: Path) -> dict[str, DiagnosticMetadata]: logger.info(f"Diagnostic metadata file at {yaml_path} is empty.") return {} - # Parse each diagnostic's metadata metadata_dict: dict[str, DiagnosticMetadata] = {} for diagnostic_key, metadata_raw in raw_data.items(): try: @@ -117,3 +93,65 @@ def load_diagnostic_metadata(yaml_path: Path) -> dict[str, DiagnosticMetadata]: except Exception as e: logger.error(f"Unexpected error loading diagnostic metadata from {yaml_path}: {e}") return {} + + +def load_diagnostic_metadata(path: Path) -> dict[str, DiagnosticMetadata]: + """ + Load diagnostic metadata from a YAML file or a directory of YAML files. + + When given a directory, all ``*.yaml`` and ``*.yml`` files in it are loaded + and merged. When given a single file, it is loaded directly (backwards + compatible with the old single-file layout). + + The YAML files should follow the structure defined in DiagnosticMetadata, + using diagnostic keys in the format ``provider-slug/diagnostic-slug``. + + Args: + path: Path to a YAML file or a directory containing YAML files + + Returns + ------- + Dictionary mapping diagnostic keys (provider/diagnostic) to their metadata. + Returns an empty dict if the path doesn't exist or cannot be parsed. + + Example YAML structure:: + + pmp/annual-cycle: + reference_datasets: + - slug: "obs4mips.CERES-EBAF.v4.2" + description: "CERES Energy Balanced and Filled" + type: "primary" + display_name: "Annual Cycle Analysis" + tags: ["atmosphere", "seasonal-cycle"] + """ + if not path.exists(): + logger.warning( + f"Diagnostic metadata path not found at {path}. " + "No metadata overrides will be applied. This is expected if you haven't " + "created the metadata files yet." + ) + return {} + + if path.is_file(): + return _load_metadata_from_file(path) + + # Load all YAML files from the directory + yaml_files = sorted(path.glob("*.yaml")) + sorted(path.glob("*.yml")) + if not yaml_files: + logger.info(f"No YAML files found in directory {path}.") + return {} + + merged: dict[str, DiagnosticMetadata] = {} + for yaml_file in yaml_files: + file_metadata = _load_metadata_from_file(yaml_file) + # Check for duplicate keys across files + duplicates = set(merged.keys()) & set(file_metadata.keys()) + if duplicates: + logger.warning( + f"Duplicate diagnostic keys found in {yaml_file.name}: " + f"{', '.join(sorted(duplicates))}. Later values will override earlier ones." + ) + merged.update(file_metadata) + + logger.info(f"Loaded metadata for {len(merged)} diagnostics from {len(yaml_files)} files in {path}") + return merged diff --git a/backend/static/diagnostics/esmvaltool.yaml b/backend/static/diagnostics/esmvaltool.yaml new file mode 100644 index 0000000..bedb254 --- /dev/null +++ b/backend/static/diagnostics/esmvaltool.yaml @@ -0,0 +1,257 @@ +# ESMValTool Diagnostic Metadata +# +# Reference datasets and display metadata for ESMValTool diagnostics. + +esmvaltool/cloud-radiative-effects: + reference_datasets: + - slug: "esmvaltool.CERES-EBAF" + description: "CERES Energy Balanced and Filled - TOA radiation fluxes (planned to move to obs4MIPs)" + type: "primary" + - slug: "esmvaltool.ESACCI-CLOUD" + description: "ESA CCI Cloud properties (planned to move to obs4MIPs)" + type: "secondary" + - slug: "esmvaltool.ISCCP-FH" + description: "ISCCP cloud properties (planned to move to obs4MIPs)" + type: "secondary" + display_name: "Cloud Radiative Effects" + description: >- + Plot climatologies and zonal mean profiles of cloud radiative effects + (shortwave + longwave) for a dataset. + tags: ["atmosphere", "clouds", "radiation"] + +esmvaltool/cloud-scatterplots-cli-ta: + reference_datasets: + - slug: "obs4mips.ERA-5" + description: "ERA5 - Air temperature (ta)" + type: "primary" + - slug: "esmvaltool.CALIPSO-ICECLOUD" + description: "Ice Cloud properties (planned to move to obs4MIPs)" + type: "secondary" + display_name: "Cloud-Temperature Scatterplots (cli vs ta)" + description: >- + Scatterplot of ice water content of cloud (cli) vs air temperature (ta). + Observational scatterplots for reference datasets are provided in the separate + "Cloud Scatterplots for Reference dataset" diagnostic. + tags: ["atmosphere", "clouds", "temperature"] + +esmvaltool/cloud-scatterplots-clivi-lwcre: + reference_datasets: + - slug: "esmvaltool.CERES-EBAF" + description: "CERES Energy Balanced and Filled - TOA radiation fluxes (planned)" + type: "primary" + - slug: "esmvaltool.ESACCI-CLOUD" + description: "ESA CCI Cloud properties (planned to move to obs4MIPs)" + type: "secondary" + display_name: "Cloud-Radiation Scatterplots (clivi vs lwcre)" + description: "Scatterplot of ice water path (clivi) vs longwave cloud radiative effect (lwcre)." + tags: ["atmosphere", "clouds", "radiation"] + +esmvaltool/cloud-scatterplots-clt-swcre: + reference_datasets: + - slug: "esmvaltool.CERES-EBAF" + description: "CERES Energy Balanced and Filled - TOA radiation fluxes (planned to move to obs4MIPs)" + type: "primary" + - slug: "esmvaltool.ESACCI-CLOUD" + description: "ESA CCI Cloud properties (planned to move to obs4MIPs)" + type: "secondary" + display_name: "Cloud-Radiation Scatterplots (clt vs swcre)" + description: "Scatterplot of total cloud fraction (clt) vs shortwave cloud radiative effect (swcre)." + tags: ["atmosphere", "clouds", "radiation"] + +esmvaltool/cloud-scatterplots-clwvi-pr: + reference_datasets: + - slug: "obs4MIPs.GPCPv2.3" + description: "GPCP v2.3 - Precipitation relationship" + type: "secondary" + - slug: "esmvaltool.ESACCI-CLOUD" + description: "ESA CCI Cloud properties (planned to move to obs4MIPs)" + type: "secondary" + - slug: "esmvaltool.CERES-EBAF" + description: "CERES - Reference cloud-radiation relationships" + type: "primary" + display_name: "Cloud-Precipitation Scatterplots (clwvi vs pr)" + description: "Scatterplot of condensed water path (clwvi) vs precipitation (pr)." + tags: ["atmosphere", "clouds", "precipitation"] + +esmvaltool/cloud-scatterplots-reference: + reference_datasets: + - slug: "obs4mips.ERA-5" + description: "ERA5 - Air temperature (ta), ingested via REF" + type: "primary" + - slug: "esmvaltool.CERES-EBAF" + description: "CERES - Reference cloud-radiation relationships, baked into recipe" + type: "primary" + - slug: "esmvaltool.ESACCI-CLOUD" + description: "ESA CCI Cloud properties (clt, clwvi, clivi), baked into recipe (planned to move to obs4MIPs)" + type: "secondary" + - slug: "esmvaltool.GPCP-V2.3" + description: "GPCP v2.3 - Precipitation (pr), hardcoded in update_recipe" + type: "secondary" + display_name: "Cloud Scatterplots for Reference dataset" + description: >- + Reference scatterplots of two cloud-relevant variables from observational datasets. + These provide the observational baseline for comparison with model cloud scatterplot diagnostics. + tags: ["atmosphere", "clouds"] + +esmvaltool/enso-basic-climatology: + reference_datasets: + - slug: "esmvaltool.GPCP-V2.3" + description: "GPCP v2.3 - Precipitation (pr), baked into ESMValTool recipe" + type: "secondary" + - slug: "esmvaltool.TROPFLUX" + description: "TROPFLUX - Sea surface temperature (tos) and wind stress (tauu), baked into ESMValTool recipe" + type: "secondary" + display_name: "ENSO Basic Climatology" + description: >- + Calculate the ENSO CLIVAR metrics for background climatology, assessing the + mean state of the tropical Pacific upon which ENSO events develop. + tags: ["ocean", "enso", "climatology"] + +esmvaltool/enso-characteristics: + reference_datasets: + - slug: "esmvaltool.TROPFLUX" + description: "TROPFLUX - Sea surface temperature (tos), hardcoded in update_recipe" + type: "primary" + display_name: "ENSO Characteristics" + description: >- + Calculate the ENSO CLIVAR metrics for basic ENSO characteristics, + evaluating the amplitude, frequency, and spatial structure of ENSO events. + tags: ["ocean", "enso", "variability"] + +esmvaltool/equilibrium-climate-sensitivity: + reference_datasets: [] + display_name: "Equilibrium Climate Sensitivity (ECS)" + description: >- + Calculate the global mean equilibrium climate sensitivity for a dataset. + ECS is defined as the long-term global mean surface temperature change + in response to a doubling of atmospheric CO2 concentration, estimated + using the Gregory regression method from abrupt-4xCO2 experiments. + tags: ["climate-sensitivity", "feedback"] + +esmvaltool/transient-climate-response: + reference_datasets: [] + display_name: "Transient Climate Response (TCR)" + description: >- + Calculate the global mean transient climate response for a dataset. + TCR is defined as the global mean surface temperature change at the time + of CO2 doubling in a 1pctCO2 experiment (1% per year CO2 increase). + tags: ["climate-sensitivity", "transient"] + +esmvaltool/transient-climate-response-emissions: + reference_datasets: [] + display_name: "Transient Climate Response to Emissions (TCRE)" + description: >- + Calculate the global mean Transient Climate Response to Cumulative CO2 Emissions. + TCRE quantifies the relationship between global warming and total cumulative + carbon emissions, a key metric for carbon budget estimation. + tags: ["climate-sensitivity", "carbon-cycle"] + +esmvaltool/zero-emission-commitment: + reference_datasets: [] + display_name: "Zero Emission Commitment (ZEC)" + description: >- + Calculate the global mean Zero Emission Commitment (ZEC) temperature. + ZEC measures the additional warming (or cooling) that occurs after all + CO2 emissions cease, reflecting the committed warming from past emissions. + tags: ["climate-sensitivity", "commitment"] + +esmvaltool/regional-historical-annual-cycle: + reference_datasets: + - slug: "obs4mips.ERA-5" + description: "ERA-5 - Sea level pressure (psl) and zonal wind (ua), ingested via REF" + type: "primary" + - slug: "esmvaltool.HadCRUT5" + description: "HadCRUT5 v5.0.1.0-analysis - Near-surface air temperature (tas), hardcoded in recipe" + type: "secondary" + - slug: "esmvaltool.GPCP-V2.3" + description: "GPCP v2.3 - Precipitation (pr), hardcoded in recipe (cannot be ingested, issue #260)" + type: "secondary" + - slug: "esmvaltool.ERA5-native6" + description: "ERA5 native6 - Specific humidity (hus), not yet on obs4MIPs" + type: "secondary" + display_name: "Regional Historical Annual Cycle" + description: >- + Plot the regional historical annual cycle of climate variables including + near-surface air temperature (tas), precipitation (pr), sea level pressure (psl), + zonal wind (ua), and specific humidity (hus). Regions are defined using + IPCC AR6 reference regions as described by Iturbide et al. (2020). + tags: ["regional", "seasonal-cycle", "historical"] + +esmvaltool/regional-historical-timeseries: + reference_datasets: + - slug: "obs4mips.ERA-5" + description: "ERA-5 - Sea level pressure (psl) and zonal wind (ua), ingested via REF" + type: "primary" + - slug: "esmvaltool.HadCRUT5" + description: "HadCRUT5 v5.0.1.0-analysis - Near-surface air temperature (tas), hardcoded in recipe" + type: "secondary" + - slug: "esmvaltool.GPCP-V2.3" + description: "GPCP v2.3 - Precipitation (pr), hardcoded in recipe (cannot be ingested, issue #260)" + type: "secondary" + - slug: "esmvaltool.ERA5-native6" + description: "ERA5 native6 - Specific humidity (hus), not yet on obs4MIPs" + type: "secondary" + display_name: "Regional Historical Timeseries" + description: >- + Plot the regional historical mean and anomaly timeseries of climate variables including + near-surface air temperature (tas), precipitation (pr), sea level pressure (psl), + zonal wind (ua), and specific humidity (hus). Regions are defined + using IPCC AR6 reference regions as described by Iturbide et al. (2020). + tags: ["regional", "timeseries", "historical"] + +esmvaltool/regional-historical-trend: + reference_datasets: + - slug: "obs4mips.ERA-5" + description: "ERA-5 - Sea level pressure (psl), near-surface air temperature (tas), and zonal wind (ua), ingested via REF" + type: "primary" + - slug: "esmvaltool.HadCRUT5" + description: "HadCRUT5 v5.0.1.0-analysis - Near-surface air temperature (tas), hardcoded in recipe" + type: "secondary" + - slug: "esmvaltool.GPCP-V2.3" + description: "GPCP v2.3 - Precipitation (pr), hardcoded in recipe (cannot be ingested, issue #260)" + type: "secondary" + - slug: "esmvaltool.ERA5-native6" + description: "ERA5 native6 - Specific humidity (hus) and precipitation (pr), hardcoded in recipe" + type: "secondary" + display_name: "Regional Historical Trends" + description: >- + Plot the regional historical trend of climate variables including near-surface + air temperature (tas), precipitation (pr), sea level pressure (psl), + zonal wind (ua), and specific humidity (hus). Regions are defined using IPCC AR6 reference + regions as described by Iturbide et al. (2020). + tags: ["regional", "trends", "historical"] + +esmvaltool/sea-ice-area-basic: + reference_datasets: + - slug: "esmvaltool.OSI-450-nh" + description: "OSI-450 v3 - Sea ice concentration (siconc), Northern Hemisphere, hardcoded in recipe" + type: "primary" + - slug: "esmvaltool.OSI-450-sh" + description: "OSI-450 v3 - Sea ice concentration (siconc), Southern Hemisphere, hardcoded in recipe" + type: "primary" + display_name: "Arctic and Antarctic Sea Ice Area Seasonal Cycle" + description: >- + Calculate the seasonal cycle and time series of Northern Hemisphere and + Southern Hemisphere sea ice area. + tags: ["sea-ice", "cryosphere"] + +esmvaltool/sea-ice-sensitivity: + reference_datasets: [] + display_name: "Sea Ice Sensitivity" + description: >- + Calculate sea ice sensitivity, quantifying the relationship between sea ice area (siconc) + and global temperature change (tas). This is an inter-model comparison diagnostic + with no observational datasets ingested. + tags: ["sea-ice", "cryosphere", "sensitivity"] + +esmvaltool/climate-at-global-warming-levels: + reference_datasets: [] + display_name: "Climate at Global Warming Levels" + description: "Calculate climate variables at global warming levels (e.g. 1.5C, 2C, 3C, 4C above pre-industrial)." + tags: ["scenarios", "warming-levels"] + +esmvaltool/climate-drivers-for-fire: + reference_datasets: [] + display_name: "Climate Drivers for Fire" + description: "Calculate diagnostics regarding climate drivers for fire, including temperature, precipitation, and humidity conditions that influence fire risk." + tags: ["fire", "climate-drivers", "impacts"] diff --git a/backend/static/diagnostics/ilamb.yaml b/backend/static/diagnostics/ilamb.yaml new file mode 100644 index 0000000..e6713f6 --- /dev/null +++ b/backend/static/diagnostics/ilamb.yaml @@ -0,0 +1,134 @@ +# ILAMB Diagnostic Metadata +# +# Reference datasets and display metadata for ILAMB (International Land Model +# Benchmarking) and IOMB (International Ocean Model Benchmarking) diagnostics. + +# Land diagnostics + +ilamb/gpp-wecann: + reference_datasets: + - slug: "obs4REF.WECANN" + description: "WECANN GPP dataset - Primary gross primary productivity reference" + type: "primary" + - slug: "obs4MIPs.GPCPv2.3" + description: "GPCP v2.3 - Precipitation relationship" + type: "secondary" + - slug: "obs4MIPs.CRU4.02" + description: "CRU TS v4.02 - Temperature relationship" + type: "secondary" + display_name: "Gross Primary Productivity (WECANN)" + description: "Apply the standard ILAMB analysis of gross primary productivity (gpp) with respect to the WECANN reference dataset." + tags: ["land", "carbon-cycle", "vegetation"] + +ilamb/gpp-fluxnet2015: + reference_datasets: + - slug: "obs4REF.FLUXNET2015" + description: "FLUXNET2015 - Tower-based GPP measurements" + type: "primary" + display_name: "Gross Primary Productivity (FLUXNET2015)" + description: "Apply the standard ILAMB analysis of gross primary productivity (gpp) with respect to the FLUXNET2015 tower-based reference dataset." + tags: ["land", "carbon-cycle", "vegetation", "flux-towers"] + +ilamb/mrro-lora: + reference_datasets: + - slug: "obs4REF.LORA" + description: "LORA dataset - Runoff observations" + type: "primary" + display_name: "Runoff (LORA)" + description: "Apply the standard ILAMB analysis of total runoff (mrro) with respect to the LORA reference dataset." + tags: ["land", "hydrology", "runoff"] + +ilamb/mrsos-wangmao: + reference_datasets: + - slug: "obs4REF.WangMao" + description: "Wang-Mao dataset - Surface soil moisture" + type: "primary" + display_name: "Surface Soil Moisture (WangMao)" + description: "Apply the standard ILAMB analysis of moisture in upper portion of soil column (mrsos) with respect to the WangMao reference dataset." + tags: ["land", "hydrology", "soil-moisture"] + +ilamb/csoil-hwsd2: + reference_datasets: + - slug: "obs4REF.HWSD2" + description: "Harmonized World Soil Database v2 - Soil carbon content" + type: "primary" + display_name: "Soil Carbon (HWSD2)" + description: "Apply the standard ILAMB analysis of carbon mass in soil pool (cSoil) with respect to the HWSD2 reference dataset." + tags: ["land", "carbon-cycle", "soil"] + +ilamb/lai-avh15c1: + reference_datasets: + - slug: "obs4REF.AVH15C1" + description: "AVHRR - Leaf area index observations" + type: "primary" + display_name: "Leaf Area Index (AVH15C1)" + description: "Apply the standard ILAMB analysis of leaf area index (lai) with respect to the AVH15C1 (AVHRR) reference dataset." + tags: ["land", "vegetation", "remote-sensing"] + +ilamb/nbp-hoffman: + reference_datasets: + - slug: "obs4REF.Hoffman" + description: "Hoffman dataset - Net biome productivity" + type: "primary" + display_name: "Net Biome Productivity (Hoffman)" + description: "Apply the standard ILAMB analysis of net biome productivity (nbp) with respect to the Hoffman reference dataset." + tags: ["land", "carbon-cycle", "net-flux"] + +ilamb/snc-esacci: + reference_datasets: + - slug: "obs4REF.CCI-CryoClim-FSC" + description: "ESA CCI Snow - Snow cover fraction" + type: "primary" + display_name: "Snow Cover (ESA CCI)" + description: "Apply the standard ILAMB analysis of snow cover fraction (snc) with respect to the ESA CCI CryoClim fractional snow cover reference dataset." + tags: ["land", "cryosphere", "snow"] + +ilamb/burntfractionall-gfed: + reference_datasets: + - slug: "obs4REF.GFED" + description: "Global Fire Emissions Database - Burnt area fraction" + type: "primary" + display_name: "Burnt Fraction (GFED)" + description: "Apply the standard ILAMB analysis of burnt area fraction (burntFractionAll) with respect to the GFED reference dataset." + tags: ["land", "fire", "disturbance"] + +ilamb/emp-gleamgpcp: + reference_datasets: + - slug: "obs4REF.GLEAMv3.3a" + description: "GLEAM v3.3a - Evapotranspiration (et)" + type: "primary" + - slug: "obs4REF.GPCP-2-3" + description: "GPCP v2.3 - Precipitation (pr)" + type: "primary" + display_name: "Evaporation minus Precipitation (GLEAM/GPCP)" + description: "Apply the standard ILAMB analysis of evaporation minus precipitation (E-P) using GLEAMv3.3a evapotranspiration and GPCP v2.3 precipitation reference datasets." + tags: ["land", "hydrology", "water-balance"] + +# Ocean diagnostics (IOMB) + +ilamb/thetao-woa2023-surface: + reference_datasets: + - slug: "obs4REF.WOA2023" + description: "World Ocean Atlas 2023 - Sea water potential temperature (surface)" + type: "primary" + display_name: "Sea Water Temperature (WOA2023 Surface)" + description: "Apply the standard ILAMB analysis of sea water potential temperature (thetao) at the surface with respect to the WOA2023 reference dataset." + tags: ["ocean", "temperature", "surface"] + +ilamb/so-woa2023-surface: + reference_datasets: + - slug: "obs4REF.WOA2023" + description: "World Ocean Atlas 2023 - Sea water salinity (surface)" + type: "primary" + display_name: "Sea Water Salinity (WOA2023 Surface)" + description: "Apply the standard ILAMB analysis of sea water salinity (so) at the surface with respect to the WOA2023 reference dataset." + tags: ["ocean", "salinity", "surface"] + +ilamb/amoc-rapid: + reference_datasets: + - slug: "obs4REF.RAPID" + description: "RAPID array - Atlantic Meridional Overturning Circulation observations" + type: "primary" + display_name: "Atlantic Meridional Overturning Circulation (RAPID)" + description: "Apply the standard ILAMB analysis of the Atlantic Meridional Overturning Circulation (AMOC) with respect to the RAPID array reference dataset." + tags: ["ocean", "circulation", "amoc"] diff --git a/backend/static/diagnostics/metadata.yaml b/backend/static/diagnostics/metadata.yaml deleted file mode 100644 index 4d2a1ad..0000000 --- a/backend/static/diagnostics/metadata.yaml +++ /dev/null @@ -1,526 +0,0 @@ -# Diagnostic Metadata Overrides -# -# This file provides additional metadata for diagnostics that supplements or overrides -# the default values from the diagnostic implementations. It's particularly useful for: -# - Exposing which reference datasets are used by each diagnostic -# - Providing display names and descriptions -# - Adding tags for categorization -# - -# ============================================================================ -# PMP DIAGNOSTICS -# ============================================================================ - -pmp/annual-cycle: - reference_datasets: - - slug: "obs4MIPs-climatology.ERA-5" - description: "ERA5 Reanalysis - Primary climatology for surface temperature (ts), winds (uas, vas), and sea level pressure (psl)" - type: "primary" - - slug: "obs4MIPs-climatology.GPCP-Monthly-3-2" - description: "Global Precipitation Climatology Project v3.2 - Precipitation reference" - type: "primary" - - slug: "obs4MIPs-climatology.CERES-EBAF-4-2" - description: "CERES Energy Balanced and Filled v4.2 - Radiation fields (rlds, rlus, rsds, rsus, rlut, rsut)" - type: "primary" - display_name: "Annual Cycle Analysis" - description: >- - Calculate the annual cycle for a dataset. Variables provided include surface temperature (ts), - precipitation (pr), winds (uas, vas), sea level pressure (psl), and radiation fields - (rlds, rlus, rsds, rsus, rlut, rsut). - tags: ["atmosphere", "seasonal-cycle", "climatology"] - -pmp/enso_perf: - reference_datasets: - - slug: "obs4mips.GPCP-Monthly-3-2" - description: "Global Precipitation Climatology Project v3.2 - Precipitation patterns" - type: "primary" - - slug: "obs4mips.TropFlux-1-0" - description: "TropFlux v1.0 - Tropical flux data" - type: "primary" - - slug: "obs4mips.HadISST-1-1" - description: "Hadley Centre Sea Ice and Sea Surface Temperature v1.1 - SST reference" - type: "primary" - display_name: "ENSO Performance Metrics" - description: >- - Calculate the ENSO performance metrics for a dataset. These metrics evaluate - how well a model simulates key aspects of El Nino-Southern Oscillation variability. - tags: ["ocean", "atmosphere", "enso", "variability"] - -pmp/enso_tel: - reference_datasets: - - slug: "obs4mips.GPCP-Monthly-3-2" - description: "Global Precipitation Climatology Project v3.2 - Precipitation teleconnections" - type: "primary" - - slug: "obs4mips.TropFlux-1-0" - description: "TropFlux v1.0 - Tropical flux data" - type: "primary" - - slug: "obs4mips.HadISST-1-1" - description: "Hadley Centre Sea Ice and SST v1.1 - SST teleconnections" - type: "primary" - display_name: "ENSO Teleconnections" - description: >- - Calculate the ENSO teleconnection metrics for a dataset. Teleconnection metrics - assess how well a model captures the remote climate impacts of ENSO events - on temperature and precipitation patterns worldwide. - tags: ["ocean", "atmosphere", "enso", "teleconnections"] - -pmp/enso_proc: - reference_datasets: - - slug: "obs4mips.GPCP-Monthly-3-2" - description: "Global Precipitation Climatology Project v3.2" - type: "primary" - - slug: "obs4mips.TropFlux-1-0" - description: "TropFlux v1.0 - Surface fluxes" - type: "primary" - - slug: "obs4mips.HadISST-1-1" - description: "Hadley Centre Sea Ice and SST v1.1" - type: "primary" - - slug: "obs4mips.CERES-EBAF-4-2" - description: "CERES Energy Balanced and Filled v4.2 - Radiation fluxes" - type: "primary" - display_name: "ENSO Processes" - description: >- - Calculate the ENSO process-oriented metrics for a dataset. These metrics evaluate - the physical processes that drive ENSO, including ocean-atmosphere coupling, - thermocline feedbacks, and energy balance. - tags: ["ocean", "atmosphere", "enso", "energy-balance"] - -# Extratropical Modes of Variability (Sea Surface Temperature modes) -pmp/extratropical-modes-of-variability-pdo: - reference_datasets: - - slug: "obs4mips.HadISST-1-1" - description: "Hadley Centre Sea Ice and SST v1.1 - SST for Pacific Decadal Oscillation" - type: "primary" - display_name: "Pacific Decadal Oscillation (PDO)" - description: >- - Calculate the extratropical modes of variability for the Pacific Decadal Oscillation (PDO), - a pattern of Pacific climate variability characterized by SST anomalies in the North Pacific. - tags: ["ocean", "variability", "pacific", "decadal"] - -pmp/extratropical-modes-of-variability-npgo: - reference_datasets: - - slug: "obs4mips.HadISST-1-1" - description: "Hadley Centre Sea Ice and SST v1.1 - SST for North Pacific Gyre Oscillation" - type: "primary" - display_name: "North Pacific Gyre Oscillation (NPGO)" - description: >- - Calculate the extratropical modes of variability for the North Pacific Gyre Oscillation (NPGO), - which tracks changes in the strength of the North Pacific subtropical and subpolar gyres. - tags: ["ocean", "variability", "pacific"] - -pmp/extratropical-modes-of-variability-amo: - reference_datasets: - - slug: "obs4mips.HadISST-1-1" - description: "Hadley Centre Sea Ice and SST v1.1 - SST for Atlantic Multidecadal Oscillation" - type: "primary" - display_name: "Atlantic Multidecadal Oscillation (AMO)" - description: >- - Calculate the extratropical modes of variability for the Atlantic Multidecadal Oscillation (AMO), - a coherent pattern of variability in North Atlantic SSTs with a period of 60-80 years. - tags: ["ocean", "variability", "atlantic", "decadal"] - -# Extratropical Modes of Variability (Sea Level Pressure modes) -pmp/extratropical-modes-of-variability-nao: - reference_datasets: - - slug: "obs4mips.20CR" - description: "20th Century Reanalysis - Sea level pressure for North Atlantic Oscillation" - type: "primary" - display_name: "North Atlantic Oscillation (NAO)" - description: >- - Calculate the extratropical modes of variability for the North Atlantic Oscillation (NAO), - a large-scale pattern of sea level pressure variability between the Icelandic Low and Azores High. - tags: ["atmosphere", "variability", "atlantic", "pressure"] - -pmp/extratropical-modes-of-variability-nam: - reference_datasets: - - slug: "obs4mips.20CR" - description: "20th Century Reanalysis - Sea level pressure for Northern Annular Mode" - type: "primary" - display_name: "Northern Annular Mode (NAM)" - description: >- - Calculate the extratropical modes of variability for the Northern Annular Mode (NAM), - the dominant mode of sea level pressure variability in the Northern Hemisphere extratropics. - tags: ["atmosphere", "variability", "hemispheric", "pressure"] - -pmp/extratropical-modes-of-variability-pna: - reference_datasets: - - slug: "obs4mips.20CR" - description: "20th Century Reanalysis - Sea level pressure for Pacific North American Pattern" - type: "primary" - display_name: "Pacific North American Pattern (PNA)" - description: >- - Calculate the extratropical modes of variability for the Pacific North American (PNA) pattern, - a prominent mode of atmospheric variability influencing weather over the Pacific and North America. - tags: ["atmosphere", "variability", "pacific", "pressure"] - -pmp/extratropical-modes-of-variability-npo: - reference_datasets: - - slug: "obs4mips.20CR" - description: "20th Century Reanalysis - Sea level pressure for North Pacific Oscillation" - type: "primary" - display_name: "North Pacific Oscillation (NPO)" - description: >- - Calculate the extratropical modes of variability for the North Pacific Oscillation (NPO), - defined by sea level pressure variability over the North Pacific. - tags: ["atmosphere", "variability", "pacific", "pressure"] - -pmp/extratropical-modes-of-variability-sam: - reference_datasets: - - slug: "obs4mips.20CR" - description: "20th Century Reanalysis - Sea level pressure for Southern Annular Mode" - type: "primary" - display_name: "Southern Annular Mode (SAM)" - description: >- - Calculate the extratropical modes of variability for the Southern Annular Mode (SAM), - the dominant mode of sea level pressure variability in the Southern Hemisphere extratropics. - tags: ["atmosphere", "variability", "southern-hemisphere", "pressure"] - -# ============================================================================ -# ESMValTool DIAGNOSTICS -# ============================================================================ - -esmvaltool/cloud-radiative-effects: - reference_datasets: - - slug: "esmvaltool.CERES-EBAF" - description: "CERES Energy Balanced and Filled - TOA radiation fluxes (planned to move to obs4MIPs)" - type: "primary" - - slug: "esmvaltool.ESACCI-CLOUD" - description: "ESA CCI Cloud properties (planned to move to obs4MIPs)" - type: "secondary" - - slug: "esmvaltool.ISCCP-FH" - description: "ISCCP cloud properties (planned to move to obs4MIPs)" - type: "secondary" - display_name: "Cloud Radiative Effects" - description: >- - Plot climatologies and zonal mean profiles of cloud radiative effects - (shortwave + longwave) for a dataset. - tags: ["atmosphere", "clouds", "radiation"] - -esmvaltool/cloud-scatterplots-cli-ta: - reference_datasets: - - slug: "obs4mips.ERA-5" - description: "ERA5 - Air temperature (ta)" - type: "primary" - - slug: "esmvaltool.CALIPSO-ICECLOUD" - description: "Ice Cloud properties (planned to move to obs4MIPs)" - type: "secondary" - display_name: "Cloud-Temperature Scatterplots (cli vs ta)" - description: >- - Scatterplot of ice water content of cloud (cli) vs air temperature (ta). - Observational scatterplots for reference datasets are provided in the separate - "Cloud Scatterplots for Reference dataset" diagnostic. - tags: ["atmosphere", "clouds", "temperature"] - -esmvaltool/cloud-scatterplots-clivi-lwcre: - reference_datasets: - - slug: "esmvaltool.CERES-EBAF" - description: "CERES Energy Balanced and Filled - TOA radiation fluxes (planned)" - type: "primary" - - slug: "esmvaltool.ESACCI-CLOUD" - description: "ESA CCI Cloud properties (planned to move to obs4MIPs)" - type: "secondary" - display_name: "Cloud-Radiation Scatterplots (clivi vs lwcre)" - description: "Scatterplot of ice water path (clivi) vs longwave cloud radiative effect (lwcre)." - tags: ["atmosphere", "clouds", "radiation"] - -esmvaltool/cloud-scatterplots-clt-swcre: - reference_datasets: - - slug: "esmvaltool.CERES-EBAF" - description: "CERES Energy Balanced and Filled - TOA radiation fluxes (planned to move to obs4MIPs)" - type: "primary" - - slug: "esmvaltool.ESACCI-CLOUD" - description: "ESA CCI Cloud properties (planned to move to obs4MIPs)" - type: "secondary" - display_name: "Cloud-Radiation Scatterplots (clt vs swcre)" - description: "Scatterplot of total cloud fraction (clt) vs shortwave cloud radiative effect (swcre)." - tags: ["atmosphere", "clouds", "radiation"] - -esmvaltool/cloud-scatterplots-clwvi-pr: - reference_datasets: - - slug: "obs4MIPs.GPCPv2.3" - description: "GPCP v2.3 - Precipitation relationship" - type: "secondary" - - slug: "esmvaltool.ESACCI-CLOUD" - description: "ESA CCI Cloud properties (planned to move to obs4MIPs)" - type: "secondary" - - slug: "esmvaltool.CERES-EBAF" - description: "CERES - Reference cloud-radiation relationships" - type: "primary" - display_name: "Cloud-Precipitation Scatterplots (clwvi vs pr)" - description: "Scatterplot of condensed water path (clwvi) vs precipitation (pr)." - tags: ["atmosphere", "clouds", "precipitation"] - -esmvaltool/cloud-scatterplots-reference: - reference_datasets: - - slug: "esmvaltool.CERES-EBAF" - description: "CERES - Reference cloud-radiation relationships" - type: "primary" - - slug: "esmvaltool.ESACCI-CLOUD" - description: "ESA CCI Cloud properties (planned to move to obs4MIPs)" - type: "secondary" - display_name: "Cloud Scatterplots for Reference dataset" - description: >- - Reference scatterplots of two cloud-relevant variables from observational datasets. - These provide the observational baseline for comparison with model cloud scatterplot diagnostics. - tags: ["atmosphere", "clouds"] - -esmvaltool/enso-basic-climatology: - reference_datasets: - - slug: "obs4MIPs.GPCPv2.3" - description: "GPCP v2.3 - Precipitation relationship" - type: "secondary" - - slug: "esmvaltool.TROPFLUX" - description: "TROPFLUX - Air-sea fluxes" - type: "secondary" - - slug: "esmvaltool.CALIPSO-ICECLOUD" - description: "Ice Cloud properties (planned to move to obs4MIPs)" - type: "secondary" - display_name: "ENSO Basic Climatology" - description: >- - Calculate the ENSO CLIVAR metrics for background climatology, assessing the - mean state of the tropical Pacific upon which ENSO events develop. - tags: ["ocean", "enso", "climatology"] - -esmvaltool/enso-characteristics: - reference_datasets: - - slug: "obs4MIPs.GPCPv2.3" - description: "GPCP v2.3 - Precipitation relationship" - type: "secondary" - - slug: "esmvaltool.TROPFLUX" - description: "TROPFLUX - Air-sea fluxes" - type: "secondary" - display_name: "ENSO Characteristics" - description: >- - Calculate the ENSO CLIVAR metrics for basic ENSO characteristics, - evaluating the amplitude, frequency, and spatial structure of ENSO events. - tags: ["ocean", "enso", "variability"] - -esmvaltool/equilibrium-climate-sensitivity: - reference_datasets: [] - display_name: "Equilibrium Climate Sensitivity (ECS)" - description: >- - Calculate the global mean equilibrium climate sensitivity for a dataset. - ECS is defined as the long-term global mean surface temperature change - in response to a doubling of atmospheric CO2 concentration, estimated - using the Gregory regression method from abrupt-4xCO2 experiments. - tags: ["climate-sensitivity", "feedback"] - -esmvaltool/transient-climate-response: - reference_datasets: [] - display_name: "Transient Climate Response (TCR)" - description: >- - Calculate the global mean transient climate response for a dataset. - TCR is defined as the global mean surface temperature change at the time - of CO2 doubling in a 1pctCO2 experiment (1% per year CO2 increase). - tags: ["climate-sensitivity", "transient"] - -esmvaltool/transient-climate-response-emissions: - reference_datasets: [] - display_name: "Transient Climate Response to Emissions (TCRE)" - description: >- - Calculate the global mean Transient Climate Response to Cumulative CO2 Emissions. - TCRE quantifies the relationship between global warming and total cumulative - carbon emissions, a key metric for carbon budget estimation. - tags: ["climate-sensitivity", "carbon-cycle"] - -esmvaltool/zero-emission-commitment: - reference_datasets: [] - display_name: "Zero Emission Commitment (ZEC)" - description: >- - Calculate the global mean Zero Emission Commitment (ZEC) temperature. - ZEC measures the additional warming (or cooling) that occurs after all - CO2 emissions cease, reflecting the committed warming from past emissions. - tags: ["climate-sensitivity", "commitment"] - -esmvaltool/regional-historical-annual-cycle: - reference_datasets: - - slug: "obs4mips.ERA-5" - description: "ERA-5 - Regional winds (uas, vas), and sea level pressure (psl)" - type: "primary" - display_name: "Regional Historical Annual Cycle" - description: >- - Plot the regional historical annual cycle of climate variables. Currently limited to - near-surface air temperature (tas) and precipitation (pr). Regions are defined using - IPCC AR6 reference regions as described by Iturbide et al. (2020). - tags: ["regional", "seasonal-cycle", "historical"] - -esmvaltool/regional-historical-timeseries: - reference_datasets: - - slug: "obs4mips.ERA-5" - description: "ERA-5 - Regional winds (uas, vas), and sea level pressure (psl)" - type: "primary" - display_name: "Regional Historical Timeseries" - description: >- - Plot the regional historical mean and anomaly timeseries of climate variables. Currently - limited to near-surface air temperature (tas) and precipitation (pr). Regions are defined - using IPCC AR6 reference regions as described by Iturbide et al. (2020). - tags: ["regional", "timeseries", "historical"] - -esmvaltool/regional-historical-trend: - reference_datasets: - - slug: "obs4mips.ERA-5" - description: "ERA-5 - Regional winds (uas, vas), and sea level pressure (psl)" - type: "primary" - display_name: "Regional Historical Trends" - description: >- - Plot the regional historical trend of climate variables. Currently limited to near-surface - air temperature (tas) and precipitation (pr). Regions are defined using IPCC AR6 reference - regions as described by Iturbide et al. (2020). - tags: ["regional", "trends", "historical"] - -esmvaltool/sea-ice-area-basic: - reference_datasets: - - slug: "esmvaltool.OSI-450" - description: "OSI-450 - Sea ice area observations" - type: "primary" - display_name: "Arctic and Antarctic Sea Ice Area Seasonal Cycle" - description: >- - Calculate the seasonal cycle and time series of Northern Hemisphere and - Southern Hemisphere sea ice area. - tags: ["sea-ice", "cryosphere"] - -esmvaltool/sea-ice-sensitivity: - reference_datasets: - - slug: "esmvaltool.OSI-450" - description: "OSI-450 - Sea ice area observations" - type: "primary" - - slug: "esmvaltool.HadCRUT5" - description: "HadCRUT5 - Global mean surface temperature for sensitivity regression" - type: "primary" - display_name: "Sea Ice Sensitivity" - description: "Calculate sea ice sensitivity, quantifying the relationship between sea ice area and global temperature change." - tags: ["sea-ice", "cryosphere", "sensitivity"] - -esmvaltool/climate-at-global-warming-levels: - reference_datasets: [] - display_name: "Climate at Global Warming Levels" - description: "Calculate climate variables at global warming levels (e.g. 1.5C, 2C, 3C, 4C above pre-industrial)." - tags: ["scenarios", "warming-levels"] - -esmvaltool/climate-drivers-for-fire: - reference_datasets: [] - display_name: "Climate Drivers for Fire" - description: "Calculate diagnostics regarding climate drivers for fire, including temperature, precipitation, and humidity conditions that influence fire risk." - tags: ["fire", "climate-drivers", "impacts"] - -# ============================================================================ -# ILAMB DIAGNOSTICS (Land) -# ============================================================================ - -ilamb/gpp-wecann: - reference_datasets: - - slug: "obs4REF.WECANN" - description: "WECANN GPP dataset - Primary gross primary productivity reference" - type: "primary" - - slug: "obs4MIPs.GPCPv2.3" - description: "GPCP v2.3 - Precipitation relationship" - type: "secondary" - - slug: "obs4MIPs.CRU4.02" - description: "CRU TS v4.02 - Temperature relationship" - type: "secondary" - display_name: "Gross Primary Productivity (WECANN)" - description: "Apply the standard ILAMB analysis of gross primary productivity (gpp) with respect to the WECANN reference dataset." - tags: ["land", "carbon-cycle", "vegetation"] - -ilamb/gpp-fluxnet2015: - reference_datasets: - - slug: "obs4REF.FLUXNET2015" - description: "FLUXNET2015 - Tower-based GPP measurements" - type: "primary" - display_name: "Gross Primary Productivity (FLUXNET2015)" - description: "Apply the standard ILAMB analysis of gross primary productivity (gpp) with respect to the FLUXNET2015 tower-based reference dataset." - tags: ["land", "carbon-cycle", "vegetation", "flux-towers"] - -ilamb/mrro-lora: - reference_datasets: - - slug: "obs4REF.LORA" - description: "LORA dataset - Runoff observations" - type: "primary" - display_name: "Runoff (LORA)" - description: "Apply the standard ILAMB analysis of total runoff (mrro) with respect to the LORA reference dataset." - tags: ["land", "hydrology", "runoff"] - -ilamb/mrsos-wangmao: - reference_datasets: - - slug: "obs4REF.WangMao" - description: "Wang-Mao dataset - Surface soil moisture" - type: "primary" - display_name: "Surface Soil Moisture (WangMao)" - description: "Apply the standard ILAMB analysis of moisture in upper portion of soil column (mrsos) with respect to the WangMao reference dataset." - tags: ["land", "hydrology", "soil-moisture"] - -ilamb/csoil-hwsd2: - reference_datasets: - - slug: "obs4REF.HWSD2" - description: "Harmonized World Soil Database v2 - Soil carbon content" - type: "primary" - display_name: "Soil Carbon (HWSD2)" - description: "Apply the standard ILAMB analysis of carbon mass in soil pool (cSoil) with respect to the HWSD2 reference dataset." - tags: ["land", "carbon-cycle", "soil"] - -ilamb/lai-avh15c1: - reference_datasets: - - slug: "obs4REF.AVH15C1" - description: "AVHRR - Leaf area index observations" - type: "primary" - display_name: "Leaf Area Index (AVH15C1)" - description: "Apply the standard ILAMB analysis of leaf area index (lai) with respect to the AVH15C1 (AVHRR) reference dataset." - tags: ["land", "vegetation", "remote-sensing"] - -ilamb/nbp-hoffman: - reference_datasets: - - slug: "obs4REF.Hoffman" - description: "Hoffman dataset - Net biome productivity" - type: "primary" - display_name: "Net Biome Productivity (Hoffman)" - description: "Apply the standard ILAMB analysis of net biome productivity (nbp) with respect to the Hoffman reference dataset." - tags: ["land", "carbon-cycle", "net-flux"] - -ilamb/snc-esacci: - reference_datasets: - - slug: "obs4REF.CCI-CryoClim-FSC" - description: "ESA CCI Snow - Snow cover fraction" - type: "primary" - display_name: "Snow Cover (ESA CCI)" - description: "Apply the standard ILAMB analysis of snow cover fraction (snc) with respect to the ESA CCI CryoClim fractional snow cover reference dataset." - tags: ["land", "cryosphere", "snow"] - -ilamb/burntfractionall-gfed: - reference_datasets: - - slug: "obs4REF.GFED" - description: "Global Fire Emissions Database - Burnt area fraction" - type: "primary" - display_name: "Burnt Fraction (GFED)" - description: "Apply the standard ILAMB analysis of burnt area fraction (burntFractionAll) with respect to the GFED reference dataset." - tags: ["land", "fire", "disturbance"] - -# ============================================================================ -# ILAMB/IOMB DIAGNOSTICS (Ocean) -# ============================================================================ - -ilamb/thetao-woa2023-surface: - reference_datasets: - - slug: "obs4REF.WOA2023" - description: "World Ocean Atlas 2023 - Sea water potential temperature (surface)" - type: "primary" - display_name: "Sea Water Temperature (WOA2023 Surface)" - description: "Apply the standard ILAMB analysis of sea water potential temperature (thetao) at the surface with respect to the WOA2023 reference dataset." - tags: ["ocean", "temperature", "surface"] - -ilamb/so-woa2023-surface: - reference_datasets: - - slug: "obs4REF.WOA2023" - description: "World Ocean Atlas 2023 - Sea water salinity (surface)" - type: "primary" - display_name: "Sea Water Salinity (WOA2023 Surface)" - description: "Apply the standard ILAMB analysis of sea water salinity (so) at the surface with respect to the WOA2023 reference dataset." - tags: ["ocean", "salinity", "surface"] - -ilamb/amoc-rapid: - reference_datasets: - - slug: "obs4REF.RAPID" - description: "RAPID array - Atlantic Meridional Overturning Circulation observations" - type: "primary" - display_name: "Atlantic Meridional Overturning Circulation (RAPID)" - description: "Apply the standard ILAMB analysis of the Atlantic Meridional Overturning Circulation (AMOC) with respect to the RAPID array reference dataset." - tags: ["ocean", "circulation", "amoc"] diff --git a/backend/static/diagnostics/pmp.yaml b/backend/static/diagnostics/pmp.yaml new file mode 100644 index 0000000..b9caf1d --- /dev/null +++ b/backend/static/diagnostics/pmp.yaml @@ -0,0 +1,168 @@ +# PMP Diagnostic Metadata +# +# Reference datasets and display metadata for PMP (PCMDI Metrics Package) diagnostics. + +pmp/annual-cycle: + reference_datasets: + - slug: "obs4MIPs-climatology.ERA-5" + description: "ERA5 Reanalysis - Primary climatology for surface temperature (ts), winds (uas, vas), sea level pressure (psl), air temperature (ta), zonal/meridional wind (ua, va), and geopotential height (zg). Period: 1981-2004." + type: "primary" + - slug: "obs4MIPs-climatology.GPCP-Monthly-3-2" + description: "Global Precipitation Climatology Project v3.2 - Precipitation reference (pr). Period: 1983-2004." + type: "primary" + - slug: "obs4MIPs-climatology.CERES-EBAF-4-2" + description: "CERES Energy Balanced and Filled v4.2 - Radiation fields (rlds, rlus, rlut, rsds, rsdt, rsus, rsut) and clear-sky variants (rldscs, rsdscs, rlutcs, rsutcs, rsuscs, rltcre, rstcre, rt). Period: 2001-2004." + type: "primary" + display_name: "Annual Cycle Analysis" + description: >- + Calculate the annual cycle for a dataset. Variables provided include surface temperature (ts), + precipitation (pr), winds (uas, vas), sea level pressure (psl), air temperature (ta), + zonal/meridional wind (ua, va), geopotential height (zg), and radiation fields + (rlds, rlus, rsds, rsdt, rsus, rlut, rsut) including clear-sky variants. + tags: ["atmosphere", "seasonal-cycle", "climatology"] + +pmp/enso_perf: + reference_datasets: + - slug: "obs4mips.GPCP-Monthly-3-2" + description: "Global Precipitation Climatology Project v3.2 - Precipitation patterns" + type: "primary" + - slug: "obs4mips.TropFlux-1-0" + description: "TropFlux v1.0 - Tropical flux data" + type: "primary" + - slug: "obs4mips.HadISST-1-1" + description: "Hadley Centre Sea Ice and Sea Surface Temperature v1.1 - SST reference" + type: "primary" + display_name: "ENSO Performance Metrics" + description: >- + Calculate the ENSO performance metrics for a dataset. These metrics evaluate + how well a model simulates key aspects of El Nino-Southern Oscillation variability. + tags: ["ocean", "atmosphere", "enso", "variability"] + +pmp/enso_tel: + reference_datasets: + - slug: "obs4mips.GPCP-Monthly-3-2" + description: "Global Precipitation Climatology Project v3.2 - Precipitation teleconnections" + type: "primary" + - slug: "obs4mips.TropFlux-1-0" + description: "TropFlux v1.0 - Tropical flux data" + type: "primary" + - slug: "obs4mips.HadISST-1-1" + description: "Hadley Centre Sea Ice and SST v1.1 - SST teleconnections" + type: "primary" + display_name: "ENSO Teleconnections" + description: >- + Calculate the ENSO teleconnection metrics for a dataset. Teleconnection metrics + assess how well a model captures the remote climate impacts of ENSO events + on temperature and precipitation patterns worldwide. + tags: ["ocean", "atmosphere", "enso", "teleconnections"] + +pmp/enso_proc: + reference_datasets: + - slug: "obs4mips.GPCP-Monthly-3-2" + description: "Global Precipitation Climatology Project v3.2" + type: "primary" + - slug: "obs4mips.TropFlux-1-0" + description: "TropFlux v1.0 - Surface fluxes" + type: "primary" + - slug: "obs4mips.HadISST-1-1" + description: "Hadley Centre Sea Ice and SST v1.1" + type: "primary" + - slug: "obs4mips.CERES-EBAF-4-2" + description: "CERES Energy Balanced and Filled v4.2 - Surface and TOA radiation fluxes (hfls, hfss, rlds, rlus, rsds, rsus)" + type: "primary" + display_name: "ENSO Processes" + description: >- + Calculate the ENSO process-oriented metrics for a dataset. These metrics evaluate + the physical processes that drive ENSO, including ocean-atmosphere coupling, + thermocline feedbacks, and energy balance. + tags: ["ocean", "atmosphere", "enso", "energy-balance"] + +# Extratropical Modes of Variability (Sea Surface Temperature modes) +pmp/extratropical-modes-of-variability-pdo: + reference_datasets: + - slug: "obs4mips.HadISST-1-1" + description: "Hadley Centre Sea Ice and SST v1.1 - SST for Pacific Decadal Oscillation" + type: "primary" + display_name: "Pacific Decadal Oscillation (PDO)" + description: >- + Calculate the extratropical modes of variability for the Pacific Decadal Oscillation (PDO), + a pattern of Pacific climate variability characterized by SST anomalies in the North Pacific. + tags: ["ocean", "variability", "pacific", "decadal"] + +pmp/extratropical-modes-of-variability-npgo: + reference_datasets: + - slug: "obs4mips.HadISST-1-1" + description: "Hadley Centre Sea Ice and SST v1.1 - SST for North Pacific Gyre Oscillation" + type: "primary" + display_name: "North Pacific Gyre Oscillation (NPGO)" + description: >- + Calculate the extratropical modes of variability for the North Pacific Gyre Oscillation (NPGO), + which tracks changes in the strength of the North Pacific subtropical and subpolar gyres. + tags: ["ocean", "variability", "pacific"] + +pmp/extratropical-modes-of-variability-amo: + reference_datasets: + - slug: "obs4mips.HadISST-1-1" + description: "Hadley Centre Sea Ice and SST v1.1 - SST for Atlantic Multidecadal Oscillation" + type: "primary" + display_name: "Atlantic Multidecadal Oscillation (AMO)" + description: >- + Calculate the extratropical modes of variability for the Atlantic Multidecadal Oscillation (AMO), + a coherent pattern of variability in North Atlantic SSTs with a period of 60-80 years. + tags: ["ocean", "variability", "atlantic", "decadal"] + +# Extratropical Modes of Variability (Sea Level Pressure modes) +pmp/extratropical-modes-of-variability-nao: + reference_datasets: + - slug: "obs4mips.20CR" + description: "20th Century Reanalysis - Sea level pressure for North Atlantic Oscillation" + type: "primary" + display_name: "North Atlantic Oscillation (NAO)" + description: >- + Calculate the extratropical modes of variability for the North Atlantic Oscillation (NAO), + a large-scale pattern of sea level pressure variability between the Icelandic Low and Azores High. + tags: ["atmosphere", "variability", "atlantic", "pressure"] + +pmp/extratropical-modes-of-variability-nam: + reference_datasets: + - slug: "obs4mips.20CR" + description: "20th Century Reanalysis - Sea level pressure for Northern Annular Mode" + type: "primary" + display_name: "Northern Annular Mode (NAM)" + description: >- + Calculate the extratropical modes of variability for the Northern Annular Mode (NAM), + the dominant mode of sea level pressure variability in the Northern Hemisphere extratropics. + tags: ["atmosphere", "variability", "hemispheric", "pressure"] + +pmp/extratropical-modes-of-variability-pna: + reference_datasets: + - slug: "obs4mips.20CR" + description: "20th Century Reanalysis - Sea level pressure for Pacific North American Pattern" + type: "primary" + display_name: "Pacific North American Pattern (PNA)" + description: >- + Calculate the extratropical modes of variability for the Pacific North American (PNA) pattern, + a prominent mode of atmospheric variability influencing weather over the Pacific and North America. + tags: ["atmosphere", "variability", "pacific", "pressure"] + +pmp/extratropical-modes-of-variability-npo: + reference_datasets: + - slug: "obs4mips.20CR" + description: "20th Century Reanalysis - Sea level pressure for North Pacific Oscillation" + type: "primary" + display_name: "North Pacific Oscillation (NPO)" + description: >- + Calculate the extratropical modes of variability for the North Pacific Oscillation (NPO), + defined by sea level pressure variability over the North Pacific. + tags: ["atmosphere", "variability", "pacific", "pressure"] + +pmp/extratropical-modes-of-variability-sam: + reference_datasets: + - slug: "obs4mips.20CR" + description: "20th Century Reanalysis - Sea level pressure for Southern Annular Mode" + type: "primary" + display_name: "Southern Annular Mode (SAM)" + description: >- + Calculate the extratropical modes of variability for the Southern Annular Mode (SAM), + the dominant mode of sea level pressure variability in the Southern Hemisphere extratropics. + tags: ["atmosphere", "variability", "southern-hemisphere", "pressure"] diff --git a/backend/tests/test_api/test_routes/test_aft.py b/backend/tests/test_api/test_routes/test_aft.py index b345604..46d779b 100644 --- a/backend/tests/test_api/test_routes/test_aft.py +++ b/backend/tests/test_api/test_routes/test_aft.py @@ -2,9 +2,9 @@ from fastapi.testclient import TestClient from ref_backend.core.aft import ( + _build_ref_to_aft_index, get_aft_diagnostic_by_id, get_aft_diagnostics_index, - get_aft_for_ref_diagnostic, load_official_aft_diagnostics, ) from ref_backend.core.collections import ( @@ -22,7 +22,7 @@ def clear_caches(): load_official_aft_diagnostics, get_aft_diagnostics_index, get_aft_diagnostic_by_id, - get_aft_for_ref_diagnostic, + _build_ref_to_aft_index, ]: fn.cache_clear() load_all_collections.cache_clear() @@ -31,7 +31,7 @@ def clear_caches(): load_official_aft_diagnostics, get_aft_diagnostics_index, get_aft_diagnostic_by_id, - get_aft_for_ref_diagnostic, + _build_ref_to_aft_index, ]: fn.cache_clear() load_all_collections.cache_clear() diff --git a/backend/tests/test_core/test_diagnostic_metadata.py b/backend/tests/test_core/test_diagnostic_metadata.py index 454c09d..53b448a 100644 --- a/backend/tests/test_core/test_diagnostic_metadata.py +++ b/backend/tests/test_core/test_diagnostic_metadata.py @@ -177,6 +177,81 @@ def test_all_reference_dataset_types(self, tmp_path: Path): assert refs[1].type == "secondary" assert refs[2].type == "comparison" + def test_directory_loading(self, tmp_path: Path): + """Test that loading from a directory merges all YAML files.""" + pmp_content = { + "pmp/annual-cycle": { + "display_name": "Annual Cycle", + "reference_datasets": [ + {"slug": "obs4mips.ERA-5", "type": "primary"}, + ], + }, + } + ilamb_content = { + "ilamb/gpp-wecann": { + "display_name": "GPP (WECANN)", + "reference_datasets": [ + {"slug": "obs4REF.WECANN", "type": "primary"}, + ], + }, + } + + metadata_dir = tmp_path / "diagnostics" + metadata_dir.mkdir() + with open(metadata_dir / "pmp.yaml", "w") as f: + yaml.dump(pmp_content, f) + with open(metadata_dir / "ilamb.yaml", "w") as f: + yaml.dump(ilamb_content, f) + + result = load_diagnostic_metadata(metadata_dir) + + assert len(result) == 2 + assert "pmp/annual-cycle" in result + assert "ilamb/gpp-wecann" in result + assert result["pmp/annual-cycle"].display_name == "Annual Cycle" + assert result["ilamb/gpp-wecann"].display_name == "GPP (WECANN)" + + def test_directory_with_no_yaml_files(self, tmp_path: Path): + """Test that an empty directory returns an empty dict.""" + metadata_dir = tmp_path / "empty_dir" + metadata_dir.mkdir() + + result = load_diagnostic_metadata(metadata_dir) + + assert result == {} + + def test_directory_duplicate_keys_warns(self, tmp_path: Path, caplog): + """Test that duplicate keys across files produce a warning and last file wins.""" + file1_content = { + "pmp/annual-cycle": { + "display_name": "From File 1", + "reference_datasets": [ + {"slug": "dataset1", "type": "primary"}, + ], + }, + } + file2_content = { + "pmp/annual-cycle": { + "display_name": "From File 2", + "reference_datasets": [ + {"slug": "dataset2", "type": "secondary"}, + ], + }, + } + + metadata_dir = tmp_path / "diagnostics" + metadata_dir.mkdir() + # Files are loaded in sorted order, so b.yaml overrides a.yaml + with open(metadata_dir / "a.yaml", "w") as f: + yaml.dump(file1_content, f) + with open(metadata_dir / "b.yaml", "w") as f: + yaml.dump(file2_content, f) + + result = load_diagnostic_metadata(metadata_dir) + + assert len(result) == 1 + assert result["pmp/annual-cycle"].display_name == "From File 2" + def test_optional_fields(self, tmp_path: Path): """Test that optional fields (display_name, tags, description) work correctly.""" yaml_content = { diff --git a/frontend/src/client/schemas.gen.ts b/frontend/src/client/schemas.gen.ts index 5db9742..2dc5c55 100644 --- a/frontend/src/client/schemas.gen.ts +++ b/frontend/src/client/schemas.gen.ts @@ -227,6 +227,20 @@ export const AFTCollectionCardContentSchema = { type: 'null' } ] + }, + reference_datasets: { + anyOf: [ + { + items: { + '$ref': '#/components/schemas/ReferenceDatasetLink' + }, + type: 'array' + }, + { + type: 'null' + } + ], + title: 'Reference Datasets' } }, type: 'object', diff --git a/frontend/src/client/types.gen.ts b/frontend/src/client/types.gen.ts index 801e7fd..a9ad98d 100644 --- a/frontend/src/client/types.gen.ts +++ b/frontend/src/client/types.gen.ts @@ -29,6 +29,7 @@ export type AftCollectionCardContent = { [key: string]: string; } | null; grouping_config?: AftCollectionGroupingConfig | null; + reference_datasets?: Array | null; }; export type AftCollectionContent = { diff --git a/frontend/src/components/explorer/explorerCardContent.tsx b/frontend/src/components/explorer/explorerCardContent.tsx index 43410c9..352e4b0 100644 --- a/frontend/src/components/explorer/explorerCardContent.tsx +++ b/frontend/src/components/explorer/explorerCardContent.tsx @@ -1,4 +1,6 @@ +import type { ReferenceDatasetLink } from "@/client/types.gen"; import { cn } from "@/lib/utils"; +import { Badge } from "../ui/badge"; import { Card, CardContent, @@ -7,6 +9,7 @@ import { CardHeader, CardTitle, } from "../ui/card"; +import { Tooltip, TooltipContent, TooltipTrigger } from "../ui/tooltip"; import { EnsembleChartContent, FigureGalleryContent, @@ -95,7 +98,7 @@ export function ExplorerCardContent({ contentItem }: ExplorerCardContentProps) { - + {contentItem.interpretation && (
@@ -104,6 +107,40 @@ export function ExplorerCardContent({ contentItem }: ExplorerCardContentProps) { {contentItem.interpretation}
)} + {contentItem.referenceDatasets && + contentItem.referenceDatasets.length > 0 && ( +
+ + Reference datasets: + + {contentItem.referenceDatasets.map( + (ref: ReferenceDatasetLink) => ( + + + + {ref.slug.split(".").pop()} + + + +
+
{ref.slug}
+ {ref.description && ( +
+ {ref.description} +
+ )} +
+ + + ), + )} +
+ )}
diff --git a/frontend/src/components/explorer/thematicContent.tsx b/frontend/src/components/explorer/thematicContent.tsx index 70e456f..903f2e4 100644 --- a/frontend/src/components/explorer/thematicContent.tsx +++ b/frontend/src/components/explorer/thematicContent.tsx @@ -48,6 +48,7 @@ function toExplorerCardContent( interpretation: apiContent.interpretation ?? undefined, span: apiContent.span ?? undefined, placeholder: apiContent.placeholder ?? undefined, + referenceDatasets: apiContent.reference_datasets ?? undefined, }; switch (apiContent.type) { case "box-whisker-chart": @@ -118,15 +119,12 @@ function hasPlainLanguageContent(theme: ThemeDetail): boolean { } function ThemeContent({ - slug, + theme, plainLanguage, }: { - slug: string; + theme: ThemeDetail; plainLanguage: boolean; }) { - const { data: theme } = useSuspenseQuery( - explorerGetThemeOptions({ path: { theme_slug: slug } }), - ); const collectionGroups = buildCollectionGroups(theme); return (
- {theme && } + {theme && ( + + )}
); diff --git a/frontend/src/components/explorer/types.ts b/frontend/src/components/explorer/types.ts index b2ae31d..21ac54f 100644 --- a/frontend/src/components/explorer/types.ts +++ b/frontend/src/components/explorer/types.ts @@ -1,4 +1,5 @@ import type { ReactNode } from "react"; +import type { ReferenceDatasetLink } from "@/client/types.gen"; import type { ChartGroupingConfig } from "./grouping"; // Base properties shared across all card types @@ -10,12 +11,12 @@ export type BaseCardContent = { interpretation?: string; span?: 1 | 2; placeholder?: boolean; + referenceDatasets?: ReferenceDatasetLink[]; }; // Card-specific content types export type BoxWhiskerChartContent = BaseCardContent & { type: "box-whisker-chart"; - interpretation?: string; metricUnits?: string; otherFilters?: Record; clipMin?: number; From e95eaaceb64f555af9f7f9c26b04abd99fc2f238 Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Sun, 8 Mar 2026 16:07:26 +0900 Subject: [PATCH 14/15] docs: update content --- backend/static/diagnostics/esmvaltool.yaml | 31 ++++++++++++---------- backend/static/diagnostics/ilamb.yaml | 20 +++++++------- backend/static/diagnostics/pmp.yaml | 18 ++++++------- 3 files changed, 36 insertions(+), 33 deletions(-) diff --git a/backend/static/diagnostics/esmvaltool.yaml b/backend/static/diagnostics/esmvaltool.yaml index bedb254..24841cc 100644 --- a/backend/static/diagnostics/esmvaltool.yaml +++ b/backend/static/diagnostics/esmvaltool.yaml @@ -5,7 +5,7 @@ esmvaltool/cloud-radiative-effects: reference_datasets: - slug: "esmvaltool.CERES-EBAF" - description: "CERES Energy Balanced and Filled - TOA radiation fluxes (planned to move to obs4MIPs)" + description: "CERES Energy Balanced and Filled - TOA radiation fluxes (rlut, rlutcs, rsut, rsutcs) (planned to move to obs4MIPs)" type: "primary" - slug: "esmvaltool.ESACCI-CLOUD" description: "ESA CCI Cloud properties (planned to move to obs4MIPs)" @@ -25,7 +25,7 @@ esmvaltool/cloud-scatterplots-cli-ta: description: "ERA5 - Air temperature (ta)" type: "primary" - slug: "esmvaltool.CALIPSO-ICECLOUD" - description: "Ice Cloud properties (planned to move to obs4MIPs)" + description: "CALIPSO - Ice cloud properties (cli), baked into recipe" type: "secondary" display_name: "Cloud-Temperature Scatterplots (cli vs ta)" description: >- @@ -37,10 +37,10 @@ esmvaltool/cloud-scatterplots-cli-ta: esmvaltool/cloud-scatterplots-clivi-lwcre: reference_datasets: - slug: "esmvaltool.CERES-EBAF" - description: "CERES Energy Balanced and Filled - TOA radiation fluxes (planned)" + description: "CERES Energy Balanced and Filled - Longwave radiation (rlut, rlutcs; lwcre derived), baked into recipe" type: "primary" - slug: "esmvaltool.ESACCI-CLOUD" - description: "ESA CCI Cloud properties (planned to move to obs4MIPs)" + description: "ESA CCI Cloud properties - Ice water path (clivi), baked into recipe" type: "secondary" display_name: "Cloud-Radiation Scatterplots (clivi vs lwcre)" description: "Scatterplot of ice water path (clivi) vs longwave cloud radiative effect (lwcre)." @@ -49,10 +49,10 @@ esmvaltool/cloud-scatterplots-clivi-lwcre: esmvaltool/cloud-scatterplots-clt-swcre: reference_datasets: - slug: "esmvaltool.CERES-EBAF" - description: "CERES Energy Balanced and Filled - TOA radiation fluxes (planned to move to obs4MIPs)" + description: "CERES Energy Balanced and Filled - Shortwave radiation (rsut, rsutcs; swcre derived), baked into recipe" type: "primary" - slug: "esmvaltool.ESACCI-CLOUD" - description: "ESA CCI Cloud properties (planned to move to obs4MIPs)" + description: "ESA CCI Cloud properties - Total cloud fraction (clt), baked into recipe" type: "secondary" display_name: "Cloud-Radiation Scatterplots (clt vs swcre)" description: "Scatterplot of total cloud fraction (clt) vs shortwave cloud radiative effect (swcre)." @@ -61,13 +61,13 @@ esmvaltool/cloud-scatterplots-clt-swcre: esmvaltool/cloud-scatterplots-clwvi-pr: reference_datasets: - slug: "obs4MIPs.GPCPv2.3" - description: "GPCP v2.3 - Precipitation relationship" + description: "GPCP v2.3 - Precipitation (pr)" type: "secondary" - slug: "esmvaltool.ESACCI-CLOUD" - description: "ESA CCI Cloud properties (planned to move to obs4MIPs)" + description: "ESA CCI Cloud properties (clwvi, planned to move to obs4MIPs)" type: "secondary" - slug: "esmvaltool.CERES-EBAF" - description: "CERES - Reference cloud-radiation relationships" + description: "CERES Energy Balanced and Filled - Radiation, baked into recipe" type: "primary" display_name: "Cloud-Precipitation Scatterplots (clwvi vs pr)" description: "Scatterplot of condensed water path (clwvi) vs precipitation (pr)." @@ -79,10 +79,13 @@ esmvaltool/cloud-scatterplots-reference: description: "ERA5 - Air temperature (ta), ingested via REF" type: "primary" - slug: "esmvaltool.CERES-EBAF" - description: "CERES - Reference cloud-radiation relationships, baked into recipe" + description: "CERES Energy Balanced and Filled - Radiation, baked into recipe" type: "primary" + - slug: "esmvaltool.CALIPSO-ICECLOUD" + description: "CALIPSO - Ice cloud properties (cli), baked into recipe" + type: "secondary" - slug: "esmvaltool.ESACCI-CLOUD" - description: "ESA CCI Cloud properties (clt, clwvi, clivi), baked into recipe (planned to move to obs4MIPs)" + description: "ESA CCI Cloud properties (clt, clwvi, clivi), baked into recipe" type: "secondary" - slug: "esmvaltool.GPCP-V2.3" description: "GPCP v2.3 - Precipitation (pr), hardcoded in update_recipe" @@ -96,10 +99,10 @@ esmvaltool/cloud-scatterplots-reference: esmvaltool/enso-basic-climatology: reference_datasets: - slug: "esmvaltool.GPCP-V2.3" - description: "GPCP v2.3 - Precipitation (pr), baked into ESMValTool recipe" + description: "GPCP v2.3 - Precipitation (pr), baked into recipe" type: "secondary" - slug: "esmvaltool.TROPFLUX" - description: "TROPFLUX - Sea surface temperature (tos) and wind stress (tauu), baked into ESMValTool recipe" + description: "TROPFLUX (OBS6) - Sea surface temperature (tos) and wind stress (tauu), baked into recipe" type: "secondary" display_name: "ENSO Basic Climatology" description: >- @@ -110,7 +113,7 @@ esmvaltool/enso-basic-climatology: esmvaltool/enso-characteristics: reference_datasets: - slug: "esmvaltool.TROPFLUX" - description: "TROPFLUX - Sea surface temperature (tos), hardcoded in update_recipe" + description: "TROPFLUX (OBS6) - Sea surface temperature (tos), hardcoded in update_recipe" type: "primary" display_name: "ENSO Characteristics" description: >- diff --git a/backend/static/diagnostics/ilamb.yaml b/backend/static/diagnostics/ilamb.yaml index e6713f6..3714288 100644 --- a/backend/static/diagnostics/ilamb.yaml +++ b/backend/static/diagnostics/ilamb.yaml @@ -8,13 +8,13 @@ ilamb/gpp-wecann: reference_datasets: - slug: "obs4REF.WECANN" - description: "WECANN GPP dataset - Primary gross primary productivity reference" + description: "WECANN GPP dataset - Gross primary productivity (gpp)" type: "primary" - - slug: "obs4MIPs.GPCPv2.3" - description: "GPCP v2.3 - Precipitation relationship" + - slug: "obs4REF.GPCPv2.3" + description: "GPCP v2.3 - Precipitation (pr), relationship dataset" type: "secondary" - - slug: "obs4MIPs.CRU4.02" - description: "CRU TS v4.02 - Temperature relationship" + - slug: "ilamb.CRU4.02" + description: "CRU TS v4.02 - Near-surface air temperature (tas), relationship dataset" type: "secondary" display_name: "Gross Primary Productivity (WECANN)" description: "Apply the standard ILAMB analysis of gross primary productivity (gpp) with respect to the WECANN reference dataset." @@ -41,7 +41,7 @@ ilamb/mrro-lora: ilamb/mrsos-wangmao: reference_datasets: - slug: "obs4REF.WangMao" - description: "Wang-Mao dataset - Surface soil moisture" + description: "Wang-Mao dataset - Soil moisture (mrsos)" type: "primary" display_name: "Surface Soil Moisture (WangMao)" description: "Apply the standard ILAMB analysis of moisture in upper portion of soil column (mrsos) with respect to the WangMao reference dataset." @@ -98,7 +98,7 @@ ilamb/emp-gleamgpcp: description: "GLEAM v3.3a - Evapotranspiration (et)" type: "primary" - slug: "obs4REF.GPCP-2-3" - description: "GPCP v2.3 - Precipitation (pr)" + description: "GPCP v2.3 (obs4REF) - Precipitation (pr)" type: "primary" display_name: "Evaporation minus Precipitation (GLEAM/GPCP)" description: "Apply the standard ILAMB analysis of evaporation minus precipitation (E-P) using GLEAMv3.3a evapotranspiration and GPCP v2.3 precipitation reference datasets." @@ -109,7 +109,7 @@ ilamb/emp-gleamgpcp: ilamb/thetao-woa2023-surface: reference_datasets: - slug: "obs4REF.WOA2023" - description: "World Ocean Atlas 2023 - Sea water potential temperature (surface)" + description: "World Ocean Atlas 2023 - Sea water potential temperature (thetao, surface, mapped to tos)" type: "primary" display_name: "Sea Water Temperature (WOA2023 Surface)" description: "Apply the standard ILAMB analysis of sea water potential temperature (thetao) at the surface with respect to the WOA2023 reference dataset." @@ -118,7 +118,7 @@ ilamb/thetao-woa2023-surface: ilamb/so-woa2023-surface: reference_datasets: - slug: "obs4REF.WOA2023" - description: "World Ocean Atlas 2023 - Sea water salinity (surface)" + description: "World Ocean Atlas 2023 - Sea water salinity (so, surface)" type: "primary" display_name: "Sea Water Salinity (WOA2023 Surface)" description: "Apply the standard ILAMB analysis of sea water salinity (so) at the surface with respect to the WOA2023 reference dataset." @@ -127,7 +127,7 @@ ilamb/so-woa2023-surface: ilamb/amoc-rapid: reference_datasets: - slug: "obs4REF.RAPID" - description: "RAPID array - Atlantic Meridional Overturning Circulation observations" + description: "RAPID array - Atlantic Meridional Overturning Circulation (amoc, from msftmz)" type: "primary" display_name: "Atlantic Meridional Overturning Circulation (RAPID)" description: "Apply the standard ILAMB analysis of the Atlantic Meridional Overturning Circulation (AMOC) with respect to the RAPID array reference dataset." diff --git a/backend/static/diagnostics/pmp.yaml b/backend/static/diagnostics/pmp.yaml index b9caf1d..0af9401 100644 --- a/backend/static/diagnostics/pmp.yaml +++ b/backend/static/diagnostics/pmp.yaml @@ -24,13 +24,13 @@ pmp/annual-cycle: pmp/enso_perf: reference_datasets: - slug: "obs4mips.GPCP-Monthly-3-2" - description: "Global Precipitation Climatology Project v3.2 - Precipitation patterns" + description: "Global Precipitation Climatology Project v3.2 - Precipitation (pr)" type: "primary" - slug: "obs4mips.TropFlux-1-0" - description: "TropFlux v1.0 - Tropical flux data" + description: "TropFlux v1.0 - Wind stress (tauu)" type: "primary" - slug: "obs4mips.HadISST-1-1" - description: "Hadley Centre Sea Ice and Sea Surface Temperature v1.1 - SST reference" + description: "Hadley Centre Sea Ice and Sea Surface Temperature v1.1 - Sea surface temperature (ts)" type: "primary" display_name: "ENSO Performance Metrics" description: >- @@ -41,13 +41,13 @@ pmp/enso_perf: pmp/enso_tel: reference_datasets: - slug: "obs4mips.GPCP-Monthly-3-2" - description: "Global Precipitation Climatology Project v3.2 - Precipitation teleconnections" + description: "Global Precipitation Climatology Project v3.2 - Precipitation teleconnections (pr)" type: "primary" - slug: "obs4mips.TropFlux-1-0" - description: "TropFlux v1.0 - Tropical flux data" + description: "TropFlux v1.0 - Wind stress (tauu)" type: "primary" - slug: "obs4mips.HadISST-1-1" - description: "Hadley Centre Sea Ice and SST v1.1 - SST teleconnections" + description: "Hadley Centre Sea Ice and SST v1.1 - Sea surface temperature teleconnections (ts)" type: "primary" display_name: "ENSO Teleconnections" description: >- @@ -59,13 +59,13 @@ pmp/enso_tel: pmp/enso_proc: reference_datasets: - slug: "obs4mips.GPCP-Monthly-3-2" - description: "Global Precipitation Climatology Project v3.2" + description: "Global Precipitation Climatology Project v3.2 - Precipitation (pr)" type: "primary" - slug: "obs4mips.TropFlux-1-0" - description: "TropFlux v1.0 - Surface fluxes" + description: "TropFlux v1.0 - Wind stress (tauu)" type: "primary" - slug: "obs4mips.HadISST-1-1" - description: "Hadley Centre Sea Ice and SST v1.1" + description: "Hadley Centre Sea Ice and SST v1.1 - Sea surface temperature (ts)" type: "primary" - slug: "obs4mips.CERES-EBAF-4-2" description: "CERES Energy Balanced and Filled v4.2 - Surface and TOA radiation fluxes (hfls, hfss, rlds, rlus, rsds, rsus)" From aa0c02cebb366f83671e2d27c54eff11d87f3fe4 Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Sun, 8 Mar 2026 16:42:03 +0900 Subject: [PATCH 15/15] fix: resolve CI failures in frontend typecheck and backend AFT tests - Add SeriesMetadata type annotation to test fixture to fix TS2345 error where heterogeneous dimension objects inferred metric?: undefined - Clear _build_ref_to_aft_index lru_cache in test fixture to prevent cached results leaking between tests --- backend/tests/test_core/test_core_aft.py | 2 ++ frontend/src/components/execution/values/series/utils.test.ts | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/backend/tests/test_core/test_core_aft.py b/backend/tests/test_core/test_core_aft.py index 1ccc29b..b944156 100644 --- a/backend/tests/test_core/test_core_aft.py +++ b/backend/tests/test_core/test_core_aft.py @@ -5,6 +5,7 @@ import pytest from ref_backend.core.aft import ( + _build_ref_to_aft_index, get_aft_diagnostic_by_id, get_aft_diagnostics_index, get_aft_for_ref_diagnostic, @@ -58,6 +59,7 @@ def clear_aft_caches(): get_aft_diagnostics_index, get_aft_diagnostic_by_id, get_aft_for_ref_diagnostic, + _build_ref_to_aft_index, ]: if hasattr(fn, "cache_clear"): fn.cache_clear() diff --git a/frontend/src/components/execution/values/series/utils.test.ts b/frontend/src/components/execution/values/series/utils.test.ts index caace13..7fca23a 100644 --- a/frontend/src/components/execution/values/series/utils.test.ts +++ b/frontend/src/components/execution/values/series/utils.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import type { SeriesValue } from "../types"; +import type { SeriesMetadata, SeriesValue } from "../types"; import { createChartData, createScaledTickFormatter, @@ -324,7 +324,7 @@ describe("getDimensionKeys", () => { }); it("collects unique dimension keys sorted alphabetically", () => { - const metadata = [ + const metadata: SeriesMetadata[] = [ { seriesIndex: 0, label: "A",