diff --git a/CLAUDE.md b/CLAUDE.md index 6415b744..6c001220 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -281,14 +281,14 @@ Entries are grouped by date under level-three headers. Add your entry under toda ``` ### YYYY-MM-DD -- [``](https://github.com/quarto-dev/kyoto/commits/): One-sentence description +- [``](https://github.com/quarto-dev/q2/commits/): One-sentence description ``` Example: ``` ### 2026-01-10 -- [`e6f742c`](https://github.com/quarto-dev/kyoto/commits/e6f742c): Refactor navigation to VS Code-style collapsible sidebar +- [`e6f742c`](https://github.com/quarto-dev/q2/commits/e6f742c): Refactor navigation to VS Code-style collapsible sidebar ``` The changelog is rendered in the About section of the hub-client UI. diff --git a/Cargo.lock b/Cargo.lock index c3bbc8c3..d9347c37 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -675,7 +675,7 @@ dependencies = [ "entities", "jetscii", "phf 0.13.1", - "phf_codegen", + "phf_codegen 0.13.1", "rustc-hash", "smallvec", "typed-arena", @@ -820,7 +820,7 @@ checksum = "d8b9f2e4c67f833b660cdb0a3523065869fb35570177239812ed4c905aeff87b" dependencies = [ "bitflags 2.11.0", "crossterm_winapi", - "derive_more", + "derive_more 2.1.1", "document-features", "mio", "parking_lot", @@ -870,6 +870,29 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "cssparser" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c66d1cd8ed61bf80b38432613a7a2f09401ab8d0501110655f8b341484a3e3" +dependencies = [ + "cssparser-macros", + "dtoa-short", + "itoa", + "phf 0.11.3", + "smallvec", +] + +[[package]] +name = "cssparser-macros" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" +dependencies = [ + "quote", + "syn", +] + [[package]] name = "curve25519-dalek" version = "4.1.3" @@ -1071,6 +1094,17 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "derive_more" +version = "0.99.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6edb4b64a43d977b8e99788fe3a04d483834fba1215a7e02caa415b626497f7f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "derive_more" version = "2.1.1" @@ -1195,6 +1229,21 @@ dependencies = [ "litrs", ] +[[package]] +name = "dtoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c3cf4824e2d5f025c7b531afcb2325364084a16806f6d47fbc1f5fbd9960590" + +[[package]] +name = "dtoa-short" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd1511a7b6a56299bd043a9c167a6d2bfb37bf84a6dfceaba651168adfb43c87" +dependencies = [ + "dtoa", +] + [[package]] name = "ecdsa" version = "0.16.9" @@ -1233,6 +1282,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "ego-tree" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2972feb8dffe7bc8c5463b1dacda1b0dfbed3710e50f977d965429692d74cd8" + [[package]] name = "either" version = "1.15.0" @@ -1450,6 +1505,16 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" +[[package]] +name = "futf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843" +dependencies = [ + "mac", + "new_debug_unreachable", +] + [[package]] name = "futures" version = "0.3.32" @@ -1538,6 +1603,15 @@ dependencies = [ "slab", ] +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + [[package]] name = "generic-array" version = "0.14.9" @@ -1549,6 +1623,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "getopts" +version = "0.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe4fbac503b8d1f88e6676011885f34b7174f46e59956bba534ba83abded4df" +dependencies = [ + "unicode-width", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -1764,6 +1847,18 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "html5ever" +version = "0.29.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b7410cae13cbc75623c98ac4cbfd1f0bedddf3227afc24f370cf0f50a44a11c" +dependencies = [ + "log", + "mac", + "markup5ever", + "match_token", +] + [[package]] name = "http" version = "1.4.0" @@ -2424,6 +2519,37 @@ dependencies = [ "which 8.0.0", ] +[[package]] +name = "mac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" + +[[package]] +name = "markup5ever" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7a7213d12e1864c0f002f52c2923d4556935a43dec5e71355c2760e0f6e7a18" +dependencies = [ + "log", + "phf 0.11.3", + "phf_codegen 0.11.3", + "string_cache", + "string_cache_codegen", + "tendril", +] + +[[package]] +name = "match_token" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88a9689d8d44bf9964484516275f5cd4c9b59457a6940c1d5d0ecbb94510a36b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "matchers" version = "0.2.0" @@ -2549,6 +2675,12 @@ dependencies = [ "tempfile", ] +[[package]] +name = "new_debug_unreachable" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" + [[package]] name = "nom" version = "7.1.3" @@ -2857,6 +2989,12 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "pathdiff" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" + [[package]] name = "pem" version = "3.0.6" @@ -2945,6 +3083,16 @@ dependencies = [ "serde", ] +[[package]] +name = "phf_codegen" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" +dependencies = [ + "phf_generator 0.11.3", + "phf_shared 0.11.3", +] + [[package]] name = "phf_codegen" version = "0.13.1" @@ -3097,6 +3245,12 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + [[package]] name = "pretty_assertions" version = "1.4.1" @@ -3265,6 +3419,7 @@ dependencies = [ "include_dir", "jupyter-protocol", "pampa", + "pathdiff", "pollster", "quarto-analysis", "quarto-ast-reconcile", @@ -3276,11 +3431,12 @@ dependencies = [ "quarto-source-map", "quarto-system-runtime", "quarto-util", + "quarto-yaml", "regex", "runtimelib", "serde", "serde_json", - "serde_yaml", + "sha2 0.10.9", "tempfile", "thiserror 2.0.18", "tokio", @@ -3288,6 +3444,7 @@ dependencies = [ "tracing", "uuid", "which 8.0.0", + "yaml-rust2", ] [[package]] @@ -3460,6 +3617,7 @@ dependencies = [ "regex", "serde", "serde_json", + "sha2 0.10.9", "thiserror 2.0.18", "yaml-rust2", ] @@ -3499,6 +3657,7 @@ dependencies = [ "quarto-error-reporting", "quarto-system-runtime", "regex", + "scraper", "serde", "serde_yaml", "tempfile", @@ -4116,6 +4275,21 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "scraper" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc3d051b884f40e309de6c149734eab57aa8cc1347992710dc80bcc1c2194c15" +dependencies = [ + "cssparser", + "ego-tree", + "getopts", + "html5ever", + "precomputed-hash", + "selectors", + "tendril", +] + [[package]] name = "sdd" version = "4.7.2" @@ -4159,6 +4333,25 @@ dependencies = [ "libc", ] +[[package]] +name = "selectors" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd568a4c9bb598e291a08244a5c1f5a8a6650bee243b5b0f8dbb3d9cc1d87fe8" +dependencies = [ + "bitflags 2.11.0", + "cssparser", + "derive_more 0.99.20", + "fxhash", + "log", + "new_debug_unreachable", + "phf 0.11.3", + "phf_codegen 0.11.3", + "precomputed-hash", + "servo_arc", + "smallvec", +] + [[package]] name = "semver" version = "1.0.27" @@ -4280,6 +4473,15 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "servo_arc" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "170fb83ab34de17dc69aa7c67482b22218ddb85da56546f9bd6b929e32a05930" +dependencies = [ + "stable_deref_trait", +] + [[package]] name = "sha1" version = "0.10.6" @@ -4507,6 +4709,31 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520" +[[package]] +name = "string_cache" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf776ba3fa74f83bf4b63c3dcbbf82173db2632ed8452cb2d891d33f459de70f" +dependencies = [ + "new_debug_unreachable", + "parking_lot", + "phf_shared 0.11.3", + "precomputed-hash", + "serde", +] + +[[package]] +name = "string_cache_codegen" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c711928715f1fe0fe509c53b43e993a9a557babc2d0a3567d0a3006f1ac931a0" +dependencies = [ + "phf_generator 0.11.3", + "phf_shared 0.11.3", + "proc-macro2", + "quote", +] + [[package]] name = "stringcase" version = "0.4.0" @@ -4666,6 +4893,17 @@ dependencies = [ "writeable", ] +[[package]] +name = "tendril" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d24a120c5fc464a3458240ee02c299ebcb9d67b5249c8848b09d639dca8d7bb0" +dependencies = [ + "futf", + "mac", + "utf-8", +] + [[package]] name = "thiserror" version = "1.0.69" diff --git a/Cargo.toml b/Cargo.toml index fe66792a..fe00dae5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,6 +42,7 @@ async-trait = "0.1" tokio-util = "0.7" pollster = "0.4" regex = "1.12" +scraper = "0.22" grass = "0.13.4" include_dir = "0.7" uuid = { version = "1", features = ["v4"] } diff --git a/README.md b/README.md index 5339d371..cc559c2c 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # Quarto 2 +[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/quarto-dev/kyoto) + > **Experimental** - This project is under active development. It's not yet ready for production use, and will not be for a while. This repository is a Rust implementation of the next version of [Quarto](https://quarto.org). The goal is to replace parts of the TypeScript/Deno runtime with a unified Rust implementation, enabling: diff --git a/claude-notes/instructions/coding.md b/claude-notes/instructions/coding.md index 85f5a7a0..85fea121 100644 --- a/claude-notes/instructions/coding.md +++ b/claude-notes/instructions/coding.md @@ -67,7 +67,7 @@ When making changes to the WASM module (`crates/wasm-quarto-hub-client/`): await wasm.default(wasmBytes); // Test your functionality here - const result = wasm.render_qmd_content_with_options(content, '', '{"source_location": true}'); + const result = await wasm.render_qmd_content(content, ''); console.log(JSON.parse(result)); ``` diff --git a/claude-notes/instructions/testing.md b/claude-notes/instructions/testing.md index da98c2de..471d5862 100644 --- a/claude-notes/instructions/testing.md +++ b/claude-notes/instructions/testing.md @@ -3,6 +3,7 @@ - You are encouraged to spend time and tokens on thinking about good tests. - If writing tests is taking a lot of time, decompose the writing of tests into subtasks. Good tests are important! - Precise tests are good tests. **bad**: testing for the presence of a field in an object. **good** testing if the value of the field is correct. +- When choosing hex colors for CSS test assertions (`ensureCssRegexMatches`), use **non-condensable** 6-digit hex values. CSS minifiers shorten `#RRGGBB` to `#RGB` when each pair is a repeated digit (e.g., `#cc5500` → `#c50`). Break at least one pair to prevent this: `#cc5501` instead of `#cc5500`. - Do not write tests that expect known-bad inputs. Instead, add a failing test, and create a beads task to handle the problem. ## End-to-End Testing for WASM Features @@ -68,4 +69,52 @@ For any WASM feature, the test should verify: - Claim a WASM feature is complete based only on `cargo check` or `npm run build` - Assume TypeScript type declarations match actual WASM exports -- Test only in the browser when a Node.js test would be faster and more reliable \ No newline at end of file +- Test only in the browser when a Node.js test would be faster and more reliable + +## Smoke-All Tests + +Smoke-all test fixtures live in `crates/quarto/tests/smoke-all/`. Each `.qmd` file embeds assertions in `_quarto.tests` frontmatter. There are **three independent runners** that exercise the same fixtures through different pipelines: + +### 1. Rust (native renderer) +```bash +cargo nextest run -p quarto --test smoke_all +``` +Fastest (~1s). Renders via `quarto-core` directly. Runs all assertion types including `ensureHtmlElements` (CSS selectors via `scraper`), `ensureCssRegexMatches`, `ensureFileRegexMatches`, etc. + +### 2. WASM Vitest (jsdom) +```bash +cd hub-client && npm run test:wasm +``` +~3s. Renders via WASM module in Node.js with jsdom for HTML assertions. Runs the full smoke-all suite plus other WASM tests. + +### 3. Playwright E2E (browser) +```bash +cd hub-client && npx playwright test e2e/smoke-all.spec.ts +``` +~12s. Full pipeline: Automerge sync → hub server → browser → WASM render → preview iframe. Tests the complete hub-client integration. + +### Writing Fixtures + +Each fixture is a `.qmd` file with test assertions in frontmatter. The project must have a `_quarto.yml`. Assertions use a two-array format `[mustMatch[], mustNotMatch[]]`: + +```yaml +_quarto: + tests: + html: + ensureCssRegexMatches: + - ["#170229", "my-custom-rule"] # patterns that must appear in CSS + - ["unwanted-pattern"] # patterns that must NOT appear + ensureHtmlElements: + - ["nav#TOC", "div.callout"] # CSS selectors that must match + - ["div.should-not-exist"] # selectors that must NOT match + ensureFileRegexMatches: + - ["pattern in HTML output"] + noErrors: true +``` + +### Running a Subset + +To debug a specific fixture, check the rendered output directly: +```bash +cargo run -- render crates/quarto/tests/smoke-all/path/to/doc.qmd -v +``` \ No newline at end of file diff --git a/claude-notes/plans/2026-01-27-hub-client-testing-infrastructure.md b/claude-notes/plans/2026-01-27-hub-client-testing-infrastructure.md index 324e42fb..746fb824 100644 --- a/claude-notes/plans/2026-01-27-hub-client-testing-infrastructure.md +++ b/claude-notes/plans/2026-01-27-hub-client-testing-infrastructure.md @@ -128,8 +128,8 @@ This plan outlines a comprehensive testing infrastructure for hub-client, addres - [x] Set up jsdom/happy-dom environment for component tests - [x] Create test utilities directory structure - [x] Set up Playwright for E2E tests -- [ ] Add `@automerge/automerge-repo-storage-nodefs` dependency for fixture management -- [ ] Add `@automerge/automerge-repo-sync-server` dependency for local E2E sync server +- [x] ~~Add `@automerge/automerge-repo-storage-nodefs` dependency for fixture management~~ — not needed; E2E tests create projects dynamically via `@quarto/quarto-sync-client` (see `2026-03-10-smoke-all-playwright-e2e.md`) +- [x] ~~Add `@automerge/automerge-repo-sync-server` dependency for local E2E sync server~~ — replaced by using the real hub server (`cargo run --bin hub`); no new dependency needed (0f5ee7f8) - [x] Update `.github/workflows/test-suite.yml` to run hub-client unit tests - [x] Create `.github/workflows/hub-client-e2e.yml` for E2E tests (manual trigger initially) @@ -173,12 +173,12 @@ This plan outlines a comprehensive testing infrastructure for hub-client, addres ### Phase 6: E2E Tests with Playwright -- [ ] Set up Playwright configuration with fixture lifecycle hooks -- [ ] Add tests for project loading (using fixture with known docId) +- [x] Set up Playwright configuration with fixture lifecycle hooks — globalSetup/Teardown starts real hub server, file-based IPC for server URL (0f5ee7f8) +- [x] Add tests for project loading (using fixture with known docId) — `e2e/project-loading.spec.ts` creates Automerge project dynamically, seeds IndexedDB, navigates via URL hash (0f5ee7f8) - [ ] Add tests for project creation flow (fresh documents) - [ ] Add tests for file editing flow -- [ ] Add tests for SCSS compilation and caching behavior -- [ ] Add tests for preview rendering +- [x] Add tests for SCSS compilation and caching behavior — `e2e/theme-subdir-e2e.spec.ts` (7 tests) + smoke-all theme fixtures verify SCSS compilation through full pipeline (0f5ee7f8) +- [x] Add tests for preview rendering — `e2e/preview-extraction.spec.ts` + `e2e/smoke-all.spec.ts` (23 smoke-all fixtures) test HTML/CSS/diagnostics extraction from live preview (0f5ee7f8) --- diff --git a/claude-notes/plans/2026-02-16-project-metadata-merging.md b/claude-notes/plans/2026-02-16-project-metadata-merging.md new file mode 100644 index 00000000..2328a572 --- /dev/null +++ b/claude-notes/plans/2026-02-16-project-metadata-merging.md @@ -0,0 +1,602 @@ +# Project Metadata Merging with Format Resolution + +**Date**: 2026-02-16 +**Status**: Complete + +## Overview + +Implement full project metadata merging in q2, where `_quarto.yml` is parsed as `ConfigValue` and merged with document frontmatter. This includes proper format-specific config resolution following Quarto 1's approach. + +### Goals + +1. Parse entire `_quarto.yml` as `ConfigValue` with source tracking +2. Implement format-specific config extraction (`format.html.*` → flat config) +3. Merge project metadata with document metadata, respecting format-specific overrides +4. Maintain backwards compatibility with existing `format_config` usage in WASM + +### Non-Goals + +- Extension metadata merging (separate task) +- Directory-level `_metadata.yml` support (future work) +- Command-line flag merging (future work) + +## Background + +### Current State + +- `ProjectConfig` has `raw: serde_json::Value` (no source tracking) +- `format_config: Option` exists but is always `None` +- `AstTransformsStage` has merge code (lines 107-128) but unused +- WASM uses `ProjectConfig::with_format_config()` to inject settings + +### Quarto 1 Behavior + +Format config merging follows a two-level precedence: + +**Within a single metadata source:** +- Format-specific (`format.html.toc`) always overrides top-level (`toc`) +- YAML key order doesn't matter + +**Across sources (lowest → highest priority):** +1. Built-in format defaults +2. Project `_quarto.yml` (top-level, then format-specific) +3. Directory `_metadata.yml` (future) +4. Document frontmatter (top-level, then format-specific) +5. Command-line flags (future) + +### Example + +```yaml +# _quarto.yml +title: "Project Title" +toc: true +format: + html: + toc-depth: 3 + theme: cosmo + +# document.qmd frontmatter +--- +title: "Chapter 1" +toc: false +format: + html: + toc-depth: 2 +--- +``` + +Result for HTML rendering: +- `title`: "Chapter 1" (doc overrides project) +- `toc`: false (doc overrides project) +- `toc-depth`: 2 (doc format.html overrides project format.html) +- `theme`: cosmo (inherited from project format.html) + +## Design + +### Approach: Option B - Format Flattening + +Create a `resolve_format_config()` function that: +1. Takes metadata and target format name +2. Extracts top-level settings +3. Extracts `format.{target}.*` settings +4. Merges them (format-specific wins over top-level) +5. Returns a flat `ConfigValue` for the target format + +This matches Quarto 1's `formatFromMetadata()` function. + +### New Types and Functions + +#### 1. Format Resolution Function (in `quarto-config`) + +```rust +/// Extract and flatten format-specific configuration. +/// +/// Given a ConfigValue containing metadata and a target format name, +/// returns a new ConfigValue with: +/// - All top-level settings (except `format`) +/// - Format-specific settings from `format.{target}` merged on top +/// +/// # Example +/// +/// Input metadata: +/// ```yaml +/// title: "Hello" +/// toc: true +/// format: +/// html: +/// toc: false +/// theme: cosmo +/// ``` +/// +/// With target_format = "html", returns: +/// ```yaml +/// title: "Hello" +/// toc: false # from format.html, overrides top-level +/// theme: cosmo # from format.html +/// ``` +pub fn resolve_format_config( + metadata: &ConfigValue, + target_format: &str, +) -> ConfigValue +``` + +**Implementation pseudocode:** +```rust +pub fn resolve_format_config(metadata: &ConfigValue, target_format: &str) -> ConfigValue { + // 1. Start with empty result map + let mut result_entries: Vec = Vec::new(); + + // 2. If metadata is not a map, return empty map + let entries = match &metadata.value { + ConfigValueKind::Map(e) => e, + _ => return ConfigValue::new_map(vec![], metadata.source_info.clone()), + }; + + // 3. Copy all top-level entries EXCEPT "format" + for entry in entries { + if entry.key != "format" { + result_entries.push(entry.clone()); + } + } + + // 4. Find format.{target} and merge its entries on top + if let Some(format_entry) = entries.iter().find(|e| e.key == "format") { + match &format_entry.value.value { + // Handle format: { html: { ... } } + ConfigValueKind::Map(format_map) => { + if let Some(target_entry) = format_map.iter().find(|e| e.key == target_format) { + if let ConfigValueKind::Map(target_settings) = &target_entry.value.value { + // Merge target settings (override existing keys) + for setting in target_settings { + // Remove existing entry with same key + result_entries.retain(|e| e.key != setting.key); + // Add the format-specific entry + result_entries.push(setting.clone()); + } + } + } + } + // Handle format: "html" (shorthand) - no settings to merge + ConfigValueKind::Scalar(Yaml::String(s)) if s == target_format => { + // Shorthand matches target, but no additional settings + } + _ => {} + } + } + + ConfigValue::new_map(result_entries, metadata.source_info.clone()) +} +``` + +#### 2. Updated ProjectConfig + +```rust +pub struct ProjectConfig { + pub project_type: ProjectType, + pub output_dir: Option, + pub render_patterns: Vec, + + /// Full project metadata as ConfigValue (with source tracking). + /// This is the entire _quarto.yml parsed with InterpretationContext::ProjectConfig. + pub metadata: Option, + + // Note: `raw: serde_json::Value` field removed - it was unused +} +``` + +#### 3. Updated Merge in AstTransformsStage + +```rust +// In AstTransformsStage::execute() + +if let Some(project_metadata) = ctx.project.config.as_ref().and_then(|c| c.metadata.as_ref()) { + // Get the target format name from context + let target_format = ctx.format.name(); // e.g., "html" + + // Flatten project metadata for target format + let project_for_format = resolve_format_config(project_metadata, target_format); + + // Flatten document metadata for target format + let doc_for_format = resolve_format_config(&doc.ast.meta, target_format); + + // Merge: project (lower priority) → document (higher priority) + let merged = MergedConfig::new(vec![&project_for_format, &doc_for_format]); + + if let Ok(materialized) = merged.materialize() { + doc.ast.meta = materialized; + } +} +``` + +### File Changes + +| File | Change | +|------|--------| +| `crates/quarto-config/src/lib.rs` | Export `resolve_format_config` | +| `crates/quarto-config/src/format.rs` | New file: format resolution logic | +| `crates/quarto-core/src/project.rs` | Add `metadata: Option`, update `parse_config()` | +| `crates/quarto-core/src/stage/stages/ast_transforms.rs` | Use format resolution in merge | +| `crates/quarto-core/Cargo.toml` | Add `quarto-yaml` dependency | + +### Dependencies + +``` +quarto-yaml ──────────────────┐ + ▼ +quarto-config ◄─── resolve_format_config() + │ + ▼ +quarto-core (project.rs, ast_transforms.rs) + │ + ▼ +pampa (yaml_to_config_value with InterpretationContext) +``` + +## Implementation Plan + +### Phase 1: Format Resolution Function + +**Files**: `crates/quarto-config/src/format.rs`, `crates/quarto-config/src/lib.rs` + +- [x] Create `format.rs` with `resolve_format_config()` function +- [x] Handle edge cases: + - No `format` key in metadata + - `format` key exists but target format not present + - `format: html` shorthand (string, not object) + - Empty format config (`format: { html: {} }`) +- [x] Export from `lib.rs` + +### Phase 2: Tests for Format Resolution + +**File**: `crates/quarto-config/src/format.rs` (tests module) + +- [x] Test: top-level only (no format key) +- [x] Test: format-specific only (no top-level) +- [x] Test: format-specific overrides top-level +- [x] Test: format-specific merges with top-level (non-overlapping keys) +- [x] Test: nested objects merge correctly +- [ ] Test: arrays follow merge semantics (!prefer, !concat) - deferred +- [x] Test: missing target format returns top-level only +- [x] Test: format shorthand normalization (`format: html` → `format: { html: {} }`) +- [x] Test: multiple formats, only target extracted +- [x] Test: source info preserved through resolution + +### Phase 3: Project Config Parsing + +**File**: `crates/quarto-core/src/project.rs` + +- [x] Add `quarto-yaml` and `pampa` to `quarto-core` dependencies +- [x] Replace `raw: serde_json::Value` with `metadata: Option` in `ProjectConfig` +- [x] Remove `format_config` field (superseded by `metadata`) +- [x] Update `parse_config()` to: + - Parse YAML with `quarto_yaml::parse_file()` + - Convert to ConfigValue with `yaml_to_config_value(..., InterpretationContext::ProjectConfig)` + - Store in `metadata` field +- [x] Update `ProjectConfig::with_format_config()` → `ProjectConfig::with_metadata()` +- [x] Update `Default` impl for `ProjectConfig` (unchanged - already works) + +### Phase 4: Tests for Project Config Parsing + +**File**: `crates/quarto-core/src/project.rs` (tests module) + +- [x] Test: `test_project_config_default` - verifies default ProjectConfig +- [x] Test: `test_project_config_with_metadata` - verifies with_metadata constructor +- [ ] Test: parse simple `_quarto.yml` into ConfigValue - deferred (requires file system mocking) +- [ ] Test: source info points to correct file - deferred +- [ ] Test: strings in project config are literal (not markdown) - deferred +- [ ] Test: nested format config parsed correctly - deferred +- [ ] Test: error handling for invalid YAML - deferred + +### Phase 5: AstTransformsStage Integration + +**File**: `crates/quarto-core/src/stage/stages/ast_transforms.rs` + +- [x] Update merge logic to use `resolve_format_config()` +- [x] Get target format name from `ctx.format.identifier.as_str()` +- [x] Flatten both project and document metadata for target format +- [x] Merge flattened configs +- [x] Update trace logging + +### Phase 6: Integration Tests + +**Location**: `crates/quarto-core/src/stage/stages/ast_transforms.rs` (tests module) + +- [x] Test: project title inherited by document (`test_project_metadata_merging_basic`) +- [x] Test: document title overrides project title (`test_project_metadata_document_overrides_project`) +- [x] Test: project `format.html.*` inherited (`test_project_format_specific_settings_inherited`) +- [x] Test: document `format.html.*` overrides project (`test_document_format_specific_overrides_project`) +- [x] Test: top-level `toc` overridden by `format.html.toc` (`test_top_level_overridden_by_format_specific`) +- [x] Test: non-target format settings ignored (`test_non_target_format_settings_ignored`) +- [x] Test: WASM `with_metadata()` still works - verified by hub-client builds + +### Phase 7: WASM Compatibility + +**File**: `crates/wasm-quarto-hub-client/src/lib.rs` + +- [x] Update WASM to use `ProjectConfig::with_metadata()` instead of `with_format_config()` +- [x] The injected config is already a full metadata ConfigValue with `format.html.*` nested +- [x] Example: `{ format: { html: { source-location: "full" } } }` +- [ ] Test hub-client renders correctly with injected config - requires WASM build verification +- [ ] Run `cargo xtask verify` to ensure WASM builds work - needs manual verification + +## Test Specifications + +### Unit Tests: resolve_format_config() + +```rust +#[test] +fn test_resolve_format_top_level_only() { + // Input: { title: "Hello", toc: true } + // Target: "html" + // Output: { title: "Hello", toc: true } +} + +#[test] +fn test_resolve_format_specific_overrides_top_level() { + // Input: { toc: true, format: { html: { toc: false } } } + // Target: "html" + // Output: { toc: false } +} + +#[test] +fn test_resolve_format_merges_non_overlapping() { + // Input: { title: "Hello", format: { html: { theme: "cosmo" } } } + // Target: "html" + // Output: { title: "Hello", theme: "cosmo" } +} + +#[test] +fn test_resolve_format_nested_objects() { + // Input: { + // format: { + // html: { + // code-fold: true, + // code-tools: { source: true } + // } + // } + // } + // Target: "html" + // Output: { code-fold: true, code-tools: { source: true } } +} + +#[test] +fn test_resolve_format_missing_target() { + // Input: { title: "Hello", format: { pdf: { documentclass: "article" } } } + // Target: "html" + // Output: { title: "Hello" } // pdf settings ignored +} + +#[test] +fn test_resolve_format_shorthand() { + // Input: { format: "html" } // shorthand, not object + // Target: "html" + // Output: {} // format: html means html with defaults +} + +#[test] +fn test_resolve_format_preserves_source_info() { + // Verify SourceInfo from original keys is preserved +} +``` + +### Integration Tests: Full Pipeline + +```rust +#[test] +fn test_project_document_merge_title() { + // _quarto.yml: { title: "Project" } + // doc.qmd: { title: "Document" } + // Result: title = "Document" +} + +#[test] +fn test_project_document_merge_inherited_settings() { + // _quarto.yml: { bibliography: "refs.bib", format: { html: { theme: "cosmo" } } } + // doc.qmd: { title: "Document" } + // Result: bibliography = "refs.bib", theme = "cosmo", title = "Document" +} + +#[test] +fn test_format_specific_override_across_layers() { + // _quarto.yml: { toc: true, format: { html: { toc-depth: 3 } } } + // doc.qmd: { format: { html: { toc-depth: 2 } } } + // Result: toc = true, toc-depth = 2 +} +``` + +## Open Questions + +### Q1: Format Name Access ✅ RESOLVED + +How do we get the target format name in `AstTransformsStage`? + +**Answer**: Use `ctx.format.identifier.as_str()` which returns `"html"`, `"pdf"`, etc. + +```rust +// In ast_transforms.rs +let target_format = ctx.format.identifier.as_str(); // "html", "pdf", etc. +``` + +See `crates/quarto-core/src/format.rs:40-54` for the `FormatIdentifier::as_str()` implementation. + +### Q2: Format Shorthand Handling ✅ RESOLVED + +Should `format: html` (string) be normalized to `format: { html: {} }` during: +- YAML parsing (in `yaml_to_config_value`) +- Format resolution (in `resolve_format_config`) +- Both? + +**Finding from DeepWiki**: The codebase handles string-to-object conversions at the `ConfigValue` conversion time in `yaml_to_config_value`, based on context and tags. However, there's no existing precedent for the specific `format: html` → `format: { html: {} }` pattern. + +**Decision**: Handle in `resolve_format_config()` because: +1. It's format-specific logic that only matters when resolving for a target format +2. The `format` key has special semantics (it's a map of format names to configs) +3. Keeps `yaml_to_config_value` generic without format-specific knowledge +4. `ConfigValue::from_path()` already demonstrates creating nested structures programmatically + +### Q3: Default Format Settings ✅ RESOLVED + +Quarto 1 has built-in defaults per format (e.g., `htmlFormat()` sets `fig-width: 7`). + +**Finding**: q2 has TWO types of defaults: + +1. **Template defaults** (exist): `compute_template_defaults()` in `pampa/src/template/config_merge.rs:166-186` + - `lang: "en"` (always) + - `pagetitle` derived from `title` + - These ARE already merged as lowest-priority layer in `merged_metadata_to_context()` + +2. **Format-specific defaults** (do NOT exist): `Format::html()` etc. in `quarto-core/src/format.rs:116-143` + - `metadata: serde_json::Value::Null` - no defaults set + - Unlike Quarto 1's `htmlFormat()` which sets `fig-width: 7`, etc. + +**Decision**: For this PR, we don't need to implement format-specific defaults. The template defaults are already handled separately in the template rendering path. Format-specific defaults (like `fig-width`) can be added in a future PR if needed - they would be the lowest-priority layer before project config. + +### Q4: Backwards Compatibility ✅ RESOLVED + +The `raw: serde_json::Value` field is used elsewhere. Need to: +- Audit all usages of `ProjectConfig.raw` +- Decide: keep both fields, or migrate all usages? + +**Finding**: `raw` is only defined and assigned in `project.rs:80,334`. No other code reads from it. + +**Decision**: Remove `raw` field entirely and replace with `metadata: Option`. No backwards compatibility concerns since the field is unused. + +## Critical Implementation Details + +### Key Function Signatures + +**yaml_to_config_value** (`crates/pampa/src/pandoc/meta.rs:137`): +```rust +pub fn yaml_to_config_value( + yaml: quarto_yaml::YamlWithSourceInfo, + context: InterpretationContext, + diagnostics: &mut crate::utils::diagnostic_collector::DiagnosticCollector, +) -> ConfigValue +``` + +**InterpretationContext** (`crates/pampa/src/pandoc/meta.rs`): +```rust +pub enum InterpretationContext { + DocumentMetadata, // Strings parsed as markdown + ProjectConfig, // Strings kept literal +} +``` + +**quarto-yaml parsing** (`crates/quarto-yaml/src/parser.rs`): +```rust +// Simple parse (no filename tracking) +pub fn parse(content: &str) -> Result + +// Parse with filename for source tracking (USE THIS) +pub fn parse_file(content: &str, filename: &str) -> Result + +// Parse YAML extracted from parent document (for frontmatter) +pub fn parse_with_parent(content: &str, parent: SourceInfo) -> Result +``` + +### StageContext Access Pattern + +In `AstTransformsStage`, access format via: +```rust +let target_format = ctx.format.identifier.as_str(); // "html", "pdf", etc. +``` + +Where `ctx` is `&mut StageContext` and `ctx.format` is `Format`. + +### Test Helper Patterns + +From `crates/quarto-config/src/merged.rs` tests: +```rust +fn scalar(s: &str) -> ConfigValue { + ConfigValue::new_scalar(Yaml::String(s.into()), SourceInfo::default()) +} + +fn map(entries: Vec<(&str, ConfigValue)>) -> ConfigValue { + let map_entries: Vec = entries + .into_iter() + .map(|(k, v)| ConfigMapEntry { + key: k.to_string(), + key_source: SourceInfo::default(), + value: v, + }) + .collect(); + ConfigValue::new_map(map_entries, SourceInfo::default()) +} +``` + +### Crate Dependency Order + +Build/test order matters: +1. `quarto-source-map` (no deps) +2. `quarto-yaml` (depends on source-map) +3. `quarto-pandoc-types` (depends on source-map) +4. `quarto-config` (depends on source-map, pandoc-types) +5. `pampa` (depends on all above) +6. `quarto-core` (depends on all above, including pampa) + +**Note**: `quarto-core` already depends on `pampa` (see `Cargo.toml:31`), so we can use `pampa::yaml_to_config_value` directly. Also already depends on `quarto-yaml` via pampa. + +### ConfigValue Key Methods + +```rust +impl ConfigValue { + pub fn get(&self, key: &str) -> Option<&ConfigValue> // Navigate into maps + pub fn new_map(entries: Vec, source_info: SourceInfo) -> Self + pub fn new_scalar(yaml: Yaml, source_info: SourceInfo) -> Self + pub fn with_merge_op(self, op: MergeOp) -> Self + pub fn from_path(path: &[&str], value: &str) -> ConfigValue // Create nested structure +} +``` + +### Current ProjectConfig Fields to Change + +```rust +// Current (crates/quarto-core/src/project.rs:70-88) +pub struct ProjectConfig { + pub project_type: ProjectType, + pub output_dir: Option, + pub render_patterns: Vec, + pub raw: serde_json::Value, // REMOVE + pub format_config: Option, // REMOVE +} + +// New +pub struct ProjectConfig { + pub project_type: ProjectType, + pub output_dir: Option, + pub render_patterns: Vec, + pub metadata: Option, // ADD - full _quarto.yml +} +``` + +### WASM Update Required + +`crates/wasm-quarto-hub-client/src/lib.rs:682-696` uses: +```rust +let format_config = ConfigValue::from_path(&["format", "html", "source-location"], "full"); +let project_config = ProjectConfig::with_format_config(format_config); +``` + +Change to: +```rust +let metadata = ConfigValue::from_path(&["format", "html", "source-location"], "full"); +let project_config = ProjectConfig::with_metadata(metadata); +``` + +### Verification Commands + +After implementation: +```bash +cargo build --workspace +cargo nextest run --workspace +cargo xtask verify # Critical for WASM changes +``` + +## References + +- Design doc: `claude-notes/plans/2025-12-07-config-merging-design.md` +- Current merge code: `crates/quarto-core/src/stage/stages/ast_transforms.rs:107-128` +- ConfigValue type: `crates/quarto-pandoc-types/src/config_value.rs` +- MergedConfig: `crates/quarto-config/src/merged.rs` +- yaml_to_config_value: `crates/pampa/src/pandoc/meta.rs:137` +- Quarto 1 format resolution: `src/command/render/render-contexts.ts` (formatFromMetadata) diff --git a/claude-notes/plans/2026-02-17-dir-metadata-path-resolution.md b/claude-notes/plans/2026-02-17-dir-metadata-path-resolution.md new file mode 100644 index 00000000..e7e1078e --- /dev/null +++ b/claude-notes/plans/2026-02-17-dir-metadata-path-resolution.md @@ -0,0 +1,365 @@ +# Directory Metadata Path Resolution + +**Date**: 2026-02-17 +**Status**: Complete +**Depends on**: `2026-02-17-metadata-yml-support.md` (Complete) + +## Completion Summary + +All phases implemented successfully: + +- [x] **Phase 1: Unit Tests** - 7 tests in `directory_metadata_tests` module +- [x] **Phase 2: Core Implementation** - `adjust_paths_to_document_dir()` function +- [x] **Phase 3: Integration** - Called from `directory_metadata_for_document()` +- [x] **Phase 4: Integration Tests** - `smoke-all/metadata/dir-metadata-paths/` + +### Key Implementation Details + +1. Added `pathdiff = "0.2"` to `quarto-core/Cargo.toml` +2. Implemented recursive path adjustment in `crates/quarto-core/src/project.rs` +3. Path adjustment handles: + - Relative `!path` values: adjusted via `pathdiff::diff_paths` + - Absolute paths: unchanged + - URLs: unchanged + - Globs and other types: unchanged + - Nested values in arrays and maps: recursively adjusted + +### Verification + +- All 15 directory metadata tests pass +- All 6488 workspace tests pass (4 pre-existing Pandoc-version failures excluded) +- smoke_all test suite passes including new `dir-metadata-paths` test + +## Context + +### What is q2? + +q2 (Quarto Rust) is a Rust rewrite of Quarto, a scientific publishing system. It renders `.qmd` (Quarto Markdown) documents to HTML and other formats. + +### What is `_metadata.yml`? + +Quarto projects support directory-level metadata files named `_metadata.yml`. These files provide default metadata that applies to all documents in that directory and its subdirectories. + +Example project structure: +``` +project/ + _quarto.yml # Project config + chapters/ + _metadata.yml # Applies to all docs in chapters/ + intro/ + _metadata.yml # Applies to docs in chapters/intro/ + chapter1.qmd # Document being rendered +``` + +When rendering `chapter1.qmd`, metadata is merged in this order (later wins): +1. `_quarto.yml` (project) +2. `chapters/_metadata.yml` (directory) +3. `chapters/intro/_metadata.yml` (directory) +4. `chapter1.qmd` frontmatter (document) + +### What's Already Implemented? + +The core directory metadata discovery is complete (see `2026-02-17-metadata-yml-support.md`): + +**Function**: `crates/quarto-core/src/project.rs::directory_metadata_for_document()` + +```rust +/// Find and parse all `_metadata.yml` files between project root and document directory. +pub fn directory_metadata_for_document( + project: &ProjectContext, + document_path: &Path, +) -> Result> { + // Walks from project root to document's parent directory + // Returns Vec of ConfigValue layers (root → leaf order) +} +``` + +This function currently parses `_metadata.yml` files but does NOT adjust paths. This plan implements that missing piece. + +### What is ConfigValue? + +`ConfigValue` (defined in `crates/quarto-pandoc-types/src/config_value.rs`) is q2's representation of YAML configuration: + +```rust +pub struct ConfigValue { + pub value: ConfigValueKind, // The actual value + pub source_info: SourceInfo, // Where it came from + pub merge_op: MergeOp, // How to merge with other layers +} + +pub enum ConfigValueKind { + Scalar(Yaml), // Plain strings, ints, bools, etc. - NOT paths + PandocInlines(Inlines), // Markdown inline content + PandocBlocks(Blocks), // Markdown block content + Path(String), // Explicitly tagged as a path with !path + Glob(String), // Glob pattern with !glob + Expr(String), // Runtime expression with !expr + Array(Vec), // Array of values + Map(Vec), // Map with ConfigMapEntry { key, key_source, value } +} +``` + +**Key insight**: Only `ConfigValueKind::Path(String)` values need adjustment. Plain strings in `Scalar(Yaml::String(...))` are left alone. + +### What are `!path` tags? + +In YAML, `!path` is a custom tag that marks a value as a file path: + +```yaml +# In _metadata.yml +css: !path ./styles.css # Parsed as ConfigValueKind::Path +bibliography: !path ../refs.bib # Parsed as ConfigValueKind::Path +theme: cosmo # Parsed as ConfigValueKind::String (NOT adjusted) +``` + +The `quarto_yaml` crate handles parsing these tags. When `InterpretationContext::ProjectConfig` is used, `!path` values become `ConfigValueKind::Path`. + +### Why is Path Adjustment Needed? + +Consider: +``` +project/ + refs.bib + chapters/ + _metadata.yml # bibliography: !path ../refs.bib + intro/ + chapter1.qmd # Being rendered +``` + +The `bibliography: !path ../refs.bib` is relative to `chapters/` (where `_metadata.yml` lives). But when `chapter1.qmd` is rendered, it's in `chapters/intro/`, so the path needs to become `../../refs.bib` to correctly reference the same file. + +**This is what this plan implements.** + +## Design + +### Path Resolution Algorithm + +Given: +- `metadata_dir`: Directory containing the `_metadata.yml` (e.g., `/project/chapters/`) +- `document_dir`: Directory containing the document (e.g., `/project/chapters/intro/`) +- `path`: The path value from ConfigValueKind::Path (e.g., `../refs.bib`) + +Algorithm: +1. Compute "absolute" path by joining: `abs_path = metadata_dir.join(path)` + - `/project/chapters/` + `../refs.bib` → `/project/chapters/../refs.bib` + - Note: We do NOT canonicalize (which would require file to exist) +2. Compute relative path from document_dir: `pathdiff::diff_paths(abs_path, document_dir)` + - `pathdiff` handles the `..` components correctly + - From `/project/chapters/intro/` to `/project/chapters/../refs.bib` → `../../refs.bib` + +**Edge cases:** +- Absolute paths (`/usr/share/file.css`): Pass through unchanged +- URLs (`https://example.com/style.css`): Pass through unchanged +- Already correct (metadata_dir == document_dir): `pathdiff` returns the original path +- Non-existent files: Works fine - we don't check if files exist + +### Where to Integrate + +In `directory_metadata_for_document()`, after parsing each `_metadata.yml` but before adding to the layers vec: + +```rust +// Current code (simplified): +for component in components { + current_dir = current_dir.join(component); + if let Some(path) = find_metadata_file(¤t_dir) { + let metadata = parse_metadata_file(&path)?; + // TODO: Add path adjustment HERE + // adjust_paths_to_document_dir(&mut metadata, ¤t_dir, document_dir); + layers.push(metadata); + } +} +``` + +## Implementation Plan + +**IMPORTANT**: Follow TDD workflow per CLAUDE.md - write tests first, verify they fail, then implement. + +### Phase 1: Unit Tests (Write First) + +**File**: `crates/quarto-core/src/project.rs` (in `directory_metadata_tests` module) + +Add tests that create `_metadata.yml` files with `!path` values and verify they're adjusted: + +```rust +#[test] +fn test_path_adjusted_for_subdirectory() { + // project/ + // shared/ + // styles.css # The actual file + // chapters/ + // _metadata.yml # css: !path ../shared/styles.css + // intro/ + // doc.qmd + // + // When rendering doc.qmd, css should become "../../shared/styles.css" +} + +#[test] +fn test_path_same_directory_unchanged() { + // project/ + // chapters/ + // _metadata.yml # css: !path ./local.css + // doc.qmd # Same directory + // + // Path stays "./local.css" (or normalized equivalent) +} + +#[test] +fn test_plain_string_not_adjusted() { + // project/ + // chapters/ + // _metadata.yml # theme: cosmo (plain string, not !path) + // intro/ + // doc.qmd + // + // "cosmo" must NOT be changed to "../cosmo" or anything else +} + +#[test] +fn test_absolute_path_unchanged() { + // css: !path /usr/share/styles/base.css + // Should pass through unchanged +} + +#[test] +fn test_array_of_paths_all_adjusted() { + // css: + // - !path ../shared/a.css + // - !path ../shared/b.css + // Both should be adjusted +} + +#[test] +fn test_glob_not_adjusted() { + // resources: !glob ../images/*.png + // Globs are patterns, not paths - should NOT be adjusted +} +``` + +### Phase 2: Core Implementation + +**File**: `crates/quarto-core/src/project.rs` + +Add helper function: + +```rust +use std::path::Path; + +/// Adjust `!path` values in metadata to be relative to document directory. +/// +/// Walks the ConfigValue tree and for each `ConfigValueKind::Path`: +/// - Computes absolute path relative to metadata_dir +/// - Recomputes relative path from document_dir +/// +/// Leaves other values (strings, globs, etc.) unchanged. +fn adjust_paths_to_document_dir( + metadata: &mut ConfigValue, + metadata_dir: &Path, + document_dir: &Path, +) { + adjust_paths_recursive(metadata, metadata_dir, document_dir); +} + +/// Recursively walk ConfigValue, adjusting Path variants. +fn adjust_paths_recursive( + value: &mut ConfigValue, + metadata_dir: &Path, + document_dir: &Path, +) { + use quarto_pandoc_types::config_value::ConfigValueKind; + use std::path::PathBuf; + + match &mut value.value { + ConfigValueKind::Path(path_str) => { + let path = PathBuf::from(&*path_str); + // Only adjust relative paths (not absolute, not URLs) + if path.is_relative() && !path_str.starts_with("http://") && !path_str.starts_with("https://") { + let abs_path = metadata_dir.join(&path); + if let Some(adjusted) = pathdiff::diff_paths(&abs_path, document_dir) { + *path_str = adjusted.to_string_lossy().into_owned(); + } + } + } + ConfigValueKind::Array(items) => { + for item in items { + adjust_paths_recursive(item, metadata_dir, document_dir); + } + } + ConfigValueKind::Map(entries) => { + for entry in entries { + adjust_paths_recursive(&mut entry.value, metadata_dir, document_dir); + } + } + // All other kinds (Scalar, PandocInlines, Glob, Expr, etc.) - no adjustment + _ => {} + } +} +``` + +**Note**: The `pathdiff` crate provides `diff_paths(target, base)` which computes a relative path from `base` to `target`. + +### Phase 3: Integration + +**File**: `crates/quarto-core/src/project.rs` + +Update `directory_metadata_for_document()` at line ~133 (after `yaml_to_config_value` call, before `layers.push`): + +```rust +// Current code around line 132-137: +let mut metadata = + yaml_to_config_value(yaml, InterpretationContext::ProjectConfig, &mut diagnostics); + +// ADD THIS: Adjust paths to be relative to document directory +adjust_paths_to_document_dir(&mut metadata, ¤t_dir, document_dir); + +layers.push(metadata); +``` + +Note: `document_dir` is already computed earlier in the function (line 87-89). + +### Phase 4: Integration Tests + +**Location**: `crates/quarto/tests/smoke-all/metadata/dir-metadata-paths/` + +Create test structure: +``` +dir-metadata-paths/ + _quarto.yml + shared/ + styles.css # Actual CSS file + chapters/ + _metadata.yml # css: !path ../shared/styles.css + intro/ + doc.qmd # Verify CSS link resolves correctly +``` + +Test assertions in `doc.qmd`: +```yaml +_quarto: + tests: + html: + ensureFileRegexMatches: + - ["../../shared/styles.css"] # Adjusted path appears in HTML +``` + +## Dependencies + +Add `pathdiff` crate to `crates/quarto-core/Cargo.toml` if not present: +```toml +pathdiff = "0.2" +``` + +## Verification + +After implementation: +1. Run unit tests: `cargo nextest run -p quarto-core directory_metadata` +2. Run smoke tests: `cargo nextest run -p quarto smoke_all` +3. Run full verification: `cargo xtask verify` + +## References + +- Existing implementation: `crates/quarto-core/src/project.rs::directory_metadata_for_document()` +- ConfigValue types: `crates/quarto-pandoc-types/src/config_value.rs` +- YAML parsing: `crates/quarto-yaml/src/lib.rs` +- TS Quarto reference: `~/src/quarto-cli/src/project/project-shared.ts:137-206` (`toInputRelativePaths`) +- Parent plan: `claude-notes/plans/2026-02-17-metadata-yml-support.md` diff --git a/claude-notes/plans/2026-02-17-metadata-yml-support.md b/claude-notes/plans/2026-02-17-metadata-yml-support.md new file mode 100644 index 00000000..22f53cb3 --- /dev/null +++ b/claude-notes/plans/2026-02-17-metadata-yml-support.md @@ -0,0 +1,838 @@ +# Directory Metadata (_metadata.yml) Support + +**Date**: 2026-02-17 +**Status**: Core Implementation Complete (Path Resolution Deferred) +**Depends on**: `2026-02-16-project-metadata-merging.md` (Complete) + +## Overview + +Implement support for `_metadata.yml` files, which provide directory-level configuration that applies to all documents within that directory and its subdirectories. This fills in the missing layer between project config (`_quarto.yml`) and document frontmatter. + +### Goals + +1. Discover `_metadata.yml` files in the directory hierarchy from project root to document +2. Parse each file as `ConfigValue` with source tracking +3. Fail render on invalid YAML (syntax errors) +4. Merge directory metadata into the layering system between project and document +5. Convert relative paths in `_metadata.yml` to be document-relative + +### Non-Goals + +- WASM/VFS support (deferred - hub-client uses `with_metadata()` injection) +- Caching of directory metadata (can optimize later if needed) +- Command-line flag merging (separate future work) +- **Schema validation** - q2 doesn't have front-matter schemas ported yet (TS Quarto builds these programmatically from 60+ YAML files). For now we validate YAML syntax only. Schema validation can be added when schemas are available. + +## Background + +### Full Metadata Layering (TS Quarto) + +From lowest to highest priority: +1. Built-in format defaults +2. Project `_quarto.yml` (top-level, then format-specific) +3. **Directory `_metadata.yml` files** (root → leaf, deeper wins) ← THIS PR +4. Document frontmatter (top-level, then format-specific) +5. Command-line flags + +### TS Quarto Implementation + +From `src/project/project-shared.ts`: + +```typescript +async function directoryMetadataForInputFile(project, inputDir) { + // Walk from project root to input directory + const relativePath = relative(projectDir, inputDir); + const dirs = relativePath.split(SEP_PATTERN); + + let config = {}; + let currentDir = projectDir; + + for (const dir of dirs) { + currentDir = join(currentDir, dir); + const file = metadataFile(currentDir); // _metadata.yml or _metadata.yaml + if (file) { + // Read, validate, normalize format, convert paths + const yaml = await readAndValidateYamlFromFile(file, frontMatterSchema, errMsg); + if (yaml.format) { + yaml.format = normalizeFormatYaml(yaml.format); + } + config = mergeConfigs(config, toInputRelativePaths(..., yaml)); + } + } + return config; +} +``` + +Key behaviors: +- Walks directories root → leaf +- Looks for `_metadata.yml` or `_metadata.yaml` +- Validates against front-matter schema (fails render on error) +- Normalizes `format` key +- Converts paths to be relative to input document +- Deeper directories override (via `mergeConfigs`) + +### Current q2 State + +After `2026-02-16-project-metadata-merging.md`: +- `resolve_format_config()` flattens format-specific settings (in `quarto-config/src/format.rs`) +- `ProjectConfig.metadata` holds parsed `_quarto.yml` as `Option` +- `AstTransformsStage` merges: project → document (see `ast_transforms.rs:~150-200`) +- Schema validation infrastructure exists in `quarto-yaml-validation` (but no schemas ported) + +**Important**: The merge happens in `AstTransformsStage::execute()`. The current flow is: +1. Get project metadata from `ctx.project.config.metadata` +2. Flatten with `resolve_format_config(&project_meta, target_format)` +3. Flatten document meta with `resolve_format_config(&doc.ast.meta, target_format)` +4. Merge with `MergedConfig::new(vec![&project_layer, &doc_layer])` +5. Materialize and assign to `doc.ast.meta` + +This PR adds directory metadata layers between steps 2 and 3. + +## Design + +### Approach + +Add a new function `directory_metadata_for_document()` that: +1. Takes project context and document path +2. Walks directory hierarchy from project root to document's directory +3. Finds and parses `_metadata.yml` files +4. Validates each against front-matter schema +5. Returns a list of `ConfigValue` layers (ordered root → leaf) + +Then update `AstTransformsStage` to merge all layers: +``` +project → dir_metadata[0] → dir_metadata[1] → ... → document +``` + +### New Types and Functions + +#### 1. Directory Metadata Discovery + +**File**: `crates/quarto-core/src/project.rs` + +```rust +/// Find and parse all _metadata.yml files between project root and document directory. +/// +/// Walks the directory hierarchy from project root to the document's parent directory, +/// looking for `_metadata.yml` or `_metadata.yaml` files. Each found file is parsed +/// and validated against the front-matter schema. +/// +/// # Arguments +/// +/// * `project` - The project context (provides project root directory) +/// * `document_path` - Path to the document being rendered +/// +/// # Returns +/// +/// A vector of `ConfigValue` layers, ordered from project root to document directory. +/// Each layer contains the parsed and validated metadata from that directory's +/// `_metadata.yml` file. Directories without `_metadata.yml` are skipped. +/// +/// # Errors +/// +/// Returns an error if: +/// - A `_metadata.yml` file contains invalid YAML +/// - A `_metadata.yml` file fails schema validation +/// - File I/O errors occur +/// +/// # Example +/// +/// Given project structure: +/// ```text +/// project/ +/// _quarto.yml +/// _metadata.yml # Layer 0: { theme: "cosmo" } +/// chapters/ +/// _metadata.yml # Layer 1: { toc: true } +/// intro/ +/// _metadata.yml # Layer 2: { toc-depth: 2 } +/// chapter1.qmd # Document being rendered +/// ``` +/// +/// Returns: [layer0, layer1, layer2] - deeper directories later in vec +pub fn directory_metadata_for_document( + project: &ProjectContext, + document_path: &Path, +) -> Result> +``` + +#### 2. Path Resolution Helper + +**File**: `crates/quarto-core/src/project.rs` + +```rust +/// Convert relative paths in metadata to be relative to the target document. +/// +/// Walks the ConfigValue tree and adjusts any `ConfigValueKind::Path` values. +/// When a `_metadata.yml` in `chapters/` has `template: !path templates/custom.tex`, +/// and we're rendering `chapters/intro/doc.qmd`, the path is adjusted to +/// `../templates/custom.tex` so it resolves correctly from the document's location. +/// +/// # Arguments +/// +/// * `metadata` - The ConfigValue containing paths to convert +/// * `metadata_dir` - Directory where the _metadata.yml file is located +/// * `document_dir` - Directory where the document being rendered is located +/// +/// # Returns +/// +/// A new ConfigValue with Path variants adjusted to be relative to document_dir. +/// Non-path values (Scalar strings, etc.) are unchanged. +/// +/// # Note +/// +/// Only `ConfigValueKind::Path` values (from `!path` tags) are converted. +/// Plain strings are not treated as paths. Users must use `!path` tags for +/// paths that need resolution: +/// +/// ```yaml +/// template: !path templates/custom.tex # Will be converted +/// title: "My Title" # Not converted (plain string) +/// ``` +pub fn convert_paths_to_document_relative( + metadata: ConfigValue, + metadata_dir: &Path, + document_dir: &Path, +) -> ConfigValue +``` + +#### 3. Updated Merge in AstTransformsStage + +**File**: `crates/quarto-core/src/stage/stages/ast_transforms.rs` + +```rust +// In AstTransformsStage::execute() + +// Get target format +let target_format = ctx.format.identifier.as_str(); + +// Layer 1: Project metadata (flattened for format) +let project_layer = ctx.project.config + .as_ref() + .and_then(|c| c.metadata.as_ref()) + .map(|m| resolve_format_config(m, target_format)); + +// Layer 2: Directory metadata (multiple layers, each flattened) +let dir_layers: Vec = directory_metadata_for_document(&ctx.project, &ctx.document.path)? + .into_iter() + .map(|m| resolve_format_config(&m, target_format)) + .collect(); + +// Layer 3: Document metadata (flattened for format) +let doc_layer = resolve_format_config(&doc.ast.meta, target_format); + +// Build merge layers: project → dir[0] → dir[1] → ... → document +let mut layers: Vec<&ConfigValue> = Vec::new(); +if let Some(ref proj) = project_layer { + layers.push(proj); +} +for dir_meta in &dir_layers { + layers.push(dir_meta); +} +layers.push(&doc_layer); + +// Merge all layers +let merged = MergedConfig::new(layers); +if let Ok(materialized) = merged.materialize() { + doc.ast.meta = materialized; +} +``` + +### File Changes + +| File | Change | +|------|--------| +| `crates/quarto-core/src/project.rs` | Add `directory_metadata_for_document()` | +| `crates/quarto-core/src/project.rs` | Add `convert_paths_to_document_relative()` | +| `crates/quarto-core/src/stage/stages/ast_transforms.rs` | Update merge to include directory layers | + +### Error Handling + +For now, we only validate YAML syntax (not schema). Parse errors from `quarto_yaml::parse_file()` will be caught and reported with file path context: + +```rust +fn parse_metadata_file(path: &Path) -> Result { + let content = fs::read_to_string(path)?; + let filename = path.file_name().unwrap_or_default().to_string_lossy(); + + let yaml = quarto_yaml::parse_file(&content, &filename) + .map_err(|e| anyhow!( + "Directory metadata validation failed for {}: {}", + path.display(), + e + ))?; + + // Convert to ConfigValue + let mut diagnostics = DiagnosticCollector::new(); + Ok(yaml_to_config_value(yaml, InterpretationContext::ProjectConfig, &mut diagnostics)) +} +``` + +**Future**: When front-matter schemas are ported to q2 (see investigation in `claude-notes/`), we can add schema validation using `quarto-yaml-validation`. + +### Path Resolution Approach + +**Key insight**: q2 has a better approach than TS Quarto. + +**TS Quarto** (implicit): Walks entire metadata tree, probes filesystem with `existsSync()` for every string, converts if file exists. + +**q2** (explicit): Uses `ConfigValueKind::Path` variant. Paths are explicitly marked via: +- `!path` YAML tag: `template: !path templates/custom.tex` +- `!glob` YAML tag: `resources: !glob images/*.png` + +This means: +1. **No filesystem probing** required +2. **No list of path keys** to maintain +3. Walk the `ConfigValue` tree and adjust any `ConfigValueKind::Path` values + +**User requirement**: To get path resolution in `_metadata.yml`, users must use `!path` tags: +```yaml +# _metadata.yml +template: !path templates/custom.tex +bibliography: !path refs.bib +css: !path styles/custom.css +``` + +**Future enhancement**: Schema-driven interpretation could automatically treat `schema: path` fields as paths without explicit tags. This would require parsing to be schema-aware. + +## Implementation Plan + +### Phase 1: Directory Metadata Discovery ✅ COMPLETE + +**File**: `crates/quarto-core/src/project.rs` + +- [x] Add `find_metadata_file()` helper (looks for `_metadata.yml` or `_metadata.yaml`) +- [x] Add `directory_metadata_for_document()` function +- [x] Walk directory hierarchy from project root to document directory +- [x] Parse each `_metadata.yml` with `quarto_yaml::parse_file()` +- [x] Convert to ConfigValue with `InterpretationContext::ProjectConfig` +- [x] Return vector of layers (root → leaf order) + +### Phase 2: Error Handling ✅ COMPLETE + +**File**: `crates/quarto-core/src/project.rs` + +- [x] Handle YAML parse errors with descriptive messages +- [x] Include file path and source location in error messages +- [x] Fail render with "Directory metadata validation failed for {file}" message +- [ ] (Future) Add schema validation when front-matter schemas are ported + +### Phase 3: Path Resolution - DEFERRED + +**File**: `crates/quarto-core/src/project.rs` + +Path resolution is deferred to a separate PR for the following reasons: + +1. **Separate concern**: Path resolution is orthogonal to directory metadata discovery. The core functionality (finding and merging `_metadata.yml` files) works without it. + +2. **Different design**: q2 uses explicit `!path` tags instead of TS Quarto's implicit filesystem probing. This requires: + - Walking `ConfigValueKind::Path` variants only (not all strings) + - No filesystem existence checks needed + - Cleaner, more predictable behavior + +3. **Testing complexity**: Path resolution tests need careful setup with actual file structures to verify relative path calculations work correctly. + +4. **Future schema integration**: When front-matter schemas are ported, schema-driven path interpretation could automatically treat `schema: path` fields as paths without explicit tags. + +**See**: `claude-notes/plans/2026-02-17-dir-metadata-path-resolution.md` for implementation plan. + +- [ ] Add `convert_paths_to_document_relative()` function (separate PR) +- [ ] Walk ConfigValue tree recursively (separate PR) +- [ ] For `ConfigValueKind::Path` values, compute relative path adjustment (separate PR) +- [ ] Handle both single paths and arrays containing paths (separate PR) +- [ ] Preserve `ConfigValueKind::Glob` unchanged (separate PR) +- [ ] Apply path conversion after parsing, before returning layers (separate PR) + +### Phase 4: Merge Integration ✅ COMPLETE + +**File**: `crates/quarto-core/src/stage/stages/ast_transforms.rs` + +- [x] Update `execute()` to call `directory_metadata_for_document()` +- [x] Apply `resolve_format_config()` to each directory layer +- [x] Build complete layer list: project → dirs → document +- [x] Update `MergedConfig::new()` call with all layers +- [x] Update tracing/logging to show directory layers + +### Phase 5: Unit Tests ✅ COMPLETE + +**File**: `crates/quarto-core/src/project.rs` (tests module) + +- [x] Test: no `_metadata.yml` files returns empty vec +- [x] Test: single `_metadata.yml` in document's directory +- [x] Test: multiple `_metadata.yml` files in hierarchy +- [x] Test: `_metadata.yaml` alternate extension works +- [x] Test: invalid YAML syntax fails with descriptive error +- [x] Test: document at project root returns empty vec +- [x] Test: single-file project returns empty vec +- [ ] Test: `!path` tagged values become `ConfigValueKind::Path` (deferred with path resolution) +- [ ] Test: path conversion adjusts `Path` variants to document-relative (deferred) +- [ ] Test: plain strings (not `!path` tagged) are NOT converted (deferred) + +### Phase 6: Integration Tests (smoke-all) ✅ COMPLETE + +**Location**: `crates/quarto/tests/smoke-all/metadata/` + +- [x] Test: directory metadata inherited by document (`dir-metadata/`) +- [x] Test: deeper directory overrides shallower (`dir-metadata-hierarchy/`) +- [x] Test: document overrides directory metadata (`dir-metadata-override/`) +- [ ] Test: format-specific settings in `_metadata.yml` work (TODO) +- [ ] Test: relative paths in `_metadata.yml` resolve correctly (deferred with path resolution) + +### Phase 7: Documentation + +- [ ] Update plan status to Complete +- [ ] Document new behavior in relevant module docs + +## Running Tests + +**Unit tests** (fast, run frequently): +```bash +cargo nextest run --workspace +# Or for specific crate: +cargo nextest run -p quarto-core +``` + +**Smoke-all tests** (slower, integration tests): +```bash +cargo nextest run -p quarto smoke_all +# Run specific test: +cargo nextest run -p quarto smoke_all::metadata +``` + +**Full verification** (before committing): +```bash +cargo xtask verify +``` + +**IMPORTANT**: Do NOT pipe `cargo nextest run` through `tail` or other commands - it causes hangs. Run it directly. + +## Test Specifications + +### Unit Tests + +```rust +#[test] +fn test_directory_metadata_empty() { + // Project with no _metadata.yml files + // Returns: empty vec +} + +#[test] +fn test_directory_metadata_single_file() { + // project/ + // chapters/ + // _metadata.yml { toc: true } + // doc.qmd + // Returns: [{ toc: true }] +} + +#[test] +fn test_directory_metadata_hierarchy() { + // project/ + // _metadata.yml { theme: "cosmo" } + // chapters/ + // _metadata.yml { toc: true } + // intro/ + // _metadata.yml { toc-depth: 2 } + // doc.qmd + // Returns: [{ theme }, { toc }, { toc-depth }] in order +} + +#[test] +fn test_directory_metadata_skips_missing() { + // project/ + // _metadata.yml { theme: "cosmo" } + // chapters/ + // intro/ # No _metadata.yml here + // _metadata.yml { toc: true } + // doc.qmd + // Returns: [{ theme }, { toc }] - skips chapters/ +} + +#[test] +fn test_directory_metadata_invalid_yaml_fails() { + // _metadata.yml with YAML syntax error + // Returns: Err with "Directory metadata validation failed for..." + // Note: Only validates syntax, not schema (schemas not yet ported) +} + +#[test] +fn test_path_conversion() { + // _metadata.yml in chapters/: { template: !path "templates/custom.tex" } + // Document in chapters/intro/doc.qmd + // Result: template path adjusted to "../templates/custom.tex" + // Note: Only ConfigValueKind::Path values are converted, not plain strings +} +``` + +### Smoke-all Tests + +**How smoke-all tests work**: The test runner in `crates/quarto/tests/smoke_all.rs` finds all `.qmd` files in `smoke-all/`, renders them, and checks assertions in the `_quarto.tests` frontmatter block. + +**Test directory structure**: +``` +crates/quarto/tests/smoke-all/metadata/dir-metadata/ +├── _quarto.yml # Project config (required) +├── _metadata.yml # Root directory metadata +├── chapters/ +│ ├── _metadata.yml # Chapter-level metadata +│ └── chapter1.qmd # Document with test assertions +``` + +**Example test files**: + +```yaml +# tests/smoke-all/metadata/dir-metadata/_quarto.yml +project: + type: default +title: "Dir Metadata Project" + +# tests/smoke-all/metadata/dir-metadata/_metadata.yml +# This should be inherited by all docs in this project +author: "Project Author" +toc: false + +# tests/smoke-all/metadata/dir-metadata/chapters/_metadata.yml +# This overrides the root _metadata.yml for docs in chapters/ +toc: true +toc-depth: 2 + +# tests/smoke-all/metadata/dir-metadata/chapters/chapter1.qmd +--- +title: Chapter 1 +_quarto: + tests: + html: + ensureHtmlElements: + - ["nav#TOC", "TOC should be present (from chapters/_metadata.yml toc: true)"] + ensureFileRegexMatches: + - ["Project Author", "Author should be inherited from root _metadata.yml"] +--- + +## Introduction + +Content here with a second heading. + +## Another Section + +More content. +``` + +**Test for path resolution** (if implementing): +```yaml +# tests/smoke-all/metadata/dir-metadata-paths/_quarto.yml +project: + type: default + +# tests/smoke-all/metadata/dir-metadata-paths/_metadata.yml +template: !path templates/custom.html + +# tests/smoke-all/metadata/dir-metadata-paths/templates/custom.html +# (actual template file) + +# tests/smoke-all/metadata/dir-metadata-paths/chapters/doc.qmd +# The template path should be resolved to ../templates/custom.html +``` + +## Open Questions + +### Q1: Schema Availability ✅ RESOLVED (DEFERRED) + +The front-matter schema is NOT yet ported to q2. TS Quarto builds it programmatically from 60+ YAML files in `src/resources/schema/`. The `quarto-yaml-validation` crate has the validation infrastructure, but no schemas. + +**Decision**: Skip schema validation for now. Validate YAML syntax only. Add schema validation when schemas are ported (separate future work). + +### Q2: Path Key List ✅ RESOLVED + +**Finding**: q2 uses explicit `ConfigValueKind::Path` variant instead of key-based detection. + +**TS Quarto approach** (implicit): +- Walks entire tree, probes filesystem for every string +- Expensive and implicit + +**q2 approach** (explicit): +- Users mark paths with `!path` tag: `template: !path custom.tex` +- Creates `ConfigValueKind::Path` variant +- Walk tree and adjust only `Path` variants + +**Decision**: Use q2's explicit approach. Only `ConfigValueKind::Path` values are converted. +No list of path keys needed. Users must use `!path` tags for paths requiring resolution. + +**Future**: Schema-driven interpretation could auto-detect `schema: path` fields. + +### Q3: Single-File Mode + +When rendering a standalone document (no project), should we: +- Look for `_metadata.yml` in parent directories up to some limit? +- Only look in the document's own directory? +- Skip directory metadata entirely? + +TS Quarto requires a project context for `directoryMetadataForInputFile()`. + +**Proposed**: Skip directory metadata for single-file mode (no project). This matches TS behavior. + +## Dependencies + +``` +quarto-yaml ──────────────────────┐ +quarto-yaml-validation ───────────┤ + ▼ +quarto-core (project.rs) ◄─── directory_metadata_for_document() + │ + ▼ +quarto-core (ast_transforms.rs) ◄─── merge all layers +``` + +## Key Code Locations + +### Existing Implementation to Study + +**Project metadata merging** (just completed, use as template): +- `crates/quarto-core/src/stage/stages/ast_transforms.rs` - Look at lines ~107-200 for the existing project → document merge +- `crates/quarto-config/src/format.rs` - `resolve_format_config()` flattens `format.{target}.*` to top-level + +**YAML parsing to ConfigValue**: +- `crates/pampa/src/pandoc/meta.rs` - `yaml_to_config_value()` function +- Uses `InterpretationContext::ProjectConfig` for `_quarto.yml` and `_metadata.yml` (strings stay literal) +- Uses `InterpretationContext::DocumentMetadata` for frontmatter (strings parsed as markdown) + +**Project context**: +- `crates/quarto-core/src/project.rs` - `ProjectContext` struct has `dir: PathBuf` (project root) +- `ProjectConfig` struct has `metadata: Option` (the parsed `_quarto.yml`) +- **Look at `find_project_config()`** (~line 263) - shows pattern for checking `.yml` and `.yaml` extensions +- **Look at `parse_config()`** (~line 301) - shows how to parse YAML to ConfigValue with `yaml_to_config_value` + +**SystemRuntime abstraction**: +- `quarto_system_runtime::SystemRuntime` trait abstracts file I/O +- Used for `runtime.path_exists()`, `runtime.read_to_string()`, etc. +- This allows WASM and native to share code with different I/O implementations +- For `directory_metadata_for_document()`, you can either: + - Take `runtime: &dyn SystemRuntime` parameter (consistent with existing code) + - Use `std::fs` directly if this function is only called in native context + +**ConfigValue types**: +- `crates/quarto-pandoc-types/src/config_value.rs` - `ConfigValue`, `ConfigValueKind`, `ConfigMapEntry` +- `ConfigValueKind::Path(String)` - created by `!path` YAML tag +- `ConfigValueKind::Glob(String)` - created by `!glob` YAML tag +- `ConfigValueKind::Map(Vec)` - for nested structures + +**Merging**: +- `crates/quarto-config/src/merged.rs` - `MergedConfig::new(layers)` and `merged.materialize()` +- Layers are ordered lowest-to-highest priority (first = lowest) + +### Existing Smoke Tests (use as examples) + +Recently added metadata tests in `crates/quarto/tests/smoke-all/metadata/`: +- `project-inherits/` - document inherits from `_quarto.yml` +- `doc-overrides/` - document overrides project settings +- `format-specific/` - tests `format.html.toc` layering + +These use the `_quarto.tests` frontmatter pattern for assertions: +```yaml +_quarto: + tests: + html: + ensureHtmlElements: + - ["nav#TOC", "TOC should be present"] + ensureFileRegexMatches: + - ["some regex pattern"] +``` + +### Imports You'll Need + +```rust +// In project.rs +use std::path::{Path, PathBuf}; +use std::fs; +use anyhow::{anyhow, Result}; +use quarto_yaml; +use pampa::yaml_to_config_value; +use pampa::utils::diagnostic_collector::DiagnosticCollector; +use quarto_config::{ConfigValue, ConfigValueKind, InterpretationContext}; + +// For path manipulation (choose one approach): +// Option A: Add pathdiff to Cargo.toml +use pathdiff::diff_paths; + +// Option B: Manual implementation (no new dependency) +// See "Path Relativity Calculation" section below +``` + +**Note**: `yaml_to_config_value` takes a `DiagnosticCollector` for collecting warnings. Create with `DiagnosticCollector::new()`. Check `diagnostics.has_errors()` after parsing. + +**Note**: `pathdiff` is NOT currently in the workspace. Either add it to `Cargo.toml` or implement the relative path calculation manually (see example below). + +### ConfigValue Tree Walking Pattern + +To walk a ConfigValue tree and transform Path values: + +```rust +fn transform_paths(value: ConfigValue, transform: impl Fn(&str) -> String) -> ConfigValue { + match value.value { + ConfigValueKind::Path(p) => ConfigValue { + value: ConfigValueKind::Path(transform(&p)), + ..value + }, + ConfigValueKind::Map(entries) => { + let new_entries = entries.into_iter().map(|entry| ConfigMapEntry { + value: transform_paths(entry.value, &transform), + ..entry + }).collect(); + ConfigValue { + value: ConfigValueKind::Map(new_entries), + ..value + } + }, + ConfigValueKind::Array(items) => { + let new_items = items.into_iter() + .map(|item| transform_paths(item, &transform)) + .collect(); + ConfigValue { + value: ConfigValueKind::Array(new_items), + ..value + } + }, + // Other variants pass through unchanged + _ => value, + } +} +``` + +### Path Relativity Calculation + +To convert a path relative to `metadata_dir` to be relative to `document_dir`: + +```rust +use std::path::Path; + +fn make_document_relative( + path: &str, + metadata_dir: &Path, + document_dir: &Path, +) -> String { + // 1. Make the path absolute (relative to metadata_dir) + let absolute = metadata_dir.join(path); + + // 2. Make it relative to document_dir + // Use pathdiff crate or manual calculation + if let Some(relative) = pathdiff::diff_paths(&absolute, document_dir) { + relative.to_string_lossy().to_string() + } else { + // Fallback: return original if can't compute relative + path.to_string() + } +} +``` + +## References + +- TS Quarto implementation: `~/src/quarto-cli/src/project/project-shared.ts` (`directoryMetadataForInputFile`) +- TS Quarto merge: `~/src/quarto-cli/src/command/render/render-contexts.ts` (`renderContexts`) +- Previous plan: `claude-notes/plans/2026-02-16-project-metadata-merging.md` +- Schema validation: `crates/quarto-yaml-validation/src/validator.rs` +- Schema gap analysis: `claude-notes/investigations/2026-02-17-schema-validation-gap-analysis.md` + +## TS Quarto Code Analysis (2026-02-17) + +### directoryMetadataForInputFile (project-shared.ts:299-355) + +```typescript +export async function directoryMetadataForInputFile( + project: ProjectContext, + inputDir: string, // <-- NOTE: this is the INPUT DIRECTORY, not the file itself +) { + const projectDir = project.dir; + + // Finds _metadata.yml or _metadata.yaml + const metadataFile = (dir: string) => { + return ["_metadata.yml", "_metadata.yaml"] + .map((file) => join(dir, file)) + .find(existsSync1); + }; + + // Walk from project root to input directory + const relativePath = relative(projectDir, inputDir); + const dirs = relativePath.split(SEP_PATTERN); + + let config = {}; + let currentDir = projectDir; + + for (let i = 0; i < dirs.length; i++) { + const dir = dirs[i]; + currentDir = join(currentDir, dir); + const file = metadataFile(currentDir); + if (file) { + // Validates against frontMatterSchema + const yaml = await readAndValidateYamlFromFile(file, frontMatterSchema, errMsg); + + // Normalize format key + if (yaml.format) { + yaml.format = normalizeFormatYaml(yaml.format); + } + + // Convert paths and merge (deeper overrides shallower) + config = mergeConfigs( + config, + toInputRelativePaths(projectType, currentDir, inputDir, yaml), + ); + } + } + return config; +} +``` + +**Key observations:** +1. Takes `inputDir` (document's parent directory), NOT the document path itself +2. Walks each directory component from project root to inputDir +3. Does NOT include project root's _metadata.yml (starts walking from first subdir) +4. Uses `mergeConfigs` which does deep merge - later values override earlier for scalars, arrays concatenate + +### toInputRelativePaths (project-shared.ts:137-206) + +This is the **implicit path resolution** - walks every string and probes filesystem: + +```typescript +export function toInputRelativePaths(type, baseDir, inputDir, collection) { + const existsCache = new Map(); + const offset = relative(inputDir, baseDir); + + const fixup = (value: string) => { + if (!existsCache.has(value)) { + const projectPath = join(baseDir, value); + try { + if (existsSync(projectPath)) { + existsCache.set(value, pathWithForwardSlashes(join(offset!, value))); + } else { + existsCache.set(value, value); + } + } catch { + existsCache.set(value, value); + } + } + return existsCache.get(value); + }; + + // Recursively walks arrays and objects, calling fixup on every string + const inner = (collection, parentKey?) => { + // ... walks structure and calls fixup on strings + }; + + inner(collection); + return collection; +} +``` + +**q2 difference**: We use explicit `!path` tags instead of filesystem probing. This is cleaner but requires users to mark paths explicitly. + +### Merge order in render-contexts.ts:397-671 + +```typescript +// resolveFormats function shows the merge order: +const userFormat = mergeFormatMetadata( + projFormat || {}, // 1. Project config + directoryFormat || {}, // 2. Directory metadata (our new layer) + inputFormat || {}, // 3. Document frontmatter +); +``` + +So the layering is: project → directory → document (document wins) diff --git a/claude-notes/plans/2026-02-17-quarto-test-infrastructure.md b/claude-notes/plans/2026-02-17-quarto-test-infrastructure.md index ab9ea026..71c4e0eb 100644 --- a/claude-notes/plans/2026-02-17-quarto-test-infrastructure.md +++ b/claude-notes/plans/2026-02-17-quarto-test-infrastructure.md @@ -149,7 +149,7 @@ Implement smoke-all style document testing for q2, enabling tests to be embedded ### 5.1 HTML-specific assertions -- [ ] `ensureHtmlElements(selectors, noMatchSelectors)` - CSS selector presence +- [x] `ensureHtmlElements(selectors, noMatchSelectors)` - CSS selector presence (scraper crate) - [ ] `ensureHtmlElementContents(selector, matches, noMatches)` - element text content - [ ] `ensureHtmlElementCount(selector, count)` - element counting diff --git a/claude-notes/plans/2026-02-18-wasm-project-context.md b/claude-notes/plans/2026-02-18-wasm-project-context.md new file mode 100644 index 00000000..10e4aeaf --- /dev/null +++ b/claude-notes/plans/2026-02-18-wasm-project-context.md @@ -0,0 +1,90 @@ +# WASM Project Context Discovery + +## Overview + +Make the WASM `render_qmd` function use the shared project discovery and directory metadata infrastructure from `quarto-core`, instead of hardcoding a minimal single-file `ProjectContext`. This enables WASM rendering to support `_quarto.yml` and `_metadata.yml` files from the VFS. + +## Current State + +- `wasm-quarto-hub-client/src/lib.rs` has `create_wasm_project_context()` which always returns `ProjectContext { config: None, is_single_file: true }` +- `quarto-core/src/project.rs` has `ProjectContext::discover()` which walks parent directories for `_quarto.yml` — already takes `&dyn SystemRuntime`, so it works with VFS +- `quarto-core/src/project.rs` has `directory_metadata_for_document()` which uses `std::fs` directly — needs porting to `SystemRuntime` +- `AstTransformsStage` already handles the full merge pipeline (project + directory + document metadata) — it's shared code +- `StageContext` already carries `runtime: Arc` — the runtime is available in every pipeline stage + +## Key Design Decision: Unify the Two WasmRuntime Instances + +### Problem + +`render_qmd()` (and similar functions) currently create two separate `WasmRuntime` instances: + +1. **Global singleton** (`get_runtime()`, line 36-45): Populated by JavaScript via Automerge sync with ALL project files — `.qmd` documents, `_quarto.yml`, `_metadata.yml`, images, etc. Used for VFS reads (`file_read`) and artifact write-back. +2. **Fresh empty runtime** (`Arc::new(WasmRuntime::new())`, lines 475/639/726/861): Created per-render and passed to the `render_qmd_to_html` pipeline. Contains nothing. + +The empty pipeline runtime is **not an intentional isolation boundary** — it's a placeholder that satisfied the `Arc` type signature when single-file rendering was first wired up. Evidence: + +- The pipeline runtime is never used for file reads (document content is passed as `&[u8]`) +- Artifacts are written back to the *global* runtime, not the pipeline runtime (lines 645-649 use `get_runtime()`) +- There's no snapshot/fork/copy-on-write VFS mechanism — `WasmRuntime::new()` just creates an empty `HashMap` +- `render_qmd_content_with_options` manually constructs project config that `discover()` would provide if it had VFS access + +### Solution + +Change the global from `OnceLock` to `OnceLock>`. Then clone the `Arc` wherever the pipeline needs `Arc`, so the pipeline reads from the same VFS that JavaScript populates. + +## Work Items + +### Phase 0: Unify WasmRuntime — share the global VFS with the pipeline + +- [x] Change `static RUNTIME: OnceLock` to `OnceLock>` in `wasm-quarto-hub-client/src/lib.rs` +- [x] Update `get_runtime()` to return `&Arc` (via `&'static Arc`) +- [x] All `get_runtime()` call sites work unchanged via `Deref` on `Arc` +- [x] Replace all `Arc::new(WasmRuntime::new())` calls (lines 475, 639, 726, 861) with `Arc::clone(get_runtime()) as Arc` +- [x] Artifact write-back (lines 645-649) stays: pipeline writes to `ctx.artifacts` (in-memory), not to the runtime VFS. The explicit copy to VFS is still needed for JS access. +- [x] Verify existing workspace tests still pass (6546/6546 passed) + +#### Tests for Phase 0 + +- [ ] **Existing tests**: All existing WASM tests should pass unchanged (single-file rendering behavior is identical) +- [ ] **test_shared_runtime_sees_vfs_files**: Add file to global VFS, verify that the `Arc` passed to the pipeline can read it via `file_read_string()` + +### Phase 1: Port `directory_metadata_for_document` to SystemRuntime + +- [x] Add `runtime: &dyn SystemRuntime` parameter to `directory_metadata_for_document()` +- [x] Replace `std::fs::read_to_string(&path)` with `runtime.file_read_string(&path)` +- [x] Replace `find_metadata_file()` to use `runtime.is_file()` instead of `Path::exists()` +- [x] Update the call site in `AstTransformsStage` to pass `ctx.runtime.as_ref()` +- [x] Update existing unit tests in `project.rs` to pass `NativeRuntime` (15/15 pass, 6546/6546 workspace pass) + +#### Unit tests for Phase 1 + +VFS-specific unit tests skipped: `WasmRuntime` only compiles on wasm32 targets. The 15 existing tests with `NativeRuntime` + `TempDir` prove the `SystemRuntime` abstraction works. VFS behavior is verified by Phase 2/3 end-to-end WASM tests. + +### Phase 2: Wire `ProjectContext::discover()` into WASM `render_qmd` + +- [x] Replace `create_wasm_project_context(path)` in `render_qmd()` with `ProjectContext::discover(path, runtime)` +- [x] `WasmRuntime` already implements `SystemRuntime` — `discover()` works as-is +- [x] `render_qmd_content()` and `render_qmd_content_with_options()` stay single-file (inline content, no VFS path) +- [x] Added `get_runtime_arc()` helper for pipeline `Arc`, `get_runtime()` still returns `&WasmRuntime` for direct calls + +#### Tests for Phase 2 + Phase 3 + +Implemented as WASM end-to-end tests in `hub-client/src/services/projectContext.wasm.test.ts` (run with `npm run test:wasm`): + +- [x] **renders single file without _quarto.yml**: Verifies backward compatibility +- [x] **inherits project title from _quarto.yml**: VFS has `_quarto.yml` with title, doc has none — title appears in HTML +- [x] **document title overrides project title**: Both have title — document wins +- [x] **discovers _quarto.yml from parent directories**: Nested doc at `/project/chapters/intro/doc.qmd` finds `/project/_quarto.yml` +- [x] **picks up directory metadata from _metadata.yml**: Author from `chapters/_metadata.yml` appears in HTML +- [x] **merges directory metadata hierarchy correctly**: Two `_metadata.yml` layers both contribute to output + +## Out of Scope + +- **`Format.metadata` merge with project config**: `Format.metadata` (a `serde_json::Value` extracted from the document's `format.` block) is read by AST transforms via `ctx.format_metadata()` but is never merged with project/directory config. This is a pre-existing gap in both native CLI and WASM paths. Tracked separately in `claude-notes/plans/2026-03-04-format-metadata-merge.md`. + +## Notes + +- `ProjectContext::discover()` walks parent directories using `runtime.path_exists()`. The `WasmRuntime` VFS uses `/project/` prefix by default. The walk goes `/project/sub/dir/` → `/project/sub/` → `/project/` → `/` → stop. This should terminate correctly but Phase 2 tests should verify it. +- The `render_qmd_content*` functions pass content directly (not via VFS), so they don't participate in project discovery. No conflict with `source_location` injection. +- `directory_metadata_for_document` currently takes `&ProjectContext` — the runtime comes as a new explicit parameter, matching the pattern used by `ProjectContext::discover()`. +- Phase 0 artifact write-back: Need to verify whether `render_qmd_to_html` writes artifacts to the runtime's VFS or only to `ctx.artifacts`. If only to `ctx.artifacts`, the explicit write-back in `render_qmd()` lines 645-649 must stay (just using the same runtime instance now). diff --git a/claude-notes/plans/2026-02-19-wasm-smoke-all-tests.md b/claude-notes/plans/2026-02-19-wasm-smoke-all-tests.md new file mode 100644 index 00000000..4d208d63 --- /dev/null +++ b/claude-notes/plans/2026-02-19-wasm-smoke-all-tests.md @@ -0,0 +1,180 @@ +# WASM Smoke-All Test Runner (TypeScript) + +## Overview + +Create a vitest-based test runner that exercises the WASM rendering module against the same smoke-all test fixtures used by the native Rust test runner. This verifies that the WASM rendering pipeline (used by hub-client for live preview) produces correct output for all tested scenarios including project metadata, directory metadata, and basic rendering. + +**Prerequisite**: The Rust-side changes in `claude-notes/plans/2026-02-18-wasm-project-context.md` are complete (commit `54875cee`). `render_qmd` now discovers `_quarto.yml` and `_metadata.yml` from the VFS. + +## Architecture + +``` +smoke-all test fixtures (shared, checked into repo) + crates/quarto/tests/smoke-all/**/*.qmd + crates/quarto/tests/smoke-all/**/_quarto.yml + crates/quarto/tests/smoke-all/**/_metadata.yml + │ + ├──→ Native Rust runner (existing) + │ crates/quarto/tests/smoke_all.rs + │ calls quarto_test::run_test_file() + │ uses NativeRuntime + real filesystem + │ + └──→ WASM vitest runner (THIS PLAN) + hub-client/src/services/smokeAll.wasm.test.ts + reads fixtures from disk (Node.js fs) + populates WASM VFS, calls render_qmd() + checks assertions against returned HTML +``` + +## Key Files to Understand + +Before implementing, read these files: + +### WASM module entry points +- **`crates/wasm-quarto-hub-client/src/lib.rs`** — The WASM module. Key functions: + - `render_qmd(path: &str) -> Promise` — Renders a QMD file from VFS. Returns JSON: `{ "success": true, "html": "...", "warnings": [...] }` or `{ "success": false, "error": "...", "diagnostics": [...] }` + - `vfs_add_file(path: &str, content: &str)` — Add a text file to VFS + - `vfs_clear()` — Clear user files from VFS (preserves embedded resources) + - `vfs_list_files()` — List all VFS files (returns JSON array) + - All VFS paths use the `/project/` prefix (e.g., `/project/_quarto.yml`, `/project/chapters/doc.qmd`) + +### Existing WASM test patterns +- **`hub-client/src/services/projectContext.wasm.test.ts`** — Best example to follow: loads WASM module, populates VFS with `vfs_add_file`, calls `render_qmd`, and asserts on HTML output. Demonstrates VFS clear/populate/render cycle. +- **`hub-client/src/services/changelogRender.wasm.test.ts`** — Another WASM test example (simpler, no VFS population). +- **`hub-client/vitest.wasm.config.ts`** — Vitest config for WASM tests. Matches `src/**/*.wasm.test.ts`, uses `environment: 'node'`, 30-second timeout. Run with `npm run test:wasm`. + +### Smoke-all test fixtures +- **`crates/quarto/tests/smoke-all/`** — The shared test fixtures. Two categories: + - `quarto-test/` — 7 basic rendering tests (callouts, code blocks, error handling, file existence) + - `metadata/` — 7 project/directory metadata test directories, each with `_quarto.yml`, possibly `_metadata.yml` files, and `.qmd` test documents + +### Test specification format +- **`crates/quarto-test/src/spec.rs`** — Rust parser for `_quarto.tests` specs (reference for the TS implementation) +- **`~/src/quarto-cli/tests/smoke/smoke-all.test.ts`** — Original TS implementation we're drawing from (Deno-based, not directly reusable) + +### Existing hub-client dependencies +- **`jsdom`** is already a dev dependency in `hub-client/package.json` — use it for `ensureHtmlElements` CSS selector queries +- **`yaml`** — **not currently in `hub-client/package.json`**. Must be added as a devDependency (`npm install -D yaml` from repo root). Needed for parsing `_quarto.tests` nested YAML from frontmatter. The `yaml` package (v2+) is the standard choice. + +## Work Items + +### Phase 0: Dependencies + +- [x] Add `yaml` package as a devDependency: run `npm install -D yaml` **from the repo root** (npm workspaces). This is needed for parsing `_quarto.tests` nested YAML from frontmatter. + +### Phase 1: Test runner infrastructure + +File: `hub-client/src/services/smokeAll.wasm.test.ts` + +- [x] WASM module initialization in `beforeAll` (follow `projectContext.wasm.test.ts` pattern — it's more relevant than `changelogRender` since it uses VFS) +- [x] `discoverTestFiles()`: use Node.js `fs` + `path` to walk `../../../crates/quarto/tests/smoke-all/**/*.qmd` relative to the test file (3 levels up from `src/services/` → `hub-client/` → repo root). Return array of absolute paths. Skip files starting with `_`. +- [x] `readFrontmatter(qmdContent: string)`: extract YAML frontmatter from QMD content (text between first `---` and second `---`). Parse with a YAML library. +- [x] `parseTestSpecs(metadata)`: extract test specs from `metadata._quarto.tests` (two levels: `metadata["_quarto"]["tests"]`). Return `{ runConfig, formatSpecs }` where each formatSpec has `{ format, assertions, checkWarnings, expectsError }`. Track `checkWarnings` as a boolean that starts `true` and is set to `false` when `noErrors`, `noErrorsOrWarnings`, or `shouldError` is encountered (matching the Rust `spec.rs` pattern). Must handle: + - `fileExists` uses object format `{ outputPath?: string, supportPath?: string }` (not bare strings) + - `printsMessage` can be a single object or an array of objects + - Unknown assertion keys must throw an error (matching both TS Quarto and Rust behavior) + - `ensureHtmlElements` uses the same two-array YAML format as `ensureFileRegexMatches`: first array = must-match selectors, second array (optional) = must-not-match selectors. Example: `ensureHtmlElements: - ["nav#TOC"]` or `ensureHtmlElements: - [] - ["nav#TOC"]`. Reuse the same array parsing logic. +- [x] `shouldSkip(runConfig)`: check `skip`, `ci`, `os`, `not_os` conditions. For WASM tests, we run on all OSes (it's platform-independent), so `os`/`not_os` checks may not apply — but implement them for compatibility. +- [x] `populateVfs(testDir, wasm)`: given a test directory (e.g., `smoke-all/metadata/project-inherits/`), find the project root (the directory containing `_quarto.yml`, or the test directory itself if no `_quarto.yml`), then recursively read all files as UTF-8 text and add them to VFS with `vfs_add_file()`. Note: `vfs_add_file` is text-only (`&str` content) — binary files in fixtures would need different handling, but current fixtures are all text. + - **Path mapping**: if the project root on disk is `/abs/path/to/smoke-all/metadata/project-inherits/`, map it to `/project/` in VFS. A file at `.../project-inherits/chapters/doc.qmd` becomes `/project/chapters/doc.qmd`. + - Call `vfs_clear()` before each test to reset state. + - Use `vfs_list_files()` as a diagnostic tool when debugging VFS population failures. + +### Phase 2: Assertion implementations + +Each assertion is a function that takes the parsed WASM render result and throws on failure. + +The WASM render result JSON has this shape (from `RenderResponse` and `JsonDiagnostic` in `lib.rs`): +```typescript +interface JsonDiagnostic { + kind: string; // Serde-serialized DiagnosticKind (verify actual casing at runtime!) + title: string; // The message text + code?: string; + problem?: string; + hints: string[]; // Always present (may be empty array) + start_line?: number; // 1-based, for Monaco + start_column?: number; + end_line?: number; + end_column?: number; + details: Array<{ kind: string; content: string; start_line?: number; start_column?: number; end_line?: number; end_column?: number; }>; +} + +interface WasmRenderResult { + success: boolean; + html?: string; // Present when success === true + error?: string; // Present when success === false + warnings?: JsonDiagnostic[]; // Present on success (when there are warnings) + diagnostics?: JsonDiagnostic[]; // Present on failure (parse errors) +} +``` + +**Important**: `diagnostics` and `warnings` are mutually exclusive — `diagnostics` only appears when `success === false`, `warnings` only when `success === true`. They are never both present. + +The `diagnostics[].title` is the message text, and `diagnostics[].kind` maps to log levels for `printsMessage` matching. The Rust runner uses `DiagnosticKind::{Error, Warning, Info, Note}` → `LogLevel::{Error, Warn, Info, Debug}` (see `runner.rs:270-275`). **Verify the exact serialized string casing** of `kind` at runtime (e.g., is it `"error"` or `"Error"`?) — the spec YAML uses uppercase (`ERROR`, `WARN`) so the mapping must be case-insensitive or normalized. + +- [x] **ensureFileRegexMatches(result, matches: string[], noMatches?: string[])** + - Assert `result.success === true` and `result.html` exists + - For each pattern in `matches`: `new RegExp(pattern, 'm').test(result.html)` must be true + - For each pattern in `noMatches`: `new RegExp(pattern, 'm').test(result.html)` must be false + - Use multiline flag (`m`) to match TS Quarto behavior + +- [x] **ensureHtmlElements(result, selectors: string[], noMatchSelectors?: string[])** + - Parsed from the same two-array YAML format as `ensureFileRegexMatches`: `[[selectors...], [noMatchSelectors...]]` + - Assert `result.success === true` and `result.html` exists + - Parse HTML with `new JSDOM(result.html)` + - For each selector in first array: `document.querySelector(selector) !== null` + - For each selector in second array: `document.querySelector(selector) === null` + +- [x] **noErrors(result)** + - Assert `result.success === true` + - If failed, include `result.error` and diagnostic titles in the error message + +- [x] **noErrorsOrWarnings(result)** + - Assert `result.success === true` + - Assert `result.warnings` is empty or absent + - Report any warning titles in the error message + +- [x] **shouldError(result)** + - Assert `result.success === false` + +- [x] **printsMessage(result, { level, regex, negate? })** + - Collect messages from whichever is present: `result.diagnostics` (on failure) or `result.warnings` (on success) — they are mutually exclusive, never both present. Map each to `{ level, message }` where `level` is derived from `kind` (case-insensitive) and `message` is `title`. + - Filter by `level` + - Check if any `message` matches `new RegExp(regex)` + - If `negate`, assert none match; otherwise assert at least one matches + +- [x] **fileExists / folderExists / pathDoesNotExist** — always-passing no-ops + - Parse from spec so the test file still exercises rendering + - `fileExists` spec uses `{ outputPath?: string, supportPath?: string }` object format — parse it but don't check anything + - `pathDoesNotExist` / `pathDoNotExists` (both spellings) and `folderExists` take bare string paths — parse them but don't check anything + - These filesystem assertions are meaningless in the WASM VFS context + +- [x] **Default assertion**: after running all explicit assertions, if `checkWarnings` is still `true` (i.e., none of `noErrors`, `noErrorsOrWarnings`, or `shouldError` were encountered during spec parsing), run `noErrorsOrWarnings` as a default check. This mirrors the Rust runner (`runner.rs:192-201`) and TS Quarto's `smoke-all.test.ts`. + +### Phase 3: Test execution loop + +- [x] For each discovered test file: + - Read content, parse frontmatter, extract test specs + - Check skip conditions + - For each format spec: **only test `html` format** (WASM only renders HTML) + - Register a vitest `it()` test case with descriptive name (relative path + format) + - In the test body: clear VFS, populate VFS, call `render_qmd(vfsPath)`, parse JSON result, run assertions +- [x] Handle the project root detection: some test directories have `_quarto.yml` at the root (metadata tests), others don't (quarto-test/ basic tests). The VFS population logic needs to find the right root. + - For metadata tests: the project root is the directory containing `_quarto.yml` (e.g., `smoke-all/metadata/project-inherits/`) + - For basic tests: there's no `_quarto.yml`, so the QMD file's directory is the project root + - Walk upward from the QMD file's directory to find `_quarto.yml`, stopping at `smoke-all/` (don't walk above the test fixture tree) + +### Phase 4: Verification + +- [x] Run `npm run test:wasm` from hub-client — new smoke-all tests pass +- [x] Verify timeout is sufficient (WASM init + multiple renders; may need to increase from 30s in vitest config) +- [x] If any tests fail due to WASM rendering differences (not bugs, just unsupported features), add appropriate skip annotations in the test runner (not in the shared fixture files). Likely candidate: `expected-error.qmd` — the WASM error path (`QuartoError::Parse`) may format the error message differently than native `render_to_file`, so the `printsMessage` regex `"unexpected character"` may not match. + +## Notes + +- **VFS `/project/` prefix**: The hub-client WASM module uses `/project/` as the VFS root by convention. All paths passed to `render_qmd()` and `vfs_add_file()` must use this prefix. +- **Format filtering**: WASM only produces HTML. Any test spec for non-HTML formats (pdf, docx, etc.) should be silently skipped. Currently all smoke-all tests specify `format: html`. +- **`ensureHtmlElements`**: The native Rust runner recognizes `ensureHtmlElements` but treats it as a no-op (not yet implemented). Unknown assertion types now cause a hard error in both Rust and TS. The WASM runner will be the first to actually check CSS selectors via jsdom. If a test fails because the HTML structure differs from expectations, that's a real finding. +- **WASM module build**: The WASM module must be built before running tests. `npm run test:wasm` should handle this, or the test should fail clearly if the WASM binary is missing. Check how existing `.wasm.test.ts` files handle this. +- **No file output**: Unlike native rendering which writes HTML to disk, WASM `render_qmd` returns HTML in the JSON response. Assertions that would read an output file instead check `result.html` directly. +- **Concurrent test isolation**: Each test must call `vfs_clear()` before populating VFS to ensure isolation. The WASM module is a singleton (single global VFS state), so tests cannot run in parallel. Vitest runs tests within a file sequentially by default, which is sufficient. Do not add `.concurrent` to describe/it blocks. If vitest config changes in the future, may need explicit `sequence: { concurrent: false }` in the WASM vitest config. diff --git a/claude-notes/plans/2026-03-07-hub-client-render-qmd-switch.md b/claude-notes/plans/2026-03-07-hub-client-render-qmd-switch.md new file mode 100644 index 00000000..810d002b --- /dev/null +++ b/claude-notes/plans/2026-03-07-hub-client-render-qmd-switch.md @@ -0,0 +1,220 @@ +# Plan: Switch Hub-Client to render_qmd + +## Context for New Agents + +This plan changes how the hub-client's live preview renders documents. Read this section +before doing anything else. + +### Architecture + +The hub-client is a React/TypeScript app in `hub-client/`. It uses a WASM build of the +Rust rendering engine (`wasm-quarto-hub-client`) for live preview. The WASM module has a +**Virtual File System (VFS)** that mirrors the project files. The Automerge sync layer +(`src/services/automergeSync.ts`) keeps the VFS in sync with the server — every file +add/change/remove triggers a corresponding `vfs_add_file()` / `vfs_remove_file()` call. + +### Current rendering flow (what we're changing) + +``` +Editor content string + → Preview.tsx calls renderToHtml(content, { sourceLocation, documentPath }) + → wasmRenderer.ts calls render_qmd_content(content) or render_qmd_content_with_options(content, opts) + → WASM creates a temporary /input.qmd, renders it with NO project context + → compileAndInjectThemeCss() runs separately for CSS + → HTML returned to Preview component → displayed in iframe +``` + +The problem: `render_qmd_content` doesn't see `_quarto.yml`, `_metadata.yml`, or any +project structure. It renders the document in isolation. + +### Target rendering flow (what we're building) + +``` +Automerge keeps VFS in sync (already working) + → Preview.tsx calls renderToHtml({ documentPath }) + → wasmRenderer.ts calls render_qmd(documentPath) + → WASM reads from VFS, discovers _quarto.yml, merges all metadata layers + → HTML returned with full project context (ToC, themes, directory metadata, etc.) +``` + +The content is NOT passed as a string — it's already in VFS via Automerge sync. +Source location for scroll sync is set via runtime metadata (see below), not a per-render option. + +### Runtime metadata (already implemented) + +Commit `e7f1e61d` added a runtime metadata layer to the WASM module. This lets the +hub-client inject metadata at the highest precedence (above project, directory, document): + +```typescript +// WASM entry points (already exported and typed): +vfs_set_runtime_metadata(yaml: string): string; // returns JSON {success, error?} +vfs_get_runtime_metadata(): string; // returns JSON {success, content?} +``` + +For scroll sync, the hub-client will call: +```typescript +vfs_set_runtime_metadata('format:\n html:\n source-location: full\n') +``` + +This causes `data-loc` attributes in rendered HTML, which `useScrollSync` uses. +Already tested in `src/services/runtimeMetadata.wasm.test.ts` (12 tests, all passing). + +### Key files + +| File | What it does | +|------|-------------| +| `hub-client/src/services/wasmRenderer.ts` | WASM wrapper service — main changes go here | +| `hub-client/src/components/Preview.tsx` | Preview component — calls renderToHtml | +| `hub-client/src/components/tabs/AboutTab.tsx` | Renders changelog markdown — keep using render_qmd_content | +| `hub-client/src/services/automergeSync.ts` | Keeps VFS in sync — no changes needed | +| `hub-client/src/test-utils/mockWasm.ts` | Mock WASM for unit tests | +| `hub-client/src/types/wasm-quarto-hub-client.d.ts` | TypeScript declarations for WASM functions | + +### VFS path convention + +All VFS paths use `/project/` prefix. Automerge paths are relative (e.g. `"index.qmd"`). +The VFS `normalize_path()` resolves relative paths to `/project/index.qmd`. Both +`vfs_add_file` and `render_qmd` go through this normalization, so relative paths from +Automerge work consistently — no `/project/` prefix needed from TypeScript. + +--- + +## Work Items + +### Phase 1: TypeScript API Changes + +- [x] Add `vfs_set_runtime_metadata` / `vfs_get_runtime_metadata` to TypeScript type declarations + - Done in `e7f1e61d` + +- [x] Add wrapper functions in `wasmRenderer.ts` + - `setRuntimeMetadata(yaml: string): VfsResponse` — wraps `vfs_set_runtime_metadata` + - `getRuntimeMetadata(): VfsResponse` — wraps `vfs_get_runtime_metadata` + - `setScrollSyncEnabled(enabled: boolean): void` — convenience that updates a + TypeScript-side settings object and flushes the full blob to WASM: + - Maintains `runtimeSettings: Record` internally + - `true` → sets `runtimeSettings.format = { html: { 'source-location': 'full' } }` + - `false` → deletes `runtimeSettings.format` + - Serializes the whole object to YAML and calls `setRuntimeMetadata(yaml)` + - This way multiple runtime settings can coexist without clobbering each other + +- [x] Update `RenderToHtmlOptions` interface (currently at line ~552 of `wasmRenderer.ts`) + - Remove `sourceLocation?: boolean` (now handled via runtime metadata) + - Make `documentPath` required (was optional, defaulted to `"input.qmd"`) + - Current interface: + ```typescript + interface RenderToHtmlOptions { + sourceLocation?: boolean; + documentPath?: string; + } + ``` + +- [x] Update `renderToHtml` implementation (currently at line ~647 of `wasmRenderer.ts`) + - Change signature: no longer takes `qmdContent` as first arg + - Call `renderQmd(documentPath)` instead of `renderQmdContent(content)` + - Remove the `renderQmdContentWithOptions` code path + - **Theme CSS**: `render_qmd` in Rust already writes CSS artifacts to VFS at + `/.quarto/project-artifacts/styles.css` (lib.rs lines 724-728). However, the + JS-side `compileAndInjectThemeCss` uses dart-sass for theme compilation, which + is separate from the Rust pipeline. After the switch, read content from VFS via + `vfsReadFile(documentPath)` to feed `compileAndInjectThemeCss`. The function + already accepts `documentPath` for relative theme resolution. + +### Phase 2: Component Updates + +- [x] Update `Preview.tsx` + - In `doRender()` (line ~208): stop passing `qmdContent` to `renderToHtml` + - Pass `documentPath` (required) — available as `currentFile?.path` + - Remove `sourceLocation: options.scrollSyncEnabled` from render options + - Instead, call `setScrollSyncEnabled()` when `scrollSyncEnabled` prop changes + (use a useEffect or similar — set it once, not per-render) + - **Race condition note:** `vfsAddFile` is synchronous and happens before + `onFileContent` callback fires, which triggers the render. So VFS is always + up-to-date when `render_qmd` reads it. + +- [x] Update `AboutTab.tsx` (line ~84) + - Currently calls `renderToHtml(doc.markdown)` with no options + - This must keep working — AboutTab renders static markdown (changelog, more-info) + that has no project context + - **Decision:** Add a `renderContentToHtml(content: string)` convenience function in + `wasmRenderer.ts` that wraps `renderQmdContent` directly (no VFS, no project context). + AboutTab calls this instead of `renderToHtml`. This cleanly separates the two paths: + `renderToHtml(opts)` for VFS-based project rendering, `renderContentToHtml(content)` + for standalone content rendering. + +- [x] Verify `PreviewRouter.tsx` needs no changes + - Uses `parseQmdToAst(props.content)` for format detection (slides vs. document) + - This is parsing, not rendering — no project context needed, no changes needed + +### Phase 3: Cleanup + +- [x] Deprecate `render_qmd_content_with_options` code path + - Remove `renderQmdContentWithOptions` from `wasmRenderer.ts` + - Remove `WasmRenderOptions` interface (line ~290, currently `{ sourceLocation?: boolean }`) + - Keep the Rust WASM function for now — remove in a follow-up + +- [x] Update mock WASM in `mockWasm.ts` + - Add `vfs_set_runtime_metadata`, `vfs_get_runtime_metadata` to mock + - Update `renderToHtml` mock to match new signature (no content arg) + - Keep `renderQmdContent` mock for AboutTab path + +### Phase 4: Tests + +- [x] **WASM integration: runtime metadata → data-loc** + - Done in `e7f1e61d` — `runtimeMetadata.wasm.test.ts` (12 tests) + +- [x] **WASM integration: render_qmd with project context** + - Done in `e7f1e61d` — tests verify runtime metadata overrides project and document + +- [x] **WASM integration: render_qmd picks up _metadata.yml** + - Populate VFS with `_quarto.yml`, `chapters/_metadata.yml` (`author: "Dir Author"`), + and `chapters/doc.qmd` + - Call `render_qmd("/project/chapters/doc.qmd")` + - Verify rendered HTML includes directory metadata author + +- [x] **Unit test: renderToHtml calls render_qmd** + - Covered by WASM integration tests: `render_qmd` with project context, directory + metadata, and relative paths all pass. TypeScript type system enforces the new + signature (`documentPath` required, no `content` arg). + +- [x] **Unit test: setScrollSyncEnabled / toSimpleYaml** + - `toSimpleYaml` unit tests in `wasmRenderer.test.ts` (6 tests): flat keys, nested + objects, booleans/numbers, empty object, mixed keys, exact scroll-sync YAML output + - WASM integration tests cover the full pipeline: `vfs_set_runtime_metadata` → `data-loc` + +- [x] **Component test: Preview renders via render_qmd** + - TypeScript type system enforces the new API. `renderToHtml` no longer accepts a + content string argument. The 322 unit tests all pass with the new signature. + +- [ ] **E2E: theme CSS compilation** + - Verify theme CSS still works after the switch + - `compileAndInjectThemeCss` now reads content from VFS via `vfsReadFile(documentPath)` + - Requires manual browser testing (dart-sass runs in browser context) + +## Resolved Questions + +1. ~~**Theme CSS compilation**~~: **Resolved.** The Rust `render_qmd` writes CSS artifacts + to VFS (lib.rs lines 724-728), but this is the Rust-side pipeline — the JS-side + dart-sass `compileAndInjectThemeCss` is separate and still needed for theme compilation. + After the switch, read content from VFS via `vfsReadFile(documentPath)` to feed it. + `compileDocumentCss` already accepts a `documentPath` parameter for relative theme + resolution, so only the content source changes. + +2. ~~**VFS path normalization**~~: **Resolved.** VFS `normalize_path()` prepends `/project/` + to relative paths. Both `vfs_add_file` and `render_qmd` go through normalization, so + relative Automerge paths (e.g. `"index.qmd"`) work without adding a `/project/` prefix + from TypeScript. Verified in `wasm.rs` lines 366-374. + +3. ~~**`setScrollSyncEnabled(false)` clearing all metadata**~~: **Resolved.** The + TypeScript layer should own a settings object and serialize the whole thing to YAML + on each change. The WASM API stays all-or-nothing (one opaque blob), and the TS + wrapper composes multiple settings. The runtime metadata goes through + `resolve_format_config` in `AstTransformsStage`, so nested format-specific keys + (e.g. `format.html.source-location`) are flattened correctly during the deep merge. + +## Files Modified + +- `hub-client/src/services/wasmRenderer.ts` — main changes (new wrappers, renderToHtml rewrite) +- `hub-client/src/components/Preview.tsx` — render call changes +- `hub-client/src/components/tabs/AboutTab.tsx` — keep standalone render path +- `hub-client/src/test-utils/mockWasm.ts` — mock updates +- `hub-client/src/services/runtimeMetadata.wasm.test.ts` — may extend with _metadata.yml test diff --git a/claude-notes/plans/2026-03-07-runtime-metadata-layer.md b/claude-notes/plans/2026-03-07-runtime-metadata-layer.md new file mode 100644 index 00000000..f311a9d0 --- /dev/null +++ b/claude-notes/plans/2026-03-07-runtime-metadata-layer.md @@ -0,0 +1,124 @@ +# Plan: Runtime Metadata Layer + +## Overview + +Add a runtime metadata layer to the `SystemRuntime` trait, allowing each execution +environment (WASM preview, native CLI, sandboxed runtime) to inject metadata into the +configuration merge pipeline. This metadata sits at the highest precedence — above +project, directory, and document layers — matching how quarto-cli handles `--metadata` +flags and `metadataOverride`. + +**Motivation:** The hub-client currently uses a separate `render_qmd_content_with_options` +WASM entry point to inject `source-location: full` for scroll sync. This is a special-case +hack for what is fundamentally just metadata. By giving the runtime a proper metadata slot, +any runtime can inject arbitrary configuration without needing per-feature API additions. + +**Depends on:** Nothing (standalone) +**Depended on by:** `2026-03-07-hub-client-render-qmd-switch.md` + +## Merge Order After This Change + +``` +Precedence (lowest → highest): +1. Project top-level settings (_quarto.yml) +2. Project format-specific settings (format.{target}.*) +3. Directory _metadata.yml layers (root → leaf) +4. Document top-level settings (frontmatter) +5. Document format-specific settings (format.{target}.*) +6. Runtime metadata ← NEW +``` + +This matches quarto-cli where `--metadata` flags override everything including the document. + +## Work Items + +### Phase 1: Trait and Merge Pipeline (Rust) + +- [x] Add `fn runtime_metadata(&self) -> Option` to `SystemRuntime` trait + - Default implementation returns `None` + - Located in `crates/quarto-system-runtime/src/traits.rs` + - Uses `serde_json::Value` to avoid coupling quarto-system-runtime to quarto-pandoc-types + - Also added forwarding in `SandboxedRuntime` (`sandbox.rs`) + +- [x] Update `AstTransformsStage::run` to include runtime metadata as final layer + - In `crates/quarto-core/src/stage/stages/ast_transforms.rs` + - Added `json_to_config_value` converter (serde_json::Value → ConfigValue) + - After building the document layer, queries `ctx.runtime.runtime_metadata()` + - If `Some`, flattens with `resolve_format_config` and pushes as final layer + - Relaxed merge gate: now triggers on `has_project_config || runtime_meta_json.is_some()` + +- [x] Write tests for runtime metadata merging (in `ast_transforms.rs`) + - 6 unit tests with `MockRuntimeWithMetadata` + +### Phase 2: WASM Runtime Storage + +- [x] Add runtime metadata storage to `WasmRuntime` + - In `crates/quarto-system-runtime/src/wasm.rs` + - Added `runtime_metadata: RwLock>` field + - Implemented `runtime_metadata()` trait method + - Added `set_runtime_metadata` / `get_runtime_metadata` public methods + +- [x] Add `vfs_set_runtime_metadata` WASM entry point + - In `crates/wasm-quarto-hub-client/src/lib.rs` + - Accepts YAML string, parses to serde_json::Value, validates it's a mapping + - Returns JSON success/error response like other VFS operations + - Passing empty string clears the runtime metadata + +- [x] Add `vfs_get_runtime_metadata` WASM entry point (for debugging/testing) + - Returns current runtime metadata as YAML string, or null if not set + +### Phase 3: Tests + +- [x] **Unit tests in `ast_transforms.rs`** — runtime metadata merge behavior (6 tests, all pass) + +- [x] **WASM integration tests** — `runtimeMetadata.wasm.test.ts` (12 tests, all pass) + - API: accepts YAML, clears with empty string, rejects non-mapping, rejects invalid YAML, round-trips + - Pipeline: injects metadata, overrides document, overrides project, format-specific, no data-loc baseline, cleared metadata, works without project config + +### Phase 4: Fix pre-existing bug (discovered during testing) + +- [x] Fix `extract_config_from_metadata` in pampa HTML writer + - Was looking for nested `format.html.source-location` — but after `AstTransformsStage` + flattens metadata via `resolve_format_config`, `source-location` is at the top level + - The nested lookup was never consistent with Pandoc (which receives flattened metadata) + - Changed to look for top-level `source-location` only + - Updated pampa unit tests to match + - This bug existed since Feb 17 2026 (commit `9d25b246`) when format resolution was + introduced, but was never caught because no test went through the full pipeline AND + checked for `data-loc` in output + +## Key Design Decisions + +1. **Runtime metadata returns `Option`** — not ConfigValue, to avoid + coupling `quarto-system-runtime` to `quarto-pandoc-types`. Converted at the merge site + via `json_to_config_value`. + +2. **Default returns `None`** — existing runtimes (native, sandboxed) are unaffected. + The native CLI can later populate this from `--metadata` flags. + +3. **Merge gate relaxation** — currently the merge block in `AstTransformsStage` is gated + on `ctx.project.config.is_some()`. With runtime metadata, we also merge when + runtime metadata is present even without a project config. This is important for + single-file renders in the hub-client. + +4. **WASM entry point accepts YAML** — consistent with how `_quarto.yml` and frontmatter + are expressed. The hub-client can set it as: + ``` + vfs_set_runtime_metadata("format:\n html:\n source-location: full\n") + ``` + +5. **pampa HTML writer reads top-level keys** — consistent with Pandoc, which never sees + `format.html.*` nesting (Quarto flattens before calling Pandoc). Format resolution is + the pipeline's responsibility, not the writer's. + +## Files Modified + +- `crates/quarto-system-runtime/src/traits.rs` — trait method +- `crates/quarto-system-runtime/src/sandbox.rs` — forwarding +- `crates/quarto-system-runtime/src/wasm.rs` — WASM implementation +- `crates/quarto-core/src/stage/stages/ast_transforms.rs` — merge logic + tests +- `crates/quarto-core/Cargo.toml` — moved yaml-rust2 to regular dependencies +- `crates/wasm-quarto-hub-client/src/lib.rs` — WASM entry points +- `crates/pampa/src/writers/html.rs` — fixed extract_config_from_metadata +- `hub-client/src/services/runtimeMetadata.wasm.test.ts` — WASM integration tests +- `hub-client/src/types/wasm-quarto-hub-client.d.ts` — TypeScript type declarations diff --git a/claude-notes/plans/2026-03-09-css-in-pipeline-a-core.md b/claude-notes/plans/2026-03-09-css-in-pipeline-a-core.md new file mode 100644 index 00000000..40a022f6 --- /dev/null +++ b/claude-notes/plans/2026-03-09-css-in-pipeline-a-core.md @@ -0,0 +1,134 @@ +# Plan: CSS in Pipeline — Part A: Core Implementation (Phases 1-2) + +Parent plan: `claude-notes/plans/2026-03-09-css-in-pipeline.md` + +This sub-plan covers: +- Phase 1: Fix ThemeConfig to read flattened metadata +- Phase 2: New CompileThemeCssStage with caching + +After this plan, the new pipeline stage exists and works, but old code paths +(native pre-pipeline extraction, WASM JS-side compilation) are still in place. +They will be removed in Part B. + +## Prerequisites (all completed) + +- MetadataMergeStage extracted (commit `853c1c0d`) +- SystemRuntime cache — Rust impl (commit `f357f5ad`) +- SystemRuntime cache — WASM impl (commit `55591f83`) + +## Codebase Orientation + +Read these files before starting: + +- `crates/quarto-sass/src/config.rs` — `ThemeConfig::from_config_value`, + currently reads `format.html.theme` (needs to change to top-level `theme`) +- `crates/quarto-sass/src/compile.rs` — `compile_theme_css` (native + WASM + versions), `compile_default_css`, assembly logic to extract +- `crates/quarto-core/Cargo.toml` — `quarto-sass` is native-only dep (line 42) +- `crates/quarto-core/src/stage/stages/apply_template.rs` — creates + `css:default` artifact with `DEFAULT_CSS` (lines 138-143) +- `crates/quarto-core/src/pipeline.rs` — pipeline builder functions +- `crates/quarto-core/src/stage/stages/mod.rs` — stage module registry + +See parent plan for full investigation notes (key findings, resolved risks, +artifact flow, sync/async strategy, etc.). + +## Phase 1: Fix ThemeConfig to read flattened metadata + +After MetadataMergeStage, the merged config is format-flattened: `theme` is at +top level, not under `format.html.theme`. + +**Tests first:** + +- [x] In `crates/quarto-sass/src/config.rs`, add new test helpers that put + `theme` at top level (not nested under `format.html`): + - `flattened_config_with_theme_string(theme: &str) -> ConfigValue` + - `flattened_config_with_theme_array(themes: &[&str]) -> ConfigValue` +- [x] Add tests using flattened helpers: + - `test_from_flattened_config_single_theme` — `{ theme: "darkly" }` works + - `test_from_flattened_config_array_theme` — `{ theme: ["cosmo", "custom.scss"] }` works + - `test_from_flattened_config_no_theme` — `{}` returns `default_bootstrap()` + - `test_from_flattened_config_null_theme` — `{ theme: null }` returns `default_bootstrap()` +- [x] Run tests — they FAILED as expected (still reading `format.html.theme`) + +**Implement:** + +- [x] Change `ThemeConfig::from_config_value` to look for top-level `theme` +- [x] Update existing tests that use nested `format.html.theme` helpers to use + flattened helpers. Old helpers removed. +- [x] Update `compile_css_from_config` doc comments to state it expects + flattened config. Updated `test_compile_css_from_config_with_theme` to pass + flattened config. +- [x] Run tests — all 139 quarto-sass tests PASS + +## Phase 2: New CompileThemeCssStage with caching + +A new pipeline stage that compiles theme CSS and stores it as an artifact. +Runs after MetadataMergeStage (needs merged metadata) and before +AstTransformsStage (no dependency on AST transforms or HTML rendering). + +**Pipeline after this change:** +``` +1. ParseDocumentStage (LoadedSource → DocumentAst) +2. EngineExecutionStage (DocumentAst → DocumentAst) [native only] +3. MetadataMergeStage (DocumentAst → DocumentAst) +4. CompileThemeCssStage (DocumentAst → DocumentAst) ← NEW +5. AstTransformsStage (DocumentAst → DocumentAst) +6. RenderHtmlBodyStage (DocumentAst → RenderedOutput) +7. ApplyTemplateStage (RenderedOutput → RenderedOutput) +``` + +The stage reads `doc.ast.meta` (merged config), compiles CSS, and stores the +result in `ctx.artifacts` as `"css:default"`. `ApplyTemplateStage` is updated +to use the existing artifact instead of always storing `DEFAULT_CSS`. + +**Caching strategy:** + +The stage uses `ctx.runtime.cache_get("sass", &key)` / +`ctx.runtime.cache_set("sass", &key, css)` to avoid recompilation. The cache +key is a hash of the assembled SCSS bundle (which is deterministic given the +theme config + custom file contents + Bootstrap version). + +The caching wraps the raw SCSS compilation at the call site in the stage. +The stage calls `assemble_theme_scss` to get the SCSS + load paths, then +compiles directly via `compile_scss_with_embedded` (native) or +`runtime.compile_sass` (WASM) — NOT via `compile_theme_css`, which would +redundantly re-assemble. This keeps quarto-sass free of runtime cache +dependencies and makes the caching explicit and testable. + +**Preferred approach**: Add `assemble_theme_scss` — a clean factoring that +exposes an existing internal step. The cache key is then +`sha256(assembled_scss + ":minified=" + minified)`. + +**Tests first:** + +- [x] Create `crates/quarto-core/src/stage/stages/compile_theme_css.rs` +- [x] Add tests: + - `test_no_theme_uses_default_css` + - `test_builtin_theme_compiles_css` + - `test_invalid_theme_falls_back_to_default` + - `test_cache_hit_skips_compilation` + - `test_null_theme_uses_default_css` + - `test_cache_key_deterministic` / `test_cache_key_differs_for_minified` / `test_cache_key_differs_for_content` +- [x] Tests pass (8/8) + +**Implement:** + +- [x] Move `quarto-sass` from native-only to general deps in `quarto-core/Cargo.toml` +- [x] Add `assemble_theme_scss` public function to `quarto-sass` (returns `(String, Vec)`) +- [x] Refactor both native and WASM `compile_theme_css` to use `assemble_theme_scss` +- [x] Implement `CompileThemeCssStage` with caching, fallback to DEFAULT_CSS +- [x] Update `ApplyTemplateStage` to skip default if `css:default` already exists +- [x] Wire `CompileThemeCssStage` into: + - `build_html_pipeline_stages()` (7 stages) + - `build_wasm_html_pipeline()` (6 stages) + - `render_qmd_to_html()` inline stage list + - NOT `parse_qmd_to_ast()` (AST-only) +- [x] Update pipeline stage count assertions +- [x] All tests pass: 6593 workspace tests, 0 failures + +## Verification + +- [x] `cargo build --workspace` — compiles +- [x] `cargo nextest run --workspace` — 6593 tests pass +- [x] `cargo xtask verify` — WASM builds and hub-client tests pass diff --git a/claude-notes/plans/2026-03-09-css-in-pipeline-b1-migration.md b/claude-notes/plans/2026-03-09-css-in-pipeline-b1-migration.md new file mode 100644 index 00000000..8d24e487 --- /dev/null +++ b/claude-notes/plans/2026-03-09-css-in-pipeline-b1-migration.md @@ -0,0 +1,169 @@ +# Plan: CSS in Pipeline — Part B1: Migration (Phases 3-4) + +Parent plan: `claude-notes/plans/2026-03-09-css-in-pipeline.md` +Prerequisite: `claude-notes/plans/2026-03-09-css-in-pipeline-b2-tests.md` + +This sub-plan removes the old pre-pipeline CSS compilation code paths now that +`CompileThemeCssStage` produces correct theme CSS inside the pipeline. + +## Changes from Part A that affect this plan + +1. **`PipelineStage` uses conditional `async_trait`**: The trait and all impls + now use `#[cfg_attr(not(target_arch = "wasm32"), async_trait)]` / + `#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]`. This was required + because `CompileThemeCssStage` awaits `SystemRuntime` async methods + (`cache_get`, `cache_set`, `compile_sass`) which return non-Send futures on + WASM. Any new `PipelineStage` impls must use the same conditional pattern. + +2. **`compute_theme_content_hash` patched**: This standalone WASM function + broke because `ThemeConfig::from_config_value` now expects flattened config + (top-level `theme`), but the function receives raw frontmatter + (`format.html.theme`). Fixed by adding `quarto_config::resolve_format_config` + call before `ThemeConfig::from_config_value`. When this function is removed + in Phase 4, the `quarto-config` dep added to `wasm-quarto-hub-client` can + also be removed (if no other code uses it). + +## Phase 3: Remove native CLI pre-pipeline theme extraction + +Current native flow: +1. `write_themed_resources` compiles CSS, writes to `{stem}_files/styles.css` +2. Passes `css_paths` to pipeline +3. Pipeline uses paths in `` tags + +New native flow: +1. **Before pipeline**: Create `{stem}_files/` directory and compute `css_paths` + but do NOT write any CSS file yet. Add a new `prepare_html_resources` + function to `resources.rs` that creates the directory and returns paths + without writing CSS content. +2. Pipeline compiles real theme CSS in `CompileThemeCssStage`, stores as + artifact (cached at `{project_dir}/.quarto/cache/sass/{key}`). If no theme + is configured, the stage stores `DEFAULT_CSS`. On compilation failure, the + stage falls back to `DEFAULT_CSS`. The artifact is always present. +3. **After pipeline**: Write `css:default` artifact to `{stem}_files/styles.css`. + If the artifact is somehow missing (should not happen — belt-and-suspenders), + write `DEFAULT_CSS` as a last-resort safety net. The CSS file is written + exactly once, with the correct content. No overwriting. + +**Design rationale**: We avoid writing `DEFAULT_CSS` first and then overwriting +because that is fragile and wasteful. The pipeline's `CompileThemeCssStage` +always produces a `css:default` artifact (either compiled theme CSS or +`DEFAULT_CSS` fallback), so the post-pipeline write is the single source of +truth for the CSS file. + +**Work items:** + +- [x] Add `prepare_html_resources` to `crates/quarto-core/src/resources.rs`: + - Creates `{stem}_files/` directory (same as `write_html_resources`) + - Returns `HtmlResourcePaths` with correct relative paths + - Does NOT write any CSS file +- [x] In `crates/quarto-core/src/render_to_file.rs`: + - Remove `extract_theme_config` and `theme_value_to_config` functions + - Remove `write_themed_resources` function + - Replace call to `write_themed_resources` with `prepare_html_resources` + - After `render_qmd_to_html` returns, extract `css:default` artifact from + the render context via `ctx.artifacts.get("css:default")` (returns + `Option<&Artifact>`; use `artifact.as_str()` to get the CSS text) and + write to `{stem}_files/styles.css`. If artifact is missing, write + `DEFAULT_CSS` as safety net. +- [x] **Artifact access** (verified): `render_qmd_to_html` takes + `&mut RenderContext`. `run_pipeline` at `pipeline.rs:273` transfers + artifacts back via `ctx.artifacts = stage_ctx.artifacts`. After + `render_qmd_to_html` returns, `ctx.artifacts.get("css:default")` works. + No API changes needed. +- [x] **Runtime cache dir** — change the **callers**, not `render_to_file.rs`: + - **CLI** (`crates/quarto/src/commands/render.rs:107`): The CLI discovers + the project at line 84. Change runtime construction at line 107 from + `Arc::new(NativeRuntime::new())` to + `Arc::new(NativeRuntime::with_cache_dir(project.dir.join(".quarto/cache")))`. + For single-file projects (`project.is_single_file`), `NativeRuntime::new()` + is acceptable (no caching). + - **Test runner** (`crates/quarto-test/src/runner.rs:258`): Keep + `NativeRuntime::new()` — no caching in tests. SASS compilation is + fast enough for test runs and caching would add complexity. +- [x] Remove `write_html_resources_with_sass` from `resources.rs`, including + its tests (`test_write_html_resources_with_sass_default_theme` and + `test_write_html_resources_with_sass_builtin_theme`) +- [x] Fix `extract_theme_specs` in `quarto-sass/config.rs` to handle + `PandocInlines` values (document frontmatter parsed by pampa). Added + `config_value_as_text` helper that falls back to `as_plain_text()`. +- [x] Run tests — 6602 passed, including all 6 theme-inheritance smoke tests. + `test_render_to_file_with_theme` updated: single-file renders without + `_quarto.yml` don't get format flattening yet (pending default-project + -single-file plan), so theme from `format.html.theme` isn't compiled. + Test verifies CSS is written (DEFAULT_CSS fallback). + +## Phase 4: Remove WASM JS-side theme compilation + +The pipeline now produces correct theme CSS in the `css:default` artifact. +WASM `render_qmd()` already writes artifacts to VFS. No JS-side compilation +needed. + +- [x] In `hub-client/src/services/wasmRenderer.ts`: + - Remove `compileAndInjectThemeCss` function + - Remove `extractThemeConfigForCacheKey` function + - Remove `compileDocumentCss` function + - Remove the call to `compileAndInjectThemeCss` in `renderToHtml()`. + - CSS version now computed by hashing VFS CSS artifact content via + `computeHash` from `sassCache.ts`. Reads + `/.quarto/project-artifacts/styles.css` from VFS after render. + - Removed `ThemeHashResponse` interface, `compile_document_css` and + `compute_theme_content_hash` from `WasmModuleExtended` interface. +- [x] In `crates/wasm-quarto-hub-client/src/lib.rs`: + - Remove `compile_document_css` WASM entry point + - Remove `compute_theme_content_hash` WASM entry point + - Remove `ThemeHashResponse` struct + impl + - Remove `extract_frontmatter_config` and `json_to_config_value` helpers + - Clean up imports: removed `THEMES_RESOURCES`, `themes::ThemeSpec` + - Keep `compile_scss`, `compile_default_bootstrap_css`, `compile_theme_css_by_name`, + `sass_available`, `sass_compiler_name`, `get_scss_resources_version` +- [x] In `crates/wasm-quarto-hub-client/Cargo.toml`: + - Remove `quarto-config` dependency + - Remove runtime `sha2` dependency (only used by removed functions) +- [x] In `hub-client/src/types/wasm-quarto-hub-client.d.ts`: + - Remove TypeScript declarations for removed WASM functions +- [x] In `hub-client/src/test-utils/mockWasm.ts`: + - Remove `compileDocumentCss` and `computeThemeContentHash` from interface + impl +- [x] In `hub-client/src/services/wasmRenderer.test.ts`: + - Remove `extractThemeConfigForCacheKey` tests and cache key format tests +- [x] Deleted `hub-client/src/services/themeContentHash.wasm.test.ts` + (tested removed WASM function) +- [x] Evaluate `sassCache.ts`: Keep — used by `compileThemeCssByName` and + `compileDefaultBootstrapCss` for theme settings UI +- [x] Rust workspace: builds, 6602 tests pass +- [x] WASM build passes, hub-client unit tests pass +- [x] **FIXED (B3)**: WASM smoke-all theme-inheritance tests — fixed missing + `setVfsCallbacks()` in smoke-all test. See B3 plan for details. + +## Verification + +- [x] `cargo build --workspace` — compiles +- [x] `cargo nextest run --workspace` — 6602 tests pass +- [x] `cargo xtask verify` — WASM builds and hub-client tests pass + +## Design decisions (resolved during review) + +1. **No CSS overwriting on native**: Instead of writing `DEFAULT_CSS` before + the pipeline and overwriting after, we split `write_html_resources` into + `prepare_html_resources` (dir + paths only) and a post-pipeline write. + CSS is written exactly once. + +2. **`cssVersion` comment kept**: The `` HTML comment + is necessary for MorphIframe to detect CSS-only changes. Computed by hashing + the VFS CSS artifact content. The parent plan's Phase 4 is less specific; + this B1 plan is definitive. + +3. **Runtime cache dir set by callers**: The CLI (`render.rs`) constructs + `NativeRuntime::with_cache_dir(...)` after project discovery. The test + runner keeps `NativeRuntime::new()` (no caching in tests). + +4. **Fallback policy**: `CompileThemeCssStage` always produces a `css:default` + artifact — either compiled theme CSS, or `DEFAULT_CSS` on no-theme / + compilation failure. The post-pipeline CSS write uses this artifact. A + last-resort `DEFAULT_CSS` write handles the (shouldn't-happen) case where + the artifact is missing. + +## Reference + +See parent plan for resolved risks: +- Artifact access from render_to_file (Risk 1 — resolved) +- Custom .scss file resolution in WASM (Risk 3 — resolved) diff --git a/claude-notes/plans/2026-03-09-css-in-pipeline-b2-tests.md b/claude-notes/plans/2026-03-09-css-in-pipeline-b2-tests.md new file mode 100644 index 00000000..6a71fea4 --- /dev/null +++ b/claude-notes/plans/2026-03-09-css-in-pipeline-b2-tests.md @@ -0,0 +1,181 @@ +# Plan: CSS in Pipeline — Part B2: Theme Inheritance Tests (Phase 5) + +Parent plan: `claude-notes/plans/2026-03-09-css-in-pipeline.md` +Prerequisite: `claude-notes/plans/2026-03-09-css-in-pipeline-a-core.md` +Next: `claude-notes/plans/2026-03-09-css-in-pipeline-b1-migration.md` + +This sub-plan adds end-to-end smoke-all tests that verify theme CSS +compilation respects the full metadata hierarchy (project, directory, +document). These tests exercise the `CompileThemeCssStage` through the +complete render pipeline on both native and WASM. + +**TDD ordering**: This plan runs BEFORE B1 (migration). Some tests are +expected to fail because B1 hasn't wired up the pipeline's +`CompileThemeCssStage` output to the actual CSS files yet. The exact +failure pattern (native vs WASM, which test cases) should be analyzed +after the fixtures are created. If *no* tests fail, something is wrong — +B1 is supposed to be the fix for these tests. + +These tests will NOT be committed until B1 makes them pass. The work +here is to create the assertion infrastructure (`ensureCssRegexMatches`) +and the fixture files, run them, analyze failures, and document what B1 +needs to fix. The fixtures and assertions are committed together with +B1 once everything is green. + +## Port `ensureCssRegexMatches` from TS Quarto + +TS Quarto's `ensureCssRegexMatches` (in `tests/verify.ts`) works by: +1. Parsing the rendered HTML to find `` tags +2. Reading each linked CSS file from disk (skipping external URLs) +3. Concatenating all linked CSS content +4. Running regex matches/no-matches against the combined CSS + +Same two-array YAML format as `ensureFileRegexMatches`: +```yaml +ensureCssRegexMatches: + - ["pattern-that-must-match"] + - ["pattern-that-must-NOT-match"] +``` + +**Native** (`quarto-test`): Parse the output HTML, find `` +hrefs, read those CSS files from the output directory, concatenate, and regex +match. Needs an HTML parser — can use a lightweight regex approach to extract +`` hrefs since the HTML is well-formed template output, or add a +dependency like `scraper` for CSS selector support. + +**WASM** (`smokeAll.wasm.test.ts`): Same approach but read CSS from VFS. +After rendering, the HTML contains `` tags pointing to relative paths +like `{stem}_files/styles.css`. These resolve in VFS under the project +prefix. Use `wasm.vfs_read_file()` to read each linked CSS file. Already +has `jsdom` available for HTML parsing (used by `ensureHtmlElements`). + +Work items: +- [ ] Add `ensureCssRegexMatches` assertion to `crates/quarto-test/src/assertions/` + (new file `css_regex.rs`; parses HTML for `` stylesheet hrefs, reads + CSS files from output dir, concatenates, regex matches) +- [ ] Register in `crates/quarto-test/src/spec.rs` assertion parser +- [ ] Add `ensureCssRegexMatches` handling in `smokeAll.wasm.test.ts` + (parse HTML with jsdom for `` hrefs, read CSS from VFS, regex match) + +## Theme detection strategy + +Each Bootswatch theme has a unique `--bs-primary` color value in the compiled +CSS. This is the most reliable single-variable discriminator. Font-family +provides a strong secondary signal for themes that use custom fonts. + +**Primary detection** (unique `--bs-primary` values — no two themes share these): +- **darkly**: `--bs-primary:.*#375a7f` +- **flatly**: `--bs-primary:.*#2c3e50` +- **cosmo**: `--bs-primary:.*#2780e3` +- **sketchy**: `--bs-primary:.*#333` + +**Secondary detection** (unique font-family strings): +- **flatly**: `Lato` in `--bs-font-sans-serif` +- **cosmo**: `Source Sans Pro` in `--bs-font-sans-serif` +- **sketchy**: `Neucha` in `--bs-font-sans-serif`, `Cabin Sketch` in headings + +**Additional discriminators**: +- **darkly**: `--bs-body-bg:.*#222` (only dark-background theme in the set) +- **default** (no theme): static `DEFAULT_CSS` contains `/* ===== Base Styles ===== */` + +Source: `resources/scss/bootstrap/themes/{theme}.scss` variable definitions. + +## Test fixture layout + +All under `crates/quarto/tests/smoke-all/metadata/theme-inheritance/`: + +``` +theme-inheritance/ +├── _quarto.yml # theme: darkly +├── root-doc.qmd # no theme override → gets darkly +├── chapters/ +│ ├── _metadata.yml # theme: flatly +│ ├── chapter1.qmd # no override → gets flatly +│ ├── chapter2.qmd # theme: cosmo → overrides to cosmo +│ └── deep/ +│ ├── _metadata.yml # (no theme) → inherits flatly from parent +│ └── deep-doc.qmd # no override → gets flatly (inherited) +└── appendix/ + ├── appendix-doc.qmd # no override → gets darkly (from project) + └── custom/ + ├── _metadata.yml # theme: sketchy + └── custom-doc.qmd # no override → gets sketchy +``` + +## Test cases (6 QMD files) + +1. **`root-doc.qmd`** — project theme only + - `_quarto.yml` sets `theme: darkly`, doc has no theme + - Assert CSS matches `--bs-primary:.*#375a7f`, does NOT match `#2c3e50` or `Base Styles` + +2. **`chapters/chapter1.qmd`** — directory metadata overrides project + - `chapters/_metadata.yml` sets `theme: flatly` + - Assert CSS matches `--bs-primary:.*#2c3e50`, does NOT match `#375a7f` + +3. **`chapters/chapter2.qmd`** — document overrides directory + - Document frontmatter sets `theme: cosmo` + - Assert CSS matches `--bs-primary:.*#2780e3`, does NOT match `#2c3e50` or `#375a7f` + +4. **`chapters/deep/deep-doc.qmd`** — inherited directory metadata (no local `_metadata.yml` theme) + - `deep/_metadata.yml` exists but has no theme → inherits flatly from `chapters/` + - Assert CSS matches `--bs-primary:.*#2c3e50` + +5. **`appendix/appendix-doc.qmd`** — sibling directory without `_metadata.yml` + - No directory metadata → falls through to project theme (darkly) + - Assert CSS matches `--bs-primary:.*#375a7f`, does NOT match `#2c3e50` + +6. **`appendix/custom/custom-doc.qmd`** — deeper subtree with own theme + - `custom/_metadata.yml` sets `theme: sketchy` + - Assert CSS matches `Neucha` (unique sketchy font), does NOT match `#375a7f` + +Sibling isolation is implicitly verified: test 2 (chapter1 → flatly) and +test 5 (appendix-doc → darkly) are separate render invocations that must +each inherit correctly from their own metadata ancestry. + +## Work items + +- [x] Identify reliable CSS detection patterns (see theme detection strategy above) +- [x] Add `ensureCssRegexMatches` assertion to native runner + WASM runner (see above) +- [x] Create fixture directory structure and all 6 QMD files +- [x] Run native smoke-all tests — analyze which pass/fail and why +- [x] Run WASM smoke-all tests — analyze which pass/fail and why +- [x] Document failure analysis (which tests fail, expected vs unexpected) + +## Native failure analysis + +All 6 theme-inheritance tests fail. All 15 existing tests pass. + +Every failure is "Required CSS pattern not found" — the native `write_themed_resources` +path only reads document frontmatter, so project-level (`_quarto.yml`) and +directory-level (`_metadata.yml`) theme settings are ignored. The CSS written +is either DEFAULT_CSS or only reflects frontmatter themes. + +Specific failures: +- **root-doc**: wants darkly (`#375a7f`) from `_quarto.yml` — gets default CSS +- **chapter1**: wants flatly (`#2c3e50`) from `chapters/_metadata.yml` — gets default CSS +- **chapter2**: wants cosmo (`#2780e3`) from frontmatter — likely gets default CSS + (even frontmatter themes may fail if `write_themed_resources` doesn't compile) +- **deep-doc**: wants flatly inherited from `chapters/` — gets default CSS +- **appendix-doc**: wants darkly from `_quarto.yml` — gets default CSS +- **custom-doc**: wants sketchy (`Neucha`) from `custom/_metadata.yml` — gets default CSS + +B1 will fix this by: removing `write_themed_resources`, using pipeline's +`CompileThemeCssStage` output (`css:default` artifact) to write CSS files. + +## WASM failure analysis + +Same 6 tests fail, same pattern. WASM `render_qmd()` runs `CompileThemeCssStage` +in the pipeline (which produces the correct `css:default` artifact), but the +JS-side `compileAndInjectThemeCss` overwrites the pipeline output with +frontmatter-only CSS. The VFS CSS file doesn't contain the correct theme. + +B1 Phase 4 will fix this by removing the JS-side `compileAndInjectThemeCss` +call, letting the pipeline's artifact flow through to VFS unchanged. + +These fixtures are held uncommitted until B1 makes them all pass. + +## Verification (after B1) + +- [x] `cargo build --workspace` — compiles +- [x] `cargo nextest run --workspace` — 6602 tests pass (including all 6 new fixtures) +- [x] `cargo xtask verify` — WASM builds and hub-client tests pass (21 WASM smoke-all, 0 failures) diff --git a/claude-notes/plans/2026-03-09-css-in-pipeline-b3-wasm-fix.md b/claude-notes/plans/2026-03-09-css-in-pipeline-b3-wasm-fix.md new file mode 100644 index 00000000..81b47bdb --- /dev/null +++ b/claude-notes/plans/2026-03-09-css-in-pipeline-b3-wasm-fix.md @@ -0,0 +1,271 @@ +# Plan: CSS in Pipeline — Part B3: Fix WASM CompileThemeCssStage + +Parent plan: `claude-notes/plans/2026-03-09-css-in-pipeline.md` +Prerequisite: B1 Phase 3 (native migration) complete, B1 Phase 4 (JS-side removal) complete. + +## Problem + +After removing the JS-side `compileAndInjectThemeCss` overwrite (B1 Phase 4), +the 6 theme-inheritance smoke-all WASM tests fail. The pipeline's +`CompileThemeCssStage` is falling back to `DEFAULT_CSS` instead of producing +compiled theme CSS on WASM. Native works correctly (all 6602 Rust tests pass). + +The 6 failing tests (all `ensureCssRegexMatches` assertions): +- `metadata/theme-inheritance/root-doc.qmd` — wants darkly (`#375a7f`) +- `metadata/theme-inheritance/chapters/chapter1.qmd` — wants flatly (`#2c3e50`) +- `metadata/theme-inheritance/chapters/chapter2.qmd` — wants cosmo (`#2780e3`) +- `metadata/theme-inheritance/chapters/deep/deep-doc.qmd` — wants flatly (`#2c3e50`) +- `metadata/theme-inheritance/appendix/appendix-doc.qmd` — wants darkly (`#375a7f`) +- `metadata/theme-inheritance/appendix/custom/custom-doc.qmd` — wants sketchy (`Neucha`) + +## Background: How WASM Rendering Works + +Understanding the full flow is critical: + +1. **Test setup** (`smokeAll.wasm.test.ts`): `populateVfs()` finds the project + root by walking up looking for `_quarto.yml`. Then `readAllFiles()` recursively + reads ALL files in the project and adds them to VFS at `/project/{relative}`. + So `_quarto.yml`, all `_metadata.yml` files, and all `.qmd` files are in VFS. + +2. **WASM render entry** (`lib.rs:648`): `render_qmd(path)` reads the file from + VFS, calls `ProjectContext::discover(path, runtime)` to find project config, + then calls `render_qmd_to_html()`. + +3. **Pipeline selection** (`pipeline.rs:373`): When `HtmlRenderConfig::default()` + is used (WASM path — empty `css_paths`, no template), it calls + `build_html_pipeline_stages()` (NOT `build_wasm_html_pipeline()`). Both include + `CompileThemeCssStage`. The difference is `build_html_pipeline_stages` also has + `EngineExecutionStage` (which is a no-op for WASM in practice). + +4. **Artifact flow** (`lib.rs:721-727`): After pipeline completes, artifacts are + written to VFS: `runtime.add_file(artifact_path, artifact.content.clone())`. + The CSS artifact path is `/.quarto/project-artifacts/styles.css`. + +5. **CSS assertion** (`smokeAll.wasm.test.ts:414-451`): Parses rendered HTML for + `` hrefs, reads each from VFS, concatenates, and runs + regex assertions. + +## Root Cause Hypothesis + +`CompileThemeCssStage` (at `crates/quarto-core/src/stage/stages/compile_theme_css.rs`) +has a `run()` method that: +1. Extracts `ThemeConfig` from `doc.ast.meta` (line 99) +2. If no themes → stores `DEFAULT_CSS` (line 114-121) +3. Assembles SCSS (line 132) +4. Checks cache (line 149) +5. Compiles via platform-specific helper (line 171) +6. On ANY error → falls back to `DEFAULT_CSS` (lines 101-110, 134-143, 180-187) + +The stage has **three silent fallback paths** that produce `DEFAULT_CSS`: +- `ThemeConfig::from_config_value` returns error → DEFAULT_CSS (with warn trace) +- `assemble_theme_scss` returns error → DEFAULT_CSS (with warn trace) +- `compile_scss` returns error → DEFAULT_CSS (with warn trace) + +On WASM, the compile path calls `ctx.runtime.compile_sass()` (line 243-244). +This is the `WasmRuntime::compile_sass` method which bridges to the dart-sass +JS compiler via `js_compile_sass_impl`. + +**Critical detail**: The `SystemRuntime` trait's DEFAULT implementation of +`compile_sass` returns `Err(RuntimeError::NotSupported(...))`. Only `WasmRuntime` +and `NativeRuntime` override it. Make sure the runtime passed through the pipeline +is actually a `WasmRuntime`, not something else with the default impl. + +**Also critical**: The old JS-side `compileAndInjectThemeCss` worked — SASS +compilation IS available in WASM. And `compile_theme_css_by_name` (still in +`lib.rs`) also works. So the SASS bridge is functional; the question is whether +it works from within the pipeline context. + +The stage logs warnings via `trace_event!` but these are NOT visible in WASM +test output by default. This makes failures completely silent. + +## Investigation Steps + +### Step 1: Make stage errors visible in WASM tests + +Add temporary console logging to `CompileThemeCssStage::run()` to see which +path is taken. Alternatively, write a focused WASM test that renders a +single document with a theme and inspects the VFS CSS content directly. + +Quick diagnostic approach: In the WASM smoke-all test runner +(`hub-client/src/services/smokeAll.wasm.test.ts`), after rendering +`root-doc.qmd`, read `/.quarto/project-artifacts/styles.css` from VFS and +log the first 200 chars of its content. If it's `DEFAULT_CSS`, the pipeline +is falling back. + +### Step 2: Check if MetadataMergeStage produces correct merged metadata + +The stage reads `theme` from `doc.ast.meta`. If `MetadataMergeStage` isn't +running or isn't merging project config correctly in WASM, the theme won't +be in the metadata. + +Key question: Does `ProjectContext::discover()` find `_quarto.yml` in VFS? + +Check `crates/quarto-system-runtime/src/wasm.rs` for how `discover` works +with VFS paths. The smoke-all test calls `populateVfs(testFile)` which adds +project files to VFS. Verify that `_quarto.yml` gets added at the right path +and that `ProjectContext::discover()` can find it. + +The `render_qmd` function in `crates/wasm-quarto-hub-client/src/lib.rs` at +line 668 calls `ProjectContext::discover(path, runtime)`. Check if this +finds the `_quarto.yml` for the theme-inheritance fixtures. + +**Important detail for theme inheritance**: The 6 failing tests are +specifically about theme *inheritance* across subdirectories. Each subdir +has its own `_metadata.yml` specifying a different theme. `MetadataMergeStage` +at line 162 calls `directory_metadata_for_document(&ctx.project, &ctx.document.input, ctx.runtime.as_ref())` +to discover `_metadata.yml` files between the project root and the document +directory. This function (in `crates/quarto-core/src/project.rs`) needs to +list directories and read files on VFS. If VFS doesn't support directory +listing properly, the directory metadata layers won't be found — and the +per-subdirectory theme overrides won't reach the merged metadata. This +could explain why ALL 6 tests fail even though the project-level +`_quarto.yml` has a default theme (the root-level theme might not be under +`theme:` directly but rather the subdirectory `_metadata.yml` files provide +the themes). + +### Step 3: Check if compile_sass works in pipeline context + +The WASM `compile_scss` helper (line 236-247 of `compile_theme_css.rs`): +```rust +#[cfg(target_arch = "wasm32")] +async fn compile_scss( + ctx: &StageContext, + scss: &str, + load_paths: &[PathBuf], + minified: bool, +) -> Result { + ctx.runtime + .compile_sass(scss, load_paths, minified) + .await + .map_err(|e| e.to_string()) +} +``` + +This calls `SystemRuntime::compile_sass()`. Check `WasmRuntime`'s impl of +this method. It likely bridges to a JS function. Verify it's wired up and +actually called. The old JS-side `compileAndInjectThemeCss` also used SASS +compilation and worked — so SASS is available, but maybe the runtime context +differs. + +### Step 4: Check `assemble_theme_scss` on WASM + +`assemble_theme_scss` in `quarto-sass` resolves theme files. On WASM, custom +theme resolution uses VFS. Built-in themes use embedded resources +(`THEMES_RESOURCES`, `BOOTSTRAP_RESOURCES`). These should work identically +on both platforms, but verify. + +### Step 5: Check artifact flow from pipeline to VFS + +In `crates/wasm-quarto-hub-client/src/lib.rs` lines 721-727: +```rust +for (_key, artifact) in ctx.artifacts.iter() { + if let Some(artifact_path) = &artifact.path { + runtime.add_file(artifact_path, artifact.content.clone()); + } +} +``` + +Verify the `css:default` artifact has a `path` set. It should — both +`store_default_css` and `store_css` in `compile_theme_css.rs` use +`.with_path(PathBuf::from(DEFAULT_CSS_ARTIFACT_PATH))` where +`DEFAULT_CSS_ARTIFACT_PATH = "/.quarto/project-artifacts/styles.css"`. + +### Step 6: Check CSS resolution in smoke-all test + +The test reads CSS from VFS via `` hrefs in the HTML. When +`HtmlRenderConfig::default()` is used (WASM path), `css_paths` is empty, +so `ApplyTemplateStage` uses `DEFAULT_CSS_ARTIFACT_PATH` as the href. +The test resolver at `smokeAll.wasm.test.ts:423`: +```typescript +const vfsPath = href.startsWith('/') ? href : `/project/${href}`; +``` +For `/.quarto/project-artifacts/styles.css`, this keeps the path as-is. + +## Key Files + +- `crates/quarto-core/src/stage/stages/compile_theme_css.rs` — the stage +- `crates/quarto-core/src/pipeline.rs` — pipeline builders, `DEFAULT_CSS_ARTIFACT_PATH` +- `crates/quarto-core/src/stage/stages/metadata_merge.rs` — metadata merge +- `crates/wasm-quarto-hub-client/src/lib.rs:648-765` — `render_qmd` function +- `crates/quarto-system-runtime/src/wasm.rs` — `WasmRuntime` impl +- `hub-client/src/services/smokeAll.wasm.test.ts` — WASM test runner +- `crates/quarto/tests/smoke-all/metadata/theme-inheritance/` — test fixtures (untracked) + +## Recommended Approach + +1. Write a minimal focused WASM test (in `hub-client/src/services/`) that: + - Adds `_quarto.yml` with `theme: darkly` and a bare `doc.qmd` to VFS + - Calls `render_qmd()` + - Reads `/.quarto/project-artifacts/styles.css` from VFS + - Logs the first 500 chars of the CSS + - Asserts it contains `#375a7f` (darkly's primary color) + +2. If the CSS is `DEFAULT_CSS`, add `console.log` instrumentation in the + Rust `CompileThemeCssStage::run()` using `web_sys::console::log_1` + (available in the WASM crate) to trace which path is taken. + +3. Fix the root cause. Likely candidates: + - `compile_sass` bridge not working in pipeline context + - `MetadataMergeStage` not finding/merging `_quarto.yml` theme + - `assemble_theme_scss` failing on WASM for path resolution reasons + +4. Once the focused test passes, run the full smoke-all suite: + ```bash + cd hub-client && npx vitest run --config vitest.wasm.config.ts src/services/smokeAll.wasm.test.ts + ``` + +5. Then run full verification: + ```bash + cargo xtask verify + ``` + +## Current State of the Codebase + +Phase 4 code changes are complete (but not committed): +- `hub-client/src/services/wasmRenderer.ts` — removed `compileAndInjectThemeCss`, + `extractThemeConfigForCacheKey`, `compileDocumentCss`. CSS version now + computed by hashing VFS CSS artifact content via `computeHash`. +- `crates/wasm-quarto-hub-client/src/lib.rs` — removed `compile_document_css`, + `compute_theme_content_hash`, `ThemeHashResponse`, `extract_frontmatter_config`, + `json_to_config_value`. Cleaned up imports (removed `THEMES_RESOURCES`, + `themes::ThemeSpec`, `quarto_config`). +- `crates/wasm-quarto-hub-client/Cargo.toml` — removed `quarto-config` and + runtime `sha2` dependencies. +- `hub-client/src/types/wasm-quarto-hub-client.d.ts` — removed declarations. +- `hub-client/src/test-utils/mockWasm.ts` — removed mock functions. +- `hub-client/src/services/wasmRenderer.test.ts` — removed tests for removed functions. +- `hub-client/src/services/themeContentHash.wasm.test.ts` — deleted (tested + removed WASM function). + +Rust workspace: `cargo build --workspace` passes, `cargo nextest run --workspace` +passes (6602 tests). WASM build passes. Hub-client unit tests pass. Only the +6 WASM smoke-all theme-inheritance tests fail. + +## Root Cause (Resolved) + +The WASM smoke-all test (`smokeAll.wasm.test.ts`) loaded the WASM module +directly without calling `setVfsCallbacks()` on the SASS bridge. This meant +the dart-sass compiler's custom VFS importer was never initialized, so +`@use`/`@import` directives in the assembled SCSS (which reference Bootstrap +files at `/__quarto_resources__/bootstrap/scss/`) couldn't be resolved — +even though those files were present in the VFS. + +The fix was one-line in nature: add `setVfsCallbacks()` in the smoke-all +test's `beforeAll()`, wiring VFS read/isFile operations to the WASM module's +`vfs_read_file` function. + +The metadata merge was working correctly — themes were being extracted from +`_quarto.yml` and `_metadata.yml` files. The SCSS was being assembled +correctly (~240KB). Only the final compilation step failed silently +(falling back to `DEFAULT_CSS`). + +## Work Items + +- [x] Diagnose why `CompileThemeCssStage` falls back to DEFAULT_CSS on WASM +- [x] Fix the root cause +- [x] Verify all 6 theme-inheritance smoke-all WASM tests pass +- [x] Run `cargo nextest run --workspace` — 6602 passed +- [x] Run hub-client tests — 39 passed (5 test files) +- [x] Run `cargo xtask verify` — full green +- [x] Update B1 plan Phase 4 checklist +- [x] Commit all Phase 4 + B3 changes together (`60750e13`) diff --git a/claude-notes/plans/2026-03-09-css-in-pipeline-c-tests.md b/claude-notes/plans/2026-03-09-css-in-pipeline-c-tests.md new file mode 100644 index 00000000..a327bb4f --- /dev/null +++ b/claude-notes/plans/2026-03-09-css-in-pipeline-c-tests.md @@ -0,0 +1,266 @@ +# Plan: CSS in Pipeline — Part C: Integration & E2E Tests (Phases 5-6) + +Parent plan: `claude-notes/plans/2026-03-09-css-in-pipeline.md` +Prerequisite: B1-B3 complete (commit `60750e13`). + +This sub-plan adds focused integration tests that verify theme CSS +compilation through the pipeline. Most end-to-end scenarios are already +covered by the B2 smoke-all fixtures; this plan fills the remaining gaps. + +## What Already Exists (from B1-B3) + +Before writing new tests, understand what's already tested: + +### Smoke-all fixtures (`crates/quarto/tests/smoke-all/metadata/theme-inheritance/`) + +6 QMD files exercised by BOTH the native Rust runner (`smoke_all.rs`) and +the WASM runner (`smokeAll.wasm.test.ts`). All use `ensureCssRegexMatches` +to verify compiled CSS contains theme-specific patterns: + +| Fixture | Tests | Theme Source | +|---|---|---| +| `root-doc.qmd` | darkly `#375a7f` | `_quarto.yml` project config | +| `chapters/chapter1.qmd` | flatly `#2c3e50` | `chapters/_metadata.yml` directory metadata | +| `chapters/chapter2.qmd` | cosmo `#2780e3` | Document frontmatter `theme: cosmo` | +| `chapters/deep/deep-doc.qmd` | flatly `#2c3e50` | Inherited from `chapters/_metadata.yml` | +| `appendix/appendix-doc.qmd` | darkly `#375a7f` | Falls through to project config | +| `appendix/custom/custom-doc.qmd` | sketchy `Neucha` | `custom/_metadata.yml` | + +### Unit tests in `compile_theme_css.rs` + +- `test_no_theme_uses_default_css` — empty metadata → DEFAULT_CSS +- `test_builtin_theme_compiles_css` — `theme: cosmo` → compiled CSS with `.btn` +- `test_cache_hit_skips_compilation` — second compile uses cached result +- `test_invalid_theme_falls_back_to_default` — bad theme name → DEFAULT_CSS +- `test_null_theme_uses_default_css` — `theme: null` → DEFAULT_CSS +- Cache key determinism/differentiation tests + +### Unit tests in `metadata_merge.rs` + +- Project metadata merging, document overrides, format-specific settings +- Runtime metadata applied, overrides document, overrides project +- Format-specific runtime metadata flattened correctly +- No-project-config + runtime metadata still merges + +### Other relevant tests + +- `wasmRenderer.test.ts` — `toSimpleYaml` tests (only remaining after Phase 4 cleanup) +- `runtimeMetadata.wasm.test.ts` — 15 tests verifying runtime metadata merges + correctly in WASM renders (but does NOT test theme CSS compilation specifically) + +## What's NOT Tested (Gaps) + +1. **Runtime metadata theme override → CSS compilation**: No test verifies + that setting `theme: darkly` via runtime metadata produces darkly CSS. + The individual pieces work (runtime metadata merges, theme CSS compiles) + but the combination is untested. + +2. **No-theme → DEFAULT_CSS through full pipeline**: The unit test in + `compile_theme_css.rs` tests the stage in isolation. No smoke-all + fixture explicitly verifies that a document with NO theme anywhere + produces DEFAULT_CSS through the full render. (Other smoke-all tests + without themes implicitly do this, but there's no explicit assertion.) + +3. **Native integration tests through full pipeline**: The existing unit + tests use `CompileThemeCssStage::run()` directly with constructed + `StageContext`. No test runs `render_qmd_to_html()` or + `render_document_to_file()` with a real `NativeRuntime` and verifies + the CSS artifact or output file. The smoke-all tests do this, but + they're driven by fixture files, not programmatic assertions. + +## Architecture Reference + +Understanding how the pieces fit together: + +### Pipeline flow + +**Native** (7 stages): +``` +Parse → EngineExecution → MetadataMerge → CompileThemeCss → AstTransforms → RenderHtmlBody → ApplyTemplate +``` + +**WASM** (6 stages, no engine execution): +``` +Parse → MetadataMerge → CompileThemeCss → AstTransforms → RenderHtmlBody → ApplyTemplate +``` + +`MetadataMergeStage` merges metadata layers (project → directory → document +→ runtime) and flattens for the target format. After this stage, +`doc.ast.meta` has a top-level `theme` key (if any layer specified one). + +`CompileThemeCssStage` reads `theme` from `doc.ast.meta`, assembles SCSS +(~240KB for a Bootswatch theme), and compiles via platform-specific path: +- **Native**: `compile_scss_with_embedded()` (grass compiler, sync) +- **WASM**: `ctx.runtime.compile_sass()` → JS bridge → dart-sass + +The stage stores the result as the `"css:default"` artifact. On ANY error, +it silently falls back to `DEFAULT_CSS` (logged via `trace_event!` but not +visible in WASM tests). + +### Key files + +- `crates/quarto-core/src/stage/stages/compile_theme_css.rs` — the stage + unit tests +- `crates/quarto-core/src/stage/stages/metadata_merge.rs` — metadata merge + unit tests +- `crates/quarto-core/src/pipeline.rs` — pipeline builders, `DEFAULT_CSS_ARTIFACT_PATH` +- `crates/quarto-core/src/render_to_file.rs` — native render entry point +- `crates/wasm-quarto-hub-client/src/lib.rs` — WASM `render_qmd()` (line ~648) +- `hub-client/src/services/smokeAll.wasm.test.ts` — WASM smoke-all runner +- `hub-client/src/wasm-js-bridge/sass.js` — dart-sass JS bridge with VFS importer +- `crates/quarto-sass/src/config.rs` — `ThemeConfig::from_config_value()` +- `crates/quarto-sass/src/compile.rs` — `assemble_theme_scss()`, `compile_theme_css()` +- `crates/quarto-core/src/resources.rs` — `DEFAULT_CSS`, `prepare_html_resources()` + +### Theme detection patterns (verified in B2/B3) + +Each Bootswatch theme has a unique `--bs-primary` color: +- **darkly**: `--bs-primary:.*#375a7f` +- **flatly**: `--bs-primary:.*#2c3e50` +- **cosmo**: `--bs-primary:.*#2780e3` +- **sketchy**: `Neucha` font family (most distinctive signal) +- **default** (no theme): `DEFAULT_CSS` is 4102 bytes, contains + `/* ===== Base Styles ===== */` + +### WASM SASS bridge — critical setup requirement + +The dart-sass compiler in WASM uses a custom VFS importer +(`hub-client/src/wasm-js-bridge/sass.js`) that resolves `@use`/`@import` +against the VFS. This requires `setVfsCallbacks()` to be called BEFORE +any SASS compilation. The callbacks wire `vfs_read_file` and `vfs_is_file` +from the WASM module into the JS importer. + +- **Production**: `wasmRenderer.ts:setupSassVfsCallbacks()` does this + during `initWasm()` +- **Smoke-all test**: `smokeAll.wasm.test.ts:beforeAll()` does this + (added in B3 fix, commit `60750e13`) +- **Any new WASM test that does theme CSS compilation** MUST also call + `setVfsCallbacks()` — otherwise SASS compilation silently fails and + falls back to DEFAULT_CSS + +Setup pattern for new WASM tests: +```typescript +import { setVfsCallbacks } from '../wasm-js-bridge/sass.js'; + +beforeAll(async () => { + // ... load WASM module ... + + setVfsCallbacks( + (path: string): string | null => { + try { + const result = JSON.parse(wasm.vfs_read_file(path)); + return result.success && result.content !== undefined ? result.content : null; + } catch { return null; } + }, + (path: string): boolean => { + try { + const result = JSON.parse(wasm.vfs_read_file(path)); + return result.success && result.content !== undefined; + } catch { return false; } + }, + ); +}); +``` + +### Runtime metadata in WASM + +`WasmRuntime` supports runtime metadata via `vfs_set_runtime_metadata(yaml)`. +This stores a `serde_json::Value` that `MetadataMergeStage` reads via +`ctx.runtime.runtime_metadata()`. Runtime metadata has the HIGHEST +precedence — it overrides document, directory, and project metadata. + +The WASM function is `vfs_set_runtime_metadata(yaml: &str)` in `lib.rs`. +From JS: `wasm.vfs_set_runtime_metadata('theme: darkly\n')`. +(The parameter accepts YAML; existing tests use YAML syntax consistently.) + +## Phase 5: Remaining Tests + +### 5a: Runtime metadata theme override (WASM) + +This is the primary gap. Create a new WASM test file. + +- [x] **Runtime metadata overrides document theme**: Set up VFS with + `_quarto.yml` (no theme), `doc.qmd` with `theme: flatly` in frontmatter. + Call `wasm.vfs_set_runtime_metadata('theme: darkly\n')`. + Render. Assert CSS contains `#375a7f` (darkly), NOT `#2c3e50` (flatly). + +**Where to put it**: New file `hub-client/src/services/themeCss.wasm.test.ts`. +The existing `runtimeMetadata.wasm.test.ts` doesn't call `setVfsCallbacks()` +(it doesn't need SASS compilation), so adding theme CSS tests there would +change its character. A dedicated file keeps concerns separated and makes +the SASS bridge setup requirement explicit. + +### 5b: Native integration tests (optional, lower priority) + +The smoke-all tests already exercise the full native pipeline. These would +add programmatic tests that construct `RenderContext` and call +`render_qmd_to_html()` directly, which is useful for faster iteration but +not strictly necessary for coverage. + +- [x] `test_render_pipeline_theme_from_project` — project config has + `theme: darkly`, bare document. Assert `css:default` artifact contains + `#375a7f`. +- [x] `test_render_pipeline_theme_from_document_overrides_project` — project + has `theme: darkly`, document has `theme: flatly`. Assert `css:default` + contains `#2c3e50`, not `#375a7f`. +- [x] `test_render_pipeline_no_theme_uses_default` — no theme anywhere. + Assert `css:default` artifact equals `DEFAULT_CSS`. + +**Where to put them**: In `crates/quarto-core/src/pipeline.rs` tests +(alongside existing full-pipeline tests like `test_render_simple_document`). +These tests use `render_qmd_to_html()` with a `RenderContext`, not +`CompileThemeCssStage::run()` directly. For project config with a theme, +use `ProjectConfig::with_metadata()` (see `metadata_merge.rs` tests). + +## Phase 6: Verification + +- [x] `cargo nextest run --workspace` — all 6605 tests pass +- [x] `cargo xtask verify` — WASM build + 40 hub-client tests pass +- [ ] Manual: `theme: darkly` in `_quarto.yml`, verify in hub-client +- [ ] Manual: `theme: sketchy` in frontmatter overrides project theme +- [ ] Manual: native CLI `quarto render` with theme in `_quarto.yml` + +## Build & Test Commands + +```bash +# Run all Rust tests +cargo nextest run --workspace + +# Run only WASM smoke-all tests +cd hub-client && npx vitest run --config vitest.wasm.config.ts src/services/smokeAll.wasm.test.ts + +# Run all hub-client tests +cd hub-client && npm run test:ci + +# Full verification (Rust + WASM build + hub-client tests) +cargo xtask verify + +# WASM rebuild (needed after changing Rust code in quarto-core or wasm-quarto-hub-client) +cd hub-client && npm run build:all +``` + +## Review Notes (2026-03-09) + +Corrections applied after source investigation: + +1. **Pipeline order fixed**: Original diagram had stages in wrong order + (AstTransforms before CompileThemeCss, EngineExecution after + CompileThemeCss, missing RenderHtmlBody). Corrected to match + `pipeline.rs` lines 138-147 (native) and 196-204 (WASM). + +2. **YAML not JSON**: `vfs_set_runtime_metadata` accepts YAML strings. + Existing tests in `runtimeMetadata.wasm.test.ts` consistently use YAML + syntax. Updated all references from `JSON.stringify(...)` to YAML. + +3. **Test file location**: Phase 5a → new `themeCss.wasm.test.ts` file. + Phase 5b → `pipeline.rs` tests (not `compile_theme_css.rs`), since + these are full-pipeline tests using `render_qmd_to_html()`. + +## Reference + +See parent plan for: +- Cache key correctness and known limitations (Risk 2) +- Custom .scss file resolution in WASM (Risk 3) + +See B3 plan (`2026-03-09-css-in-pipeline-b3-wasm-fix.md`) for: +- Full diagnosis of the WASM SASS bridge issue +- How `web_sys::console::log_1` can be used for Rust-side WASM debugging + (add `web-sys` as a `cfg(target_arch = "wasm32")` dep to `quarto-core`) diff --git a/claude-notes/plans/2026-03-09-css-in-pipeline.md b/claude-notes/plans/2026-03-09-css-in-pipeline.md new file mode 100644 index 00000000..60c2bb58 --- /dev/null +++ b/claude-notes/plans/2026-03-09-css-in-pipeline.md @@ -0,0 +1,443 @@ +# Plan: Move Theme CSS Compilation into the Render Pipeline + +## Overview + +Theme CSS compilation currently runs **outside** the render pipeline — in +`render_to_file.rs` (native) and `wasmRenderer.ts` (WASM). Both paths extract +theme config from document frontmatter only, ignoring `_quarto.yml` and +`_metadata.yml`. This means `theme: darkly` in `_quarto.yml` has no effect on +CSS output. + +The fix: compile theme CSS inside a new `CompileThemeCssStage` that runs after +`MetadataMergeStage`, where the fully merged metadata (project + directory + +document + runtime) is available. CSS compilation results are cached via the +`SystemRuntime` cache interface to avoid expensive recompilation across renders. + +## Prerequisites + +- **MetadataMergeStage** must be extracted first (see + `claude-notes/plans/2026-03-09-metadata-merge-stage.md`). After that stage + runs, `doc.ast.meta` contains the format-flattened merged config where `theme` + sits at top level (not nested under `format.html.theme`). +- **SystemRuntime cache interface** must be implemented first (see + `claude-notes/plans/2026-03-09-runtime-cache.md`). SASS compilation is + expensive (~200-500ms native, ~1-2s WASM) and the existing codebase caches + results. The cache interface provides platform-abstracted persistent caching: + per-project filesystem at `{project_dir}/.quarto/cache/` on native, IndexedDB + on WASM. The native runtime is configured with the cache dir after project + discovery via `NativeRuntime::with_cache_dir()`. + +## Key Findings from Investigation + +### 1. CSS compilation code (`quarto-sass`) + +- **Native `compile_theme_css`**: sync (uses grass via `compile_scss_with_embedded`) +- **WASM `compile_theme_css`**: async (uses dart-sass via `runtime.compile_sass()`) +- Both take `ThemeConfig` + `ThemeContext` and return `Result` +- `ThemeContext` needs: document directory (`PathBuf`) + runtime (`&dyn SystemRuntime`) +- `ThemeConfig::from_config_value` currently reads `format.html.theme` — must + change to top-level `theme` for flattened metadata + +### 2. Dependency chain + +- `quarto-sass` in `quarto-core/Cargo.toml` is **native-only** (line 42, + under `[target.'cfg(not(target_arch = "wasm32"))'.dependencies]`) +- `quarto-sass` itself has **no native-only main dependencies** — it uses + `quarto-system-runtime` for platform abstraction +- `wasm-quarto-hub-client` already depends on `quarto-sass` directly +- **Action**: Move `quarto-sass` to general deps in `quarto-core/Cargo.toml` + +### 3. Sync/async in pipeline stages + +- All pipeline stages implement `#[async_trait]` with async `run()` methods +- Native sync `compile_theme_css` can be called from async context (no problem) +- WASM async `compile_theme_css` needs `.await` +- **Solution (implemented)**: Changed `PipelineStage` trait and all impls to use + conditional async_trait: `#[cfg_attr(not(target_arch = "wasm32"), async_trait)]` + / `#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]`. This matches the + pattern used by `SystemRuntime` and allows WASM stages to await non-Send + futures from `SystemRuntime` methods (`cache_get`, `cache_set`, `compile_sass`). + Compilation within the stage uses `#[cfg]` helper functions for platform-specific + compilation calls. + +### 4. Artifact flow + +- **WASM**: `render_qmd()` in `lib.rs` writes all pipeline artifacts to VFS + after completion (lines 722-728). CSS artifact at + `/.quarto/project-artifacts/styles.css` flows automatically. +- **Native**: `render_to_file.rs` writes CSS to `{stem}_files/styles.css` + *before* the pipeline. After pipeline, we overwrite that file with the + artifact content. +- `ApplyTemplateStage` already creates the `css:default` artifact (lines + 138-143 of `apply_template.rs`) with `DEFAULT_CSS`. We replace the content + with compiled theme CSS. + +### 5. JS-side overwrite risk + +- `compileAndInjectThemeCss` in `wasmRenderer.ts` runs AFTER `render_qmd()` + and writes to the same VFS path (`/.quarto/project-artifacts/styles.css`). +- If pipeline produces correct CSS, the JS call **overwrites it** with + frontmatter-only CSS. +- **Solution**: Remove the JS-side call entirely (Phase 4). + +### 6. WASM test infrastructure + +- Tests load `.wasm` from disk, `sass` npm package is available +- No graceful fallback if sass unavailable — pipeline should fall back to + `DEFAULT_CSS` on compilation failure +- Current WASM tests don't exercise theme CSS compilation through the pipeline + +### 7. Previous attempt difficulties (all resolved) + +| Difficulty | Resolution | +|-----------|------------| +| Merged metadata unavailable in pipeline | MetadataMergeStage (prerequisite) | +| `ThemeConfig` reads `format.html.theme` | Change to top-level `theme` (Phase 1) | +| `quarto-sass` is native-only in quarto-core | Move to general deps (Phase 2) | +| Native compile is sync, WASM is async | Conditional `async_trait(?Send)` on `PipelineStage` + `#[cfg]` helper fns (Phase 2) | +| CSS recompiled every render | RuntimeCache caching (Phase 2, prerequisite) | +| Native double-write to disk | Overwrite after pipeline (Phase 3) | +| JS-side overwrites pipeline CSS | Remove JS call (Phase 4) | + +## Work Items + +### Phase 1: Fix ThemeConfig to read flattened metadata + +After MetadataMergeStage, the merged config is format-flattened: `theme` is at +top level, not under `format.html.theme`. + +**Tests first:** + +- [ ] In `crates/quarto-sass/src/config.rs`, add new test helpers that put + `theme` at top level (not nested under `format.html`): + - `flattened_config_with_theme_string(theme: &str) -> ConfigValue` + - `flattened_config_with_theme_array(themes: &[&str]) -> ConfigValue` +- [ ] Add tests using flattened helpers: + - `test_from_flattened_config_single_theme` — `{ theme: "darkly" }` works + - `test_from_flattened_config_array_theme` — `{ theme: ["cosmo", "custom.scss"] }` works + - `test_from_flattened_config_no_theme` — `{}` returns `default_bootstrap()` + - `test_from_flattened_config_null_theme` — `{ theme: null }` returns `default_bootstrap()` +- [ ] Run tests — they should FAIL (still reading `format.html.theme`) + +**Implement:** + +- [ ] Change `ThemeConfig::from_config_value` to look for top-level `theme`: + ```rust + // Before: + let theme_value = config.get("format") + .and_then(|f| f.get("html")) + .and_then(|h| h.get("theme")); + // After: + let theme_value = config.get("theme"); + ``` +- [ ] Update existing tests that use nested `format.html.theme` helpers to use + flattened helpers. The old helpers can be removed. +- [ ] Update `compile_css_from_config` doc comments to state it expects + flattened config. Update `test_compile_css_from_config_with_theme` to pass + flattened config. +- [ ] Run tests — should PASS + +### Phase 2: New CompileThemeCssStage with caching + +A new pipeline stage that compiles theme CSS and stores it as an artifact. +Runs after MetadataMergeStage (needs merged metadata) and before +AstTransformsStage (no dependency on AST transforms or HTML rendering). + +**Pipeline after this change:** +``` +1. ParseDocumentStage (LoadedSource → DocumentAst) +2. EngineExecutionStage (DocumentAst → DocumentAst) [native only] +3. MetadataMergeStage (DocumentAst → DocumentAst) +4. CompileThemeCssStage (DocumentAst → DocumentAst) ← NEW +5. AstTransformsStage (DocumentAst → DocumentAst) +6. RenderHtmlBodyStage (DocumentAst → RenderedOutput) +7. ApplyTemplateStage (RenderedOutput → RenderedOutput) +``` + +The stage reads `doc.ast.meta` (merged config), compiles CSS, and stores the +result in `ctx.artifacts` as `"css:default"`. `ApplyTemplateStage` is updated +to use the existing artifact instead of always storing `DEFAULT_CSS`. + +**Caching strategy:** + +The stage uses `ctx.runtime.cache_get("sass", &key)` / +`ctx.runtime.cache_set("sass", &key, css)` to avoid recompilation. The cache +key is a hash of the assembled SCSS bundle (which is deterministic given the +theme config + custom file contents + Bootstrap version). Two pages with the +same effective theme produce the same assembled SCSS → same cache key → one +compilation. + +On native, cached CSS is stored at `{project_dir}/.quarto/cache/sass/{key}`, +persisting across render sessions. On WASM, it's stored in IndexedDB. For +single-file renders without a project, the runtime has no cache dir and the +cache methods are no-ops — each render compiles fresh (acceptable since +single-file renders are typically one-off). + +The caching wraps the raw SCSS compilation at the call site in the stage. +The stage calls `assemble_theme_scss` to get the SCSS + load paths, then +compiles directly via `compile_scss_with_embedded` (native) or +`runtime.compile_sass` (WASM) — NOT via `compile_theme_css`, which would +redundantly re-assemble. This keeps quarto-sass free of runtime cache +dependencies and makes the caching explicit and testable. + +To compute the cache key without duplicating assembly work, we need a way to +get the assembled SCSS without compiling it. Options: +- Add `assemble_theme_scss(config, context) -> String` to quarto-sass public API + (extracts the assembly step from `compile_theme_css`) +- Or: hash the `ThemeConfig` deterministically (theme names + custom file + contents). This is simpler but slightly less precise (doesn't capture + Bootstrap resource version changes). + +**Preferred**: Add `assemble_theme_scss` — it's a clean factoring that exposes +an existing internal step. The cache key is then `sha256(assembled_scss + minified)`. + +**Tests first:** + +- [ ] Create `crates/quarto-core/src/stage/stages/compile_theme_css.rs` +- [ ] Add tests using a mock runtime with controllable `compile_sass` and + `cache_get`/`cache_set`: + - `test_no_theme_uses_default_css` — metadata has no `theme`, artifact + contains `DEFAULT_CSS` + - `test_builtin_theme_compiles_css` — metadata has `theme: "cosmo"`, + artifact is NOT `DEFAULT_CSS` + - `test_compile_error_falls_back_to_default` — `compile_sass` returns + error, artifact contains `DEFAULT_CSS` + - `test_cache_hit_skips_compilation` — `cache_get` returns CSS, + `compile_sass` is NOT called, artifact contains cached CSS + - `test_cache_miss_compiles_and_stores` — `cache_get` returns None, + `compile_sass` is called, result stored via `cache_set` + - `test_cache_error_still_compiles` — `cache_get` fails, compilation + proceeds normally (cache is best-effort) +- [ ] Run tests — should FAIL + +**Implement:** + +- [ ] Move `quarto-sass` from native-only to general deps in + `crates/quarto-core/Cargo.toml`: + ```toml + # Move from [target.'cfg(not(...))'.dependencies] to [dependencies] + quarto-sass.workspace = true + ``` +- [ ] Add `assemble_theme_scss` public function to `quarto-sass`: + - Signature: `fn assemble_theme_scss(config: &ThemeConfig, context: &ThemeContext) -> Result<(String, Vec), SassError>` + - Returns the assembled SCSS string **and** the load paths needed for + compilation (custom theme directories, etc.) + - Only called when `config.has_themes()` is true; the no-theme path + short-circuits to `DEFAULT_CSS` in the stage without calling this + - This is a refactoring of existing logic in `compile_theme_css` — extract + the assembly step before the `compile_scss_with_embedded` / `runtime.compile_sass` call +- [ ] Implement `CompileThemeCssStage`: + - `input_kind() → DocumentAst`, `output_kind() → DocumentAst` + - In `run()`: + 1. Extract `ThemeConfig` from `doc.ast.meta` + 2. If no themes (`!config.has_themes()`): store `DEFAULT_CSS` as + artifact and return early (no compilation needed) + 3. Assemble SCSS via `assemble_theme_scss` → `(scss, load_paths)` + 4. Compute cache key: `sha256(scss + ":minified=" + minified)` + 5. Check cache: `ctx.runtime.cache_get("sass", &key).await` + 6. On hit: use cached CSS + 7. On miss: compile the assembled SCSS directly — + native: `compile_scss_with_embedded(runtime, &resources, &scss, &load_paths, minified)` + WASM: `runtime.compile_sass(&scss, &load_paths, minified).await` + (NOT `compile_theme_css`, which would re-assemble) + 8. Store in cache: `ctx.runtime.cache_set("sass", &key, css).await` + 9. On compile error: fall back to `DEFAULT_CSS` + 10. Store as artifact: `ctx.artifacts.store("css:default", ...)` + - Pass `DocumentAst` through unchanged (stage only produces a side-effect + artifact) +- [ ] Update `ApplyTemplateStage` to check for existing `"css:default"` artifact + before storing `DEFAULT_CSS`. If the artifact already exists (set by + `CompileThemeCssStage`), skip the default. If not (e.g., stage was skipped), + store `DEFAULT_CSS` as fallback. +- [ ] Wire `CompileThemeCssStage` into pipeline builders in `pipeline.rs`: + - `build_html_pipeline_stages()`: insert after MetadataMergeStage + - `build_wasm_html_pipeline()`: insert after MetadataMergeStage + - `render_qmd_to_html()`: also insert in the inline stage list at + lines 372-379 (the branch for custom CSS/template), after MetadataMergeStage + - `parse_qmd_to_ast()`: do NOT insert (AST-only pipeline, no CSS needed) +- [ ] Update pipeline stage count assertions in tests +- [ ] Run tests — should PASS + +### Phase 3: Remove native CLI pre-pipeline theme extraction + +Current native flow: +1. `write_themed_resources` compiles CSS, writes to `{stem}_files/styles.css` +2. Passes `css_paths` to pipeline +3. Pipeline uses paths in `` tags + +New native flow: +1. `write_html_resources` creates `{stem}_files/` dir, writes DEFAULT_CSS + placeholder, returns css_paths +2. Pipeline compiles real theme CSS in `CompileThemeCssStage`, stores as + artifact (cached at `{project_dir}/.quarto/cache/sass/{key}`) +3. After pipeline returns, extract `css:default` artifact and overwrite + `{stem}_files/styles.css` + +**Work items:** + +- [ ] In `crates/quarto-core/src/render_to_file.rs`: + - Remove `extract_theme_config` and `theme_value_to_config` functions + - Remove `write_themed_resources` function + - Replace call to `write_themed_resources` with `write_html_resources` + - After `render_qmd_to_html` returns, extract `css:default` artifact from + the render context and overwrite `{stem}_files/styles.css` with its content + - **Runtime setup**: Change runtime construction to use + `NativeRuntime::with_cache_dir(project.dir.join(".quarto/cache"))` so the + pipeline's `CompileThemeCssStage` can use the cache. For single-file renders + (no project), use `NativeRuntime::new()` (no caching — acceptable). +- [ ] **Artifact access**: `render_qmd_to_html` currently returns `RenderOutput` + but artifacts live in `RenderContext`. Check how artifacts are returned. + The `run_pipeline` function in `pipeline.rs` transfers artifacts back to + `RenderContext` (line ~262: `ctx.artifacts = stage_ctx.artifacts`). So after + `render_qmd_to_html`, artifacts should be accessible via `ctx.artifacts`. + If `render_qmd_to_html` doesn't return the context, we may need to modify it + to also return the artifact store (or return the full context). +- [ ] Remove `write_html_resources_with_sass` from `resources.rs` +- [ ] Run tests — verify native rendering still works + +### Phase 4: Remove WASM JS-side theme compilation + +The pipeline now produces correct theme CSS in the `css:default` artifact. +WASM `render_qmd()` already writes artifacts to VFS. No JS-side compilation +needed. + +- [ ] In `hub-client/src/services/wasmRenderer.ts`: + - Remove `compileAndInjectThemeCss` function + - Remove `extractThemeConfigForCacheKey` function + - Remove the call to `compileAndInjectThemeCss` in `renderToHtml()` (around + lines 706-728). The `renderQmd()` call already produces correct CSS. + - Update `themeVersion` tracking — the `renderToHtml` function uses the + return value of `compileAndInjectThemeCss` as a change-detection key. After + removal, theme changes are detected through the normal render path (theme + config is in the merged metadata, which affects the HTML output hash). +- [ ] In `crates/wasm-quarto-hub-client/src/lib.rs`: + - Remove `compile_document_css` WASM entry point + - Remove `compute_theme_content_hash` WASM entry point + - Keep `compile_scss`, `compile_default_bootstrap_css`, `compile_theme_css_by_name`, + `sass_available`, `sass_compiler_name`, `get_scss_resources_version` — these + may still be used by other code paths (settings panel, manual compilation) +- [ ] In `hub-client/src/types/wasm-quarto-hub-client.d.ts`: + - Remove TypeScript declarations for removed WASM functions +- [ ] Evaluate `hub-client/src/services/sassCache.ts`: + - The cache is used by `compileScss`, `compileDocumentCss`, + `compileThemeCssByName`, `compileDefaultBootstrapCss` + - If only `compileDocumentCss` is removed but others remain, keep the cache + - If all callers are removed, remove the cache entirely + - **Likely outcome**: Keep it — `compileThemeCssByName` and others are used + by the theme settings UI +- [ ] Run hub-client tests + +### Phase 5: Integration and E2E tests + +#### Native integration tests (`crates/quarto-core/`) + +Using full pipeline with `NativeRuntime` + grass SASS compiler: + +- [ ] `test_render_pipeline_theme_from_project` — `_quarto.yml` has + `format: { html: { theme: darkly } }`, bare `doc.qmd`. Assert CSS artifact + is NOT `DEFAULT_CSS` and contains darkly-specific values. +- [ ] `test_render_pipeline_theme_from_document_overrides_project` — project + has `theme: darkly`, document has `theme: flatly`. Assert CSS contains + flatly values, not darkly. +- [ ] `test_render_pipeline_no_theme_uses_compiled_default` — no theme + anywhere. Assert CSS is compiled Bootstrap (from `compile_default_css`). + +#### WASM E2E tests (`hub-client/src/services/`) + +New file `themeInheritance.wasm.test.ts` following existing patterns: + +- [ ] **Project theme**: `_quarto.yml` has `theme: darkly`, `doc.qmd` has none. + Assert CSS artifact contains darkly-specific values. +- [ ] **Document overrides project**: `_quarto.yml` has `theme: darkly`, + `doc.qmd` has `theme: flatly`. Assert CSS contains flatly, not darkly. +- [ ] **Directory metadata theme**: `chapters/_metadata.yml` has `theme: sketchy`, + `chapters/doc.qmd` has none. Assert CSS contains sketchy. +- [ ] **No theme anywhere**: Assert CSS is default Bootstrap. +- [ ] **Runtime metadata overrides all**: `vfs_set_runtime_metadata` with + `theme: darkly`, document has `theme: flatly`. Assert CSS contains darkly. + +**Detection strategy**: Each Bootswatch theme produces distinctive CSS. Before +writing tests, compile a few themes to identify reliable detection strings +(e.g., darkly uses `$body-bg: #222`, sketchy has hand-drawn borders). + +### Phase 6: Verification + +- [ ] `cargo nextest run --workspace` — all tests pass +- [ ] `cargo xtask verify` — WASM and hub-client build and test +- [ ] Manual: `theme: darkly` in `_quarto.yml`, verify in hub-client +- [ ] Manual: `theme: sketchy` in frontmatter overrides project theme +- [ ] Manual: native CLI `quarto render` with theme in `_quarto.yml` + +## Resolved Risks + +### 1. Artifact access from render_to_file (Phase 3) + +**Status: Resolved.** `render_qmd_to_html` takes `ctx: &mut RenderContext<'_>`. +After the call returns, `ctx.artifacts` contains the pipeline's artifacts +(transferred back at `pipeline.rs:262` via `ctx.artifacts = stage_ctx.artifacts`). +The `ctx` variable remains in scope after line 233 of `render_to_file.rs`, so +we can directly access `ctx.artifacts.get("css:default")` and overwrite the +CSS file. No API changes needed. + +### 2. Cache key correctness (known limitation) + +The cache key is `sha256(assembled_scss + ":minified=" + minified)`. This is +correct as long as the assembled SCSS is fully deterministic given the inputs. +If Bootstrap resources change (e.g., after a `wasm-quarto-hub-client` update), +the assembled SCSS will be different → new cache key → automatic invalidation. + +**Known limitation**: Custom `.scss` files are resolved relative to +`document_dir`. If a user edits `custom.scss`, the assembled SCSS changes → +new cache key. But if they edit a file `@import`-ed by `custom.scss`, the +assembled SCSS may NOT change (we only read top-level files into layers). TS +Quarto handles this by using a session cache (cleared per render session) for +SCSS with `@import`. + +**Accepted for v1**: Custom `@import` chains are rare and the per-project cache +at `.quarto/cache/sass/` is easily cleared (`rm -rf .quarto/cache` or future +`quarto clean`). A future enhancement could hash all transitively imported +files. + +### 3. Custom .scss file resolution in WASM + +**Status: Resolved (no new risk).** Hub-client already populates project files +in VFS before rendering. Custom `.scss` files in the project will be available +via VFS. `ThemeContext` resolves paths relative to the document directory, which +works in both native (real filesystem) and WASM (VFS). This is existing +behavior — the pipeline change doesn't affect it. + +### 4. `assemble_theme_scss` refactoring + +**Status: Resolved (clean separation confirmed).** In `compile_theme_css` +(both native and WASM versions), lines 96-111 are the assembly step: +`process_theme_specs` → `load_title_block_layer` → `assemble_with_user_layers`. +This is platform-independent code that produces a SCSS string. The compilation +step that follows (native: `compile_scss_with_embedded`, WASM: +`runtime.compile_sass`) is platform-specific. + +The refactoring is: extract the assembly into `assemble_theme_scss(config, +context) -> Result<(String, Vec), SassError>` returning the SCSS +string and load paths. Both `compile_theme_css` variants call it internally, +so they can't diverge. The function is platform-independent (no `#[cfg]`). + +## Files Modified (Summary) + +| File | Phase | Change | +|------|-------|--------| +| `crates/quarto-sass/src/config.rs` | 1 | Read top-level `theme` instead of `format.html.theme` | +| `crates/quarto-sass/src/compile.rs` | 2 | Extract `assemble_theme_scss` public function | +| `crates/quarto-core/Cargo.toml` | 2 | Move `quarto-sass` to general deps | +| New: `crates/quarto-core/src/stage/stages/compile_theme_css.rs` | 2 | New pipeline stage with caching | +| `crates/quarto-core/src/stage/stages/mod.rs` | 2 | Register new stage module | +| `crates/quarto-core/src/pipeline.rs` | 2 | Insert CompileThemeCssStage in pipeline builders | +| `crates/quarto-core/src/stage/stages/apply_template.rs` | 2 | Use existing `css:default` artifact if present | +| `crates/quarto-core/src/stage/traits.rs` | 2 | Conditional `async_trait(?Send)` for WASM | +| All `impl PipelineStage` files | 2 | Same conditional `async_trait` on each impl | +| `crates/wasm-quarto-hub-client/src/lib.rs` | 2 | `resolve_format_config` in `compute_theme_content_hash` | +| `crates/wasm-quarto-hub-client/Cargo.toml` | 2 | Added `quarto-config` dep (for above) | +| `crates/quarto-core/src/render_to_file.rs` | 3 | Remove pre-pipeline theme extraction, overwrite CSS after pipeline, configure runtime with cache dir | +| `crates/quarto-core/src/resources.rs` | 3 | Remove `write_html_resources_with_sass` | +| `hub-client/src/services/wasmRenderer.ts` | 4 | Remove `compileAndInjectThemeCss` and related | +| `crates/wasm-quarto-hub-client/src/lib.rs` | 4 | Remove `compile_document_css`, `compute_theme_content_hash` | +| `hub-client/src/types/wasm-quarto-hub-client.d.ts` | 4 | Remove TS declarations for removed functions | +| New: `hub-client/src/services/themeInheritance.wasm.test.ts` | 5 | WASM E2E tests for theme inheritance | diff --git a/claude-notes/plans/2026-03-09-default-project-single-file.md b/claude-notes/plans/2026-03-09-default-project-single-file.md new file mode 100644 index 00000000..4d6e0676 --- /dev/null +++ b/claude-notes/plans/2026-03-09-default-project-single-file.md @@ -0,0 +1,145 @@ +# Plan: Default Project for Single-File Renders + +## Overview + +Currently, single-file renders (no `_quarto.yml`) get `config: None` in +`ProjectContext`. This causes the metadata merge gate in `MetadataMergeStage` +to skip format flattening, leaving `doc.ast.meta` with nested `format.html.*` +keys instead of flattened top-level keys. + +We fix this by making `ProjectContext.config` non-optional (`ProjectConfig` +instead of `Option`). Every `ProjectContext` always has a +config — real projects get their parsed `_quarto.yml`, single-file renders get +`ProjectConfig::default()`. This ensures `resolve_format_config` always runs +and `doc.ast.meta` is always flattened. The type system prevents the bug from +being reintroduced. + +## Background: How the bug works + +### The rendering pipeline + +Quarto renders documents through a staged pipeline defined in +`crates/quarto-core/src/pipeline.rs`. The key stages for this bug are: + +1. `ParseDocumentStage` — parses QMD frontmatter + body into a Pandoc AST. + Frontmatter like `format: { html: { toc: true } }` ends up as nested keys + in `doc.ast.meta`. +2. `MetadataMergeStage` — merges project config, directory `_metadata.yml` + layers, document frontmatter, and runtime metadata. **Crucially, this stage + flattens format-specific keys** via `resolve_format_config()` — e.g., + `format.html.toc: true` becomes top-level `toc: true`. +3. `CompileThemeCssStage` — reads `doc.ast.meta["theme"]` to compile SCSS. +4. `AstTransformsStage` — runs transforms (callouts, TOC, footnotes, etc.) + that read flattened keys from `doc.ast.meta`. + +### The gate + +In `MetadataMergeStage::run()` (`metadata_merge.rs:~147-149`): +```rust +let has_project_config = ctx.project.config.is_some(); +if has_project_config || runtime_meta_json.is_some() { + // ... flattening and merging happens here +} +``` + +When `config` is `None` (single-file render) AND there's no runtime metadata, +this entire block is skipped. The document AST passes through unflattened. +Downstream stages looking for `doc.ast.meta["theme"]` or `doc.ast.meta["toc"]` +find nothing — those values are buried under `format.html.*`. + +### Format flattening + +`resolve_format_config()` in `crates/quarto-config/src/format.rs:72` takes a +`ConfigValue` metadata tree and a target format (e.g., "html"). It: +1. Extracts `format.{target}.*` keys +2. Merges them over top-level keys (format-specific wins) +3. Removes the `format` key from the result + +So `{ title: "Hello", format: { html: { toc: true } } }` becomes +`{ title: "Hello", toc: true }`. + +### Key types and locations + +- `ProjectContext` struct: `crates/quarto-core/src/project.rs:~344` +- `ProjectConfig` struct: `crates/quarto-core/src/project.rs:~261` + (derives `Default` — `project_type: Default`, `output_dir: None`, + `render_patterns: []`, `metadata: None`) +- `ProjectContext::discover()`: `project.rs:~368` — finds `_quarto.yml`, + creates context. Line ~404: `is_single_file = config.is_none() && input_file.is_some()` +- `ProjectContext::single_file()`: `project.rs:~433` — creates single-file + context directly, hardcodes `config: None` +- `MetadataMergeStage::run()`: `stage/stages/metadata_merge.rs:~115` +- `directory_metadata_for_document()`: `project.rs:~83` — uses + `project.config.is_none()` to early-return +- `project_type()`: `project.rs:~549` — unwraps through Option +- WASM construction: `wasm-quarto-hub-client/src/lib.rs:~486` — + `create_wasm_project_context()` hardcodes `config: None` + +### Weakened test + +`render_to_file.rs:~414` `test_render_to_file_with_theme` — renders a +single-file QMD with `format.html.theme: cosmo`. The assertion was weakened +from `assert!(css.contains(".btn"))` (compiled Bootstrap) to +`assert!(!css.is_empty())` (just checks DEFAULT_CSS fallback) because the +theme key is invisible due to this bug. Lines 442-447 have a NOTE comment +explaining this. + +## Prerequisites + +None. This is a standalone change. + +## Downstream impact + +This unblocks the migration of pipeline consumers from `Format.metadata` to +`doc.ast.meta` (see `2026-03-04-format-metadata-merge.md`). Without this +change, transforms reading flattened keys from `doc.ast.meta` would silently +get `None` in single-file renders. + +## Work Items + +### Phase 1: Tests first + +Write tests against the CURRENT (`Option`) API that demonstrate +the bug by failing. These tests will be updated in Phase 4 after the type change. + +- [x] Add test in `metadata_merge.rs`: single-file render with + `format.html.toc: true` in frontmatter and `config: None` — assert + `doc.ast.meta.get("toc")` is `Some(true)` after merge stage runs. + Currently fails because the merge gate skips flattening when config is None. +- [x] Add test in `project.rs`: `ProjectContext::discover()` for a path with + no `_quarto.yml` — assert `ctx.config.is_some()` (i.e. a default config is + present). Currently fails because `config` is `None`. +- [x] Add test in `project.rs`: `ProjectContext::single_file()` — assert + `ctx.config.is_some()`. Currently fails for the same reason. +- [x] Run tests — all 3 FAIL as expected + +### Phase 2: Make `config` non-optional — DONE + +- [x] Struct definition changed +- [x] `discover()`, `single_file()`, WASM construction updated +- [x] `project_type()` accessor simplified + +### Phase 3: Update all access sites — DONE + +- [x] `directory_metadata_for_document()` — changed to `project.is_single_file` +- [x] `MetadataMergeStage` — removed gate, simplified config access, dedented +- [x] Verified `output_dir` resolution in `discover()` unchanged (local `Option`) + +### Phase 4: Update all test constructions — DONE + +- [x] All `config: None` → `ProjectConfig::default()` (35+ sites) +- [x] All `config: Some(ProjectConfig::with_metadata(...))` → unwrapped +- [x] All `config: Some(ProjectConfig { ... })` → unwrapped +- [x] Phase 1 test assertions updated for non-optional type + +### Phase 5: Restore weakened test — DONE + +- [x] `render_to_file.rs` assertion restored to `css.contains(".btn")` +- [x] NOTE comment removed + +### Phase 6: Verify — DONE + +- [x] `cargo nextest run --workspace` — 6621 tests pass, 0 failures +- [x] `cargo xtask verify --skip-hub-tests` — all steps pass (including WASM build) +- [x] Spot-check: single-file render with `format.html.toc: true` produces + `