From 8058a03829a100a7b99afcd25f9cb0715973b350 Mon Sep 17 00:00:00 2001 From: Nathaniel Tucker Date: Sat, 21 Mar 2026 21:06:25 -0400 Subject: [PATCH 1/3] demo(benchmark-react): report ops/s instead of ms (#3808) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * demo(benchmark-react): report ops/s instead of ms for CI consistency The Node benchmark (benchmark.yml) reports ops/sec via Benchmark.js, while the React benchmark reported milliseconds. Switch to ops/s (1000/ms) with customBiggerIsBetter so both benchmark graphs use the same units. Non-duration metrics (ref-stability counts, heap bytes) are unchanged. Made-with: Cursor * demo(benchmark-react): parameterize list-detail-switch navigations Rename list-detail-switch → list-detail-switch-10 to include the navigation count in the scenario name. Refactor listDetailSwitch() to accept (navigations, seedCount) instead of a single n that was used for both seeding and a hardcoded loop. Add machine specs after the results table. Made-with: Cursor * docs: Add specs to bench table * demo(benchmark-react): exclude ref-stability from CI runs Ref-stability scenarios emit `count` (lower is better) which is incompatible with the customBiggerIsBetter CI tool, causing silent missed regressions. Exclude deterministic scenarios from CI; they remain available for local comparison runs. Made-with: Cursor * bugbot * enhance: Better design --- .cursor/rules/benchmarking.mdc | 2 +- .github/workflows/benchmark-react.yml | 4 +- examples/benchmark-react/README.md | 40 ++++++++++--------- .../benchmark-react/bench/report-viewer.html | 14 ++++--- examples/benchmark-react/bench/report.ts | 2 +- examples/benchmark-react/bench/runner.ts | 40 ++++++++++++------- examples/benchmark-react/bench/scenarios.ts | 4 +- examples/benchmark-react/bench/validate.ts | 2 +- .../src/shared/benchHarness.tsx | 10 ++--- examples/benchmark-react/src/shared/types.ts | 4 +- 10 files changed, 69 insertions(+), 53 deletions(-) diff --git a/.cursor/rules/benchmarking.mdc b/.cursor/rules/benchmarking.mdc index cf1fac14c50b..c9f8873b42e7 100644 --- a/.cursor/rules/benchmarking.mdc +++ b/.cursor/rules/benchmarking.mdc @@ -17,7 +17,7 @@ When working on **`packages/react`** or comparing data-client to other React dat - **Where it lives**: `examples/benchmark-react/` - **How to run**: From repo root: `yarn build:benchmark-react`, then `yarn workspace example-benchmark-react preview &` and in another terminal `cd examples/benchmark-react && yarn bench` - **What it measures**: Browser-based init/update duration, ref-stability counts, sorted-view (Query memoization), optional memory (heap delta), startup metrics (FCP/TBT), and React Profiler commit times. Compares data-client, TanStack Query, and SWR. -- **CI**: `.github/workflows/benchmark-react.yml` runs on changes to `packages/react/src/**`, `packages/core/src/**`, `packages/endpoint/src/schemas/**`, `packages/normalizr/src/**`, or `examples/benchmark-react/**` and reports via `rhysd/github-action-benchmark` (customSmallerIsBetter). CI runs **data-client only** (hot-path scenarios) to track regressions; competitor libraries (TanStack Query, SWR) are for local comparison only. +- **CI**: `.github/workflows/benchmark-react.yml` runs on changes to `packages/react/src/**`, `packages/core/src/**`, `packages/endpoint/src/schemas/**`, `packages/normalizr/src/**`, or `examples/benchmark-react/**` and reports via `rhysd/github-action-benchmark` (customBiggerIsBetter). CI runs **data-client only** (hot-path scenarios) to track regressions; competitor libraries (TanStack Query, SWR) are for local comparison only. - **Report viewer**: Open `examples/benchmark-react/bench/report-viewer.html` in a browser and paste `react-bench-output.json` to view a comparison table and charts. Toggle "React commit" and "Trace" filters. Use "Load history" for time-series. See `@examples/benchmark-react/README.md` for methodology, adding a new library, and interpreting results. diff --git a/.github/workflows/benchmark-react.yml b/.github/workflows/benchmark-react.yml index ee9f6eaaabca..a66ccd655a39 100644 --- a/.github/workflows/benchmark-react.yml +++ b/.github/workflows/benchmark-react.yml @@ -60,7 +60,7 @@ jobs: uses: rhysd/github-action-benchmark@v1 with: name: 'Benchmark React' - tool: 'customSmallerIsBetter' + tool: 'customBiggerIsBetter' output-file-path: examples/benchmark-react/react-bench-output.json github-token: "${{ secrets.GITHUB_TOKEN }}" gh-pages-branch: 'gh-pages-bench' @@ -78,7 +78,7 @@ jobs: uses: rhysd/github-action-benchmark@v1 with: name: 'Benchmark React' - tool: 'customSmallerIsBetter' + tool: 'customBiggerIsBetter' output-file-path: examples/benchmark-react/react-bench-output.json github-token: "${{ secrets.GITHUB_TOKEN }}" gh-pages-branch: 'gh-pages-bench' diff --git a/examples/benchmark-react/README.md b/examples/benchmark-react/README.md index 72b37bc9c18c..8bc88b9b03b9 100644 --- a/examples/benchmark-react/README.md +++ b/examples/benchmark-react/README.md @@ -27,9 +27,9 @@ The repo has two benchmark suites: **Hot path (CI)** -- **Get list** (`getlist-100`, `getlist-500`) — Time to show a ListView component that auto-fetches 100 or 500 issues from the list endpoint, then renders (unit: ms). Exercises the full fetch + normalization + render pipeline. +- **Get list** (`getlist-100`, `getlist-500`) — Time to show a ListView component that auto-fetches 100 or 500 issues from the list endpoint, then renders (unit: ops/s). Exercises the full fetch + normalization + render pipeline. - **Get list sorted** (`getlist-500-sorted`) — Mount 500 issues through a sorted/derived view. data-client uses `useQuery(sortedIssuesQuery)` with `Query` schema memoization; competitors use `useMemo` + sort. -- **Update entity** (`update-entity`) — Time to update one issue and propagate to the UI (unit: ms). +- **Update entity** (`update-entity`) — Time to update one issue and propagate to the UI (unit: ops/s). - **Update entity sorted** (`update-entity-sorted`) — After mounting a sorted view, update one entity. data-client's `Query` memoization avoids re-sorting when sort keys are unchanged. - **Update entity multi-view** (`update-entity-multi-view`) — Update one issue that appears simultaneously in a list, a detail panel, and a pinned-cards strip. Exercises cross-query entity propagation: normalized cache updates once and all three views reflect the change; non-normalized libraries must invalidate and refetch each query independently. - **Update user (scaling)** (`update-user`, `update-user-10000`) — Update one shared user with 1,000 or 10,000 mounted issues to test subscriber scaling. Normalized cache: one store update, all views of that user update. @@ -51,12 +51,12 @@ The repo has two benchmark suites: ## Expected results -Illustrative **relative** results with **SWR = 100%** (baseline). For **duration** rows, each value is (library median ms ÷ SWR median ms) × 100 — **lower is faster**. For **ref-stability** rows, the same idea uses the “refs changed” count — **lower is fewer components that saw a new object reference**. Figures are rounded from the **Latest measured results** table below (network simulation on); absolute milliseconds will vary by machine, but **library-to-library ratios** are usually similar. +Illustrative **relative** results with **SWR = 100%** (baseline). For **duration** rows, each value is (library wall-clock time ÷ SWR wall-clock time) × 100 — **lower is faster**. For **ref-stability** rows, the same idea uses the “refs changed” count — **lower is fewer components that saw a new object reference**. Figures are rounded from the **Latest measured results** table below (network simulation on); absolute ops/s will vary by machine, but **library-to-library ratios** are usually similar. | Category | Scenarios (representative) | data-client | tanstack-query | swr | |---|---|---:|---:|---:| | Navigation | `getlist-100`, `getlist-500`, `getlist-500-sorted` | ~103% | ~102% | **100%** | -| Navigation | `list-detail-switch` | ~21% | ~102% | **100%** | +| Navigation | `list-detail-switch-10` | ~21% | ~102% | **100%** | | Mutations | `update-entity`, `update-user`, `update-entity-sorted`, `update-entity-multi-view`, `unshift-item`, `delete-item`, `move-item` | ~2% | ~102% | **100%** | | Scaling (10k items) | `update-user-10000` | ~5% | ~122% | **100%** | @@ -69,18 +69,20 @@ Run: **2026-03-21**, Linux (WSL2), `yarn build:benchmark-react`, static preview | Scenario | Unit | data-client | tanstack-query | swr | |---|---|---:|---:|---:| -| `getlist-100` | ms | 89.3 ± 0.22 | 88.7 ± 0.15 | 87.5 ± 0.50 | -| `getlist-500` | ms | 102.3 ± 1.25 | 99.9 ± 1.25 | 98.4 ± 1.25 | -| `getlist-500-sorted` | ms | 101.8 ± 1.61 | 99.2 ± 1.29 | 97.9 ± 0.63 | -| `list-detail-switch` | ms | 144.4 ± 21.22 | 689.4 ± 20.83 | 674.5 ± 35.67 | -| `update-entity` | ms | 2.8 ± 0.09 | 142.6 ± 0.31 | 142.4 ± 0.34 | -| `update-user` | ms | 3.0 ± 0.13 | 142.7 ± 0.43 | 139.4 ± 0.51 | -| `update-entity-sorted` | ms | 3.2 ± 0.24 | 141.3 ± 0.07 | 141.4 ± 0.56 | -| `update-entity-multi-view` | ms | 2.8 ± 0.41 | 146.6 ± 7.25 | 145.3 ± 8.21 | -| `update-user-10000` | ms | 10.3 ± 0.82 | 246.0 ± 1.35 | 201.2 ± 0.75 | -| `unshift-item` | ms | 3.5 ± 0.06 | 144.5 ± 0.38 | 139.7 ± 0.07 | -| `delete-item` | ms | 3.2 ± 0.10 | 144.4 ± 0.11 | 139.9 ± 0.11 | -| `move-item` | ms | 3.5 ± 0.13 | 156.4 ± 0.50 | 146.4 ± 0.05 | +| `getlist-100` | ops/s | 11.20 ± 0.03 | 11.27 ± 0.02 | 11.43 ± 0.07 | +| `getlist-500` | ops/s | 9.78 ± 0.12 | 10.01 ± 0.13 | 10.16 ± 0.13 | +| `getlist-500-sorted` | ops/s | 9.82 ± 0.16 | 10.08 ± 0.13 | 10.21 ± 0.07 | +| `list-detail-switch-10` | ops/s | 6.93 ± 1.02 | 1.45 ± 0.04 | 1.48 ± 0.08 | +| `update-entity` | ops/s | 357.14 ± 11.48 | 7.01 ± 0.02 | 7.02 ± 0.02 | +| `update-user` | ops/s | 333.33 ± 14.44 | 7.01 ± 0.02 | 7.17 ± 0.03 | +| `update-entity-sorted` | ops/s | 312.50 ± 23.44 | 7.08 ± 0.00 | 7.07 ± 0.03 | +| `update-entity-multi-view` | ops/s | 357.14 ± 52.30 | 6.82 ± 0.34 | 6.88 ± 0.39 | +| `update-user-10000` | ops/s | 97.09 ± 7.73 | 4.07 ± 0.02 | 4.97 ± 0.02 | +| `unshift-item` | ops/s | 285.71 ± 4.90 | 6.92 ± 0.02 | 7.16 ± 0.00 | +| `delete-item` | ops/s | 312.50 ± 9.77 | 6.93 ± 0.01 | 7.15 ± 0.01 | +| `move-item` | ops/s | 285.71 ± 10.61 | 6.39 ± 0.02 | 6.83 ± 0.00 | + +[Measured on a Ryzen 9 7950X; 64 GB RAM; Ubuntu (WSL2); Node 24.12.0; Chromium (Playwright)] ## Expected variance @@ -94,7 +96,7 @@ Regressions >5% on stable scenarios or >15% on volatile scenarios are worth inve ## Interpreting results -- **Lower is better** for duration (ms), ref-stability counts, and heap delta (bytes). +- **Higher is better** for throughput (ops/s). **Lower is better** for ref-stability counts and heap delta (bytes). - **Ref-stability:** data-client's normalized cache keeps referential equality for unchanged entities, so `issueRefChanged` and `userRefChanged` should stay low. Non-normalized libs typically show higher counts because they create new object references for every cache write. - **React commit:** Reported as `(react commit)` suffix entries. These measure React Profiler `actualDuration` and isolate React reconciliation cost from layout/paint. - **Report viewer:** Toggle the "Base metrics", "React commit", and "Trace" checkboxes to filter the comparison table. Use "Load history" to compare multiple runs over time. @@ -186,14 +188,14 @@ Regressions >5% on stable scenarios or >15% on volatile scenarios are worth inve Scenarios are classified as `small` or `large` based on their cost: - **Small** (3 warmup + 15 measurement): `getlist-100`, `update-entity`, `ref-stability-*`, `invalidate-and-resolve`, `unshift-item`, `delete-item` - - **Large** (1 warmup + 4 measurement): `getlist-500`, `getlist-500-sorted`, `update-user`, `update-user-10000`, `update-entity-sorted`, `update-entity-multi-view`, `list-detail-switch` + - **Large** (1 warmup + 4 measurement): `getlist-500`, `getlist-500-sorted`, `update-user`, `update-user-10000`, `update-entity-sorted`, `update-entity-multi-view`, `list-detail-switch-10` - **Memory** (opt-in, 1 warmup + 3 measurement): `memory-mount-unmount-cycle` — run with `--action memory` When running all scenarios (`yarn bench`), each group runs with its own warmup/measurement count. Use `--size` to run only one group. ## Output -The runner prints a JSON array in `customSmallerIsBetter` format (name, unit, value, range) to stdout. In CI this is written to `react-bench-output.json` and sent to the benchmark action. +The runner prints a JSON array in `customBiggerIsBetter` format (name, unit, value, range) to stdout. In CI this is written to `react-bench-output.json` and sent to the benchmark action. To view results locally, open `bench/report-viewer.html` in a browser and paste the JSON (or upload `react-bench-output.json`) to see a comparison table and bar chart. diff --git a/examples/benchmark-react/bench/report-viewer.html b/examples/benchmark-react/bench/report-viewer.html index 66631d13daef..6dfe5b9d124f 100644 --- a/examples/benchmark-react/bench/report-viewer.html +++ b/examples/benchmark-react/bench/report-viewer.html @@ -34,7 +34,7 @@

React benchmark report



- +
@@ -150,7 +150,7 @@

Time-series (load multiple runs)

var minV = Infinity, maxV = -Infinity; libList.forEach(function (lib) { var r = byLib[lib]; - if (r && typeof r.value === 'number' && (r.unit === 'bytes' || r.unit === 'ms' || r.unit === 'count')) { + if (r && typeof r.value === 'number' && (r.unit === 'bytes' || r.unit === 'ops/s' || r.unit === 'ms' || r.unit === 'count')) { if (r.value < minV) minV = r.value; if (r.value > maxV) maxV = r.value; } @@ -159,11 +159,15 @@

Time-series (load multiple runs)

var r = byLib[lib]; if (!r) { cells.push('\u2014'); return; } var cls = ''; - if (typeof r.value === 'number' && minV !== maxV && (r.unit === 'ms' || r.unit === 'bytes')) { - cls = r.value <= minV ? 'fast' : (r.value >= maxV ? 'slow' : ''); + if (typeof r.value === 'number' && minV !== maxV) { + if (r.unit === 'ops/s') { + cls = r.value >= maxV ? 'fast' : (r.value <= minV ? 'slow' : ''); + } else if (r.unit === 'ms' || r.unit === 'bytes') { + cls = r.value <= minV ? 'fast' : (r.value >= maxV ? 'slow' : ''); + } } var range = r.range ? ' ' + r.range : ''; - var unitLabel = r.unit === 'ms' ? ' ms' : r.unit === 'bytes' ? ' B' : ''; + var unitLabel = r.unit === 'ops/s' ? ' ops/s' : r.unit === 'ms' ? ' ms' : r.unit === 'bytes' ? ' B' : ''; cells.push('' + (r.value != null ? Number(r.value) + unitLabel + range : '\u2014') + ''); }); tbody += '' + cells.join('') + ''; diff --git a/examples/benchmark-react/bench/report.ts b/examples/benchmark-react/bench/report.ts index 849b3d88712e..58979812f651 100644 --- a/examples/benchmark-react/bench/report.ts +++ b/examples/benchmark-react/bench/report.ts @@ -1,5 +1,5 @@ /** - * Format results as customSmallerIsBetter JSON for rhysd/github-action-benchmark. + * Format results as customBiggerIsBetter JSON for rhysd/github-action-benchmark. */ export interface BenchmarkResult { name: string; diff --git a/examples/benchmark-react/bench/runner.ts b/examples/benchmark-react/bench/runner.ts index 479f1b7d4fcf..3f10ced6f879 100644 --- a/examples/benchmark-react/bench/runner.ts +++ b/examples/benchmark-react/bench/runner.ts @@ -73,7 +73,8 @@ function filterScenarios(scenarios: Scenario[]): { s => s.name.startsWith('data-client:') && s.category !== 'memory' && - s.category !== 'startup', + s.category !== 'startup' && + !s.deterministic, ); } else if ( !actions || @@ -389,7 +390,11 @@ function shuffle(arr: T[]): T[] { function scenarioUnit(scenario: Scenario): string { if (isRefStabilityScenario(scenario)) return 'count'; if (scenario.resultMetric === 'heapDelta') return 'bytes'; - return 'ms'; + return 'ops/s'; +} + +function msToOps(ms: number): number { + return ms > 0 ? 1000 / ms : 0; } function recordResult( @@ -443,12 +448,17 @@ async function runRound( try { const result = await runScenario(page, lib, scenario, networkSim, cdp); recordResult(samples, scenario, result); + const unit = scenarioUnit(scenario); + const displayValue = + unit === 'ops/s' ? + `${msToOps(result.value).toFixed(2)} ops/s` + : `${result.value.toFixed(2)} ${unit}`; const commitSuffix = result.reactCommit != null ? - ` (commit ${result.reactCommit.toFixed(2)} ms)` + ` (commit ${msToOps(result.reactCommit).toFixed(2)} ops/s)` : ''; process.stderr.write( - ` ${prefix}${scenario.name}: ${result.value.toFixed(2)} ${scenarioUnit(scenario)}${commitSuffix}\n`, + ` ${prefix}${scenario.name}: ${displayValue}${commitSuffix}\n`, ); } catch (err) { console.error( @@ -603,8 +613,11 @@ async function main() { const warmup = warmupCount(scenario); if (s.value.length <= warmup) continue; - const { median, range } = computeStats(s.value, warmup); const unit = scenarioUnit(scenario); + const isOps = unit === 'ops/s'; + const statSamples = + isOps ? s.value.slice(warmup).map(msToOps) : s.value.slice(warmup); + const { median, range } = computeStats(statSamples, 0); report.push({ name: scenario.name, unit, @@ -617,13 +630,11 @@ async function main() { .slice(warmup) .filter(x => !Number.isNaN(x)); if (reactSamples.length > 0 && !scenario.resultMetric) { - const { median: rcMedian, range: rcRange } = computeStats( - reactSamples, - 0, - ); + const rcOps = reactSamples.map(msToOps); + const { median: rcMedian, range: rcRange } = computeStats(rcOps, 0); report.push({ name: `${scenario.name} (react commit)`, - unit: 'ms', + unit: 'ops/s', value: Math.round(rcMedian * 100) / 100, range: rcRange, }); @@ -632,13 +643,11 @@ async function main() { // Chrome trace durations (opt-in via BENCH_TRACE=true) const traceSamples = s.trace.slice(warmup).filter(x => !Number.isNaN(x)); if (traceSamples.length > 0) { - const { median: trMedian, range: trRange } = computeStats( - traceSamples, - 0, - ); + const trOps = traceSamples.map(msToOps); + const { median: trMedian, range: trRange } = computeStats(trOps, 0); report.push({ name: `${scenario.name} (trace)`, - unit: 'ms', + unit: 'ops/s', value: Math.round(trMedian * 100) / 100, range: trRange, }); @@ -657,6 +666,7 @@ async function main() { ); } process.stderr.write('\n'); + process.stdout.write(formatReport(report)); } diff --git a/examples/benchmark-react/bench/scenarios.ts b/examples/benchmark-react/bench/scenarios.ts index e90b0efb1c6b..a984d5fae3cc 100644 --- a/examples/benchmark-react/bench/scenarios.ts +++ b/examples/benchmark-react/bench/scenarios.ts @@ -134,9 +134,9 @@ const BASE_SCENARIOS: BaseScenario[] = [ size: 'large', }, { - nameSuffix: 'list-detail-switch', + nameSuffix: 'list-detail-switch-10', action: 'listDetailSwitch', - args: [1000], + args: [10, 1000], category: 'hotPath', size: 'large', renderLimit: 100, diff --git a/examples/benchmark-react/bench/validate.ts b/examples/benchmark-react/bench/validate.ts index f155befaf1e2..427d13293b7c 100644 --- a/examples/benchmark-react/bench/validate.ts +++ b/examples/benchmark-react/bench/validate.ts @@ -498,7 +498,7 @@ test('listDetailSwitch completes with correct DOM transitions', async (page, lib return; await clearComplete(page); - await page.evaluate(() => window.__BENCH__!.listDetailSwitch!(20)); + await page.evaluate(() => window.__BENCH__!.listDetailSwitch!(5, 20)); await waitForComplete(page, 30000); const hasSortedList = await page.evaluate( diff --git a/examples/benchmark-react/src/shared/benchHarness.tsx b/examples/benchmark-react/src/shared/benchHarness.tsx index 425ba678f47b..dd8d41dbe2d5 100644 --- a/examples/benchmark-react/src/shared/benchHarness.tsx +++ b/examples/benchmark-react/src/shared/benchHarness.tsx @@ -249,9 +249,9 @@ export function useBenchState() { ); const listDetailSwitch = useCallback( - async (n: number) => { - await seedIssueList(FIXTURE_ISSUES.slice(0, n)); - setSortedViewCount(n); + async (navigations: number, seedCount: number) => { + await seedIssueList(FIXTURE_ISSUES.slice(0, seedCount)); + setSortedViewCount(seedCount); setShowSortedView(true); await waitForElement('[data-sorted-list]'); @@ -264,9 +264,9 @@ export function useBenchState() { await waitForElement('[data-sorted-list]'); performance.mark('mount-start'); - for (let i = 2; i <= 11; i++) { + for (let i = 0; i < navigations; i++) { setShowSortedView(false); - setDetailIssueNumber(i); + setDetailIssueNumber(i + 2); await waitForElement('[data-detail-view]'); setDetailIssueNumber(null); diff --git a/examples/benchmark-react/src/shared/types.ts b/examples/benchmark-react/src/shared/types.ts index 24e15cf0556d..38a37df9e4fb 100644 --- a/examples/benchmark-react/src/shared/types.ts +++ b/examples/benchmark-react/src/shared/types.ts @@ -43,8 +43,8 @@ export interface BenchAPI { initDoubleList?(count: number): void; /** Move an issue from one state-filtered list to another. Exercises Collection.move (data-client) vs invalidate+refetch (others). */ moveItem?(id: number): void; - /** Switch between sorted list view and individual issue detail views 10 times (20 renders). Exercises normalized cache lookup (data-client) vs per-navigation fetch (others). */ - listDetailSwitch?(count: number): void; + /** Switch between sorted list view and individual issue detail views. Exercises normalized cache lookup (data-client) vs per-navigation fetch (others). */ + listDetailSwitch?(navigations: number, seedCount: number): Promise; /** Mount list + detail panel + pinned card strip for multi-view entity propagation. */ initMultiView?(count: number): void; /** Update an entity that appears in list + detail + pinned views; waits for all three to reflect the change. */ From e75a1810c8538512f61e183e8f46663343ec7dc0 Mon Sep 17 00:00:00 2001 From: Nathaniel Tucker Date: Sat, 21 Mar 2026 21:19:03 -0400 Subject: [PATCH 2/3] docs: Improve bench presentation --- examples/benchmark-react/README.md | 42 ++++++++++++++++-------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/examples/benchmark-react/README.md b/examples/benchmark-react/README.md index 8bc88b9b03b9..90b59dd57927 100644 --- a/examples/benchmark-react/README.md +++ b/examples/benchmark-react/README.md @@ -51,36 +51,38 @@ The repo has two benchmark suites: ## Expected results -Illustrative **relative** results with **SWR = 100%** (baseline). For **duration** rows, each value is (library wall-clock time ÷ SWR wall-clock time) × 100 — **lower is faster**. For **ref-stability** rows, the same idea uses the “refs changed” count — **lower is fewer components that saw a new object reference**. Figures are rounded from the **Latest measured results** table below (network simulation on); absolute ops/s will vary by machine, but **library-to-library ratios** are usually similar. +Illustrative **relative** results with **SWR = 100%** (baseline). For **throughput** rows, each value is (library ops/s ÷ SWR ops/s) × 100 — **higher is faster**. For **ref-stability** rows, the ratio uses the “refs changed” count — **lower is fewer components that saw a new object reference**. Figures are rounded from the **Latest measured results** table below (network simulation on); absolute ops/s will vary by machine, but **library-to-library ratios** are usually similar. | Category | Scenarios (representative) | data-client | tanstack-query | swr | |---|---|---:|---:|---:| -| Navigation | `getlist-100`, `getlist-500`, `getlist-500-sorted` | ~103% | ~102% | **100%** | -| Navigation | `list-detail-switch-10` | ~21% | ~102% | **100%** | -| Mutations | `update-entity`, `update-user`, `update-entity-sorted`, `update-entity-multi-view`, `unshift-item`, `delete-item`, `move-item` | ~2% | ~102% | **100%** | -| Scaling (10k items) | `update-user-10000` | ~5% | ~122% | **100%** | +| Navigation | `getlist-100`, `getlist-500`, `getlist-500-sorted` | ~97% | ~99% | **100%** | +| Navigation | `list-detail-switch-10` | **~468%** | ~98% | 100% | +| Mutations | `update-entity`, `update-user`, `update-entity-sorted`, `update-entity-multi-view`, `unshift-item`, `delete-item`, `move-item` | **~4600%** | ~98% | 100% | +| Scaling (10k items) | `update-user-10000` | **~1953%** | ~82% | 100% | ## Latest measured results (network simulation on) -Median per metric; range is approximate 95% CI margin from the runner (`stats.ts`). **Network simulation** applies the per-RPC delays in `bench/scenarios.ts` (`NETWORK_SIM_DELAYS`, e.g. `fetchIssueList` 80 ms, `updateUser` 50 ms) so list refetches after an author update pay extra latency compared to normalized propagation. +Median ops/s per scenario; range is approximate 95% CI margin from the runner (`stats.ts`). **Network simulation** applies the per-RPC delays in `bench/scenarios.ts` (`NETWORK_SIM_DELAYS`, e.g. `fetchIssueList` 80 ms, `updateUser` 50 ms) so list refetches after an author update pay extra latency compared to normalized propagation. Run: **2026-03-21**, Linux (WSL2), `yarn build:benchmark-react`, static preview + `env -u CI npx tsx bench/runner.ts --network-sim true` (all libraries; memory scenarios not included). Numbers are **machine-specific**; use them for relative comparison between libraries, not as absolutes. -| Scenario | Unit | data-client | tanstack-query | swr | -|---|---|---:|---:|---:| -| `getlist-100` | ops/s | 11.20 ± 0.03 | 11.27 ± 0.02 | 11.43 ± 0.07 | -| `getlist-500` | ops/s | 9.78 ± 0.12 | 10.01 ± 0.13 | 10.16 ± 0.13 | -| `getlist-500-sorted` | ops/s | 9.82 ± 0.16 | 10.08 ± 0.13 | 10.21 ± 0.07 | -| `list-detail-switch-10` | ops/s | 6.93 ± 1.02 | 1.45 ± 0.04 | 1.48 ± 0.08 | -| `update-entity` | ops/s | 357.14 ± 11.48 | 7.01 ± 0.02 | 7.02 ± 0.02 | -| `update-user` | ops/s | 333.33 ± 14.44 | 7.01 ± 0.02 | 7.17 ± 0.03 | -| `update-entity-sorted` | ops/s | 312.50 ± 23.44 | 7.08 ± 0.00 | 7.07 ± 0.03 | -| `update-entity-multi-view` | ops/s | 357.14 ± 52.30 | 6.82 ± 0.34 | 6.88 ± 0.39 | -| `update-user-10000` | ops/s | 97.09 ± 7.73 | 4.07 ± 0.02 | 4.97 ± 0.02 | -| `unshift-item` | ops/s | 285.71 ± 4.90 | 6.92 ± 0.02 | 7.16 ± 0.00 | -| `delete-item` | ops/s | 312.50 ± 9.77 | 6.93 ± 0.01 | 7.15 ± 0.01 | -| `move-item` | ops/s | 285.71 ± 10.61 | 6.39 ± 0.02 | 6.83 ± 0.00 | +| Scenario | data-client | tanstack-query | swr | +|---|---:|---:|---:| +| **Navigation** | | | | +| `getlist-100` | 11.20 ± 0.03 | 11.27 ± 0.02 | 11.43 ± 0.07 | +| `getlist-500` | 9.78 ± 0.12 | 10.01 ± 0.13 | 10.16 ± 0.13 | +| `getlist-500-sorted` | 9.82 ± 0.16 | 10.08 ± 0.13 | 10.21 ± 0.07 | +| `list-detail-switch-10` | 6.93 ± 1.02 | 1.45 ± 0.04 | 1.48 ± 0.08 | +| **Mutations** | | | | +| `update-entity` | 357.14 ± 11.48 | 7.01 ± 0.02 | 7.02 ± 0.02 | +| `update-user` | 333.33 ± 14.44 | 7.01 ± 0.02 | 7.17 ± 0.03 | +| `update-entity-sorted` | 312.50 ± 23.44 | 7.08 ± 0.00 | 7.07 ± 0.03 | +| `update-entity-multi-view` | 357.14 ± 52.30 | 6.82 ± 0.34 | 6.88 ± 0.39 | +| `update-user-10000` | 97.09 ± 7.73 | 4.07 ± 0.02 | 4.97 ± 0.02 | +| `unshift-item` | 285.71 ± 4.90 | 6.92 ± 0.02 | 7.16 ± 0.00 | +| `delete-item` | 312.50 ± 9.77 | 6.93 ± 0.01 | 7.15 ± 0.01 | +| `move-item` | 285.71 ± 10.61 | 6.39 ± 0.02 | 6.83 ± 0.00 | [Measured on a Ryzen 9 7950X; 64 GB RAM; Ubuntu (WSL2); Node 24.12.0; Chromium (Playwright)] From ee441c65a86fa5a8528aa00f810b567bd643974d Mon Sep 17 00:00:00 2001 From: Nathaniel Tucker Date: Sat, 21 Mar 2026 22:45:56 -0400 Subject: [PATCH 3/3] demo(benchmark-react): add baseline framework for raw React comparison (#3809) * demo(benchmark-react): add baseline framework for raw React comparison Add a "baseline" framework that uses only React primitives (useState + useEffect) with zero global caching. Each component independently fetches its own data, providing a true zero-library comparison point for all existing benchmark scenarios. Made-with: Cursor * docs: Update readme with results --- examples/benchmark-react/README.md | 46 +-- examples/benchmark-react/bench/scenarios.ts | 7 +- .../benchmark-react/src/baseline/index.tsx | 304 ++++++++++++++++++ examples/benchmark-react/webpack.config.cjs | 2 +- 4 files changed, 334 insertions(+), 25 deletions(-) create mode 100644 examples/benchmark-react/src/baseline/index.tsx diff --git a/examples/benchmark-react/README.md b/examples/benchmark-react/README.md index 90b59dd57927..522667542fe5 100644 --- a/examples/benchmark-react/README.md +++ b/examples/benchmark-react/README.md @@ -51,14 +51,14 @@ The repo has two benchmark suites: ## Expected results -Illustrative **relative** results with **SWR = 100%** (baseline). For **throughput** rows, each value is (library ops/s ÷ SWR ops/s) × 100 — **higher is faster**. For **ref-stability** rows, the ratio uses the “refs changed” count — **lower is fewer components that saw a new object reference**. Figures are rounded from the **Latest measured results** table below (network simulation on); absolute ops/s will vary by machine, but **library-to-library ratios** are usually similar. +Illustrative **relative** results with **baseline = 100%** (plain React useState/useEffect, no data library). For **throughput** rows, each value is (library ops/s ÷ baseline ops/s) × 100 — **higher is faster**. For **ref-stability** rows, the ratio uses the “refs changed” count — **lower is fewer components that saw a new object reference**. Figures are rounded from the **Latest measured results** table below (network simulation on); absolute ops/s will vary by machine, but **library-to-library ratios** are usually similar. -| Category | Scenarios (representative) | data-client | tanstack-query | swr | -|---|---|---:|---:|---:| -| Navigation | `getlist-100`, `getlist-500`, `getlist-500-sorted` | ~97% | ~99% | **100%** | -| Navigation | `list-detail-switch-10` | **~468%** | ~98% | 100% | -| Mutations | `update-entity`, `update-user`, `update-entity-sorted`, `update-entity-multi-view`, `unshift-item`, `delete-item`, `move-item` | **~4600%** | ~98% | 100% | -| Scaling (10k items) | `update-user-10000` | **~1953%** | ~82% | 100% | +| Category | Scenarios (representative) | data-client | tanstack-query | swr | baseline | +|---|---|---:|---:|---:|---:| +| Navigation | `getlist-100`, `getlist-500`, `getlist-500-sorted` | ~96% | ~98% | ~99% | **100%** | +| Navigation | `list-detail-switch-10` | **~949%** | ~199% | ~203% | 100% | +| Mutations | `update-entity`, `update-user`, `update-entity-sorted`, `update-entity-multi-view`, `unshift-item`, `delete-item`, `move-item` | **~4486%** | ~96% | ~99% | 100% | +| Scaling (10k items) | `update-user-10000` | **~2006%** | ~84% | ~103% | 100% | ## Latest measured results (network simulation on) @@ -67,22 +67,22 @@ Median ops/s per scenario; range is approximate 95% CI margin from the runner (` Run: **2026-03-21**, Linux (WSL2), `yarn build:benchmark-react`, static preview + `env -u CI npx tsx bench/runner.ts --network-sim true` (all libraries; memory scenarios not included). Numbers are **machine-specific**; use them for relative comparison between libraries, not as absolutes. -| Scenario | data-client | tanstack-query | swr | -|---|---:|---:|---:| -| **Navigation** | | | | -| `getlist-100` | 11.20 ± 0.03 | 11.27 ± 0.02 | 11.43 ± 0.07 | -| `getlist-500` | 9.78 ± 0.12 | 10.01 ± 0.13 | 10.16 ± 0.13 | -| `getlist-500-sorted` | 9.82 ± 0.16 | 10.08 ± 0.13 | 10.21 ± 0.07 | -| `list-detail-switch-10` | 6.93 ± 1.02 | 1.45 ± 0.04 | 1.48 ± 0.08 | -| **Mutations** | | | | -| `update-entity` | 357.14 ± 11.48 | 7.01 ± 0.02 | 7.02 ± 0.02 | -| `update-user` | 333.33 ± 14.44 | 7.01 ± 0.02 | 7.17 ± 0.03 | -| `update-entity-sorted` | 312.50 ± 23.44 | 7.08 ± 0.00 | 7.07 ± 0.03 | -| `update-entity-multi-view` | 357.14 ± 52.30 | 6.82 ± 0.34 | 6.88 ± 0.39 | -| `update-user-10000` | 97.09 ± 7.73 | 4.07 ± 0.02 | 4.97 ± 0.02 | -| `unshift-item` | 285.71 ± 4.90 | 6.92 ± 0.02 | 7.16 ± 0.00 | -| `delete-item` | 312.50 ± 9.77 | 6.93 ± 0.01 | 7.15 ± 0.01 | -| `move-item` | 285.71 ± 10.61 | 6.39 ± 0.02 | 6.83 ± 0.00 | +| Scenario | data-client | tanstack-query | swr | baseline | +|---|---:|---:|---:|---:| +| **Navigation** | | | | | +| `getlist-100` | 11.20 ± 0.03 | 11.27 ± 0.02 | 11.43 ± 0.07 | 11.55 ± 0.02 | +| `getlist-500` | 9.78 ± 0.12 | 10.01 ± 0.13 | 10.16 ± 0.13 | 10.22 ± 0.07 | +| `getlist-500-sorted` | 9.82 ± 0.16 | 10.08 ± 0.13 | 10.21 ± 0.07 | 10.29 ± 0.06 | +| `list-detail-switch-10` | 6.93 ± 1.02 | 1.45 ± 0.04 | 1.48 ± 0.08 | 0.73 ± 0.00 | +| **Mutations** | | | | | +| `update-entity` | 357.14 ± 11.48 | 7.01 ± 0.02 | 7.02 ± 0.02 | 7.22 ± 0.00 | +| `update-user` | 333.33 ± 14.44 | 7.01 ± 0.02 | 7.17 ± 0.03 | 7.22 ± 0.01 | +| `update-entity-sorted` | 312.50 ± 23.44 | 7.08 ± 0.00 | 7.07 ± 0.03 | 7.28 ± 0.01 | +| `update-entity-multi-view` | 357.14 ± 52.30 | 6.82 ± 0.34 | 6.88 ± 0.39 | 7.14 ± 0.36 | +| `update-user-10000` | 97.09 ± 7.73 | 4.07 ± 0.02 | 4.97 ± 0.02 | 4.84 ± 0.03 | +| `unshift-item` | 285.71 ± 4.90 | 6.92 ± 0.02 | 7.16 ± 0.00 | 7.16 ± 0.02 | +| `delete-item` | 312.50 ± 9.77 | 6.93 ± 0.01 | 7.15 ± 0.01 | 7.16 ± 0.01 | +| `move-item` | 285.71 ± 10.61 | 6.39 ± 0.02 | 6.83 ± 0.00 | 6.82 ± 0.00 | [Measured on a Ryzen 9 7950X; 64 GB RAM; Ubuntu (WSL2); Node 24.12.0; Chromium (Playwright)] diff --git a/examples/benchmark-react/bench/scenarios.ts b/examples/benchmark-react/bench/scenarios.ts index a984d5fae3cc..887340777ace 100644 --- a/examples/benchmark-react/bench/scenarios.ts +++ b/examples/benchmark-react/bench/scenarios.ts @@ -186,7 +186,12 @@ const BASE_SCENARIOS: BaseScenario[] = [ }, ]; -export const LIBRARIES = ['data-client', 'tanstack-query', 'swr'] as const; +export const LIBRARIES = [ + 'data-client', + 'tanstack-query', + 'swr', + 'baseline', +] as const; export const SCENARIOS: Scenario[] = LIBRARIES.flatMap(lib => BASE_SCENARIOS.filter( diff --git a/examples/benchmark-react/src/baseline/index.tsx b/examples/benchmark-react/src/baseline/index.tsx new file mode 100644 index 000000000000..3011a175811c --- /dev/null +++ b/examples/benchmark-react/src/baseline/index.tsx @@ -0,0 +1,304 @@ +import { + moveItemIsReady, + renderBenchApp, + useBenchState, +} from '@shared/benchHarness'; +import { + DOUBLE_LIST_STYLE, + IssueRow, + PINNED_STRIP_STYLE, + PinnedCardView, + PlainIssueList, +} from '@shared/components'; +import { + FIXTURE_USERS, + FIXTURE_USERS_BY_LOGIN, + FIXTURE_ISSUES_BY_NUMBER, + sortByTitle, +} from '@shared/data'; +import { setCurrentIssues } from '@shared/refStability'; +import { + fetchIssue, + fetchIssueList, + updateIssue, + updateUser as serverUpdateUser, + createIssue, + deleteIssue, +} from '@shared/server'; +import type { Issue } from '@shared/types'; +import React, { useCallback, useEffect, useMemo, useState } from 'react'; + +function SortedListView({ + limit, + refetchKey, +}: { + limit?: number; + refetchKey: number; +}) { + const [issues, setIssues] = useState(null); + useEffect(() => { + fetchIssueList().then(setIssues); + }, [refetchKey]); + const sorted = useMemo(() => (issues ? sortByTitle(issues) : []), [issues]); + if (!sorted.length) return null; + return ( +
+ +
+ ); +} + +function DetailView({ + number, + refetchKey, +}: { + number: number; + refetchKey: number; +}) { + const [issue, setIssue] = useState(null); + useEffect(() => { + fetchIssue({ number }).then(setIssue); + }, [number, refetchKey]); + if (!issue) return null; + return ( +
+ +
+ ); +} + +function PinnedCard({ + number, + refetchKey, +}: { + number: number; + refetchKey: number; +}) { + const [issue, setIssue] = useState(null); + useEffect(() => { + fetchIssue({ number }).then(setIssue); + }, [number, refetchKey]); + if (!issue) return null; + return ; +} + +function PinnedStrip({ + numbers, + refetchKey, +}: { + numbers: number[]; + refetchKey: number; +}) { + return ( +
+ {numbers.map(n => ( + + ))} +
+ ); +} + +function ListView({ + count, + limit, + refetchKey, +}: { + count: number; + limit?: number; + refetchKey: number; +}) { + const [issues, setIssues] = useState(null); + useEffect(() => { + fetchIssueList({ count }).then(setIssues); + }, [count, refetchKey]); + if (!issues) return null; + setCurrentIssues(issues); + return ; +} + +function StateListView({ + state, + count, + limit, + refetchKey, +}: { + state: string; + count: number; + limit?: number; + refetchKey: number; +}) { + const [issues, setIssues] = useState(null); + useEffect(() => { + fetchIssueList({ state, count }).then(setIssues); + }, [state, count, refetchKey]); + if (!issues) return null; + return ( +
+ {issues.length} + +
+ ); +} + +function DoubleListView({ + count, + limit, + refetchKey, +}: { + count: number; + limit?: number; + refetchKey: number; +}) { + return ( +
+ + +
+ ); +} + +function BenchmarkHarness() { + const [refetchKey, setRefetchKey] = useState(0); + const triggerRefetch = useCallback(() => setRefetchKey(k => k + 1), []); + + const { + listViewCount, + showSortedView, + showDoubleList, + doubleListCount, + detailIssueNumber, + pinnedNumbers, + renderLimit, + containerRef, + measureUpdate, + registerAPI, + } = useBenchState(); + + const updateEntity = useCallback( + (number: number) => { + const issue = FIXTURE_ISSUES_BY_NUMBER.get(number); + if (!issue) return; + measureUpdate(() => + updateIssue({ + number, + title: `${issue.title} (updated)`, + }).then(triggerRefetch), + ); + }, + [measureUpdate, triggerRefetch], + ); + + const updateUser = useCallback( + (login: string) => { + const user = FIXTURE_USERS_BY_LOGIN.get(login); + if (!user) return; + measureUpdate(() => + serverUpdateUser({ + login, + name: `${user.name} (updated)`, + }).then(triggerRefetch), + ); + }, + [measureUpdate, triggerRefetch], + ); + + const unshiftItem = useCallback(() => { + const user = FIXTURE_USERS[0]; + measureUpdate(() => + createIssue({ title: 'New Issue', user }).then(triggerRefetch), + ); + }, [measureUpdate, triggerRefetch]); + + const deleteEntity = useCallback( + (number: number) => { + measureUpdate(() => deleteIssue({ number }).then(triggerRefetch)); + }, + [measureUpdate, triggerRefetch], + ); + + const moveItem = useCallback( + (number: number) => { + measureUpdate( + () => updateIssue({ number, state: 'closed' }).then(triggerRefetch), + () => moveItemIsReady(containerRef, number), + ); + }, + [measureUpdate, triggerRefetch, containerRef], + ); + + const updateEntityMultiView = useCallback( + (number: number) => { + const issue = FIXTURE_ISSUES_BY_NUMBER.get(number); + if (!issue) return; + const expected = `${issue.title} (updated)`; + measureUpdate( + () => updateIssue({ number, title: expected }).then(triggerRefetch), + () => { + const container = containerRef.current!; + const listTitle = container.querySelector( + `[data-issue-number="${number}"] [data-title]`, + ); + const detailTitle = container.querySelector( + '[data-detail-view] [data-title]', + ); + const pinnedTitle = container.querySelector( + `[data-pinned-number="${number}"] [data-title]`, + ); + return [listTitle, detailTitle, pinnedTitle].every( + el => el?.textContent === expected, + ); + }, + ); + }, + [measureUpdate, triggerRefetch, containerRef], + ); + + registerAPI({ + updateEntity, + updateUser, + updateEntityMultiView, + unshiftItem, + deleteEntity, + moveItem, + }); + + return ( +
+ {listViewCount != null && ( + + )} + {showSortedView && ( + + )} + {showDoubleList && doubleListCount != null && ( + + )} + {detailIssueNumber != null && ( + + )} + {pinnedNumbers.length > 0 && ( + + )} +
+ ); +} + +renderBenchApp(BenchmarkHarness); diff --git a/examples/benchmark-react/webpack.config.cjs b/examples/benchmark-react/webpack.config.cjs index ffa8f5868793..91e5f676a9d5 100644 --- a/examples/benchmark-react/webpack.config.cjs +++ b/examples/benchmark-react/webpack.config.cjs @@ -2,7 +2,7 @@ const { makeConfig } = require('@anansi/webpack-config'); const HtmlWebpackPlugin = require('html-webpack-plugin'); const path = require('path'); -const LIBRARIES = ['data-client', 'tanstack-query', 'swr']; +const LIBRARIES = ['data-client', 'tanstack-query', 'swr', 'baseline']; const entries = {}; for (const lib of LIBRARIES) {