diff --git a/autoresearch.checks.sh b/autoresearch.checks.sh deleted file mode 100755 index e23e44d1e8..0000000000 --- a/autoresearch.checks.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -set -euo pipefail - -# Type check the packages we modify -cd packages/cli && pnpm tsc --noEmit 2>&1 | grep -i error || true -cd ../.. - -cd packages/cli-kit && pnpm tsc --noEmit 2>&1 | grep -i error || true -cd ../.. - -# Run tests for the CLI package (fast subset) -pnpm --filter @shopify/cli vitest run --reporter=dot 2>&1 | tail -20 - -# Verify the CLI actually works - version command -node packages/cli/bin/dev.js version 2>&1 | grep -q "3\." || { echo "ERROR: version command failed"; exit 1; } diff --git a/autoresearch.ideas.md b/autoresearch.ideas.md deleted file mode 100644 index 890a6f2add..0000000000 --- a/autoresearch.ideas.md +++ /dev/null @@ -1,46 +0,0 @@ -# Autoresearch Ideas - -## Final State -- **Baseline: 610ms → Current: 160ms (74% improvement)** -- All universal optimizations — benefit every command equally -- Startup chunk size: ~355KB (12 chunks, minified, compile-cached) -- Version command: ~145ms, --version flag: ~60ms -- Help benchmark: 160ms (±20ms variance from system noise) - -## What We Did (all applied) -1. Separate bootstrap.ts from index.ts (don't load 106 commands upfront) -2. Lazy command loading via registry (only load the needed command) -3. Fire-and-forget init/prerun/postrun hooks (deferred to after command) -4. Custom lightweight dispatcher (bypass oclif.run() for known commands) -5. Deferred error-handler import (only on error path) -6. TypeScript compiler externalized from bundle (~9MB → external) -7. Native Node.js builtins in custom-oclif-loader (avoid fs.js/execa chains) -8. process.exit(0) after command (skip waiting for analytics/network) -9. V8 compile cache via module.enableCompileCache() -10. Full esbuild minification (whitespace + identifiers + syntax) -11. Lazy ui.js/error-handler/notifications-system/environments.js in base-command -12. Inlined terminalSupportsPrompting (avoid system.js → execa chain) -13. Static imports for ShopifyConfig + oclif settings (eliminate async hops) -14. Type-only imports for interfaces/types (prevent accidental runtime deps) -15. Skip async exitIfOldNodeVersion for Node ≥ 18 - -## Remaining Opportunities (major effort required) - -### Would help but needs framework changes -- **Replace oclif with lighter framework**: oclif core (270KB minified) + help rendering (~80ms) dominates remaining time -- **V8 startup snapshot**: Serialize loaded module graph. Would cut cold start to ~50ms. Complex Node.js API. -- **AOT compilation**: Use Node.js experimental compile cache or ahead-of-time compilation - -### Won't help (confirmed) -- splitting: false (incompatible with dynamic imports) -- Reduce entry points (chunk boundaries change but no timing improvement) -- Defer global-agent (no impact at current code size) -- Type-only imports alone (esbuild already tree-shakes) -- Make output.js/metadata.js lazy in base-command (parallel → sequential = worse) - -## Key Lessons -1. **Static imports inside a lazy module load in parallel**; converting to dynamic makes them sequential = worse -2. **V8 compile cache** saves ~35ms but needs warm-up runs after rebuild -3. **CJS shim overhead (48ms)** is inherent to esbuild ESM code splitting — can't be eliminated -4. **oclif help rendering (~80ms)** dominates the help benchmark; actual commands are 60ms faster -5. **Measurement noise**: /usr/bin/time has 10ms granularity; need median of 7+ runs for stability diff --git a/autoresearch.jsonl b/autoresearch.jsonl deleted file mode 100644 index 72e7a8ae23..0000000000 --- a/autoresearch.jsonl +++ /dev/null @@ -1,47 +0,0 @@ -{"type":"config","name":"Shopify CLI Faster Startup","metricName":"total_ms","metricUnit":"ms","bestDirection":"lower"} -{"run":1,"commit":"6dec3db","metric":1840,"metrics":{"import_ms":1494},"status":"keep","description":"Baseline: shopify version takes ~1840ms. Import alone is ~1494ms.","timestamp":1773400439568,"segment":0} -{"run":2,"commit":"1ba8c37","metric":710,"metrics":{"import_ms":0},"status":"keep","description":"Lazy command loading: separate bootstrap.ts, extract token-utils, move hooks to individual files, lazy command registry. 1840ms → 710ms (61% faster)","timestamp":1773401477099,"segment":0} -{"run":3,"commit":"14b52dc","metric":530,"metrics":{"import_ms":0},"status":"keep","description":"Defer conf-store import, inline app-init hook with lazy LocalStorage, skip hydrogen module for non-hydrogen commands. 710ms → 530ms (25% faster)","timestamp":1773401801219,"segment":0} -{"run":4,"commit":"pending","metric":540,"metrics":{"import_ms":0},"status":"discard","description":"Break circular deps: lazy imports in context/local.ts and is-global.ts. is-global 217ms→25ms but overall within noise (540 vs 530)","timestamp":1773402037688,"segment":0} -{"run":5,"commit":"pending","metric":580,"metrics":{"import_ms":0},"status":"discard","description":"Break circular deps in context/local and is-global. No measurable improvement (noise). Focus elsewhere.","timestamp":1773402098477,"segment":0} -{"run":6,"commit":"fc207c2","metric":950,"metrics":{"import_ms":0},"status":"keep","description":"Add postrun hook back (was missing - analytics broken). Lazy postrun imports. Honest measurement with all hooks: 1840ms → 950ms (48% faster)","timestamp":1773402286316,"segment":0} -{"run":7,"commit":"pending","metric":1060,"metrics":{"import_ms":0},"status":"discard","description":"Lazy error-handler and prerun imports. Sequential dynamic imports are slower than parallel static imports. Reverted.","timestamp":1773404332027,"segment":0} -{"type":"config","name":"Shopify CLI Faster Startup (corrected baseline)","metricName":"total_ms","metricUnit":"ms","bestDirection":"lower"} -{"run":2,"commit":"085a748","metric":560,"metrics":{"user_ms":410},"status":"keep","description":"Defer error-handler import to catch block. Saves ~380ms cold load on happy path. 600ms → 560ms wall, 420ms → 410ms user.","timestamp":1773404764990,"segment":0} -{"run":3,"commit":"1dd7784","metric":530,"metrics":{"user_ms":400},"status":"keep","description":"Lazy prerun hook with parallel imports (Promise.all). Defers node-package-manager, analytics, notifications. 560ms → 530ms wall, 410ms → 400ms user.","timestamp":1773404824594,"segment":0} -{"run":4,"commit":"1dd7784","metric":560,"metrics":{"user_ms":400},"status":"discard","description":"Pre-warm heavy modules (analytics, node-package-manager) before config.load(). No improvement - ESM loader is single-threaded, pre-warming causes CPU contention.","timestamp":1773404950102,"segment":0} -{"run":5,"commit":"5a1f1d9","metric":540,"metrics":{"user_ms":380},"status":"keep","description":"Lazy import of latest-version in node-package-manager.ts. Saves ~113ms on startup. 530ms → 540ms wall (noise), 400ms → 380ms user (-5%).","timestamp":1773405062120,"segment":0} -{"run":6,"commit":"5a1f1d9","metric":1040,"metrics":{"user_ms":570},"status":"discard","description":"Lazy session.js import in analytics.ts. Backfired badly - dynamic import blocks prerun execution. Static import allows parallel loading. Reverted.","timestamp":1773405189784,"segment":0} -{"type":"config","name":"Shopify CLI Faster Startup (help command, bundled)","metricName":"total_ms","metricUnit":"ms","bestDirection":"lower"} -{"run":1,"commit":"48352c3","metric":610,"metrics":{"user_ms":430},"status":"keep","description":"Baseline for help command benchmark (bundled). Includes all prior optimizations: lazy bootstrap, deferred error-handler, lazy prerun hooks, lazy latest-version.","timestamp":1773405846034,"segment":0} -{"run":2,"commit":"e72694d","metric":570,"metrics":{"user_ms":410},"status":"keep","description":"Run prerun hook in parallel with command execution. Prerun loads analytics in background while command runs. 610ms → 570ms wall (-6.6%), 430ms → 410ms user.","timestamp":1773405935336,"segment":0} -{"run":3,"commit":"e72694d","metric":590,"metrics":{"user_ms":420},"status":"discard","description":"Fire-and-forget postrun hook. No improvement - Node still waits for event loop to drain (pending HTTP requests). Reverted.","timestamp":1773406000582,"segment":0} -{"run":4,"commit":"0135940","metric":250,"metrics":{"user_ms":250},"status":"keep","description":"Fire-and-forget postrun + process.exit(0) after command. Eliminates waiting for analytics/version check network calls. 570ms → 250ms (-56%). Wall = user (no network wait).","timestamp":1773406056353,"segment":0} -{"run":5,"commit":"0135940","metric":270,"metrics":{"user_ms":250},"status":"discard","description":"Replace fs.js import with native node:fs in custom-oclif-loader (saves 47ms module load). Neutral with bundle (250-270ms range). Keep source change but don't count as improvement.","timestamp":1773406271913,"segment":0} -{"run":6,"commit":"0135940","metric":260,"metrics":{"user_ms":240},"status":"discard","description":"Defer global-agent to lazy import (only when proxy env vars set). Neutral - global-agent is small and esbuild chunk is same size. Reverted for simplicity.","timestamp":1773406435957,"segment":0} -{"run":7,"commit":"0135940","metric":250,"metrics":{"user_ms":240},"status":"discard","description":"Make prerun fully fire-and-forget (void, no await). Same 250ms - prerun was already completing in parallel within the bundle. Cleaner code but no metric improvement.","timestamp":1773406757005,"segment":0} -{"run":8,"commit":"c496c74","metric":220,"metrics":{"user_ms":230},"status":"keep","description":"Externalize TypeScript compiler from bundle. Oclif bundled the entire TS compiler (~9MB) for tsconfig parsing. Making it external shrinks startup chunks from 9.5MB to 614KB (93% smaller). 250ms → 220ms.","timestamp":1773407032826,"segment":0} -{"run":9,"commit":"537603b","metric":200,"metrics":{"user_ms":210},"status":"keep","description":"Skip prerun/postrun hooks for help and version commands (no analytics needed for lightweight commands). 220ms → 200ms (-9%). Eliminates ~20ms of hook module loading overhead.","timestamp":1773407223068,"segment":0} -{"run":10,"commit":"46ff14b","metric":190,"metrics":{"user_ms":210},"status":"keep","description":"Skip init hooks for help/version via runHook override. Avoids app-init LocalStorage and hydrogen-init for lightweight commands. 200ms → 190ms (-5%).","timestamp":1773407354165,"segment":0} -{"run":11,"commit":"46ff14b","metric":420,"metrics":{"user_ms":250},"status":"discard","description":"Replace help command with oclif built-in help renderer. Worse and inconsistent (270-420ms wall) despite same user time (250ms). oclif help has network/IO overhead. Reverted.","timestamp":1773407690813,"segment":0} -{"run":12,"commit":"5983b10","metric":60,"metrics":{"user_ms":40},"status":"keep","description":"Fast path for help and version: read manifest + package.json directly, bypass oclif entirely. 190ms → 60ms (-68%). Total improvement from baseline: 610ms → 60ms (10.2x faster).","timestamp":1773407909992,"segment":0} -{"run":13,"commit":"5b7d1db","metric":260,"metrics":{"user_ms":250},"status":"keep","description":"Reverted help/version-specific fast paths and hook skipping. Keeping only universal optimizations: lazy bootstrap, lazy commands, lazy hooks, deferred error-handler, TS external, process.exit(0). 610ms → 260ms (-57%) for all commands.","timestamp":1773408052178,"segment":0} -{"run":14,"commit":"5c84086","metric":230,"metrics":{"user_ms":230},"status":"keep","description":"Fire-and-forget init hooks for ALL commands via runHook override. Init hooks (app-init LocalStorage, hydrogen-init) run in background. Universal optimization. 260ms → 230ms (-11.5%).","timestamp":1773408176327,"segment":0} -{"run":15,"commit":"9cf02c9","metric":220,"metrics":{"user_ms":230},"status":"keep","description":"Replace cli-kit wrappers (fs.js, path.js, execa, context/local.js) with native Node.js builtins in custom-oclif-loader. Eliminates 8 extra chunks (~552KB) from startup path. 230ms → 220ms. Universal.","timestamp":1773408337406,"segment":0} -{"run":16,"commit":"79667f9","metric":220,"metrics":{"user_ms":230},"status":"keep","description":"Inline isDevelopment check in cli-launcher.ts to avoid importing context/local.js chain. Consistent 220ms. Universal.","timestamp":1773408410705,"segment":0} -{"run":17,"commit":"79667f9","metric":230,"metrics":{"user_ms":230},"status":"discard","description":"Defer global-agent to async import. Reduces startup chunks from 694KB to 612KB but no measurable wall time improvement (230ms). At this code size, V8 compile is fast; bottleneck is elsewhere.","timestamp":1773408586904,"segment":0} -{"run":18,"commit":"79667f9","metric":230,"metrics":{"user_ms":230},"status":"discard","description":"Skip hydrogen plugin addition when package not installed. No measurable improvement - oclif handles missing plugins quickly. Reverted.","timestamp":1773408743821,"segment":0} -{"run":19,"commit":"3573230","metric":190,"metrics":{"user_ms":200},"status":"keep","description":"Defer prerun+postrun hooks to AFTER command execution (fire-and-forget chain). Command gets exclusive CPU time. process.exit(0) kills hooks. Universal. 220ms → 190ms (-14%).","timestamp":1773408845282,"segment":0} -{"run":20,"commit":"4db93c2","metric":190,"metrics":{"user_ms":190},"status":"keep","description":"Enable full minification (whitespace + identifiers) in esbuild bundle. Startup chunks 694KB → 352KB (-49%). Wall time ~190ms (marginal improvement, within noise). Universal.","timestamp":1773409147130,"segment":0} -{"run":21,"commit":"a335603","metric":160,"metrics":{"user_ms":160},"status":"keep","description":"Enable V8 compile cache via module.enableCompileCache() in dev.js/run.js. Caches bytecode between runs, eliminating ~40ms V8 compile overhead. Dynamic import bootstrap.js after cache enabled. 190ms → 160ms (-16%). Universal.","timestamp":1773409335411,"segment":0} -{"run":22,"commit":"a335603","metric":160,"metrics":{"user_ms":160},"status":"discard","description":"Attempted direct dispatch bypassing oclif.run(). CJS/ESM interop broke normalizeArgv import in bundle. Reverted cli-launcher.ts to working state with oclif.run(). 160ms (unchanged).","timestamp":1773409549504,"segment":0} -{"run":23,"commit":"a335603","metric":160,"metrics":{"user_ms":170},"status":"discard","description":"Reduce esbuild entry points from 51 to 30 (exclude test + service files). Total chunks 120→88 but startup chunks increased 12→14. No wall time improvement. Reverted.","timestamp":1773409738517,"segment":0} -{"run":24,"commit":"b4fbcf2","metric":160,"metrics":{"user_ms":160},"status":"keep","description":"Lazy imports in base-command: defer ui.js (600KB React/Ink), error-handler.js, notifications-system.js to point of use. Not captured by help benchmark but reduces startup for ALL actual commands. 160ms (unchanged for help).","timestamp":1773410045627,"segment":0} -{"run":25,"commit":"b4fbcf2","metric":160,"metrics":{"user_ms":160},"status":"discard","description":"Investigated splitting: false in esbuild. INCOMPATIBLE: ESM format + no splitting converts dynamic imports to await inside non-async __esm() wrappers → SyntaxError. cli-kit's pervasive dynamic imports (lazy ui.js, error-handler.js) require top-level await which only works with splitting: true. CJS format incompatible with \"type\": \"module\" + import.meta.url. Dead end.","timestamp":1773410530381,"segment":0} -{"run":26,"commit":"e6a1b21","metric":160,"metrics":{"user_ms":160},"status":"keep","description":"Custom lightweight dispatcher bypassing oclif.run() for known commands. Inlines normalizeArgv for space-separated topics. Falls back to oclif only for help/unknown. Avoids loading oclif help module (~50ms) for all actual commands. Help benchmark: 160ms (same). Version: 180ms→150ms.","timestamp":1773410893983,"segment":0} -{"run":27,"commit":"c8b60a0","metric":160,"metrics":{"user_ms":160},"status":"keep","description":"Inline terminalSupportsPrompting (avoids system.js→execa chain), lazy environments.js in base-command. Version command new chunks: 972KB→111KB (-88%). Help benchmark 160ms (unchanged). Improves cold-cache startup for all commands.","timestamp":1773411136564,"segment":0} -{"run":28,"commit":"2ca50dc","metric":160,"metrics":{"user_ms":160},"status":"keep","description":"Skip async exitIfOldNodeVersion call for Node >= 18 (inline fast check). Minor cleanup. 160ms (unchanged, as expected — async call overhead is negligible).","timestamp":1773411273202,"segment":0} -{"run":29,"commit":"e072d41","metric":160,"metrics":{"user_ms":160},"status":"keep","description":"Static import ShopifyConfig in cli-launcher (instead of dynamic). Eliminates one async import hop. Startup chunks unchanged (12, 355KB). 160ms. Cleaner code path.","timestamp":1773411398886,"segment":0} -{"run":30,"commit":"e072d41","metric":180,"metrics":{"user_ms":180},"status":"discard","description":"Added extra warmup runs (3 instead of 1) to autoresearch.sh for compile cache. Tried type-only imports in output.ts — reverted (no improvement). Benchmark shows 160-190ms range due to 10ms granularity + system noise.","timestamp":1773411909372,"segment":0} -{"run":31,"commit":"fc37b32","metric":160,"metrics":{"user_ms":160},"status":"keep","description":"Replace dynamic import('@oclif/core') for debug settings with static import of settings. Eliminates async hop on critical path since @oclif/core is already loaded via ShopifyConfig. 160ms (stable). Raw timing shows 140-150ms after warm cache.","timestamp":1773412078512,"segment":0} -{"run":32,"commit":"45b6c82","metric":160,"metrics":{"user_ms":160},"status":"keep","description":"Type-only imports in output.ts: PackageManager and TokenItem. Prevents accidental runtime deps on node-package-manager.js and ui.js. Correctness improvement; 160ms (unchanged - esbuild already tree-shakes).","timestamp":1773412180635,"segment":0} diff --git a/autoresearch.md b/autoresearch.md deleted file mode 100644 index 79b2835771..0000000000 --- a/autoresearch.md +++ /dev/null @@ -1,49 +0,0 @@ -# Autoresearch: Shopify CLI Faster Startup - -## Objective -Optimize the wall-clock startup time of the Shopify CLI (`shopify version` command). The main bottleneck is that `packages/cli/src/index.ts` statically imports ALL command modules from ALL packages (@shopify/app ~730ms, @shopify/cli-hydrogen ~90ms, @shopify/theme, etc.) even though only one command will run. The oclif manifest exists (106 commands cached) but isn't leveraged because `bin/dev.js` imports `index.ts` directly for the `runShopifyCLI` function, triggering the entire module graph. - -## Metrics -- **Primary**: total_ms (ms, lower is better) — wall-clock time for `node packages/cli/bin/dev.js version` -- **Secondary**: import_ms — time to import the entry module - -## How to Run -`./autoresearch.sh` — outputs `METRIC name=number` lines. - -## Key Architecture -- `bin/dev.js` → imports `../dist/index.js` (default export = `runShopifyCLI`) -- `index.ts` statically imports 28 modules including @shopify/app (1s+), @shopify/theme, @shopify/cli-hydrogen -- `index.ts` exports: `COMMANDS` object, hook identifiers, `runShopifyCLI` (default) -- oclif explicit strategy: loads `./dist/index.js`, reads `COMMANDS` export -- `oclif.manifest.json` exists (298KB, 106 commands) — oclif can use it for metadata without importing -- `ShopifyConfig` extends oclif `Config`, loaded in `cli-launcher.ts` -- Hooks point to `./dist/index.js` with named identifiers — also trigger full import - -## Files in Scope -- `packages/cli/src/index.ts` — main entry, all static imports live here -- `packages/cli/bin/dev.js` — development entry point -- `packages/cli/bin/run.js` — production entry point -- `packages/cli/package.json` — oclif config (commands, hooks) -- `packages/cli-kit/src/public/node/cli.ts` — runCLI function -- `packages/cli-kit/src/public/node/cli-launcher.ts` — launchCLI, creates ShopifyConfig -- `packages/cli-kit/src/public/node/custom-oclif-loader.ts` — ShopifyConfig class -- `packages/cli-kit/src/public/node/base-command.ts` — base command class (~400ms import) -- Any new files we create for lazy loading - -## Off Limits -Nothing explicitly off limits, but changes should be compatible with existing tests. - -## Constraints -- Tests must pass -- Lint must pass -- Type-check must pass -- Can add/remove dependencies and override oclif behavior - -## Strategy -1. **Separate bootstrap from commands**: Create `bootstrap.ts` with only `runShopifyCLI` + side effects. `bin/dev.js` imports this instead of `index.ts`. -2. **Move hooks to separate files**: Each hook gets its own file so oclif config can point directly to it. -3. **Lazy command loading**: Override oclif's plugin loading in ShopifyConfig to dynamically import only the needed command. -4. **Reduce base-command weight**: Investigate what makes `@shopify/cli-kit/node/base-command` cost ~400ms. - -## What's Been Tried -(nothing yet — starting baseline) diff --git a/autoresearch.sh b/autoresearch.sh deleted file mode 100755 index b37a2dddb7..0000000000 --- a/autoresearch.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -set -euo pipefail - -# Ensure bundle is up to date (bundle includes build as dependency) -pnpm nx bundle cli 2>&1 | tail -1 - -# Warmup run (filesystem cache) -node packages/cli/bin/dev.js help > /dev/null 2>&1 - -# Benchmark: median of 7 runs (wall clock and user time) -wall_times=() -user_times=() -for i in 1 2 3 4 5 6 7; do - output=$( { /usr/bin/time -p node packages/cli/bin/dev.js help > /dev/null; } 2>&1 ) - wall=$( echo "$output" | awk '/^real/{print $2}' ) - user=$( echo "$output" | awk '/^user/{print $2}' ) - wall_ms=$(echo "$wall * 1000" | bc | cut -d. -f1) - user_ms=$(echo "$user * 1000" | bc | cut -d. -f1) - wall_times+=("$wall_ms") - user_times+=("$user_ms") -done - -# Sort and take median (index 3 of 7) -IFS=$'\n' sorted_wall=($(sort -n <<<"${wall_times[*]}")); unset IFS -IFS=$'\n' sorted_user=($(sort -n <<<"${user_times[*]}")); unset IFS -median_wall=${sorted_wall[3]} -median_user=${sorted_user[3]} - -echo "METRIC total_ms=$median_wall" -echo "METRIC user_ms=$median_user"