From b14a1cc657809027542e2b54d8fcd27ab31b2f88 Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Tue, 24 Mar 2026 08:57:43 -0400 Subject: [PATCH 01/11] Use dynamic software factory harness ports --- ...-for-software-factory-test-harness-plan.md | 99 ++ packages/host/app/router.ts | 2 +- packages/matrix/docker/synapse/index.ts | 42 +- packages/matrix/helpers/environment-config.ts | 21 +- packages/realm-server/main.ts | 25 + packages/realm-server/middleware/index.ts | 10 + packages/realm-server/prerender/page-pool.ts | 2 +- packages/realm-server/routes.ts | 2 + packages/realm-server/server.ts | 54 +- .../tests/virtual-network-test.ts | 24 + packages/realm-server/worker-manager.ts | 23 + packages/runtime-common/virtual-network.ts | 11 + packages/software-factory/README.md | 16 +- .../software-factory/playwright.config.ts | 4 +- .../playwright.global-setup.ts | 33 + .../software-factory/src/cli/cache-realm.ts | 1 + .../software-factory/src/cli/serve-realm.ts | 8 + .../software-factory/src/cli/serve-support.ts | 6 + .../software-factory/src/factory-bootstrap.ts | 41 + .../src/factory-entrypoint.ts | 8 +- .../src/factory-target-realm.ts | 82 +- packages/software-factory/src/harness.ts | 899 +++++++++++++++--- packages/software-factory/src/realm-auth.ts | 22 +- .../AgentProfile/demo-agent.json | 2 +- .../DarkFactory/demo-factory.json | 2 +- .../KnowledgeArticle/agent-onboarding.json | 2 +- .../Project/demo-project.json | 2 +- .../Ticket/ticket-001.json | 2 +- .../darkfactory-adopter/agent-demo.json | 2 +- .../darkfactory-adopter/factory-demo.json | 2 +- .../knowledge-article-demo.json | 2 +- .../darkfactory-adopter/project-demo.json | 2 +- .../darkfactory-adopter/ticket-demo.json | 2 +- .../tests/darkfactory.spec.ts | 37 +- .../tests/factory-bootstrap.spec.ts | 19 +- .../factory-entrypoint.integration.test.ts | 15 + .../tests/factory-entrypoint.test.ts | 55 ++ .../tests/factory-target-realm.spec.ts | 11 +- .../tests/factory-target-realm.test.ts | 43 +- packages/software-factory/tests/fixtures.ts | 86 +- .../tests/helpers/browser-auth.ts | 39 +- 41 files changed, 1484 insertions(+), 276 deletions(-) create mode 100644 docs/cs-10485-dynamic-port-allocation-for-software-factory-test-harness-plan.md diff --git a/docs/cs-10485-dynamic-port-allocation-for-software-factory-test-harness-plan.md b/docs/cs-10485-dynamic-port-allocation-for-software-factory-test-harness-plan.md new file mode 100644 index 00000000000..a9a24370708 --- /dev/null +++ b/docs/cs-10485-dynamic-port-allocation-for-software-factory-test-harness-plan.md @@ -0,0 +1,99 @@ +# CS-10485 Dynamic Port Allocation For Software Factory Test Harness Plan + +## Goal + +Remove the software-factory harness's hardcoded runtime ports so isolated realm stacks can run in parallel and the harness no longer collides with `mise run dev-all`. + +Success for this ticket means: + +- isolated realm servers do not assume fixed ports for realm-server, compat proxy, or worker-manager +- the harness support stack starts Synapse on a non-conflicting dynamic port +- Playwright fixtures discover actual runtime ports from metadata instead of reconstructing them from constants +- `playwright.config.ts` is updated to `workers: 2` after we confirm the targeted Playwright suite runs cleanly at that worker count + +## Current State + +The current implementation still hardcodes the same ports in multiple places: + +- [`packages/software-factory/src/harness.ts`](/home/hassan/codez/boxel-cs-10485-codex/packages/software-factory/src/harness.ts) + - `REALM_SERVER_PORT=4205` + - `COMPAT_REALM_SERVER_PORT=4201` + - `WORKER_MANAGER_PORT=4232` + - published realm domains and base/skills URL mappings are built from those constants +- [`packages/software-factory/src/cli/serve-realm.ts`](/home/hassan/codez/boxel-cs-10485-codex/packages/software-factory/src/cli/serve-realm.ts) + - writes metadata for `realmURL` and auth only, not the allocated ports that fixtures need +- [`packages/software-factory/tests/fixtures.ts`](/home/hassan/codez/boxel-cs-10485-codex/packages/software-factory/tests/fixtures.ts) + - assumes fixed ports for shutdown waiting and Playwright request rewrites +- [`packages/software-factory/playwright.config.ts`](/home/hassan/codez/boxel-cs-10485-codex/packages/software-factory/playwright.config.ts) + - pins `workers: 1` +- [`packages/matrix/docker/synapse/index.ts`](/home/hassan/codez/boxel-cs-10485-codex/packages/matrix/docker/synapse/index.ts) and [`packages/matrix/helpers/environment-config.ts`](/home/hassan/codez/boxel-cs-10485-codex/packages/matrix/helpers/environment-config.ts) + - the current non-environment path still uses fixed Synapse host port `8008` and fixed `getSynapseURL()` output of `http://localhost:8008` + +## Assumptions To Validate While Implementing + +- The realm-server and worker-manager CLIs tolerate `--port=0` and continue to emit their existing `ready` IPC message. +- The cleanest source of truth for actual bound ports may be a well-known runtime metadata file written after bind completes, instead of reconstructing ports indirectly. +- The compat port is only a compatibility surface for browser rewrites; nothing important should depend on `4201` specifically. +- The Synapse helper does not yet provide dynamic host-port behavior for the harness's current execution mode, so this needs a code change rather than just propagation. + +## Implementation Plan + +### 1. Make port allocation explicit in the harness + +- Update [`packages/software-factory/src/harness.ts`](/home/hassan/codez/boxel-cs-10485-codex/packages/software-factory/src/harness.ts) so the default realm-server, compat-proxy, and worker-manager ports are dynamic instead of fixed. +- Refactor `startIsolatedRealmStack()` to return the effective runtime ports in addition to the child processes. +- Add a single well-known runtime metadata contract for the spawned realm stack so the actual bound ports are published explicitly and consumed consistently. +- Make all internal URL construction use those effective ports instead of module-level constants: + - published realm domains + - base realm mappings + - optional skills realm mappings + - compat proxy target/listen ports + +### 2. Propagate support-stack runtime data + +- Update `startFactorySupportServices()` in [`packages/software-factory/src/harness.ts`](/home/hassan/codez/boxel-cs-10485-codex/packages/software-factory/src/harness.ts) to avoid fixed Synapse ports and to preserve the actual `matrixURL` in support context. +- Extend the Synapse startup path so the harness can obtain a dynamic host port in normal local test mode, not only in `BOXEL_ENVIRONMENT` mode. +- If the support metadata contract needs more structure, extend [`packages/software-factory/src/runtime-metadata.ts`](/home/hassan/codez/boxel-cs-10485-codex/packages/software-factory/src/runtime-metadata.ts) and the Playwright setup flow accordingly. + +### 3. Expand serve-realm metadata + +- Extend [`packages/software-factory/src/cli/serve-realm.ts`](/home/hassan/codez/boxel-cs-10485-codex/packages/software-factory/src/cli/serve-realm.ts) so the metadata JSON includes the actual runtime ports and any derived origins/prefixes the tests need. +- Keep the metadata payload as the single source of truth for isolated test realms. + +### 4. Switch tests to metadata-driven rewrites and cleanup + +- Update [`packages/software-factory/tests/fixtures.ts`](/home/hassan/codez/boxel-cs-10485-codex/packages/software-factory/tests/fixtures.ts) to read the real runtime ports from `serve-realm.ts` metadata. +- Remove the fixed-port assumptions from: + - shutdown waiting + - base realm redirect registration + - optional skills redirect registration +- Preserve the shared-realm cache behavior per Playwright worker and test file. + +### 5. Re-enable parallel Playwright workers + +- Raise `workers` in [`packages/software-factory/playwright.config.ts`](/home/hassan/codez/boxel-cs-10485-codex/packages/software-factory/playwright.config.ts) from `1` to `2` only after local confirmation that the targeted suite runs cleanly with two workers. +- Keep `fullyParallel: false` unless the updated test behavior proves broader parallelism is safe. + +## Target Files + +- [`packages/software-factory/src/harness.ts`](/home/hassan/codez/boxel-cs-10485-codex/packages/software-factory/src/harness.ts) +- [`packages/software-factory/src/cli/serve-realm.ts`](/home/hassan/codez/boxel-cs-10485-codex/packages/software-factory/src/cli/serve-realm.ts) +- [`packages/software-factory/src/runtime-metadata.ts`](/home/hassan/codez/boxel-cs-10485-codex/packages/software-factory/src/runtime-metadata.ts) if support metadata needs a contract update +- [`packages/software-factory/tests/fixtures.ts`](/home/hassan/codez/boxel-cs-10485-codex/packages/software-factory/tests/fixtures.ts) +- [`packages/software-factory/playwright.config.ts`](/home/hassan/codez/boxel-cs-10485-codex/packages/software-factory/playwright.config.ts) +- Possibly [`packages/software-factory/playwright.global-setup.ts`](/home/hassan/codez/boxel-cs-10485-codex/packages/software-factory/playwright.global-setup.ts) if support metadata wiring needs an adjustment +- [`packages/matrix/docker/synapse/index.ts`](/home/hassan/codez/boxel-cs-10485-codex/packages/matrix/docker/synapse/index.ts) +- [`packages/matrix/helpers/environment-config.ts`](/home/hassan/codez/boxel-cs-10485-codex/packages/matrix/helpers/environment-config.ts) if `getSynapseURL()` needs to stop assuming `8008` + +## Testing Notes + +- Run `pnpm lint` in [`packages/software-factory`](/home/hassan/codez/boxel-cs-10485-codex/packages/software-factory) before any commit. +- Run targeted Playwright coverage in [`packages/software-factory`](/home/hassan/codez/boxel-cs-10485-codex/packages/software-factory), starting with: + - `pnpm test -- --grep "darkfactory|factory target realm|factory bootstrap"` +- Verify the targeted suite runs with `workers=2` without realm-stack or Synapse port collisions before keeping that config change. +- If local parallel execution is noisy or environment-dependent, capture the residual risk explicitly and leave full confirmation to CI. + +## Open Questions + +- What exact metadata shape should the realm stack publish so both the harness and Playwright fixtures can consume it without duplicating URL construction logic? +- Are there any remaining browser-side assumptions that still reference the compat origin directly instead of using runtime metadata? diff --git a/packages/host/app/router.ts b/packages/host/app/router.ts index 7c1d18615af..5cdfab0ecb1 100644 --- a/packages/host/app/router.ts +++ b/packages/host/app/router.ts @@ -17,7 +17,7 @@ Router.map(function () { }); this.route('module', { path: '/module/:id/:nonce/:options' }); this.route('connect', { path: '/connect/:origin' }); - this.route('standby'); + this.route('standby', { path: '/_standby' }); this.route('command-runner', { path: '/command-runner/:request_id/:nonce', }); diff --git a/packages/matrix/docker/synapse/index.ts b/packages/matrix/docker/synapse/index.ts index 095de13bb64..979cc56357f 100644 --- a/packages/matrix/docker/synapse/index.ts +++ b/packages/matrix/docker/synapse/index.ts @@ -110,7 +110,18 @@ interface StartOptions { dataDir?: string; containerName?: string; suppressRegistrationSecretFile?: true; + dynamicHostPort?: true; } + +async function resolveHostPort(synapseId: string): Promise { + let { execSync } = await import('child_process'); + let portOutput = execSync(`docker port ${synapseId} 8008/tcp`, { + encoding: 'utf-8', + }).trim(); + let firstLine = portOutput.split('\n')[0]; + return parseInt(firstLine.split(':').pop()!, 10); +} + export async function synapseStart( opts?: StartOptions, stopExisting = true, @@ -146,8 +157,11 @@ export async function synapseStart( '-v', `${path.join(__dirname, 'templates')}:/custom/templates/`, ]; - if (isEnvironmentMode()) { - // Branch mode: dynamic host port, no fixed IP + if (isEnvironmentMode() || opts?.dynamicHostPort) { + // Dynamic host port, with fixed container IP only when not running in branch mode + if (!isEnvironmentMode()) { + dockerParams.push(`--ip=${synCfg.host}`); + } dockerParams.push('-p', '0:8008/tcp', '--network=boxel'); } else { dockerParams.push( @@ -185,19 +199,23 @@ export async function synapseStart( ], }); - // In branch mode, read the dynamic host port and register with Traefik - if (isEnvironmentMode()) { - let { execSync } = await import('child_process'); - let portOutput = execSync(`docker port ${synapseId} 8008/tcp`, { - encoding: 'utf-8', - }).trim(); - let firstLine = portOutput.split('\n')[0]; - let hostPort = parseInt(firstLine.split(':').pop()!, 10); + let hostPort = synCfg.port; + if (isEnvironmentMode() || opts?.dynamicHostPort) { + hostPort = await resolveHostPort(synapseId); console.log(`Synapse dynamic host port: ${hostPort}`); + } + + if (isEnvironmentMode()) { registerSynapseWithTraefik(hostPort); } - const synapse: SynapseInstance = { synapseId, ...synCfg }; + const synapse: SynapseInstance = { + synapseId, + ...synCfg, + host: '127.0.0.1', + port: hostPort, + baseUrl: `http://localhost:${hostPort}`, + }; synapses.set(synapseId, synapse); function cleanupRegistrationSecret() { @@ -250,7 +268,7 @@ export async function registerUser( admin = false, displayName?: string, ): Promise { - const url = `${getSynapseURL()}/_synapse/admin/v1/register`; + const url = `${getSynapseURL(synapse)}/_synapse/admin/v1/register`; const context = await request.newContext({ baseURL: url }); const { nonce } = await (await context.get(url)).json(); const mac = admin diff --git a/packages/matrix/helpers/environment-config.ts b/packages/matrix/helpers/environment-config.ts index dff1ad4f6e5..d3ebf7682d5 100644 --- a/packages/matrix/helpers/environment-config.ts +++ b/packages/matrix/helpers/environment-config.ts @@ -1,9 +1,5 @@ import { execSync } from 'child_process'; -import { - writeFileSync, - renameSync, - unlinkSync, -} from 'fs'; +import { writeFileSync, renameSync, unlinkSync } from 'fs'; import { join, resolve } from 'path'; import yaml from 'yaml'; @@ -64,7 +60,16 @@ export function getSynapseContainerName(): string { return 'boxel-synapse'; } -export function getSynapseURL(): string { +export function getSynapseURL(synapse?: { + baseUrl?: string; + port?: number; +}): string { + if (synapse?.baseUrl) { + return synapse.baseUrl; + } + if (synapse?.port != null) { + return `http://localhost:${synapse.port}`; + } if (!isEnvironmentMode()) { return 'http://localhost:8008'; } @@ -110,9 +115,7 @@ export function registerSynapseWithTraefik(hostPort: number): void { }; atomicWrite(configPath, yaml.stringify(config)); - console.log( - `Registered Synapse at ${hostname} -> localhost:${hostPort}`, - ); + console.log(`Registered Synapse at ${hostname} -> localhost:${hostPort}`); } export function deregisterSynapseFromTraefik(): void { diff --git a/packages/realm-server/main.ts b/packages/realm-server/main.ts index de573520a52..abd1343cd3a 100644 --- a/packages/realm-server/main.ts +++ b/packages/realm-server/main.ts @@ -15,6 +15,8 @@ import { NodeAdapter } from './node-realm'; import yargs from 'yargs'; import { RealmServer } from './server'; import { resolve } from 'path'; +import { dirname, join } from 'path'; +import { mkdirSync, renameSync, writeFileSync } from 'fs'; import * as Sentry from '@sentry/node'; import { PgAdapter, PgQueuePublisher } from '@cardstack/postgres'; import { MatrixClient } from '@cardstack/runtime-common/matrix-client'; @@ -35,6 +37,23 @@ import { (globalThis as any).ContentTagGlobal = ContentTagGlobal; let log = logger('main'); +const runtimeMetadataFile = + process.env.SOFTWARE_FACTORY_REALM_SERVER_METADATA_FILE; + +function writeRuntimeMetadata(payload: unknown): void { + if (!runtimeMetadataFile) { + return; + } + + mkdirSync(dirname(runtimeMetadataFile), { recursive: true }); + let tempFile = join( + dirname(runtimeMetadataFile), + `.realm-server.${process.pid}.${Date.now()}.tmp`, + ); + writeFileSync(tempFile, JSON.stringify(payload, null, 2)); + renameSync(tempFile, runtimeMetadataFile); +} + if (process.env.NODE_ENV === 'test') { (globalThis as any).__environment = 'test'; } @@ -386,6 +405,12 @@ const getIndexHTML = async () => { let httpServer = server.listen(port); httpServer.on('listening', () => { + let actualPort = + (httpServer.address() as import('net').AddressInfo | null)?.port ?? port; + writeRuntimeMetadata({ + pid: process.pid, + port: actualPort, + }); if (isEnvironmentMode()) { registerService(httpServer, serviceName, { wildcardSubdomains: true }); } diff --git a/packages/realm-server/middleware/index.ts b/packages/realm-server/middleware/index.ts index f8dd294f9a4..b631c613c99 100644 --- a/packages/realm-server/middleware/index.ts +++ b/packages/realm-server/middleware/index.ts @@ -17,6 +17,7 @@ import { const REQUEST_BODY_STATE = 'requestBody'; interface ProxyOptions { + requestHeaders?: Record; responseHeaders?: Record; } @@ -33,6 +34,11 @@ export function proxyAsset( return `/${filename}`; }, events: { + proxyReq: (proxyReq) => { + for (let [key, value] of Object.entries(opts?.requestHeaders ?? {})) { + proxyReq.setHeader(key, value); + } + }, proxyRes: (_proxyRes, _req, res) => { for (let [key, value] of Object.entries(opts?.responseHeaders ?? {})) { res.setHeader(key, value); @@ -103,6 +109,10 @@ export function ecsMetadata(ctxt: Koa.Context, next: Koa.Next) { } export function fullRequestURL(ctxt: Koa.Context): URL { + let forwardedURL = ctxt.req.headers['x-boxel-forwarded-url']; + if (typeof forwardedURL === 'string' && forwardedURL.trim() !== '') { + return new URL(forwardedURL); + } let protocol = ctxt.req.headers['x-forwarded-proto'] === 'https' ? 'https' : 'http'; return new URL(`${protocol}://${ctxt.req.headers.host}${ctxt.req.url}`); diff --git a/packages/realm-server/prerender/page-pool.ts b/packages/realm-server/prerender/page-pool.ts index 536948f2198..0bcb792885b 100644 --- a/packages/realm-server/prerender/page-pool.ts +++ b/packages/realm-server/prerender/page-pool.ts @@ -384,7 +384,7 @@ export class PagePool { } async #loadStandbyPage(page: Page, pageId: string): Promise { - await page.goto(`${this.#boxelHostURL}/standby`, { + await page.goto(`${this.#boxelHostURL}/_standby`, { waitUntil: 'domcontentloaded', timeout: this.#standbyTimeoutMs, }); diff --git a/packages/realm-server/routes.ts b/packages/realm-server/routes.ts index adfc9904ae8..0cc4be1096f 100644 --- a/packages/realm-server/routes.ts +++ b/packages/realm-server/routes.ts @@ -102,6 +102,7 @@ export type CreateRoutesArgs = { backgroundURL?: string; iconURL?: string; }) => Promise<{ realm: Realm; info: Partial }>; + serveHostApp: (ctxt: Koa.Context, next: Koa.Next) => Promise; serveIndex: (ctxt: Koa.Context, next: Koa.Next) => Promise; serveFromRealm: (ctxt: Koa.Context, next: Koa.Next) => Promise; sendEvent: ( @@ -126,6 +127,7 @@ export function createRoutes(args: CreateRoutesArgs) { router.head('/', livenessCheck); router.get('/', healthCheck, args.serveIndex, args.serveFromRealm); + router.get('/_standby', healthCheck, args.serveHostApp, args.serveFromRealm); router.post('/_server-session', handleCreateSessionRequest(args)); router.post( '/_create-realm', diff --git a/packages/realm-server/server.ts b/packages/realm-server/server.ts index e40c98e13e9..4a08890d434 100644 --- a/packages/realm-server/server.ts +++ b/packages/realm-server/server.ts @@ -225,6 +225,7 @@ export class RealmServer { grafanaSecret: this.grafanaSecret, virtualNetwork: this.virtualNetwork, createRealm: this.createRealm, + serveHostApp: this.serveHostApp, serveIndex: this.serveIndex, serveFromRealm: this.serveFromRealm, sendEvent: this.sendEvent, @@ -238,7 +239,13 @@ export class RealmServer { prerenderer: this.prerenderer, }), ) - .use(proxyAsset('/auth-service-worker.js', this.assetsURL)) + .use( + proxyAsset('/auth-service-worker.js', this.assetsURL, { + requestHeaders: { + 'accept-encoding': 'identity', + }, + }), + ) .use(this.serveIndex) .use(this.serveFromRealm); @@ -506,6 +513,19 @@ export class RealmServer { return; }; + private serveHostApp = async (ctxt: Koa.Context, next: Koa.Next) => { + let acceptHeader = (ctxt.header.accept ?? '').toLowerCase(); + if (!acceptHeader.includes('text/html')) { + return next(); + } + + ctxt.type = 'html'; + ctxt.body = injectHeadHTML( + await this.retrieveIndexHTML(), + `Boxel\n${this.defaultIconLinks().join('\n')}`, + ); + }; + private findRealmForRequestURL(requestURL: URL): Realm | undefined { return this.realms.find((candidate) => { let realmURL = new URL(candidate.url); @@ -652,6 +672,18 @@ export class RealmServer { this.promiseForIndexHTML = deferred.promise; } + let rewriteRealmURL = (url?: string) => { + if (!url) { + return url; + } + + let parsed = new URL(url); + return new URL( + `${parsed.pathname}${parsed.search}${parsed.hash}`, + this.serverURL, + ).href; + }; + let indexHTML = (await this.getIndexHTML()).replace( /()/, (_match, g1, g2, g3) => { @@ -674,7 +706,23 @@ export class RealmServer { config = merge({}, config, { hostsOwnAssets: false, assetsURL: this.assetsURL.href, + matrixURL: this.matrixClient.matrixURL.href.replace(/\/$/, ''), + matrixServerName: this.matrixClient.matrixURL.hostname, realmServerURL: this.serverURL.href, + resolvedBaseRealmURL: rewriteRealmURL(config.resolvedBaseRealmURL), + resolvedCatalogRealmURL: rewriteRealmURL( + config.resolvedCatalogRealmURL, + ), + resolvedExternalCatalogRealmURL: rewriteRealmURL( + config.resolvedExternalCatalogRealmURL, + ), + resolvedSkillsRealmURL: rewriteRealmURL( + config.resolvedSkillsRealmURL, + ), + resolvedOpenRouterRealmURL: rewriteRealmURL( + config.resolvedOpenRouterRealmURL, + ), + defaultSystemCardId: rewriteRealmURL(config.defaultSystemCardId), cardSizeLimitBytes: this.cardSizeLimitBytes, fileSizeLimitBytes: this.fileSizeLimitBytes, publishedRealmDomainOverrides: @@ -824,6 +872,10 @@ export class RealmServer { undefined, userInitiatedPriority, ); + let actualRealmURL = this.virtualNetwork.mapURL(url, 'virtual-to-real'); + if (actualRealmURL && actualRealmURL.href !== url) { + this.virtualNetwork.addURLMapping(new URL(url), actualRealmURL); + } return { realm, diff --git a/packages/realm-server/tests/virtual-network-test.ts b/packages/realm-server/tests/virtual-network-test.ts index 9aa1432d800..06f9cc705e5 100644 --- a/packages/realm-server/tests/virtual-network-test.ts +++ b/packages/realm-server/tests/virtual-network-test.ts @@ -2,6 +2,7 @@ import type { ResponseWithNodeStream } from '@cardstack/runtime-common'; import { VirtualNetwork } from '@cardstack/runtime-common'; import { module, test } from 'qunit'; import { basename } from 'path'; +import '../setup-logger'; module(basename(__filename), function () { module('virtual-network', function () { @@ -62,5 +63,28 @@ module(basename(__filename), function () { assert.strictEqual(response.url, 'http://test-realm/test/person.gts'); assert.true(response.redirected); }); + + test('can resolve mapped URLs in both directions', function (assert) { + let virtualNetwork = new VirtualNetwork(); + virtualNetwork.addURLMapping( + new URL('http://localhost:4205/test/'), + new URL('http://localhost:45123/test/'), + ); + + assert.strictEqual( + virtualNetwork.mapURL( + 'http://localhost:4205/test/hassan/personal/_readiness-check', + 'virtual-to-real', + )?.href, + 'http://localhost:45123/test/hassan/personal/_readiness-check', + ); + assert.strictEqual( + virtualNetwork.mapURL( + 'http://localhost:45123/test/hassan/personal/_readiness-check', + 'real-to-virtual', + )?.href, + 'http://localhost:4205/test/hassan/personal/_readiness-check', + ); + }); }); }); diff --git a/packages/realm-server/worker-manager.ts b/packages/realm-server/worker-manager.ts index 6e1d15f5c4b..0ae74fa2239 100644 --- a/packages/realm-server/worker-manager.ts +++ b/packages/realm-server/worker-manager.ts @@ -20,6 +20,8 @@ import { spawn, type ChildProcess } from 'child_process'; import pluralize from 'pluralize'; import Koa from 'koa'; import Router from '@koa/router'; +import { mkdirSync, renameSync, writeFileSync } from 'fs'; +import { dirname, join } from 'path'; import { ecsMetadata, fullRequestURL, livenessCheck } from './middleware'; import type { Server } from 'http'; import { PgAdapter } from '@cardstack/postgres'; @@ -43,6 +45,22 @@ import { */ let log = logger('worker-manager'); +const runtimeMetadataFile = + process.env.SOFTWARE_FACTORY_WORKER_MANAGER_METADATA_FILE; + +function writeRuntimeMetadata(payload: unknown): void { + if (!runtimeMetadataFile) { + return; + } + + mkdirSync(dirname(runtimeMetadataFile), { recursive: true }); + let tempFile = join( + dirname(runtimeMetadataFile), + `.worker-manager.${process.pid}.${Date.now()}.tmp`, + ); + writeFileSync(tempFile, JSON.stringify(payload, null, 2)); + renameSync(tempFile, runtimeMetadataFile); +} // This is an ENV var we get from ECS that looks like: // http://169.254.170.2/v3/a1de500d004f49bea02ace30cefb0f01-3236013547 where the @@ -232,6 +250,11 @@ if (port != null) { webServerInstance.on('listening', () => { let actualPort = (webServerInstance!.address() as import('net').AddressInfo).port ?? port; + writeRuntimeMetadata({ + pid: process.pid, + port: actualPort, + url: `http://127.0.0.1:${actualPort}`, + }); if (isEnvironmentMode()) { registerService(webServerInstance!, serviceName); } diff --git a/packages/runtime-common/virtual-network.ts b/packages/runtime-common/virtual-network.ts index 3073dc6adde..6254224cdbb 100644 --- a/packages/runtime-common/virtual-network.ts +++ b/packages/runtime-common/virtual-network.ts @@ -51,6 +51,17 @@ export class VirtualNetwork { this.urlMappings.push([from.href, to.href]); } + mapURL( + url: string | URL, + direction: 'virtual-to-real' | 'real-to-virtual', + ): URL | undefined { + let resolved = this.resolveURLMapping( + typeof url === 'string' ? url : url.href, + direction, + ); + return resolved ? new URL(resolved) : undefined; + } + addImportMap(prefix: string, handler: (rest: string) => string): void { this.importMap.set(prefix, handler); } diff --git a/packages/software-factory/README.md b/packages/software-factory/README.md index 2661103ee3d..6fb374928b8 100644 --- a/packages/software-factory/README.md +++ b/packages/software-factory/README.md @@ -2,9 +2,9 @@ Local card-development harness for fast Boxel iteration. -This package gives you a cached local realm fixture, a fixed-port isolated realm -server, and a Playwright loop that exercises cards in the real browser app -shell. +This package gives you a cached local realm fixture, an isolated realm server +with harness-managed ports, and a Playwright loop that exercises cards in the +real browser app shell. ## Prerequisites @@ -24,9 +24,11 @@ isolated realm server. By default it serves the test realm and base realm from the same fixed realm-server origin. The skills realm can be enabled when needed with `SOFTWARE_FACTORY_INCLUDE_SKILLS=1`. -For the software-factory Playwright flow, the isolated realm stack is intended -to be self-contained on `http://localhost:4205/`. The fixture realms and test -startup do not require a separate external realm server on `http://localhost:4201/`. +For the software-factory Playwright flow, the isolated realm stack is +self-contained and writes its actual runtime URLs and ports to harness metadata. +The fixture realms use the placeholder origin `https://sf.boxel.test/`, which +the harness rewrites to the live source-realm URL at startup. The Playwright +flow does not require a separate external realm server on `http://localhost:4201/`. ## Commands @@ -35,7 +37,7 @@ startup do not require a separate external realm server on `http://localhost:420 - `pnpm serve:support` - Starts shared support services and prepares a reusable runtime context in the background - `pnpm serve:realm` - - Starts the isolated realm server on `http://localhost:4205/test/` + - Starts the isolated realm server for `/test/` on a dynamically assigned realm-server URL - `pnpm smoke:realm` - Boots the isolated realm server, fetches `project-demo` as card JSON, and exits - `pnpm factory:go -- --brief-url --target-realm-url ` diff --git a/packages/software-factory/playwright.config.ts b/packages/software-factory/playwright.config.ts index bcdaf90189b..e57ef5cfa6b 100644 --- a/packages/software-factory/playwright.config.ts +++ b/packages/software-factory/playwright.config.ts @@ -10,8 +10,8 @@ export default defineConfig({ testMatch: ['**/*.spec.ts'], fullyParallel: false, reporter: process.env.CI ? [['list']] : undefined, - workers: 1, - timeout: 60_000, + workers: 2, + timeout: 300_000, expect: { timeout: 15_000, }, diff --git a/packages/software-factory/playwright.global-setup.ts b/packages/software-factory/playwright.global-setup.ts index c60f806f286..598678fc0f8 100644 --- a/packages/software-factory/playwright.global-setup.ts +++ b/packages/software-factory/playwright.global-setup.ts @@ -54,6 +54,33 @@ function prefixChunk(label: string, chunk: string): string { .join('\n'); } +function maybeLogCacheProgress( + log: ReturnType, + chunk: string, +): void { + let trimmed = chunk.replace(/\s+$/, ''); + if (!trimmed) { + return; + } + + for (let line of trimmed.split('\n')) { + if ( + /\b(begin visiting file|completed visiting file|starting from-scratch indexing|completed from scratch indexing|starting indexing|has completed indexing)\b/.test( + line, + ) + ) { + log.info(line); + continue; + } + + if ( + /encountered error indexing|Render timed-out|missing file /.test(line) + ) { + log.warn(line); + } + } +} + function mirrorChildOutput( child: ReturnType, log: ReturnType, @@ -217,6 +244,12 @@ export default async function globalSetup() { }, () => cacheLogs, ); + cacheChild.stdout?.on('data', (chunk) => { + maybeLogCacheProgress(cacheLog, String(chunk)); + }); + cacheChild.stderr?.on('data', (chunk) => { + maybeLogCacheProgress(cacheLog, String(chunk)); + }); let cacheStartedAt = Date.now(); await waitForCommand(cacheChild, () => cacheLogs); diff --git a/packages/software-factory/src/cli/cache-realm.ts b/packages/software-factory/src/cli/cache-realm.ts index 15644c1f3fc..1a46c634a49 100644 --- a/packages/software-factory/src/cli/cache-realm.ts +++ b/packages/software-factory/src/cli/cache-realm.ts @@ -39,6 +39,7 @@ async function main(): Promise { ); } console.log(JSON.stringify(payload, null, 2)); + process.exit(0); } main().catch((error: unknown) => { diff --git a/packages/software-factory/src/cli/serve-realm.ts b/packages/software-factory/src/cli/serve-realm.ts index 232059cf8b5..ef10e1a2b44 100644 --- a/packages/software-factory/src/cli/serve-realm.ts +++ b/packages/software-factory/src/cli/serve-realm.ts @@ -25,7 +25,9 @@ async function main(): Promise { let payload = { realmDir, realmURL: runtime.realmURL.href, + realmServerURL: runtime.realmServerURL.href, databaseName: runtime.databaseName, + ports: runtime.ports, sampleCardURL: runtime.cardURL('project-demo'), ownerBearerToken: runtime.createBearerToken(), }; @@ -40,6 +42,7 @@ async function main(): Promise { console.log(JSON.stringify(payload, null, 2)); let cleanExit = false; + let keepAlive = setInterval(() => {}, 60_000); process.on('exit', () => { if (!cleanExit) { for (let pid of runtime.childPids) { @@ -53,6 +56,7 @@ async function main(): Promise { }); let stop = async () => { + clearInterval(keepAlive); await runtime.stop(); cleanExit = true; process.exit(0); @@ -60,6 +64,10 @@ async function main(): Promise { process.on('SIGINT', () => void stop()); process.on('SIGTERM', () => void stop()); + + // Keep the harness process alive so its managed children stay attached until + // the test fixture explicitly shuts the stack down. + await new Promise(() => {}); } main().catch((error: unknown) => { diff --git a/packages/software-factory/src/cli/serve-support.ts b/packages/software-factory/src/cli/serve-support.ts index a3317c5c199..73a53962641 100644 --- a/packages/software-factory/src/cli/serve-support.ts +++ b/packages/software-factory/src/cli/serve-support.ts @@ -21,14 +21,20 @@ async function main(): Promise { writeSupportMetadata(payload); console.log(JSON.stringify(payload, null, 2)); + let keepAlive = setInterval(() => {}, 60_000); let stop = async () => { + clearInterval(keepAlive); await support.stop(); process.exit(0); }; process.on('SIGINT', () => void stop()); process.on('SIGTERM', () => void stop()); + + // Keep the wrapper alive so test teardown can signal it and so the shared + // support processes remain attached to this parent process. + await new Promise(() => {}); } main().catch((error: unknown) => { diff --git a/packages/software-factory/src/factory-bootstrap.ts b/packages/software-factory/src/factory-bootstrap.ts index eb98bfbb7da..e68d2e38113 100644 --- a/packages/software-factory/src/factory-bootstrap.ts +++ b/packages/software-factory/src/factory-bootstrap.ts @@ -485,6 +485,8 @@ async function createCardIfMissing( ); } + await waitForCardToBeReadable(realmUrl, cardPath, fetchImpl); + return { id: cardPath, status: 'created' }; } @@ -554,4 +556,43 @@ async function patchTicketStatus( `Failed to patch ticket status for ${ticketPath}: HTTP ${patchResponse.status} ${text}`.trim(), ); } + + await waitForCardToBeReadable(realmUrl, ticketPath, fetchImpl); +} + +async function waitForCardToBeReadable( + realmUrl: string, + cardPath: string, + fetchImpl: typeof globalThis.fetch, +): Promise { + let cardUrl = new URL(cardPath, realmUrl).href; + let timeoutMs = 15_000; + let retryDelayMs = 250; + let startedAt = Date.now(); + let lastError: string | undefined; + + while (Date.now() - startedAt < timeoutMs) { + try { + let response = await fetchImpl(cardUrl, { + method: 'GET', + headers: { Accept: cardSourceMimeType }, + }); + + if (response.ok) { + return; + } + + lastError = `HTTP ${response.status} ${await response.text()}`.trim(); + } catch (error) { + lastError = error instanceof Error ? error.message : String(error); + } + + await new Promise((resolve) => setTimeout(resolve, retryDelayMs)); + } + + throw new Error( + `Timed out waiting for card ${cardPath} in ${realmUrl} to become readable${ + lastError ? `: ${lastError}` : '' + }`, + ); } diff --git a/packages/software-factory/src/factory-entrypoint.ts b/packages/software-factory/src/factory-entrypoint.ts index 198d7f34181..d483796f776 100644 --- a/packages/software-factory/src/factory-entrypoint.ts +++ b/packages/software-factory/src/factory-entrypoint.ts @@ -2,7 +2,6 @@ import { parseArgs as parseNodeArgs } from 'node:util'; import { bootstrapProjectArtifacts, - inferDarkfactoryModuleUrl, type FactoryBootstrapOptions, type FactoryBootstrapResult, } from './factory-bootstrap'; @@ -191,14 +190,19 @@ export async function runFactoryEntrypoint( )(targetRealmResolution); let realmFetch = createBoxelRealmFetch(targetRealm.url, { + authorization: targetRealm.authorization, fetch: dependencies?.fetch, + primeRealmURL: targetRealm.url, }); let artifacts = await ( dependencies?.bootstrapArtifacts ?? bootstrapProjectArtifacts )(brief, targetRealm.url, { fetch: realmFetch, - darkfactoryModuleUrl: inferDarkfactoryModuleUrl(targetRealm.url), + darkfactoryModuleUrl: new URL( + 'software-factory/darkfactory', + targetRealm.serverUrl, + ).href, }); return buildFactoryEntrypointSummary(options, brief, targetRealm, artifacts); diff --git a/packages/software-factory/src/factory-target-realm.ts b/packages/software-factory/src/factory-target-realm.ts index 31a57837ccb..46179bd2322 100644 --- a/packages/software-factory/src/factory-target-realm.ts +++ b/packages/software-factory/src/factory-target-realm.ts @@ -4,6 +4,7 @@ import { ensureTrailingSlash } from '@cardstack/runtime-common/paths'; import { SupportedMimeType } from '@cardstack/runtime-common/router'; import { + getAccessibleRealmTokens, getActiveProfile, getRealmServerToken, matrixLogin, @@ -25,11 +26,13 @@ export interface FactoryTargetRealmResolution { export interface FactoryTargetRealmBootstrapResult extends FactoryTargetRealmResolution { createdRealm: boolean; + authorization: string; } interface CreateRealmResult { createdRealm: boolean; url: string; + authorization: string; } export interface FactoryTargetRealmBootstrapActions { @@ -37,6 +40,11 @@ export interface FactoryTargetRealmBootstrapActions { resolution: FactoryTargetRealmResolution, ) => Promise; fetch?: typeof globalThis.fetch; + waitForRealmReady?: ( + realmUrl: string, + authorization: string, + fetchImpl: typeof globalThis.fetch, + ) => Promise; } export function resolveFactoryTargetRealm( @@ -62,6 +70,7 @@ export async function bootstrapFactoryTargetRealm( ((targetRealm) => createRealm(targetRealm, { fetch: actions?.fetch, + waitForRealmReady: actions?.waitForRealmReady, })) )(resolution); @@ -69,12 +78,16 @@ export async function bootstrapFactoryTargetRealm( ...resolution, url: createRealmResult.url, createdRealm: createRealmResult.createdRealm, + authorization: createRealmResult.authorization, }; } async function createRealm( resolution: FactoryTargetRealmResolution, - dependencies?: { fetch?: typeof globalThis.fetch }, + dependencies?: { + fetch?: typeof globalThis.fetch; + waitForRealmReady?: FactoryTargetRealmBootstrapActions['waitForRealmReady']; + }, ): Promise { let fetchImpl = dependencies?.fetch ?? globalThis.fetch; @@ -129,19 +142,31 @@ async function createRealm( canonicalRealmUrl, fetchImpl, ); + let authorization = await getRealmAuthorization( + matrixAuth, + canonicalRealmUrl, + ); + await (dependencies?.waitForRealmReady ?? waitForRealmReady)( + canonicalRealmUrl, + authorization, + fetchImpl, + ); return { createdRealm: true, url: canonicalRealmUrl, + authorization, }; } let text = await response.text(); if (response.status === 400 && /already exists on this server/.test(text)) { + let authorization = await getRealmAuthorization(matrixAuth, resolution.url); return { createdRealm: false, url: resolution.url, + authorization, }; } @@ -191,6 +216,61 @@ async function appendRealmToMatrixAccountData( } } +async function waitForRealmReady( + realmUrl: string, + authorization: string, + fetchImpl: typeof globalThis.fetch, +): Promise { + let readinessUrl = new URL('_readiness-check', realmUrl).href; + let timeoutMs = 15_000; + let retryDelayMs = 250; + let startedAt = Date.now(); + let lastError: string | undefined; + + while (Date.now() - startedAt < timeoutMs) { + try { + let response = await fetchImpl(readinessUrl, { + headers: { + Accept: SupportedMimeType.RealmInfo, + Authorization: authorization, + }, + }); + + if (response.ok) { + return; + } + + lastError = `HTTP ${response.status} ${await response.text()}`.trim(); + } catch (error) { + lastError = error instanceof Error ? error.message : String(error); + } + + await new Promise((resolve) => setTimeout(resolve, retryDelayMs)); + } + + throw new Error( + `Timed out waiting for target realm ${realmUrl} to become ready${ + lastError ? `: ${lastError}` : '' + }`, + ); +} + +async function getRealmAuthorization( + matrixAuth: MatrixAuth, + realmUrl: string, +): Promise { + let realmTokens = await getAccessibleRealmTokens(matrixAuth); + let authorization = realmTokens[ensureTrailingSlash(realmUrl)]; + + if (!authorization) { + throw new Error( + `Realm auth lookup did not include ${ensureTrailingSlash(realmUrl)}`, + ); + } + + return authorization; +} + function resolveRealmServerProfile( ownerUsername: string, serverUrl: string, diff --git a/packages/software-factory/src/harness.ts b/packages/software-factory/src/harness.ts index 34741c161a6..b5dc79bc63b 100644 --- a/packages/software-factory/src/harness.ts +++ b/packages/software-factory/src/harness.ts @@ -11,7 +11,9 @@ import { readFileSync, rmSync, statSync, + writeFileSync, } from 'node:fs'; +import { createServer as createNetServer } from 'node:net'; import { tmpdir } from 'node:os'; import { join, relative, resolve } from 'node:path'; @@ -31,7 +33,6 @@ type RealmPermissions = Record; type FactorySupportContext = { matrixURL: string; matrixRegistrationSecret: string; - prerenderURL: string; }; type SynapseInstance = { @@ -43,6 +44,7 @@ type SynapseInstance = { export interface FactoryRealmOptions { realmDir?: string; realmURL?: URL; + realmServerURL?: URL; permissions?: RealmPermissions; useCache?: boolean; cacheSalt?: string; @@ -62,14 +64,21 @@ export interface FactoryTestContext extends FactorySupportContext { fixtureHash: string; realmDir: string; realmURL: string; + realmServerURL: string; templateDatabaseName: string; } export interface StartedFactoryRealm { realmDir: string; realmURL: URL; + realmServerURL: URL; databaseName: string; childPids: number[]; + ports: { + publicPort: number; + realmServerPort: number; + workerManagerPort: number; + }; cardURL(path: string): string; createBearerToken(user?: string, permissions?: RealmAction[]): string; authorizationHeaders( @@ -88,11 +97,24 @@ type SpawnedProcess = ChildProcess & { send(message: string): boolean; }; +type StartedCompatRealmProxy = { + listenPort: number; + setTargetPort(targetPort: number): void; + stop(): Promise; +}; + type RunningFactoryStack = { + prerender: { + stop(): Promise; + }; realmServer: SpawnedProcess; + realmServerURL: URL; workerManager: SpawnedProcess; - compatProxy?: { - stop(): Promise; + compatProxy?: StartedCompatRealmProxy; + ports: { + publicPort: number; + realmServerPort: number; + workerManagerPort: number; }; rootDir: string; }; @@ -100,6 +122,7 @@ type RunningFactoryStack = { const packageRoot = resolve(process.cwd()); const workspaceRoot = resolve(packageRoot, '..', '..'); const realmServerDir = resolve(packageRoot, '..', 'realm-server'); +const hostDir = resolve(packageRoot, '..', 'host'); const baseRealmDir = resolve(packageRoot, '..', 'base'); const skillsRealmDir = resolve(packageRoot, '..', 'skills-realm', 'contents'); const sourceRealmDir = resolve( @@ -114,23 +137,23 @@ const prepareTestPgScript = resolve( 'prepare-test-pg.sh', ); -const CACHE_VERSION = 7; -const REALM_SERVER_PORT = Number( - process.env.SOFTWARE_FACTORY_REALM_PORT ?? 4205, -); -const COMPAT_REALM_SERVER_PORT = Number( - process.env.SOFTWARE_FACTORY_COMPAT_REALM_PORT ?? 4201, +const CACHE_VERSION = 8; +const DEFAULT_REALM_SERVER_PORT = Number( + process.env.SOFTWARE_FACTORY_REALM_PORT ?? 0, ); -const WORKER_MANAGER_PORT = Number( - process.env.SOFTWARE_FACTORY_WORKER_MANAGER_PORT ?? 4232, +const DEFAULT_COMPAT_REALM_SERVER_PORT = Number( + process.env.SOFTWARE_FACTORY_COMPAT_REALM_PORT ?? 0, ); -const DEFAULT_REALM_URL = new URL( - process.env.SOFTWARE_FACTORY_REALM_URL ?? - `http://localhost:${REALM_SERVER_PORT}/test/`, -); -const LOCAL_SOFTWARE_FACTORY_SOURCE_URL = new URL( - `http://localhost:${REALM_SERVER_PORT}/software-factory/`, +const DEFAULT_WORKER_MANAGER_PORT = Number( + process.env.SOFTWARE_FACTORY_WORKER_MANAGER_PORT ?? 0, ); +const CONFIGURED_REALM_URL = process.env.SOFTWARE_FACTORY_REALM_URL + ? new URL(process.env.SOFTWARE_FACTORY_REALM_URL) + : undefined; +const CONFIGURED_REALM_SERVER_URL = process.env + .SOFTWARE_FACTORY_REALM_SERVER_URL + ? new URL(process.env.SOFTWARE_FACTORY_REALM_SERVER_URL) + : undefined; const DEFAULT_REALM_DIR = resolve( packageRoot, process.env.SOFTWARE_FACTORY_REALM_DIR ?? 'test-fixtures/darkfactory-adopter', @@ -144,6 +167,9 @@ const DEFAULT_ICONS_PROBE_URL = new URL( const DEFAULT_PG_PORT = process.env.SOFTWARE_FACTORY_PGPORT ?? '55436'; const DEFAULT_PG_HOST = process.env.SOFTWARE_FACTORY_PGHOST ?? '127.0.0.1'; const DEFAULT_PG_USER = process.env.SOFTWARE_FACTORY_PGUSER ?? 'postgres'; +const DEFAULT_PRERENDER_PORT = Number( + process.env.SOFTWARE_FACTORY_PRERENDER_PORT ?? 4231, +); const DEFAULT_MIGRATED_TEMPLATE_DB = process.env.SOFTWARE_FACTORY_MIGRATED_TEMPLATE_DB ?? 'boxel_migrated_template'; @@ -154,6 +180,7 @@ const DEFAULT_REALM_OWNER = '@software-factory-owner:localhost'; const REALM_SECRET_SEED = "shhh! it's a secret"; const REALM_SERVER_SECRET_SEED = "mum's the word"; const GRAFANA_SECRET = "shhh! it's a secret"; +const FIXTURE_SOURCE_REALM_URL_PLACEHOLDER = 'https://sf.boxel.test/'; const DEFAULT_MATRIX_SERVER_USERNAME = process.env.SOFTWARE_FACTORY_MATRIX_SERVER_USERNAME ?? 'realm_server'; const DEFAULT_MATRIX_BROWSER_USERNAME = @@ -168,6 +195,8 @@ const DEFAULT_SOURCE_REALM_PERMISSIONS: RealmPermissions = { '*': ['read'], [DEFAULT_REALM_OWNER]: ['read', 'write', 'realm-owner'], }; +const DEFAULT_BASE_REALM_PERMISSIONS: RealmPermissions = + DEFAULT_SOURCE_REALM_PERMISSIONS; const managedProcessStdio: StdioOptions = process.env.SOFTWARE_FACTORY_DEBUG_SERVER === '1' ? (['ignore', 'inherit', 'inherit', 'ipc'] as const) @@ -230,6 +259,121 @@ function hashString(value: string): string { return createHash('sha256').update(value).digest('hex'); } +async function findAvailablePort(): Promise { + return await new Promise((resolve, reject) => { + let server = createNetServer(); + server.once('error', reject); + server.listen(0, '127.0.0.1', () => { + let address = server.address(); + if (!address || typeof address === 'string') { + reject(new Error('Unable to determine allocated port')); + return; + } + server.close((error) => { + if (error) { + reject(error); + } else { + resolve(address.port); + } + }); + }); + }); +} + +async function resolveFactoryRealmServerURL( + realmServerURL?: URL, +): Promise { + if (realmServerURL) { + return new URL(realmServerURL.href); + } + + if (CONFIGURED_REALM_SERVER_URL) { + return new URL(CONFIGURED_REALM_SERVER_URL.href); + } + + let port = + DEFAULT_COMPAT_REALM_SERVER_PORT === 0 + ? await findAvailablePort() + : DEFAULT_COMPAT_REALM_SERVER_PORT; + return new URL(`http://localhost:${port}/`); +} + +async function resolveFactoryRealmLocation(options: { + realmURL?: URL; + realmServerURL?: URL; +}): Promise<{ + realmURL: URL; + realmServerURL: URL; +}> { + let realmURL = options.realmURL + ? new URL(options.realmURL.href) + : CONFIGURED_REALM_URL + ? new URL(CONFIGURED_REALM_URL.href) + : undefined; + let realmServerURL = options.realmServerURL + ? new URL(options.realmServerURL.href) + : CONFIGURED_REALM_SERVER_URL + ? new URL(CONFIGURED_REALM_SERVER_URL.href) + : undefined; + + if (!realmURL && !realmServerURL) { + realmServerURL = await resolveFactoryRealmServerURL(); + realmURL = new URL('test/', realmServerURL); + } else if (!realmServerURL) { + throw new Error( + 'An explicit realm server URL is required when a realm URL is provided. Set options.realmServerURL or SOFTWARE_FACTORY_REALM_SERVER_URL.', + ); + } else if (!realmURL) { + realmURL = new URL('test/', realmServerURL); + } + + return { + realmURL, + realmServerURL, + }; +} + +function baseRealmURLFor(realmServerURL: URL): URL { + return new URL('base/', realmServerURL); +} + +function skillsRealmURLFor(realmServerURL: URL): URL { + return new URL('skills/', realmServerURL); +} + +function sourceRealmURLFor(realmServerURL: URL): URL { + return new URL('software-factory/', realmServerURL); +} + +function withPort(url: URL, port: number): URL { + let next = new URL(url.href); + next.port = String(port); + return next; +} + +function realmRelativePath(realmURL: URL, realmServerURL: URL): string { + if (realmURL.origin !== realmServerURL.origin) { + throw new Error( + `Realm URL ${realmURL.href} does not share an origin with realm server URL ${realmServerURL.href}`, + ); + } + + let serverPath = realmServerURL.pathname.endsWith('/') + ? realmServerURL.pathname + : `${realmServerURL.pathname}/`; + if (!realmURL.pathname.startsWith(serverPath)) { + throw new Error( + `Realm URL ${realmURL.href} is not mounted under realm server URL ${realmServerURL.href}`, + ); + } + + return realmURL.pathname.slice(serverPath.length); +} + +function realmURLWithinServer(realmServerURL: URL, realmPath: string): URL { + return new URL(realmPath || '.', realmServerURL); +} + function shouldIgnoreFixturePath(relativePath: string): boolean { if (relativePath === '.DS_Store') { return true; @@ -329,6 +473,42 @@ async function waitUntil( throw new Error(options.timeoutMessage ?? 'Timed out waiting for condition'); } +async function waitForJsonFile( + file: string, + getLogs: () => string, + options: { + timeout?: number; + label: string; + process?: SpawnedProcess; + }, +): Promise { + let timeout = options.timeout ?? DEFAULT_REALM_STARTUP_TIMEOUT_MS; + let startedAt = Date.now(); + + while (Date.now() - startedAt < timeout) { + try { + return JSON.parse(readFileSync(file, 'utf8')) as T; + } catch (error) { + let nodeError = error as NodeJS.ErrnoException; + if (nodeError.code !== 'ENOENT' && !(error instanceof SyntaxError)) { + throw error; + } + } + + if (options.process && options.process.exitCode !== null) { + throw new Error( + `${options.label} exited early with code ${options.process.exitCode}\n${getLogs()}`, + ); + } + + await new Promise((resolve) => setTimeout(resolve, 100)); + } + + throw new Error( + `Timed out waiting for ${options.label} metadata in ${file}\n${getLogs()}`, + ); +} + async function canConnectToPg(): Promise { let client = new PgClient({ ...pgAdminConnectionConfig(), @@ -408,6 +588,42 @@ function maybeRequire(specifier: string) { return undefined; } +function fileExists(path: string): boolean { + try { + return statSync(path).isFile(); + } catch { + return false; + } +} + +function findHostDistPackageDir(): string | undefined { + let siblingRoot = resolve(workspaceRoot, '..'); + let candidates = [ + process.env.SOFTWARE_FACTORY_HOST_DIST_PACKAGE_DIR, + resolve(siblingRoot, 'boxel', 'packages', 'host'), + ...readdirSync(siblingRoot, { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .map((entry) => resolve(siblingRoot, entry.name, 'packages', 'host')), + hostDir, + ] + .filter((value): value is string => Boolean(value)) + .map((value) => resolve(value)); + + let seen = new Set(); + for (let candidate of candidates) { + if (seen.has(candidate)) { + continue; + } + seen.add(candidate); + + if (fileExists(join(candidate, 'dist', 'index.html'))) { + return candidate; + } + } + + return undefined; +} + async function loadSynapseModule() { let moduleSpecifier = '../../matrix/docker/synapse/index.ts'; return (maybeRequire(moduleSpecifier) ?? (await import(moduleSpecifier))) as { @@ -419,7 +635,10 @@ async function loadSynapseModule() { displayName?: string, ) => Promise; synapseStart: ( - opts?: { suppressRegistrationSecretFile?: true }, + opts?: { + suppressRegistrationSecretFile?: true; + dynamicHostPort?: true; + }, stopExisting?: boolean, ) => Promise; synapseStop: (id: string) => Promise; @@ -429,42 +648,220 @@ async function loadSynapseModule() { async function loadMatrixEnvironmentConfigModule() { let moduleSpecifier = '../../matrix/helpers/environment-config.ts'; return (maybeRequire(moduleSpecifier) ?? (await import(moduleSpecifier))) as { - getSynapseURL: () => string; + getSynapseURL: (synapse?: { baseUrl?: string; port?: number }) => string; }; } -async function loadIsolatedRealmServerModule() { - let moduleSpecifier = '../../matrix/helpers/isolated-realm-server.ts'; - return (maybeRequire(moduleSpecifier) ?? (await import(moduleSpecifier))) as { - startPrerenderServer: () => Promise<{ - url: string; - stop(): Promise; - }>; - }; -} - -async function ensureHostReady(): Promise { - await logTimed( +async function ensureHostReady(matrixURL: string): Promise<{ + stop?: () => Promise; +}> { + return await logTimed( supportLog, `ensureHostReady ${DEFAULT_HOST_URL}`, async () => { let response: Response; try { response = await fetch(DEFAULT_HOST_URL); + if (response.ok) { + return {}; + } } catch (error) { - throw new Error( - `Host app is not reachable at ${DEFAULT_HOST_URL}: ${ + supportLog.debug( + `host app not reachable at ${DEFAULT_HOST_URL}, starting fallback host service: ${ error instanceof Error ? error.message : String(error) }`, ); } - if (!response.ok) { - throw new Error( - `Host app is not ready at ${DEFAULT_HOST_URL}: status ${response.status}`, + + let hostPackageDir = findHostDistPackageDir(); + let command = ['start']; + let cwd = hostDir; + if (hostPackageDir) { + supportLog.debug(`serving built host dist from ${hostPackageDir}`); + command = ['serve:dist']; + cwd = hostPackageDir; + } else { + supportLog.warn( + 'no built host dist found; falling back to pnpm start in packages/host', ); } + + let child = spawn('pnpm', command, { + cwd, + detached: true, + stdio: ['ignore', 'pipe', 'pipe'], + env: { + ...process.env, + MATRIX_URL: matrixURL, + }, + }); + + let logs = ''; + child.stdout?.on('data', (chunk) => { + logs = `${logs}${String(chunk)}`.slice(-20_000); + }); + child.stderr?.on('data', (chunk) => { + logs = `${logs}${String(chunk)}`.slice(-20_000); + }); + + await waitUntil( + async () => { + if (child.exitCode !== null) { + throw new Error( + `host app exited early with code ${child.exitCode}\n${logs}`, + ); + } + try { + let readyResponse = await fetch(DEFAULT_HOST_URL); + return readyResponse.ok; + } catch { + return false; + } + }, + { + timeout: 180_000, + interval: 500, + timeoutMessage: `Timed out waiting for host app at ${DEFAULT_HOST_URL}\n${logs}`, + }, + ); + + return { + async stop() { + if (child.exitCode === null) { + try { + process.kill(-child.pid!, 'SIGTERM'); + } catch { + // best effort cleanup + } + } + }, + }; + }, + ); +} + +async function waitForHttpReady(url: string, timeoutMs = 60_000) { + let startedAt = Date.now(); + while (Date.now() - startedAt < timeoutMs) { + try { + let response = await fetch(url); + if (response.ok) { + return; + } + } catch { + // server not ready yet + } + await new Promise((resolve) => setTimeout(resolve, 200)); + } + throw new Error(`timed out waiting for ${url} to become ready`); +} + +async function stopChildProcess( + child: ChildProcess, + signal: NodeJS.Signals = 'SIGINT', +): Promise { + if (child.exitCode !== null || child.killed) { + return; + } + + await new Promise((resolve) => { + let settled = false; + let timeout: NodeJS.Timeout | undefined; + let cleanup = () => { + if (timeout) { + clearTimeout(timeout); + } + child.removeAllListeners('exit'); + child.removeAllListeners('error'); + }; + + child.once('exit', () => { + if (!settled) { + settled = true; + cleanup(); + resolve(); + } + }); + child.once('error', () => { + if (!settled) { + settled = true; + cleanup(); + resolve(); + } + }); + + timeout = setTimeout(() => { + if (!settled) { + child.kill('SIGTERM'); + } + }, 5_000); + + child.kill(signal); + }); +} + +async function startHarnessPrerenderServer(options: { + boxelHostURL: string; + port?: number; +}): Promise<{ + url: string; + stop(): Promise; +}> { + let port = options.port ?? DEFAULT_PRERENDER_PORT; + if (port === 0) { + port = await findAvailablePort(); + } + let url = `http://localhost:${port}`; + let silent = process.env.SOFTWARE_FACTORY_PRERENDER_SILENT !== '0'; + let child = spawn( + 'ts-node', + [ + '--transpileOnly', + 'prerender/prerender-server', + `--port=${port}`, + ...(silent ? ['--silent'] : []), + ], + { + cwd: realmServerDir, + stdio: ['pipe', 'pipe', 'pipe'], + env: { + ...process.env, + NODE_ENV: process.env.NODE_ENV ?? 'development', + NODE_NO_WARNINGS: '1', + BOXEL_HOST_URL: options.boxelHostURL, + LOG_LEVELS: + process.env.SOFTWARE_FACTORY_PRERENDER_LOG_LEVELS ?? + process.env.LOG_LEVELS, + }, }, ); + + child.stdout?.on('data', (data: Buffer) => { + console.log(`prerender: ${data.toString()}`); + }); + child.stderr?.on('data', (data: Buffer) => { + console.error(`prerender: ${data.toString()}`); + }); + + let exitPromise = new Promise((_, reject) => { + child.once('exit', (code, signal) => { + reject( + new Error( + `prerender server exited before it became ready (code: ${code}, signal: ${signal})`, + ), + ); + }); + child.once('error', reject); + }); + + await Promise.race([waitForHttpReady(url), exitPromise]); + + return { + url, + async stop() { + await stopChildProcess(child); + }, + }; } async function ensureIconsReady(): Promise<{ @@ -790,6 +1187,28 @@ async function resetMountedRealmState( ); } +async function resetQueueState(databaseName: string): Promise { + await logTimed(templateLog, `resetQueueState ${databaseName}`, async () => { + let client = new PgClient(pgAdminConnectionConfig(databaseName)); + try { + await client.connect(); + await client.query('BEGIN'); + await client.query(`DELETE FROM job_reservations`); + await client.query(`DELETE FROM jobs`); + await client.query('COMMIT'); + } catch (error) { + try { + await client.query('ROLLBACK'); + } catch { + // best effort cleanup + } + throw error; + } finally { + await client.end(); + } + }); +} + async function waitForQueueIdle(databaseName: string): Promise { await logTimed(templateLog, `waitForQueueIdle ${databaseName}`, async () => { await waitUntil( @@ -861,6 +1280,7 @@ function hasTemplateDatabaseName( function buildRealmToken( realmURL: URL, + realmServerURL: URL, user = DEFAULT_REALM_OWNER, permissions = DEFAULT_PERMISSIONS[DEFAULT_REALM_OWNER] ?? [ 'read', @@ -874,7 +1294,7 @@ function buildRealmToken( realm: realmURL.href, permissions, sessionRoom: `software-factory-session-room-for-${user}`, - realmServerURL: new URL(realmURL.origin).href, + realmServerURL: realmServerURL.href, }, REALM_SECRET_SEED, { expiresIn: '7d' }, @@ -985,30 +1405,51 @@ async function readIncomingRequestBody( return chunks.length ? Buffer.concat(chunks) : undefined; } +function describeCompatProxyError(error: unknown): string { + if (!(error instanceof Error)) { + return String(error); + } + + let parts: string[] = []; + let current: unknown = error; + + while (current) { + if (current instanceof Error) { + let code = + 'code' in current && typeof current.code === 'string' + ? ` (${current.code})` + : ''; + parts.push(`${current.message}${code}`); + current = current.cause; + } else { + parts.push(String(current)); + break; + } + } + + return parts.join(' <- '); +} + async function startCompatRealmProxy({ listenPort, - targetPort, }: { listenPort: number; - targetPort: number; -}): Promise< - | { - stop(): Promise; - } - | undefined -> { - if (listenPort === targetPort) { - return undefined; - } - - realmLog.debug( - `startCompatRealmProxy: ${listenPort} -> ${targetPort} starting`, - ); +}): Promise { + realmLog.debug(`startCompatRealmProxy: requested listenPort=${listenPort}`); + let targetPort: number | undefined; let server = createServer( async (req: IncomingMessage, res: ServerResponse) => { + if (targetPort == null) { + res.statusCode = 503; + res.setHeader('content-type', 'text/plain; charset=utf-8'); + res.end('software-factory compat proxy target is not ready'); + return; + } let incomingURL = new URL( req.url ?? '/', - `http://127.0.0.1:${listenPort}`, + `${ + req.headers['x-forwarded-proto'] === 'https' ? 'https' : 'http' + }://${req.headers.host ?? `127.0.0.1:${actualListenPort}`}`, ); let upstreamURL = new URL( `${incomingURL.pathname}${incomingURL.search}`, @@ -1022,10 +1463,14 @@ async function startCompatRealmProxy({ ([key]) => key.toLowerCase() !== 'host', ), ) as Record; + headers['x-boxel-forwarded-url'] = incomingURL.href; let response = await fetch(upstreamURL, { method: req.method, headers, body: body as BodyInit | undefined, + // Preserve upstream redirects so the client follows them against the + // public compat URL with a fresh forwarded URL header. + redirect: 'manual', }); let responseHeaders = new Headers(response.headers); @@ -1051,47 +1496,39 @@ async function startCompatRealmProxy({ }); res.end(Buffer.from(await response.arrayBuffer())); } catch (error) { + let description = describeCompatProxyError(error); + realmLog.warn( + `startCompatRealmProxy: upstream fetch failed for ${upstreamURL.href}: ${description}`, + ); res.statusCode = 502; res.setHeader('content-type', 'text/plain; charset=utf-8'); res.end( - `software-factory compat proxy failed for ${upstreamURL.href}: ${ - error instanceof Error ? error.message : String(error) - }`, + `software-factory compat proxy failed for ${upstreamURL.href}: ${description}`, ); } }, ); - - let started = false; - for (let attempt = 0; attempt < 20; attempt++) { - try { - await new Promise((resolve, reject) => { - server.once('error', reject); - server.listen(listenPort, '127.0.0.1', () => resolve()); - }); - started = true; - break; - } catch (error) { - let nodeError = error as NodeJS.ErrnoException; - if (nodeError.code !== 'EADDRINUSE') { - throw error; - } - await new Promise((resolve) => setTimeout(resolve, 100)); - } - } - - if (!started) { - realmLog.debug( - `startCompatRealmProxy: ${listenPort} -> ${targetPort} skipped (port unavailable)`, - ); - return undefined; + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(listenPort, '127.0.0.1', () => resolve()); + }); + let address = server.address(); + if (!address || typeof address === 'string') { + throw new Error('Unable to determine compat proxy port'); } - - realmLog.debug(`startCompatRealmProxy: ${listenPort} -> ${targetPort} ready`); + let actualListenPort = address.port; + realmLog.debug(`startCompatRealmProxy: listening on ${actualListenPort}`); return { + listenPort: actualListenPort, + setTargetPort(nextTargetPort: number) { + targetPort = nextTargetPort; + realmLog.debug( + `startCompatRealmProxy: ${actualListenPort} -> ${nextTargetPort} ready`, + ); + }, async stop() { realmLog.debug( - `startCompatRealmProxy: ${listenPort} -> ${targetPort} stopping`, + `startCompatRealmProxy: ${actualListenPort} -> ${targetPort ?? 'unset'} stopping`, ); await new Promise((resolve, reject) => { server.close((error) => { @@ -1106,7 +1543,51 @@ async function startCompatRealmProxy({ }; } -function copyRealmFixture(realmDir: string, destination: string): void { +function rewriteFixtureSourceModuleUrls( + destination: string, + sourceRealmURL: URL, +): void { + let rewrittenFiles = 0; + + function visit(currentDir: string) { + for (let entry of readdirSync(currentDir, { withFileTypes: true })) { + let absolutePath = join(currentDir, entry.name); + if (entry.isDirectory()) { + visit(absolutePath); + continue; + } + if (!entry.isFile() || !entry.name.endsWith('.json')) { + continue; + } + + let contents = readFileSync(absolutePath, 'utf8'); + if (!contents.includes(FIXTURE_SOURCE_REALM_URL_PLACEHOLDER)) { + continue; + } + + writeFileSync( + absolutePath, + contents + .split(FIXTURE_SOURCE_REALM_URL_PLACEHOLDER) + .join(sourceRealmURL.href), + ); + rewrittenFiles++; + } + } + + visit(destination); + if (rewrittenFiles > 0) { + realmLog.debug( + `rewriteFixtureSourceModuleUrls: rewrote ${rewrittenFiles} files to ${sourceRealmURL.href}`, + ); + } +} + +function copyRealmFixture( + realmDir: string, + destination: string, + sourceRealmURL: URL, +): void { copySync(realmDir, destination, { preserveTimestamps: true, filter(src) { @@ -1114,11 +1595,13 @@ function copyRealmFixture(realmDir: string, destination: string): void { return relativePath === '' || !shouldIgnoreFixturePath(relativePath); }, }); + rewriteFixtureSourceModuleUrls(destination, sourceRealmURL); } async function startIsolatedRealmStack({ realmDir, realmURL, + realmServerURL, databaseName, context, migrateDB, @@ -1126,6 +1609,7 @@ async function startIsolatedRealmStack({ }: { realmDir: string; realmURL: URL; + realmServerURL: URL; databaseName: string; context: FactorySupportContext; migrateDB: boolean; @@ -1137,11 +1621,45 @@ async function startIsolatedRealmStack({ async () => { let rootDir = mkdtempSync(join(tmpdir(), 'software-factory-realms-')); let testRealmDir = join(rootDir, 'test'); + let workerManagerMetadataFile = join( + rootDir, + 'worker-manager.runtime.json', + ); + let realmServerMetadataFile = join(rootDir, 'realm-server.runtime.json'); + let actualRealmServerPort = + DEFAULT_REALM_SERVER_PORT === 0 + ? await findAvailablePort() + : DEFAULT_REALM_SERVER_PORT; + let actualRealmServerURL = withPort( + realmServerURL, + actualRealmServerPort, + ); + let actualRealmPath = realmRelativePath(realmURL, realmServerURL); + let actualRealmURL = realmURLWithinServer( + actualRealmServerURL, + actualRealmPath, + ); + let legacyRealmServerURL = new URL('http://localhost:4205/'); + let legacyRealmURL = new URL('test/', legacyRealmServerURL); + let publicBaseRealmURL = baseRealmURLFor(realmServerURL); + let actualBaseRealmURL = baseRealmURLFor(actualRealmServerURL); + let sourceRealmURL = sourceRealmURLFor(realmServerURL); + let actualSourceRealmURL = sourceRealmURLFor(actualRealmServerURL); + let legacySourceRealmURL = sourceRealmURLFor(legacyRealmServerURL); + let skillsRealmURL = skillsRealmURLFor(realmServerURL); + let actualSkillsRealmURL = skillsRealmURLFor(actualRealmServerURL); + let legacySkillsRealmURL = skillsRealmURLFor(legacyRealmServerURL); ensureDirSync(testRealmDir); - copyRealmFixture(realmDir, testRealmDir); + copyRealmFixture(realmDir, testRealmDir, sourceRealmURL); realmLog.debug( `startIsolatedRealmStack: copied fixture ${realmDir} -> ${testRealmDir}`, ); + let compatProxy = await startCompatRealmProxy({ + listenPort: Number(realmServerURL.port), + }); + let prerender = await startHarnessPrerenderServer({ + boxelHostURL: realmServerURL.href.replace(/\/$/, ''), + }); let env = { ...process.env, @@ -1160,25 +1678,44 @@ async function startIsolatedRealmStack({ REALM_SERVER_FULL_INDEX_ON_STARTUP: String(fullIndexOnStartup), LOW_CREDIT_THRESHOLD: '2000', LOG_LEVELS: DEFAULT_REALM_LOG_LEVELS, - PUBLISHED_REALM_BOXEL_SPACE_DOMAIN: `localhost:${REALM_SERVER_PORT}`, - PUBLISHED_REALM_BOXEL_SITE_DOMAIN: `localhost:${REALM_SERVER_PORT}`, + PUBLISHED_REALM_BOXEL_SPACE_DOMAIN: `localhost:${compatProxy.listenPort}`, + PUBLISHED_REALM_BOXEL_SITE_DOMAIN: `localhost:${compatProxy.listenPort}`, + SOFTWARE_FACTORY_WORKER_MANAGER_METADATA_FILE: + workerManagerMetadataFile, + SOFTWARE_FACTORY_REALM_SERVER_METADATA_FILE: realmServerMetadataFile, }; let workerArgs = [ '--transpileOnly', 'worker-manager', - `--port=${WORKER_MANAGER_PORT}`, + `--port=${DEFAULT_WORKER_MANAGER_PORT}`, `--matrixURL=${context.matrixURL}`, - `--prerendererUrl=${context.prerenderURL}`, + `--prerendererUrl=${prerender.url}`, `--fromUrl=${realmURL.href}`, - `--toUrl=${realmURL.href}`, + `--toUrl=${actualRealmURL.href}`, + `--fromUrl=${publicBaseRealmURL.href}`, + `--toUrl=${actualBaseRealmURL.href}`, '--fromUrl=https://cardstack.com/base/', - `--toUrl=http://localhost:${REALM_SERVER_PORT}/base/`, + `--toUrl=${publicBaseRealmURL.href}`, + `--fromUrl=${sourceRealmURL.href}`, + `--toUrl=${actualSourceRealmURL.href}`, ]; if (INCLUDE_SKILLS) { workerArgs.push( - `--fromUrl=http://localhost:${REALM_SERVER_PORT}/skills/`, - `--toUrl=http://localhost:${REALM_SERVER_PORT}/skills/`, + `--fromUrl=${skillsRealmURL.href}`, + `--toUrl=${actualSkillsRealmURL.href}`, + ); + } + workerArgs.push( + `--fromUrl=${legacyRealmURL.href}`, + `--toUrl=${actualRealmURL.href}`, + `--fromUrl=${legacySourceRealmURL.href}`, + `--toUrl=${actualSourceRealmURL.href}`, + ); + if (INCLUDE_SKILLS) { + workerArgs.push( + `--fromUrl=${legacySkillsRealmURL.href}`, + `--toUrl=${actualSkillsRealmURL.href}`, ); } if (migrateDB) { @@ -1194,36 +1731,57 @@ async function startIsolatedRealmStack({ stdio: managedProcessStdio, }) as SpawnedProcess; let getWorkerLogs = captureProcessLogs(workerManager); + let workerManagerRuntime = await waitForJsonFile<{ + pid: number; + port: number; + url: string; + }>(workerManagerMetadataFile, getWorkerLogs, { + label: 'worker manager', + process: workerManager, + }); let serverArgs = [ '--transpileOnly', 'main', - `--port=${REALM_SERVER_PORT}`, + `--port=${actualRealmServerPort}`, + `--serverURL=${realmServerURL.href}`, `--matrixURL=${context.matrixURL}`, `--realmsRootPath=${rootDir}`, - `--workerManagerPort=${WORKER_MANAGER_PORT}`, - `--prerendererUrl=${context.prerenderURL}`, - `--path=${testRealmDir}`, - '--username=test_realm', - `--fromUrl=${realmURL.href}`, - `--toUrl=${realmURL.href}`, + `--workerManagerUrl=${workerManagerRuntime.url}`, + `--prerendererUrl=${prerender.url}`, '--username=base_realm', `--path=${baseRealmDir}`, - '--fromUrl=https://cardstack.com/base/', - `--toUrl=http://localhost:${REALM_SERVER_PORT}/base/`, + `--fromUrl=${publicBaseRealmURL.href}`, + `--toUrl=${actualBaseRealmURL.href}`, '--username=software_factory_realm', `--path=${sourceRealmDir}`, - `--fromUrl=${LOCAL_SOFTWARE_FACTORY_SOURCE_URL.href}`, - `--toUrl=${LOCAL_SOFTWARE_FACTORY_SOURCE_URL.href}`, + `--fromUrl=${sourceRealmURL.href}`, + `--toUrl=${actualSourceRealmURL.href}`, + '--username=test_realm', + `--path=${testRealmDir}`, + `--fromUrl=${realmURL.href}`, + `--toUrl=${actualRealmURL.href}`, ]; if (INCLUDE_SKILLS) { serverArgs.splice( - 11, + 16, 0, '--username=skills_realm', `--path=${skillsRealmDir}`, - `--fromUrl=http://localhost:${REALM_SERVER_PORT}/skills/`, - `--toUrl=http://localhost:${REALM_SERVER_PORT}/skills/`, + `--fromUrl=${skillsRealmURL.href}`, + `--toUrl=${actualSkillsRealmURL.href}`, + ); + } + serverArgs.push( + `--fromUrl=${legacyRealmURL.href}`, + `--toUrl=${actualRealmURL.href}`, + `--fromUrl=${legacySourceRealmURL.href}`, + `--toUrl=${actualSourceRealmURL.href}`, + ); + if (INCLUDE_SKILLS) { + serverArgs.push( + `--fromUrl=${legacySkillsRealmURL.href}`, + `--toUrl=${actualSkillsRealmURL.href}`, ); } @@ -1234,12 +1792,20 @@ async function startIsolatedRealmStack({ stdio: managedProcessStdio, }) as SpawnedProcess; let getServerLogs = captureProcessLogs(realmServer); - let compatProxy = await startCompatRealmProxy({ - listenPort: COMPAT_REALM_SERVER_PORT, - targetPort: REALM_SERVER_PORT, - }); + let realmServerRuntime: { + pid: number; + port: number; + }; try { + realmServerRuntime = await waitForJsonFile<{ + pid: number; + port: number; + }>(realmServerMetadataFile, getServerLogs, { + label: 'realm server', + process: realmServer, + }); + compatProxy.setTargetPort(realmServerRuntime.port); await Promise.race([ waitForReady( realmServer, @@ -1260,6 +1826,11 @@ async function startIsolatedRealmStack({ createProcessExitPromise(workerManager, 'worker manager'), ]); } catch (error) { + try { + await prerender.stop(); + } catch { + // best effort cleanup + } try { await stopManagedProcess(realmServer); } catch { @@ -1281,7 +1852,14 @@ async function startIsolatedRealmStack({ return { compatProxy, + prerender, realmServer, + realmServerURL, + ports: { + publicPort: compatProxy.listenPort, + realmServerPort: realmServerRuntime.port, + workerManagerPort: workerManagerRuntime.port, + }, workerManager, rootDir, }; @@ -1295,6 +1873,12 @@ async function stopIsolatedRealmStack( await logTimed(realmLog, 'stopIsolatedRealmStack', async () => { let cleanupError: unknown; + try { + await stack.prerender.stop(); + } catch (error) { + cleanupError ??= error; + } + try { await stopManagedProcess(stack.realmServer); } catch (error) { @@ -1328,6 +1912,7 @@ async function stopIsolatedRealmStack( async function buildTemplateDatabase({ realmDir, realmURL, + realmServerURL, permissions, context, cacheKey, @@ -1335,6 +1920,7 @@ async function buildTemplateDatabase({ }: { realmDir: string; realmURL: URL; + realmServerURL: URL; permissions: RealmPermissions; context: FactorySupportContext; cacheKey: string; @@ -1361,21 +1947,31 @@ async function buildTemplateDatabase({ builderDatabaseName, ); } + let baseRealmURL = baseRealmURLFor(realmServerURL); + let sourceRealmURL = sourceRealmURLFor(realmServerURL); await resetMountedRealmState(builderDatabaseName, [ realmURL, - LOCAL_SOFTWARE_FACTORY_SOURCE_URL, + baseRealmURL, + sourceRealmURL, ]); + await resetQueueState(builderDatabaseName); await seedRealmPermissions(builderDatabaseName, realmURL, permissions); await seedRealmPermissions( builderDatabaseName, - LOCAL_SOFTWARE_FACTORY_SOURCE_URL, + baseRealmURL, + DEFAULT_BASE_REALM_PERMISSIONS, + ); + await seedRealmPermissions( + builderDatabaseName, + sourceRealmURL, DEFAULT_SOURCE_REALM_PERMISSIONS, ); let stack = await startIsolatedRealmStack({ realmDir, realmURL, + realmServerURL, databaseName: builderDatabaseName, context, migrateDB: !hasMigratedTemplate, @@ -1400,39 +1996,33 @@ export async function startFactorySupportServices(): Promise<{ }> { return await logTimed(supportLog, 'startFactorySupportServices', async () => { await ensurePgReady(); - await ensureHostReady(); - let icons = await ensureIconsReady(); cleanupStaleSynapseContainers(); let { synapseStart, synapseStop } = await loadSynapseModule(); let { getSynapseURL } = await loadMatrixEnvironmentConfigModule(); - let { startPrerenderServer } = await loadIsolatedRealmServerModule(); let synapseStartedAt = Date.now(); let synapse = await synapseStart( - { suppressRegistrationSecretFile: true }, + { suppressRegistrationSecretFile: true, dynamicHostPort: true }, true, ); supportLog.debug( `synapse started in ${formatElapsedMs(Date.now() - synapseStartedAt)} on port ${synapse.port}`, ); + let matrixURL = + process.env.SOFTWARE_FACTORY_MATRIX_URL ?? getSynapseURL(synapse); + let host = await ensureHostReady(matrixURL); + let icons = await ensureIconsReady(); await ensureSupportUsers(synapse); - let prerenderStartedAt = Date.now(); - let prerender = await startPrerenderServer(); - supportLog.debug( - `prerender started in ${formatElapsedMs(Date.now() - prerenderStartedAt)} at ${prerender.url}`, - ); - let matrixURL = process.env.SOFTWARE_FACTORY_MATRIX_URL ?? getSynapseURL(); return { context: { matrixURL, matrixRegistrationSecret: synapse.registrationSecret, - prerenderURL: prerender.url, }, async stop() { await logTimed(supportLog, 'stopFactorySupportServices', async () => { - await prerender.stop(); await synapseStop(synapse.synapseId); + await host.stop?.(); await icons.stop?.(); }); }, @@ -1453,13 +2043,17 @@ export async function startFactoryGlobalContext( ): Promise { return await logTimed(harnessLog, 'startFactoryGlobalContext', async () => { let realmDir = resolve(options.realmDir ?? DEFAULT_REALM_DIR); - let realmURL = new URL((options.realmURL ?? DEFAULT_REALM_URL).href); + let { realmURL, realmServerURL } = await resolveFactoryRealmLocation({ + realmURL: options.realmURL, + realmServerURL: options.realmServerURL, + }); let support = await startFactorySupportServices(); try { let template = await ensureFactoryRealmTemplate({ ...options, realmDir, realmURL, + realmServerURL, context: support.context, }); @@ -1469,6 +2063,7 @@ export async function startFactoryGlobalContext( fixtureHash: template.fixtureHash, realmDir, realmURL: realmURL.href, + realmServerURL: realmServerURL.href, templateDatabaseName: template.templateDatabaseName, }; @@ -1488,7 +2083,18 @@ export async function ensureFactoryRealmTemplate( ): Promise { return await logTimed(harnessLog, 'ensureFactoryRealmTemplate', async () => { let realmDir = resolve(options.realmDir ?? DEFAULT_REALM_DIR); - let realmURL = new URL((options.realmURL ?? DEFAULT_REALM_URL).href); + let contextRealmURL = + options.context && hasTemplateDatabaseName(options.context) + ? new URL(options.context.realmURL) + : undefined; + let contextRealmServerURL = + options.context && hasTemplateDatabaseName(options.context) + ? new URL(options.context.realmServerURL) + : undefined; + let { realmURL, realmServerURL } = await resolveFactoryRealmLocation({ + realmURL: options.realmURL ?? contextRealmURL, + realmServerURL: options.realmServerURL ?? contextRealmServerURL, + }); let permissions = options.permissions ?? DEFAULT_PERMISSIONS; let fixtureHash = hashRealmFixture(realmDir); let cacheKey = hashString( @@ -1534,6 +2140,7 @@ export async function ensureFactoryRealmTemplate( await buildTemplateDatabase({ realmDir, realmURL, + realmServerURL, permissions, context, cacheKey, @@ -1557,17 +2164,30 @@ export async function startFactoryRealmServer( ): Promise { return await logTimed(harnessLog, 'startFactoryRealmServer', async () => { let realmDir = resolve(options.realmDir ?? DEFAULT_REALM_DIR); - let realmURL = new URL((options.realmURL ?? DEFAULT_REALM_URL).href); + let existingContext = options.context ?? parseFactoryContext(); + let contextRealmURL = + existingContext && hasTemplateDatabaseName(existingContext) + ? new URL(existingContext.realmURL) + : undefined; + let contextRealmServerURL = + existingContext && hasTemplateDatabaseName(existingContext) + ? new URL(existingContext.realmServerURL) + : undefined; + let { realmURL, realmServerURL } = await resolveFactoryRealmLocation({ + realmURL: options.realmURL ?? contextRealmURL, + realmServerURL: options.realmServerURL ?? contextRealmServerURL, + }); let templateDatabaseName = options.templateDatabaseName; let databaseName = runtimeDatabaseName(); let ownedGlobalContext: FactoryGlobalContextHandle | undefined; - let context = options.context ?? parseFactoryContext(); + let context = existingContext; if (!context) { ownedGlobalContext = await startFactoryGlobalContext({ ...options, realmDir, realmURL, + realmServerURL, }); context = ownedGlobalContext.context; } @@ -1580,6 +2200,7 @@ export async function startFactoryRealmServer( ...options, realmDir, realmURL, + realmServerURL, context, }) ).templateDatabaseName; @@ -1590,20 +2211,27 @@ export async function startFactoryRealmServer( ); let stack: RunningFactoryStack; try { + let baseRealmURL = baseRealmURLFor(realmServerURL); + let sourceRealmURL = sourceRealmURLFor(realmServerURL); await dropDatabase(databaseName); await cloneDatabaseFromTemplate(templateDatabaseName, databaseName); - await resetMountedRealmState(databaseName, [ - LOCAL_SOFTWARE_FACTORY_SOURCE_URL, - ]); + await resetQueueState(databaseName); + await seedRealmPermissions( + databaseName, + baseRealmURL, + DEFAULT_BASE_REALM_PERMISSIONS, + ); + await resetMountedRealmState(databaseName, [sourceRealmURL]); await seedRealmPermissions( databaseName, - LOCAL_SOFTWARE_FACTORY_SOURCE_URL, + sourceRealmURL, DEFAULT_SOURCE_REALM_PERMISSIONS, ); stack = await startIsolatedRealmStack({ realmDir, realmURL, + realmServerURL, databaseName, context, migrateDB: false, @@ -1634,7 +2262,9 @@ export async function startFactoryRealmServer( return { realmDir, realmURL, + realmServerURL, databaseName, + ports: stack.ports, childPids: [stack.realmServer.pid, stack.workerManager.pid].filter( (pid): pid is number => pid != null, ), @@ -1645,12 +2275,13 @@ export async function startFactoryRealmServer( user = DEFAULT_REALM_OWNER, permissions?: RealmAction[], ) { - return buildRealmToken(realmURL, user, permissions); + return buildRealmToken(realmURL, realmServerURL, user, permissions); }, authorizationHeaders(user?: string, permissions?: RealmAction[]) { return { Authorization: `Bearer ${buildRealmToken( realmURL, + realmServerURL, user, permissions, )}`, diff --git a/packages/software-factory/src/realm-auth.ts b/packages/software-factory/src/realm-auth.ts index 730e3e6e759..38f5d09e4c0 100644 --- a/packages/software-factory/src/realm-auth.ts +++ b/packages/software-factory/src/realm-auth.ts @@ -36,6 +36,7 @@ export interface CreateBoxelRealmFetchOptions { authorization?: string; fetch?: typeof globalThis.fetch; profile?: ActiveBoxelProfile | null; + primeRealmURL?: string; } export function createBoxelRealmFetch( @@ -72,8 +73,27 @@ export function createBoxelRealmFetch( matrixClient, () => fetchImpl, ); + let authedFetch = fetcher(fetchImpl, [ + authorizationMiddleware(realmAuthDataSource), + ]); + let primedRealmURL = normalizeOptionalString(options?.primeRealmURL); + let primeRequest = + primedRealmURL != null + ? realmAuthDataSource.reauthenticate( + ensureTrailingSlash( + normalizeProfileUrl(primedRealmURL, 'primeRealmURL'), + ), + ) + : undefined; + + if (!primeRequest) { + return authedFetch; + } - return fetcher(fetchImpl, [authorizationMiddleware(realmAuthDataSource)]); + return async (input, init) => { + await primeRequest; + return await authedFetch(input, init); + }; } function getOptionalActiveProfile( diff --git a/packages/software-factory/test-fixtures/darkfactory-adopter/AgentProfile/demo-agent.json b/packages/software-factory/test-fixtures/darkfactory-adopter/AgentProfile/demo-agent.json index 812e0513917..d5b6fb381c4 100644 --- a/packages/software-factory/test-fixtures/darkfactory-adopter/AgentProfile/demo-agent.json +++ b/packages/software-factory/test-fixtures/darkfactory-adopter/AgentProfile/demo-agent.json @@ -10,7 +10,7 @@ "relationships": {}, "meta": { "adoptsFrom": { - "module": "http://localhost:4205/software-factory/darkfactory", + "module": "https://sf.boxel.test/darkfactory", "name": "AgentProfile" } } diff --git a/packages/software-factory/test-fixtures/darkfactory-adopter/DarkFactory/demo-factory.json b/packages/software-factory/test-fixtures/darkfactory-adopter/DarkFactory/demo-factory.json index 6217d754df8..3c1387ee649 100644 --- a/packages/software-factory/test-fixtures/darkfactory-adopter/DarkFactory/demo-factory.json +++ b/packages/software-factory/test-fixtures/darkfactory-adopter/DarkFactory/demo-factory.json @@ -14,7 +14,7 @@ }, "meta": { "adoptsFrom": { - "module": "http://localhost:4205/software-factory/darkfactory", + "module": "https://sf.boxel.test/darkfactory", "name": "DarkFactory" } } diff --git a/packages/software-factory/test-fixtures/darkfactory-adopter/KnowledgeArticle/agent-onboarding.json b/packages/software-factory/test-fixtures/darkfactory-adopter/KnowledgeArticle/agent-onboarding.json index ae4f2f57769..e19bafa62c6 100644 --- a/packages/software-factory/test-fixtures/darkfactory-adopter/KnowledgeArticle/agent-onboarding.json +++ b/packages/software-factory/test-fixtures/darkfactory-adopter/KnowledgeArticle/agent-onboarding.json @@ -17,7 +17,7 @@ }, "meta": { "adoptsFrom": { - "module": "http://localhost:4205/software-factory/darkfactory", + "module": "https://sf.boxel.test/darkfactory", "name": "KnowledgeArticle" } } diff --git a/packages/software-factory/test-fixtures/darkfactory-adopter/Project/demo-project.json b/packages/software-factory/test-fixtures/darkfactory-adopter/Project/demo-project.json index 472c06bb653..364b804e916 100644 --- a/packages/software-factory/test-fixtures/darkfactory-adopter/Project/demo-project.json +++ b/packages/software-factory/test-fixtures/darkfactory-adopter/Project/demo-project.json @@ -25,7 +25,7 @@ }, "meta": { "adoptsFrom": { - "module": "http://localhost:4205/software-factory/darkfactory", + "module": "https://sf.boxel.test/darkfactory", "name": "Project" } } diff --git a/packages/software-factory/test-fixtures/darkfactory-adopter/Ticket/ticket-001.json b/packages/software-factory/test-fixtures/darkfactory-adopter/Ticket/ticket-001.json index ecda0994260..fdc8d303ea0 100644 --- a/packages/software-factory/test-fixtures/darkfactory-adopter/Ticket/ticket-001.json +++ b/packages/software-factory/test-fixtures/darkfactory-adopter/Ticket/ticket-001.json @@ -34,7 +34,7 @@ }, "meta": { "adoptsFrom": { - "module": "http://localhost:4205/software-factory/darkfactory", + "module": "https://sf.boxel.test/darkfactory", "name": "Ticket" } } diff --git a/packages/software-factory/test-fixtures/darkfactory-adopter/agent-demo.json b/packages/software-factory/test-fixtures/darkfactory-adopter/agent-demo.json index 812e0513917..d5b6fb381c4 100644 --- a/packages/software-factory/test-fixtures/darkfactory-adopter/agent-demo.json +++ b/packages/software-factory/test-fixtures/darkfactory-adopter/agent-demo.json @@ -10,7 +10,7 @@ "relationships": {}, "meta": { "adoptsFrom": { - "module": "http://localhost:4205/software-factory/darkfactory", + "module": "https://sf.boxel.test/darkfactory", "name": "AgentProfile" } } diff --git a/packages/software-factory/test-fixtures/darkfactory-adopter/factory-demo.json b/packages/software-factory/test-fixtures/darkfactory-adopter/factory-demo.json index 7f6bd587c20..6cdd653db1d 100644 --- a/packages/software-factory/test-fixtures/darkfactory-adopter/factory-demo.json +++ b/packages/software-factory/test-fixtures/darkfactory-adopter/factory-demo.json @@ -14,7 +14,7 @@ }, "meta": { "adoptsFrom": { - "module": "http://localhost:4205/software-factory/darkfactory", + "module": "https://sf.boxel.test/darkfactory", "name": "DarkFactory" } } diff --git a/packages/software-factory/test-fixtures/darkfactory-adopter/knowledge-article-demo.json b/packages/software-factory/test-fixtures/darkfactory-adopter/knowledge-article-demo.json index f628510d3ab..b074e3d66b9 100644 --- a/packages/software-factory/test-fixtures/darkfactory-adopter/knowledge-article-demo.json +++ b/packages/software-factory/test-fixtures/darkfactory-adopter/knowledge-article-demo.json @@ -17,7 +17,7 @@ }, "meta": { "adoptsFrom": { - "module": "http://localhost:4205/software-factory/darkfactory", + "module": "https://sf.boxel.test/darkfactory", "name": "KnowledgeArticle" } } diff --git a/packages/software-factory/test-fixtures/darkfactory-adopter/project-demo.json b/packages/software-factory/test-fixtures/darkfactory-adopter/project-demo.json index e459cc2be80..58cdd9963d3 100644 --- a/packages/software-factory/test-fixtures/darkfactory-adopter/project-demo.json +++ b/packages/software-factory/test-fixtures/darkfactory-adopter/project-demo.json @@ -25,7 +25,7 @@ }, "meta": { "adoptsFrom": { - "module": "http://localhost:4205/software-factory/darkfactory", + "module": "https://sf.boxel.test/darkfactory", "name": "Project" } } diff --git a/packages/software-factory/test-fixtures/darkfactory-adopter/ticket-demo.json b/packages/software-factory/test-fixtures/darkfactory-adopter/ticket-demo.json index a9933dfc78a..ba8c1c4efcf 100644 --- a/packages/software-factory/test-fixtures/darkfactory-adopter/ticket-demo.json +++ b/packages/software-factory/test-fixtures/darkfactory-adopter/ticket-demo.json @@ -34,7 +34,7 @@ }, "meta": { "adoptsFrom": { - "module": "http://localhost:4205/software-factory/darkfactory", + "module": "https://sf.boxel.test/darkfactory", "name": "Ticket" } } diff --git a/packages/software-factory/tests/darkfactory.spec.ts b/packages/software-factory/tests/darkfactory.spec.ts index 3e55f9143f3..b09016dca98 100644 --- a/packages/software-factory/tests/darkfactory.spec.ts +++ b/packages/software-factory/tests/darkfactory.spec.ts @@ -11,17 +11,22 @@ const adopterRealmDir = resolve( test.use({ realmDir: adopterRealmDir }); test.use({ realmServerMode: 'shared' }); +async function gotoCard( + page: Parameters[0]['authedPage'], + url: string, +) { + await page.goto(url, { waitUntil: 'commit' }); +} + test('renders a project adopted from the public DarkFactory module', async ({ authedPage, cardURL, }) => { - await authedPage.goto(cardURL('project-demo'), { - waitUntil: 'domcontentloaded', - }); + await gotoCard(authedPage, cardURL('project-demo')); await expect( authedPage.getByRole('heading', { name: 'DarkFactory Adoption Harness' }), - ).toBeVisible(); + ).toBeVisible({ timeout: 120_000 }); await expect( authedPage.getByRole('heading', { name: 'Objective' }), ).toBeVisible(); @@ -38,15 +43,13 @@ test('renders a ticket adopted from the public DarkFactory module', async ({ authedPage, cardURL, }) => { - await authedPage.goto(cardURL('ticket-demo'), { - waitUntil: 'domcontentloaded', - }); + await gotoCard(authedPage, cardURL('ticket-demo')); await expect( authedPage.getByRole('heading', { name: 'Verify public DarkFactory adoption', }), - ).toBeVisible(); + ).toBeVisible({ timeout: 120_000 }); await expect( authedPage.getByRole('heading', { name: 'Project' }), ).toBeVisible(); @@ -68,13 +71,11 @@ test('renders a knowledge article and agent profile adopted from the public Dark authedPage, cardURL, }) => { - await authedPage.goto(cardURL('knowledge-article-demo'), { - waitUntil: 'domcontentloaded', - }); + await gotoCard(authedPage, cardURL('knowledge-article-demo')); await expect( authedPage.getByRole('heading', { name: 'Agent Onboarding' }).first(), - ).toBeVisible(); + ).toBeVisible({ timeout: 120_000 }); await expect( authedPage.getByText('onboarding', { exact: true }).first(), ).toBeVisible(); @@ -84,13 +85,11 @@ test('renders a knowledge article and agent profile adopted from the public Dark ), ).toBeVisible(); - await authedPage.goto(cardURL('agent-demo'), { - waitUntil: 'domcontentloaded', - }); + await gotoCard(authedPage, cardURL('agent-demo')); await expect( authedPage.getByRole('heading', { name: 'codex-darkfactory' }), - ).toBeVisible(); + ).toBeVisible({ timeout: 120_000 }); await expect(authedPage.getByText('Boxel tracker workflows')).toBeVisible(); await expect(authedPage.getByText('ticket triage')).toBeVisible(); }); @@ -99,13 +98,11 @@ test('renders a DarkFactory card with active projects from the adopter realm', a authedPage, cardURL, }) => { - await authedPage.goto(cardURL('factory-demo'), { - waitUntil: 'domcontentloaded', - }); + await gotoCard(authedPage, cardURL('factory-demo')); await expect( authedPage.getByRole('heading', { name: 'DarkFactory Test Fixture' }), - ).toBeVisible(); + ).toBeVisible({ timeout: 120_000 }); await expect( authedPage.getByRole('heading', { name: 'Active Projects' }), ).toBeVisible(); diff --git a/packages/software-factory/tests/factory-bootstrap.spec.ts b/packages/software-factory/tests/factory-bootstrap.spec.ts index 42f3670dd3c..e580cb39d77 100644 --- a/packages/software-factory/tests/factory-bootstrap.spec.ts +++ b/packages/software-factory/tests/factory-bootstrap.spec.ts @@ -43,7 +43,10 @@ test.use({ realmServerMode: 'isolated' }); test('bootstrap creates actual card instances in a live realm', async ({ realm, }) => { - let darkfactoryModuleUrl = `${realm.realmURL.origin}/software-factory/darkfactory`; + let darkfactoryModuleUrl = new URL( + '../software-factory/darkfactory', + realm.realmURL, + ).href; let authenticatedFetch = buildAuthenticatedFetch( realm.ownerBearerToken, fetch, @@ -120,7 +123,10 @@ test('bootstrap creates actual card instances in a live realm', async ({ test('bootstrap is idempotent — rerun does not duplicate cards', async ({ realm, }) => { - let darkfactoryModuleUrl = `${realm.realmURL.origin}/software-factory/darkfactory`; + let darkfactoryModuleUrl = new URL( + '../software-factory/darkfactory', + realm.realmURL, + ).href; let authenticatedFetch = buildAuthenticatedFetch( realm.ownerBearerToken, fetch, @@ -152,7 +158,10 @@ test('bootstrapped project card renders correctly in the browser', async ({ realm, authedPage, }) => { - let darkfactoryModuleUrl = `${realm.realmURL.origin}/software-factory/darkfactory`; + let darkfactoryModuleUrl = new URL( + '../software-factory/darkfactory', + realm.realmURL, + ).href; let authenticatedFetch = buildAuthenticatedFetch( realm.ownerBearerToken, fetch, @@ -164,12 +173,12 @@ test('bootstrapped project card renders correctly in the browser', async ({ }); await authedPage.goto(realm.cardURL('Project/sticky-note-mvp'), { - waitUntil: 'domcontentloaded', + waitUntil: 'commit', }); await expect( authedPage.getByRole('heading', { name: 'Sticky Note MVP' }), - ).toBeVisible(); + ).toBeVisible({ timeout: 120_000 }); await expect( authedPage.getByRole('heading', { name: 'Objective' }), ).toBeVisible(); diff --git a/packages/software-factory/tests/factory-entrypoint.integration.test.ts b/packages/software-factory/tests/factory-entrypoint.integration.test.ts index 87220f4000f..07a8e5d961b 100644 --- a/packages/software-factory/tests/factory-entrypoint.integration.test.ts +++ b/packages/software-factory/tests/factory-entrypoint.integration.test.ts @@ -118,6 +118,21 @@ module('factory-entrypoint integration', function () { ) { response.writeHead(200, { 'content-type': 'application/json' }); response.end('{}'); + } else if (request.url === '/_realm-auth' && request.method === 'POST') { + response.writeHead(200, { 'content-type': 'application/json' }); + response.end( + JSON.stringify({ + [canonicalTargetRealmUrl]: 'Bearer target-realm-token', + }), + ); + } else if ( + request.url === '/hassan/personal/_readiness-check' && + request.method === 'GET' + ) { + response.writeHead(200, { + 'content-type': 'text/html', + }); + response.end(''); } else if ( request.url === '/hassan/personal/_session' && request.method === 'POST' diff --git a/packages/software-factory/tests/factory-entrypoint.test.ts b/packages/software-factory/tests/factory-entrypoint.test.ts index a5a9a4a7363..3465e59099e 100644 --- a/packages/software-factory/tests/factory-entrypoint.test.ts +++ b/packages/software-factory/tests/factory-entrypoint.test.ts @@ -227,4 +227,59 @@ module('factory-entrypoint', function (hooks) { ); assert.true(summary.brief.content.includes('structured drafting')); }); + + test('runFactoryEntrypoint uses the resolved realm server URL for darkfactory artifacts', async function (assert) { + process.env.MATRIX_USERNAME = 'hassan'; + + let capturedDarkfactoryModuleUrl: string | undefined; + + await runFactoryEntrypoint( + { + briefUrl, + targetRealmUrl, + realmServerUrl: 'https://realms.example.test/app/', + mode: 'implement', + }, + { + bootstrapTargetRealm: async (resolution) => ({ + ...bootstrappedTargetRealm, + url: resolution.url, + serverUrl: resolution.serverUrl, + createdRealm: false, + authorization: 'Bearer target-realm-token', + }), + bootstrapArtifacts: async (_brief, _targetRealmUrl, options) => { + capturedDarkfactoryModuleUrl = options?.darkfactoryModuleUrl; + return mockBootstrapResult; + }, + fetch: async () => + new Response( + JSON.stringify({ + data: { + attributes: { + content: 'Brief content', + cardInfo: { + name: 'Sticky Note', + summary: + 'Colorful, short-form note designed for spatial arrangement on boards and artboards.', + }, + tags: ['documents-content', 'sticky', 'note'], + }, + }, + }), + { + status: 200, + headers: { + 'content-type': 'application/json', + }, + }, + ), + }, + ); + + assert.strictEqual( + capturedDarkfactoryModuleUrl, + 'https://realms.example.test/app/software-factory/darkfactory', + ); + }); }); diff --git a/packages/software-factory/tests/factory-target-realm.spec.ts b/packages/software-factory/tests/factory-target-realm.spec.ts index 9707297c781..6f8459f4625 100644 --- a/packages/software-factory/tests/factory-target-realm.spec.ts +++ b/packages/software-factory/tests/factory-target-realm.spec.ts @@ -57,9 +57,12 @@ test('factory:go creates a target realm and bootstraps project artifacts end-to- let briefPort = (briefServer.address() as AddressInfo).port; let briefUrl = `http://127.0.0.1:${briefPort}/brief/sticky-note`; - let serverOrigin = realm.realmURL.origin; + let realmServerURL = realm.realmServerURL.href; let newEndpoint = `e2e-realm-${Date.now()}`; - let targetRealmUrl = `${serverOrigin}/${targetUsername}/${newEndpoint}/`; + let targetRealmUrl = new URL( + `${targetUsername}/${newEndpoint}/`, + realmServerURL, + ).href; try { let result = await runCommand( @@ -74,7 +77,7 @@ test('factory:go creates a target realm and bootstraps project artifacts end-to- '--target-realm-url', targetRealmUrl, '--realm-server-url', - `${serverOrigin}/`, + realmServerURL, '--mode', 'implement', ], @@ -85,7 +88,7 @@ test('factory:go creates a target realm and bootstraps project artifacts end-to- MATRIX_USERNAME: targetUsername, MATRIX_PASSWORD: targetPassword, MATRIX_URL: matrixURL, - REALM_SERVER_URL: `${serverOrigin}/`, + REALM_SERVER_URL: realmServerURL, }, timeoutMs: 120_000, }, diff --git a/packages/software-factory/tests/factory-target-realm.test.ts b/packages/software-factory/tests/factory-target-realm.test.ts index 76c038ab129..2a68cc6f9c3 100644 --- a/packages/software-factory/tests/factory-target-realm.test.ts +++ b/packages/software-factory/tests/factory-target-realm.test.ts @@ -96,12 +96,14 @@ module('factory-target-realm', function (hooks) { return { createdRealm: true, url: resolution.url, + authorization: 'Bearer target-realm-token', }; }, }); assert.strictEqual(createCalls, 1); assert.true(result.createdRealm); + assert.strictEqual(result.authorization, 'Bearer target-realm-token'); }); test('bootstrapFactoryTargetRealm reports when the realm already exists', async function (assert) { @@ -115,10 +117,12 @@ module('factory-target-realm', function (hooks) { createRealm: async () => ({ createdRealm: false, url: resolution.url, + authorization: 'Bearer target-realm-token', }), }); assert.false(result.createdRealm); + assert.strictEqual(result.authorization, 'Bearer target-realm-token'); }); test('bootstrapFactoryTargetRealm uses the canonical realm URL returned by create-realm', async function (assert) { @@ -132,6 +136,7 @@ module('factory-target-realm', function (hooks) { createRealm: async () => ({ createdRealm: true, url: 'https://realms.example.test/hassan/personal/', + authorization: 'Bearer target-realm-token', }), }); @@ -139,10 +144,11 @@ module('factory-target-realm', function (hooks) { result.url, 'https://realms.example.test/hassan/personal/', ); + assert.strictEqual(result.authorization, 'Bearer target-realm-token'); }); test('bootstrapFactoryTargetRealm sends the realm-server JWT to create-realm', async function (assert) { - assert.expect(11); + assert.expect(17); process.env.MATRIX_URL = 'https://matrix.example.test/'; process.env.MATRIX_USERNAME = 'hassan'; @@ -266,6 +272,40 @@ module('factory-target-realm', function (hooks) { 'Bearer matrix-access-token', ); response = new Response('{}', { status: 200 }); + } else if (request.url === 'https://realms.example.test/_realm-auth') { + assert.strictEqual( + request.headers.get('Authorization'), + 'Bearer realm-server-token', + ); + response = new Response( + JSON.stringify({ + [targetRealmUrl]: 'Bearer target-realm-token', + }), + { + status: 200, + headers: { + 'content-type': 'application/json', + }, + }, + ); + } else if ( + request.url === + 'https://realms.example.test/hassan/personal/_readiness-check' + ) { + assert.strictEqual( + request.headers.get('Authorization'), + 'Bearer target-realm-token', + ); + assert.strictEqual( + request.headers.get('Accept'), + 'application/vnd.api+json', + ); + response = new Response(null, { + status: 200, + headers: { + 'content-type': 'text/html', + }, + }); } else { throw new Error(`Unexpected url: ${request.method} ${request.url}`); } @@ -276,6 +316,7 @@ module('factory-target-realm', function (hooks) { let result = await bootstrapFactoryTargetRealm(resolution); assert.true(result.createdRealm); + assert.strictEqual(result.authorization, 'Bearer target-realm-token'); }); }); diff --git a/packages/software-factory/tests/fixtures.ts b/packages/software-factory/tests/fixtures.ts index 0ba54f74211..a671f947b93 100644 --- a/packages/software-factory/tests/fixtures.ts +++ b/packages/software-factory/tests/fixtures.ts @@ -13,7 +13,13 @@ import { buildBrowserState, installBrowserState } from './helpers/browser-auth'; type StartedFactoryRealm = { realmDir: string; realmURL: URL; + realmServerURL: URL; ownerBearerToken: string; + ports: { + publicPort: number; + realmServerPort: number; + workerManagerPort: number; + }; cardURL(path: string): string; authorizationHeaders(): Record; stop(): Promise; @@ -47,15 +53,6 @@ const defaultRealmDir = resolve( packageRoot, process.env.SOFTWARE_FACTORY_REALM_DIR ?? 'test-fixtures/darkfactory-adopter', ); -const realmPort = Number(process.env.SOFTWARE_FACTORY_REALM_PORT ?? 4205); -const compatPort = Number( - process.env.SOFTWARE_FACTORY_COMPAT_REALM_PORT ?? 4201, -); -const workerManagerPort = Number( - process.env.SOFTWARE_FACTORY_WORKER_MANAGER_PORT ?? 4232, -); -const localBasePrefix = `http://localhost:${realmPort}/base/`; -const localSkillsPrefix = `http://localhost:${realmPort}/skills/`; const testSourceRealmDir = resolve( packageRoot, 'test-fixtures/public-software-factory-source', @@ -117,7 +114,7 @@ async function waitForMetadataFile( metadataFile: string, child: ReturnType, getLogs: () => string, - timeoutMs = 120_000, + timeoutMs = 300_000, ): Promise { let startedAt = Date.now(); @@ -147,7 +144,6 @@ async function startRealmProcess(realmDir = defaultRealmDir) { let supportMetadata = existsSync(defaultSupportMetadataFile) ? (JSON.parse(readFileSync(defaultSupportMetadataFile, 'utf8')) as { context?: Record; - templateDatabaseName?: string; }) : undefined; @@ -167,12 +163,6 @@ async function startRealmProcess(realmDir = defaultRealmDir) { SOFTWARE_FACTORY_CONTEXT: JSON.stringify(supportMetadata.context), } : {}), - ...(supportMetadata?.templateDatabaseName - ? { - SOFTWARE_FACTORY_TEMPLATE_DATABASE_NAME: - supportMetadata.templateDatabaseName, - } - : {}), }, stdio: ['ignore', 'pipe', 'pipe'], }, @@ -188,6 +178,12 @@ async function startRealmProcess(realmDir = defaultRealmDir) { let metadata: { realmDir: string; realmURL: string; + realmServerURL: string; + ports: { + publicPort: number; + realmServerPort: number; + workerManagerPort: number; + }; sampleCardURL: string; ownerBearerToken: string; }; @@ -196,6 +192,12 @@ async function startRealmProcess(realmDir = defaultRealmDir) { metadata = await waitForMetadataFile<{ realmDir: string; realmURL: string; + realmServerURL: string; + ports: { + publicPort: number; + realmServerPort: number; + workerManagerPort: number; + }; sampleCardURL: string; ownerBearerToken: string; }>(metadataFile, child, () => logs); @@ -224,9 +226,9 @@ async function startRealmProcess(realmDir = defaultRealmDir) { }); } await Promise.all([ - waitForPortFree(realmPort), - waitForPortFree(compatPort), - waitForPortFree(workerManagerPort), + waitForPortFree(metadata.ports.realmServerPort), + waitForPortFree(metadata.ports.publicPort), + waitForPortFree(metadata.ports.workerManagerPort), ]); } finally { rmSync(tempDir, { recursive: true, force: true }); @@ -236,7 +238,9 @@ async function startRealmProcess(realmDir = defaultRealmDir) { return { realmDir: metadata.realmDir, realmURL: new URL(metadata.realmURL), + realmServerURL: new URL(metadata.realmServerURL), ownerBearerToken: metadata.ownerBearerToken, + ports: metadata.ports, cardURL(path: string) { return new URL(path, metadata.realmURL).href; }, @@ -290,44 +294,12 @@ async function releaseSharedRealm(key: string): Promise { } } -async function registerRealmRedirect( - page: Page, - fromPrefix: string, - toPrefix: string, -) { - await page.route(`${fromPrefix}**`, async (route) => { - let url = route.request().url(); - let suffix = url.slice(fromPrefix.length); - await route.continue({ url: `${toPrefix}${suffix}` }); - }); -} - -async function setRealmRedirects(page: Page) { - await registerRealmRedirect( - page, - 'http://localhost:4201/base/', - localBasePrefix, - ); - if (process.env.SOFTWARE_FACTORY_INCLUDE_SKILLS === '1') { - await registerRealmRedirect( - page, - 'http://localhost:4201/skills/', - localSkillsPrefix, - ); - } -} - export const test = base.extend< FactoryRealmFixtures & FactoryRealmOptions & FactoryRealmInternalFixtures >({ realmDir: [defaultRealmDir, { option: true }], realmServerMode: ['shared', { option: true }], - page: async ({ page }, use) => { - await setRealmRedirects(page); - await use(page); - }, - realm: async ( { browserName: _browserName, realmDir, realmServerMode }, use, @@ -360,12 +332,14 @@ export const test = base.extend< await use((path: string) => realm.cardURL(path)); }, - authedPage: async ({ browser, realmURL }, use) => { - let state = await buildBrowserState(realmURL.href); + authedPage: async ({ browser, realm }, use) => { + let state = await buildBrowserState( + realm.realmURL.href, + realm.realmServerURL.href, + ); let context = await browser.newContext(); await installBrowserState(context, state); let page = await context.newPage(); - await setRealmRedirects(page); try { await use(page); @@ -375,6 +349,6 @@ export const test = base.extend< }, }); -test.setTimeout(120_000); +test.setTimeout(300_000); export { expect }; diff --git a/packages/software-factory/tests/helpers/browser-auth.ts b/packages/software-factory/tests/helpers/browser-auth.ts index 1c6efadcc1b..e857c960d02 100644 --- a/packages/software-factory/tests/helpers/browser-auth.ts +++ b/packages/software-factory/tests/helpers/browser-auth.ts @@ -24,6 +24,7 @@ type BoxelProfile = { type SupportContext = { matrixURL?: string; + realmServerURL?: string; }; function getSupportMatrixURL(): string | undefined { @@ -31,6 +32,11 @@ function getSupportMatrixURL(): string | undefined { return context?.matrixURL; } +function getSupportRealmServerURL(): string | undefined { + let context = readSupportContext() as SupportContext | undefined; + return context?.realmServerURL; +} + const defaultMatrixUrl = ensureTrailingSlash( process.env.SOFTWARE_FACTORY_BROWSER_MATRIX_URL ?? getSupportMatrixURL() ?? @@ -191,8 +197,13 @@ async function getRealmAuthTokens( export async function buildBrowserState( realmURL: string, - realmServerURL = new URL('/', realmURL).href, + realmServerURL = getSupportRealmServerURL(), ): Promise { + if (!realmServerURL) { + throw new Error( + 'A realmServerURL is required to build browser state for software-factory tests', + ); + } let matrixAuth = await matrixLogin(); let realmTokens = await getRealmAuthTokens(matrixAuth, realmServerURL); @@ -219,16 +230,26 @@ export async function installBrowserState( state: FactoryBrowserState, ) { await target.addInitScript((payload: FactoryBrowserState) => { - window.localStorage.clear(); - window.localStorage.setItem('auth', JSON.stringify(payload.auth)); - window.localStorage.setItem( - 'boxel-session', - JSON.stringify(payload.boxelSession), - ); + try { + window.localStorage.clear(); + window.localStorage.setItem('auth', JSON.stringify(payload.auth)); + window.localStorage.setItem( + 'boxel-session', + JSON.stringify(payload.boxelSession), + ); + } catch { + // Init scripts also run on bootstrap documents where localStorage is not + // accessible yet (for example, the initial about:blank page). The script + // will run again for the actual realm page, where storage is available. + } }, state); } -export async function seedBrowserSession(page: Page, realmURL: string) { - let state = await buildBrowserState(realmURL); +export async function seedBrowserSession( + page: Page, + realmURL: string, + realmServerURL?: string, +) { + let state = await buildBrowserState(realmURL, realmServerURL); await installBrowserState(page, state); } From bf5e4c6fed7aa803ffbc0538c30070fd65b6c659 Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Tue, 24 Mar 2026 13:17:34 -0400 Subject: [PATCH 02/11] Stabilize software factory harness for parallel tests --- packages/postgres/pg-adapter.ts | 12 + packages/runtime-common/router.ts | 38 +- .../playwright.global-setup.ts | 124 +- .../software-factory/scripts/lib/boxel.ts | 10 +- .../software-factory/src/cli/cache-realm.ts | 2 + .../software-factory/src/cli/serve-realm.ts | 4 + packages/software-factory/src/error-format.ts | 64 + .../software-factory/src/factory-bootstrap.ts | 11 +- .../src/factory-target-realm.ts | 11 +- packages/software-factory/src/harness.ts | 2361 +---------------- packages/software-factory/src/harness/api.ts | 376 +++ .../software-factory/src/harness/database.ts | 636 +++++ .../src/harness/isolated-realm-stack.ts | 567 ++++ .../software-factory/src/harness/shared.ts | 752 ++++++ .../src/harness/support-services.ts | 418 +++ .../software-factory/src/runtime-metadata.ts | 13 + .../darkfactory.gts | 593 +---- .../tests/factory-bootstrap.test.ts | 67 +- .../factory-entrypoint.integration.test.ts | 31 +- .../tests/factory-target-realm.test.ts | 81 + packages/software-factory/tests/fixtures.ts | 46 +- 21 files changed, 3202 insertions(+), 3015 deletions(-) create mode 100644 packages/software-factory/src/error-format.ts create mode 100644 packages/software-factory/src/harness/api.ts create mode 100644 packages/software-factory/src/harness/database.ts create mode 100644 packages/software-factory/src/harness/isolated-realm-stack.ts create mode 100644 packages/software-factory/src/harness/shared.ts create mode 100644 packages/software-factory/src/harness/support-services.ts mode change 100644 => 120000 packages/software-factory/test-fixtures/public-software-factory-source/darkfactory.gts diff --git a/packages/postgres/pg-adapter.ts b/packages/postgres/pg-adapter.ts index 9fcaae8fe3f..c5c733818b3 100644 --- a/packages/postgres/pg-adapter.ts +++ b/packages/postgres/pg-adapter.ts @@ -31,6 +31,16 @@ function config() { type Config = ReturnType; +function configuredPoolMax(): number | undefined { + let rawValue = process.env.PG_POOL_MAX ?? process.env.PGPOOLMAX; + if (!rawValue) { + return undefined; + } + + let value = Number(rawValue); + return Number.isInteger(value) && value > 0 ? value : undefined; +} + export class PgAdapter implements DBAdapter { readonly kind = 'pg'; #isClosed = false; @@ -46,6 +56,7 @@ export class PgAdapter implements DBAdapter { } this.config = config(); let { user, host, database, password, port } = this.config; + let max = configuredPoolMax(); log.debug(`connecting to DB ${this.url}`); this.pool = new Pool({ user, @@ -53,6 +64,7 @@ export class PgAdapter implements DBAdapter { database, password, port, + ...(max ? { max } : {}), }); } diff --git a/packages/runtime-common/router.ts b/packages/runtime-common/router.ts index 8f4112a0ad6..1a0d632d7a8 100644 --- a/packages/runtime-common/router.ts +++ b/packages/runtime-common/router.ts @@ -16,6 +16,35 @@ type Handler = ( requestContext: RequestContext, ) => Promise; +function formatUnknownError(error: unknown): string { + if (error instanceof Error) { + return error.stack?.trim() || error.message; + } + + if ( + error === null || + error === undefined || + typeof error === 'string' || + typeof error === 'number' || + typeof error === 'boolean' || + typeof error === 'bigint' + ) { + return String(error); + } + + try { + let serialized = JSON.stringify(error); + if (serialized && serialized !== '{}') { + return serialized; + } + } catch { + // fall through to object tag + } + + let tag = Object.prototype.toString.call(error); + return tag === '[object Object]' ? 'non-Error object thrown' : tag; +} + export type Method = 'GET' | 'QUERY' | 'POST' | 'PATCH' | 'DELETE' | 'HEAD'; /* eslint-disable @typescript-eslint/no-duplicate-enum-values */ @@ -194,9 +223,12 @@ export class Router { this.log.error(err); - return new Response(`unexpected exception in realm ${err}`, { - status: 500, - }); + return new Response( + `unexpected exception in realm ${formatUnknownError(err)}`, + { + status: 500, + }, + ); } } diff --git a/packages/software-factory/playwright.global-setup.ts b/packages/software-factory/playwright.global-setup.ts index 598678fc0f8..cfb3405c76a 100644 --- a/packages/software-factory/playwright.global-setup.ts +++ b/packages/software-factory/playwright.global-setup.ts @@ -7,6 +7,7 @@ import { sharedRuntimeDir, writeSupportMetadata, getSupportMetadataFile, + type PreparedTemplateMetadata, } from './src/runtime-metadata'; const packageRoot = resolve(__dirname); @@ -23,6 +24,10 @@ const testSourceRealmDir = resolve( packageRoot, 'test-fixtures/public-software-factory-source', ); +const bootstrapTargetRealmDir = resolve( + packageRoot, + 'test-fixtures/bootstrap-target', +); const realmDir = existsSync(configuredRealmDir) ? configuredRealmDir : fallbackRealmDir; @@ -175,6 +180,64 @@ async function waitForMetadataFile( ); } +async function prepareTemplateForRealm( + realmDir: string, + context: Record, + metadataFile: string, +): Promise { + let cacheLogs = ''; + setupLog.warn( + `starting cache:prepare for ${realmDir}; this can take a while on cold startup or in CI`, + ); + let cacheChild = spawn('pnpm', ['cache:prepare', realmDir], { + cwd: packageRoot, + stdio: ['ignore', 'pipe', 'pipe'], + env: { + ...process.env, + SOFTWARE_FACTORY_CONTEXT: JSON.stringify(context), + SOFTWARE_FACTORY_METADATA_FILE: metadataFile, + SOFTWARE_FACTORY_SOURCE_REALM_DIR: testSourceRealmDir, + }, + }); + + mirrorChildOutput( + cacheChild, + cacheLog, + (next) => { + cacheLogs = next; + }, + () => cacheLogs, + ); + cacheChild.stdout?.on('data', (chunk) => { + maybeLogCacheProgress(cacheLog, String(chunk)); + }); + cacheChild.stderr?.on('data', (chunk) => { + maybeLogCacheProgress(cacheLog, String(chunk)); + }); + + let cacheStartedAt = Date.now(); + await waitForCommand(cacheChild, () => cacheLogs); + let cachePayload = await waitForMetadataFile<{ + realmDir: string; + templateDatabaseName: string; + realmURL: string; + realmServerURL: string; + }>(metadataFile, cacheChild, () => cacheLogs, 5_000); + setupLog.info( + `cache:prepare finished for ${realmDir} in ${( + (Date.now() - cacheStartedAt) / + 1000 + ).toFixed(1)}s`, + ); + + return { + realmDir: cachePayload.realmDir, + templateDatabaseName: cachePayload.templateDatabaseName, + templateRealmURL: cachePayload.realmURL, + templateRealmServerURL: cachePayload.realmServerURL, + }; +} + export default async function globalSetup() { let setupStartedAt = Date.now(); rmSync(sharedRuntimeDir, { recursive: true, force: true }); @@ -219,53 +282,28 @@ export default async function globalSetup() { )}s`, ); - let cacheLogs = ''; - let cacheMetadataFile = resolve(sharedRuntimeDir, 'cache.json'); - setupLog.warn( - 'starting cache:prepare; this can take a while on cold startup or in CI', - ); - setupLog.info(`starting cache:prepare for realm ${realmDir}`); - let cacheChild = spawn('pnpm', ['cache:prepare', realmDir], { - cwd: packageRoot, - stdio: ['ignore', 'pipe', 'pipe'], - env: { - ...process.env, - SOFTWARE_FACTORY_CONTEXT: JSON.stringify(payload.context), - SOFTWARE_FACTORY_METADATA_FILE: cacheMetadataFile, - SOFTWARE_FACTORY_SOURCE_REALM_DIR: testSourceRealmDir, - }, - }); - - mirrorChildOutput( - cacheChild, - cacheLog, - (next) => { - cacheLogs = next; - }, - () => cacheLogs, - ); - cacheChild.stdout?.on('data', (chunk) => { - maybeLogCacheProgress(cacheLog, String(chunk)); - }); - cacheChild.stderr?.on('data', (chunk) => { - maybeLogCacheProgress(cacheLog, String(chunk)); - }); - - let cacheStartedAt = Date.now(); - await waitForCommand(cacheChild, () => cacheLogs); - let cachePayload = await waitForMetadataFile<{ - templateDatabaseName: string; - }>(cacheMetadataFile, cacheChild, () => cacheLogs, 5_000); - setupLog.info( - `cache:prepare finished in ${((Date.now() - cacheStartedAt) / 1000).toFixed( - 1, - )}s`, - ); + let preparedRealmDirs = [...new Set([realmDir, bootstrapTargetRealmDir])]; + let preparedTemplates: PreparedTemplateMetadata[] = []; + for (let [index, preparedRealmDir] of preparedRealmDirs.entries()) { + preparedTemplates.push( + await prepareTemplateForRealm( + preparedRealmDir, + payload.context, + resolve(sharedRuntimeDir, `cache-${index}.json`), + ), + ); + } + let primaryTemplate = + preparedTemplates.find((entry) => entry.realmDir === realmDir) ?? + preparedTemplates[0]; writeSupportMetadata({ ...payload, pid: child.pid, - templateDatabaseName: cachePayload.templateDatabaseName, + templateDatabaseName: primaryTemplate?.templateDatabaseName, + templateRealmURL: primaryTemplate?.templateRealmURL, + templateRealmServerURL: primaryTemplate?.templateRealmServerURL, + preparedTemplates, }); setupLog.info( diff --git a/packages/software-factory/scripts/lib/boxel.ts b/packages/software-factory/scripts/lib/boxel.ts index e161f2ea760..c0f1b820d6d 100644 --- a/packages/software-factory/scripts/lib/boxel.ts +++ b/packages/software-factory/scripts/lib/boxel.ts @@ -2,6 +2,8 @@ import { existsSync, readFileSync } from 'node:fs'; import { homedir } from 'node:os'; import { join } from 'node:path'; +import { formatErrorResponse } from '../../src/error-format'; + const PROFILES_FILE = join(homedir(), '.boxel-cli', 'profiles.json'); type BoxelStoredProfile = { @@ -182,7 +184,7 @@ export async function getOpenIdToken( ); if (!response.ok) { - let text = await response.text(); + let text = await formatErrorResponse(response); throw new Error(`OpenID token request failed: ${response.status} ${text}`); } @@ -206,7 +208,7 @@ export async function getRealmServerToken( ); if (!response.ok) { - let text = await response.text(); + let text = await formatErrorResponse(response); throw new Error( `Realm server session request failed: ${response.status} ${text}`, ); @@ -238,7 +240,7 @@ export async function getAccessibleRealmTokens( ); if (!response.ok) { - let text = await response.text(); + let text = await formatErrorResponse(response); throw new Error(`Realm auth lookup failed: ${response.status} ${text}`); } @@ -291,7 +293,7 @@ export async function searchRealm(input: { ); if (!response.ok) { - let text = await response.text(); + let text = await formatErrorResponse(response); throw new Error(`Search failed: ${response.status} ${text}`); } diff --git a/packages/software-factory/src/cli/cache-realm.ts b/packages/software-factory/src/cli/cache-realm.ts index 1a46c634a49..02599091740 100644 --- a/packages/software-factory/src/cli/cache-realm.ts +++ b/packages/software-factory/src/cli/cache-realm.ts @@ -28,6 +28,8 @@ async function main(): Promise { templateDatabaseName: template.templateDatabaseName, fixtureHash: template.fixtureHash, cacheHit: template.cacheHit, + realmURL: template.realmURL.href, + realmServerURL: template.realmServerURL.href, }; if (process.env.SOFTWARE_FACTORY_METADATA_FILE) { mkdirSync(dirname(process.env.SOFTWARE_FACTORY_METADATA_FILE), { diff --git a/packages/software-factory/src/cli/serve-realm.ts b/packages/software-factory/src/cli/serve-realm.ts index ef10e1a2b44..87160a9788b 100644 --- a/packages/software-factory/src/cli/serve-realm.ts +++ b/packages/software-factory/src/cli/serve-realm.ts @@ -20,6 +20,10 @@ async function main(): Promise { let runtime = await startFactoryRealmServer({ realmDir, templateDatabaseName: process.env.SOFTWARE_FACTORY_TEMPLATE_DATABASE_NAME, + templateRealmServerURL: process.env + .SOFTWARE_FACTORY_TEMPLATE_REALM_SERVER_URL + ? new URL(process.env.SOFTWARE_FACTORY_TEMPLATE_REALM_SERVER_URL) + : undefined, }); let payload = { diff --git a/packages/software-factory/src/error-format.ts b/packages/software-factory/src/error-format.ts new file mode 100644 index 00000000000..ea6a58728c8 --- /dev/null +++ b/packages/software-factory/src/error-format.ts @@ -0,0 +1,64 @@ +export function formatUnknownError(error: unknown): string { + if (error instanceof Error) { + return error.stack?.trim() || error.message; + } + + if ( + error === null || + error === undefined || + typeof error === 'string' || + typeof error === 'number' || + typeof error === 'boolean' || + typeof error === 'bigint' + ) { + return String(error); + } + + try { + let serialized = JSON.stringify(error); + if (serialized && serialized !== '{}') { + return serialized; + } + } catch { + // fall through to object tag + } + + let tag = Object.prototype.toString.call(error); + return tag === '[object Object]' ? 'non-Error object thrown' : tag; +} + +export function formatErrorBody( + body: string, + contentType?: string | null, +): string { + let trimmed = body.trim(); + if (!trimmed) { + return ''; + } + + let looksJson = + contentType?.includes('json') || + trimmed.startsWith('{') || + trimmed.startsWith('['); + + if (looksJson) { + try { + return formatUnknownError(JSON.parse(trimmed)); + } catch { + // keep the original text when the body is malformed JSON + } + } + + if (trimmed === '[object Object]') { + return 'server returned a non-serialized object body'; + } + + return trimmed; +} + +export async function formatErrorResponse(response: Response): Promise { + return formatErrorBody( + await response.text(), + response.headers.get('content-type'), + ); +} diff --git a/packages/software-factory/src/factory-bootstrap.ts b/packages/software-factory/src/factory-bootstrap.ts index e68d2e38113..0c79394f97f 100644 --- a/packages/software-factory/src/factory-bootstrap.ts +++ b/packages/software-factory/src/factory-bootstrap.ts @@ -1,4 +1,5 @@ import type { FactoryBrief } from './factory-brief'; +import { formatErrorResponse, formatUnknownError } from './error-format'; const cardSourceMimeType = 'application/vnd.card+source'; @@ -479,7 +480,7 @@ async function createCardIfMissing( }); if (!writeResponse.ok) { - let text = await writeResponse.text(); + let text = await formatErrorResponse(writeResponse); throw new Error( `Failed to create card ${cardPath} in ${realmUrl}: HTTP ${writeResponse.status} ${text}`.trim(), ); @@ -551,7 +552,7 @@ async function patchTicketStatus( }); if (!patchResponse.ok) { - let text = await patchResponse.text(); + let text = await formatErrorResponse(patchResponse); throw new Error( `Failed to patch ticket status for ${ticketPath}: HTTP ${patchResponse.status} ${text}`.trim(), ); @@ -582,9 +583,11 @@ async function waitForCardToBeReadable( return; } - lastError = `HTTP ${response.status} ${await response.text()}`.trim(); + lastError = `HTTP ${response.status} ${await formatErrorResponse( + response, + )}`.trim(); } catch (error) { - lastError = error instanceof Error ? error.message : String(error); + lastError = formatUnknownError(error); } await new Promise((resolve) => setTimeout(resolve, retryDelayMs)); diff --git a/packages/software-factory/src/factory-target-realm.ts b/packages/software-factory/src/factory-target-realm.ts index 46179bd2322..a7a8e60c322 100644 --- a/packages/software-factory/src/factory-target-realm.ts +++ b/packages/software-factory/src/factory-target-realm.ts @@ -11,6 +11,7 @@ import { type ActiveBoxelProfile, type MatrixAuth, } from '../scripts/lib/boxel'; +import { formatErrorResponse, formatUnknownError } from './error-format'; import { FactoryEntrypointUsageError } from './factory-entrypoint-errors'; export interface ResolveFactoryTargetRealmOptions { @@ -159,7 +160,7 @@ async function createRealm( }; } - let text = await response.text(); + let text = await formatErrorResponse(response); if (response.status === 400 && /already exists on this server/.test(text)) { let authorization = await getRealmAuthorization(matrixAuth, resolution.url); @@ -209,7 +210,7 @@ async function appendRealmToMatrixAccountData( }); if (!putResponse.ok) { - let text = await putResponse.text(); + let text = await formatErrorResponse(putResponse); throw new Error( `Failed to update Matrix account data with realm ${realmUrl}: HTTP ${putResponse.status} ${text}`.trim(), ); @@ -240,9 +241,11 @@ async function waitForRealmReady( return; } - lastError = `HTTP ${response.status} ${await response.text()}`.trim(); + lastError = `HTTP ${response.status} ${await formatErrorResponse( + response, + )}`.trim(); } catch (error) { - lastError = error instanceof Error ? error.message : String(error); + lastError = formatUnknownError(error); } await new Promise((resolve) => setTimeout(resolve, retryDelayMs)); diff --git a/packages/software-factory/src/harness.ts b/packages/software-factory/src/harness.ts index b5dc79bc63b..8f966508937 100644 --- a/packages/software-factory/src/harness.ts +++ b/packages/software-factory/src/harness.ts @@ -1,2346 +1,15 @@ -import { spawn, spawnSync, type ChildProcess } from 'node:child_process'; -import { createHash } from 'node:crypto'; -import { - createServer, - type IncomingMessage, - type ServerResponse, -} from 'node:http'; -import { - mkdtempSync, - readdirSync, - readFileSync, - rmSync, - statSync, - writeFileSync, -} from 'node:fs'; -import { createServer as createNetServer } from 'node:net'; -import { tmpdir } from 'node:os'; -import { join, relative, resolve } from 'node:path'; - -import type { StdioOptions } from 'node:child_process'; - -import fsExtra from 'fs-extra'; -import jwt from 'jsonwebtoken'; -import { Client as PgClient } from 'pg'; -import './setup-logger'; -import { logger } from './logger'; - -type RealmAction = 'read' | 'write' | 'realm-owner' | 'assume-user'; -const { copySync, ensureDirSync } = fsExtra; - -type RealmPermissions = Record; - -type FactorySupportContext = { - matrixURL: string; - matrixRegistrationSecret: string; -}; - -type SynapseInstance = { - synapseId: string; - port: number; - registrationSecret: string; -}; - -export interface FactoryRealmOptions { - realmDir?: string; - realmURL?: URL; - realmServerURL?: URL; - permissions?: RealmPermissions; - useCache?: boolean; - cacheSalt?: string; - templateDatabaseName?: string; - context?: FactoryTestContext | FactorySupportContext; -} - -export interface FactoryRealmTemplate { - cacheKey: string; - templateDatabaseName: string; - fixtureHash: string; - cacheHit: boolean; -} - -export interface FactoryTestContext extends FactorySupportContext { - cacheKey: string; - fixtureHash: string; - realmDir: string; - realmURL: string; - realmServerURL: string; - templateDatabaseName: string; -} - -export interface StartedFactoryRealm { - realmDir: string; - realmURL: URL; - realmServerURL: URL; - databaseName: string; - childPids: number[]; - ports: { - publicPort: number; - realmServerPort: number; - workerManagerPort: number; - }; - cardURL(path: string): string; - createBearerToken(user?: string, permissions?: RealmAction[]): string; - authorizationHeaders( - user?: string, - permissions?: RealmAction[], - ): Record; - stop(): Promise; -} - -type FactoryGlobalContextHandle = { - context: FactoryTestContext; - stop(): Promise; -}; - -type SpawnedProcess = ChildProcess & { - send(message: string): boolean; -}; - -type StartedCompatRealmProxy = { - listenPort: number; - setTargetPort(targetPort: number): void; - stop(): Promise; -}; - -type RunningFactoryStack = { - prerender: { - stop(): Promise; - }; - realmServer: SpawnedProcess; - realmServerURL: URL; - workerManager: SpawnedProcess; - compatProxy?: StartedCompatRealmProxy; - ports: { - publicPort: number; - realmServerPort: number; - workerManagerPort: number; - }; - rootDir: string; -}; - -const packageRoot = resolve(process.cwd()); -const workspaceRoot = resolve(packageRoot, '..', '..'); -const realmServerDir = resolve(packageRoot, '..', 'realm-server'); -const hostDir = resolve(packageRoot, '..', 'host'); -const baseRealmDir = resolve(packageRoot, '..', 'base'); -const skillsRealmDir = resolve(packageRoot, '..', 'skills-realm', 'contents'); -const sourceRealmDir = resolve( - packageRoot, - process.env.SOFTWARE_FACTORY_SOURCE_REALM_DIR ?? 'realm', -); -const boxelIconsDir = resolve(packageRoot, '..', 'boxel-icons'); -const prepareTestPgScript = resolve( - realmServerDir, - 'tests', - 'scripts', - 'prepare-test-pg.sh', -); - -const CACHE_VERSION = 8; -const DEFAULT_REALM_SERVER_PORT = Number( - process.env.SOFTWARE_FACTORY_REALM_PORT ?? 0, -); -const DEFAULT_COMPAT_REALM_SERVER_PORT = Number( - process.env.SOFTWARE_FACTORY_COMPAT_REALM_PORT ?? 0, -); -const DEFAULT_WORKER_MANAGER_PORT = Number( - process.env.SOFTWARE_FACTORY_WORKER_MANAGER_PORT ?? 0, -); -const CONFIGURED_REALM_URL = process.env.SOFTWARE_FACTORY_REALM_URL - ? new URL(process.env.SOFTWARE_FACTORY_REALM_URL) - : undefined; -const CONFIGURED_REALM_SERVER_URL = process.env - .SOFTWARE_FACTORY_REALM_SERVER_URL - ? new URL(process.env.SOFTWARE_FACTORY_REALM_SERVER_URL) - : undefined; -const DEFAULT_REALM_DIR = resolve( - packageRoot, - process.env.SOFTWARE_FACTORY_REALM_DIR ?? 'test-fixtures/darkfactory-adopter', -); -const DEFAULT_HOST_URL = process.env.HOST_URL ?? 'http://localhost:4200/'; -const DEFAULT_ICONS_URL = process.env.ICONS_URL ?? 'http://localhost:4206/'; -const DEFAULT_ICONS_PROBE_URL = new URL( - '@cardstack/boxel-icons/v1/icons/code.js', - DEFAULT_ICONS_URL, -).href; -const DEFAULT_PG_PORT = process.env.SOFTWARE_FACTORY_PGPORT ?? '55436'; -const DEFAULT_PG_HOST = process.env.SOFTWARE_FACTORY_PGHOST ?? '127.0.0.1'; -const DEFAULT_PG_USER = process.env.SOFTWARE_FACTORY_PGUSER ?? 'postgres'; -const DEFAULT_PRERENDER_PORT = Number( - process.env.SOFTWARE_FACTORY_PRERENDER_PORT ?? 4231, -); -const DEFAULT_MIGRATED_TEMPLATE_DB = - process.env.SOFTWARE_FACTORY_MIGRATED_TEMPLATE_DB ?? - 'boxel_migrated_template'; -const DEFAULT_REALM_LOG_LEVELS = - process.env.SOFTWARE_FACTORY_REALM_LOG_LEVELS ?? - '*=info,realm:requests=warn,realm-index-updater=debug,index-runner=debug,index-perf=debug,index-writer=debug,worker=debug,worker-manager=debug,realm=debug,perf=debug'; -const DEFAULT_REALM_OWNER = '@software-factory-owner:localhost'; -const REALM_SECRET_SEED = "shhh! it's a secret"; -const REALM_SERVER_SECRET_SEED = "mum's the word"; -const GRAFANA_SECRET = "shhh! it's a secret"; -const FIXTURE_SOURCE_REALM_URL_PLACEHOLDER = 'https://sf.boxel.test/'; -const DEFAULT_MATRIX_SERVER_USERNAME = - process.env.SOFTWARE_FACTORY_MATRIX_SERVER_USERNAME ?? 'realm_server'; -const DEFAULT_MATRIX_BROWSER_USERNAME = - process.env.SOFTWARE_FACTORY_BROWSER_MATRIX_USERNAME ?? - 'software-factory-browser'; -const INCLUDE_SKILLS = process.env.SOFTWARE_FACTORY_INCLUDE_SKILLS === '1'; -const DEFAULT_PERMISSIONS: RealmPermissions = { - '*': ['read'], - [DEFAULT_REALM_OWNER]: ['read', 'write', 'realm-owner'], -}; -const DEFAULT_SOURCE_REALM_PERMISSIONS: RealmPermissions = { - '*': ['read'], - [DEFAULT_REALM_OWNER]: ['read', 'write', 'realm-owner'], -}; -const DEFAULT_BASE_REALM_PERMISSIONS: RealmPermissions = - DEFAULT_SOURCE_REALM_PERMISSIONS; -const managedProcessStdio: StdioOptions = - process.env.SOFTWARE_FACTORY_DEBUG_SERVER === '1' - ? (['ignore', 'inherit', 'inherit', 'ipc'] as const) - : (['ignore', 'pipe', 'pipe', 'ipc'] as const); -const DEFAULT_REALM_STARTUP_TIMEOUT_MS = Number( - process.env.SOFTWARE_FACTORY_REALM_STARTUP_TIMEOUT_MS ?? 120_000, -); -const FULL_INDEX_REALM_STARTUP_TIMEOUT_MS = Number( - process.env.SOFTWARE_FACTORY_FULL_INDEX_REALM_STARTUP_TIMEOUT_MS ?? 600_000, -); - -let preparePgPromise: Promise | undefined; -const harnessLog = logger('software-factory:harness'); -const supportLog = logger('software-factory:harness:support'); -const templateLog = logger('software-factory:harness:template'); -const realmLog = logger('software-factory:harness:realm'); - -function formatElapsedMs(elapsedMs: number): string { - return `${(elapsedMs / 1000).toFixed(1)}s`; -} - -async function logTimed( - log: ReturnType, - label: string, - callback: () => Promise, -): Promise { - let startedAt = Date.now(); - log.debug(`${label}: starting`); - try { - let result = await callback(); - log.debug( - `${label}: finished in ${formatElapsedMs(Date.now() - startedAt)}`, - ); - return result; - } catch (error) { - log.warn( - `${label}: failed after ${formatElapsedMs(Date.now() - startedAt)}: ${ - error instanceof Error ? error.message : String(error) - }`, - ); - throw error; - } -} - -function stableStringify(value: unknown): string { - if (value === null || typeof value !== 'object') { - return JSON.stringify(value); - } - if (Array.isArray(value)) { - return `[${value.map((item) => stableStringify(item)).join(',')}]`; - } - let record = value as Record; - let keys = Object.keys(record).sort(); - return `{${keys - .map((key) => `${JSON.stringify(key)}:${stableStringify(record[key])}`) - .join(',')}}`; -} - -function hashString(value: string): string { - return createHash('sha256').update(value).digest('hex'); -} - -async function findAvailablePort(): Promise { - return await new Promise((resolve, reject) => { - let server = createNetServer(); - server.once('error', reject); - server.listen(0, '127.0.0.1', () => { - let address = server.address(); - if (!address || typeof address === 'string') { - reject(new Error('Unable to determine allocated port')); - return; - } - server.close((error) => { - if (error) { - reject(error); - } else { - resolve(address.port); - } - }); - }); - }); -} - -async function resolveFactoryRealmServerURL( - realmServerURL?: URL, -): Promise { - if (realmServerURL) { - return new URL(realmServerURL.href); - } - - if (CONFIGURED_REALM_SERVER_URL) { - return new URL(CONFIGURED_REALM_SERVER_URL.href); - } - - let port = - DEFAULT_COMPAT_REALM_SERVER_PORT === 0 - ? await findAvailablePort() - : DEFAULT_COMPAT_REALM_SERVER_PORT; - return new URL(`http://localhost:${port}/`); -} - -async function resolveFactoryRealmLocation(options: { - realmURL?: URL; - realmServerURL?: URL; -}): Promise<{ - realmURL: URL; - realmServerURL: URL; -}> { - let realmURL = options.realmURL - ? new URL(options.realmURL.href) - : CONFIGURED_REALM_URL - ? new URL(CONFIGURED_REALM_URL.href) - : undefined; - let realmServerURL = options.realmServerURL - ? new URL(options.realmServerURL.href) - : CONFIGURED_REALM_SERVER_URL - ? new URL(CONFIGURED_REALM_SERVER_URL.href) - : undefined; - - if (!realmURL && !realmServerURL) { - realmServerURL = await resolveFactoryRealmServerURL(); - realmURL = new URL('test/', realmServerURL); - } else if (!realmServerURL) { - throw new Error( - 'An explicit realm server URL is required when a realm URL is provided. Set options.realmServerURL or SOFTWARE_FACTORY_REALM_SERVER_URL.', - ); - } else if (!realmURL) { - realmURL = new URL('test/', realmServerURL); - } - - return { - realmURL, - realmServerURL, - }; -} - -function baseRealmURLFor(realmServerURL: URL): URL { - return new URL('base/', realmServerURL); -} - -function skillsRealmURLFor(realmServerURL: URL): URL { - return new URL('skills/', realmServerURL); -} - -function sourceRealmURLFor(realmServerURL: URL): URL { - return new URL('software-factory/', realmServerURL); -} - -function withPort(url: URL, port: number): URL { - let next = new URL(url.href); - next.port = String(port); - return next; -} - -function realmRelativePath(realmURL: URL, realmServerURL: URL): string { - if (realmURL.origin !== realmServerURL.origin) { - throw new Error( - `Realm URL ${realmURL.href} does not share an origin with realm server URL ${realmServerURL.href}`, - ); - } - - let serverPath = realmServerURL.pathname.endsWith('/') - ? realmServerURL.pathname - : `${realmServerURL.pathname}/`; - if (!realmURL.pathname.startsWith(serverPath)) { - throw new Error( - `Realm URL ${realmURL.href} is not mounted under realm server URL ${realmServerURL.href}`, - ); - } - - return realmURL.pathname.slice(serverPath.length); -} - -function realmURLWithinServer(realmServerURL: URL, realmPath: string): URL { - return new URL(realmPath || '.', realmServerURL); -} - -function shouldIgnoreFixturePath(relativePath: string): boolean { - if (relativePath === '.DS_Store') { - return true; - } - return relativePath - .split('/') - .some((segment) => - [ - 'node_modules', - '.git', - '.boxel-history', - 'playwright-report', - 'test-results', - ].includes(segment), - ); -} - -function hashRealmFixture(realmDir: string): string { - let entries: string[] = []; - - function visit(currentDir: string) { - for (let entry of readdirSync(currentDir, { withFileTypes: true })) { - let absolutePath = join(currentDir, entry.name); - let relativePath = relative(realmDir, absolutePath).replace(/\\/g, '/'); - if (shouldIgnoreFixturePath(relativePath)) { - continue; - } - if (entry.isDirectory()) { - visit(absolutePath); - continue; - } - if (!entry.isFile()) { - continue; - } - let stats = statSync(absolutePath); - let contentsHash = createHash('sha256') - .update(readFileSync(absolutePath)) - .digest('hex'); - entries.push(`${relativePath}:${stats.size}:${contentsHash}`); - } - } - - visit(realmDir); - entries.sort(); - return hashString(entries.join('|')); -} - -function templateDatabaseNameForCacheKey(cacheKey: string): string { - return `sf_tpl_${cacheKey.slice(0, 24)}`; -} - -function builderDatabaseNameForCacheKey(cacheKey: string): string { - return `sf_bld_${cacheKey.slice(0, 16)}`; -} - -function runtimeDatabaseName(): string { - return `sf_run_${Date.now().toString(36)}_${Math.random() - .toString(36) - .slice(2, 8)}`; -} - -function pgAdminConnectionConfig(database = 'postgres') { - return { - host: DEFAULT_PG_HOST, - port: Number(DEFAULT_PG_PORT), - user: DEFAULT_PG_USER, - password: process.env.PGPASSWORD || undefined, - database, - }; -} - -function quotePgIdentifier(identifier: string): string { - if (!/^[a-zA-Z0-9_]+$/.test(identifier)) { - throw new Error(`unsafe postgres identifier: ${identifier}`); - } - return `"${identifier}"`; -} - -async function waitUntil( - condition: () => Promise, - options: { - timeout?: number; - interval?: number; - timeoutMessage?: string; - } = {}, -): Promise { - let timeout = options.timeout ?? 30_000; - let interval = options.interval ?? 250; - let start = Date.now(); - while (Date.now() - start < timeout) { - let result = await condition(); - if (result) { - return result; - } - await new Promise((resolve) => setTimeout(resolve, interval)); - } - throw new Error(options.timeoutMessage ?? 'Timed out waiting for condition'); -} - -async function waitForJsonFile( - file: string, - getLogs: () => string, - options: { - timeout?: number; - label: string; - process?: SpawnedProcess; - }, -): Promise { - let timeout = options.timeout ?? DEFAULT_REALM_STARTUP_TIMEOUT_MS; - let startedAt = Date.now(); - - while (Date.now() - startedAt < timeout) { - try { - return JSON.parse(readFileSync(file, 'utf8')) as T; - } catch (error) { - let nodeError = error as NodeJS.ErrnoException; - if (nodeError.code !== 'ENOENT' && !(error instanceof SyntaxError)) { - throw error; - } - } - - if (options.process && options.process.exitCode !== null) { - throw new Error( - `${options.label} exited early with code ${options.process.exitCode}\n${getLogs()}`, - ); - } - - await new Promise((resolve) => setTimeout(resolve, 100)); - } - - throw new Error( - `Timed out waiting for ${options.label} metadata in ${file}\n${getLogs()}`, - ); -} - -async function canConnectToPg(): Promise { - let client = new PgClient({ - ...pgAdminConnectionConfig(), - connectionTimeoutMillis: 1000, - }); - try { - await client.connect(); - await client.query('SELECT 1'); - return true; - } catch { - return false; - } finally { - try { - await client.end(); - } catch { - // best effort cleanup - } - } -} - -function runCommand(command: string, args: string[], cwd: string) { - let result = spawnSync(command, args, { - cwd, - stdio: 'inherit', - env: { - ...process.env, - PGHOST: DEFAULT_PG_HOST, - PGPORT: DEFAULT_PG_PORT, - PGUSER: DEFAULT_PG_USER, - }, - }); - if (result.status !== 0) { - throw new Error(`command failed: ${command} ${args.join(' ')}`); - } -} - -function cleanupStaleSynapseContainers() { - let result = spawnSync( - 'docker', - [ - 'ps', - '-aq', - '--filter', - 'name=synapsedocker-', - '--filter', - 'name=boxel-synapse', - ], - { - cwd: workspaceRoot, - encoding: 'utf8', - }, - ); - - if (result.status !== 0) { - return; - } - - let containerIds = result.stdout - .split(/\s+/) - .map((id) => id.trim()) - .filter(Boolean); - - if (containerIds.length === 0) { - return; - } - - spawnSync('docker', ['rm', '-f', ...containerIds], { - cwd: workspaceRoot, - stdio: 'ignore', - }); -} - -function maybeRequire(specifier: string) { - if (typeof require === 'function') { - return require(specifier); - } - return undefined; -} - -function fileExists(path: string): boolean { - try { - return statSync(path).isFile(); - } catch { - return false; - } -} - -function findHostDistPackageDir(): string | undefined { - let siblingRoot = resolve(workspaceRoot, '..'); - let candidates = [ - process.env.SOFTWARE_FACTORY_HOST_DIST_PACKAGE_DIR, - resolve(siblingRoot, 'boxel', 'packages', 'host'), - ...readdirSync(siblingRoot, { withFileTypes: true }) - .filter((entry) => entry.isDirectory()) - .map((entry) => resolve(siblingRoot, entry.name, 'packages', 'host')), - hostDir, - ] - .filter((value): value is string => Boolean(value)) - .map((value) => resolve(value)); - - let seen = new Set(); - for (let candidate of candidates) { - if (seen.has(candidate)) { - continue; - } - seen.add(candidate); - - if (fileExists(join(candidate, 'dist', 'index.html'))) { - return candidate; - } - } - - return undefined; -} - -async function loadSynapseModule() { - let moduleSpecifier = '../../matrix/docker/synapse/index.ts'; - return (maybeRequire(moduleSpecifier) ?? (await import(moduleSpecifier))) as { - registerUser: ( - synapse: SynapseInstance, - username: string, - password: string, - admin?: boolean, - displayName?: string, - ) => Promise; - synapseStart: ( - opts?: { - suppressRegistrationSecretFile?: true; - dynamicHostPort?: true; - }, - stopExisting?: boolean, - ) => Promise; - synapseStop: (id: string) => Promise; - }; -} - -async function loadMatrixEnvironmentConfigModule() { - let moduleSpecifier = '../../matrix/helpers/environment-config.ts'; - return (maybeRequire(moduleSpecifier) ?? (await import(moduleSpecifier))) as { - getSynapseURL: (synapse?: { baseUrl?: string; port?: number }) => string; - }; -} - -async function ensureHostReady(matrixURL: string): Promise<{ - stop?: () => Promise; -}> { - return await logTimed( - supportLog, - `ensureHostReady ${DEFAULT_HOST_URL}`, - async () => { - let response: Response; - try { - response = await fetch(DEFAULT_HOST_URL); - if (response.ok) { - return {}; - } - } catch (error) { - supportLog.debug( - `host app not reachable at ${DEFAULT_HOST_URL}, starting fallback host service: ${ - error instanceof Error ? error.message : String(error) - }`, - ); - } - - let hostPackageDir = findHostDistPackageDir(); - let command = ['start']; - let cwd = hostDir; - if (hostPackageDir) { - supportLog.debug(`serving built host dist from ${hostPackageDir}`); - command = ['serve:dist']; - cwd = hostPackageDir; - } else { - supportLog.warn( - 'no built host dist found; falling back to pnpm start in packages/host', - ); - } - - let child = spawn('pnpm', command, { - cwd, - detached: true, - stdio: ['ignore', 'pipe', 'pipe'], - env: { - ...process.env, - MATRIX_URL: matrixURL, - }, - }); - - let logs = ''; - child.stdout?.on('data', (chunk) => { - logs = `${logs}${String(chunk)}`.slice(-20_000); - }); - child.stderr?.on('data', (chunk) => { - logs = `${logs}${String(chunk)}`.slice(-20_000); - }); - - await waitUntil( - async () => { - if (child.exitCode !== null) { - throw new Error( - `host app exited early with code ${child.exitCode}\n${logs}`, - ); - } - try { - let readyResponse = await fetch(DEFAULT_HOST_URL); - return readyResponse.ok; - } catch { - return false; - } - }, - { - timeout: 180_000, - interval: 500, - timeoutMessage: `Timed out waiting for host app at ${DEFAULT_HOST_URL}\n${logs}`, - }, - ); - - return { - async stop() { - if (child.exitCode === null) { - try { - process.kill(-child.pid!, 'SIGTERM'); - } catch { - // best effort cleanup - } - } - }, - }; - }, - ); -} - -async function waitForHttpReady(url: string, timeoutMs = 60_000) { - let startedAt = Date.now(); - while (Date.now() - startedAt < timeoutMs) { - try { - let response = await fetch(url); - if (response.ok) { - return; - } - } catch { - // server not ready yet - } - await new Promise((resolve) => setTimeout(resolve, 200)); - } - throw new Error(`timed out waiting for ${url} to become ready`); -} - -async function stopChildProcess( - child: ChildProcess, - signal: NodeJS.Signals = 'SIGINT', -): Promise { - if (child.exitCode !== null || child.killed) { - return; - } - - await new Promise((resolve) => { - let settled = false; - let timeout: NodeJS.Timeout | undefined; - let cleanup = () => { - if (timeout) { - clearTimeout(timeout); - } - child.removeAllListeners('exit'); - child.removeAllListeners('error'); - }; - - child.once('exit', () => { - if (!settled) { - settled = true; - cleanup(); - resolve(); - } - }); - child.once('error', () => { - if (!settled) { - settled = true; - cleanup(); - resolve(); - } - }); - - timeout = setTimeout(() => { - if (!settled) { - child.kill('SIGTERM'); - } - }, 5_000); - - child.kill(signal); - }); -} - -async function startHarnessPrerenderServer(options: { - boxelHostURL: string; - port?: number; -}): Promise<{ - url: string; - stop(): Promise; -}> { - let port = options.port ?? DEFAULT_PRERENDER_PORT; - if (port === 0) { - port = await findAvailablePort(); - } - let url = `http://localhost:${port}`; - let silent = process.env.SOFTWARE_FACTORY_PRERENDER_SILENT !== '0'; - let child = spawn( - 'ts-node', - [ - '--transpileOnly', - 'prerender/prerender-server', - `--port=${port}`, - ...(silent ? ['--silent'] : []), - ], - { - cwd: realmServerDir, - stdio: ['pipe', 'pipe', 'pipe'], - env: { - ...process.env, - NODE_ENV: process.env.NODE_ENV ?? 'development', - NODE_NO_WARNINGS: '1', - BOXEL_HOST_URL: options.boxelHostURL, - LOG_LEVELS: - process.env.SOFTWARE_FACTORY_PRERENDER_LOG_LEVELS ?? - process.env.LOG_LEVELS, - }, - }, - ); - - child.stdout?.on('data', (data: Buffer) => { - console.log(`prerender: ${data.toString()}`); - }); - child.stderr?.on('data', (data: Buffer) => { - console.error(`prerender: ${data.toString()}`); - }); - - let exitPromise = new Promise((_, reject) => { - child.once('exit', (code, signal) => { - reject( - new Error( - `prerender server exited before it became ready (code: ${code}, signal: ${signal})`, - ), - ); - }); - child.once('error', reject); - }); - - await Promise.race([waitForHttpReady(url), exitPromise]); - - return { - url, - async stop() { - await stopChildProcess(child); - }, - }; -} - -async function ensureIconsReady(): Promise<{ - stop?: () => Promise; -}> { - return await logTimed( - supportLog, - `ensureIconsReady ${DEFAULT_ICONS_PROBE_URL}`, - async () => { - try { - let response = await fetch(DEFAULT_ICONS_PROBE_URL); - if (response.ok) { - supportLog.debug('icons server already available'); - return {}; - } - } catch { - // fall through and start the local icon server - } - - let child = spawn('pnpm', ['serve'], { - cwd: boxelIconsDir, - detached: true, - stdio: ['ignore', 'pipe', 'pipe'], - env: process.env, - }); - - let logs = ''; - child.stdout?.on('data', (chunk) => { - logs = `${logs}${String(chunk)}`.slice(-20_000); - }); - child.stderr?.on('data', (chunk) => { - logs = `${logs}${String(chunk)}`.slice(-20_000); - }); - - await waitUntil( - async () => { - if (child.exitCode !== null) { - throw new Error( - `icons server exited early with code ${child.exitCode}\n${logs}`, - ); - } - try { - let response = await fetch(DEFAULT_ICONS_PROBE_URL); - return response.ok; - } catch { - return false; - } - }, - { - timeout: 30_000, - interval: 250, - timeoutMessage: `Timed out waiting for icons server at ${DEFAULT_ICONS_PROBE_URL}\n${logs}`, - }, - ); - - supportLog.debug('started local icons server'); - return { - async stop() { - if (child.exitCode === null) { - try { - process.kill(-child.pid!, 'SIGTERM'); - } catch { - // best effort cleanup - } - } - }, - }; - }, - ); -} - -async function ensurePgReady(): Promise { - if (!preparePgPromise) { - preparePgPromise = logTimed( - supportLog, - `ensurePgReady ${DEFAULT_PG_HOST}:${DEFAULT_PG_PORT}`, - async () => { - if (await canConnectToPg()) { - supportLog.debug('postgres already available'); - return; - } - runCommand('bash', [prepareTestPgScript], workspaceRoot); - await waitUntil(() => canConnectToPg(), { - timeout: 30_000, - interval: 250, - timeoutMessage: `Timed out waiting for Postgres on ${DEFAULT_PG_HOST}:${DEFAULT_PG_PORT}`, - }); - }, - ).catch((error) => { - preparePgPromise = undefined; - throw error; - }); - } - - await preparePgPromise; -} - -async function databaseExists(databaseName: string): Promise { - let client = new PgClient(pgAdminConnectionConfig()); - try { - await client.connect(); - let result = await client.query<{ exists: boolean }>( - 'SELECT EXISTS(SELECT 1 FROM pg_database WHERE datname = $1) AS exists', - [databaseName], - ); - return Boolean(result.rows[0]?.exists); - } finally { - await client.end(); - } -} - -async function dropDatabase(databaseName: string): Promise { - await logTimed(templateLog, `dropDatabase ${databaseName}`, async () => { - let client = new PgClient(pgAdminConnectionConfig()); - try { - await client.connect(); - await client.query( - `SELECT pg_terminate_backend(pid) - FROM pg_stat_activity - WHERE datname = $1 AND pid <> pg_backend_pid()`, - [databaseName], - ); - await client.query( - `DROP DATABASE IF EXISTS ${quotePgIdentifier(databaseName)}`, - ); - } finally { - await client.end(); - } - }); -} - -async function cloneDatabaseFromTemplate( - templateDatabaseName: string, - databaseName: string, -): Promise { - await logTimed( - templateLog, - `cloneDatabaseFromTemplate ${templateDatabaseName} -> ${databaseName}`, - async () => { - let client = new PgClient(pgAdminConnectionConfig()); - try { - await client.connect(); - await client.query( - `CREATE DATABASE ${quotePgIdentifier(databaseName)} TEMPLATE ${quotePgIdentifier( - templateDatabaseName, - )}`, - ); - } finally { - await client.end(); - } - }, - ); -} - -async function createTemplateSnapshot( - sourceDatabaseName: string, - templateDatabaseName: string, -): Promise { - await logTimed( - templateLog, - `createTemplateSnapshot ${sourceDatabaseName} -> ${templateDatabaseName}`, - async () => { - let client = new PgClient(pgAdminConnectionConfig()); - try { - await client.connect(); - await client.query( - `SELECT pg_terminate_backend(pid) - FROM pg_stat_activity - WHERE datname = $1 AND pid <> pg_backend_pid()`, - [templateDatabaseName], - ); - await client.query( - `DROP DATABASE IF EXISTS ${quotePgIdentifier(templateDatabaseName)}`, - ); - await client.query( - `CREATE DATABASE ${quotePgIdentifier(templateDatabaseName)} TEMPLATE ${quotePgIdentifier( - sourceDatabaseName, - )}`, - ); - await client.query( - `ALTER DATABASE ${quotePgIdentifier(templateDatabaseName)} WITH IS_TEMPLATE true`, - ); - } finally { - await client.end(); - } - }, - ); -} - -async function seedRealmPermissions( - databaseName: string, - realmURL: URL, - permissions: RealmPermissions, -): Promise { - await logTimed( - templateLog, - `seedRealmPermissions ${databaseName} ${realmURL.href}`, - async () => { - let client = new PgClient(pgAdminConnectionConfig(databaseName)); - try { - await client.connect(); - await client.query('BEGIN'); - - for (let [username, actions] of Object.entries(permissions)) { - if (!actions || actions.length === 0) { - await client.query( - `DELETE FROM realm_user_permissions - WHERE realm_url = $1 AND username = $2`, - [realmURL.href, username], - ); - continue; - } - - if (username !== '*') { - await client.query( - `INSERT INTO users (matrix_user_id) - VALUES ($1) - ON CONFLICT (matrix_user_id) DO NOTHING`, - [username], - ); - } - - await client.query( - `INSERT INTO realm_user_permissions ( - realm_url, - username, - read, - write, - realm_owner - ) VALUES ($1, $2, $3, $4, $5) - ON CONFLICT (realm_url, username) DO UPDATE - SET read = EXCLUDED.read, - write = EXCLUDED.write, - realm_owner = EXCLUDED.realm_owner`, - [ - realmURL.href, - username, - actions.includes('read'), - actions.includes('write'), - actions.includes('realm-owner'), - ], - ); - } - - await client.query('COMMIT'); - } catch (error) { - try { - await client.query('ROLLBACK'); - } catch { - // best effort cleanup - } - throw error; - } finally { - await client.end(); - } - }, - ); -} - -async function resetRealmState( - databaseName: string, - realmURL: URL, -): Promise { - await logTimed( - templateLog, - `resetRealmState ${databaseName} ${realmURL.href}`, - async () => { - let client = new PgClient(pgAdminConnectionConfig(databaseName)); - try { - await client.connect(); - await client.query('BEGIN'); - - await client.query( - `DELETE FROM modules WHERE resolved_realm_url = $1`, - [realmURL.href], - ); - await client.query(`DELETE FROM boxel_index WHERE realm_url = $1`, [ - realmURL.href, - ]); - await client.query( - `DELETE FROM boxel_index_working WHERE realm_url = $1`, - [realmURL.href], - ); - await client.query(`DELETE FROM realm_versions WHERE realm_url = $1`, [ - realmURL.href, - ]); - await client.query(`DELETE FROM realm_file_meta WHERE realm_url = $1`, [ - realmURL.href, - ]); - await client.query( - `DELETE FROM published_realms - WHERE source_realm_url = $1 OR published_realm_url = $1`, - [realmURL.href], - ); - - await client.query('COMMIT'); - } catch (error) { - try { - await client.query('ROLLBACK'); - } catch { - // best effort cleanup - } - throw error; - } finally { - await client.end(); - } - }, - ); -} - -async function resetMountedRealmState( - databaseName: string, - realmURLs: URL[], -): Promise { - await logTimed( - templateLog, - `resetMountedRealmState ${databaseName} (${realmURLs.length} realms)`, - async () => { - for (let realmURL of realmURLs) { - await resetRealmState(databaseName, realmURL); - } - }, - ); -} - -async function resetQueueState(databaseName: string): Promise { - await logTimed(templateLog, `resetQueueState ${databaseName}`, async () => { - let client = new PgClient(pgAdminConnectionConfig(databaseName)); - try { - await client.connect(); - await client.query('BEGIN'); - await client.query(`DELETE FROM job_reservations`); - await client.query(`DELETE FROM jobs`); - await client.query('COMMIT'); - } catch (error) { - try { - await client.query('ROLLBACK'); - } catch { - // best effort cleanup - } - throw error; - } finally { - await client.end(); - } - }); -} - -async function waitForQueueIdle(databaseName: string): Promise { - await logTimed(templateLog, `waitForQueueIdle ${databaseName}`, async () => { - await waitUntil( - async () => { - let client = new PgClient(pgAdminConnectionConfig(databaseName)); - try { - await client.connect(); - let { - rows: [{ count: unfulfilledJobs }], - } = await client.query<{ count: number }>( - `SELECT COUNT(*)::int AS count FROM jobs WHERE status = 'unfulfilled'`, - ); - let { - rows: [{ count: activeReservations }], - } = await client.query<{ count: number }>( - `SELECT COUNT(*)::int AS count FROM job_reservations WHERE completed_at IS NULL`, - ); - return unfulfilledJobs === 0 && activeReservations === 0; - } finally { - await client.end(); - } - }, - { - timeout: 30_000, - interval: 100, - timeoutMessage: `Timed out waiting for queue to become idle in ${databaseName}`, - }, - ); - }); -} - -async function ensureSupportUsers(synapse: SynapseInstance): Promise { - await logTimed(supportLog, 'ensureSupportUsers', async () => { - let { registerUser } = await loadSynapseModule(); - - await registerUser( - synapse, - DEFAULT_MATRIX_SERVER_USERNAME, - browserPassword(DEFAULT_MATRIX_SERVER_USERNAME), - ); - await registerUser( - synapse, - DEFAULT_MATRIX_BROWSER_USERNAME, - browserPassword(DEFAULT_MATRIX_BROWSER_USERNAME), - ); - }); -} -function browserPassword(username: string): string { - let cleanUsername = username.replace(/^@/, '').replace(/:.*$/, ''); - return createHash('sha256') - .update(cleanUsername) - .update(REALM_SECRET_SEED) - .digest('hex'); -} - -function parseFactoryContext(): FactoryTestContext | undefined { - let raw = process.env.SOFTWARE_FACTORY_CONTEXT; - if (!raw) { - return undefined; - } - return JSON.parse(raw) as FactoryTestContext; -} - -function hasTemplateDatabaseName( - context: FactorySupportContext | FactoryTestContext, -): context is FactoryTestContext { - return 'templateDatabaseName' in context; -} - -function buildRealmToken( - realmURL: URL, - realmServerURL: URL, - user = DEFAULT_REALM_OWNER, - permissions = DEFAULT_PERMISSIONS[DEFAULT_REALM_OWNER] ?? [ - 'read', - 'write', - 'realm-owner', - ], -): string { - return jwt.sign( - { - user, - realm: realmURL.href, - permissions, - sessionRoom: `software-factory-session-room-for-${user}`, - realmServerURL: realmServerURL.href, - }, - REALM_SECRET_SEED, - { expiresIn: '7d' }, - ); -} - -function createProcessExitPromise( - proc: SpawnedProcess, - label: string, -): Promise { - return new Promise((_, reject) => { - proc.once('exit', (code, signal) => { - reject( - new Error( - `${label} exited before it became ready (code: ${code}, signal: ${signal})`, - ), - ); - }); - proc.once('error', reject); - }); -} - -function appendProcessLogs(buffer: string, chunk: Buffer | string): string { - return `${buffer}${String(chunk)}`.slice(-100_000); -} - -function captureProcessLogs(proc: SpawnedProcess) { - let stdout = ''; - let stderr = ''; - - proc.stdout?.on('data', (chunk) => { - stdout = appendProcessLogs(stdout, chunk); - }); - proc.stderr?.on('data', (chunk) => { - stderr = appendProcessLogs(stderr, chunk); - }); - - return () => { - let logs = []; - if (stdout) { - logs.push(`stdout:\n${stdout}`); - } - if (stderr) { - logs.push(`stderr:\n${stderr}`); - } - return logs.join('\n\n'); - }; -} - -async function waitForReady( - proc: SpawnedProcess, - label: string, - timeoutMs = 120_000, - getLogs?: () => string, -): Promise { - let timedOut = await Promise.race([ - new Promise((resolve) => { - let onMessage = (message: unknown) => { - if (message === 'ready') { - proc.off('message', onMessage); - resolve(); - } - }; - proc.on('message', onMessage); - }), - createProcessExitPromise(proc, label), - new Promise((resolve) => setTimeout(() => resolve(true), timeoutMs)), - ]); - - if (timedOut) { - let logOutput = getLogs?.(); - throw new Error( - `Timed out waiting for ${label} to start${ - logOutput ? `\n\n${logOutput}` : '' - }`, - ); - } -} - -async function stopManagedProcess(proc: SpawnedProcess): Promise { - if (proc.exitCode !== null) { - return; - } - let stopped = new Promise((resolve) => { - let onMessage = (message: unknown) => { - if (message === 'stopped') { - proc.off('message', onMessage); - resolve(); - } - }; - proc.on('message', onMessage); - }); - proc.send('stop'); - await Promise.race([ - stopped, - new Promise((resolve) => setTimeout(resolve, 15_000)), - ]); - proc.send('kill'); -} - -async function readIncomingRequestBody( - req: IncomingMessage, -): Promise { - let chunks: Buffer[] = []; - for await (let chunk of req) { - chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); - } - return chunks.length ? Buffer.concat(chunks) : undefined; -} - -function describeCompatProxyError(error: unknown): string { - if (!(error instanceof Error)) { - return String(error); - } - - let parts: string[] = []; - let current: unknown = error; - - while (current) { - if (current instanceof Error) { - let code = - 'code' in current && typeof current.code === 'string' - ? ` (${current.code})` - : ''; - parts.push(`${current.message}${code}`); - current = current.cause; - } else { - parts.push(String(current)); - break; - } - } - - return parts.join(' <- '); -} - -async function startCompatRealmProxy({ - listenPort, -}: { - listenPort: number; -}): Promise { - realmLog.debug(`startCompatRealmProxy: requested listenPort=${listenPort}`); - let targetPort: number | undefined; - let server = createServer( - async (req: IncomingMessage, res: ServerResponse) => { - if (targetPort == null) { - res.statusCode = 503; - res.setHeader('content-type', 'text/plain; charset=utf-8'); - res.end('software-factory compat proxy target is not ready'); - return; - } - let incomingURL = new URL( - req.url ?? '/', - `${ - req.headers['x-forwarded-proto'] === 'https' ? 'https' : 'http' - }://${req.headers.host ?? `127.0.0.1:${actualListenPort}`}`, - ); - let upstreamURL = new URL( - `${incomingURL.pathname}${incomingURL.search}`, - `http://localhost:${targetPort}`, - ); - - try { - let body = await readIncomingRequestBody(req); - let headers = Object.fromEntries( - Object.entries(req.headers).filter( - ([key]) => key.toLowerCase() !== 'host', - ), - ) as Record; - headers['x-boxel-forwarded-url'] = incomingURL.href; - let response = await fetch(upstreamURL, { - method: req.method, - headers, - body: body as BodyInit | undefined, - // Preserve upstream redirects so the client follows them against the - // public compat URL with a fresh forwarded URL header. - redirect: 'manual', - }); - - let responseHeaders = new Headers(response.headers); - let location = responseHeaders.get('location'); - if (location) { - responseHeaders.set( - 'location', - location - .replace( - `http://localhost:${targetPort}/`, - `http://127.0.0.1:${listenPort}/`, - ) - .replace( - `http://localhost:${targetPort}/`, - `http://localhost:${listenPort}/`, - ), - ); - } - - res.statusCode = response.status; - responseHeaders.forEach((value, key) => { - res.setHeader(key, value); - }); - res.end(Buffer.from(await response.arrayBuffer())); - } catch (error) { - let description = describeCompatProxyError(error); - realmLog.warn( - `startCompatRealmProxy: upstream fetch failed for ${upstreamURL.href}: ${description}`, - ); - res.statusCode = 502; - res.setHeader('content-type', 'text/plain; charset=utf-8'); - res.end( - `software-factory compat proxy failed for ${upstreamURL.href}: ${description}`, - ); - } - }, - ); - await new Promise((resolve, reject) => { - server.once('error', reject); - server.listen(listenPort, '127.0.0.1', () => resolve()); - }); - let address = server.address(); - if (!address || typeof address === 'string') { - throw new Error('Unable to determine compat proxy port'); - } - let actualListenPort = address.port; - realmLog.debug(`startCompatRealmProxy: listening on ${actualListenPort}`); - return { - listenPort: actualListenPort, - setTargetPort(nextTargetPort: number) { - targetPort = nextTargetPort; - realmLog.debug( - `startCompatRealmProxy: ${actualListenPort} -> ${nextTargetPort} ready`, - ); - }, - async stop() { - realmLog.debug( - `startCompatRealmProxy: ${actualListenPort} -> ${targetPort ?? 'unset'} stopping`, - ); - await new Promise((resolve, reject) => { - server.close((error) => { - if (error) { - reject(error); - } else { - resolve(); - } - }); - }); - }, - }; -} - -function rewriteFixtureSourceModuleUrls( - destination: string, - sourceRealmURL: URL, -): void { - let rewrittenFiles = 0; - - function visit(currentDir: string) { - for (let entry of readdirSync(currentDir, { withFileTypes: true })) { - let absolutePath = join(currentDir, entry.name); - if (entry.isDirectory()) { - visit(absolutePath); - continue; - } - if (!entry.isFile() || !entry.name.endsWith('.json')) { - continue; - } - - let contents = readFileSync(absolutePath, 'utf8'); - if (!contents.includes(FIXTURE_SOURCE_REALM_URL_PLACEHOLDER)) { - continue; - } - - writeFileSync( - absolutePath, - contents - .split(FIXTURE_SOURCE_REALM_URL_PLACEHOLDER) - .join(sourceRealmURL.href), - ); - rewrittenFiles++; - } - } - - visit(destination); - if (rewrittenFiles > 0) { - realmLog.debug( - `rewriteFixtureSourceModuleUrls: rewrote ${rewrittenFiles} files to ${sourceRealmURL.href}`, - ); - } -} - -function copyRealmFixture( - realmDir: string, - destination: string, - sourceRealmURL: URL, -): void { - copySync(realmDir, destination, { - preserveTimestamps: true, - filter(src) { - let relativePath = relative(realmDir, src).replace(/\\/g, '/'); - return relativePath === '' || !shouldIgnoreFixturePath(relativePath); - }, - }); - rewriteFixtureSourceModuleUrls(destination, sourceRealmURL); -} - -async function startIsolatedRealmStack({ - realmDir, - realmURL, - realmServerURL, - databaseName, - context, - migrateDB, - fullIndexOnStartup, -}: { - realmDir: string; - realmURL: URL; - realmServerURL: URL; - databaseName: string; - context: FactorySupportContext; - migrateDB: boolean; - fullIndexOnStartup: boolean; -}): Promise { - return await logTimed( - realmLog, - `startIsolatedRealmStack ${databaseName} ${realmURL.href}`, - async () => { - let rootDir = mkdtempSync(join(tmpdir(), 'software-factory-realms-')); - let testRealmDir = join(rootDir, 'test'); - let workerManagerMetadataFile = join( - rootDir, - 'worker-manager.runtime.json', - ); - let realmServerMetadataFile = join(rootDir, 'realm-server.runtime.json'); - let actualRealmServerPort = - DEFAULT_REALM_SERVER_PORT === 0 - ? await findAvailablePort() - : DEFAULT_REALM_SERVER_PORT; - let actualRealmServerURL = withPort( - realmServerURL, - actualRealmServerPort, - ); - let actualRealmPath = realmRelativePath(realmURL, realmServerURL); - let actualRealmURL = realmURLWithinServer( - actualRealmServerURL, - actualRealmPath, - ); - let legacyRealmServerURL = new URL('http://localhost:4205/'); - let legacyRealmURL = new URL('test/', legacyRealmServerURL); - let publicBaseRealmURL = baseRealmURLFor(realmServerURL); - let actualBaseRealmURL = baseRealmURLFor(actualRealmServerURL); - let sourceRealmURL = sourceRealmURLFor(realmServerURL); - let actualSourceRealmURL = sourceRealmURLFor(actualRealmServerURL); - let legacySourceRealmURL = sourceRealmURLFor(legacyRealmServerURL); - let skillsRealmURL = skillsRealmURLFor(realmServerURL); - let actualSkillsRealmURL = skillsRealmURLFor(actualRealmServerURL); - let legacySkillsRealmURL = skillsRealmURLFor(legacyRealmServerURL); - ensureDirSync(testRealmDir); - copyRealmFixture(realmDir, testRealmDir, sourceRealmURL); - realmLog.debug( - `startIsolatedRealmStack: copied fixture ${realmDir} -> ${testRealmDir}`, - ); - let compatProxy = await startCompatRealmProxy({ - listenPort: Number(realmServerURL.port), - }); - let prerender = await startHarnessPrerenderServer({ - boxelHostURL: realmServerURL.href.replace(/\/$/, ''), - }); - - let env = { - ...process.env, - PGHOST: DEFAULT_PG_HOST, - PGPORT: DEFAULT_PG_PORT, - PGUSER: DEFAULT_PG_USER, - PGDATABASE: databaseName, - NODE_NO_WARNINGS: '1', - NODE_ENV: 'test', - REALM_SERVER_SECRET_SEED, - REALM_SECRET_SEED, - GRAFANA_SECRET, - MATRIX_URL: context.matrixURL, - MATRIX_REGISTRATION_SHARED_SECRET: context.matrixRegistrationSecret, - REALM_SERVER_MATRIX_USERNAME: DEFAULT_MATRIX_SERVER_USERNAME, - REALM_SERVER_FULL_INDEX_ON_STARTUP: String(fullIndexOnStartup), - LOW_CREDIT_THRESHOLD: '2000', - LOG_LEVELS: DEFAULT_REALM_LOG_LEVELS, - PUBLISHED_REALM_BOXEL_SPACE_DOMAIN: `localhost:${compatProxy.listenPort}`, - PUBLISHED_REALM_BOXEL_SITE_DOMAIN: `localhost:${compatProxy.listenPort}`, - SOFTWARE_FACTORY_WORKER_MANAGER_METADATA_FILE: - workerManagerMetadataFile, - SOFTWARE_FACTORY_REALM_SERVER_METADATA_FILE: realmServerMetadataFile, - }; - - let workerArgs = [ - '--transpileOnly', - 'worker-manager', - `--port=${DEFAULT_WORKER_MANAGER_PORT}`, - `--matrixURL=${context.matrixURL}`, - `--prerendererUrl=${prerender.url}`, - `--fromUrl=${realmURL.href}`, - `--toUrl=${actualRealmURL.href}`, - `--fromUrl=${publicBaseRealmURL.href}`, - `--toUrl=${actualBaseRealmURL.href}`, - '--fromUrl=https://cardstack.com/base/', - `--toUrl=${publicBaseRealmURL.href}`, - `--fromUrl=${sourceRealmURL.href}`, - `--toUrl=${actualSourceRealmURL.href}`, - ]; - if (INCLUDE_SKILLS) { - workerArgs.push( - `--fromUrl=${skillsRealmURL.href}`, - `--toUrl=${actualSkillsRealmURL.href}`, - ); - } - workerArgs.push( - `--fromUrl=${legacyRealmURL.href}`, - `--toUrl=${actualRealmURL.href}`, - `--fromUrl=${legacySourceRealmURL.href}`, - `--toUrl=${actualSourceRealmURL.href}`, - ); - if (INCLUDE_SKILLS) { - workerArgs.push( - `--fromUrl=${legacySkillsRealmURL.href}`, - `--toUrl=${actualSkillsRealmURL.href}`, - ); - } - if (migrateDB) { - workerArgs.splice(5, 0, '--migrateDB'); - } - - realmLog.debug( - `startIsolatedRealmStack: starting worker-manager for ${databaseName}`, - ); - let workerManager = spawn('ts-node', workerArgs, { - cwd: realmServerDir, - env, - stdio: managedProcessStdio, - }) as SpawnedProcess; - let getWorkerLogs = captureProcessLogs(workerManager); - let workerManagerRuntime = await waitForJsonFile<{ - pid: number; - port: number; - url: string; - }>(workerManagerMetadataFile, getWorkerLogs, { - label: 'worker manager', - process: workerManager, - }); - - let serverArgs = [ - '--transpileOnly', - 'main', - `--port=${actualRealmServerPort}`, - `--serverURL=${realmServerURL.href}`, - `--matrixURL=${context.matrixURL}`, - `--realmsRootPath=${rootDir}`, - `--workerManagerUrl=${workerManagerRuntime.url}`, - `--prerendererUrl=${prerender.url}`, - '--username=base_realm', - `--path=${baseRealmDir}`, - `--fromUrl=${publicBaseRealmURL.href}`, - `--toUrl=${actualBaseRealmURL.href}`, - '--username=software_factory_realm', - `--path=${sourceRealmDir}`, - `--fromUrl=${sourceRealmURL.href}`, - `--toUrl=${actualSourceRealmURL.href}`, - '--username=test_realm', - `--path=${testRealmDir}`, - `--fromUrl=${realmURL.href}`, - `--toUrl=${actualRealmURL.href}`, - ]; - if (INCLUDE_SKILLS) { - serverArgs.splice( - 16, - 0, - '--username=skills_realm', - `--path=${skillsRealmDir}`, - `--fromUrl=${skillsRealmURL.href}`, - `--toUrl=${actualSkillsRealmURL.href}`, - ); - } - serverArgs.push( - `--fromUrl=${legacyRealmURL.href}`, - `--toUrl=${actualRealmURL.href}`, - `--fromUrl=${legacySourceRealmURL.href}`, - `--toUrl=${actualSourceRealmURL.href}`, - ); - if (INCLUDE_SKILLS) { - serverArgs.push( - `--fromUrl=${legacySkillsRealmURL.href}`, - `--toUrl=${actualSkillsRealmURL.href}`, - ); - } - - realmLog.debug(`startIsolatedRealmStack: starting realm server`); - let realmServer = spawn('ts-node', serverArgs, { - cwd: realmServerDir, - env, - stdio: managedProcessStdio, - }) as SpawnedProcess; - let getServerLogs = captureProcessLogs(realmServer); - let realmServerRuntime: { - pid: number; - port: number; - }; - - try { - realmServerRuntime = await waitForJsonFile<{ - pid: number; - port: number; - }>(realmServerMetadataFile, getServerLogs, { - label: 'realm server', - process: realmServer, - }); - compatProxy.setTargetPort(realmServerRuntime.port); - await Promise.race([ - waitForReady( - realmServer, - 'realm server', - fullIndexOnStartup - ? FULL_INDEX_REALM_STARTUP_TIMEOUT_MS - : DEFAULT_REALM_STARTUP_TIMEOUT_MS, - () => - [ - 'realm server logs:', - getServerLogs(), - 'worker manager logs:', - getWorkerLogs(), - ] - .filter((entry) => entry && entry.trim().length > 0) - .join('\n\n'), - ), - createProcessExitPromise(workerManager, 'worker manager'), - ]); - } catch (error) { - try { - await prerender.stop(); - } catch { - // best effort cleanup - } - try { - await stopManagedProcess(realmServer); - } catch { - // best effort cleanup - } - try { - await stopManagedProcess(workerManager); - } catch { - // best effort cleanup - } - try { - await compatProxy?.stop(); - } catch { - // best effort cleanup - } - rmSync(rootDir, { recursive: true, force: true }); - throw error; - } - - return { - compatProxy, - prerender, - realmServer, - realmServerURL, - ports: { - publicPort: compatProxy.listenPort, - realmServerPort: realmServerRuntime.port, - workerManagerPort: workerManagerRuntime.port, - }, - workerManager, - rootDir, - }; - }, - ); -} - -async function stopIsolatedRealmStack( - stack: RunningFactoryStack, -): Promise { - await logTimed(realmLog, 'stopIsolatedRealmStack', async () => { - let cleanupError: unknown; - - try { - await stack.prerender.stop(); - } catch (error) { - cleanupError ??= error; - } - - try { - await stopManagedProcess(stack.realmServer); - } catch (error) { - cleanupError ??= error; - } - - try { - await stopManagedProcess(stack.workerManager); - } catch (error) { - cleanupError ??= error; - } - - try { - await stack.compatProxy?.stop(); - } catch (error) { - cleanupError ??= error; - } - - try { - rmSync(stack.rootDir, { recursive: true, force: true }); - } catch (error) { - cleanupError ??= error; - } - - if (cleanupError) { - throw cleanupError; - } - }); -} - -async function buildTemplateDatabase({ - realmDir, - realmURL, - realmServerURL, - permissions, - context, - cacheKey, - templateDatabaseName, -}: { - realmDir: string; - realmURL: URL; - realmServerURL: URL; - permissions: RealmPermissions; - context: FactorySupportContext; - cacheKey: string; - templateDatabaseName: string; -}): Promise { - await logTimed( - templateLog, - `buildTemplateDatabase ${templateDatabaseName}`, - async () => { - let builderDatabaseName = builderDatabaseNameForCacheKey(cacheKey); - let hasMigratedTemplate = await databaseExists( - DEFAULT_MIGRATED_TEMPLATE_DB, - ); - - templateLog.debug( - `buildTemplateDatabase: builder=${builderDatabaseName} migratedTemplate=${hasMigratedTemplate}`, - ); - await dropDatabase(templateDatabaseName); - await dropDatabase(builderDatabaseName); - - if (hasMigratedTemplate) { - await cloneDatabaseFromTemplate( - DEFAULT_MIGRATED_TEMPLATE_DB, - builderDatabaseName, - ); - } - let baseRealmURL = baseRealmURLFor(realmServerURL); - let sourceRealmURL = sourceRealmURLFor(realmServerURL); - - await resetMountedRealmState(builderDatabaseName, [ - realmURL, - baseRealmURL, - sourceRealmURL, - ]); - await resetQueueState(builderDatabaseName); - await seedRealmPermissions(builderDatabaseName, realmURL, permissions); - await seedRealmPermissions( - builderDatabaseName, - baseRealmURL, - DEFAULT_BASE_REALM_PERMISSIONS, - ); - await seedRealmPermissions( - builderDatabaseName, - sourceRealmURL, - DEFAULT_SOURCE_REALM_PERMISSIONS, - ); - - let stack = await startIsolatedRealmStack({ - realmDir, - realmURL, - realmServerURL, - databaseName: builderDatabaseName, - context, - migrateDB: !hasMigratedTemplate, - fullIndexOnStartup: true, - }); - - try { - await waitForQueueIdle(builderDatabaseName); - } finally { - await stopIsolatedRealmStack(stack); - } - - await createTemplateSnapshot(builderDatabaseName, templateDatabaseName); - await dropDatabase(builderDatabaseName); - }, - ); -} - -export async function startFactorySupportServices(): Promise<{ - context: FactorySupportContext; - stop(): Promise; -}> { - return await logTimed(supportLog, 'startFactorySupportServices', async () => { - await ensurePgReady(); - cleanupStaleSynapseContainers(); - let { synapseStart, synapseStop } = await loadSynapseModule(); - let { getSynapseURL } = await loadMatrixEnvironmentConfigModule(); - - let synapseStartedAt = Date.now(); - let synapse = await synapseStart( - { suppressRegistrationSecretFile: true, dynamicHostPort: true }, - true, - ); - supportLog.debug( - `synapse started in ${formatElapsedMs(Date.now() - synapseStartedAt)} on port ${synapse.port}`, - ); - let matrixURL = - process.env.SOFTWARE_FACTORY_MATRIX_URL ?? getSynapseURL(synapse); - let host = await ensureHostReady(matrixURL); - let icons = await ensureIconsReady(); - await ensureSupportUsers(synapse); - - return { - context: { - matrixURL, - matrixRegistrationSecret: synapse.registrationSecret, - }, - async stop() { - await logTimed(supportLog, 'stopFactorySupportServices', async () => { - await synapseStop(synapse.synapseId); - await host.stop?.(); - await icons.stop?.(); - }); - }, - }; - }); -} - -export function getFactoryTestContext(): FactoryTestContext { - let context = parseFactoryContext(); - if (!context) { - throw new Error('SOFTWARE_FACTORY_CONTEXT is not defined'); - } - return context; -} - -export async function startFactoryGlobalContext( - options: FactoryRealmOptions = {}, -): Promise { - return await logTimed(harnessLog, 'startFactoryGlobalContext', async () => { - let realmDir = resolve(options.realmDir ?? DEFAULT_REALM_DIR); - let { realmURL, realmServerURL } = await resolveFactoryRealmLocation({ - realmURL: options.realmURL, - realmServerURL: options.realmServerURL, - }); - let support = await startFactorySupportServices(); - try { - let template = await ensureFactoryRealmTemplate({ - ...options, - realmDir, - realmURL, - realmServerURL, - context: support.context, - }); - - let context: FactoryTestContext = { - ...support.context, - cacheKey: template.cacheKey, - fixtureHash: template.fixtureHash, - realmDir, - realmURL: realmURL.href, - realmServerURL: realmServerURL.href, - templateDatabaseName: template.templateDatabaseName, - }; - - return { - context, - stop: support.stop, - }; - } catch (error) { - await support.stop(); - throw error; - } - }); -} - -export async function ensureFactoryRealmTemplate( - options: FactoryRealmOptions = {}, -): Promise { - return await logTimed(harnessLog, 'ensureFactoryRealmTemplate', async () => { - let realmDir = resolve(options.realmDir ?? DEFAULT_REALM_DIR); - let contextRealmURL = - options.context && hasTemplateDatabaseName(options.context) - ? new URL(options.context.realmURL) - : undefined; - let contextRealmServerURL = - options.context && hasTemplateDatabaseName(options.context) - ? new URL(options.context.realmServerURL) - : undefined; - let { realmURL, realmServerURL } = await resolveFactoryRealmLocation({ - realmURL: options.realmURL ?? contextRealmURL, - realmServerURL: options.realmServerURL ?? contextRealmServerURL, - }); - let permissions = options.permissions ?? DEFAULT_PERMISSIONS; - let fixtureHash = hashRealmFixture(realmDir); - let cacheKey = hashString( - stableStringify({ - version: CACHE_VERSION, - realmURL: realmURL.href, - permissions, - fixtureHash, - cacheSalt: - options.cacheSalt ?? process.env.SOFTWARE_FACTORY_CACHE_SALT ?? null, - }), - ); - let templateDatabaseName = templateDatabaseNameForCacheKey(cacheKey); - - let ownedSupport: - | { - context: FactorySupportContext; - stop(): Promise; - } - | undefined; - let context = options.context; - if (!context) { - ownedSupport = await startFactorySupportServices(); - context = ownedSupport.context; - } - - try { - if (await databaseExists(templateDatabaseName)) { - templateLog.debug( - `ensureFactoryRealmTemplate: cache hit ${templateDatabaseName}`, - ); - return { - cacheKey, - templateDatabaseName, - fixtureHash, - cacheHit: true, - }; - } - - templateLog.debug( - `ensureFactoryRealmTemplate: cache miss ${templateDatabaseName}`, - ); - await buildTemplateDatabase({ - realmDir, - realmURL, - realmServerURL, - permissions, - context, - cacheKey, - templateDatabaseName, - }); - - return { - cacheKey, - templateDatabaseName, - fixtureHash, - cacheHit: false, - }; - } finally { - await ownedSupport?.stop(); - } - }); -} - -export async function startFactoryRealmServer( - options: FactoryRealmOptions = {}, -): Promise { - return await logTimed(harnessLog, 'startFactoryRealmServer', async () => { - let realmDir = resolve(options.realmDir ?? DEFAULT_REALM_DIR); - let existingContext = options.context ?? parseFactoryContext(); - let contextRealmURL = - existingContext && hasTemplateDatabaseName(existingContext) - ? new URL(existingContext.realmURL) - : undefined; - let contextRealmServerURL = - existingContext && hasTemplateDatabaseName(existingContext) - ? new URL(existingContext.realmServerURL) - : undefined; - let { realmURL, realmServerURL } = await resolveFactoryRealmLocation({ - realmURL: options.realmURL ?? contextRealmURL, - realmServerURL: options.realmServerURL ?? contextRealmServerURL, - }); - let templateDatabaseName = options.templateDatabaseName; - let databaseName = runtimeDatabaseName(); - - let ownedGlobalContext: FactoryGlobalContextHandle | undefined; - let context = existingContext; - if (!context) { - ownedGlobalContext = await startFactoryGlobalContext({ - ...options, - realmDir, - realmURL, - realmServerURL, - }); - context = ownedGlobalContext.context; - } - - if (!templateDatabaseName) { - templateDatabaseName = hasTemplateDatabaseName(context) - ? context.templateDatabaseName - : ( - await ensureFactoryRealmTemplate({ - ...options, - realmDir, - realmURL, - realmServerURL, - context, - }) - ).templateDatabaseName; - } - - realmLog.debug( - `startFactoryRealmServer: database=${databaseName} template=${templateDatabaseName}`, - ); - let stack: RunningFactoryStack; - try { - let baseRealmURL = baseRealmURLFor(realmServerURL); - let sourceRealmURL = sourceRealmURLFor(realmServerURL); - await dropDatabase(databaseName); - await cloneDatabaseFromTemplate(templateDatabaseName, databaseName); - await resetQueueState(databaseName); - await seedRealmPermissions( - databaseName, - baseRealmURL, - DEFAULT_BASE_REALM_PERMISSIONS, - ); - await resetMountedRealmState(databaseName, [sourceRealmURL]); - await seedRealmPermissions( - databaseName, - sourceRealmURL, - DEFAULT_SOURCE_REALM_PERMISSIONS, - ); - - stack = await startIsolatedRealmStack({ - realmDir, - realmURL, - realmServerURL, - databaseName, - context, - migrateDB: false, - fullIndexOnStartup: false, - }); - } catch (error) { - let cleanupError: unknown; - - try { - await dropDatabase(databaseName); - } catch (cleanupFailure) { - cleanupError ??= cleanupFailure; - } - - try { - await ownedGlobalContext?.stop(); - } catch (cleanupFailure) { - cleanupError ??= cleanupFailure; - } - - if (cleanupError) { - throw cleanupError; - } - - throw error; - } - - return { - realmDir, - realmURL, - realmServerURL, - databaseName, - ports: stack.ports, - childPids: [stack.realmServer.pid, stack.workerManager.pid].filter( - (pid): pid is number => pid != null, - ), - cardURL(path: string) { - return new URL(path, realmURL).href; - }, - createBearerToken( - user = DEFAULT_REALM_OWNER, - permissions?: RealmAction[], - ) { - return buildRealmToken(realmURL, realmServerURL, user, permissions); - }, - authorizationHeaders(user?: string, permissions?: RealmAction[]) { - return { - Authorization: `Bearer ${buildRealmToken( - realmURL, - realmServerURL, - user, - permissions, - )}`, - }; - }, - async stop() { - await logTimed( - realmLog, - `stopFactoryRealmServer ${databaseName}`, - async () => { - let cleanupError: unknown; - - try { - await stopIsolatedRealmStack(stack); - } catch (error) { - cleanupError ??= error; - } - - try { - await dropDatabase(databaseName); - } catch (error) { - cleanupError ??= error; - } - - try { - await ownedGlobalContext?.stop(); - } catch (error) { - cleanupError ??= error; - } - - if (cleanupError) { - throw cleanupError; - } - }, - ); - }, - }; - }); -} - -export async function fetchRealmCardJson( - path: string, - options: FactoryRealmOptions = {}, -) { - return await logTimed(harnessLog, `fetchRealmCardJson ${path}`, async () => { - let runtime = await startFactoryRealmServer(options); - try { - let response = await fetch(runtime.cardURL(path), { - headers: { - Accept: 'application/vnd.card+json', - }, - }); - return { - status: response.status, - body: await response.text(), - url: response.url, - }; - } finally { - await runtime.stop(); - } - }); -} +export { + ensureFactoryRealmTemplate, + fetchRealmCardJson, + getFactoryTestContext, + startFactoryGlobalContext, + startFactoryRealmServer, + startFactorySupportServices, +} from './harness/api'; + +export type { + FactoryRealmOptions, + FactoryRealmTemplate, + FactoryTestContext, + StartedFactoryRealm, +} from './harness/shared'; diff --git a/packages/software-factory/src/harness/api.ts b/packages/software-factory/src/harness/api.ts new file mode 100644 index 00000000000..2e515998a0a --- /dev/null +++ b/packages/software-factory/src/harness/api.ts @@ -0,0 +1,376 @@ +import { resolve } from 'node:path'; + +import { + buildRealmToken, + CACHE_VERSION, + DEFAULT_BASE_REALM_PERMISSIONS, + DEFAULT_PERMISSIONS, + DEFAULT_REALM_DIR, + DEFAULT_REALM_OWNER, + DEFAULT_SOURCE_REALM_PERMISSIONS, + hashRealmFixture, + sourceRealmDir, + harnessLog, + hasTemplateDatabaseName, + logTimed, + parseFactoryContext, + resolveFactoryRealmLocation, + runtimeDatabaseName, + stableStringify, + templateDatabaseNameForCacheKey, + hashString, + baseRealmURLFor, + sourceRealmURLFor, + realmLog, + type FactoryGlobalContextHandle, + type FactoryRealmOptions, + type FactoryRealmTemplate, + type FactorySupportContext, + type FactoryTestContext, + type RealmAction, + type StartedFactoryRealm, +} from './shared'; +import { + buildTemplateDatabase, + clearModuleCache, + cloneDatabaseFromTemplate, + databaseExists, + dropDatabase, + rebuildWorkingIndexFromIndex, + resetQueueState, + rewriteClonedRealmServerUrls, + seedRealmPermissions, + warnIfSnapshotLooksCold, +} from './database'; +import { startFactorySupportServices } from './support-services'; +import { + startIsolatedRealmStack, + stopIsolatedRealmStack, +} from './isolated-realm-stack'; + +export function getFactoryTestContext(): FactoryTestContext { + let context = parseFactoryContext(); + if (!context) { + throw new Error('SOFTWARE_FACTORY_CONTEXT is not defined'); + } + return context; +} + +export { startFactorySupportServices }; + +export async function startFactoryGlobalContext( + options: FactoryRealmOptions = {}, +): Promise { + return await logTimed(harnessLog, 'startFactoryGlobalContext', async () => { + let realmDir = resolve(options.realmDir ?? DEFAULT_REALM_DIR); + let { realmURL, realmServerURL } = await resolveFactoryRealmLocation({ + realmURL: options.realmURL, + realmServerURL: options.realmServerURL, + }); + let support = await startFactorySupportServices(); + try { + let template = await ensureFactoryRealmTemplate({ + ...options, + realmDir, + realmURL, + realmServerURL, + context: support.context, + }); + + let context: FactoryTestContext = { + ...support.context, + cacheKey: template.cacheKey, + fixtureHash: template.fixtureHash, + realmDir, + realmURL: realmURL.href, + realmServerURL: realmServerURL.href, + templateDatabaseName: template.templateDatabaseName, + }; + + return { + context, + stop: support.stop, + }; + } catch (error) { + await support.stop(); + throw error; + } + }); +} + +export async function ensureFactoryRealmTemplate( + options: FactoryRealmOptions = {}, +): Promise { + return await logTimed(harnessLog, 'ensureFactoryRealmTemplate', async () => { + let realmDir = resolve(options.realmDir ?? DEFAULT_REALM_DIR); + let contextRealmURL = + options.context && hasTemplateDatabaseName(options.context) + ? new URL(options.context.realmURL) + : undefined; + let contextRealmServerURL = + options.context && hasTemplateDatabaseName(options.context) + ? new URL(options.context.realmServerURL) + : undefined; + let { realmURL, realmServerURL } = await resolveFactoryRealmLocation({ + realmURL: options.realmURL ?? contextRealmURL, + realmServerURL: options.realmServerURL ?? contextRealmServerURL, + }); + let permissions = options.permissions ?? DEFAULT_PERMISSIONS; + let fixtureHash = hashRealmFixture(realmDir); + let sourceRealmHash = hashRealmFixture(sourceRealmDir); + let cacheKey = hashString( + stableStringify({ + version: CACHE_VERSION, + realmURL: realmURL.href, + permissions, + fixtureHash, + sourceRealmHash, + cacheSalt: + options.cacheSalt ?? process.env.SOFTWARE_FACTORY_CACHE_SALT ?? null, + }), + ); + let templateDatabaseName = templateDatabaseNameForCacheKey(cacheKey); + + let ownedSupport: + | { + context: FactorySupportContext; + stop(): Promise; + } + | undefined; + let context = options.context; + if (!context) { + ownedSupport = await startFactorySupportServices(); + context = ownedSupport.context; + } + + try { + if (await databaseExists(templateDatabaseName)) { + return { + cacheKey, + templateDatabaseName, + fixtureHash, + cacheHit: true, + realmURL, + realmServerURL, + }; + } + + await buildTemplateDatabase({ + realmDir, + realmURL, + realmServerURL, + permissions, + context, + cacheKey, + templateDatabaseName, + }); + + return { + cacheKey, + templateDatabaseName, + fixtureHash, + cacheHit: false, + realmURL, + realmServerURL, + }; + } finally { + await ownedSupport?.stop(); + } + }); +} + +export async function startFactoryRealmServer( + options: FactoryRealmOptions = {}, +): Promise { + return await logTimed(harnessLog, 'startFactoryRealmServer', async () => { + let realmDir = resolve(options.realmDir ?? DEFAULT_REALM_DIR); + let existingContext = options.context ?? parseFactoryContext(); + let contextRealmURL = + existingContext && hasTemplateDatabaseName(existingContext) + ? new URL(existingContext.realmURL) + : undefined; + let contextRealmServerURL = + existingContext && hasTemplateDatabaseName(existingContext) + ? new URL(existingContext.realmServerURL) + : undefined; + let { realmURL, realmServerURL } = await resolveFactoryRealmLocation({ + realmURL: options.realmURL ?? contextRealmURL, + realmServerURL: options.realmServerURL ?? contextRealmServerURL, + }); + let templateDatabaseName = options.templateDatabaseName; + let databaseName = runtimeDatabaseName(); + + let ownedGlobalContext: FactoryGlobalContextHandle | undefined; + let context = existingContext; + if (!context) { + ownedGlobalContext = await startFactoryGlobalContext({ + ...options, + realmDir, + realmURL, + realmServerURL, + }); + context = ownedGlobalContext.context; + } + + if (!templateDatabaseName) { + templateDatabaseName = hasTemplateDatabaseName(context) + ? context.templateDatabaseName + : ( + await ensureFactoryRealmTemplate({ + ...options, + realmDir, + realmURL, + realmServerURL, + context, + }) + ).templateDatabaseName; + } + + let stack; + try { + let baseRealmURL = baseRealmURLFor(realmServerURL); + let sourceRealmURL = sourceRealmURLFor(realmServerURL); + await dropDatabase(databaseName); + await cloneDatabaseFromTemplate(templateDatabaseName, databaseName); + if (options.templateRealmServerURL) { + await rewriteClonedRealmServerUrls( + databaseName, + options.templateRealmServerURL, + realmServerURL, + ); + } + await resetQueueState(databaseName); + await clearModuleCache(databaseName); + await rebuildWorkingIndexFromIndex(databaseName); + await warnIfSnapshotLooksCold(databaseName, [ + realmURL, + baseRealmURL, + sourceRealmURL, + ]); + await seedRealmPermissions( + databaseName, + baseRealmURL, + DEFAULT_BASE_REALM_PERMISSIONS, + ); + await seedRealmPermissions( + databaseName, + sourceRealmURL, + DEFAULT_SOURCE_REALM_PERMISSIONS, + ); + + stack = await startIsolatedRealmStack({ + realmDir, + realmURL, + realmServerURL, + databaseName, + context, + migrateDB: false, + fullIndexOnStartup: false, + }); + } catch (error) { + let cleanupError: unknown; + + try { + await dropDatabase(databaseName); + } catch (cleanupFailure) { + cleanupError ??= cleanupFailure; + } + + try { + await ownedGlobalContext?.stop(); + } catch (cleanupFailure) { + cleanupError ??= cleanupFailure; + } + + if (cleanupError) { + throw cleanupError; + } + + throw error; + } + + return { + realmDir, + realmURL, + realmServerURL, + databaseName, + ports: stack.ports, + childPids: [stack.realmServer.pid, stack.workerManager.pid].filter( + (pid): pid is number => pid != null, + ), + cardURL(path: string) { + return new URL(path, realmURL).href; + }, + createBearerToken( + user = DEFAULT_REALM_OWNER, + permissions?: RealmAction[], + ) { + return buildRealmToken(realmURL, realmServerURL, user, permissions); + }, + authorizationHeaders(user?: string, permissions?: RealmAction[]) { + return { + Authorization: `Bearer ${buildRealmToken( + realmURL, + realmServerURL, + user, + permissions, + )}`, + }; + }, + async stop() { + await logTimed( + realmLog, + `stopFactoryRealmServer ${databaseName}`, + async () => { + let cleanupError: unknown; + + try { + await stopIsolatedRealmStack(stack); + } catch (error) { + cleanupError ??= error; + } + + try { + await dropDatabase(databaseName); + } catch (error) { + cleanupError ??= error; + } + + try { + await ownedGlobalContext?.stop(); + } catch (error) { + cleanupError ??= error; + } + + if (cleanupError) { + throw cleanupError; + } + }, + ); + }, + }; + }); +} + +export async function fetchRealmCardJson( + path: string, + options: FactoryRealmOptions = {}, +) { + return await logTimed(harnessLog, `fetchRealmCardJson ${path}`, async () => { + let runtime = await startFactoryRealmServer(options); + try { + let response = await fetch(runtime.cardURL(path), { + headers: { + Accept: 'application/vnd.card+json', + }, + }); + return { + status: response.status, + body: await response.text(), + url: response.url, + }; + } finally { + await runtime.stop(); + } + }); +} diff --git a/packages/software-factory/src/harness/database.ts b/packages/software-factory/src/harness/database.ts new file mode 100644 index 00000000000..703b5bebaac --- /dev/null +++ b/packages/software-factory/src/harness/database.ts @@ -0,0 +1,636 @@ +import { Client as PgClient } from 'pg'; + +import { + baseRealmURLFor, + builderDatabaseNameForCacheKey, + DEFAULT_BASE_REALM_PERMISSIONS, + DEFAULT_MIGRATED_TEMPLATE_DB, + DEFAULT_SOURCE_REALM_PERMISSIONS, + logTimed, + pgAdminConnectionConfig, + quotePgIdentifier, + sourceRealmURLFor, + templateLog, + waitUntil, + type FactorySupportContext, + type RealmPermissions, +} from './shared'; +import { + startIsolatedRealmStack, + stopIsolatedRealmStack, +} from './isolated-realm-stack'; + +export async function canConnectToPg(): Promise { + let client = new PgClient({ + ...pgAdminConnectionConfig(), + connectionTimeoutMillis: 1000, + }); + try { + await client.connect(); + await client.query('SELECT 1'); + return true; + } catch { + return false; + } finally { + try { + await client.end(); + } catch { + // best effort cleanup + } + } +} + +export async function databaseExists(databaseName: string): Promise { + let client = new PgClient(pgAdminConnectionConfig()); + try { + await client.connect(); + let result = await client.query<{ exists: boolean }>( + 'SELECT EXISTS(SELECT 1 FROM pg_database WHERE datname = $1) AS exists', + [databaseName], + ); + return Boolean(result.rows[0]?.exists); + } finally { + await client.end(); + } +} + +export async function dropDatabase(databaseName: string): Promise { + await logTimed(templateLog, `dropDatabase ${databaseName}`, async () => { + let client = new PgClient(pgAdminConnectionConfig()); + try { + await client.connect(); + await client.query( + `SELECT pg_terminate_backend(pid) + FROM pg_stat_activity + WHERE datname = $1 AND pid <> pg_backend_pid()`, + [databaseName], + ); + await client.query( + `DROP DATABASE IF EXISTS ${quotePgIdentifier(databaseName)}`, + ); + } finally { + await client.end(); + } + }); +} + +export async function cloneDatabaseFromTemplate( + templateDatabaseName: string, + databaseName: string, +): Promise { + await logTimed( + templateLog, + `cloneDatabaseFromTemplate ${templateDatabaseName} -> ${databaseName}`, + async () => { + let client = new PgClient(pgAdminConnectionConfig()); + try { + await client.connect(); + await client.query( + `CREATE DATABASE ${quotePgIdentifier(databaseName)} TEMPLATE ${quotePgIdentifier( + templateDatabaseName, + )}`, + ); + } finally { + await client.end(); + } + }, + ); +} + +export async function createTemplateSnapshot( + sourceDatabaseName: string, + templateDatabaseName: string, +): Promise { + await logTimed( + templateLog, + `createTemplateSnapshot ${sourceDatabaseName} -> ${templateDatabaseName}`, + async () => { + let client = new PgClient(pgAdminConnectionConfig()); + try { + await client.connect(); + await client.query( + `SELECT pg_terminate_backend(pid) + FROM pg_stat_activity + WHERE datname = $1 AND pid <> pg_backend_pid()`, + [templateDatabaseName], + ); + await client.query( + `DROP DATABASE IF EXISTS ${quotePgIdentifier(templateDatabaseName)}`, + ); + await client.query( + `CREATE DATABASE ${quotePgIdentifier(templateDatabaseName)} TEMPLATE ${quotePgIdentifier( + sourceDatabaseName, + )}`, + ); + await client.query( + `ALTER DATABASE ${quotePgIdentifier(templateDatabaseName)} WITH IS_TEMPLATE true`, + ); + } finally { + await client.end(); + } + }, + ); +} + +export async function seedRealmPermissions( + databaseName: string, + realmURL: URL, + permissions: RealmPermissions, +): Promise { + await logTimed( + templateLog, + `seedRealmPermissions ${databaseName} ${realmURL.href}`, + async () => { + let client = new PgClient(pgAdminConnectionConfig(databaseName)); + try { + await client.connect(); + await client.query('BEGIN'); + + for (let [username, actions] of Object.entries(permissions)) { + if (!actions || actions.length === 0) { + await client.query( + `DELETE FROM realm_user_permissions + WHERE realm_url = $1 AND username = $2`, + [realmURL.href, username], + ); + continue; + } + + if (username !== '*') { + await client.query( + `INSERT INTO users (matrix_user_id) + VALUES ($1) + ON CONFLICT (matrix_user_id) DO NOTHING`, + [username], + ); + } + + await client.query( + `INSERT INTO realm_user_permissions ( + realm_url, + username, + read, + write, + realm_owner + ) VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (realm_url, username) DO UPDATE + SET read = EXCLUDED.read, + write = EXCLUDED.write, + realm_owner = EXCLUDED.realm_owner`, + [ + realmURL.href, + username, + actions.includes('read'), + actions.includes('write'), + actions.includes('realm-owner'), + ], + ); + } + + await client.query('COMMIT'); + } catch (error) { + try { + await client.query('ROLLBACK'); + } catch { + // best effort cleanup + } + throw error; + } finally { + await client.end(); + } + }, + ); +} + +export async function resetRealmState( + databaseName: string, + realmURL: URL, +): Promise { + await logTimed( + templateLog, + `resetRealmState ${databaseName} ${realmURL.href}`, + async () => { + let client = new PgClient(pgAdminConnectionConfig(databaseName)); + try { + await client.connect(); + await client.query('BEGIN'); + + await client.query( + `DELETE FROM modules WHERE resolved_realm_url = $1`, + [realmURL.href], + ); + await client.query(`DELETE FROM boxel_index WHERE realm_url = $1`, [ + realmURL.href, + ]); + await client.query( + `DELETE FROM boxel_index_working WHERE realm_url = $1`, + [realmURL.href], + ); + await client.query(`DELETE FROM realm_versions WHERE realm_url = $1`, [ + realmURL.href, + ]); + await client.query(`DELETE FROM realm_file_meta WHERE realm_url = $1`, [ + realmURL.href, + ]); + await client.query( + `DELETE FROM published_realms + WHERE source_realm_url = $1 OR published_realm_url = $1`, + [realmURL.href], + ); + + await client.query('COMMIT'); + } catch (error) { + try { + await client.query('ROLLBACK'); + } catch { + // best effort cleanup + } + throw error; + } finally { + await client.end(); + } + }, + ); +} + +export async function resetMountedRealmState( + databaseName: string, + realmURLs: URL[], +): Promise { + await logTimed( + templateLog, + `resetMountedRealmState ${databaseName} (${realmURLs.length} realms)`, + async () => { + for (let realmURL of realmURLs) { + await resetRealmState(databaseName, realmURL); + } + }, + ); +} + +export async function resetQueueState(databaseName: string): Promise { + await logTimed(templateLog, `resetQueueState ${databaseName}`, async () => { + let client = new PgClient(pgAdminConnectionConfig(databaseName)); + try { + await client.connect(); + await client.query('BEGIN'); + await client.query(`DELETE FROM job_reservations`); + await client.query(`DELETE FROM jobs`); + await client.query('COMMIT'); + } catch (error) { + try { + await client.query('ROLLBACK'); + } catch { + // best effort cleanup + } + throw error; + } finally { + await client.end(); + } + }); +} + +export async function clearModuleCache(databaseName: string): Promise { + await logTimed(templateLog, `clearModuleCache ${databaseName}`, async () => { + let client = new PgClient(pgAdminConnectionConfig(databaseName)); + try { + await client.connect(); + await client.query(`DELETE FROM modules`); + } finally { + await client.end(); + } + }); +} + +// When schema changes add new persisted URL-bearing columns or tables, this +// rewrite pass needs to be updated as well. If that coverage drifts, cloned +// harness databases quietly fall back toward cold indexing instead of reusing +// the prepared snapshot. +export async function rewriteClonedRealmServerUrls( + databaseName: string, + fromRealmServerURL: URL, + toRealmServerURL: URL, +): Promise { + if (fromRealmServerURL.href === toRealmServerURL.href) { + return; + } + + await logTimed( + templateLog, + `rewriteClonedRealmServerUrls ${databaseName} ${fromRealmServerURL.href} -> ${toRealmServerURL.href}`, + async () => { + let client = new PgClient(pgAdminConnectionConfig(databaseName)); + let fromURL = fromRealmServerURL.href; + let toURL = toRealmServerURL.href; + try { + await client.connect(); + await client.query('BEGIN'); + + await client.query( + `UPDATE boxel_index + SET url = replace(url, $1, $2), + file_alias = replace(file_alias, $1, $2), + realm_url = replace(realm_url, $1, $2), + pristine_doc = replace(pristine_doc::text, $1, $2)::jsonb, + search_doc = replace(search_doc::text, $1, $2)::jsonb, + error_doc = replace(error_doc::text, $1, $2)::jsonb, + deps = replace(deps::text, $1, $2)::jsonb, + types = replace(types::text, $1, $2)::jsonb, + isolated_html = replace(isolated_html, $1, $2), + embedded_html = replace(embedded_html::text, $1, $2)::jsonb, + atom_html = replace(atom_html, $1, $2), + fitted_html = replace(fitted_html::text, $1, $2)::jsonb, + display_names = replace(display_names::text, $1, $2)::jsonb, + icon_html = replace(icon_html, $1, $2), + head_html = replace(head_html, $1, $2), + last_known_good_deps = replace(last_known_good_deps::text, $1, $2)::jsonb`, + [fromURL, toURL], + ); + + await client.query( + `UPDATE realm_versions + SET realm_url = replace(realm_url, $1, $2)`, + [fromURL, toURL], + ); + await client.query( + `UPDATE realm_file_meta + SET realm_url = replace(realm_url, $1, $2)`, + [fromURL, toURL], + ); + await client.query( + `UPDATE realm_user_permissions + SET realm_url = replace(realm_url, $1, $2)`, + [fromURL, toURL], + ); + await client.query( + `UPDATE realm_meta + SET realm_url = replace(realm_url, $1, $2), + value = replace(value::text, $1, $2)::jsonb`, + [fromURL, toURL], + ); + await client.query( + `UPDATE published_realms + SET source_realm_url = replace(source_realm_url, $1, $2), + published_realm_url = replace(published_realm_url, $1, $2)`, + [fromURL, toURL], + ); + await client.query( + `UPDATE session_rooms + SET realm_url = replace(realm_url, $1, $2)`, + [fromURL, toURL], + ); + + await client.query('COMMIT'); + } catch (error) { + try { + await client.query('ROLLBACK'); + } catch { + // best effort cleanup + } + throw error; + } finally { + await client.end(); + } + }, + ); +} + +export async function rebuildWorkingIndexFromIndex( + databaseName: string, +): Promise { + await logTimed( + templateLog, + `rebuildWorkingIndexFromIndex ${databaseName}`, + async () => { + let client = new PgClient(pgAdminConnectionConfig(databaseName)); + try { + await client.connect(); + await client.query('BEGIN'); + await client.query(`DELETE FROM boxel_index_working`); + await client.query( + `INSERT INTO boxel_index_working ( + url, + file_alias, + type, + realm_version, + realm_url, + pristine_doc, + search_doc, + error_doc, + deps, + types, + icon_html, + isolated_html, + indexed_at, + is_deleted, + last_modified, + embedded_html, + atom_html, + fitted_html, + display_names, + resource_created_at, + head_html, + has_error, + last_known_good_deps + ) + SELECT + url, + file_alias, + type, + realm_version, + realm_url, + pristine_doc, + search_doc, + error_doc, + deps, + types, + icon_html, + isolated_html, + indexed_at, + is_deleted, + last_modified, + embedded_html, + atom_html, + fitted_html, + display_names, + resource_created_at, + head_html, + has_error, + last_known_good_deps + FROM boxel_index`, + ); + await client.query('COMMIT'); + } catch (error) { + try { + await client.query('ROLLBACK'); + } catch { + // best effort cleanup + } + throw error; + } finally { + await client.end(); + } + }, + ); +} + +export async function warnIfSnapshotLooksCold( + databaseName: string, + realmURLs: URL[], +): Promise { + await logTimed( + templateLog, + `warnIfSnapshotLooksCold ${databaseName}`, + async () => { + let client = new PgClient(pgAdminConnectionConfig(databaseName)); + try { + await client.connect(); + + let missing: string[] = []; + for (let realmURL of realmURLs) { + let indexResult = await client.query<{ count: number }>( + `SELECT COUNT(*)::int AS count + FROM boxel_index + WHERE realm_url = $1`, + [realmURL.href], + ); + let versionResult = await client.query<{ count: number }>( + `SELECT COUNT(*)::int AS count + FROM realm_versions + WHERE realm_url = $1`, + [realmURL.href], + ); + + if ( + (indexResult.rows[0]?.count ?? 0) === 0 || + (versionResult.rows[0]?.count ?? 0) === 0 + ) { + missing.push(realmURL.href); + } + } + + if (missing.length > 0) { + templateLog.warn( + `cloned harness snapshot is missing preindexed coverage for ${missing.join( + ', ', + )}; runtime may do a cold/full index. If schema or persisted index fields changed, update rewriteClonedRealmServerUrls() and rebuildWorkingIndexFromIndex().`, + ); + } + } finally { + await client.end(); + } + }, + ); +} + +export async function waitForQueueIdle(databaseName: string): Promise { + await logTimed(templateLog, `waitForQueueIdle ${databaseName}`, async () => { + await waitUntil( + async () => { + let client = new PgClient(pgAdminConnectionConfig(databaseName)); + try { + await client.connect(); + let { + rows: [{ count: unfulfilledJobs }], + } = await client.query<{ count: number }>( + `SELECT COUNT(*)::int AS count FROM jobs WHERE status = 'unfulfilled'`, + ); + let { + rows: [{ count: activeReservations }], + } = await client.query<{ count: number }>( + `SELECT COUNT(*)::int AS count FROM job_reservations WHERE completed_at IS NULL`, + ); + return unfulfilledJobs === 0 && activeReservations === 0; + } finally { + await client.end(); + } + }, + { + timeout: 30_000, + interval: 100, + timeoutMessage: `Timed out waiting for queue to become idle in ${databaseName}`, + }, + ); + }); +} + +export async function buildTemplateDatabase({ + realmDir, + realmURL, + realmServerURL, + permissions, + context, + cacheKey, + templateDatabaseName, +}: { + realmDir: string; + realmURL: URL; + realmServerURL: URL; + permissions: RealmPermissions; + context: FactorySupportContext; + cacheKey: string; + templateDatabaseName: string; +}): Promise { + await logTimed( + templateLog, + `buildTemplateDatabase ${templateDatabaseName}`, + async () => { + let builderDatabaseName = builderDatabaseNameForCacheKey(cacheKey); + let hasMigratedTemplate = await databaseExists( + DEFAULT_MIGRATED_TEMPLATE_DB, + ); + + templateLog.debug( + `buildTemplateDatabase: builder=${builderDatabaseName} migratedTemplate=${hasMigratedTemplate}`, + ); + await dropDatabase(templateDatabaseName); + await dropDatabase(builderDatabaseName); + + if (hasMigratedTemplate) { + await cloneDatabaseFromTemplate( + DEFAULT_MIGRATED_TEMPLATE_DB, + builderDatabaseName, + ); + } + let baseRealmURL = baseRealmURLFor(realmServerURL); + let sourceRealmURL = sourceRealmURLFor(realmServerURL); + + await resetMountedRealmState(builderDatabaseName, [ + realmURL, + baseRealmURL, + sourceRealmURL, + ]); + await resetQueueState(builderDatabaseName); + await seedRealmPermissions(builderDatabaseName, realmURL, permissions); + await seedRealmPermissions( + builderDatabaseName, + baseRealmURL, + DEFAULT_BASE_REALM_PERMISSIONS, + ); + await seedRealmPermissions( + builderDatabaseName, + sourceRealmURL, + DEFAULT_SOURCE_REALM_PERMISSIONS, + ); + + let stack = await startIsolatedRealmStack({ + realmDir, + realmURL, + realmServerURL, + databaseName: builderDatabaseName, + context, + migrateDB: !hasMigratedTemplate, + fullIndexOnStartup: true, + }); + + try { + await waitForQueueIdle(builderDatabaseName); + } finally { + await stopIsolatedRealmStack(stack); + } + + await createTemplateSnapshot(builderDatabaseName, templateDatabaseName); + await dropDatabase(builderDatabaseName); + }, + ); +} diff --git a/packages/software-factory/src/harness/isolated-realm-stack.ts b/packages/software-factory/src/harness/isolated-realm-stack.ts new file mode 100644 index 00000000000..556b89e1d80 --- /dev/null +++ b/packages/software-factory/src/harness/isolated-realm-stack.ts @@ -0,0 +1,567 @@ +import { + createServer, + type IncomingMessage, + type ServerResponse, +} from 'node:http'; +import { + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { join, relative } from 'node:path'; +import { tmpdir } from 'node:os'; +import fsExtra from 'fs-extra'; +import { spawn } from 'node:child_process'; + +import { + baseRealmDir, + baseRealmURLFor, + captureProcessLogs, + createProcessExitPromise, + DEFAULT_MATRIX_SERVER_USERNAME, + DEFAULT_PG_HOST, + DEFAULT_PG_POOL_MAX, + DEFAULT_PG_PORT, + DEFAULT_PG_USER, + DEFAULT_REALM_LOG_LEVELS, + DEFAULT_REALM_SERVER_PORT, + DEFAULT_WORKER_MANAGER_PORT, + findAvailablePort, + FIXTURE_SOURCE_REALM_URL_PLACEHOLDER, + FULL_INDEX_REALM_STARTUP_TIMEOUT_MS, + INCLUDE_SKILLS, + managedProcessStdio, + realmLog, + realmRelativePath, + realmServerDir, + realmURLWithinServer, + REALM_SECRET_SEED, + REALM_SERVER_SECRET_SEED, + shouldIgnoreFixturePath, + skillsRealmDir, + skillsRealmURLFor, + sourceRealmDir, + sourceRealmURLFor, + GRAFANA_SECRET, + waitForJsonFile, + waitForReady, + withPort, + DEFAULT_REALM_STARTUP_TIMEOUT_MS, + stopManagedProcess, + type FactorySupportContext, + type RunningFactoryStack, + type SpawnedProcess, + type StartedCompatRealmProxy, +} from './shared'; +import { startHarnessPrerenderServer } from './support-services'; + +const { copySync, ensureDirSync } = fsExtra; + +async function readIncomingRequestBody( + req: IncomingMessage, +): Promise { + let chunks: Buffer[] = []; + for await (let chunk of req) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + return chunks.length ? Buffer.concat(chunks) : undefined; +} + +function describeCompatProxyError(error: unknown): string { + if (!(error instanceof Error)) { + return String(error); + } + + let parts: string[] = []; + let current: unknown = error; + + while (current) { + if (current instanceof Error) { + let code = + 'code' in current && typeof current.code === 'string' + ? ` (${current.code})` + : ''; + parts.push(`${current.message}${code}`); + current = current.cause; + } else { + parts.push(String(current)); + break; + } + } + + return parts.join(' <- '); +} + +async function startCompatRealmProxy({ + listenPort, +}: { + listenPort: number; +}): Promise { + realmLog.debug(`startCompatRealmProxy: requested listenPort=${listenPort}`); + let targetPort: number | undefined; + let actualListenPort = listenPort; + let server = createServer( + async (req: IncomingMessage, res: ServerResponse) => { + if (targetPort == null) { + res.statusCode = 503; + res.setHeader('content-type', 'text/plain; charset=utf-8'); + res.end('software-factory compat proxy target is not ready'); + return; + } + let incomingURL = new URL( + req.url ?? '/', + `${ + req.headers['x-forwarded-proto'] === 'https' ? 'https' : 'http' + }://${req.headers.host ?? `127.0.0.1:${actualListenPort}`}`, + ); + let upstreamURL = new URL( + `${incomingURL.pathname}${incomingURL.search}`, + `http://localhost:${targetPort}`, + ); + + try { + let body = await readIncomingRequestBody(req); + let headers = Object.fromEntries( + Object.entries(req.headers).filter( + ([key]) => key.toLowerCase() !== 'host', + ), + ) as Record; + headers['x-boxel-forwarded-url'] = incomingURL.href; + let response = await fetch(upstreamURL, { + method: req.method, + headers, + body: body as BodyInit | undefined, + redirect: 'manual', + }); + + let responseHeaders = new Headers(response.headers); + let location = responseHeaders.get('location'); + if (location) { + responseHeaders.set( + 'location', + location + .replace( + `http://localhost:${targetPort}/`, + `http://127.0.0.1:${listenPort}/`, + ) + .replace( + `http://localhost:${targetPort}/`, + `http://localhost:${listenPort}/`, + ), + ); + } + + res.statusCode = response.status; + responseHeaders.forEach((value, key) => { + res.setHeader(key, value); + }); + res.end(Buffer.from(await response.arrayBuffer())); + } catch (error) { + let description = describeCompatProxyError(error); + realmLog.warn( + `startCompatRealmProxy: upstream fetch failed for ${upstreamURL.href}: ${description}`, + ); + res.statusCode = 502; + res.setHeader('content-type', 'text/plain; charset=utf-8'); + res.end( + `software-factory compat proxy failed for ${upstreamURL.href}: ${description}`, + ); + } + }, + ); + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(listenPort, '127.0.0.1', () => resolve()); + }); + let address = server.address(); + if (!address || typeof address === 'string') { + throw new Error('Unable to determine compat proxy port'); + } + actualListenPort = address.port; + realmLog.debug(`startCompatRealmProxy: listening on ${actualListenPort}`); + return { + listenPort: actualListenPort, + setTargetPort(nextTargetPort: number) { + targetPort = nextTargetPort; + realmLog.debug( + `startCompatRealmProxy: ${actualListenPort} -> ${nextTargetPort} ready`, + ); + }, + async stop() { + realmLog.debug( + `startCompatRealmProxy: ${actualListenPort} -> ${targetPort ?? 'unset'} stopping`, + ); + await new Promise((resolve, reject) => { + server.close((error) => { + if (error) { + reject(error); + } else { + resolve(); + } + }); + }); + }, + }; +} + +function rewriteFixtureSourceModuleUrls( + destination: string, + sourceRealmURL: URL, +): void { + let rewrittenFiles = 0; + + function visit(currentDir: string) { + for (let entry of readdirSync(currentDir, { withFileTypes: true })) { + let absolutePath = join(currentDir, entry.name); + if (entry.isDirectory()) { + visit(absolutePath); + continue; + } + if (!entry.isFile() || !entry.name.endsWith('.json')) { + continue; + } + + let contents = readFileSync(absolutePath, 'utf8'); + if (!contents.includes(FIXTURE_SOURCE_REALM_URL_PLACEHOLDER)) { + continue; + } + + writeFileSync( + absolutePath, + contents + .split(FIXTURE_SOURCE_REALM_URL_PLACEHOLDER) + .join(sourceRealmURL.href), + ); + rewrittenFiles++; + } + } + + visit(destination); + if (rewrittenFiles > 0) { + realmLog.debug( + `rewriteFixtureSourceModuleUrls: rewrote ${rewrittenFiles} files to ${sourceRealmURL.href}`, + ); + } +} + +function copyRealmFixture( + realmDir: string, + destination: string, + sourceRealmURL: URL, +): void { + copySync(realmDir, destination, { + preserveTimestamps: true, + filter(src) { + let relativePath = relative(realmDir, src).replace(/\\/g, '/'); + return relativePath === '' || !shouldIgnoreFixturePath(relativePath); + }, + }); + rewriteFixtureSourceModuleUrls(destination, sourceRealmURL); +} + +export async function startIsolatedRealmStack({ + realmDir, + realmURL, + realmServerURL, + databaseName, + context, + migrateDB, + fullIndexOnStartup, +}: { + realmDir: string; + realmURL: URL; + realmServerURL: URL; + databaseName: string; + context: FactorySupportContext; + migrateDB: boolean; + fullIndexOnStartup: boolean; +}): Promise { + let rootDir = mkdtempSync(join(tmpdir(), 'software-factory-realms-')); + let testRealmDir = join(rootDir, 'test'); + let workerManagerMetadataFile = join(rootDir, 'worker-manager.runtime.json'); + let realmServerMetadataFile = join(rootDir, 'realm-server.runtime.json'); + let actualRealmServerPort = + DEFAULT_REALM_SERVER_PORT === 0 + ? await findAvailablePort() + : DEFAULT_REALM_SERVER_PORT; + let actualRealmServerURL = withPort(realmServerURL, actualRealmServerPort); + let actualRealmPath = realmRelativePath(realmURL, realmServerURL); + let actualRealmURL = realmURLWithinServer( + actualRealmServerURL, + actualRealmPath, + ); + let legacyRealmServerURL = new URL('http://localhost:4205/'); + let legacyRealmURL = new URL('test/', legacyRealmServerURL); + let publicBaseRealmURL = baseRealmURLFor(realmServerURL); + let actualBaseRealmURL = baseRealmURLFor(actualRealmServerURL); + let sourceRealmURL = sourceRealmURLFor(realmServerURL); + let actualSourceRealmURL = sourceRealmURLFor(actualRealmServerURL); + let legacySourceRealmURL = sourceRealmURLFor(legacyRealmServerURL); + let skillsRealmURL = skillsRealmURLFor(realmServerURL); + let actualSkillsRealmURL = skillsRealmURLFor(actualRealmServerURL); + let legacySkillsRealmURL = skillsRealmURLFor(legacyRealmServerURL); + ensureDirSync(testRealmDir); + copyRealmFixture(realmDir, testRealmDir, sourceRealmURL); + realmLog.debug( + `startIsolatedRealmStack: copied fixture ${realmDir} -> ${testRealmDir}`, + ); + let compatProxy = await startCompatRealmProxy({ + listenPort: Number(realmServerURL.port), + }); + let prerender = await startHarnessPrerenderServer({ + boxelHostURL: realmServerURL.href.replace(/\/$/, ''), + }); + + let env = { + ...process.env, + PGHOST: DEFAULT_PG_HOST, + PGPORT: DEFAULT_PG_PORT, + PGUSER: DEFAULT_PG_USER, + PG_POOL_MAX: String(DEFAULT_PG_POOL_MAX), + PGDATABASE: databaseName, + NODE_NO_WARNINGS: '1', + NODE_ENV: 'test', + REALM_SERVER_SECRET_SEED, + REALM_SECRET_SEED, + GRAFANA_SECRET, + MATRIX_URL: context.matrixURL, + MATRIX_REGISTRATION_SHARED_SECRET: context.matrixRegistrationSecret, + REALM_SERVER_MATRIX_USERNAME: DEFAULT_MATRIX_SERVER_USERNAME, + REALM_SERVER_FULL_INDEX_ON_STARTUP: String(fullIndexOnStartup), + LOW_CREDIT_THRESHOLD: '2000', + LOG_LEVELS: DEFAULT_REALM_LOG_LEVELS, + PUBLISHED_REALM_BOXEL_SPACE_DOMAIN: `localhost:${compatProxy.listenPort}`, + PUBLISHED_REALM_BOXEL_SITE_DOMAIN: `localhost:${compatProxy.listenPort}`, + SOFTWARE_FACTORY_WORKER_MANAGER_METADATA_FILE: workerManagerMetadataFile, + SOFTWARE_FACTORY_REALM_SERVER_METADATA_FILE: realmServerMetadataFile, + }; + + let workerArgs = [ + '--transpileOnly', + 'worker-manager', + `--port=${DEFAULT_WORKER_MANAGER_PORT}`, + `--matrixURL=${context.matrixURL}`, + `--prerendererUrl=${prerender.url}`, + `--fromUrl=${realmURL.href}`, + `--toUrl=${actualRealmURL.href}`, + `--fromUrl=${publicBaseRealmURL.href}`, + `--toUrl=${actualBaseRealmURL.href}`, + '--fromUrl=https://cardstack.com/base/', + `--toUrl=${publicBaseRealmURL.href}`, + `--fromUrl=${sourceRealmURL.href}`, + `--toUrl=${actualSourceRealmURL.href}`, + ]; + if (INCLUDE_SKILLS) { + workerArgs.push( + `--fromUrl=${skillsRealmURL.href}`, + `--toUrl=${actualSkillsRealmURL.href}`, + ); + } + workerArgs.push( + `--fromUrl=${legacyRealmURL.href}`, + `--toUrl=${actualRealmURL.href}`, + `--fromUrl=${legacySourceRealmURL.href}`, + `--toUrl=${actualSourceRealmURL.href}`, + ); + if (INCLUDE_SKILLS) { + workerArgs.push( + `--fromUrl=${legacySkillsRealmURL.href}`, + `--toUrl=${actualSkillsRealmURL.href}`, + ); + } + if (migrateDB) { + workerArgs.splice(5, 0, '--migrateDB'); + } + + let workerManager = spawn('ts-node', workerArgs, { + cwd: realmServerDir, + env, + stdio: managedProcessStdio, + }) as SpawnedProcess; + let getWorkerLogs = captureProcessLogs(workerManager); + workerManager.on('exit', (code, signal) => { + if (code === 0 || signal === 'SIGTERM' || signal === 'SIGINT') { + return; + } + realmLog.warn( + `worker manager exited unexpectedly (code: ${code}, signal: ${signal})\n${getWorkerLogs()}`, + ); + }); + let workerManagerRuntime = await waitForJsonFile<{ + pid: number; + port: number; + url: string; + }>(workerManagerMetadataFile, getWorkerLogs, { + label: 'worker manager', + process: workerManager, + }); + + let serverArgs = [ + '--transpileOnly', + 'main', + `--port=${actualRealmServerPort}`, + `--serverURL=${realmServerURL.href}`, + `--matrixURL=${context.matrixURL}`, + `--realmsRootPath=${rootDir}`, + `--workerManagerUrl=${workerManagerRuntime.url}`, + `--prerendererUrl=${prerender.url}`, + '--username=base_realm', + `--path=${baseRealmDir}`, + `--fromUrl=${publicBaseRealmURL.href}`, + `--toUrl=${actualBaseRealmURL.href}`, + '--username=software_factory_realm', + `--path=${sourceRealmDir}`, + `--fromUrl=${sourceRealmURL.href}`, + `--toUrl=${actualSourceRealmURL.href}`, + '--username=test_realm', + `--path=${testRealmDir}`, + `--fromUrl=${realmURL.href}`, + `--toUrl=${actualRealmURL.href}`, + ]; + if (INCLUDE_SKILLS) { + serverArgs.splice( + 16, + 0, + '--username=skills_realm', + `--path=${skillsRealmDir}`, + `--fromUrl=${skillsRealmURL.href}`, + `--toUrl=${actualSkillsRealmURL.href}`, + ); + } + serverArgs.push( + `--fromUrl=${legacyRealmURL.href}`, + `--toUrl=${actualRealmURL.href}`, + `--fromUrl=${legacySourceRealmURL.href}`, + `--toUrl=${actualSourceRealmURL.href}`, + ); + if (INCLUDE_SKILLS) { + serverArgs.push( + `--fromUrl=${legacySkillsRealmURL.href}`, + `--toUrl=${actualSkillsRealmURL.href}`, + ); + } + + let realmServer = spawn('ts-node', serverArgs, { + cwd: realmServerDir, + env, + stdio: managedProcessStdio, + }) as SpawnedProcess; + let getServerLogs = captureProcessLogs(realmServer); + realmServer.on('exit', (code, signal) => { + if (code === 0 || signal === 'SIGTERM' || signal === 'SIGINT') { + return; + } + realmLog.warn( + `realm server exited unexpectedly (code: ${code}, signal: ${signal})\n${getServerLogs()}`, + ); + }); + + try { + let realmServerRuntime = await waitForJsonFile<{ + pid: number; + port: number; + }>(realmServerMetadataFile, getServerLogs, { + label: 'realm server', + process: realmServer, + }); + compatProxy.setTargetPort(realmServerRuntime.port); + await Promise.race([ + waitForReady( + realmServer, + 'realm server', + fullIndexOnStartup + ? FULL_INDEX_REALM_STARTUP_TIMEOUT_MS + : DEFAULT_REALM_STARTUP_TIMEOUT_MS, + () => + [ + 'realm server logs:', + getServerLogs(), + 'worker manager logs:', + getWorkerLogs(), + ] + .filter((entry) => entry && entry.trim().length > 0) + .join('\n\n'), + ), + createProcessExitPromise(workerManager, 'worker manager'), + ]); + + return { + compatProxy, + prerender, + realmServer, + realmServerURL, + ports: { + publicPort: compatProxy.listenPort, + realmServerPort: realmServerRuntime.port, + workerManagerPort: workerManagerRuntime.port, + }, + workerManager, + rootDir, + }; + } catch (error) { + try { + await prerender.stop(); + } catch { + // best effort cleanup + } + try { + await stopManagedProcess(realmServer); + } catch { + // best effort cleanup + } + try { + await stopManagedProcess(workerManager); + } catch { + // best effort cleanup + } + try { + await compatProxy?.stop(); + } catch { + // best effort cleanup + } + rmSync(rootDir, { recursive: true, force: true }); + throw error; + } +} + +export async function stopIsolatedRealmStack( + stack: RunningFactoryStack, +): Promise { + let cleanupError: unknown; + + try { + await stack.prerender.stop(); + } catch (error) { + cleanupError ??= error; + } + + try { + await stopManagedProcess(stack.realmServer); + } catch (error) { + cleanupError ??= error; + } + + try { + await stopManagedProcess(stack.workerManager); + } catch (error) { + cleanupError ??= error; + } + + try { + await stack.compatProxy?.stop(); + } catch (error) { + cleanupError ??= error; + } + + try { + rmSync(stack.rootDir, { recursive: true, force: true }); + } catch (error) { + cleanupError ??= error; + } + + if (cleanupError) { + throw cleanupError; + } +} diff --git a/packages/software-factory/src/harness/shared.ts b/packages/software-factory/src/harness/shared.ts new file mode 100644 index 00000000000..e2eb7187a17 --- /dev/null +++ b/packages/software-factory/src/harness/shared.ts @@ -0,0 +1,752 @@ +import { + spawnSync, + type ChildProcess, + type StdioOptions, +} from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { createServer as createNetServer } from 'node:net'; +import { readdirSync, readFileSync, statSync } from 'node:fs'; +import { join, relative, resolve } from 'node:path'; + +import jwt from 'jsonwebtoken'; +import '../setup-logger'; +import { logger } from '../logger'; + +export type RealmAction = 'read' | 'write' | 'realm-owner' | 'assume-user'; + +export type RealmPermissions = Record; + +export type FactorySupportContext = { + matrixURL: string; + matrixRegistrationSecret: string; +}; + +export type SynapseInstance = { + synapseId: string; + port: number; + registrationSecret: string; +}; + +export interface FactoryRealmOptions { + realmDir?: string; + realmURL?: URL; + realmServerURL?: URL; + templateRealmServerURL?: URL; + permissions?: RealmPermissions; + useCache?: boolean; + cacheSalt?: string; + templateDatabaseName?: string; + context?: FactoryTestContext | FactorySupportContext; +} + +export interface FactoryRealmTemplate { + cacheKey: string; + templateDatabaseName: string; + fixtureHash: string; + cacheHit: boolean; + realmURL: URL; + realmServerURL: URL; +} + +export interface FactoryTestContext extends FactorySupportContext { + cacheKey: string; + fixtureHash: string; + realmDir: string; + realmURL: string; + realmServerURL: string; + templateDatabaseName: string; +} + +export interface StartedFactoryRealm { + realmDir: string; + realmURL: URL; + realmServerURL: URL; + databaseName: string; + childPids: number[]; + ports: { + publicPort: number; + realmServerPort: number; + workerManagerPort: number; + }; + cardURL(path: string): string; + createBearerToken(user?: string, permissions?: RealmAction[]): string; + authorizationHeaders( + user?: string, + permissions?: RealmAction[], + ): Record; + stop(): Promise; +} + +export type FactoryGlobalContextHandle = { + context: FactoryTestContext; + stop(): Promise; +}; + +export type SpawnedProcess = ChildProcess & { + send(message: string): boolean; +}; + +export type StartedCompatRealmProxy = { + listenPort: number; + setTargetPort(targetPort: number): void; + stop(): Promise; +}; + +export type RunningFactoryStack = { + prerender: { + stop(): Promise; + }; + realmServer: SpawnedProcess; + realmServerURL: URL; + workerManager: SpawnedProcess; + compatProxy?: StartedCompatRealmProxy; + ports: { + publicPort: number; + realmServerPort: number; + workerManagerPort: number; + }; + rootDir: string; +}; + +export const packageRoot = resolve(process.cwd()); +export const workspaceRoot = resolve(packageRoot, '..', '..'); +export const realmServerDir = resolve(packageRoot, '..', 'realm-server'); +export const hostDir = resolve(packageRoot, '..', 'host'); +export const baseRealmDir = resolve(packageRoot, '..', 'base'); +export const skillsRealmDir = resolve( + packageRoot, + '..', + 'skills-realm', + 'contents', +); +export const sourceRealmDir = resolve( + packageRoot, + process.env.SOFTWARE_FACTORY_SOURCE_REALM_DIR ?? 'realm', +); +export const boxelIconsDir = resolve(packageRoot, '..', 'boxel-icons'); +export const prepareTestPgScript = resolve( + realmServerDir, + 'tests', + 'scripts', + 'prepare-test-pg.sh', +); + +export const CACHE_VERSION = 8; +export const DEFAULT_REALM_SERVER_PORT = Number( + process.env.SOFTWARE_FACTORY_REALM_PORT ?? 0, +); +export const DEFAULT_COMPAT_REALM_SERVER_PORT = Number( + process.env.SOFTWARE_FACTORY_COMPAT_REALM_PORT ?? 0, +); +export const DEFAULT_WORKER_MANAGER_PORT = Number( + process.env.SOFTWARE_FACTORY_WORKER_MANAGER_PORT ?? 0, +); +export const CONFIGURED_REALM_URL = process.env.SOFTWARE_FACTORY_REALM_URL + ? new URL(process.env.SOFTWARE_FACTORY_REALM_URL) + : undefined; +export const CONFIGURED_REALM_SERVER_URL = process.env + .SOFTWARE_FACTORY_REALM_SERVER_URL + ? new URL(process.env.SOFTWARE_FACTORY_REALM_SERVER_URL) + : undefined; +export const DEFAULT_REALM_DIR = resolve( + packageRoot, + process.env.SOFTWARE_FACTORY_REALM_DIR ?? 'test-fixtures/darkfactory-adopter', +); +export const DEFAULT_HOST_URL = + process.env.HOST_URL ?? 'http://localhost:4200/'; +export const DEFAULT_ICONS_URL = + process.env.ICONS_URL ?? 'http://localhost:4206/'; +export const DEFAULT_ICONS_PROBE_URL = new URL( + '@cardstack/boxel-icons/v1/icons/code.js', + DEFAULT_ICONS_URL, +).href; +export const DEFAULT_PG_PORT = process.env.SOFTWARE_FACTORY_PGPORT ?? '55436'; +export const DEFAULT_PG_HOST = + process.env.SOFTWARE_FACTORY_PGHOST ?? '127.0.0.1'; +export const DEFAULT_PG_USER = + process.env.SOFTWARE_FACTORY_PGUSER ?? 'postgres'; +export const DEFAULT_PRERENDER_PORT = Number( + process.env.SOFTWARE_FACTORY_PRERENDER_PORT ?? 0, +); +// The seeded test Postgres used by the harness runs with max_connections=20, so +// isolated workers need a smaller per-process pool cap to keep workers=2 stable. +export const DEFAULT_PG_POOL_MAX = Number( + process.env.SOFTWARE_FACTORY_PG_POOL_MAX ?? 2, +); +export const DEFAULT_MIGRATED_TEMPLATE_DB = + process.env.SOFTWARE_FACTORY_MIGRATED_TEMPLATE_DB ?? + 'boxel_migrated_template'; +export const DEFAULT_REALM_LOG_LEVELS = + process.env.SOFTWARE_FACTORY_REALM_LOG_LEVELS ?? + '*=info,realm:requests=warn,realm-index-updater=debug,index-runner=debug,index-perf=debug,index-writer=debug,worker=debug,worker-manager=debug,realm=debug,perf=debug'; +export const DEFAULT_REALM_OWNER = '@software-factory-owner:localhost'; +export const REALM_SECRET_SEED = "shhh! it's a secret"; +export const REALM_SERVER_SECRET_SEED = "mum's the word"; +export const GRAFANA_SECRET = "shhh! it's a secret"; +export const FIXTURE_SOURCE_REALM_URL_PLACEHOLDER = 'https://sf.boxel.test/'; +export const DEFAULT_MATRIX_SERVER_USERNAME = + process.env.SOFTWARE_FACTORY_MATRIX_SERVER_USERNAME ?? 'realm_server'; +export const DEFAULT_MATRIX_BROWSER_USERNAME = + process.env.SOFTWARE_FACTORY_BROWSER_MATRIX_USERNAME ?? + 'software-factory-browser'; +export const INCLUDE_SKILLS = + process.env.SOFTWARE_FACTORY_INCLUDE_SKILLS === '1'; +export const DEFAULT_PERMISSIONS: RealmPermissions = { + '*': ['read'], + [DEFAULT_REALM_OWNER]: ['read', 'write', 'realm-owner'], +}; +export const DEFAULT_SOURCE_REALM_PERMISSIONS: RealmPermissions = { + '*': ['read'], + [DEFAULT_REALM_OWNER]: ['read', 'write', 'realm-owner'], +}; +export const DEFAULT_BASE_REALM_PERMISSIONS = DEFAULT_SOURCE_REALM_PERMISSIONS; +export const managedProcessStdio: StdioOptions = + process.env.SOFTWARE_FACTORY_DEBUG_SERVER === '1' + ? (['ignore', 'inherit', 'inherit', 'ipc'] as const) + : (['ignore', 'pipe', 'pipe', 'ipc'] as const); +export const DEFAULT_REALM_STARTUP_TIMEOUT_MS = Number( + process.env.SOFTWARE_FACTORY_REALM_STARTUP_TIMEOUT_MS ?? 120_000, +); +export const FULL_INDEX_REALM_STARTUP_TIMEOUT_MS = Number( + process.env.SOFTWARE_FACTORY_FULL_INDEX_REALM_STARTUP_TIMEOUT_MS ?? 600_000, +); + +export const harnessLog = logger('software-factory:harness'); +export const supportLog = logger('software-factory:harness:support'); +export const templateLog = logger('software-factory:harness:template'); +export const realmLog = logger('software-factory:harness:realm'); + +export function formatElapsedMs(elapsedMs: number): string { + return `${(elapsedMs / 1000).toFixed(1)}s`; +} + +export async function logTimed( + log: ReturnType, + label: string, + callback: () => Promise, +): Promise { + let startedAt = Date.now(); + log.debug(`${label}: starting`); + try { + let result = await callback(); + log.debug( + `${label}: finished in ${formatElapsedMs(Date.now() - startedAt)}`, + ); + return result; + } catch (error) { + log.warn( + `${label}: failed after ${formatElapsedMs(Date.now() - startedAt)}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + throw error; + } +} + +export function stableStringify(value: unknown): string { + if (value === null || typeof value !== 'object') { + return JSON.stringify(value); + } + if (Array.isArray(value)) { + return `[${value.map((item) => stableStringify(item)).join(',')}]`; + } + let record = value as Record; + let keys = Object.keys(record).sort(); + return `{${keys + .map((key) => `${JSON.stringify(key)}:${stableStringify(record[key])}`) + .join(',')}}`; +} + +export function hashString(value: string): string { + return createHash('sha256').update(value).digest('hex'); +} + +export async function findAvailablePort(): Promise { + return await new Promise((resolve, reject) => { + let server = createNetServer(); + server.once('error', reject); + server.listen(0, '127.0.0.1', () => { + let address = server.address(); + if (!address || typeof address === 'string') { + reject(new Error('Unable to determine allocated port')); + return; + } + server.close((error) => { + if (error) { + reject(error); + } else { + resolve(address.port); + } + }); + }); + }); +} + +export async function resolveFactoryRealmServerURL( + realmServerURL?: URL, +): Promise { + if (realmServerURL) { + return new URL(realmServerURL.href); + } + + if (CONFIGURED_REALM_SERVER_URL) { + return new URL(CONFIGURED_REALM_SERVER_URL.href); + } + + let port = + DEFAULT_COMPAT_REALM_SERVER_PORT === 0 + ? await findAvailablePort() + : DEFAULT_COMPAT_REALM_SERVER_PORT; + return new URL(`http://localhost:${port}/`); +} + +export async function resolveFactoryRealmLocation(options: { + realmURL?: URL; + realmServerURL?: URL; +}): Promise<{ + realmURL: URL; + realmServerURL: URL; +}> { + let realmURL = options.realmURL + ? new URL(options.realmURL.href) + : CONFIGURED_REALM_URL + ? new URL(CONFIGURED_REALM_URL.href) + : undefined; + let realmServerURL = options.realmServerURL + ? new URL(options.realmServerURL.href) + : CONFIGURED_REALM_SERVER_URL + ? new URL(CONFIGURED_REALM_SERVER_URL.href) + : undefined; + + if (!realmURL && !realmServerURL) { + realmServerURL = await resolveFactoryRealmServerURL(); + realmURL = new URL('test/', realmServerURL); + } else if (!realmServerURL) { + throw new Error( + 'An explicit realm server URL is required when a realm URL is provided. Set options.realmServerURL or SOFTWARE_FACTORY_REALM_SERVER_URL.', + ); + } else if (!realmURL) { + realmURL = new URL('test/', realmServerURL); + } + + return { + realmURL, + realmServerURL, + }; +} + +export function baseRealmURLFor(realmServerURL: URL): URL { + return new URL('base/', realmServerURL); +} + +export function skillsRealmURLFor(realmServerURL: URL): URL { + return new URL('skills/', realmServerURL); +} + +export function sourceRealmURLFor(realmServerURL: URL): URL { + return new URL('software-factory/', realmServerURL); +} + +export function withPort(url: URL, port: number): URL { + let next = new URL(url.href); + next.port = String(port); + return next; +} + +export function realmRelativePath(realmURL: URL, realmServerURL: URL): string { + if (realmURL.origin !== realmServerURL.origin) { + throw new Error( + `Realm URL ${realmURL.href} does not share an origin with realm server URL ${realmServerURL.href}`, + ); + } + + let serverPath = realmServerURL.pathname.endsWith('/') + ? realmServerURL.pathname + : `${realmServerURL.pathname}/`; + if (!realmURL.pathname.startsWith(serverPath)) { + throw new Error( + `Realm URL ${realmURL.href} is not mounted under realm server URL ${realmServerURL.href}`, + ); + } + + return realmURL.pathname.slice(serverPath.length); +} + +export function realmURLWithinServer( + realmServerURL: URL, + realmPath: string, +): URL { + return new URL(realmPath || '.', realmServerURL); +} + +export function shouldIgnoreFixturePath(relativePath: string): boolean { + if (relativePath === '.DS_Store') { + return true; + } + return relativePath + .split('/') + .some((segment) => + [ + 'node_modules', + '.git', + '.boxel-history', + 'playwright-report', + 'test-results', + ].includes(segment), + ); +} + +export function hashRealmFixture(realmDir: string): string { + let entries: string[] = []; + + function visit(currentDir: string) { + for (let entry of readdirSync(currentDir, { withFileTypes: true })) { + let absolutePath = join(currentDir, entry.name); + let relativePath = relative(realmDir, absolutePath).replace(/\\/g, '/'); + if (shouldIgnoreFixturePath(relativePath)) { + continue; + } + if (entry.isDirectory()) { + visit(absolutePath); + continue; + } + if (!entry.isFile()) { + continue; + } + let stats = statSync(absolutePath); + let contentsHash = createHash('sha256') + .update(readFileSync(absolutePath)) + .digest('hex'); + entries.push(`${relativePath}:${stats.size}:${contentsHash}`); + } + } + + visit(realmDir); + entries.sort(); + return hashString(entries.join('|')); +} + +export function templateDatabaseNameForCacheKey(cacheKey: string): string { + return `sf_tpl_${cacheKey.slice(0, 24)}`; +} + +export function builderDatabaseNameForCacheKey(cacheKey: string): string { + return `sf_bld_${cacheKey.slice(0, 16)}`; +} + +export function runtimeDatabaseName(): string { + return `sf_run_${Date.now().toString(36)}_${Math.random() + .toString(36) + .slice(2, 8)}`; +} + +export function pgAdminConnectionConfig(database = 'postgres') { + return { + host: DEFAULT_PG_HOST, + port: Number(DEFAULT_PG_PORT), + user: DEFAULT_PG_USER, + password: process.env.PGPASSWORD || undefined, + database, + }; +} + +export function quotePgIdentifier(identifier: string): string { + if (!/^[a-zA-Z0-9_]+$/.test(identifier)) { + throw new Error(`unsafe postgres identifier: ${identifier}`); + } + return `"${identifier}"`; +} + +export async function waitUntil( + condition: () => Promise, + options: { + timeout?: number; + interval?: number; + timeoutMessage?: string; + } = {}, +): Promise { + let timeout = options.timeout ?? 30_000; + let interval = options.interval ?? 250; + let start = Date.now(); + while (Date.now() - start < timeout) { + let result = await condition(); + if (result) { + return result; + } + await new Promise((resolve) => setTimeout(resolve, interval)); + } + throw new Error(options.timeoutMessage ?? 'Timed out waiting for condition'); +} + +export async function waitForJsonFile( + file: string, + getLogs: () => string, + options: { + timeout?: number; + label: string; + process?: SpawnedProcess; + }, +): Promise { + let timeout = options.timeout ?? DEFAULT_REALM_STARTUP_TIMEOUT_MS; + let startedAt = Date.now(); + + while (Date.now() - startedAt < timeout) { + try { + return JSON.parse(readFileSync(file, 'utf8')) as T; + } catch (error) { + let nodeError = error as NodeJS.ErrnoException; + if (nodeError.code !== 'ENOENT' && !(error instanceof SyntaxError)) { + throw error; + } + } + + if (options.process && options.process.exitCode !== null) { + throw new Error( + `${options.label} exited early with code ${options.process.exitCode}\n${getLogs()}`, + ); + } + + await new Promise((resolve) => setTimeout(resolve, 100)); + } + + throw new Error( + `Timed out waiting for ${options.label} metadata in ${file}\n${getLogs()}`, + ); +} + +export function runCommand(command: string, args: string[], cwd: string) { + let result = spawnSync(command, args, { + cwd, + stdio: 'inherit', + env: { + ...process.env, + PGHOST: DEFAULT_PG_HOST, + PGPORT: DEFAULT_PG_PORT, + PGUSER: DEFAULT_PG_USER, + }, + }); + if (result.status !== 0) { + throw new Error(`command failed: ${command} ${args.join(' ')}`); + } +} + +export function cleanupStaleSynapseContainers() { + let result = spawnSync( + 'docker', + [ + 'ps', + '-aq', + '--filter', + 'name=synapsedocker-', + '--filter', + 'name=boxel-synapse', + ], + { + cwd: workspaceRoot, + encoding: 'utf8', + }, + ); + + if (result.status !== 0) { + return; + } + + let containerIds = result.stdout + .split(/\s+/) + .map((id) => id.trim()) + .filter(Boolean); + + if (containerIds.length === 0) { + return; + } + + spawnSync('docker', ['rm', '-f', ...containerIds], { + cwd: workspaceRoot, + stdio: 'ignore', + }); +} + +export function maybeRequire(specifier: string) { + if (typeof require === 'function') { + return require(specifier); + } + return undefined; +} + +export function fileExists(path: string): boolean { + try { + return statSync(path).isFile(); + } catch { + return false; + } +} + +export function findHostDistPackageDir(): string | undefined { + let siblingRoot = resolve(workspaceRoot, '..'); + let candidates = [ + process.env.SOFTWARE_FACTORY_HOST_DIST_PACKAGE_DIR, + resolve(siblingRoot, 'boxel', 'packages', 'host'), + ...readdirSync(siblingRoot, { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .map((entry) => resolve(siblingRoot, entry.name, 'packages', 'host')), + hostDir, + ] + .filter((value): value is string => Boolean(value)) + .map((value) => resolve(value)); + + let seen = new Set(); + for (let candidate of candidates) { + if (seen.has(candidate)) { + continue; + } + seen.add(candidate); + + if (fileExists(join(candidate, 'dist', 'index.html'))) { + return candidate; + } + } + + return undefined; +} + +export function browserPassword(username: string): string { + let cleanUsername = username.replace(/^@/, '').replace(/:.*$/, ''); + return createHash('sha256') + .update(cleanUsername) + .update(REALM_SECRET_SEED) + .digest('hex'); +} + +export function parseFactoryContext(): FactoryTestContext | undefined { + let raw = process.env.SOFTWARE_FACTORY_CONTEXT; + if (!raw) { + return undefined; + } + return JSON.parse(raw) as FactoryTestContext; +} + +export function hasTemplateDatabaseName( + context: FactorySupportContext | FactoryTestContext, +): context is FactoryTestContext { + return 'templateDatabaseName' in context; +} + +export function buildRealmToken( + realmURL: URL, + realmServerURL: URL, + user = DEFAULT_REALM_OWNER, + permissions = DEFAULT_PERMISSIONS[DEFAULT_REALM_OWNER] ?? [ + 'read', + 'write', + 'realm-owner', + ], +): string { + return jwt.sign( + { + user, + realm: realmURL.href, + permissions, + sessionRoom: `software-factory-session-room-for-${user}`, + realmServerURL: realmServerURL.href, + }, + REALM_SECRET_SEED, + { expiresIn: '7d' }, + ); +} + +export function createProcessExitPromise( + proc: SpawnedProcess, + label: string, +): Promise { + return new Promise((_, reject) => { + proc.once('exit', (code, signal) => { + reject( + new Error( + `${label} exited before it became ready (code: ${code}, signal: ${signal})`, + ), + ); + }); + proc.once('error', reject); + }); +} + +export function appendProcessLogs( + buffer: string, + chunk: Buffer | string, +): string { + return `${buffer}${String(chunk)}`.slice(-100_000); +} + +export function captureProcessLogs(proc: SpawnedProcess) { + let stdout = ''; + let stderr = ''; + + proc.stdout?.on('data', (chunk) => { + stdout = appendProcessLogs(stdout, chunk); + }); + proc.stderr?.on('data', (chunk) => { + stderr = appendProcessLogs(stderr, chunk); + }); + + return () => { + let logs = []; + if (stdout) { + logs.push(`stdout:\n${stdout}`); + } + if (stderr) { + logs.push(`stderr:\n${stderr}`); + } + return logs.join('\n\n'); + }; +} + +export async function waitForReady( + proc: SpawnedProcess, + label: string, + timeoutMs = 120_000, + getLogs?: () => string, +): Promise { + let timedOut = await Promise.race([ + new Promise((resolve) => { + let onMessage = (message: unknown) => { + if (message === 'ready') { + proc.off('message', onMessage); + resolve(); + } + }; + proc.on('message', onMessage); + }), + createProcessExitPromise(proc, label), + new Promise((resolve) => setTimeout(() => resolve(true), timeoutMs)), + ]); + + if (timedOut) { + let logOutput = getLogs?.(); + throw new Error( + `Timed out waiting for ${label} to start${ + logOutput ? `\n\n${logOutput}` : '' + }`, + ); + } +} + +export async function stopManagedProcess(proc: SpawnedProcess): Promise { + if (proc.exitCode !== null) { + return; + } + let stopped = new Promise((resolve) => { + let onMessage = (message: unknown) => { + if (message === 'stopped') { + proc.off('message', onMessage); + resolve(); + } + }; + proc.on('message', onMessage); + }); + proc.send('stop'); + await Promise.race([ + stopped, + new Promise((resolve) => setTimeout(resolve, 15_000)), + ]); + proc.send('kill'); +} diff --git a/packages/software-factory/src/harness/support-services.ts b/packages/software-factory/src/harness/support-services.ts new file mode 100644 index 00000000000..2fb0e3527f7 --- /dev/null +++ b/packages/software-factory/src/harness/support-services.ts @@ -0,0 +1,418 @@ +import { spawn, type ChildProcess } from 'node:child_process'; + +import { + boxelIconsDir, + browserPassword, + cleanupStaleSynapseContainers, + DEFAULT_HOST_URL, + DEFAULT_ICONS_PROBE_URL, + DEFAULT_MATRIX_BROWSER_USERNAME, + DEFAULT_MATRIX_SERVER_USERNAME, + DEFAULT_PG_HOST, + DEFAULT_PG_PORT, + DEFAULT_PRERENDER_PORT, + findAvailablePort, + findHostDistPackageDir, + hostDir, + logTimed, + maybeRequire, + prepareTestPgScript, + realmServerDir, + runCommand, + supportLog, + waitUntil, + workspaceRoot, + type FactorySupportContext, + type SynapseInstance, +} from './shared'; +import { canConnectToPg } from './database'; + +let preparePgPromise: Promise | undefined; + +async function loadSynapseModule() { + let moduleSpecifier = '../../../matrix/docker/synapse/index.ts'; + return (maybeRequire(moduleSpecifier) ?? (await import(moduleSpecifier))) as { + registerUser: ( + synapse: SynapseInstance, + username: string, + password: string, + admin?: boolean, + displayName?: string, + ) => Promise; + synapseStart: ( + opts?: { + suppressRegistrationSecretFile?: true; + dynamicHostPort?: true; + }, + stopExisting?: boolean, + ) => Promise; + synapseStop: (id: string) => Promise; + }; +} + +async function loadMatrixEnvironmentConfigModule() { + let moduleSpecifier = '../../../matrix/helpers/environment-config.ts'; + return (maybeRequire(moduleSpecifier) ?? (await import(moduleSpecifier))) as { + getSynapseURL: (synapse?: { baseUrl?: string; port?: number }) => string; + }; +} + +async function ensureHostReady(matrixURL: string): Promise<{ + stop?: () => Promise; +}> { + return await logTimed( + supportLog, + `ensureHostReady ${DEFAULT_HOST_URL}`, + async () => { + let response: Response; + try { + response = await fetch(DEFAULT_HOST_URL); + if (response.ok) { + return {}; + } + } catch (error) { + supportLog.debug( + `host app not reachable at ${DEFAULT_HOST_URL}, starting fallback host service: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + + let hostPackageDir = findHostDistPackageDir(); + let command = ['start']; + let cwd = hostDir; + if (hostPackageDir) { + supportLog.debug(`serving built host dist from ${hostPackageDir}`); + command = ['serve:dist']; + cwd = hostPackageDir; + } else { + supportLog.warn( + 'no built host dist found; falling back to pnpm start in packages/host', + ); + } + + let child = spawn('pnpm', command, { + cwd, + detached: true, + stdio: ['ignore', 'pipe', 'pipe'], + env: { + ...process.env, + MATRIX_URL: matrixURL, + }, + }); + + let logs = ''; + child.stdout?.on('data', (chunk) => { + logs = `${logs}${String(chunk)}`.slice(-20_000); + }); + child.stderr?.on('data', (chunk) => { + logs = `${logs}${String(chunk)}`.slice(-20_000); + }); + + await waitUntil( + async () => { + if (child.exitCode !== null) { + throw new Error( + `host app exited early with code ${child.exitCode}\n${logs}`, + ); + } + try { + let readyResponse = await fetch(DEFAULT_HOST_URL); + return readyResponse.ok; + } catch { + return false; + } + }, + { + timeout: 180_000, + interval: 500, + timeoutMessage: `Timed out waiting for host app at ${DEFAULT_HOST_URL}\n${logs}`, + }, + ); + + return { + async stop() { + if (child.exitCode === null) { + try { + process.kill(-child.pid!, 'SIGTERM'); + } catch { + // best effort cleanup + } + } + }, + }; + }, + ); +} + +async function waitForHttpReady(url: string, timeoutMs = 60_000) { + let startedAt = Date.now(); + while (Date.now() - startedAt < timeoutMs) { + try { + let response = await fetch(url); + if (response.ok) { + return; + } + } catch { + // server not ready yet + } + await new Promise((resolve) => setTimeout(resolve, 200)); + } + throw new Error(`timed out waiting for ${url} to become ready`); +} + +async function stopChildProcess( + child: ChildProcess, + signal: NodeJS.Signals = 'SIGINT', +): Promise { + if (child.exitCode !== null || child.killed) { + return; + } + + await new Promise((resolve) => { + let settled = false; + let timeout: NodeJS.Timeout | undefined; + let cleanup = () => { + if (timeout) { + clearTimeout(timeout); + } + child.removeAllListeners('exit'); + child.removeAllListeners('error'); + }; + + child.once('exit', () => { + if (!settled) { + settled = true; + cleanup(); + resolve(); + } + }); + child.once('error', () => { + if (!settled) { + settled = true; + cleanup(); + resolve(); + } + }); + + timeout = setTimeout(() => { + if (!settled) { + child.kill('SIGTERM'); + } + }, 5_000); + + child.kill(signal); + }); +} + +export async function startHarnessPrerenderServer(options: { + boxelHostURL: string; + port?: number; +}): Promise<{ + url: string; + stop(): Promise; +}> { + let port = options.port ?? DEFAULT_PRERENDER_PORT; + if (port === 0) { + port = await findAvailablePort(); + } + let url = `http://localhost:${port}`; + let silent = process.env.SOFTWARE_FACTORY_PRERENDER_SILENT !== '0'; + let child = spawn( + 'ts-node', + [ + '--transpileOnly', + 'prerender/prerender-server', + `--port=${port}`, + ...(silent ? ['--silent'] : []), + ], + { + cwd: realmServerDir, + stdio: ['pipe', 'pipe', 'pipe'], + env: { + ...process.env, + NODE_ENV: process.env.NODE_ENV ?? 'development', + NODE_NO_WARNINGS: '1', + BOXEL_HOST_URL: options.boxelHostURL, + LOG_LEVELS: + process.env.SOFTWARE_FACTORY_PRERENDER_LOG_LEVELS ?? + process.env.LOG_LEVELS, + }, + }, + ); + + child.stdout?.on('data', (data: Buffer) => { + console.log(`prerender: ${data.toString()}`); + }); + child.stderr?.on('data', (data: Buffer) => { + console.error(`prerender: ${data.toString()}`); + }); + + let exitPromise = new Promise((_, reject) => { + child.once('exit', (code, signal) => { + reject( + new Error( + `prerender server exited before it became ready (code: ${code}, signal: ${signal})`, + ), + ); + }); + child.once('error', reject); + }); + + await Promise.race([waitForHttpReady(url), exitPromise]); + + return { + url, + async stop() { + await stopChildProcess(child); + }, + }; +} + +async function ensureIconsReady(): Promise<{ + stop?: () => Promise; +}> { + return await logTimed( + supportLog, + `ensureIconsReady ${DEFAULT_ICONS_PROBE_URL}`, + async () => { + try { + let response = await fetch(DEFAULT_ICONS_PROBE_URL); + if (response.ok) { + supportLog.debug('icons server already available'); + return {}; + } + } catch { + // fall through and start the local icon server + } + + let child = spawn('pnpm', ['serve'], { + cwd: boxelIconsDir, + detached: true, + stdio: ['ignore', 'pipe', 'pipe'], + env: process.env, + }); + + let logs = ''; + child.stdout?.on('data', (chunk) => { + logs = `${logs}${String(chunk)}`.slice(-20_000); + }); + child.stderr?.on('data', (chunk) => { + logs = `${logs}${String(chunk)}`.slice(-20_000); + }); + + await waitUntil( + async () => { + if (child.exitCode !== null) { + throw new Error( + `icons server exited early with code ${child.exitCode}\n${logs}`, + ); + } + try { + let response = await fetch(DEFAULT_ICONS_PROBE_URL); + return response.ok; + } catch { + return false; + } + }, + { + timeout: 30_000, + interval: 250, + timeoutMessage: `Timed out waiting for icons server at ${DEFAULT_ICONS_PROBE_URL}\n${logs}`, + }, + ); + + supportLog.debug('started local icons server'); + return { + async stop() { + if (child.exitCode === null) { + try { + process.kill(-child.pid!, 'SIGTERM'); + } catch { + // best effort cleanup + } + } + }, + }; + }, + ); +} + +async function ensurePgReady(): Promise { + if (!preparePgPromise) { + preparePgPromise = logTimed( + supportLog, + `ensurePgReady ${DEFAULT_PG_HOST}:${DEFAULT_PG_PORT}`, + async () => { + if (await canConnectToPg()) { + supportLog.debug('postgres already available'); + return; + } + runCommand('bash', [prepareTestPgScript], workspaceRoot); + await waitUntil(() => canConnectToPg(), { + timeout: 30_000, + interval: 250, + timeoutMessage: `Timed out waiting for Postgres on ${DEFAULT_PG_HOST}:${DEFAULT_PG_PORT}`, + }); + }, + ).catch((error) => { + preparePgPromise = undefined; + throw error; + }); + } + + await preparePgPromise; +} + +async function ensureSupportUsers(synapse: SynapseInstance): Promise { + await logTimed(supportLog, 'ensureSupportUsers', async () => { + let { registerUser } = await loadSynapseModule(); + + await registerUser( + synapse, + DEFAULT_MATRIX_SERVER_USERNAME, + browserPassword(DEFAULT_MATRIX_SERVER_USERNAME), + ); + await registerUser( + synapse, + DEFAULT_MATRIX_BROWSER_USERNAME, + browserPassword(DEFAULT_MATRIX_BROWSER_USERNAME), + ); + }); +} + +export async function startFactorySupportServices(): Promise<{ + context: FactorySupportContext; + stop(): Promise; +}> { + return await logTimed(supportLog, 'startFactorySupportServices', async () => { + await ensurePgReady(); + cleanupStaleSynapseContainers(); + let { synapseStart, synapseStop } = await loadSynapseModule(); + let { getSynapseURL } = await loadMatrixEnvironmentConfigModule(); + + let synapse = await synapseStart( + { suppressRegistrationSecretFile: true, dynamicHostPort: true }, + true, + ); + let matrixURL = + process.env.SOFTWARE_FACTORY_MATRIX_URL ?? getSynapseURL(synapse); + let host = await ensureHostReady(matrixURL); + let icons = await ensureIconsReady(); + await ensureSupportUsers(synapse); + + return { + context: { + matrixURL, + matrixRegistrationSecret: synapse.registrationSecret, + }, + async stop() { + await logTimed(supportLog, 'stopFactorySupportServices', async () => { + await synapseStop(synapse.synapseId); + await host.stop?.(); + await icons.stop?.(); + }); + }, + }; + }); +} diff --git a/packages/software-factory/src/runtime-metadata.ts b/packages/software-factory/src/runtime-metadata.ts index d54763d7176..aecaad767ed 100644 --- a/packages/software-factory/src/runtime-metadata.ts +++ b/packages/software-factory/src/runtime-metadata.ts @@ -8,6 +8,13 @@ export const defaultSupportMetadataFile = join( 'support.json', ); +export interface PreparedTemplateMetadata { + realmDir: string; + templateDatabaseName: string; + templateRealmURL: string; + templateRealmServerURL: string; +} + export function getSupportMetadataFile() { return ( process.env.SOFTWARE_FACTORY_SUPPORT_METADATA_FILE ?? @@ -22,6 +29,9 @@ export function readSupportMetadata(): pid?: number; realmDir?: string; templateDatabaseName?: string; + templateRealmURL?: string; + templateRealmServerURL?: string; + preparedTemplates?: PreparedTemplateMetadata[]; } | undefined { let metadataFile = getSupportMetadataFile(); @@ -35,6 +45,9 @@ export function readSupportMetadata(): pid?: number; realmDir?: string; templateDatabaseName?: string; + templateRealmURL?: string; + templateRealmServerURL?: string; + preparedTemplates?: PreparedTemplateMetadata[]; }; } catch (error) { throw new Error( diff --git a/packages/software-factory/test-fixtures/public-software-factory-source/darkfactory.gts b/packages/software-factory/test-fixtures/public-software-factory-source/darkfactory.gts deleted file mode 100644 index 2d57a4f6f78..00000000000 --- a/packages/software-factory/test-fixtures/public-software-factory-source/darkfactory.gts +++ /dev/null @@ -1,592 +0,0 @@ -import { - CardDef, - Component, - field, - contains, - containsMany, - linksTo, - linksToMany, -} from 'https://cardstack.com/base/card-api'; -import StringField from 'https://cardstack.com/base/string'; -import NumberField from 'https://cardstack.com/base/number'; -import DateTimeField from 'https://cardstack.com/base/datetime'; -import DateField from 'https://cardstack.com/base/date'; -import MarkdownField from 'https://cardstack.com/base/markdown'; -import TextAreaField from 'https://cardstack.com/base/text-area'; -import enumField from 'https://cardstack.com/base/enum'; - -export const TicketStatusField = enumField(StringField, { - options: [ - { value: 'backlog', label: 'Backlog' }, - { value: 'in_progress', label: 'In Progress' }, - { value: 'blocked', label: 'Blocked' }, - { value: 'review', label: 'In Review' }, - { value: 'done', label: 'Done' }, - ], -}); - -export const TicketPriorityField = enumField(StringField, { - options: [ - { value: 'critical', label: 'Critical' }, - { value: 'high', label: 'High' }, - { value: 'medium', label: 'Medium' }, - { value: 'low', label: 'Low' }, - ], -}); - -export const TicketTypeField = enumField(StringField, { - options: [ - { value: 'feature', label: 'Feature' }, - { value: 'bug', label: 'Bug' }, - { value: 'task', label: 'Task' }, - { value: 'research', label: 'Research' }, - { value: 'infrastructure', label: 'Infrastructure' }, - ], -}); - -export const ProjectStatusField = enumField(StringField, { - options: [ - { value: 'planning', label: 'Planning' }, - { value: 'active', label: 'Active' }, - { value: 'on_hold', label: 'On Hold' }, - { value: 'completed', label: 'Completed' }, - { value: 'archived', label: 'Archived' }, - ], -}); - -export const KnowledgeTypeField = enumField(StringField, { - options: [ - { value: 'architecture', label: 'Architecture' }, - { value: 'decision', label: 'Decision (ADR)' }, - { value: 'runbook', label: 'Runbook' }, - { value: 'context', label: 'Context' }, - { value: 'api', label: 'API Reference' }, - { value: 'onboarding', label: 'Onboarding' }, - ], -}); - -export class AgentProfile extends CardDef { - static displayName = 'Agent Profile'; - - @field agentId = contains(StringField); - @field capabilities = containsMany(StringField); - @field specialization = contains(StringField); - @field notes = contains(MarkdownField); - - @field title = contains(StringField, { - computeVia: function (this: AgentProfile) { - return this.cardInfo.name?.trim()?.length - ? this.cardInfo.name - : (this.agentId ?? 'Unnamed Agent'); - }, - }); - - static fitted = class Fitted extends Component { - - }; - - static embedded = this.fitted; - - static isolated = class Isolated extends Component { - - }; -} - -export class KnowledgeArticle extends CardDef { - static displayName = 'Knowledge Article'; - - @field articleTitle = contains(StringField); - @field articleType = contains(KnowledgeTypeField); - @field content = contains(MarkdownField); - @field tags = containsMany(StringField); - @field lastUpdatedBy = linksTo(() => AgentProfile); - @field updatedAt = contains(DateTimeField); - - @field title = contains(StringField, { - computeVia: function (this: KnowledgeArticle) { - return this.cardInfo.name?.trim()?.length - ? this.cardInfo.name - : (this.articleTitle ?? 'Untitled Article'); - }, - }); - - static fitted = class Fitted extends Component { - - }; - - static embedded = this.fitted; - - static isolated = class Isolated extends Component { - - }; -} - -export class Ticket extends CardDef { - static displayName = 'Ticket'; - - @field ticketId = contains(StringField); - @field summary = contains(StringField); - @field description = contains(MarkdownField); - @field ticketType = contains(TicketTypeField); - @field status = contains(TicketStatusField); - @field priority = contains(TicketPriorityField); - @field project = linksTo(() => Project); - @field assignedAgent = linksTo(() => AgentProfile); - @field relatedTickets = linksToMany(() => Ticket); - @field relatedKnowledge = linksToMany(() => KnowledgeArticle); - @field acceptanceCriteria = contains(MarkdownField); - @field agentNotes = contains(MarkdownField); - @field estimatedHours = contains(NumberField); - @field actualHours = contains(NumberField); - @field createdAt = contains(DateTimeField); - @field updatedAt = contains(DateTimeField); - - @field title = contains(StringField, { - computeVia: function (this: Ticket) { - return this.cardInfo.name?.trim()?.length - ? this.cardInfo.name - : (this.summary ?? 'Untitled Ticket'); - }, - }); - - static fitted = class Fitted extends Component { - - }; - - static embedded = this.fitted; - - static isolated = class Isolated extends Component { - - }; -} - -export class Project extends CardDef { - static displayName = 'Project'; - static prefersWideFormat = true; - - @field projectCode = contains(StringField); - @field projectName = contains(StringField); - @field projectStatus = contains(ProjectStatusField); - @field deadline = contains(DateField); - @field objective = contains(TextAreaField); - @field scope = contains(MarkdownField); - @field technicalContext = contains(MarkdownField); - @field tickets = linksToMany(() => Ticket, { - query: { - filter: { - on: { - // @ts-ignore this is not a CJS file, import.meta is allowed - module: new URL('./darkfactory', import.meta.url).href, - name: 'Ticket', - }, - eq: { 'project.id': '$this.id' }, - }, - }, - }); - @field knowledgeBase = linksToMany(() => KnowledgeArticle); - @field teamAgents = linksToMany(() => AgentProfile); - @field successCriteria = contains(MarkdownField); - @field risks = contains(MarkdownField); - @field createdAt = contains(DateTimeField); - - @field title = contains(StringField, { - computeVia: function (this: Project) { - return this.cardInfo.name?.trim()?.length - ? this.cardInfo.name - : (this.projectName ?? 'Untitled Project'); - }, - }); - - static fitted = class Fitted extends Component { - - }; - - static embedded = this.fitted; - - static isolated = class Isolated extends Component { - - }; -} - -export class DarkFactory extends CardDef { - static displayName = 'Dark Factory'; - - @field factoryName = contains(StringField); - @field description = contains(MarkdownField); - @field activeProjects = linksToMany(() => Project); - - @field title = contains(StringField, { - computeVia: function (this: DarkFactory) { - return this.cardInfo.name?.trim()?.length - ? this.cardInfo.name - : (this.factoryName ?? 'Dark Factory'); - }, - }); - - static fitted = class Fitted extends Component { - - }; - - static embedded = this.fitted; - - static isolated = class Isolated extends Component { - - }; -} diff --git a/packages/software-factory/test-fixtures/public-software-factory-source/darkfactory.gts b/packages/software-factory/test-fixtures/public-software-factory-source/darkfactory.gts new file mode 120000 index 00000000000..cf4d0995a32 --- /dev/null +++ b/packages/software-factory/test-fixtures/public-software-factory-source/darkfactory.gts @@ -0,0 +1 @@ +../../realm/darkfactory.gts \ No newline at end of file diff --git a/packages/software-factory/tests/factory-bootstrap.test.ts b/packages/software-factory/tests/factory-bootstrap.test.ts index 5425ea8ea07..2e5b4e123b0 100644 --- a/packages/software-factory/tests/factory-bootstrap.test.ts +++ b/packages/software-factory/tests/factory-bootstrap.test.ts @@ -440,6 +440,40 @@ module('factory-bootstrap', function () { 'Ticket/my-app-v2-0-beta-define-core', ); }); + + test('does not surface non-serialized response objects as [object Object]', async function (assert) { + assert.expect(2); + + let failingFetch = (async ( + input: RequestInfo | URL, + init?: RequestInit, + ) => { + let request = new Request(input, init); + if (request.method === 'GET') { + return new Response('Not found', { status: 404 }); + } + + return new Response({ errors: ['boom'] } as unknown as BodyInit, { + status: 500, + }); + }) as typeof globalThis.fetch; + + await assert.rejects( + bootstrapProjectArtifacts(stickyNoteBrief, targetRealmUrl, { + darkfactoryModuleUrl, + fetch: failingFetch, + }), + (error: unknown) => { + assert.false(String(error).includes('[object Object]')); + return ( + error instanceof Error && + error.message.includes( + 'server returned a non-serialized object body', + ) + ); + }, + ); + }); }); }); @@ -455,7 +489,9 @@ function buildMockFetch( calls: { url: string; method: string }[], options: MockFetchOptions, ): typeof globalThis.fetch { - let existingCards = options.existingPaths ?? new Set(); + let initialExistingCards = new Set(options.existingPaths ?? []); + let createdCards = new Set(); + let storedBodies: Record = {}; return (async (input: RequestInfo | URL, init?: RequestInit) => { let request = new Request(input, init); @@ -469,16 +505,19 @@ function buildMockFetch( if (method === 'GET') { let exists = options.allExist || - (!options.allMissing && existingCards.has(cardPath)); + createdCards.has(cardPath) || + (!options.allMissing && initialExistingCards.has(cardPath)); if (exists) { - let ticketStatus = options.existingTicketStatus ?? 'backlog'; - let mockAttributes: Record = { status: ticketStatus }; - return new Response( - JSON.stringify({ + let existingBody = storedBodies[cardPath]; + let payload = + existingBody ?? + ({ data: { type: 'card', - attributes: mockAttributes, + attributes: { + status: options.existingTicketStatus ?? 'backlog', + }, meta: { adoptsFrom: { module: darkfactoryModuleUrl, @@ -486,18 +525,22 @@ function buildMockFetch( }, }, }, - }), - { status: 200, headers: { 'content-type': 'application/json' } }, - ); + } satisfies Record); + return new Response(JSON.stringify(payload), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); } return new Response('Not found', { status: 404 }); } if (method === 'POST') { + createdCards.add(cardPath); + let body = await request.text(); + storedBodies[cardPath] = JSON.parse(body); if (options.captureWrites) { - let body = await request.text(); - options.captureWrites[cardPath] = JSON.parse(body); + options.captureWrites[cardPath] = storedBodies[cardPath]; } return new Response(null, { status: 204 }); } diff --git a/packages/software-factory/tests/factory-entrypoint.integration.test.ts b/packages/software-factory/tests/factory-entrypoint.integration.test.ts index 07a8e5d961b..231dcff21d8 100644 --- a/packages/software-factory/tests/factory-entrypoint.integration.test.ts +++ b/packages/software-factory/tests/factory-entrypoint.integration.test.ts @@ -48,6 +48,7 @@ module('factory-entrypoint integration', function () { test('factory:go package script prints a structured JSON summary', async function (assert) { let canonicalTargetRealmUrl: string; let targetRealmUrl: string; + let createdCardPaths = new Set(); let server = createServer((request, response) => { if (request.url === '/software-factory/Wiki/sticky-note') { response.writeHead(200, { 'content-type': 'application/json' }); @@ -147,13 +148,37 @@ module('factory-entrypoint integration', function () { request.url?.startsWith('/hassan/personal/') && request.method === 'GET' ) { - // Card existence check — return 404 for first run - response.writeHead(404, { 'content-type': 'text/plain' }); - response.end('not found'); + let cardPath = request.url + .replace('/hassan/personal/', '') + .replace(/\.json$/, ''); + if (createdCardPaths.has(cardPath)) { + response.writeHead(200, { 'content-type': 'application/json' }); + response.end( + JSON.stringify({ + data: { + type: 'card', + attributes: {}, + meta: { + adoptsFrom: { + module: `${origin}/software-factory/darkfactory`, + name: 'Project', + }, + }, + }, + }), + ); + } else { + // Card existence check — return 404 for first run + response.writeHead(404, { 'content-type': 'text/plain' }); + response.end('not found'); + } } else if ( request.url?.startsWith('/hassan/personal/') && request.method === 'POST' ) { + createdCardPaths.add( + request.url.replace('/hassan/personal/', '').replace(/\.json$/, ''), + ); // Card creation — accept it response.writeHead(204); response.end(); diff --git a/packages/software-factory/tests/factory-target-realm.test.ts b/packages/software-factory/tests/factory-target-realm.test.ts index 2a68cc6f9c3..5fcefcd8873 100644 --- a/packages/software-factory/tests/factory-target-realm.test.ts +++ b/packages/software-factory/tests/factory-target-realm.test.ts @@ -318,6 +318,87 @@ module('factory-target-realm', function (hooks) { assert.true(result.createdRealm); assert.strictEqual(result.authorization, 'Bearer target-realm-token'); }); + + test('bootstrapFactoryTargetRealm does not surface non-serialized response objects as [object Object]', async function (assert) { + assert.expect(2); + + process.env.MATRIX_URL = 'https://matrix.example.test/'; + process.env.MATRIX_USERNAME = 'hassan'; + process.env.MATRIX_PASSWORD = 'secret'; + process.env.REALM_SERVER_URL = 'https://realms.example.test/'; + + let resolution = resolveFactoryTargetRealm({ + targetRealmUrl, + realmServerUrl: null, + }); + + globalThis.fetch = (async (input, init) => { + let request = new Request(input, init); + + if ( + request.url === 'https://matrix.example.test/_matrix/client/v3/login' + ) { + return new Response( + JSON.stringify({ + access_token: 'matrix-access-token', + device_id: 'device-id', + user_id: '@hassan:localhost', + }), + { + status: 200, + headers: { 'content-type': 'application/json' }, + }, + ); + } + + if ( + request.url === + 'https://matrix.example.test/_matrix/client/v3/user/%40hassan%3Alocalhost/openid/request_token' + ) { + return new Response( + JSON.stringify({ + access_token: 'openid-token', + expires_in: 300, + matrix_server_name: 'localhost', + token_type: 'Bearer', + }), + { + status: 200, + headers: { 'content-type': 'application/json' }, + }, + ); + } + + if (request.url === 'https://realms.example.test/_server-session') { + return new Response('{}', { + status: 200, + headers: { + 'content-type': 'application/json', + Authorization: 'Bearer realm-server-token', + }, + }); + } + + if (request.url === 'https://realms.example.test/_create-realm') { + return new Response({ errors: ['boom'] } as unknown as BodyInit, { + status: 500, + }); + } + + throw new Error(`Unexpected url: ${request.method} ${request.url}`); + }) as typeof globalThis.fetch; + + await assert.rejects( + bootstrapFactoryTargetRealm(resolution), + (error: unknown) => { + assert.false(String(error).includes('[object Object]')); + return ( + error instanceof Error && + error.message.includes('server returned a non-serialized object body') + ); + }, + ); + }); }); function restoreEnv(name: string, value: string | undefined): void { diff --git a/packages/software-factory/tests/fixtures.ts b/packages/software-factory/tests/fixtures.ts index a671f947b93..e1cf3188311 100644 --- a/packages/software-factory/tests/fixtures.ts +++ b/packages/software-factory/tests/fixtures.ts @@ -7,7 +7,11 @@ import { join, resolve } from 'node:path'; import type { Page } from '@playwright/test'; import { test as base, expect } from '@playwright/test'; -import { defaultSupportMetadataFile } from '../src/runtime-metadata'; +import { + defaultSupportMetadataFile, + type PreparedTemplateMetadata, + readSupportMetadata, +} from '../src/runtime-metadata'; import { buildBrowserState, installBrowserState } from './helpers/browser-auth'; type StartedFactoryRealm = { @@ -142,10 +146,32 @@ async function startRealmProcess(realmDir = defaultRealmDir) { let metadataFile = join(tempDir, 'runtime.json'); let logs = ''; let supportMetadata = existsSync(defaultSupportMetadataFile) - ? (JSON.parse(readFileSync(defaultSupportMetadataFile, 'utf8')) as { - context?: Record; - }) + ? (readSupportMetadata() as + | { + context?: Record; + templateDatabaseName?: string; + templateRealmURL?: string; + templateRealmServerURL?: string; + realmDir?: string; + preparedTemplates?: PreparedTemplateMetadata[]; + } + | undefined) : undefined; + let preparedTemplate = + supportMetadata?.preparedTemplates?.find( + (entry) => resolve(entry.realmDir) === resolve(realmDir), + ) ?? + (supportMetadata?.realmDir != null && + resolve(supportMetadata.realmDir) === resolve(realmDir) && + supportMetadata.templateDatabaseName && + supportMetadata.templateRealmServerURL + ? { + realmDir: supportMetadata.realmDir, + templateDatabaseName: supportMetadata.templateDatabaseName, + templateRealmURL: supportMetadata.templateRealmURL ?? '', + templateRealmServerURL: supportMetadata.templateRealmServerURL, + } + : undefined); let child = spawn( tsNodeBin, @@ -163,6 +189,18 @@ async function startRealmProcess(realmDir = defaultRealmDir) { SOFTWARE_FACTORY_CONTEXT: JSON.stringify(supportMetadata.context), } : {}), + ...(preparedTemplate + ? { + SOFTWARE_FACTORY_TEMPLATE_DATABASE_NAME: + preparedTemplate.templateDatabaseName, + } + : {}), + ...(preparedTemplate + ? { + SOFTWARE_FACTORY_TEMPLATE_REALM_SERVER_URL: + preparedTemplate.templateRealmServerURL, + } + : {}), }, stdio: ['ignore', 'pipe', 'pipe'], }, From fa442e56b74351b36d4756c4d922c6190bdddb59 Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Tue, 24 Mar 2026 13:33:15 -0400 Subject: [PATCH 03/11] Fix software factory test typing after main merge --- packages/software-factory/tests/darkfactory.spec.ts | 7 +++---- packages/software-factory/tests/factory-entrypoint.test.ts | 1 + 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/software-factory/tests/darkfactory.spec.ts b/packages/software-factory/tests/darkfactory.spec.ts index b09016dca98..fcb014bd8f3 100644 --- a/packages/software-factory/tests/darkfactory.spec.ts +++ b/packages/software-factory/tests/darkfactory.spec.ts @@ -1,5 +1,7 @@ import { resolve } from 'node:path'; +import type { Page } from '@playwright/test'; + import { expect, test } from './fixtures'; const adopterRealmDir = resolve( @@ -11,10 +13,7 @@ const adopterRealmDir = resolve( test.use({ realmDir: adopterRealmDir }); test.use({ realmServerMode: 'shared' }); -async function gotoCard( - page: Parameters[0]['authedPage'], - url: string, -) { +async function gotoCard(page: Page, url: string) { await page.goto(url, { waitUntil: 'commit' }); } diff --git a/packages/software-factory/tests/factory-entrypoint.test.ts b/packages/software-factory/tests/factory-entrypoint.test.ts index 3465e59099e..44c5ebf5742 100644 --- a/packages/software-factory/tests/factory-entrypoint.test.ts +++ b/packages/software-factory/tests/factory-entrypoint.test.ts @@ -29,6 +29,7 @@ const bootstrappedTargetRealm: FactoryTargetRealmBootstrapResult = { serverUrl: 'https://realms.example.test/', ownerUsername: 'hassan', createdRealm: true, + authorization: 'Bearer token', }; const mockBootstrapResult: FactoryBootstrapResult = { project: { id: 'Project/sticky-note-mvp', status: 'created' }, From ce2bd37826d75c6bfa979c9f9cc0a134a442426c Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Tue, 24 Mar 2026 14:14:42 -0400 Subject: [PATCH 04/11] Cache software factory templates across runs --- packages/software-factory/src/harness/api.ts | 41 ++++++++++----- .../software-factory/src/harness/database.ts | 10 ++++ .../software-factory/src/runtime-metadata.ts | 51 ++++++++++++++++++- 3 files changed, 89 insertions(+), 13 deletions(-) diff --git a/packages/software-factory/src/harness/api.ts b/packages/software-factory/src/harness/api.ts index 2e515998a0a..3daede618a4 100644 --- a/packages/software-factory/src/harness/api.ts +++ b/packages/software-factory/src/harness/api.ts @@ -1,5 +1,9 @@ import { resolve } from 'node:path'; +import { + readPreparedTemplateMetadata, + writePreparedTemplateMetadata, +} from '../runtime-metadata'; import { buildRealmToken, CACHE_VERSION, @@ -20,6 +24,7 @@ import { templateDatabaseNameForCacheKey, hashString, baseRealmURLFor, + realmRelativePath, sourceRealmURLFor, realmLog, type FactoryGlobalContextHandle, @@ -118,10 +123,11 @@ export async function ensureFactoryRealmTemplate( let permissions = options.permissions ?? DEFAULT_PERMISSIONS; let fixtureHash = hashRealmFixture(realmDir); let sourceRealmHash = hashRealmFixture(sourceRealmDir); + let realmPath = realmRelativePath(realmURL, realmServerURL); let cacheKey = hashString( stableStringify({ version: CACHE_VERSION, - realmURL: realmURL.href, + realmPath, permissions, fixtureHash, sourceRealmHash, @@ -130,6 +136,22 @@ export async function ensureFactoryRealmTemplate( }), ); let templateDatabaseName = templateDatabaseNameForCacheKey(cacheKey); + let cachedTemplateMetadata = + readPreparedTemplateMetadata(templateDatabaseName); + + if ( + (await databaseExists(templateDatabaseName)) && + cachedTemplateMetadata + ) { + return { + cacheKey, + templateDatabaseName, + fixtureHash, + cacheHit: true, + realmURL: new URL(cachedTemplateMetadata.templateRealmURL), + realmServerURL: new URL(cachedTemplateMetadata.templateRealmServerURL), + }; + } let ownedSupport: | { @@ -144,17 +166,6 @@ export async function ensureFactoryRealmTemplate( } try { - if (await databaseExists(templateDatabaseName)) { - return { - cacheKey, - templateDatabaseName, - fixtureHash, - cacheHit: true, - realmURL, - realmServerURL, - }; - } - await buildTemplateDatabase({ realmDir, realmURL, @@ -164,6 +175,12 @@ export async function ensureFactoryRealmTemplate( cacheKey, templateDatabaseName, }); + writePreparedTemplateMetadata({ + realmDir, + templateDatabaseName, + templateRealmURL: realmURL.href, + templateRealmServerURL: realmServerURL.href, + }); return { cacheKey, diff --git a/packages/software-factory/src/harness/database.ts b/packages/software-factory/src/harness/database.ts index 703b5bebaac..773e1266b04 100644 --- a/packages/software-factory/src/harness/database.ts +++ b/packages/software-factory/src/harness/database.ts @@ -59,6 +59,16 @@ export async function dropDatabase(databaseName: string): Promise { let client = new PgClient(pgAdminConnectionConfig()); try { await client.connect(); + let result = await client.query<{ exists: boolean }>( + 'SELECT EXISTS(SELECT 1 FROM pg_database WHERE datname = $1) AS exists', + [databaseName], + ); + if (!result.rows[0]?.exists) { + return; + } + await client.query( + `ALTER DATABASE ${quotePgIdentifier(databaseName)} WITH IS_TEMPLATE false`, + ); await client.query( `SELECT pg_terminate_backend(pid) FROM pg_stat_activity diff --git a/packages/software-factory/src/runtime-metadata.ts b/packages/software-factory/src/runtime-metadata.ts index aecaad767ed..ab4ead6cbf8 100644 --- a/packages/software-factory/src/runtime-metadata.ts +++ b/packages/software-factory/src/runtime-metadata.ts @@ -1,8 +1,18 @@ -import { existsSync, readFileSync, renameSync, writeFileSync } from 'node:fs'; +import { + existsSync, + mkdirSync, + readFileSync, + renameSync, + writeFileSync, +} from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; export const sharedRuntimeDir = join(tmpdir(), 'software-factory-runtime'); +export const templateMetadataDir = join( + tmpdir(), + 'software-factory-template-cache', +); export const defaultSupportMetadataFile = join( sharedRuntimeDir, 'support.json', @@ -62,8 +72,47 @@ export function readSupportContext(): Record | undefined { return readSupportMetadata()?.context; } +function getTemplateMetadataFile(templateDatabaseName: string): string { + return join(templateMetadataDir, `${templateDatabaseName}.json`); +} + +export function readPreparedTemplateMetadata( + templateDatabaseName: string, +): PreparedTemplateMetadata | undefined { + let metadataFile = getTemplateMetadataFile(templateDatabaseName); + if (!existsSync(metadataFile)) { + return undefined; + } + + try { + return JSON.parse( + readFileSync(metadataFile, 'utf8'), + ) as PreparedTemplateMetadata; + } catch (error) { + throw new Error( + `Unable to parse software-factory template metadata at ${metadataFile}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } +} + +export function writePreparedTemplateMetadata( + payload: PreparedTemplateMetadata, +): void { + mkdirSync(templateMetadataDir, { recursive: true }); + let metadataFile = getTemplateMetadataFile(payload.templateDatabaseName); + let tempFile = join( + dirname(metadataFile), + `.template.${process.pid}.${Date.now()}.tmp`, + ); + writeFileSync(tempFile, JSON.stringify(payload, null, 2)); + renameSync(tempFile, metadataFile); +} + export function writeSupportMetadata(payload: unknown): void { let metadataFile = getSupportMetadataFile(); + mkdirSync(dirname(metadataFile), { recursive: true }); let tempFile = join( dirname(metadataFile), `.support.${process.pid}.${Date.now()}.tmp`, From 13be2d9e3c978d8eb6d45ebed44b795d52c325c1 Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Tue, 24 Mar 2026 16:41:04 -0400 Subject: [PATCH 05/11] Align harness ports with Playwright worker lifetime --- packages/software-factory/tests/fixtures.ts | 105 +++++++++++++++++++- 1 file changed, 100 insertions(+), 5 deletions(-) diff --git a/packages/software-factory/tests/fixtures.ts b/packages/software-factory/tests/fixtures.ts index e1cf3188311..2389c6de450 100644 --- a/packages/software-factory/tests/fixtures.ts +++ b/packages/software-factory/tests/fixtures.ts @@ -44,6 +44,7 @@ type FactoryRealmOptions = { type FactoryRealmInternalFixtures = { realm: StartedFactoryRealm; + testWorkerPortSet: TestWorkerPortSet; }; type SharedRealmHandle = { @@ -51,6 +52,13 @@ type SharedRealmHandle = { refCount: number; }; +type TestWorkerPortSet = { + compatRealmServerPort: number; + realmServerPort: number; + workerManagerPort: number; + prerenderPort: number; +}; + const packageRoot = resolve(process.cwd()); const tsNodeBin = resolve(packageRoot, 'node_modules', '.bin', 'ts-node'); const defaultRealmDir = resolve( @@ -62,6 +70,11 @@ const testSourceRealmDir = resolve( 'test-fixtures/public-software-factory-source', ); const sharedRealms = new Map>(); +const testWorkerPortBlockSize = 10; +const testWorkerPortSearchStride = 200; +const testWorkerPortBase = Number( + process.env.SOFTWARE_FACTORY_TEST_WORKER_PORT_BASE ?? 43100, +); function appendLog(buffer: string, chunk: string): string { let combined = `${buffer}${chunk}`; @@ -114,6 +127,63 @@ async function waitForPortFree( throw new Error(`Port ${port} still in use after ${timeoutMs}ms`); } +async function isPortFree(port: number): Promise { + return await new Promise((resolve, reject) => { + let server = createServer(); + server.once('error', (error: NodeJS.ErrnoException) => { + server.close(() => { + if (error.code === 'EADDRINUSE') { + resolve(false); + } else { + reject(error); + } + }); + }); + server.listen(port, '127.0.0.1', () => { + server.close((closeError) => { + if (closeError) { + reject(closeError); + } else { + resolve(true); + } + }); + }); + }); +} + +async function allocateTestWorkerPortSet( + testWorkerIndex: number, +): Promise { + // Reserve one stable port block per Playwright testWorker. The isolated + // harness still restarts realm-server, worker-manager, compat proxy, and + // prerender between tests, but those services should come back on the same + // URLs for the lifetime of the testWorker. That keeps BOXEL_HOST_URL and the + // prerender standby target stable within a worker even when the realm stack + // itself is recreated per test. + for (let attempt = 0; attempt < 100; attempt++) { + let blockStart = + testWorkerPortBase + + testWorkerIndex * testWorkerPortBlockSize + + attempt * testWorkerPortSearchStride; + let candidate: TestWorkerPortSet = { + compatRealmServerPort: blockStart, + realmServerPort: blockStart + 1, + workerManagerPort: blockStart + 2, + prerenderPort: blockStart + 3, + }; + let ports = Object.values(candidate); + if ( + (await Promise.all(ports.map((port) => isPortFree(port)))).every(Boolean) + ) { + return candidate; + } + } + + throw new Error( + `Unable to allocate a stable software-factory port block for testWorker ${testWorkerIndex}`, + ); +} + async function waitForMetadataFile( metadataFile: string, child: ReturnType, @@ -141,7 +211,10 @@ async function waitForMetadataFile( ); } -async function startRealmProcess(realmDir = defaultRealmDir) { +async function startRealmProcess( + realmDir = defaultRealmDir, + testWorkerPortSet: TestWorkerPortSet, +) { let tempDir = mkdtempSync(join(tmpdir(), 'software-factory-realm-')); let metadataFile = join(tempDir, 'runtime.json'); let logs = ''; @@ -184,6 +257,16 @@ async function startRealmProcess(realmDir = defaultRealmDir) { NODE_NO_WARNINGS: '1', SOFTWARE_FACTORY_METADATA_FILE: metadataFile, SOFTWARE_FACTORY_SOURCE_REALM_DIR: testSourceRealmDir, + SOFTWARE_FACTORY_COMPAT_REALM_PORT: String( + testWorkerPortSet.compatRealmServerPort, + ), + SOFTWARE_FACTORY_REALM_PORT: String(testWorkerPortSet.realmServerPort), + SOFTWARE_FACTORY_WORKER_MANAGER_PORT: String( + testWorkerPortSet.workerManagerPort, + ), + SOFTWARE_FACTORY_PRERENDER_PORT: String( + testWorkerPortSet.prerenderPort, + ), ...(supportMetadata?.context ? { SOFTWARE_FACTORY_CONTEXT: JSON.stringify(supportMetadata.context), @@ -302,10 +385,11 @@ function sharedRealmKey( async function acquireSharedRealm( key: string, realmDir: string, + testWorkerPortSet: TestWorkerPortSet, ): Promise { let existing = sharedRealms.get(key); if (!existing) { - existing = startRealmProcess(realmDir).then((realm) => ({ + existing = startRealmProcess(realmDir, testWorkerPortSet).then((realm) => ({ realm, refCount: 0, })); @@ -337,15 +421,26 @@ export const test = base.extend< >({ realmDir: [defaultRealmDir, { option: true }], realmServerMode: ['shared', { option: true }], + testWorkerPortSet: [ + async ({ browserName: _browserName }, use, workerInfo) => { + // These services are ephemeral per test, but we intentionally keep their + // port assignments stable for the lifetime of a Playwright testWorker. + // That gives each testWorker a consistent harness URL set even as the + // underlying realm stack is torn down and recreated between tests. + let portSet = await allocateTestWorkerPortSet(workerInfo.parallelIndex); + await use(portSet); + }, + { scope: 'worker' }, + ], realm: async ( - { browserName: _browserName, realmDir, realmServerMode }, + { browserName: _browserName, realmDir, realmServerMode, testWorkerPortSet }, use, testInfo, ) => { if (realmServerMode === 'shared') { let key = sharedRealmKey(testInfo.workerIndex, testInfo.file, realmDir); - let realm = await acquireSharedRealm(key, realmDir); + let realm = await acquireSharedRealm(key, realmDir, testWorkerPortSet); try { await use(realm); } finally { @@ -354,7 +449,7 @@ export const test = base.extend< return; } - let realm = await startRealmProcess(realmDir); + let realm = await startRealmProcess(realmDir, testWorkerPortSet); try { await use(realm); } finally { From e4a86a19ab0cf3ffb6be7187962a61da1f05d9fd Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Tue, 24 Mar 2026 16:54:35 -0400 Subject: [PATCH 06/11] Make bootstrap idempotency spec reuse one realm boot --- .../tests/factory-bootstrap.spec.ts | 75 ++++++++----------- 1 file changed, 31 insertions(+), 44 deletions(-) diff --git a/packages/software-factory/tests/factory-bootstrap.spec.ts b/packages/software-factory/tests/factory-bootstrap.spec.ts index e580cb39d77..bbb817b9afa 100644 --- a/packages/software-factory/tests/factory-bootstrap.spec.ts +++ b/packages/software-factory/tests/factory-bootstrap.spec.ts @@ -40,9 +40,10 @@ const cardSourceMimeType = 'application/vnd.card+source'; test.use({ realmDir: bootstrapTargetDir }); test.use({ realmServerMode: 'isolated' }); -test('bootstrap creates actual card instances in a live realm', async ({ - realm, -}) => { +function buildBootstrapContext(realm: { + realmURL: URL; + ownerBearerToken: string; +}) { let darkfactoryModuleUrl = new URL( '../software-factory/darkfactory', realm.realmURL, @@ -51,18 +52,32 @@ test('bootstrap creates actual card instances in a live realm', async ({ realm.ownerBearerToken, fetch, ); + let bootstrapOptions = { fetch: authenticatedFetch, darkfactoryModuleUrl }; + + return { + authenticatedFetch, + bootstrapOptions, + darkfactoryModuleUrl, + }; +} + +test('bootstrap creates card instances and reruns idempotently in a live realm', async ({ + realm, +}) => { + let { authenticatedFetch, bootstrapOptions, darkfactoryModuleUrl } = + buildBootstrapContext(realm); - let result = await bootstrapProjectArtifacts( + let result1 = await bootstrapProjectArtifacts( stickyNoteBrief, realm.realmURL.href, - { fetch: authenticatedFetch, darkfactoryModuleUrl }, + bootstrapOptions, ); - expect(result.project.id).toBe('Project/sticky-note-mvp'); - expect(result.project.status).toBe('created'); - expect(result.knowledgeArticles).toHaveLength(2); - expect(result.tickets).toHaveLength(3); - expect(result.activeTicket.id).toBe('Ticket/sticky-note-define-core'); + expect(result1.project.id).toBe('Project/sticky-note-mvp'); + expect(result1.project.status).toBe('created'); + expect(result1.knowledgeArticles).toHaveLength(2); + expect(result1.tickets).toHaveLength(3); + expect(result1.activeTicket.id).toBe('Ticket/sticky-note-define-core'); let projectResponse = await authenticatedFetch( realm.cardURL('Project/sticky-note-mvp'), @@ -118,28 +133,6 @@ test('bootstrap creates actual card instances in a live realm', async ({ 'Sticky Note — Brief Context', ); expect(contextJson.data.attributes.articleType).toBe('context'); -}); - -test('bootstrap is idempotent — rerun does not duplicate cards', async ({ - realm, -}) => { - let darkfactoryModuleUrl = new URL( - '../software-factory/darkfactory', - realm.realmURL, - ).href; - let authenticatedFetch = buildAuthenticatedFetch( - realm.ownerBearerToken, - fetch, - ); - let bootstrapOptions = { fetch: authenticatedFetch, darkfactoryModuleUrl }; - - let result1 = await bootstrapProjectArtifacts( - stickyNoteBrief, - realm.realmURL.href, - bootstrapOptions, - ); - expect(result1.project.status).toBe('created'); - expect(result1.tickets[0].status).toBe('created'); let result2 = await bootstrapProjectArtifacts( stickyNoteBrief, @@ -158,19 +151,13 @@ test('bootstrapped project card renders correctly in the browser', async ({ realm, authedPage, }) => { - let darkfactoryModuleUrl = new URL( - '../software-factory/darkfactory', - realm.realmURL, - ).href; - let authenticatedFetch = buildAuthenticatedFetch( - realm.ownerBearerToken, - fetch, - ); + let { bootstrapOptions } = buildBootstrapContext(realm); - await bootstrapProjectArtifacts(stickyNoteBrief, realm.realmURL.href, { - fetch: authenticatedFetch, - darkfactoryModuleUrl, - }); + await bootstrapProjectArtifacts( + stickyNoteBrief, + realm.realmURL.href, + bootstrapOptions, + ); await authedPage.goto(realm.cardURL('Project/sticky-note-mvp'), { waitUntil: 'commit', From 736be27179a5a2abf41fb7c6917ace6d336d625a Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Tue, 24 Mar 2026 17:07:00 -0400 Subject: [PATCH 07/11] Keep prerender alive for the Playwright worker --- packages/realm-server/prerender/page-pool.ts | 58 ++++++++++++++++--- .../realm-server/prerender/prerenderer.ts | 9 ++- .../src/harness/isolated-realm-stack.ts | 25 ++++++-- .../software-factory/src/harness/shared.ts | 6 +- packages/software-factory/tests/fixtures.ts | 54 +++++++++++++++-- 5 files changed, 131 insertions(+), 21 deletions(-) diff --git a/packages/realm-server/prerender/page-pool.ts b/packages/realm-server/prerender/page-pool.ts index 0bcb792885b..896720cd690 100644 --- a/packages/realm-server/prerender/page-pool.ts +++ b/packages/realm-server/prerender/page-pool.ts @@ -75,6 +75,19 @@ const STANDBY_BACKOFF_MS = 500; const STANDBY_BACKOFF_CAP_MS = 4000; const CONSOLE_ERROR_LIMIT = 50; +export class StandbyTargetNotReadyError extends Error {} + +function isExpectedStandbyTargetNotReadyError(error: unknown): boolean { + if (!(error instanceof Error)) { + return false; + } + + return ( + error instanceof StandbyTargetNotReadyError || + /ERR_CONNECTION_REFUSED|returned HTTP 50[23]/.test(error.message) + ); +} + export class PagePool { #affinityPages = new Map>(); #standbys = new Set(); @@ -329,10 +342,12 @@ export class PagePool { try { return await this.#createStandby(); } catch (e) { - log.error( - `Standby creation attempt ${attempt} failed (page pool capacity ${this.#totalContextCount()}/${this.#maxPages + 1}):`, - e, - ); + let message = `Standby creation attempt ${attempt} failed (page pool capacity ${this.#totalContextCount()}/${this.#maxPages + 1}):`; + if (isExpectedStandbyTargetNotReadyError(e)) { + log.debug(message, e); + } else { + log.error(message, e); + } if (attempt >= STANDBY_CREATION_RETRIES) { return undefined; } @@ -369,7 +384,11 @@ export class PagePool { this.#standbys.add(entry); return entry; } catch (e) { - log.error('Error creating standby page:', e); + if (isExpectedStandbyTargetNotReadyError(e)) { + log.debug('Standby page target is not ready yet:', e); + } else { + log.error('Error creating standby page:', e); + } if (context) { try { await context.close(); @@ -384,10 +403,31 @@ export class PagePool { } async #loadStandbyPage(page: Page, pageId: string): Promise { - await page.goto(`${this.#boxelHostURL}/_standby`, { - waitUntil: 'domcontentloaded', - timeout: this.#standbyTimeoutMs, - }); + let standbyURL = `${this.#boxelHostURL}/_standby`; + try { + let response = await page.goto(standbyURL, { + waitUntil: 'domcontentloaded', + timeout: this.#standbyTimeoutMs, + }); + let status = response?.status(); + if (status === 502 || status === 503) { + throw new StandbyTargetNotReadyError( + `Standby target ${standbyURL} returned HTTP ${status}`, + ); + } + } catch (error) { + if ( + error instanceof Error && + /ERR_CONNECTION_REFUSED|ERR_CONNECTION_RESET|ERR_CONNECTION_CLOSED/.test( + error.message, + ) + ) { + throw new StandbyTargetNotReadyError( + `Standby target ${standbyURL} is not reachable yet: ${error.message}`, + ); + } + throw error; + } await this.#withStandbyTimeout( () => page.waitForFunction(() => { diff --git a/packages/realm-server/prerender/prerenderer.ts b/packages/realm-server/prerender/prerenderer.ts index 11f9d267c97..4aecd1a7c48 100644 --- a/packages/realm-server/prerender/prerenderer.ts +++ b/packages/realm-server/prerender/prerenderer.ts @@ -10,7 +10,7 @@ import { type RunCommandResponse, } from '@cardstack/runtime-common'; import { BrowserManager } from './browser-manager'; -import { PagePool } from './page-pool'; +import { PagePool, StandbyTargetNotReadyError } from './page-pool'; import { RenderRunner } from './render-runner'; import { isEnvironmentMode, serviceURL } from '../lib/dev-service-registry'; import { toAffinityKey } from './affinity'; @@ -86,6 +86,13 @@ export class Prerenderer { this.#affinityIdleEvictMs = this.#resolveAffinityIdleEvictMs(); this.#startCleanupLoop(); void this.#pagePool.warmStandbys().catch((e) => { + if (e instanceof StandbyTargetNotReadyError) { + log.debug( + 'Prerenderer startup skipped standby warmup because the Boxel host target is not ready yet:', + e, + ); + return; + } log.warn('Failed to warm standby pages during prerenderer startup:', e); }); } diff --git a/packages/software-factory/src/harness/isolated-realm-stack.ts b/packages/software-factory/src/harness/isolated-realm-stack.ts index 556b89e1d80..6ac28797f81 100644 --- a/packages/software-factory/src/harness/isolated-realm-stack.ts +++ b/packages/software-factory/src/harness/isolated-realm-stack.ts @@ -19,6 +19,7 @@ import { baseRealmDir, baseRealmURLFor, captureProcessLogs, + CONFIGURED_PRERENDER_URL, createProcessExitPromise, DEFAULT_MATRIX_SERVER_USERNAME, DEFAULT_PG_HOST, @@ -310,9 +311,21 @@ export async function startIsolatedRealmStack({ let compatProxy = await startCompatRealmProxy({ listenPort: Number(realmServerURL.port), }); - let prerender = await startHarnessPrerenderServer({ - boxelHostURL: realmServerURL.href.replace(/\/$/, ''), - }); + // The software-factory Playwright harness can keep prerender alive for the + // lifetime of a Playwright testWorker even though the realm stack itself is + // recreated per test. When provided, reuse that long-lived prerender URL so + // we only restart realm-server and worker-manager here. + let prerender = CONFIGURED_PRERENDER_URL + ? undefined + : await startHarnessPrerenderServer({ + boxelHostURL: realmServerURL.href.replace(/\/$/, ''), + }); + let prerenderURL = CONFIGURED_PRERENDER_URL?.href ?? prerender?.url; + if (!prerenderURL) { + throw new Error( + 'Unable to determine prerender URL for isolated realm stack', + ); + } let env = { ...process.env, @@ -343,7 +356,7 @@ export async function startIsolatedRealmStack({ 'worker-manager', `--port=${DEFAULT_WORKER_MANAGER_PORT}`, `--matrixURL=${context.matrixURL}`, - `--prerendererUrl=${prerender.url}`, + `--prerendererUrl=${prerenderURL}`, `--fromUrl=${realmURL.href}`, `--toUrl=${actualRealmURL.href}`, `--fromUrl=${publicBaseRealmURL.href}`, @@ -406,7 +419,7 @@ export async function startIsolatedRealmStack({ `--matrixURL=${context.matrixURL}`, `--realmsRootPath=${rootDir}`, `--workerManagerUrl=${workerManagerRuntime.url}`, - `--prerendererUrl=${prerender.url}`, + `--prerendererUrl=${prerenderURL}`, '--username=base_realm', `--path=${baseRealmDir}`, `--fromUrl=${publicBaseRealmURL.href}`, @@ -532,7 +545,7 @@ export async function stopIsolatedRealmStack( let cleanupError: unknown; try { - await stack.prerender.stop(); + await stack.prerender?.stop(); } catch (error) { cleanupError ??= error; } diff --git a/packages/software-factory/src/harness/shared.ts b/packages/software-factory/src/harness/shared.ts index e2eb7187a17..79455622ac9 100644 --- a/packages/software-factory/src/harness/shared.ts +++ b/packages/software-factory/src/harness/shared.ts @@ -93,7 +93,7 @@ export type StartedCompatRealmProxy = { }; export type RunningFactoryStack = { - prerender: { + prerender?: { stop(): Promise; }; realmServer: SpawnedProcess; @@ -168,6 +168,10 @@ export const DEFAULT_PG_USER = export const DEFAULT_PRERENDER_PORT = Number( process.env.SOFTWARE_FACTORY_PRERENDER_PORT ?? 0, ); +export const CONFIGURED_PRERENDER_URL = process.env + .SOFTWARE_FACTORY_PRERENDER_URL + ? new URL(process.env.SOFTWARE_FACTORY_PRERENDER_URL) + : undefined; // The seeded test Postgres used by the harness runs with max_connections=20, so // isolated workers need a smaller per-process pool cap to keep workers=2 stable. export const DEFAULT_PG_POOL_MAX = Number( diff --git a/packages/software-factory/tests/fixtures.ts b/packages/software-factory/tests/fixtures.ts index 2389c6de450..c9ed1a77a4f 100644 --- a/packages/software-factory/tests/fixtures.ts +++ b/packages/software-factory/tests/fixtures.ts @@ -12,6 +12,7 @@ import { type PreparedTemplateMetadata, readSupportMetadata, } from '../src/runtime-metadata'; +import { startHarnessPrerenderServer } from '../src/harness/support-services'; import { buildBrowserState, installBrowserState } from './helpers/browser-auth'; type StartedFactoryRealm = { @@ -45,6 +46,10 @@ type FactoryRealmOptions = { type FactoryRealmInternalFixtures = { realm: StartedFactoryRealm; testWorkerPortSet: TestWorkerPortSet; + testWorkerPrerender: { + url: string; + stop(): Promise; + }; }; type SharedRealmHandle = { @@ -214,6 +219,7 @@ async function waitForMetadataFile( async function startRealmProcess( realmDir = defaultRealmDir, testWorkerPortSet: TestWorkerPortSet, + testWorkerPrerenderURL: string, ) { let tempDir = mkdtempSync(join(tmpdir(), 'software-factory-realm-')); let metadataFile = join(tempDir, 'runtime.json'); @@ -267,6 +273,7 @@ async function startRealmProcess( SOFTWARE_FACTORY_PRERENDER_PORT: String( testWorkerPortSet.prerenderPort, ), + SOFTWARE_FACTORY_PRERENDER_URL: testWorkerPrerenderURL, ...(supportMetadata?.context ? { SOFTWARE_FACTORY_CONTEXT: JSON.stringify(supportMetadata.context), @@ -386,10 +393,15 @@ async function acquireSharedRealm( key: string, realmDir: string, testWorkerPortSet: TestWorkerPortSet, + testWorkerPrerenderURL: string, ): Promise { let existing = sharedRealms.get(key); if (!existing) { - existing = startRealmProcess(realmDir, testWorkerPortSet).then((realm) => ({ + existing = startRealmProcess( + realmDir, + testWorkerPortSet, + testWorkerPrerenderURL, + ).then((realm) => ({ realm, refCount: 0, })); @@ -432,15 +444,45 @@ export const test = base.extend< }, { scope: 'worker' }, ], + testWorkerPrerender: [ + async ({ browserName: _browserName, testWorkerPortSet }, use) => { + // Prerender is intentionally testWorker-scoped instead of test-scoped. + // It is stateless, and now that the compat/realm ports are also stable + // for the same Playwright testWorker, each restarted realm stack can + // point back to the same prerender process without changing BOXEL_HOST_URL. + let boxelHostURL = `http://localhost:${testWorkerPortSet.compatRealmServerPort}`; + let prerender = await startHarnessPrerenderServer({ + boxelHostURL, + port: testWorkerPortSet.prerenderPort, + }); + try { + await use(prerender); + } finally { + await prerender.stop(); + } + }, + { scope: 'worker' }, + ], realm: async ( - { browserName: _browserName, realmDir, realmServerMode, testWorkerPortSet }, + { + browserName: _browserName, + realmDir, + realmServerMode, + testWorkerPortSet, + testWorkerPrerender, + }, use, testInfo, ) => { if (realmServerMode === 'shared') { let key = sharedRealmKey(testInfo.workerIndex, testInfo.file, realmDir); - let realm = await acquireSharedRealm(key, realmDir, testWorkerPortSet); + let realm = await acquireSharedRealm( + key, + realmDir, + testWorkerPortSet, + testWorkerPrerender.url, + ); try { await use(realm); } finally { @@ -449,7 +491,11 @@ export const test = base.extend< return; } - let realm = await startRealmProcess(realmDir, testWorkerPortSet); + let realm = await startRealmProcess( + realmDir, + testWorkerPortSet, + testWorkerPrerender.url, + ); try { await use(realm); } finally { From bba63c5cb62bb7f270d2cd04550320f74aff5b1f Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Tue, 24 Mar 2026 17:20:47 -0400 Subject: [PATCH 08/11] Fix software factory harness lint issues --- ...-for-software-factory-test-harness-plan.md | 99 ------------------- .../src/harness/isolated-realm-stack.ts | 2 +- packages/software-factory/tests/fixtures.ts | 10 +- 3 files changed, 8 insertions(+), 103 deletions(-) delete mode 100644 docs/cs-10485-dynamic-port-allocation-for-software-factory-test-harness-plan.md diff --git a/docs/cs-10485-dynamic-port-allocation-for-software-factory-test-harness-plan.md b/docs/cs-10485-dynamic-port-allocation-for-software-factory-test-harness-plan.md deleted file mode 100644 index a9a24370708..00000000000 --- a/docs/cs-10485-dynamic-port-allocation-for-software-factory-test-harness-plan.md +++ /dev/null @@ -1,99 +0,0 @@ -# CS-10485 Dynamic Port Allocation For Software Factory Test Harness Plan - -## Goal - -Remove the software-factory harness's hardcoded runtime ports so isolated realm stacks can run in parallel and the harness no longer collides with `mise run dev-all`. - -Success for this ticket means: - -- isolated realm servers do not assume fixed ports for realm-server, compat proxy, or worker-manager -- the harness support stack starts Synapse on a non-conflicting dynamic port -- Playwright fixtures discover actual runtime ports from metadata instead of reconstructing them from constants -- `playwright.config.ts` is updated to `workers: 2` after we confirm the targeted Playwright suite runs cleanly at that worker count - -## Current State - -The current implementation still hardcodes the same ports in multiple places: - -- [`packages/software-factory/src/harness.ts`](/home/hassan/codez/boxel-cs-10485-codex/packages/software-factory/src/harness.ts) - - `REALM_SERVER_PORT=4205` - - `COMPAT_REALM_SERVER_PORT=4201` - - `WORKER_MANAGER_PORT=4232` - - published realm domains and base/skills URL mappings are built from those constants -- [`packages/software-factory/src/cli/serve-realm.ts`](/home/hassan/codez/boxel-cs-10485-codex/packages/software-factory/src/cli/serve-realm.ts) - - writes metadata for `realmURL` and auth only, not the allocated ports that fixtures need -- [`packages/software-factory/tests/fixtures.ts`](/home/hassan/codez/boxel-cs-10485-codex/packages/software-factory/tests/fixtures.ts) - - assumes fixed ports for shutdown waiting and Playwright request rewrites -- [`packages/software-factory/playwright.config.ts`](/home/hassan/codez/boxel-cs-10485-codex/packages/software-factory/playwright.config.ts) - - pins `workers: 1` -- [`packages/matrix/docker/synapse/index.ts`](/home/hassan/codez/boxel-cs-10485-codex/packages/matrix/docker/synapse/index.ts) and [`packages/matrix/helpers/environment-config.ts`](/home/hassan/codez/boxel-cs-10485-codex/packages/matrix/helpers/environment-config.ts) - - the current non-environment path still uses fixed Synapse host port `8008` and fixed `getSynapseURL()` output of `http://localhost:8008` - -## Assumptions To Validate While Implementing - -- The realm-server and worker-manager CLIs tolerate `--port=0` and continue to emit their existing `ready` IPC message. -- The cleanest source of truth for actual bound ports may be a well-known runtime metadata file written after bind completes, instead of reconstructing ports indirectly. -- The compat port is only a compatibility surface for browser rewrites; nothing important should depend on `4201` specifically. -- The Synapse helper does not yet provide dynamic host-port behavior for the harness's current execution mode, so this needs a code change rather than just propagation. - -## Implementation Plan - -### 1. Make port allocation explicit in the harness - -- Update [`packages/software-factory/src/harness.ts`](/home/hassan/codez/boxel-cs-10485-codex/packages/software-factory/src/harness.ts) so the default realm-server, compat-proxy, and worker-manager ports are dynamic instead of fixed. -- Refactor `startIsolatedRealmStack()` to return the effective runtime ports in addition to the child processes. -- Add a single well-known runtime metadata contract for the spawned realm stack so the actual bound ports are published explicitly and consumed consistently. -- Make all internal URL construction use those effective ports instead of module-level constants: - - published realm domains - - base realm mappings - - optional skills realm mappings - - compat proxy target/listen ports - -### 2. Propagate support-stack runtime data - -- Update `startFactorySupportServices()` in [`packages/software-factory/src/harness.ts`](/home/hassan/codez/boxel-cs-10485-codex/packages/software-factory/src/harness.ts) to avoid fixed Synapse ports and to preserve the actual `matrixURL` in support context. -- Extend the Synapse startup path so the harness can obtain a dynamic host port in normal local test mode, not only in `BOXEL_ENVIRONMENT` mode. -- If the support metadata contract needs more structure, extend [`packages/software-factory/src/runtime-metadata.ts`](/home/hassan/codez/boxel-cs-10485-codex/packages/software-factory/src/runtime-metadata.ts) and the Playwright setup flow accordingly. - -### 3. Expand serve-realm metadata - -- Extend [`packages/software-factory/src/cli/serve-realm.ts`](/home/hassan/codez/boxel-cs-10485-codex/packages/software-factory/src/cli/serve-realm.ts) so the metadata JSON includes the actual runtime ports and any derived origins/prefixes the tests need. -- Keep the metadata payload as the single source of truth for isolated test realms. - -### 4. Switch tests to metadata-driven rewrites and cleanup - -- Update [`packages/software-factory/tests/fixtures.ts`](/home/hassan/codez/boxel-cs-10485-codex/packages/software-factory/tests/fixtures.ts) to read the real runtime ports from `serve-realm.ts` metadata. -- Remove the fixed-port assumptions from: - - shutdown waiting - - base realm redirect registration - - optional skills redirect registration -- Preserve the shared-realm cache behavior per Playwright worker and test file. - -### 5. Re-enable parallel Playwright workers - -- Raise `workers` in [`packages/software-factory/playwright.config.ts`](/home/hassan/codez/boxel-cs-10485-codex/packages/software-factory/playwright.config.ts) from `1` to `2` only after local confirmation that the targeted suite runs cleanly with two workers. -- Keep `fullyParallel: false` unless the updated test behavior proves broader parallelism is safe. - -## Target Files - -- [`packages/software-factory/src/harness.ts`](/home/hassan/codez/boxel-cs-10485-codex/packages/software-factory/src/harness.ts) -- [`packages/software-factory/src/cli/serve-realm.ts`](/home/hassan/codez/boxel-cs-10485-codex/packages/software-factory/src/cli/serve-realm.ts) -- [`packages/software-factory/src/runtime-metadata.ts`](/home/hassan/codez/boxel-cs-10485-codex/packages/software-factory/src/runtime-metadata.ts) if support metadata needs a contract update -- [`packages/software-factory/tests/fixtures.ts`](/home/hassan/codez/boxel-cs-10485-codex/packages/software-factory/tests/fixtures.ts) -- [`packages/software-factory/playwright.config.ts`](/home/hassan/codez/boxel-cs-10485-codex/packages/software-factory/playwright.config.ts) -- Possibly [`packages/software-factory/playwright.global-setup.ts`](/home/hassan/codez/boxel-cs-10485-codex/packages/software-factory/playwright.global-setup.ts) if support metadata wiring needs an adjustment -- [`packages/matrix/docker/synapse/index.ts`](/home/hassan/codez/boxel-cs-10485-codex/packages/matrix/docker/synapse/index.ts) -- [`packages/matrix/helpers/environment-config.ts`](/home/hassan/codez/boxel-cs-10485-codex/packages/matrix/helpers/environment-config.ts) if `getSynapseURL()` needs to stop assuming `8008` - -## Testing Notes - -- Run `pnpm lint` in [`packages/software-factory`](/home/hassan/codez/boxel-cs-10485-codex/packages/software-factory) before any commit. -- Run targeted Playwright coverage in [`packages/software-factory`](/home/hassan/codez/boxel-cs-10485-codex/packages/software-factory), starting with: - - `pnpm test -- --grep "darkfactory|factory target realm|factory bootstrap"` -- Verify the targeted suite runs with `workers=2` without realm-stack or Synapse port collisions before keeping that config change. -- If local parallel execution is noisy or environment-dependent, capture the residual risk explicitly and leave full confirmation to CI. - -## Open Questions - -- What exact metadata shape should the realm stack publish so both the harness and Playwright fixtures can consume it without duplicating URL construction logic? -- Are there any remaining browser-side assumptions that still reference the compat origin directly instead of using runtime metadata? diff --git a/packages/software-factory/src/harness/isolated-realm-stack.ts b/packages/software-factory/src/harness/isolated-realm-stack.ts index 6ac28797f81..3302d314a15 100644 --- a/packages/software-factory/src/harness/isolated-realm-stack.ts +++ b/packages/software-factory/src/harness/isolated-realm-stack.ts @@ -515,7 +515,7 @@ export async function startIsolatedRealmStack({ }; } catch (error) { try { - await prerender.stop(); + await prerender?.stop(); } catch { // best effort cleanup } diff --git a/packages/software-factory/tests/fixtures.ts b/packages/software-factory/tests/fixtures.ts index c9ed1a77a4f..646207e786f 100644 --- a/packages/software-factory/tests/fixtures.ts +++ b/packages/software-factory/tests/fixtures.ts @@ -43,8 +43,7 @@ type FactoryRealmOptions = { realmServerMode: RealmServerMode; }; -type FactoryRealmInternalFixtures = { - realm: StartedFactoryRealm; +type FactoryRealmWorkerFixtures = { testWorkerPortSet: TestWorkerPortSet; testWorkerPrerender: { url: string; @@ -52,6 +51,10 @@ type FactoryRealmInternalFixtures = { }; }; +type FactoryRealmTestFixtures = { + realm: StartedFactoryRealm; +}; + type SharedRealmHandle = { realm: StartedFactoryRealm; refCount: number; @@ -429,7 +432,8 @@ async function releaseSharedRealm(key: string): Promise { } export const test = base.extend< - FactoryRealmFixtures & FactoryRealmOptions & FactoryRealmInternalFixtures + FactoryRealmFixtures & FactoryRealmOptions & FactoryRealmTestFixtures, + FactoryRealmWorkerFixtures >({ realmDir: [defaultRealmDir, { option: true }], realmServerMode: ['shared', { option: true }], From 30d4eee62b2c3403ba5f31075152fec58de34a35 Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Tue, 24 Mar 2026 17:46:57 -0400 Subject: [PATCH 09/11] Address PR review feedback --- .../realm-server/lib/runtime-metadata-file.ts | 20 +++++++++++ packages/realm-server/main.ts | 15 ++------ packages/realm-server/middleware/index.ts | 36 ++++++++++++++++--- packages/realm-server/prerender/page-pool.ts | 28 +++++++++++++-- packages/realm-server/worker-manager.ts | 15 ++------ .../software-factory/src/cli/cache-realm.ts | 3 +- .../software-factory/src/cli/serve-realm.ts | 22 ++++++------ .../software-factory/src/cli/serve-support.ts | 22 ++++++------ .../src/harness/isolated-realm-stack.ts | 1 + .../src/harness/support-services.ts | 22 ++++++++---- packages/software-factory/tests/fixtures.ts | 9 ++++- 11 files changed, 131 insertions(+), 62 deletions(-) create mode 100644 packages/realm-server/lib/runtime-metadata-file.ts diff --git a/packages/realm-server/lib/runtime-metadata-file.ts b/packages/realm-server/lib/runtime-metadata-file.ts new file mode 100644 index 00000000000..6bf9bd48ca3 --- /dev/null +++ b/packages/realm-server/lib/runtime-metadata-file.ts @@ -0,0 +1,20 @@ +import { mkdirSync, renameSync, writeFileSync } from 'fs'; +import { dirname, join } from 'path'; + +export function writeRuntimeMetadataFile( + runtimeMetadataFile: string | undefined, + tempFilePrefix: string, + payload: unknown, +): void { + if (!runtimeMetadataFile) { + return; + } + + mkdirSync(dirname(runtimeMetadataFile), { recursive: true }); + let tempFile = join( + dirname(runtimeMetadataFile), + `.${tempFilePrefix}.${process.pid}.${Date.now()}.tmp`, + ); + writeFileSync(tempFile, JSON.stringify(payload, null, 2)); + renameSync(tempFile, runtimeMetadataFile); +} diff --git a/packages/realm-server/main.ts b/packages/realm-server/main.ts index abd1343cd3a..cfa78fb1503 100644 --- a/packages/realm-server/main.ts +++ b/packages/realm-server/main.ts @@ -15,8 +15,6 @@ import { NodeAdapter } from './node-realm'; import yargs from 'yargs'; import { RealmServer } from './server'; import { resolve } from 'path'; -import { dirname, join } from 'path'; -import { mkdirSync, renameSync, writeFileSync } from 'fs'; import * as Sentry from '@sentry/node'; import { PgAdapter, PgQueuePublisher } from '@cardstack/postgres'; import { MatrixClient } from '@cardstack/runtime-common/matrix-client'; @@ -33,6 +31,7 @@ import { registerService, deregisterEnvironment, } from './lib/dev-service-registry'; +import { writeRuntimeMetadataFile } from './lib/runtime-metadata-file'; (globalThis as any).ContentTagGlobal = ContentTagGlobal; @@ -41,17 +40,7 @@ const runtimeMetadataFile = process.env.SOFTWARE_FACTORY_REALM_SERVER_METADATA_FILE; function writeRuntimeMetadata(payload: unknown): void { - if (!runtimeMetadataFile) { - return; - } - - mkdirSync(dirname(runtimeMetadataFile), { recursive: true }); - let tempFile = join( - dirname(runtimeMetadataFile), - `.realm-server.${process.pid}.${Date.now()}.tmp`, - ); - writeFileSync(tempFile, JSON.stringify(payload, null, 2)); - renameSync(tempFile, runtimeMetadataFile); + writeRuntimeMetadataFile(runtimeMetadataFile, 'realm-server', payload); } if (process.env.NODE_ENV === 'test') { diff --git a/packages/realm-server/middleware/index.ts b/packages/realm-server/middleware/index.ts index b631c613c99..6cdd1077711 100644 --- a/packages/realm-server/middleware/index.ts +++ b/packages/realm-server/middleware/index.ts @@ -108,14 +108,40 @@ export function ecsMetadata(ctxt: Koa.Context, next: Koa.Next) { return next(); } +function isLoopbackAddress(address: string | undefined): boolean { + return ( + address === '127.0.0.1' || + address === '::1' || + address === '::ffff:127.0.0.1' + ); +} + export function fullRequestURL(ctxt: Koa.Context): URL { - let forwardedURL = ctxt.req.headers['x-boxel-forwarded-url']; - if (typeof forwardedURL === 'string' && forwardedURL.trim() !== '') { - return new URL(forwardedURL); - } let protocol = ctxt.req.headers['x-forwarded-proto'] === 'https' ? 'https' : 'http'; - return new URL(`${protocol}://${ctxt.req.headers.host}${ctxt.req.url}`); + let computedURL = new URL( + `${protocol}://${ctxt.req.headers.host}${ctxt.req.url}`, + ); + let forwardedURL = ctxt.req.headers['x-boxel-forwarded-url']; + if ( + process.env.BOXEL_TRUST_FORWARDED_URL === 'true' && + typeof forwardedURL === 'string' && + forwardedURL.trim() !== '' && + isLoopbackAddress(ctxt.req.socket?.remoteAddress) + ) { + try { + let parsed = new URL(forwardedURL); + if ( + parsed.pathname === computedURL.pathname && + parsed.search === computedURL.search + ) { + return parsed; + } + } catch { + // Ignore malformed forwarded URLs and fall back to the computed request. + } + } + return computedURL; } export async function fetchRequestFromContext( diff --git a/packages/realm-server/prerender/page-pool.ts b/packages/realm-server/prerender/page-pool.ts index 896720cd690..654842c5630 100644 --- a/packages/realm-server/prerender/page-pool.ts +++ b/packages/realm-server/prerender/page-pool.ts @@ -88,6 +88,20 @@ function isExpectedStandbyTargetNotReadyError(error: unknown): boolean { ); } +function isExpectedStandbyConsoleError(args: { + affinityKey: string; + type: ReturnType; + formatted: string; + locationURL?: string; +}): boolean { + return ( + args.affinityKey === 'standby' && + args.type === 'error' && + /status of 50[23]/.test(args.formatted) && + args.locationURL?.includes('/_standby') === true + ); +} + export class PagePool { #affinityPages = new Map>(); #standbys = new Set(); @@ -681,15 +695,25 @@ export class PagePool { let suffix = segments.length ? `:${segments.join(':')}` : ''; locationInfo = ` (${locationData.url}${suffix})`; } + let type = message.type(); + if ( + isExpectedStandbyConsoleError({ + affinityKey, + type, + formatted, + locationURL: locationData?.url, + }) + ) { + return; + } logFn( 'Console[%s] affinity=%s pageId=%s%s %s', - message.type(), + type, affinityKey, pageId, locationInfo, formatted, ); - let type = message.type(); if (type === 'error' || type === 'assert') { this.#recordConsoleError(pageId, { type, diff --git a/packages/realm-server/worker-manager.ts b/packages/realm-server/worker-manager.ts index 0ae74fa2239..ba0c4643caa 100644 --- a/packages/realm-server/worker-manager.ts +++ b/packages/realm-server/worker-manager.ts @@ -20,8 +20,6 @@ import { spawn, type ChildProcess } from 'child_process'; import pluralize from 'pluralize'; import Koa from 'koa'; import Router from '@koa/router'; -import { mkdirSync, renameSync, writeFileSync } from 'fs'; -import { dirname, join } from 'path'; import { ecsMetadata, fullRequestURL, livenessCheck } from './middleware'; import type { Server } from 'http'; import { PgAdapter } from '@cardstack/postgres'; @@ -36,6 +34,7 @@ import { renderIndexingDashboard, type PendingJob, } from './handlers/handle-indexing-dashboard'; +import { writeRuntimeMetadataFile } from './lib/runtime-metadata-file'; /* About the Worker Manager * @@ -49,17 +48,7 @@ const runtimeMetadataFile = process.env.SOFTWARE_FACTORY_WORKER_MANAGER_METADATA_FILE; function writeRuntimeMetadata(payload: unknown): void { - if (!runtimeMetadataFile) { - return; - } - - mkdirSync(dirname(runtimeMetadataFile), { recursive: true }); - let tempFile = join( - dirname(runtimeMetadataFile), - `.worker-manager.${process.pid}.${Date.now()}.tmp`, - ); - writeFileSync(tempFile, JSON.stringify(payload, null, 2)); - renameSync(tempFile, runtimeMetadataFile); + writeRuntimeMetadataFile(runtimeMetadataFile, 'worker-manager', payload); } // This is an ENV var we get from ECS that looks like: diff --git a/packages/software-factory/src/cli/cache-realm.ts b/packages/software-factory/src/cli/cache-realm.ts index 02599091740..9a4a3cea1f5 100644 --- a/packages/software-factory/src/cli/cache-realm.ts +++ b/packages/software-factory/src/cli/cache-realm.ts @@ -41,10 +41,9 @@ async function main(): Promise { ); } console.log(JSON.stringify(payload, null, 2)); - process.exit(0); } main().catch((error: unknown) => { console.error(error); - process.exit(1); + process.exitCode = 1; }); diff --git a/packages/software-factory/src/cli/serve-realm.ts b/packages/software-factory/src/cli/serve-realm.ts index 87160a9788b..36e53941723 100644 --- a/packages/software-factory/src/cli/serve-realm.ts +++ b/packages/software-factory/src/cli/serve-realm.ts @@ -46,7 +46,6 @@ async function main(): Promise { console.log(JSON.stringify(payload, null, 2)); let cleanExit = false; - let keepAlive = setInterval(() => {}, 60_000); process.on('exit', () => { if (!cleanExit) { for (let pid of runtime.childPids) { @@ -60,21 +59,24 @@ async function main(): Promise { }); let stop = async () => { - clearInterval(keepAlive); await runtime.stop(); cleanExit = true; - process.exit(0); }; - process.on('SIGINT', () => void stop()); - process.on('SIGTERM', () => void stop()); - - // Keep the harness process alive so its managed children stay attached until - // the test fixture explicitly shuts the stack down. - await new Promise(() => {}); + await new Promise((resolve, reject) => { + let handleSignal = () => { + process.removeListener('SIGINT', onSigint); + process.removeListener('SIGTERM', onSigterm); + void stop().then(resolve).catch(reject); + }; + let onSigint = () => handleSignal(); + let onSigterm = () => handleSignal(); + process.on('SIGINT', onSigint); + process.on('SIGTERM', onSigterm); + }); } main().catch((error: unknown) => { console.error(error); - process.exit(1); + process.exitCode = 1; }); diff --git a/packages/software-factory/src/cli/serve-support.ts b/packages/software-factory/src/cli/serve-support.ts index 73a53962641..e75522d7dd7 100644 --- a/packages/software-factory/src/cli/serve-support.ts +++ b/packages/software-factory/src/cli/serve-support.ts @@ -21,23 +21,25 @@ async function main(): Promise { writeSupportMetadata(payload); console.log(JSON.stringify(payload, null, 2)); - let keepAlive = setInterval(() => {}, 60_000); let stop = async () => { - clearInterval(keepAlive); await support.stop(); - process.exit(0); }; - process.on('SIGINT', () => void stop()); - process.on('SIGTERM', () => void stop()); - - // Keep the wrapper alive so test teardown can signal it and so the shared - // support processes remain attached to this parent process. - await new Promise(() => {}); + await new Promise((resolve, reject) => { + let handleSignal = () => { + process.removeListener('SIGINT', onSigint); + process.removeListener('SIGTERM', onSigterm); + void stop().then(resolve).catch(reject); + }; + let onSigint = () => handleSignal(); + let onSigterm = () => handleSignal(); + process.on('SIGINT', onSigint); + process.on('SIGTERM', onSigterm); + }); } main().catch((error: unknown) => { console.error(error); - process.exit(1); + process.exitCode = 1; }); diff --git a/packages/software-factory/src/harness/isolated-realm-stack.ts b/packages/software-factory/src/harness/isolated-realm-stack.ts index 3302d314a15..74a621d297f 100644 --- a/packages/software-factory/src/harness/isolated-realm-stack.ts +++ b/packages/software-factory/src/harness/isolated-realm-stack.ts @@ -345,6 +345,7 @@ export async function startIsolatedRealmStack({ REALM_SERVER_FULL_INDEX_ON_STARTUP: String(fullIndexOnStartup), LOW_CREDIT_THRESHOLD: '2000', LOG_LEVELS: DEFAULT_REALM_LOG_LEVELS, + BOXEL_TRUST_FORWARDED_URL: 'true', PUBLISHED_REALM_BOXEL_SPACE_DOMAIN: `localhost:${compatProxy.listenPort}`, PUBLISHED_REALM_BOXEL_SITE_DOMAIN: `localhost:${compatProxy.listenPort}`, SOFTWARE_FACTORY_WORKER_MANAGER_METADATA_FILE: workerManagerMetadataFile, diff --git a/packages/software-factory/src/harness/support-services.ts b/packages/software-factory/src/harness/support-services.ts index 2fb0e3527f7..e2625847dce 100644 --- a/packages/software-factory/src/harness/support-services.ts +++ b/packages/software-factory/src/harness/support-services.ts @@ -29,6 +29,10 @@ import { canConnectToPg } from './database'; let preparePgPromise: Promise | undefined; +function hostStartupLooksLikePortContention(logs: string): boolean { + return /EADDRINUSE|address already in use/i.test(logs); +} + async function loadSynapseModule() { let moduleSpecifier = '../../../matrix/docker/synapse/index.ts'; return (maybeRequire(moduleSpecifier) ?? (await import(moduleSpecifier))) as { @@ -111,17 +115,23 @@ async function ensureHostReady(matrixURL: string): Promise<{ await waitUntil( async () => { + try { + let readyResponse = await fetch(DEFAULT_HOST_URL); + if (readyResponse.ok) { + return true; + } + } catch { + // host not ready yet + } if (child.exitCode !== null) { + if (hostStartupLooksLikePortContention(logs)) { + return false; + } throw new Error( `host app exited early with code ${child.exitCode}\n${logs}`, ); } - try { - let readyResponse = await fetch(DEFAULT_HOST_URL); - return readyResponse.ok; - } catch { - return false; - } + return false; }, { timeout: 180_000, diff --git a/packages/software-factory/tests/fixtures.ts b/packages/software-factory/tests/fixtures.ts index 646207e786f..6ee8889bf79 100644 --- a/packages/software-factory/tests/fixtures.ts +++ b/packages/software-factory/tests/fixtures.ts @@ -80,6 +80,10 @@ const testSourceRealmDir = resolve( const sharedRealms = new Map>(); const testWorkerPortBlockSize = 10; const testWorkerPortSearchStride = 200; +const testWorkerRunOffset = Number( + process.env.SOFTWARE_FACTORY_TEST_WORKER_RUN_OFFSET ?? + ((process.pid * 31 + process.ppid) % 1000) * testWorkerPortBlockSize, +); const testWorkerPortBase = Number( process.env.SOFTWARE_FACTORY_TEST_WORKER_PORT_BASE ?? 43100, ); @@ -167,10 +171,13 @@ async function allocateTestWorkerPortSet( // prerender between tests, but those services should come back on the same // URLs for the lifetime of the testWorker. That keeps BOXEL_HOST_URL and the // prerender standby target stable within a worker even when the realm stack - // itself is recreated per test. + // itself is recreated per test. Include a per-process offset so concurrent + // Playwright runs with the same worker index do not all probe the same block + // first. for (let attempt = 0; attempt < 100; attempt++) { let blockStart = testWorkerPortBase + + testWorkerRunOffset + testWorkerIndex * testWorkerPortBlockSize + attempt * testWorkerPortSearchStride; let candidate: TestWorkerPortSet = { From 63d72f12336c0bdb03f492714115603cd44f5d38 Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Tue, 24 Mar 2026 18:21:56 -0400 Subject: [PATCH 10/11] Consolidate software factory cache prepare --- .../playwright.global-setup.ts | 45 +++++++--------- .../software-factory/src/cli/cache-realm.ts | 53 +++++++++++++------ 2 files changed, 57 insertions(+), 41 deletions(-) diff --git a/packages/software-factory/playwright.global-setup.ts b/packages/software-factory/playwright.global-setup.ts index cfb3405c76a..1e24da66581 100644 --- a/packages/software-factory/playwright.global-setup.ts +++ b/packages/software-factory/playwright.global-setup.ts @@ -180,16 +180,16 @@ async function waitForMetadataFile( ); } -async function prepareTemplateForRealm( - realmDir: string, +async function prepareTemplatesForRealms( + realmDirs: string[], context: Record, metadataFile: string, -): Promise { +): Promise { let cacheLogs = ''; setupLog.warn( - `starting cache:prepare for ${realmDir}; this can take a while on cold startup or in CI`, + `starting cache:prepare for ${realmDirs.length} realm(s); this can take a while on cold startup or in CI`, ); - let cacheChild = spawn('pnpm', ['cache:prepare', realmDir], { + let cacheChild = spawn('pnpm', ['cache:prepare', ...realmDirs], { cwd: packageRoot, stdio: ['ignore', 'pipe', 'pipe'], env: { @@ -218,24 +218,22 @@ async function prepareTemplateForRealm( let cacheStartedAt = Date.now(); await waitForCommand(cacheChild, () => cacheLogs); let cachePayload = await waitForMetadataFile<{ - realmDir: string; - templateDatabaseName: string; - realmURL: string; - realmServerURL: string; + preparedTemplates?: PreparedTemplateMetadata[]; }>(metadataFile, cacheChild, () => cacheLogs, 5_000); setupLog.info( - `cache:prepare finished for ${realmDir} in ${( + `cache:prepare finished for ${realmDirs.length} realm(s) in ${( (Date.now() - cacheStartedAt) / 1000 ).toFixed(1)}s`, ); - return { - realmDir: cachePayload.realmDir, - templateDatabaseName: cachePayload.templateDatabaseName, - templateRealmURL: cachePayload.realmURL, - templateRealmServerURL: cachePayload.realmServerURL, - }; + if (!cachePayload.preparedTemplates?.length) { + throw new Error( + `cache:prepare did not return preparedTemplates for ${realmDirs.join(', ')}`, + ); + } + + return cachePayload.preparedTemplates; } export default async function globalSetup() { @@ -283,16 +281,11 @@ export default async function globalSetup() { ); let preparedRealmDirs = [...new Set([realmDir, bootstrapTargetRealmDir])]; - let preparedTemplates: PreparedTemplateMetadata[] = []; - for (let [index, preparedRealmDir] of preparedRealmDirs.entries()) { - preparedTemplates.push( - await prepareTemplateForRealm( - preparedRealmDir, - payload.context, - resolve(sharedRuntimeDir, `cache-${index}.json`), - ), - ); - } + let preparedTemplates = await prepareTemplatesForRealms( + preparedRealmDirs, + payload.context, + resolve(sharedRuntimeDir, 'cache.json'), + ); let primaryTemplate = preparedTemplates.find((entry) => entry.realmDir === realmDir) ?? preparedTemplates[0]; diff --git a/packages/software-factory/src/cli/cache-realm.ts b/packages/software-factory/src/cli/cache-realm.ts index 9a4a3cea1f5..693948fc4cd 100644 --- a/packages/software-factory/src/cli/cache-realm.ts +++ b/packages/software-factory/src/cli/cache-realm.ts @@ -8,28 +8,51 @@ import { import { readSupportContext } from '../runtime-metadata'; async function main(): Promise { - let realmDir = resolve( - process.cwd(), - process.argv[2] ?? 'test-fixtures/darkfactory-adopter', - ); + let realmDirs = [ + ...new Set( + (process.argv.slice(2).length > 0 + ? process.argv.slice(2) + : ['test-fixtures/darkfactory-adopter'] + ).map((realmDir) => resolve(process.cwd(), realmDir)), + ), + ]; let serializedSupportContext = process.env.SOFTWARE_FACTORY_CONTEXT; let supportContext: FactoryRealmOptions['context'] = serializedSupportContext ? (JSON.parse(serializedSupportContext) as FactoryRealmOptions['context']) : (readSupportContext() as FactoryRealmOptions['context']); - let template = await ensureFactoryRealmTemplate({ - realmDir, - context: supportContext, - }); + let preparedTemplates = []; + for (let realmDir of realmDirs) { + let template = await ensureFactoryRealmTemplate({ + realmDir, + context: supportContext, + }); + preparedTemplates.push({ + realmDir, + cacheKey: template.cacheKey, + templateDatabaseName: template.templateDatabaseName, + fixtureHash: template.fixtureHash, + cacheHit: template.cacheHit, + realmURL: template.realmURL.href, + realmServerURL: template.realmServerURL.href, + }); + } + let primaryTemplate = preparedTemplates[0]; let payload = { - realmDir, - cacheKey: template.cacheKey, - templateDatabaseName: template.templateDatabaseName, - fixtureHash: template.fixtureHash, - cacheHit: template.cacheHit, - realmURL: template.realmURL.href, - realmServerURL: template.realmServerURL.href, + realmDir: primaryTemplate.realmDir, + cacheKey: primaryTemplate.cacheKey, + templateDatabaseName: primaryTemplate.templateDatabaseName, + fixtureHash: primaryTemplate.fixtureHash, + cacheHit: primaryTemplate.cacheHit, + realmURL: primaryTemplate.realmURL, + realmServerURL: primaryTemplate.realmServerURL, + preparedTemplates: preparedTemplates.map((template) => ({ + realmDir: template.realmDir, + templateDatabaseName: template.templateDatabaseName, + templateRealmURL: template.realmURL, + templateRealmServerURL: template.realmServerURL, + })), }; if (process.env.SOFTWARE_FACTORY_METADATA_FILE) { mkdirSync(dirname(process.env.SOFTWARE_FACTORY_METADATA_FILE), { From 8f45d6fafae5b229d99d0cd5ea70095ae8770ec3 Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Tue, 24 Mar 2026 19:23:46 -0400 Subject: [PATCH 11/11] Use log levels for prerender test output --- .../matrix/helpers/isolated-realm-server.ts | 4 --- packages/postgres/pg-adapter.ts | 2 +- .../realm-server/prerender/prerender-app.ts | 1 - .../prerender/prerender-server.ts | 8 +----- packages/realm-server/tests/helpers/index.ts | 1 - .../tests/scripts/run-qunit-with-test-pg.sh | 2 +- packages/runtime-common/log.ts | 20 ++++++++++---- .../software-factory/playwright.config.ts | 4 +++ .../playwright.global-setup.ts | 15 +++++++++++ packages/software-factory/scripts/test.ts | 7 ++++- .../software-factory/src/cli/cache-realm.ts | 4 +++ packages/software-factory/src/harness/api.ts | 13 +++++++--- .../software-factory/src/harness/shared.ts | 1 + .../src/harness/support-services.ts | 8 +----- packages/software-factory/src/logger.ts | 26 ++++++++++++------- .../software-factory/src/runtime-metadata.ts | 2 ++ 16 files changed, 76 insertions(+), 42 deletions(-) diff --git a/packages/matrix/helpers/isolated-realm-server.ts b/packages/matrix/helpers/isolated-realm-server.ts index e4055dd79bf..c9263e838b9 100644 --- a/packages/matrix/helpers/isolated-realm-server.ts +++ b/packages/matrix/helpers/isolated-realm-server.ts @@ -138,7 +138,6 @@ export async function startPrerenderServer( ): Promise { let port = await findAvailablePort(options?.port ?? DEFAULT_PRERENDER_PORT); let url = `http://localhost:${port}`; - let silent = process.env.SOFTWARE_FACTORY_PRERENDER_SILENT !== '0'; let env = { ...process.env, NODE_ENV: process.env.NODE_ENV ?? 'development', @@ -152,9 +151,6 @@ export async function startPrerenderServer( 'prerender/prerender-server', `--port=${port}`, ]; - if (silent) { - prerenderArgs.push('--silent'); - } let child = spawn('ts-node', prerenderArgs, { cwd: realmServerDir, diff --git a/packages/postgres/pg-adapter.ts b/packages/postgres/pg-adapter.ts index c5c733818b3..2c8c8734d4a 100644 --- a/packages/postgres/pg-adapter.ts +++ b/packages/postgres/pg-adapter.ts @@ -32,7 +32,7 @@ function config() { type Config = ReturnType; function configuredPoolMax(): number | undefined { - let rawValue = process.env.PG_POOL_MAX ?? process.env.PGPOOLMAX; + let rawValue = process.env.PG_POOL_MAX; if (!rawValue) { return undefined; } diff --git a/packages/realm-server/prerender/prerender-app.ts b/packages/realm-server/prerender/prerender-app.ts index 0291f353a40..6ac3eaa0d8c 100644 --- a/packages/realm-server/prerender/prerender-app.ts +++ b/packages/realm-server/prerender/prerender-app.ts @@ -726,7 +726,6 @@ async function unregisterWithManager(serverURL: string) { export function createPrerenderHttpServer(options?: { maxPages?: number; - silent?: boolean; port?: number; }): Server { let draining = false; diff --git a/packages/realm-server/prerender/prerender-server.ts b/packages/realm-server/prerender/prerender-server.ts index 2e8a762d982..690ee9b3d68 100644 --- a/packages/realm-server/prerender/prerender-server.ts +++ b/packages/realm-server/prerender/prerender-server.ts @@ -11,7 +11,7 @@ import { let log = logger('prerender-server'); -let { port, silent } = yargs(process.argv.slice(2)) +let { port } = yargs(process.argv.slice(2)) .usage('Start prerender server') .options({ port: { @@ -19,17 +19,11 @@ let { port, silent } = yargs(process.argv.slice(2)) demandOption: true, type: 'number', }, - silent: { - description: 'Disable forwarding Puppeteer console output to server logs', - type: 'boolean', - default: false, - }, }) .parseSync(); let webServerInstance: Server | undefined; webServerInstance = createPrerenderHttpServer({ - silent, port, }).listen(port); let actualPort = port; diff --git a/packages/realm-server/tests/helpers/index.ts b/packages/realm-server/tests/helpers/index.ts index fd374a299d1..682c15398e5 100644 --- a/packages/realm-server/tests/helpers/index.ts +++ b/packages/realm-server/tests/helpers/index.ts @@ -569,7 +569,6 @@ async function startTestPrerenderServer(): Promise { return testPrerenderURL; } let server = createPrerenderHttpServer({ - silent: Boolean(process.env.SILENT_PRERENDERER), maxPages: 1, }); prerenderServer = server; diff --git a/packages/realm-server/tests/scripts/run-qunit-with-test-pg.sh b/packages/realm-server/tests/scripts/run-qunit-with-test-pg.sh index d4e3422b975..4ef285c291e 100755 --- a/packages/realm-server/tests/scripts/run-qunit-with-test-pg.sh +++ b/packages/realm-server/tests/scripts/run-qunit-with-test-pg.sh @@ -6,7 +6,7 @@ SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" "${SCRIPT_DIR}/prepare-test-pg.sh" trap '"${SCRIPT_DIR}/stop-test-pg.sh" >/dev/null 2>&1 || true' EXIT INT TERM -BASE_LOG_LEVELS="*=error,prerenderer-chrome=silent,pg-adapter=warn,realm:requests=warn" +BASE_LOG_LEVELS="*=error,prerenderer-chrome=none,pg-adapter=warn,realm:requests=warn" EXTRA_LOG_LEVELS="${LOG_LEVELS-}" if [ -n "$EXTRA_LOG_LEVELS" ]; then EFFECTIVE_LOG_LEVELS="${BASE_LOG_LEVELS},${EXTRA_LOG_LEVELS}" diff --git a/packages/runtime-common/log.ts b/packages/runtime-common/log.ts index 5b31516d81b..bd3188fa8a3 100644 --- a/packages/runtime-common/log.ts +++ b/packages/runtime-common/log.ts @@ -23,13 +23,16 @@ import LogLevel, { type LogLevelDesc } from 'loglevel'; const DEFAULT_LOG_LEVEL = 'info'; -const validLevels = [ +// We historically documented @cardstack/logger-style `none`, while the +// underlying loglevel package uses `silent`. Accept both spellings and +// normalize to `silent` before configuring loglevel. +const canonicalLevels = [ 'trace', 'debug', 'info', 'warn', 'error', - 'silent', + 'none', 0, 1, 2, @@ -37,6 +40,7 @@ const validLevels = [ 4, 5, ]; +const validLevels = [...canonicalLevels, 'silent']; interface LogDefinitions { [logName: string]: LogLevel.LogLevelDesc; @@ -49,7 +53,7 @@ export function makeLogDefinitions( serializedLogLevels.split(',').map((pattern) => { let [logName, level] = pattern.split('='); assertLogLevelDesc(level); - return [logName, level]; + return [logName, normalizeLogLevelDesc(level)]; }), ); } @@ -87,10 +91,16 @@ function getLevelForLog( return matchingLogName ? logDefinitions[matchingLogName] : DEFAULT_LOG_LEVEL; } -function assertLogLevelDesc(level: any): asserts level is LogLevelDesc { +function normalizeLogLevelDesc(level: LogLevelDesc | 'none'): LogLevelDesc { + return level === 'none' ? 'silent' : level; +} + +function assertLogLevelDesc( + level: any, +): asserts level is LogLevelDesc | 'none' { if (!validLevels.includes(level)) { throw new Error( - `${level} is not a valid log level. valid values are ${validLevels.join()}`, + `${level} is not a valid log level. valid values are ${canonicalLevels.join()} (silent is accepted as an alias for none)`, ); } } diff --git a/packages/software-factory/playwright.config.ts b/packages/software-factory/playwright.config.ts index e57ef5cfa6b..28a3002f4f1 100644 --- a/packages/software-factory/playwright.config.ts +++ b/packages/software-factory/playwright.config.ts @@ -1,5 +1,9 @@ import { defineConfig } from '@playwright/test'; +const defaultPlaywrightLogLevels = + '*=warn,software-factory:playwright=info,software-factory:playwright:support=info,software-factory:playwright:cache=info,prerenderer-chrome=none'; +process.env.LOG_LEVELS ??= defaultPlaywrightLogLevels; + const realmPort = Number(process.env.SOFTWARE_FACTORY_REALM_PORT ?? 4205); const realmURL = process.env.SOFTWARE_FACTORY_REALM_URL ?? diff --git a/packages/software-factory/playwright.global-setup.ts b/packages/software-factory/playwright.global-setup.ts index 1e24da66581..98d3620be42 100644 --- a/packages/software-factory/playwright.global-setup.ts +++ b/packages/software-factory/playwright.global-setup.ts @@ -233,6 +233,21 @@ async function prepareTemplatesForRealms( ); } + for (let template of cachePayload.preparedTemplates) { + let realmLabel = template.realmDir.replace(`${packageRoot}/`, ''); + if (template.cacheHit) { + setupLog.info( + `cache hit for ${realmLabel} -> ${template.templateDatabaseName}`, + ); + } else { + setupLog.warn( + `cache miss for ${realmLabel} -> ${template.templateDatabaseName}: ${ + template.cacheMissReason ?? 'unknown reason' + }`, + ); + } + } + return cachePayload.preparedTemplates; } diff --git a/packages/software-factory/scripts/test.ts b/packages/software-factory/scripts/test.ts index e71e0a07193..f6692c84ede 100644 --- a/packages/software-factory/scripts/test.ts +++ b/packages/software-factory/scripts/test.ts @@ -9,6 +9,8 @@ type TestRunnerOptions = { headed: boolean; }; +const defaultNodeTestLogLevels = '*=info,prerenderer-chrome=none'; + function parseArgs(argv: string[]): TestRunnerOptions { let options: TestRunnerOptions = { nodeOnly: false, @@ -70,8 +72,11 @@ async function main(): Promise { let shouldRunPlaywright = !options.nodeOnly; if (shouldRunNode) { + let logLevels = process.env.LOG_LEVELS ?? defaultNodeTestLogLevels; await runCommand( - 'NODE_NO_WARNINGS=1 qunit --require ts-node/register/transpile-only tests/index.ts', + `LOG_LEVELS=${JSON.stringify( + logLevels, + )} NODE_NO_WARNINGS=1 qunit --require ts-node/register/transpile-only tests/index.ts`, ); } diff --git a/packages/software-factory/src/cli/cache-realm.ts b/packages/software-factory/src/cli/cache-realm.ts index 693948fc4cd..7cb7a9436ca 100644 --- a/packages/software-factory/src/cli/cache-realm.ts +++ b/packages/software-factory/src/cli/cache-realm.ts @@ -34,6 +34,7 @@ async function main(): Promise { templateDatabaseName: template.templateDatabaseName, fixtureHash: template.fixtureHash, cacheHit: template.cacheHit, + cacheMissReason: template.cacheMissReason, realmURL: template.realmURL.href, realmServerURL: template.realmServerURL.href, }); @@ -45,6 +46,7 @@ async function main(): Promise { templateDatabaseName: primaryTemplate.templateDatabaseName, fixtureHash: primaryTemplate.fixtureHash, cacheHit: primaryTemplate.cacheHit, + cacheMissReason: primaryTemplate.cacheMissReason, realmURL: primaryTemplate.realmURL, realmServerURL: primaryTemplate.realmServerURL, preparedTemplates: preparedTemplates.map((template) => ({ @@ -52,6 +54,8 @@ async function main(): Promise { templateDatabaseName: template.templateDatabaseName, templateRealmURL: template.realmURL, templateRealmServerURL: template.realmServerURL, + cacheHit: template.cacheHit, + cacheMissReason: template.cacheMissReason, })), }; if (process.env.SOFTWARE_FACTORY_METADATA_FILE) { diff --git a/packages/software-factory/src/harness/api.ts b/packages/software-factory/src/harness/api.ts index 3daede618a4..23b3d11b474 100644 --- a/packages/software-factory/src/harness/api.ts +++ b/packages/software-factory/src/harness/api.ts @@ -138,11 +138,9 @@ export async function ensureFactoryRealmTemplate( let templateDatabaseName = templateDatabaseNameForCacheKey(cacheKey); let cachedTemplateMetadata = readPreparedTemplateMetadata(templateDatabaseName); + let hasTemplateDatabase = await databaseExists(templateDatabaseName); - if ( - (await databaseExists(templateDatabaseName)) && - cachedTemplateMetadata - ) { + if (hasTemplateDatabase && cachedTemplateMetadata) { return { cacheKey, templateDatabaseName, @@ -153,6 +151,12 @@ export async function ensureFactoryRealmTemplate( }; } + let cacheMissReason = !cachedTemplateMetadata + ? hasTemplateDatabase + ? 'template metadata is missing' + : 'template database has not been prepared yet' + : 'template database is missing'; + let ownedSupport: | { context: FactorySupportContext; @@ -187,6 +191,7 @@ export async function ensureFactoryRealmTemplate( templateDatabaseName, fixtureHash, cacheHit: false, + cacheMissReason, realmURL, realmServerURL, }; diff --git a/packages/software-factory/src/harness/shared.ts b/packages/software-factory/src/harness/shared.ts index 79455622ac9..cf7dda21015 100644 --- a/packages/software-factory/src/harness/shared.ts +++ b/packages/software-factory/src/harness/shared.ts @@ -44,6 +44,7 @@ export interface FactoryRealmTemplate { templateDatabaseName: string; fixtureHash: string; cacheHit: boolean; + cacheMissReason?: string; realmURL: URL; realmServerURL: URL; } diff --git a/packages/software-factory/src/harness/support-services.ts b/packages/software-factory/src/harness/support-services.ts index e2625847dce..5c24de5e405 100644 --- a/packages/software-factory/src/harness/support-services.ts +++ b/packages/software-factory/src/harness/support-services.ts @@ -227,15 +227,9 @@ export async function startHarnessPrerenderServer(options: { port = await findAvailablePort(); } let url = `http://localhost:${port}`; - let silent = process.env.SOFTWARE_FACTORY_PRERENDER_SILENT !== '0'; let child = spawn( 'ts-node', - [ - '--transpileOnly', - 'prerender/prerender-server', - `--port=${port}`, - ...(silent ? ['--silent'] : []), - ], + ['--transpileOnly', 'prerender/prerender-server', `--port=${port}`], { cwd: realmServerDir, stdio: ['pipe', 'pipe', 'pipe'], diff --git a/packages/software-factory/src/logger.ts b/packages/software-factory/src/logger.ts index 7219ab2a981..ee703f936eb 100644 --- a/packages/software-factory/src/logger.ts +++ b/packages/software-factory/src/logger.ts @@ -1,4 +1,5 @@ export type LogLevel = 'trace' | 'debug' | 'info' | 'warn' | 'error' | 'none'; +type AcceptedLogLevel = LogLevel | 'silent'; type LoggerConfiguration = { defaultLevel?: LogLevel; @@ -36,37 +37,42 @@ export function logger(logName: string): Logger { function parseLogConfiguration( serializedLogLevels: string, ): LoggerConfiguration { - let parsedLevels = serializedLogLevels + // Keep pattern ordering intact to match @cardstack/logger. In particular, + // `*` is just another pattern rule rather than a special default-level + // signal, so later rules can still override earlier ones exactly as the + // package does. + let logLevels = serializedLogLevels .split(',') .map((pattern) => pattern.trim()) .filter(Boolean) .map((pattern) => { let [logName, level] = pattern.split('='); assertLogLevel(level); - return [logName, level] as [string, LogLevel]; + return [logName, normalizeLogLevel(level)] as [string, LogLevel]; }); - let defaultLevel = - parsedLevels.find(([pattern]) => pattern === '*')?.[1] ?? 'info'; - let logLevels = parsedLevels.filter(([pattern]) => pattern !== '*'); - return { - defaultLevel, + defaultLevel: 'info', logLevels, }; } -function assertLogLevel(level: unknown): asserts level is LogLevel { +function normalizeLogLevel(level: AcceptedLogLevel): LogLevel { + return level === 'silent' ? 'none' : level; +} + +function assertLogLevel(level: unknown): asserts level is AcceptedLogLevel { if ( level !== 'trace' && level !== 'debug' && level !== 'info' && level !== 'warn' && level !== 'error' && - level !== 'none' + level !== 'none' && + level !== 'silent' ) { throw new Error( - `${String(level)} is not a valid log level. valid values are trace,debug,info,warn,error,none`, + `${String(level)} is not a valid log level. valid values are trace,debug,info,warn,error,none (silent is accepted as an alias for none)`, ); } } diff --git a/packages/software-factory/src/runtime-metadata.ts b/packages/software-factory/src/runtime-metadata.ts index ab4ead6cbf8..538e7a8508b 100644 --- a/packages/software-factory/src/runtime-metadata.ts +++ b/packages/software-factory/src/runtime-metadata.ts @@ -23,6 +23,8 @@ export interface PreparedTemplateMetadata { templateDatabaseName: string; templateRealmURL: string; templateRealmServerURL: string; + cacheHit?: boolean; + cacheMissReason?: string; } export function getSupportMetadataFile() {