feat(gastown): staging batch — circuit breaker, drain/eviction, token refresh, Cloudflare links, fullscreen

cloudflare-gastown/AGENTS.md

@@ -58,6 +58,7 @@
   - `${beads}` → bare table name. Use for `FROM`, `INSERT INTO`, `DELETE FROM`.
   - `${beads.columns.status}` → bare column name. Use for `SET` clauses and `INSERT` column lists where the table is already implied.
   - `${beads.status}` → qualified `table.column`. Use for `SELECT`, `WHERE`, `JOIN ON`, `ORDER BY`, and anywhere a column could be ambiguous.
+- **Do not alias tables in SQL queries.** Always use the full table name and the qualified `${table.column}` interpolator. Aliases like `FROM beads b` combined with the qualified interpolator produce double-qualified names (`b.beads.bead_id`) that SQLite rejects. If a self-join requires disambiguation, use a raw string alias only for the second copy and reference its columns with `${table.columns.col}` (bare) prefixed manually.
 - Prefer static queries over dynamically constructed ones. Move conditional logic into the query itself using SQL constructs like `COALESCE`, `CASE`, `NULLIF`, or `WHERE (? IS NULL OR col = ?)` patterns so the full query is always visible as a single readable string.
 - Always parse query results with the Zod `Record` schemas from `db/tables/*.table.ts`. Never use ad-hoc `as Record<string, unknown>` casts or `String(row.col)` to extract fields — use `.pick()` for partial selects and `.array()` for lists, e.g. `BeadRecord.pick({ bead_id: true }).array().parse(rows)`. This keeps row parsing type-safe and co-located with the schema definition.
 - When a column has a SQL `CHECK` constraint that restricts it to a set of values (i.e. an enum), mirror that in the Record schema using `z.enum()` rather than `z.string()`, e.g. `role: z.enum(['polecat', 'refinery', 'mayor', 'witness'])`.

cloudflare-gastown/container/src/completion-reporter.ts

@@ -7,6 +7,47 @@
 
 import type { ManagedAgent } from './types';
 
+/**
+ * Notify the TownDO that the mayor has finished processing a prompt and
+ * is now waiting for user input. This lets the TownDO transition the
+ * mayor from "working" to "waiting", which drops the alarm to the idle
+ * cadence and stops health-check pings that reset the container's
+ * sleepAfter timer.
+ *
+ * Best-effort: errors are logged but do not propagate.
+ */
+export async function reportMayorWaiting(agent: ManagedAgent): Promise<void> {
+  const apiUrl = agent.gastownApiUrl;
+  const authToken =
+    process.env.GASTOWN_CONTAINER_TOKEN ?? agent.gastownContainerToken ?? agent.gastownSessionToken;
+  if (!apiUrl || !authToken) {
+    console.warn(
+      `Cannot report mayor ${agent.agentId} waiting: no API credentials on agent record`
+    );
+    return;
+  }
+
+  const url = `${apiUrl}/api/towns/${agent.townId}/rigs/${agent.rigId}/agents/${agent.agentId}/waiting`;
+  try {
+    const response = await fetch(url, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        Authorization: `Bearer ${authToken}`,
+      },
+      body: JSON.stringify({ agentId: agent.agentId, firedAt: Date.now() }),
+    });
+
+    if (!response.ok) {
+      console.warn(
+        `Failed to report mayor ${agent.agentId} waiting: ${response.status} ${response.statusText}`
+      );
+    }
+  } catch (err) {
+    console.warn(`Error reporting mayor ${agent.agentId} waiting:`, err);
+  }
+}
+
 /**
  * Notify the Rig DO that an agent session has completed or failed.
  * Best-effort: errors are logged but do not propagate.

cloudflare-gastown/container/src/control-server.ts

@@ -10,10 +10,12 @@ import {
   activeServerCount,
   getUptime,
   stopAll,
+  drainAll,
+  isDraining,
   getAgentEvents,
   registerEventSink,
 } from './process-manager';
-import { startHeartbeat, stopHeartbeat } from './heartbeat';
+import { startHeartbeat, stopHeartbeat, notifyContainerReady } from './heartbeat';
 import { pushContext as pushDashboardContext } from './dashboard-context';
 import { mergeBranch, setupRigBrowseWorktree } from './git-manager';
 import {
@@ -46,6 +48,53 @@ export function getCurrentTownConfig(): Record<string, unknown> | null {
   return lastKnownTownConfig;
 }
 
+/**
+ * Sync config-derived env vars from the last-known town config into
+ * process.env. Safe to call at any time — no-ops when no config is cached.
+ */
+function syncTownConfigToProcessEnv(): void {
+  const cfg = getCurrentTownConfig();
+  if (!cfg) return;
+
+  const CONFIG_ENV_MAP: Array<[string, string]> = [
+    ['github_cli_pat', 'GITHUB_CLI_PAT'],
+    ['git_author_name', 'GASTOWN_GIT_AUTHOR_NAME'],
+    ['git_author_email', 'GASTOWN_GIT_AUTHOR_EMAIL'],
+    ['kilocode_token', 'KILOCODE_TOKEN'],
+  ];
+  for (const [cfgKey, envKey] of CONFIG_ENV_MAP) {
+    const val = cfg[cfgKey];
+    if (typeof val === 'string' && val) {
+      process.env[envKey] = val;
+    } else {
+      delete process.env[envKey];
+    }
+  }
+
+  const gitAuth = cfg.git_auth;
+  if (typeof gitAuth === 'object' && gitAuth !== null) {
+    const auth = gitAuth as Record<string, unknown>;
+    for (const [authKey, envKey] of [
+      ['github_token', 'GIT_TOKEN'],
+      ['gitlab_token', 'GITLAB_TOKEN'],
+      ['gitlab_instance_url', 'GITLAB_INSTANCE_URL'],
+    ] as const) {
+      const val = auth[authKey];
+      if (typeof val === 'string' && val) {
+        process.env[envKey] = val;
+      } else {
+        delete process.env[envKey];
+      }
+    }
+  }
+
+  if (cfg.disable_ai_coauthor) {
+    process.env.GASTOWN_DISABLE_AI_COAUTHOR = '1';
+  } else {
+    delete process.env.GASTOWN_DISABLE_AI_COAUTHOR;
+  }
+}
+
 export const app = new Hono();
 
 // Parse and validate town config from X-Town-Config header (sent by TownDO on
@@ -92,11 +141,21 @@ app.use('*', async (c, next) => {
 
 // GET /health
 app.get('/health', c => {
+  // When the TownDO is draining, it passes the drain nonce and town
+  // ID via headers so idle containers (no running agents) can
+  // acknowledge readiness and clear the drain flag.
+  const drainNonce = c.req.header('X-Drain-Nonce');
+  const townId = c.req.header('X-Town-Id');
+  if (drainNonce && townId) {
+    void notifyContainerReady(townId, drainNonce);
+  }
+
   const response: HealthResponse = {
     status: 'ok',
     agents: activeAgentCount(),
     servers: activeServerCount(),
     uptime: getUptime(),
+    draining: isDraining() || undefined,
   };
   return c.json(response);
 });
@@ -133,8 +192,23 @@ app.post('/refresh-token', async c => {
   return c.json({ refreshed: true });
 });
 
+// POST /sync-config
+// Push config-derived env vars from X-Town-Config into process.env on
+// the running container. Called by TownDO.syncConfigToContainer() after
+// persisting env vars to DO storage, so the live process picks up
+// changes (e.g. refreshed KILOCODE_TOKEN) without a container restart.
+app.post('/sync-config', async c => {
+  syncTownConfigToProcessEnv();
+  return c.json({ synced: true });
+});
+
 // POST /agents/start
 app.post('/agents/start', async c => {
+  if (isDraining()) {
+    console.warn('[control-server] /agents/start: rejected — container is draining');
+    return c.json({ error: 'Container is draining, cannot start new agents' }, 503);
+  }
+
   const body: unknown = await c.req.json().catch(() => null);
   const parsed = StartAgentRequest.safeParse(body);
   if (!parsed.success) {
@@ -214,45 +288,7 @@ app.patch('/agents/:agentId/model', async c => {
   // Sync config-derived env vars from X-Town-Config into process.env so
   // the SDK server restart picks up fresh tokens and git identity.
   // The middleware already parsed the header into lastKnownTownConfig.
-  const cfg = getCurrentTownConfig();
-  if (cfg) {
-    const CONFIG_ENV_MAP: Array<[string, string]> = [
-      ['github_cli_pat', 'GITHUB_CLI_PAT'],
-      ['git_author_name', 'GASTOWN_GIT_AUTHOR_NAME'],
-      ['git_author_email', 'GASTOWN_GIT_AUTHOR_EMAIL'],
-    ];
-    for (const [cfgKey, envKey] of CONFIG_ENV_MAP) {
-      const val = cfg[cfgKey];
-      if (typeof val === 'string' && val) {
-        process.env[envKey] = val;
-      } else {
-        delete process.env[envKey];
-      }
-    }
-    // git_auth tokens
-    const gitAuth = cfg.git_auth;
-    if (typeof gitAuth === 'object' && gitAuth !== null) {
-      const auth = gitAuth as Record<string, unknown>;
-      for (const [authKey, envKey] of [
-        ['github_token', 'GIT_TOKEN'],
-        ['gitlab_token', 'GITLAB_TOKEN'],
-        ['gitlab_instance_url', 'GITLAB_INSTANCE_URL'],
-      ] as const) {
-        const val = auth[authKey];
-        if (typeof val === 'string' && val) {
-          process.env[envKey] = val;
-        } else {
-          delete process.env[envKey];
-        }
-      }
-    }
-    // disable_ai_coauthor
-    if (cfg.disable_ai_coauthor) {
-      process.env.GASTOWN_DISABLE_AI_COAUTHOR = '1';
-    } else {
-      delete process.env.GASTOWN_DISABLE_AI_COAUTHOR;
-    }
-  }
+  syncTownConfigToProcessEnv();
 
   await updateAgentModel(
     agentId,
@@ -723,15 +759,26 @@ export function startControlServer(): void {
     startHeartbeat(apiUrl, authToken);
   }
 
-  // Handle graceful shutdown
+  // Handle graceful shutdown (immediate, no drain — used by SIGINT for dev)
   const shutdown = async () => {
     console.log('Shutting down control server...');
     stopHeartbeat();
     await stopAll();
     process.exit(0);
   };
 
-  process.on('SIGTERM', () => void shutdown());
+  process.on(
+    'SIGTERM',
+    () =>
+      void (async () => {
+        console.log('[control-server] SIGTERM received — starting graceful drain...');
+        stopHeartbeat();
+        await drainAll();
+        await stopAll();
+        process.exit(0);
+      })()
+  );
+
   process.on('SIGINT', () => void shutdown());
 
   // Track connected WebSocket clients with optional agent filter

cloudflare-gastown/container/src/git-manager.ts

@@ -275,9 +275,45 @@ async function cloneRepoInner(
     `Cloning repo for rig ${options.rigId}: hasAuth=${hasAuth} envKeys=[${Object.keys(options.envVars ?? {}).join(',')}]`
   );
 
+  // Omit --branch: on empty repos (no commits) the default branch doesn't
+  // exist yet, so `git clone --branch <branch>` would fail with
+  // "Remote branch <branch> not found in upstream origin".
   await mkdir(dir, { recursive: true });
-  await exec('git', ['clone', '--no-checkout', '--branch', options.defaultBranch, authUrl, dir]);
+  await exec('git', ['clone', '--no-checkout', authUrl, dir]);
   await configureRepoCredentials(dir, options.gitUrl, options.envVars);
+
+  // Detect empty repo: git rev-parse HEAD fails when there are no commits.
+  const isEmpty = await exec('git', ['rev-parse', 'HEAD'], dir)
+    .then(() => false)
+    .catch(() => true);
+
+  if (isEmpty) {
+    console.log(`Detected empty repo for rig ${options.rigId}, creating initial commit`);
+    // Create an initial empty commit so branches/worktrees can be created.
+    // Use -c flags for user identity (the repo has no config yet and the
+    // container may not have GIT_AUTHOR_NAME set).
+    await exec(
+      'git',
+      [
+        '-c',
+        'user.name=Gastown',
+        '-c',
+        'user.email=gastown@kilo.ai',
+        'commit',
+        '--allow-empty',
+        '-m',
+        'Initial commit',
+      ],
+      dir
+    );
+    await exec('git', ['push', 'origin', `HEAD:${options.defaultBranch}`], dir);
+    // Best-effort: set remote HEAD so future operations know the default branch
+    await exec('git', ['remote', 'set-head', 'origin', options.defaultBranch], dir).catch(() => {});
+    // Fetch so origin/<defaultBranch> ref is available locally
+    await exec('git', ['fetch', 'origin'], dir);
+    console.log(`Created initial commit on empty repo for rig ${options.rigId}`);
+  }
+
   console.log(`Cloned repo for rig ${options.rigId}`);
   return dir;
 }
@@ -303,6 +339,18 @@ async function createWorktreeInner(options: WorktreeOptions): Promise<string> {
     return dir;
   }
 
+  // Verify the repo has at least one commit. If cloneRepoInner's initial
+  // commit push failed, there's no HEAD and we can't create branches.
+  const hasHead = await exec('git', ['rev-parse', '--verify', 'HEAD'], repo)
+    .then(() => true)
+    .catch(() => false);
+
+  if (!hasHead) {
+    throw new Error(
+      `Cannot create worktree: repo has no commits. Push an initial commit first or re-connect the rig.`
+    );
+  }
+
   // When a startPoint is provided (e.g. a convoy feature branch), create
   // the new branch from that ref so the agent begins with the latest
   // merged work from upstream. Without a startPoint, try to track the
@@ -398,6 +446,24 @@ async function setupBrowseWorktreeInner(rigId: string, defaultBranch: string): P
     return browseDir;
   }
 
+  // Check whether origin/<defaultBranch> exists. On a repo that was just
+  // initialized with an empty commit in cloneRepoInner the ref should
+  // exist, but if the push failed (network, permissions) it may not.
+  const hasRemoteBranch = await exec(
+    'git',
+    ['rev-parse', '--verify', `origin/${defaultBranch}`],
+    repo
+  )
+    .then(() => true)
+    .catch(() => false);
+
+  if (!hasRemoteBranch) {
+    console.log(
+      `Skipping browse worktree for rig ${rigId}: origin/${defaultBranch} not found (repo may be empty), will create on next fetch`
+    );
+    return browseDir;
+  }
+
   // Create a worktree on the default branch for browsing.
   // Force-create (or reset) the tracking branch to origin/<defaultBranch>
   // so a recreated browse worktree always starts from the latest remote