feat: add shared mirror cache for repeated repository clones (#51)

* feat(clone): add shared mirror cache for repeated repo clones

* fix(lib): reduce state gitignore sync complexity for lint

* test(e2e): verify clone mirror cache reuse

* fix(e2e): detect mirror path from generated cache directory

* fix(e2e): assert mirror path under projects root cache

Author: skulidropek

.github/workflows/check.yml

@@ -98,6 +98,19 @@ jobs:
       - name: OpenCode autoconnect
         run: bash scripts/e2e/opencode-autoconnect.sh
 
+  e2e-clone-cache:
+    name: E2E (Clone cache)
+    runs-on: ubuntu-latest
+    timeout-minutes: 25
+    steps:
+      - uses: actions/checkout@v6
+      - name: Install dependencies
+        uses: ./.github/actions/setup
+      - name: Docker info
+        run: docker version && docker compose version
+      - name: Clone cache reuse
+        run: bash scripts/e2e/clone-cache.sh
+
   e2e-login-context:
     name: E2E (Login context)
     runs-on: ubuntu-latest

packages/docker-git/tests/core/templates.test.ts

@@ -71,6 +71,11 @@ describe("planFiles", () => {
           "GIT_CREDENTIAL_HELPER_PATH=\"/usr/local/bin/docker-git-credential-helper\""
         )
         expect(entrypointSpec.contents).toContain("token=\"$GITHUB_TOKEN\"")
+        expect(entrypointSpec.contents).toContain("CACHE_ROOT=\"/home/dev/.docker-git/.cache/git-mirrors\"")
+        expect(entrypointSpec.contents).toContain("CLONE_CACHE_ARGS=\"--reference-if-able '$CACHE_REPO_DIR' --dissociate\"")
+        expect(entrypointSpec.contents).toContain("[clone-cache] using mirror: $CACHE_REPO_DIR")
+        expect(entrypointSpec.contents).toContain("git clone --progress $CLONE_CACHE_ARGS")
+        expect(entrypointSpec.contents).toContain("[clone-cache] mirror created: $CACHE_REPO_DIR")
       }
     }))
 

packages/lib/src/core/templates-entrypoint/tasks.ts

@@ -46,13 +46,42 @@ else
   AUTH_REPO_URL="$REPO_URL"
   if [[ -n "$GIT_AUTH_TOKEN" && "$REPO_URL" == https://* ]]; then
     AUTH_REPO_URL="$(printf "%s" "$REPO_URL" | sed "s#^https://#https://\${GIT_AUTH_USER}:\${GIT_AUTH_TOKEN}@#")"
+  fi
+
+  CLONE_CACHE_ARGS=""
+  CACHE_REPO_DIR=""
+  CACHE_ROOT="/home/${config.sshUser}/.docker-git/.cache/git-mirrors"
+  if command -v sha256sum >/dev/null 2>&1; then
+    REPO_CACHE_KEY="$(printf "%s" "$REPO_URL" | sha256sum | awk '{print $1}')"
+  elif command -v shasum >/dev/null 2>&1; then
+    REPO_CACHE_KEY="$(printf "%s" "$REPO_URL" | shasum -a 256 | awk '{print $1}')"
+  else
+    REPO_CACHE_KEY="$(printf "%s" "$REPO_URL" | tr '/:@' '_' | tr -cd '[:alnum:]_.-')"
+  fi
+
+  if [[ -n "$REPO_CACHE_KEY" ]]; then
+    CACHE_REPO_DIR="$CACHE_ROOT/$REPO_CACHE_KEY.git"
+    mkdir -p "$CACHE_ROOT"
+    chown 1000:1000 "$CACHE_ROOT" || true
+    if [[ -d "$CACHE_REPO_DIR" ]]; then
+      if su - ${config.sshUser} -c "git --git-dir '$CACHE_REPO_DIR' rev-parse --is-bare-repository >/dev/null 2>&1"; then
+        if ! su - ${config.sshUser} -c "GIT_TERMINAL_PROMPT=0 git --git-dir '$CACHE_REPO_DIR' fetch --progress --prune '$REPO_URL' '+refs/*:refs/*'"; then
+          echo "[clone-cache] mirror refresh failed for $REPO_URL"
+        fi
+        CLONE_CACHE_ARGS="--reference-if-able '$CACHE_REPO_DIR' --dissociate"
+        echo "[clone-cache] using mirror: $CACHE_REPO_DIR"
+      else
+        echo "[clone-cache] invalid mirror removed: $CACHE_REPO_DIR"
+        rm -rf "$CACHE_REPO_DIR"
+      fi
+    fi
   fi`
 
 const renderCloneBodyRef = (config: TemplateConfig): string =>
   `  if [[ -n "$REPO_REF" ]]; then
     if [[ "$REPO_REF" == refs/pull/* ]]; then
       REF_BRANCH="pr-$(printf "%s" "$REPO_REF" | tr '/:' '--')"
-      if ! su - ${config.sshUser} -c "GIT_TERMINAL_PROMPT=0 git clone --progress '$AUTH_REPO_URL' '$TARGET_DIR'"; then
+      if ! su - ${config.sshUser} -c "GIT_TERMINAL_PROMPT=0 git clone --progress $CLONE_CACHE_ARGS '$AUTH_REPO_URL' '$TARGET_DIR'"; then
         echo "[clone] git clone failed for $REPO_URL"
         CLONE_OK=0
       else
@@ -62,12 +91,12 @@ const renderCloneBodyRef = (config: TemplateConfig): string =>
         fi
       fi
     else
-      if ! su - ${config.sshUser} -c "GIT_TERMINAL_PROMPT=0 git clone --progress --branch '$REPO_REF' '$AUTH_REPO_URL' '$TARGET_DIR'"; then
+      if ! su - ${config.sshUser} -c "GIT_TERMINAL_PROMPT=0 git clone --progress $CLONE_CACHE_ARGS --branch '$REPO_REF' '$AUTH_REPO_URL' '$TARGET_DIR'"; then
         DEFAULT_REF="$(git ls-remote --symref "$AUTH_REPO_URL" HEAD 2>/dev/null | awk '/^ref:/ {print $2}' | head -n 1 || true)"
         DEFAULT_BRANCH="$(printf "%s" "$DEFAULT_REF" | sed 's#^refs/heads/##')"
         if [[ -n "$DEFAULT_BRANCH" ]]; then
           echo "[clone] branch '$REPO_REF' missing; retrying with '$DEFAULT_BRANCH'"
-          if ! su - ${config.sshUser} -c "GIT_TERMINAL_PROMPT=0 git clone --progress --branch '$DEFAULT_BRANCH' '$AUTH_REPO_URL' '$TARGET_DIR'"; then
+          if ! su - ${config.sshUser} -c "GIT_TERMINAL_PROMPT=0 git clone --progress $CLONE_CACHE_ARGS --branch '$DEFAULT_BRANCH' '$AUTH_REPO_URL' '$TARGET_DIR'"; then
             echo "[clone] git clone failed for $REPO_URL"
             CLONE_OK=0
           elif [[ "$REPO_REF" == issue-* ]]; then
@@ -83,12 +112,27 @@ const renderCloneBodyRef = (config: TemplateConfig): string =>
       fi
     fi
   else
-    if ! su - ${config.sshUser} -c "GIT_TERMINAL_PROMPT=0 git clone --progress '$AUTH_REPO_URL' '$TARGET_DIR'"; then
+    if ! su - ${config.sshUser} -c "GIT_TERMINAL_PROMPT=0 git clone --progress $CLONE_CACHE_ARGS '$AUTH_REPO_URL' '$TARGET_DIR'"; then
       echo "[clone] git clone failed for $REPO_URL"
       CLONE_OK=0
     fi
   fi`
 
+const renderCloneCacheFinalize = (config: TemplateConfig): string =>
+  `if [[ "$CLONE_OK" -eq 1 && -d "$TARGET_DIR/.git" && -n "$CACHE_REPO_DIR" && ! -d "$CACHE_REPO_DIR" ]]; then
+  CACHE_TMP_DIR="$CACHE_REPO_DIR.tmp-$$"
+  if su - ${config.sshUser} -c "rm -rf '$CACHE_TMP_DIR' && GIT_TERMINAL_PROMPT=0 git clone --mirror --progress '$TARGET_DIR/.git' '$CACHE_TMP_DIR'"; then
+    if mv "$CACHE_TMP_DIR" "$CACHE_REPO_DIR" 2>/dev/null; then
+      echo "[clone-cache] mirror created: $CACHE_REPO_DIR"
+    else
+      rm -rf "$CACHE_TMP_DIR"
+    fi
+  else
+    echo "[clone-cache] mirror bootstrap failed for $REPO_URL"
+    rm -rf "$CACHE_TMP_DIR"
+  fi
+fi`
+
 const renderIssueWorkspaceAgentsResolve = (): string =>
   `ISSUE_ID="$(printf "%s" "$REPO_REF" | sed -E 's#^issue-##')"
 ISSUE_URL=""
@@ -180,6 +224,8 @@ const renderCloneBody = (config: TemplateConfig): string =>
     "",
     renderCloneRemotes(config),
     "",
+    renderCloneCacheFinalize(config),
+    "",
     renderIssueWorkspaceAgents()
   ].join("\n")
 

packages/lib/src/usecases/docker-git-config-search.ts

@@ -10,7 +10,7 @@ type DockerGitConfigSearchState = {
 
 const isDockerGitConfig = (entry: string): boolean => entry.endsWith("docker-git.json")
 
-const shouldSkipDir = (entry: string): boolean => entry === ".git" || entry === ".orch"
+const shouldSkipDir = (entry: string): boolean => entry === ".git" || entry === ".orch" || entry === ".docker-git"
 
 const processDockerGitEntry = (
   fs: FileSystem.FileSystem,

packages/lib/src/usecases/state-repo/gitignore.ts

@@ -17,11 +17,18 @@ const volatileCodexIgnorePatterns: ReadonlyArray<string> = [
   "**/.orch/auth/codex/models_cache.json"
 ]
 
+const repositoryCacheIgnorePatterns: ReadonlyArray<string> = [
+  ".cache/git-mirrors/"
+]
+
 const defaultStateGitignore = [
   stateGitignoreMarker,
   "# NOTE: this repo intentionally tracks EVERYTHING under the state dir, including .orch/env and .orch/auth.",
   "# Keep the remote private; treat it as sensitive infrastructure state.",
   "",
+  "# Shared git mirrors cache (do not commit)",
+  ...repositoryCacheIgnorePatterns,
+  "",
   "# Volatile Codex artifacts (do not commit)",
   ...volatileCodexIgnorePatterns,
   ""
@@ -32,6 +39,34 @@ const normalizeGitignoreText = (text: string): string =>
     .replaceAll("\r\n", "\n")
     .trim()
 
+type MissingManagedPatterns = {
+  readonly repositoryCache: ReadonlyArray<string>
+  readonly volatileCodex: ReadonlyArray<string>
+}
+
+const collectMissingManagedPatterns = (prevLines: ReadonlySet<string>): MissingManagedPatterns => ({
+  repositoryCache: repositoryCacheIgnorePatterns.filter((p) => !prevLines.has(p)),
+  volatileCodex: volatileCodexIgnorePatterns.filter((p) => !prevLines.has(p))
+})
+
+const hasMissingManagedPatterns = (missing: MissingManagedPatterns): boolean =>
+  missing.repositoryCache.length > 0 || missing.volatileCodex.length > 0
+
+const appendManagedBlocks = (
+  prev: string,
+  missing: MissingManagedPatterns
+): string => {
+  const blocks = [
+    missing.repositoryCache.length > 0
+      ? `# Shared git mirrors cache (do not commit)\n${missing.repositoryCache.join("\n")}`
+      : "",
+    missing.volatileCodex.length > 0
+      ? `# Volatile Codex artifacts (do not commit)\n${missing.volatileCodex.join("\n")}`
+      : ""
+  ].filter((block) => block.length > 0)
+  return `${[prev.trimEnd(), ...blocks].join("\n\n")}\n`
+}
+
 export const ensureStateGitignore = (
   fs: FileSystem.FileSystem,
   path: Path.Path,
@@ -65,11 +100,10 @@ export const ensureStateGitignore = (
       return
     }
 
-    // Ensure volatile Codex artifacts are ignored; append if missing.
-    const missingVolatile = volatileCodexIgnorePatterns.filter((p) => !prevLines.has(p))
-    if (missingVolatile.length === 0) {
+    // Ensure managed ignore patterns exist; append any missing entries.
+    const missing = collectMissingManagedPatterns(prevLines)
+    if (!hasMissingManagedPatterns(missing)) {
       return
     }
-    const next = `${prev.trimEnd()}\n\n# Volatile Codex artifacts (do not commit)\n${missingVolatile.join("\n")}\n`
-    yield* _(fs.writeFileString(gitignorePath, next))
+    yield* _(fs.writeFileString(gitignorePath, appendManagedBlocks(prev, missing)))
   })

packages/lib/tests/usecases/docker-git-config-search.test.ts

@@ -0,0 +1,57 @@
+import * as FileSystem from "@effect/platform/FileSystem"
+import * as Path from "@effect/platform/Path"
+import { NodeContext } from "@effect/platform-node"
+import { describe, expect, it } from "@effect/vitest"
+import { Effect } from "effect"
+
+import { findDockerGitConfigPaths } from "../../src/usecases/docker-git-config-search.js"
+
+const withTempDir = <A, E, R>(
+  use: (tempDir: string) => Effect.Effect<A, E, R>
+): Effect.Effect<A, E, R | FileSystem.FileSystem> =>
+  Effect.scoped(
+    Effect.gen(function*(_) {
+      const fs = yield* _(FileSystem.FileSystem)
+      const tempDir = yield* _(
+        fs.makeTempDirectoryScoped({
+          prefix: "docker-git-config-search-"
+        })
+      )
+      return yield* _(use(tempDir))
+    })
+  )
+
+const writeFileWithParents = (
+  fs: FileSystem.FileSystem,
+  path: Path.Path,
+  filePath: string
+) =>
+  Effect.gen(function*(_) {
+    const parent = path.dirname(filePath)
+    yield* _(fs.makeDirectory(parent, { recursive: true }))
+    yield* _(fs.writeFileString(filePath, "{}\n"))
+  })
+
+describe("findDockerGitConfigPaths", () => {
+  it.effect("skips metadata and shared docker-git cache directories", () =>
+    withTempDir((root) =>
+      Effect.gen(function*(_) {
+        const fs = yield* _(FileSystem.FileSystem)
+        const path = yield* _(Path.Path)
+        const includedMain = path.join(root, "org/repo-a/docker-git.json")
+        const includedNested = path.join(root, "org/repo-b/nested/docker-git.json")
+        const ignoredGit = path.join(root, "org/repo-a/.git/docker-git.json")
+        const ignoredOrch = path.join(root, "org/repo-a/.orch/docker-git.json")
+        const ignoredDockerGit = path.join(root, ".docker-git/.cache/git-mirrors/docker-git.json")
+
+        yield* _(writeFileWithParents(fs, path, includedMain))
+        yield* _(writeFileWithParents(fs, path, includedNested))
+        yield* _(writeFileWithParents(fs, path, ignoredGit))
+        yield* _(writeFileWithParents(fs, path, ignoredOrch))
+        yield* _(writeFileWithParents(fs, path, ignoredDockerGit))
+
+        const found = yield* _(findDockerGitConfigPaths(fs, path, root))
+        expect([...found].sort()).toEqual([includedMain, includedNested].sort())
+      })
+    ).pipe(Effect.provide(NodeContext.layer)))
+})

packages/lib/tests/usecases/state-repo-gitignore.test.ts

@@ -0,0 +1,64 @@
+import * as FileSystem from "@effect/platform/FileSystem"
+import * as Path from "@effect/platform/Path"
+import { NodeContext } from "@effect/platform-node"
+import { describe, expect, it } from "@effect/vitest"
+import { Effect } from "effect"
+
+import { ensureStateGitignore } from "../../src/usecases/state-repo/gitignore.js"
+
+const withTempDir = <A, E, R>(
+  use: (tempDir: string) => Effect.Effect<A, E, R>
+): Effect.Effect<A, E, R | FileSystem.FileSystem> =>
+  Effect.scoped(
+    Effect.gen(function*(_) {
+      const fs = yield* _(FileSystem.FileSystem)
+      const tempDir = yield* _(
+        fs.makeTempDirectoryScoped({
+          prefix: "docker-git-state-gitignore-"
+        })
+      )
+      return yield* _(use(tempDir))
+    })
+  )
+
+describe("ensureStateGitignore", () => {
+  it.effect("creates managed .gitignore with repository mirror cache ignored", () =>
+    withTempDir((root) =>
+      Effect.gen(function*(_) {
+        const fs = yield* _(FileSystem.FileSystem)
+        const path = yield* _(Path.Path)
+
+        yield* _(ensureStateGitignore(fs, path, root))
+
+        const gitignore = yield* _(fs.readFileString(path.join(root, ".gitignore")))
+        expect(gitignore).toContain("# docker-git state repository")
+        expect(gitignore).toContain(".cache/git-mirrors/")
+        expect(gitignore).toContain("**/.orch/auth/codex/models_cache.json")
+      })
+    ).pipe(Effect.provide(NodeContext.layer)))
+
+  it.effect("appends missing cache ignore pattern for managed files", () =>
+    withTempDir((root) =>
+      Effect.gen(function*(_) {
+        const fs = yield* _(FileSystem.FileSystem)
+        const path = yield* _(Path.Path)
+        const gitignorePath = path.join(root, ".gitignore")
+        const existing = [
+          "# docker-git state repository",
+          "# NOTE: this repo intentionally tracks EVERYTHING under the state dir, including .orch/env and .orch/auth.",
+          "# Keep the remote private; treat it as sensitive infrastructure state.",
+          "",
+          "custom-ignore/",
+          ""
+        ].join("\n")
+
+        yield* _(fs.writeFileString(gitignorePath, existing))
+        yield* _(ensureStateGitignore(fs, path, root))
+
+        const gitignore = yield* _(fs.readFileString(gitignorePath))
+        expect(gitignore).toContain("custom-ignore/")
+        expect(gitignore).toContain("# Shared git mirrors cache (do not commit)")
+        expect(gitignore).toContain(".cache/git-mirrors/")
+      })
+    ).pipe(Effect.provide(NodeContext.layer)))
+})