first test on fixing locks for integration tests with lookups

packages/@aws-cdk/integ-runner/lib/engines/toolkit-lib.ts

@@ -103,14 +103,21 @@ export class ToolkitLibRunnerEngine implements ICdk {
     });
 
     try {
-      await using lock = await this.toolkit.synth(cx, {
+      // Explicitly manage the lock lifecycle instead of using 'await using'
+      // to ensure it's properly released before any subsequent synth operations
+      const lock = await this.toolkit.synth(cx, {
         validateStacks: false,
         stacks: {
           strategy: StackSelectionStrategy.ALL_STACKS,
           failOnEmpty: false,
         },
       });
-      await this.validateRegion(lock);
+      try {
+        await this.validateRegion(lock);
+      } finally {
+        // Always dispose the lock to avoid conflicts with subsequent synth operations
+        await lock.dispose();
+      }
     } catch (e: any) {
       if (e.message.includes('Missing context keys')) {
         // @TODO - silently ignore missing context
@@ -140,15 +147,26 @@ export class ToolkitLibRunnerEngine implements ICdk {
    */
   public async deploy(options: DeployOptions) {
     const cx = await this.cx(options);
-    await this.toolkit.deploy(cx, {
-      roleArn: options.roleArn,
-      traceLogs: options.traceLogs,
-      stacks: this.stackSelector(options),
-      deploymentMethod: {
-        method: 'change-set',
-      },
-      outputsFile: options.outputsFile ? path.join(this.options.workingDirectory, options.outputsFile) : undefined,
-    });
+    try {
+      await this.toolkit.deploy(cx, {
+        roleArn: options.roleArn,
+        traceLogs: options.traceLogs,
+        stacks: this.stackSelector(options),
+        deploymentMethod: {
+          method: 'change-set',
+        },
+        outputsFile: options.outputsFile ? path.join(this.options.workingDirectory, options.outputsFile) : undefined,
+      });
+    } finally {
+      // Always dispose the cloud assembly source to release any locks
+      if (this.isDisposableCloudAssemblySource(cx)) {
+        await cx.dispose();
+      }
+
+      if (options.output) {
+        this.cleanupReadLocks(path.join(this.options.workingDirectory, options.output));
+      }
+    }
   }
 
   /**
@@ -202,7 +220,7 @@ export class ToolkitLibRunnerEngine implements ICdk {
     // check if the app is a path to existing snapshot and then use it as an assembly directory
     const potentialCxPath = path.join(this.options.workingDirectory, options.app);
     if (fs.pathExistsSync(potentialCxPath) && fs.statSync(potentialCxPath).isDirectory()) {
-      return this.toolkit.fromAssemblyDirectory(potentialCxPath);
+      return this.toolkit.fromAssemblyDirectory(potentialCxPath, { holdLock: false });
     }
 
     let outdir;
@@ -241,6 +259,25 @@ export class ToolkitLibRunnerEngine implements ICdk {
     };
   }
 
+  private cleanupReadLocks(directory: string): void {
+    if (!fs.pathExistsSync(directory)) {
+      return;
+    }
+
+    const prefix = `read.${process.pid}.`;
+    for (const entry of fs.readdirSync(directory)) {
+      if (entry.startsWith(prefix) && entry.endsWith('.lock')) {
+        fs.removeSync(path.join(directory, entry));
+      }
+    }
+  }
+
+  private isDisposableCloudAssemblySource(
+    source: ICloudAssemblySource,
+  ): source is ICloudAssemblySource & { dispose: () => Promise<void> } {
+    return typeof (source as { dispose?: unknown }).dispose === 'function';
+  }
+
   /**
    * Check that the regions for the stacks in the CloudAssembly match the regions requested on the engine
    *

packages/@aws-cdk/integ-runner/lib/runner/integ-test-runner.ts

@@ -493,6 +493,7 @@ export class IntegTestRunner extends IntegRunner {
           app: path.relative(this.directory, this.snapshotDir),
           lookups: expectedTestSuite?.enableLookups,
         });
+        this.cleanupSnapshotReadLocks();
       }
       // now deploy the "actual" test.
       await this.cdk.deploy({
@@ -605,4 +606,5 @@ export class IntegTestRunner extends IntegRunner {
       throw e;
     }
   }
+
 }

packages/@aws-cdk/integ-runner/lib/runner/runner-base.ts

@@ -354,6 +354,8 @@ export abstract class IntegRunner {
       fs.removeSync(this.snapshotDir);
     }
 
+    this.cleanupSnapshotReadLocks();
+
     const actualTestSuite = await this.actualTestSuite();
 
     // if lookups are enabled then we need to synth again
@@ -392,6 +394,19 @@ export abstract class IntegRunner {
     }
   }
 
+  protected cleanupSnapshotReadLocks(): void {
+    if (!fs.existsSync(this.snapshotDir)) {
+      return;
+    }
+
+    const prefix = `read.${process.pid}.`;
+    for (const entry of fs.readdirSync(this.snapshotDir)) {
+      if (entry.startsWith(prefix) && entry.endsWith('.lock')) {
+        fs.removeSync(path.join(this.snapshotDir, entry));
+      }
+    }
+  }
+
   protected getContext(additionalContext?: Record<string, any>): Record<string, any> {
     return {
       ...currentlyRecommendedAwsCdkLibFlags(),

packages/@aws-cdk/integ-runner/test/engines/toolkit-lib-snapshot-path.test.ts

@@ -48,7 +48,7 @@ describe('ToolkitLibRunnerEngine - Snapshot Path Handling', () => {
 
     expect(mockedFs.pathExistsSync).toHaveBeenCalledWith(fullSnapshotPath);
     expect(mockedFs.statSync).toHaveBeenCalledWith(fullSnapshotPath);
-    expect(mockToolkit.fromAssemblyDirectory).toHaveBeenCalledWith(fullSnapshotPath);
+    expect(mockToolkit.fromAssemblyDirectory).toHaveBeenCalledWith(fullSnapshotPath, { holdLock: false });
     expect(mockToolkit.fromCdkApp).not.toHaveBeenCalled();
   });
 

packages/@aws-cdk/integ-runner/test/engines/toolkit-lib.test.ts

@@ -1,18 +1,22 @@
 /* eslint-disable @typescript-eslint/unbound-method */
 import { Toolkit, BaseCredentials } from '@aws-cdk/toolkit-lib';
+import * as fs from 'fs-extra';
 import { ToolkitLibRunnerEngine } from '../../lib/engines/toolkit-lib';
 
 jest.mock('@aws-cdk/toolkit-lib');
+jest.mock('fs-extra');
 
 const MockedToolkit = Toolkit as jest.MockedClass<typeof Toolkit>;
 const MockedBaseCredentials = BaseCredentials as jest.Mocked<typeof BaseCredentials>;
+const mockedFs = fs as jest.Mocked<typeof fs>;
 
 describe('ToolkitLibRunnerEngine', () => {
   let mockToolkit: jest.Mocked<Toolkit>;
   let engine: ToolkitLibRunnerEngine;
 
   beforeEach(() => {
     jest.clearAllMocks();
+    mockedFs.pathExistsSync.mockReturnValue(false);
     mockToolkit = {
       synth: jest.fn(),
       fromCdkApp: jest.fn(),
@@ -32,7 +36,7 @@ describe('ToolkitLibRunnerEngine', () => {
   describe('synth', () => {
     it('should use fromCdkApp and produce for fast synthesis', async () => {
       const mockCx = { produce: jest.fn() };
-      const mockLock = { [Symbol.asyncDispose]: jest.fn(), cloudAssembly: { stacksRecursively: [] } };
+      const mockLock = { dispose: jest.fn(), [Symbol.asyncDispose]: jest.fn(), cloudAssembly: { stacksRecursively: [] } };
       mockCx.produce.mockResolvedValue(mockLock);
       mockToolkit.fromCdkApp.mockResolvedValue(mockCx as any);
       mockToolkit.synth.mockImplementation((cx) => cx.produce() as any);
@@ -57,7 +61,7 @@ describe('ToolkitLibRunnerEngine', () => {
           assetMetadata: false,
         },
       });
-      expect(mockLock[Symbol.asyncDispose]).toHaveBeenCalled();
+      expect(mockLock.dispose).toHaveBeenCalled();
     });
 
     it('should handle missing context error silently', async () => {
@@ -144,6 +148,29 @@ describe('ToolkitLibRunnerEngine', () => {
         outputsFile: '/test/dir/assertion-results.json',
       });
     });
+
+    it('cleans up read lock files in output directory after deploy', async () => {
+      const mockCx = {};
+      const outputDir = '/test/dir/cdk.out';
+      const pidPrefix = `read.${process.pid}.`;
+      const matchingLock = `${pidPrefix}1.lock`;
+      const otherLock = 'read.99999.1.lock';
+
+      mockToolkit.fromCdkApp.mockResolvedValue(mockCx as any);
+      mockToolkit.deploy.mockResolvedValue({} as any);
+      mockedFs.pathExistsSync.mockReturnValueOnce(false).mockReturnValue(true);
+      mockedFs.readdirSync.mockReturnValue([matchingLock, otherLock, 'other.txt'] as any);
+
+      await engine.deploy({
+        app: 'test-app',
+        stacks: ['stack1'],
+        output: 'cdk.out',
+      });
+
+      expect(mockedFs.removeSync).toHaveBeenCalledWith(`${outputDir}/${matchingLock}`);
+      expect(mockedFs.removeSync).not.toHaveBeenCalledWith(`${outputDir}/${otherLock}`);
+      expect(mockedFs.removeSync).not.toHaveBeenCalledWith(`${outputDir}/other.txt`);
+    });
   });
 
   describe('watch', () => {

packages/@aws-cdk/toolkit-lib/lib/api/cloud-assembly/source-builder.ts

@@ -70,6 +70,13 @@ export interface AssemblyDirectoryProps {
    * @default true
    */
   readonly failOnMissingContext?: boolean;
+
+  /**
+   * Whether to hold a read lock for the lifetime of the produced assembly.
+   *
+   * @default true
+   */
+  readonly holdLock?: boolean;
 }
 
 /**
@@ -410,6 +417,25 @@ export abstract class CloudAssemblySourceBuilder {
         const readLock = await new RWLock(directory).acquireRead();
         try {
           const asm = await assemblyFromDirectory(directory, services.ioHelper, props.loadAssemblyOptions);
+          if (props.holdLock === false) {
+            await readLock.release();
+            const noOpLock = { release: async () => undefined };
+            const assembly = new ReadableCloudAssembly(asm, noOpLock, { deleteOnDispose: false });
+            if (assembly.cloudAssembly.manifest.missing && assembly.cloudAssembly.manifest.missing.length > 0) {
+              if (props.failOnMissingContext ?? true) {
+                const missingKeysSet = missingContextKeys(assembly.cloudAssembly.manifest.missing);
+                const missingKeys = Array.from(missingKeysSet);
+                throw AssemblyError.withCause(
+                  'Assembly contains missing context. ' +
+                    "Make sure all necessary context is already in 'cdk.context.json' by running 'cdk synth' on a machine with sufficient AWS credentials and committing the result. " +
+                    `Missing context keys: '${missingKeys.join(', ')}'`,
+                  'Error producing assembly',
+                );
+              }
+            }
+            return new CachedCloudAssembly(assembly);
+          }
+
           const assembly = new ReadableCloudAssembly(asm, readLock, { deleteOnDispose: false });
           if (assembly.cloudAssembly.manifest.missing && assembly.cloudAssembly.manifest.missing.length > 0) {
             if (props.failOnMissingContext ?? true) {

packages/@aws-cdk/toolkit-lib/test/api/cloud-assembly/source-builder.test.ts

@@ -412,4 +412,19 @@ describe('fromAssemblyDirectory', () => {
     // THEN
     expect(assembly.cloudAssembly.stacksRecursively.map(s => s.hierarchicalId)).toEqual(['Stack1']);
   });
+
+  test('does not hold read lock when holdLock is false', async () => {
+    await using outdir = autoCleanOutDir();
+    const fixtureDir = path.join(__dirname, '..', '..', '_fixtures', 'two-empty-stacks', 'cdk.out');
+    await fs.copy(fixtureDir, outdir.dir);
+
+    const cx = await toolkit.fromAssemblyDirectory(outdir.dir, { holdLock: false });
+    const assembly = await cx.produce();
+
+    const lockFiles = (await fs.readdir(outdir.dir)).filter(name =>
+      name.startsWith(`read.${process.pid}.`) && name.endsWith('.lock'));
+
+    expect(lockFiles).toEqual([]);
+    await assembly.dispose();
+  });
 });