From 3722c12b9b4be8da35a1db463a641a17b55abb92 Mon Sep 17 00:00:00 2001 From: Mike Voets Date: Fri, 13 Mar 2026 15:20:38 +0900 Subject: [PATCH 1/2] feat(cli): add --concurrency flag to cdk destroy --- .../toolkit-lib/lib/actions/destroy/index.ts | 7 +++ .../work-graph/build-destroy-work-graph.ts | 36 ++++++++++++ .../toolkit-lib/lib/api/work-graph/index.ts | 1 + .../lib/api/work-graph/work-graph-builder.ts | 1 + .../lib/api/work-graph/work-graph.ts | 15 +++++ .../toolkit-lib/lib/toolkit/toolkit.ts | 22 ++++--- .../toolkit-lib/test/actions/destroy.test.ts | 12 ++++ packages/aws-cdk/lib/cli/cdk-toolkit.ts | 28 +++++++-- packages/aws-cdk/lib/cli/cli-config.ts | 1 + .../aws-cdk/lib/cli/cli-type-registry.json | 6 ++ packages/aws-cdk/lib/cli/cli.ts | 1 + .../aws-cdk/lib/cli/convert-to-user-input.ts | 1 + .../lib/cli/parse-command-line-arguments.ts | 6 ++ packages/aws-cdk/lib/cli/user-input.ts | 7 +++ packages/aws-cdk/test/cli/cdk-toolkit.test.ts | 57 +++++++++++++++++++ 15 files changed, 189 insertions(+), 12 deletions(-) create mode 100644 packages/@aws-cdk/toolkit-lib/lib/api/work-graph/build-destroy-work-graph.ts diff --git a/packages/@aws-cdk/toolkit-lib/lib/actions/destroy/index.ts b/packages/@aws-cdk/toolkit-lib/lib/actions/destroy/index.ts index d0086ee95..c838aa1be 100644 --- a/packages/@aws-cdk/toolkit-lib/lib/actions/destroy/index.ts +++ b/packages/@aws-cdk/toolkit-lib/lib/actions/destroy/index.ts @@ -12,4 +12,11 @@ export interface DestroyOptions { * The arn of the IAM role to use for the stack destroy operation */ readonly roleArn?: string; + + /** + * Maximum number of simultaneous destroys (dependency permitting) to execute. + * + * @default 1 + */ + readonly concurrency?: number; } diff --git a/packages/@aws-cdk/toolkit-lib/lib/api/work-graph/build-destroy-work-graph.ts b/packages/@aws-cdk/toolkit-lib/lib/api/work-graph/build-destroy-work-graph.ts new file mode 100644 index 000000000..a3e308487 --- /dev/null +++ b/packages/@aws-cdk/toolkit-lib/lib/api/work-graph/build-destroy-work-graph.ts @@ -0,0 +1,36 @@ +import * as cxapi from '@aws-cdk/cloud-assembly-api'; +import { WorkGraph } from './work-graph'; +import { DeploymentState } from './work-graph-types'; +import type { IoHelper } from '../io/private'; + +/** + * Build a WorkGraph for destroy with reversed dependencies. + * + * In deploy order, if A depends on B, B is deployed first. For destroy, + * the arrows are reversed: A must be destroyed before B. + */ +export function buildDestroyWorkGraph(stacks: cxapi.CloudFormationStackArtifact[], ioHelper: IoHelper): WorkGraph { + const graph = new WorkGraph({}, ioHelper); + const selectedIds = new Set(stacks.map((s) => s.id)); + + for (const stack of stacks) { + graph.addNodes({ + type: 'stack', + id: stack.id, + dependencies: new Set(), + stack, + deploymentState: DeploymentState.PENDING, + priority: 0, + }); + } + + for (const stack of stacks) { + for (const dep of stack.dependencies) { + if (cxapi.CloudFormationStackArtifact.isCloudFormationStackArtifact(dep) && selectedIds.has(dep.id)) { + graph.addDependency(dep.id, stack.id); + } + } + } + + return graph; +} diff --git a/packages/@aws-cdk/toolkit-lib/lib/api/work-graph/index.ts b/packages/@aws-cdk/toolkit-lib/lib/api/work-graph/index.ts index f06d9a3b5..3d7d32afa 100644 --- a/packages/@aws-cdk/toolkit-lib/lib/api/work-graph/index.ts +++ b/packages/@aws-cdk/toolkit-lib/lib/api/work-graph/index.ts @@ -1,3 +1,4 @@ +export * from './build-destroy-work-graph'; export * from './work-graph'; export * from './work-graph-builder'; export * from './work-graph-types'; diff --git a/packages/@aws-cdk/toolkit-lib/lib/api/work-graph/work-graph-builder.ts b/packages/@aws-cdk/toolkit-lib/lib/api/work-graph/work-graph-builder.ts index 101d6e972..9b07442fb 100644 --- a/packages/@aws-cdk/toolkit-lib/lib/api/work-graph/work-graph-builder.ts +++ b/packages/@aws-cdk/toolkit-lib/lib/api/work-graph/work-graph-builder.ts @@ -189,3 +189,4 @@ function stacksFromAssets(artifacts: cxapi.CloudArtifact[]) { function onlyStacks(artifacts: cxapi.CloudArtifact[]) { return artifacts.filter(x => cxapi.CloudFormationStackArtifact.isCloudFormationStackArtifact(x)); } + diff --git a/packages/@aws-cdk/toolkit-lib/lib/api/work-graph/work-graph.ts b/packages/@aws-cdk/toolkit-lib/lib/api/work-graph/work-graph.ts index ff03e25b7..4e2fefd8f 100644 --- a/packages/@aws-cdk/toolkit-lib/lib/api/work-graph/work-graph.ts +++ b/packages/@aws-cdk/toolkit-lib/lib/api/work-graph/work-graph.ts @@ -118,6 +118,21 @@ export class WorkGraph { }); } + /** + * Execute all stack nodes in dependency order with the given concurrency. + * + * Unlike `doParallel`, this method only handles stack nodes and takes a + * simple callback. Intended for destroy where there are no asset nodes. + */ + public processStacks(concurrency: number, fn: (stackNode: StackNode) => Promise) { + return this.forAllArtifacts(concurrency, async (x: WorkNode) => { + if (x.type !== 'stack') { + return; + } + await fn(x); + }); + } + /** * Return the set of unblocked nodes */ diff --git a/packages/@aws-cdk/toolkit-lib/lib/toolkit/toolkit.ts b/packages/@aws-cdk/toolkit-lib/lib/toolkit/toolkit.ts index d4f06bc5c..ae2217293 100644 --- a/packages/@aws-cdk/toolkit-lib/lib/toolkit/toolkit.ts +++ b/packages/@aws-cdk/toolkit-lib/lib/toolkit/toolkit.ts @@ -89,7 +89,7 @@ import { ResourceMigrator } from '../api/resource-import'; import { tagsForStack } from '../api/tags/private'; import { DEFAULT_TOOLKIT_STACK_NAME } from '../api/toolkit-info'; import type { AssetBuildNode, AssetPublishNode, Concurrency, StackNode } from '../api/work-graph'; -import { WorkGraphBuilder } from '../api/work-graph'; +import { WorkGraphBuilder, buildDestroyWorkGraph } from '../api/work-graph'; import type { AssemblyData, RefactorResult, StackDetails, SuccessfulDeployStackResult } from '../payloads'; import { PermissionChangeType } from '../payloads'; import { formatErrorMessage, formatTime, obscureTemplate, serializeStructure, validateSnsTopicArn } from '../util'; @@ -1247,8 +1247,7 @@ export class Toolkit extends CloudAssemblySourceBuilder { private async _destroy(assembly: StackAssembly, action: 'deploy' | 'destroy', options: DestroyOptions): Promise { const selectStacks = stacksOpt(options); const ioHelper = asIoHelper(this.ioHost, action); - // The stacks will have been ordered for deployment, so reverse them for deletion. - const stacks = (await assembly.selectStacksV2(selectStacks)).reversed(); + const stacks = await assembly.selectStacksV2(selectStacks); const ret: DestroyResult = { stacks: [], @@ -1262,16 +1261,21 @@ export class Toolkit extends CloudAssemblySourceBuilder { return ret; } + const concurrency = options.concurrency || 1; + let destroyCount = 0; + const destroySpan = await ioHelper.span(SPAN.DESTROY_ACTION).begin({ stacks: stacks.stackArtifacts, }); try { - for (const [index, stack] of stacks.stackArtifacts.entries()) { + const destroyStack = async (stackNode: StackNode) => { + const stack = stackNode.stack; + destroyCount++; try { const singleDestroySpan = await ioHelper.span(SPAN.DESTROY_STACK) - .begin(chalk.green(`${chalk.blue(stack.displayName)}: destroying... [${index + 1}/${stacks.stackCount}]`), { + .begin(chalk.green(`${chalk.blue(stack.displayName)}: destroying... [${destroyCount}/${stacks.stackCount}]`), { total: stacks.stackCount, - current: index + 1, + current: destroyCount, stack, }); const deployments = await this.deploymentsForAction(action); @@ -1297,7 +1301,10 @@ export class Toolkit extends CloudAssemblySourceBuilder { await ioHelper.notify(IO.CDK_TOOLKIT_E7900.msg(`\n ❌ ${chalk.blue(stack.displayName)}: ${action} failed ${e}`, { error: e })); throw e; } - } + }; + + const workGraph = buildDestroyWorkGraph(stacks.stackArtifacts, ioHelper); + await workGraph.processStacks(concurrency, destroyStack); return ret; } finally { @@ -1439,3 +1446,4 @@ async function synthAndMeasure( function zeroTime(): ElapsedTime { return { asMs: 0, asSec: 0 }; } + diff --git a/packages/@aws-cdk/toolkit-lib/test/actions/destroy.test.ts b/packages/@aws-cdk/toolkit-lib/test/actions/destroy.test.ts index b63fa432e..d638b1f95 100644 --- a/packages/@aws-cdk/toolkit-lib/test/actions/destroy.test.ts +++ b/packages/@aws-cdk/toolkit-lib/test/actions/destroy.test.ts @@ -82,6 +82,18 @@ describe('destroy', () => { })); }); + test('destroy with concurrency', async () => { + // WHEN + const cx = await builderFixture(toolkit, 'two-empty-stacks'); + await toolkit.destroy(cx, { + stacks: { strategy: StackSelectionStrategy.ALL_STACKS }, + concurrency: 3, + }); + + // THEN + expect(mockDestroyStack).toHaveBeenCalledTimes(2); + }); + test('action disposes of assembly produced by source', async () => { // GIVEN const [assemblySource, mockDispose, realDispose] = await disposableCloudAssemblySource(toolkit); diff --git a/packages/aws-cdk/lib/cli/cdk-toolkit.ts b/packages/aws-cdk/lib/cli/cdk-toolkit.ts index 2afff6093..1497ca905 100644 --- a/packages/aws-cdk/lib/cli/cdk-toolkit.ts +++ b/packages/aws-cdk/lib/cli/cdk-toolkit.ts @@ -16,6 +16,7 @@ import type { ActionLessRequest, IoHelper } from '../../lib/api-private'; import { asIoHelper, cfnApi, createIgnoreMatcher, IO, tagsForStack } from '../../lib/api-private'; import type { AssetBuildNode, AssetPublishNode, Concurrency, StackNode, WorkGraph } from '../api'; import { + buildDestroyWorkGraph, CloudWatchLogEventMonitor, DEFAULT_TOOLKIT_STACK_NAME, DiffFormatter, @@ -1000,8 +1001,7 @@ export class CdkToolkit { public async destroy(options: DestroyOptions) { const ioHelper = this.ioHost.asIoHelper(); - // The stacks will have been ordered for deployment, so reverse them for deletion. - const stacks = (await this.selectStacksForDestroy(options.selector, options.exclusively)).reversed(); + const stacks = await this.selectStacksForDestroy(options.selector, options.exclusively); if (!options.force) { const motivation = 'Destroying stacks is an irreversible action'; @@ -1017,9 +1017,18 @@ export class CdkToolkit { } } + const concurrency = options.concurrency || 1; const action = options.fromDeploy ? 'deploy' : 'destroy'; - for (const [index, stack] of stacks.stackArtifacts.entries()) { - await ioHelper.defaults.info(chalk.green('%s: destroying... [%s/%s]'), chalk.blue(stack.displayName), index + 1, stacks.stackCount); + let destroyCount = 0; + + if (concurrency > 1) { + this.ioHost.stackProgress = StackActivityProgress.EVENTS; + } + + const destroyStack = async (stackNode: StackNode) => { + const stack = stackNode.stack; + destroyCount++; + await ioHelper.defaults.info(chalk.green('%s: destroying... [%s/%s]'), chalk.blue(stack.displayName), destroyCount, stacks.stackCount); try { await this.props.deployments.destroyStack({ stack, @@ -1031,7 +1040,10 @@ export class CdkToolkit { await ioHelper.defaults.error(`\n ❌ %s: ${action} failed`, chalk.blue(stack.displayName), e); throw e; } - } + }; + + const workGraph = buildDestroyWorkGraph(stacks.stackArtifacts, ioHelper); + await workGraph.processStacks(concurrency, destroyStack); } public async list( @@ -1914,6 +1926,11 @@ export interface DestroyOptions { * Whether the destroy request came from a deploy. */ fromDeploy?: boolean; + + /** + * Maximum number of simultaneous destroys (dependency permitting) to execute. + */ + concurrency?: number; } /** @@ -2210,3 +2227,4 @@ function requiresApproval(requireApproval: RequireApproval, permissionChangeType return requireApproval === RequireApproval.ANYCHANGE || requireApproval === RequireApproval.BROADENING && permissionChangeType === PermissionChangeType.BROADENING; } + diff --git a/packages/aws-cdk/lib/cli/cli-config.ts b/packages/aws-cdk/lib/cli/cli-config.ts index 1096f1921..766f80cfa 100644 --- a/packages/aws-cdk/lib/cli/cli-config.ts +++ b/packages/aws-cdk/lib/cli/cli-config.ts @@ -344,6 +344,7 @@ export async function makeConfig(): Promise { all: { type: 'boolean', default: false, desc: 'Destroy all available stacks' }, exclusively: { type: 'boolean', alias: 'e', desc: 'Only destroy requested stacks, don\'t include dependees' }, force: { type: 'boolean', alias: 'f', desc: 'Do not ask for confirmation before destroying the stacks' }, + concurrency: { type: 'number', desc: 'Maximum number of simultaneous destroys (dependency permitting) to execute.', default: 1, requiresArg: true }, }, }, 'diff': { diff --git a/packages/aws-cdk/lib/cli/cli-type-registry.json b/packages/aws-cdk/lib/cli/cli-type-registry.json index 999b33d9e..1b8bd218e 100644 --- a/packages/aws-cdk/lib/cli/cli-type-registry.json +++ b/packages/aws-cdk/lib/cli/cli-type-registry.json @@ -755,6 +755,12 @@ "type": "boolean", "alias": "f", "desc": "Do not ask for confirmation before destroying the stacks" + }, + "concurrency": { + "type": "number", + "desc": "Maximum number of simultaneous destroys (dependency permitting) to execute.", + "default": 1, + "requiresArg": true } } }, diff --git a/packages/aws-cdk/lib/cli/cli.ts b/packages/aws-cdk/lib/cli/cli.ts index 913d42d61..6c3d01461 100644 --- a/packages/aws-cdk/lib/cli/cli.ts +++ b/packages/aws-cdk/lib/cli/cli.ts @@ -470,6 +470,7 @@ export async function exec(args: string[], synthesizer?: Synthesizer): Promise): any { type: 'boolean', alias: 'f', desc: 'Do not ask for confirmation before destroying the stacks', + }) + .option('concurrency', { + default: 1, + type: 'number', + desc: 'Maximum number of simultaneous destroys (dependency permitting) to execute.', + requiresArg: true, }), ) .command( diff --git a/packages/aws-cdk/lib/cli/user-input.ts b/packages/aws-cdk/lib/cli/user-input.ts index 01a15d9a4..4e224c93f 100644 --- a/packages/aws-cdk/lib/cli/user-input.ts +++ b/packages/aws-cdk/lib/cli/user-input.ts @@ -1213,6 +1213,13 @@ export interface DestroyOptions { */ readonly force?: boolean; + /** + * Maximum number of simultaneous destroys (dependency permitting) to execute. + * + * @default - 1 + */ + readonly concurrency?: number; + /** * Positional argument for destroy */ diff --git a/packages/aws-cdk/test/cli/cdk-toolkit.test.ts b/packages/aws-cdk/test/cli/cdk-toolkit.test.ts index a4a377654..b7a88d876 100644 --- a/packages/aws-cdk/test/cli/cdk-toolkit.test.ts +++ b/packages/aws-cdk/test/cli/cdk-toolkit.test.ts @@ -1205,6 +1205,63 @@ describe('destroy', () => { }); }).resolves; }); + + test('destroy with concurrency', async () => { + const toolkit = defaultToolkitSetup(); + + await toolkit.destroy({ + selector: { patterns: ['*'] }, + exclusively: false, + force: true, + concurrency: 5, + }); + }); + + test('destroy respects dependency order with concurrency', async () => { + const stackC: TestStackArtifact = { + stackName: 'Test-Stack-C', + template: { Resources: { TemplateName: 'Test-Stack-C' } }, + env: 'aws://123456789012/bermuda-triangle-1', + }; + const stackD: TestStackArtifact = { + stackName: 'Test-Stack-D', + template: { Resources: { TemplateName: 'Test-Stack-D' } }, + env: 'aws://123456789012/bermuda-triangle-1', + depends: [stackC.stackName], + }; + cloudExecutable = await MockCloudExecutable.create({ + stacks: [stackC, stackD], + }); + + const destroyOrder: string[] = []; + const fakeDeployments = new FakeCloudFormation({ + 'Test-Stack-C': { Baz: 'Zinga!' }, + 'Test-Stack-D': { Baz: 'Zinga!' }, + }); + const originalDestroyStack = fakeDeployments.destroyStack.bind(fakeDeployments); + fakeDeployments.destroyStack = async (options: DestroyStackOptions) => { + destroyOrder.push(options.stack.stackName); + return originalDestroyStack(options); + }; + + const toolkit = new CdkToolkit({ + ioHost, + cloudExecutable, + configuration: cloudExecutable.configuration, + sdkProvider: cloudExecutable.sdkProvider, + deployments: fakeDeployments, + }); + + await toolkit.destroy({ + selector: { allTopLevel: true, patterns: [] }, + exclusively: false, + force: true, + concurrency: 10, + }); + + // stackD depends on stackC, so D must be destroyed before C + expect(destroyOrder.indexOf('Test-Stack-D')).toBeLessThan(destroyOrder.indexOf('Test-Stack-C')); + }); }); describe('watch', () => { From 25be8f096c0a12b52e7eddd48f31d63c03ce9564 Mon Sep 17 00:00:00 2001 From: Mike Voets Date: Fri, 13 Mar 2026 20:10:56 +0900 Subject: [PATCH 2/2] test(cli-integ): add integration tests for cdk destroy --concurrency --- .../cdk-destroy-all-concurrently.integtest.ts | 24 ++++++++++++++++++ .../cdk-destroy-with-concurrency.integtest.ts | 25 +++++++++++++++++++ .../aws-cdk/lib/cli/convert-to-user-input.ts | 1 + 3 files changed, 50 insertions(+) create mode 100644 packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/destroy/cdk-destroy-all-concurrently.integtest.ts create mode 100644 packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/destroy/cdk-destroy-with-concurrency.integtest.ts diff --git a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/destroy/cdk-destroy-all-concurrently.integtest.ts b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/destroy/cdk-destroy-all-concurrently.integtest.ts new file mode 100644 index 000000000..ac6fbdf97 --- /dev/null +++ b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/destroy/cdk-destroy-all-concurrently.integtest.ts @@ -0,0 +1,24 @@ +import { DescribeStacksCommand } from '@aws-sdk/client-cloudformation'; +import { integTest, withDefaultFixture } from '../../../lib'; + +jest.setTimeout(2 * 60 * 60_000); // Includes the time to acquire locks, worst-case single-threaded runtime + +integTest( + 'destroy all concurrently', + withDefaultFixture(async (fixture) => { + // Deploy two independent stacks + await fixture.cdkDeploy(['test-1', 'test-2']); + + // Destroy both concurrently + await fixture.cdkDestroy('test-*', { options: ['--concurrency', '2'] }); + + // Assert both stacks are gone + await expect(fixture.aws.cloudFormation.send(new DescribeStacksCommand({ + StackName: fixture.fullStackName('test-1'), + }))).rejects.toThrow(/does not exist/); + + await expect(fixture.aws.cloudFormation.send(new DescribeStacksCommand({ + StackName: fixture.fullStackName('test-2'), + }))).rejects.toThrow(/does not exist/); + }), +); diff --git a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/destroy/cdk-destroy-with-concurrency.integtest.ts b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/destroy/cdk-destroy-with-concurrency.integtest.ts new file mode 100644 index 000000000..b87a36c58 --- /dev/null +++ b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/destroy/cdk-destroy-with-concurrency.integtest.ts @@ -0,0 +1,25 @@ +import { DescribeStacksCommand } from '@aws-sdk/client-cloudformation'; +import { integTest, withDefaultFixture } from '../../../lib'; + +jest.setTimeout(2 * 60 * 60_000); // Includes the time to acquire locks, worst-case single-threaded runtime + +integTest( + 'destroy with concurrency respects dependency ordering', + withDefaultFixture(async (fixture) => { + // Deploy the consuming stack which will include the producing stack + await fixture.cdkDeploy('order-consuming'); + + // Destroy the providing stack with concurrency, which must destroy + // the consuming stack first due to reversed dependency ordering + await fixture.cdkDestroy('order-providing', { options: ['--concurrency', '2'] }); + + // Assert both stacks are gone + await expect(fixture.aws.cloudFormation.send(new DescribeStacksCommand({ + StackName: fixture.fullStackName('order-consuming'), + }))).rejects.toThrow(/does not exist/); + + await expect(fixture.aws.cloudFormation.send(new DescribeStacksCommand({ + StackName: fixture.fullStackName('order-providing'), + }))).rejects.toThrow(/does not exist/); + }), +); diff --git a/packages/aws-cdk/lib/cli/convert-to-user-input.ts b/packages/aws-cdk/lib/cli/convert-to-user-input.ts index feee36a47..a3c47960f 100644 --- a/packages/aws-cdk/lib/cli/convert-to-user-input.ts +++ b/packages/aws-cdk/lib/cli/convert-to-user-input.ts @@ -473,6 +473,7 @@ export function convertConfigToUserInput(config: any): UserInput { all: config.destroy?.all, exclusively: config.destroy?.exclusively, force: config.destroy?.force, + concurrency: config.destroy?.concurrency, }; const diffOptions = { exclusively: config.diff?.exclusively,