feat(cli): add --concurrency flag to cdk destroy

packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/destroy/cdk-destroy-all-concurrently.integtest.ts

@@ -0,0 +1,24 @@
+import { DescribeStacksCommand } from '@aws-sdk/client-cloudformation';
+import { integTest, withDefaultFixture } from '../../../lib';
+
+jest.setTimeout(2 * 60 * 60_000); // Includes the time to acquire locks, worst-case single-threaded runtime
+
+integTest(
+  'destroy all concurrently',
+  withDefaultFixture(async (fixture) => {
+    // Deploy two independent stacks
+    await fixture.cdkDeploy(['test-1', 'test-2']);
+
+    // Destroy both concurrently
+    await fixture.cdkDestroy('test-*', { options: ['--concurrency', '2'] });
+
+    // Assert both stacks are gone
+    await expect(fixture.aws.cloudFormation.send(new DescribeStacksCommand({
+      StackName: fixture.fullStackName('test-1'),
+    }))).rejects.toThrow(/does not exist/);
+
+    await expect(fixture.aws.cloudFormation.send(new DescribeStacksCommand({
+      StackName: fixture.fullStackName('test-2'),
+    }))).rejects.toThrow(/does not exist/);
+  }),
+);

packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/destroy/cdk-destroy-with-concurrency.integtest.ts

@@ -0,0 +1,25 @@
+import { DescribeStacksCommand } from '@aws-sdk/client-cloudformation';
+import { integTest, withDefaultFixture } from '../../../lib';
+
+jest.setTimeout(2 * 60 * 60_000); // Includes the time to acquire locks, worst-case single-threaded runtime
+
+integTest(
+  'destroy with concurrency respects dependency ordering',
+  withDefaultFixture(async (fixture) => {
+    // Deploy the consuming stack which will include the producing stack
+    await fixture.cdkDeploy('order-consuming');
+
+    // Destroy the providing stack with concurrency, which must destroy
+    // the consuming stack first due to reversed dependency ordering
+    await fixture.cdkDestroy('order-providing', { options: ['--concurrency', '2'] });
+
+    // Assert both stacks are gone
+    await expect(fixture.aws.cloudFormation.send(new DescribeStacksCommand({
+      StackName: fixture.fullStackName('order-consuming'),
+    }))).rejects.toThrow(/does not exist/);
+
+    await expect(fixture.aws.cloudFormation.send(new DescribeStacksCommand({
+      StackName: fixture.fullStackName('order-providing'),
+    }))).rejects.toThrow(/does not exist/);
+  }),
+);

packages/@aws-cdk/toolkit-lib/lib/actions/destroy/index.ts

@@ -12,4 +12,11 @@ export interface DestroyOptions {
    * The arn of the IAM role to use for the stack destroy operation
    */
   readonly roleArn?: string;
+
+  /**
+   * Maximum number of simultaneous destroys (dependency permitting) to execute.
+   *
+   * @default 1
+   */
+  readonly concurrency?: number;
 }

packages/@aws-cdk/toolkit-lib/lib/api/work-graph/build-destroy-work-graph.ts

@@ -0,0 +1,36 @@
+import * as cxapi from '@aws-cdk/cloud-assembly-api';
+import { WorkGraph } from './work-graph';
+import { DeploymentState } from './work-graph-types';
+import type { IoHelper } from '../io/private';
+
+/**
+ * Build a WorkGraph for destroy with reversed dependencies.
+ *
+ * In deploy order, if A depends on B, B is deployed first. For destroy,
+ * the arrows are reversed: A must be destroyed before B.
+ */
+export function buildDestroyWorkGraph(stacks: cxapi.CloudFormationStackArtifact[], ioHelper: IoHelper): WorkGraph {
+  const graph = new WorkGraph({}, ioHelper);
+  const selectedIds = new Set(stacks.map((s) => s.id));
+
+  for (const stack of stacks) {
+    graph.addNodes({
+      type: 'stack',
+      id: stack.id,
+      dependencies: new Set<string>(),
+      stack,
+      deploymentState: DeploymentState.PENDING,
+      priority: 0,
+    });
+  }
+
+  for (const stack of stacks) {
+    for (const dep of stack.dependencies) {
+      if (cxapi.CloudFormationStackArtifact.isCloudFormationStackArtifact(dep) && selectedIds.has(dep.id)) {
+        graph.addDependency(dep.id, stack.id);
+      }
+    }
+  }
+
+  return graph;
+}

packages/@aws-cdk/toolkit-lib/lib/api/work-graph/index.ts

@@ -1,3 +1,4 @@
+export * from './build-destroy-work-graph';
 export * from './work-graph';
 export * from './work-graph-builder';
 export * from './work-graph-types';

packages/@aws-cdk/toolkit-lib/lib/api/work-graph/work-graph-builder.ts

@@ -189,3 +189,4 @@ function stacksFromAssets(artifacts: cxapi.CloudArtifact[]) {
 function onlyStacks(artifacts: cxapi.CloudArtifact[]) {
   return artifacts.filter(x => cxapi.CloudFormationStackArtifact.isCloudFormationStackArtifact(x));
 }
+

packages/@aws-cdk/toolkit-lib/lib/api/work-graph/work-graph.ts

@@ -118,6 +118,21 @@ export class WorkGraph {
     });
   }
 
+  /**
+   * Execute all stack nodes in dependency order with the given concurrency.
+   *
+   * Unlike `doParallel`, this method only handles stack nodes and takes a
+   * simple callback. Intended for destroy where there are no asset nodes.
+   */
+  public processStacks(concurrency: number, fn: (stackNode: StackNode) => Promise<void>) {
+    return this.forAllArtifacts(concurrency, async (x: WorkNode) => {
+      if (x.type !== 'stack') {
+        return;
+      }
+      await fn(x);
+    });
+  }
+
   /**
    * Return the set of unblocked nodes
    */

packages/@aws-cdk/toolkit-lib/lib/toolkit/toolkit.ts

@@ -89,7 +89,7 @@ import { ResourceMigrator } from '../api/resource-import';
 import { tagsForStack } from '../api/tags/private';
 import { DEFAULT_TOOLKIT_STACK_NAME } from '../api/toolkit-info';
 import type { AssetBuildNode, AssetPublishNode, Concurrency, StackNode } from '../api/work-graph';
-import { WorkGraphBuilder } from '../api/work-graph';
+import { WorkGraphBuilder, buildDestroyWorkGraph } from '../api/work-graph';
 import type { AssemblyData, RefactorResult, StackDetails, SuccessfulDeployStackResult } from '../payloads';
 import { PermissionChangeType } from '../payloads';
 import { formatErrorMessage, formatTime, obscureTemplate, serializeStructure, validateSnsTopicArn } from '../util';
@@ -1247,8 +1247,7 @@ export class Toolkit extends CloudAssemblySourceBuilder {
   private async _destroy(assembly: StackAssembly, action: 'deploy' | 'destroy', options: DestroyOptions): Promise<DestroyResult> {
     const selectStacks = stacksOpt(options);
     const ioHelper = asIoHelper(this.ioHost, action);
-    // The stacks will have been ordered for deployment, so reverse them for deletion.
-    const stacks = (await assembly.selectStacksV2(selectStacks)).reversed();
+    const stacks = await assembly.selectStacksV2(selectStacks);
 
     const ret: DestroyResult = {
       stacks: [],
@@ -1262,16 +1261,21 @@ export class Toolkit extends CloudAssemblySourceBuilder {
       return ret;
     }
 
+    const concurrency = options.concurrency || 1;
+    let destroyCount = 0;
+
     const destroySpan = await ioHelper.span(SPAN.DESTROY_ACTION).begin({
       stacks: stacks.stackArtifacts,
     });
     try {
-      for (const [index, stack] of stacks.stackArtifacts.entries()) {
+      const destroyStack = async (stackNode: StackNode) => {
+        const stack = stackNode.stack;
+        destroyCount++;
         try {
           const singleDestroySpan = await ioHelper.span(SPAN.DESTROY_STACK)
-            .begin(chalk.green(`${chalk.blue(stack.displayName)}: destroying... [${index + 1}/${stacks.stackCount}]`), {
+            .begin(chalk.green(`${chalk.blue(stack.displayName)}: destroying... [${destroyCount}/${stacks.stackCount}]`), {
               total: stacks.stackCount,
-              current: index + 1,
+              current: destroyCount,
               stack,
             });
           const deployments = await this.deploymentsForAction(action);
@@ -1297,7 +1301,10 @@ export class Toolkit extends CloudAssemblySourceBuilder {
           await ioHelper.notify(IO.CDK_TOOLKIT_E7900.msg(`\n ❌  ${chalk.blue(stack.displayName)}: ${action} failed ${e}`, { error: e }));
           throw e;
         }
-      }
+      };
+
+      const workGraph = buildDestroyWorkGraph(stacks.stackArtifacts, ioHelper);
+      await workGraph.processStacks(concurrency, destroyStack);
 
       return ret;
     } finally {
@@ -1439,3 +1446,4 @@ async function synthAndMeasure(
 function zeroTime(): ElapsedTime {
   return { asMs: 0, asSec: 0 };
 }
+

packages/@aws-cdk/toolkit-lib/test/actions/destroy.test.ts

@@ -82,6 +82,18 @@ describe('destroy', () => {
     }));
   });
 
+  test('destroy with concurrency', async () => {
+    // WHEN
+    const cx = await builderFixture(toolkit, 'two-empty-stacks');
+    await toolkit.destroy(cx, {
+      stacks: { strategy: StackSelectionStrategy.ALL_STACKS },
+      concurrency: 3,
+    });
+
+    // THEN
+    expect(mockDestroyStack).toHaveBeenCalledTimes(2);
+  });
+
   test('action disposes of assembly produced by source', async () => {
     // GIVEN
     const [assemblySource, mockDispose, realDispose] = await disposableCloudAssemblySource(toolkit);

packages/aws-cdk/lib/cli/cdk-toolkit.ts

@@ -16,6 +16,7 @@ import type { ActionLessRequest, IoHelper } from '../../lib/api-private';
 import { asIoHelper, cfnApi, createIgnoreMatcher, IO, tagsForStack } from '../../lib/api-private';
 import type { AssetBuildNode, AssetPublishNode, Concurrency, StackNode, WorkGraph } from '../api';
 import {
+  buildDestroyWorkGraph,
   CloudWatchLogEventMonitor,
   DEFAULT_TOOLKIT_STACK_NAME,
   DiffFormatter,
@@ -1000,8 +1001,7 @@ export class CdkToolkit {
   public async destroy(options: DestroyOptions) {
     const ioHelper = this.ioHost.asIoHelper();
 
-    // The stacks will have been ordered for deployment, so reverse them for deletion.
-    const stacks = (await this.selectStacksForDestroy(options.selector, options.exclusively)).reversed();
+    const stacks = await this.selectStacksForDestroy(options.selector, options.exclusively);
 
     if (!options.force) {
       const motivation = 'Destroying stacks is an irreversible action';
@@ -1017,9 +1017,18 @@ export class CdkToolkit {
       }
     }
 
+    const concurrency = options.concurrency || 1;
     const action = options.fromDeploy ? 'deploy' : 'destroy';
-    for (const [index, stack] of stacks.stackArtifacts.entries()) {
-      await ioHelper.defaults.info(chalk.green('%s: destroying... [%s/%s]'), chalk.blue(stack.displayName), index + 1, stacks.stackCount);
+    let destroyCount = 0;
+
+    if (concurrency > 1) {
+      this.ioHost.stackProgress = StackActivityProgress.EVENTS;
+    }
+
+    const destroyStack = async (stackNode: StackNode) => {
+      const stack = stackNode.stack;
+      destroyCount++;
+      await ioHelper.defaults.info(chalk.green('%s: destroying... [%s/%s]'), chalk.blue(stack.displayName), destroyCount, stacks.stackCount);
       try {
         await this.props.deployments.destroyStack({
           stack,
@@ -1031,7 +1040,10 @@ export class CdkToolkit {
         await ioHelper.defaults.error(`\n ❌  %s: ${action} failed`, chalk.blue(stack.displayName), e);
         throw e;
       }
-    }
+    };
+
+    const workGraph = buildDestroyWorkGraph(stacks.stackArtifacts, ioHelper);
+    await workGraph.processStacks(concurrency, destroyStack);
   }
 
   public async list(
@@ -1914,6 +1926,11 @@ export interface DestroyOptions {
    * Whether the destroy request came from a deploy.
    */
   fromDeploy?: boolean;
+
+  /**
+   * Maximum number of simultaneous destroys (dependency permitting) to execute.
+   */
+  concurrency?: number;
 }
 
 /**
@@ -2210,3 +2227,4 @@ function requiresApproval(requireApproval: RequireApproval, permissionChangeType
   return requireApproval === RequireApproval.ANYCHANGE ||
     requireApproval === RequireApproval.BROADENING && permissionChangeType === PermissionChangeType.BROADENING;
 }
+

packages/aws-cdk/lib/cli/cli-config.ts

@@ -344,6 +344,7 @@ export async function makeConfig(): Promise<CliConfig> {
           all: { type: 'boolean', default: false, desc: 'Destroy all available stacks' },
           exclusively: { type: 'boolean', alias: 'e', desc: 'Only destroy requested stacks, don\'t include dependees' },
           force: { type: 'boolean', alias: 'f', desc: 'Do not ask for confirmation before destroying the stacks' },
+          concurrency: { type: 'number', desc: 'Maximum number of simultaneous destroys (dependency permitting) to execute.', default: 1, requiresArg: true },
         },
       },
       'diff': {

packages/aws-cdk/lib/cli/cli-type-registry.json

@@ -755,6 +755,12 @@
           "type": "boolean",
           "alias": "f",
           "desc": "Do not ask for confirmation before destroying the stacks"
+        },
+        "concurrency": {
+          "type": "number",
+          "desc": "Maximum number of simultaneous destroys (dependency permitting) to execute.",
+          "default": 1,
+          "requiresArg": true
         }
       }
     },

packages/aws-cdk/lib/cli/cli.ts

@@ -470,6 +470,7 @@ export async function exec(args: string[], synthesizer?: Synthesizer): Promise<n
           exclusively: args.exclusively,
           force: args.force,
           roleArn: args.roleArn,
+          concurrency: args.concurrency,
         });
 
       case 'gc':

packages/aws-cdk/lib/cli/convert-to-user-input.ts

@@ -198,6 +198,7 @@ export function convertYargsToUserInput(args: any): UserInput {
         all: args.all,
         exclusively: args.exclusively,
         force: args.force,
+        concurrency: args.concurrency,
         STACKS: args.STACKS,
       };
       break;
@@ -472,6 +473,7 @@ export function convertConfigToUserInput(config: any): UserInput {
     all: config.destroy?.all,
     exclusively: config.destroy?.exclusively,
     force: config.destroy?.force,
+    concurrency: config.destroy?.concurrency,
   };
   const diffOptions = {
     exclusively: config.diff?.exclusively,

packages/aws-cdk/lib/cli/parse-command-line-arguments.ts

@@ -787,6 +787,12 @@ export function parseCommandLineArguments(args: Array<string>): any {
           type: 'boolean',
           alias: 'f',
           desc: 'Do not ask for confirmation before destroying the stacks',
+        })
+        .option('concurrency', {
+          default: 1,
+          type: 'number',
+          desc: 'Maximum number of simultaneous destroys (dependency permitting) to execute.',
+          requiresArg: true,
         }),
     )
     .command(

packages/aws-cdk/lib/cli/user-input.ts

@@ -1213,6 +1213,13 @@ export interface DestroyOptions {
    */
   readonly force?: boolean;
 
+  /**
+   * Maximum number of simultaneous destroys (dependency permitting) to execute.
+   *
+   * @default - 1
+   */
+  readonly concurrency?: number;
+
   /**
    * Positional argument for destroy
    */