From 60c09f064977e2191c6b5735a1448edbfa059826 Mon Sep 17 00:00:00 2001 From: Peter Amiri Date: Tue, 14 Apr 2026 18:50:30 -0700 Subject: [PATCH 1/2] feat(test): add parallel test execution runner Adds ParallelRunner.cfc that discovers test bundles, partitions them across N workers, fires parallel HTTP requests to the test endpoint, and aggregates JSON results. Includes comprehensive BDD spec. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../specs/internal/parallelRunnerSpec.cfc | 283 +++++++++++++ vendor/wheels/wheelstest/ParallelRunner.cfc | 383 ++++++++++++++++++ 2 files changed, 666 insertions(+) create mode 100644 vendor/wheels/tests/specs/internal/parallelRunnerSpec.cfc create mode 100644 vendor/wheels/wheelstest/ParallelRunner.cfc diff --git a/vendor/wheels/tests/specs/internal/parallelRunnerSpec.cfc b/vendor/wheels/tests/specs/internal/parallelRunnerSpec.cfc new file mode 100644 index 000000000..f3b8fb4b1 --- /dev/null +++ b/vendor/wheels/tests/specs/internal/parallelRunnerSpec.cfc @@ -0,0 +1,283 @@ +component extends="wheels.WheelsTest" { + + function run() { + + describe("ParallelRunner", () => { + + beforeEach(() => { + runner = new wheels.wheelstest.ParallelRunner( + baseUrl = "http://localhost:8080", + workers = 4 + ); + }); + + describe("discoverBundles()", () => { + + it("finds Spec files in the core test directory", () => { + var bundles = runner.discoverBundles(baseDirectory = "wheels.tests.specs"); + expect(bundles).toBeArray(); + expect(arrayLen(bundles)).toBeGT(0); + }); + + it("returns dotted paths ending with the bundle name", () => { + var bundles = runner.discoverBundles(baseDirectory = "wheels.tests.specs"); + // Every path should start with the base directory + for (var b in bundles) { + expect(b).toInclude("wheels.tests.specs."); + } + }); + + it("returns an empty array for a nonexistent directory", () => { + var bundles = runner.discoverBundles(baseDirectory = "wheels.tests.specs.nonexistent_abc123"); + expect(bundles).toBeArray(); + expect(arrayLen(bundles)).toBe(0); + }); + + it("finds bundles in a subdirectory", () => { + var bundles = runner.discoverBundles(baseDirectory = "wheels.tests.specs.model"); + expect(bundles).toBeArray(); + expect(arrayLen(bundles)).toBeGT(0); + for (var b in bundles) { + expect(b).toInclude("wheels.tests.specs.model."); + } + }); + + }); + + describe("partitionBundles()", () => { + + it("distributes bundles evenly via round-robin", () => { + var bundles = ["a", "b", "c", "d", "e", "f"]; + var partitions = runner.partitionBundles(bundles = bundles, partitionCount = 3); + + expect(partitions).toBeArray(); + expect(arrayLen(partitions)).toBe(3); + // Round-robin: [a,d], [b,e], [c,f] + expect(partitions[1]).toBe(["a", "d"]); + expect(partitions[2]).toBe(["b", "e"]); + expect(partitions[3]).toBe(["c", "f"]); + }); + + it("handles fewer bundles than workers", () => { + var bundles = ["a", "b"]; + var partitions = runner.partitionBundles(bundles = bundles, partitionCount = 5); + + // Should cap at 2 partitions since we only have 2 bundles + expect(arrayLen(partitions)).toBe(2); + expect(partitions[1]).toBe(["a"]); + expect(partitions[2]).toBe(["b"]); + }); + + it("handles a single bundle", () => { + var bundles = ["only"]; + var partitions = runner.partitionBundles(bundles = bundles, partitionCount = 4); + + expect(arrayLen(partitions)).toBe(1); + expect(partitions[1]).toBe(["only"]); + }); + + it("handles an empty bundle list", () => { + var bundles = []; + var partitions = runner.partitionBundles(bundles = bundles, partitionCount = 4); + + expect(arrayLen(partitions)).toBe(0); + }); + + it("preserves all bundles across partitions", () => { + var bundles = ["a", "b", "c", "d", "e", "f", "g"]; + var partitions = runner.partitionBundles(bundles = bundles, partitionCount = 3); + + var allBundles = []; + for (var p in partitions) { + for (var b in p) { + arrayAppend(allBundles, b); + } + } + arraySort(allBundles, "textNoCase"); + arraySort(bundles, "textNoCase"); + expect(allBundles).toBe(bundles); + }); + + }); + + describe("aggregateResults()", () => { + + it("sums totals correctly from multiple partitions", () => { + var partitionResults = [ + { + success = true, + data = {totalPass = 10, totalFail = 1, totalError = 0, totalSkipped = 2, totalDuration = 1000, bundleStats = []}, + error = "", duration = 1000, partition = 1, status = "COMPLETED" + }, + { + success = true, + data = {totalPass = 20, totalFail = 0, totalError = 1, totalSkipped = 3, totalDuration = 2000, bundleStats = []}, + error = "", duration = 2000, partition = 2, status = "COMPLETED" + } + ]; + + var result = runner.aggregateResults(partitionResults = partitionResults); + + expect(result.totalPass).toBe(30); + expect(result.totalFail).toBe(1); + expect(result.totalError).toBe(1); + expect(result.totalSkipped).toBe(5); + expect(result.totalDuration).toBe(3000); + }); + + it("merges bundleStats arrays", () => { + var partitionResults = [ + { + success = true, + data = { + totalPass = 5, totalFail = 0, totalError = 0, totalSkipped = 0, + totalDuration = 500, + bundleStats = [{name = "BundleA", totalPass = 5, totalFail = 0, suiteStats = []}] + }, + error = "", duration = 500, partition = 1, status = "COMPLETED" + }, + { + success = true, + data = { + totalPass = 3, totalFail = 0, totalError = 0, totalSkipped = 0, + totalDuration = 300, + bundleStats = [{name = "BundleB", totalPass = 3, totalFail = 0, suiteStats = []}] + }, + error = "", duration = 300, partition = 2, status = "COMPLETED" + } + ]; + + var result = runner.aggregateResults(partitionResults = partitionResults); + + expect(arrayLen(result.bundleStats)).toBe(2); + expect(result.bundleStats[1].name).toBe("BundleA"); + expect(result.bundleStats[2].name).toBe("BundleB"); + }); + + it("collects failures from spec results", () => { + var partitionResults = [ + { + success = true, + data = { + totalPass = 5, totalFail = 1, totalError = 0, totalSkipped = 0, + totalDuration = 500, + bundleStats = [{ + name = "FailBundle", + totalPass = 5, totalFail = 1, + suiteStats = [{ + name = "Suite1", + specStats = [ + {name = "passing test", status = "Passed", failMessage = ""}, + {name = "failing test", status = "Failed", failMessage = "expected true but got false"} + ], + suiteStats = [] + }] + }] + }, + error = "", duration = 500, partition = 1, status = "COMPLETED" + } + ]; + + var result = runner.aggregateResults(partitionResults = partitionResults); + + expect(arrayLen(result.failures)).toBe(1); + expect(result.failures[1].bundle).toBe("FailBundle"); + expect(result.failures[1].spec).toBe("failing test"); + expect(result.failures[1].status).toBe("Failed"); + expect(result.failures[1].message).toInclude("expected true"); + }); + + it("records partition-level errors", () => { + var partitionResults = [ + { + success = true, + data = {totalPass = 5, totalFail = 0, totalError = 0, totalSkipped = 0, totalDuration = 500, bundleStats = []}, + error = "", duration = 500, partition = 1, status = "COMPLETED" + }, + { + success = false, + data = {}, + error = "HTTP 500: Internal Server Error", duration = 100, partition = 2, status = "COMPLETED" + } + ]; + + var result = runner.aggregateResults(partitionResults = partitionResults); + + expect(arrayLen(result.partitionErrors)).toBe(1); + expect(result.partitionErrors[1].partition).toBe(2); + expect(result.partitionErrors[1].error).toInclude("500"); + // Partition errors count toward totalError + expect(result.totalError).toBe(1); + }); + + it("handles all partitions failing", () => { + var partitionResults = [ + {success = false, data = {}, error = "timeout", duration = 600000, partition = 1, status = "COMPLETED"}, + {success = false, data = {}, error = "timeout", duration = 600000, partition = 2, status = "COMPLETED"} + ]; + + var result = runner.aggregateResults(partitionResults = partitionResults); + + expect(result.totalPass).toBe(0); + expect(result.totalError).toBe(2); + expect(arrayLen(result.partitionErrors)).toBe(2); + }); + + }); + + describe("formatReport()", () => { + + it("produces readable text output for passing results", () => { + var results = { + totalPass = 100, totalFail = 0, totalError = 0, totalSkipped = 5, + totalDuration = 5000, bundleStats = [], failures = [], + partitionErrors = [], workers = 4, partitions = 4, wallTime = 2000 + }; + + var report = runner.formatReport(results = results, format = "text"); + + expect(report).toInclude("Parallel Test Results"); + expect(report).toInclude("Workers: 4"); + expect(report).toInclude("Passed: 100"); + expect(report).toInclude("RESULT: PASSED"); + }); + + it("includes failure details in text output", () => { + var results = { + totalPass = 90, totalFail = 2, totalError = 0, totalSkipped = 0, + totalDuration = 5000, bundleStats = [], + failures = [ + {bundle = "MyBundle", spec = "my test", status = "Failed", message = "assertion failed"} + ], + partitionErrors = [], workers = 4, partitions = 4, wallTime = 2000 + }; + + var report = runner.formatReport(results = results, format = "text"); + + expect(report).toInclude("Failures"); + expect(report).toInclude("MyBundle"); + expect(report).toInclude("assertion failed"); + expect(report).toInclude("RESULT: FAILED"); + }); + + it("produces valid JSON output", () => { + var results = { + totalPass = 50, totalFail = 0, totalError = 0, totalSkipped = 0, + totalDuration = 2000, bundleStats = [], failures = [], + partitionErrors = [], workers = 2, partitions = 2, wallTime = 1200 + }; + + var report = runner.formatReport(results = results, format = "json"); + var parsed = deserializeJSON(report); + + expect(parsed.totalPass).toBe(50); + expect(parsed.workers).toBe(2); + }); + + }); + + }); + + } + +} diff --git a/vendor/wheels/wheelstest/ParallelRunner.cfc b/vendor/wheels/wheelstest/ParallelRunner.cfc new file mode 100644 index 000000000..d1383eccf --- /dev/null +++ b/vendor/wheels/wheelstest/ParallelRunner.cfc @@ -0,0 +1,383 @@ +/** + * Parallel Test Runner for Wheels + * + * Partitions test bundles and runs them concurrently via multiple HTTP requests, + * then aggregates the results. Uses cfthread for parallelism and the existing + * TestBox JSON reporter endpoint for each partition. + * + * Usage: + * var runner = new wheels.wheelstest.ParallelRunner(baseUrl="http://localhost:8080", workers=4); + * var results = runner.run(type="core", db="sqlite"); + */ +component { + + variables.baseUrl = ""; + variables.workers = 4; + variables.timeoutMs = 600000; + + /** + * Initialize the parallel runner. + * + * @baseUrl Base URL of the running Wheels application (e.g. "http://localhost:8080") + * @workers Number of parallel workers (threads) to use + */ + public ParallelRunner function init(string baseUrl = "http://localhost:8080", numeric workers = 4) { + variables.baseUrl = arguments.baseUrl; + variables.workers = arguments.workers; + return this; + } + + /** + * Main entry point. Discovers bundles, partitions them, executes in parallel, and aggregates results. + * + * @type "core" or "app" — determines which test suite to run + * @directory Optional subdirectory filter (dotted path, e.g. "wheels.tests.specs.model") + * @db Database to test against (default "sqlite") + * @workers Override the default worker count for this run + */ + public struct function run( + required string type, + string directory = "", + string db = "sqlite", + numeric workers = variables.workers + ) { + var startTick = getTickCount(); + + // Determine base spec directory + var baseDir = ""; + if (len(arguments.directory)) { + baseDir = arguments.directory; + } else if (arguments.type == "core") { + baseDir = "wheels.tests.specs"; + } else { + baseDir = "tests.specs"; + } + + // Discover all bundle paths + var bundles = discoverBundles(baseDirectory = baseDir); + if (arrayLen(bundles) == 0) { + return { + totalPass = 0, totalFail = 0, totalError = 0, totalSkipped = 0, + totalDuration = 0, bundleStats = [], failures = [], + workers = arguments.workers, partitions = 0, + wallTime = getTickCount() - startTick, + message = "No test bundles found in #baseDir#" + }; + } + + // Partition bundles across workers + var effectiveWorkers = min(arguments.workers, arrayLen(bundles)); + var partitions = partitionBundles(bundles = bundles, partitionCount = effectiveWorkers); + + // Execute partitions in parallel + var partitionResults = executePartitions( + partitions = partitions, + type = arguments.type, + db = arguments.db + ); + + // Aggregate results + var aggregated = aggregateResults(partitionResults = partitionResults); + aggregated.workers = effectiveWorkers; + aggregated.partitions = arrayLen(partitions); + aggregated.wallTime = getTickCount() - startTick; + aggregated.totalBundles = arrayLen(bundles); + + return aggregated; + } + + /** + * Discover test bundle paths by scanning the filesystem for *Spec.cfc files. + * + * @baseDirectory Dotted path to the base directory (e.g. "wheels.tests.specs") + * @return Array of dotted bundle paths (e.g. ["wheels.tests.specs.model.CreateSpec", ...]) + */ + public array function discoverBundles(required string baseDirectory) { + var results = []; + var fsPath = expandPath("/" & replace(arguments.baseDirectory, ".", "/", "all")); + + if (!directoryExists(fsPath)) { + return results; + } + + var files = directoryList(fsPath, true, "path", "*.cfc", "name asc", "file"); + + var baseFsPath = replace(fsPath, "\", "/", "all"); + if (right(baseFsPath, 1) == "/") { + baseFsPath = left(baseFsPath, len(baseFsPath) - 1); + } + + for (var filePath in files) { + var fileName = listLast(replace(filePath, "\", "/", "all"), "/"); + if (reFindNoCase("(Spec|Test)\.cfc$", fileName)) { + // Convert filesystem path to dotted bundle path + var normalized = replace(filePath, "\", "/", "all"); + normalized = reReplaceNoCase(normalized, "\.cfc$", ""); + normalized = replace(normalized, baseFsPath, ""); + var dottedRelative = replace(normalized, "/", ".", "all"); + if (left(dottedRelative, 1) == ".") { + dottedRelative = right(dottedRelative, len(dottedRelative) - 1); + } + arrayAppend(results, arguments.baseDirectory & "." & dottedRelative); + } + } + + return results; + } + + /** + * Partition an array of bundles into N groups using round-robin distribution. + * + * @bundles Array of bundle paths + * @partitionCount Number of partitions to create + * @return Array of arrays, each containing a subset of bundle paths + */ + public array function partitionBundles(required array bundles, required numeric partitionCount) { + var effectiveCount = min(arguments.partitionCount, arrayLen(arguments.bundles)); + if (effectiveCount <= 0) { + return []; + } + + var partitions = []; + for (var i = 1; i <= effectiveCount; i++) { + arrayAppend(partitions, []); + } + + for (var i = 1; i <= arrayLen(arguments.bundles); i++) { + var partitionIndex = ((i - 1) mod effectiveCount) + 1; + arrayAppend(partitions[partitionIndex], arguments.bundles[i]); + } + + return partitions; + } + + /** + * Execute test partitions in parallel using cfthread. + * Each thread fires an HTTP request to the test runner with its assigned bundles. + * + * @partitions Array of arrays, each containing bundle paths for one worker + * @type "core" or "app" + * @db Database name + * @return Array of structs, each containing {success, data, error, duration, partition} + */ + public array function executePartitions( + required array partitions, + required string type, + required string db + ) { + var threadNames = []; + var runId = replace(createUUID(), "-", "", "all"); + + for (var i = 1; i <= arrayLen(arguments.partitions); i++) { + var threadName = "parallelTest_#runId#_#i#"; + arrayAppend(threadNames, threadName); + + var bundleList = arrayToList(arguments.partitions[i]); + + thread + name="#threadName#" + action="run" + baseUrl="#variables.baseUrl#" + testType="#arguments.type#" + testDb="#arguments.db#" + bundleList="#bundleList#" + partitionIndex="#i#" + { + var partitionStart = getTickCount(); + try { + var testPath = (attributes.testType == "app") ? "/wheels/app/tests" : "/wheels/core/tests"; + var testUrl = attributes.baseUrl + & testPath & "?db=" & attributes.testDb + & "&format=json&cli=true" + & "&testBundles=" & urlEncodedFormat(attributes.bundleList); + + cfhttp( + url = testUrl, + method = "GET", + timeout = 600, + result = "local.httpResult" + ); + + if (listFirst(local.httpResult.statusCode, " ") == "200" || listFirst(local.httpResult.statusCode, " ") == "417") { + thread.success = true; + thread.data = deserializeJSON(local.httpResult.fileContent); + thread.error = ""; + } else { + thread.success = false; + thread.data = {}; + thread.error = "HTTP #local.httpResult.statusCode#: #left(local.httpResult.fileContent, 500)#"; + } + } catch (any e) { + thread.success = false; + thread.data = {}; + thread.error = "Thread error: #e.message# #e.detail#"; + } + thread.duration = getTickCount() - partitionStart; + thread.partition = attributes.partitionIndex; + } + } + + // Join all threads — wait up to 10 minutes + var nameList = arrayToList(threadNames); + thread action="join" name="#nameList#" timeout="#variables.timeoutMs#"; + + // Collect results + var results = []; + for (var tName in threadNames) { + var t = cfthread[tName]; + arrayAppend(results, { + success = structKeyExists(t, "success") ? t.success : false, + data = structKeyExists(t, "data") ? t.data : {}, + error = structKeyExists(t, "error") ? t.error : "Thread did not complete", + duration = structKeyExists(t, "duration") ? t.duration : 0, + partition = structKeyExists(t, "partition") ? t.partition : 0, + status = structKeyExists(t, "status") ? t.status : "UNKNOWN" + }); + } + + return results; + } + + /** + * Aggregate results from multiple partition runs into a single result struct. + * + * @partitionResults Array of partition result structs from executePartitions() + * @return Aggregated result struct + */ + public struct function aggregateResults(required array partitionResults) { + var aggregated = { + totalPass = 0, + totalFail = 0, + totalError = 0, + totalSkipped = 0, + totalDuration = 0, + bundleStats = [], + failures = [], + partitionErrors = [] + }; + + for (var pr in arguments.partitionResults) { + if (pr.success && isStruct(pr.data) && !structIsEmpty(pr.data)) { + var d = pr.data; + aggregated.totalPass += val(structKeyExists(d, "totalPass") ? d.totalPass : 0); + aggregated.totalFail += val(structKeyExists(d, "totalFail") ? d.totalFail : 0); + aggregated.totalError += val(structKeyExists(d, "totalError") ? d.totalError : 0); + aggregated.totalSkipped += val(structKeyExists(d, "totalSkipped") ? d.totalSkipped : 0); + aggregated.totalDuration += val(structKeyExists(d, "totalDuration") ? d.totalDuration : 0); + + // Merge bundle stats + if (structKeyExists(d, "bundleStats") && isArray(d.bundleStats)) { + for (var bs in d.bundleStats) { + arrayAppend(aggregated.bundleStats, bs); + + // Collect failures from bundle suiteStats + if (structKeyExists(bs, "suiteStats") && isArray(bs.suiteStats)) { + $collectFailures( + suiteStats = bs.suiteStats, + bundleName = structKeyExists(bs, "name") ? bs.name : "unknown", + failures = aggregated.failures + ); + } + } + } + } else { + // Partition-level error + arrayAppend(aggregated.partitionErrors, { + partition = pr.partition, + error = pr.error, + duration = pr.duration + }); + // Count partition errors as test errors so they surface in totals + aggregated.totalError++; + } + } + + return aggregated; + } + + /** + * Format aggregated results as a human-readable report or JSON. + * + * @results Aggregated result struct from aggregateResults() + * @format "text" for human-readable, "json" for JSON + * @return Formatted report string + */ + public string function formatReport(required struct results, string format = "text") { + if (arguments.format == "json") { + return serializeJSON(arguments.results); + } + + var r = arguments.results; + var out = []; + + arrayAppend(out, "=== Parallel Test Results ==="); + arrayAppend(out, "Workers: #val(structKeyExists(r, 'workers') ? r.workers : 0)# | Partitions: #val(structKeyExists(r, 'partitions') ? r.partitions : 0)#"); + arrayAppend(out, "Wall time: #val(structKeyExists(r, 'wallTime') ? r.wallTime : 0)#ms | Sum of partition durations: #r.totalDuration#ms"); + arrayAppend(out, ""); + + var total = r.totalPass + r.totalFail + r.totalError + r.totalSkipped; + arrayAppend(out, "Total: #total# | Passed: #r.totalPass# | Failed: #r.totalFail# | Errors: #r.totalError# | Skipped: #r.totalSkipped#"); + + if (r.totalFail > 0 || r.totalError > 0) { + arrayAppend(out, ""); + arrayAppend(out, "--- Failures ---"); + for (var f in r.failures) { + arrayAppend(out, " [#f.status#] #f.bundle# > #f.spec#"); + if (len(f.message)) { + arrayAppend(out, " #left(f.message, 200)#"); + } + } + } + + if (structKeyExists(r, "partitionErrors") && arrayLen(r.partitionErrors) > 0) { + arrayAppend(out, ""); + arrayAppend(out, "--- Partition Errors ---"); + for (var pe in r.partitionErrors) { + arrayAppend(out, " Partition #pe.partition#: #pe.error#"); + } + } + + arrayAppend(out, ""); + if (r.totalFail == 0 && r.totalError == 0 && (!structKeyExists(r, "partitionErrors") || arrayLen(r.partitionErrors) == 0)) { + arrayAppend(out, "RESULT: PASSED"); + } else { + arrayAppend(out, "RESULT: FAILED"); + } + + return arrayToList(out, chr(10)); + } + + /** + * Recursively collect failure/error specs from suiteStats. + */ + public void function $collectFailures( + required array suiteStats, + required string bundleName, + required array failures + ) { + for (var suite in arguments.suiteStats) { + if (structKeyExists(suite, "specStats") && isArray(suite.specStats)) { + for (var spec in suite.specStats) { + var status = structKeyExists(spec, "status") ? spec.status : ""; + if (status == "Failed" || status == "Error") { + arrayAppend(arguments.failures, { + bundle = arguments.bundleName, + spec = structKeyExists(spec, "name") ? spec.name : "unknown", + status = status, + message = structKeyExists(spec, "failMessage") ? spec.failMessage : "" + }); + } + } + } + // Recurse into nested suites + if (structKeyExists(suite, "suiteStats") && isArray(suite.suiteStats)) { + $collectFailures( + suiteStats = suite.suiteStats, + bundleName = arguments.bundleName, + failures = arguments.failures + ); + } + } + } + +} From 9b9da6162bd2dd75350eb034f79122123623f96a Mon Sep 17 00:00:00 2001 From: Peter Amiri Date: Tue, 14 Apr 2026 22:25:15 -0700 Subject: [PATCH 2/2] fix(test): make $collectFailures private in ParallelRunner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Standalone CFC, not a mixin — private is the correct access modifier. Co-Authored-By: Claude Opus 4.6 (1M context) --- vendor/wheels/wheelstest/ParallelRunner.cfc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/wheels/wheelstest/ParallelRunner.cfc b/vendor/wheels/wheelstest/ParallelRunner.cfc index d1383eccf..bfccbd4d4 100644 --- a/vendor/wheels/wheelstest/ParallelRunner.cfc +++ b/vendor/wheels/wheelstest/ParallelRunner.cfc @@ -350,7 +350,7 @@ component { /** * Recursively collect failure/error specs from suiteStats. */ - public void function $collectFailures( + private void function $collectFailures( required array suiteStats, required string bundleName, required array failures