From a900a5f13be91774ac32cbd6d9a2672da68e4e28 Mon Sep 17 00:00:00 2001 From: Guillaume Nodet Date: Fri, 27 Mar 2026 19:55:07 +0100 Subject: [PATCH 1/9] CAMEL-23264: Enhance Splitter EIP with chunking, error threshold, and failure tracking - Add group(int) option to chunk split items into List batches using GroupIterator - Add errorThreshold(double) to abort when failure ratio exceeds threshold - Add maxFailedRecords(int) to abort after N failures - Add SplitResult exchange property with structured failure details - Extract shouldContinueOnFailure() in MulticastProcessor for subclass override - Mutually exclusive validation: stopOnException vs errorThreshold/maxFailedRecords Co-Authored-By: Claude Opus 4.6 --- .../camel/ExchangeConstantProvider.java | 3 +- .../main/java/org/apache/camel/Exchange.java | 4 + .../org/apache/camel/ExchangePropertyKey.java | 3 + .../java/org/apache/camel/SplitResult.java | 93 ++++++++++++ .../org/apache/camel/model/split.json | 5 +- .../apache/camel/model/SplitDefinition.java | 129 ++++++++++++++++ .../camel/processor/MulticastProcessor.java | 20 ++- .../org/apache/camel/processor/Splitter.java | 143 +++++++++++++++++- .../apache/camel/reifier/SplitReifier.java | 21 +++ .../processor/SplitterErrorThresholdTest.java | 110 ++++++++++++++ .../camel/processor/SplitterGroupTest.java | 109 +++++++++++++ .../SplitterMaxFailedRecordsTest.java | 112 ++++++++++++++ .../processor/SplitterSplitResultTest.java | 124 +++++++++++++++ .../deserializers/ModelDeserializers.java | 18 +++ 14 files changed, 886 insertions(+), 8 deletions(-) create mode 100644 core/camel-api/src/main/java/org/apache/camel/SplitResult.java create mode 100644 core/camel-core/src/test/java/org/apache/camel/processor/SplitterErrorThresholdTest.java create mode 100644 core/camel-core/src/test/java/org/apache/camel/processor/SplitterGroupTest.java create mode 100644 core/camel-core/src/test/java/org/apache/camel/processor/SplitterMaxFailedRecordsTest.java create mode 100644 core/camel-core/src/test/java/org/apache/camel/processor/SplitterSplitResultTest.java diff --git a/core/camel-api/src/generated/java/org/apache/camel/ExchangeConstantProvider.java b/core/camel-api/src/generated/java/org/apache/camel/ExchangeConstantProvider.java index da304199b08b8..2e181224ce8c9 100644 --- a/core/camel-api/src/generated/java/org/apache/camel/ExchangeConstantProvider.java +++ b/core/camel-api/src/generated/java/org/apache/camel/ExchangeConstantProvider.java @@ -13,7 +13,7 @@ public class ExchangeConstantProvider { private static final Map MAP; static { - Map map = new HashMap<>(149); + Map map = new HashMap<>(150); map.put("AGGREGATED_COLLECTION_GUARD", "CamelAggregatedCollectionGuard"); map.put("AGGREGATED_COMPLETED_BY", "CamelAggregatedCompletedBy"); map.put("AGGREGATED_CORRELATION_KEY", "CamelAggregatedCorrelationKey"); @@ -142,6 +142,7 @@ public class ExchangeConstantProvider { map.put("SLIP_PRODUCER", "CamelSlipProducer"); map.put("SPLIT_COMPLETE", "CamelSplitComplete"); map.put("SPLIT_INDEX", "CamelSplitIndex"); + map.put("SPLIT_RESULT", "CamelSplitResult"); map.put("SPLIT_SIZE", "CamelSplitSize"); map.put("STEP_ID", "CamelStepId"); map.put("STREAM_CACHE_UNIT_OF_WORK", "CamelStreamCacheUnitOfWork"); diff --git a/core/camel-api/src/main/java/org/apache/camel/Exchange.java b/core/camel-api/src/main/java/org/apache/camel/Exchange.java index 55ec9532b5823..de55e3fca53f1 100644 --- a/core/camel-api/src/main/java/org/apache/camel/Exchange.java +++ b/core/camel-api/src/main/java/org/apache/camel/Exchange.java @@ -282,6 +282,10 @@ public interface Exchange extends VariableAware { javaType = "int", important = true) String SPLIT_SIZE = "CamelSplitSize"; + @Metadata(label = "split", + description = "The result of a Splitter EIP operation with error thresholds, providing structured failure details.", + javaType = "org.apache.camel.SplitResult") + String SPLIT_RESULT = "CamelSplitResult"; @Metadata(label = "step", description = "The id of the Step EIP", javaType = "String") String STEP_ID = "CamelStepId"; diff --git a/core/camel-api/src/main/java/org/apache/camel/ExchangePropertyKey.java b/core/camel-api/src/main/java/org/apache/camel/ExchangePropertyKey.java index 491a48f3a2d06..a5c2bd39fb294 100644 --- a/core/camel-api/src/main/java/org/apache/camel/ExchangePropertyKey.java +++ b/core/camel-api/src/main/java/org/apache/camel/ExchangePropertyKey.java @@ -78,6 +78,7 @@ public enum ExchangePropertyKey { SLIP_PRODUCER(Exchange.SLIP_PRODUCER), SPLIT_COMPLETE(Exchange.SPLIT_COMPLETE), SPLIT_INDEX(Exchange.SPLIT_INDEX), + SPLIT_RESULT(Exchange.SPLIT_RESULT), SPLIT_SIZE(Exchange.SPLIT_SIZE), STEP_ID(Exchange.STEP_ID), STREAM_CACHE_UNIT_OF_WORK(Exchange.STREAM_CACHE_UNIT_OF_WORK), @@ -208,6 +209,8 @@ public static ExchangePropertyKey asExchangePropertyKey(String name) { return SPLIT_COMPLETE; case Exchange.SPLIT_INDEX: return SPLIT_INDEX; + case Exchange.SPLIT_RESULT: + return SPLIT_RESULT; case Exchange.SPLIT_SIZE: return SPLIT_SIZE; case Exchange.STEP_ID: diff --git a/core/camel-api/src/main/java/org/apache/camel/SplitResult.java b/core/camel-api/src/main/java/org/apache/camel/SplitResult.java new file mode 100644 index 0000000000000..a859e7d8b8461 --- /dev/null +++ b/core/camel-api/src/main/java/org/apache/camel/SplitResult.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel; + +import java.util.Collections; +import java.util.List; + +/** + * Result of a Splitter EIP operation that provides structured information about the outcome, including failure details + * when error thresholds ({@code errorThreshold} or {@code maxFailedRecords}) are configured. + *

+ * This result is available as an exchange property ({@link Exchange#SPLIT_RESULT}) after the split operation completes. + */ +public final class SplitResult { + + /** + * Details of a single split item failure. + * + * @param index the 0-based index of the failed item in the split sequence + * @param exception the exception that caused the failure + */ + public record Failure(int index, Exception exception) { + } + + private final int totalItems; + private final int failureCount; + private final List failures; + private final boolean aborted; + + public SplitResult(int totalItems, int failureCount, List failures, boolean aborted) { + this.totalItems = totalItems; + this.failureCount = failureCount; + this.failures = failures != null ? Collections.unmodifiableList(failures) : Collections.emptyList(); + this.aborted = aborted; + } + + /** + * The total number of items that were prepared for splitting. + */ + public int getTotalItems() { + return totalItems; + } + + /** + * The number of items that completed successfully. + */ + public int getSuccessCount() { + return totalItems - failureCount; + } + + /** + * The number of items that failed during processing. + */ + public int getFailureCount() { + return failureCount; + } + + /** + * The list of individual failures with their index and exception details. + */ + public List getFailures() { + return failures; + } + + /** + * Whether the split operation was aborted early because an error threshold was exceeded. + */ + public boolean isAborted() { + return aborted; + } + + @Override + public String toString() { + return "SplitResult[total=" + totalItems + + ", success=" + getSuccessCount() + + ", failures=" + failureCount + + ", aborted=" + aborted + "]"; + } +} diff --git a/core/camel-core-model/src/generated/resources/META-INF/org/apache/camel/model/split.json b/core/camel-core-model/src/generated/resources/META-INF/org/apache/camel/model/split.json index 8c51d8a7afe80..c8ffe70dfc724 100644 --- a/core/camel-core-model/src/generated/resources/META-INF/org/apache/camel/model/split.json +++ b/core/camel-core-model/src/generated/resources/META-INF/org/apache/camel/model/split.json @@ -30,7 +30,10 @@ "executorService": { "index": 15, "kind": "attribute", "displayName": "Executor Service", "group": "advanced", "label": "advanced", "required": false, "type": "object", "javaType": "java.util.concurrent.ExecutorService", "deprecated": false, "autowired": false, "secret": false, "description": "To use a custom Thread Pool to be used for parallel processing. Notice if you set this option, then parallel processing is automatically implied, and you do not have to enable that option as well." }, "onPrepare": { "index": 16, "kind": "attribute", "displayName": "On Prepare", "group": "advanced", "label": "advanced", "required": false, "type": "object", "javaType": "org.apache.camel.Processor", "deprecated": false, "autowired": false, "secret": false, "description": "Uses the Processor when preparing the org.apache.camel.Exchange to be sent. This can be used to deep-clone messages that should be sent, or any custom logic needed before the exchange is sent." }, "shareUnitOfWork": { "index": 17, "kind": "attribute", "displayName": "Share Unit Of Work", "group": "advanced", "label": "advanced", "required": false, "type": "boolean", "javaType": "java.lang.Boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "description": "Shares the org.apache.camel.spi.UnitOfWork with the parent and each of the sub messages. Splitter will by default not share unit of work between the parent exchange and each split exchange. This means each split exchange has its own individual unit of work." }, - "outputs": { "index": 18, "kind": "element", "displayName": "Outputs", "group": "common", "required": true, "type": "array", "javaType": "java.util.List>", "oneOf": [ "aggregate", "bean", "choice", "circuitBreaker", "claimCheck", "convertBodyTo", "convertHeaderTo", "convertVariableTo", "delay", "doCatch", "doFinally", "doTry", "dynamicRouter", "enrich", "filter", "idempotentConsumer", "intercept", "interceptFrom", "interceptSendToEndpoint", "kamelet", "loadBalance", "log", "loop", "marshal", "multicast", "onCompletion", "onException", "pausable", "pipeline", "policy", "poll", "pollEnrich", "process", "recipientList", "removeHeader", "removeHeaders", "removeProperties", "removeProperty", "removeVariable", "resequence", "resumable", "rollback", "routingSlip", "saga", "sample", "script", "setBody", "setExchangePattern", "setHeader", "setHeaders", "setProperty", "setVariable", "setVariables", "sort", "split", "step", "stop", "threads", "throttle", "throwException", "to", "toD", "tokenizer", "transacted", "transform", "transformDataType", "unmarshal", "validate", "wireTap" ], "deprecated": false, "autowired": false, "secret": false } + "group": { "index": 18, "kind": "attribute", "displayName": "Group", "group": "advanced", "label": "advanced", "required": false, "type": "integer", "javaType": "java.lang.Integer", "deprecated": false, "autowired": false, "secret": false, "description": "Groups N split messages into a single message with a java.util.List body. This allows processing items in chunks instead of one at a time." }, + "errorThreshold": { "index": 19, "kind": "attribute", "displayName": "Error Threshold", "group": "advanced", "label": "advanced", "required": false, "type": "number", "javaType": "java.lang.Double", "deprecated": false, "autowired": false, "secret": false, "description": "Sets the error threshold as a fraction (0.0-1.0) of failed items before aborting the split operation." }, + "maxFailedRecords": { "index": 20, "kind": "attribute", "displayName": "Max Failed Records", "group": "advanced", "label": "advanced", "required": false, "type": "integer", "javaType": "java.lang.Integer", "deprecated": false, "autowired": false, "secret": false, "description": "Sets the maximum number of failed records before aborting the split operation." }, + "outputs": { "index": 21, "kind": "element", "displayName": "Outputs", "group": "common", "required": true, "type": "array", "javaType": "java.util.List>", "oneOf": [ "aggregate", "bean", "choice", "circuitBreaker", "claimCheck", "convertBodyTo", "convertHeaderTo", "convertVariableTo", "delay", "doCatch", "doFinally", "doTry", "dynamicRouter", "enrich", "filter", "idempotentConsumer", "intercept", "interceptFrom", "interceptSendToEndpoint", "kamelet", "loadBalance", "log", "loop", "marshal", "multicast", "onCompletion", "onException", "pausable", "pipeline", "policy", "poll", "pollEnrich", "process", "recipientList", "removeHeader", "removeHeaders", "removeProperties", "removeProperty", "removeVariable", "resequence", "resumable", "rollback", "routingSlip", "saga", "sample", "script", "setBody", "setExchangePattern", "setHeader", "setHeaders", "setProperty", "setVariable", "setVariables", "sort", "split", "step", "stop", "threads", "throttle", "throwException", "to", "toD", "tokenizer", "transacted", "transform", "transformDataType", "unmarshal", "validate", "wireTap" ], "deprecated": false, "autowired": false, "secret": false } }, "exchangeProperties": { "CamelSplitIndex": { "index": 0, "kind": "exchangeProperty", "displayName": "Split Index", "label": "producer", "required": false, "javaType": "int", "deprecated": false, "autowired": false, "secret": false, "important": true, "description": "A split counter that increases for each Exchange being split. The counter starts from 0." }, diff --git a/core/camel-core-model/src/main/java/org/apache/camel/model/SplitDefinition.java b/core/camel-core-model/src/main/java/org/apache/camel/model/SplitDefinition.java index a32db4e58602a..1f86484d8cd00 100644 --- a/core/camel-core-model/src/main/java/org/apache/camel/model/SplitDefinition.java +++ b/core/camel-core-model/src/main/java/org/apache/camel/model/SplitDefinition.java @@ -85,6 +85,15 @@ public class SplitDefinition extends OutputExpressionNode implements ExecutorSer @XmlAttribute @Metadata(label = "advanced", javaType = "java.lang.Boolean") private String shareUnitOfWork; + @XmlAttribute + @Metadata(label = "advanced", javaType = "java.lang.Integer") + private String group; + @XmlAttribute + @Metadata(label = "advanced", javaType = "java.lang.Double") + private String errorThreshold; + @XmlAttribute + @Metadata(label = "advanced", javaType = "java.lang.Integer") + private String maxFailedRecords; public SplitDefinition() { } @@ -107,6 +116,9 @@ public SplitDefinition(SplitDefinition source) { this.executorService = source.executorService; this.onPrepare = source.onPrepare; this.shareUnitOfWork = source.shareUnitOfWork; + this.group = source.group; + this.errorThreshold = source.errorThreshold; + this.maxFailedRecords = source.maxFailedRecords; } public SplitDefinition(Expression expression) { @@ -566,6 +578,89 @@ public SplitDefinition shareUnitOfWork(String shareUnitOfWork) { return this; } + /** + * Groups N split messages into a single message with a {@link java.util.List} body. This allows processing items in + * chunks instead of one at a time. + * + * @param group the number of items per group + * @return the builder + */ + public SplitDefinition group(int group) { + return group(Integer.toString(group)); + } + + /** + * Groups N split messages into a single message with a {@link java.util.List} body. This allows processing items in + * chunks instead of one at a time. + * + * @param group the number of items per group + * @return the builder + */ + public SplitDefinition group(String group) { + setGroup(group); + return this; + } + + /** + * Sets the error threshold as a fraction (0.0-1.0) of failed items before aborting the split operation. For + * example, 0.1 means abort if more than 10% of items fail. When the threshold is exceeded, a + * {@link org.apache.camel.CamelExchangeException} is thrown. + *

+ * This option is mutually exclusive with {@code stopOnException}. When set, individual item failures are tracked + * but processing continues until the threshold is exceeded. + * + * @param errorThreshold the failure ratio threshold (0.0-1.0) + * @return the builder + */ + public SplitDefinition errorThreshold(double errorThreshold) { + return errorThreshold(Double.toString(errorThreshold)); + } + + /** + * Sets the error threshold as a fraction (0.0-1.0) of failed items before aborting the split operation. For + * example, 0.1 means abort if more than 10% of items fail. When the threshold is exceeded, a + * {@link org.apache.camel.CamelExchangeException} is thrown. + *

+ * This option is mutually exclusive with {@code stopOnException}. When set, individual item failures are tracked + * but processing continues until the threshold is exceeded. + * + * @param errorThreshold the failure ratio threshold (0.0-1.0) + * @return the builder + */ + public SplitDefinition errorThreshold(String errorThreshold) { + setErrorThreshold(errorThreshold); + return this; + } + + /** + * Sets the maximum number of failed records before aborting the split operation. When the count is exceeded, a + * {@link org.apache.camel.CamelExchangeException} is thrown. + *

+ * This option is mutually exclusive with {@code stopOnException}. Can be combined with {@code errorThreshold} — + * processing aborts when either threshold is exceeded. + * + * @param maxFailedRecords the maximum number of allowed failures + * @return the builder + */ + public SplitDefinition maxFailedRecords(int maxFailedRecords) { + return maxFailedRecords(Integer.toString(maxFailedRecords)); + } + + /** + * Sets the maximum number of failed records before aborting the split operation. When the count is exceeded, a + * {@link org.apache.camel.CamelExchangeException} is thrown. + *

+ * This option is mutually exclusive with {@code stopOnException}. Can be combined with {@code errorThreshold} — + * processing aborts when either threshold is exceeded. + * + * @param maxFailedRecords the maximum number of allowed failures + * @return the builder + */ + public SplitDefinition maxFailedRecords(String maxFailedRecords) { + setMaxFailedRecords(maxFailedRecords); + return this; + } + // Properties // ------------------------------------------------------------------------- @@ -723,4 +818,38 @@ public String getExecutorService() { public void setExecutorService(String executorService) { this.executorService = executorService; } + + public String getGroup() { + return group; + } + + /** + * Groups N split messages into a single message with a {@link java.util.List} body. This allows processing items in + * chunks instead of one at a time. + */ + public void setGroup(String group) { + this.group = group; + } + + public String getErrorThreshold() { + return errorThreshold; + } + + /** + * Sets the error threshold as a fraction (0.0-1.0) of failed items before aborting the split operation. + */ + public void setErrorThreshold(String errorThreshold) { + this.errorThreshold = errorThreshold; + } + + public String getMaxFailedRecords() { + return maxFailedRecords; + } + + /** + * Sets the maximum number of failed records before aborting the split operation. + */ + public void setMaxFailedRecords(String maxFailedRecords) { + this.maxFailedRecords = maxFailedRecords; + } } diff --git a/core/camel-core-processor/src/main/java/org/apache/camel/processor/MulticastProcessor.java b/core/camel-core-processor/src/main/java/org/apache/camel/processor/MulticastProcessor.java index e4fc297d8566d..173bbb59c98ee 100644 --- a/core/camel-core-processor/src/main/java/org/apache/camel/processor/MulticastProcessor.java +++ b/core/camel-core-processor/src/main/java/org/apache/camel/processor/MulticastProcessor.java @@ -587,7 +587,7 @@ public void run() { msg = "Multicast processing failed for number " + index; } boolean continueProcessing = PipelineHelper.continueProcessing(exchange, msg, LOG); - if (stopOnException && !continueProcessing) { + if (!continueProcessing && !shouldContinueOnFailure(exchange, original, index)) { if (exchange.getException() != null) { // wrap in exception to explain where it failed exchange.setException(new CamelExchangeException( @@ -723,7 +723,7 @@ boolean doRun() throws Exception { msg = "Multicast processing failed for number " + index; } boolean continueProcessing = PipelineHelper.continueProcessing(exchange, msg, LOG); - if (stopOnException && !continueProcessing) { + if (!continueProcessing && !shouldContinueOnFailure(exchange, original, index)) { if (exchange.getException() != null) { // wrap in exception to explain where it failed exchange.setException(new CamelExchangeException( @@ -759,6 +759,22 @@ boolean doRun() throws Exception { } } + /** + * Determines whether processing should continue after a sub-exchange has failed. + *

+ * The default implementation returns {@code false} when {@code stopOnException} is enabled (meaning processing + * should stop). Subclasses (e.g., Splitter) can override this to implement more sophisticated failure policies such + * as error threshold checking. + * + * @param subExchange the failed sub-exchange + * @param original the original exchange + * @param index the index of the failed sub-exchange + * @return {@code true} to continue processing despite the failure, {@code false} to stop + */ + protected boolean shouldContinueOnFailure(Exchange subExchange, Exchange original, int index) { + return !stopOnException; + } + protected ScheduledFuture schedule(Executor executor, Runnable runnable, long delay, TimeUnit unit) { if (executor instanceof ScheduledExecutorService scheduledExecutorService) { return scheduledExecutorService.schedule(runnable, delay, unit); diff --git a/core/camel-core-processor/src/main/java/org/apache/camel/processor/Splitter.java b/core/camel-core-processor/src/main/java/org/apache/camel/processor/Splitter.java index 4620ecf945f42..e402bc1478867 100644 --- a/core/camel-core-processor/src/main/java/org/apache/camel/processor/Splitter.java +++ b/core/camel-core-processor/src/main/java/org/apache/camel/processor/Splitter.java @@ -25,7 +25,9 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ExecutorService; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.camel.AggregationStrategy; import org.apache.camel.AsyncCallback; @@ -37,9 +39,11 @@ import org.apache.camel.Processor; import org.apache.camel.Route; import org.apache.camel.RuntimeCamelException; +import org.apache.camel.SplitResult; import org.apache.camel.processor.aggregate.ShareUnitOfWorkAggregationStrategy; import org.apache.camel.processor.aggregate.UseOriginalAggregationStrategy; import org.apache.camel.support.ExchangeHelper; +import org.apache.camel.support.GroupIterator; import org.apache.camel.support.ObjectHelper; import org.apache.camel.util.IOHelper; import org.apache.camel.util.StringHelper; @@ -54,8 +58,12 @@ public class Splitter extends MulticastProcessor { private static final String IGNORE_DELIMITER_MARKER = "false"; private static final String SINGLE_DELIMITER_MARKER = "single"; + private static final String SPLIT_FAILURE_TRACKER = "CamelSplitFailureTracker"; private final Expression expression; private final String delimiter; + private int group; + private double errorThreshold; + private int maxFailedRecords; public Splitter(CamelContext camelContext, Route route, Expression expression, Processor destination, AggregationStrategy aggregationStrategy, boolean parallelProcessing, @@ -123,7 +131,21 @@ public boolean process(Exchange exchange, final AsyncCallback callback) { setAggregationStrategyOnExchange(exchange, original); } - return super.process(exchange, callback); + // create failure tracker if error thresholds are configured + boolean hasErrorThreshold = errorThreshold > 0 || maxFailedRecords > 0; + if (hasErrorThreshold) { + exchange.setProperty(SPLIT_FAILURE_TRACKER, new SplitFailureTracker()); + } + + // wrap callback to build SplitResult after all items are processed + AsyncCallback wrappedCallback = hasErrorThreshold + ? doneSync -> { + buildSplitResult(exchange); + callback.done(doneSync); + } + : callback; + + return super.process(exchange, wrappedCallback); } @Override @@ -145,6 +167,14 @@ protected Iterable createProcessorExchangePairs(Exchange throw exchange.getException(); } + // store total items count in tracker for SplitResult reporting + if (answer instanceof Collection coll) { + SplitFailureTracker tracker = exchange.getProperty(SPLIT_FAILURE_TRACKER, SplitFailureTracker.class); + if (tracker != null) { + tracker.setTotalItems(coll.size()); + } + } + return answer; } @@ -166,14 +196,17 @@ private SplitterIterable(Exchange exchange, Object value) { this.original = exchange; this.value = value; + Iterator rawIterator; if (IGNORE_DELIMITER_MARKER.equalsIgnoreCase(delimiter)) { - this.iterator = ObjectHelper.createIterator(value, null); + rawIterator = ObjectHelper.createIterator(value, null); } else if (SINGLE_DELIMITER_MARKER.equalsIgnoreCase(delimiter)) { // force single element - this.iterator = ObjectHelper.createIterator(List.of(value)); + rawIterator = ObjectHelper.createIterator(List.of(value)); } else { - this.iterator = ObjectHelper.createIterator(value, delimiter); + rawIterator = ObjectHelper.createIterator(value, delimiter); } + // wrap with GroupIterator if group > 0 to chunk items into List batches + this.iterator = group > 0 ? new GroupIterator(rawIterator, group) : rawIterator; this.copy = copyAndPrepareSubExchange(exchange); this.route = ExchangeHelper.getRoute(exchange); @@ -310,6 +343,108 @@ public Expression getExpression() { return expression; } + public int getGroup() { + return group; + } + + public void setGroup(int group) { + this.group = group; + } + + public double getErrorThreshold() { + return errorThreshold; + } + + public void setErrorThreshold(double errorThreshold) { + this.errorThreshold = errorThreshold; + } + + public int getMaxFailedRecords() { + return maxFailedRecords; + } + + public void setMaxFailedRecords(int maxFailedRecords) { + this.maxFailedRecords = maxFailedRecords; + } + + @Override + protected boolean shouldContinueOnFailure(Exchange subExchange, Exchange original, int index) { + SplitFailureTracker tracker = original.getProperty(SPLIT_FAILURE_TRACKER, SplitFailureTracker.class); + if (tracker == null) { + return super.shouldContinueOnFailure(subExchange, original, index); + } + + // record the failure + tracker.recordFailure(index, subExchange.getException()); + + // check if we've exceeded the max failed records + if (maxFailedRecords > 0 && tracker.getFailureCount() >= maxFailedRecords) { + return false; + } + // check if we've exceeded the error ratio threshold + if (errorThreshold > 0) { + double ratio = (double) tracker.getFailureCount() / (index + 1); + if (ratio >= errorThreshold) { + return false; + } + } + + // continue processing - clear the exception so aggregation proceeds normally + subExchange.setException(null); + return true; + } + + static final class SplitFailureTracker { + private final AtomicInteger failureCount = new AtomicInteger(); + private final AtomicInteger totalItems = new AtomicInteger(); + private final CopyOnWriteArrayList failures = new CopyOnWriteArrayList<>(); + + record SplitFailure(int index, Exception exception) { + } + + void recordFailure(int index, Exception exception) { + failureCount.incrementAndGet(); + failures.add(new SplitFailure(index, exception)); + } + + void setTotalItems(int count) { + totalItems.set(count); + } + + int getTotalItems() { + return totalItems.get(); + } + + int getFailureCount() { + return failureCount.get(); + } + + List getFailures() { + return Collections.unmodifiableList(failures); + } + } + + private void buildSplitResult(Exchange exchange) { + SplitFailureTracker tracker = exchange.getProperty(SPLIT_FAILURE_TRACKER, SplitFailureTracker.class); + if (tracker == null) { + return; + } + + int totalItems = tracker.getTotalItems(); + boolean aborted = exchange.getException() != null; + + List failures = new ArrayList<>(); + for (SplitFailureTracker.SplitFailure f : tracker.getFailures()) { + failures.add(new SplitResult.Failure(f.index(), f.exception())); + } + + SplitResult result = new SplitResult(totalItems, tracker.getFailureCount(), failures, aborted); + exchange.setProperty(ExchangePropertyKey.SPLIT_RESULT, result); + + // remove internal tracker + exchange.removeProperty(SPLIT_FAILURE_TRACKER); + } + private Exchange copyAndPrepareSubExchange(Exchange exchange) { Exchange answer = processorExchangeFactory.createCopy(exchange); // must preserve exchange id diff --git a/core/camel-core-reifier/src/main/java/org/apache/camel/reifier/SplitReifier.java b/core/camel-core-reifier/src/main/java/org/apache/camel/reifier/SplitReifier.java index 77d0fb583dfbf..49afb1a83427b 100644 --- a/core/camel-core-reifier/src/main/java/org/apache/camel/reifier/SplitReifier.java +++ b/core/camel-core-reifier/src/main/java/org/apache/camel/reifier/SplitReifier.java @@ -78,6 +78,27 @@ public Processor createProcessor() throws Exception { } answer.setSynchronous(isSynchronous); answer.setDisabled(isDisabled(camelContext, definition)); + + int group = parseInt(definition.getGroup(), 0); + if (group > 0) { + answer.setGroup(group); + } + + String etStr = parseString(definition.getErrorThreshold()); + double errorThreshold = etStr != null ? Double.parseDouble(etStr) : 0; + int maxFailedRecords = parseInt(definition.getMaxFailedRecords(), 0); + boolean hasErrorThreshold = errorThreshold > 0 || maxFailedRecords > 0; + if (hasErrorThreshold && isStopOnException) { + throw new IllegalArgumentException( + "Cannot use both stopOnException and errorThreshold/maxFailedRecords on the Splitter EIP"); + } + if (errorThreshold > 0) { + answer.setErrorThreshold(errorThreshold); + } + if (maxFailedRecords > 0) { + answer.setMaxFailedRecords(maxFailedRecords); + } + return answer; } diff --git a/core/camel-core/src/test/java/org/apache/camel/processor/SplitterErrorThresholdTest.java b/core/camel-core/src/test/java/org/apache/camel/processor/SplitterErrorThresholdTest.java new file mode 100644 index 0000000000000..09770dbcdd59f --- /dev/null +++ b/core/camel-core/src/test/java/org/apache/camel/processor/SplitterErrorThresholdTest.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.processor; + +import java.util.Arrays; + +import org.apache.camel.ContextTestSupport; +import org.apache.camel.Exchange; +import org.apache.camel.builder.RouteBuilder; +import org.apache.camel.component.mock.MockEndpoint; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class SplitterErrorThresholdTest extends ContextTestSupport { + + @Test + void testErrorThresholdStopsWhenRatioExceeded() throws Exception { + // items: FAIL, FAIL, a, b, c + // errorThreshold=0.5 (50%) + // item 0: FAIL → failCount=1, ratio=1/1=100% ≥ 50% → stop + MockEndpoint mock = getMockEndpoint("mock:split"); + mock.expectedMinimumMessageCount(0); + + Exchange result = template.send("direct:start", + e -> e.getIn().setBody(Arrays.asList("FAIL", "FAIL", "a", "b", "c"))); + + mock.assertIsSatisfied(); + + assertNotNull(result.getException(), "Should have an exception when error threshold exceeded"); + } + + @Test + void testErrorThresholdBelowRatio() throws Exception { + // items: a, b, FAIL, c, d, e, f, g, h, i + // errorThreshold=0.5 (50%) + // item 0: a → ok + // item 1: b → ok + // item 2: FAIL → failCount=1, ratio=1/3=33% < 50% → continue + // items 3-9: ok + MockEndpoint mock = getMockEndpoint("mock:split"); + mock.expectedMessageCount(9); // 10 items minus 1 failure + + template.sendBody("direct:start", Arrays.asList("a", "b", "FAIL", "c", "d", "e", "f", "g", "h", "i")); + + mock.assertIsSatisfied(); + } + + @Test + void testErrorThresholdAllSucceed() throws Exception { + MockEndpoint mock = getMockEndpoint("mock:split"); + mock.expectedMessageCount(5); + + template.sendBody("direct:start", Arrays.asList("a", "b", "c", "d", "e")); + + mock.assertIsSatisfied(); + } + + @Test + void testStopOnExceptionAndErrorThresholdAreMutuallyExclusive() throws Exception { + try { + context.addRoutes(new RouteBuilder() { + @Override + public void configure() { + from("direct:invalid") + .split(body()).stopOnException().errorThreshold(0.5) + .to("mock:invalid"); + } + }); + throw new AssertionError("Expected IllegalArgumentException"); + } catch (Exception e) { + assertTrue(e.getCause() instanceof IllegalArgumentException + || e instanceof IllegalArgumentException, + "Should throw IllegalArgumentException"); + } + } + + @Override + protected RouteBuilder createRouteBuilder() { + return new RouteBuilder() { + @Override + public void configure() { + from("direct:start") + .split(body()).errorThreshold(0.5) + .process(exchange -> { + String body = exchange.getIn().getBody(String.class); + if ("FAIL".equals(body)) { + throw new IllegalArgumentException("Simulated failure for: " + body); + } + }) + .to("mock:split"); + } + }; + } +} diff --git a/core/camel-core/src/test/java/org/apache/camel/processor/SplitterGroupTest.java b/core/camel-core/src/test/java/org/apache/camel/processor/SplitterGroupTest.java new file mode 100644 index 0000000000000..1da7beeb775ed --- /dev/null +++ b/core/camel-core/src/test/java/org/apache/camel/processor/SplitterGroupTest.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.processor; + +import java.util.Arrays; +import java.util.List; + +import org.apache.camel.ContextTestSupport; +import org.apache.camel.builder.RouteBuilder; +import org.apache.camel.component.mock.MockEndpoint; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; + +class SplitterGroupTest extends ContextTestSupport { + + @Test + void testSplitterGroup() throws Exception { + MockEndpoint mock = getMockEndpoint("mock:split"); + mock.expectedMessageCount(3); + + template.sendBody("direct:start", Arrays.asList("a", "b", "c", "d", "e", "f", "g")); + + mock.assertIsSatisfied(); + + // First group: [a, b, c] + Object body0 = mock.getReceivedExchanges().get(0).getIn().getBody(); + assertInstanceOf(List.class, body0); + assertEquals(3, ((List) body0).size()); + + // Second group: [d, e, f] + Object body1 = mock.getReceivedExchanges().get(1).getIn().getBody(); + assertInstanceOf(List.class, body1); + assertEquals(3, ((List) body1).size()); + + // Third group: [g] (remainder) + Object body2 = mock.getReceivedExchanges().get(2).getIn().getBody(); + assertInstanceOf(List.class, body2); + assertEquals(1, ((List) body2).size()); + } + + @Test + void testSplitterGroupExactMultiple() throws Exception { + MockEndpoint mock = getMockEndpoint("mock:split"); + mock.expectedMessageCount(2); + + template.sendBody("direct:start", Arrays.asList("a", "b", "c", "d", "e", "f")); + + mock.assertIsSatisfied(); + + assertEquals(3, ((List) mock.getReceivedExchanges().get(0).getIn().getBody()).size()); + assertEquals(3, ((List) mock.getReceivedExchanges().get(1).getIn().getBody()).size()); + } + + @Test + void testSplitterGroupSingleItem() throws Exception { + MockEndpoint mock = getMockEndpoint("mock:split"); + mock.expectedMessageCount(1); + + template.sendBody("direct:start", List.of("only")); + + mock.assertIsSatisfied(); + + List body = (List) mock.getReceivedExchanges().get(0).getIn().getBody(); + assertEquals(1, body.size()); + assertEquals("only", body.get(0)); + } + + @Test + void testSplitterGroupWithParallelProcessing() throws Exception { + MockEndpoint mock = getMockEndpoint("mock:parallel-split"); + mock.expectedMessageCount(3); + + template.sendBody("direct:parallel", Arrays.asList(1, 2, 3, 4, 5, 6, 7)); + + mock.assertIsSatisfied(); + } + + @Override + protected RouteBuilder createRouteBuilder() { + return new RouteBuilder() { + @Override + public void configure() { + from("direct:start") + .split(body()).group(3) + .to("mock:split"); + + from("direct:parallel") + .split(body()).group(3).parallelProcessing() + .to("mock:parallel-split"); + } + }; + } +} diff --git a/core/camel-core/src/test/java/org/apache/camel/processor/SplitterMaxFailedRecordsTest.java b/core/camel-core/src/test/java/org/apache/camel/processor/SplitterMaxFailedRecordsTest.java new file mode 100644 index 0000000000000..69b36f39a4139 --- /dev/null +++ b/core/camel-core/src/test/java/org/apache/camel/processor/SplitterMaxFailedRecordsTest.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.processor; + +import java.util.Arrays; + +import org.apache.camel.ContextTestSupport; +import org.apache.camel.Exchange; +import org.apache.camel.builder.RouteBuilder; +import org.apache.camel.component.mock.MockEndpoint; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class SplitterMaxFailedRecordsTest extends ContextTestSupport { + + @Test + void testMaxFailedRecordsStopsAfterThreshold() throws Exception { + // items: a, FAIL, b, FAIL, c, FAIL, d + // maxFailedRecords=2, so processing should stop after the 2nd failure + MockEndpoint mock = getMockEndpoint("mock:split"); + // With maxFailedRecords=2: items a (ok), FAIL (fail #1, continue), b (ok), FAIL (fail #2, stop) + // Items c, FAIL, d should NOT be processed + mock.expectedMinimumMessageCount(2); // at least a, b + + Exchange result = template.send("direct:start", + e -> e.getIn().setBody(Arrays.asList("a", "FAIL", "b", "FAIL", "c", "FAIL", "d"))); + + mock.assertIsSatisfied(); + + // the exchange should have an exception because threshold was exceeded + assertNotNull(result.getException(), "Should have an exception when max failed records exceeded"); + + // verify that not all items were processed + assertTrue(mock.getReceivedCounter() < 7, "Should have stopped before processing all items"); + } + + @Test + void testMaxFailedRecordsAllSucceed() throws Exception { + MockEndpoint mock = getMockEndpoint("mock:split"); + mock.expectedMessageCount(4); + + template.sendBody("direct:start", Arrays.asList("a", "b", "c", "d")); + + mock.assertIsSatisfied(); + } + + @Test + void testMaxFailedRecordsSingleFailure() throws Exception { + // maxFailedRecords=2, only 1 failure, should process all items + // only 3 items reach mock:split (the failed one throws before reaching the endpoint) + MockEndpoint mock = getMockEndpoint("mock:split"); + mock.expectedMessageCount(3); + + template.sendBody("direct:start", Arrays.asList("a", "FAIL", "b", "c")); + + mock.assertIsSatisfied(); + } + + @Test + void testStopOnExceptionAndMaxFailedRecordsAreMutuallyExclusive() throws Exception { + try { + context.addRoutes(new RouteBuilder() { + @Override + public void configure() { + from("direct:invalid") + .split(body()).stopOnException().maxFailedRecords(3) + .to("mock:invalid"); + } + }); + // should not reach here + throw new AssertionError("Expected IllegalArgumentException"); + } catch (Exception e) { + assertTrue(e.getCause() instanceof IllegalArgumentException + || e instanceof IllegalArgumentException, + "Should throw IllegalArgumentException"); + } + } + + @Override + protected RouteBuilder createRouteBuilder() { + return new RouteBuilder() { + @Override + public void configure() { + from("direct:start") + .split(body()).maxFailedRecords(2) + .process(exchange -> { + String body = exchange.getIn().getBody(String.class); + if ("FAIL".equals(body)) { + throw new IllegalArgumentException("Simulated failure for: " + body); + } + }) + .to("mock:split"); + } + }; + } +} diff --git a/core/camel-core/src/test/java/org/apache/camel/processor/SplitterSplitResultTest.java b/core/camel-core/src/test/java/org/apache/camel/processor/SplitterSplitResultTest.java new file mode 100644 index 0000000000000..012e6d75e0d3a --- /dev/null +++ b/core/camel-core/src/test/java/org/apache/camel/processor/SplitterSplitResultTest.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.processor; + +import java.util.Arrays; + +import org.apache.camel.ContextTestSupport; +import org.apache.camel.Exchange; +import org.apache.camel.SplitResult; +import org.apache.camel.builder.RouteBuilder; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class SplitterSplitResultTest extends ContextTestSupport { + + @Test + void testSplitResultWithFailures() throws Exception { + // items: a, FAIL, b, FAIL, c — maxFailedRecords=5 so all items are processed + Exchange result = template.send("direct:tolerant", + e -> e.getIn().setBody(Arrays.asList("a", "FAIL", "b", "FAIL", "c"))); + + SplitResult splitResult = result.getProperty(Exchange.SPLIT_RESULT, SplitResult.class); + assertNotNull(splitResult, "SplitResult should be set as exchange property"); + + assertEquals(5, splitResult.getTotalItems()); + assertEquals(2, splitResult.getFailureCount()); + assertEquals(3, splitResult.getSuccessCount()); + assertFalse(splitResult.isAborted()); + + // verify failure details + assertEquals(2, splitResult.getFailures().size()); + assertEquals(1, splitResult.getFailures().get(0).index()); + assertEquals(3, splitResult.getFailures().get(1).index()); + assertNotNull(splitResult.getFailures().get(0).exception()); + } + + @Test + void testSplitResultWhenAborted() throws Exception { + // items: FAIL, FAIL, a, b — maxFailedRecords=2 so aborted after 2nd failure + Exchange result = template.send("direct:strict", + e -> e.getIn().setBody(Arrays.asList("FAIL", "FAIL", "a", "b"))); + + SplitResult splitResult = result.getProperty(Exchange.SPLIT_RESULT, SplitResult.class); + assertNotNull(splitResult, "SplitResult should be set even when aborted"); + assertTrue(splitResult.isAborted()); + assertEquals(2, splitResult.getFailureCount()); + } + + @Test + void testSplitResultAllSuccess() throws Exception { + Exchange result = template.send("direct:tolerant", + e -> e.getIn().setBody(Arrays.asList("a", "b", "c"))); + + SplitResult splitResult = result.getProperty(Exchange.SPLIT_RESULT, SplitResult.class); + assertNotNull(splitResult, "SplitResult should be set even with no failures"); + + assertEquals(3, splitResult.getTotalItems()); + assertEquals(0, splitResult.getFailureCount()); + assertEquals(3, splitResult.getSuccessCount()); + assertFalse(splitResult.isAborted()); + assertTrue(splitResult.getFailures().isEmpty()); + } + + @Test + void testNoSplitResultWithoutErrorThreshold() throws Exception { + // plain split without error threshold should not set SplitResult + Exchange result = template.send("direct:plain", + e -> e.getIn().setBody(Arrays.asList("a", "b", "c"))); + + SplitResult splitResult = result.getProperty(Exchange.SPLIT_RESULT, SplitResult.class); + assertNull(splitResult, "SplitResult should not be set without error threshold"); + } + + @Override + protected RouteBuilder createRouteBuilder() { + return new RouteBuilder() { + @Override + public void configure() { + from("direct:tolerant") + .split(body()).maxFailedRecords(5) + .process(exchange -> { + String body = exchange.getIn().getBody(String.class); + if ("FAIL".equals(body)) { + throw new IllegalArgumentException("Simulated failure for: " + body); + } + }) + .to("mock:split"); + + from("direct:strict") + .split(body()).maxFailedRecords(2) + .process(exchange -> { + String body = exchange.getIn().getBody(String.class); + if ("FAIL".equals(body)) { + throw new IllegalArgumentException("Simulated failure for: " + body); + } + }) + .to("mock:split"); + + from("direct:plain") + .split(body()) + .to("mock:split"); + } + }; + } +} diff --git a/dsl/camel-yaml-dsl/camel-yaml-dsl-deserializers/src/generated/java/org/apache/camel/dsl/yaml/deserializers/ModelDeserializers.java b/dsl/camel-yaml-dsl/camel-yaml-dsl-deserializers/src/generated/java/org/apache/camel/dsl/yaml/deserializers/ModelDeserializers.java index db0750c4bd7f6..4f35d28c62dda 100644 --- a/dsl/camel-yaml-dsl/camel-yaml-dsl-deserializers/src/generated/java/org/apache/camel/dsl/yaml/deserializers/ModelDeserializers.java +++ b/dsl/camel-yaml-dsl/camel-yaml-dsl-deserializers/src/generated/java/org/apache/camel/dsl/yaml/deserializers/ModelDeserializers.java @@ -16833,9 +16833,12 @@ protected boolean setProperty(SpELExpression target, String propertyKey, @YamlProperty(name = "delimiter", type = "string", defaultValue = ",", description = "Delimiter used in splitting messages. Can be turned off using the value false. To force not splitting then the delimiter can be set to single to use the value as a single list, this can be needed in some special situations. The default value is comma.", displayName = "Delimiter"), @YamlProperty(name = "description", type = "string", description = "Sets the description of this node", displayName = "Description"), @YamlProperty(name = "disabled", type = "boolean", defaultValue = "false", description = "Disables this EIP from the route.", displayName = "Disabled"), + @YamlProperty(name = "errorThreshold", type = "number"), @YamlProperty(name = "executorService", type = "string", description = "To use a custom Thread Pool to be used for parallel processing. Notice if you set this option, then parallel processing is automatically implied, and you do not have to enable that option as well.", displayName = "Executor Service"), @YamlProperty(name = "expression", type = "object:org.apache.camel.model.language.ExpressionDefinition", description = "Expression of how to split the message body, such as as-is, using a tokenizer, or using a xpath.", displayName = "Expression", oneOf = "expression"), + @YamlProperty(name = "group", type = "number"), @YamlProperty(name = "id", type = "string", description = "Sets the id of this node", displayName = "Id"), + @YamlProperty(name = "maxFailedRecords", type = "number"), @YamlProperty(name = "note", type = "string", description = "Sets the note of this node", displayName = "Note"), @YamlProperty(name = "onPrepare", type = "string", description = "Uses the Processor when preparing the org.apache.camel.Exchange to be sent. This can be used to deep-clone messages that should be sent, or any custom logic needed before the exchange is sent.", displayName = "On Prepare"), @YamlProperty(name = "parallelAggregate", type = "boolean", deprecated = true, defaultValue = "false", description = "If enabled then the aggregate method on AggregationStrategy can be called concurrently. Notice that this would require the implementation of AggregationStrategy to be implemented as thread-safe. By default this is false meaning that Camel synchronizes the call to the aggregate method. Though in some use-cases this can be used to archive higher performance when the AggregationStrategy is implemented as thread-safe.", displayName = "Parallel Aggregate"), @@ -16888,6 +16891,11 @@ protected boolean setProperty(SplitDefinition target, String propertyKey, target.setDisabled(val); break; } + case "errorThreshold": { + String val = asText(node); + target.setErrorThreshold(val); + break; + } case "executorService": { String val = asText(node); target.setExecutorService(val); @@ -16898,6 +16906,16 @@ protected boolean setProperty(SplitDefinition target, String propertyKey, target.setExpression(val); break; } + case "group": { + String val = asText(node); + target.setGroup(val); + break; + } + case "maxFailedRecords": { + String val = asText(node); + target.setMaxFailedRecords(val); + break; + } case "onPrepare": { String val = asText(node); target.setOnPrepare(val); From 46efaa95ed737bc5c55a83612cb63c8ac2a17945 Mon Sep 17 00:00:00 2001 From: Guillaume Nodet Date: Fri, 27 Mar 2026 20:31:34 +0100 Subject: [PATCH 2/9] CAMEL-23264: Add watermark tracking to Splitter EIP - Index-based watermarking: skip already-processed items on subsequent runs - Value-based watermarking: expose stored watermark as exchange property, evaluate Simple expression after completion to determine new value - Watermark is only updated on successful completion (not on abort) - Add SPLIT_WATERMARK exchange property constant Co-Authored-By: Claude Opus 4.6 --- .../camel/ExchangeConstantProvider.java | 3 +- .../main/java/org/apache/camel/Exchange.java | 4 + .../org/apache/camel/model/split.json | 9 +- .../apache/camel/model/SplitDefinition.java | 111 +++++++++++++ .../org/apache/camel/processor/Splitter.java | 103 +++++++++++- .../apache/camel/reifier/SplitReifier.java | 17 ++ .../processor/SplitterWatermarkTest.java | 152 ++++++++++++++++++ .../deserializers/ModelDeserializers.java | 20 ++- 8 files changed, 412 insertions(+), 7 deletions(-) create mode 100644 core/camel-core/src/test/java/org/apache/camel/processor/SplitterWatermarkTest.java diff --git a/core/camel-api/src/generated/java/org/apache/camel/ExchangeConstantProvider.java b/core/camel-api/src/generated/java/org/apache/camel/ExchangeConstantProvider.java index 2e181224ce8c9..36b6d5353c1ba 100644 --- a/core/camel-api/src/generated/java/org/apache/camel/ExchangeConstantProvider.java +++ b/core/camel-api/src/generated/java/org/apache/camel/ExchangeConstantProvider.java @@ -13,7 +13,7 @@ public class ExchangeConstantProvider { private static final Map MAP; static { - Map map = new HashMap<>(150); + Map map = new HashMap<>(151); map.put("AGGREGATED_COLLECTION_GUARD", "CamelAggregatedCollectionGuard"); map.put("AGGREGATED_COMPLETED_BY", "CamelAggregatedCompletedBy"); map.put("AGGREGATED_CORRELATION_KEY", "CamelAggregatedCorrelationKey"); @@ -144,6 +144,7 @@ public class ExchangeConstantProvider { map.put("SPLIT_INDEX", "CamelSplitIndex"); map.put("SPLIT_RESULT", "CamelSplitResult"); map.put("SPLIT_SIZE", "CamelSplitSize"); + map.put("SPLIT_WATERMARK", "CamelSplitWatermark"); map.put("STEP_ID", "CamelStepId"); map.put("STREAM_CACHE_UNIT_OF_WORK", "CamelStreamCacheUnitOfWork"); map.put("TIMER_COUNTER", "CamelTimerCounter"); diff --git a/core/camel-api/src/main/java/org/apache/camel/Exchange.java b/core/camel-api/src/main/java/org/apache/camel/Exchange.java index de55e3fca53f1..0c2dd456661fc 100644 --- a/core/camel-api/src/main/java/org/apache/camel/Exchange.java +++ b/core/camel-api/src/main/java/org/apache/camel/Exchange.java @@ -286,6 +286,10 @@ public interface Exchange extends VariableAware { description = "The result of a Splitter EIP operation with error thresholds, providing structured failure details.", javaType = "org.apache.camel.SplitResult") String SPLIT_RESULT = "CamelSplitResult"; + @Metadata(label = "split", + description = "The current watermark value from the watermark store, set before split processing begins.", + javaType = "String") + String SPLIT_WATERMARK = "CamelSplitWatermark"; @Metadata(label = "step", description = "The id of the Step EIP", javaType = "String") String STEP_ID = "CamelStepId"; diff --git a/core/camel-core-model/src/generated/resources/META-INF/org/apache/camel/model/split.json b/core/camel-core-model/src/generated/resources/META-INF/org/apache/camel/model/split.json index c8ffe70dfc724..0f7987150865b 100644 --- a/core/camel-core-model/src/generated/resources/META-INF/org/apache/camel/model/split.json +++ b/core/camel-core-model/src/generated/resources/META-INF/org/apache/camel/model/split.json @@ -33,11 +33,16 @@ "group": { "index": 18, "kind": "attribute", "displayName": "Group", "group": "advanced", "label": "advanced", "required": false, "type": "integer", "javaType": "java.lang.Integer", "deprecated": false, "autowired": false, "secret": false, "description": "Groups N split messages into a single message with a java.util.List body. This allows processing items in chunks instead of one at a time." }, "errorThreshold": { "index": 19, "kind": "attribute", "displayName": "Error Threshold", "group": "advanced", "label": "advanced", "required": false, "type": "number", "javaType": "java.lang.Double", "deprecated": false, "autowired": false, "secret": false, "description": "Sets the error threshold as a fraction (0.0-1.0) of failed items before aborting the split operation." }, "maxFailedRecords": { "index": 20, "kind": "attribute", "displayName": "Max Failed Records", "group": "advanced", "label": "advanced", "required": false, "type": "integer", "javaType": "java.lang.Integer", "deprecated": false, "autowired": false, "secret": false, "description": "Sets the maximum number of failed records before aborting the split operation." }, - "outputs": { "index": 21, "kind": "element", "displayName": "Outputs", "group": "common", "required": true, "type": "array", "javaType": "java.util.List>", "oneOf": [ "aggregate", "bean", "choice", "circuitBreaker", "claimCheck", "convertBodyTo", "convertHeaderTo", "convertVariableTo", "delay", "doCatch", "doFinally", "doTry", "dynamicRouter", "enrich", "filter", "idempotentConsumer", "intercept", "interceptFrom", "interceptSendToEndpoint", "kamelet", "loadBalance", "log", "loop", "marshal", "multicast", "onCompletion", "onException", "pausable", "pipeline", "policy", "poll", "pollEnrich", "process", "recipientList", "removeHeader", "removeHeaders", "removeProperties", "removeProperty", "removeVariable", "resequence", "resumable", "rollback", "routingSlip", "saga", "sample", "script", "setBody", "setExchangePattern", "setHeader", "setHeaders", "setProperty", "setVariable", "setVariables", "sort", "split", "step", "stop", "threads", "throttle", "throwException", "to", "toD", "tokenizer", "transacted", "transform", "transformDataType", "unmarshal", "validate", "wireTap" ], "deprecated": false, "autowired": false, "secret": false } + "watermarkStore": { "index": 21, "kind": "attribute", "displayName": "Watermark Store", "group": "advanced", "label": "advanced", "required": false, "type": "object", "javaType": "java.util.Map", "deprecated": false, "autowired": false, "secret": false, "description": "Sets a reference to a watermark store (a Map ) in the registry." }, + "watermarkKey": { "index": 22, "kind": "attribute", "displayName": "Watermark Key", "group": "advanced", "label": "advanced", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "description": "Sets the key to use in the watermark store." }, + "watermarkExpression": { "index": 23, "kind": "attribute", "displayName": "Watermark Expression", "group": "advanced", "label": "advanced", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "description": "Sets a Simple expression to evaluate on the exchange after split completion to determine the new watermark value." }, + "outputs": { "index": 24, "kind": "element", "displayName": "Outputs", "group": "common", "required": true, "type": "array", "javaType": "java.util.List>", "oneOf": [ "aggregate", "bean", "choice", "circuitBreaker", "claimCheck", "convertBodyTo", "convertHeaderTo", "convertVariableTo", "delay", "doCatch", "doFinally", "doTry", "dynamicRouter", "enrich", "filter", "idempotentConsumer", "intercept", "interceptFrom", "interceptSendToEndpoint", "kamelet", "loadBalance", "log", "loop", "marshal", "multicast", "onCompletion", "onException", "pausable", "pipeline", "policy", "poll", "pollEnrich", "process", "recipientList", "removeHeader", "removeHeaders", "removeProperties", "removeProperty", "removeVariable", "resequence", "resumable", "rollback", "routingSlip", "saga", "sample", "script", "setBody", "setExchangePattern", "setHeader", "setHeaders", "setProperty", "setVariable", "setVariables", "sort", "split", "step", "stop", "threads", "throttle", "throwException", "to", "toD", "tokenizer", "transacted", "transform", "transformDataType", "unmarshal", "validate", "wireTap" ], "deprecated": false, "autowired": false, "secret": false } }, "exchangeProperties": { "CamelSplitIndex": { "index": 0, "kind": "exchangeProperty", "displayName": "Split Index", "label": "producer", "required": false, "javaType": "int", "deprecated": false, "autowired": false, "secret": false, "important": true, "description": "A split counter that increases for each Exchange being split. The counter starts from 0." }, "CamelSplitComplete": { "index": 1, "kind": "exchangeProperty", "displayName": "Split Complete", "label": "producer", "required": false, "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "description": "Whether this Exchange is the last." }, - "CamelSplitSize": { "index": 2, "kind": "exchangeProperty", "displayName": "Split Size", "label": "producer", "required": false, "javaType": "int", "deprecated": false, "autowired": false, "secret": false, "important": true, "description": "The total number of Exchanges that was split. This property is not applied for stream based splitting, except for the very last message because then Camel knows the total size." } + "CamelSplitSize": { "index": 2, "kind": "exchangeProperty", "displayName": "Split Size", "label": "producer", "required": false, "javaType": "int", "deprecated": false, "autowired": false, "secret": false, "important": true, "description": "The total number of Exchanges that was split. This property is not applied for stream based splitting, except for the very last message because then Camel knows the total size." }, + "CamelSplitResult": { "index": 3, "kind": "exchangeProperty", "displayName": "Split Result", "label": "producer", "required": false, "javaType": "org.apache.camel.SplitResult", "deprecated": false, "autowired": false, "secret": false, "description": "The result of a Splitter EIP operation with error thresholds, providing structured failure details." }, + "CamelSplitWatermark": { "index": 4, "kind": "exchangeProperty", "displayName": "Split Watermark", "label": "producer", "required": false, "javaType": "String", "deprecated": false, "autowired": false, "secret": false, "description": "The current watermark value from the watermark store, set before split processing begins." } } } diff --git a/core/camel-core-model/src/main/java/org/apache/camel/model/SplitDefinition.java b/core/camel-core-model/src/main/java/org/apache/camel/model/SplitDefinition.java index 1f86484d8cd00..3e57d53f196a0 100644 --- a/core/camel-core-model/src/main/java/org/apache/camel/model/SplitDefinition.java +++ b/core/camel-core-model/src/main/java/org/apache/camel/model/SplitDefinition.java @@ -16,6 +16,7 @@ */ package org.apache.camel.model; +import java.util.Map; import java.util.concurrent.ExecutorService; import jakarta.xml.bind.annotation.XmlAccessType; @@ -44,6 +45,9 @@ public class SplitDefinition extends OutputExpressionNode implements ExecutorSer private AggregationStrategy aggregationStrategyBean; @XmlTransient private Processor onPrepareProcessor; + @XmlTransient + @SuppressWarnings("rawtypes") + private Map watermarkStoreBean; @XmlAttribute @Metadata(defaultValue = ",") @@ -94,6 +98,15 @@ public class SplitDefinition extends OutputExpressionNode implements ExecutorSer @XmlAttribute @Metadata(label = "advanced", javaType = "java.lang.Integer") private String maxFailedRecords; + @XmlAttribute + @Metadata(label = "advanced", javaType = "java.util.Map") + private String watermarkStore; + @XmlAttribute + @Metadata(label = "advanced") + private String watermarkKey; + @XmlAttribute + @Metadata(label = "advanced") + private String watermarkExpression; public SplitDefinition() { } @@ -119,6 +132,10 @@ public SplitDefinition(SplitDefinition source) { this.group = source.group; this.errorThreshold = source.errorThreshold; this.maxFailedRecords = source.maxFailedRecords; + this.watermarkStoreBean = source.watermarkStoreBean; + this.watermarkStore = source.watermarkStore; + this.watermarkKey = source.watermarkKey; + this.watermarkExpression = source.watermarkExpression; } public SplitDefinition(Expression expression) { @@ -661,6 +678,62 @@ public SplitDefinition maxFailedRecords(String maxFailedRecords) { return this; } + /** + * Sets a watermark store and key for resume-from-last-position support. When configured, the Splitter tracks + * progress and can skip already-processed items on subsequent runs. + *

+ * With index-based watermarking (no {@code watermarkExpression}), items up to the stored index are automatically + * skipped. With value-based watermarking (with {@code watermarkExpression}), the stored value is exposed as an + * exchange property ({@link org.apache.camel.Exchange#SPLIT_WATERMARK}) for upstream filtering. + *

+ * The watermark is only updated on successful completion — aborted runs preserve the previous watermark to allow + * retry. + * + * @param store the Map to store watermark state in + * @param key the key to use in the store + * @return the builder + */ + @SuppressWarnings("rawtypes") + public SplitDefinition watermarkStore(Map store, String key) { + this.watermarkStoreBean = store; + setWatermarkKey(key); + return this; + } + + /** + * Sets a reference to a watermark store (a {@code Map}) in the registry. + * + * @param watermarkStore reference to the store bean + * @return the builder + */ + public SplitDefinition watermarkStore(String watermarkStore) { + setWatermarkStore(watermarkStore); + return this; + } + + /** + * Sets the key to use in the watermark store. + * + * @param watermarkKey the key + * @return the builder + */ + public SplitDefinition watermarkKey(String watermarkKey) { + setWatermarkKey(watermarkKey); + return this; + } + + /** + * Sets a Simple expression to evaluate on the exchange after split completion to determine the new watermark value. + * When set, enables value-based watermarking instead of index-based. + * + * @param watermarkExpression the Simple expression + * @return the builder + */ + public SplitDefinition watermarkExpression(String watermarkExpression) { + setWatermarkExpression(watermarkExpression); + return this; + } + // Properties // ------------------------------------------------------------------------- @@ -852,4 +925,42 @@ public String getMaxFailedRecords() { public void setMaxFailedRecords(String maxFailedRecords) { this.maxFailedRecords = maxFailedRecords; } + + @SuppressWarnings("rawtypes") + public Map getWatermarkStoreBean() { + return watermarkStoreBean; + } + + public String getWatermarkStore() { + return watermarkStore; + } + + /** + * Sets a reference to a watermark store (a {@code Map}) in the registry. + */ + public void setWatermarkStore(String watermarkStore) { + this.watermarkStore = watermarkStore; + } + + public String getWatermarkKey() { + return watermarkKey; + } + + /** + * Sets the key to use in the watermark store. + */ + public void setWatermarkKey(String watermarkKey) { + this.watermarkKey = watermarkKey; + } + + public String getWatermarkExpression() { + return watermarkExpression; + } + + /** + * Sets a Simple expression to evaluate on the exchange after split completion to determine the new watermark value. + */ + public void setWatermarkExpression(String watermarkExpression) { + this.watermarkExpression = watermarkExpression; + } } diff --git a/core/camel-core-processor/src/main/java/org/apache/camel/processor/Splitter.java b/core/camel-core-processor/src/main/java/org/apache/camel/processor/Splitter.java index e402bc1478867..5287e5ba0c1f1 100644 --- a/core/camel-core-processor/src/main/java/org/apache/camel/processor/Splitter.java +++ b/core/camel-core-processor/src/main/java/org/apache/camel/processor/Splitter.java @@ -59,11 +59,16 @@ public class Splitter extends MulticastProcessor { private static final String IGNORE_DELIMITER_MARKER = "false"; private static final String SINGLE_DELIMITER_MARKER = "single"; private static final String SPLIT_FAILURE_TRACKER = "CamelSplitFailureTracker"; + private static final String SPLIT_WATERMARK_OFFSET = "CamelSplitWatermarkOffset"; + private static final String SPLIT_WATERMARK_COUNT = "CamelSplitWatermarkCount"; private final Expression expression; private final String delimiter; private int group; private double errorThreshold; private int maxFailedRecords; + private Map watermarkStore; + private String watermarkKey; + private Expression watermarkExpression; public Splitter(CamelContext camelContext, Route route, Expression expression, Processor destination, AggregationStrategy aggregationStrategy, boolean parallelProcessing, @@ -104,6 +109,9 @@ protected void doBuild() throws Exception { protected void doInit() throws Exception { super.doInit(); expression.init(getCamelContext()); + if (watermarkExpression != null) { + watermarkExpression.init(getCamelContext()); + } } @Override @@ -137,10 +145,25 @@ public boolean process(Exchange exchange, final AsyncCallback callback) { exchange.setProperty(SPLIT_FAILURE_TRACKER, new SplitFailureTracker()); } - // wrap callback to build SplitResult after all items are processed - AsyncCallback wrappedCallback = hasErrorThreshold + // set current watermark value as exchange property before processing + boolean hasWatermark = watermarkStore != null && watermarkKey != null; + if (hasWatermark) { + String currentWatermark = watermarkStore.get(watermarkKey); + if (currentWatermark != null) { + exchange.setProperty(Exchange.SPLIT_WATERMARK, currentWatermark); + } + } + + // wrap callback to build SplitResult and/or update watermark after all items are processed + boolean needsWrapping = hasErrorThreshold || hasWatermark; + AsyncCallback wrappedCallback = needsWrapping ? doneSync -> { - buildSplitResult(exchange); + if (hasErrorThreshold) { + buildSplitResult(exchange); + } + if (hasWatermark) { + updateWatermark(exchange); + } callback.done(doneSync); } : callback; @@ -191,6 +214,7 @@ private final class SplitterIterable implements Iterable, private Exchange copy; private final Route route; private final Exchange original; + private final int watermarkOffset; private SplitterIterable(Exchange exchange, Object value) { this.original = exchange; @@ -205,6 +229,24 @@ private SplitterIterable(Exchange exchange, Object value) { } else { rawIterator = ObjectHelper.createIterator(value, delimiter); } + + // index-based watermarking: skip items up to the stored watermark index + int skipCount = 0; + if (watermarkStore != null && watermarkKey != null && watermarkExpression == null) { + String storedIndex = watermarkStore.get(watermarkKey); + if (storedIndex != null) { + int skipTo = Integer.parseInt(storedIndex); + while (rawIterator.hasNext() && skipCount <= skipTo) { + rawIterator.next(); + skipCount++; + } + } + } + this.watermarkOffset = skipCount; + if (skipCount > 0) { + exchange.setProperty(SPLIT_WATERMARK_OFFSET, skipCount); + } + // wrap with GroupIterator if group > 0 to chunk items into List batches this.iterator = group > 0 ? new GroupIterator(rawIterator, group) : rawIterator; @@ -230,6 +272,10 @@ public boolean hasNext() { if (!answer) { // we are now closed closed = true; + // store item count for watermark tracking + if (watermarkStore != null && watermarkKey != null && watermarkExpression == null) { + original.setProperty(SPLIT_WATERMARK_COUNT, index); + } // nothing more so we need to close the expression value in case it needs to be try { close(); @@ -367,6 +413,30 @@ public void setMaxFailedRecords(int maxFailedRecords) { this.maxFailedRecords = maxFailedRecords; } + public Map getWatermarkStore() { + return watermarkStore; + } + + public void setWatermarkStore(Map watermarkStore) { + this.watermarkStore = watermarkStore; + } + + public String getWatermarkKey() { + return watermarkKey; + } + + public void setWatermarkKey(String watermarkKey) { + this.watermarkKey = watermarkKey; + } + + public Expression getWatermarkExpression() { + return watermarkExpression; + } + + public void setWatermarkExpression(Expression watermarkExpression) { + this.watermarkExpression = watermarkExpression; + } + @Override protected boolean shouldContinueOnFailure(Exchange subExchange, Exchange original, int index) { SplitFailureTracker tracker = original.getProperty(SPLIT_FAILURE_TRACKER, SplitFailureTracker.class); @@ -424,6 +494,33 @@ List getFailures() { } } + private void updateWatermark(Exchange exchange) { + // don't update watermark if processing was aborted (allows retry) + if (exchange.getException() != null) { + return; + } + + if (watermarkExpression != null) { + // value-based: evaluate expression on the exchange after split completion + String newValue = watermarkExpression.evaluate(exchange, String.class); + if (newValue != null) { + watermarkStore.put(watermarkKey, newValue); + } + } else { + // index-based: compute absolute last index and store it + int offset = exchange.getProperty(SPLIT_WATERMARK_OFFSET, 0, Integer.class); + Integer count = exchange.getProperty(SPLIT_WATERMARK_COUNT, Integer.class); + if (count != null && count > 0) { + int lastAbsoluteIndex = offset + count - 1; + watermarkStore.put(watermarkKey, String.valueOf(lastAbsoluteIndex)); + } + } + + // clean up internal properties + exchange.removeProperty(SPLIT_WATERMARK_OFFSET); + exchange.removeProperty(SPLIT_WATERMARK_COUNT); + } + private void buildSplitResult(Exchange exchange) { SplitFailureTracker tracker = exchange.getProperty(SPLIT_FAILURE_TRACKER, SplitFailureTracker.class); if (tracker == null) { diff --git a/core/camel-core-reifier/src/main/java/org/apache/camel/reifier/SplitReifier.java b/core/camel-core-reifier/src/main/java/org/apache/camel/reifier/SplitReifier.java index 49afb1a83427b..35c9273bef63f 100644 --- a/core/camel-core-reifier/src/main/java/org/apache/camel/reifier/SplitReifier.java +++ b/core/camel-core-reifier/src/main/java/org/apache/camel/reifier/SplitReifier.java @@ -16,6 +16,7 @@ */ package org.apache.camel.reifier; +import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.function.BiFunction; @@ -99,6 +100,22 @@ public Processor createProcessor() throws Exception { answer.setMaxFailedRecords(maxFailedRecords); } + @SuppressWarnings("unchecked") + Map watermarkStore = definition.getWatermarkStoreBean(); + if (watermarkStore == null && definition.getWatermarkStore() != null) { + watermarkStore = mandatoryLookup(definition.getWatermarkStore(), Map.class); + } + String watermarkKey = parseString(definition.getWatermarkKey()); + if (watermarkStore != null && watermarkKey != null) { + answer.setWatermarkStore(watermarkStore); + answer.setWatermarkKey(watermarkKey); + String watermarkExprStr = parseString(definition.getWatermarkExpression()); + if (watermarkExprStr != null) { + Expression watermarkExpr = camelContext.resolveLanguage("simple").createExpression(watermarkExprStr); + answer.setWatermarkExpression(watermarkExpr); + } + } + return answer; } diff --git a/core/camel-core/src/test/java/org/apache/camel/processor/SplitterWatermarkTest.java b/core/camel-core/src/test/java/org/apache/camel/processor/SplitterWatermarkTest.java new file mode 100644 index 0000000000000..53e678f401933 --- /dev/null +++ b/core/camel-core/src/test/java/org/apache/camel/processor/SplitterWatermarkTest.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.processor; + +import java.util.Arrays; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.camel.ContextTestSupport; +import org.apache.camel.Exchange; +import org.apache.camel.builder.RouteBuilder; +import org.apache.camel.component.mock.MockEndpoint; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +class SplitterWatermarkTest extends ContextTestSupport { + + private final Map store = new ConcurrentHashMap<>(); + + @Test + void testIndexBasedWatermarkFirstRun() throws Exception { + MockEndpoint mock = getMockEndpoint("mock:split"); + mock.expectedMessageCount(5); + + template.sendBody("direct:index", Arrays.asList("a", "b", "c", "d", "e")); + + mock.assertIsSatisfied(); + + // watermark should be stored as last index (4) + assertEquals("4", store.get("testJob")); + } + + @Test + void testIndexBasedWatermarkSecondRun() throws Exception { + // simulate a previous run that processed items 0-2 + store.put("testJob", "2"); + + MockEndpoint mock = getMockEndpoint("mock:split"); + mock.expectedMessageCount(2); // only items 3 and 4 + + template.sendBody("direct:index", Arrays.asList("a", "b", "c", "d", "e")); + + mock.assertIsSatisfied(); + + // bodies should be d, e (items at index 3 and 4) + assertEquals("d", mock.getReceivedExchanges().get(0).getIn().getBody(String.class)); + assertEquals("e", mock.getReceivedExchanges().get(1).getIn().getBody(String.class)); + + // watermark updated to 4 + assertEquals("4", store.get("testJob")); + } + + @Test + void testIndexBasedWatermarkNoUpdateOnAbort() throws Exception { + // watermark at 0, maxFailedRecords=1 + store.put("testJob2", "0"); + + MockEndpoint mock = getMockEndpoint("mock:split2"); + mock.expectedMinimumMessageCount(0); + + // items after skipping index 0: FAIL, c, d — FAIL triggers abort + template.send("direct:index-abort", + e -> e.getIn().setBody(Arrays.asList("a", "FAIL", "c", "d"))); + + mock.assertIsSatisfied(); + + // watermark should NOT be updated (still 0) + assertEquals("0", store.get("testJob2")); + } + + @Test + void testValueBasedWatermark() throws Exception { + MockEndpoint mock = getMockEndpoint("mock:split3"); + mock.expectedMessageCount(3); + + Exchange result = template.send("direct:value", + e -> e.getIn().setBody(Arrays.asList("2024-01-01", "2024-01-02", "2024-01-03"))); + + mock.assertIsSatisfied(); + + // no previous watermark, so SPLIT_WATERMARK should not be set + assertNull(result.getProperty(Exchange.SPLIT_WATERMARK)); + + // watermark expression evaluates ${body} on the aggregated exchange (last item) + assertEquals("2024-01-03", store.get("dateJob")); + } + + @Test + void testValueBasedWatermarkWithPreviousValue() throws Exception { + store.put("dateJob", "2024-01-01"); + + MockEndpoint mock = getMockEndpoint("mock:split3"); + mock.expectedMessageCount(3); + + Exchange result = template.send("direct:value", + e -> e.getIn().setBody(Arrays.asList("2024-01-02", "2024-01-03", "2024-01-04"))); + + mock.assertIsSatisfied(); + + // previous watermark should be exposed as exchange property + assertEquals("2024-01-01", result.getProperty(Exchange.SPLIT_WATERMARK)); + + // watermark updated to the last processed date + assertEquals("2024-01-04", store.get("dateJob")); + } + + @Override + protected RouteBuilder createRouteBuilder() { + return new RouteBuilder() { + @Override + public void configure() { + from("direct:index") + .split(body()).watermarkStore(store, "testJob") + .to("mock:split"); + + from("direct:index-abort") + .split(body()).watermarkStore(store, "testJob2").maxFailedRecords(1) + .process(exchange -> { + String body = exchange.getIn().getBody(String.class); + if ("FAIL".equals(body)) { + throw new IllegalArgumentException("Simulated failure"); + } + }) + .to("mock:split2"); + + from("direct:value") + .split(body()) + // use aggregation strategy that keeps the last exchange, so ${body} reflects the last item + .aggregationStrategy((oldExchange, newExchange) -> newExchange) + .watermarkStore(store, "dateJob") + .watermarkExpression("${body}") + .to("mock:split3"); + } + }; + } +} diff --git a/dsl/camel-yaml-dsl/camel-yaml-dsl-deserializers/src/generated/java/org/apache/camel/dsl/yaml/deserializers/ModelDeserializers.java b/dsl/camel-yaml-dsl/camel-yaml-dsl-deserializers/src/generated/java/org/apache/camel/dsl/yaml/deserializers/ModelDeserializers.java index 4f35d28c62dda..56a976cfde849 100644 --- a/dsl/camel-yaml-dsl/camel-yaml-dsl-deserializers/src/generated/java/org/apache/camel/dsl/yaml/deserializers/ModelDeserializers.java +++ b/dsl/camel-yaml-dsl/camel-yaml-dsl-deserializers/src/generated/java/org/apache/camel/dsl/yaml/deserializers/ModelDeserializers.java @@ -16848,7 +16848,10 @@ protected boolean setProperty(SpELExpression target, String propertyKey, @YamlProperty(name = "stopOnException", type = "boolean", defaultValue = "false", description = "Will now stop further processing if an exception or failure occurred during processing of an org.apache.camel.Exchange and the caused exception will be thrown. Will also stop if processing the exchange failed (has a fault message) or an exception was thrown and handled by the error handler (such as using onException). In all situations the splitter will stop further processing. This is the same behavior as in pipeline, which is used by the routing engine. The default behavior is to not stop but continue processing till the end", displayName = "Stop On Exception"), @YamlProperty(name = "streaming", type = "boolean", defaultValue = "false", description = "When in streaming mode, then the splitter splits the original message on-demand, and each split message is processed one by one. This reduces memory usage as the splitter do not split all the messages first, but then we do not know the total size, and therefore the org.apache.camel.Exchange#SPLIT_SIZE is empty. In non-streaming mode (default) the splitter will split each message first, to know the total size, and then process each message one by one. This requires to keep all the split messages in memory and therefore requires more memory. The total size is provided in the org.apache.camel.Exchange#SPLIT_SIZE header. The streaming mode also affects the aggregation behavior. If enabled then Camel will process replies out-of-order, e.g. in the order they come back. If disabled, Camel will process replies in the same order as the messages was split.", displayName = "Streaming"), @YamlProperty(name = "synchronous", type = "boolean", defaultValue = "false", description = "Sets whether synchronous processing should be strictly used. When enabled then the same thread is used to continue routing after the split is complete, even if parallel processing is enabled.", displayName = "Synchronous"), - @YamlProperty(name = "timeout", type = "string", defaultValue = "0", description = "Sets a total timeout specified in millis, when using parallel processing. If the Splitter hasn't been able to send and process all replies within the given timeframe, then the timeout triggers and the Splitter breaks out and continues. The timeout method is invoked before breaking out. If the timeout is reached with running tasks still remaining, certain tasks for which it is difficult for Camel to shut down in a graceful manner may continue to run. So use this option with a bit of care.", displayName = "Timeout") + @YamlProperty(name = "timeout", type = "string", defaultValue = "0", description = "Sets a total timeout specified in millis, when using parallel processing. If the Splitter hasn't been able to send and process all replies within the given timeframe, then the timeout triggers and the Splitter breaks out and continues. The timeout method is invoked before breaking out. If the timeout is reached with running tasks still remaining, certain tasks for which it is difficult for Camel to shut down in a graceful manner may continue to run. So use this option with a bit of care.", displayName = "Timeout"), + @YamlProperty(name = "watermarkExpression", type = "string"), + @YamlProperty(name = "watermarkKey", type = "string"), + @YamlProperty(name = "watermarkStore", type = "string") } ) public static class SplitDefinitionDeserializer extends YamlDeserializerBase { @@ -16956,6 +16959,21 @@ protected boolean setProperty(SplitDefinition target, String propertyKey, target.setTimeout(val); break; } + case "watermarkExpression": { + String val = asText(node); + target.setWatermarkExpression(val); + break; + } + case "watermarkKey": { + String val = asText(node); + target.setWatermarkKey(val); + break; + } + case "watermarkStore": { + String val = asText(node); + target.setWatermarkStore(val); + break; + } case "id": { String val = asText(node); target.setId(val); From 5abcdcf799de67602a3bfb3b33d5993ae2b1a7cc Mon Sep 17 00:00:00 2001 From: Guillaume Nodet Date: Fri, 27 Mar 2026 20:54:53 +0100 Subject: [PATCH 3/9] CAMEL-23264: Improve value-based watermark and add transacted tests - Evaluate watermark expression per-item via ProcessorExchangePair.done() hook instead of requiring a custom aggregation strategy - Add WatermarkProcessorExchangePair wrapper for thread-safe per-item watermark tracking using AtomicReference - Add SplitterTransactedTest with 6 tests covering group, error threshold, split result, and watermark features using the MulticastTransactedTask code path Co-Authored-By: Claude Opus 4.6 --- .../org/apache/camel/processor/Splitter.java | 86 ++++++- .../processor/SplitterTransactedTest.java | 211 ++++++++++++++++++ .../processor/SplitterWatermarkTest.java | 4 +- 3 files changed, 294 insertions(+), 7 deletions(-) create mode 100644 core/camel-core/src/test/java/org/apache/camel/processor/SplitterTransactedTest.java diff --git a/core/camel-core-processor/src/main/java/org/apache/camel/processor/Splitter.java b/core/camel-core-processor/src/main/java/org/apache/camel/processor/Splitter.java index 5287e5ba0c1f1..8bdd3e929e156 100644 --- a/core/camel-core-processor/src/main/java/org/apache/camel/processor/Splitter.java +++ b/core/camel-core-processor/src/main/java/org/apache/camel/processor/Splitter.java @@ -28,6 +28,7 @@ import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ExecutorService; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import org.apache.camel.AggregationStrategy; import org.apache.camel.AsyncCallback; @@ -61,6 +62,7 @@ public class Splitter extends MulticastProcessor { private static final String SPLIT_FAILURE_TRACKER = "CamelSplitFailureTracker"; private static final String SPLIT_WATERMARK_OFFSET = "CamelSplitWatermarkOffset"; private static final String SPLIT_WATERMARK_COUNT = "CamelSplitWatermarkCount"; + private static final String SPLIT_WATERMARK_LATEST = "CamelSplitWatermarkLatest"; private final Expression expression; private final String delimiter; private int group; @@ -152,6 +154,10 @@ public boolean process(Exchange exchange, final AsyncCallback callback) { if (currentWatermark != null) { exchange.setProperty(Exchange.SPLIT_WATERMARK, currentWatermark); } + // pre-initialize AtomicReference for value-based watermark tracking (thread-safe) + if (watermarkExpression != null) { + exchange.setProperty(SPLIT_WATERMARK_LATEST, new AtomicReference()); + } } // wrap callback to build SplitResult and/or update watermark after all items are processed @@ -171,6 +177,17 @@ public boolean process(Exchange exchange, final AsyncCallback callback) { return super.process(exchange, wrappedCallback); } + @Override + protected ProcessorExchangePair createProcessorExchangePair( + int index, Processor processor, Exchange exchange, Route route) { + ProcessorExchangePair pair = super.createProcessorExchangePair(index, processor, exchange, route); + // wrap to evaluate watermark expression on each completed sub-exchange + if (watermarkExpression != null) { + return new WatermarkProcessorExchangePair(pair, exchange); + } + return pair; + } + @Override protected Iterable createProcessorExchangePairs(Exchange exchange) throws Exception { @@ -494,6 +511,63 @@ List getFailures() { } } + @SuppressWarnings("unchecked") + /** + * Wraps a {@link ProcessorExchangePair} to evaluate the watermark expression on each completed sub-exchange. + */ + private final class WatermarkProcessorExchangePair implements ProcessorExchangePair { + private final ProcessorExchangePair delegate; + private final Exchange original; + + WatermarkProcessorExchangePair(ProcessorExchangePair delegate, Exchange original) { + this.delegate = delegate; + this.original = original; + } + + @Override + public int getIndex() { + return delegate.getIndex(); + } + + @Override + public Exchange getExchange() { + return delegate.getExchange(); + } + + @Override + public org.apache.camel.Producer getProducer() { + return delegate.getProducer(); + } + + @Override + public Processor getProcessor() { + return delegate.getProcessor(); + } + + @Override + public void begin() { + delegate.begin(); + } + + @Override + @SuppressWarnings("unchecked") + public void done() { + delegate.done(); + // evaluate watermark expression on completed sub-exchange (only if no exception) + Exchange subExchange = delegate.getExchange(); + if (subExchange.getException() == null) { + String value = watermarkExpression.evaluate(subExchange, String.class); + if (value != null) { + // AtomicReference is pre-initialized in process() — safe for parallel use + AtomicReference latestRef + = (AtomicReference) original.getProperty(SPLIT_WATERMARK_LATEST); + latestRef.set(value); + } + } + } + } + + @SuppressWarnings("unchecked") private void updateWatermark(Exchange exchange) { // don't update watermark if processing was aborted (allows retry) if (exchange.getException() != null) { @@ -501,10 +575,13 @@ private void updateWatermark(Exchange exchange) { } if (watermarkExpression != null) { - // value-based: evaluate expression on the exchange after split completion - String newValue = watermarkExpression.evaluate(exchange, String.class); - if (newValue != null) { - watermarkStore.put(watermarkKey, newValue); + // value-based: use the latest value tracked per-item during processing + AtomicReference latestRef = exchange.getProperty(SPLIT_WATERMARK_LATEST, AtomicReference.class); + if (latestRef != null) { + String newValue = latestRef.get(); + if (newValue != null) { + watermarkStore.put(watermarkKey, newValue); + } } } else { // index-based: compute absolute last index and store it @@ -519,6 +596,7 @@ private void updateWatermark(Exchange exchange) { // clean up internal properties exchange.removeProperty(SPLIT_WATERMARK_OFFSET); exchange.removeProperty(SPLIT_WATERMARK_COUNT); + exchange.removeProperty(SPLIT_WATERMARK_LATEST); } private void buildSplitResult(Exchange exchange) { diff --git a/core/camel-core/src/test/java/org/apache/camel/processor/SplitterTransactedTest.java b/core/camel-core/src/test/java/org/apache/camel/processor/SplitterTransactedTest.java new file mode 100644 index 0000000000000..1bb78546bd9c8 --- /dev/null +++ b/core/camel-core/src/test/java/org/apache/camel/processor/SplitterTransactedTest.java @@ -0,0 +1,211 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.processor; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.camel.ContextTestSupport; +import org.apache.camel.Exchange; +import org.apache.camel.SplitResult; +import org.apache.camel.builder.RouteBuilder; +import org.apache.camel.component.mock.MockEndpoint; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests for Splitter EIP enhancements (group, error threshold, watermark) using the transacted code path + * ({@link MulticastProcessor.MulticastTransactedTask}). + */ +class SplitterTransactedTest extends ContextTestSupport { + + private final Map store = new ConcurrentHashMap<>(); + + @Test + void testGroupTransacted() throws Exception { + MockEndpoint mock = getMockEndpoint("mock:group"); + mock.expectedMessageCount(3); + + Exchange exchange = createExchangeWithBody(Arrays.asList("a", "b", "c", "d", "e", "f", "g")); + exchange.getExchangeExtension().setTransacted(true); + + template.send("direct:group", exchange); + + mock.assertIsSatisfied(); + + // verify chunks + assertEquals(List.of("a", "b", "c"), mock.getReceivedExchanges().get(0).getIn().getBody()); + assertEquals(List.of("d", "e", "f"), mock.getReceivedExchanges().get(1).getIn().getBody()); + assertEquals(List.of("g"), mock.getReceivedExchanges().get(2).getIn().getBody()); + } + + @Test + void testMaxFailedRecordsTransacted() throws Exception { + MockEndpoint mock = getMockEndpoint("mock:maxfail"); + mock.expectedMinimumMessageCount(0); + + Exchange exchange = createExchangeWithBody(Arrays.asList("ok", "FAIL", "ok2", "FAIL2", "ok3")); + exchange.getExchangeExtension().setTransacted(true); + + Exchange result = template.send("direct:maxfail", exchange); + + mock.assertIsSatisfied(); + + SplitResult splitResult = result.getProperty(Exchange.SPLIT_RESULT, SplitResult.class); + assertNotNull(splitResult); + assertEquals(2, splitResult.getFailureCount()); + assertTrue(splitResult.isAborted()); + } + + @Test + void testErrorThresholdTransacted() throws Exception { + MockEndpoint mock = getMockEndpoint("mock:threshold"); + mock.expectedMinimumMessageCount(0); + + // 2 out of 4 fail = 50% error rate, threshold is 0.4 (40%) + Exchange exchange = createExchangeWithBody(Arrays.asList("ok", "FAIL", "FAIL2", "ok2")); + exchange.getExchangeExtension().setTransacted(true); + + Exchange result = template.send("direct:threshold", exchange); + + mock.assertIsSatisfied(); + + SplitResult splitResult = result.getProperty(Exchange.SPLIT_RESULT, SplitResult.class); + assertNotNull(splitResult); + assertTrue(splitResult.isAborted()); + } + + @Test + void testSplitResultTransacted() throws Exception { + MockEndpoint mock = getMockEndpoint("mock:result"); + mock.expectedMessageCount(3); + + Exchange exchange = createExchangeWithBody(Arrays.asList("ok", "FAIL", "ok2", "ok3")); + exchange.getExchangeExtension().setTransacted(true); + + Exchange result = template.send("direct:result", exchange); + + mock.assertIsSatisfied(); + + SplitResult splitResult = result.getProperty(Exchange.SPLIT_RESULT, SplitResult.class); + assertNotNull(splitResult); + assertEquals(4, splitResult.getTotalItems()); + assertEquals(1, splitResult.getFailureCount()); + assertEquals(3, splitResult.getSuccessCount()); + assertFalse(splitResult.isAborted()); + } + + @Test + void testIndexWatermarkTransacted() throws Exception { + store.put("txJob", "1"); + + MockEndpoint mock = getMockEndpoint("mock:watermark"); + mock.expectedMessageCount(3); + + Exchange exchange = createExchangeWithBody(Arrays.asList("a", "b", "c", "d", "e")); + exchange.getExchangeExtension().setTransacted(true); + + Exchange result = template.send("direct:watermark", exchange); + + mock.assertIsSatisfied(); + + // should skip items 0 and 1, process c, d, e + assertEquals("c", mock.getReceivedExchanges().get(0).getIn().getBody(String.class)); + assertEquals("d", mock.getReceivedExchanges().get(1).getIn().getBody(String.class)); + assertEquals("e", mock.getReceivedExchanges().get(2).getIn().getBody(String.class)); + + // watermark updated to 4 + assertEquals("4", store.get("txJob")); + + // previous watermark exposed + assertEquals("1", result.getProperty(Exchange.SPLIT_WATERMARK)); + } + + @Test + void testValueWatermarkTransacted() throws Exception { + MockEndpoint mock = getMockEndpoint("mock:valuewm"); + mock.expectedMessageCount(3); + + Exchange exchange = createExchangeWithBody(Arrays.asList("2024-01-01", "2024-01-02", "2024-01-03")); + exchange.getExchangeExtension().setTransacted(true); + + template.send("direct:valuewm", exchange); + + mock.assertIsSatisfied(); + + // watermark should be the last processed item + assertEquals("2024-01-03", store.get("txDateJob")); + } + + @Override + protected RouteBuilder createRouteBuilder() { + return new RouteBuilder() { + @Override + public void configure() { + from("direct:group") + .split(body()).group(3) + .to("mock:group"); + + from("direct:maxfail") + .split(body()).maxFailedRecords(2) + .process(exchange -> { + String body = exchange.getIn().getBody(String.class); + if (body.startsWith("FAIL")) { + throw new IllegalArgumentException("Simulated failure: " + body); + } + }) + .to("mock:maxfail"); + + from("direct:threshold") + .split(body()).errorThreshold(0.4) + .process(exchange -> { + String body = exchange.getIn().getBody(String.class); + if (body.startsWith("FAIL")) { + throw new IllegalArgumentException("Simulated failure: " + body); + } + }) + .to("mock:threshold"); + + from("direct:result") + .split(body()).maxFailedRecords(10) + .process(exchange -> { + String body = exchange.getIn().getBody(String.class); + if (body.startsWith("FAIL")) { + throw new IllegalArgumentException("Simulated failure: " + body); + } + }) + .to("mock:result"); + + from("direct:watermark") + .split(body()).watermarkStore(store, "txJob") + .to("mock:watermark"); + + from("direct:valuewm") + .split(body()) + .watermarkStore(store, "txDateJob") + .watermarkExpression("${body}") + .to("mock:valuewm"); + } + }; + } +} diff --git a/core/camel-core/src/test/java/org/apache/camel/processor/SplitterWatermarkTest.java b/core/camel-core/src/test/java/org/apache/camel/processor/SplitterWatermarkTest.java index 53e678f401933..81fee77d43a1f 100644 --- a/core/camel-core/src/test/java/org/apache/camel/processor/SplitterWatermarkTest.java +++ b/core/camel-core/src/test/java/org/apache/camel/processor/SplitterWatermarkTest.java @@ -97,7 +97,7 @@ void testValueBasedWatermark() throws Exception { // no previous watermark, so SPLIT_WATERMARK should not be set assertNull(result.getProperty(Exchange.SPLIT_WATERMARK)); - // watermark expression evaluates ${body} on the aggregated exchange (last item) + // watermark expression evaluated per-item: last successful item's body is stored assertEquals("2024-01-03", store.get("dateJob")); } @@ -141,8 +141,6 @@ public void configure() { from("direct:value") .split(body()) - // use aggregation strategy that keeps the last exchange, so ${body} reflects the last item - .aggregationStrategy((oldExchange, newExchange) -> newExchange) .watermarkStore(store, "dateJob") .watermarkExpression("${body}") .to("mock:split3"); From 9e62f8080bfda84dd7b76c235f54ce2461e8d266 Mon Sep 17 00:00:00 2001 From: Guillaume Nodet Date: Fri, 27 Mar 2026 21:24:03 +0100 Subject: [PATCH 4/9] CAMEL-23264: Regenerate downstream artifacts for Splitter EIP changes Regenerated: catalog split.json, Spring/XML-IO XSD schemas, XML/YAML ModelParser/ModelWriter, YAML DSL ModelDeserializers. Co-Authored-By: Claude Opus 4.6 --- .../apache/camel/catalog/models/split.json | 12 +++- .../camel/catalog/schemas/camel-spring.xsd | 55 +++++++++++++++++++ .../camel/catalog/schemas/camel-xml-io.xsd | 55 +++++++++++++++++++ .../org/apache/camel/xml/in/ModelParser.java | 6 ++ .../org/apache/camel/xml/out/ModelWriter.java | 6 ++ .../apache/camel/yaml/out/ModelWriter.java | 6 ++ .../deserializers/ModelDeserializers.java | 12 ++-- 7 files changed, 144 insertions(+), 8 deletions(-) diff --git a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/models/split.json b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/models/split.json index 8c51d8a7afe80..0f7987150865b 100644 --- a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/models/split.json +++ b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/models/split.json @@ -30,11 +30,19 @@ "executorService": { "index": 15, "kind": "attribute", "displayName": "Executor Service", "group": "advanced", "label": "advanced", "required": false, "type": "object", "javaType": "java.util.concurrent.ExecutorService", "deprecated": false, "autowired": false, "secret": false, "description": "To use a custom Thread Pool to be used for parallel processing. Notice if you set this option, then parallel processing is automatically implied, and you do not have to enable that option as well." }, "onPrepare": { "index": 16, "kind": "attribute", "displayName": "On Prepare", "group": "advanced", "label": "advanced", "required": false, "type": "object", "javaType": "org.apache.camel.Processor", "deprecated": false, "autowired": false, "secret": false, "description": "Uses the Processor when preparing the org.apache.camel.Exchange to be sent. This can be used to deep-clone messages that should be sent, or any custom logic needed before the exchange is sent." }, "shareUnitOfWork": { "index": 17, "kind": "attribute", "displayName": "Share Unit Of Work", "group": "advanced", "label": "advanced", "required": false, "type": "boolean", "javaType": "java.lang.Boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "description": "Shares the org.apache.camel.spi.UnitOfWork with the parent and each of the sub messages. Splitter will by default not share unit of work between the parent exchange and each split exchange. This means each split exchange has its own individual unit of work." }, - "outputs": { "index": 18, "kind": "element", "displayName": "Outputs", "group": "common", "required": true, "type": "array", "javaType": "java.util.List>", "oneOf": [ "aggregate", "bean", "choice", "circuitBreaker", "claimCheck", "convertBodyTo", "convertHeaderTo", "convertVariableTo", "delay", "doCatch", "doFinally", "doTry", "dynamicRouter", "enrich", "filter", "idempotentConsumer", "intercept", "interceptFrom", "interceptSendToEndpoint", "kamelet", "loadBalance", "log", "loop", "marshal", "multicast", "onCompletion", "onException", "pausable", "pipeline", "policy", "poll", "pollEnrich", "process", "recipientList", "removeHeader", "removeHeaders", "removeProperties", "removeProperty", "removeVariable", "resequence", "resumable", "rollback", "routingSlip", "saga", "sample", "script", "setBody", "setExchangePattern", "setHeader", "setHeaders", "setProperty", "setVariable", "setVariables", "sort", "split", "step", "stop", "threads", "throttle", "throwException", "to", "toD", "tokenizer", "transacted", "transform", "transformDataType", "unmarshal", "validate", "wireTap" ], "deprecated": false, "autowired": false, "secret": false } + "group": { "index": 18, "kind": "attribute", "displayName": "Group", "group": "advanced", "label": "advanced", "required": false, "type": "integer", "javaType": "java.lang.Integer", "deprecated": false, "autowired": false, "secret": false, "description": "Groups N split messages into a single message with a java.util.List body. This allows processing items in chunks instead of one at a time." }, + "errorThreshold": { "index": 19, "kind": "attribute", "displayName": "Error Threshold", "group": "advanced", "label": "advanced", "required": false, "type": "number", "javaType": "java.lang.Double", "deprecated": false, "autowired": false, "secret": false, "description": "Sets the error threshold as a fraction (0.0-1.0) of failed items before aborting the split operation." }, + "maxFailedRecords": { "index": 20, "kind": "attribute", "displayName": "Max Failed Records", "group": "advanced", "label": "advanced", "required": false, "type": "integer", "javaType": "java.lang.Integer", "deprecated": false, "autowired": false, "secret": false, "description": "Sets the maximum number of failed records before aborting the split operation." }, + "watermarkStore": { "index": 21, "kind": "attribute", "displayName": "Watermark Store", "group": "advanced", "label": "advanced", "required": false, "type": "object", "javaType": "java.util.Map", "deprecated": false, "autowired": false, "secret": false, "description": "Sets a reference to a watermark store (a Map ) in the registry." }, + "watermarkKey": { "index": 22, "kind": "attribute", "displayName": "Watermark Key", "group": "advanced", "label": "advanced", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "description": "Sets the key to use in the watermark store." }, + "watermarkExpression": { "index": 23, "kind": "attribute", "displayName": "Watermark Expression", "group": "advanced", "label": "advanced", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "description": "Sets a Simple expression to evaluate on the exchange after split completion to determine the new watermark value." }, + "outputs": { "index": 24, "kind": "element", "displayName": "Outputs", "group": "common", "required": true, "type": "array", "javaType": "java.util.List>", "oneOf": [ "aggregate", "bean", "choice", "circuitBreaker", "claimCheck", "convertBodyTo", "convertHeaderTo", "convertVariableTo", "delay", "doCatch", "doFinally", "doTry", "dynamicRouter", "enrich", "filter", "idempotentConsumer", "intercept", "interceptFrom", "interceptSendToEndpoint", "kamelet", "loadBalance", "log", "loop", "marshal", "multicast", "onCompletion", "onException", "pausable", "pipeline", "policy", "poll", "pollEnrich", "process", "recipientList", "removeHeader", "removeHeaders", "removeProperties", "removeProperty", "removeVariable", "resequence", "resumable", "rollback", "routingSlip", "saga", "sample", "script", "setBody", "setExchangePattern", "setHeader", "setHeaders", "setProperty", "setVariable", "setVariables", "sort", "split", "step", "stop", "threads", "throttle", "throwException", "to", "toD", "tokenizer", "transacted", "transform", "transformDataType", "unmarshal", "validate", "wireTap" ], "deprecated": false, "autowired": false, "secret": false } }, "exchangeProperties": { "CamelSplitIndex": { "index": 0, "kind": "exchangeProperty", "displayName": "Split Index", "label": "producer", "required": false, "javaType": "int", "deprecated": false, "autowired": false, "secret": false, "important": true, "description": "A split counter that increases for each Exchange being split. The counter starts from 0." }, "CamelSplitComplete": { "index": 1, "kind": "exchangeProperty", "displayName": "Split Complete", "label": "producer", "required": false, "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "description": "Whether this Exchange is the last." }, - "CamelSplitSize": { "index": 2, "kind": "exchangeProperty", "displayName": "Split Size", "label": "producer", "required": false, "javaType": "int", "deprecated": false, "autowired": false, "secret": false, "important": true, "description": "The total number of Exchanges that was split. This property is not applied for stream based splitting, except for the very last message because then Camel knows the total size." } + "CamelSplitSize": { "index": 2, "kind": "exchangeProperty", "displayName": "Split Size", "label": "producer", "required": false, "javaType": "int", "deprecated": false, "autowired": false, "secret": false, "important": true, "description": "The total number of Exchanges that was split. This property is not applied for stream based splitting, except for the very last message because then Camel knows the total size." }, + "CamelSplitResult": { "index": 3, "kind": "exchangeProperty", "displayName": "Split Result", "label": "producer", "required": false, "javaType": "org.apache.camel.SplitResult", "deprecated": false, "autowired": false, "secret": false, "description": "The result of a Splitter EIP operation with error thresholds, providing structured failure details." }, + "CamelSplitWatermark": { "index": 4, "kind": "exchangeProperty", "displayName": "Split Watermark", "label": "producer", "required": false, "javaType": "String", "deprecated": false, "autowired": false, "secret": false, "description": "The current watermark value from the watermark store, set before split processing begins." } } } diff --git a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/schemas/camel-spring.xsd b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/schemas/camel-spring.xsd index 5e6156580ce51..49d161bc57c43 100644 --- a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/schemas/camel-spring.xsd +++ b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/schemas/camel-spring.xsd @@ -13952,6 +13952,61 @@ should be sent, or any custom logic needed before the exchange is sent. Shares the org.apache.camel.spi.UnitOfWork with the parent and each of the sub messages. Splitter will by default not share unit of work between the parent exchange and each split exchange. This means each split exchange has its own individual unit of work. Default value: false +]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/schemas/camel-xml-io.xsd b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/schemas/camel-xml-io.xsd index 17e0bed614ed7..0313bc934413d 100644 --- a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/schemas/camel-xml-io.xsd +++ b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/schemas/camel-xml-io.xsd @@ -12636,6 +12636,61 @@ should be sent, or any custom logic needed before the exchange is sent. Shares the org.apache.camel.spi.UnitOfWork with the parent and each of the sub messages. Splitter will by default not share unit of work between the parent exchange and each split exchange. This means each split exchange has its own individual unit of work. Default value: false +]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/core/camel-xml-io/src/generated/java/org/apache/camel/xml/in/ModelParser.java b/core/camel-xml-io/src/generated/java/org/apache/camel/xml/in/ModelParser.java index 4f8089312fcde..7b535a5df8c78 100644 --- a/core/camel-xml-io/src/generated/java/org/apache/camel/xml/in/ModelParser.java +++ b/core/camel-xml-io/src/generated/java/org/apache/camel/xml/in/ModelParser.java @@ -1133,7 +1133,10 @@ protected SplitDefinition doParseSplitDefinition() throws IOException, XmlPullPa case "aggregationStrategyMethodAllowNull": def.setAggregationStrategyMethodAllowNull(val); yield true; case "aggregationStrategyMethodName": def.setAggregationStrategyMethodName(val); yield true; case "delimiter": def.setDelimiter(val); yield true; + case "errorThreshold": def.setErrorThreshold(val); yield true; case "executorService": def.setExecutorService(val); yield true; + case "group": def.setGroup(val); yield true; + case "maxFailedRecords": def.setMaxFailedRecords(val); yield true; case "onPrepare": def.setOnPrepare(val); yield true; case "parallelAggregate": def.setParallelAggregate(val); yield true; case "parallelProcessing": def.setParallelProcessing(val); yield true; @@ -1142,6 +1145,9 @@ protected SplitDefinition doParseSplitDefinition() throws IOException, XmlPullPa case "streaming": def.setStreaming(val); yield true; case "synchronous": def.setSynchronous(val); yield true; case "timeout": def.setTimeout(val); yield true; + case "watermarkExpression": def.setWatermarkExpression(val); yield true; + case "watermarkKey": def.setWatermarkKey(val); yield true; + case "watermarkStore": def.setWatermarkStore(val); yield true; default: yield processorDefinitionAttributeHandler().accept(def, key, val); }, outputExpressionNodeElementHandler(), noValueHandler()); } diff --git a/core/camel-xml-io/src/generated/java/org/apache/camel/xml/out/ModelWriter.java b/core/camel-xml-io/src/generated/java/org/apache/camel/xml/out/ModelWriter.java index 8b43c453afae1..3df1aa815b709 100644 --- a/core/camel-xml-io/src/generated/java/org/apache/camel/xml/out/ModelWriter.java +++ b/core/camel-xml-io/src/generated/java/org/apache/camel/xml/out/ModelWriter.java @@ -1758,6 +1758,12 @@ protected void doWriteSplitDefinition(String name, SplitDefinition def) throws I doWriteAttribute("executorService", def.getExecutorService(), null); doWriteAttribute("onPrepare", def.getOnPrepare(), null); doWriteAttribute("shareUnitOfWork", def.getShareUnitOfWork(), null); + doWriteAttribute("group", def.getGroup(), null); + doWriteAttribute("errorThreshold", def.getErrorThreshold(), null); + doWriteAttribute("maxFailedRecords", def.getMaxFailedRecords(), null); + doWriteAttribute("watermarkStore", def.getWatermarkStore(), null); + doWriteAttribute("watermarkKey", def.getWatermarkKey(), null); + doWriteAttribute("watermarkExpression", def.getWatermarkExpression(), null); doWriteOutputExpressionNodeElements(def); endElement(name); } diff --git a/core/camel-yaml-io/src/generated/java/org/apache/camel/yaml/out/ModelWriter.java b/core/camel-yaml-io/src/generated/java/org/apache/camel/yaml/out/ModelWriter.java index aab9ec7671d58..57f735fc5360a 100644 --- a/core/camel-yaml-io/src/generated/java/org/apache/camel/yaml/out/ModelWriter.java +++ b/core/camel-yaml-io/src/generated/java/org/apache/camel/yaml/out/ModelWriter.java @@ -1758,6 +1758,12 @@ protected void doWriteSplitDefinition(String name, SplitDefinition def) throws I doWriteAttribute("executorService", def.getExecutorService(), null); doWriteAttribute("onPrepare", def.getOnPrepare(), null); doWriteAttribute("shareUnitOfWork", def.getShareUnitOfWork(), null); + doWriteAttribute("group", def.getGroup(), null); + doWriteAttribute("errorThreshold", def.getErrorThreshold(), null); + doWriteAttribute("maxFailedRecords", def.getMaxFailedRecords(), null); + doWriteAttribute("watermarkStore", def.getWatermarkStore(), null); + doWriteAttribute("watermarkKey", def.getWatermarkKey(), null); + doWriteAttribute("watermarkExpression", def.getWatermarkExpression(), null); doWriteOutputExpressionNodeElements(def); endElement(name); } diff --git a/dsl/camel-yaml-dsl/camel-yaml-dsl-deserializers/src/generated/java/org/apache/camel/dsl/yaml/deserializers/ModelDeserializers.java b/dsl/camel-yaml-dsl/camel-yaml-dsl-deserializers/src/generated/java/org/apache/camel/dsl/yaml/deserializers/ModelDeserializers.java index 56a976cfde849..8109b3440f0b2 100644 --- a/dsl/camel-yaml-dsl/camel-yaml-dsl-deserializers/src/generated/java/org/apache/camel/dsl/yaml/deserializers/ModelDeserializers.java +++ b/dsl/camel-yaml-dsl/camel-yaml-dsl-deserializers/src/generated/java/org/apache/camel/dsl/yaml/deserializers/ModelDeserializers.java @@ -16833,12 +16833,12 @@ protected boolean setProperty(SpELExpression target, String propertyKey, @YamlProperty(name = "delimiter", type = "string", defaultValue = ",", description = "Delimiter used in splitting messages. Can be turned off using the value false. To force not splitting then the delimiter can be set to single to use the value as a single list, this can be needed in some special situations. The default value is comma.", displayName = "Delimiter"), @YamlProperty(name = "description", type = "string", description = "Sets the description of this node", displayName = "Description"), @YamlProperty(name = "disabled", type = "boolean", defaultValue = "false", description = "Disables this EIP from the route.", displayName = "Disabled"), - @YamlProperty(name = "errorThreshold", type = "number"), + @YamlProperty(name = "errorThreshold", type = "number", description = "Sets the error threshold as a fraction (0.0-1.0) of failed items before aborting the split operation.", displayName = "Error Threshold"), @YamlProperty(name = "executorService", type = "string", description = "To use a custom Thread Pool to be used for parallel processing. Notice if you set this option, then parallel processing is automatically implied, and you do not have to enable that option as well.", displayName = "Executor Service"), @YamlProperty(name = "expression", type = "object:org.apache.camel.model.language.ExpressionDefinition", description = "Expression of how to split the message body, such as as-is, using a tokenizer, or using a xpath.", displayName = "Expression", oneOf = "expression"), - @YamlProperty(name = "group", type = "number"), + @YamlProperty(name = "group", type = "number", description = "Groups N split messages into a single message with a java.util.List body. This allows processing items in chunks instead of one at a time.", displayName = "Group"), @YamlProperty(name = "id", type = "string", description = "Sets the id of this node", displayName = "Id"), - @YamlProperty(name = "maxFailedRecords", type = "number"), + @YamlProperty(name = "maxFailedRecords", type = "number", description = "Sets the maximum number of failed records before aborting the split operation.", displayName = "Max Failed Records"), @YamlProperty(name = "note", type = "string", description = "Sets the note of this node", displayName = "Note"), @YamlProperty(name = "onPrepare", type = "string", description = "Uses the Processor when preparing the org.apache.camel.Exchange to be sent. This can be used to deep-clone messages that should be sent, or any custom logic needed before the exchange is sent.", displayName = "On Prepare"), @YamlProperty(name = "parallelAggregate", type = "boolean", deprecated = true, defaultValue = "false", description = "If enabled then the aggregate method on AggregationStrategy can be called concurrently. Notice that this would require the implementation of AggregationStrategy to be implemented as thread-safe. By default this is false meaning that Camel synchronizes the call to the aggregate method. Though in some use-cases this can be used to archive higher performance when the AggregationStrategy is implemented as thread-safe.", displayName = "Parallel Aggregate"), @@ -16849,9 +16849,9 @@ protected boolean setProperty(SpELExpression target, String propertyKey, @YamlProperty(name = "streaming", type = "boolean", defaultValue = "false", description = "When in streaming mode, then the splitter splits the original message on-demand, and each split message is processed one by one. This reduces memory usage as the splitter do not split all the messages first, but then we do not know the total size, and therefore the org.apache.camel.Exchange#SPLIT_SIZE is empty. In non-streaming mode (default) the splitter will split each message first, to know the total size, and then process each message one by one. This requires to keep all the split messages in memory and therefore requires more memory. The total size is provided in the org.apache.camel.Exchange#SPLIT_SIZE header. The streaming mode also affects the aggregation behavior. If enabled then Camel will process replies out-of-order, e.g. in the order they come back. If disabled, Camel will process replies in the same order as the messages was split.", displayName = "Streaming"), @YamlProperty(name = "synchronous", type = "boolean", defaultValue = "false", description = "Sets whether synchronous processing should be strictly used. When enabled then the same thread is used to continue routing after the split is complete, even if parallel processing is enabled.", displayName = "Synchronous"), @YamlProperty(name = "timeout", type = "string", defaultValue = "0", description = "Sets a total timeout specified in millis, when using parallel processing. If the Splitter hasn't been able to send and process all replies within the given timeframe, then the timeout triggers and the Splitter breaks out and continues. The timeout method is invoked before breaking out. If the timeout is reached with running tasks still remaining, certain tasks for which it is difficult for Camel to shut down in a graceful manner may continue to run. So use this option with a bit of care.", displayName = "Timeout"), - @YamlProperty(name = "watermarkExpression", type = "string"), - @YamlProperty(name = "watermarkKey", type = "string"), - @YamlProperty(name = "watermarkStore", type = "string") + @YamlProperty(name = "watermarkExpression", type = "string", description = "Sets a Simple expression to evaluate on the exchange after split completion to determine the new watermark value.", displayName = "Watermark Expression"), + @YamlProperty(name = "watermarkKey", type = "string", description = "Sets the key to use in the watermark store.", displayName = "Watermark Key"), + @YamlProperty(name = "watermarkStore", type = "string", description = "Sets a reference to a watermark store (a Map ) in the registry.", displayName = "Watermark Store") } ) public static class SplitDefinitionDeserializer extends YamlDeserializerBase { From a50edf613dd6ab04054742f9ea7144777392012d Mon Sep 17 00:00:00 2001 From: Guillaume Nodet Date: Fri, 27 Mar 2026 21:47:14 +0100 Subject: [PATCH 5/9] CAMEL-23264: Regenerate camelYamlDsl.json schema with field descriptions Co-Authored-By: Claude Opus 4.6 --- .../resources/schema/camelYamlDsl.json | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/dsl/camel-yaml-dsl/camel-yaml-dsl/src/generated/resources/schema/camelYamlDsl.json b/dsl/camel-yaml-dsl/camel-yaml-dsl/src/generated/resources/schema/camelYamlDsl.json index 7ca5baed769af..655114bb44117 100644 --- a/dsl/camel-yaml-dsl/camel-yaml-dsl/src/generated/resources/schema/camelYamlDsl.json +++ b/dsl/camel-yaml-dsl/camel-yaml-dsl/src/generated/resources/schema/camelYamlDsl.json @@ -7253,16 +7253,31 @@ "description" : "Disables this EIP from the route.", "default" : false }, + "errorThreshold" : { + "type" : "number", + "title" : "Error Threshold", + "description" : "Sets the error threshold as a fraction (0.0-1.0) of failed items before aborting the split operation." + }, "executorService" : { "type" : "string", "title" : "Executor Service", "description" : "To use a custom Thread Pool to be used for parallel processing. Notice if you set this option, then parallel processing is automatically implied, and you do not have to enable that option as well." }, + "group" : { + "type" : "number", + "title" : "Group", + "description" : "Groups N split messages into a single message with a java.util.List body. This allows processing items in chunks instead of one at a time." + }, "id" : { "type" : "string", "title" : "Id", "description" : "Sets the id of this node" }, + "maxFailedRecords" : { + "type" : "number", + "title" : "Max Failed Records", + "description" : "Sets the maximum number of failed records before aborting the split operation." + }, "note" : { "type" : "string", "title" : "Note", @@ -7322,6 +7337,21 @@ "description" : "Sets a total timeout specified in millis, when using parallel processing. If the Splitter hasn't been able to send and process all replies within the given timeframe, then the timeout triggers and the Splitter breaks out and continues. The timeout method is invoked before breaking out. If the timeout is reached with running tasks still remaining, certain tasks for which it is difficult for Camel to shut down in a graceful manner may continue to run. So use this option with a bit of care.", "default" : "0" }, + "watermarkExpression" : { + "type" : "string", + "title" : "Watermark Expression", + "description" : "Sets a Simple expression to evaluate on the exchange after split completion to determine the new watermark value." + }, + "watermarkKey" : { + "type" : "string", + "title" : "Watermark Key", + "description" : "Sets the key to use in the watermark store." + }, + "watermarkStore" : { + "type" : "string", + "title" : "Watermark Store", + "description" : "Sets a reference to a watermark store (a Map ) in the registry." + }, "constant" : { }, "csimple" : { }, "datasonnet" : { }, From b149e006a8c54cf2e55f6b224c7bf92138abae37 Mon Sep 17 00:00:00 2001 From: Guillaume Nodet Date: Sat, 28 Mar 2026 07:43:09 +0100 Subject: [PATCH 6/9] CAMEL-23264: Regenerate camelYamlDsl.json schema with field descriptions Co-Authored-By: Claude Opus 4.6 --- .../src/generated/resources/schema/camelYamlDsl.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dsl/camel-yaml-dsl/camel-yaml-dsl/src/generated/resources/schema/camelYamlDsl.json b/dsl/camel-yaml-dsl/camel-yaml-dsl/src/generated/resources/schema/camelYamlDsl.json index 655114bb44117..6fab11ce8d20a 100644 --- a/dsl/camel-yaml-dsl/camel-yaml-dsl/src/generated/resources/schema/camelYamlDsl.json +++ b/dsl/camel-yaml-dsl/camel-yaml-dsl/src/generated/resources/schema/camelYamlDsl.json @@ -7301,6 +7301,11 @@ "description" : "If enabled then processing each split messages occurs concurrently. Note the caller thread will still wait until all messages has been fully processed, before it continues. It's only processing the sub messages from the splitter which happens concurrently. When parallel processing is enabled, then the Camel routing engin will continue processing using last used thread from the parallel thread pool. However, if you want to use the original thread that called the splitter, then make sure to enable the synchronous option as well. In parallel processing mode, you may want to also synchronous = true to force this EIP to process the sub-tasks using the upper bounds of the thread-pool. If using synchronous = false then Camel will allow its reactive routing engine to use as many threads as possible, which may be available due to sub-tasks using other thread-pools such as CompletableFuture.runAsync or others.", "default" : false }, + "resumeStrategy" : { + "type" : "string", + "title" : "Resume Strategy", + "description" : "Sets a reference to a ResumeStrategy in the registry for resume-from-last-position support." + }, "shareUnitOfWork" : { "type" : "boolean", "title" : "Share Unit Of Work", @@ -7347,11 +7352,6 @@ "title" : "Watermark Key", "description" : "Sets the key to use in the watermark store." }, - "watermarkStore" : { - "type" : "string", - "title" : "Watermark Store", - "description" : "Sets a reference to a watermark store (a Map ) in the registry." - }, "constant" : { }, "csimple" : { }, "datasonnet" : { }, From 0b7a7ef2be83083afba3a57490d8a3b6c980b4e3 Mon Sep 17 00:00:00 2001 From: Guillaume Nodet Date: Sat, 28 Mar 2026 09:08:35 +0100 Subject: [PATCH 7/9] CAMEL-23264: Fix group+watermark bug, add docs, tests, and validation - Fix index-based watermark producing wrong offset when combined with group(): track raw item count independently of chunk index - Optimize double readCurrentWatermark() call by reusing exchange property - Refactor watermark from Map to ResumeStrategy SPI - Add validation for errorThreshold range and watermarkKey/resumeStrategy completeness - Add documentation for all new features (group, errorThreshold, maxFailedRecords, SplitResult, watermark tracking) to split-eip.adoc - Document parallel processing limitations for errorThreshold - Document concurrent exchange limitations for watermark tracking - Add upgrade guide section for Split EIP enhancements - Add tests: group+watermark, parallel+errorThreshold, group+errorThreshold, streaming+parallel+maxFailedRecords, streaming mode, validation, and ResumeStrategy test helper Co-Authored-By: Claude Opus 4.6 --- .../apache/camel/catalog/models/split.json | 2 +- .../docs/modules/eips/pages/split-eip.adoc | 349 ++++++++++++++++++ .../org/apache/camel/model/split.json | 2 +- .../apache/camel/model/SplitDefinition.java | 70 ++-- .../org/apache/camel/processor/Splitter.java | 204 +++++++--- .../apache/camel/reifier/SplitReifier.java | 35 +- .../processor/SplitterErrorThresholdTest.java | 48 +++ .../camel/processor/SplitterGroupTest.java | 35 ++ .../SplitterMaxFailedRecordsTest.java | 24 ++ .../SplitterParallelErrorThresholdTest.java | 137 +++++++ .../processor/SplitterSplitResultTest.java | 46 +++ .../processor/SplitterStreamingTest.java | 182 +++++++++ .../processor/SplitterTestResumeStrategy.java | 173 +++++++++ .../processor/SplitterTransactedTest.java | 6 +- .../processor/SplitterValidationTest.java | 119 ++++++ .../processor/SplitterWatermarkTest.java | 88 ++++- .../pages/camel-4x-upgrade-guide-4_19.adoc | 21 ++ .../deserializers/ModelDeserializers.java | 14 +- 18 files changed, 1449 insertions(+), 106 deletions(-) create mode 100644 core/camel-core/src/test/java/org/apache/camel/processor/SplitterParallelErrorThresholdTest.java create mode 100644 core/camel-core/src/test/java/org/apache/camel/processor/SplitterStreamingTest.java create mode 100644 core/camel-core/src/test/java/org/apache/camel/processor/SplitterTestResumeStrategy.java create mode 100644 core/camel-core/src/test/java/org/apache/camel/processor/SplitterValidationTest.java diff --git a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/models/split.json b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/models/split.json index 0f7987150865b..5ab3ad27f8378 100644 --- a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/models/split.json +++ b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/models/split.json @@ -33,7 +33,7 @@ "group": { "index": 18, "kind": "attribute", "displayName": "Group", "group": "advanced", "label": "advanced", "required": false, "type": "integer", "javaType": "java.lang.Integer", "deprecated": false, "autowired": false, "secret": false, "description": "Groups N split messages into a single message with a java.util.List body. This allows processing items in chunks instead of one at a time." }, "errorThreshold": { "index": 19, "kind": "attribute", "displayName": "Error Threshold", "group": "advanced", "label": "advanced", "required": false, "type": "number", "javaType": "java.lang.Double", "deprecated": false, "autowired": false, "secret": false, "description": "Sets the error threshold as a fraction (0.0-1.0) of failed items before aborting the split operation." }, "maxFailedRecords": { "index": 20, "kind": "attribute", "displayName": "Max Failed Records", "group": "advanced", "label": "advanced", "required": false, "type": "integer", "javaType": "java.lang.Integer", "deprecated": false, "autowired": false, "secret": false, "description": "Sets the maximum number of failed records before aborting the split operation." }, - "watermarkStore": { "index": 21, "kind": "attribute", "displayName": "Watermark Store", "group": "advanced", "label": "advanced", "required": false, "type": "object", "javaType": "java.util.Map", "deprecated": false, "autowired": false, "secret": false, "description": "Sets a reference to a watermark store (a Map ) in the registry." }, + "resumeStrategy": { "index": 21, "kind": "attribute", "displayName": "Resume Strategy", "group": "advanced", "label": "advanced", "required": false, "type": "object", "javaType": "org.apache.camel.resume.ResumeStrategy", "deprecated": false, "autowired": false, "secret": false, "description": "Sets a reference to a ResumeStrategy in the registry for resume-from-last-position support." }, "watermarkKey": { "index": 22, "kind": "attribute", "displayName": "Watermark Key", "group": "advanced", "label": "advanced", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "description": "Sets the key to use in the watermark store." }, "watermarkExpression": { "index": 23, "kind": "attribute", "displayName": "Watermark Expression", "group": "advanced", "label": "advanced", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "description": "Sets a Simple expression to evaluate on the exchange after split completion to determine the new watermark value." }, "outputs": { "index": 24, "kind": "element", "displayName": "Outputs", "group": "common", "required": true, "type": "array", "javaType": "java.util.List>", "oneOf": [ "aggregate", "bean", "choice", "circuitBreaker", "claimCheck", "convertBodyTo", "convertHeaderTo", "convertVariableTo", "delay", "doCatch", "doFinally", "doTry", "dynamicRouter", "enrich", "filter", "idempotentConsumer", "intercept", "interceptFrom", "interceptSendToEndpoint", "kamelet", "loadBalance", "log", "loop", "marshal", "multicast", "onCompletion", "onException", "pausable", "pipeline", "policy", "poll", "pollEnrich", "process", "recipientList", "removeHeader", "removeHeaders", "removeProperties", "removeProperty", "removeVariable", "resequence", "resumable", "rollback", "routingSlip", "saga", "sample", "script", "setBody", "setExchangePattern", "setHeader", "setHeaders", "setProperty", "setVariable", "setVariables", "sort", "split", "step", "stop", "threads", "throttle", "throwException", "to", "toD", "tokenizer", "transacted", "transform", "transformDataType", "unmarshal", "validate", "wireTap" ], "deprecated": false, "autowired": false, "secret": false } diff --git a/core/camel-core-engine/src/main/docs/modules/eips/pages/split-eip.adoc b/core/camel-core-engine/src/main/docs/modules/eips/pages/split-eip.adoc index 067d2b24eb0a0..04ecdd891f5b8 100644 --- a/core/camel-core-engine/src/main/docs/modules/eips/pages/split-eip.adoc +++ b/core/camel-core-engine/src/main/docs/modules/eips/pages/split-eip.adoc @@ -1038,6 +1038,355 @@ BuildCombinedResponse: (id=1,item=A);(id=2,item=B);(id=3,item=C) Response to caller: Response[(id=1,item=A);(id=2,item=B);(id=3,item=C)] ---- +=== Chunking with group + +The `group` option on the Split EIP allows grouping N split items together into a single message +with a `java.util.List` body. This is useful when you want to process items in batches rather than +one at a time. + +[tabs] +==== +Java:: ++ +[source,java] +---- +from("direct:start") + .split(body()).group(3) + .to("mock:batch"); +---- + +XML:: ++ +[source,xml] +---- + + + + ${body} + + + +---- + +YAML:: ++ +[source,yaml] +---- +- route: + from: + uri: direct:start + steps: + - split: + group: "3" + expression: + simple: + expression: "${body}" + steps: + - to: + uri: mock:batch +---- +==== + +If the input has 7 items, the route above produces 3 exchanges: one with items 1-3, one with items 4-6, +and one with item 7. + +NOTE: This `group` option is on the Split EIP definition itself and works with any expression. +It is different from the `group` option on the xref:components:languages:tokenize-language.adoc[Tokenize] language, +which groups tokenized text lines together. + +=== Error handling with maxFailedRecords and errorThreshold + +The `stopOnException` option is all-or-nothing: a single failure stops the entire split. +For more fine-grained control, the Splitter provides two error threshold options that let processing +continue through some failures while stopping when the error rate becomes unacceptable. + +==== maxFailedRecords + +The `maxFailedRecords` option sets the maximum number of failed split items before aborting. +Processing continues as long as the failure count stays below this threshold. When the threshold +is reached, the splitter stops and sets an exception on the exchange. + +[tabs] +==== +Java:: ++ +[source,java] +---- +from("direct:start") + .split(body()).maxFailedRecords(5) + .process(exchange -> { + // processing logic that may throw exceptions + }) + .to("mock:result"); +---- + +XML:: ++ +[source,xml] +---- + + + + ${body} + + + + +---- + +YAML:: ++ +[source,yaml] +---- +- route: + from: + uri: direct:start + steps: + - split: + maxFailedRecords: "5" + expression: + simple: + expression: "${body}" + steps: + - process: + ref: myProcessor + - to: + uri: mock:result +---- +==== + +In this example, the first 4 failures are tolerated and processing continues. +When the 5th failure occurs, the splitter stops and the exchange will have an exception set. + +==== errorThreshold + +The `errorThreshold` option sets the maximum allowed failure ratio as a fraction between 0.0 and 1.0. +After each failure, the ratio of failed items to total processed items is calculated. If this ratio +meets or exceeds the threshold, the splitter stops. + +[tabs] +==== +Java:: ++ +[source,java] +---- +from("direct:start") + .split(body()).errorThreshold(0.5) + .process(exchange -> { + // processing logic that may throw exceptions + }) + .to("mock:result"); +---- + +XML:: ++ +[source,xml] +---- + + + + ${body} + + + + +---- + +YAML:: ++ +[source,yaml] +---- +- route: + from: + uri: direct:start + steps: + - split: + errorThreshold: "0.5" + expression: + simple: + expression: "${body}" + steps: + - process: + ref: myProcessor + - to: + uri: mock:result +---- +==== + +In this example, if 50% or more of the processed items have failed, the splitter stops. + +Both `maxFailedRecords` and `errorThreshold` can be combined. The splitter stops when either +threshold is exceeded. + +IMPORTANT: The `stopOnException` option is mutually exclusive with `maxFailedRecords` and +`errorThreshold`. You cannot use `stopOnException` together with either of these options. + +NOTE: When using `errorThreshold` with `parallelProcessing`, the failure ratio may vary slightly +between runs because the ratio is calculated as failures are reported, and the order in which +parallel items complete is non-deterministic. For deterministic abort behavior with parallel +processing, prefer `maxFailedRecords` (absolute count) over `errorThreshold` (ratio). + +==== SplitResult + +When `maxFailedRecords` or `errorThreshold` is configured, the splitter makes a +`SplitResult` object available as an exchange property (`CamelSplitResult`) after the split completes. +This provides structured information about the outcome: + +[source,java] +---- +Exchange result = template.send("direct:start", + e -> e.getIn().setBody(myItems)); + +SplitResult splitResult = result.getProperty(Exchange.SPLIT_RESULT, SplitResult.class); +if (splitResult != null) { + int total = splitResult.getTotalItems(); // total items processed + int success = splitResult.getSuccessCount(); // successful items + int failures = splitResult.getFailureCount(); // failed items + boolean aborted = splitResult.isAborted(); // true if a threshold was exceeded + + // inspect individual failures + for (SplitResult.Failure failure : splitResult.getFailures()) { + int index = failure.index(); // 0-based index of failed item + Exception ex = failure.exception(); // the exception that occurred + } +} +---- + +=== Watermark tracking + +The Splitter supports watermark tracking for incremental processing scenarios using Camel's +`ResumeStrategy` SPI. A watermark records how far processing has progressed, so subsequent +runs can skip already-processed items. + +This is useful when processing a data source repeatedly (e.g., polling a database or file) +where you want to resume from where the last run left off. You can use any `ResumeStrategy` +implementation for persistence — from a simple in-memory strategy for testing to a Kafka-backed +strategy for production use. + +==== Index-based watermark + +The simplest form uses the split index as the watermark. On each run, items up to and including +the stored watermark index are skipped. After successful processing, the watermark is updated +to the last processed index. + +[tabs] +==== +Java:: ++ +[source,java] +---- +ResumeStrategy strategy = ... // any ResumeStrategy implementation + +from("direct:start") + .split(body()).resumeStrategy(strategy, "myJob") + .to("mock:result"); +---- + +XML:: ++ +[source,xml] +---- + + + + ${body} + + + +---- + +YAML:: ++ +[source,yaml] +---- +- route: + from: + uri: direct:start + steps: + - split: + resumeStrategy: "#myStrategy" + watermarkKey: myJob + expression: + simple: + expression: "${body}" + steps: + - to: + uri: mock:result +---- +==== + +On the first run with 5 items, all are processed and the watermark is stored as `"4"` (the last 0-based index). +On the next run with the same 5 items, items 0 through 4 are skipped and nothing is processed. +If the data source grows to 8 items, only items 5, 6, and 7 are processed. + +==== Value-based watermark + +For more control, you can use a `watermarkExpression` to extract a watermark value from each processed item. +The value from the last successfully processed item (by index order) is stored. + +[tabs] +==== +Java:: ++ +[source,java] +---- +ResumeStrategy strategy = ... // any ResumeStrategy implementation + +from("direct:start") + .split(body()) + .resumeStrategy(strategy, "dateJob") + .watermarkExpression("${body}") + .to("mock:result"); +---- + +XML:: ++ +[source,xml] +---- + + + + ${body} + + + +---- + +YAML:: ++ +[source,yaml] +---- +- route: + from: + uri: direct:start + steps: + - split: + resumeStrategy: "#myStrategy" + watermarkKey: dateJob + watermarkExpression: "${body}" + expression: + simple: + expression: "${body}" + steps: + - to: + uri: mock:result +---- +==== + +With value-based watermarking, the previous watermark value is exposed as the exchange property +`CamelSplitWatermark` before split processing begins. You can use this to filter items in your +processing logic. + +NOTE: The `watermarkExpression` option uses the Simple language for expression evaluation. + +NOTE: The watermark is only updated when the split completes successfully. If the split is aborted +(e.g., due to exceeding `maxFailedRecords`), the watermark is not updated, which allows the +failed batch to be retried. + +IMPORTANT: Watermark tracking assumes sequential route invocations (e.g., batch jobs triggered by +a `timer` or `scheduler`). If multiple exchanges hit the same route concurrently, they will read +the same watermark and may process duplicate items. Use a single-consumer pattern for watermark-based routes. + === Stop processing in case of exception The Splitter will by default continue to process diff --git a/core/camel-core-model/src/generated/resources/META-INF/org/apache/camel/model/split.json b/core/camel-core-model/src/generated/resources/META-INF/org/apache/camel/model/split.json index 0f7987150865b..5ab3ad27f8378 100644 --- a/core/camel-core-model/src/generated/resources/META-INF/org/apache/camel/model/split.json +++ b/core/camel-core-model/src/generated/resources/META-INF/org/apache/camel/model/split.json @@ -33,7 +33,7 @@ "group": { "index": 18, "kind": "attribute", "displayName": "Group", "group": "advanced", "label": "advanced", "required": false, "type": "integer", "javaType": "java.lang.Integer", "deprecated": false, "autowired": false, "secret": false, "description": "Groups N split messages into a single message with a java.util.List body. This allows processing items in chunks instead of one at a time." }, "errorThreshold": { "index": 19, "kind": "attribute", "displayName": "Error Threshold", "group": "advanced", "label": "advanced", "required": false, "type": "number", "javaType": "java.lang.Double", "deprecated": false, "autowired": false, "secret": false, "description": "Sets the error threshold as a fraction (0.0-1.0) of failed items before aborting the split operation." }, "maxFailedRecords": { "index": 20, "kind": "attribute", "displayName": "Max Failed Records", "group": "advanced", "label": "advanced", "required": false, "type": "integer", "javaType": "java.lang.Integer", "deprecated": false, "autowired": false, "secret": false, "description": "Sets the maximum number of failed records before aborting the split operation." }, - "watermarkStore": { "index": 21, "kind": "attribute", "displayName": "Watermark Store", "group": "advanced", "label": "advanced", "required": false, "type": "object", "javaType": "java.util.Map", "deprecated": false, "autowired": false, "secret": false, "description": "Sets a reference to a watermark store (a Map ) in the registry." }, + "resumeStrategy": { "index": 21, "kind": "attribute", "displayName": "Resume Strategy", "group": "advanced", "label": "advanced", "required": false, "type": "object", "javaType": "org.apache.camel.resume.ResumeStrategy", "deprecated": false, "autowired": false, "secret": false, "description": "Sets a reference to a ResumeStrategy in the registry for resume-from-last-position support." }, "watermarkKey": { "index": 22, "kind": "attribute", "displayName": "Watermark Key", "group": "advanced", "label": "advanced", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "description": "Sets the key to use in the watermark store." }, "watermarkExpression": { "index": 23, "kind": "attribute", "displayName": "Watermark Expression", "group": "advanced", "label": "advanced", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "description": "Sets a Simple expression to evaluate on the exchange after split completion to determine the new watermark value." }, "outputs": { "index": 24, "kind": "element", "displayName": "Outputs", "group": "common", "required": true, "type": "array", "javaType": "java.util.List>", "oneOf": [ "aggregate", "bean", "choice", "circuitBreaker", "claimCheck", "convertBodyTo", "convertHeaderTo", "convertVariableTo", "delay", "doCatch", "doFinally", "doTry", "dynamicRouter", "enrich", "filter", "idempotentConsumer", "intercept", "interceptFrom", "interceptSendToEndpoint", "kamelet", "loadBalance", "log", "loop", "marshal", "multicast", "onCompletion", "onException", "pausable", "pipeline", "policy", "poll", "pollEnrich", "process", "recipientList", "removeHeader", "removeHeaders", "removeProperties", "removeProperty", "removeVariable", "resequence", "resumable", "rollback", "routingSlip", "saga", "sample", "script", "setBody", "setExchangePattern", "setHeader", "setHeaders", "setProperty", "setVariable", "setVariables", "sort", "split", "step", "stop", "threads", "throttle", "throwException", "to", "toD", "tokenizer", "transacted", "transform", "transformDataType", "unmarshal", "validate", "wireTap" ], "deprecated": false, "autowired": false, "secret": false } diff --git a/core/camel-core-model/src/main/java/org/apache/camel/model/SplitDefinition.java b/core/camel-core-model/src/main/java/org/apache/camel/model/SplitDefinition.java index 3e57d53f196a0..d951e32b56140 100644 --- a/core/camel-core-model/src/main/java/org/apache/camel/model/SplitDefinition.java +++ b/core/camel-core-model/src/main/java/org/apache/camel/model/SplitDefinition.java @@ -16,7 +16,6 @@ */ package org.apache.camel.model; -import java.util.Map; import java.util.concurrent.ExecutorService; import jakarta.xml.bind.annotation.XmlAccessType; @@ -29,6 +28,7 @@ import org.apache.camel.Expression; import org.apache.camel.Processor; import org.apache.camel.model.language.ExpressionDefinition; +import org.apache.camel.resume.ResumeStrategy; import org.apache.camel.spi.Metadata; /** @@ -46,8 +46,7 @@ public class SplitDefinition extends OutputExpressionNode implements ExecutorSer @XmlTransient private Processor onPrepareProcessor; @XmlTransient - @SuppressWarnings("rawtypes") - private Map watermarkStoreBean; + private ResumeStrategy resumeStrategyBean; @XmlAttribute @Metadata(defaultValue = ",") @@ -99,8 +98,8 @@ public class SplitDefinition extends OutputExpressionNode implements ExecutorSer @Metadata(label = "advanced", javaType = "java.lang.Integer") private String maxFailedRecords; @XmlAttribute - @Metadata(label = "advanced", javaType = "java.util.Map") - private String watermarkStore; + @Metadata(label = "advanced", javaType = "org.apache.camel.resume.ResumeStrategy") + private String resumeStrategy; @XmlAttribute @Metadata(label = "advanced") private String watermarkKey; @@ -132,8 +131,8 @@ public SplitDefinition(SplitDefinition source) { this.group = source.group; this.errorThreshold = source.errorThreshold; this.maxFailedRecords = source.maxFailedRecords; - this.watermarkStoreBean = source.watermarkStoreBean; - this.watermarkStore = source.watermarkStore; + this.resumeStrategyBean = source.resumeStrategyBean; + this.resumeStrategy = source.resumeStrategy; this.watermarkKey = source.watermarkKey; this.watermarkExpression = source.watermarkExpression; } @@ -625,6 +624,10 @@ public SplitDefinition group(String group) { *

* This option is mutually exclusive with {@code stopOnException}. When set, individual item failures are tracked * but processing continues until the threshold is exceeded. + *

+ * Note: When combined with {@code parallelProcessing}, the failure ratio may vary between runs because + * parallel items complete in non-deterministic order. For deterministic abort behavior with parallel processing, + * prefer {@code maxFailedRecords} (absolute count) over {@code errorThreshold} (ratio). * * @param errorThreshold the failure ratio threshold (0.0-1.0) * @return the builder @@ -679,7 +682,7 @@ public SplitDefinition maxFailedRecords(String maxFailedRecords) { } /** - * Sets a watermark store and key for resume-from-last-position support. When configured, the Splitter tracks + * Sets a {@link ResumeStrategy} and key for resume-from-last-position support. When configured, the Splitter tracks * progress and can skip already-processed items on subsequent runs. *

* With index-based watermarking (no {@code watermarkExpression}), items up to the stored index are automatically @@ -689,25 +692,36 @@ public SplitDefinition maxFailedRecords(String maxFailedRecords) { * The watermark is only updated on successful completion — aborted runs preserve the previous watermark to allow * retry. * - * @param store the Map to store watermark state in - * @param key the key to use in the store - * @return the builder + * @param strategy the ResumeStrategy to persist watermark state + * @param key the key to use in the strategy + * @return the builder */ - @SuppressWarnings("rawtypes") - public SplitDefinition watermarkStore(Map store, String key) { - this.watermarkStoreBean = store; + public SplitDefinition resumeStrategy(ResumeStrategy strategy, String key) { + this.resumeStrategyBean = strategy; setWatermarkKey(key); return this; } /** - * Sets a reference to a watermark store (a {@code Map}) in the registry. + * Sets a {@link ResumeStrategy} for resume-from-last-position support. The watermark key must also be configured + * via {@link #watermarkKey(String)}. * - * @param watermarkStore reference to the store bean + * @param strategy the ResumeStrategy to persist watermark state + * @return the builder + */ + public SplitDefinition resumeStrategy(ResumeStrategy strategy) { + this.resumeStrategyBean = strategy; + return this; + } + + /** + * Sets a reference to a {@link ResumeStrategy} in the registry. + * + * @param resumeStrategy reference to the strategy bean * @return the builder */ - public SplitDefinition watermarkStore(String watermarkStore) { - setWatermarkStore(watermarkStore); + public SplitDefinition resumeStrategy(String resumeStrategy) { + setResumeStrategy(resumeStrategy); return this; } @@ -723,8 +737,9 @@ public SplitDefinition watermarkKey(String watermarkKey) { } /** - * Sets a Simple expression to evaluate on the exchange after split completion to determine the new watermark value. - * When set, enables value-based watermarking instead of index-based. + * Sets a Simple expression to evaluate on each completed sub-exchange to determine the new watermark value. When + * set, enables value-based watermarking instead of index-based. The expression is evaluated using the Simple + * language. * * @param watermarkExpression the Simple expression * @return the builder @@ -926,20 +941,19 @@ public void setMaxFailedRecords(String maxFailedRecords) { this.maxFailedRecords = maxFailedRecords; } - @SuppressWarnings("rawtypes") - public Map getWatermarkStoreBean() { - return watermarkStoreBean; + public ResumeStrategy getResumeStrategyBean() { + return resumeStrategyBean; } - public String getWatermarkStore() { - return watermarkStore; + public String getResumeStrategy() { + return resumeStrategy; } /** - * Sets a reference to a watermark store (a {@code Map}) in the registry. + * Sets a reference to a {@link ResumeStrategy} in the registry for resume-from-last-position support. */ - public void setWatermarkStore(String watermarkStore) { - this.watermarkStore = watermarkStore; + public void setResumeStrategy(String resumeStrategy) { + this.resumeStrategy = resumeStrategy; } public String getWatermarkKey() { diff --git a/core/camel-core-processor/src/main/java/org/apache/camel/processor/Splitter.java b/core/camel-core-processor/src/main/java/org/apache/camel/processor/Splitter.java index 8bdd3e929e156..8cda2374b9849 100644 --- a/core/camel-core-processor/src/main/java/org/apache/camel/processor/Splitter.java +++ b/core/camel-core-processor/src/main/java/org/apache/camel/processor/Splitter.java @@ -43,9 +43,17 @@ import org.apache.camel.SplitResult; import org.apache.camel.processor.aggregate.ShareUnitOfWorkAggregationStrategy; import org.apache.camel.processor.aggregate.UseOriginalAggregationStrategy; +import org.apache.camel.resume.Offset; +import org.apache.camel.resume.OffsetKey; +import org.apache.camel.resume.ResumeStrategy; +import org.apache.camel.resume.ResumeStrategyConfiguration; +import org.apache.camel.resume.cache.ResumeCache; import org.apache.camel.support.ExchangeHelper; import org.apache.camel.support.GroupIterator; import org.apache.camel.support.ObjectHelper; +import org.apache.camel.support.resume.OffsetKeys; +import org.apache.camel.support.resume.Offsets; +import org.apache.camel.support.service.ServiceHelper; import org.apache.camel.util.IOHelper; import org.apache.camel.util.StringHelper; @@ -63,12 +71,17 @@ public class Splitter extends MulticastProcessor { private static final String SPLIT_WATERMARK_OFFSET = "CamelSplitWatermarkOffset"; private static final String SPLIT_WATERMARK_COUNT = "CamelSplitWatermarkCount"; private static final String SPLIT_WATERMARK_LATEST = "CamelSplitWatermarkLatest"; + + private record IndexedWatermark(int index, String value) { + } + private final Expression expression; private final String delimiter; private int group; private double errorThreshold; private int maxFailedRecords; - private Map watermarkStore; + private ResumeStrategy resumeStrategy; + private final ConcurrentHashMap watermarkCache = new ConcurrentHashMap<>(); private String watermarkKey; private Expression watermarkExpression; @@ -102,11 +115,6 @@ public String getTraceLabel() { return "split[" + expression + "]"; } - @Override - protected void doBuild() throws Exception { - super.doBuild(); - } - @Override protected void doInit() throws Exception { super.doInit(); @@ -114,6 +122,59 @@ protected void doInit() throws Exception { if (watermarkExpression != null) { watermarkExpression.init(getCamelContext()); } + if (resumeStrategy != null && watermarkKey != null) { + ServiceHelper.startService(resumeStrategy); + resumeStrategy.loadCache(); + } + } + + @Override + protected void doStop() throws Exception { + if (resumeStrategy != null) { + ServiceHelper.stopService(resumeStrategy); + } + super.doStop(); + } + + /** + * Reads the current watermark value. Checks the local cache first (populated from previous exchanges), then falls + * back to the strategy's resume cache (populated by {@code loadCache()} on init or backed by external storage). + */ + private String readCurrentWatermark() { + String value = watermarkCache.get(watermarkKey); + if (value != null) { + return value; + } + return readFromStrategyCache(); + } + + private String readFromStrategyCache() { + try { + ResumeStrategyConfiguration config = resumeStrategy.getResumeStrategyConfiguration(); + if (config != null) { + ResumeCache cache = config.getResumeCache(); + if (cache != null) { + String[] result = new String[1]; + cache.forEach((key, value) -> { + String keyStr = key instanceof OffsetKey ok + ? String.valueOf(ok.getValue()) : String.valueOf(key); + if (watermarkKey.equals(keyStr) && value != null) { + result[0] = toWatermarkString(value); + return false; + } + return true; + }); + return result[0]; + } + } + } catch (Exception e) { + // best-effort + } + return null; + } + + private static String toWatermarkString(Object value) { + return value instanceof Offset o ? String.valueOf(o.getValue()) : String.valueOf(value); } @Override @@ -148,15 +209,15 @@ public boolean process(Exchange exchange, final AsyncCallback callback) { } // set current watermark value as exchange property before processing - boolean hasWatermark = watermarkStore != null && watermarkKey != null; + boolean hasWatermark = resumeStrategy != null && watermarkKey != null; if (hasWatermark) { - String currentWatermark = watermarkStore.get(watermarkKey); + String currentWatermark = readCurrentWatermark(); if (currentWatermark != null) { exchange.setProperty(Exchange.SPLIT_WATERMARK, currentWatermark); } // pre-initialize AtomicReference for value-based watermark tracking (thread-safe) if (watermarkExpression != null) { - exchange.setProperty(SPLIT_WATERMARK_LATEST, new AtomicReference()); + exchange.setProperty(SPLIT_WATERMARK_LATEST, new AtomicReference()); } } @@ -207,14 +268,6 @@ protected Iterable createProcessorExchangePairs(Exchange throw exchange.getException(); } - // store total items count in tracker for SplitResult reporting - if (answer instanceof Collection coll) { - SplitFailureTracker tracker = exchange.getProperty(SPLIT_FAILURE_TRACKER, SplitFailureTracker.class); - if (tracker != null) { - tracker.setTotalItems(coll.size()); - } - } - return answer; } @@ -232,6 +285,8 @@ private final class SplitterIterable implements Iterable, private final Route route; private final Exchange original; private final int watermarkOffset; + // tracks individual (raw) item count, independent of grouping + private final AtomicInteger rawItemCount = new AtomicInteger(); private SplitterIterable(Exchange exchange, Object value) { this.original = exchange; @@ -249,8 +304,12 @@ private SplitterIterable(Exchange exchange, Object value) { // index-based watermarking: skip items up to the stored watermark index int skipCount = 0; - if (watermarkStore != null && watermarkKey != null && watermarkExpression == null) { - String storedIndex = watermarkStore.get(watermarkKey); + if (resumeStrategy != null && watermarkKey != null && watermarkExpression == null) { + // reuse watermark already read in process() and set as exchange property + String storedIndex = exchange.getProperty(Exchange.SPLIT_WATERMARK, String.class); + if (storedIndex == null) { + storedIndex = readCurrentWatermark(); + } if (storedIndex != null) { int skipTo = Integer.parseInt(storedIndex); while (rawIterator.hasNext() && skipCount <= skipTo) { @@ -264,8 +323,23 @@ private SplitterIterable(Exchange exchange, Object value) { exchange.setProperty(SPLIT_WATERMARK_OFFSET, skipCount); } + // wrap rawIterator in a counting decorator to track individual items + // consumed, independent of any GroupIterator chunking + Iterator countingIterator = new Iterator<>() { + @Override + public boolean hasNext() { + return rawIterator.hasNext(); + } + + @Override + public Object next() { + rawItemCount.incrementAndGet(); + return rawIterator.next(); + } + }; + // wrap with GroupIterator if group > 0 to chunk items into List batches - this.iterator = group > 0 ? new GroupIterator(rawIterator, group) : rawIterator; + this.iterator = group > 0 ? new GroupIterator(countingIterator, group) : countingIterator; this.copy = copyAndPrepareSubExchange(exchange); this.route = ExchangeHelper.getRoute(exchange); @@ -289,9 +363,13 @@ public boolean hasNext() { if (!answer) { // we are now closed closed = true; - // store item count for watermark tracking - if (watermarkStore != null && watermarkKey != null && watermarkExpression == null) { - original.setProperty(SPLIT_WATERMARK_COUNT, index); + // store raw item count for watermark tracking (guard against + // re-iteration in MulticastProcessor.doDone which re-creates + // the iterator to release exchanges). + // Use rawItemCount (individual items) not index (which counts chunks when group > 0) + if (resumeStrategy != null && watermarkKey != null && watermarkExpression == null + && original.getProperty(SPLIT_WATERMARK_COUNT) == null) { + original.setProperty(SPLIT_WATERMARK_COUNT, rawItemCount.get()); } // nothing more so we need to close the expression value in case it needs to be try { @@ -336,6 +414,12 @@ public ProcessorExchangePair next() { Message in = newExchange.getIn(); in.setBody(part); } + // track total items for SplitResult (works in both streaming and non-streaming mode) + SplitFailureTracker tracker + = original.getProperty(SPLIT_FAILURE_TRACKER, SplitFailureTracker.class); + if (tracker != null) { + tracker.incrementTotalItems(); + } return createProcessorExchangePair(index++, processor, newExchange, route); } else { return null; @@ -430,12 +514,12 @@ public void setMaxFailedRecords(int maxFailedRecords) { this.maxFailedRecords = maxFailedRecords; } - public Map getWatermarkStore() { - return watermarkStore; + public ResumeStrategy getResumeStrategy() { + return resumeStrategy; } - public void setWatermarkStore(Map watermarkStore) { - this.watermarkStore = watermarkStore; + public void setResumeStrategy(ResumeStrategy resumeStrategy) { + this.resumeStrategy = resumeStrategy; } public String getWatermarkKey() { @@ -476,7 +560,9 @@ protected boolean shouldContinueOnFailure(Exchange subExchange, Exchange origina } } - // continue processing - clear the exception so aggregation proceeds normally + // Continue processing — clear the exception from the sub-exchange so that aggregation + // proceeds normally. The failure is already recorded in the tracker above, so the + // SplitResult will still contain the failure details even though the exception is cleared. subExchange.setException(null); return true; } @@ -484,18 +570,15 @@ protected boolean shouldContinueOnFailure(Exchange subExchange, Exchange origina static final class SplitFailureTracker { private final AtomicInteger failureCount = new AtomicInteger(); private final AtomicInteger totalItems = new AtomicInteger(); - private final CopyOnWriteArrayList failures = new CopyOnWriteArrayList<>(); - - record SplitFailure(int index, Exception exception) { - } + private final CopyOnWriteArrayList failures = new CopyOnWriteArrayList<>(); void recordFailure(int index, Exception exception) { failureCount.incrementAndGet(); - failures.add(new SplitFailure(index, exception)); + failures.add(new SplitResult.Failure(index, exception)); } - void setTotalItems(int count) { - totalItems.set(count); + void incrementTotalItems() { + totalItems.incrementAndGet(); } int getTotalItems() { @@ -506,12 +589,11 @@ int getFailureCount() { return failureCount.get(); } - List getFailures() { - return Collections.unmodifiableList(failures); + List getFailures() { + return List.copyOf(failures); } } - @SuppressWarnings("unchecked") /** * Wraps a {@link ProcessorExchangePair} to evaluate the watermark expression on each completed sub-exchange. */ @@ -558,10 +640,16 @@ public void done() { if (subExchange.getException() == null) { String value = watermarkExpression.evaluate(subExchange, String.class); if (value != null) { - // AtomicReference is pre-initialized in process() — safe for parallel use - AtomicReference latestRef - = (AtomicReference) original.getProperty(SPLIT_WATERMARK_LATEST); - latestRef.set(value); + int itemIndex = delegate.getIndex(); + // Use accumulateAndGet to keep the value from the highest-indexed item. + // This ensures deterministic behavior in parallel processing mode. + AtomicReference latestRef + = (AtomicReference) original.getProperty(SPLIT_WATERMARK_LATEST); + latestRef.accumulateAndGet( + new IndexedWatermark(itemIndex, value), + (existing, candidate) -> existing == null || candidate.index() > existing.index() + ? candidate + : existing); } } } @@ -575,12 +663,13 @@ private void updateWatermark(Exchange exchange) { } if (watermarkExpression != null) { - // value-based: use the latest value tracked per-item during processing - AtomicReference latestRef = exchange.getProperty(SPLIT_WATERMARK_LATEST, AtomicReference.class); + // value-based: use the value from the highest-indexed item tracked during processing + AtomicReference latestRef + = exchange.getProperty(SPLIT_WATERMARK_LATEST, AtomicReference.class); if (latestRef != null) { - String newValue = latestRef.get(); - if (newValue != null) { - watermarkStore.put(watermarkKey, newValue); + IndexedWatermark latest = latestRef.get(); + if (latest != null) { + persistWatermark(latest.value()); } } } else { @@ -589,7 +678,7 @@ private void updateWatermark(Exchange exchange) { Integer count = exchange.getProperty(SPLIT_WATERMARK_COUNT, Integer.class); if (count != null && count > 0) { int lastAbsoluteIndex = offset + count - 1; - watermarkStore.put(watermarkKey, String.valueOf(lastAbsoluteIndex)); + persistWatermark(String.valueOf(lastAbsoluteIndex)); } } @@ -599,21 +688,24 @@ private void updateWatermark(Exchange exchange) { exchange.removeProperty(SPLIT_WATERMARK_LATEST); } + private void persistWatermark(String value) { + watermarkCache.put(watermarkKey, value); + try { + resumeStrategy.updateLastOffset(OffsetKeys.of(watermarkKey), Offsets.of(value)); + } catch (Exception e) { + throw RuntimeCamelException.wrapRuntimeCamelException(e); + } + } + private void buildSplitResult(Exchange exchange) { SplitFailureTracker tracker = exchange.getProperty(SPLIT_FAILURE_TRACKER, SplitFailureTracker.class); if (tracker == null) { return; } - int totalItems = tracker.getTotalItems(); boolean aborted = exchange.getException() != null; - - List failures = new ArrayList<>(); - for (SplitFailureTracker.SplitFailure f : tracker.getFailures()) { - failures.add(new SplitResult.Failure(f.index(), f.exception())); - } - - SplitResult result = new SplitResult(totalItems, tracker.getFailureCount(), failures, aborted); + SplitResult result + = new SplitResult(tracker.getTotalItems(), tracker.getFailureCount(), tracker.getFailures(), aborted); exchange.setProperty(ExchangePropertyKey.SPLIT_RESULT, result); // remove internal tracker diff --git a/core/camel-core-reifier/src/main/java/org/apache/camel/reifier/SplitReifier.java b/core/camel-core-reifier/src/main/java/org/apache/camel/reifier/SplitReifier.java index 35c9273bef63f..c742409f09ed5 100644 --- a/core/camel-core-reifier/src/main/java/org/apache/camel/reifier/SplitReifier.java +++ b/core/camel-core-reifier/src/main/java/org/apache/camel/reifier/SplitReifier.java @@ -16,7 +16,6 @@ */ package org.apache.camel.reifier; -import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.function.BiFunction; @@ -31,6 +30,7 @@ import org.apache.camel.processor.aggregate.AggregationStrategyBeanAdapter; import org.apache.camel.processor.aggregate.AggregationStrategyBiFunctionAdapter; import org.apache.camel.processor.aggregate.ShareUnitOfWorkAggregationStrategy; +import org.apache.camel.resume.ResumeStrategy; public class SplitReifier extends ExpressionReifier { @@ -93,6 +93,14 @@ public Processor createProcessor() throws Exception { throw new IllegalArgumentException( "Cannot use both stopOnException and errorThreshold/maxFailedRecords on the Splitter EIP"); } + if (errorThreshold != 0 && (errorThreshold < 0 || errorThreshold > 1.0)) { + throw new IllegalArgumentException( + "errorThreshold must be between 0.0 and 1.0, but was: " + errorThreshold); + } + if (maxFailedRecords < 0) { + throw new IllegalArgumentException( + "maxFailedRecords must not be negative, but was: " + maxFailedRecords); + } if (errorThreshold > 0) { answer.setErrorThreshold(errorThreshold); } @@ -100,16 +108,27 @@ public Processor createProcessor() throws Exception { answer.setMaxFailedRecords(maxFailedRecords); } - @SuppressWarnings("unchecked") - Map watermarkStore = definition.getWatermarkStoreBean(); - if (watermarkStore == null && definition.getWatermarkStore() != null) { - watermarkStore = mandatoryLookup(definition.getWatermarkStore(), Map.class); + ResumeStrategy resumeStrategy = definition.getResumeStrategyBean(); + if (resumeStrategy == null && definition.getResumeStrategy() != null) { + resumeStrategy = mandatoryLookup(definition.getResumeStrategy(), ResumeStrategy.class); + } + if (resumeStrategy != null) { + CamelContextAware.trySetCamelContext(resumeStrategy, camelContext); } String watermarkKey = parseString(definition.getWatermarkKey()); - if (watermarkStore != null && watermarkKey != null) { - answer.setWatermarkStore(watermarkStore); + String watermarkExprStr = parseString(definition.getWatermarkExpression()); + // validate watermark configuration completeness + if ((resumeStrategy != null) != (watermarkKey != null)) { + throw new IllegalArgumentException( + "Both resumeStrategy and watermarkKey must be configured together on the Splitter EIP"); + } + if (watermarkExprStr != null && resumeStrategy == null) { + throw new IllegalArgumentException( + "watermarkExpression requires resumeStrategy and watermarkKey on the Splitter EIP"); + } + if (resumeStrategy != null && watermarkKey != null) { + answer.setResumeStrategy(resumeStrategy); answer.setWatermarkKey(watermarkKey); - String watermarkExprStr = parseString(definition.getWatermarkExpression()); if (watermarkExprStr != null) { Expression watermarkExpr = camelContext.resolveLanguage("simple").createExpression(watermarkExprStr); answer.setWatermarkExpression(watermarkExpr); diff --git a/core/camel-core/src/test/java/org/apache/camel/processor/SplitterErrorThresholdTest.java b/core/camel-core/src/test/java/org/apache/camel/processor/SplitterErrorThresholdTest.java index 09770dbcdd59f..76d07fb74b743 100644 --- a/core/camel-core/src/test/java/org/apache/camel/processor/SplitterErrorThresholdTest.java +++ b/core/camel-core/src/test/java/org/apache/camel/processor/SplitterErrorThresholdTest.java @@ -90,6 +90,34 @@ public void configure() { } } + @Test + void testErrorThresholdWithParallelProcessing() throws Exception { + MockEndpoint mock = getMockEndpoint("mock:parallel-split"); + mock.expectedMinimumMessageCount(0); + + Exchange result = template.send("direct:parallel", + e -> e.getIn().setBody(Arrays.asList("FAIL", "FAIL", "a", "b", "c"))); + + mock.assertIsSatisfied(); + + assertNotNull(result.getException(), "Should have an exception when error threshold exceeded in parallel mode"); + } + + @Test + void testCombinedErrorThresholdAndMaxFailedRecords() throws Exception { + // maxFailedRecords=10 (very high), errorThreshold=0.3 (30%) + // First item FAIL: ratio=1/1=100% >= 30% -> stop due to errorThreshold + MockEndpoint mock = getMockEndpoint("mock:combined"); + mock.expectedMinimumMessageCount(0); + + Exchange result = template.send("direct:combined", + e -> e.getIn().setBody(Arrays.asList("FAIL", "a", "b", "c"))); + + mock.assertIsSatisfied(); + + assertNotNull(result.getException(), "Should stop when errorThreshold exceeded even with high maxFailedRecords"); + } + @Override protected RouteBuilder createRouteBuilder() { return new RouteBuilder() { @@ -104,6 +132,26 @@ public void configure() { } }) .to("mock:split"); + + from("direct:parallel") + .split(body()).errorThreshold(0.5).parallelProcessing() + .process(exchange -> { + String body = exchange.getIn().getBody(String.class); + if ("FAIL".equals(body)) { + throw new IllegalArgumentException("Simulated failure for: " + body); + } + }) + .to("mock:parallel-split"); + + from("direct:combined") + .split(body()).maxFailedRecords(10).errorThreshold(0.3) + .process(exchange -> { + String body = exchange.getIn().getBody(String.class); + if ("FAIL".equals(body)) { + throw new IllegalArgumentException("Simulated failure for: " + body); + } + }) + .to("mock:combined"); } }; } diff --git a/core/camel-core/src/test/java/org/apache/camel/processor/SplitterGroupTest.java b/core/camel-core/src/test/java/org/apache/camel/processor/SplitterGroupTest.java index 1da7beeb775ed..afedb2b5c5bf9 100644 --- a/core/camel-core/src/test/java/org/apache/camel/processor/SplitterGroupTest.java +++ b/core/camel-core/src/test/java/org/apache/camel/processor/SplitterGroupTest.java @@ -20,12 +20,16 @@ import java.util.List; import org.apache.camel.ContextTestSupport; +import org.apache.camel.Exchange; +import org.apache.camel.SplitResult; import org.apache.camel.builder.RouteBuilder; import org.apache.camel.component.mock.MockEndpoint; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; class SplitterGroupTest extends ContextTestSupport { @@ -91,6 +95,25 @@ void testSplitterGroupWithParallelProcessing() throws Exception { mock.assertIsSatisfied(); } + @Test + void testGroupWithMaxFailedRecords() throws Exception { + // 9 items grouped by 3. Processor throws if the chunk contains "FAIL". + // Items: a, b, FAIL, d, e, f, g, h, i → chunks [a,b,FAIL], [d,e,f], [g,h,i] + // First chunk triggers failure, maxFailedRecords=2, so processing continues + MockEndpoint mock = getMockEndpoint("mock:group-fail"); + mock.expectedMinimumMessageCount(2); // at least the 2 non-failing chunks + + Exchange result = template.send("direct:group-fail", + e -> e.getIn().setBody(Arrays.asList("a", "b", "FAIL", "d", "e", "f", "g", "h", "i"))); + + mock.assertIsSatisfied(); + + SplitResult splitResult = result.getProperty(Exchange.SPLIT_RESULT, SplitResult.class); + assertNotNull(splitResult); + assertEquals(1, splitResult.getFailureCount()); + assertFalse(splitResult.isAborted()); + } + @Override protected RouteBuilder createRouteBuilder() { return new RouteBuilder() { @@ -103,6 +126,18 @@ public void configure() { from("direct:parallel") .split(body()).group(3).parallelProcessing() .to("mock:parallel-split"); + + from("direct:group-fail") + .split(body()).group(3).maxFailedRecords(2) + .process(exchange -> { + List chunk = exchange.getIn().getBody(List.class); + for (Object item : chunk) { + if ("FAIL".equals(item)) { + throw new IllegalArgumentException("Chunk contains FAIL"); + } + } + }) + .to("mock:group-fail"); } }; } diff --git a/core/camel-core/src/test/java/org/apache/camel/processor/SplitterMaxFailedRecordsTest.java b/core/camel-core/src/test/java/org/apache/camel/processor/SplitterMaxFailedRecordsTest.java index 69b36f39a4139..6c8f63f806374 100644 --- a/core/camel-core/src/test/java/org/apache/camel/processor/SplitterMaxFailedRecordsTest.java +++ b/core/camel-core/src/test/java/org/apache/camel/processor/SplitterMaxFailedRecordsTest.java @@ -92,6 +92,20 @@ public void configure() { } } + @Test + void testMaxFailedRecordsWithParallelProcessing() throws Exception { + MockEndpoint mock = getMockEndpoint("mock:parallel"); + mock.expectedMinimumMessageCount(0); + + Exchange result = template.send("direct:parallel", + e -> e.getIn().setBody(Arrays.asList("a", "FAIL", "b", "FAIL", "c", "FAIL", "d"))); + + mock.assertIsSatisfied(); + + assertNotNull(result.getException(), "Should have exception in parallel mode"); + assertTrue(mock.getReceivedCounter() < 7, "Should stop before processing all items in parallel mode"); + } + @Override protected RouteBuilder createRouteBuilder() { return new RouteBuilder() { @@ -106,6 +120,16 @@ public void configure() { } }) .to("mock:split"); + + from("direct:parallel") + .split(body()).maxFailedRecords(2).parallelProcessing() + .process(exchange -> { + String body = exchange.getIn().getBody(String.class); + if ("FAIL".equals(body)) { + throw new IllegalArgumentException("Simulated failure for: " + body); + } + }) + .to("mock:parallel"); } }; } diff --git a/core/camel-core/src/test/java/org/apache/camel/processor/SplitterParallelErrorThresholdTest.java b/core/camel-core/src/test/java/org/apache/camel/processor/SplitterParallelErrorThresholdTest.java new file mode 100644 index 0000000000000..3efcf15f5c1dc --- /dev/null +++ b/core/camel-core/src/test/java/org/apache/camel/processor/SplitterParallelErrorThresholdTest.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.processor; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.camel.ContextTestSupport; +import org.apache.camel.Exchange; +import org.apache.camel.SplitResult; +import org.apache.camel.builder.RouteBuilder; +import org.apache.camel.component.mock.MockEndpoint; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests for error threshold features with parallel processing. + */ +class SplitterParallelErrorThresholdTest extends ContextTestSupport { + + @Test + void testMaxFailedRecordsWithParallel() throws Exception { + // 20 items, every 4th fails (indices 3, 7, 11, 15, 19) + // maxFailedRecords=3 → should abort after 3rd failure + List items = new ArrayList<>(); + for (int i = 0; i < 20; i++) { + items.add(i % 4 == 3 ? "FAIL" : "item-" + i); + } + + MockEndpoint mock = getMockEndpoint("mock:parallel-max"); + mock.expectedMinimumMessageCount(0); + + Exchange result = template.send("direct:parallel-max", + e -> e.getIn().setBody(items)); + + mock.assertIsSatisfied(); + + assertNotNull(result.getException(), "Should have exception when maxFailedRecords exceeded"); + + SplitResult splitResult = result.getProperty(Exchange.SPLIT_RESULT, SplitResult.class); + assertNotNull(splitResult, "SplitResult should be set"); + assertTrue(splitResult.isAborted()); + assertTrue(splitResult.getFailureCount() >= 3, "Should have at least 3 failures"); + } + + @Test + void testMaxFailedRecordsAllSucceedWithParallel() throws Exception { + MockEndpoint mock = getMockEndpoint("mock:parallel-max"); + mock.expectedMessageCount(10); + + List items = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + items.add("item-" + i); + } + + Exchange result = template.send("direct:parallel-max", + e -> e.getIn().setBody(items)); + + mock.assertIsSatisfied(); + + SplitResult splitResult = result.getProperty(Exchange.SPLIT_RESULT, SplitResult.class); + assertNotNull(splitResult); + assertEquals(10, splitResult.getTotalItems()); + assertEquals(0, splitResult.getFailureCount()); + assertFalse(splitResult.isAborted()); + } + + @Test + void testErrorThresholdWithParallelHighFailureRate() throws Exception { + // 10 items, first 8 fail → very high failure rate + // errorThreshold=0.5 → should abort early + List items = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + items.add(i < 8 ? "FAIL" : "item-" + i); + } + + MockEndpoint mock = getMockEndpoint("mock:parallel-threshold"); + mock.expectedMinimumMessageCount(0); + + Exchange result = template.send("direct:parallel-threshold", + e -> e.getIn().setBody(items)); + + mock.assertIsSatisfied(); + + assertNotNull(result.getException(), "Should abort with high failure rate"); + + SplitResult splitResult = result.getProperty(Exchange.SPLIT_RESULT, SplitResult.class); + assertNotNull(splitResult); + assertTrue(splitResult.isAborted()); + } + + @Override + protected RouteBuilder createRouteBuilder() { + return new RouteBuilder() { + @Override + public void configure() { + from("direct:parallel-max") + .split(body()).parallelProcessing().maxFailedRecords(3) + .process(exchange -> { + String body = exchange.getIn().getBody(String.class); + if ("FAIL".equals(body)) { + throw new IllegalArgumentException("Simulated failure: " + body); + } + }) + .to("mock:parallel-max"); + + from("direct:parallel-threshold") + .split(body()).parallelProcessing().errorThreshold(0.5) + .process(exchange -> { + String body = exchange.getIn().getBody(String.class); + if ("FAIL".equals(body)) { + throw new IllegalArgumentException("Simulated failure: " + body); + } + }) + .to("mock:parallel-threshold"); + } + }; + } +} diff --git a/core/camel-core/src/test/java/org/apache/camel/processor/SplitterSplitResultTest.java b/core/camel-core/src/test/java/org/apache/camel/processor/SplitterSplitResultTest.java index 012e6d75e0d3a..c15b5440b39c8 100644 --- a/core/camel-core/src/test/java/org/apache/camel/processor/SplitterSplitResultTest.java +++ b/core/camel-core/src/test/java/org/apache/camel/processor/SplitterSplitResultTest.java @@ -80,6 +80,32 @@ void testSplitResultAllSuccess() throws Exception { assertTrue(splitResult.getFailures().isEmpty()); } + @Test + void testSplitResultInStreamingMode() throws Exception { + Exchange result = template.send("direct:streaming", + e -> e.getIn().setBody(Arrays.asList("a", "FAIL", "b", "c", "d"))); + + SplitResult splitResult = result.getProperty(Exchange.SPLIT_RESULT, SplitResult.class); + assertNotNull(splitResult, "SplitResult should be set in streaming mode"); + assertEquals(5, splitResult.getTotalItems()); + assertEquals(1, splitResult.getFailureCount()); + assertEquals(4, splitResult.getSuccessCount()); + assertFalse(splitResult.isAborted()); + } + + @Test + void testSplitResultWithParallelProcessing() throws Exception { + Exchange result = template.send("direct:parallel", + e -> e.getIn().setBody(Arrays.asList("a", "FAIL", "b", "c"))); + + SplitResult splitResult = result.getProperty(Exchange.SPLIT_RESULT, SplitResult.class); + assertNotNull(splitResult, "SplitResult should be set in parallel mode"); + assertEquals(4, splitResult.getTotalItems()); + assertEquals(1, splitResult.getFailureCount()); + assertEquals(3, splitResult.getSuccessCount()); + assertFalse(splitResult.isAborted()); + } + @Test void testNoSplitResultWithoutErrorThreshold() throws Exception { // plain split without error threshold should not set SplitResult @@ -118,6 +144,26 @@ public void configure() { from("direct:plain") .split(body()) .to("mock:split"); + + from("direct:streaming") + .split(body()).streaming().maxFailedRecords(10) + .process(exchange -> { + String body = exchange.getIn().getBody(String.class); + if ("FAIL".equals(body)) { + throw new IllegalArgumentException("Simulated failure for: " + body); + } + }) + .to("mock:split"); + + from("direct:parallel") + .split(body()).parallelProcessing().maxFailedRecords(10) + .process(exchange -> { + String body = exchange.getIn().getBody(String.class); + if ("FAIL".equals(body)) { + throw new IllegalArgumentException("Simulated failure for: " + body); + } + }) + .to("mock:split"); } }; } diff --git a/core/camel-core/src/test/java/org/apache/camel/processor/SplitterStreamingTest.java b/core/camel-core/src/test/java/org/apache/camel/processor/SplitterStreamingTest.java new file mode 100644 index 0000000000000..a2b1ad9feaabe --- /dev/null +++ b/core/camel-core/src/test/java/org/apache/camel/processor/SplitterStreamingTest.java @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.processor; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.camel.ContextTestSupport; +import org.apache.camel.Exchange; +import org.apache.camel.SplitResult; +import org.apache.camel.builder.RouteBuilder; +import org.apache.camel.component.mock.MockEndpoint; +import org.apache.camel.resume.ResumeStrategy; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class SplitterStreamingTest extends ContextTestSupport { + + private final Map store = new ConcurrentHashMap<>(); + private final ResumeStrategy strategy = new SplitterTestResumeStrategy(store); + + @Test + void testStreamingSplitResultTotalItems() throws Exception { + MockEndpoint mock = getMockEndpoint("mock:streaming"); + mock.expectedMessageCount(3); + + Exchange result = template.send("direct:streaming", + e -> e.getIn().setBody(Arrays.asList("a", "FAIL", "b", "c"))); + + mock.assertIsSatisfied(); + + SplitResult splitResult = result.getProperty(Exchange.SPLIT_RESULT, SplitResult.class); + assertNotNull(splitResult, "SplitResult should be set in streaming mode"); + assertEquals(4, splitResult.getTotalItems(), "totalItems should reflect all iterated items"); + assertEquals(1, splitResult.getFailureCount()); + assertEquals(3, splitResult.getSuccessCount()); + assertFalse(splitResult.isAborted()); + } + + @Test + void testStreamingSplitResultWhenAborted() throws Exception { + MockEndpoint mock = getMockEndpoint("mock:streaming-abort"); + mock.expectedMinimumMessageCount(0); + + Exchange result = template.send("direct:streaming-abort", + e -> e.getIn().setBody(Arrays.asList("FAIL", "FAIL", "a", "b"))); + + mock.assertIsSatisfied(); + + SplitResult splitResult = result.getProperty(Exchange.SPLIT_RESULT, SplitResult.class); + assertNotNull(splitResult, "SplitResult should be set even when streaming and aborted"); + assertTrue(splitResult.isAborted()); + assertEquals(2, splitResult.getFailureCount()); + // totalItems reflects how many were iterated before abort (at least 2) + assertTrue(splitResult.getTotalItems() >= 2, "totalItems should reflect items iterated before abort"); + } + + @Test + void testStreamingGrouping() throws Exception { + MockEndpoint mock = getMockEndpoint("mock:streaming-group"); + mock.expectedMessageCount(3); + + template.sendBody("direct:streaming-group", Arrays.asList("a", "b", "c", "d", "e", "f", "g")); + + mock.assertIsSatisfied(); + + assertInstanceOf(List.class, mock.getReceivedExchanges().get(0).getIn().getBody()); + assertEquals(3, ((List) mock.getReceivedExchanges().get(0).getIn().getBody()).size()); + assertEquals(1, ((List) mock.getReceivedExchanges().get(2).getIn().getBody()).size()); + } + + @Test + void testStreamingWatermark() throws Exception { + MockEndpoint mock = getMockEndpoint("mock:streaming-wm"); + mock.expectedMessageCount(5); + + template.sendBody("direct:streaming-wm", Arrays.asList("a", "b", "c", "d", "e")); + + mock.assertIsSatisfied(); + + assertEquals("4", store.get("streamJob")); + } + + @Test + void testStreamingWatermarkSecondRun() throws Exception { + store.put("streamJob", "2"); + + MockEndpoint mock = getMockEndpoint("mock:streaming-wm"); + mock.expectedMessageCount(2); + + template.sendBody("direct:streaming-wm", Arrays.asList("a", "b", "c", "d", "e")); + + mock.assertIsSatisfied(); + + assertEquals("d", mock.getReceivedExchanges().get(0).getIn().getBody(String.class)); + assertEquals("e", mock.getReceivedExchanges().get(1).getIn().getBody(String.class)); + assertEquals("4", store.get("streamJob")); + } + + @Test + void testStreamingParallelMaxFailedRecords() throws Exception { + // streaming + parallel + maxFailedRecords combination + MockEndpoint mock = getMockEndpoint("mock:streaming-parallel"); + mock.expectedMinimumMessageCount(0); + + Exchange result = template.send("direct:streaming-parallel", + e -> e.getIn().setBody(Arrays.asList("a", "FAIL", "b", "FAIL", "FAIL", "c"))); + + mock.assertIsSatisfied(); + + assertNotNull(result.getException(), "Should abort after 3 failures"); + + SplitResult splitResult = result.getProperty(Exchange.SPLIT_RESULT, SplitResult.class); + assertNotNull(splitResult); + assertTrue(splitResult.isAborted()); + assertTrue(splitResult.getFailureCount() >= 3); + } + + @Override + protected RouteBuilder createRouteBuilder() { + return new RouteBuilder() { + @Override + public void configure() { + from("direct:streaming") + .split(body()).streaming().maxFailedRecords(10) + .process(exchange -> { + if ("FAIL".equals(exchange.getIn().getBody(String.class))) { + throw new IllegalArgumentException("Simulated"); + } + }) + .to("mock:streaming"); + + from("direct:streaming-abort") + .split(body()).streaming().maxFailedRecords(2) + .process(exchange -> { + if ("FAIL".equals(exchange.getIn().getBody(String.class))) { + throw new IllegalArgumentException("Simulated"); + } + }) + .to("mock:streaming-abort"); + + from("direct:streaming-group") + .split(body()).streaming().group(3) + .to("mock:streaming-group"); + + from("direct:streaming-wm") + .split(body()).streaming().resumeStrategy(strategy, "streamJob") + .to("mock:streaming-wm"); + + from("direct:streaming-parallel") + .split(body()).streaming().parallelProcessing().maxFailedRecords(3) + .process(exchange -> { + if ("FAIL".equals(exchange.getIn().getBody(String.class))) { + throw new IllegalArgumentException("Simulated"); + } + }) + .to("mock:streaming-parallel"); + } + }; + } +} diff --git a/core/camel-core/src/test/java/org/apache/camel/processor/SplitterTestResumeStrategy.java b/core/camel-core/src/test/java/org/apache/camel/processor/SplitterTestResumeStrategy.java new file mode 100644 index 0000000000000..5771d8ca91859 --- /dev/null +++ b/core/camel-core/src/test/java/org/apache/camel/processor/SplitterTestResumeStrategy.java @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.processor; + +import java.util.Map; +import java.util.function.BiFunction; +import java.util.function.Function; + +import org.apache.camel.resume.Offset; +import org.apache.camel.resume.OffsetKey; +import org.apache.camel.resume.Resumable; +import org.apache.camel.resume.ResumeAdapter; +import org.apache.camel.resume.ResumeStrategy; +import org.apache.camel.resume.ResumeStrategyConfiguration; +import org.apache.camel.resume.cache.ResumeCache; + +/** + * A simple in-memory {@link ResumeStrategy} for Splitter watermark tests. Wraps a {@link Map} that tests can + * pre-populate and assert against. The cache is a live view backed by the store, so changes to the store are + * immediately visible. + */ +class SplitterTestResumeStrategy implements ResumeStrategy { + + private final Map store; + private final ResumeStrategyConfiguration config; + + SplitterTestResumeStrategy(Map store) { + this.store = store; + ResumeCache cache = new LiveStoreCache(store); + ResumeStrategyConfiguration cfg = new ResumeStrategyConfiguration() { + @Override + public String resumeStrategyService() { + return "test"; + } + }; + cfg.setResumeCache(cache); + this.config = cfg; + } + + @Override + public void loadCache() { + // no-op: cache is a live view of the store + } + + @Override + public void updateLastOffset(OffsetKey offsetKey, Offset offsetValue) { + store.put(offsetKey.getValue().toString(), offsetValue.getValue().toString()); + } + + @Override + public void updateLastOffset(T offset) { + // no-op + } + + @Override + public void updateLastOffset(T offset, UpdateCallBack updateCallBack) { + // no-op + } + + @Override + public void updateLastOffset(OffsetKey offsetKey, Offset offset, UpdateCallBack updateCallBack) + throws Exception { + updateLastOffset(offsetKey, offset); + } + + @Override + public void setAdapter(ResumeAdapter adapter) { + // no-op + } + + @Override + public ResumeAdapter getAdapter() { + return null; + } + + @Override + public void setResumeStrategyConfiguration(ResumeStrategyConfiguration resumeStrategyConfiguration) { + // no-op + } + + @Override + public ResumeStrategyConfiguration getResumeStrategyConfiguration() { + return config; + } + + @Override + public void start() { + // no-op + } + + @Override + public void stop() { + // no-op + } + + /** + * A {@link ResumeCache} that directly delegates to the backing store map, providing a live view. + */ + private static class LiveStoreCache implements ResumeCache { + private final Map store; + + LiveStoreCache(Map store) { + this.store = store; + } + + @Override + public Object get(Object key) { + return store.get(String.valueOf(key)); + } + + @Override + @SuppressWarnings("unchecked") + public T get(Object key, Class clazz) { + Object value = get(key); + return value != null ? clazz.cast(value) : null; + } + + @Override + public void add(Object key, Object offsetValue) { + store.put(String.valueOf(key), String.valueOf(offsetValue)); + } + + @Override + public boolean contains(Object key, Object entry) { + Object value = get(key); + return value != null && value.equals(entry); + } + + @Override + public Object computeIfAbsent(Object key, Function mapping) { + return store.computeIfAbsent(String.valueOf(key), k -> String.valueOf(mapping.apply(k))); + } + + @Override + public Object computeIfPresent(Object key, BiFunction remapping) { + return store.computeIfPresent(String.valueOf(key), + (k, v) -> String.valueOf(remapping.apply(k, v))); + } + + @Override + public boolean isFull() { + return false; + } + + @Override + public long capacity() { + return Integer.MAX_VALUE; + } + + @Override + public void forEach(BiFunction action) { + for (Map.Entry e : store.entrySet()) { + if (!action.apply(e.getKey(), e.getValue())) { + break; + } + } + } + } +} diff --git a/core/camel-core/src/test/java/org/apache/camel/processor/SplitterTransactedTest.java b/core/camel-core/src/test/java/org/apache/camel/processor/SplitterTransactedTest.java index 1bb78546bd9c8..8f5dab810e236 100644 --- a/core/camel-core/src/test/java/org/apache/camel/processor/SplitterTransactedTest.java +++ b/core/camel-core/src/test/java/org/apache/camel/processor/SplitterTransactedTest.java @@ -26,6 +26,7 @@ import org.apache.camel.SplitResult; import org.apache.camel.builder.RouteBuilder; import org.apache.camel.component.mock.MockEndpoint; +import org.apache.camel.resume.ResumeStrategy; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -40,6 +41,7 @@ class SplitterTransactedTest extends ContextTestSupport { private final Map store = new ConcurrentHashMap<>(); + private final ResumeStrategy strategy = new SplitterTestResumeStrategy(store); @Test void testGroupTransacted() throws Exception { @@ -197,12 +199,12 @@ public void configure() { .to("mock:result"); from("direct:watermark") - .split(body()).watermarkStore(store, "txJob") + .split(body()).resumeStrategy(strategy, "txJob") .to("mock:watermark"); from("direct:valuewm") .split(body()) - .watermarkStore(store, "txDateJob") + .resumeStrategy(strategy, "txDateJob") .watermarkExpression("${body}") .to("mock:valuewm"); } diff --git a/core/camel-core/src/test/java/org/apache/camel/processor/SplitterValidationTest.java b/core/camel-core/src/test/java/org/apache/camel/processor/SplitterValidationTest.java new file mode 100644 index 0000000000000..d13ed1c7e2c04 --- /dev/null +++ b/core/camel-core/src/test/java/org/apache/camel/processor/SplitterValidationTest.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.processor; + +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.camel.ContextTestSupport; +import org.apache.camel.builder.RouteBuilder; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class SplitterValidationTest extends ContextTestSupport { + + @Test + void testErrorThresholdNegativeIsRejected() { + Exception e = assertThrows(Exception.class, () -> context.addRoutes(new RouteBuilder() { + @Override + public void configure() { + from("direct:start") + .split(body()).errorThreshold(-0.1) + .to("mock:split"); + } + })); + assertTrue(findIllegalArgumentException(e, "errorThreshold")); + } + + @Test + void testErrorThresholdAboveOneIsRejected() { + Exception e = assertThrows(Exception.class, () -> context.addRoutes(new RouteBuilder() { + @Override + public void configure() { + from("direct:start") + .split(body()).errorThreshold(1.5) + .to("mock:split"); + } + })); + assertTrue(findIllegalArgumentException(e, "errorThreshold")); + } + + @Test + void testMaxFailedRecordsNegativeIsRejected() { + Exception e = assertThrows(Exception.class, () -> context.addRoutes(new RouteBuilder() { + @Override + public void configure() { + from("direct:start") + .split(body()).maxFailedRecords(-1) + .to("mock:split"); + } + })); + assertTrue(findIllegalArgumentException(e, "maxFailedRecords")); + } + + @Test + void testWatermarkKeyWithoutResumeStrategyIsRejected() { + Exception e = assertThrows(Exception.class, () -> context.addRoutes(new RouteBuilder() { + @Override + public void configure() { + from("direct:start") + .split(body()).watermarkKey("someKey") + .to("mock:split"); + } + })); + assertTrue(findIllegalArgumentException(e, "resumeStrategy")); + } + + @Test + void testResumeStrategyWithoutKeyIsRejected() { + context.getRegistry().bind("myStrategy", + new SplitterTestResumeStrategy(new ConcurrentHashMap<>())); + Exception e = assertThrows(Exception.class, () -> context.addRoutes(new RouteBuilder() { + @Override + public void configure() { + from("direct:start") + .split(body()).resumeStrategy("myStrategy") + .to("mock:split"); + } + })); + assertTrue(findIllegalArgumentException(e, "resumeStrategy")); + } + + @Test + void testWatermarkExpressionWithoutResumeStrategyIsRejected() { + Exception e = assertThrows(Exception.class, () -> context.addRoutes(new RouteBuilder() { + @Override + public void configure() { + from("direct:start") + .split(body()).watermarkExpression("${body}") + .to("mock:split"); + } + })); + assertTrue(findIllegalArgumentException(e, "watermarkExpression")); + } + + private boolean findIllegalArgumentException(Throwable e, String keyword) { + while (e != null) { + if (e instanceof IllegalArgumentException && e.getMessage() != null && e.getMessage().contains(keyword)) { + return true; + } + e = e.getCause(); + } + return false; + } +} diff --git a/core/camel-core/src/test/java/org/apache/camel/processor/SplitterWatermarkTest.java b/core/camel-core/src/test/java/org/apache/camel/processor/SplitterWatermarkTest.java index 81fee77d43a1f..6bb6bb779b64a 100644 --- a/core/camel-core/src/test/java/org/apache/camel/processor/SplitterWatermarkTest.java +++ b/core/camel-core/src/test/java/org/apache/camel/processor/SplitterWatermarkTest.java @@ -17,6 +17,7 @@ package org.apache.camel.processor; import java.util.Arrays; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -24,6 +25,7 @@ import org.apache.camel.Exchange; import org.apache.camel.builder.RouteBuilder; import org.apache.camel.component.mock.MockEndpoint; +import org.apache.camel.resume.ResumeStrategy; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -32,6 +34,7 @@ class SplitterWatermarkTest extends ContextTestSupport { private final Map store = new ConcurrentHashMap<>(); + private final ResumeStrategy strategy = new SplitterTestResumeStrategy(store); @Test void testIndexBasedWatermarkFirstRun() throws Exception { @@ -120,17 +123,81 @@ void testValueBasedWatermarkWithPreviousValue() throws Exception { assertEquals("2024-01-04", store.get("dateJob")); } + @Test + void testValueBasedWatermarkWithParallelProcessing() throws Exception { + MockEndpoint mock = getMockEndpoint("mock:parallel-wm"); + mock.expectedMessageCount(5); + + template.sendBody("direct:parallel-value", + Arrays.asList("val-0", "val-1", "val-2", "val-3", "val-4")); + + mock.assertIsSatisfied(); + + // In parallel mode, the watermark should be from the highest-indexed item (val-4) + assertEquals("val-4", store.get("parallelJob")); + } + + @Test + void testIndexBasedWatermarkWithGroup() throws Exception { + // 10 items, group=3 → 4 chunks: [a,b,c], [d,e,f], [g,h,i], [j] + // Watermark should store "9" (last raw item index), not "3" (last chunk index) + MockEndpoint mock = getMockEndpoint("mock:group-wm"); + mock.expectedMessageCount(4); + + template.sendBody("direct:group-index", + Arrays.asList("a", "b", "c", "d", "e", "f", "g", "h", "i", "j")); + + mock.assertIsSatisfied(); + + assertEquals("9", store.get("groupJob"), "Watermark should be last raw item index, not chunk index"); + } + + @Test + void testIndexBasedWatermarkWithGroupSecondRun() throws Exception { + // Pre-populate watermark at 4 → skip items 0-4 (a,b,c,d,e) + // Remaining: f,g,h,i,j → chunks [f,g,h], [i,j] + store.put("groupJob", "4"); + + MockEndpoint mock = getMockEndpoint("mock:group-wm"); + mock.expectedMessageCount(2); + + template.sendBody("direct:group-index", + Arrays.asList("a", "b", "c", "d", "e", "f", "g", "h", "i", "j")); + + mock.assertIsSatisfied(); + + // first chunk should be [f,g,h] + List firstChunk = mock.getReceivedExchanges().get(0).getIn().getBody(List.class); + assertEquals(Arrays.asList("f", "g", "h"), firstChunk); + + // watermark updated to 9 (last raw item index) + assertEquals("9", store.get("groupJob")); + } + + @Test + void testIndexBasedWatermarkWithParallelProcessing() throws Exception { + MockEndpoint mock = getMockEndpoint("mock:parallel-idx"); + mock.expectedMessageCount(5); + + template.sendBody("direct:parallel-index", + Arrays.asList("a", "b", "c", "d", "e")); + + mock.assertIsSatisfied(); + + assertEquals("4", store.get("parallelIdxJob")); + } + @Override protected RouteBuilder createRouteBuilder() { return new RouteBuilder() { @Override public void configure() { from("direct:index") - .split(body()).watermarkStore(store, "testJob") + .split(body()).resumeStrategy(strategy, "testJob") .to("mock:split"); from("direct:index-abort") - .split(body()).watermarkStore(store, "testJob2").maxFailedRecords(1) + .split(body()).resumeStrategy(strategy, "testJob2").maxFailedRecords(1) .process(exchange -> { String body = exchange.getIn().getBody(String.class); if ("FAIL".equals(body)) { @@ -141,9 +208,24 @@ public void configure() { from("direct:value") .split(body()) - .watermarkStore(store, "dateJob") + .resumeStrategy(strategy, "dateJob") .watermarkExpression("${body}") .to("mock:split3"); + + from("direct:group-index") + .split(body()).group(3).resumeStrategy(strategy, "groupJob") + .to("mock:group-wm"); + + from("direct:parallel-value") + .split(body()).parallelProcessing() + .resumeStrategy(strategy, "parallelJob") + .watermarkExpression("${body}") + .to("mock:parallel-wm"); + + from("direct:parallel-index") + .split(body()).parallelProcessing() + .resumeStrategy(strategy, "parallelIdxJob") + .to("mock:parallel-idx"); } }; } diff --git a/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_19.adoc b/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_19.adoc index 2e505a6ecd73e..5169509aeac0a 100644 --- a/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_19.adoc +++ b/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_19.adoc @@ -45,6 +45,27 @@ uses `InOnly` pattern. Removed 2 deprecated methods in Java DSL for `throttler` EIP. +==== Split EIP enhancements + +The Split EIP has been enhanced with the following new features: + +* **Chunking with `group`**: Groups N split items together into a single message with a `java.util.List` body, + allowing batch processing of split items. +* **Error thresholds**: Two new options `maxFailedRecords` (max failure count) and `errorThreshold` + (max failure ratio 0.0-1.0) allow the splitter to continue through individual failures while stopping + when error rates become unacceptable. These are mutually exclusive with `stopOnException`. +* **SplitResult**: When error thresholds are configured, a `SplitResult` object is available as the + `CamelSplitResult` exchange property after the split completes, providing total items, success/failure + counts, and individual failure details (index and exception). +* **Watermark tracking**: New options `resumeStrategy`, `watermarkKey`, and `watermarkExpression` enable + incremental processing using Camel's `ResumeStrategy` SPI to track progress. Index-based watermarking + automatically skips previously processed items; value-based watermarking uses a custom expression + to extract watermark values. The previous watermark value is exposed as the `CamelSplitWatermark` + exchange property. Any `ResumeStrategy` implementation can be used — from in-memory for testing to + Kafka-backed for production. + +See the xref:components:eips:split-eip.adoc[Split EIP] documentation for details and examples. + ==== ErrorRegistry A new `ErrorRegistry` SPI has been added to `CamelContext` for capturing routing errors. diff --git a/dsl/camel-yaml-dsl/camel-yaml-dsl-deserializers/src/generated/java/org/apache/camel/dsl/yaml/deserializers/ModelDeserializers.java b/dsl/camel-yaml-dsl/camel-yaml-dsl-deserializers/src/generated/java/org/apache/camel/dsl/yaml/deserializers/ModelDeserializers.java index 8109b3440f0b2..295f2c45d98e3 100644 --- a/dsl/camel-yaml-dsl/camel-yaml-dsl-deserializers/src/generated/java/org/apache/camel/dsl/yaml/deserializers/ModelDeserializers.java +++ b/dsl/camel-yaml-dsl/camel-yaml-dsl-deserializers/src/generated/java/org/apache/camel/dsl/yaml/deserializers/ModelDeserializers.java @@ -16843,6 +16843,7 @@ protected boolean setProperty(SpELExpression target, String propertyKey, @YamlProperty(name = "onPrepare", type = "string", description = "Uses the Processor when preparing the org.apache.camel.Exchange to be sent. This can be used to deep-clone messages that should be sent, or any custom logic needed before the exchange is sent.", displayName = "On Prepare"), @YamlProperty(name = "parallelAggregate", type = "boolean", deprecated = true, defaultValue = "false", description = "If enabled then the aggregate method on AggregationStrategy can be called concurrently. Notice that this would require the implementation of AggregationStrategy to be implemented as thread-safe. By default this is false meaning that Camel synchronizes the call to the aggregate method. Though in some use-cases this can be used to archive higher performance when the AggregationStrategy is implemented as thread-safe.", displayName = "Parallel Aggregate"), @YamlProperty(name = "parallelProcessing", type = "boolean", defaultValue = "false", description = "If enabled then processing each split messages occurs concurrently. Note the caller thread will still wait until all messages has been fully processed, before it continues. It's only processing the sub messages from the splitter which happens concurrently. When parallel processing is enabled, then the Camel routing engin will continue processing using last used thread from the parallel thread pool. However, if you want to use the original thread that called the splitter, then make sure to enable the synchronous option as well. In parallel processing mode, you may want to also synchronous = true to force this EIP to process the sub-tasks using the upper bounds of the thread-pool. If using synchronous = false then Camel will allow its reactive routing engine to use as many threads as possible, which may be available due to sub-tasks using other thread-pools such as CompletableFuture.runAsync or others.", displayName = "Parallel Processing"), + @YamlProperty(name = "resumeStrategy", type = "string", description = "Sets a reference to a ResumeStrategy in the registry for resume-from-last-position support.", displayName = "Resume Strategy"), @YamlProperty(name = "shareUnitOfWork", type = "boolean", defaultValue = "false", description = "Shares the org.apache.camel.spi.UnitOfWork with the parent and each of the sub messages. Splitter will by default not share unit of work between the parent exchange and each split exchange. This means each split exchange has its own individual unit of work.", displayName = "Share Unit Of Work"), @YamlProperty(name = "steps", type = "array:org.apache.camel.model.ProcessorDefinition"), @YamlProperty(name = "stopOnException", type = "boolean", defaultValue = "false", description = "Will now stop further processing if an exception or failure occurred during processing of an org.apache.camel.Exchange and the caused exception will be thrown. Will also stop if processing the exchange failed (has a fault message) or an exception was thrown and handled by the error handler (such as using onException). In all situations the splitter will stop further processing. This is the same behavior as in pipeline, which is used by the routing engine. The default behavior is to not stop but continue processing till the end", displayName = "Stop On Exception"), @@ -16850,8 +16851,7 @@ protected boolean setProperty(SpELExpression target, String propertyKey, @YamlProperty(name = "synchronous", type = "boolean", defaultValue = "false", description = "Sets whether synchronous processing should be strictly used. When enabled then the same thread is used to continue routing after the split is complete, even if parallel processing is enabled.", displayName = "Synchronous"), @YamlProperty(name = "timeout", type = "string", defaultValue = "0", description = "Sets a total timeout specified in millis, when using parallel processing. If the Splitter hasn't been able to send and process all replies within the given timeframe, then the timeout triggers and the Splitter breaks out and continues. The timeout method is invoked before breaking out. If the timeout is reached with running tasks still remaining, certain tasks for which it is difficult for Camel to shut down in a graceful manner may continue to run. So use this option with a bit of care.", displayName = "Timeout"), @YamlProperty(name = "watermarkExpression", type = "string", description = "Sets a Simple expression to evaluate on the exchange after split completion to determine the new watermark value.", displayName = "Watermark Expression"), - @YamlProperty(name = "watermarkKey", type = "string", description = "Sets the key to use in the watermark store.", displayName = "Watermark Key"), - @YamlProperty(name = "watermarkStore", type = "string", description = "Sets a reference to a watermark store (a Map ) in the registry.", displayName = "Watermark Store") + @YamlProperty(name = "watermarkKey", type = "string", description = "Sets the key to use in the watermark store.", displayName = "Watermark Key") } ) public static class SplitDefinitionDeserializer extends YamlDeserializerBase { @@ -16934,6 +16934,11 @@ protected boolean setProperty(SplitDefinition target, String propertyKey, target.setParallelProcessing(val); break; } + case "resumeStrategy": { + String val = asText(node); + target.setResumeStrategy(val); + break; + } case "shareUnitOfWork": { String val = asText(node); target.setShareUnitOfWork(val); @@ -16969,11 +16974,6 @@ protected boolean setProperty(SplitDefinition target, String propertyKey, target.setWatermarkKey(val); break; } - case "watermarkStore": { - String val = asText(node); - target.setWatermarkStore(val); - break; - } case "id": { String val = asText(node); target.setId(val); From 2de20cd6fb70958adba8320f8141267ae24cef3b Mon Sep 17 00:00:00 2001 From: Guillaume Nodet Date: Sat, 28 Mar 2026 10:29:06 +0100 Subject: [PATCH 8/9] CAMEL-23264: Improve javadoc, logging, and docs for Splitter enhancements - Clarify SplitResult.totalItems counts chunks (not individual items) when group() is used - Add LOG.debug in readFromStrategyCache() catch block for debuggability - Document that custom AggregationStrategy won't see individual item exceptions when error thresholds are configured (exceptions cleared after recording) - Update SplitResult code example to note chunk counting with group() Co-Authored-By: Claude Opus 4.6 --- .../src/main/java/org/apache/camel/SplitResult.java | 3 ++- .../src/main/docs/modules/eips/pages/split-eip.adoc | 6 +++++- .../src/main/java/org/apache/camel/processor/Splitter.java | 7 ++++++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/core/camel-api/src/main/java/org/apache/camel/SplitResult.java b/core/camel-api/src/main/java/org/apache/camel/SplitResult.java index a859e7d8b8461..b04b06acae827 100644 --- a/core/camel-api/src/main/java/org/apache/camel/SplitResult.java +++ b/core/camel-api/src/main/java/org/apache/camel/SplitResult.java @@ -49,7 +49,8 @@ public SplitResult(int totalItems, int failureCount, List failures, boo } /** - * The total number of items that were prepared for splitting. + * The total number of items that were prepared for splitting. When {@code group()} is used, this counts the number + * of chunks (groups), not the number of individual elements within them. */ public int getTotalItems() { return totalItems; diff --git a/core/camel-core-engine/src/main/docs/modules/eips/pages/split-eip.adoc b/core/camel-core-engine/src/main/docs/modules/eips/pages/split-eip.adoc index 04ecdd891f5b8..821e4cf283657 100644 --- a/core/camel-core-engine/src/main/docs/modules/eips/pages/split-eip.adoc +++ b/core/camel-core-engine/src/main/docs/modules/eips/pages/split-eip.adoc @@ -1221,6 +1221,10 @@ threshold is exceeded. IMPORTANT: The `stopOnException` option is mutually exclusive with `maxFailedRecords` and `errorThreshold`. You cannot use `stopOnException` together with either of these options. +NOTE: When error thresholds are configured, individual item exceptions are cleared from sub-exchanges +after being recorded in the `SplitResult`. This means a custom `AggregationStrategy` will not see +individual item exceptions — use the `SplitResult` exchange property to access failure details instead. + NOTE: When using `errorThreshold` with `parallelProcessing`, the failure ratio may vary slightly between runs because the ratio is calculated as failures are reported, and the order in which parallel items complete is non-deterministic. For deterministic abort behavior with parallel @@ -1239,7 +1243,7 @@ Exchange result = template.send("direct:start", SplitResult splitResult = result.getProperty(Exchange.SPLIT_RESULT, SplitResult.class); if (splitResult != null) { - int total = splitResult.getTotalItems(); // total items processed + int total = splitResult.getTotalItems(); // total items (or chunks when group() is used) int success = splitResult.getSuccessCount(); // successful items int failures = splitResult.getFailureCount(); // failed items boolean aborted = splitResult.isAborted(); // true if a threshold was exceeded diff --git a/core/camel-core-processor/src/main/java/org/apache/camel/processor/Splitter.java b/core/camel-core-processor/src/main/java/org/apache/camel/processor/Splitter.java index 8cda2374b9849..6af230fc3b69d 100644 --- a/core/camel-core-processor/src/main/java/org/apache/camel/processor/Splitter.java +++ b/core/camel-core-processor/src/main/java/org/apache/camel/processor/Splitter.java @@ -56,6 +56,8 @@ import org.apache.camel.support.service.ServiceHelper; import org.apache.camel.util.IOHelper; import org.apache.camel.util.StringHelper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import static org.apache.camel.util.ObjectHelper.notNull; @@ -65,6 +67,8 @@ */ public class Splitter extends MulticastProcessor { + private static final Logger LOG = LoggerFactory.getLogger(Splitter.class); + private static final String IGNORE_DELIMITER_MARKER = "false"; private static final String SINGLE_DELIMITER_MARKER = "single"; private static final String SPLIT_FAILURE_TRACKER = "CamelSplitFailureTracker"; @@ -168,7 +172,8 @@ private String readFromStrategyCache() { } } } catch (Exception e) { - // best-effort + LOG.debug("Failed to read watermark from resume strategy cache for key '{}': {}", + watermarkKey, e.getMessage(), e); } return null; } From 9dd3a7961493f52da414e5eb6110fd78d355d304 Mon Sep 17 00:00:00 2001 From: Guillaume Nodet Date: Sat, 28 Mar 2026 10:52:24 +0100 Subject: [PATCH 9/9] =?UTF-8?q?CAMEL-23264:=20Regenerate=20generated=20fil?= =?UTF-8?q?es=20for=20watermarkStore=E2=86=92resumeStrategy=20rename?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix stale watermarkStore references in XSD schemas, ModelParser, and ModelWriter files that were generated before the field was renamed to resumeStrategy. Co-Authored-By: Claude Opus 4.6 --- .../org/apache/camel/catalog/schemas/camel-spring.xsd | 4 ++-- .../org/apache/camel/catalog/schemas/camel-xml-io.xsd | 4 ++-- .../generated/java/org/apache/camel/xml/in/ModelParser.java | 2 +- .../generated/java/org/apache/camel/xml/out/ModelWriter.java | 2 +- .../generated/java/org/apache/camel/yaml/out/ModelWriter.java | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/schemas/camel-spring.xsd b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/schemas/camel-spring.xsd index 49d161bc57c43..59ab26c88d6b8 100644 --- a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/schemas/camel-spring.xsd +++ b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/schemas/camel-spring.xsd @@ -13984,11 +13984,11 @@ Sets the maximum number of failed records before aborting the split operation. - + diff --git a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/schemas/camel-xml-io.xsd b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/schemas/camel-xml-io.xsd index 0313bc934413d..82e2108576647 100644 --- a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/schemas/camel-xml-io.xsd +++ b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/schemas/camel-xml-io.xsd @@ -12668,11 +12668,11 @@ Sets the maximum number of failed records before aborting the split operation. - + diff --git a/core/camel-xml-io/src/generated/java/org/apache/camel/xml/in/ModelParser.java b/core/camel-xml-io/src/generated/java/org/apache/camel/xml/in/ModelParser.java index 7b535a5df8c78..f1fc566d15abb 100644 --- a/core/camel-xml-io/src/generated/java/org/apache/camel/xml/in/ModelParser.java +++ b/core/camel-xml-io/src/generated/java/org/apache/camel/xml/in/ModelParser.java @@ -1140,6 +1140,7 @@ protected SplitDefinition doParseSplitDefinition() throws IOException, XmlPullPa case "onPrepare": def.setOnPrepare(val); yield true; case "parallelAggregate": def.setParallelAggregate(val); yield true; case "parallelProcessing": def.setParallelProcessing(val); yield true; + case "resumeStrategy": def.setResumeStrategy(val); yield true; case "shareUnitOfWork": def.setShareUnitOfWork(val); yield true; case "stopOnException": def.setStopOnException(val); yield true; case "streaming": def.setStreaming(val); yield true; @@ -1147,7 +1148,6 @@ protected SplitDefinition doParseSplitDefinition() throws IOException, XmlPullPa case "timeout": def.setTimeout(val); yield true; case "watermarkExpression": def.setWatermarkExpression(val); yield true; case "watermarkKey": def.setWatermarkKey(val); yield true; - case "watermarkStore": def.setWatermarkStore(val); yield true; default: yield processorDefinitionAttributeHandler().accept(def, key, val); }, outputExpressionNodeElementHandler(), noValueHandler()); } diff --git a/core/camel-xml-io/src/generated/java/org/apache/camel/xml/out/ModelWriter.java b/core/camel-xml-io/src/generated/java/org/apache/camel/xml/out/ModelWriter.java index 3df1aa815b709..cd1bafb03536e 100644 --- a/core/camel-xml-io/src/generated/java/org/apache/camel/xml/out/ModelWriter.java +++ b/core/camel-xml-io/src/generated/java/org/apache/camel/xml/out/ModelWriter.java @@ -1761,7 +1761,7 @@ protected void doWriteSplitDefinition(String name, SplitDefinition def) throws I doWriteAttribute("group", def.getGroup(), null); doWriteAttribute("errorThreshold", def.getErrorThreshold(), null); doWriteAttribute("maxFailedRecords", def.getMaxFailedRecords(), null); - doWriteAttribute("watermarkStore", def.getWatermarkStore(), null); + doWriteAttribute("resumeStrategy", def.getResumeStrategy(), null); doWriteAttribute("watermarkKey", def.getWatermarkKey(), null); doWriteAttribute("watermarkExpression", def.getWatermarkExpression(), null); doWriteOutputExpressionNodeElements(def); diff --git a/core/camel-yaml-io/src/generated/java/org/apache/camel/yaml/out/ModelWriter.java b/core/camel-yaml-io/src/generated/java/org/apache/camel/yaml/out/ModelWriter.java index 57f735fc5360a..bed17d72673af 100644 --- a/core/camel-yaml-io/src/generated/java/org/apache/camel/yaml/out/ModelWriter.java +++ b/core/camel-yaml-io/src/generated/java/org/apache/camel/yaml/out/ModelWriter.java @@ -1761,7 +1761,7 @@ protected void doWriteSplitDefinition(String name, SplitDefinition def) throws I doWriteAttribute("group", def.getGroup(), null); doWriteAttribute("errorThreshold", def.getErrorThreshold(), null); doWriteAttribute("maxFailedRecords", def.getMaxFailedRecords(), null); - doWriteAttribute("watermarkStore", def.getWatermarkStore(), null); + doWriteAttribute("resumeStrategy", def.getResumeStrategy(), null); doWriteAttribute("watermarkKey", def.getWatermarkKey(), null); doWriteAttribute("watermarkExpression", def.getWatermarkExpression(), null); doWriteOutputExpressionNodeElements(def);