From 52840c7417def2747ccece07e700573990606a80 Mon Sep 17 00:00:00 2001 From: Zihan Dai <1436286758@qq.com> Date: Wed, 1 Apr 2026 03:19:21 +1100 Subject: [PATCH] Fix last_by returning null row after deleting all table data (#16985) After deleting all data of a device in table model, queries like `select last_by(s0, time)` could still return one row with null values instead of returning no rows. Three related issues were fixed: 1. LastQueryAggTableScanOperator: skip emitting a result row when all aggregators are LastByAccumulator and none has an initialized result. Applied to both the normal aggregation path and the cache-hit path. 2. FileLoaderUtils: read time-column modifications using AlignedFullPath.VECTOR_PLACEHOLDER instead of the concrete measurement ID, matching aligned deletion semantics used in other paths. 3. ModificationUtils: set modified=true when an aligned value chunk is fully deleted, preventing statistics-based shortcuts from using stale metadata. Signed-off-by: Zihan Dai <1436286758@qq.com> --- .../it/db/it/IoTDBDeletionTableIT.java | 80 +++++++++++++++++++ .../operator/source/FileLoaderUtils.java | 2 +- .../LastQueryAggTableScanOperator.java | 41 +++++++++- .../iotdb/db/utils/ModificationUtils.java | 1 + 4 files changed, 121 insertions(+), 3 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java index e6939c5226a00..4b2c8ed5e8cd3 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java @@ -476,6 +476,86 @@ public void testSuccessfullyInvalidateCache() throws SQLException { cleanData(4); } + @Test + public void testLastByWithTimeShouldReturnNoRowsAfterDeletingAllData() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use test"); + statement.execute("create table last_by_delete_0(deviceId string tag, s0 int32 field)"); + statement.execute( + "insert into last_by_delete_0(time, deviceId, s0) values (1, 'd0', 1)"); + statement.execute("flush"); + + try (ResultSet resultSet = + statement.executeQuery( + "select last_by(s0, time) from last_by_delete_0 where deviceId = 'd0'")) { + assertTrue(resultSet.next()); + } + + statement.execute("delete from last_by_delete_0 where deviceId = 'd0'"); + + try (ResultSet resultSet = + statement.executeQuery( + "select last_by(s0, time) from last_by_delete_0 where deviceId = 'd0'")) { + assertResultSetEmpty(resultSet); + } + + try (ResultSet resultSet = + statement.executeQuery( + "select last_by(s0, time) from last_by_delete_0 where deviceId = 'd0'")) { + assertResultSetEmpty(resultSet); + } + } + } + + @Test + public void testThreeArgLastByShouldReturnNoRowsAfterDeletingAllData() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use test"); + statement.execute("create table last_by_delete_1(deviceId string tag, s0 int32 field)"); + statement.execute( + "insert into last_by_delete_1(time, deviceId, s0) values (1, 'd0', 1)"); + statement.execute("flush"); + + try (ResultSet resultSet = + statement.executeQuery( + "select last_by(time, s0, time) from last_by_delete_1 where deviceId = 'd0'")) { + assertTrue(resultSet.next()); + } + + statement.execute("delete from last_by_delete_1 where deviceId = 'd0'"); + + try (ResultSet resultSet = + statement.executeQuery( + "select last_by(time, s0, time) from last_by_delete_1 where deviceId = 'd0'")) { + assertResultSetEmpty(resultSet); + } + + try (ResultSet resultSet = + statement.executeQuery( + "select last_by(time, s0, time) from last_by_delete_1 where deviceId = 'd0'")) { + assertResultSetEmpty(resultSet); + } + } + } + + private void assertResultSetEmpty(ResultSet resultSet) throws SQLException { + if (!resultSet.next()) { + return; + } + + StringBuilder rowBuilder = new StringBuilder(); + int columnCount = resultSet.getMetaData().getColumnCount(); + for (int i = 1; i <= columnCount; i++) { + if (i > 1) { + rowBuilder.append(", "); + } + rowBuilder.append(resultSet.getString(i)); + } + fail("Expected no rows, but got: " + rowBuilder); + } + @Test public void testFullDeleteWithoutWhereClause() throws SQLException { prepareData(5, 1); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java index 254c061187a43..7c0c8b1afb0a8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java @@ -370,7 +370,7 @@ private static AbstractAlignedTimeSeriesMetadata setModifications( // deal with time column List timeModifications = context.getPathModifications( - resource, alignedPath.getDeviceId(), timeColumnMetadata.getMeasurementId()); + resource, alignedPath.getDeviceId(), AlignedFullPath.VECTOR_PLACEHOLDER); // all rows are deleted, just return null to skip device data in this file if (ModificationUtils.isAllDeletedByMods( timeModifications, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/LastQueryAggTableScanOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/LastQueryAggTableScanOperator.java index 8b4719bc1f13f..f80ef1fac068c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/LastQueryAggTableScanOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/LastQueryAggTableScanOperator.java @@ -24,6 +24,7 @@ import org.apache.iotdb.db.queryengine.execution.fragment.DataNodeQueryContext; import org.apache.iotdb.db.queryengine.execution.operator.process.last.LastQueryUtil; import org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.LastAccumulator; +import org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.LastByAccumulator; import org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.LastByDescAccumulator; import org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.LastDescAccumulator; import org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.TableAggregator; @@ -323,7 +324,9 @@ private void buildResultUseLastValuesCache() { // when there is no data, no need to append result if the query is GROUP BY or output of // aggregator is partial (final operator will produce NULL result) if (timeLastValue == EMPTY_PRIMITIVE_TYPE - && (groupingKeySize != 0 || tableAggregators.get(0).getStep().isOutputPartial())) { + && (groupingKeySize != 0 + || tableAggregators.get(0).getStep().isOutputPartial() + || shouldSkipEmptyLastByResult(currentHitResult))) { outputDeviceIndex++; currentHitCacheIndex++; return; @@ -706,7 +709,9 @@ private void checkIfUpdated( @Override protected void updateResultTsBlock() { - appendAggregationResult(); + if (!shouldSkipEmptyLastByResult()) { + appendAggregationResult(); + } if (needUpdateCache && timeIterator.hasCachedTimeRange()) { if (lastRowCacheResults != null) { @@ -723,6 +728,38 @@ protected void updateResultTsBlock() { resetTableAggregators(); } + private boolean shouldSkipEmptyLastByResult() { + if (groupingKeySize != 0 || tableAggregators.isEmpty()) { + return false; + } + for (TableAggregator aggregator : tableAggregators) { + if (!(aggregator.getAccumulator() instanceof LastByAccumulator)) { + return false; + } + if (((LastByAccumulator) aggregator.getAccumulator()).hasInitResult()) { + return false; + } + } + return true; + } + + private boolean shouldSkipEmptyLastByResult(TimeValuePair[] currentHitResult) { + if (groupingKeySize != 0 || tableAggregators.isEmpty()) { + return false; + } + for (TableAggregator aggregator : tableAggregators) { + if (!(aggregator.getAccumulator() instanceof LastByAccumulator)) { + return false; + } + } + for (TimeValuePair timeValuePair : currentHitResult) { + if (timeValuePair.getValue() != EMPTY_PRIMITIVE_TYPE) { + return false; + } + } + return true; + } + @Override String getNthIdColumnValue(DeviceEntry deviceEntry, int idColumnIndex) { // +1 for skipping the table name segment diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java index 351e64650a7d5..571f836a4fc7f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java @@ -140,6 +140,7 @@ private static boolean areAllValueColumnsDeleted( for (TimeRange range : valueChunkMetadata.getDeleteIntervalList()) { if (range.contains(valueChunkMetadata.getStartTime(), valueChunkMetadata.getEndTime())) { valueChunkMetadataList.set(i, null); + modified = true; currentRemoved = true; break; } else {