From e2e91d5a22eb3e37cc7bc189b80135fb9beae0cb Mon Sep 17 00:00:00 2001 From: JackieTien97 Date: Wed, 1 Apr 2026 19:20:48 +0800 Subject: [PATCH 1/8] First step by opus-4.6 --- .../query/recent/IoTExplainJsonFormatIT.java | 251 ++++++++++++ .../queryengine/common/MPPQueryContext.java | 10 + .../operator/ExplainAnalyzeOperator.java | 37 ++ ...ableModelStatementMemorySourceVisitor.java | 65 ++- .../plan/planner/TableOperatorGenerator.java | 7 +- .../plan/node/PlanGraphJsonPrinter.java | 220 +++++++++++ .../analyzer/StatementAnalyzer.java | 2 + .../planner/TableLogicalPlanner.java | 3 +- .../planner/node/ExplainAnalyzeNode.java | 39 +- .../plan/relational/sql/ast/Explain.java | 22 +- .../relational/sql/ast/ExplainAnalyze.java | 24 +- .../sql/ast/ExplainOutputFormat.java | 26 ++ .../relational/sql/parser/AstBuilder.java | 35 +- .../FragmentInstanceStatisticsJsonDrawer.java | 371 ++++++++++++++++++ .../node/PlanGraphJsonPrinterTest.java | 120 ++++++ .../relational/sql/ExplainFormatTest.java | 127 ++++++ .../relational/grammar/sql/RelationalSql.g4 | 4 +- 17 files changed, 1345 insertions(+), 18 deletions(-) create mode 100644 integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTExplainJsonFormatIT.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphJsonPrinter.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExplainOutputFormat.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/FragmentInstanceStatisticsJsonDrawer.java create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/PlanGraphJsonPrinterTest.java create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ExplainFormatTest.java diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTExplainJsonFormatIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTExplainJsonFormatIT.java new file mode 100644 index 0000000000000..94ed692ab6410 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTExplainJsonFormatIT.java @@ -0,0 +1,251 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one + * * or more contributor license agreements. See the NOTICE file + * * distributed with this work for additional information + * * regarding copyright ownership. The ASF licenses this file + * * to you under the Apache License, Version 2.0 (the + * * "License"); you may not use this file except in compliance + * * with the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, + * * software distributed under the License is distributed on an + * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * * KIND, either express or implied. See the License for the + * * specific language governing permissions and limitations + * * under the License. + * + */ + +package org.apache.iotdb.relational.it.query.recent; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Locale; + +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class}) +public class IoTExplainJsonFormatIT { + private static final String DATABASE_NAME = "testdb_json"; + + @BeforeClass + public static void setUp() { + Locale.setDefault(Locale.ENGLISH); + + EnvFactory.getEnv().getConfig().getCommonConfig().setPartitionInterval(1000); + EnvFactory.getEnv().initClusterEnvironment(); + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE IF NOT EXISTS " + DATABASE_NAME); + statement.execute("USE " + DATABASE_NAME); + statement.execute( + "CREATE TABLE IF NOT EXISTS testtb(deviceid STRING TAG, voltage FLOAT FIELD)"); + statement.execute("INSERT INTO testtb VALUES(1000, 'd1', 100.0)"); + statement.execute("INSERT INTO testtb VALUES(2000, 'd1', 200.0)"); + statement.execute("INSERT INTO testtb VALUES(1000, 'd2', 300.0)"); + } catch (Exception e) { + fail(e.getMessage()); + } + } + + @AfterClass + public static void tearDown() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("DROP DATABASE IF EXISTS " + DATABASE_NAME); + } catch (Exception e) { + // ignore + } + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testExplainJsonFormat() { + String sql = "EXPLAIN (FORMAT JSON) SELECT * FROM testtb"; + try (Connection conn = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = conn.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = statement.executeQuery(sql); + + Assert.assertTrue("Should have at least one row", resultSet.next()); + String jsonStr = resultSet.getString(1); + Assert.assertNotNull(jsonStr); + + // Verify it's valid JSON + JsonObject root = JsonParser.parseString(jsonStr).getAsJsonObject(); + Assert.assertTrue("JSON should have 'name' field", root.has("name")); + Assert.assertTrue("JSON should have 'id' field", root.has("id")); + + resultSet.close(); + } catch (SQLException e) { + fail(e.getMessage()); + } + } + + @Test + public void testExplainDefaultFormatIsNotJson() { + String sql = "EXPLAIN SELECT * FROM testtb"; + try (Connection conn = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = conn.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = statement.executeQuery(sql); + + Assert.assertTrue("Should have at least one row", resultSet.next()); + String firstLine = resultSet.getString(1); + // Default format (GRAPHVIZ) produces box-drawing characters, not JSON + Assert.assertFalse( + "Default EXPLAIN should not produce JSON", firstLine.trim().startsWith("{")); + + resultSet.close(); + } catch (SQLException e) { + fail(e.getMessage()); + } + } + + @Test + public void testExplainAnalyzeJsonFormat() { + String sql = "EXPLAIN ANALYZE (FORMAT JSON) SELECT * FROM testtb"; + try (Connection conn = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = conn.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = statement.executeQuery(sql); + + // JSON format should produce a single row with full JSON + Assert.assertTrue("Should have at least one row", resultSet.next()); + String jsonStr = resultSet.getString(1); + Assert.assertNotNull(jsonStr); + + // Verify it's valid JSON + JsonObject root = JsonParser.parseString(jsonStr).getAsJsonObject(); + Assert.assertTrue("JSON should have 'planStatistics' field", root.has("planStatistics")); + Assert.assertTrue( + "JSON should have 'fragmentInstances' field", root.has("fragmentInstances")); + Assert.assertTrue( + "JSON should have 'fragmentInstancesCount' field", root.has("fragmentInstancesCount")); + + // Plan statistics should contain known keys + JsonObject planStats = root.getAsJsonObject("planStatistics"); + Assert.assertTrue(planStats.has("analyzeCostMs")); + Assert.assertTrue(planStats.has("logicalPlanCostMs")); + Assert.assertTrue(planStats.has("distributionPlanCostMs")); + Assert.assertTrue(planStats.has("dispatchCostMs")); + + resultSet.close(); + } catch (SQLException e) { + fail(e.getMessage()); + } + } + + @Test + public void testExplainAnalyzeVerboseJsonFormat() { + String sql = "EXPLAIN ANALYZE VERBOSE (FORMAT JSON) SELECT * FROM testtb"; + try (Connection conn = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = conn.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = statement.executeQuery(sql); + + Assert.assertTrue("Should have at least one row", resultSet.next()); + String jsonStr = resultSet.getString(1); + Assert.assertNotNull(jsonStr); + + // Verify it's valid JSON + JsonObject root = JsonParser.parseString(jsonStr).getAsJsonObject(); + Assert.assertTrue(root.has("planStatistics")); + Assert.assertTrue(root.has("fragmentInstances")); + + resultSet.close(); + } catch (SQLException e) { + fail(e.getMessage()); + } + } + + @Test + public void testExplainAnalyzeDefaultIsText() { + String sql = "EXPLAIN ANALYZE SELECT * FROM testtb"; + try (Connection conn = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = conn.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = statement.executeQuery(sql); + + Assert.assertTrue("Should have at least one row", resultSet.next()); + String firstLine = resultSet.getString(1); + // Default format (TEXT) should not start with '{' + Assert.assertFalse( + "Default EXPLAIN ANALYZE should not produce JSON", firstLine.trim().startsWith("{")); + + resultSet.close(); + } catch (SQLException e) { + fail(e.getMessage()); + } + } + + @Test + public void testExplainGraphvizFormatExplicit() { + String sql = "EXPLAIN (FORMAT GRAPHVIZ) SELECT * FROM testtb"; + try (Connection conn = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = conn.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = statement.executeQuery(sql); + + Assert.assertTrue("Should have at least one row", resultSet.next()); + String firstLine = resultSet.getString(1); + // GRAPHVIZ format produces box-drawing characters, not JSON + Assert.assertFalse( + "GRAPHVIZ EXPLAIN should not produce JSON", firstLine.trim().startsWith("{")); + + resultSet.close(); + } catch (SQLException e) { + fail(e.getMessage()); + } + } + + @Test + public void testExplainAnalyzeTextFormatExplicit() { + String sql = "EXPLAIN ANALYZE (FORMAT TEXT) SELECT * FROM testtb"; + try (Connection conn = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = conn.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = statement.executeQuery(sql); + + Assert.assertTrue("Should have at least one row", resultSet.next()); + String firstLine = resultSet.getString(1); + Assert.assertFalse( + "TEXT EXPLAIN ANALYZE should not produce JSON", firstLine.trim().startsWith("{")); + + resultSet.close(); + } catch (SQLException e) { + fail(e.getMessage()); + } + } + + @Test(expected = SQLException.class) + public void testExplainInvalidFormat() throws SQLException { + String sql = "EXPLAIN (FORMAT XML) SELECT * FROM testtb"; + try (Connection conn = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = conn.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + statement.executeQuery(sql); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java index 88bd1998f683c..43b214f226cc3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java @@ -34,6 +34,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.memory.MemoryReservationManager; import org.apache.iotdb.db.queryengine.plan.planner.memory.NotThreadSafeMemoryReservationManager; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.NodeRef; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExplainOutputFormat; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Query; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Table; @@ -108,6 +109,7 @@ public enum ExplainType { // - EXPLAIN: Show the logical and physical query plan without execution // - EXPLAIN_ANALYZE: Execute the query and collect detailed execution statistics private ExplainType explainType = ExplainType.NONE; + private ExplainOutputFormat explainOutputFormat = null; private boolean verbose = false; private QueryPlanStatistics queryPlanStatistics = null; @@ -346,6 +348,14 @@ public ExplainType getExplainType() { return explainType; } + public void setExplainOutputFormat(ExplainOutputFormat explainOutputFormat) { + this.explainOutputFormat = explainOutputFormat; + } + + public ExplainOutputFormat getExplainOutputFormat() { + return explainOutputFormat; + } + public boolean isExplainAnalyze() { return explainType == ExplainType.EXPLAIN_ANALYZE; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/ExplainAnalyzeOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/ExplainAnalyzeOperator.java index b4f1a5c7261bf..52af0bb745583 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/ExplainAnalyzeOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/ExplainAnalyzeOperator.java @@ -33,8 +33,10 @@ import org.apache.iotdb.db.queryengine.plan.execution.QueryExecution; import org.apache.iotdb.db.queryengine.plan.planner.plan.FragmentInstance; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.NodeRef; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExplainOutputFormat; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Table; import org.apache.iotdb.db.queryengine.statistics.FragmentInstanceStatisticsDrawer; +import org.apache.iotdb.db.queryengine.statistics.FragmentInstanceStatisticsJsonDrawer; import org.apache.iotdb.db.queryengine.statistics.QueryStatisticsFetcher; import org.apache.iotdb.db.queryengine.statistics.StatisticLine; import org.apache.iotdb.db.utils.SetThreadName; @@ -78,9 +80,12 @@ public class ExplainAnalyzeOperator implements ProcessOperator { private final boolean verbose; private boolean outputResult = false; private final List instances; + private final ExplainOutputFormat outputFormat; private final FragmentInstanceStatisticsDrawer fragmentInstanceStatisticsDrawer = new FragmentInstanceStatisticsDrawer(); + private final FragmentInstanceStatisticsJsonDrawer fragmentInstanceStatisticsJsonDrawer = + new FragmentInstanceStatisticsJsonDrawer(); private final ScheduledFuture logRecordTask; private final IClientManager clientManager; @@ -92,9 +97,20 @@ public ExplainAnalyzeOperator( long queryId, boolean verbose, long timeout) { + this(operatorContext, child, queryId, verbose, timeout, ExplainOutputFormat.TEXT); + } + + public ExplainAnalyzeOperator( + OperatorContext operatorContext, + Operator child, + long queryId, + boolean verbose, + long timeout, + ExplainOutputFormat outputFormat) { this.operatorContext = operatorContext; this.child = child; this.verbose = verbose; + this.outputFormat = outputFormat; Coordinator coordinator = Coordinator.getInstance(); this.clientManager = coordinator.getInternalServiceClientManager(); @@ -103,6 +119,7 @@ public ExplainAnalyzeOperator( this.instances = queryExecution.getDistributedPlan().getInstances(); mppQueryContext = queryExecution.getContext(); fragmentInstanceStatisticsDrawer.renderPlanStatistics(mppQueryContext); + fragmentInstanceStatisticsJsonDrawer.renderPlanStatistics(mppQueryContext); // The time interval guarantees the result of EXPLAIN ANALYZE will be printed at least three // times. @@ -130,6 +147,7 @@ public TsBlock next() throws Exception { } fragmentInstanceStatisticsDrawer.renderDispatchCost(mppQueryContext); + fragmentInstanceStatisticsJsonDrawer.renderDispatchCost(mppQueryContext); // fetch statics from all fragment instances TsBlock result = buildResult(); @@ -182,6 +200,9 @@ private void logIntermediateResultIfTimeout() { } private TsBlock buildResult() throws FragmentInstanceFetchException { + if (outputFormat == ExplainOutputFormat.JSON) { + return buildJsonResult(); + } Map, Pair>> cteAnalyzeResults = mppQueryContext.getCteExplainResults(); List mainAnalyzeResult = buildFragmentInstanceStatistics(instances, verbose); @@ -199,6 +220,22 @@ private TsBlock buildResult() throws FragmentInstanceFetchException { return builder.build(); } + private TsBlock buildJsonResult() throws FragmentInstanceFetchException { + Map allStatistics = + QueryStatisticsFetcher.fetchAllStatistics(instances, clientManager); + String jsonResult = + fragmentInstanceStatisticsJsonDrawer.renderFragmentInstancesAsJson( + instances, allStatistics, verbose); + + TsBlockBuilder builder = new TsBlockBuilder(Collections.singletonList(TSDataType.TEXT)); + TimeColumnBuilder timeColumnBuilder = builder.getTimeColumnBuilder(); + ColumnBuilder columnBuilder = builder.getColumnBuilder(0); + timeColumnBuilder.writeLong(0); + columnBuilder.writeBinary(new Binary(jsonResult.getBytes())); + builder.declarePosition(); + return builder.build(); + } + private List mergeAnalyzeResults( Map, Pair>> cteAnalyzeResults, List mainAnalyzeResult) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/memory/TableModelStatementMemorySourceVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/memory/TableModelStatementMemorySourceVisitor.java index 932d941979223..c6a7692395a37 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/memory/TableModelStatementMemorySourceVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/memory/TableModelStatementMemorySourceVisitor.java @@ -25,6 +25,7 @@ import org.apache.iotdb.db.queryengine.plan.Coordinator; import org.apache.iotdb.db.queryengine.plan.planner.LocalExecutionPlanner; import org.apache.iotdb.db.queryengine.plan.planner.plan.LogicalQueryPlan; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanGraphJsonPrinter; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanGraphPrinter; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.NodeRef; @@ -35,6 +36,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AstVisitor; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CountDevice; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Explain; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExplainOutputFormat; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Node; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowDevice; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Table; @@ -103,15 +105,26 @@ public StatementMemorySource visitExplain( Coordinator.getInstance().getDataNodeLocationSupplier()) .generateDistributedPlanWithOptimize(planContext); - List mainExplainResult = - outputNodeWithExchange.accept( - new PlanGraphPrinter(), - new PlanGraphPrinter.GraphContext( - context.getQueryContext().getTypeProvider().getTemplatedInfo())); + List mainExplainResult; + if (node.getOutputFormat() == ExplainOutputFormat.JSON) { + mainExplainResult = PlanGraphJsonPrinter.getJsonLines(outputNodeWithExchange); + } else { + mainExplainResult = + outputNodeWithExchange.accept( + new PlanGraphPrinter(), + new PlanGraphPrinter.GraphContext( + context.getQueryContext().getTypeProvider().getTemplatedInfo())); + } Map, Pair>> cteExplainResults = context.getQueryContext().getCteExplainResults(); - List lines = mergeExplainResults(cteExplainResults, mainExplainResult); + List lines; + if (node.getOutputFormat() == ExplainOutputFormat.JSON) { + // For JSON format, we merge CTE results into the JSON output + lines = mergeExplainResultsJson(cteExplainResults, mainExplainResult); + } else { + lines = mergeExplainResults(cteExplainResults, mainExplainResult); + } return getStatementMemorySource(header, lines); } @@ -149,4 +162,44 @@ private List mergeExplainResults( return analyzeResult; } + + private List mergeExplainResultsJson( + Map, Pair>> cteExplainResults, + List mainExplainResult) { + if (cteExplainResults.isEmpty()) { + return mainExplainResult; + } + + // For JSON format with CTEs, wrap everything in a combined JSON object + StringBuilder sb = new StringBuilder(); + sb.append("{\n"); + sb.append(" \"cteQueries\": [\n"); + int cteIndex = 0; + int cteSize = cteExplainResults.size(); + for (Map.Entry, Pair>> entry : + cteExplainResults.entrySet()) { + sb.append(" {\n"); + sb.append(" \"name\": \"").append(entry.getKey().getNode().getName()).append("\",\n"); + sb.append(" \"plan\": "); + // Each CTE's plan is already a JSON string + for (String line : entry.getValue().getRight()) { + sb.append(line); + } + sb.append("\n }"); + if (++cteIndex < cteSize) { + sb.append(","); + } + sb.append("\n"); + } + sb.append(" ],\n"); + sb.append(" \"mainQuery\": "); + for (String line : mainExplainResult) { + sb.append(line); + } + sb.append("\n}"); + + List result = new ArrayList<>(); + result.add(sb.toString()); + return result; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java index c31a2061f49ec..fe96177ee5f2c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java @@ -3472,7 +3472,12 @@ public Operator visitExplainAnalyze(ExplainAnalyzeNode node, LocalExecutionPlanC node.getPlanNodeId(), ExplainAnalyzeOperator.class.getSimpleName()); return new ExplainAnalyzeOperator( - operatorContext, operator, node.getQueryId(), node.isVerbose(), node.getTimeout()); + operatorContext, + operator, + node.getQueryId(), + node.isVerbose(), + node.getTimeout(), + node.getOutputFormat()); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphJsonPrinter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphJsonPrinter.java new file mode 100644 index 0000000000000..e35c23d466059 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphJsonPrinter.java @@ -0,0 +1,220 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.planner.plan.node; + +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.DeviceTableScanNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ExchangeNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ExplainAnalyzeNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.OutputNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeDeviceViewScanNode; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; + +import java.util.ArrayList; +import java.util.List; + +/** + * Converts a plan tree into a JSON representation. Each plan node becomes a JSON object with: - + * "name": the node type with its plan node ID - "id": the plan node ID - "properties": key-value + * pairs - "children": array of child nodes + */ +public class PlanGraphJsonPrinter { + + private static final Gson GSON = new GsonBuilder().setPrettyPrinting().create(); + + private static final String REGION_NOT_ASSIGNED = "Not Assigned"; + + public static String toPrettyJson(PlanNode node) { + JsonObject root = buildJsonNode(node); + return GSON.toJson(root); + } + + private static JsonObject buildJsonNode(PlanNode node) { + JsonObject jsonNode = new JsonObject(); + String simpleName = node.getClass().getSimpleName(); + String nodeId = node.getPlanNodeId().getId(); + + jsonNode.addProperty("name", simpleName + "-" + nodeId); + jsonNode.addProperty("id", nodeId); + + JsonObject properties = buildProperties(node); + if (properties.size() > 0) { + jsonNode.add("properties", properties); + } + + List children = node.getChildren(); + if (children != null && !children.isEmpty()) { + JsonArray childrenArray = new JsonArray(); + for (PlanNode child : children) { + childrenArray.add(buildJsonNode(child)); + } + jsonNode.add("children", childrenArray); + } + + return jsonNode; + } + + private static JsonObject buildProperties(PlanNode node) { + JsonObject properties = new JsonObject(); + + if (node instanceof OutputNode) { + OutputNode n = (OutputNode) node; + properties.addProperty("OutputColumns", String.valueOf(n.getOutputColumnNames())); + properties.addProperty("OutputSymbols", String.valueOf(n.getOutputSymbols())); + } else if (node instanceof ExplainAnalyzeNode) { + ExplainAnalyzeNode n = (ExplainAnalyzeNode) node; + properties.addProperty("ChildPermittedOutputs", String.valueOf(n.getChildPermittedOutputs())); + } else if (node instanceof TableScanNode) { + buildTableScanProperties(properties, (TableScanNode) node); + } else if (node instanceof ExchangeNode) { + // No extra properties needed + } else if (node + instanceof org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode) { + buildAggregationProperties( + properties, + (org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode) node); + } else if (node + instanceof org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode) { + org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode n = + (org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode) node; + properties.addProperty("Predicate", String.valueOf(n.getPredicate())); + } else if (node + instanceof org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode) { + org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode n = + (org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode) node; + properties.addProperty("OutputSymbols", String.valueOf(n.getOutputSymbols())); + properties.addProperty("Expressions", String.valueOf(n.getAssignments().getMap().values())); + } else if (node + instanceof org.apache.iotdb.db.queryengine.plan.relational.planner.node.LimitNode) { + org.apache.iotdb.db.queryengine.plan.relational.planner.node.LimitNode n = + (org.apache.iotdb.db.queryengine.plan.relational.planner.node.LimitNode) node; + properties.addProperty("Count", String.valueOf(n.getCount())); + } else if (node + instanceof org.apache.iotdb.db.queryengine.plan.relational.planner.node.OffsetNode) { + org.apache.iotdb.db.queryengine.plan.relational.planner.node.OffsetNode n = + (org.apache.iotdb.db.queryengine.plan.relational.planner.node.OffsetNode) node; + properties.addProperty("Count", String.valueOf(n.getCount())); + } else if (node + instanceof org.apache.iotdb.db.queryengine.plan.relational.planner.node.SortNode) { + org.apache.iotdb.db.queryengine.plan.relational.planner.node.SortNode n = + (org.apache.iotdb.db.queryengine.plan.relational.planner.node.SortNode) node; + properties.addProperty("OrderBy", String.valueOf(n.getOrderingScheme())); + } else if (node + instanceof org.apache.iotdb.db.queryengine.plan.relational.planner.node.MergeSortNode) { + org.apache.iotdb.db.queryengine.plan.relational.planner.node.MergeSortNode n = + (org.apache.iotdb.db.queryengine.plan.relational.planner.node.MergeSortNode) node; + properties.addProperty("OrderBy", String.valueOf(n.getOrderingScheme())); + } else if (node + instanceof org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode) { + org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode n = + (org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode) node; + properties.addProperty("JoinType", String.valueOf(n.getJoinType())); + properties.addProperty("Criteria", String.valueOf(n.getCriteria())); + properties.addProperty("OutputSymbols", String.valueOf(n.getOutputSymbols())); + } else if (node + instanceof org.apache.iotdb.db.queryengine.plan.relational.planner.node.UnionNode) { + org.apache.iotdb.db.queryengine.plan.relational.planner.node.UnionNode n = + (org.apache.iotdb.db.queryengine.plan.relational.planner.node.UnionNode) node; + properties.addProperty("OutputSymbols", String.valueOf(n.getOutputSymbols())); + } + + return properties; + } + + private static void buildTableScanProperties(JsonObject properties, TableScanNode node) { + properties.addProperty("QualifiedTableName", node.getQualifiedObjectName().toString()); + properties.addProperty("OutputSymbols", String.valueOf(node.getOutputSymbols())); + + if (node instanceof DeviceTableScanNode) { + DeviceTableScanNode deviceNode = (DeviceTableScanNode) node; + properties.addProperty("DeviceNumber", String.valueOf(deviceNode.getDeviceEntries().size())); + properties.addProperty("ScanOrder", String.valueOf(deviceNode.getScanOrder())); + if (deviceNode.getTimePredicate().isPresent()) { + properties.addProperty( + "TimePredicate", String.valueOf(deviceNode.getTimePredicate().get())); + } + } + + if (node.getPushDownPredicate() != null) { + properties.addProperty("PushDownPredicate", String.valueOf(node.getPushDownPredicate())); + } + properties.addProperty("PushDownOffset", String.valueOf(node.getPushDownOffset())); + properties.addProperty("PushDownLimit", String.valueOf(node.getPushDownLimit())); + + if (node instanceof DeviceTableScanNode) { + properties.addProperty( + "PushDownLimitToEachDevice", + String.valueOf(((DeviceTableScanNode) node).isPushLimitToEachDevice())); + } + + properties.addProperty( + "RegionId", + node.getRegionReplicaSet() == null || node.getRegionReplicaSet().getRegionId() == null + ? REGION_NOT_ASSIGNED + : String.valueOf(node.getRegionReplicaSet().getRegionId().getId())); + + if (node instanceof TreeDeviceViewScanNode) { + TreeDeviceViewScanNode treeNode = (TreeDeviceViewScanNode) node; + properties.addProperty("TreeDB", treeNode.getTreeDBName()); + properties.addProperty( + "MeasurementToColumnName", String.valueOf(treeNode.getMeasurementColumnNameMap())); + } + } + + private static void buildAggregationProperties( + JsonObject properties, + org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode node) { + properties.addProperty("OutputSymbols", String.valueOf(node.getOutputSymbols())); + + JsonArray aggregators = new JsonArray(); + int i = 0; + for (org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode.Aggregation + aggregation : node.getAggregations().values()) { + JsonObject agg = new JsonObject(); + agg.addProperty("index", i++); + agg.addProperty("function", aggregation.getResolvedFunction().toString()); + if (aggregation.hasMask()) { + agg.addProperty("mask", String.valueOf(aggregation.getMask().get())); + } + if (aggregation.isDistinct()) { + agg.addProperty("distinct", true); + } + aggregators.add(agg); + } + properties.add("Aggregators", aggregators); + + properties.addProperty("GroupingKeys", String.valueOf(node.getGroupingKeys())); + if (node.isStreamable()) { + properties.addProperty("Streamable", true); + properties.addProperty("PreGroupedSymbols", String.valueOf(node.getPreGroupedSymbols())); + } + properties.addProperty("Step", String.valueOf(node.getStep())); + } + + public static List getJsonLines(PlanNode node) { + List lines = new ArrayList<>(); + lines.add(toPrettyJson(node)); + return lines; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index cff29c0d83a1a..eb4e1241d1400 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -847,6 +847,7 @@ protected Scope visitLoadTsFile(final LoadTsFile node, final Optional sco @Override protected Scope visitExplain(Explain node, Optional context) { queryContext.setExplainType(ExplainType.EXPLAIN); + queryContext.setExplainOutputFormat(node.getOutputFormat()); analysis.setFinishQueryAfterAnalyze(); return visitQuery((Query) node.getStatement(), context); } @@ -863,6 +864,7 @@ protected Scope visitCopyTo(CopyTo node, Optional context) { protected Scope visitExplainAnalyze(ExplainAnalyze node, Optional context) { queryContext.setExplainType(ExplainType.EXPLAIN_ANALYZE); queryContext.setVerbose(node.isVerbose()); + queryContext.setExplainOutputFormat(node.getOutputFormat()); return visitQuery((Query) node.getStatement(), context); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableLogicalPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableLogicalPlanner.java index c33cb05250bd1..538890b15682a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableLogicalPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableLogicalPlanner.java @@ -625,7 +625,8 @@ private RelationPlan planExplainAnalyze(final ExplainAnalyze statement, final An queryContext.getTimeOut(), symbol, // recording permittedOutputs of ExplainAnalyzeNode's child - getChildPermittedOutputs(analysis, statement.getStatement(), originalQueryPlan)); + getChildPermittedOutputs(analysis, statement.getStatement(), originalQueryPlan), + statement.getOutputFormat()); return new RelationPlan( newRoot, originalQueryPlan.getScope(), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/ExplainAnalyzeNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/ExplainAnalyzeNode.java index 23aa8baad2dfd..d50d07521ec23 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/ExplainAnalyzeNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/ExplainAnalyzeNode.java @@ -24,6 +24,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.SingleChildProcessNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExplainOutputFormat; import java.io.DataOutputStream; import java.io.IOException; @@ -38,6 +39,7 @@ public class ExplainAnalyzeNode extends SingleChildProcessNode { private final long timeout; private final Symbol outputSymbol; private final List childPermittedOutputs; + private final ExplainOutputFormat outputFormat; public ExplainAnalyzeNode( PlanNodeId id, @@ -47,18 +49,46 @@ public ExplainAnalyzeNode( long timeout, Symbol outputSymbol, List childPermittedOutputs) { + this( + id, + child, + verbose, + queryId, + timeout, + outputSymbol, + childPermittedOutputs, + ExplainOutputFormat.TEXT); + } + + public ExplainAnalyzeNode( + PlanNodeId id, + PlanNode child, + boolean verbose, + long queryId, + long timeout, + Symbol outputSymbol, + List childPermittedOutputs, + ExplainOutputFormat outputFormat) { super(id, child); this.verbose = verbose; this.timeout = timeout; this.queryId = queryId; this.outputSymbol = outputSymbol; this.childPermittedOutputs = childPermittedOutputs; + this.outputFormat = outputFormat; } @Override public PlanNode clone() { return new ExplainAnalyzeNode( - getPlanNodeId(), child, verbose, queryId, timeout, outputSymbol, childPermittedOutputs); + getPlanNodeId(), + child, + verbose, + queryId, + timeout, + outputSymbol, + childPermittedOutputs, + outputFormat); } @Override @@ -89,7 +119,8 @@ public PlanNode replaceChildren(List newChildren) { queryId, timeout, outputSymbol, - childPermittedOutputs); + childPermittedOutputs, + outputFormat); } // ExplainAnalyze should be at the same region as Coordinator all the time. Therefore, there will @@ -116,6 +147,10 @@ public long getTimeout() { return timeout; } + public ExplainOutputFormat getOutputFormat() { + return outputFormat; + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Explain.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Explain.java index f3a50175269e8..6e51e9bae607d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Explain.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Explain.java @@ -33,21 +33,34 @@ public class Explain extends Statement { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(Explain.class); private final Statement statement; + private final ExplainOutputFormat outputFormat; public Explain(Statement statement) { super(null); this.statement = requireNonNull(statement, "statement is null"); + this.outputFormat = ExplainOutputFormat.GRAPHVIZ; } public Explain(NodeLocation location, Statement statement) { super(requireNonNull(location, "location is null")); this.statement = requireNonNull(statement, "statement is null"); + this.outputFormat = ExplainOutputFormat.GRAPHVIZ; + } + + public Explain(NodeLocation location, Statement statement, ExplainOutputFormat outputFormat) { + super(requireNonNull(location, "location is null")); + this.statement = requireNonNull(statement, "statement is null"); + this.outputFormat = requireNonNull(outputFormat, "outputFormat is null"); } public Statement getStatement() { return statement; } + public ExplainOutputFormat getOutputFormat() { + return outputFormat; + } + @Override public R accept(AstVisitor visitor, C context) { return visitor.visitExplain(this, context); @@ -60,7 +73,7 @@ public List getChildren() { @Override public int hashCode() { - return Objects.hash(statement); + return Objects.hash(statement, outputFormat); } @Override @@ -72,12 +85,15 @@ public boolean equals(Object obj) { return false; } Explain o = (Explain) obj; - return Objects.equals(statement, o.statement); + return Objects.equals(statement, o.statement) && outputFormat == o.outputFormat; } @Override public String toString() { - return toStringHelper(this).add("statement", statement).toString(); + return toStringHelper(this) + .add("statement", statement) + .add("outputFormat", outputFormat) + .toString(); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExplainAnalyze.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExplainAnalyze.java index cd3df879fc55e..dc7798967e4d8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExplainAnalyze.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExplainAnalyze.java @@ -35,17 +35,31 @@ public class ExplainAnalyze extends Statement { private final Statement statement; private final boolean verbose; + private final ExplainOutputFormat outputFormat; public ExplainAnalyze(Statement statement, boolean verbose) { super(null); this.statement = requireNonNull(statement, "statement is null"); this.verbose = verbose; + this.outputFormat = ExplainOutputFormat.TEXT; } public ExplainAnalyze(NodeLocation location, boolean verbose, Statement statement) { super(requireNonNull(location, "location is null")); this.statement = requireNonNull(statement, "statement is null"); this.verbose = verbose; + this.outputFormat = ExplainOutputFormat.TEXT; + } + + public ExplainAnalyze( + NodeLocation location, + boolean verbose, + Statement statement, + ExplainOutputFormat outputFormat) { + super(requireNonNull(location, "location is null")); + this.statement = requireNonNull(statement, "statement is null"); + this.verbose = verbose; + this.outputFormat = requireNonNull(outputFormat, "outputFormat is null"); } public Statement getStatement() { @@ -56,6 +70,10 @@ public boolean isVerbose() { return verbose; } + public ExplainOutputFormat getOutputFormat() { + return outputFormat; + } + @Override public R accept(AstVisitor visitor, C context) { return visitor.visitExplainAnalyze(this, context); @@ -85,7 +103,11 @@ public boolean equals(Object obj) { @Override public String toString() { - return toStringHelper(this).add("statement", statement).add("verbose", verbose).toString(); + return toStringHelper(this) + .add("statement", statement) + .add("verbose", verbose) + .add("outputFormat", outputFormat) + .toString(); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExplainOutputFormat.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExplainOutputFormat.java new file mode 100644 index 0000000000000..9dba3649a7f7f --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExplainOutputFormat.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +public enum ExplainOutputFormat { + GRAPHVIZ, + TEXT, + JSON +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java index a31b48c24b4a5..c825d6b92cb26 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java @@ -98,6 +98,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExistsPredicate; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Explain; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExplainAnalyze; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExplainOutputFormat; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExtendRegion; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Extract; @@ -1804,7 +1805,22 @@ public Node visitExplain(RelationalSqlParser.ExplainContext ctx) { } else { innerStatement = (Statement) visit(ctx.executeImmediateStatement()); } - return new Explain(getLocation(ctx), innerStatement); + ExplainOutputFormat format = ExplainOutputFormat.GRAPHVIZ; + if (ctx.identifier() != null) { + String formatStr = ((Identifier) visit(ctx.identifier())).getValue().toUpperCase(); + switch (formatStr) { + case "GRAPHVIZ": + format = ExplainOutputFormat.GRAPHVIZ; + break; + case "JSON": + format = ExplainOutputFormat.JSON; + break; + default: + throw new SemanticException( + "Invalid EXPLAIN format: " + formatStr + ". Supported formats: GRAPHVIZ, JSON"); + } + } + return new Explain(getLocation(ctx), innerStatement, format); } @Override @@ -1817,7 +1833,22 @@ public Node visitExplainAnalyze(RelationalSqlParser.ExplainAnalyzeContext ctx) { } else { innerStatement = (Statement) visit(ctx.executeImmediateStatement()); } - return new ExplainAnalyze(getLocation(ctx), ctx.VERBOSE() != null, innerStatement); + ExplainOutputFormat format = ExplainOutputFormat.TEXT; + if (ctx.identifier() != null) { + String formatStr = ((Identifier) visit(ctx.identifier())).getValue().toUpperCase(); + switch (formatStr) { + case "TEXT": + format = ExplainOutputFormat.TEXT; + break; + case "JSON": + format = ExplainOutputFormat.JSON; + break; + default: + throw new SemanticException( + "Invalid EXPLAIN ANALYZE format: " + formatStr + ". Supported formats: TEXT, JSON"); + } + } + return new ExplainAnalyze(getLocation(ctx), ctx.VERBOSE() != null, innerStatement, format); } // ********************** author expressions ******************** diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/FragmentInstanceStatisticsJsonDrawer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/FragmentInstanceStatisticsJsonDrawer.java new file mode 100644 index 0000000000000..3c6ff7bfb41b2 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/FragmentInstanceStatisticsJsonDrawer.java @@ -0,0 +1,371 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.statistics; + +import org.apache.iotdb.db.queryengine.common.FragmentInstanceId; +import org.apache.iotdb.db.queryengine.common.MPPQueryContext; +import org.apache.iotdb.db.queryengine.plan.planner.plan.FragmentInstance; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.mpp.rpc.thrift.TFetchFragmentInstanceStatisticsResp; +import org.apache.iotdb.mpp.rpc.thrift.TOperatorStatistics; +import org.apache.iotdb.mpp.rpc.thrift.TQueryStatistics; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Produces JSON output for EXPLAIN ANALYZE results, mirroring the same data as {@link + * FragmentInstanceStatisticsDrawer} but in JSON format. + */ +public class FragmentInstanceStatisticsJsonDrawer { + + private static final Gson GSON = new GsonBuilder().setPrettyPrinting().create(); + private static final double NS_TO_MS_FACTOR = 1.0 / 1000000; + private static final double EPSILON = 1e-10; + + private final JsonObject planStatistics = new JsonObject(); + + public void renderPlanStatistics(MPPQueryContext context) { + planStatistics.addProperty( + "analyzeCostMs", formatMs(context.getAnalyzeCost() * NS_TO_MS_FACTOR)); + planStatistics.addProperty( + "fetchPartitionCostMs", formatMs(context.getFetchPartitionCost() * NS_TO_MS_FACTOR)); + planStatistics.addProperty( + "fetchSchemaCostMs", formatMs(context.getFetchSchemaCost() * NS_TO_MS_FACTOR)); + planStatistics.addProperty( + "logicalPlanCostMs", formatMs(context.getLogicalPlanCost() * NS_TO_MS_FACTOR)); + planStatistics.addProperty( + "logicalOptimizationCostMs", + formatMs(context.getLogicalOptimizationCost() * NS_TO_MS_FACTOR)); + planStatistics.addProperty( + "distributionPlanCostMs", formatMs(context.getDistributionPlanCost() * NS_TO_MS_FACTOR)); + } + + public void renderDispatchCost(MPPQueryContext context) { + planStatistics.addProperty( + "dispatchCostMs", formatMs(context.getDispatchCost() * NS_TO_MS_FACTOR)); + } + + public String renderFragmentInstancesAsJson( + List instancesToBeRendered, + Map allStatistics, + boolean verbose) { + + JsonObject root = new JsonObject(); + root.add("planStatistics", planStatistics); + + List validInstances = + instancesToBeRendered.stream() + .filter( + instance -> { + TFetchFragmentInstanceStatisticsResp statistics = + allStatistics.get(instance.getId()); + return statistics != null && statistics.getDataRegion() != null; + }) + .collect(Collectors.toList()); + + root.addProperty("fragmentInstancesCount", validInstances.size()); + + JsonArray fragmentInstancesArray = new JsonArray(); + for (FragmentInstance instance : validInstances) { + TFetchFragmentInstanceStatisticsResp statistics = allStatistics.get(instance.getId()); + JsonObject fiJson = new JsonObject(); + + fiJson.addProperty("id", instance.getId().toString()); + fiJson.addProperty("ip", statistics.getIp()); + fiJson.addProperty("dataRegion", statistics.getDataRegion()); + fiJson.addProperty("state", statistics.getState()); + fiJson.addProperty( + "totalWallTimeMs", statistics.getEndTimeInMS() - statistics.getStartTimeInMS()); + fiJson.addProperty( + "initDataQuerySourceCostMs", + formatMs(statistics.getInitDataQuerySourceCost() * NS_TO_MS_FACTOR)); + + if (statistics.isSetInitDataQuerySourceRetryCount() + && statistics.getInitDataQuerySourceRetryCount() > 0) { + fiJson.addProperty( + "initDataQuerySourceRetryCount", statistics.getInitDataQuerySourceRetryCount()); + } + + fiJson.addProperty("seqFileUnclosed", statistics.getSeqUnclosedNum()); + fiJson.addProperty("seqFileClosed", statistics.getSeqClosednNum()); + fiJson.addProperty("unseqFileUnclosed", statistics.getUnseqUnclosedNum()); + fiJson.addProperty("unseqFileClosed", statistics.getUnseqClosedNum()); + fiJson.addProperty( + "readyQueuedTimeMs", formatMs(statistics.getReadyQueuedTime() * NS_TO_MS_FACTOR)); + fiJson.addProperty( + "blockQueuedTimeMs", formatMs(statistics.getBlockQueuedTime() * NS_TO_MS_FACTOR)); + + // Query statistics + JsonObject queryStats = renderQueryStatisticsJson(statistics.getQueryStatistics(), verbose); + fiJson.add("queryStatistics", queryStats); + + // Operators + PlanNode planNodeTree = instance.getFragment().getPlanNodeTree(); + JsonObject operatorTree = + renderOperatorJson(planNodeTree, statistics.getOperatorStatisticsMap()); + if (operatorTree != null) { + fiJson.add("operators", operatorTree); + } + + fragmentInstancesArray.add(fiJson); + } + + root.add("fragmentInstances", fragmentInstancesArray); + return GSON.toJson(root); + } + + private JsonObject renderQueryStatisticsJson(TQueryStatistics qs, boolean verbose) { + JsonObject stats = new JsonObject(); + + if (verbose) { + stats.addProperty("loadBloomFilterFromCacheCount", qs.loadBloomFilterFromCacheCount); + stats.addProperty("loadBloomFilterFromDiskCount", qs.loadBloomFilterFromDiskCount); + stats.addProperty("loadBloomFilterActualIOSize", qs.loadBloomFilterActualIOSize); + stats.addProperty( + "loadBloomFilterTimeMs", formatMs(qs.loadBloomFilterTime * NS_TO_MS_FACTOR)); + + addIfNonZero( + stats, "loadTimeSeriesMetadataDiskSeqCount", qs.loadTimeSeriesMetadataDiskSeqCount); + addIfNonZero( + stats, "loadTimeSeriesMetadataDiskUnSeqCount", qs.loadTimeSeriesMetadataDiskUnSeqCount); + addIfNonZero( + stats, "loadTimeSeriesMetadataMemSeqCount", qs.loadTimeSeriesMetadataMemSeqCount); + addIfNonZero( + stats, "loadTimeSeriesMetadataMemUnSeqCount", qs.loadTimeSeriesMetadataMemUnSeqCount); + addIfNonZero( + stats, + "loadTimeSeriesMetadataAlignedDiskSeqCount", + qs.loadTimeSeriesMetadataAlignedDiskSeqCount); + addIfNonZero( + stats, + "loadTimeSeriesMetadataAlignedDiskUnSeqCount", + qs.loadTimeSeriesMetadataAlignedDiskUnSeqCount); + addIfNonZero( + stats, + "loadTimeSeriesMetadataAlignedMemSeqCount", + qs.loadTimeSeriesMetadataAlignedMemSeqCount); + addIfNonZero( + stats, + "loadTimeSeriesMetadataAlignedMemUnSeqCount", + qs.loadTimeSeriesMetadataAlignedMemUnSeqCount); + + addMsIfNonZero( + stats, + "loadTimeSeriesMetadataDiskSeqTimeMs", + qs.loadTimeSeriesMetadataDiskSeqTime * NS_TO_MS_FACTOR); + addMsIfNonZero( + stats, + "loadTimeSeriesMetadataDiskUnSeqTimeMs", + qs.loadTimeSeriesMetadataDiskUnSeqTime * NS_TO_MS_FACTOR); + addMsIfNonZero( + stats, + "loadTimeSeriesMetadataMemSeqTimeMs", + qs.loadTimeSeriesMetadataMemSeqTime * NS_TO_MS_FACTOR); + addMsIfNonZero( + stats, + "loadTimeSeriesMetadataMemUnSeqTimeMs", + qs.loadTimeSeriesMetadataMemUnSeqTime * NS_TO_MS_FACTOR); + addMsIfNonZero( + stats, + "loadTimeSeriesMetadataAlignedDiskSeqTimeMs", + qs.loadTimeSeriesMetadataAlignedDiskSeqTime * NS_TO_MS_FACTOR); + addMsIfNonZero( + stats, + "loadTimeSeriesMetadataAlignedDiskUnSeqTimeMs", + qs.loadTimeSeriesMetadataAlignedDiskUnSeqTime * NS_TO_MS_FACTOR); + addMsIfNonZero( + stats, + "loadTimeSeriesMetadataAlignedMemSeqTimeMs", + qs.loadTimeSeriesMetadataAlignedMemSeqTime * NS_TO_MS_FACTOR); + addMsIfNonZero( + stats, + "loadTimeSeriesMetadataAlignedMemUnSeqTimeMs", + qs.loadTimeSeriesMetadataAlignedMemUnSeqTime * NS_TO_MS_FACTOR); + + stats.addProperty( + "loadTimeSeriesMetadataFromCacheCount", qs.loadTimeSeriesMetadataFromCacheCount); + stats.addProperty( + "loadTimeSeriesMetadataFromDiskCount", qs.loadTimeSeriesMetadataFromDiskCount); + stats.addProperty( + "loadTimeSeriesMetadataActualIOSize", qs.loadTimeSeriesMetadataActualIOSize); + + addIfNonZero( + stats, + "alignedTimeSeriesMetadataModificationCount", + qs.getAlignedTimeSeriesMetadataModificationCount()); + addMsIfNonZero( + stats, + "alignedTimeSeriesMetadataModificationTimeMs", + qs.getAlignedTimeSeriesMetadataModificationTime() * NS_TO_MS_FACTOR); + addIfNonZero( + stats, + "nonAlignedTimeSeriesMetadataModificationCount", + qs.getNonAlignedTimeSeriesMetadataModificationCount()); + addMsIfNonZero( + stats, + "nonAlignedTimeSeriesMetadataModificationTimeMs", + qs.getNonAlignedTimeSeriesMetadataModificationTime() * NS_TO_MS_FACTOR); + + addIfNonZero( + stats, + "constructNonAlignedChunkReadersDiskCount", + qs.constructNonAlignedChunkReadersDiskCount); + addIfNonZero( + stats, + "constructNonAlignedChunkReadersMemCount", + qs.constructNonAlignedChunkReadersMemCount); + addIfNonZero( + stats, "constructAlignedChunkReadersDiskCount", qs.constructAlignedChunkReadersDiskCount); + addIfNonZero( + stats, "constructAlignedChunkReadersMemCount", qs.constructAlignedChunkReadersMemCount); + addMsIfNonZero( + stats, + "constructNonAlignedChunkReadersDiskTimeMs", + qs.constructNonAlignedChunkReadersDiskTime * NS_TO_MS_FACTOR); + addMsIfNonZero( + stats, + "constructNonAlignedChunkReadersMemTimeMs", + qs.constructNonAlignedChunkReadersMemTime * NS_TO_MS_FACTOR); + addMsIfNonZero( + stats, + "constructAlignedChunkReadersDiskTimeMs", + qs.constructAlignedChunkReadersDiskTime * NS_TO_MS_FACTOR); + addMsIfNonZero( + stats, + "constructAlignedChunkReadersMemTimeMs", + qs.constructAlignedChunkReadersMemTime * NS_TO_MS_FACTOR); + + stats.addProperty("loadChunkFromCacheCount", qs.loadChunkFromCacheCount); + stats.addProperty("loadChunkFromDiskCount", qs.loadChunkFromDiskCount); + stats.addProperty("loadChunkActualIOSize", qs.loadChunkActualIOSize); + + addIfNonZero( + stats, "pageReadersDecodeAlignedDiskCount", qs.pageReadersDecodeAlignedDiskCount); + addMsIfNonZero( + stats, + "pageReadersDecodeAlignedDiskTimeMs", + qs.pageReadersDecodeAlignedDiskTime * NS_TO_MS_FACTOR); + addIfNonZero(stats, "pageReadersDecodeAlignedMemCount", qs.pageReadersDecodeAlignedMemCount); + addMsIfNonZero( + stats, + "pageReadersDecodeAlignedMemTimeMs", + qs.pageReadersDecodeAlignedMemTime * NS_TO_MS_FACTOR); + addIfNonZero( + stats, "pageReadersDecodeNonAlignedDiskCount", qs.pageReadersDecodeNonAlignedDiskCount); + addMsIfNonZero( + stats, + "pageReadersDecodeNonAlignedDiskTimeMs", + qs.pageReadersDecodeNonAlignedDiskTime * NS_TO_MS_FACTOR); + addIfNonZero( + stats, "pageReadersDecodeNonAlignedMemCount", qs.pageReadersDecodeNonAlignedMemCount); + addMsIfNonZero( + stats, + "pageReadersDecodeNonAlignedMemTimeMs", + qs.pageReadersDecodeNonAlignedMemTime * NS_TO_MS_FACTOR); + addIfNonZero(stats, "pageReaderMaxUsedMemorySize", qs.pageReaderMaxUsedMemorySize); + addIfNonZero(stats, "chunkWithMetadataErrorsCount", qs.chunkWithMetadataErrorsCount); + } + + stats.addProperty("timeSeriesIndexFilteredRows", qs.timeSeriesIndexFilteredRows); + stats.addProperty("chunkIndexFilteredRows", qs.chunkIndexFilteredRows); + stats.addProperty("pageIndexFilteredRows", qs.pageIndexFilteredRows); + + if (verbose) { + stats.addProperty("rowScanFilteredRows", qs.rowScanFilteredRows); + } + + return stats; + } + + private JsonObject renderOperatorJson( + PlanNode planNodeTree, Map operatorStatistics) { + if (planNodeTree == null) { + return null; + } + + JsonObject operatorJson = new JsonObject(); + TOperatorStatistics opStats = operatorStatistics.get(planNodeTree.getPlanNodeId().toString()); + + operatorJson.addProperty("planNodeId", planNodeTree.getPlanNodeId().toString()); + operatorJson.addProperty("nodeType", planNodeTree.getClass().getSimpleName()); + + if (opStats != null) { + operatorJson.addProperty("operatorType", opStats.getOperatorType()); + if (opStats.isSetCount()) { + operatorJson.addProperty("count", opStats.getCount()); + } + operatorJson.addProperty( + "cpuTimeMs", formatMs(opStats.getTotalExecutionTimeInNanos() * NS_TO_MS_FACTOR)); + operatorJson.addProperty("outputRows", opStats.getOutputRows()); + operatorJson.addProperty("hasNextCalledCount", opStats.hasNextCalledCount); + operatorJson.addProperty("nextCalledCount", opStats.nextCalledCount); + if (opStats.getMemoryUsage() != 0) { + operatorJson.addProperty("estimatedMemorySize", opStats.getMemoryUsage()); + } + + if (opStats.getSpecifiedInfoSize() != 0) { + JsonObject specifiedInfo = new JsonObject(); + for (Map.Entry entry : opStats.getSpecifiedInfo().entrySet()) { + specifiedInfo.addProperty(entry.getKey(), entry.getValue()); + } + operatorJson.add("specifiedInfo", specifiedInfo); + } + } + + List children = planNodeTree.getChildren(); + if (children != null && !children.isEmpty()) { + JsonArray childrenArray = new JsonArray(); + for (PlanNode child : children) { + JsonObject childJson = renderOperatorJson(child, operatorStatistics); + if (childJson != null) { + childrenArray.add(childJson); + } + } + if (childrenArray.size() > 0) { + operatorJson.add("children", childrenArray); + } + } + + return operatorJson; + } + + private static double formatMs(double ms) { + return Math.round(ms * 1000.0) / 1000.0; + } + + private static void addIfNonZero(JsonObject obj, String key, long value) { + if (value != 0) { + obj.addProperty(key, value); + } + } + + private static void addMsIfNonZero(JsonObject obj, String key, double value) { + if (Math.abs(value) > EPSILON) { + obj.addProperty(key, formatMs(value)); + } + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/PlanGraphJsonPrinterTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/PlanGraphJsonPrinterTest.java new file mode 100644 index 0000000000000..a600957b25fa3 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/PlanGraphJsonPrinterTest.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.planner.node; + +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanGraphJsonPrinter; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.LimitNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.OutputNode; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.GenericLiteral; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; + +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.Optional; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +public class PlanGraphJsonPrinterTest { + + @Test + public void testSimplePlanToJson() { + // Build a simple plan: Output -> Limit -> Filter + LimitNode placeholder = new LimitNode(new PlanNodeId("placeholder"), null, 0, Optional.empty()); + FilterNode filterNode = + new FilterNode( + new PlanNodeId("3"), + placeholder, + new ComparisonExpression( + ComparisonExpression.Operator.EQUAL, + new SymbolReference("s1"), + new GenericLiteral("INT32", "1"))); + + LimitNode limitNode = new LimitNode(new PlanNodeId("2"), filterNode, 10, Optional.empty()); + + List outputSymbols = Arrays.asList(new Symbol("s1"), new Symbol("s2")); + OutputNode outputNode = + new OutputNode(new PlanNodeId("1"), limitNode, Arrays.asList("s1", "s2"), outputSymbols); + + String json = PlanGraphJsonPrinter.toPrettyJson(outputNode); + + assertNotNull(json); + assertTrue(json.length() > 0); + + // Parse the JSON and verify structure + JsonObject root = JsonParser.parseString(json).getAsJsonObject(); + assertEquals("OutputNode-1", root.get("name").getAsString()); + assertEquals("1", root.get("id").getAsString()); + assertTrue(root.has("properties")); + assertTrue(root.has("children")); + + // Verify child (LimitNode) + JsonObject limitJson = root.getAsJsonArray("children").get(0).getAsJsonObject(); + assertEquals("LimitNode-2", limitJson.get("name").getAsString()); + + // Verify grandchild (FilterNode) + JsonObject filterJson = limitJson.getAsJsonArray("children").get(0).getAsJsonObject(); + assertEquals("FilterNode-3", filterJson.get("name").getAsString()); + } + + @Test + public void testGetJsonLinesReturnsSingleElement() { + LimitNode limitNode = + new LimitNode( + new PlanNodeId("1"), + new LimitNode(new PlanNodeId("placeholder"), null, 0, Optional.empty()), + 5, + Optional.empty()); + + List lines = PlanGraphJsonPrinter.getJsonLines(limitNode); + assertEquals(1, lines.size()); + + // The single element should be valid JSON + JsonObject root = JsonParser.parseString(lines.get(0)).getAsJsonObject(); + assertNotNull(root.get("name")); + } + + @Test + public void testJsonIsValidFormat() { + LimitNode limitNode = + new LimitNode( + new PlanNodeId("1"), + new LimitNode(new PlanNodeId("placeholder"), null, 0, Optional.empty()), + 42, + Optional.empty()); + + String json = PlanGraphJsonPrinter.toPrettyJson(limitNode); + + // Should parse without errors + JsonObject root = JsonParser.parseString(json).getAsJsonObject(); + assertEquals("LimitNode-1", root.get("name").getAsString()); + assertTrue(root.has("properties")); + assertEquals("42", root.getAsJsonObject("properties").get("Count").getAsString()); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ExplainFormatTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ExplainFormatTest.java new file mode 100644 index 0000000000000..56e204b38df9d --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ExplainFormatTest.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql; + +import org.apache.iotdb.db.protocol.session.IClientSession; +import org.apache.iotdb.db.protocol.session.InternalClientSession; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Explain; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExplainAnalyze; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExplainOutputFormat; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Statement; +import org.apache.iotdb.db.queryengine.plan.relational.sql.parser.SqlParser; + +import org.junit.Before; +import org.junit.Test; + +import java.time.ZoneId; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class ExplainFormatTest { + + private SqlParser sqlParser; + private IClientSession clientSession; + + @Before + public void setUp() { + sqlParser = new SqlParser(); + clientSession = new InternalClientSession("testClient"); + clientSession.setDatabaseName("testdb"); + } + + private Statement parseSQL(String sql) { + return sqlParser.createStatement(sql, ZoneId.systemDefault(), clientSession); + } + + @Test + public void testExplainDefaultFormat() { + Statement stmt = parseSQL("EXPLAIN SELECT * FROM table1"); + assertTrue(stmt instanceof Explain); + assertEquals(ExplainOutputFormat.GRAPHVIZ, ((Explain) stmt).getOutputFormat()); + } + + @Test + public void testExplainGraphvizFormat() { + Statement stmt = parseSQL("EXPLAIN (FORMAT GRAPHVIZ) SELECT * FROM table1"); + assertTrue(stmt instanceof Explain); + assertEquals(ExplainOutputFormat.GRAPHVIZ, ((Explain) stmt).getOutputFormat()); + } + + @Test + public void testExplainJsonFormat() { + Statement stmt = parseSQL("EXPLAIN (FORMAT JSON) SELECT * FROM table1"); + assertTrue(stmt instanceof Explain); + assertEquals(ExplainOutputFormat.JSON, ((Explain) stmt).getOutputFormat()); + } + + @Test + public void testExplainJsonFormatCaseInsensitive() { + Statement stmt = parseSQL("EXPLAIN (FORMAT json) SELECT * FROM table1"); + assertTrue(stmt instanceof Explain); + assertEquals(ExplainOutputFormat.JSON, ((Explain) stmt).getOutputFormat()); + } + + @Test(expected = Exception.class) + public void testExplainInvalidFormat() { + parseSQL("EXPLAIN (FORMAT XML) SELECT * FROM table1"); + } + + @Test + public void testExplainAnalyzeDefaultFormat() { + Statement stmt = parseSQL("EXPLAIN ANALYZE SELECT * FROM table1"); + assertTrue(stmt instanceof ExplainAnalyze); + assertEquals(ExplainOutputFormat.TEXT, ((ExplainAnalyze) stmt).getOutputFormat()); + } + + @Test + public void testExplainAnalyzeTextFormat() { + Statement stmt = parseSQL("EXPLAIN ANALYZE (FORMAT TEXT) SELECT * FROM table1"); + assertTrue(stmt instanceof ExplainAnalyze); + assertEquals(ExplainOutputFormat.TEXT, ((ExplainAnalyze) stmt).getOutputFormat()); + } + + @Test + public void testExplainAnalyzeJsonFormat() { + Statement stmt = parseSQL("EXPLAIN ANALYZE (FORMAT JSON) SELECT * FROM table1"); + assertTrue(stmt instanceof ExplainAnalyze); + assertEquals(ExplainOutputFormat.JSON, ((ExplainAnalyze) stmt).getOutputFormat()); + } + + @Test + public void testExplainAnalyzeVerboseJsonFormat() { + Statement stmt = parseSQL("EXPLAIN ANALYZE VERBOSE (FORMAT JSON) SELECT * FROM table1"); + assertTrue(stmt instanceof ExplainAnalyze); + ExplainAnalyze ea = (ExplainAnalyze) stmt; + assertEquals(ExplainOutputFormat.JSON, ea.getOutputFormat()); + assertTrue(ea.isVerbose()); + } + + @Test(expected = Exception.class) + public void testExplainAnalyzeInvalidFormat() { + parseSQL("EXPLAIN ANALYZE (FORMAT GRAPHVIZ) SELECT * FROM table1"); + } + + @Test(expected = Exception.class) + public void testExplainTextFormatInvalid() { + // TEXT is not valid for EXPLAIN (only GRAPHVIZ and JSON) + parseSQL("EXPLAIN (FORMAT TEXT) SELECT * FROM table1"); + } +} diff --git a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 index 9b09d6ebad043..c7f1cf4b9bcfc 100644 --- a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 +++ b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 @@ -932,8 +932,8 @@ copyToStatementOption // ------------------------------------------- Query Statement --------------------------------------------------------- queryStatement : query #statementDefault - | EXPLAIN (query | executeStatement | executeImmediateStatement) #explain - | EXPLAIN ANALYZE VERBOSE? (query | executeStatement | executeImmediateStatement) #explainAnalyze + | EXPLAIN ('(' FORMAT identifier ')')? (query | executeStatement | executeImmediateStatement) #explain + | EXPLAIN ANALYZE VERBOSE? ('(' FORMAT identifier ')')? (query | executeStatement | executeImmediateStatement) #explainAnalyze ; query From 196df02de9d6eea8a923d3ed67a3f15872defdb3 Mon Sep 17 00:00:00 2001 From: JackieTien97 Date: Thu, 2 Apr 2026 12:14:20 +0800 Subject: [PATCH 2/8] Address review feedback for EXPLAIN FORMAT JSON feature - Fix ExplainAnalyzeOperator to only instantiate the needed drawer (TEXT or JSON), avoiding wasted work - Replace hand-concatenated JSON in mergeExplainResultsJson with Gson to prevent injection from unescaped CTE names - Add proper imports in PlanGraphJsonPrinter, replace FQN with simple class names - Use JsonArray for list-type properties instead of String.valueOf() - Fix ExplainAnalyze.equals()/hashCode() to include outputFormat and verbose - Add Javadoc to ExplainOutputFormat documenting valid formats per statement - Default MPPQueryContext.explainOutputFormat to TEXT instead of null - Mark old 5-arg ExplainAnalyzeOperator constructor @Deprecated - Improve testExplainInvalidFormat to assert on error message content - Add common pitfalls section to CLAUDE.md Co-Authored-By: Claude Opus 4.6 (1M context) --- CLAUDE.md | 137 ++++++++++++++++++ .../query/recent/IoTExplainJsonFormatIT.java | 10 +- .../queryengine/common/MPPQueryContext.java | 2 +- .../operator/ExplainAnalyzeOperator.java | 26 +++- ...ableModelStatementMemorySourceVisitor.java | 43 ++---- .../plan/node/PlanGraphJsonPrinter.java | 116 ++++++++------- .../relational/sql/ast/ExplainAnalyze.java | 6 +- .../sql/ast/ExplainOutputFormat.java | 9 ++ .../FragmentInstanceStatisticsJsonDrawer.java | 1 + 9 files changed, 254 insertions(+), 96 deletions(-) create mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000000000..421c68ad8ba54 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,137 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +Apache IoTDB is a time series database for IoT data. It uses a distributed architecture with ConfigNodes (metadata/coordination) and DataNodes (storage/query). Data is stored in TsFile columnar format (separate repo: https://github.com/apache/tsfile). Current version is 2.0.7-SNAPSHOT. + +## Build Commands + +```bash +# Full build (skip tests) +mvn clean package -pl distribution -am -DskipTests + +# Build a specific module (e.g., datanode) +mvn clean package -pl iotdb-core/datanode -am -DskipTests + +# Run unit tests for a specific module +mvn clean test -pl iotdb-core/datanode + +# Run a single test class +mvn clean test -pl iotdb-core/datanode -Dtest=ClassName + +# Run a single test method +mvn clean test -pl iotdb-core/datanode -Dtest=ClassName#methodName + +# Format code (requires JDK 17+; auto-skipped on JDK <17) +mvn spotless:apply + +# Format code in integration-test module +mvn spotless:apply -P with-integration-tests + +# Check formatting without applying +mvn spotless:check +``` + +## Integration Tests + +Integration tests live in `integration-test/` (not included in default build). They require the `with-integration-tests` profile: + +```bash +# Build template-node first (needed once, or after code changes) +mvn clean package -DskipTests -pl integration-test -am -P with-integration-tests + +# Run tree-model ITs (simple: 1 ConfigNode + 1 DataNode) +mvn clean verify -DskipUTs -pl integration-test -am -P with-integration-tests + +# Run tree-model ITs (cluster: 1 ConfigNode + 3 DataNodes) +mvn clean verify -DskipUTs -pl integration-test -am -PClusterIT -P with-integration-tests + +# Run table-model ITs (simple) +mvn clean verify -DskipUTs -pl integration-test -am -PTableSimpleIT -P with-integration-tests + +# Run table-model ITs (cluster) +mvn clean verify -DskipUTs -pl integration-test -am -PTableClusterIT -P with-integration-tests +``` + +To run integration tests from IntelliJ: enable the `with-integration-tests` profile in Maven sidebar, then run test cases directly. + +## Code Style + +- **Spotless** with Google Java Format (GOOGLE style). Import order: `org.apache.iotdb`, blank, `javax`, `java`, static. +- **Checkstyle** is also configured (see `checkstyle.xml` at project root). +- Java source/target level is 1.8 (compiled with `maven.compiler.release=8` on JDK 9+). + +## Architecture + +### Node Types + +- **ConfigNode** (`iotdb-core/confignode`): Manages cluster metadata, schema regions, data regions, partition tables. Coordinates via Ratis consensus. +- **DataNode** (`iotdb-core/datanode`): Handles data storage, query execution, and client connections. The main server component. +- **AINode** (`iotdb-core/ainode`): Python-based node for AI/ML inference tasks. + +### Dual Data Model + +IoTDB supports two data models operating on the same storage: +- **Tree model**: Traditional IoT hierarchy (e.g., `root.ln.wf01.wt01.temperature`). SQL uses path-based addressing. +- **Table model** (relational): SQL table semantics. Grammar lives in `iotdb-core/relational-grammar/`. Query plan code under `queryengine/plan/relational/`. + +### Key DataNode Subsystems (`iotdb-core/datanode`) + +- **queryengine**: SQL parsing, planning, optimization, and execution. + - `plan/parser/` - ANTLR-based SQL parser + - `plan/statement/` - AST statement nodes + - `plan/planner/` - Logical and physical planning (tree model: `TreeModelPlanner`, table model: under `plan/relational/`) + - `plan/optimization/` - Query optimization rules + - `execution/operator/` - Physical operators (volcano-style iterator model) + - `execution/exchange/` - Inter-node data exchange + - `execution/fragment/` - Distributed query fragment management +- **storageengine**: Write path, memtable, flush, WAL, compaction, TsFile management. + - `dataregion/` - DataRegion lifecycle, memtable, flush, compaction + - `dataregion/wal/` - Write-ahead log + - `buffer/` - Memory buffer management +- **schemaengine**: Schema (timeseries metadata) management. +- **pipe**: Data sync/replication framework (source -> processor -> sink pipeline). +- **consensus**: DataNode-side consensus integration. +- **subscription**: Client subscription service for streaming data changes. + +### Consensus (`iotdb-core/consensus`) + +Pluggable consensus protocols: Simple (single-node), Ratis (Raft-based), IoT Consensus (optimized for IoT writes). Factory pattern via `ConsensusFactory`. + +### Protocol Layer (`iotdb-protocol/`) + +Thrift IDL definitions for RPC between nodes. Generated sources are produced automatically during build. Sub-modules: `thrift-commons`, `thrift-confignode`, `thrift-datanode`, `thrift-consensus`, `thrift-ainode`. + +### Client Libraries (`iotdb-client/`) + +- `session/` - Java Session API (primary client interface) +- `jdbc/` - JDBC driver +- `cli/` - Command-line client +- `client-cpp/`, `client-go/`, `client-py/` - Multi-language clients +- `service-rpc/` - Shared Thrift service definitions + +### API Layer (`iotdb-api/`) + +Extension point interfaces: `udf-api` (user-defined functions), `trigger-api` (event triggers), `pipe-api` (data sync plugins), `external-api`, `external-service-api`. + +## IDE Setup + +After `mvn package`, right-click the root project in IntelliJ and choose "Maven -> Reload Project" to add generated source roots (Thrift and ANTLR). + +Generated source directories that need to be on the source path: +- `**/target/generated-sources/thrift` +- `**/target/generated-sources/antlr4` + +## Common Pitfalls + +### Build + +- **Missing Thrift compiler**: The local machine may not have the `thrift` binary installed. Running `mvn clean package -pl -am -DskipTests` will fail at the `iotdb-thrift` module. **Workaround**: To verify your changes compile, use `mvn compile -pl ` (without `-am` or `clean`) to leverage existing target caches. +- **Pre-existing compilation errors in unrelated modules**: The datanode module may have pre-existing compile errors in other subsystems (e.g., pipe, copyto) that cause `mvn clean test -pl iotdb-core/datanode -Dtest=XxxTest` to fail during compilation. **Workaround**: First run `mvn compile -pl iotdb-core/datanode` to confirm your changed files compile successfully. If the errors are in files you did not modify, they are pre-existing and do not affect your changes. + +### Code Style + +- **Always run `mvn spotless:apply` after editing Java files**: Spotless runs `spotless:check` automatically during the `compile` phase. Format violations cause an immediate BUILD FAILURE. Make it a habit to run `mvn spotless:apply -pl ` right after editing, not at the end. For files under `integration-test/`, add `-P with-integration-tests`. +- **Gson version compatibility**: `JsonObject.isEmpty()` / `JsonArray.isEmpty()` may not be available in the Gson version used by this project. Use `size() > 0` instead and add a comment explaining why. diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTExplainJsonFormatIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTExplainJsonFormatIT.java index 94ed692ab6410..e023d6ce38653 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTExplainJsonFormatIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTExplainJsonFormatIT.java @@ -239,13 +239,19 @@ public void testExplainAnalyzeTextFormatExplicit() { } } - @Test(expected = SQLException.class) - public void testExplainInvalidFormat() throws SQLException { + @Test + public void testExplainInvalidFormat() { String sql = "EXPLAIN (FORMAT XML) SELECT * FROM testtb"; try (Connection conn = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); Statement statement = conn.createStatement()) { statement.execute("USE " + DATABASE_NAME); statement.executeQuery(sql); + fail("Expected SQLException for invalid format"); + } catch (SQLException e) { + Assert.assertTrue( + "Error message should mention the invalid format", + e.getMessage().toUpperCase().contains("FORMAT") + || e.getMessage().toUpperCase().contains("XML")); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java index 43b214f226cc3..8f248c2936361 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java @@ -109,7 +109,7 @@ public enum ExplainType { // - EXPLAIN: Show the logical and physical query plan without execution // - EXPLAIN_ANALYZE: Execute the query and collect detailed execution statistics private ExplainType explainType = ExplainType.NONE; - private ExplainOutputFormat explainOutputFormat = null; + private ExplainOutputFormat explainOutputFormat = ExplainOutputFormat.TEXT; private boolean verbose = false; private QueryPlanStatistics queryPlanStatistics = null; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/ExplainAnalyzeOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/ExplainAnalyzeOperator.java index 52af0bb745583..9676c94844a8a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/ExplainAnalyzeOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/ExplainAnalyzeOperator.java @@ -82,15 +82,14 @@ public class ExplainAnalyzeOperator implements ProcessOperator { private final List instances; private final ExplainOutputFormat outputFormat; - private final FragmentInstanceStatisticsDrawer fragmentInstanceStatisticsDrawer = - new FragmentInstanceStatisticsDrawer(); - private final FragmentInstanceStatisticsJsonDrawer fragmentInstanceStatisticsJsonDrawer = - new FragmentInstanceStatisticsJsonDrawer(); + private final FragmentInstanceStatisticsDrawer fragmentInstanceStatisticsDrawer; + private final FragmentInstanceStatisticsJsonDrawer fragmentInstanceStatisticsJsonDrawer; private final ScheduledFuture logRecordTask; private final IClientManager clientManager; private final MPPQueryContext mppQueryContext; + @Deprecated public ExplainAnalyzeOperator( OperatorContext operatorContext, Operator child, @@ -118,8 +117,16 @@ public ExplainAnalyzeOperator( QueryExecution queryExecution = (QueryExecution) coordinator.getQueryExecution(queryId); this.instances = queryExecution.getDistributedPlan().getInstances(); mppQueryContext = queryExecution.getContext(); - fragmentInstanceStatisticsDrawer.renderPlanStatistics(mppQueryContext); - fragmentInstanceStatisticsJsonDrawer.renderPlanStatistics(mppQueryContext); + + if (outputFormat == ExplainOutputFormat.JSON) { + this.fragmentInstanceStatisticsDrawer = null; + this.fragmentInstanceStatisticsJsonDrawer = new FragmentInstanceStatisticsJsonDrawer(); + fragmentInstanceStatisticsJsonDrawer.renderPlanStatistics(mppQueryContext); + } else { + this.fragmentInstanceStatisticsDrawer = new FragmentInstanceStatisticsDrawer(); + this.fragmentInstanceStatisticsJsonDrawer = null; + fragmentInstanceStatisticsDrawer.renderPlanStatistics(mppQueryContext); + } // The time interval guarantees the result of EXPLAIN ANALYZE will be printed at least three // times. @@ -146,8 +153,11 @@ public TsBlock next() throws Exception { return null; } - fragmentInstanceStatisticsDrawer.renderDispatchCost(mppQueryContext); - fragmentInstanceStatisticsJsonDrawer.renderDispatchCost(mppQueryContext); + if (outputFormat == ExplainOutputFormat.JSON) { + fragmentInstanceStatisticsJsonDrawer.renderDispatchCost(mppQueryContext); + } else { + fragmentInstanceStatisticsDrawer.renderDispatchCost(mppQueryContext); + } // fetch statics from all fragment instances TsBlock result = buildResult(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/memory/TableModelStatementMemorySourceVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/memory/TableModelStatementMemorySourceVisitor.java index c6a7692395a37..2633805074d07 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/memory/TableModelStatementMemorySourceVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/memory/TableModelStatementMemorySourceVisitor.java @@ -41,6 +41,11 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowDevice; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Table; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.utils.Pair; @@ -60,6 +65,8 @@ public class TableModelStatementMemorySourceVisitor extends AstVisitor { + private static final Gson GSON = new GsonBuilder().setPrettyPrinting().create(); + @Override public StatementMemorySource visitNode( final Node node, final TableModelStatementMemorySourceContext context) { @@ -170,36 +177,18 @@ private List mergeExplainResultsJson( return mainExplainResult; } - // For JSON format with CTEs, wrap everything in a combined JSON object - StringBuilder sb = new StringBuilder(); - sb.append("{\n"); - sb.append(" \"cteQueries\": [\n"); - int cteIndex = 0; - int cteSize = cteExplainResults.size(); + JsonObject wrapper = new JsonObject(); + JsonArray cteArray = new JsonArray(); for (Map.Entry, Pair>> entry : cteExplainResults.entrySet()) { - sb.append(" {\n"); - sb.append(" \"name\": \"").append(entry.getKey().getNode().getName()).append("\",\n"); - sb.append(" \"plan\": "); - // Each CTE's plan is already a JSON string - for (String line : entry.getValue().getRight()) { - sb.append(line); - } - sb.append("\n }"); - if (++cteIndex < cteSize) { - sb.append(","); - } - sb.append("\n"); - } - sb.append(" ],\n"); - sb.append(" \"mainQuery\": "); - for (String line : mainExplainResult) { - sb.append(line); + JsonObject cte = new JsonObject(); + cte.addProperty("name", entry.getKey().getNode().getName()); + cte.add("plan", JsonParser.parseString(entry.getValue().getRight().get(0))); + cteArray.add(cte); } - sb.append("\n}"); + wrapper.add("cteQueries", cteArray); + wrapper.add("mainQuery", JsonParser.parseString(mainExplainResult.get(0))); - List result = new ArrayList<>(); - result.add(sb.toString()); - return result; + return Collections.singletonList(GSON.toJson(wrapper)); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphJsonPrinter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphJsonPrinter.java index e35c23d466059..97e7d88328f58 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphJsonPrinter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphJsonPrinter.java @@ -19,12 +19,21 @@ package org.apache.iotdb.db.queryengine.plan.planner.plan.node; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.DeviceTableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ExchangeNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ExplainAnalyzeNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.LimitNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.MergeSortNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.OffsetNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.OutputNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SortNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeDeviceViewScanNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.UnionNode; import com.google.gson.Gson; import com.google.gson.GsonBuilder; @@ -59,6 +68,7 @@ private static JsonObject buildJsonNode(PlanNode node) { jsonNode.addProperty("id", nodeId); JsonObject properties = buildProperties(node); + // JsonObject.isEmpty() is not available in all Gson versions if (properties.size() > 0) { jsonNode.add("properties", properties); } @@ -80,71 +90,68 @@ private static JsonObject buildProperties(PlanNode node) { if (node instanceof OutputNode) { OutputNode n = (OutputNode) node; - properties.addProperty("OutputColumns", String.valueOf(n.getOutputColumnNames())); - properties.addProperty("OutputSymbols", String.valueOf(n.getOutputSymbols())); + properties.add("OutputColumns", toJsonArray(n.getOutputColumnNames())); + properties.add("OutputSymbols", toJsonArray(n.getOutputSymbols())); } else if (node instanceof ExplainAnalyzeNode) { ExplainAnalyzeNode n = (ExplainAnalyzeNode) node; - properties.addProperty("ChildPermittedOutputs", String.valueOf(n.getChildPermittedOutputs())); + properties.add("ChildPermittedOutputs", toJsonArray(n.getChildPermittedOutputs())); } else if (node instanceof TableScanNode) { buildTableScanProperties(properties, (TableScanNode) node); } else if (node instanceof ExchangeNode) { // No extra properties needed - } else if (node - instanceof org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode) { - buildAggregationProperties( - properties, - (org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode) node); - } else if (node - instanceof org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode) { - org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode n = - (org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode) node; + } else if (node instanceof AggregationNode) { + buildAggregationProperties(properties, (AggregationNode) node); + } else if (node instanceof FilterNode) { + FilterNode n = (FilterNode) node; properties.addProperty("Predicate", String.valueOf(n.getPredicate())); - } else if (node - instanceof org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode) { - org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode n = - (org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode) node; - properties.addProperty("OutputSymbols", String.valueOf(n.getOutputSymbols())); - properties.addProperty("Expressions", String.valueOf(n.getAssignments().getMap().values())); - } else if (node - instanceof org.apache.iotdb.db.queryengine.plan.relational.planner.node.LimitNode) { - org.apache.iotdb.db.queryengine.plan.relational.planner.node.LimitNode n = - (org.apache.iotdb.db.queryengine.plan.relational.planner.node.LimitNode) node; - properties.addProperty("Count", String.valueOf(n.getCount())); - } else if (node - instanceof org.apache.iotdb.db.queryengine.plan.relational.planner.node.OffsetNode) { - org.apache.iotdb.db.queryengine.plan.relational.planner.node.OffsetNode n = - (org.apache.iotdb.db.queryengine.plan.relational.planner.node.OffsetNode) node; - properties.addProperty("Count", String.valueOf(n.getCount())); - } else if (node - instanceof org.apache.iotdb.db.queryengine.plan.relational.planner.node.SortNode) { - org.apache.iotdb.db.queryengine.plan.relational.planner.node.SortNode n = - (org.apache.iotdb.db.queryengine.plan.relational.planner.node.SortNode) node; + } else if (node instanceof ProjectNode) { + ProjectNode n = (ProjectNode) node; + properties.add("OutputSymbols", toJsonArray(n.getOutputSymbols())); + properties.add("Expressions", toJsonArray(n.getAssignments().getMap().values())); + } else if (node instanceof LimitNode) { + LimitNode n = (LimitNode) node; + properties.addProperty("Count", n.getCount()); + } else if (node instanceof OffsetNode) { + OffsetNode n = (OffsetNode) node; + properties.addProperty("Count", n.getCount()); + } else if (node instanceof SortNode) { + SortNode n = (SortNode) node; properties.addProperty("OrderBy", String.valueOf(n.getOrderingScheme())); - } else if (node - instanceof org.apache.iotdb.db.queryengine.plan.relational.planner.node.MergeSortNode) { - org.apache.iotdb.db.queryengine.plan.relational.planner.node.MergeSortNode n = - (org.apache.iotdb.db.queryengine.plan.relational.planner.node.MergeSortNode) node; + } else if (node instanceof MergeSortNode) { + MergeSortNode n = (MergeSortNode) node; properties.addProperty("OrderBy", String.valueOf(n.getOrderingScheme())); - } else if (node - instanceof org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode) { - org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode n = - (org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode) node; + } else if (node instanceof JoinNode) { + JoinNode n = (JoinNode) node; properties.addProperty("JoinType", String.valueOf(n.getJoinType())); - properties.addProperty("Criteria", String.valueOf(n.getCriteria())); - properties.addProperty("OutputSymbols", String.valueOf(n.getOutputSymbols())); - } else if (node - instanceof org.apache.iotdb.db.queryengine.plan.relational.planner.node.UnionNode) { - org.apache.iotdb.db.queryengine.plan.relational.planner.node.UnionNode n = - (org.apache.iotdb.db.queryengine.plan.relational.planner.node.UnionNode) node; - properties.addProperty("OutputSymbols", String.valueOf(n.getOutputSymbols())); + properties.add("Criteria", toJsonArray(n.getCriteria())); + properties.add("OutputSymbols", toJsonArray(n.getOutputSymbols())); + } else if (node instanceof UnionNode) { + UnionNode n = (UnionNode) node; + properties.add("OutputSymbols", toJsonArray(n.getOutputSymbols())); } return properties; } + private static JsonArray toJsonArray(java.util.Collection items) { + JsonArray array = new JsonArray(); + for (T item : items) { + array.add(String.valueOf(item)); + } + return array; + } + + private static JsonArray toJsonArray(List items) { + JsonArray array = new JsonArray(); + for (T item : items) { + array.add(String.valueOf(item)); + } + return array; + } + private static void buildTableScanProperties(JsonObject properties, TableScanNode node) { properties.addProperty("QualifiedTableName", node.getQualifiedObjectName().toString()); - properties.addProperty("OutputSymbols", String.valueOf(node.getOutputSymbols())); + properties.add("OutputSymbols", toJsonArray(node.getOutputSymbols())); if (node instanceof DeviceTableScanNode) { DeviceTableScanNode deviceNode = (DeviceTableScanNode) node; @@ -182,15 +189,12 @@ private static void buildTableScanProperties(JsonObject properties, TableScanNod } } - private static void buildAggregationProperties( - JsonObject properties, - org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode node) { - properties.addProperty("OutputSymbols", String.valueOf(node.getOutputSymbols())); + private static void buildAggregationProperties(JsonObject properties, AggregationNode node) { + properties.add("OutputSymbols", toJsonArray(node.getOutputSymbols())); JsonArray aggregators = new JsonArray(); int i = 0; - for (org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode.Aggregation - aggregation : node.getAggregations().values()) { + for (AggregationNode.Aggregation aggregation : node.getAggregations().values()) { JsonObject agg = new JsonObject(); agg.addProperty("index", i++); agg.addProperty("function", aggregation.getResolvedFunction().toString()); @@ -204,10 +208,10 @@ private static void buildAggregationProperties( } properties.add("Aggregators", aggregators); - properties.addProperty("GroupingKeys", String.valueOf(node.getGroupingKeys())); + properties.add("GroupingKeys", toJsonArray(node.getGroupingKeys())); if (node.isStreamable()) { properties.addProperty("Streamable", true); - properties.addProperty("PreGroupedSymbols", String.valueOf(node.getPreGroupedSymbols())); + properties.add("PreGroupedSymbols", toJsonArray(node.getPreGroupedSymbols())); } properties.addProperty("Step", String.valueOf(node.getStep())); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExplainAnalyze.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExplainAnalyze.java index dc7798967e4d8..b824705df1839 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExplainAnalyze.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExplainAnalyze.java @@ -86,7 +86,7 @@ public List getChildren() { @Override public int hashCode() { - return Objects.hash(statement, verbose); + return Objects.hash(statement, verbose, outputFormat); } @Override @@ -98,7 +98,9 @@ public boolean equals(Object obj) { return false; } ExplainAnalyze o = (ExplainAnalyze) obj; - return Objects.equals(statement, o.statement); + return Objects.equals(statement, o.statement) + && verbose == o.verbose + && outputFormat == o.outputFormat; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExplainOutputFormat.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExplainOutputFormat.java index 9dba3649a7f7f..d7d1fc08bb395 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExplainOutputFormat.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExplainOutputFormat.java @@ -19,6 +19,15 @@ package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; +/** + * Output format for EXPLAIN and EXPLAIN ANALYZE statements. + * + *
    + *
  • {@link #GRAPHVIZ} - Box-drawing plan visualization. Valid for EXPLAIN only (default). + *
  • {@link #TEXT} - Text-based output. Valid for EXPLAIN ANALYZE only (default). + *
  • {@link #JSON} - Structured JSON output. Valid for both EXPLAIN and EXPLAIN ANALYZE. + *
+ */ public enum ExplainOutputFormat { GRAPHVIZ, TEXT, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/FragmentInstanceStatisticsJsonDrawer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/FragmentInstanceStatisticsJsonDrawer.java index 3c6ff7bfb41b2..2b00d6191fe78 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/FragmentInstanceStatisticsJsonDrawer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/FragmentInstanceStatisticsJsonDrawer.java @@ -345,6 +345,7 @@ private JsonObject renderOperatorJson( childrenArray.add(childJson); } } + // JsonArray.isEmpty() is not available in all Gson versions if (childrenArray.size() > 0) { operatorJson.add("children", childrenArray); } From 936694831fed04e680196b28caf4a7b5b1f56a5d Mon Sep 17 00:00:00 2001 From: JackieTien97 Date: Thu, 2 Apr 2026 12:20:33 +0800 Subject: [PATCH 3/8] Remove CLAUDE.md from tracking and add it to .gitignore Co-Authored-By: Claude Opus 4.6 (1M context) --- .gitignore | 4 ++ CLAUDE.md | 137 ----------------------------------------------------- 2 files changed, 4 insertions(+), 137 deletions(-) delete mode 100644 CLAUDE.md diff --git a/.gitignore b/.gitignore index 2c19b1b3a2cd5..8698059eb1fb9 100644 --- a/.gitignore +++ b/.gitignore @@ -127,3 +127,7 @@ iotdb-core/tsfile/src/main/antlr4/org/apache/tsfile/parser/gen/ # Relational Grammar ANTLR iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/.antlr/ + +# Claude Code +CLAUDE.md +.claude/ diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index 421c68ad8ba54..0000000000000 --- a/CLAUDE.md +++ /dev/null @@ -1,137 +0,0 @@ -# CLAUDE.md - -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. - -## Project Overview - -Apache IoTDB is a time series database for IoT data. It uses a distributed architecture with ConfigNodes (metadata/coordination) and DataNodes (storage/query). Data is stored in TsFile columnar format (separate repo: https://github.com/apache/tsfile). Current version is 2.0.7-SNAPSHOT. - -## Build Commands - -```bash -# Full build (skip tests) -mvn clean package -pl distribution -am -DskipTests - -# Build a specific module (e.g., datanode) -mvn clean package -pl iotdb-core/datanode -am -DskipTests - -# Run unit tests for a specific module -mvn clean test -pl iotdb-core/datanode - -# Run a single test class -mvn clean test -pl iotdb-core/datanode -Dtest=ClassName - -# Run a single test method -mvn clean test -pl iotdb-core/datanode -Dtest=ClassName#methodName - -# Format code (requires JDK 17+; auto-skipped on JDK <17) -mvn spotless:apply - -# Format code in integration-test module -mvn spotless:apply -P with-integration-tests - -# Check formatting without applying -mvn spotless:check -``` - -## Integration Tests - -Integration tests live in `integration-test/` (not included in default build). They require the `with-integration-tests` profile: - -```bash -# Build template-node first (needed once, or after code changes) -mvn clean package -DskipTests -pl integration-test -am -P with-integration-tests - -# Run tree-model ITs (simple: 1 ConfigNode + 1 DataNode) -mvn clean verify -DskipUTs -pl integration-test -am -P with-integration-tests - -# Run tree-model ITs (cluster: 1 ConfigNode + 3 DataNodes) -mvn clean verify -DskipUTs -pl integration-test -am -PClusterIT -P with-integration-tests - -# Run table-model ITs (simple) -mvn clean verify -DskipUTs -pl integration-test -am -PTableSimpleIT -P with-integration-tests - -# Run table-model ITs (cluster) -mvn clean verify -DskipUTs -pl integration-test -am -PTableClusterIT -P with-integration-tests -``` - -To run integration tests from IntelliJ: enable the `with-integration-tests` profile in Maven sidebar, then run test cases directly. - -## Code Style - -- **Spotless** with Google Java Format (GOOGLE style). Import order: `org.apache.iotdb`, blank, `javax`, `java`, static. -- **Checkstyle** is also configured (see `checkstyle.xml` at project root). -- Java source/target level is 1.8 (compiled with `maven.compiler.release=8` on JDK 9+). - -## Architecture - -### Node Types - -- **ConfigNode** (`iotdb-core/confignode`): Manages cluster metadata, schema regions, data regions, partition tables. Coordinates via Ratis consensus. -- **DataNode** (`iotdb-core/datanode`): Handles data storage, query execution, and client connections. The main server component. -- **AINode** (`iotdb-core/ainode`): Python-based node for AI/ML inference tasks. - -### Dual Data Model - -IoTDB supports two data models operating on the same storage: -- **Tree model**: Traditional IoT hierarchy (e.g., `root.ln.wf01.wt01.temperature`). SQL uses path-based addressing. -- **Table model** (relational): SQL table semantics. Grammar lives in `iotdb-core/relational-grammar/`. Query plan code under `queryengine/plan/relational/`. - -### Key DataNode Subsystems (`iotdb-core/datanode`) - -- **queryengine**: SQL parsing, planning, optimization, and execution. - - `plan/parser/` - ANTLR-based SQL parser - - `plan/statement/` - AST statement nodes - - `plan/planner/` - Logical and physical planning (tree model: `TreeModelPlanner`, table model: under `plan/relational/`) - - `plan/optimization/` - Query optimization rules - - `execution/operator/` - Physical operators (volcano-style iterator model) - - `execution/exchange/` - Inter-node data exchange - - `execution/fragment/` - Distributed query fragment management -- **storageengine**: Write path, memtable, flush, WAL, compaction, TsFile management. - - `dataregion/` - DataRegion lifecycle, memtable, flush, compaction - - `dataregion/wal/` - Write-ahead log - - `buffer/` - Memory buffer management -- **schemaengine**: Schema (timeseries metadata) management. -- **pipe**: Data sync/replication framework (source -> processor -> sink pipeline). -- **consensus**: DataNode-side consensus integration. -- **subscription**: Client subscription service for streaming data changes. - -### Consensus (`iotdb-core/consensus`) - -Pluggable consensus protocols: Simple (single-node), Ratis (Raft-based), IoT Consensus (optimized for IoT writes). Factory pattern via `ConsensusFactory`. - -### Protocol Layer (`iotdb-protocol/`) - -Thrift IDL definitions for RPC between nodes. Generated sources are produced automatically during build. Sub-modules: `thrift-commons`, `thrift-confignode`, `thrift-datanode`, `thrift-consensus`, `thrift-ainode`. - -### Client Libraries (`iotdb-client/`) - -- `session/` - Java Session API (primary client interface) -- `jdbc/` - JDBC driver -- `cli/` - Command-line client -- `client-cpp/`, `client-go/`, `client-py/` - Multi-language clients -- `service-rpc/` - Shared Thrift service definitions - -### API Layer (`iotdb-api/`) - -Extension point interfaces: `udf-api` (user-defined functions), `trigger-api` (event triggers), `pipe-api` (data sync plugins), `external-api`, `external-service-api`. - -## IDE Setup - -After `mvn package`, right-click the root project in IntelliJ and choose "Maven -> Reload Project" to add generated source roots (Thrift and ANTLR). - -Generated source directories that need to be on the source path: -- `**/target/generated-sources/thrift` -- `**/target/generated-sources/antlr4` - -## Common Pitfalls - -### Build - -- **Missing Thrift compiler**: The local machine may not have the `thrift` binary installed. Running `mvn clean package -pl -am -DskipTests` will fail at the `iotdb-thrift` module. **Workaround**: To verify your changes compile, use `mvn compile -pl ` (without `-am` or `clean`) to leverage existing target caches. -- **Pre-existing compilation errors in unrelated modules**: The datanode module may have pre-existing compile errors in other subsystems (e.g., pipe, copyto) that cause `mvn clean test -pl iotdb-core/datanode -Dtest=XxxTest` to fail during compilation. **Workaround**: First run `mvn compile -pl iotdb-core/datanode` to confirm your changed files compile successfully. If the errors are in files you did not modify, they are pre-existing and do not affect your changes. - -### Code Style - -- **Always run `mvn spotless:apply` after editing Java files**: Spotless runs `spotless:check` automatically during the `compile` phase. Format violations cause an immediate BUILD FAILURE. Make it a habit to run `mvn spotless:apply -pl ` right after editing, not at the end. For files under `integration-test/`, add `-P with-integration-tests`. -- **Gson version compatibility**: `JsonObject.isEmpty()` / `JsonArray.isEmpty()` may not be available in the Gson version used by this project. Use `size() > 0` instead and add a comment explaining why. From 14d31578f49aca7c57d14836dff8f03583783843 Mon Sep 17 00:00:00 2001 From: JackieTien97 Date: Thu, 2 Apr 2026 16:06:15 +0800 Subject: [PATCH 4/8] add IT --- .../query/recent/IoTExplainJsonFormatIT.java | 142 +++++++++++++++++- ...ableModelStatementMemorySourceVisitor.java | 2 +- 2 files changed, 142 insertions(+), 2 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTExplainJsonFormatIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTExplainJsonFormatIT.java index e023d6ce38653..0f3d031312414 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTExplainJsonFormatIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTExplainJsonFormatIT.java @@ -26,6 +26,7 @@ import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; import org.apache.iotdb.itbase.env.BaseEnv; +import com.google.gson.JsonArray; import com.google.gson.JsonObject; import com.google.gson.JsonParser; import org.junit.AfterClass; @@ -52,7 +53,12 @@ public class IoTExplainJsonFormatIT { public static void setUp() { Locale.setDefault(Locale.ENGLISH); - EnvFactory.getEnv().getConfig().getCommonConfig().setPartitionInterval(1000); + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setPartitionInterval(1000) + .setMemtableSizeThreshold(10000) + .setMaxRowsInCteBuffer(100); EnvFactory.getEnv().initClusterEnvironment(); try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); @@ -61,9 +67,12 @@ public static void setUp() { statement.execute("USE " + DATABASE_NAME); statement.execute( "CREATE TABLE IF NOT EXISTS testtb(deviceid STRING TAG, voltage FLOAT FIELD)"); + // Insert data across multiple time partitions (partitionInterval=1000) statement.execute("INSERT INTO testtb VALUES(1000, 'd1', 100.0)"); statement.execute("INSERT INTO testtb VALUES(2000, 'd1', 200.0)"); + statement.execute("INSERT INTO testtb VALUES(3000, 'd1', 150.0)"); statement.execute("INSERT INTO testtb VALUES(1000, 'd2', 300.0)"); + statement.execute("INSERT INTO testtb VALUES(2000, 'd2', 250.0)"); } catch (Exception e) { fail(e.getMessage()); } @@ -254,4 +263,135 @@ public void testExplainInvalidFormat() { || e.getMessage().toUpperCase().contains("XML")); } } + + @Test + public void testExplainAnalyzeJsonMultipleFragmentInstances() { + String sql = "EXPLAIN ANALYZE (FORMAT JSON) SELECT * FROM testtb"; + try (Connection conn = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = conn.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = statement.executeQuery(sql); + + Assert.assertTrue(resultSet.next()); + String jsonStr = resultSet.getString(1); + JsonObject root = JsonParser.parseString(jsonStr).getAsJsonObject(); + + // Verify fragmentInstancesCount matches the size of fragmentInstances array + int declaredCount = root.get("fragmentInstancesCount").getAsInt(); + JsonArray fragmentInstances = root.getAsJsonArray("fragmentInstances"); + Assert.assertNotNull("fragmentInstances array should be present", fragmentInstances); + Assert.assertEquals( + "fragmentInstancesCount should match fragmentInstances array size", + declaredCount, + fragmentInstances.size()); + Assert.assertTrue( + "Should have at least 2 fragment instances for multi-partition data", declaredCount >= 2); + + // Verify each fragment instance has required fields + for (int i = 0; i < fragmentInstances.size(); i++) { + JsonObject fi = fragmentInstances.get(i).getAsJsonObject(); + Assert.assertTrue("Fragment instance should have 'id'", fi.has("id")); + Assert.assertTrue("Fragment instance should have 'state'", fi.has("state")); + Assert.assertTrue("Fragment instance should have 'dataRegion'", fi.has("dataRegion")); + } + + resultSet.close(); + } catch (SQLException e) { + fail(e.getMessage()); + } + } + + @Test + public void testExplainJsonWithCte() { + try (Connection conn = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = conn.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + statement.execute( + "CREATE TABLE IF NOT EXISTS cte_tb(deviceid STRING TAG, voltage FLOAT FIELD)"); + statement.execute("INSERT INTO cte_tb VALUES(1000, 'd1', 50.0)"); + + String sql = + "EXPLAIN (FORMAT JSON) WITH cte1 AS MATERIALIZED (SELECT * FROM cte_tb) " + + "SELECT * FROM testtb WHERE testtb.deviceid IN (SELECT deviceid FROM cte1)"; + ResultSet resultSet = statement.executeQuery(sql); + + Assert.assertTrue(resultSet.next()); + String jsonStr = resultSet.getString(1); + JsonObject root = JsonParser.parseString(jsonStr).getAsJsonObject(); + + // When CTEs are present, the JSON should have cteQueries and mainQuery + Assert.assertTrue("JSON with CTE should have 'cteQueries' field", root.has("cteQueries")); + Assert.assertTrue("JSON with CTE should have 'mainQuery' field", root.has("mainQuery")); + + JsonArray cteQueries = root.getAsJsonArray("cteQueries"); + Assert.assertEquals("Should have exactly 1 CTE query", 1, cteQueries.size()); + + JsonObject cte = cteQueries.get(0).getAsJsonObject(); + Assert.assertTrue("CTE should have 'name' field", cte.has("name")); + Assert.assertEquals("cte1", cte.get("name").getAsString()); + Assert.assertTrue("CTE should have 'plan' field", cte.has("plan")); + + // The main query plan should be a JSON object with 'name' field (plan node) + JsonObject mainQuery = root.getAsJsonObject("mainQuery"); + Assert.assertTrue("Main query plan should have 'name' field", mainQuery.has("name")); + + statement.execute("DROP TABLE IF EXISTS cte_tb"); + resultSet.close(); + } catch (SQLException e) { + fail(e.getMessage()); + } + } + + @Test + public void testExplainJsonWithScalarSubquery() { + String sql = + "EXPLAIN (FORMAT JSON) SELECT * FROM testtb " + + "WHERE voltage > (SELECT avg(voltage) FROM testtb)"; + try (Connection conn = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = conn.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = statement.executeQuery(sql); + + Assert.assertTrue(resultSet.next()); + String jsonStr = resultSet.getString(1); + JsonObject root = JsonParser.parseString(jsonStr).getAsJsonObject(); + + // Verify it's a valid plan tree with children (subquery creates a more complex plan) + Assert.assertTrue("JSON should have 'name' field", root.has("name")); + Assert.assertTrue("JSON should have 'id' field", root.has("id")); + Assert.assertTrue("Plan with scalar subquery should have 'children'", root.has("children")); + + resultSet.close(); + } catch (SQLException e) { + fail(e.getMessage()); + } + } + + @Test + public void testExplainAnalyzeJsonWithScalarSubquery() { + String sql = + "EXPLAIN ANALYZE (FORMAT JSON) SELECT * FROM testtb " + + "WHERE voltage > (SELECT avg(voltage) FROM testtb)"; + try (Connection conn = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = conn.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = statement.executeQuery(sql); + + Assert.assertTrue(resultSet.next()); + String jsonStr = resultSet.getString(1); + JsonObject root = JsonParser.parseString(jsonStr).getAsJsonObject(); + + Assert.assertTrue(root.has("planStatistics")); + Assert.assertTrue(root.has("fragmentInstances")); + Assert.assertTrue(root.has("fragmentInstancesCount")); + + int declaredCount = root.get("fragmentInstancesCount").getAsInt(); + JsonArray fragmentInstances = root.getAsJsonArray("fragmentInstances"); + Assert.assertEquals(declaredCount, fragmentInstances.size()); + + resultSet.close(); + } catch (SQLException e) { + fail(e.getMessage()); + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/memory/TableModelStatementMemorySourceVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/memory/TableModelStatementMemorySourceVisitor.java index 2633805074d07..35ec7bcb1c4a7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/memory/TableModelStatementMemorySourceVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/memory/TableModelStatementMemorySourceVisitor.java @@ -182,7 +182,7 @@ private List mergeExplainResultsJson( for (Map.Entry, Pair>> entry : cteExplainResults.entrySet()) { JsonObject cte = new JsonObject(); - cte.addProperty("name", entry.getKey().getNode().getName()); + cte.addProperty("name", entry.getKey().getNode().getName().toString()); cte.add("plan", JsonParser.parseString(entry.getValue().getRight().get(0))); cteArray.add(cte); } From b76978c6dced1f2b724b7b2aa9da2c2907cab0ec Mon Sep 17 00:00:00 2001 From: JackieTien97 Date: Thu, 2 Apr 2026 16:23:02 +0800 Subject: [PATCH 5/8] ignore some interp files of antlr in rat check --- iotdb-core/relational-grammar/pom.xml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/iotdb-core/relational-grammar/pom.xml b/iotdb-core/relational-grammar/pom.xml index f4fb18bee2c83..289972ce30fb6 100644 --- a/iotdb-core/relational-grammar/pom.xml +++ b/iotdb-core/relational-grammar/pom.xml @@ -96,8 +96,10 @@ true - **/SqlLexer.java - **/SqlLexer.interp + **/RelationalSqlParser.java + **/RelationalSqlLexer.java + **/RelationalSqlLexer.interp + **/RelationalSql.interp From a586954dde7d49b73c7a499ffb1572e98718cc9a Mon Sep 17 00:00:00 2001 From: JackieTien97 Date: Thu, 2 Apr 2026 18:45:06 +0800 Subject: [PATCH 6/8] add comments for expected json format for it --- .../query/recent/IoTExplainJsonFormatIT.java | 293 ++++++++++++++++++ .../UnaliasSymbolReferences.java | 3 +- 2 files changed, 295 insertions(+), 1 deletion(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTExplainJsonFormatIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTExplainJsonFormatIT.java index 0f3d031312414..4cbfd02bf5d7d 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTExplainJsonFormatIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTExplainJsonFormatIT.java @@ -91,6 +91,54 @@ public static void tearDown() { @Test public void testExplainJsonFormat() { + // Expected output (single row, single JSON object representing the distributed plan tree): + // { + // "name": "OutputNode-", + // "id": "", + // "properties": { + // "OutputColumns": ["time", "deviceid", "voltage"], + // "OutputSymbols": ["time", "deviceid", "voltage"] + // }, + // "children": [ + // { + // "name": "CollectNode-", + // "id": "", + // "children": [ + // { + // "name": "ExchangeNode-", + // "id": "", + // "children": [ + // { + // "name": "DeviceTableScanNode-", + // "id": "", + // "properties": { + // "QualifiedTableName": "testdb_json.testtb", + // "OutputSymbols": ["time", "deviceid", "voltage"], + // "DeviceNumber": "1", + // "ScanOrder": "ASC", + // "PushDownOffset": "0", + // "PushDownLimit": "0", + // "PushDownLimitToEachDevice": "false", + // "RegionId": "" + // } + // } + // ] + // }, + // { + // "name": "ExchangeNode-", + // "id": "", + // "children": [ + // { + // "name": "DeviceTableScanNode-", + // "id": "", + // "properties": { ... } + // } + // ] + // } + // ] + // } + // ] + // } String sql = "EXPLAIN (FORMAT JSON) SELECT * FROM testtb"; try (Connection conn = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); Statement statement = conn.createStatement()) { @@ -134,6 +182,76 @@ public void testExplainDefaultFormatIsNotJson() { @Test public void testExplainAnalyzeJsonFormat() { + // Expected output (single row, single JSON object with plan statistics + fragment instances): + // { + // "planStatistics": { + // "analyzeCostMs": , + // "fetchPartitionCostMs": , + // "fetchSchemaCostMs": , + // "logicalPlanCostMs": , + // "logicalOptimizationCostMs": , + // "distributionPlanCostMs": , + // "dispatchCostMs": + // }, + // "fragmentInstancesCount": 3, + // "fragmentInstances": [ + // { + // "id": "..", + // "ip": "127.0.0.1:", + // "dataRegion": "virtual_data_region", + // "state": "FINISHED", + // "totalWallTimeMs": , + // "initDataQuerySourceCostMs": , + // "seqFileUnclosed": 0, "seqFileClosed": 0, + // "unseqFileUnclosed": 0, "unseqFileClosed": 0, + // "readyQueuedTimeMs": , + // "blockQueuedTimeMs": , + // "queryStatistics": { + // "timeSeriesIndexFilteredRows": 0, + // "chunkIndexFilteredRows": 0, + // "pageIndexFilteredRows": 0 + // }, + // "operators": { + // "planNodeId": "", + // "nodeType": "IdentitySinkNode", + // "operatorType": "IdentitySinkOperator", + // "cpuTimeMs": , + // "outputRows": 5, + // "hasNextCalledCount": 5, + // "nextCalledCount": 4, + // "estimatedMemorySize": 1024, + // "specifiedInfo": { "DownStreamPlanNodeId": "" }, + // "children": [ + // { + // "planNodeId": "", + // "nodeType": "CollectNode", + // "operatorType": "CollectOperator", + // ... + // "children": [ + // { "planNodeId": "", "nodeType": "ExchangeNode", ... }, + // { "planNodeId": "", "nodeType": "ExchangeNode", ... } + // ] + // } + // ] + // } + // }, + // { + // "id": "...", "dataRegion": "4", "state": "FINISHED", + // ... + // "operators": { + // "nodeType": "IdentitySinkNode", ... + // "children": [ + // { "nodeType": "DeviceTableScanNode", "operatorType": "TableScanOperator", ... } + // ] + // } + // }, + // { + // "id": "...", "dataRegion": "3", "state": "FINISHED", + // ... + // "operators": { ... } + // } + // ] + // } String sql = "EXPLAIN ANALYZE (FORMAT JSON) SELECT * FROM testtb"; try (Connection conn = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); Statement statement = conn.createStatement()) { @@ -168,6 +286,47 @@ public void testExplainAnalyzeJsonFormat() { @Test public void testExplainAnalyzeVerboseJsonFormat() { + // Expected output (same structure as testExplainAnalyzeJsonFormat, but queryStatistics + // includes verbose fields like bloom filter, metadata, chunk reader, and page decoder stats): + // { + // "planStatistics": { ... }, + // "fragmentInstancesCount": 3, + // "fragmentInstances": [ + // { + // "id": "...", "dataRegion": "virtual_data_region", ... + // "queryStatistics": { + // "loadBloomFilterFromCacheCount": 0, + // "loadBloomFilterFromDiskCount": 0, + // "loadBloomFilterActualIOSize": 0, + // "loadBloomFilterTimeMs": 0.0, + // "loadTimeSeriesMetadataFromCacheCount": 0, + // "loadTimeSeriesMetadataFromDiskCount": 0, + // "loadTimeSeriesMetadataActualIOSize": 0, + // "loadChunkFromCacheCount": 0, + // "loadChunkFromDiskCount": 0, + // "loadChunkActualIOSize": 0, + // "timeSeriesIndexFilteredRows": 0, + // "chunkIndexFilteredRows": 0, + // "pageIndexFilteredRows": 0, + // "rowScanFilteredRows": 0 + // }, + // "operators": { ... } + // }, + // { + // "id": "...", "dataRegion": "4", ... + // "queryStatistics": { + // ... (same as above, plus non-zero fields like:) + // "loadTimeSeriesMetadataAlignedMemSeqCount": 2, + // "loadTimeSeriesMetadataAlignedMemSeqTimeMs": , + // "pageReadersDecodeAlignedMemCount": 2, + // "pageReadersDecodeAlignedMemTimeMs": , + // ... + // }, + // "operators": { ... } + // }, + // { "id": "...", "dataRegion": "3", ... } + // ] + // } String sql = "EXPLAIN ANALYZE VERBOSE (FORMAT JSON) SELECT * FROM testtb"; try (Connection conn = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); Statement statement = conn.createStatement()) { @@ -266,6 +425,17 @@ public void testExplainInvalidFormat() { @Test public void testExplainAnalyzeJsonMultipleFragmentInstances() { + // Expected output (same structure as testExplainAnalyzeJsonFormat): + // { + // "planStatistics": { ... }, + // "fragmentInstancesCount": 3, // >= 2 due to multi-partition data + // "fragmentInstances": [ + // { "id": "...", "dataRegion": "virtual_data_region", "state": "FINISHED", ... }, + // { "id": "...", "dataRegion": "4", "state": "FINISHED", ... }, + // { "id": "...", "dataRegion": "3", "state": "FINISHED", ... } + // ] + // } + // Each fragment instance must have "id", "state", and "dataRegion" fields. String sql = "EXPLAIN ANALYZE (FORMAT JSON) SELECT * FROM testtb"; try (Connection conn = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); Statement statement = conn.createStatement()) { @@ -303,6 +473,42 @@ public void testExplainAnalyzeJsonMultipleFragmentInstances() { @Test public void testExplainJsonWithCte() { + // Expected output (when CTE is present, the JSON wraps cteQueries + mainQuery): + // { + // "cteQueries": [ + // { + // "name": "cte1", + // "plan": + // } + // ], + // "mainQuery": { + // "name": "OutputNode-", + // "id": "", + // "properties": { + // "OutputColumns": ["time", "deviceid", "voltage"], + // "OutputSymbols": ["time", "deviceid", "voltage"] + // }, + // "children": [ + // { + // "name": "ProjectNode-", ... + // "children": [ + // { + // "name": "FilterNode-", ... + // "children": [ + // { + // "name": "SemiJoinNode-", ... + // "children": [ + // { "name": "ExchangeNode-", ... }, // main table scan branch + // { "name": "ExchangeNode-", ... } // CTE scan branch + // ] + // } + // ] + // } + // ] + // } + // ] + // } + // } try (Connection conn = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); Statement statement = conn.createStatement()) { statement.execute("USE " + DATABASE_NAME); @@ -344,6 +550,43 @@ public void testExplainJsonWithCte() { @Test public void testExplainJsonWithScalarSubquery() { + // Expected output (scalar subquery is inlined during optimization, so the plan is a simple + // tree with the constant predicate pushed down to DeviceTableScanNode): + // { + // "name": "OutputNode-", + // "id": "", + // "properties": { + // "OutputColumns": ["time", "deviceid", "voltage"], + // "OutputSymbols": ["time", "deviceid", "voltage"] + // }, + // "children": [ + // { + // "name": "CollectNode-", + // "id": "", + // "children": [ + // { + // "name": "ExchangeNode-", ... + // "children": [ + // { + // "name": "DeviceTableScanNode-", + // "properties": { + // "QualifiedTableName": "testdb_json.testtb", + // "PushDownPredicate": "(\"voltage\" > 2E2)", + // ... + // } + // } + // ] + // }, + // { + // "name": "ExchangeNode-", ... + // "children": [ + // { "name": "DeviceTableScanNode-", ... } + // ] + // } + // ] + // } + // ] + // } String sql = "EXPLAIN (FORMAT JSON) SELECT * FROM testtb " + "WHERE voltage > (SELECT avg(voltage) FROM testtb)"; @@ -369,6 +612,56 @@ public void testExplainJsonWithScalarSubquery() { @Test public void testExplainAnalyzeJsonWithScalarSubquery() { + // Expected output (same EXPLAIN ANALYZE JSON structure, but the scalar subquery is resolved + // at planning time, so the executed plan only scans with a constant predicate): + // { + // "planStatistics": { + // "analyzeCostMs": , + // "fetchPartitionCostMs": , + // "fetchSchemaCostMs": , + // "logicalPlanCostMs": , // higher than simple query due to subquery planning + // "logicalOptimizationCostMs": , + // "distributionPlanCostMs": , + // "dispatchCostMs": + // }, + // "fragmentInstancesCount": 3, + // "fragmentInstances": [ + // { + // "id": "...", "dataRegion": "virtual_data_region", "state": "FINISHED", + // ... + // "operators": { + // "nodeType": "IdentitySinkNode", ... + // "children": [ + // { + // "nodeType": "CollectNode", ... + // "children": [ + // { "nodeType": "ExchangeNode", "outputRows": 2, ... }, + // { "nodeType": "ExchangeNode", "outputRows": 0, ... } + // ] + // } + // ] + // } + // }, + // { + // "dataRegion": "4", ... + // "operators": { + // "nodeType": "IdentitySinkNode", ... + // "children": [ + // { "nodeType": "DeviceTableScanNode", "operatorType": "TableScanOperator", ... } + // ] + // } + // }, + // { + // "dataRegion": "3", ... + // "operators": { + // "nodeType": "IdentitySinkNode", ... + // "children": [ + // { "nodeType": "DeviceTableScanNode", "outputRows": 0, ... } + // ] + // } + // } + // ] + // } String sql = "EXPLAIN ANALYZE (FORMAT JSON) SELECT * FROM testtb " + "WHERE voltage > (SELECT avg(voltage) FROM testtb)"; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java index 24adb746df4e1..4b7ab3f3ab43d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java @@ -385,7 +385,8 @@ public PlanAndMappings visitExplainAnalyze(ExplainAnalyzeNode node, UnaliasConte node.getQueryId(), node.getTimeout(), node.getOutputSymbols().get(0), - newChildPermittedOutputs), + newChildPermittedOutputs, + node.getOutputFormat()), mapping); } From d76f8e0138c5e866c1cc7720047cef37799432bd Mon Sep 17 00:00:00 2001 From: JackieTien97 Date: Fri, 3 Apr 2026 18:50:04 +0800 Subject: [PATCH 7/8] Fix testExplain IT to match updated EXPLAIN grammar with FORMAT clause The grammar change added optional '(' FORMAT identifier ')' to EXPLAIN and EXPLAIN ANALYZE rules, so '(' is now a valid expected token in parser error messages. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../iotdb/relational/it/insertquery/IoTDBInsertQueryIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/insertquery/IoTDBInsertQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/insertquery/IoTDBInsertQueryIT.java index b67fe732f16b4..174b2d4711ae8 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/insertquery/IoTDBInsertQueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/insertquery/IoTDBInsertQueryIT.java @@ -460,7 +460,7 @@ public void testExplain() throws SQLException { e.getMessage(), e.getMessage() .contains( - "700: line 1:9: mismatched input 'INSERT'. Expecting: 'ANALYZE', 'EXECUTE', ")); + "700: line 1:9: mismatched input 'INSERT'. Expecting: '(', 'ANALYZE', 'EXECUTE', ")); } try { @@ -472,7 +472,7 @@ public void testExplain() throws SQLException { e.getMessage(), e.getMessage() .contains( - "700: line 1:17: mismatched input 'INSERT'. Expecting: 'EXECUTE', 'VERBOSE', ")); + "700: line 1:17: mismatched input 'INSERT'. Expecting: '(', 'EXECUTE', 'VERBOSE', ")); } } From c8cc5d781491feaed358c29eb492648e912ab3bf Mon Sep 17 00:00:00 2001 From: JackieTien97 Date: Fri, 3 Apr 2026 19:40:26 +0800 Subject: [PATCH 8/8] add more exclude files in rat check --- iotdb-core/relational-grammar/pom.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/iotdb-core/relational-grammar/pom.xml b/iotdb-core/relational-grammar/pom.xml index 289972ce30fb6..76acbf3355057 100644 --- a/iotdb-core/relational-grammar/pom.xml +++ b/iotdb-core/relational-grammar/pom.xml @@ -100,6 +100,8 @@ **/RelationalSqlLexer.java **/RelationalSqlLexer.interp **/RelationalSql.interp + **/RelationalSqlBaseListener.java + **/RelationalSqlListener.java