diff --git a/.gitignore b/.gitignore index 2c19b1b3a2cd5..8698059eb1fb9 100644 --- a/.gitignore +++ b/.gitignore @@ -127,3 +127,7 @@ iotdb-core/tsfile/src/main/antlr4/org/apache/tsfile/parser/gen/ # Relational Grammar ANTLR iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/.antlr/ + +# Claude Code +CLAUDE.md +.claude/ diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/insertquery/IoTDBInsertQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/insertquery/IoTDBInsertQueryIT.java index b67fe732f16b4..174b2d4711ae8 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/insertquery/IoTDBInsertQueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/insertquery/IoTDBInsertQueryIT.java @@ -460,7 +460,7 @@ public void testExplain() throws SQLException { e.getMessage(), e.getMessage() .contains( - "700: line 1:9: mismatched input 'INSERT'. Expecting: 'ANALYZE', 'EXECUTE', ")); + "700: line 1:9: mismatched input 'INSERT'. Expecting: '(', 'ANALYZE', 'EXECUTE', ")); } try { @@ -472,7 +472,7 @@ public void testExplain() throws SQLException { e.getMessage(), e.getMessage() .contains( - "700: line 1:17: mismatched input 'INSERT'. Expecting: 'EXECUTE', 'VERBOSE', ")); + "700: line 1:17: mismatched input 'INSERT'. Expecting: '(', 'EXECUTE', 'VERBOSE', ")); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTExplainJsonFormatIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTExplainJsonFormatIT.java new file mode 100644 index 0000000000000..4cbfd02bf5d7d --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTExplainJsonFormatIT.java @@ -0,0 +1,690 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one + * * or more contributor license agreements. See the NOTICE file + * * distributed with this work for additional information + * * regarding copyright ownership. The ASF licenses this file + * * to you under the Apache License, Version 2.0 (the + * * "License"); you may not use this file except in compliance + * * with the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, + * * software distributed under the License is distributed on an + * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * * KIND, either express or implied. See the License for the + * * specific language governing permissions and limitations + * * under the License. + * + */ + +package org.apache.iotdb.relational.it.query.recent; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Locale; + +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class}) +public class IoTExplainJsonFormatIT { + private static final String DATABASE_NAME = "testdb_json"; + + @BeforeClass + public static void setUp() { + Locale.setDefault(Locale.ENGLISH); + + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setPartitionInterval(1000) + .setMemtableSizeThreshold(10000) + .setMaxRowsInCteBuffer(100); + EnvFactory.getEnv().initClusterEnvironment(); + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE IF NOT EXISTS " + DATABASE_NAME); + statement.execute("USE " + DATABASE_NAME); + statement.execute( + "CREATE TABLE IF NOT EXISTS testtb(deviceid STRING TAG, voltage FLOAT FIELD)"); + // Insert data across multiple time partitions (partitionInterval=1000) + statement.execute("INSERT INTO testtb VALUES(1000, 'd1', 100.0)"); + statement.execute("INSERT INTO testtb VALUES(2000, 'd1', 200.0)"); + statement.execute("INSERT INTO testtb VALUES(3000, 'd1', 150.0)"); + statement.execute("INSERT INTO testtb VALUES(1000, 'd2', 300.0)"); + statement.execute("INSERT INTO testtb VALUES(2000, 'd2', 250.0)"); + } catch (Exception e) { + fail(e.getMessage()); + } + } + + @AfterClass + public static void tearDown() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("DROP DATABASE IF EXISTS " + DATABASE_NAME); + } catch (Exception e) { + // ignore + } + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testExplainJsonFormat() { + // Expected output (single row, single JSON object representing the distributed plan tree): + // { + // "name": "OutputNode-", + // "id": "", + // "properties": { + // "OutputColumns": ["time", "deviceid", "voltage"], + // "OutputSymbols": ["time", "deviceid", "voltage"] + // }, + // "children": [ + // { + // "name": "CollectNode-", + // "id": "", + // "children": [ + // { + // "name": "ExchangeNode-", + // "id": "", + // "children": [ + // { + // "name": "DeviceTableScanNode-", + // "id": "", + // "properties": { + // "QualifiedTableName": "testdb_json.testtb", + // "OutputSymbols": ["time", "deviceid", "voltage"], + // "DeviceNumber": "1", + // "ScanOrder": "ASC", + // "PushDownOffset": "0", + // "PushDownLimit": "0", + // "PushDownLimitToEachDevice": "false", + // "RegionId": "" + // } + // } + // ] + // }, + // { + // "name": "ExchangeNode-", + // "id": "", + // "children": [ + // { + // "name": "DeviceTableScanNode-", + // "id": "", + // "properties": { ... } + // } + // ] + // } + // ] + // } + // ] + // } + String sql = "EXPLAIN (FORMAT JSON) SELECT * FROM testtb"; + try (Connection conn = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = conn.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = statement.executeQuery(sql); + + Assert.assertTrue("Should have at least one row", resultSet.next()); + String jsonStr = resultSet.getString(1); + Assert.assertNotNull(jsonStr); + + // Verify it's valid JSON + JsonObject root = JsonParser.parseString(jsonStr).getAsJsonObject(); + Assert.assertTrue("JSON should have 'name' field", root.has("name")); + Assert.assertTrue("JSON should have 'id' field", root.has("id")); + + resultSet.close(); + } catch (SQLException e) { + fail(e.getMessage()); + } + } + + @Test + public void testExplainDefaultFormatIsNotJson() { + String sql = "EXPLAIN SELECT * FROM testtb"; + try (Connection conn = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = conn.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = statement.executeQuery(sql); + + Assert.assertTrue("Should have at least one row", resultSet.next()); + String firstLine = resultSet.getString(1); + // Default format (GRAPHVIZ) produces box-drawing characters, not JSON + Assert.assertFalse( + "Default EXPLAIN should not produce JSON", firstLine.trim().startsWith("{")); + + resultSet.close(); + } catch (SQLException e) { + fail(e.getMessage()); + } + } + + @Test + public void testExplainAnalyzeJsonFormat() { + // Expected output (single row, single JSON object with plan statistics + fragment instances): + // { + // "planStatistics": { + // "analyzeCostMs": , + // "fetchPartitionCostMs": , + // "fetchSchemaCostMs": , + // "logicalPlanCostMs": , + // "logicalOptimizationCostMs": , + // "distributionPlanCostMs": , + // "dispatchCostMs": + // }, + // "fragmentInstancesCount": 3, + // "fragmentInstances": [ + // { + // "id": "..", + // "ip": "127.0.0.1:", + // "dataRegion": "virtual_data_region", + // "state": "FINISHED", + // "totalWallTimeMs": , + // "initDataQuerySourceCostMs": , + // "seqFileUnclosed": 0, "seqFileClosed": 0, + // "unseqFileUnclosed": 0, "unseqFileClosed": 0, + // "readyQueuedTimeMs": , + // "blockQueuedTimeMs": , + // "queryStatistics": { + // "timeSeriesIndexFilteredRows": 0, + // "chunkIndexFilteredRows": 0, + // "pageIndexFilteredRows": 0 + // }, + // "operators": { + // "planNodeId": "", + // "nodeType": "IdentitySinkNode", + // "operatorType": "IdentitySinkOperator", + // "cpuTimeMs": , + // "outputRows": 5, + // "hasNextCalledCount": 5, + // "nextCalledCount": 4, + // "estimatedMemorySize": 1024, + // "specifiedInfo": { "DownStreamPlanNodeId": "" }, + // "children": [ + // { + // "planNodeId": "", + // "nodeType": "CollectNode", + // "operatorType": "CollectOperator", + // ... + // "children": [ + // { "planNodeId": "", "nodeType": "ExchangeNode", ... }, + // { "planNodeId": "", "nodeType": "ExchangeNode", ... } + // ] + // } + // ] + // } + // }, + // { + // "id": "...", "dataRegion": "4", "state": "FINISHED", + // ... + // "operators": { + // "nodeType": "IdentitySinkNode", ... + // "children": [ + // { "nodeType": "DeviceTableScanNode", "operatorType": "TableScanOperator", ... } + // ] + // } + // }, + // { + // "id": "...", "dataRegion": "3", "state": "FINISHED", + // ... + // "operators": { ... } + // } + // ] + // } + String sql = "EXPLAIN ANALYZE (FORMAT JSON) SELECT * FROM testtb"; + try (Connection conn = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = conn.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = statement.executeQuery(sql); + + // JSON format should produce a single row with full JSON + Assert.assertTrue("Should have at least one row", resultSet.next()); + String jsonStr = resultSet.getString(1); + Assert.assertNotNull(jsonStr); + + // Verify it's valid JSON + JsonObject root = JsonParser.parseString(jsonStr).getAsJsonObject(); + Assert.assertTrue("JSON should have 'planStatistics' field", root.has("planStatistics")); + Assert.assertTrue( + "JSON should have 'fragmentInstances' field", root.has("fragmentInstances")); + Assert.assertTrue( + "JSON should have 'fragmentInstancesCount' field", root.has("fragmentInstancesCount")); + + // Plan statistics should contain known keys + JsonObject planStats = root.getAsJsonObject("planStatistics"); + Assert.assertTrue(planStats.has("analyzeCostMs")); + Assert.assertTrue(planStats.has("logicalPlanCostMs")); + Assert.assertTrue(planStats.has("distributionPlanCostMs")); + Assert.assertTrue(planStats.has("dispatchCostMs")); + + resultSet.close(); + } catch (SQLException e) { + fail(e.getMessage()); + } + } + + @Test + public void testExplainAnalyzeVerboseJsonFormat() { + // Expected output (same structure as testExplainAnalyzeJsonFormat, but queryStatistics + // includes verbose fields like bloom filter, metadata, chunk reader, and page decoder stats): + // { + // "planStatistics": { ... }, + // "fragmentInstancesCount": 3, + // "fragmentInstances": [ + // { + // "id": "...", "dataRegion": "virtual_data_region", ... + // "queryStatistics": { + // "loadBloomFilterFromCacheCount": 0, + // "loadBloomFilterFromDiskCount": 0, + // "loadBloomFilterActualIOSize": 0, + // "loadBloomFilterTimeMs": 0.0, + // "loadTimeSeriesMetadataFromCacheCount": 0, + // "loadTimeSeriesMetadataFromDiskCount": 0, + // "loadTimeSeriesMetadataActualIOSize": 0, + // "loadChunkFromCacheCount": 0, + // "loadChunkFromDiskCount": 0, + // "loadChunkActualIOSize": 0, + // "timeSeriesIndexFilteredRows": 0, + // "chunkIndexFilteredRows": 0, + // "pageIndexFilteredRows": 0, + // "rowScanFilteredRows": 0 + // }, + // "operators": { ... } + // }, + // { + // "id": "...", "dataRegion": "4", ... + // "queryStatistics": { + // ... (same as above, plus non-zero fields like:) + // "loadTimeSeriesMetadataAlignedMemSeqCount": 2, + // "loadTimeSeriesMetadataAlignedMemSeqTimeMs": , + // "pageReadersDecodeAlignedMemCount": 2, + // "pageReadersDecodeAlignedMemTimeMs": , + // ... + // }, + // "operators": { ... } + // }, + // { "id": "...", "dataRegion": "3", ... } + // ] + // } + String sql = "EXPLAIN ANALYZE VERBOSE (FORMAT JSON) SELECT * FROM testtb"; + try (Connection conn = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = conn.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = statement.executeQuery(sql); + + Assert.assertTrue("Should have at least one row", resultSet.next()); + String jsonStr = resultSet.getString(1); + Assert.assertNotNull(jsonStr); + + // Verify it's valid JSON + JsonObject root = JsonParser.parseString(jsonStr).getAsJsonObject(); + Assert.assertTrue(root.has("planStatistics")); + Assert.assertTrue(root.has("fragmentInstances")); + + resultSet.close(); + } catch (SQLException e) { + fail(e.getMessage()); + } + } + + @Test + public void testExplainAnalyzeDefaultIsText() { + String sql = "EXPLAIN ANALYZE SELECT * FROM testtb"; + try (Connection conn = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = conn.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = statement.executeQuery(sql); + + Assert.assertTrue("Should have at least one row", resultSet.next()); + String firstLine = resultSet.getString(1); + // Default format (TEXT) should not start with '{' + Assert.assertFalse( + "Default EXPLAIN ANALYZE should not produce JSON", firstLine.trim().startsWith("{")); + + resultSet.close(); + } catch (SQLException e) { + fail(e.getMessage()); + } + } + + @Test + public void testExplainGraphvizFormatExplicit() { + String sql = "EXPLAIN (FORMAT GRAPHVIZ) SELECT * FROM testtb"; + try (Connection conn = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = conn.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = statement.executeQuery(sql); + + Assert.assertTrue("Should have at least one row", resultSet.next()); + String firstLine = resultSet.getString(1); + // GRAPHVIZ format produces box-drawing characters, not JSON + Assert.assertFalse( + "GRAPHVIZ EXPLAIN should not produce JSON", firstLine.trim().startsWith("{")); + + resultSet.close(); + } catch (SQLException e) { + fail(e.getMessage()); + } + } + + @Test + public void testExplainAnalyzeTextFormatExplicit() { + String sql = "EXPLAIN ANALYZE (FORMAT TEXT) SELECT * FROM testtb"; + try (Connection conn = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = conn.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = statement.executeQuery(sql); + + Assert.assertTrue("Should have at least one row", resultSet.next()); + String firstLine = resultSet.getString(1); + Assert.assertFalse( + "TEXT EXPLAIN ANALYZE should not produce JSON", firstLine.trim().startsWith("{")); + + resultSet.close(); + } catch (SQLException e) { + fail(e.getMessage()); + } + } + + @Test + public void testExplainInvalidFormat() { + String sql = "EXPLAIN (FORMAT XML) SELECT * FROM testtb"; + try (Connection conn = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = conn.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + statement.executeQuery(sql); + fail("Expected SQLException for invalid format"); + } catch (SQLException e) { + Assert.assertTrue( + "Error message should mention the invalid format", + e.getMessage().toUpperCase().contains("FORMAT") + || e.getMessage().toUpperCase().contains("XML")); + } + } + + @Test + public void testExplainAnalyzeJsonMultipleFragmentInstances() { + // Expected output (same structure as testExplainAnalyzeJsonFormat): + // { + // "planStatistics": { ... }, + // "fragmentInstancesCount": 3, // >= 2 due to multi-partition data + // "fragmentInstances": [ + // { "id": "...", "dataRegion": "virtual_data_region", "state": "FINISHED", ... }, + // { "id": "...", "dataRegion": "4", "state": "FINISHED", ... }, + // { "id": "...", "dataRegion": "3", "state": "FINISHED", ... } + // ] + // } + // Each fragment instance must have "id", "state", and "dataRegion" fields. + String sql = "EXPLAIN ANALYZE (FORMAT JSON) SELECT * FROM testtb"; + try (Connection conn = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = conn.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = statement.executeQuery(sql); + + Assert.assertTrue(resultSet.next()); + String jsonStr = resultSet.getString(1); + JsonObject root = JsonParser.parseString(jsonStr).getAsJsonObject(); + + // Verify fragmentInstancesCount matches the size of fragmentInstances array + int declaredCount = root.get("fragmentInstancesCount").getAsInt(); + JsonArray fragmentInstances = root.getAsJsonArray("fragmentInstances"); + Assert.assertNotNull("fragmentInstances array should be present", fragmentInstances); + Assert.assertEquals( + "fragmentInstancesCount should match fragmentInstances array size", + declaredCount, + fragmentInstances.size()); + Assert.assertTrue( + "Should have at least 2 fragment instances for multi-partition data", declaredCount >= 2); + + // Verify each fragment instance has required fields + for (int i = 0; i < fragmentInstances.size(); i++) { + JsonObject fi = fragmentInstances.get(i).getAsJsonObject(); + Assert.assertTrue("Fragment instance should have 'id'", fi.has("id")); + Assert.assertTrue("Fragment instance should have 'state'", fi.has("state")); + Assert.assertTrue("Fragment instance should have 'dataRegion'", fi.has("dataRegion")); + } + + resultSet.close(); + } catch (SQLException e) { + fail(e.getMessage()); + } + } + + @Test + public void testExplainJsonWithCte() { + // Expected output (when CTE is present, the JSON wraps cteQueries + mainQuery): + // { + // "cteQueries": [ + // { + // "name": "cte1", + // "plan": + // } + // ], + // "mainQuery": { + // "name": "OutputNode-", + // "id": "", + // "properties": { + // "OutputColumns": ["time", "deviceid", "voltage"], + // "OutputSymbols": ["time", "deviceid", "voltage"] + // }, + // "children": [ + // { + // "name": "ProjectNode-", ... + // "children": [ + // { + // "name": "FilterNode-", ... + // "children": [ + // { + // "name": "SemiJoinNode-", ... + // "children": [ + // { "name": "ExchangeNode-", ... }, // main table scan branch + // { "name": "ExchangeNode-", ... } // CTE scan branch + // ] + // } + // ] + // } + // ] + // } + // ] + // } + // } + try (Connection conn = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = conn.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + statement.execute( + "CREATE TABLE IF NOT EXISTS cte_tb(deviceid STRING TAG, voltage FLOAT FIELD)"); + statement.execute("INSERT INTO cte_tb VALUES(1000, 'd1', 50.0)"); + + String sql = + "EXPLAIN (FORMAT JSON) WITH cte1 AS MATERIALIZED (SELECT * FROM cte_tb) " + + "SELECT * FROM testtb WHERE testtb.deviceid IN (SELECT deviceid FROM cte1)"; + ResultSet resultSet = statement.executeQuery(sql); + + Assert.assertTrue(resultSet.next()); + String jsonStr = resultSet.getString(1); + JsonObject root = JsonParser.parseString(jsonStr).getAsJsonObject(); + + // When CTEs are present, the JSON should have cteQueries and mainQuery + Assert.assertTrue("JSON with CTE should have 'cteQueries' field", root.has("cteQueries")); + Assert.assertTrue("JSON with CTE should have 'mainQuery' field", root.has("mainQuery")); + + JsonArray cteQueries = root.getAsJsonArray("cteQueries"); + Assert.assertEquals("Should have exactly 1 CTE query", 1, cteQueries.size()); + + JsonObject cte = cteQueries.get(0).getAsJsonObject(); + Assert.assertTrue("CTE should have 'name' field", cte.has("name")); + Assert.assertEquals("cte1", cte.get("name").getAsString()); + Assert.assertTrue("CTE should have 'plan' field", cte.has("plan")); + + // The main query plan should be a JSON object with 'name' field (plan node) + JsonObject mainQuery = root.getAsJsonObject("mainQuery"); + Assert.assertTrue("Main query plan should have 'name' field", mainQuery.has("name")); + + statement.execute("DROP TABLE IF EXISTS cte_tb"); + resultSet.close(); + } catch (SQLException e) { + fail(e.getMessage()); + } + } + + @Test + public void testExplainJsonWithScalarSubquery() { + // Expected output (scalar subquery is inlined during optimization, so the plan is a simple + // tree with the constant predicate pushed down to DeviceTableScanNode): + // { + // "name": "OutputNode-", + // "id": "", + // "properties": { + // "OutputColumns": ["time", "deviceid", "voltage"], + // "OutputSymbols": ["time", "deviceid", "voltage"] + // }, + // "children": [ + // { + // "name": "CollectNode-", + // "id": "", + // "children": [ + // { + // "name": "ExchangeNode-", ... + // "children": [ + // { + // "name": "DeviceTableScanNode-", + // "properties": { + // "QualifiedTableName": "testdb_json.testtb", + // "PushDownPredicate": "(\"voltage\" > 2E2)", + // ... + // } + // } + // ] + // }, + // { + // "name": "ExchangeNode-", ... + // "children": [ + // { "name": "DeviceTableScanNode-", ... } + // ] + // } + // ] + // } + // ] + // } + String sql = + "EXPLAIN (FORMAT JSON) SELECT * FROM testtb " + + "WHERE voltage > (SELECT avg(voltage) FROM testtb)"; + try (Connection conn = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = conn.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = statement.executeQuery(sql); + + Assert.assertTrue(resultSet.next()); + String jsonStr = resultSet.getString(1); + JsonObject root = JsonParser.parseString(jsonStr).getAsJsonObject(); + + // Verify it's a valid plan tree with children (subquery creates a more complex plan) + Assert.assertTrue("JSON should have 'name' field", root.has("name")); + Assert.assertTrue("JSON should have 'id' field", root.has("id")); + Assert.assertTrue("Plan with scalar subquery should have 'children'", root.has("children")); + + resultSet.close(); + } catch (SQLException e) { + fail(e.getMessage()); + } + } + + @Test + public void testExplainAnalyzeJsonWithScalarSubquery() { + // Expected output (same EXPLAIN ANALYZE JSON structure, but the scalar subquery is resolved + // at planning time, so the executed plan only scans with a constant predicate): + // { + // "planStatistics": { + // "analyzeCostMs": , + // "fetchPartitionCostMs": , + // "fetchSchemaCostMs": , + // "logicalPlanCostMs": , // higher than simple query due to subquery planning + // "logicalOptimizationCostMs": , + // "distributionPlanCostMs": , + // "dispatchCostMs": + // }, + // "fragmentInstancesCount": 3, + // "fragmentInstances": [ + // { + // "id": "...", "dataRegion": "virtual_data_region", "state": "FINISHED", + // ... + // "operators": { + // "nodeType": "IdentitySinkNode", ... + // "children": [ + // { + // "nodeType": "CollectNode", ... + // "children": [ + // { "nodeType": "ExchangeNode", "outputRows": 2, ... }, + // { "nodeType": "ExchangeNode", "outputRows": 0, ... } + // ] + // } + // ] + // } + // }, + // { + // "dataRegion": "4", ... + // "operators": { + // "nodeType": "IdentitySinkNode", ... + // "children": [ + // { "nodeType": "DeviceTableScanNode", "operatorType": "TableScanOperator", ... } + // ] + // } + // }, + // { + // "dataRegion": "3", ... + // "operators": { + // "nodeType": "IdentitySinkNode", ... + // "children": [ + // { "nodeType": "DeviceTableScanNode", "outputRows": 0, ... } + // ] + // } + // } + // ] + // } + String sql = + "EXPLAIN ANALYZE (FORMAT JSON) SELECT * FROM testtb " + + "WHERE voltage > (SELECT avg(voltage) FROM testtb)"; + try (Connection conn = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = conn.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = statement.executeQuery(sql); + + Assert.assertTrue(resultSet.next()); + String jsonStr = resultSet.getString(1); + JsonObject root = JsonParser.parseString(jsonStr).getAsJsonObject(); + + Assert.assertTrue(root.has("planStatistics")); + Assert.assertTrue(root.has("fragmentInstances")); + Assert.assertTrue(root.has("fragmentInstancesCount")); + + int declaredCount = root.get("fragmentInstancesCount").getAsInt(); + JsonArray fragmentInstances = root.getAsJsonArray("fragmentInstances"); + Assert.assertEquals(declaredCount, fragmentInstances.size()); + + resultSet.close(); + } catch (SQLException e) { + fail(e.getMessage()); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java index 88bd1998f683c..8f248c2936361 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java @@ -34,6 +34,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.memory.MemoryReservationManager; import org.apache.iotdb.db.queryengine.plan.planner.memory.NotThreadSafeMemoryReservationManager; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.NodeRef; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExplainOutputFormat; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Query; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Table; @@ -108,6 +109,7 @@ public enum ExplainType { // - EXPLAIN: Show the logical and physical query plan without execution // - EXPLAIN_ANALYZE: Execute the query and collect detailed execution statistics private ExplainType explainType = ExplainType.NONE; + private ExplainOutputFormat explainOutputFormat = ExplainOutputFormat.TEXT; private boolean verbose = false; private QueryPlanStatistics queryPlanStatistics = null; @@ -346,6 +348,14 @@ public ExplainType getExplainType() { return explainType; } + public void setExplainOutputFormat(ExplainOutputFormat explainOutputFormat) { + this.explainOutputFormat = explainOutputFormat; + } + + public ExplainOutputFormat getExplainOutputFormat() { + return explainOutputFormat; + } + public boolean isExplainAnalyze() { return explainType == ExplainType.EXPLAIN_ANALYZE; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/ExplainAnalyzeOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/ExplainAnalyzeOperator.java index b4f1a5c7261bf..9676c94844a8a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/ExplainAnalyzeOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/ExplainAnalyzeOperator.java @@ -33,8 +33,10 @@ import org.apache.iotdb.db.queryengine.plan.execution.QueryExecution; import org.apache.iotdb.db.queryengine.plan.planner.plan.FragmentInstance; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.NodeRef; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExplainOutputFormat; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Table; import org.apache.iotdb.db.queryengine.statistics.FragmentInstanceStatisticsDrawer; +import org.apache.iotdb.db.queryengine.statistics.FragmentInstanceStatisticsJsonDrawer; import org.apache.iotdb.db.queryengine.statistics.QueryStatisticsFetcher; import org.apache.iotdb.db.queryengine.statistics.StatisticLine; import org.apache.iotdb.db.utils.SetThreadName; @@ -78,23 +80,36 @@ public class ExplainAnalyzeOperator implements ProcessOperator { private final boolean verbose; private boolean outputResult = false; private final List instances; + private final ExplainOutputFormat outputFormat; - private final FragmentInstanceStatisticsDrawer fragmentInstanceStatisticsDrawer = - new FragmentInstanceStatisticsDrawer(); + private final FragmentInstanceStatisticsDrawer fragmentInstanceStatisticsDrawer; + private final FragmentInstanceStatisticsJsonDrawer fragmentInstanceStatisticsJsonDrawer; private final ScheduledFuture logRecordTask; private final IClientManager clientManager; private final MPPQueryContext mppQueryContext; + @Deprecated public ExplainAnalyzeOperator( OperatorContext operatorContext, Operator child, long queryId, boolean verbose, long timeout) { + this(operatorContext, child, queryId, verbose, timeout, ExplainOutputFormat.TEXT); + } + + public ExplainAnalyzeOperator( + OperatorContext operatorContext, + Operator child, + long queryId, + boolean verbose, + long timeout, + ExplainOutputFormat outputFormat) { this.operatorContext = operatorContext; this.child = child; this.verbose = verbose; + this.outputFormat = outputFormat; Coordinator coordinator = Coordinator.getInstance(); this.clientManager = coordinator.getInternalServiceClientManager(); @@ -102,7 +117,16 @@ public ExplainAnalyzeOperator( QueryExecution queryExecution = (QueryExecution) coordinator.getQueryExecution(queryId); this.instances = queryExecution.getDistributedPlan().getInstances(); mppQueryContext = queryExecution.getContext(); - fragmentInstanceStatisticsDrawer.renderPlanStatistics(mppQueryContext); + + if (outputFormat == ExplainOutputFormat.JSON) { + this.fragmentInstanceStatisticsDrawer = null; + this.fragmentInstanceStatisticsJsonDrawer = new FragmentInstanceStatisticsJsonDrawer(); + fragmentInstanceStatisticsJsonDrawer.renderPlanStatistics(mppQueryContext); + } else { + this.fragmentInstanceStatisticsDrawer = new FragmentInstanceStatisticsDrawer(); + this.fragmentInstanceStatisticsJsonDrawer = null; + fragmentInstanceStatisticsDrawer.renderPlanStatistics(mppQueryContext); + } // The time interval guarantees the result of EXPLAIN ANALYZE will be printed at least three // times. @@ -129,7 +153,11 @@ public TsBlock next() throws Exception { return null; } - fragmentInstanceStatisticsDrawer.renderDispatchCost(mppQueryContext); + if (outputFormat == ExplainOutputFormat.JSON) { + fragmentInstanceStatisticsJsonDrawer.renderDispatchCost(mppQueryContext); + } else { + fragmentInstanceStatisticsDrawer.renderDispatchCost(mppQueryContext); + } // fetch statics from all fragment instances TsBlock result = buildResult(); @@ -182,6 +210,9 @@ private void logIntermediateResultIfTimeout() { } private TsBlock buildResult() throws FragmentInstanceFetchException { + if (outputFormat == ExplainOutputFormat.JSON) { + return buildJsonResult(); + } Map, Pair>> cteAnalyzeResults = mppQueryContext.getCteExplainResults(); List mainAnalyzeResult = buildFragmentInstanceStatistics(instances, verbose); @@ -199,6 +230,22 @@ private TsBlock buildResult() throws FragmentInstanceFetchException { return builder.build(); } + private TsBlock buildJsonResult() throws FragmentInstanceFetchException { + Map allStatistics = + QueryStatisticsFetcher.fetchAllStatistics(instances, clientManager); + String jsonResult = + fragmentInstanceStatisticsJsonDrawer.renderFragmentInstancesAsJson( + instances, allStatistics, verbose); + + TsBlockBuilder builder = new TsBlockBuilder(Collections.singletonList(TSDataType.TEXT)); + TimeColumnBuilder timeColumnBuilder = builder.getTimeColumnBuilder(); + ColumnBuilder columnBuilder = builder.getColumnBuilder(0); + timeColumnBuilder.writeLong(0); + columnBuilder.writeBinary(new Binary(jsonResult.getBytes())); + builder.declarePosition(); + return builder.build(); + } + private List mergeAnalyzeResults( Map, Pair>> cteAnalyzeResults, List mainAnalyzeResult) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/memory/TableModelStatementMemorySourceVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/memory/TableModelStatementMemorySourceVisitor.java index 932d941979223..35ec7bcb1c4a7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/memory/TableModelStatementMemorySourceVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/memory/TableModelStatementMemorySourceVisitor.java @@ -25,6 +25,7 @@ import org.apache.iotdb.db.queryengine.plan.Coordinator; import org.apache.iotdb.db.queryengine.plan.planner.LocalExecutionPlanner; import org.apache.iotdb.db.queryengine.plan.planner.plan.LogicalQueryPlan; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanGraphJsonPrinter; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanGraphPrinter; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.NodeRef; @@ -35,10 +36,16 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AstVisitor; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CountDevice; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Explain; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExplainOutputFormat; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Node; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowDevice; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Table; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.utils.Pair; @@ -58,6 +65,8 @@ public class TableModelStatementMemorySourceVisitor extends AstVisitor { + private static final Gson GSON = new GsonBuilder().setPrettyPrinting().create(); + @Override public StatementMemorySource visitNode( final Node node, final TableModelStatementMemorySourceContext context) { @@ -103,15 +112,26 @@ public StatementMemorySource visitExplain( Coordinator.getInstance().getDataNodeLocationSupplier()) .generateDistributedPlanWithOptimize(planContext); - List mainExplainResult = - outputNodeWithExchange.accept( - new PlanGraphPrinter(), - new PlanGraphPrinter.GraphContext( - context.getQueryContext().getTypeProvider().getTemplatedInfo())); + List mainExplainResult; + if (node.getOutputFormat() == ExplainOutputFormat.JSON) { + mainExplainResult = PlanGraphJsonPrinter.getJsonLines(outputNodeWithExchange); + } else { + mainExplainResult = + outputNodeWithExchange.accept( + new PlanGraphPrinter(), + new PlanGraphPrinter.GraphContext( + context.getQueryContext().getTypeProvider().getTemplatedInfo())); + } Map, Pair>> cteExplainResults = context.getQueryContext().getCteExplainResults(); - List lines = mergeExplainResults(cteExplainResults, mainExplainResult); + List lines; + if (node.getOutputFormat() == ExplainOutputFormat.JSON) { + // For JSON format, we merge CTE results into the JSON output + lines = mergeExplainResultsJson(cteExplainResults, mainExplainResult); + } else { + lines = mergeExplainResults(cteExplainResults, mainExplainResult); + } return getStatementMemorySource(header, lines); } @@ -149,4 +169,26 @@ private List mergeExplainResults( return analyzeResult; } + + private List mergeExplainResultsJson( + Map, Pair>> cteExplainResults, + List mainExplainResult) { + if (cteExplainResults.isEmpty()) { + return mainExplainResult; + } + + JsonObject wrapper = new JsonObject(); + JsonArray cteArray = new JsonArray(); + for (Map.Entry, Pair>> entry : + cteExplainResults.entrySet()) { + JsonObject cte = new JsonObject(); + cte.addProperty("name", entry.getKey().getNode().getName().toString()); + cte.add("plan", JsonParser.parseString(entry.getValue().getRight().get(0))); + cteArray.add(cte); + } + wrapper.add("cteQueries", cteArray); + wrapper.add("mainQuery", JsonParser.parseString(mainExplainResult.get(0))); + + return Collections.singletonList(GSON.toJson(wrapper)); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java index c31a2061f49ec..fe96177ee5f2c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java @@ -3472,7 +3472,12 @@ public Operator visitExplainAnalyze(ExplainAnalyzeNode node, LocalExecutionPlanC node.getPlanNodeId(), ExplainAnalyzeOperator.class.getSimpleName()); return new ExplainAnalyzeOperator( - operatorContext, operator, node.getQueryId(), node.isVerbose(), node.getTimeout()); + operatorContext, + operator, + node.getQueryId(), + node.isVerbose(), + node.getTimeout(), + node.getOutputFormat()); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphJsonPrinter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphJsonPrinter.java new file mode 100644 index 0000000000000..97e7d88328f58 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphJsonPrinter.java @@ -0,0 +1,224 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.planner.plan.node; + +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.DeviceTableScanNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ExchangeNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ExplainAnalyzeNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.LimitNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.MergeSortNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.OffsetNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.OutputNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SortNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeDeviceViewScanNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.UnionNode; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; + +import java.util.ArrayList; +import java.util.List; + +/** + * Converts a plan tree into a JSON representation. Each plan node becomes a JSON object with: - + * "name": the node type with its plan node ID - "id": the plan node ID - "properties": key-value + * pairs - "children": array of child nodes + */ +public class PlanGraphJsonPrinter { + + private static final Gson GSON = new GsonBuilder().setPrettyPrinting().create(); + + private static final String REGION_NOT_ASSIGNED = "Not Assigned"; + + public static String toPrettyJson(PlanNode node) { + JsonObject root = buildJsonNode(node); + return GSON.toJson(root); + } + + private static JsonObject buildJsonNode(PlanNode node) { + JsonObject jsonNode = new JsonObject(); + String simpleName = node.getClass().getSimpleName(); + String nodeId = node.getPlanNodeId().getId(); + + jsonNode.addProperty("name", simpleName + "-" + nodeId); + jsonNode.addProperty("id", nodeId); + + JsonObject properties = buildProperties(node); + // JsonObject.isEmpty() is not available in all Gson versions + if (properties.size() > 0) { + jsonNode.add("properties", properties); + } + + List children = node.getChildren(); + if (children != null && !children.isEmpty()) { + JsonArray childrenArray = new JsonArray(); + for (PlanNode child : children) { + childrenArray.add(buildJsonNode(child)); + } + jsonNode.add("children", childrenArray); + } + + return jsonNode; + } + + private static JsonObject buildProperties(PlanNode node) { + JsonObject properties = new JsonObject(); + + if (node instanceof OutputNode) { + OutputNode n = (OutputNode) node; + properties.add("OutputColumns", toJsonArray(n.getOutputColumnNames())); + properties.add("OutputSymbols", toJsonArray(n.getOutputSymbols())); + } else if (node instanceof ExplainAnalyzeNode) { + ExplainAnalyzeNode n = (ExplainAnalyzeNode) node; + properties.add("ChildPermittedOutputs", toJsonArray(n.getChildPermittedOutputs())); + } else if (node instanceof TableScanNode) { + buildTableScanProperties(properties, (TableScanNode) node); + } else if (node instanceof ExchangeNode) { + // No extra properties needed + } else if (node instanceof AggregationNode) { + buildAggregationProperties(properties, (AggregationNode) node); + } else if (node instanceof FilterNode) { + FilterNode n = (FilterNode) node; + properties.addProperty("Predicate", String.valueOf(n.getPredicate())); + } else if (node instanceof ProjectNode) { + ProjectNode n = (ProjectNode) node; + properties.add("OutputSymbols", toJsonArray(n.getOutputSymbols())); + properties.add("Expressions", toJsonArray(n.getAssignments().getMap().values())); + } else if (node instanceof LimitNode) { + LimitNode n = (LimitNode) node; + properties.addProperty("Count", n.getCount()); + } else if (node instanceof OffsetNode) { + OffsetNode n = (OffsetNode) node; + properties.addProperty("Count", n.getCount()); + } else if (node instanceof SortNode) { + SortNode n = (SortNode) node; + properties.addProperty("OrderBy", String.valueOf(n.getOrderingScheme())); + } else if (node instanceof MergeSortNode) { + MergeSortNode n = (MergeSortNode) node; + properties.addProperty("OrderBy", String.valueOf(n.getOrderingScheme())); + } else if (node instanceof JoinNode) { + JoinNode n = (JoinNode) node; + properties.addProperty("JoinType", String.valueOf(n.getJoinType())); + properties.add("Criteria", toJsonArray(n.getCriteria())); + properties.add("OutputSymbols", toJsonArray(n.getOutputSymbols())); + } else if (node instanceof UnionNode) { + UnionNode n = (UnionNode) node; + properties.add("OutputSymbols", toJsonArray(n.getOutputSymbols())); + } + + return properties; + } + + private static JsonArray toJsonArray(java.util.Collection items) { + JsonArray array = new JsonArray(); + for (T item : items) { + array.add(String.valueOf(item)); + } + return array; + } + + private static JsonArray toJsonArray(List items) { + JsonArray array = new JsonArray(); + for (T item : items) { + array.add(String.valueOf(item)); + } + return array; + } + + private static void buildTableScanProperties(JsonObject properties, TableScanNode node) { + properties.addProperty("QualifiedTableName", node.getQualifiedObjectName().toString()); + properties.add("OutputSymbols", toJsonArray(node.getOutputSymbols())); + + if (node instanceof DeviceTableScanNode) { + DeviceTableScanNode deviceNode = (DeviceTableScanNode) node; + properties.addProperty("DeviceNumber", String.valueOf(deviceNode.getDeviceEntries().size())); + properties.addProperty("ScanOrder", String.valueOf(deviceNode.getScanOrder())); + if (deviceNode.getTimePredicate().isPresent()) { + properties.addProperty( + "TimePredicate", String.valueOf(deviceNode.getTimePredicate().get())); + } + } + + if (node.getPushDownPredicate() != null) { + properties.addProperty("PushDownPredicate", String.valueOf(node.getPushDownPredicate())); + } + properties.addProperty("PushDownOffset", String.valueOf(node.getPushDownOffset())); + properties.addProperty("PushDownLimit", String.valueOf(node.getPushDownLimit())); + + if (node instanceof DeviceTableScanNode) { + properties.addProperty( + "PushDownLimitToEachDevice", + String.valueOf(((DeviceTableScanNode) node).isPushLimitToEachDevice())); + } + + properties.addProperty( + "RegionId", + node.getRegionReplicaSet() == null || node.getRegionReplicaSet().getRegionId() == null + ? REGION_NOT_ASSIGNED + : String.valueOf(node.getRegionReplicaSet().getRegionId().getId())); + + if (node instanceof TreeDeviceViewScanNode) { + TreeDeviceViewScanNode treeNode = (TreeDeviceViewScanNode) node; + properties.addProperty("TreeDB", treeNode.getTreeDBName()); + properties.addProperty( + "MeasurementToColumnName", String.valueOf(treeNode.getMeasurementColumnNameMap())); + } + } + + private static void buildAggregationProperties(JsonObject properties, AggregationNode node) { + properties.add("OutputSymbols", toJsonArray(node.getOutputSymbols())); + + JsonArray aggregators = new JsonArray(); + int i = 0; + for (AggregationNode.Aggregation aggregation : node.getAggregations().values()) { + JsonObject agg = new JsonObject(); + agg.addProperty("index", i++); + agg.addProperty("function", aggregation.getResolvedFunction().toString()); + if (aggregation.hasMask()) { + agg.addProperty("mask", String.valueOf(aggregation.getMask().get())); + } + if (aggregation.isDistinct()) { + agg.addProperty("distinct", true); + } + aggregators.add(agg); + } + properties.add("Aggregators", aggregators); + + properties.add("GroupingKeys", toJsonArray(node.getGroupingKeys())); + if (node.isStreamable()) { + properties.addProperty("Streamable", true); + properties.add("PreGroupedSymbols", toJsonArray(node.getPreGroupedSymbols())); + } + properties.addProperty("Step", String.valueOf(node.getStep())); + } + + public static List getJsonLines(PlanNode node) { + List lines = new ArrayList<>(); + lines.add(toPrettyJson(node)); + return lines; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index cff29c0d83a1a..eb4e1241d1400 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -847,6 +847,7 @@ protected Scope visitLoadTsFile(final LoadTsFile node, final Optional sco @Override protected Scope visitExplain(Explain node, Optional context) { queryContext.setExplainType(ExplainType.EXPLAIN); + queryContext.setExplainOutputFormat(node.getOutputFormat()); analysis.setFinishQueryAfterAnalyze(); return visitQuery((Query) node.getStatement(), context); } @@ -863,6 +864,7 @@ protected Scope visitCopyTo(CopyTo node, Optional context) { protected Scope visitExplainAnalyze(ExplainAnalyze node, Optional context) { queryContext.setExplainType(ExplainType.EXPLAIN_ANALYZE); queryContext.setVerbose(node.isVerbose()); + queryContext.setExplainOutputFormat(node.getOutputFormat()); return visitQuery((Query) node.getStatement(), context); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableLogicalPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableLogicalPlanner.java index c33cb05250bd1..538890b15682a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableLogicalPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableLogicalPlanner.java @@ -625,7 +625,8 @@ private RelationPlan planExplainAnalyze(final ExplainAnalyze statement, final An queryContext.getTimeOut(), symbol, // recording permittedOutputs of ExplainAnalyzeNode's child - getChildPermittedOutputs(analysis, statement.getStatement(), originalQueryPlan)); + getChildPermittedOutputs(analysis, statement.getStatement(), originalQueryPlan), + statement.getOutputFormat()); return new RelationPlan( newRoot, originalQueryPlan.getScope(), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/ExplainAnalyzeNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/ExplainAnalyzeNode.java index 23aa8baad2dfd..d50d07521ec23 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/ExplainAnalyzeNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/ExplainAnalyzeNode.java @@ -24,6 +24,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.SingleChildProcessNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExplainOutputFormat; import java.io.DataOutputStream; import java.io.IOException; @@ -38,6 +39,7 @@ public class ExplainAnalyzeNode extends SingleChildProcessNode { private final long timeout; private final Symbol outputSymbol; private final List childPermittedOutputs; + private final ExplainOutputFormat outputFormat; public ExplainAnalyzeNode( PlanNodeId id, @@ -47,18 +49,46 @@ public ExplainAnalyzeNode( long timeout, Symbol outputSymbol, List childPermittedOutputs) { + this( + id, + child, + verbose, + queryId, + timeout, + outputSymbol, + childPermittedOutputs, + ExplainOutputFormat.TEXT); + } + + public ExplainAnalyzeNode( + PlanNodeId id, + PlanNode child, + boolean verbose, + long queryId, + long timeout, + Symbol outputSymbol, + List childPermittedOutputs, + ExplainOutputFormat outputFormat) { super(id, child); this.verbose = verbose; this.timeout = timeout; this.queryId = queryId; this.outputSymbol = outputSymbol; this.childPermittedOutputs = childPermittedOutputs; + this.outputFormat = outputFormat; } @Override public PlanNode clone() { return new ExplainAnalyzeNode( - getPlanNodeId(), child, verbose, queryId, timeout, outputSymbol, childPermittedOutputs); + getPlanNodeId(), + child, + verbose, + queryId, + timeout, + outputSymbol, + childPermittedOutputs, + outputFormat); } @Override @@ -89,7 +119,8 @@ public PlanNode replaceChildren(List newChildren) { queryId, timeout, outputSymbol, - childPermittedOutputs); + childPermittedOutputs, + outputFormat); } // ExplainAnalyze should be at the same region as Coordinator all the time. Therefore, there will @@ -116,6 +147,10 @@ public long getTimeout() { return timeout; } + public ExplainOutputFormat getOutputFormat() { + return outputFormat; + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java index 24adb746df4e1..4b7ab3f3ab43d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java @@ -385,7 +385,8 @@ public PlanAndMappings visitExplainAnalyze(ExplainAnalyzeNode node, UnaliasConte node.getQueryId(), node.getTimeout(), node.getOutputSymbols().get(0), - newChildPermittedOutputs), + newChildPermittedOutputs, + node.getOutputFormat()), mapping); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Explain.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Explain.java index f3a50175269e8..6e51e9bae607d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Explain.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Explain.java @@ -33,21 +33,34 @@ public class Explain extends Statement { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(Explain.class); private final Statement statement; + private final ExplainOutputFormat outputFormat; public Explain(Statement statement) { super(null); this.statement = requireNonNull(statement, "statement is null"); + this.outputFormat = ExplainOutputFormat.GRAPHVIZ; } public Explain(NodeLocation location, Statement statement) { super(requireNonNull(location, "location is null")); this.statement = requireNonNull(statement, "statement is null"); + this.outputFormat = ExplainOutputFormat.GRAPHVIZ; + } + + public Explain(NodeLocation location, Statement statement, ExplainOutputFormat outputFormat) { + super(requireNonNull(location, "location is null")); + this.statement = requireNonNull(statement, "statement is null"); + this.outputFormat = requireNonNull(outputFormat, "outputFormat is null"); } public Statement getStatement() { return statement; } + public ExplainOutputFormat getOutputFormat() { + return outputFormat; + } + @Override public R accept(AstVisitor visitor, C context) { return visitor.visitExplain(this, context); @@ -60,7 +73,7 @@ public List getChildren() { @Override public int hashCode() { - return Objects.hash(statement); + return Objects.hash(statement, outputFormat); } @Override @@ -72,12 +85,15 @@ public boolean equals(Object obj) { return false; } Explain o = (Explain) obj; - return Objects.equals(statement, o.statement); + return Objects.equals(statement, o.statement) && outputFormat == o.outputFormat; } @Override public String toString() { - return toStringHelper(this).add("statement", statement).toString(); + return toStringHelper(this) + .add("statement", statement) + .add("outputFormat", outputFormat) + .toString(); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExplainAnalyze.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExplainAnalyze.java index cd3df879fc55e..b824705df1839 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExplainAnalyze.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExplainAnalyze.java @@ -35,17 +35,31 @@ public class ExplainAnalyze extends Statement { private final Statement statement; private final boolean verbose; + private final ExplainOutputFormat outputFormat; public ExplainAnalyze(Statement statement, boolean verbose) { super(null); this.statement = requireNonNull(statement, "statement is null"); this.verbose = verbose; + this.outputFormat = ExplainOutputFormat.TEXT; } public ExplainAnalyze(NodeLocation location, boolean verbose, Statement statement) { super(requireNonNull(location, "location is null")); this.statement = requireNonNull(statement, "statement is null"); this.verbose = verbose; + this.outputFormat = ExplainOutputFormat.TEXT; + } + + public ExplainAnalyze( + NodeLocation location, + boolean verbose, + Statement statement, + ExplainOutputFormat outputFormat) { + super(requireNonNull(location, "location is null")); + this.statement = requireNonNull(statement, "statement is null"); + this.verbose = verbose; + this.outputFormat = requireNonNull(outputFormat, "outputFormat is null"); } public Statement getStatement() { @@ -56,6 +70,10 @@ public boolean isVerbose() { return verbose; } + public ExplainOutputFormat getOutputFormat() { + return outputFormat; + } + @Override public R accept(AstVisitor visitor, C context) { return visitor.visitExplainAnalyze(this, context); @@ -68,7 +86,7 @@ public List getChildren() { @Override public int hashCode() { - return Objects.hash(statement, verbose); + return Objects.hash(statement, verbose, outputFormat); } @Override @@ -80,12 +98,18 @@ public boolean equals(Object obj) { return false; } ExplainAnalyze o = (ExplainAnalyze) obj; - return Objects.equals(statement, o.statement); + return Objects.equals(statement, o.statement) + && verbose == o.verbose + && outputFormat == o.outputFormat; } @Override public String toString() { - return toStringHelper(this).add("statement", statement).add("verbose", verbose).toString(); + return toStringHelper(this) + .add("statement", statement) + .add("verbose", verbose) + .add("outputFormat", outputFormat) + .toString(); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExplainOutputFormat.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExplainOutputFormat.java new file mode 100644 index 0000000000000..d7d1fc08bb395 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExplainOutputFormat.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +/** + * Output format for EXPLAIN and EXPLAIN ANALYZE statements. + * + *
    + *
  • {@link #GRAPHVIZ} - Box-drawing plan visualization. Valid for EXPLAIN only (default). + *
  • {@link #TEXT} - Text-based output. Valid for EXPLAIN ANALYZE only (default). + *
  • {@link #JSON} - Structured JSON output. Valid for both EXPLAIN and EXPLAIN ANALYZE. + *
+ */ +public enum ExplainOutputFormat { + GRAPHVIZ, + TEXT, + JSON +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java index a31b48c24b4a5..c825d6b92cb26 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java @@ -98,6 +98,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExistsPredicate; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Explain; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExplainAnalyze; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExplainOutputFormat; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExtendRegion; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Extract; @@ -1804,7 +1805,22 @@ public Node visitExplain(RelationalSqlParser.ExplainContext ctx) { } else { innerStatement = (Statement) visit(ctx.executeImmediateStatement()); } - return new Explain(getLocation(ctx), innerStatement); + ExplainOutputFormat format = ExplainOutputFormat.GRAPHVIZ; + if (ctx.identifier() != null) { + String formatStr = ((Identifier) visit(ctx.identifier())).getValue().toUpperCase(); + switch (formatStr) { + case "GRAPHVIZ": + format = ExplainOutputFormat.GRAPHVIZ; + break; + case "JSON": + format = ExplainOutputFormat.JSON; + break; + default: + throw new SemanticException( + "Invalid EXPLAIN format: " + formatStr + ". Supported formats: GRAPHVIZ, JSON"); + } + } + return new Explain(getLocation(ctx), innerStatement, format); } @Override @@ -1817,7 +1833,22 @@ public Node visitExplainAnalyze(RelationalSqlParser.ExplainAnalyzeContext ctx) { } else { innerStatement = (Statement) visit(ctx.executeImmediateStatement()); } - return new ExplainAnalyze(getLocation(ctx), ctx.VERBOSE() != null, innerStatement); + ExplainOutputFormat format = ExplainOutputFormat.TEXT; + if (ctx.identifier() != null) { + String formatStr = ((Identifier) visit(ctx.identifier())).getValue().toUpperCase(); + switch (formatStr) { + case "TEXT": + format = ExplainOutputFormat.TEXT; + break; + case "JSON": + format = ExplainOutputFormat.JSON; + break; + default: + throw new SemanticException( + "Invalid EXPLAIN ANALYZE format: " + formatStr + ". Supported formats: TEXT, JSON"); + } + } + return new ExplainAnalyze(getLocation(ctx), ctx.VERBOSE() != null, innerStatement, format); } // ********************** author expressions ******************** diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/FragmentInstanceStatisticsJsonDrawer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/FragmentInstanceStatisticsJsonDrawer.java new file mode 100644 index 0000000000000..2b00d6191fe78 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/FragmentInstanceStatisticsJsonDrawer.java @@ -0,0 +1,372 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.statistics; + +import org.apache.iotdb.db.queryengine.common.FragmentInstanceId; +import org.apache.iotdb.db.queryengine.common.MPPQueryContext; +import org.apache.iotdb.db.queryengine.plan.planner.plan.FragmentInstance; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.mpp.rpc.thrift.TFetchFragmentInstanceStatisticsResp; +import org.apache.iotdb.mpp.rpc.thrift.TOperatorStatistics; +import org.apache.iotdb.mpp.rpc.thrift.TQueryStatistics; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Produces JSON output for EXPLAIN ANALYZE results, mirroring the same data as {@link + * FragmentInstanceStatisticsDrawer} but in JSON format. + */ +public class FragmentInstanceStatisticsJsonDrawer { + + private static final Gson GSON = new GsonBuilder().setPrettyPrinting().create(); + private static final double NS_TO_MS_FACTOR = 1.0 / 1000000; + private static final double EPSILON = 1e-10; + + private final JsonObject planStatistics = new JsonObject(); + + public void renderPlanStatistics(MPPQueryContext context) { + planStatistics.addProperty( + "analyzeCostMs", formatMs(context.getAnalyzeCost() * NS_TO_MS_FACTOR)); + planStatistics.addProperty( + "fetchPartitionCostMs", formatMs(context.getFetchPartitionCost() * NS_TO_MS_FACTOR)); + planStatistics.addProperty( + "fetchSchemaCostMs", formatMs(context.getFetchSchemaCost() * NS_TO_MS_FACTOR)); + planStatistics.addProperty( + "logicalPlanCostMs", formatMs(context.getLogicalPlanCost() * NS_TO_MS_FACTOR)); + planStatistics.addProperty( + "logicalOptimizationCostMs", + formatMs(context.getLogicalOptimizationCost() * NS_TO_MS_FACTOR)); + planStatistics.addProperty( + "distributionPlanCostMs", formatMs(context.getDistributionPlanCost() * NS_TO_MS_FACTOR)); + } + + public void renderDispatchCost(MPPQueryContext context) { + planStatistics.addProperty( + "dispatchCostMs", formatMs(context.getDispatchCost() * NS_TO_MS_FACTOR)); + } + + public String renderFragmentInstancesAsJson( + List instancesToBeRendered, + Map allStatistics, + boolean verbose) { + + JsonObject root = new JsonObject(); + root.add("planStatistics", planStatistics); + + List validInstances = + instancesToBeRendered.stream() + .filter( + instance -> { + TFetchFragmentInstanceStatisticsResp statistics = + allStatistics.get(instance.getId()); + return statistics != null && statistics.getDataRegion() != null; + }) + .collect(Collectors.toList()); + + root.addProperty("fragmentInstancesCount", validInstances.size()); + + JsonArray fragmentInstancesArray = new JsonArray(); + for (FragmentInstance instance : validInstances) { + TFetchFragmentInstanceStatisticsResp statistics = allStatistics.get(instance.getId()); + JsonObject fiJson = new JsonObject(); + + fiJson.addProperty("id", instance.getId().toString()); + fiJson.addProperty("ip", statistics.getIp()); + fiJson.addProperty("dataRegion", statistics.getDataRegion()); + fiJson.addProperty("state", statistics.getState()); + fiJson.addProperty( + "totalWallTimeMs", statistics.getEndTimeInMS() - statistics.getStartTimeInMS()); + fiJson.addProperty( + "initDataQuerySourceCostMs", + formatMs(statistics.getInitDataQuerySourceCost() * NS_TO_MS_FACTOR)); + + if (statistics.isSetInitDataQuerySourceRetryCount() + && statistics.getInitDataQuerySourceRetryCount() > 0) { + fiJson.addProperty( + "initDataQuerySourceRetryCount", statistics.getInitDataQuerySourceRetryCount()); + } + + fiJson.addProperty("seqFileUnclosed", statistics.getSeqUnclosedNum()); + fiJson.addProperty("seqFileClosed", statistics.getSeqClosednNum()); + fiJson.addProperty("unseqFileUnclosed", statistics.getUnseqUnclosedNum()); + fiJson.addProperty("unseqFileClosed", statistics.getUnseqClosedNum()); + fiJson.addProperty( + "readyQueuedTimeMs", formatMs(statistics.getReadyQueuedTime() * NS_TO_MS_FACTOR)); + fiJson.addProperty( + "blockQueuedTimeMs", formatMs(statistics.getBlockQueuedTime() * NS_TO_MS_FACTOR)); + + // Query statistics + JsonObject queryStats = renderQueryStatisticsJson(statistics.getQueryStatistics(), verbose); + fiJson.add("queryStatistics", queryStats); + + // Operators + PlanNode planNodeTree = instance.getFragment().getPlanNodeTree(); + JsonObject operatorTree = + renderOperatorJson(planNodeTree, statistics.getOperatorStatisticsMap()); + if (operatorTree != null) { + fiJson.add("operators", operatorTree); + } + + fragmentInstancesArray.add(fiJson); + } + + root.add("fragmentInstances", fragmentInstancesArray); + return GSON.toJson(root); + } + + private JsonObject renderQueryStatisticsJson(TQueryStatistics qs, boolean verbose) { + JsonObject stats = new JsonObject(); + + if (verbose) { + stats.addProperty("loadBloomFilterFromCacheCount", qs.loadBloomFilterFromCacheCount); + stats.addProperty("loadBloomFilterFromDiskCount", qs.loadBloomFilterFromDiskCount); + stats.addProperty("loadBloomFilterActualIOSize", qs.loadBloomFilterActualIOSize); + stats.addProperty( + "loadBloomFilterTimeMs", formatMs(qs.loadBloomFilterTime * NS_TO_MS_FACTOR)); + + addIfNonZero( + stats, "loadTimeSeriesMetadataDiskSeqCount", qs.loadTimeSeriesMetadataDiskSeqCount); + addIfNonZero( + stats, "loadTimeSeriesMetadataDiskUnSeqCount", qs.loadTimeSeriesMetadataDiskUnSeqCount); + addIfNonZero( + stats, "loadTimeSeriesMetadataMemSeqCount", qs.loadTimeSeriesMetadataMemSeqCount); + addIfNonZero( + stats, "loadTimeSeriesMetadataMemUnSeqCount", qs.loadTimeSeriesMetadataMemUnSeqCount); + addIfNonZero( + stats, + "loadTimeSeriesMetadataAlignedDiskSeqCount", + qs.loadTimeSeriesMetadataAlignedDiskSeqCount); + addIfNonZero( + stats, + "loadTimeSeriesMetadataAlignedDiskUnSeqCount", + qs.loadTimeSeriesMetadataAlignedDiskUnSeqCount); + addIfNonZero( + stats, + "loadTimeSeriesMetadataAlignedMemSeqCount", + qs.loadTimeSeriesMetadataAlignedMemSeqCount); + addIfNonZero( + stats, + "loadTimeSeriesMetadataAlignedMemUnSeqCount", + qs.loadTimeSeriesMetadataAlignedMemUnSeqCount); + + addMsIfNonZero( + stats, + "loadTimeSeriesMetadataDiskSeqTimeMs", + qs.loadTimeSeriesMetadataDiskSeqTime * NS_TO_MS_FACTOR); + addMsIfNonZero( + stats, + "loadTimeSeriesMetadataDiskUnSeqTimeMs", + qs.loadTimeSeriesMetadataDiskUnSeqTime * NS_TO_MS_FACTOR); + addMsIfNonZero( + stats, + "loadTimeSeriesMetadataMemSeqTimeMs", + qs.loadTimeSeriesMetadataMemSeqTime * NS_TO_MS_FACTOR); + addMsIfNonZero( + stats, + "loadTimeSeriesMetadataMemUnSeqTimeMs", + qs.loadTimeSeriesMetadataMemUnSeqTime * NS_TO_MS_FACTOR); + addMsIfNonZero( + stats, + "loadTimeSeriesMetadataAlignedDiskSeqTimeMs", + qs.loadTimeSeriesMetadataAlignedDiskSeqTime * NS_TO_MS_FACTOR); + addMsIfNonZero( + stats, + "loadTimeSeriesMetadataAlignedDiskUnSeqTimeMs", + qs.loadTimeSeriesMetadataAlignedDiskUnSeqTime * NS_TO_MS_FACTOR); + addMsIfNonZero( + stats, + "loadTimeSeriesMetadataAlignedMemSeqTimeMs", + qs.loadTimeSeriesMetadataAlignedMemSeqTime * NS_TO_MS_FACTOR); + addMsIfNonZero( + stats, + "loadTimeSeriesMetadataAlignedMemUnSeqTimeMs", + qs.loadTimeSeriesMetadataAlignedMemUnSeqTime * NS_TO_MS_FACTOR); + + stats.addProperty( + "loadTimeSeriesMetadataFromCacheCount", qs.loadTimeSeriesMetadataFromCacheCount); + stats.addProperty( + "loadTimeSeriesMetadataFromDiskCount", qs.loadTimeSeriesMetadataFromDiskCount); + stats.addProperty( + "loadTimeSeriesMetadataActualIOSize", qs.loadTimeSeriesMetadataActualIOSize); + + addIfNonZero( + stats, + "alignedTimeSeriesMetadataModificationCount", + qs.getAlignedTimeSeriesMetadataModificationCount()); + addMsIfNonZero( + stats, + "alignedTimeSeriesMetadataModificationTimeMs", + qs.getAlignedTimeSeriesMetadataModificationTime() * NS_TO_MS_FACTOR); + addIfNonZero( + stats, + "nonAlignedTimeSeriesMetadataModificationCount", + qs.getNonAlignedTimeSeriesMetadataModificationCount()); + addMsIfNonZero( + stats, + "nonAlignedTimeSeriesMetadataModificationTimeMs", + qs.getNonAlignedTimeSeriesMetadataModificationTime() * NS_TO_MS_FACTOR); + + addIfNonZero( + stats, + "constructNonAlignedChunkReadersDiskCount", + qs.constructNonAlignedChunkReadersDiskCount); + addIfNonZero( + stats, + "constructNonAlignedChunkReadersMemCount", + qs.constructNonAlignedChunkReadersMemCount); + addIfNonZero( + stats, "constructAlignedChunkReadersDiskCount", qs.constructAlignedChunkReadersDiskCount); + addIfNonZero( + stats, "constructAlignedChunkReadersMemCount", qs.constructAlignedChunkReadersMemCount); + addMsIfNonZero( + stats, + "constructNonAlignedChunkReadersDiskTimeMs", + qs.constructNonAlignedChunkReadersDiskTime * NS_TO_MS_FACTOR); + addMsIfNonZero( + stats, + "constructNonAlignedChunkReadersMemTimeMs", + qs.constructNonAlignedChunkReadersMemTime * NS_TO_MS_FACTOR); + addMsIfNonZero( + stats, + "constructAlignedChunkReadersDiskTimeMs", + qs.constructAlignedChunkReadersDiskTime * NS_TO_MS_FACTOR); + addMsIfNonZero( + stats, + "constructAlignedChunkReadersMemTimeMs", + qs.constructAlignedChunkReadersMemTime * NS_TO_MS_FACTOR); + + stats.addProperty("loadChunkFromCacheCount", qs.loadChunkFromCacheCount); + stats.addProperty("loadChunkFromDiskCount", qs.loadChunkFromDiskCount); + stats.addProperty("loadChunkActualIOSize", qs.loadChunkActualIOSize); + + addIfNonZero( + stats, "pageReadersDecodeAlignedDiskCount", qs.pageReadersDecodeAlignedDiskCount); + addMsIfNonZero( + stats, + "pageReadersDecodeAlignedDiskTimeMs", + qs.pageReadersDecodeAlignedDiskTime * NS_TO_MS_FACTOR); + addIfNonZero(stats, "pageReadersDecodeAlignedMemCount", qs.pageReadersDecodeAlignedMemCount); + addMsIfNonZero( + stats, + "pageReadersDecodeAlignedMemTimeMs", + qs.pageReadersDecodeAlignedMemTime * NS_TO_MS_FACTOR); + addIfNonZero( + stats, "pageReadersDecodeNonAlignedDiskCount", qs.pageReadersDecodeNonAlignedDiskCount); + addMsIfNonZero( + stats, + "pageReadersDecodeNonAlignedDiskTimeMs", + qs.pageReadersDecodeNonAlignedDiskTime * NS_TO_MS_FACTOR); + addIfNonZero( + stats, "pageReadersDecodeNonAlignedMemCount", qs.pageReadersDecodeNonAlignedMemCount); + addMsIfNonZero( + stats, + "pageReadersDecodeNonAlignedMemTimeMs", + qs.pageReadersDecodeNonAlignedMemTime * NS_TO_MS_FACTOR); + addIfNonZero(stats, "pageReaderMaxUsedMemorySize", qs.pageReaderMaxUsedMemorySize); + addIfNonZero(stats, "chunkWithMetadataErrorsCount", qs.chunkWithMetadataErrorsCount); + } + + stats.addProperty("timeSeriesIndexFilteredRows", qs.timeSeriesIndexFilteredRows); + stats.addProperty("chunkIndexFilteredRows", qs.chunkIndexFilteredRows); + stats.addProperty("pageIndexFilteredRows", qs.pageIndexFilteredRows); + + if (verbose) { + stats.addProperty("rowScanFilteredRows", qs.rowScanFilteredRows); + } + + return stats; + } + + private JsonObject renderOperatorJson( + PlanNode planNodeTree, Map operatorStatistics) { + if (planNodeTree == null) { + return null; + } + + JsonObject operatorJson = new JsonObject(); + TOperatorStatistics opStats = operatorStatistics.get(planNodeTree.getPlanNodeId().toString()); + + operatorJson.addProperty("planNodeId", planNodeTree.getPlanNodeId().toString()); + operatorJson.addProperty("nodeType", planNodeTree.getClass().getSimpleName()); + + if (opStats != null) { + operatorJson.addProperty("operatorType", opStats.getOperatorType()); + if (opStats.isSetCount()) { + operatorJson.addProperty("count", opStats.getCount()); + } + operatorJson.addProperty( + "cpuTimeMs", formatMs(opStats.getTotalExecutionTimeInNanos() * NS_TO_MS_FACTOR)); + operatorJson.addProperty("outputRows", opStats.getOutputRows()); + operatorJson.addProperty("hasNextCalledCount", opStats.hasNextCalledCount); + operatorJson.addProperty("nextCalledCount", opStats.nextCalledCount); + if (opStats.getMemoryUsage() != 0) { + operatorJson.addProperty("estimatedMemorySize", opStats.getMemoryUsage()); + } + + if (opStats.getSpecifiedInfoSize() != 0) { + JsonObject specifiedInfo = new JsonObject(); + for (Map.Entry entry : opStats.getSpecifiedInfo().entrySet()) { + specifiedInfo.addProperty(entry.getKey(), entry.getValue()); + } + operatorJson.add("specifiedInfo", specifiedInfo); + } + } + + List children = planNodeTree.getChildren(); + if (children != null && !children.isEmpty()) { + JsonArray childrenArray = new JsonArray(); + for (PlanNode child : children) { + JsonObject childJson = renderOperatorJson(child, operatorStatistics); + if (childJson != null) { + childrenArray.add(childJson); + } + } + // JsonArray.isEmpty() is not available in all Gson versions + if (childrenArray.size() > 0) { + operatorJson.add("children", childrenArray); + } + } + + return operatorJson; + } + + private static double formatMs(double ms) { + return Math.round(ms * 1000.0) / 1000.0; + } + + private static void addIfNonZero(JsonObject obj, String key, long value) { + if (value != 0) { + obj.addProperty(key, value); + } + } + + private static void addMsIfNonZero(JsonObject obj, String key, double value) { + if (Math.abs(value) > EPSILON) { + obj.addProperty(key, formatMs(value)); + } + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/PlanGraphJsonPrinterTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/PlanGraphJsonPrinterTest.java new file mode 100644 index 0000000000000..a600957b25fa3 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/PlanGraphJsonPrinterTest.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.planner.node; + +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanGraphJsonPrinter; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.LimitNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.OutputNode; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.GenericLiteral; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; + +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.Optional; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +public class PlanGraphJsonPrinterTest { + + @Test + public void testSimplePlanToJson() { + // Build a simple plan: Output -> Limit -> Filter + LimitNode placeholder = new LimitNode(new PlanNodeId("placeholder"), null, 0, Optional.empty()); + FilterNode filterNode = + new FilterNode( + new PlanNodeId("3"), + placeholder, + new ComparisonExpression( + ComparisonExpression.Operator.EQUAL, + new SymbolReference("s1"), + new GenericLiteral("INT32", "1"))); + + LimitNode limitNode = new LimitNode(new PlanNodeId("2"), filterNode, 10, Optional.empty()); + + List outputSymbols = Arrays.asList(new Symbol("s1"), new Symbol("s2")); + OutputNode outputNode = + new OutputNode(new PlanNodeId("1"), limitNode, Arrays.asList("s1", "s2"), outputSymbols); + + String json = PlanGraphJsonPrinter.toPrettyJson(outputNode); + + assertNotNull(json); + assertTrue(json.length() > 0); + + // Parse the JSON and verify structure + JsonObject root = JsonParser.parseString(json).getAsJsonObject(); + assertEquals("OutputNode-1", root.get("name").getAsString()); + assertEquals("1", root.get("id").getAsString()); + assertTrue(root.has("properties")); + assertTrue(root.has("children")); + + // Verify child (LimitNode) + JsonObject limitJson = root.getAsJsonArray("children").get(0).getAsJsonObject(); + assertEquals("LimitNode-2", limitJson.get("name").getAsString()); + + // Verify grandchild (FilterNode) + JsonObject filterJson = limitJson.getAsJsonArray("children").get(0).getAsJsonObject(); + assertEquals("FilterNode-3", filterJson.get("name").getAsString()); + } + + @Test + public void testGetJsonLinesReturnsSingleElement() { + LimitNode limitNode = + new LimitNode( + new PlanNodeId("1"), + new LimitNode(new PlanNodeId("placeholder"), null, 0, Optional.empty()), + 5, + Optional.empty()); + + List lines = PlanGraphJsonPrinter.getJsonLines(limitNode); + assertEquals(1, lines.size()); + + // The single element should be valid JSON + JsonObject root = JsonParser.parseString(lines.get(0)).getAsJsonObject(); + assertNotNull(root.get("name")); + } + + @Test + public void testJsonIsValidFormat() { + LimitNode limitNode = + new LimitNode( + new PlanNodeId("1"), + new LimitNode(new PlanNodeId("placeholder"), null, 0, Optional.empty()), + 42, + Optional.empty()); + + String json = PlanGraphJsonPrinter.toPrettyJson(limitNode); + + // Should parse without errors + JsonObject root = JsonParser.parseString(json).getAsJsonObject(); + assertEquals("LimitNode-1", root.get("name").getAsString()); + assertTrue(root.has("properties")); + assertEquals("42", root.getAsJsonObject("properties").get("Count").getAsString()); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ExplainFormatTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ExplainFormatTest.java new file mode 100644 index 0000000000000..56e204b38df9d --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ExplainFormatTest.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql; + +import org.apache.iotdb.db.protocol.session.IClientSession; +import org.apache.iotdb.db.protocol.session.InternalClientSession; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Explain; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExplainAnalyze; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExplainOutputFormat; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Statement; +import org.apache.iotdb.db.queryengine.plan.relational.sql.parser.SqlParser; + +import org.junit.Before; +import org.junit.Test; + +import java.time.ZoneId; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class ExplainFormatTest { + + private SqlParser sqlParser; + private IClientSession clientSession; + + @Before + public void setUp() { + sqlParser = new SqlParser(); + clientSession = new InternalClientSession("testClient"); + clientSession.setDatabaseName("testdb"); + } + + private Statement parseSQL(String sql) { + return sqlParser.createStatement(sql, ZoneId.systemDefault(), clientSession); + } + + @Test + public void testExplainDefaultFormat() { + Statement stmt = parseSQL("EXPLAIN SELECT * FROM table1"); + assertTrue(stmt instanceof Explain); + assertEquals(ExplainOutputFormat.GRAPHVIZ, ((Explain) stmt).getOutputFormat()); + } + + @Test + public void testExplainGraphvizFormat() { + Statement stmt = parseSQL("EXPLAIN (FORMAT GRAPHVIZ) SELECT * FROM table1"); + assertTrue(stmt instanceof Explain); + assertEquals(ExplainOutputFormat.GRAPHVIZ, ((Explain) stmt).getOutputFormat()); + } + + @Test + public void testExplainJsonFormat() { + Statement stmt = parseSQL("EXPLAIN (FORMAT JSON) SELECT * FROM table1"); + assertTrue(stmt instanceof Explain); + assertEquals(ExplainOutputFormat.JSON, ((Explain) stmt).getOutputFormat()); + } + + @Test + public void testExplainJsonFormatCaseInsensitive() { + Statement stmt = parseSQL("EXPLAIN (FORMAT json) SELECT * FROM table1"); + assertTrue(stmt instanceof Explain); + assertEquals(ExplainOutputFormat.JSON, ((Explain) stmt).getOutputFormat()); + } + + @Test(expected = Exception.class) + public void testExplainInvalidFormat() { + parseSQL("EXPLAIN (FORMAT XML) SELECT * FROM table1"); + } + + @Test + public void testExplainAnalyzeDefaultFormat() { + Statement stmt = parseSQL("EXPLAIN ANALYZE SELECT * FROM table1"); + assertTrue(stmt instanceof ExplainAnalyze); + assertEquals(ExplainOutputFormat.TEXT, ((ExplainAnalyze) stmt).getOutputFormat()); + } + + @Test + public void testExplainAnalyzeTextFormat() { + Statement stmt = parseSQL("EXPLAIN ANALYZE (FORMAT TEXT) SELECT * FROM table1"); + assertTrue(stmt instanceof ExplainAnalyze); + assertEquals(ExplainOutputFormat.TEXT, ((ExplainAnalyze) stmt).getOutputFormat()); + } + + @Test + public void testExplainAnalyzeJsonFormat() { + Statement stmt = parseSQL("EXPLAIN ANALYZE (FORMAT JSON) SELECT * FROM table1"); + assertTrue(stmt instanceof ExplainAnalyze); + assertEquals(ExplainOutputFormat.JSON, ((ExplainAnalyze) stmt).getOutputFormat()); + } + + @Test + public void testExplainAnalyzeVerboseJsonFormat() { + Statement stmt = parseSQL("EXPLAIN ANALYZE VERBOSE (FORMAT JSON) SELECT * FROM table1"); + assertTrue(stmt instanceof ExplainAnalyze); + ExplainAnalyze ea = (ExplainAnalyze) stmt; + assertEquals(ExplainOutputFormat.JSON, ea.getOutputFormat()); + assertTrue(ea.isVerbose()); + } + + @Test(expected = Exception.class) + public void testExplainAnalyzeInvalidFormat() { + parseSQL("EXPLAIN ANALYZE (FORMAT GRAPHVIZ) SELECT * FROM table1"); + } + + @Test(expected = Exception.class) + public void testExplainTextFormatInvalid() { + // TEXT is not valid for EXPLAIN (only GRAPHVIZ and JSON) + parseSQL("EXPLAIN (FORMAT TEXT) SELECT * FROM table1"); + } +} diff --git a/iotdb-core/relational-grammar/pom.xml b/iotdb-core/relational-grammar/pom.xml index f4fb18bee2c83..76acbf3355057 100644 --- a/iotdb-core/relational-grammar/pom.xml +++ b/iotdb-core/relational-grammar/pom.xml @@ -96,8 +96,12 @@ true - **/SqlLexer.java - **/SqlLexer.interp + **/RelationalSqlParser.java + **/RelationalSqlLexer.java + **/RelationalSqlLexer.interp + **/RelationalSql.interp + **/RelationalSqlBaseListener.java + **/RelationalSqlListener.java diff --git a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 index 9b09d6ebad043..c7f1cf4b9bcfc 100644 --- a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 +++ b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 @@ -932,8 +932,8 @@ copyToStatementOption // ------------------------------------------- Query Statement --------------------------------------------------------- queryStatement : query #statementDefault - | EXPLAIN (query | executeStatement | executeImmediateStatement) #explain - | EXPLAIN ANALYZE VERBOSE? (query | executeStatement | executeImmediateStatement) #explainAnalyze + | EXPLAIN ('(' FORMAT identifier ')')? (query | executeStatement | executeImmediateStatement) #explain + | EXPLAIN ANALYZE VERBOSE? ('(' FORMAT identifier ')')? (query | executeStatement | executeImmediateStatement) #explainAnalyze ; query