Branch 4.1

be/CMakeLists.txt

@@ -524,20 +524,18 @@ set(DORIS_LINK_LIBS
     ${WL_START_GROUP}
     Agent
     Common
+    Core
     Exec
     Exprs
+    Format
     IO
-    Olap
+    Storage
     Runtime
-    RuntimeFilter
     Service
     Util
     DorisGen
-    Webserver
-    Geo
-    GeoType
-    Vec
-    Pipeline
+    Load
+    InformationSchema
     Cloud
     ${WL_END_GROUP}
     CommonCPP
@@ -868,21 +866,23 @@ endfunction(pch_reuse target)
 
 add_subdirectory(${SRC_DIR}/agent)
 add_subdirectory(${SRC_DIR}/common)
+add_subdirectory(${SRC_DIR}/core)
 add_subdirectory(${SRC_DIR}/exec)
 add_subdirectory(${SRC_DIR}/exprs)
+add_subdirectory(${SRC_DIR}/format)
 add_subdirectory(${SRC_DIR}/gen_cpp)
-add_subdirectory(${SRC_DIR}/geo)
-add_subdirectory(${SRC_DIR}/http)
 add_subdirectory(${SRC_DIR}/io)
-add_subdirectory(${SRC_DIR}/olap/rowset/segment_v2/ann_index)
-add_subdirectory(${SRC_DIR}/olap)
+add_subdirectory(${SRC_DIR}/storage/index/ann)
+add_subdirectory(${SRC_DIR}/storage)
 add_subdirectory(${SRC_DIR}/runtime)
-add_subdirectory(${SRC_DIR}/runtime_filter)
 add_subdirectory(${SRC_DIR}/service) # this include doris_be
 add_subdirectory(${SRC_DIR}/cloud)
+add_subdirectory(${SRC_DIR}/load)
+add_subdirectory(${SRC_DIR}/information_schema)
 
 option(BUILD_META_TOOL "Build meta tool" OFF)
-if (BUILD_META_TOOL)
+option(BUILD_INDEX_TOOL "Build index tool" OFF)
+if (BUILD_META_TOOL OR BUILD_INDEX_TOOL)
     add_subdirectory(${SRC_DIR}/tools)
 endif()
 
@@ -897,14 +897,7 @@ if (BUILD_FILE_CACHE_MICROBENCH_TOOL)
         DESTINATION ${OUTPUT_DIR}/bin)
 endif()
 
-option(BUILD_INDEX_TOOL "Build index tool" OFF)
-if (BUILD_INDEX_TOOL)
-    add_subdirectory(${SRC_DIR}/index-tools)
-endif()
-
 add_subdirectory(${SRC_DIR}/util)
-add_subdirectory(${SRC_DIR}/vec)
-add_subdirectory(${SRC_DIR}/pipeline)
 
 # this include doris_be_test
 if (MAKE_TEST)

be/benchmark/benchmark_fastunion.hpp

@@ -23,7 +23,7 @@
 
 using Roaring64Map = doris::detail::Roaring64Map;
 
-namespace doris::vectorized {
+namespace doris {
 
 const uint32_t num_bitmaps = 100;
 const uint32_t num_outer_slots = 1000;
@@ -79,26 +79,26 @@ static void CustomCounters(benchmark::State& state) {
     state.counters["TotalValues"] = num_bitmaps * num_outer_slots * num_inner_values;
     state.counters["Overlap%"] = overlap_percent;
 }
-} // namespace doris::vectorized
+} // namespace doris
 
 static void BM_FastUnionOptimized(benchmark::State& state) {
     for (auto _ : state) {
-        auto result = Roaring64Map::fastunion(doris::vectorized::test_map_ptrs.size(),
-                                              doris::vectorized::test_map_ptrs.data());
+        auto result =
+                Roaring64Map::fastunion(doris::test_map_ptrs.size(), doris::test_map_ptrs.data());
         benchmark::DoNotOptimize(result);
     }
-    doris::vectorized::CustomCounters(state);
-    state.SetComplexityN(doris::vectorized::num_bitmaps);
+    doris::CustomCounters(state);
+    state.SetComplexityN(doris::num_bitmaps);
 }
 
 static void BM_FastUnionLegacy(benchmark::State& state) {
     for (auto _ : state) {
-        auto result = doris::vectorized::legacy_fastunion(doris::vectorized::test_map_ptrs.size(),
-                                                          doris::vectorized::test_map_ptrs.data());
+        auto result =
+                doris::legacy_fastunion(doris::test_map_ptrs.size(), doris::test_map_ptrs.data());
         benchmark::DoNotOptimize(result);
     }
-    doris::vectorized::CustomCounters(state);
-    state.SetComplexityN(doris::vectorized::num_bitmaps);
+    doris::CustomCounters(state);
+    state.SetComplexityN(doris::num_bitmaps);
 }
 
 BENCHMARK(BM_FastUnionOptimized)

be/benchmark/benchmark_main.cpp

@@ -26,7 +26,7 @@
 #include "vec/data_types/data_type.h"
 #include "vec/data_types/data_type_string.h"
 
-namespace doris::vectorized { // change if need
+namespace doris { // change if need
 
 static void Example1(benchmark::State& state) {
     // init. dont time it.
@@ -48,6 +48,6 @@ static void Example1(benchmark::State& state) {
 }
 // could BENCHMARK many functions to compare them together.
 BENCHMARK(Example1);
-} // namespace doris::vectorized
+} // namespace doris
 
 BENCHMARK_MAIN();

be/benchmark/benchmark_plain_text_line_reader.hpp

@@ -16,88 +16,98 @@
 // under the License.
 
 #include <benchmark/benchmark.h>
+
 #include <string>
 #include <vector>
 
 #include "vec/exec/format/file_reader/new_plain_text_line_reader.h"
 
 namespace doris {
 
-static std::string create_test_data(size_t length, const std::string& delimiter = "", char fill_char = 'a') {
+static std::string create_test_data(size_t length, const std::string& delimiter = "",
+                                    char fill_char = 'a') {
     return std::string(length, fill_char) + delimiter;
 }
 
 static void BM_FindLfCrlfLineSep(benchmark::State& state) {
     size_t data_size = state.range(0);
     size_t delimiter_type = state.range(1);
-    
+
     std::string test_data;
-    switch(delimiter_type) {
-        case 0: // No delimiter
-            test_data = create_test_data(data_size);
-            break;
-        case 1: // Delimiter is \n
-            test_data = create_test_data(data_size, "\n");
-            break;
-        case 2: // Delimiter is \r\n
-            test_data = create_test_data(data_size, "\r\n");
-            break;
-        default:
-            test_data = create_test_data(data_size);
-            break;
+    switch (delimiter_type) {
+    case 0: // No delimiter
+        test_data = create_test_data(data_size);
+        break;
+    case 1: // Delimiter is \n
+        test_data = create_test_data(data_size, "\n");
+        break;
+    case 2: // Delimiter is \r\n
+        test_data = create_test_data(data_size, "\r\n");
+        break;
+    default:
+        test_data = create_test_data(data_size);
+        break;
     }
 
     PlainTextLineReaderCtx ctx("\n", 1, false);
     const auto* data = reinterpret_cast<const uint8_t*>(test_data.c_str());
     const size_t size = test_data.size();
-    
+
     for (auto _ : state) {
         const auto* result = ctx.find_lf_crlf_line_sep(data, size);
         benchmark::DoNotOptimize(result);
     }
-    
+
     state.SetBytesProcessed(state.iterations() * test_data.size());
-    
+
     std::string label = "size_" + std::to_string(data_size);
     switch (delimiter_type) {
-        case 0: label += "_delim_no"; break;
-        case 1: label += "_delim_lf"; break;
-        case 2: label += "_delim_crlf"; break;
-        default: label += "_delim_no"; break;
+    case 0:
+        label += "_delim_no";
+        break;
+    case 1:
+        label += "_delim_lf";
+        break;
+    case 2:
+        label += "_delim_crlf";
+        break;
+    default:
+        label += "_delim_no";
+        break;
     }
     state.SetLabel(label);
 }
 
 BENCHMARK(BM_FindLfCrlfLineSep)
-    ->Unit(benchmark::kNanosecond)
-    ->Args({16, 0}) // 16 bytes, no delimiter
-    ->Args({16, 1}) // 16 bytes, delimiter is \n
-    ->Args({16, 2}) // 16 bytes, delimiter is \r\n
-    ->Args({32, 0}) // 32 bytes, no delimiter
-    ->Args({32, 1}) // 32 bytes, delimiter is \n
-    ->Args({32, 2}) // 32 bytes, delimiter is \r\n
-    ->Args({64, 0}) // 64 bytes, no delimiter
-    ->Args({64, 1}) // 64 bytes, delimiter is \n
-    ->Args({64, 2}) // 64 bytes, delimiter is \r\n
-    ->Args({128, 0}) // 128 bytes, no delimiter
-    ->Args({128, 1}) // 128 bytes, delimiter is \n
-    ->Args({128, 2}) // 128 bytes, delimiter is \r\n
-    ->Args({256, 0}) // 256 bytes, no delimiter
-    ->Args({256, 1}) // 256 bytes, delimiter is \n
-    ->Args({256, 2}) // 256 bytes, delimiter is \r\n
-    ->Args({512, 0}) // 512 bytes, no delimiter
-    ->Args({512, 1}) // 512 bytes, delimiter is \n
-    ->Args({512, 2}) // 512 bytes, delimiter is \r\n
-    ->Args({1024, 0}) // 1KB, no delimiter
-    ->Args({1024, 1}) // 1KB, delimiter is \n
-    ->Args({1024, 2}) // 1KB, delimiter is \r\n
-    ->Args({64 * 1024, 0}) // 64KB, no delimiter
-    ->Args({64 * 1024, 1}) // 64KB, delimiter is \n
-    ->Args({64 * 1024, 2}) // 64KB, delimiter is \r\n
-    ->Args({1024 * 1024, 0}) // 1MB, no delimiter
-    ->Args({1024 * 1024, 1}) // 1MB, delimiter is \n
-    ->Args({1024 * 1024, 2}) // 1MB, delimiter is \r\n
-    ->Repetitions(5)
-    ->DisplayAggregatesOnly();
+        ->Unit(benchmark::kNanosecond)
+        ->Args({16, 0})          // 16 bytes, no delimiter
+        ->Args({16, 1})          // 16 bytes, delimiter is \n
+        ->Args({16, 2})          // 16 bytes, delimiter is \r\n
+        ->Args({32, 0})          // 32 bytes, no delimiter
+        ->Args({32, 1})          // 32 bytes, delimiter is \n
+        ->Args({32, 2})          // 32 bytes, delimiter is \r\n
+        ->Args({64, 0})          // 64 bytes, no delimiter
+        ->Args({64, 1})          // 64 bytes, delimiter is \n
+        ->Args({64, 2})          // 64 bytes, delimiter is \r\n
+        ->Args({128, 0})         // 128 bytes, no delimiter
+        ->Args({128, 1})         // 128 bytes, delimiter is \n
+        ->Args({128, 2})         // 128 bytes, delimiter is \r\n
+        ->Args({256, 0})         // 256 bytes, no delimiter
+        ->Args({256, 1})         // 256 bytes, delimiter is \n
+        ->Args({256, 2})         // 256 bytes, delimiter is \r\n
+        ->Args({512, 0})         // 512 bytes, no delimiter
+        ->Args({512, 1})         // 512 bytes, delimiter is \n
+        ->Args({512, 2})         // 512 bytes, delimiter is \r\n
+        ->Args({1024, 0})        // 1KB, no delimiter
+        ->Args({1024, 1})        // 1KB, delimiter is \n
+        ->Args({1024, 2})        // 1KB, delimiter is \r\n
+        ->Args({64 * 1024, 0})   // 64KB, no delimiter
+        ->Args({64 * 1024, 1})   // 64KB, delimiter is \n
+        ->Args({64 * 1024, 2})   // 64KB, delimiter is \r\n
+        ->Args({1024 * 1024, 0}) // 1MB, no delimiter
+        ->Args({1024 * 1024, 1}) // 1MB, delimiter is \n
+        ->Args({1024 * 1024, 2}) // 1MB, delimiter is \r\n
+        ->Repetitions(5)
+        ->DisplayAggregatesOnly();
 
 } // namespace doris

be/src/agent/agent_server.cpp

@@ -37,11 +37,11 @@
 #include "common/config.h"
 #include "common/logging.h"
 #include "common/status.h"
-#include "olap/olap_define.h"
-#include "olap/options.h"
-#include "olap/snapshot_manager.h"
-#include "olap/storage_engine.h"
 #include "runtime/exec_env.h"
+#include "storage/olap_define.h"
+#include "storage/options.h"
+#include "storage/snapshot/snapshot_manager.h"
+#include "storage/storage_engine.h"
 #include "util/work_thread_pool.hpp"
 
 namespace doris {

be/src/agent/heartbeat_server.cpp

@@ -32,12 +32,12 @@
 #include "cloud/config.h"
 #include "common/config.h"
 #include "common/status.h"
-#include "olap/storage_engine.h"
 #include "runtime/cluster_info.h"
 #include "runtime/exec_env.h"
 #include "runtime/fragment_mgr.h"
 #include "runtime/heartbeat_flags.h"
 #include "service/backend_options.h"
+#include "storage/storage_engine.h"
 #include "util/debug_util.h"
 #include "util/mem_info.h"
 #include "util/network_util.h"

be/src/agent/task_worker_pool.cpp

@@ -55,6 +55,7 @@
 #include "cloud/config.h"
 #include "common/config.h"
 #include "common/logging.h"
+#include "common/metrics/doris_metrics.h"
 #include "common/status.h"
 #include "io/fs/file_system.h"
 #include "io/fs/hdfs_file_system.h"
@@ -63,35 +64,34 @@
 #include "io/fs/path.h"
 #include "io/fs/remote_file_system.h"
 #include "io/fs/s3_file_system.h"
-#include "olap/cumulative_compaction_time_series_policy.h"
-#include "olap/data_dir.h"
-#include "olap/olap_common.h"
-#include "olap/rowset/rowset_meta.h"
-#include "olap/snapshot_manager.h"
-#include "olap/storage_engine.h"
-#include "olap/storage_policy.h"
-#include "olap/tablet.h"
-#include "olap/tablet_manager.h"
-#include "olap/tablet_meta.h"
-#include "olap/tablet_schema.h"
-#include "olap/task/engine_batch_load_task.h"
-#include "olap/task/engine_checksum_task.h"
-#include "olap/task/engine_clone_task.h"
-#include "olap/task/engine_cloud_index_change_task.h"
-#include "olap/task/engine_index_change_task.h"
-#include "olap/task/engine_publish_version_task.h"
-#include "olap/task/engine_storage_migration_task.h"
-#include "olap/txn_manager.h"
-#include "olap/utils.h"
 #include "runtime/exec_env.h"
 #include "runtime/fragment_mgr.h"
 #include "runtime/index_policy/index_policy_mgr.h"
 #include "runtime/memory/global_memory_arbitrator.h"
 #include "runtime/snapshot_loader.h"
 #include "service/backend_options.h"
+#include "storage/compaction/cumulative_compaction_time_series_policy.h"
+#include "storage/data_dir.h"
+#include "storage/olap_common.h"
+#include "storage/rowset/rowset_meta.h"
+#include "storage/snapshot/snapshot_manager.h"
+#include "storage/storage_engine.h"
+#include "storage/storage_policy.h"
+#include "storage/tablet/tablet.h"
+#include "storage/tablet/tablet_manager.h"
+#include "storage/tablet/tablet_meta.h"
+#include "storage/tablet/tablet_schema.h"
+#include "storage/task/engine_batch_load_task.h"
+#include "storage/task/engine_checksum_task.h"
+#include "storage/task/engine_clone_task.h"
+#include "storage/task/engine_cloud_index_change_task.h"
+#include "storage/task/engine_index_change_task.h"
+#include "storage/task/engine_publish_version_task.h"
+#include "storage/task/engine_storage_migration_task.h"
+#include "storage/txn/txn_manager.h"
+#include "storage/utils.h"
 #include "util/brpc_client_cache.h"
 #include "util/debug_points.h"
-#include "util/doris_metrics.h"
 #include "util/jni-util.h"
 #include "util/mem_info.h"
 #include "util/random.h"

be/src/agent/utils.cpp

@@ -42,8 +42,8 @@
 
 #include "common/config.h"
 #include "common/status.h"
-#include "runtime/client_cache.h"
 #include "runtime/cluster_info.h"
+#include "util/client_cache.h"
 
 namespace doris {
 class TConfirmUnusedRemoteFilesRequest;

be/src/agent/utils.h

@@ -23,7 +23,7 @@
 #include <string>
 
 #include "common/status.h"
-#include "runtime/client_cache.h"
+#include "util/client_cache.h"
 
 namespace doris {
 class TConfirmUnusedRemoteFilesRequest;