C Scan API

Cargo.toml

@@ -263,6 +263,7 @@ vortex-decimal-byte-parts = { version = "0.1.0", path = "encodings/decimal-byte-
 vortex-error = { version = "0.1.0", path = "./vortex-error", default-features = false }
 vortex-fastlanes = { version = "0.1.0", path = "./encodings/fastlanes", default-features = false }
 vortex-file = { version = "0.1.0", path = "./vortex-file", default-features = false }
+vortex-ffi = { version = "0.1.0", path = "./vortex-ffi", default-features = false }
 vortex-flatbuffers = { version = "0.1.0", path = "./vortex-flatbuffers", default-features = false }
 vortex-fsst = { version = "0.1.0", path = "./encodings/fsst", default-features = false }
 vortex-io = { version = "0.1.0", path = "./vortex-io", default-features = false }

vortex-array/src/dtype/field_names.rs

@@ -341,6 +341,12 @@ impl From<Vec<&str>> for FieldNames {
     }
 }
 
+impl From<Vec<String>> for FieldNames {
+    fn from(value: Vec<String>) -> Self {
+        Self(value.into_iter().map(FieldName::from).collect())
+    }
+}
+
 impl From<&[FieldName]> for FieldNames {
     fn from(value: &[FieldName]) -> Self {
         Self(Arc::from(value))

vortex-cxx/cpp/include/vortex/scan.hpp

@@ -105,4 +105,4 @@ class ScanBuilder {
 
     rust::Box<ffi::VortexScanBuilder> impl_;
 };
-} // namespace vortex
+} // namespace vortex

vortex-duckdb/Cargo.toml

@@ -37,6 +37,7 @@ parking_lot = { workspace = true }
 paste = { workspace = true }
 tracing = { workspace = true }
 url = { workspace = true }
+vortex-ffi = { workspace = true }
 vortex = { workspace = true, features = ["files", "tokio", "object_store"] }
 vortex-utils = { workspace = true, features = ["dashmap"] }
 
@@ -55,6 +56,6 @@ workspace = true
 [build-dependencies]
 bindgen = { workspace = true }
 cbindgen = { workspace = true }
-cc = { workspace = true }
+cc = { workspace = true , features = ["parallel"] }
 reqwest = { workspace = true }
 zip = { workspace = true }

vortex-duckdb/build.rs

@@ -41,6 +41,8 @@ const SOURCE_FILES: [&str; 18] = [
     "cpp/vector_buffer.cpp",
 ];
 
+const FFI_INCLUDE: &str = "../vortex-ffi/cinclude";
+
 const DOWNLOAD_MAX_RETRIES: i32 = 3;
 const DOWNLOAD_TIMEOUT: u64 = 90;
 
@@ -302,6 +304,7 @@ fn c2rust(crate_dir: &Path, duckdb_include_dir: &Path) {
         .rustified_non_exhaustive_enum("DUCKDB_TYPE")
         .size_t_is_usize(true)
         .clang_arg(format!("-I{}", duckdb_include_dir.display()))
+        .clang_arg(format!("-I{}", crate_dir.join(FFI_INCLUDE).display()))
         .clang_arg(format!("-I{}", crate_dir.join("cpp/include").display()))
         .generate_comments(true)
         // Tell cargo to invalidate the built crate whenever any of the
@@ -323,13 +326,18 @@ fn c2rust(crate_dir: &Path, duckdb_include_dir: &Path) {
 }
 
 fn cpp(duckdb_include_dir: &Path) {
+    //println!("cargo:rustc-link-arg=-fsanitize=address");
     cc::Build::new()
         .std("c++20")
         // Duckdb sources fail -Wno-unused-parameter
         .flags(["-Wall", "-Wextra", "-Wpedantic", "-Wno-unused-parameter"])
+        // TODO
+        //.flag("-fsanitize=address")
         .cpp(true)
+        .debug(true)
         .include(duckdb_include_dir)
         .include("cpp/include")
+        .include(FFI_INCLUDE)
         .files(SOURCE_FILES)
         .compile("vortex-duckdb-extras");
     // bindgen generates rerun-if-changed for .h/.hpp files

vortex-duckdb/cpp/CMakeLists.txt

@@ -23,7 +23,7 @@ if (NOT CMAKE_BUILD_TYPE)
 endif()
 
 # Enable compiler warnings (matching build.rs flags).
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Wpedantic")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Wpedantic -Wno-unused-parameter")
 
 # Find DuckDB include directory via the symlink created by build.rs.
 # The symlink points to target/duckdb-source-vX.Y.Z which contains duckdb-X.Y.Z/
@@ -39,7 +39,7 @@ else()
     )
 endif()
 
-include_directories(include ${DUCKDB_INCLUDE})
+include_directories(include ${DUCKDB_INCLUDE} ../../vortex-ffi/cinclude)
 
 # Auto-discover C++ source files
 file(GLOB CPP_SOURCES "*.cpp")

vortex-duckdb/cpp/include/duckdb_vx/duckdb_diagnostics.h

@@ -18,9 +18,10 @@
     _Pragma("GCC diagnostic ignored \"-Wall\"")                                \
     _Pragma("GCC diagnostic ignored \"-Wextra\"")                              \
     _Pragma("GCC diagnostic ignored \"-Wpedantic\"")
+    _Pragma("GCC diagnostic ignored \"-Wunused-parameter\"")
 #define DUCKDB_INCLUDES_END _Pragma("GCC diagnostic pop")
 #else
 #define DUCKDB_INCLUDES_BEGIN
 #define DUCKDB_INCLUDES_END
 #endif
-// clang-format on
+// clang-format on

vortex-duckdb/cpp/include/duckdb_vx/table_function.h

@@ -12,7 +12,22 @@
 #include "error.h"
 #include "table_filter.h"
 #include "duckdb_vx/data.h"
-#include "duckdb_vx/client_context.h"
+#include "duckdb_vx/duckdb_diagnostics.h"
+
+#ifdef __cplusplus
+    DUCKDB_INCLUDES_BEGIN
+    #include "duckdb/common/arrow/arrow.hpp"
+    DUCKDB_INCLUDES_END
+
+    using FFI_ArrowSchema = ArrowSchema;
+    using FFI_ArrowArrayStream = ArrowArrayStream;
+#else
+    // TODO nanoarrow
+    typedef void FFI_ArrowSchema;
+    typedef void FFI_ArrowArrayStream;
+#endif
+
+#include "vortex.h"
 
 #ifdef __cplusplus /* If compiled as C++, use C ABI */
 extern "C" {
@@ -150,8 +165,6 @@ typedef struct {
     duckdb_vx_string_map (*to_string)(void *bind_data);
     // void *dynamic_to_string;
 
-    double (*table_scan_progress)(duckdb_client_context ctx, void *bind_data, void *global_state);
-
     idx_t (*get_partition_data)(const void *bind_data,
                                 void *init_global_data,
                                 void *init_local_data,
@@ -171,10 +184,18 @@ typedef struct {
     bool sampling_pushdown;
     bool late_materialization;
     idx_t max_threads;
+
+    // PoC hack: retain Rust exporter code
+    // return local batch id
+    uint64_t (*export_array)(const vx_array* arr, duckdb_data_chunk chunk);
+
 } duckdb_vx_tfunc_vtab_t;
 
 // A single function for configuring the DuckDB table function vtable.
-duckdb_state duckdb_vx_tfunc_register(duckdb_database ffi_db, const duckdb_vx_tfunc_vtab_t *vtab);
+duckdb_state duckdb_vx_tfunc_register(
+    duckdb_database ffi_db,
+    const duckdb_vx_tfunc_vtab_t *vtab
+);
 
 #ifdef __cplusplus /* End C ABI */
 }