Skip to content

Commit aa5fca6

Browse files
DolceTriadeDolceTriade
committed
autumnus -> lumis. update stuff. split off indexer-types
1 parent 9bd0dd8 commit aa5fca6

12 files changed

Lines changed: 834 additions & 940 deletions

File tree

‎Cargo.lock‎

Lines changed: 675 additions & 809 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 39 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
[workspace]
2-
members = ["indexer", "backend", "reposerver"]
2+
members = ["indexer", "backend", "reposerver", "indexer-types"]
33
resolver = "2"
44

55
[package]
@@ -11,14 +11,14 @@ edition = "2024"
1111
crate-type = ["cdylib", "rlib"]
1212

1313
[dependencies]
14-
leptos = { version = "0.8.0", features = ["nightly", "tracing"] }
15-
leptos_router = { version = "0.8.0", features = ["nightly"] }
16-
axum = { version = "0.8.0", optional = true }
14+
leptos = { version = "0.8.17", features = ["nightly", "tracing"] }
15+
leptos_router = { version = "0.8.13", features = ["nightly"] }
16+
axum = { version = "0.8.8", optional = true }
1717
console_error_panic_hook = { version = "0.1", optional = true }
18-
leptos_axum = { version = "0.8.0", optional = true }
19-
leptos_meta = { version = "0.8.0" }
18+
leptos_axum = { version = "0.8.8", optional = true }
19+
leptos_meta = { version = "0.8.6" }
2020
tokio = { version = "1", features = ["full"], optional = true }
21-
wasm-bindgen = { version = "=0.2.104", optional = true }
21+
wasm-bindgen = { version = "=0.2.108", optional = true }
2222
serde = { version = "1", features = ["derive"] }
2323
serde_json = "1"
2424
reqwest = { version = "0.12", default-features = false, features = [
@@ -35,7 +35,7 @@ colored = { version = "2.1", optional = true }
3535
console_log = { version = "1.0", optional = true }
3636
log = "0.4.28"
3737
tracing-web = { version = "0.1.3", optional = true }
38-
web-sys = { version = "=0.3.81", features = [
38+
web-sys = { version = "=0.3.85", features = [
3939
"Blob",
4040
"BlobPropertyBag",
4141
"CanvasRenderingContext2d",
@@ -74,12 +74,37 @@ sqlx = { version = "0.8", optional = true }
7474
async-trait = { version = "0.1" }
7575
base64 = { version = "0.22", optional = true }
7676
zstd = { version = "0.13", optional = true }
77-
pointer-indexer = { path = "./indexer", optional = true }
77+
pointer-indexer-types = { path = "./indexer-types", optional = true }
7878
anyhow = "1.0.100"
7979
server_fn_macro_default = { version = "0.8.5", optional = true }
8080
pulldown-cmark = { version = "0.11.0", optional = true, default-features = false }
8181
lazy_static = { version = "1.5.0", optional = true }
82-
autumnus = { version = "0.8.0-beta.3", optional = true, features = ["all-languages"] }
82+
lumis = { git = "https://github.com/leandrocp/lumis", branch = "main", optional = true, default-features = false, features = [
83+
"lang-bash",
84+
"lang-c",
85+
"lang-cpp",
86+
"lang-css",
87+
"lang-glsl",
88+
"lang-go",
89+
"lang-html",
90+
"lang-java",
91+
"lang-javascript",
92+
"lang-json",
93+
"lang-lua",
94+
"lang-markdown",
95+
"lang-nix",
96+
"lang-objc",
97+
"lang-php",
98+
"lang-protobuf",
99+
"lang-python",
100+
"lang-rust",
101+
"lang-sql",
102+
"lang-swift",
103+
"lang-toml",
104+
"lang-typescript",
105+
"lang-xml",
106+
"lang-yaml",
107+
] }
83108

84109
thiserror = { version = "1.0.63", optional = true }
85110
tower-http = { version = "0.6.6", optional = true, features = ["compression-gzip", "compression-zstd"] }
@@ -94,7 +119,6 @@ hydrate = [
94119
"dep:wasm-bindgen",
95120
"leptos/hydrate",
96121
]
97-
pointer-indexer = ["dep:pointer-indexer"]
98122
ssr = [
99123
"dep:axum",
100124
"dep:colored",
@@ -104,10 +128,10 @@ ssr = [
104128
"dep:tokio",
105129
"dep:base64",
106130
"dep:zstd",
107-
"dep:pointer-indexer",
131+
"dep:pointer-indexer-types",
108132
"dep:lazy_static",
109133
"pulldown-cmark/html",
110-
"autumnus",
134+
"lumis",
111135
"server_fn_macro_default/axum",
112136
"clap/derive",
113137
"clap/wrap_help",
@@ -133,8 +157,8 @@ pulldown-cmark = ["dep:pulldown-cmark"]
133157
thiserror = ["dep:thiserror"]
134158

135159
server_fn_macro_default = ["dep:server_fn_macro_default"]
136-
autumnus = ["dep:autumnus"]
137-
vendored = ["pointer-indexer/vendored"]
160+
pointer-indexer-types = ["dep:pointer-indexer-types"]
161+
lumis = ["dep:lumis"]
138162

139163
# Defines a size-optimized profile for the WASM bundle in release mode
140164
[profile.wasm-release]

‎backend/Cargo.toml‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,9 @@ tokio = { version = "1", features = ["macros", "rt-multi-thread", "signal"] }
1010
serde = { version = "1", features = ["derive"] }
1111
serde_json = "1"
1212
sqlx = { version = "0.8", features = ["runtime-tokio", "postgres", "macros", "chrono"] }
13-
pointer-indexer = { path = "../indexer" }
13+
pointer-indexer-types = { path = "../indexer-types" }
1414
zstd = "0.13"
15-
clap = { version = "4.5", features = ["derive"] }
15+
clap = { version = "4.5", features = ["derive", "env"] }
1616
tracing = "0.1"
1717
tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] }
1818
thiserror = "1"
@@ -26,4 +26,3 @@ tempfile = "3"
2626
chrono = { version = "0.4", features = ["serde"] }
2727

2828
[features]
29-
vendored = ["pointer-indexer/vendored"]

‎backend/src/main.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ use base64::Engine;
2121
use base64::engine::general_purpose::STANDARD as BASE64;
2222
use clap::Parser;
2323
use futures::{StreamExt, TryStreamExt, stream::FuturesUnordered};
24-
use pointer_indexer::models::{
24+
use pointer_indexer_types::{
2525
BranchHead, ChunkMapping, ContentBlob, FilePointer, ReferenceRecord, SymbolNamespaceRecord,
2626
SymbolRecord, UniqueChunk,
2727
};

‎flake.lock‎

Lines changed: 3 additions & 3 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎indexer-types/Cargo.toml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
[package]
2+
name = "pointer-indexer-types"
3+
version = "0.1.0"
4+
edition = "2024"
5+
6+
[dependencies]
7+
serde = { version = "1", features = ["derive"] }

‎indexer-types/src/lib.rs‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
use serde::{Deserialize, Serialize};
2+
3+
#[derive(Debug, Clone, Serialize, Deserialize)]
4+
pub struct ContentBlob {
5+
pub hash: String,
6+
pub language: Option<String>,
7+
pub byte_len: i64,
8+
pub line_count: i32,
9+
}
10+
11+
#[derive(Debug, Clone, Serialize, Deserialize)]
12+
pub struct SymbolRecord {
13+
pub content_hash: String,
14+
pub name: String,
15+
}
16+
17+
#[derive(Debug, Clone, Serialize, Deserialize)]
18+
pub struct ReferenceRecord {
19+
pub content_hash: String,
20+
pub namespace: Option<String>,
21+
pub name: String,
22+
pub fully_qualified: String,
23+
pub kind: Option<String>,
24+
pub line: usize,
25+
pub column: usize,
26+
}
27+
28+
#[derive(Debug, Clone, Serialize, Deserialize)]
29+
pub struct SymbolNamespaceRecord {
30+
pub namespace: String,
31+
}
32+
33+
#[derive(Debug, Clone, Serialize, Deserialize)]
34+
pub struct FilePointer {
35+
pub repository: String,
36+
pub commit_sha: String,
37+
pub file_path: String,
38+
pub content_hash: String,
39+
}
40+
41+
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
42+
pub struct IndexReport {
43+
pub content_blobs: Vec<ContentBlob>,
44+
pub symbol_records: Vec<SymbolRecord>,
45+
pub file_pointers: Vec<FilePointer>,
46+
pub reference_records: Vec<ReferenceRecord>,
47+
pub branches: Vec<BranchHead>,
48+
}
49+
50+
#[derive(Debug, Clone, Serialize, Deserialize)]
51+
pub struct BranchSnapshotPolicy {
52+
pub interval_seconds: u64,
53+
pub keep_count: u32,
54+
}
55+
56+
#[derive(Debug, Clone, Serialize, Deserialize)]
57+
pub struct BranchPolicy {
58+
pub latest_keep_count: u32,
59+
#[serde(skip_serializing_if = "Option::is_none")]
60+
pub is_live: Option<bool>,
61+
#[serde(default, skip_serializing_if = "Vec::is_empty")]
62+
pub snapshot_policies: Vec<BranchSnapshotPolicy>,
63+
}
64+
65+
#[derive(Debug, Clone, Serialize, Deserialize)]
66+
pub struct BranchHead {
67+
pub repository: String,
68+
pub branch: String,
69+
pub commit_sha: String,
70+
#[serde(skip_serializing_if = "Option::is_none")]
71+
pub policy: Option<BranchPolicy>,
72+
}
73+
74+
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)]
75+
pub struct UniqueChunk {
76+
pub chunk_hash: String,
77+
pub text_content: String,
78+
}
79+
80+
#[derive(Debug, Clone, Serialize, Deserialize)]
81+
pub struct ChunkMapping {
82+
pub content_hash: String,
83+
pub chunk_hash: String,
84+
pub chunk_index: usize,
85+
pub chunk_line_count: i32,
86+
}

‎indexer/Cargo.toml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,13 +12,14 @@ ignore = "0.4"
1212
serde = { version = "1", features = ["derive"] }
1313
serde_json = "1"
1414
sha2 = "0.10"
15+
pointer-indexer-types = { path = "../indexer-types" }
1516
tree-sitter = "0.25"
1617
tree-sitter-c = "0.24"
1718
tree-sitter-cpp = "0.23"
1819
tree-sitter-go = "0.25"
1920
tree-sitter-java = "0.23"
2021
tree-sitter-nix = "0.3"
21-
tree-sitter-proto = "0.3"
22+
tree-sitter-proto = "0.4"
2223
tree-sitter-python = "0.25"
2324
tree-sitter-rust = "0.24"
2425
tree-sitter-typescript = "0.23"

‎indexer/src/models.rs‎

Lines changed: 6 additions & 90 deletions
Original file line numberDiff line numberDiff line change
@@ -7,103 +7,19 @@ use std::sync::{Arc, Mutex};
77

88
use anyhow::{Context, Result, anyhow};
99
use serde::de::DeserializeOwned;
10-
use serde::{Deserialize, Serialize};
10+
use serde::Serialize;
1111
use tempfile::{Builder, NamedTempFile, TempPath};
1212

1313
use crate::chunk_store::ChunkStore;
1414

15+
pub use pointer_indexer_types::{
16+
BranchHead, BranchPolicy, BranchSnapshotPolicy, ChunkMapping, ContentBlob, FilePointer,
17+
IndexReport, ReferenceRecord, SymbolNamespaceRecord, SymbolRecord, UniqueChunk,
18+
};
19+
1520
const NEWLINE: &[u8] = b"\n";
1621
const BUFFER_FLUSH_BYTES: usize = 512 * 1024;
1722

18-
// Represents a file's metadata. Content is stored separately.
19-
#[derive(Debug, Clone, Serialize, Deserialize)]
20-
pub struct ContentBlob {
21-
pub hash: String,
22-
pub language: Option<String>,
23-
pub byte_len: i64,
24-
pub line_count: i32,
25-
}
26-
27-
#[derive(Debug, Clone, Serialize, Deserialize)]
28-
pub struct SymbolRecord {
29-
pub content_hash: String,
30-
pub name: String,
31-
}
32-
33-
#[derive(Debug, Clone, Serialize, Deserialize)]
34-
pub struct ReferenceRecord {
35-
pub content_hash: String,
36-
pub namespace: Option<String>,
37-
pub name: String,
38-
pub fully_qualified: String,
39-
pub kind: Option<String>,
40-
pub line: usize,
41-
pub column: usize,
42-
}
43-
44-
#[derive(Debug, Clone, Serialize, Deserialize)]
45-
pub struct SymbolNamespaceRecord {
46-
pub namespace: String,
47-
}
48-
49-
#[derive(Debug, Clone, Serialize, Deserialize)]
50-
pub struct FilePointer {
51-
pub repository: String,
52-
pub commit_sha: String,
53-
pub file_path: String,
54-
pub content_hash: String,
55-
}
56-
57-
// A report containing all the metadata extracted from a repository.
58-
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
59-
pub struct IndexReport {
60-
pub content_blobs: Vec<ContentBlob>,
61-
pub symbol_records: Vec<SymbolRecord>,
62-
pub file_pointers: Vec<FilePointer>,
63-
pub reference_records: Vec<ReferenceRecord>,
64-
pub branches: Vec<BranchHead>,
65-
}
66-
67-
#[derive(Debug, Clone, Serialize, Deserialize)]
68-
pub struct BranchSnapshotPolicy {
69-
pub interval_seconds: u64,
70-
pub keep_count: u32,
71-
}
72-
73-
#[derive(Debug, Clone, Serialize, Deserialize)]
74-
pub struct BranchPolicy {
75-
pub latest_keep_count: u32,
76-
#[serde(skip_serializing_if = "Option::is_none")]
77-
pub is_live: Option<bool>,
78-
#[serde(default, skip_serializing_if = "Vec::is_empty")]
79-
pub snapshot_policies: Vec<BranchSnapshotPolicy>,
80-
}
81-
82-
#[derive(Debug, Clone, Serialize, Deserialize)]
83-
pub struct BranchHead {
84-
pub repository: String,
85-
pub branch: String,
86-
pub commit_sha: String,
87-
#[serde(skip_serializing_if = "Option::is_none")]
88-
pub policy: Option<BranchPolicy>,
89-
}
90-
91-
// A unique, deduplicated chunk of text content.
92-
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)]
93-
pub struct UniqueChunk {
94-
pub chunk_hash: String,
95-
pub text_content: String,
96-
}
97-
98-
// Maps a file's content hash to a sequence of chunks.
99-
#[derive(Debug, Clone, Serialize, Deserialize)]
100-
pub struct ChunkMapping {
101-
pub content_hash: String,
102-
pub chunk_hash: String,
103-
pub chunk_index: usize,
104-
pub chunk_line_count: i32,
105-
}
106-
10723
#[derive(Debug)]
10824
struct WriterState {
10925
file: NamedTempFile,

‎nix/pointer.nix‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
rustPlatform,
77
rustc,
88
tailwindcss,
9-
wasm-bindgen-cli_0_2_104,
9+
wasm-bindgen-cli_0_2_108,
1010
openssl,
1111
pkg-config,
1212
autoPatchelfHook,
@@ -30,7 +30,7 @@ rustPlatform.buildRustPackage rec {
3030
enableParallelBuild = true;
3131

3232
nativeBuildInputs = [
33-
wasm-bindgen-cli_0_2_104
33+
wasm-bindgen-cli_0_2_108
3434
binaryen
3535
cargo-leptos
3636
rustc.llvmPackages.lld

0 commit comments

Comments
 (0)