From 9ee134a1fb83784154f47c62076c72f7ed7b356d Mon Sep 17 00:00:00 2001 From: Fricoben Date: Fri, 30 Jan 2026 16:35:52 +0000 Subject: [PATCH] feat: sync OpenClaw support and headless mode from tokscale MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add OpenClaw AI tool parser, headless scanning infrastructure, and scanner robustness improvements synced from upstream tokscale. - Add OpenClaw session parser (sessions.json index → JSONL files) - Add headless roots scanning (VIBETRACKING_HEADLESS_DIR env var) - Add usage*.csv pattern with archive/backup filtering - Add sessions.json exact match pattern for OpenClaw - Use .extend() for multi-path scanner aggregation - Add openclaw to all source lists, types, colors, and DB constraints - Add shared parsing utilities (utils.rs) - Add serial_test for environment-dependent tests Co-Authored-By: Claude Opus 4.5 --- packages/cli/src/graph-types.ts | 2 +- packages/cli/src/native.ts | 2 + packages/core/Cargo.lock | 89 ++++++ packages/core/Cargo.toml | 1 + packages/core/index.d.ts | 1 + packages/core/src/lib.rs | 71 ++++- packages/core/src/scanner.rs | 192 ++++++++++++- packages/core/src/sessions/mod.rs | 2 + packages/core/src/sessions/openclaw.rs | 256 ++++++++++++++++++ packages/core/src/sessions/utils.rs | 57 ++++ pnpm-lock.yaml | 94 +++---- src/app/api/import/route.ts | 1 + src/components/ui/charts/constants.ts | 2 + src/lib/graph-types.ts | 2 +- supabase/migrations/009_add_openclaw_tool.sql | 12 + 15 files changed, 717 insertions(+), 67 deletions(-) create mode 100644 packages/core/src/sessions/openclaw.rs create mode 100644 packages/core/src/sessions/utils.rs create mode 100644 supabase/migrations/009_add_openclaw_tool.sql diff --git a/packages/cli/src/graph-types.ts b/packages/cli/src/graph-types.ts index b9baefd..43d7850 100644 --- a/packages/cli/src/graph-types.ts +++ b/packages/cli/src/graph-types.ts @@ -6,7 +6,7 @@ /** * Valid source identifiers */ -export type SourceType = "opencode" | "claude" | "codex" | "gemini" | "cursor" | "amp" | "droid"; +export type SourceType = "opencode" | "claude" | "codex" | "gemini" | "cursor" | "amp" | "droid" | "openclaw"; /** * Token breakdown by category diff --git a/packages/cli/src/native.ts b/packages/cli/src/native.ts index d1d83a4..03d447b 100644 --- a/packages/cli/src/native.ts +++ b/packages/cli/src/native.ts @@ -154,6 +154,7 @@ interface NativeParsedMessages { geminiCount: number; ampCount: number; droidCount?: number; + openclawCount?: number; processingTimeMs: number; } @@ -362,6 +363,7 @@ export interface ParsedMessages { geminiCount: number; ampCount: number; droidCount: number; + openclawCount: number; processingTimeMs: number; } diff --git a/packages/core/Cargo.lock b/packages/core/Cargo.lock index 9da96b5..fda5565 100644 --- a/packages/core/Cargo.lock +++ b/packages/core/Cargo.lock @@ -886,6 +886,15 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.4.28" @@ -1073,6 +1082,29 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -1178,6 +1210,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + [[package]] name = "redox_users" version = "0.4.6" @@ -1323,6 +1364,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "scc" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46e6f046b7fef48e2660c57ed794263155d713de679057f2d0c169bfc6e756cc" +dependencies = [ + "sdd", +] + [[package]] name = "schannel" version = "0.1.28" @@ -1332,6 +1382,18 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sdd" +version = "3.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490dcfcbfef26be6800d11870ff2df8774fa6e86d047e3e8c8a76b25655e41ca" + [[package]] name = "security-framework" version = "2.11.1" @@ -1416,6 +1478,32 @@ dependencies = [ "serde", ] +[[package]] +name = "serial_test" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d0b343e184fc3b7bb44dff0705fffcf4b3756ba6aff420dddd8b24ca145e555" +dependencies = [ + "futures-executor", + "futures-util", + "log", + "once_cell", + "parking_lot", + "scc", + "serial_test_derive", +] + +[[package]] +name = "serial_test_derive" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f50427f258fb77356e4cd4aa0e87e2bd2c66dbcee41dc405282cae2bfc26c83" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "shlex" version = "1.3.0" @@ -1753,6 +1841,7 @@ dependencies = [ "reqwest", "serde", "serde_json", + "serial_test", "simd-json", "tempfile", "thiserror 2.0.17", diff --git a/packages/core/Cargo.toml b/packages/core/Cargo.toml index 8b1115a..fe0b3cb 100644 --- a/packages/core/Cargo.toml +++ b/packages/core/Cargo.toml @@ -55,6 +55,7 @@ napi-build = "2" [dev-dependencies] tempfile = "3" criterion = { version = "0.5", features = ["html_reports"] } +serial_test = "3" [[bench]] name = "json_parsing" diff --git a/packages/core/index.d.ts b/packages/core/index.d.ts index fbcd0c8..2abdb4d 100644 --- a/packages/core/index.d.ts +++ b/packages/core/index.d.ts @@ -189,6 +189,7 @@ export interface ParsedMessages { geminiCount: number ampCount: number droidCount: number + openclawCount: number processingTimeMs: number } diff --git a/packages/core/src/lib.rs b/packages/core/src/lib.rs index 9b7e0fc..edfd02a 100644 --- a/packages/core/src/lib.rs +++ b/packages/core/src/lib.rs @@ -72,6 +72,7 @@ pub struct ParsedMessages { pub gemini_count: i32, pub amp_count: i32, pub droid_count: i32, + pub openclaw_count: i32, pub processing_time_ms: u32, } @@ -182,6 +183,7 @@ pub struct GraphResult { use rayon::prelude::*; use sessions::UnifiedMessage; +use std::path::{Path, PathBuf}; use std::time::Instant; fn get_home_dir(home_dir_option: &Option) -> napi::Result { @@ -449,6 +451,29 @@ fn parse_all_messages_with_pricing( .collect(); all_messages.extend(droid_messages); + // Parse OpenClaw index files + let openclaw_messages: Vec = scan_result + .openclaw_files + .par_iter() + .flat_map(|path| { + sessions::openclaw::parse_openclaw_index(path) + .into_iter() + .map(|mut msg| { + msg.cost = pricing.calculate_cost( + &msg.model_id, + msg.tokens.input, + msg.tokens.output, + msg.tokens.cache_read, + msg.tokens.cache_write, + msg.tokens.reasoning, + ); + msg + }) + .collect::>() + }) + .collect(); + all_messages.extend(openclaw_messages); + all_messages } @@ -468,6 +493,7 @@ pub async fn get_model_report(options: ReportOptions) -> napi::Result napi::Result napi::Result "cursor".to_string(), "amp".to_string(), "droid".to_string(), + "openclaw".to_string(), ] }); @@ -692,6 +720,16 @@ fn filter_messages_for_report( filtered } +fn is_headless_path(path: &Path, headless_roots: &[PathBuf]) -> bool { + headless_roots.iter().any(|root| path.starts_with(root)) +} + +fn apply_headless_agent(message: &mut UnifiedMessage, is_headless: bool) { + if is_headless && message.agent.is_none() { + message.agent = Some("headless".to_string()); + } +} + // ============================================================================= // Two-Phase Processing Functions (for parallel execution optimization) // ============================================================================= @@ -713,6 +751,7 @@ pub fn parse_local_sources(options: LocalParseOptions) -> napi::Result napi::Result = sources.into_iter().filter(|s| s != "cursor").collect(); let scan_result = scanner::scan_all_sources(&home_dir, &local_sources); + let headless_roots = scanner::headless_roots(&home_dir); let mut messages: Vec = Vec::new(); @@ -760,14 +800,18 @@ pub fn parse_local_sources(options: LocalParseOptions) -> napi::Result = scan_result .codex_files .par_iter() .flat_map(|path| { + let is_headless = is_headless_path(path, &headless_roots); sessions::codex::parse_codex_file(path) .into_iter() - .map(|msg| unified_to_parsed(&msg)) + .map(|mut msg| { + apply_headless_agent(&mut msg, is_headless); + unified_to_parsed(&msg) + }) .collect::>() }) .collect(); @@ -816,6 +860,20 @@ pub fn parse_local_sources(options: LocalParseOptions) -> napi::Result = scan_result + .openclaw_files + .par_iter() + .flat_map(|path| { + sessions::openclaw::parse_openclaw_index(path) + .into_iter() + .map(|msg| unified_to_parsed(&msg)) + .collect::>() + }) + .collect(); + let openclaw_count = openclaw_msgs.len() as i32; + messages.extend(openclaw_msgs); + // Apply date filters let filtered = filter_parsed_messages(messages, &options); @@ -827,6 +885,7 @@ pub fn parse_local_sources(options: LocalParseOptions) -> napi::Result napi::Result = cursor_files .par_iter() @@ -1071,7 +1130,7 @@ pub async fn finalize_monthly_report(options: FinalizeMonthlyOptions) -> napi::R // Add Cursor messages if enabled if options.include_cursor { let cursor_cache_dir = format!("{}/.vibetracking/cursor-cache", home_dir); - let cursor_files = scanner::scan_directory(&cursor_cache_dir, "*.csv"); + let cursor_files = scanner::scan_directory(&cursor_cache_dir, "usage*.csv"); let cursor_messages: Vec = cursor_files .par_iter() @@ -1203,7 +1262,7 @@ pub async fn finalize_graph(options: FinalizeGraphOptions) -> napi::Result = cursor_files .par_iter() @@ -1297,7 +1356,7 @@ pub async fn finalize_report_and_graph(options: FinalizeReportOptions) -> napi:: // Add Cursor messages if enabled if options.include_cursor { let cursor_cache_dir = format!("{}/.vibetracking/cursor-cache", home_dir); - let cursor_files = scanner::scan_directory(&cursor_cache_dir, "*.csv"); + let cursor_files = scanner::scan_directory(&cursor_cache_dir, "usage*.csv"); let cursor_messages: Vec = cursor_files .par_iter() diff --git a/packages/core/src/scanner.rs b/packages/core/src/scanner.rs index 90c7d80..0738de4 100644 --- a/packages/core/src/scanner.rs +++ b/packages/core/src/scanner.rs @@ -16,6 +16,7 @@ pub enum SessionType { Cursor, Amp, Droid, + OpenClaw, } /// Result of scanning all session directories @@ -28,6 +29,7 @@ pub struct ScanResult { pub cursor_files: Vec, pub amp_files: Vec, pub droid_files: Vec, + pub openclaw_files: Vec, } impl ScanResult { @@ -40,6 +42,7 @@ impl ScanResult { + self.cursor_files.len() + self.amp_files.len() + self.droid_files.len() + + self.openclaw_files.len() } /// Get all files as a single vector @@ -67,11 +70,34 @@ impl ScanResult { for path in &self.droid_files { result.push((SessionType::Droid, path.clone())); } + for path in &self.openclaw_files { + result.push((SessionType::OpenClaw, path.clone())); + } result } } +pub fn headless_roots(home_dir: &str) -> Vec { + if let Ok(path) = std::env::var("VIBETRACKING_HEADLESS_DIR") { + return vec![PathBuf::from(path)]; + } + + let mut roots = Vec::new(); + roots.push(PathBuf::from(format!( + "{}/.config/vibetracking/headless", + home_dir + ))); + + let mac_root = PathBuf::from(format!( + "{}/Library/Application Support/vibetracking/headless", + home_dir + )); + roots.push(mac_root); + + roots +} + /// Scan a single directory for session files pub fn scan_directory(root: &str, pattern: &str) -> Vec { if !std::path::Path::new(root).exists() { @@ -90,17 +116,43 @@ pub fn scan_directory(root: &str, pattern: &str) -> Vec { let file_name = path.file_name().and_then(|n| n.to_str()).unwrap_or(""); + let is_in_archive_dir = path.components().any(|c| { + c.as_os_str() + .to_string_lossy() + .eq_ignore_ascii_case("archive") + }); + match pattern { "*.json" => file_name.ends_with(".json"), "*.jsonl" => file_name.ends_with(".jsonl"), "*.csv" => file_name.ends_with(".csv"), + "usage*.csv" => { + if is_in_archive_dir { + return false; + } + + if file_name == "usage.csv" { + return true; + } + + // Accept only per-account files: usage..csv + if !file_name.starts_with("usage.") || !file_name.ends_with(".csv") { + return false; + } + + // Exclude legacy backups like usage.backup-.csv + if file_name.starts_with("usage.backup") { + return false; + } + + true + } "session-*.json" => { file_name.starts_with("session-") && file_name.ends_with(".json") } - "T-*.json" => { - file_name.starts_with("T-") && file_name.ends_with(".json") - } + "T-*.json" => file_name.starts_with("T-") && file_name.ends_with(".json"), "*.settings.json" => file_name.ends_with(".settings.json"), + "sessions.json" => file_name == "sessions.json", _ => false, } }) @@ -120,6 +172,9 @@ pub fn scan_all_sources(home_dir: &str, sources: &[String]) -> ScanResult { let include_cursor = include_all || sources.iter().any(|s| s == "cursor"); let include_amp = include_all || sources.iter().any(|s| s == "amp"); let include_droid = include_all || sources.iter().any(|s| s == "droid"); + let include_openclaw = include_all || sources.iter().any(|s| s == "openclaw"); + + let headless_roots = headless_roots(home_dir); // Define scan tasks let mut tasks: Vec<(SessionType, String, &str)> = Vec::new(); @@ -144,6 +199,13 @@ pub fn scan_all_sources(home_dir: &str, sources: &[String]) -> ScanResult { std::env::var("CODEX_HOME").unwrap_or_else(|_| format!("{}/.codex", home_dir)); let codex_path = format!("{}/sessions", codex_home); tasks.push((SessionType::Codex, codex_path, "*.jsonl")); + + // Codex headless: /codex/*.jsonl + for root in &headless_roots { + let codex_headless_path = root.join("codex"); + let path = codex_headless_path.to_string_lossy().to_string(); + tasks.push((SessionType::Codex, path, "*.jsonl")); + } } if include_gemini { @@ -153,9 +215,9 @@ pub fn scan_all_sources(home_dir: &str, sources: &[String]) -> ScanResult { } if include_cursor { - // Cursor: ~/.vibetracking/cursor-cache/*.csv + // Cursor: ~/.vibetracking/cursor-cache/usage*.csv let cursor_path = format!("{}/.vibetracking/cursor-cache", home_dir); - tasks.push((SessionType::Cursor, cursor_path, "*.csv")); + tasks.push((SessionType::Cursor, cursor_path, "usage*.csv")); } if include_amp { @@ -172,6 +234,19 @@ pub fn scan_all_sources(home_dir: &str, sources: &[String]) -> ScanResult { tasks.push((SessionType::Droid, droid_path, "*.settings.json")); } + if include_openclaw { + // Current path + let openclaw_path = format!("{}/.openclaw/agents", home_dir); + tasks.push((SessionType::OpenClaw, openclaw_path, "sessions.json")); + // Legacy paths (Clawd -> Moltbot -> OpenClaw rebrand history) + let clawdbot_path = format!("{}/.clawdbot/agents", home_dir); + tasks.push((SessionType::OpenClaw, clawdbot_path, "sessions.json")); + let moltbot_path = format!("{}/.moltbot/agents", home_dir); + tasks.push((SessionType::OpenClaw, moltbot_path, "sessions.json")); + let moldbot_path = format!("{}/.moldbot/agents", home_dir); + tasks.push((SessionType::OpenClaw, moldbot_path, "sessions.json")); + } + // Execute scans in parallel let scan_results: Vec<(SessionType, Vec)> = tasks .into_par_iter() @@ -184,13 +259,14 @@ pub fn scan_all_sources(home_dir: &str, sources: &[String]) -> ScanResult { // Aggregate results for (session_type, files) in scan_results { match session_type { - SessionType::OpenCode => result.opencode_files = files, - SessionType::Claude => result.claude_files = files, - SessionType::Codex => result.codex_files = files, - SessionType::Gemini => result.gemini_files = files, - SessionType::Cursor => result.cursor_files = files, - SessionType::Amp => result.amp_files = files, - SessionType::Droid => result.droid_files = files, + SessionType::OpenCode => result.opencode_files.extend(files), + SessionType::Claude => result.claude_files.extend(files), + SessionType::Codex => result.codex_files.extend(files), + SessionType::Gemini => result.gemini_files.extend(files), + SessionType::Cursor => result.cursor_files.extend(files), + SessionType::Amp => result.amp_files.extend(files), + SessionType::Droid => result.droid_files.extend(files), + SessionType::OpenClaw => result.openclaw_files.extend(files), } } @@ -200,10 +276,18 @@ pub fn scan_all_sources(home_dir: &str, sources: &[String]) -> ScanResult { #[cfg(test)] mod tests { use super::*; + use serial_test::serial; use std::fs::{self, File}; use std::io::Write; use tempfile::TempDir; + fn restore_env(var: &str, previous: Option) { + match previous { + Some(value) => std::env::set_var(var, value), + None => std::env::remove_var(var), + } + } + #[test] fn test_scan_result_total_files() { let result = ScanResult { @@ -214,6 +298,7 @@ mod tests { cursor_files: vec![], amp_files: vec![], droid_files: vec![], + openclaw_files: vec![], }; assert_eq!(result.total_files(), 4); } @@ -228,6 +313,7 @@ mod tests { cursor_files: vec![PathBuf::from("e.csv")], amp_files: vec![], droid_files: vec![], + openclaw_files: vec![], }; let all = result.all_files(); @@ -377,7 +463,43 @@ mod tests { } #[test] + #[serial] + fn test_headless_roots_default() { + let previous = std::env::var("VIBETRACKING_HEADLESS_DIR").ok(); + std::env::remove_var("VIBETRACKING_HEADLESS_DIR"); + + let home = "/tmp/vibetracking-test-home"; + let roots = headless_roots(home); + let config_root = PathBuf::from(format!("{}/.config/vibetracking/headless", home)); + let mac_root = PathBuf::from(format!( + "{}/Library/Application Support/vibetracking/headless", + home + )); + + assert_eq!(roots.len(), 2); + assert!(roots.contains(&config_root)); + assert!(roots.contains(&mac_root)); + + restore_env("VIBETRACKING_HEADLESS_DIR", previous); + } + + #[test] + #[serial] + fn test_headless_roots_override() { + let previous = std::env::var("VIBETRACKING_HEADLESS_DIR").ok(); + std::env::set_var("VIBETRACKING_HEADLESS_DIR", "/custom/headless"); + + let roots = headless_roots("/tmp/home"); + assert_eq!(roots, vec![PathBuf::from("/custom/headless")]); + + restore_env("VIBETRACKING_HEADLESS_DIR", previous); + } + + #[test] + #[serial] fn test_scan_all_sources_opencode() { + let previous_xdg = std::env::var("XDG_DATA_HOME").ok(); + let dir = TempDir::new().unwrap(); let home = dir.path(); setup_mock_opencode_dir(home); @@ -390,6 +512,8 @@ mod tests { assert!(result.claude_files.is_empty()); assert!(result.codex_files.is_empty()); assert!(result.gemini_files.is_empty()); + + restore_env("XDG_DATA_HOME", previous_xdg); } #[test] @@ -434,7 +558,49 @@ mod tests { } #[test] + #[serial] + fn test_scan_all_sources_headless_paths() { + let previous_headless = std::env::var("VIBETRACKING_HEADLESS_DIR").ok(); + let previous_codex = std::env::var("CODEX_HOME").ok(); + std::env::remove_var("VIBETRACKING_HEADLESS_DIR"); + + let dir = TempDir::new().unwrap(); + let home = dir.path(); + + // Isolate CODEX_HOME so we don't scan the developer's real ~/.codex + std::env::set_var("CODEX_HOME", home.join(".codex-nonexistent")); + + let mac_root = home + .join("Library") + .join("Application Support") + .join("vibetracking") + .join("headless"); + + fs::create_dir_all(mac_root.join("codex")).unwrap(); + File::create(mac_root.join("codex").join("codex.jsonl")).unwrap(); + + let result = scan_all_sources( + home.to_str().unwrap(), + &[ + "claude".to_string(), + "codex".to_string(), + "gemini".to_string(), + ], + ); + + assert!(result.claude_files.is_empty()); + assert_eq!(result.codex_files.len(), 1); + assert!(result.gemini_files.is_empty()); + + restore_env("VIBETRACKING_HEADLESS_DIR", previous_headless); + restore_env("CODEX_HOME", previous_codex); + } + + #[test] + #[serial] fn test_scan_all_sources_codex_with_env() { + let previous_codex = std::env::var("CODEX_HOME").ok(); + let dir = TempDir::new().unwrap(); let home = dir.path(); setup_mock_codex_dir(home); @@ -444,5 +610,7 @@ mod tests { let result = scan_all_sources(home.to_str().unwrap(), &["codex".to_string()]); assert_eq!(result.codex_files.len(), 1); + + restore_env("CODEX_HOME", previous_codex); } } diff --git a/packages/core/src/sessions/mod.rs b/packages/core/src/sessions/mod.rs index fe6e6e8..4cce648 100644 --- a/packages/core/src/sessions/mod.rs +++ b/packages/core/src/sessions/mod.rs @@ -8,7 +8,9 @@ pub mod codex; pub mod cursor; pub mod droid; pub mod gemini; +pub mod openclaw; pub mod opencode; +pub(crate) mod utils; use crate::TokenBreakdown; diff --git a/packages/core/src/sessions/openclaw.rs b/packages/core/src/sessions/openclaw.rs new file mode 100644 index 0000000..3069d7e --- /dev/null +++ b/packages/core/src/sessions/openclaw.rs @@ -0,0 +1,256 @@ +//! OpenClaw session parser +//! +//! Parses JSONL files from ~/.openclaw/ or ~/.clawdbot/ +//! Uses sessions.json index to find actual session file paths + +use super::UnifiedMessage; +use crate::TokenBreakdown; +use serde::Deserialize; +use std::collections::HashMap; +use std::io::{BufRead, BufReader}; +use std::path::Path; + +#[derive(Debug, Deserialize)] +struct SessionIndex { + #[serde(flatten)] + sessions: HashMap, +} + +#[derive(Debug, Deserialize)] +struct SessionEntry { + #[serde(rename = "sessionId")] + session_id: String, + #[serde(rename = "sessionFile")] + session_file: Option, +} + +#[derive(Debug, Deserialize)] +struct OpenClawEntry { + #[serde(rename = "type")] + entry_type: String, + message: Option, + #[serde(rename = "modelId")] + model_id: Option, + provider: Option, +} + +#[derive(Debug, Deserialize)] +struct OpenClawMessage { + role: Option, + usage: Option, + timestamp: Option, +} + +#[derive(Debug, Deserialize)] +struct OpenClawUsage { + input: Option, + output: Option, + #[serde(rename = "cacheRead")] + cache_read: Option, + #[serde(rename = "cacheWrite")] + cache_write: Option, + #[serde(rename = "totalTokens")] + total_tokens: Option, + cost: Option, +} + +#[derive(Debug, Deserialize)] +struct OpenClawCost { + total: Option, +} + +pub fn parse_openclaw_index(index_path: &Path) -> Vec { + let data = match std::fs::read(index_path) { + Ok(d) => d, + Err(_) => return Vec::new(), + }; + + let mut bytes = data; + let index: SessionIndex = match simd_json::from_slice(&mut bytes) { + Ok(i) => i, + Err(_) => return Vec::new(), + }; + + let mut all_messages = Vec::new(); + + for (_key, entry) in index.sessions { + if let Some(session_file) = entry.session_file { + let session_path = Path::new(&session_file); + if session_path.exists() { + let messages = parse_openclaw_session(session_path, &entry.session_id); + all_messages.extend(messages); + } + } + } + + all_messages +} + +fn parse_openclaw_session(session_path: &Path, session_id: &str) -> Vec { + let file = match std::fs::File::open(session_path) { + Ok(f) => f, + Err(_) => return Vec::new(), + }; + + let reader = BufReader::new(file); + let mut messages = Vec::new(); + let mut current_model: Option = None; + let mut current_provider: Option = None; + + for line in reader.lines() { + let line = match line { + Ok(l) => l, + Err(_) => continue, + }; + + let trimmed = line.trim(); + if trimmed.is_empty() { + continue; + } + + let mut bytes = trimmed.as_bytes().to_vec(); + let entry: OpenClawEntry = match simd_json::from_slice(&mut bytes) { + Ok(e) => e, + Err(_) => continue, + }; + + match entry.entry_type.as_str() { + "model_change" => { + if let Some(model) = entry.model_id { + current_model = Some(model); + } + if let Some(provider) = entry.provider { + current_provider = Some(provider); + } + } + "message" => { + if let Some(msg) = entry.message { + if msg.role.as_deref() != Some("assistant") { + continue; + } + + let usage = match msg.usage { + Some(u) => u, + None => continue, + }; + + let model = match ¤t_model { + Some(m) => m.clone(), + None => continue, + }; + + let provider = current_provider + .clone() + .unwrap_or_else(|| "unknown".to_string()); + let timestamp = msg.timestamp.unwrap_or(0); + let cost = usage.cost.and_then(|c| c.total).unwrap_or(0.0); + + messages.push(UnifiedMessage::new( + "openclaw", + model, + provider, + session_id.to_string(), + timestamp, + TokenBreakdown { + input: usage.input.unwrap_or(0), + output: usage.output.unwrap_or(0), + cache_read: usage.cache_read.unwrap_or(0), + cache_write: usage.cache_write.unwrap_or(0), + reasoning: 0, + }, + cost, + )); + } + } + _ => {} + } + } + + messages +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs::File; + use std::io::Write; + use tempfile::TempDir; + + fn create_test_session(dir: &TempDir, filename: &str, content: &str) -> String { + let path = dir.path().join(filename); + let mut file = File::create(&path).unwrap(); + file.write_all(content.as_bytes()).unwrap(); + path.to_string_lossy().to_string() + } + + #[test] + fn test_parse_openclaw_session_with_model_change() { + let dir = TempDir::new().unwrap(); + let content = r#"{"type":"model_change","id":"abc","provider":"openai-codex","modelId":"gpt-5.2"} +{"type":"message","id":"msg1","message":{"role":"assistant","content":[],"usage":{"input":100,"output":50,"cacheRead":200,"totalTokens":350,"cost":{"total":0.05}},"timestamp":1700000000000}}"#; + + let session_path = create_test_session(&dir, "session.jsonl", content); + let messages = parse_openclaw_session(Path::new(&session_path), "test-session"); + + assert_eq!(messages.len(), 1); + assert_eq!(messages[0].model_id, "gpt-5.2"); + assert_eq!(messages[0].provider_id, "openai-codex"); + assert_eq!(messages[0].tokens.input, 100); + assert_eq!(messages[0].tokens.output, 50); + assert_eq!(messages[0].tokens.cache_read, 200); + assert_eq!(messages[0].cost, 0.05); + } + + #[test] + fn test_parse_openclaw_session_user_messages_ignored() { + let dir = TempDir::new().unwrap(); + let content = r#"{"type":"model_change","provider":"anthropic","modelId":"claude-3.5-sonnet"} +{"type":"message","id":"msg1","message":{"role":"user","content":[{"type":"text","text":"hello"}]}} +{"type":"message","id":"msg2","message":{"role":"assistant","content":[],"usage":{"input":50,"output":25},"timestamp":1700000000000}}"#; + + let session_path = create_test_session(&dir, "session.jsonl", content); + let messages = parse_openclaw_session(Path::new(&session_path), "test-session"); + + assert_eq!(messages.len(), 1); + assert_eq!(messages[0].tokens.input, 50); + } + + #[test] + fn test_parse_openclaw_session_no_model_change() { + let dir = TempDir::new().unwrap(); + let content = r#"{"type":"message","id":"msg1","message":{"role":"assistant","content":[],"usage":{"input":100,"output":50},"timestamp":1700000000000}}"#; + + let session_path = create_test_session(&dir, "session.jsonl", content); + let messages = parse_openclaw_session(Path::new(&session_path), "test-session"); + + assert_eq!(messages.len(), 0); + } + + #[test] + fn test_parse_openclaw_index() { + let dir = TempDir::new().unwrap(); + + let session_content = r#"{"type":"model_change","provider":"anthropic","modelId":"claude-3.5-sonnet"} +{"type":"message","id":"msg1","message":{"role":"assistant","content":[],"usage":{"input":100,"output":50,"cacheRead":0,"cacheWrite":0},"timestamp":1700000000000}}"#; + let session_path = create_test_session(&dir, "session-abc.jsonl", session_content); + + let index_content = format!( + r#"{{ + "agent:main:main": {{ + "sessionId": "abc-123", + "sessionFile": "{}" + }} + }}"#, + session_path.replace('\\', "\\\\") + ); + + let index_path = dir.path().join("sessions.json"); + let mut file = File::create(&index_path).unwrap(); + file.write_all(index_content.as_bytes()).unwrap(); + + let messages = parse_openclaw_index(&index_path); + assert_eq!(messages.len(), 1); + assert_eq!(messages[0].model_id, "claude-3.5-sonnet"); + assert_eq!(messages[0].session_id, "abc-123"); + } +} diff --git a/packages/core/src/sessions/utils.rs b/packages/core/src/sessions/utils.rs new file mode 100644 index 0000000..36e6718 --- /dev/null +++ b/packages/core/src/sessions/utils.rs @@ -0,0 +1,57 @@ +//! Shared parsing helpers for session logs. + +use serde_json::Value; +use std::path::Path; +use std::time::SystemTime; + +pub(crate) fn extract_i64(value: Option<&Value>) -> Option { + value.and_then(|val| { + val.as_i64() + .or_else(|| val.as_u64().map(|v| v as i64)) + .or_else(|| val.as_str().and_then(|s| s.parse::().ok())) + }) +} + +pub(crate) fn extract_string(value: Option<&Value>) -> Option { + value.and_then(|val| val.as_str().map(|s| s.to_string())) +} + +pub(crate) fn parse_timestamp_value(value: &Value) -> Option { + if let Some(ts) = value.as_str() { + return parse_timestamp_str(ts); + } + + let numeric = value + .as_i64() + .or_else(|| value.as_u64().map(|v| v as i64))?; + // Heuristic: values >= 1e12 are treated as milliseconds, smaller values as seconds. + if numeric >= 1_000_000_000_000 { + Some(numeric) + } else { + Some(numeric * 1000) + } +} + +pub(crate) fn parse_timestamp_str(value: &str) -> Option { + if let Ok(dt) = chrono::DateTime::parse_from_rfc3339(value) { + return Some(dt.timestamp_millis()); + } + + if let Ok(numeric) = value.parse::() { + if numeric >= 1_000_000_000_000 { + return Some(numeric); + } + return Some(numeric * 1000); + } + + None +} + +pub(crate) fn file_modified_timestamp_ms(path: &Path) -> i64 { + std::fs::metadata(path) + .and_then(|meta| meta.modified()) + .ok() + .and_then(|time| time.duration_since(SystemTime::UNIX_EPOCH).ok()) + .map(|duration| duration.as_millis() as i64) + .unwrap_or_else(|| chrono::Utc::now().timestamp_millis()) +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1fb6ed0..6a2d62b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -64,8 +64,8 @@ importers: packages/cli: dependencies: '@starknetid/vibetracking-core': - specifier: 0.2.4 - version: 0.2.4 + specifier: 0.2.5 + version: 0.2.5 clipboardy: specifier: ^5.0.2 version: 5.0.2 @@ -108,26 +108,26 @@ importers: version: 6.4.1 optionalDependencies: '@starknetid/vibetracking-core-darwin-arm64': - specifier: 0.2.4 - version: 0.2.4 + specifier: 0.2.5 + version: 0.2.5 '@starknetid/vibetracking-core-darwin-universal': - specifier: 0.2.4 - version: 0.2.4 + specifier: 0.2.5 + version: 0.2.5 '@starknetid/vibetracking-core-darwin-x64': - specifier: 0.2.4 - version: 0.2.4 + specifier: 0.2.5 + version: 0.2.5 '@starknetid/vibetracking-core-linux-arm64-gnu': - specifier: 0.2.4 - version: 0.2.4 + specifier: 0.2.5 + version: 0.2.5 '@starknetid/vibetracking-core-linux-x64-gnu': - specifier: 0.2.4 - version: 0.2.4 + specifier: 0.2.5 + version: 0.2.5 '@starknetid/vibetracking-core-win32-arm64-msvc': - specifier: 0.2.4 - version: 0.2.4 + specifier: 0.2.5 + version: 0.2.5 '@starknetid/vibetracking-core-win32-x64-msvc': - specifier: 0.2.4 - version: 0.2.4 + specifier: 0.2.5 + version: 0.2.5 packages: @@ -1067,49 +1067,49 @@ packages: '@standard-schema/utils@0.3.0': resolution: {integrity: sha512-e7Mew686owMaPJVNNLs55PUvgz371nKgwsc4vxE49zsODpJEnxgxRo2y/OKrqueavXgZNMDVj3DdHFlaSAeU8g==} - '@starknetid/vibetracking-core-darwin-arm64@0.2.4': - resolution: {integrity: sha512-99QXjIqXJGm0fymWAwVw6Q0k7/wk8RNAhDOZ2L/yCr+ZZwj4Di55DxrVnW9h1FHefG5BP00mSB6MMZKGHCifJA==} + '@starknetid/vibetracking-core-darwin-arm64@0.2.5': + resolution: {integrity: sha512-1NY/uL+4asVceVO1OkcvzkVK9fsvEa1QeaXwSju/42hJOmIDR7FDUxG7BkdvWR009HU4roNgBDGZCQwm7baBng==} engines: {node: '>= 16'} cpu: [arm64] os: [darwin] - '@starknetid/vibetracking-core-darwin-universal@0.2.4': - resolution: {integrity: sha512-daG3aTxefLqYNEWI9Nxo/tTJH6yo0vF7gn+kyIKshBUw09ge+HPzOsYMkzG9gwpqbtUZNjmKlwyYkHeiKJqaEw==} + '@starknetid/vibetracking-core-darwin-universal@0.2.5': + resolution: {integrity: sha512-LUBqC4os8jQehJPFieP6khDt13+1WiBjg619dc5/W7NI4tIayaKwuOoqENzCOnajZhIleuPqMvNAEcJndLOyGQ==} engines: {node: '>= 16'} os: [darwin] - '@starknetid/vibetracking-core-darwin-x64@0.2.4': - resolution: {integrity: sha512-U/YYOXiMIICShSnbJR6tXAWfY3MRm6fJI/QkDnw8aIU+BvTzjPqR5UwJLGTYXea1mXGzhuyhJd48Ima4BGuc8A==} + '@starknetid/vibetracking-core-darwin-x64@0.2.5': + resolution: {integrity: sha512-raCq1CfjjfpQywmpw2t8VHz1LhbdHFwQ78B30XytAsDe8gkeRWewEwpjAswa+MST8LwYsWadQf3M6gaTiWGy+Q==} engines: {node: '>= 16'} cpu: [x64] os: [darwin] - '@starknetid/vibetracking-core-linux-arm64-gnu@0.2.4': - resolution: {integrity: sha512-kaSOVx/OifGbEYEnmk9TuQ6Fu2kE+kRaD/g4k9TawCq3eHzORFyTEddToSkkrP8pZ3JVaG8J4iQfiHOvpv9xdw==} + '@starknetid/vibetracking-core-linux-arm64-gnu@0.2.5': + resolution: {integrity: sha512-nTIpP0tDM9Z9WoanMKkOshS0E30RizFd1qQTjriuFldJ09XJtB5hlZzFYewS4YKrz3bJSILRXo6I+yQyiz/IFg==} engines: {node: '>= 16'} cpu: [arm64] os: [linux] - '@starknetid/vibetracking-core-linux-x64-gnu@0.2.4': - resolution: {integrity: sha512-CkjmUmSbdptrkEEFKfVjbZct3dzO2RUGLmYPkPHBuEXAxEyLmMc7GaB162mk1aMEx2UUnGdMTTDrykHKjdg6VA==} + '@starknetid/vibetracking-core-linux-x64-gnu@0.2.5': + resolution: {integrity: sha512-X7v8G1piY6YROjia4Iqtytx3D9h+jAAEwpT7uUE/H0kGuqBhEPoD1UgJe7MG5NAxh8iyBj6KXGoTz00CRs+waw==} engines: {node: '>= 16'} cpu: [x64] os: [linux] - '@starknetid/vibetracking-core-win32-arm64-msvc@0.2.4': - resolution: {integrity: sha512-sa55CpgS7D0L27IRFZtFXIgij5vP1m6r68N5v+8yiUkvsaFK5eENUa3/ihEDoIsWDguleNffdaOR7SGIcvmI1g==} + '@starknetid/vibetracking-core-win32-arm64-msvc@0.2.5': + resolution: {integrity: sha512-+H/hCvFRaRetqO3qTVdF9RLxe+3iIIpt0BSXGbqX5w1O1AYpQvAX+Z3L6tVgoxJYjastati5Ocs3trvjee7C7A==} engines: {node: '>= 16'} cpu: [arm64] os: [win32] - '@starknetid/vibetracking-core-win32-x64-msvc@0.2.4': - resolution: {integrity: sha512-CUaL309bjcT2jABGU6U6WnnTU8DFytaBLNGfcbxQUjLRfuyuJbUcb7Ggyft6l60CkfETh/D1KayhD+D4EIozAw==} + '@starknetid/vibetracking-core-win32-x64-msvc@0.2.5': + resolution: {integrity: sha512-aj1zPhMGtLE1RjitdBDdq6Wj1voG1WIm5efruuXgyCwvDsMRPAD4EqkNPkNnXlrHh2/K8z4xHEHBi0VznhKFQw==} engines: {node: '>= 16'} cpu: [x64] os: [win32] - '@starknetid/vibetracking-core@0.2.4': - resolution: {integrity: sha512-vhkLmTSGyv6QcJGrInNdYx6xFF6WNWktMZSGx5jwBj1oJ2ZzvBrHjw+zck4bbc8+Oc9jPaoIL3g6zmLwABTLfQ==} + '@starknetid/vibetracking-core@0.2.5': + resolution: {integrity: sha512-v0Y9pQ2DgrBHGoup6jkizvTtOgGHqeNWN7kosbVBYmltlsrxEKlCKEIJ0S12KEhBzmCx1OXZ0YPeL8V34hfonA==} engines: {node: '>= 16'} '@supabase/auth-js@2.89.0': @@ -4255,36 +4255,36 @@ snapshots: '@standard-schema/utils@0.3.0': {} - '@starknetid/vibetracking-core-darwin-arm64@0.2.4': + '@starknetid/vibetracking-core-darwin-arm64@0.2.5': optional: true - '@starknetid/vibetracking-core-darwin-universal@0.2.4': + '@starknetid/vibetracking-core-darwin-universal@0.2.5': optional: true - '@starknetid/vibetracking-core-darwin-x64@0.2.4': + '@starknetid/vibetracking-core-darwin-x64@0.2.5': optional: true - '@starknetid/vibetracking-core-linux-arm64-gnu@0.2.4': + '@starknetid/vibetracking-core-linux-arm64-gnu@0.2.5': optional: true - '@starknetid/vibetracking-core-linux-x64-gnu@0.2.4': + '@starknetid/vibetracking-core-linux-x64-gnu@0.2.5': optional: true - '@starknetid/vibetracking-core-win32-arm64-msvc@0.2.4': + '@starknetid/vibetracking-core-win32-arm64-msvc@0.2.5': optional: true - '@starknetid/vibetracking-core-win32-x64-msvc@0.2.4': + '@starknetid/vibetracking-core-win32-x64-msvc@0.2.5': optional: true - '@starknetid/vibetracking-core@0.2.4': + '@starknetid/vibetracking-core@0.2.5': optionalDependencies: - '@starknetid/vibetracking-core-darwin-arm64': 0.2.4 - '@starknetid/vibetracking-core-darwin-universal': 0.2.4 - '@starknetid/vibetracking-core-darwin-x64': 0.2.4 - '@starknetid/vibetracking-core-linux-arm64-gnu': 0.2.4 - '@starknetid/vibetracking-core-linux-x64-gnu': 0.2.4 - '@starknetid/vibetracking-core-win32-arm64-msvc': 0.2.4 - '@starknetid/vibetracking-core-win32-x64-msvc': 0.2.4 + '@starknetid/vibetracking-core-darwin-arm64': 0.2.5 + '@starknetid/vibetracking-core-darwin-universal': 0.2.5 + '@starknetid/vibetracking-core-darwin-x64': 0.2.5 + '@starknetid/vibetracking-core-linux-arm64-gnu': 0.2.5 + '@starknetid/vibetracking-core-linux-x64-gnu': 0.2.5 + '@starknetid/vibetracking-core-win32-arm64-msvc': 0.2.5 + '@starknetid/vibetracking-core-win32-x64-msvc': 0.2.5 '@supabase/auth-js@2.89.0': dependencies: diff --git a/src/app/api/import/route.ts b/src/app/api/import/route.ts index 5224c39..d6f0a37 100644 --- a/src/app/api/import/route.ts +++ b/src/app/api/import/route.ts @@ -27,6 +27,7 @@ function normalizeToolName(source: string): string { cursor: "cursor", amp: "amp", droid: "droid", + openclaw: "openclaw", claude_code: "claude_code", }; return toolMap[source.toLowerCase()] || source.toLowerCase(); diff --git a/src/components/ui/charts/constants.ts b/src/components/ui/charts/constants.ts index a282a7e..eb90d43 100644 --- a/src/components/ui/charts/constants.ts +++ b/src/components/ui/charts/constants.ts @@ -20,6 +20,7 @@ export const TOOL_COLORS: Record = { gemini: "#20C997", // Teal amp: "#CC9A06", // Yellow droid: "#DC3545", // Red + openclaw: "#6C757D", // Gray }; export const TOOL_LABELS: Record = { @@ -31,6 +32,7 @@ export const TOOL_LABELS: Record = { gemini: "Gemini", amp: "Amp", droid: "Droid", + openclaw: "OpenClaw", }; // GitHub contribution colors diff --git a/src/lib/graph-types.ts b/src/lib/graph-types.ts index 1b89e82..2eacb98 100644 --- a/src/lib/graph-types.ts +++ b/src/lib/graph-types.ts @@ -3,7 +3,7 @@ * Mirrors packages/cli/src/graph-types.ts. */ -export type SourceType = "opencode" | "claude" | "codex" | "gemini" | "cursor" | "amp" | "droid"; +export type SourceType = "opencode" | "claude" | "codex" | "gemini" | "cursor" | "amp" | "droid" | "openclaw"; export interface TokenBreakdown { input: number; diff --git a/supabase/migrations/009_add_openclaw_tool.sql b/supabase/migrations/009_add_openclaw_tool.sql new file mode 100644 index 0000000..b8855f6 --- /dev/null +++ b/supabase/migrations/009_add_openclaw_tool.sql @@ -0,0 +1,12 @@ +-- Add openclaw as a valid tool in daily_activity and token_usage tables + +-- Drop existing constraints +ALTER TABLE public.daily_activity DROP CONSTRAINT IF EXISTS daily_activity_tool_check; +ALTER TABLE public.token_usage DROP CONSTRAINT IF EXISTS token_usage_tool_check; + +-- Add new constraints with openclaw included +ALTER TABLE public.daily_activity ADD CONSTRAINT daily_activity_tool_check + CHECK (tool IN ('claude_code', 'codex', 'cursor', 'opencode', 'claude', 'gemini', 'amp', 'droid', 'openclaw')); + +ALTER TABLE public.token_usage ADD CONSTRAINT token_usage_tool_check + CHECK (tool IN ('claude_code', 'codex', 'cursor', 'opencode', 'claude', 'gemini', 'amp', 'droid', 'openclaw'));