diff --git a/package.json b/package.json
index daaee5b..c7dfbec 100644
--- a/package.json
+++ b/package.json
@@ -10,7 +10,7 @@
"dev": "vite",
"format": "prettier --write .",
"check": "prettier --check .",
- "start": "electron .",
+ "start": "electron . --no-sandbox",
"pack": "electron-builder --dir",
"dist": "electron-builder"
},
diff --git a/src/dataprocessor.js b/src/dataprocessor.js
index 28c9008..da20cf9 100644
--- a/src/dataprocessor.js
+++ b/src/dataprocessor.js
@@ -30,7 +30,6 @@ class DataProcessor {
/**
* Initializes anomaly detection templates.
- * @param {Object} providedTemplates - Template definitions (defaults to templates.json)
*/
async loadConfiguration(providedTemplates = templates) {
try {
@@ -51,14 +50,9 @@ class DataProcessor {
// --- Local File Handling ---
- /**
- * Orchestrates the reading of multiple local files from a file input event.
- */
handleLocalFile(event) {
const files = Array.from(event.target.files);
- if (files.length === 0) {
- return;
- }
+ if (files.length === 0) return;
messenger.emit('ui:set-loading', {
message: `Parsing ${files.length} Files...`,
@@ -66,89 +60,95 @@ class DataProcessor {
let loadedCount = 0;
- files.forEach((file) => {
- const reader = new FileReader();
+ files.forEach(async (file) => {
+ try {
+ const fileText = await this.#readFileContent(file);
- reader.onload = async (e) => {
- try {
- let rawData;
- if (file.name.endsWith('.csv')) {
- const parsedCSV = this.#parseCSV(e.target.result);
-
- // NEW: Explicitly route AlfaOBD files
- if (this.#isAlfaOBD(parsedCSV)) {
- rawData = this.#normalizeAlfaOBD(parsedCSV);
- } else {
- rawData = this.#normalizeWideCSV(parsedCSV);
- }
+ let rawData;
+ if (file.name.includes('.csv')) {
+ const parsedCSV = this.#parseCSV(fileText);
+ if (this.#isAlfaOBD(parsedCSV)) {
+ rawData = this.#normalizeAlfaOBD(parsedCSV);
} else {
- rawData = JSON.parse(e.target.result);
+ rawData = this.#normalizeWideCSV(parsedCSV);
}
- await this.#process(rawData, file.name);
- } catch (err) {
- const msg = `Error parsing ${file.name}: ${err.message}`;
- console.error(msg);
- Alert.showAlert(msg);
- } finally {
- loadedCount++;
- if (loadedCount === files.length) this.#finalizeBatchLoad();
+ } else {
+ // Pass the raw JSON straight to process; it will detect columnar internally
+ rawData = JSON.parse(fileText);
}
- };
+
+ await this.#process(rawData, file.name);
+ } catch (err) {
+ const msg = `Error parsing ${file.name}: ${err.message}`;
+ console.error(msg);
+ Alert.showAlert(msg);
+ } finally {
+ loadedCount++;
+ if (loadedCount === files.length) this.#finalizeBatchLoad();
+ }
+ });
+ }
+
+ async #readFileContent(file) {
+ if (file.name.endsWith('.gz')) {
+ const ds = new DecompressionStream('gzip');
+ const decompressedStream = file.stream().pipeThrough(ds);
+ return await new Response(decompressedStream).text();
+ }
+
+ return new Promise((resolve, reject) => {
+ const reader = new FileReader();
+ reader.onload = (e) => resolve(e.target.result);
+ reader.onerror = () => reject(new Error('Failed to read file'));
reader.readAsText(file);
});
}
// --- Data Transformation & State Sync ---
- /**
- * Processes raw telemetry array into a structured log entry.
- * @param {Array} data - Array of {s, t, v} points
- * @param {string} fileName - Source file identifier
- */
async process(data, fileName) {
const result = await this.#process(data, fileName);
this.#finalizeBatchLoad();
return result;
}
- /**
- * Processes raw telemetry array into a structured log entry.
- * @private
- */
async #process(data, fileName) {
try {
- if (!Array.isArray(data)) throw new Error('Input data must be an array');
+ let telemetryData = data;
- let telemetryPoints = data;
+ // Auto-detect and unpack the highly compressed columnar format
+ if (this.#isColumnarJSON(telemetryData)) {
+ telemetryData = this.#normalizeColumnarJSON(telemetryData);
+ }
+
+ if (!Array.isArray(telemetryData))
+ throw new Error('Input data must be an array');
+
+ let telemetryPoints = telemetryData;
let fileMetadata = {};
- if (data.length > 0 && data[0].metadata) {
- fileMetadata = data[0].metadata;
- telemetryPoints = data.slice(1);
+ if (telemetryData.length > 0 && telemetryData[0].metadata) {
+ fileMetadata = telemetryData[0].metadata;
+ telemetryPoints = telemetryData.slice(1);
}
- // If there are no data points after removing metadata, handle gracefully
if (telemetryPoints.length === 0) {
console.warn(
'Preprocessing: File contains metadata but no telemetry points.'
);
}
- // Detect schema based on the first actual data point
const schema = this.#detectSchema(telemetryPoints[0]);
- // Use flatMap to handle 1-to-many expansion (e.g. Object -> Multiple Signals)
const processedPoints = telemetryPoints.flatMap((item) =>
this.#applyMappingAndCleaning(item, schema)
);
const result = this.#transformRawData(processedPoints, fileName);
- // Attach the extracted metadata to the result object
result.metadata = fileMetadata;
result.size = telemetryPoints.length;
- // --- CHANGED: Check for duplicates in Library before saving ---
const allLibraryFiles = await dbManager.getAllFiles();
const existingFile = allLibraryFiles.find(
(f) => f.name === fileName && f.size === result.size
@@ -156,7 +156,7 @@ class DataProcessor {
if (existingFile) {
console.log(
- `File '${fileName}' already exists in library (ID: ${existingFile.id}). Skipping DB save.`
+ `File '${fileName}' already exists in library. Skipping DB save.`
);
result.dbId = existingFile.id;
} else {
@@ -171,7 +171,6 @@ class DataProcessor {
AppState.files.push(result);
}
- // Register with project manager (it handles its own duplicate checks for resources)
projectManager.registerFile({
name: fileName,
dbId: result.dbId,
@@ -186,15 +185,46 @@ class DataProcessor {
}
}
- /**
- * Detects if the parsed CSV matches the AlfaOBD log format.
- * @private
- */
+ #isColumnarJSON(data) {
+ return (
+ data &&
+ typeof data === 'object' &&
+ !Array.isArray(data) &&
+ 'series' in data
+ );
+ }
+
+ #normalizeColumnarJSON(data) {
+ const normalized = [];
+
+ if (data.metadata) {
+ normalized.push({ metadata: data.metadata });
+ }
+
+ const dictionary = data.signal_dictionary || {};
+ const series = data.series || {};
+
+ for (const [signalId, vectors] of Object.entries(series)) {
+ const signalName = dictionary[signalId] || signalId;
+ const times = vectors.t || [];
+ const values = vectors.v || [];
+
+ const length = Math.min(times.length, values.length);
+ for (let i = 0; i < length; i++) {
+ normalized.push({
+ s: signalName,
+ t: times[i],
+ v: values[i],
+ });
+ }
+ }
+
+ return normalized;
+ }
+
#isAlfaOBD(rows) {
if (!rows || rows.length === 0) return false;
-
const keys = Object.keys(rows[0]);
- // AlfaOBD files specifically use a 'Time' column formatted as 'HH:MM:SS.mmm'
const hasTimeColumn = keys.includes('Time');
const firstTimeValue = rows[0]['Time'];
@@ -205,11 +235,6 @@ class DataProcessor {
);
}
- /**
- * Explicit normalizer for AlfaOBD Wide CSVs.
- * Converts 'HH:MM:SS.mmm' to absolute milliseconds and flattens data.
- * @private
- */
#normalizeAlfaOBD(rows) {
const normalized = [];
if (!rows || rows.length === 0) return normalized;
@@ -222,7 +247,6 @@ class DataProcessor {
const rawTime = row[timeKey];
if (!rawTime) return;
- // Parse HH:MM:SS.mmm into milliseconds
const parts = rawTime.split(':');
if (parts.length !== 3) return;
@@ -234,7 +258,6 @@ class DataProcessor {
const timestampMs = (hours * 3600 + minutes * 60 + seconds) * 1000;
- // Flatten the wide row into Long Format
signalKeys.forEach((sigKey) => {
const val = row[sigKey];
if (val !== '' && val !== null && val !== undefined) {
@@ -250,52 +273,33 @@ class DataProcessor {
return normalized;
}
- /**
- * Determines which schema to use based on the keys present in the first data point.
- * @private
- */
#detectSchema(samplePoint) {
if (!samplePoint) return this.SCHEMA_REGISTRY.JSON;
-
if ('SensorName' in samplePoint) return this.SCHEMA_REGISTRY.CSV;
-
return this.SCHEMA_REGISTRY.JSON;
}
- /**
- * Combines key mapping, object flattening, and data sanitization.
- * Returns an array of points to support 1-to-many mapping.
- * @private
- */
#applyMappingAndCleaning(rawPoint, schema) {
try {
const baseSignal = rawPoint[schema.signal];
const timestamp = Number(rawPoint[schema.timestamp]);
const rawValue = rawPoint[schema.value];
- // Validate Timestamp
if (isNaN(timestamp)) return [];
- // Clean base signal name
let prefix = '';
if (typeof baseSignal === 'string') {
prefix = baseSignal.replace(/\n/g, ' ').trim();
}
- // Supports GPS, Accelerometer, or any complex object structure
if (typeof rawValue === 'object' && rawValue !== null) {
const derivedPoints = [];
for (const [key, val] of Object.entries(rawValue)) {
const numVal = Number(val);
-
- // Strict check: we only want to graph numbers
if (isNaN(numVal)) continue;
- // Format Key: "latitude" -> "Latitude"
const formattedKey = key.charAt(0).toUpperCase() + key.slice(1);
-
- // Construct Composite Signal Name: "GPS" + "Latitude" -> "GPS Latitude"
const finalSignal = prefix
? `${prefix}-${formattedKey}`
: formattedKey;
@@ -310,9 +314,7 @@ class DataProcessor {
}
const numValue = Number(rawValue);
- if (isNaN(numValue)) {
- return [];
- }
+ if (isNaN(numValue)) return [];
return [
{
@@ -327,10 +329,6 @@ class DataProcessor {
}
}
- /**
- * Simple CSV to Object parser (Helper)
- * @private
- */
#parseCSV(csvText) {
const lines = csvText.split('\n').filter((line) => line.trim());
if (lines.length === 0) return [];
@@ -346,16 +344,11 @@ class DataProcessor {
});
}
- /**
- * Converts Wide Format (Time, Sig1, Sig2...) to Long Format (SensorName, Time_ms, Reading)
- * @private
- */
#normalizeWideCSV(rows) {
if (!rows || rows.length === 0) return rows;
const keys = Object.keys(rows[0]);
- // If it already has the standard columns, return as is.
if (
keys.includes('SensorName') &&
(keys.includes('Time_ms') || keys.includes('time'))
@@ -363,7 +356,6 @@ class DataProcessor {
return rows;
}
- // Detect Time Column
const timeKey = keys.find((k) => k.toLowerCase().includes('time'));
if (!timeKey) return rows;
@@ -391,10 +383,6 @@ class DataProcessor {
return normalized;
}
- /**
- * Transforms raw telemetry points into a structured file entry.
- * @private
- */
#transformRawData(data, fileName) {
const sorted = [...data].sort((a, b) => a.timestamp - b.timestamp);
const signals = {};
@@ -425,10 +413,6 @@ class DataProcessor {
};
}
- /**
- * Handles cleanup tasks after a batch of files has been parsed.
- * @private
- */
#finalizeBatchLoad() {
messenger.emit('dataprocessor:batch-load-completed', {});
const input = DOM.get('fileInput');
diff --git a/src/drive.js b/src/drive.js
index 8651e77..1ab4482 100644
--- a/src/drive.js
+++ b/src/drive.js
@@ -199,25 +199,62 @@ class DriveManager {
localStorage.setItem('recent_logs', JSON.stringify(recent));
const currentToken = ++this.activeLoadToken;
- UI.setLoading(true, 'Fetching from Drive...', () => {
+ UI.setLoading(true, 'Downloading from Drive...', () => {
this.activeLoadToken++;
UI.setLoading(false);
});
try {
- const response = await gapi.client.drive.files.get({
- fileId: id,
- alt: 'media',
- });
- if (currentToken !== this.activeLoadToken) return;
- dataProcessor.process(response.result, fileName);
- } catch (error) {
- if (currentToken === this.activeLoadToken)
- Alert.showAlert(
- `Drive Error: ${error.result?.error?.message || error.message}`
+ let dataToProcess;
+
+ if (fileName.endsWith('.gz')) {
+ const tokenObj = gapi.client.getToken();
+ if (!tokenObj)
+ throw new Error(
+ 'No active Google session found. Please sign in again.'
+ );
+
+ const response = await fetch(
+ `https://www.googleapis.com/drive/v3/files/${id}?alt=media`,
+ {
+ headers: { Authorization: `Bearer ${tokenObj.access_token}` },
+ }
);
+
+ if (!response.ok) throw new Error(`HTTP error: ${response.status}`);
+ const blob = await response.blob();
+
+ if (currentToken !== this.activeLoadToken) return;
+
+ const ds = new DecompressionStream('gzip');
+ const decompressedStream = blob.stream().pipeThrough(ds);
+ const fileText = await new Response(decompressedStream).text();
+
+ dataToProcess = JSON.parse(fileText);
+ } else {
+ const response = await gapi.client.drive.files.get({
+ fileId: id,
+ alt: 'media',
+ });
+
+ if (currentToken !== this.activeLoadToken) return;
+
+ // Handle GAPI optionally parsing JSON strings automatically
+ dataToProcess =
+ typeof response.result === 'string'
+ ? JSON.parse(response.result)
+ : response.result;
+ }
+
+ dataProcessor.process(dataToProcess, fileName);
+ } catch (error) {
+ if (currentToken === this.activeLoadToken) {
+ Alert.showAlert(`Drive Error: ${error.message}`);
+ }
} finally {
- if (currentToken === this.activeLoadToken) UI.setLoading(false);
+ if (currentToken === this.activeLoadToken) {
+ UI.setLoading(false);
+ }
}
}
@@ -481,14 +518,18 @@ class DriveManager {
}
getFileMetadata(fileName) {
- const match = fileName.match(/-(\d+)-(\d+)\.json$/);
+ // Robust regex handles both legacy .json and the new .json.gz / .json.json.gz
+ const match = fileName.match(/-(\d+)-(\d+)(?:\.json|\.gz)+$/);
if (!match) return { date: 'Unknown', length: '?' };
+
const date = new Date(parseInt(match[1]));
+
+ // Restored backward compatibility: return standard ISO string
return { date: date.toISOString(), length: match[2] };
}
extractTimestamp(fileName) {
- const match = fileName.match(/-(\d+)-(\d+)\.json$/);
+ const match = fileName.match(/-(\d+)-(\d+)(?:\.json|\.gz)+$/);
return match ? parseInt(match[1]) : 0;
}
diff --git a/tests/dataprocessor.test.js b/tests/dataprocessor.test.js
index cf69b29..3cab16d 100644
--- a/tests/dataprocessor.test.js
+++ b/tests/dataprocessor.test.js
@@ -126,7 +126,6 @@ describe('DataProcessor - handleLocalFile', () => {
dataProcessor.handleLocalFile(mockEvent);
- // Increase timeout slightly to allow async process() to finish
setTimeout(() => {
try {
expect(messenger.emit).toHaveBeenCalledWith(
@@ -134,14 +133,6 @@ describe('DataProcessor - handleLocalFile', () => {
{ message: 'Parsing 1 Files...' }
);
- // Note: process() fails on "dummy" data structure in tests usually,
- // triggering ui:updateDataLoadedState -> false.
- // If it succeeds, it triggers batch-load-completed.
- // Based on previous test logic, we expect it to fail or finish.
- // We verify calls are made.
-
- // This expectation might vary based on whether 'dummy' json structure throws in #process
- // But the main goal is ensuring it ran.
expect(messenger.emit).toHaveBeenCalled();
done();
@@ -365,7 +356,7 @@ Battery,100,12.6
} catch (error) {
done(error);
}
- }, 50);
+ }, 200);
});
test('should correctly preprocess and map CSV data using LEGACY_CSV schema', async () => {
@@ -732,9 +723,6 @@ describe('DataProcessor: Nested Object Support', () => {
});
test('should detect and parse AlfaOBD HH:MM:SS.mmm timestamps into milliseconds', (done) => {
- // Math verification for 13:48:35.666
- // 13 hours = 46,800 sec | 48 min = 2,880 sec | 35.666 sec
- // Total seconds: 49715.666 -> * 1000 = 49715666 milliseconds
const alfaOBD_CSV = `Time,Engine speed rpm,Spark advance °
13:48:35.666,1584,5.938
13:48:37.223,1858,14.938`;
@@ -821,3 +809,78 @@ invalid_time_string,2000
});
});
});
+
+describe('DataProcessor: Columnar JSON Support', () => {
+ beforeEach(() => {
+ AppState.files = [];
+ jest.clearAllMocks();
+
+ document.body.innerHTML = `
+
+
+ `;
+
+ DOM.get = jest.fn((id) => document.getElementById(id));
+ dbManager.getAllFiles = jest.fn().mockResolvedValue([]);
+ dbManager.saveTelemetry = jest.fn().mockResolvedValue(1);
+ projectManager.registerFile = jest.fn();
+ });
+
+ test('should normalize highly-compressed Columnar JSON back to structured format', (done) => {
+ const columnarData = {
+ metadata: {
+ 'trip.duration': '3600',
+ },
+ signal_dictionary: {
+ 12: 'Boost Pressure',
+ 14: 'Engine RPM',
+ },
+ series: {
+ 12: {
+ t: [1000, 2000],
+ v: [14.1, 15.2],
+ },
+ 14: {
+ t: [1000, 2000],
+ v: [2000.0, 2100.0],
+ },
+ },
+ };
+
+ const event = {
+ target: {
+ files: [
+ new File([JSON.stringify(columnarData)], 'trip.json', {
+ type: 'application/json',
+ }),
+ ],
+ },
+ };
+
+ dataProcessor.handleLocalFile(event);
+
+ setTimeout(() => {
+ try {
+ expect(AppState.files.length).toBe(1);
+ const file = AppState.files[0];
+
+ // Metadata check
+ expect(file.metadata['trip.duration']).toBe('3600');
+
+ // Available signals check (should map IDs to human-readable names)
+ expect(file.availableSignals).toContain('Boost Pressure');
+ expect(file.availableSignals).toContain('Engine RPM');
+
+ // Series data check (un-pivoted successfully)
+ const boostData = file.signals['Boost Pressure'];
+ expect(boostData).toHaveLength(2);
+ expect(boostData[0].x).toBe(1000); // timestamp
+ expect(boostData[0].y).toBe(14.1); // value
+
+ done();
+ } catch (error) {
+ done(error);
+ }
+ }, 200);
+ });
+});