From db2d6bf2ec10fe90d3c2a95958b85607b897189f Mon Sep 17 00:00:00 2001 From: Tomek Zebrowski Date: Fri, 3 Apr 2026 22:50:51 +0200 Subject: [PATCH 1/2] perf(gdrive): optimize trip log exports with columnar JSON - Refactored `TripLogTransformer` to generate a Columnar Series-based JSON format instead of a flat row-based array. This eliminates redundant keys and string repetitions, further shrinking the payload. - Introduced a `signal_dictionary` to the JSON schema. Time-series data is now keyed by raw numeric PIDs to guarantee consistency across app language changes, while the dictionary provides the translated names. - Updated `TripLogTransformerTest` assertions to validate the new schema and dictionary mapping. --- app/build.gradle | 6 +- .../integrations/log/TripLogTransformer.kt | 151 ++++++++++-------- .../gcp/gdrive/TripLogTransformerTest.kt | 29 ++-- 3 files changed, 107 insertions(+), 79 deletions(-) diff --git a/app/build.gradle b/app/build.gradle index 4e94492b..f535ce86 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -81,7 +81,7 @@ android { resValue "string", "DEFAULT_PROFILE", "profile_8" resValue "string", "applicationId", "org.obd.graphs.my.giulia.aa" applicationId "org.obd.graphs.my.giulia.aa" - versionCode 220 + versionCode 221 } giuliaPerformanceMonitor { @@ -89,7 +89,7 @@ android { resValue "string", "DEFAULT_PROFILE", "profile_8" resValue "string", "applicationId", "org.obd.graphs.my.giulia.performance_monitor" applicationId "org.obd.graphs.my.giulia.performance_monitor" - versionCode 101 + versionCode 102 } giulia { @@ -97,7 +97,7 @@ android { resValue "string", "DEFAULT_PROFILE", "profile_3" resValue "string", "applicationId", "org.obd.graphs.my.giulia" applicationId "org.obd.graphs.my.giulia" - versionCode 74 + versionCode 75 } } diff --git a/integrations/src/main/java/org/obd/graphs/integrations/log/TripLogTransformer.kt b/integrations/src/main/java/org/obd/graphs/integrations/log/TripLogTransformer.kt index e263ac7f..9ca217eb 100644 --- a/integrations/src/main/java/org/obd/graphs/integrations/log/TripLogTransformer.kt +++ b/integrations/src/main/java/org/obd/graphs/integrations/log/TripLogTransformer.kt @@ -48,29 +48,81 @@ private class DefaultJSONOutput( private val valueMapper: (signal: Int, value: Any) -> Any ) : TripLogTransformer { + private class SeriesData { + val timestamps = mutableListOf() + val values = mutableListOf() + } + override fun transform(file: File, metadata: Map): File = file.inputStream().use { input -> process(JsonReader(InputStreamReader(input)), metadata) } - override fun transform(log: String, metadata: Map): File = process(JsonReader(StringReader(log)), metadata) + override fun transform(log: String, metadata: Map): File = + process(JsonReader(StringReader(log)), metadata) private fun process(reader: JsonReader, metadata: Map): File { Log.d("DefaultJSONOutput", "Received $metadata") val tempFile = File.createTempFile("json_buffer_", ".tmp").apply { - // Ensures the file is cleaned up if the JVM shuts down deleteOnExit() } + val seriesMap = mutableMapOf() + try { - // Nested .use calls ensure all streams are closed even if an exception occurs + reader.isLenient = true + parseRootToMemory(reader, seriesMap) + tempFile.outputStream().bufferedWriter().use { fileWriter -> JsonWriter(fileWriter).use { writer -> - reader.isLenient = true - writer.beginArray() - parseRoot(reader, writer, metadata) - writer.endArray() + writer.beginObject() // Root object + + // Write Metadata + if (metadata.isNotEmpty()) { + writer.name("metadata") + writer.beginObject() + metadata.forEach { (key, value) -> + writer.name(key).value(value) + } + writer.endObject() + } + + // Write Signal Dictionary + writer.name("signal_dictionary") + writer.beginObject() + seriesMap.keys.forEach { signalKey -> + val idAsInt = signalKey.toIntOrNull() + val translatedName = if (idAsInt != null) { + signalMapper[idAsInt] ?: signalKey + } else { + signalKey + } + writer.name(signalKey).value(translatedName.toString()) + } + writer.endObject() + + // Write Series Data + writer.name("series") + writer.beginObject() + seriesMap.forEach { (signalId, seriesData) -> + writer.name(signalId) + writer.beginObject() + + writer.name("t") + writer.beginArray() + seriesData.timestamps.forEach { writer.value(it) } + writer.endArray() + + writer.name("v") + writer.beginArray() + seriesData.values.forEach { writer.writeDynamicValue(it) } + writer.endArray() + + writer.endObject() + } + writer.endObject() // end series + writer.endObject() // end root } } return tempFile @@ -85,26 +137,14 @@ private class DefaultJSONOutput( } } - private fun parseRoot( + private fun parseRootToMemory( reader: JsonReader, - writer: JsonWriter, - metadata: Map + seriesMap: MutableMap ) { - if (metadata.isNotEmpty()) { - writer.beginObject() - writer.name("metadata") - writer.beginObject() - metadata.forEach { (key, value) -> - writer.name(key).value(value) - } - writer.endObject() - writer.endObject() - } - reader.beginObject() while (reader.hasNext()) { if (reader.nextName() == "entries") { - parseEntries(reader, writer) + parseEntriesToMemory(reader, seriesMap) } else { reader.skipValue() } @@ -112,58 +152,58 @@ private class DefaultJSONOutput( reader.endObject() } - private fun parseEntries( + private fun parseEntriesToMemory( reader: JsonReader, - writer: JsonWriter + seriesMap: MutableMap ) { - reader.beginObject() // Start "entries" map + reader.beginObject() while (reader.hasNext()) { reader.nextName() // Skip the dynamic key ("12", "99") - parseEntryGroup(reader, writer) + parseEntryGroupToMemory(reader, seriesMap) } reader.endObject() } - private fun parseEntryGroup( + private fun parseEntryGroupToMemory( reader: JsonReader, - writer: JsonWriter + seriesMap: MutableMap ) { - reader.beginObject() // Inside "12": { + reader.beginObject() while (reader.hasNext()) { if (reader.nextName() == "metrics") { - parseMetricsArray(reader, writer) + parseMetricsArrayToMemory(reader, seriesMap) } else { - reader.skipValue() // Skip "id", "mean", etc. + reader.skipValue() } } reader.endObject() } - private fun parseMetricsArray( + private fun parseMetricsArrayToMemory( reader: JsonReader, - writer: JsonWriter + seriesMap: MutableMap ) { - reader.beginArray() // [ + reader.beginArray() while (reader.hasNext()) { - parseSingleMetric(reader, writer) + parseSingleMetricToMemory(reader, seriesMap) } reader.endArray() } - private fun parseSingleMetric( + private fun parseSingleMetricToMemory( reader: JsonReader, - writer: JsonWriter + seriesMap: MutableMap ) { var ts: Long = 0 var signal = 0 var value: Any = 0.0 - reader.beginObject() // Metric object { + reader.beginObject() while (reader.hasNext()) { when (reader.nextName()) { "ts" -> ts = reader.nextLong() "entry" -> { - reader.beginObject() // Nested "entry": { + reader.beginObject() while (reader.hasNext()) { when (reader.nextName()) { "data" -> signal = reader.nextInt() @@ -179,25 +219,19 @@ private class DefaultJSONOutput( } reader.endObject() } - else -> reader.skipValue() } } - reader.endObject() - writer.beginObject() - writer.name("t").value(ts) - writer.name("s").value((signalMapper[signal] ?: signal).toString()) - val mappedResult: Any = valueMapper(signal, value) - writer.name("v") - writer.writeDynamicValue(mappedResult) - writer.endObject() + val signalKey = signal.toString() // Group purely by ID + val mappedResult = valueMapper(signal, value) + + val series = seriesMap.getOrPut(signalKey) { SeriesData() } + series.timestamps.add(ts) + series.values.add(mappedResult) } - /** - * Recursively reads a JSON object from the reader and returns it as a Map. - */ private fun JsonReader.readMap(): Map { val map = mutableMapOf() @@ -205,10 +239,8 @@ private class DefaultJSONOutput( while (this.hasNext()) { val key = this.nextName() val value: Any? = when (this.peek()) { - JsonToken.BEGIN_OBJECT -> readMap() // Recursive call for nested maps + JsonToken.BEGIN_OBJECT -> readMap() JsonToken.BEGIN_ARRAY -> { - // Optional: Handle arrays if your source map has lists - // For now we just skip or you can implement readList() similarly this.skipValue() null } @@ -227,27 +259,20 @@ private class DefaultJSONOutput( return map } - /** - * Extension to write mixed types (Number, String, Map, List) to JsonWriter. - */ private fun JsonWriter.writeDynamicValue(value: Any?) { when (value) { null -> this.nullValue() is Number -> this.value(value) is String -> this.value(value) is Boolean -> this.value(value) - - // Handle Map -> JSON Object is Map<*, *> -> { this.beginObject() for ((k, v) in value) { this.name(k.toString()) - writeDynamicValue(v) // Recursive call for nested values + writeDynamicValue(v) } this.endObject() } - - // Handle List/Array -> JSON Array (Optional, but good for safety) is Collection<*> -> { this.beginArray() for (item in value) { @@ -255,8 +280,6 @@ private class DefaultJSONOutput( } this.endArray() } - - // Fallback for unknown objects else -> this.value(value.toString()) } } diff --git a/integrations/src/test/java/org/obd/graphs/integrations/gcp/gdrive/TripLogTransformerTest.kt b/integrations/src/test/java/org/obd/graphs/integrations/gcp/gdrive/TripLogTransformerTest.kt index fe0ee756..f1a4b72b 100644 --- a/integrations/src/test/java/org/obd/graphs/integrations/gcp/gdrive/TripLogTransformerTest.kt +++ b/integrations/src/test/java/org/obd/graphs/integrations/gcp/gdrive/TripLogTransformerTest.kt @@ -70,7 +70,10 @@ class TripLogTransformerTest { val transformer: TripLogTransformer = TripLog.transformer { s, v -> v } val result = transformer.transform(file).readText() - Assertions.assertThat(result).startsWith("[{\"t\":1765481896083,\"s\":\"12\",\"v\":3298.0767},{\"t\":1765481896267,\"s\":\"12\",\"v\":3298.0767},{\"t\":1765481896463,\"s\":\"12\",\"v\":3298.0767},{\"t\":1765481896666,\"s\":\"12\"") + Assertions.assertThat(result).contains("\"signal_dictionary\":{") + Assertions.assertThat(result).contains("\"series\":{") + Assertions.assertThat(result).contains("\"12\":{\"t\":[1765481896083,1765481896267") + Assertions.assertThat(result).contains("\"v\":[3298.0767,3298.0767") } @Test @@ -111,10 +114,11 @@ class TripLogTransformerTest { val result = transformer.transform(rawJson, meta).readText() + // Notice 12 is mapped to "Boost" in dictionary, but 13 defaults to "13" since it's unmapped. val expectedJson = - """[{"metadata":{"key1":"value1","key2":"value2"}},{"t":1000,"s":"Boost","v":101.0},{"t":2000,"s":"13","v":121.0}]""" + """{"metadata":{"key1":"value1","key2":"value2"},"signal_dictionary":{"12":"Boost","13":"13"},"series":{"12":{"t":[1000],"v":[101.0]},"13":{"t":[2000],"v":[121.0]}}}""" - Assertions.assertThat(expectedJson).isEqualTo(result) + Assertions.assertThat(result).isEqualTo(expectedJson) } @Test @@ -148,7 +152,7 @@ class TripLogTransformerTest { val transformer: TripLogTransformer = TripLog.transformer { s, v -> v } val result = transformer.transform(rawJson).readText() - val expectedJson = """[{"t":1500,"s":"99","v":{"GPS altitude":57.10662841796875,"GPS Location":{"altitude":57.10662841796875,"accuracy":46.843723,"latitude":54.16406183,"longitude":16.29066863}}}]""" + val expectedJson = """{"signal_dictionary":{"99":"99"},"series":{"99":{"t":[1500],"v":[{"GPS altitude":57.10662841796875,"GPS Location":{"altitude":57.10662841796875,"accuracy":46.843723,"latitude":54.16406183,"longitude":16.29066863}}]}}}""" Assertions.assertThat(result).isEqualTo(expectedJson) } @@ -186,13 +190,13 @@ class TripLogTransformerTest { val result = transformer.transform(rawJson).readText() val expectedJson = - """[{"t":1000,"s":"Boost","v":101.0},{"t":2000,"s":"13","v":121.0}]""" + """{"signal_dictionary":{"12":"Boost","13":"13"},"series":{"12":{"t":[1000],"v":[101.0]},"13":{"t":[2000],"v":[121.0]}}}""" - Assertions.assertThat(expectedJson).isEqualTo(result) + Assertions.assertThat(result).isEqualTo(expectedJson) } @Test - fun `optimize should convert complex json to optimized flat format`() { + fun `optimize should convert complex json to optimized columnar format`() { val rawJson = """ { @@ -224,9 +228,9 @@ class TripLogTransformerTest { val result = transformer.transform(rawJson).readText() val expectedJson = - """[{"t":1000,"s":"12","v":50.5},{"t":2000,"s":"12","v":60.5}]""" + """{"signal_dictionary":{"12":"12"},"series":{"12":{"t":[1000,2000],"v":[50.5,60.5]}}}""" - Assertions.assertThat(expectedJson).isEqualTo(result) + Assertions.assertThat(result).isEqualTo(expectedJson) } @Test @@ -253,9 +257,10 @@ class TripLogTransformerTest { assertFalse(result.contains("\"rawAnswer\"")) assertFalse(result.contains("\"entry\"")) - assertTrue(result.contains("\"s\"")) + assertTrue(result.contains("\"signal_dictionary\"")) + assertTrue(result.contains("\"series\"")) assertTrue(result.contains("\"t\"")) - assertTrue(result.contains("\"v\":2.0")) + assertTrue(result.contains("\"v\":[2.0]")) } @Test @@ -271,7 +276,7 @@ class TripLogTransformerTest { val transformer: TripLogTransformer = TripLog.transformer { s, v -> v } val result = transformer.transform(rawJson).readText() - val expected = """[]""" + val expected = """{"signal_dictionary":{},"series":{}}""" assertEquals(expected, result) } } From 3288d4eb33e02f0eebf7fb3277473f770a1a2008 Mon Sep 17 00:00:00 2001 From: Tomek Zebrowski Date: Sat, 4 Apr 2026 10:37:42 +0200 Subject: [PATCH 2/2] feat: reduce log files size by using gzip compression --- .../gcp/gdrive/DefaultTripLogDriveManager.kt | 34 +++++++++++++------ .../integrations/log/TripLogTransformer.kt | 1 - 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/integrations/src/main/java/org/obd/graphs.integrations/gcp/gdrive/DefaultTripLogDriveManager.kt b/integrations/src/main/java/org/obd/graphs.integrations/gcp/gdrive/DefaultTripLogDriveManager.kt index dcbc3abb..4a3e2330 100644 --- a/integrations/src/main/java/org/obd/graphs.integrations/gcp/gdrive/DefaultTripLogDriveManager.kt +++ b/integrations/src/main/java/org/obd/graphs.integrations/gcp/gdrive/DefaultTripLogDriveManager.kt @@ -29,6 +29,7 @@ import org.obd.graphs.integrations.log.OutputType import org.obd.graphs.integrations.log.TripLog import org.obd.graphs.sendBroadcastEvent import java.io.File +import java.util.zip.GZIPOutputStream internal open class DefaultTripLogDriveManager( webClientId: String, @@ -68,18 +69,31 @@ internal open class DefaultTripLogDriveManager( metadata["trip.profileId"] = tripDesc.profileId metadata["trip.startTime"] = tripDesc.startTime metadata["trip.profileLabel"] = tripDesc.profileLabel - metadata["trip.profileId"] = tripDesc.profileId - transformer.transform(inFile, metadata).inputStream().use { outFile -> - drive.uploadFile( - InputStreamContent( - "text/plain", - outFile, - "$deviceId-${inFile.name.removePrefix("trip-profile_")}" - ), - folderId - ) + val transformedFile = transformer.transform(inFile, metadata) + + val tempGzipFile = File(activity.cacheDir, "${inFile.name}.gz") + + tempGzipFile.outputStream().use { fos -> + GZIPOutputStream(fos).use { gzipOs -> + transformedFile.inputStream().use { inputStream -> + inputStream.copyTo(gzipOs) + } + } } + + val originalName = inFile.name.removePrefix("trip-profile_") + val fileName = "$deviceId-$originalName.json.gz" + + drive.uploadFile( + localFile = tempGzipFile, + fileName = fileName, + parentFolderId = folderId, + mimeType = "application/gzip" + ) + + tempGzipFile.delete() + transformedFile.delete() } sendBroadcastEvent(TRIPS_UPLOAD_SUCCESSFUL) } diff --git a/integrations/src/main/java/org/obd/graphs/integrations/log/TripLogTransformer.kt b/integrations/src/main/java/org/obd/graphs/integrations/log/TripLogTransformer.kt index 9ca217eb..56c0e7a4 100644 --- a/integrations/src/main/java/org/obd/graphs/integrations/log/TripLogTransformer.kt +++ b/integrations/src/main/java/org/obd/graphs/integrations/log/TripLogTransformer.kt @@ -32,7 +32,6 @@ internal interface TripLogTransformer { internal enum class OutputType { JSON } object TripLog { - @Suppress("UNUSED_EXPRESSION") internal fun transformer( outputType: OutputType = OutputType.JSON, signalMapper: Map = mapOf(),