From ab840fc72ac1889c2cd69253e50ef185b3bb9093 Mon Sep 17 00:00:00 2001 From: Lucas Faria Date: Mon, 27 Oct 2025 13:42:04 -0300 Subject: [PATCH 1/3] feat: add AI task extraction for notetaker recordings - Add task extraction IPC handler using OpenAI GPT-4o-mini - Create useExtractTasks hook with TanStack Query mutation - Add Extract Tasks button in LiveTranscriptView - Display extracted tasks with title and description - Show loading, error, and empty states - Require OpenAI API key in settings This allows users to extract actionable tasks from meeting transcripts with a single click, perfect for the demo today! --- src/main/preload.ts | 5 + src/main/services/recallRecording.ts | 56 +++++++++++ .../components/LiveTranscriptView.tsx | 97 ++++++++++++++++++- .../notetaker/hooks/useExtractTasks.ts | 42 ++++++++ src/renderer/types/electron.d.ts | 4 + 5 files changed, 203 insertions(+), 1 deletion(-) create mode 100644 src/renderer/features/notetaker/hooks/useExtractTasks.ts diff --git a/src/main/preload.ts b/src/main/preload.ts index 658a3ec4d..810c10ac5 100644 --- a/src/main/preload.ts +++ b/src/main/preload.ts @@ -202,6 +202,11 @@ contextBridge.exposeInMainWorld("electronAPI", { ipcRenderer.invoke("notetaker:get-recording", recordingId), notetakerDeleteRecording: (recordingId: string): Promise => ipcRenderer.invoke("notetaker:delete-recording", recordingId), + notetakerExtractTasks: ( + transcriptText: string, + openaiApiKey: string, + ): Promise> => + ipcRenderer.invoke("notetaker:extract-tasks", transcriptText, openaiApiKey), // Real-time transcript listener onTranscriptSegment: ( listener: (segment: { diff --git a/src/main/services/recallRecording.ts b/src/main/services/recallRecording.ts index 3ca45de9e..2f75d5860 100644 --- a/src/main/services/recallRecording.ts +++ b/src/main/services/recallRecording.ts @@ -1,6 +1,10 @@ +import { createOpenAI } from "@ai-sdk/openai"; import { PostHogAPIClient } from "@api/posthogClient"; import RecallAiSdk from "@recallai/desktop-sdk"; +import { generateObject } from "ai"; import { ipcMain } from "electron"; +import { z } from "zod"; +import { TASK_EXTRACTION_PROMPT } from "./transcription-prompts"; interface RecordingSession { windowId: string; @@ -428,6 +432,45 @@ export function getActiveSessions() { return Array.from(activeSessions.values()); } +async function extractTasksFromTranscript( + transcriptText: string, + openaiApiKey: string, +): Promise> { + try { + const openai = createOpenAI({ apiKey: openaiApiKey }); + + const schema = z.object({ + tasks: z.array( + z.object({ + title: z.string().describe("Brief task title"), + description: z.string().describe("Detailed description with context"), + }), + ), + }); + + const { object } = await generateObject({ + model: openai("gpt-4o-mini"), + schema, + messages: [ + { + role: "system", + content: + "You are a helpful assistant that extracts actionable tasks from conversation transcripts. Be generous in identifying work items - include feature requests, requirements, and any work that needs to be done.", + }, + { + role: "user", + content: `${TASK_EXTRACTION_PROMPT}\n${transcriptText}`, + }, + ], + }); + + return object.tasks || []; + } catch (error) { + console.error("[Task Extraction] Error:", error); + throw error; + } +} + export function registerRecallIPCHandlers() { ipcMain.handle( "recall:initialize", @@ -483,4 +526,17 @@ export function registerRecallIPCHandlers() { } return await posthogClient.deleteDesktopRecording(recordingId); }); + + ipcMain.handle( + "notetaker:extract-tasks", + async (_event, transcriptText, openaiApiKey) => { + console.log("[Task Extraction] Starting task extraction..."); + const tasks = await extractTasksFromTranscript( + transcriptText, + openaiApiKey, + ); + console.log(`[Task Extraction] Extracted ${tasks.length} tasks`); + return tasks; + }, + ); } diff --git a/src/renderer/features/notetaker/components/LiveTranscriptView.tsx b/src/renderer/features/notetaker/components/LiveTranscriptView.tsx index 7e8a764fe..408e54d71 100644 --- a/src/renderer/features/notetaker/components/LiveTranscriptView.tsx +++ b/src/renderer/features/notetaker/components/LiveTranscriptView.tsx @@ -1,5 +1,18 @@ -import { Badge, Box, Card, Flex, ScrollArea, Text } from "@radix-ui/themes"; +import { ListChecksIcon, SparkleIcon } from "@phosphor-icons/react"; +import { + Badge, + Box, + Button, + Card, + Flex, + Heading, + ScrollArea, + Separator, + Text, +} from "@radix-ui/themes"; import { useEffect, useRef, useState } from "react"; +import { useAuthStore } from "../../../stores/authStore"; +import { useExtractTasks } from "../hooks/useExtractTasks"; import { useLiveTranscript } from "../hooks/useTranscript"; interface LiveTranscriptViewProps { @@ -11,10 +24,21 @@ export function LiveTranscriptView({ }: LiveTranscriptViewProps) { const scrollRef = useRef(null); const [autoScroll, setAutoScroll] = useState(true); + const openaiApiKey = useAuthStore((state) => state.openaiApiKey); const { segments, addSegment, forceUpload, pendingCount } = useLiveTranscript(posthogRecordingId); + const extractTasksMutation = useExtractTasks(); + + const handleExtractTasks = () => { + const fullText = segments.map((s) => s.text).join(" "); + extractTasksMutation.mutate({ + recordingId: posthogRecordingId, + transcriptText: fullText, + }); + }; + // Auto-scroll to bottom when new segments arrive useEffect(() => { if (autoScroll && scrollRef.current && segments.length > 0) { @@ -177,6 +201,77 @@ export function LiveTranscriptView({ )} + + {/* Extract Tasks Section */} + {segments.length > 0 && ( + <> + + + + + + + Extracted tasks + + + + + + {!openaiApiKey && ( + + + Add OpenAI API key in settings to extract tasks + + + )} + + {extractTasksMutation.isSuccess && extractTasksMutation.data && ( + + {extractTasksMutation.data.tasks.length === 0 ? ( + + + No tasks found in transcript + + + ) : ( + extractTasksMutation.data.tasks.map((task, idx) => ( + + + + {task.title} + + + {task.description} + + + + )) + )} + + )} + + {extractTasksMutation.isError && ( + + + Failed to extract tasks:{" "} + {extractTasksMutation.error instanceof Error + ? extractTasksMutation.error.message + : "Unknown error"} + + + )} + + + )} ); } diff --git a/src/renderer/features/notetaker/hooks/useExtractTasks.ts b/src/renderer/features/notetaker/hooks/useExtractTasks.ts new file mode 100644 index 000000000..82f7f3cb9 --- /dev/null +++ b/src/renderer/features/notetaker/hooks/useExtractTasks.ts @@ -0,0 +1,42 @@ +import { useMutation, useQueryClient } from "@tanstack/react-query"; +import { useAuthStore } from "../../../stores/authStore"; + +export interface ExtractedTask { + title: string; + description: string; +} + +export function useExtractTasks() { + const openaiApiKey = useAuthStore((state) => state.openaiApiKey); + const queryClient = useQueryClient(); + + return useMutation({ + mutationFn: async ({ + recordingId, + transcriptText, + }: { + recordingId: string; + transcriptText: string; + }) => { + if (!openaiApiKey) { + throw new Error("OpenAI API key not configured"); + } + + const tasks = await window.electronAPI.notetakerExtractTasks( + transcriptText, + openaiApiKey, + ); + + return { tasks, recordingId }; + }, + onSuccess: (data) => { + // Invalidate recording query to refetch with new tasks + queryClient.invalidateQueries({ + queryKey: ["notetaker-recording", data.recordingId], + }); + queryClient.invalidateQueries({ + queryKey: ["notetaker-transcript", data.recordingId], + }); + }, + }); +} diff --git a/src/renderer/types/electron.d.ts b/src/renderer/types/electron.d.ts index 02730f745..7d0c6ecb2 100644 --- a/src/renderer/types/electron.d.ts +++ b/src/renderer/types/electron.d.ts @@ -148,6 +148,10 @@ export interface IElectronAPI { recall_recording_id: string | null; }>; notetakerDeleteRecording: (recordingId: string) => Promise; + notetakerExtractTasks: ( + transcriptText: string, + openaiApiKey: string, + ) => Promise>; // Real-time transcript listener onTranscriptSegment: ( listener: (segment: { From d9353b63b73d35be2e72cc4a5c16488bd62b9879 Mon Sep 17 00:00:00 2001 From: Lucas Faria Date: Mon, 27 Oct 2025 13:45:23 -0300 Subject: [PATCH 2/3] feat: auto-extract tasks immediately when meeting ends - Automatically trigger task extraction when meeting-ended IPC event fires - No need to wait for upload - extract tasks from local transcript segments - Only auto-extracts if OpenAI API key is configured - Users see tasks instantly when they return to check the recording This creates a magical UX - tasks are already extracted by the time users look at their recordings! --- .../components/LiveTranscriptView.tsx | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/src/renderer/features/notetaker/components/LiveTranscriptView.tsx b/src/renderer/features/notetaker/components/LiveTranscriptView.tsx index 408e54d71..cacf1dbed 100644 --- a/src/renderer/features/notetaker/components/LiveTranscriptView.tsx +++ b/src/renderer/features/notetaker/components/LiveTranscriptView.tsx @@ -79,7 +79,7 @@ export function LiveTranscriptView({ return cleanup; }, [posthogRecordingId, addSegment]); - // Listen for meeting-ended event to force upload remaining segments + // Listen for meeting-ended event to force upload remaining segments and extract tasks useEffect(() => { console.log( `[LiveTranscript] Setting up meeting-ended listener for ${posthogRecordingId}`, @@ -87,13 +87,33 @@ export function LiveTranscriptView({ const cleanup = window.electronAPI.onMeetingEnded((event) => { if (event.posthog_recording_id === posthogRecordingId) { - console.log(`[LiveTranscript] Meeting ended, force uploading segments`); + console.log( + `[LiveTranscript] Meeting ended, force uploading segments and extracting tasks`, + ); forceUpload(); + + // Automatically extract tasks when meeting ends (if OpenAI key is configured) + if (openaiApiKey && segments.length > 0) { + console.log( + `[LiveTranscript] Auto-extracting tasks for ${segments.length} segments`, + ); + const fullText = segments.map((s) => s.text).join(" "); + extractTasksMutation.mutate({ + recordingId: posthogRecordingId, + transcriptText: fullText, + }); + } } }); return cleanup; - }, [posthogRecordingId, forceUpload]); + }, [ + posthogRecordingId, + forceUpload, + openaiApiKey, + segments, + extractTasksMutation, + ]); // Detect manual scroll to disable auto-scroll const handleScroll = () => { From b6ce79da8107f3da82476cb7b2708763206919a2 Mon Sep 17 00:00:00 2001 From: Lucas Faria Date: Mon, 27 Oct 2025 13:52:07 -0300 Subject: [PATCH 3/3] fix: store extracted tasks per-recording in backend - Update uploadDesktopRecordingTranscript to accept extracted_tasks - Save extracted tasks to backend RecordingTranscript model - Fetch tasks from backend transcript instead of mutation state - Each recording now has its own tasks properly isolated This fixes the bug where all recordings were showing the same tasks. --- src/api/posthogClient.ts | 4 +++ .../components/LiveTranscriptView.tsx | 35 ++++++++++--------- .../notetaker/hooks/useExtractTasks.ts | 15 +++++++- .../features/notetaker/hooks/useTranscript.ts | 4 +++ 4 files changed, 41 insertions(+), 17 deletions(-) diff --git a/src/api/posthogClient.ts b/src/api/posthogClient.ts index 3712e0e82..809442293 100644 --- a/src/api/posthogClient.ts +++ b/src/api/posthogClient.ts @@ -411,6 +411,10 @@ export class PostHogAPIClient { text: string; confidence: number | null; }>; + extracted_tasks?: Array<{ + title: string; + description: string; + }>; }, ) { this.validateRecordingId(recordingId); diff --git a/src/renderer/features/notetaker/components/LiveTranscriptView.tsx b/src/renderer/features/notetaker/components/LiveTranscriptView.tsx index cacf1dbed..24e8e68fe 100644 --- a/src/renderer/features/notetaker/components/LiveTranscriptView.tsx +++ b/src/renderer/features/notetaker/components/LiveTranscriptView.tsx @@ -13,7 +13,7 @@ import { import { useEffect, useRef, useState } from "react"; import { useAuthStore } from "../../../stores/authStore"; import { useExtractTasks } from "../hooks/useExtractTasks"; -import { useLiveTranscript } from "../hooks/useTranscript"; +import { useLiveTranscript, useTranscript } from "../hooks/useTranscript"; interface LiveTranscriptViewProps { posthogRecordingId: string; // PostHog UUID @@ -29,6 +29,7 @@ export function LiveTranscriptView({ const { segments, addSegment, forceUpload, pendingCount } = useLiveTranscript(posthogRecordingId); + const { data: transcriptData } = useTranscript(posthogRecordingId); const extractTasksMutation = useExtractTasks(); const handleExtractTasks = () => { @@ -246,7 +247,7 @@ export function LiveTranscriptView({ - {!openaiApiKey && ( + {!openaiApiKey && !transcriptData?.extracted_tasks && ( Add OpenAI API key in settings to extract tasks @@ -254,16 +255,10 @@ export function LiveTranscriptView({ )} - {extractTasksMutation.isSuccess && extractTasksMutation.data && ( - - {extractTasksMutation.data.tasks.length === 0 ? ( - - - No tasks found in transcript - - - ) : ( - extractTasksMutation.data.tasks.map((task, idx) => ( + {transcriptData?.extracted_tasks && + transcriptData.extracted_tasks.length > 0 && ( + + {transcriptData.extracted_tasks.map((task, idx) => ( @@ -274,10 +269,18 @@ export function LiveTranscriptView({ - )) - )} - - )} + ))} + + )} + + {transcriptData?.extracted_tasks && + transcriptData.extracted_tasks.length === 0 && ( + + + No tasks found in transcript + + + )} {extractTasksMutation.isError && ( diff --git a/src/renderer/features/notetaker/hooks/useExtractTasks.ts b/src/renderer/features/notetaker/hooks/useExtractTasks.ts index 82f7f3cb9..259cece64 100644 --- a/src/renderer/features/notetaker/hooks/useExtractTasks.ts +++ b/src/renderer/features/notetaker/hooks/useExtractTasks.ts @@ -8,6 +8,7 @@ export interface ExtractedTask { export function useExtractTasks() { const openaiApiKey = useAuthStore((state) => state.openaiApiKey); + const { client } = useAuthStore(); const queryClient = useQueryClient(); return useMutation({ @@ -22,15 +23,27 @@ export function useExtractTasks() { throw new Error("OpenAI API key not configured"); } + if (!client) { + throw new Error("Not authenticated"); + } + + // Extract tasks using AI const tasks = await window.electronAPI.notetakerExtractTasks( transcriptText, openaiApiKey, ); + // Upload tasks to backend + await client.uploadDesktopRecordingTranscript(recordingId, { + full_text: transcriptText, + segments: [], // Segments already uploaded + extracted_tasks: tasks, + }); + return { tasks, recordingId }; }, onSuccess: (data) => { - // Invalidate recording query to refetch with new tasks + // Invalidate queries to refetch with new tasks from backend queryClient.invalidateQueries({ queryKey: ["notetaker-recording", data.recordingId], }); diff --git a/src/renderer/features/notetaker/hooks/useTranscript.ts b/src/renderer/features/notetaker/hooks/useTranscript.ts index ba69f72de..b99c8a190 100644 --- a/src/renderer/features/notetaker/hooks/useTranscript.ts +++ b/src/renderer/features/notetaker/hooks/useTranscript.ts @@ -12,6 +12,10 @@ export interface TranscriptSegment { interface TranscriptData { full_text: string; segments: TranscriptSegment[]; + extracted_tasks?: Array<{ + title: string; + description: string; + }>; } export function useTranscript(recordingId: string | null) {