feat: add markdown formatting and session compacts to search results

apps/webapp/app/routes/api.v1.search.tsx

@@ -20,6 +20,7 @@ export const SearchBodyRequest = z.object({
   scoreThreshold: z.number().optional(),
   minResults: z.number().optional(),
   adaptiveFiltering: z.boolean().optional(),
+  structured: z.boolean().optional(),
 });
 
 const searchService = new SearchService();
@@ -47,6 +48,7 @@ const { action, loader } = createHybridActionApiRoute(
         minResults: body.minResults,
         spaceIds: body.spaceIds,
         adaptiveFiltering: body.adaptiveFiltering,
+        structured: body.structured,
       },
     );
     return json(results);

apps/webapp/app/services/search.server.ts

@@ -25,15 +25,20 @@ export class SearchService {
    * @param query The search query
    * @param userId The user ID for personalization
    * @param options Search options
-   * @returns Array of relevant statements
+   * @returns Markdown formatted context (default) or structured JSON (if structured: true)
    */
   public async search(
     query: string,
     userId: string,
     options: SearchOptions = {},
     source?: string,
-  ): Promise<{
+  ): Promise<string | {
-    episodes: {content: string; createdAt: Date; spaceIds: string[]}[];
+    episodes: {
+      content: string;
+      createdAt: Date;
+      spaceIds: string[];
+      isCompact?: boolean;
+    }[];
     facts: {
       fact: string;
       validAt: Date;
@@ -57,6 +62,7 @@ export class SearchService {
       minResults: options.minResults || 10,
       spaceIds: options.spaceIds || [],
       adaptiveFiltering: options.adaptiveFiltering || false,
+      structured: options.structured || false,
     };
 
     const queryVector = await this.getEmbedding(query);
@@ -107,21 +113,28 @@ export class SearchService {
       filteredResults.map((item) => item.statement),
     );
 
-    return {
+    // Replace session episodes with compacts automatically
-      episodes: episodes.map((episode) => ({
+    const unifiedEpisodes = await this.replaceWithCompacts(episodes, userId);
-        content: episode.originalContent,
+
-        createdAt: episode.createdAt,
+    const factsData = filteredResults.map((statement) => ({
-        spaceIds: episode.spaceIds || [],
-      })),
-      facts: filteredResults.map((statement) => ({
       fact: statement.statement.fact,
       validAt: statement.statement.validAt,
       invalidAt: statement.statement.invalidAt || null,
       relevantScore: statement.score,
-      })),
+    }));
+
+    // Return markdown by default, structured JSON if requested
+    if (opts.structured) {
+      return {
+        episodes: unifiedEpisodes,
+        facts: factsData,
       };
     }
 
+    // Return markdown formatted context
+    return this.formatAsMarkdown(unifiedEpisodes, factsData);
+  }
+
   /**
    * Apply adaptive filtering to ranked results
    * Uses a minimum quality threshold to filter out low-quality results
@@ -363,114 +376,175 @@ export class SearchService {
   }
 
   /**
-   * Enhanced search that includes compacted sessions
+   * Format search results as markdown for agent consumption
-   * This method searches both regular episodes/statements AND compacted sessions
-   * Compacted sessions are preferred when they have high confidence matches
    */
-  public async searchWithCompacts(
+  private formatAsMarkdown(
-    query: string,
+    episodes: Array<{
-    userId: string,
+      content: string;
-    options: SearchOptions = {},
+      createdAt: Date;
-    source?: string,
+      spaceIds: string[];
-  ): Promise<ExtendedSearchResult> {
+      isCompact?: boolean;
-    const startTime = Date.now();
+    }>,
+    facts: Array<{
+      fact: string;
+      validAt: Date;
+      invalidAt: Date | null;
+      relevantScore: number;
+    }>,
+  ): string {
+    const sections: string[] = [];
 
-    // First, run the standard search for episodes and facts
+    // Add episodes/compacts section
-    const standardResults = await this.search(query, userId, options, source);
+    if (episodes.length > 0) {
+      sections.push("## Recalled Relevant Context\n");
 
-    // Search compacted sessions
+      episodes.forEach((episode, index) => {
-    try {
+        const date = episode.createdAt.toLocaleString("en-US", {
-      const queryVector = await this.getEmbedding(query);
+          month: "short",
-      const { searchCompactedSessionsByEmbedding } = await import(
+          day: "numeric",
-        "~/services/graphModels/compactedSession"
+          year: "numeric",
-      );
+          hour: "2-digit",
-
+          minute: "2-digit",
-      const compactResults = await searchCompactedSessionsByEmbedding(
-        queryVector,
-        userId,
-        options.limit || 10,
-        options.scoreThreshold || 0.7,
-      );
-
-      logger.info(`Found ${compactResults.length} matching compacted sessions`, {
-        query,
-        userId,
         });
 
-      // Format compact results
+        if (episode.isCompact) {
-      const formattedCompacts = compactResults.map((result) => ({
+          sections.push(`### 📦 Session Compact`);
-        summary: result.compact.summary,
+          sections.push(`**Created**: ${date}\n`);
-        sessionId: result.compact.sessionId,
+          sections.push(episode.content);
-        episodeCount: result.compact.episodeCount,
+          sections.push(""); // Empty line
-        confidence: result.compact.confidence,
+        } else {
-        relevantScore: result.score,
+          sections.push(`### Episode ${index + 1}`);
-      }));
+          sections.push(`**Created**: ${date}`);
-
+          if (episode.spaceIds.length > 0) {
-      // Log compact recall
+            sections.push(`**Spaces**: ${episode.spaceIds.join(", ")}`);
-      await this.logCompactRecallAsync(
-        query,
-        userId,
-        compactResults,
-        Date.now() - startTime,
-        source,
-      ).catch((error) => {
-        logger.error("Failed to log compact recall event:", error);
-      });
-
-      return {
-        ...standardResults,
-        compacts: formattedCompacts,
-      };
-    } catch (error) {
-      logger.error("Error searching compacted sessions:", { error });
-      // Return standard results if compact search fails
-      return {
-        ...standardResults,
-        compacts: [],
-      };
           }
+          sections.push(""); // Empty line before content
+          sections.push(episode.content);
+          sections.push(""); // Empty line after
+        }
+      });
+    }
+
+    // Add facts section
+    if (facts.length > 0) {
+      sections.push("## Key Facts\n");
+
+      facts.forEach((fact) => {
+        const validDate = fact.validAt.toLocaleString("en-US", {
+          month: "short",
+          day: "numeric",
+          year: "numeric",
+        });
+        const invalidInfo = fact.invalidAt
+          ? ` → Invalidated ${fact.invalidAt.toLocaleString("en-US", { month: "short", day: "numeric", year: "numeric" })}`
+          : "";
+
+        sections.push(`- ${fact.fact}`);
+        sections.push(`  *Valid from ${validDate}${invalidInfo}*`);
+      });
+      sections.push(""); // Empty line after facts
+    }
+
+    // Handle empty results
+    if (episodes.length === 0 && facts.length === 0) {
+      sections.push("*No relevant memories found.*\n");
+    }
+
+    return sections.join("\n");
   }
 
   /**
-   * Log recall event for compacted sessions
+   * Replace session episodes with their compacted sessions
+   * Returns unified array with both regular episodes and compacts
    */
-  private async logCompactRecallAsync(
+  private async replaceWithCompacts(
-    query: string,
+    episodes: EpisodicNode[],
     userId: string,
-    compacts: Array<{ compact: any; score: number }>,
+  ): Promise<Array<{
-    responseTime: number,
+    content: string;
-    source?: string,
+    createdAt: Date;
-  ): Promise<void> {
+    spaceIds: string[];
-    try {
+    isCompact?: boolean;
-      const averageScore =
+  }>> {
-        compacts.length > 0
+    // Group episodes by sessionId
-          ? compacts.reduce((sum, c) => sum + c.score, 0) / compacts.length
+    const sessionEpisodes = new Map<string, EpisodicNode[]>();
-          : 0;
+    const nonSessionEpisodes: EpisodicNode[] = [];
 
-      await prisma.recallLog.create({
+    for (const episode of episodes) {
-        data: {
+      // Skip episodes with documentId (these are document chunks, not session episodes)
-          accessType: "search",
+      if (episode.metadata?.documentUuid) {
-          query,
+        nonSessionEpisodes.push(episode);
-          targetType: "compacted_session",
+        continue;
-          searchMethod: "vector_similarity",
+      }
-          resultCount: compacts.length,
+
-          similarityScore: averageScore,
+      // Episodes with sessionId - group them
-          context: JSON.stringify({
+      if (episode.sessionId) {
-            compactedSessionSearch: true,
+        if (!sessionEpisodes.has(episode.sessionId)) {
-          }),
+          sessionEpisodes.set(episode.sessionId, []);
-          source: source ?? "search_with_compacts",
+        }
-          responseTimeMs: responseTime,
+        sessionEpisodes.get(episode.sessionId)!.push(episode);
-          userId,
+      } else {
-        },
+        // No sessionId - keep as regular episode
+        nonSessionEpisodes.push(episode);
+      }
+    }
+
+    // Build unified result array
+    const result: Array<{
+      content: string;
+      createdAt: Date;
+      spaceIds: string[];
+      isCompact?: boolean;
+    }> = [];
+
+    // Add non-session episodes first
+    for (const episode of nonSessionEpisodes) {
+      result.push({
+        content: episode.originalContent,
+        createdAt: episode.createdAt,
+        spaceIds: episode.spaceIds || [],
+      });
+    }
+
+    // Check each session for compacts
+    const { getCompactedSessionBySessionId } = await import(
+      "~/services/graphModels/compactedSession"
+    );
+
+    const sessionIds = Array.from(sessionEpisodes.keys());
+
+    for (const sessionId of sessionIds) {
+      const sessionEps = sessionEpisodes.get(sessionId)!;
+      const compact = await getCompactedSessionBySessionId(sessionId, userId);
+
+      if (compact) {
+        // Compact exists - add compact as episode, skip original episodes
+        result.push({
+          content: compact.summary,
+          createdAt: compact.startTime, // Use session start time
+          spaceIds: [], // Compacts don't have spaceIds directly
+          isCompact: true,
         });
 
-      logger.debug(
+        logger.info(`Replaced ${sessionEps.length} episodes with compact`, {
-        `Logged compact recall event for user ${userId}: ${compacts.length} compacts in ${responseTime}ms`,
+          sessionId,
-      );
+          episodeCount: sessionEps.length,
-    } catch (error) {
+        });
-      logger.error("Error creating compact recall log entry:", { error });
+      } else {
+        // No compact - add original episodes
+        for (const episode of sessionEps) {
+          result.push({
+            content: episode.originalContent,
+            createdAt: episode.createdAt,
+            spaceIds: episode.spaceIds || [],
+          });
         }
       }
+    }
+
+    return result;
+  }
+
 }
 
 /**
@@ -489,24 +563,5 @@ export interface SearchOptions {
   minResults?: number;
   spaceIds?: string[]; // Filter results by specific spaces
   adaptiveFiltering?: boolean;
-}
+  structured?: boolean; // Return structured JSON instead of markdown (default: false)
-
-/**
- * Extended search result that includes compacted sessions
- */
-export interface ExtendedSearchResult {
-  episodes: { content: string; createdAt: Date; spaceIds: string[] }[];
-  facts: {
-    fact: string;
-    validAt: Date;
-    invalidAt: Date | null;
-    relevantScore: number;
-  }[];
-  compacts?: {
-    summary: string;
-    sessionId: string;
-    episodeCount: number;
-    confidence: number;
-    relevantScore: number;
-  }[];
 }

apps/webapp/app/trigger/session/session-compaction.ts

@@ -27,7 +27,7 @@ const CompactionResultSchema = z.object({
 });
 
 const CONFIG = {
-  minEpisodesForCompaction: 3, // Minimum episodes to trigger compaction
+  minEpisodesForCompaction: 5, // Minimum episodes to trigger compaction
   compactionThreshold: 1, // Trigger after N new episodes
   maxEpisodesPerBatch: 50, // Process in batches if needed
 };
@@ -52,10 +52,7 @@ export const sessionCompactionTask = task({
 
     try {
       // Check if compaction already exists
-      // const existingCompact = await getCompactedSessionBySessionId(sessionId, userId);
+      const existingCompact = await getCompactedSessionBySessionId(sessionId, userId);
-      const existingCompact = {} as CompactedSessionNode;
-
-      
 
       // Fetch all episodes for this session
       const episodes = await getSessionEpisodes(sessionId, userId, existingCompact?.endTime);
@@ -87,10 +84,9 @@ export const sessionCompactionTask = task({
       }
 
       // Generate or update compaction
-      const compactionResult = await createCompaction(sessionId, episodes, userId, source)
+      const compactionResult = existingCompact
-      // const compactionResult = existingCompact
+        ? await updateCompaction(existingCompact, episodes, userId)
-      //   ? await updateCompaction(existingCompact, episodes, userId)
+        : await createCompaction(sessionId, episodes, userId, source);
-      //   : await createCompaction(sessionId, episodes, userId, source);
 
       logger.info(`Session compaction completed`, {
         sessionId,

apps/webapp/app/utils/mcp/memory.ts

@@ -77,7 +77,7 @@ export const memoryTools = [
   {
     name: "memory_search",
     description:
-      "Search stored memories for past conversations, user preferences, project context, and decisions. USE THIS TOOL: 1) At start of every conversation to find related context, 2) When user mentions past work or projects, 3) Before answering questions that might have previous context. HOW TO USE: Write a simple query describing what to find (e.g., 'user code preferences', 'authentication bugs', 'API setup steps'). Returns: Episodes (past conversations) and Facts (extracted knowledge) as JSON.",
+      "Search stored memories for past conversations, user preferences, project context, and decisions. USE THIS TOOL: 1) At start of every conversation to find related context, 2) When user mentions past work or projects, 3) Before answering questions that might have previous context. HOW TO USE: Write a simple query describing what to find (e.g., 'user code preferences', 'authentication bugs', 'API setup steps'). Returns: Markdown-formatted context optimized for LLM consumption, including session compacts, episodes, and key facts with temporal metadata.",
     inputSchema: SearchParamsSchema,
   },
   {