Fix: improve knowledge graph and recall

apps/webapp/app/env.server.ts

@@ -80,8 +80,9 @@ const EnvironmentSchema = z.object({
 
   // Model envs
   MODEL: z.string().default(LLMModelEnum.GPT41),
-  EMBEDDING_MODEL: z.string().default("bge-m3"),
+  EMBEDDING_MODEL: z.string().default("mxbai-embed-large"),
   OLLAMA_URL: z.string().optional(),
+  COHERE_API_KEY: z.string().optional(),
 });
 
 export type Environment = z.infer<typeof EnvironmentSchema>;

apps/webapp/app/routes/api.v1.evaluate.tsx

@@ -0,0 +1,121 @@
+import { z } from "zod";
+import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server";
+import { makeModelCall } from "~/lib/model.server";
+import { json } from "@remix-run/node";
+
+export const EvaluateBodyRequest = z.object({
+  question: z.string(),
+  standard_answer: z.string(),
+  generated_answer: z.string(),
+});
+
+const { action, loader } = createActionApiRoute(
+  {
+    body: EvaluateBodyRequest,
+    allowJWT: true,
+    authorization: {
+      action: "search", // Using same permission as search
+    },
+    corsStrategy: "all",
+  },
+  async ({ body, authentication: _ }) => {
+    const { question, standard_answer, generated_answer } = body;
+
+    const evaluationPrompt = `Your task is to label an answer to a question as 'CORRECT' or 'WRONG'. You will be given the following data:
+    (1) a question (posed by one user to another user),
+    (2) a 'gold' (ground truth) answer,
+    (3) a generated answer
+which you will score as CORRECT/WRONG.
+
+The point of the question is to ask about something one user should know about the other user based on their prior conversations.
+The gold answer will usually be a concise and short answer that includes the referenced topic, for example:
+Question: Do you remember what I got the last time I went to Hawaii?
+Gold answer: A shell necklace
+The generated answer might be much longer, but you should be generous with your grading - as long as it touches on the same topic as the gold answer, it should be counted as CORRECT.
+
+For time related questions, the gold answer will be a specific date, month, year, etc. The generated answer might be much longer or use relative time references (like "last Tuesday" or "next month"), but you should be generous with your grading - as long as it refers to the same date or time period as the gold answer, it should be counted as CORRECT. Even if the format differs (e.g., "May 7th" vs "7 May"), consider it CORRECT if it's the same date.
+
+Now it's time for the real question:
+Question: ${question}
+Gold answer: ${standard_answer}
+Generated answer: ${generated_answer}
+
+First, provide a short (one sentence) explanation of your reasoning, then finish with CORRECT or WRONG.
+Do NOT include both CORRECT and WRONG in your response, or it will break the evaluation script.
+
+Just return the label CORRECT or WRONG in a json format with the key as "label".`;
+
+    try {
+      // Use the LLM to evaluate the answer
+      const llmResponse = await makeModelCall(
+        false, // Don't stream
+        [{ role: "user", content: evaluationPrompt }],
+        (_text: string, _model: string) => {
+          // onFinish callback - we can log model usage here if needed
+        }
+      ) as string;
+
+      // Parse the LLM response to extract the label
+      const response = llmResponse.trim();
+      let label = "WRONG";
+      let reasoning = response;
+
+      // Try to parse as JSON first
+      try {
+        const jsonResponse = JSON.parse(response);
+        if (jsonResponse.label && (jsonResponse.label === "CORRECT" || jsonResponse.label === "WRONG")) {
+          label = jsonResponse.label;
+          reasoning = jsonResponse.reasoning || response;
+        }
+      } catch (jsonError) {
+        // If not JSON, look for CORRECT/WRONG in the text
+        if (response.includes("CORRECT") && !response.includes("WRONG")) {
+          label = "CORRECT";
+        } else if (response.includes("WRONG") && !response.includes("CORRECT")) {
+          label = "WRONG";
+        }
+        // Extract reasoning (everything before the final CORRECT/WRONG)
+        const parts = response.split(/(CORRECT|WRONG)$/);
+        if (parts.length > 1) {
+          reasoning = parts[0].trim();
+        }
+      }
+
+      // Calculate match ratio for additional metrics
+      const generatedLower = generated_answer.toLowerCase();
+      const standardLower = standard_answer.toString().toLowerCase();
+      const standardWords = standardLower.split(/\s+/).filter(word => word.length > 2);
+      const matchingWords = standardWords.filter(word => generatedLower.includes(word));
+      const matchRatio = standardWords.length > 0 ? matchingWords.length / standardWords.length : 0;
+
+      return json({
+        label: label,
+        reasoning: reasoning,
+        matchRatio: matchRatio,
+        method: "llm"
+      });
+
+    } catch (error) {
+      console.error("Error in LLM evaluation:", error);
+      
+      // Fallback to heuristic evaluation
+      const generatedLower = generated_answer.toLowerCase();
+      const standardLower = standard_answer.toString().toLowerCase();
+      
+      const standardWords = standardLower.split(/\s+/).filter(word => word.length > 2);
+      const matchingWords = standardWords.filter(word => generatedLower.includes(word));
+      const matchRatio = standardWords.length > 0 ? matchingWords.length / standardWords.length : 0;
+      
+      const isCorrect = matchRatio > 0.3; // If 30% of important words match
+      
+      return json({
+        label: isCorrect ? "CORRECT" : "WRONG",
+        reasoning: `Generated answer ${isCorrect ? 'contains' : 'does not contain'} sufficient matching content with the gold standard (${matchRatio.toFixed(2)} match ratio)`,
+        matchRatio: matchRatio,
+        method: "heuristic_fallback"
+      });
+    }
+  },
+);
+
+export { action, loader };

apps/webapp/app/routes/api.v1.qa.tsx

@@ -0,0 +1,139 @@
+import { z } from "zod";
+import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server";
+import { SearchService } from "~/services/search.server";
+import { makeModelCall } from "~/lib/model.server";
+import { json } from "@remix-run/node";
+
+export const QABodyRequest = z.object({
+  question: z.string(),
+  startTime: z.string().optional(),
+  endTime: z.string().optional(),
+  spaceId: z.string().optional(),
+  limit: z.number().optional(),
+  maxBfsDepth: z.number().optional(),
+  includeInvalidated: z.boolean().optional(),
+  entityTypes: z.array(z.string()).optional(),
+  scoreThreshold: z.number().optional(),
+  minResults: z.number().optional(),
+});
+
+const searchService = new SearchService();
+const { action, loader } = createActionApiRoute(
+  {
+    body: QABodyRequest,
+    allowJWT: true,
+    authorization: {
+      action: "search",
+    },
+    corsStrategy: "all",
+  },
+  async ({ body, authentication }) => {
+    // First, search for relevant information
+    const searchResults = await searchService.search(
+      body.question,
+      authentication.userId,
+      {
+        startTime: body.startTime ? new Date(body.startTime) : undefined,
+        endTime: body.endTime ? new Date(body.endTime) : undefined,
+        limit: body.limit || 20, // Get more results for better context
+        maxBfsDepth: body.maxBfsDepth,
+        includeInvalidated: body.includeInvalidated,
+        entityTypes: body.entityTypes,
+        scoreThreshold: body.scoreThreshold,
+        minResults: body.minResults,
+      },
+    );
+
+    // Combine episodes and facts into context
+    let context = [...searchResults.episodes].join("\n\n");
+
+    searchResults.facts.map((fact) => {
+      context += `\n\nfact: ${fact.fact}\n validAt: ${fact.validAt}`;
+    });
+
+    // console.log("Context:", context);
+
+    if (!context.trim()) {
+      return json({
+        question: body.question,
+        generated_answer: "I couldn't find any relevant information to answer this question.",
+      });
+    }
+
+    // Generate answer using LLM
+    const prompt = `You are an analytical AI that reasons deeply about context before answering questions. Your task is to:
+
+1. FIRST: Look for direct, explicit answers in the context
+2. ANALYZE the context thoroughly for relevant information
+3. IDENTIFY patterns, connections, and implications 
+4. REASON about what the context suggests or implies
+5. ANSWER based on direct evidence OR analysis
+
+<reasoning>
+- Scan through ALL episodes and facts completely before answering
+- Look for every explicit statement that relates to the question
+- NEVER stop after finding the first answer - continue scanning for more
+- When asking "what did X show Y", look for ALL items X showed Y on that date
+- Collect multiple items, events, or details that answer the same question
+- If not found directly, identify all context elements related to the question
+- Look for patterns, themes, and implicit information in the context
+- Consider what the context suggests beyond explicit statements
+- Note any contradictions or missing information that affects the answer
+- Pay close attention to temporal information and dates (validAt timestamps)
+- For time-sensitive questions, prioritize more recent information
+- Consider the chronological sequence of events when relevant
+- CRITICAL: Ensure completeness by including ALL relevant items found
+- If you find 2+ items for the same question, mention them all in your answer
+- Be precise with details (specific types, colors, descriptions when available)
+- Draw logical conclusions based on available evidence
+- Don't give reasoning in the output
+</reasoning>
+
+Follow this output format. don't give the JSON with \`\`\`json
+<output>
+{"answer" : "Your direct, short(max 2 sentences) answer based on your analysis"}
+</output>
+`;
+
+    const userPrompt = `<context>
+    ${context}
+    </context>
+
+    <question>
+    Question: ${body.question}
+    </question>
+    `;
+    let responseText = "";
+    let generated_answer = "";
+    try {
+      await makeModelCall(
+        false, // Don't stream
+        [{ role: "system", content: prompt }, { role: "user", content: userPrompt }],
+        (text) => {
+          responseText = text;
+        }
+      );
+
+      const outputMatch = responseText.match(/<output>([\s\S]*?)<\/output>/);
+      if (outputMatch && outputMatch[1]) {
+        try {
+          const parsedOutput = JSON.parse(outputMatch[1].trim());
+          generated_answer = parsedOutput.answer || "No answer provided";
+        } catch (jsonError) {
+          console.error("Error parsing JSON output:", jsonError);
+          generated_answer = outputMatch[1].trim();
+        }
+      }
+    } catch (error) {
+      console.error("Error generating answer:", error);
+      generated_answer = "I encountered an error while generating an answer to this question.";
+    }
+
+    return json({
+      question: body.question,
+      generated_answer,
+    });
+  },
+);
+
+export { action, loader };

apps/webapp/app/services/knowledgeGraph.server.ts

@@ -451,13 +451,32 @@ export class KnowledgeGraphService {
         const predicateNode = predicateMap.get(triple.predicate.toLowerCase());
 
         if (subjectNode && objectNode && predicateNode) {
+          // Determine the correct validAt date (when the fact actually occurred/occurs)
+          let validAtDate = episode.validAt; // Default fallback to episode date
+
+          // Check if statement has event_date indicating when the fact actually happened/happens
+          if (triple.attributes?.event_date) {
+            try {
+              const eventDate = new Date(triple.attributes.event_date);
+              // Use the event date as validAt (when the fact is actually true)
+              if (!isNaN(eventDate.getTime())) {
+                validAtDate = eventDate;
+              }
+            } catch (error) {
+              // If parsing fails, use episode validAt as fallback
+              logger.log(
+                `Failed to parse event_date: ${triple.attributes.event_date}, using episode validAt`,
+              );
+            }
+          }
+
           // Create a statement node
           const statement: StatementNode = {
             uuid: crypto.randomUUID(),
             fact: triple.fact,
             factEmbedding: factEmbeddings[tripleIndex],
             createdAt: new Date(),
-            validAt: episode.validAt,
+            validAt: validAtDate,
             invalidAt: null,
             attributes: triple.attributes || {},
             userId: episode.userId,
@@ -1145,7 +1164,8 @@ export class KnowledgeGraphService {
       source,
       relatedMemories,
       ingestionRules,
-      episodeTimestamp: episodeTimestamp?.toISOString(),
+      episodeTimestamp:
+        episodeTimestamp?.toISOString() || new Date().toISOString(),
       sessionContext,
     };
     const messages = normalizePrompt(context);
@@ -1157,6 +1177,30 @@ export class KnowledgeGraphService {
     const outputMatch = responseText.match(/<output>([\s\S]*?)<\/output>/);
     if (outputMatch && outputMatch[1]) {
       normalizedEpisodeBody = outputMatch[1].trim();
+    } else {
+      // Log format violation and use fallback
+      logger.warn("Normalization response missing <output> tags", {
+        responseText: responseText.substring(0, 200) + "...",
+        source,
+        episodeLength: episodeBody.length,
+      });
+
+      // Fallback: use raw response if it's not empty and seems meaningful
+      const trimmedResponse = responseText.trim();
+      if (
+        trimmedResponse &&
+        trimmedResponse !== "NOTHING_TO_REMEMBER" &&
+        trimmedResponse.length > 10
+      ) {
+        normalizedEpisodeBody = trimmedResponse;
+        logger.info("Using raw response as fallback for normalization", {
+          fallbackLength: trimmedResponse.length,
+        });
+      } else {
+        logger.warn("No usable normalization content found", {
+          responseText: responseText,
+        });
+      }
     }
 
     return normalizedEpisodeBody;

apps/webapp/app/services/prompts/nodes.ts

@@ -34,7 +34,8 @@ You are given a conversation context and a CURRENT EPISODE. Your task is to extr
 
 3. **Exclusions**:
    - Do NOT extract entities representing relationships or actions (predicates will be handled separately).
-   - Do NOT extract dates, times, or other temporal information—these will be handled separately.
+   - Do NOT extract absolute dates, timestamps, or specific time points—these will be handled separately.
+   - Do NOT extract relative time expressions that resolve to specific dates ("last week", "yesterday", "3pm").
 
 4. **Entity Name Extraction**:
    - Extract ONLY the core entity name, WITHOUT any type descriptors or qualifiers
@@ -45,9 +46,37 @@ You are given a conversation context and a CURRENT EPISODE. Your task is to extr
    - **FULL NAMES**: Use complete names when available (e.g., "John Smith" not "John")
    - **NO TYPE SUFFIXES**: Never append the entity type to the entity name
 
+5. **Temporal and Relationship Context Extraction**:
+   - EXTRACT duration expressions that describe relationship spans ("4 years", "2 months", "5 years")
+   - EXTRACT temporal context that anchors relationships ("since moving", "after graduation", "during college")
+   - EXTRACT relationship qualifiers ("close friends", "support system", "work team", "family members")
+   - DO NOT extract absolute dates, timestamps, or specific time points ("June 9, 2023", "3pm", "last Saturday")
+   - DO NOT extract relative time expressions that resolve to specific dates ("last week", "yesterday")
+
 ## Examples of Correct Entity Extraction:
 
-**CORRECT Examples:**
+**TEMPORAL INFORMATION - What to EXTRACT vs EXCLUDE:**
+
+✅ **EXTRACT - Relationship Temporal Information:**
+- Text: "I've known these friends for 4 years" → Extract: "4 years" (Duration)
+- Text: "since I moved from my home country" → Extract: "since moving" (TemporalContext)  
+- Text: "after that tough breakup" → Extract: "after breakup" (TemporalContext)
+- Text: "we've been married for 5 years" → Extract: "5 years" (Duration)
+- Text: "during college" → Extract: "during college" (TemporalContext)
+
+❌ **EXCLUDE - Absolute Dates/Times:**
+- Text: "on June 9, 2023" → Don't extract "June 9, 2023" 
+- Text: "last Saturday" → Don't extract "last Saturday"
+- Text: "at 3pm yesterday" → Don't extract "3pm" or "yesterday"
+- Text: "next week" → Don't extract "next week"
+
+**RELATIONSHIP CONTEXT ENTITIES:**
+- Text: "my close friends" → Extract: "close friends" (QualifiedGroup)
+- Text: "strong support system" → Extract: "support system" (RelationshipType)
+- Text: "work colleagues" → Extract: "work colleagues" (ProfessionalGroup)
+- Text: "family members" → Extract: "family members" (FamilyGroup)
+
+**STANDARD ENTITY EXTRACTION:**
 - Text: "Tesla car" → Name: "Tesla", Type: "Vehicle"
 - Text: "Google's search engine" → Name: "Google", Type: "Company" + Name: "Search Engine", Type: "Product"
 - Text: "Microsoft Office suite" → Name: "Microsoft Office", Type: "Software"
@@ -123,7 +152,8 @@ You are given a TEXT. Your task is to extract **entity nodes** mentioned **expli
 
 3. **Exclusions**:
    - Do NOT extract entities representing relationships or actions (predicates will be handled separately).
-   - Do NOT extract dates, times, or other temporal information—these will be handled separately.
+   - Do NOT extract absolute dates, timestamps, or specific time points—these will be handled separately.
+   - Do NOT extract relative time expressions that resolve to specific dates ("last week", "yesterday", "3pm").
 
 4. **Entity Name Extraction**:
    - Extract ONLY the core entity name, WITHOUT any type descriptors or qualifiers
@@ -134,9 +164,37 @@ You are given a TEXT. Your task is to extract **entity nodes** mentioned **expli
    - **FULL NAMES**: Use complete names when available (e.g., "John Smith" not "John")
    - **NO TYPE SUFFIXES**: Never append the entity type to the entity name
 
+5. **Temporal and Relationship Context Extraction**:
+   - EXTRACT duration expressions that describe relationship spans ("4 years", "2 months", "5 years")
+   - EXTRACT temporal context that anchors relationships ("since moving", "after graduation", "during college")
+   - EXTRACT relationship qualifiers ("close friends", "support system", "work team", "family members")
+   - DO NOT extract absolute dates, timestamps, or specific time points ("June 9, 2023", "3pm", "last Saturday")
+   - DO NOT extract relative time expressions that resolve to specific dates ("last week", "yesterday")
+
 ## Examples of Correct Entity Extraction:
 
-**CORRECT Examples:**
+**TEMPORAL INFORMATION - What to EXTRACT vs EXCLUDE:**
+
+✅ **EXTRACT - Relationship Temporal Information:**
+- Text: "I've known these friends for 4 years" → Extract: "4 years" (Duration)
+- Text: "since I moved from my home country" → Extract: "since moving" (TemporalContext)  
+- Text: "after that tough breakup" → Extract: "after breakup" (TemporalContext)
+- Text: "we've been married for 5 years" → Extract: "5 years" (Duration)
+- Text: "during college" → Extract: "during college" (TemporalContext)
+
+❌ **EXCLUDE - Absolute Dates/Times:**
+- Text: "on June 9, 2023" → Don't extract "June 9, 2023" 
+- Text: "last Saturday" → Don't extract "last Saturday"
+- Text: "at 3pm yesterday" → Don't extract "3pm" or "yesterday"
+- Text: "next week" → Don't extract "next week"
+
+**RELATIONSHIP CONTEXT ENTITIES:**
+- Text: "my close friends" → Extract: "close friends" (QualifiedGroup)
+- Text: "strong support system" → Extract: "support system" (RelationshipType)
+- Text: "work colleagues" → Extract: "work colleagues" (ProfessionalGroup)
+- Text: "family members" → Extract: "family members" (FamilyGroup)
+
+**STANDARD ENTITY EXTRACTION:**
 - Text: "Tesla car" → Name: "Tesla", Type: "Vehicle"
 - Text: "Google's search engine" → Name: "Google", Type: "Company" + Name: "Search Engine", Type: "Product"
 - Text: "Microsoft Office suite" → Name: "Microsoft Office", Type: "Software"
@@ -174,7 +232,6 @@ ${JSON.stringify(context.entityTypes || {}, null, 2)}
     { role: "user", content: userPrompt },
   ];
 };
-
 /**
  * Extract entities from an episode using JSON-based approach
  */

apps/webapp/app/services/prompts/normalize.ts

@@ -3,229 +3,235 @@ import { type CoreMessage } from "ai";
 export const normalizePrompt = (
   context: Record<string, any>,
 ): CoreMessage[] => {
-  const sysPrompt = `
-You are C.O.R.E. (Contextual Observation & Recall Engine), a memory extraction system. Convert input information into clear, concise, third-person factual statements that EVOLVE the memory graph by forming new relationships and capturing new information.
+  const sysPrompt = `You are C.O.R.E. (Contextual Observation & Recall Engine), a smart memory enrichment system.
 
-## Core Processing Philosophy
-When related memories are provided, make memory graph evolution your PRIMARY GOAL, NOT information storage:
-- **EVOLVE**: Focus on new information that adds relationships or updates existing knowledge
-- **CONNECT**: Form explicit relationships between new and existing information
-- **FILTER**: Aggressively exclude information already captured in related memories
-- **ENHANCE**: Use existing knowledge to clarify new information and form connections
+Create ONE enriched sentence that transforms the episode into a contextually-rich memory using SELECTIVE enrichment.
 
-## Memory Processing Guidelines
-- Output all memory statements in the third person (e.g., "User prefers...", "The assistant performed...", "The system detected...").
-- Convert input information into clear, concise memory statements.
-- Maintain a neutral, factual tone in all memory entries.
-- Structure memories as factual statements, not questions.
-- Include relevant context and temporal information when available.
-- When ingesting from assistant's perspective, capture the complete user-assistant interaction context.
+<smart_enrichment_process>
+Evaluate the episode and apply enrichment ONLY where it adds significant value:
 
-## Temporal Resolution
-When processing episodes with relative time references, resolve them to absolute dates based on the episode timestamp:
-- "yesterday" → resolve to the day before the episode date
-- "today" → resolve to the episode date
-- "last week" → resolve to the week before the episode date
-- "two days ago" → resolve to two days before the episode date
-- "this morning/afternoon/evening" → resolve to the episode date with time context
+1. PRIMARY FACTS - always preserve the core information from the episode
+2. TEMPORAL RESOLUTION - convert relative dates to absolute dates using episode timestamp  
+3. STRATEGIC ENRICHMENT - add context only for HIGH VALUE cases (see guidelines below)
+4. VISUAL CONTENT - capture exact text on signs, objects shown, specific details from images
+5. EMOTIONAL PRESERVATION - maintain the tone and feeling of emotional exchanges
+6. IDENTITY PRESERVATION - preserve definitional and possessive relationships that establish entity connections
 
-Include these resolved dates in the extracted statements for precise temporal information.
-Example: If episode is from May 8th, 2024, and content mentions "yesterday", convert to "on May 7th, 2024".
+ENRICHMENT DECISION MATRIX:
+- Clear, complete statement → minimal enrichment (just temporal + attribution)
+- Unclear references → resolve with context
+- Emotional support → preserve feeling, avoid historical dumping
+- New developments → connect to ongoing narrative
+- Visual content → extract specific details as primary facts
+</smart_enrichment_process>
 
-## Complete Conversational Context
-- IMPORTANT: Preserve the complete context of conversations, including BOTH:
-  - What the user said, asked, or requested
-  - How the assistant responded or what it suggested
-  - Any decisions, conclusions, or agreements reached
-- Do not focus solely on the assistant's contributions while ignoring user context
-- Capture the cause-and-effect relationship between user inputs and assistant responses
-- For multi-turn conversations, preserve the logical flow and key points from each turn
-- When the user provides information, record that information directly, not just how the assistant used it
+<context_usage_decision>
+When related memories/previous episodes are provided, evaluate if they improve understanding:
 
-## Node Entity Types
+USE CONTEXT when current episode has:
+- Unclear pronouns ("she", "it", "they" without clear antecedent)
+- Vague references ("the agency", "the event" without definition in current episode)
+- Continuation phrases ("following up", "as we discussed")
+- Incomplete information that context clarifies
+
+IGNORE CONTEXT when current episode is:
+- Clear and self-contained ("I got a job in New York")
+- Simple emotional responses ("Thanks, that's great!")
+- Generic encouragement ("You're doing awesome!")
+- Complete statements with all necessary information
+
+DECISION RULE: If the current episode can be understood perfectly without context, don't use it. Only use context when it genuinely clarifies or
+resolves ambiguity.
+</context_usage_decision>
+
+<temporal_resolution>
+Using episode timestamp as anchor, convert ALL relative time references:
+- "yesterday" → calculate exact date (e.g., "June 26, 2023")
+- "last week" → date range (e.g., "around June 19-25, 2023")
+- "next month" → future date (e.g., "July 2023")
+- "recently" → approximate timeframe with uncertainty
+</temporal_resolution>
+
+<visual_content_capture>
+For episodes with images/photos, EXTRACT:
+- Exact text on signs, posters, labels (e.g., "Trans Lives Matter")
+- Objects, people, settings, activities shown
+- Specific visual details that add context
+Integrate visual content as primary facts, not descriptions.
+</visual_content_capture>
+
+<strategic_enrichment>
+When related memories are provided, apply SELECTIVE enrichment:
+
+HIGH VALUE ENRICHMENT (always include):
+- Temporal resolution: "last week" → "June 20, 2023"
+- Entity disambiguation: "she" → "Caroline" when unclear
+- Missing critical context: "the agency" → "Bright Futures Adoption Agency" (first mention only)
+- New developments: connecting current facts to ongoing storylines
+- Identity-defining possessives: "my X, Y" → preserve the relationship between person and Y as their X
+- Definitional phrases: maintain the defining relationship, not just the entity reference
+- Origin/source connections: preserve "from my X" relationships
+
+LOW VALUE ENRICHMENT (usually skip):
+- Obvious references: "Thanks, Mel!" doesn't need Melanie's full context
+- Support/encouragement statements: emotional exchanges rarely need historical anchoring
+- Already clear entities: don't replace pronouns when reference is obvious
+- Repetitive context: never repeat the same descriptive phrase within a conversation
+- Ongoing conversations: don't re-establish context that's already been set
+- Emotional responses: keep supportive statements simple and warm
+- Sequential topics: reference previous topics minimally ("recent X" not full description)
+
+ANTI-BLOAT RULES:
+- If the original statement is clear and complete, add minimal enrichment
+- Never use the same contextual phrase twice in one conversation
+- Focus on what's NEW, not what's already established
+- Preserve emotional tone - don't bury feelings in facts
+- ONE CONTEXT REFERENCE PER TOPIC: Don't keep referencing "the charity race" with full details
+- STOP AT CLARITY: If original meaning is clear, don't add backstory
+- AVOID COMPOUND ENRICHMENT: Don't chain multiple contextual additions in one sentence
+
+CONTEXT FATIGUE PREVENTION:
+- After mentioning a topic once with full context, subsequent references should be minimal
+- Use "recent" instead of repeating full details: "recent charity race" not "the May 20, 2023 charity race for mental health"
+- Focus on CURRENT episode facts, not historical anchoring
+- Don't re-explain what's already been established in the conversation
+
+ENRICHMENT SATURATION RULE:
+Once a topic has been enriched with full context in the conversation, subsequent mentions should be minimal:
+- First mention: "May 20, 2023 charity race for mental health"
+- Later mentions: "the charity race" or "recent race"
+- Don't re-explain established context
+
+IDENTITY AND DEFINITIONAL RELATIONSHIP PRESERVATION:
+- Preserve possessive phrases that define relationships: "my X, Y" → "Y, [person]'s X"
+- Keep origin/source relationships: "from my X" → preserve the X connection
+- Preserve family/professional/institutional relationships expressed through possessives
+- Don't reduce identity-rich phrases to simple location/entity references
+</strategic_enrichment>
+
+<entity_types>
 ${context.entityTypes}
+</entity_types>
 
-## Ingestion Rules
-${context.ingestionRules ? `The following rules apply to content from ${context.source}:
+<ingestion_rules>
+${
+  context.ingestionRules
+    ? `Apply these rules for content from ${context.source}:
 ${context.ingestionRules}
 
-IMPORTANT: If the content does NOT satisfy these rules, respond with "NOTHING_TO_REMEMBER" regardless of other criteria.` : 'No specific ingestion rules defined for this source.'}
+CRITICAL: If content does NOT satisfy these rules, respond with "NOTHING_TO_REMEMBER" regardless of other criteria.`
+    : "No specific ingestion rules defined for this source."
+}
+</ingestion_rules>
 
-## Related Memory Processing Strategy
-When related memories are provided, apply this filtering and enhancement strategy:
+<quality_control>
+RETURN "NOTHING_TO_REMEMBER" if content consists ONLY of:
+- Pure generic responses without context ("awesome", "thanks", "okay" with no subject)
+- Empty pleasantries with no substance ("how are you", "have a good day")
+- Standalone acknowledgments without topic reference ("got it", "will do")
+- Truly vague encouragement with no specific subject matter ("great job" with no context)
+- Already captured information without new connections
+- Technical noise or system messages
 
-### 1. INFORMATION FILTERING (What NOT to Include)
-- **Already Captured Facts**: Do not repeat information already present in related memories unless it adds new context
-- **Static Relationships**: Skip relationships already established (e.g., "John is co-founder" if already captured)
-- **Redundant Details**: Exclude details that don't add new understanding or connections
-- **Background Context**: Filter out explanatory information that's already in the memory graph
+STORE IN MEMORY if content contains:
+- Specific facts, names, dates, or detailed information
+- Personal details, preferences, or decisions
+- Concrete plans, commitments, or actions
+- Visual content with specific details
+- Temporal information that can be resolved
+- New connections to existing knowledge
+- Encouragement that references specific activities or topics
+- Statements expressing personal values or beliefs
+- Support that's contextually relevant to ongoing conversations
+- Responses that reveal relationship dynamics or personal characteristics
 
-### 2. RELATIONSHIP FORMATION (What TO Include)
-- **New Connections**: Include explicit relationships between entities mentioned in current and related episodes
-- **Evolving Relationships**: Capture changes or updates to existing relationships
-- **Cross-Context Links**: Form connections that bridge different contexts or time periods
-- **Causal Relationships**: Extract how current information affects or is affected by existing knowledge
+MEANINGFUL ENCOURAGEMENT EXAMPLES (STORE these):
+- "Taking time for yourself is so important" → Shows personal values about self-care
+- "You're doing an awesome job looking after yourself and your family" → Specific topic reference
+- "That charity race sounds great" → Contextually relevant support
+- "Your future family is gonna be so lucky" → Values-based encouragement about specific situation
 
-### 3. NEW INFORMATION EXTRACTION (Priority Focus)
-- **Fresh Facts**: Extract information not present in any related memory
-- **Updated Status**: Capture changes to previously captured information
-- **New Attributes**: Add additional properties or characteristics of known entities
-- **Temporal Updates**: Record time-based changes or progressions
-- **Contextual Additions**: Include new contexts or situations involving known entities
+EMPTY ENCOURAGEMENT EXAMPLES (DON'T STORE these):
+- "Great job!" (no context)
+- "Awesome!" (no subject)
+- "Keep it up!" (no specific reference)
+</quality_control>
 
-### 4. MEMORY GRAPH EVOLUTION PATTERNS
-- **Entity Enhancement**: Add new properties to existing entities without repeating known ones
-- **Relationship Expansion**: Create new relationship types between known entities
-- **Network Growth**: Connect previously isolated memory clusters
-- **Knowledge Refinement**: Update or correct existing information with new insights
+<enrichment_examples>
+HIGH VALUE enrichment:
+- Original: "She said yes!" 
+- Enriched: "On June 27, 2023, Caroline received approval from Bright Futures Agency for her adoption application."
+- Why: Resolves unclear pronoun, adds temporal context, identifies the approving entity
 
-## Memory Selection Criteria
-Evaluate conversations using these priority categories:
+MINIMAL enrichment (emotional support):
+- Original: "You'll be an awesome mom! Good luck!"
+- Enriched: "On May 25, 2023, Melanie encouraged Caroline about her adoption plans, affirming she would be an awesome mother."
+- Why: Simple temporal context, preserve emotional tone, no historical dumping
 
-### 1. High Priority (Always Remember)
-- **User Preferences**: Explicit likes, dislikes, settings, or preferences
-- **Personal Information**: Names, relationships, contact details, important dates
-- **Commitments**: Promises, agreements, or obligations made by either party
-- **Recurring Patterns**: Regular activities, habits, or routines mentioned
-- **Explicit Instructions**: "Remember X" or "Don't forget about Y" statements
-- **Important Decisions**: Key choices or conclusions reached
+ANTI-BLOAT example (what NOT to do):
+- Wrong: "On May 25, 2023, Melanie praised Caroline for her commitment to creating a family for children in need through adoption—supported by the inclusive Adoption Agency whose brochure and signs reading 'new arrival' and 'information and domestic building' Caroline had shared earlier that day—and encouraged her by affirming she would be an awesome mom."
+- Right: "On May 25, 2023, Melanie encouraged Caroline about her adoption plans, affirming she would be an awesome mother."
 
-### 2. Medium Priority (Remember if Significant)
-- **Task Context**: Background information relevant to ongoing tasks
-- **Problem Statements**: Issues or challenges the user is facing
-- **Learning & Growth**: Skills being developed, topics being studied
-- **Emotional Responses**: Strong reactions to suggestions or information
-- **Time-Sensitive Information**: Details that will be relevant for a limited period
+CLEAR REFERENCE (minimal enrichment):
+- Original: "Thanks, Caroline! The event was really thought-provoking."
+- Enriched: "On May 25, 2023, Melanie thanked Caroline and described the charity race as thought-provoking."
+- Why: Clear context doesn't need repetitive anchoring
 
-### 3. Low Priority (Rarely Remember)
-- **Casual Exchanges**: Greetings, acknowledgments, or social pleasantries
-- **Clarification Questions**: Questions asked to understand instructions
-- **Immediate Task Execution**: Simple commands and their direct execution
-- **Repeated Information**: Content already stored in memory
-- **Ephemeral Context**: Information only relevant to the current exchange
+CONVERSATION FLOW EXAMPLES:
+❌ WRONG (context fatigue): "reinforcing their ongoing conversation about mental health following Melanie's participation in the recent charity race for mental health"
+✅ RIGHT (minimal reference): "reinforcing their conversation about mental health"
 
-### 4. Do Not Remember (Forgettable Conversations)
-#### Transient Interactions
-- **Simple acknowledgments**: "Thanks", "OK", "Got it"
-- **Greetings and farewells**: "Hello", "Good morning", "Goodbye", "Talk to you later"
-- **Filler conversations**: Small talk about weather with no specific preferences mentioned
-- **Routine status updates** without meaningful information: "Still working on it"
+❌ WRONG (compound enrichment): "as she begins the process of turning her dream of giving children a loving home into reality and considers specific adoption agencies"
+✅ RIGHT (focused): "as she begins pursuing her adoption plans"
 
-#### Redundant Information
-- **Repeated requests** for the same information within a short timeframe
-- **Clarifications** that don't add new information: "What did you mean by that?"
-- **Confirmations** of already established facts: "Yes, as I mentioned earlier..."
-- **Information already stored** in memory in the same or similar form
+❌ WRONG (over-contextualization): "following her participation in the May 20, 2023 charity race for mental health awareness"
+✅ RIGHT (after first mention): "following the recent charity race"
 
-#### Temporary Operational Exchanges
-- **System commands** without context: "Open this file", "Run this code"
-- **Simple navigational instructions**: "Go back", "Scroll down"
-- **Format adjustments**: "Make this bigger", "Change the color"
-- **Immediate task execution** without long-term relevance
+GENERIC IDENTITY PRESERVATION EXAMPLES:
+- Original: "my hometown, Boston" → Enriched: "Boston, [person]'s hometown" 
+- Original: "my workplace, Google" → Enriched: "Google, [person]'s workplace"
+- Original: "my sister, Sarah" → Enriched: "Sarah, [person]'s sister"
+- Original: "from my university, MIT" → Enriched: "from MIT, [person]'s university"
 
-#### Low-Information Content
-- **Vague statements** without specific details: "That looks interesting"
-- **Ambiguous questions** that were later clarified in the conversation
-- **Incomplete thoughts** that were abandoned or redirected
-- **Hypothetical scenarios** that weren't pursued further
+POSSESSIVE + APPOSITIVE PATTERNS (Critical for Relations):
+- Original: "my colleague at my office, Microsoft" 
+- Enriched: "his colleague at Microsoft, David's workplace"
+- Why: Preserves both the work relationship AND the employment identity
 
-#### Technical Noise
-- **Error messages** or technical issues that were resolved
-- **Connection problems** or temporary disruptions
-- **Interface feedback**: "Loading...", "Processing complete"
-- **Formatting issues** that were corrected
+- Original: "my friend from my university, Stanford"
+- Enriched: "her friend from Stanford, Lisa's alma mater"
+- Why: Establishes both the friendship and educational institution identity
 
-#### Context-Dependent Ephemera
-- **Time-sensitive information** that quickly becomes irrelevant: "I'll be back in 5 minutes"
-- **Temporary states**: "I'm currently looking at the document"
-- **Attention-directing statements** without content: "Look at this part"
-- **Intermediate steps** in a process where only the conclusion matters
+- Original: "my neighbor in my city, Chicago"
+- Enriched: "his neighbor in Chicago, Mark's hometown"
+- Why: Maintains both the neighbor relationship and residence identity
 
-### 5. Do Not Remember (Privacy and System Noise)
-- **Sensitive Credentials**: Passwords, API keys, tokens, or authentication details
-- **Personal Data**: Unless the user explicitly asks to store it
-- **System Meta-commentary**: Update notices, version information, system status messages
-- **Debug Information**: Logs, error traces, or diagnostic information
-- **QA/Troubleshooting**: Conversations clearly intended for testing or debugging purposes
-- **Internal Processing**: Comments about the assistant's own thinking process
+❌ WRONG (loses relationships): reduces to just entity names without preserving the defining relationship
+✅ RIGHT (preserves identity): maintains the possessive/definitional connection that establishes entity relationships
+</enrichment_examples>
 
-## Enhanced Processing for Related Memories
-When related memories are provided:
-
-### Step 1: Analyze Existing Knowledge
-- Identify all entities, relationships, and facts already captured
-- Map the existing knowledge structure
-- Note any gaps or areas for enhancement
-
-### Step 2: Extract Novel Information
-- Filter current episode for information NOT in related memories
-- Identify new entities, attributes, or relationships
-- Focus on information that adds value to the memory graph
-
-### Step 3: Form Strategic Relationships
-- Connect new entities to existing ones through explicit relationships
-- Convert implicit connections into explicit memory statements
-- Bridge knowledge gaps using new information
-
-### Step 4: Evolve Existing Knowledge
-- Update outdated information with new details
-- Add new attributes to known entities
-- Expand relationship networks with new connections
-
-## Making Implicit Relationships Explicit
-- **Entity Disambiguation**: When same names appear across contexts, use related memories to clarify relationships
-- **Possessive Language**: Convert possessive forms into explicit relationships using related memory context
-- **Cross-Reference Formation**: Create explicit links between entities that appear in multiple episodes
-- **Temporal Relationship**: Establish time-based connections between related events or decisions
-
-## Information Prioritization with Related Memories
-- **HIGHEST PRIORITY**: New relationships between known entities
-- **HIGH PRIORITY**: New attributes or properties of known entities
-- **MEDIUM PRIORITY**: New entities with connections to existing knowledge
-- **LOW PRIORITY**: Standalone new information without clear connections
-- **EXCLUDE**: Information already captured in related memories that doesn't add new connections
-
-## Output Format
-When extracting memory-worthy information:
-
-1. If nothing meets the criteria for storage (especially after filtering against related memories), respond with exactly: "NOTHING_TO_REMEMBER"
-
-2. Otherwise, provide a summary that:
-   - **Prioritizes NEW information**: Focus on facts not present in related memories
-   - **Emphasizes relationships**: Highlight connections between new and existing information
-   - **Scales with novelty**: Make length reflect amount of genuinely new, valuable information
-   - **Uses third person perspective**: Maintain neutral, factual tone
-   - **Includes specific details**: Include names, dates, numbers when they add new value
-   - **Avoids redundancy**: Skip information already captured in related memories
-   - **Forms explicit connections**: Make relationships between entities clear and direct
-
-## Examples of Memory Graph Evolution
-
-### Before (Redundant Approach):
-Related Memory: "John Smith is the co-founder of TechCorp."
-Current Episode: "User discussed project timeline with John, the co-founder."
-BAD Output: "User discussed project timeline with John Smith, who is the co-founder of TechCorp."
-
-### After (Evolution Approach):
-Related Memory: "John Smith is the co-founder of TechCorp."
-Current Episode: "User discussed project timeline with John, the co-founder."
-GOOD Output: "User discussed project timeline with John Smith. The project timeline discussion involved TechCorp's co-founder."
-
-### Relationship Formation Example:
-Related Memory: "User prefers morning meetings."
-Current Episode: "User scheduled a meeting with John for 9 AM."
-Output: "User scheduled a 9 AM meeting with John Smith, aligning with their preference for morning meetings."
-
-Process information with related memories by focusing on evolving the memory graph through new connections and information rather than repeating already captured facts.
+CRITICAL OUTPUT FORMAT REQUIREMENT:
+You MUST wrap your response in <output> tags. This is MANDATORY - no exceptions.
 
+If the episode should be stored in memory:
 <output>
-{{processed_statement}}
+{{your_enriched_sentence_here}}
 </output>
 
-if there is nothing to remember 
+If there is nothing worth remembering:
 <output>
 NOTHING_TO_REMEMBER
 </output>
+
+FAILURE TO USE <output> TAGS WILL RESULT IN EMPTY NORMALIZATION AND SYSTEM FAILURE.
+
+FORMAT EXAMPLES:
+✅ CORRECT: <output>On May 25, 2023, Caroline shared her adoption plans with Melanie.</output>
+✅ CORRECT: <output>NOTHING_TO_REMEMBER</output>
+❌ WRONG: On May 25, 2023, Caroline shared her adoption plans with Melanie.
+❌ WRONG: NOTHING_TO_REMEMBER
+
+ALWAYS include opening <output> and closing </output> tags around your entire response.
 `;
 
   const userPrompt = `
@@ -238,11 +244,11 @@ ${context.source}
 </SOURCE>
 
 <EPISODE_TIMESTAMP>
-${context.episodeTimestamp || 'Not provided'}
+${context.episodeTimestamp || "Not provided"}
 </EPISODE_TIMESTAMP>
 
 <SAME_SESSION_CONTEXT>
-${context.sessionContext || 'No previous episodes in this session'}
+${context.sessionContext || "No previous episodes in this session"}
 </SAME_SESSION_CONTEXT>
 
 <RELATED_MEMORIES>

apps/webapp/app/services/prompts/statements.ts

@@ -70,6 +70,9 @@ EXTRACT NEW MEANINGFUL RELATIONSHIPS:
   * Product-organization relationships (e.g., "Software" "developed by" "Company")
   * Technical dependencies and usage (e.g., "Application" "uses" "Database")
   * Hierarchical relationships (e.g., "Manager" "supervises" "Employee")
+  * Duration relationships (e.g., "Caroline" "has known" "friends" [duration: "4 years"])
+  * Temporal sequence relationships (e.g., "Caroline" "met" "friends" [context: "since moving"])
+  * Contextual support relationships (e.g., "friends" "supported" "Caroline" [context: "during breakup"])
 
 ## SAME-NAME ENTITY RELATIONSHIP FORMATION
 When entities share identical names but have different types, CREATE explicit relationship statements:
@@ -80,6 +83,19 @@ When entities share identical names but have different types, CREATE explicit re
 - **MANDATORY**: Always create at least one relationship statement for same-name entities
 - **CONTEXT-DRIVEN**: Choose predicates that accurately reflect the most likely relationship based on available context
 
+## DURATION AND TEMPORAL CONTEXT ENTITY USAGE
+When Duration or TemporalContext entities are available in AVAILABLE ENTITIES:
+- **Duration entities** (e.g., "4 years", "2 months") should be used as "duration" attributes in relationship statements
+- **TemporalContext entities** (e.g., "since moving", "after breakup") should be used as "temporal_context" attributes
+- **DO NOT** use Duration/TemporalContext entities as direct subjects or objects in relationships
+- **DO USE** them to enrich relationship statements with temporal information
+
+EXAMPLES of correct Duration/TemporalContext usage:
+- If AVAILABLE ENTITIES contains ["Caroline", "friends", "4 years", "since moving"]:
+  * CREATE: "Caroline" "has known" "friends" [attributes: {"duration": "4 years", "temporal_context": "since moving"}]
+  * DO NOT CREATE: "Caroline" "relates to" "4 years" (Duration as object)
+  * DO NOT CREATE: "since moving" "describes" "friendship" (TemporalContext as subject)
+
 ## PREVIOUS EPISODE FILTERING
 Before creating any relationship statement:
 - **CHECK**: Review previous episodes to see if this exact relationship already exists
@@ -87,11 +103,24 @@ Before creating any relationship statement:
 - **ENHANCE**: Only create statements if they add new information or represent updates
 - **FOCUS**: Prioritize completely new connections not represented in the knowledge graph
 
-ABOUT TEMPORAL INFORMATION:
-- For events with dates/times, DO NOT create a separate statement with the event as both source and target.
-- Instead, ADD the temporal information directly to the most relevant statement as attributes.
-- Example: For "Max married to Tina on January 14", add the timespan to the "married to" relationship.
-- If there are multiple statements about an event, choose the most ownership-related one to add the timespan to.
+CRITICAL TEMPORAL INFORMATION HANDLING:
+- For events with specific dates/times, ALWAYS capture temporal information in statement attributes
+- Use the "event_date" attribute to specify when the fact/event actually occurred (not when it was mentioned)
+- Use the "temporal_context" attribute for temporal descriptions like "last week", "recently", etc.
+- MANDATORY: Use the REFERENCE_TIME to resolve relative temporal expressions to absolute ISO dates
+- Calculate event_date by using REFERENCE_TIME as the anchor point for relative time calculations
+- Example: For "Max married to Tina on January 14", add:
+  - "event_date": "January 14" (or fully resolved date if available)
+  - "temporal_context": "specific date mentioned"
+- For recent events: "went camping last week" → add:
+  - "event_date": "[resolved ISO date ~7 days before episode date, e.g., '2023-06-20T00:00:00.000Z']"  
+  - "temporal_context": "last week"
+- For past events: "read book last year" → add:
+  - "event_date": "[resolved ISO date ~1 year before episode date, e.g., '2022-06-27T00:00:00.000Z']"
+  - "temporal_context": "last year"
+- For future events: "going to Paris next month" → add:
+  - "event_date": "[resolved ISO date ~1 month after episode date, e.g., '2023-07-27T00:00:00.000Z']"
+  - "temporal_context": "next month"
 
 Format your response as a JSON object with the following structure:
 <output>
@@ -105,8 +134,12 @@ Format your response as a JSON object with the following structure:
       "targetType": "[Target Entity Type]",
       "fact": "[Natural language representation of the fact]",
       "attributes": { 
-        "confidence": confidence of the fact
+        "confidence": confidence of the fact,
         "source": "explicit or implicit source type",
+        "event_date": "ISO date when the fact/event actually occurred (if applicable)",
+        "temporal_context": "original temporal description (e.g., 'last week', 'recently')",
+        "duration": "duration information from Duration entities (e.g., '4 years', '2 months')",
+        "context": "contextual information from TemporalContext entities (e.g., 'since moving', 'after breakup')"
       }
     }
   ]
@@ -129,11 +162,18 @@ If AVAILABLE ENTITIES contains ["John", "Max", "Wedding", "John (Company)"], you
 - "Max" "married to" "Tina" with timespan attribute ✓ (if new relationship)
 - "John" "founded" "John (Company)" ✓ (PRIORITY: same name, different types)
 
+Example of CORRECT Duration/TemporalContext usage:
+If AVAILABLE ENTITIES contains ["Caroline", "friends", "4 years", "since moving", "breakup"]:
+- "Caroline" "has known" "friends" [attributes: {"duration": "4 years", "context": "since moving"}] ✓
+- "friends" "supported" "Caroline" [attributes: {"context": "during breakup"}] ✓
+- "Caroline" "met" "friends" [attributes: {"context": "since moving"}] ✓
+
 Example of INCORRECT usage:
 - "John" "attends" "Party" ✗ (if "Party" is not in AVAILABLE ENTITIES)
 - "Marriage" "occurs on" "Marriage" ✗ (NEVER create self-loops)
 - "John" "attends" "Wedding" ✗ (if already captured in previous episodes)
-- "January 14" "is" "Marriage date" ✗ (if "January 14" or "Marriage date" is not in AVAILABLE ENTITIES)`,
+- "Caroline" "relates to" "4 years" ✗ (Duration entity used as direct object)
+- "since moving" "describes" "friendship" ✗ (TemporalContext entity used as direct subject)`,
     },
     {
       role: "user",
@@ -171,9 +211,12 @@ export const resolveStatementPrompt = (
   return [
     {
       role: "system",
-      content: `You are a knowledge graph expert that analyzes statements to detect duplications and contradictions. 
-You analyze multiple new statements against existing statements to determine whether the new statement duplicates any existing statement or contradicts any existing statement.
-Pay special attention to temporal aspects, event updates, and context changes. If an event changes (like a date shift), statements about the original event are likely contradicted by statements about the updated event.
+      content: `You are a knowledge graph expert that analyzes statements to detect duplications and TRUE contradictions. 
+You analyze multiple new statements against existing statements to determine whether the new statement duplicates any existing statement or ACTUALLY contradicts any existing statement.
+
+CRITICAL: Distinguish between CONTRADICTIONS vs PROGRESSIONS:
+- CONTRADICTIONS: Statements that CANNOT both be true (mutually exclusive facts)  
+- PROGRESSIONS: Sequential states or developments that CAN both be true (e.g., planning → execution, researching → deciding)
 
 
 I need to analyze whether a new statement duplicates or contradicts existing statements in a knowledge graph.
@@ -185,32 +228,60 @@ Follow these instructions carefully:
    - Two statements are duplicates if they express the same meaning even with different wording
    - Consider entity resolution has already been done, so different entity names are NOT an issue
 
-2. Determine if the new statement contradicts any existing valid statements
-   - Contradictions occur when statements cannot both be true at the same time
-   - Pay special attention to negations, opposites, and mutually exclusive facts
-   - Consider temporal validity - statements may only be contradictions within specific time periods
-   
-3. IMPORTANT: For events that change (like rescheduled appointments, moved dates, changed locations):
-   - When an event changes date/time/location, new statements about the updated event likely contradict statements about the original event
-   - Look for contextual clues about event changes, cancellations, or rescheduling
-   - Example: If "Concert on June 10" moved to "Concert on June 12", then "John attends June 10 concert" contradicts "John doesn't attend June 12 concert"
+2. Determine if the new statement ACTUALLY contradicts any existing valid statements
+   - TRUE CONTRADICTIONS: Statements that cannot both be true simultaneously
+   - Pay attention to direct negations, opposites, and mutually exclusive facts
+   - Consider temporal context - statements may be contradictory only within specific time periods
+
+3. CRITICAL DISTINCTION - What are NOT contradictions:
+   - PROGRESSIONS: "researching X" → "decided on X" (both can be true - research led to decision)
+   - TEMPORAL SEQUENCES: "planning camping" → "went camping" (both can be true - plan was executed)  
+   - STATE CHANGES: "single" → "married" (both can be true at different times)
+   - LEARNING/GROWTH: "studying topic X" → "expert in topic X" (both can be true - progression)
+
+4. SPECIFIC EXAMPLES:
+
+TRUE CONTRADICTIONS (mark as contradictions):
+   - "John lives in New York" vs "John lives in San Francisco" (same time period, can't be both)
+   - "Meeting at 3pm" vs "Meeting at 5pm" (same meeting, conflicting times)
+   - "Project completed" vs "Project cancelled" (mutually exclusive outcomes) 
+   - "Caroline is single" vs "Caroline is married" (same time period, opposite states)
+
+NOT CONTRADICTIONS (do NOT mark as contradictions):
+   - "Caroline researching adoption agencies" vs "Caroline finalized adoption agency" (research → decision progression)
+   - "Caroline planning camping next week" vs "Caroline went camping" (planning → execution progression)
+   - "User studying Python" vs "User completed Python course" (learning progression)
+   - "Meeting scheduled for 3pm" vs "Meeting was held at 3pm" (planning → execution)
+   - "Considering job offers" vs "Accepted job offer" (consideration → decision)
+
+5. MANDATORY OUTPUT FORMAT:
+
+You MUST wrap your response in <output> tags. Do not include any text outside these tags.
 
-  4. Format your response as a JSON object with the following structure:
 <output>
 [{
     "statementId": "new_statement_uuid",
-    "isDuplicate": true/false,
-    "duplicateId": "existing_statement_uuid-if-duplicate-exists",
-    "contradictions": ["existing_statement_uuid-1", "existing_statement_uuid-2"], // UUIDs of any contradicted statements
-    }]
+    "isDuplicate": false,
+    "duplicateId": null,
+    "contradictions": []
+  },
+  {
+    "statementId": "another_statement_uuid",
+    "isDuplicate": true,
+    "duplicateId": "existing_duplicate_uuid",
+    "contradictions": ["contradicted_statement_uuid"]
+  }]
 </output>
-  
-  Important guidelines:
+
+CRITICAL FORMATTING RULES:
+- ALWAYS use <output> and </output> tags
+- Include NO text before <output> or after </output>
+- Return valid JSON array with all statement IDs from NEW_STATEMENTS
 - If the new statement is a duplicate, include the UUID of the duplicate statement
-- For contradictions, list all statement UUIDs that the new statement contradicts
-- If a statement is both a contradiction AND a duplicate (rare case), mark it as a duplicate
-- Identify temporal and contextual shifts that may create implicit contradictions
-- Don't give any reason, just give the final output.
+- For TRUE contradictions only, list statement UUIDs that the new statement contradicts
+- If a statement is both a contradiction AND a duplicate (rare case), mark it as a duplicate  
+- DO NOT mark progressions, temporal sequences, or state developments as contradictions
+- ONLY mark genuine mutually exclusive facts as contradictions
 `,
     },
     {

apps/webapp/app/services/search.server.ts

@@ -1,6 +1,7 @@
 import type { EpisodicNode, StatementNode } from "@core/types";
 import { logger } from "./logger.service";
 import {
+  applyCohereReranking,
   applyCrossEncoderReranking,
   applyMultiFactorMMRReranking,
 } from "./search/rerank";
@@ -13,6 +14,7 @@ import {
 import { getEmbedding } from "~/lib/model.server";
 import { prisma } from "~/db.server";
 import { runQuery } from "~/lib/neo4j.server";
+import { env } from "~/env.server";
 
 /**
  * SearchService provides methods to search the reified + temporal knowledge graph
@@ -34,7 +36,7 @@ export class SearchService {
     query: string,
     userId: string,
     options: SearchOptions = {},
-  ): Promise<{ episodes: string[]; facts: string[] }> {
+  ): Promise<{ episodes: string[]; facts: { fact: string; validAt: Date }[] }> {
     const startTime = Date.now();
     // Default options
 
@@ -95,7 +97,10 @@ export class SearchService {
 
     return {
       episodes: episodes.map((episode) => episode.content),
-      facts: filteredResults.map((statement) => statement.fact),
+      facts: filteredResults.map((statement) => ({
+        fact: statement.fact,
+        validAt: statement.validAt,
+      })),
     };
   }
 
@@ -108,6 +113,9 @@ export class SearchService {
     options: Required<SearchOptions>,
   ): StatementNode[] {
     if (results.length === 0) return [];
+    if (results.length <= 5) {
+      return results;
+    }
 
     let isRRF = false;
     // Extract scores from results
@@ -129,6 +137,8 @@ export class SearchService {
         score = (result as any).combinedScore;
       } else if ((result as any).mmrScore !== undefined) {
         score = (result as any).mmrScore;
+      } else if ((result as any).cohereScore !== undefined) {
+        score = (result as any).cohereScore;
       }
 
       return { result, score };
@@ -227,6 +237,11 @@ export class SearchService {
       results.bfs.length > 0,
     ].filter(Boolean).length;
 
+    if (env.COHERE_API_KEY) {
+      logger.info("Using Cohere reranking");
+      return applyCohereReranking(query, results, options);
+    }
+
     // If results are coming from only one source, use cross-encoder reranking
     if (nonEmptySources <= 1) {
       logger.info(

apps/webapp/app/services/search/rerank.ts

@@ -3,6 +3,7 @@ import { combineAndDeduplicateStatements } from "./utils";
 import { type CoreMessage } from "ai";
 import { makeModelCall } from "~/lib/model.server";
 import { logger } from "../logger.service";
+import { CohereClientV2 } from "cohere-ai";
 
 // Utility function to safely convert BigInt values to Number
 function safeNumber(value: any): number {
@@ -439,3 +440,93 @@ export function applyMultiFactorReranking(results: {
 
   return sortedResults;
 }
+
+/**
+ * Apply Cohere Rerank 3.5 to search results for improved question-to-fact matching
+ * This is particularly effective for bridging the semantic gap between questions and factual statements
+ */
+export async function applyCohereReranking(
+  query: string,
+  results: {
+    bm25: StatementNode[];
+    vector: StatementNode[];
+    bfs: StatementNode[];
+  },
+  options?: {
+    limit?: number;
+    model?: string;
+  },
+): Promise<StatementNode[]> {
+  const { model = "rerank-v3.5" } = options || {};
+  const limit = 100;
+
+  try {
+    const startTime = Date.now();
+    // Combine and deduplicate all results
+    const allResults = [
+      ...results.bm25.slice(0, 100),
+      ...results.vector.slice(0, 100),
+      ...results.bfs.slice(0, 100),
+    ];
+    const uniqueResults = combineAndDeduplicateStatements(allResults);
+    console.log("Unique results:", uniqueResults.length);
+
+    if (uniqueResults.length === 0) {
+      logger.info("No results to rerank with Cohere");
+      return [];
+    }
+
+    // Check for API key
+    const apiKey = process.env.COHERE_API_KEY;
+    if (!apiKey) {
+      logger.warn("COHERE_API_KEY not found, falling back to original results");
+      return uniqueResults.slice(0, limit);
+    }
+
+    // Initialize Cohere client
+    const cohere = new CohereClientV2({
+      token: apiKey,
+    });
+
+    // Prepare documents for Cohere API
+    const documents = uniqueResults.map((statement) => statement.fact);
+
+    logger.info(
+      `Cohere reranking ${documents.length} statements with model ${model}`,
+    );
+
+    // Call Cohere Rerank API
+    const response = await cohere.rerank({
+      query,
+      documents,
+      model,
+      topN: Math.min(limit, documents.length),
+    });
+
+    console.log("Cohere reranking billed units:", response.meta?.billedUnits);
+
+    // Map results back to StatementNodes with Cohere scores
+    const rerankedResults = response.results
+      .map((result, index) => ({
+        ...uniqueResults[result.index],
+        cohereScore: result.relevanceScore,
+        cohereRank: index + 1,
+      }))
+      .filter((result) => result.cohereScore > 0.3);
+
+    const responseTime = Date.now() - startTime;
+    logger.info(
+      `Cohere reranking completed: ${rerankedResults.length} results returned in ${responseTime}ms`,
+    );
+
+    return rerankedResults;
+  } catch (error) {
+    logger.error("Cohere reranking failed:", { error });
+
+    // Graceful fallback to original results
+    const allResults = [...results.bm25, ...results.vector, ...results.bfs];
+    const uniqueResults = combineAndDeduplicateStatements(allResults);
+
+    return uniqueResults.slice(0, limit);
+  }
+}

apps/webapp/app/trigger/extension/search.ts

@@ -50,7 +50,7 @@ export const extensionSearch = task({
           const searchResult = response.data;
 
           return {
-            facts: searchResult.facts || [],
+            facts: searchResult.facts || {},
             episodes: searchResult.episodes || [],
           };
         } catch (error) {

apps/webapp/package.json

@@ -86,6 +86,7 @@
     "class-variance-authority": "^0.7.1",
     "clsx": "^2.1.1",
     "cmdk": "^0.2.1",
+    "cohere-ai": "^7.18.1",
     "compression": "^1.7.4",
     "cross-env": "^7.0.3",
     "d3": "^7.9.0",
@@ -181,4 +182,4 @@
   "engines": {
     "node": ">=20.0.0"
   }
-}
+}