fix: add check to wait for neo4j

* feat: remove trigger and run base on bullmq
* fix: telemetry and trigger deploymen
* feat: add Ollama container and update ingestion status for unchanged documents
* feat: add logger to bullmq workers
* 1. Remove chat and deep-search from trigger
2. Add ai/sdk for chat UI
3. Added a better model manager

* refactor: simplify clustered graph query and add stop conditions for AI responses

* fix: streaming

* fix: docker docs

---------

Co-authored-by: Manoj <saimanoj58@gmail.com>

.env.example

@@ -1,4 +1,4 @@
-VERSION=0.1.25
+VERSION=0.1.26
 
 # Nest run in docker, change host to database container name
 DB_HOST=localhost
@@ -56,7 +56,5 @@ AWS_ACCESS_KEY_ID=
 AWS_SECRET_ACCESS_KEY=
 AWS_REGION=us-east-1
 
-## Trigger ##
-TRIGGER_PROJECT_ID=
-TRIGGER_SECRET_KEY=
-TRIGGER_API_URL=http://host.docker.internal:8030
+QUEUE_PROVIDER=bullmq
+

.vscode/settings.json

@@ -1,7 +0,0 @@
-{
-  "eslint.workingDirectories": [
-    {
-      "mode": "auto"
-    }
-  ]
-}

README.md

@@ -66,6 +66,10 @@ CORE is an open-source unified, persistent memory layer for all your AI tools. Y
 ## 🚀 CORE Self-Hosting
 Want to run CORE on your own infrastructure? Self-hosting gives you complete control over your data and deployment.
 
+**Quick Deploy Options:**
+
+[![Deploy on Railway](https://railway.com/button.svg)](https://railway.com/deploy/core?referralCode=LHvbIb&utm_medium=integration&utm_source=template&utm_campaign=generic)
+
 **Prerequisites**:
 
 - Docker (20.10.0+) and Docker Compose (2.20.0+) installed
@@ -258,3 +262,8 @@ Have questions or feedback? We're here to help:
 
 
 
+
+
+
+
+

apps/webapp/app/bullmq/connection.ts

@@ -0,0 +1,50 @@
+import Redis, { type RedisOptions } from "ioredis";
+
+let redisConnection: Redis | null = null;
+
+/**
+ * Get or create a Redis connection for BullMQ
+ * This connection is shared across all queues and workers
+ */
+export function getRedisConnection() {
+  if (redisConnection) {
+    return redisConnection;
+  }
+
+  // Dynamically import ioredis only when needed
+
+  const redisConfig: RedisOptions = {
+    host: process.env.REDIS_HOST,
+    port: parseInt(process.env.REDIS_PORT as string),
+    password: process.env.REDIS_PASSWORD,
+    maxRetriesPerRequest: null, // Required for BullMQ
+    enableReadyCheck: false, // Required for BullMQ
+  };
+
+  // Add TLS configuration if not disabled
+  if (!process.env.REDIS_TLS_DISABLED) {
+    redisConfig.tls = {};
+  }
+
+  redisConnection = new Redis(redisConfig);
+
+  redisConnection.on("error", (error) => {
+    console.error("Redis connection error:", error);
+  });
+
+  redisConnection.on("connect", () => {
+    console.log("Redis connected successfully");
+  });
+
+  return redisConnection;
+}
+
+/**
+ * Close the Redis connection (useful for graceful shutdown)
+ */
+export async function closeRedisConnection(): Promise<void> {
+  if (redisConnection) {
+    await redisConnection.quit();
+    redisConnection = null;
+  }
+}

apps/webapp/app/bullmq/queues/index.ts

@@ -0,0 +1,94 @@
+/**
+ * BullMQ Queues
+ *
+ * All queue definitions for the BullMQ implementation
+ */
+
+import { Queue } from "bullmq";
+import { getRedisConnection } from "../connection";
+
+/**
+ * Episode ingestion queue
+ * Handles individual episode ingestion (including document chunks)
+ */
+export const ingestQueue = new Queue("ingest-queue", {
+  connection: getRedisConnection(),
+  defaultJobOptions: {
+    attempts: 3,
+    backoff: {
+      type: "exponential",
+      delay: 2000,
+    },
+    removeOnComplete: {
+      age: 3600, // Keep completed jobs for 1 hour
+      count: 1000, // Keep last 1000 completed jobs
+    },
+    removeOnFail: {
+      age: 86400, // Keep failed jobs for 24 hours
+    },
+  },
+});
+
+/**
+ * Document ingestion queue
+ * Handles document-level ingestion with differential processing
+ */
+export const documentIngestQueue = new Queue("document-ingest-queue", {
+  connection: getRedisConnection(),
+  defaultJobOptions: {
+    attempts: 3,
+    backoff: {
+      type: "exponential",
+      delay: 2000,
+    },
+    removeOnComplete: {
+      age: 3600,
+      count: 1000,
+    },
+    removeOnFail: {
+      age: 86400,
+    },
+  },
+});
+
+/**
+ * Conversation title creation queue
+ */
+export const conversationTitleQueue = new Queue("conversation-title-queue", {
+  connection: getRedisConnection(),
+  defaultJobOptions: {
+    attempts: 3,
+    backoff: {
+      type: "exponential",
+      delay: 2000,
+    },
+    removeOnComplete: {
+      age: 3600,
+      count: 1000,
+    },
+    removeOnFail: {
+      age: 86400,
+    },
+  },
+});
+
+/**
+ * Session compaction queue
+ */
+export const sessionCompactionQueue = new Queue("session-compaction-queue", {
+  connection: getRedisConnection(),
+  defaultJobOptions: {
+    attempts: 3,
+    backoff: {
+      type: "exponential",
+      delay: 2000,
+    },
+    removeOnComplete: {
+      age: 3600,
+      count: 1000,
+    },
+    removeOnFail: {
+      age: 86400,
+    },
+  },
+});

apps/webapp/app/bullmq/start-workers.ts

@@ -0,0 +1,121 @@
+/**
+ * BullMQ Worker Startup Script
+ *
+ * This script starts all BullMQ workers for processing background jobs.
+ * Run this as a separate process alongside your main application.
+ *
+ * Usage:
+ *   tsx apps/webapp/app/bullmq/start-workers.ts
+ */
+
+import { logger } from "~/services/logger.service";
+import {
+  ingestWorker,
+  documentIngestWorker,
+  conversationTitleWorker,
+  sessionCompactionWorker,
+  closeAllWorkers,
+} from "./workers";
+import {
+  ingestQueue,
+  documentIngestQueue,
+  conversationTitleQueue,
+  sessionCompactionQueue,
+} from "./queues";
+import {
+  setupWorkerLogging,
+  startPeriodicMetricsLogging,
+} from "./utils/worker-logger";
+
+let metricsInterval: NodeJS.Timeout | null = null;
+
+/**
+ * Initialize and start all BullMQ workers with comprehensive logging
+ */
+export async function initWorkers(): Promise<void> {
+  // Setup comprehensive logging for all workers
+  setupWorkerLogging(ingestWorker, ingestQueue, "ingest-episode");
+  setupWorkerLogging(
+    documentIngestWorker,
+    documentIngestQueue,
+    "ingest-document",
+  );
+  setupWorkerLogging(
+    conversationTitleWorker,
+    conversationTitleQueue,
+    "conversation-title",
+  );
+
+  setupWorkerLogging(
+    sessionCompactionWorker,
+    sessionCompactionQueue,
+    "session-compaction",
+  );
+
+  // Start periodic metrics logging (every 60 seconds)
+  metricsInterval = startPeriodicMetricsLogging(
+    [
+      { worker: ingestWorker, queue: ingestQueue, name: "ingest-episode" },
+      {
+        worker: documentIngestWorker,
+        queue: documentIngestQueue,
+        name: "ingest-document",
+      },
+      {
+        worker: conversationTitleWorker,
+        queue: conversationTitleQueue,
+        name: "conversation-title",
+      },
+
+      {
+        worker: sessionCompactionWorker,
+        queue: sessionCompactionQueue,
+        name: "session-compaction",
+      },
+    ],
+    60000, // Log metrics every 60 seconds
+  );
+
+  // Log worker startup
+  logger.log("\n🚀 Starting BullMQ workers...");
+  logger.log("─".repeat(80));
+  logger.log(`✓ Ingest worker: ${ingestWorker.name} (concurrency: 5)`);
+  logger.log(
+    `✓ Document ingest worker: ${documentIngestWorker.name} (concurrency: 3)`,
+  );
+  logger.log(
+    `✓ Conversation title worker: ${conversationTitleWorker.name} (concurrency: 10)`,
+  );
+
+  logger.log(
+    `✓ Session compaction worker: ${sessionCompactionWorker.name} (concurrency: 3)`,
+  );
+  logger.log("─".repeat(80));
+  logger.log("✅ All BullMQ workers started and listening for jobs");
+  logger.log("📊 Metrics will be logged every 60 seconds\n");
+}
+
+/**
+ * Shutdown all workers gracefully
+ */
+export async function shutdownWorkers(): Promise<void> {
+  logger.log("Shutdown signal received, closing workers gracefully...");
+  if (metricsInterval) {
+    clearInterval(metricsInterval);
+  }
+  await closeAllWorkers();
+}
+
+// If running as standalone script, initialize workers
+if (import.meta.url === `file://${process.argv[1]}`) {
+  initWorkers();
+
+  // Handle graceful shutdown
+  const shutdown = async () => {
+    await shutdownWorkers();
+    process.exit(0);
+  };
+
+  process.on("SIGTERM", shutdown);
+  process.on("SIGINT", shutdown);
+}

apps/webapp/app/bullmq/utils/job-finder.ts

@@ -0,0 +1,132 @@
+/**
+ * BullMQ Job Finder Utilities
+ *
+ * Helper functions to find, retrieve, and cancel BullMQ jobs
+ */
+
+interface JobInfo {
+  id: string;
+  isCompleted: boolean;
+  status?: string;
+}
+
+/**
+ * Get all active queues
+ */
+async function getAllQueues() {
+  const {
+    ingestQueue,
+    documentIngestQueue,
+    conversationTitleQueue,
+    sessionCompactionQueue,
+  } = await import("../queues");
+
+  return [
+    ingestQueue,
+    documentIngestQueue,
+    conversationTitleQueue,
+    sessionCompactionQueue,
+  ];
+}
+
+/**
+ * Find jobs by tags (metadata stored in job data)
+ * Since BullMQ doesn't have native tag support like Trigger.dev,
+ * we search through jobs and check if their data contains the required identifiers
+ */
+export async function getJobsByTags(
+  tags: string[],
+  taskIdentifier?: string,
+): Promise<JobInfo[]> {
+  const queues = await getAllQueues();
+  const matchingJobs: JobInfo[] = [];
+
+  for (const queue of queues) {
+    // Skip if taskIdentifier is specified and doesn't match queue name
+    if (taskIdentifier && !queue.name.includes(taskIdentifier)) {
+      continue;
+    }
+
+    // Get all active and waiting jobs
+    const [active, waiting, delayed] = await Promise.all([
+      queue.getActive(),
+      queue.getWaiting(),
+      queue.getDelayed(),
+    ]);
+
+    const allJobs = [...active, ...waiting, ...delayed];
+
+    for (const job of allJobs) {
+      // Check if job data contains all required tags
+      const jobData = job.data as any;
+      const matchesTags = tags.every(
+        (tag) =>
+          job.id?.includes(tag) ||
+          jobData.userId === tag ||
+          jobData.workspaceId === tag ||
+          jobData.queueId === tag,
+      );
+
+      if (matchesTags) {
+        const state = await job.getState();
+        matchingJobs.push({
+          id: job.id!,
+          isCompleted: state === "completed" || state === "failed",
+          status: state,
+        });
+      }
+    }
+  }
+
+  return matchingJobs;
+}
+
+/**
+ * Get a specific job by ID across all queues
+ */
+export async function getJobById(jobId: string): Promise<JobInfo | null> {
+  const queues = await getAllQueues();
+
+  for (const queue of queues) {
+    try {
+      const job = await queue.getJob(jobId);
+      if (job) {
+        const state = await job.getState();
+        return {
+          id: job.id!,
+          isCompleted: state === "completed" || state === "failed",
+          status: state,
+        };
+      }
+    } catch {
+      // Job not in this queue, continue
+      continue;
+    }
+  }
+
+  return null;
+}
+
+/**
+ * Cancel a job by ID
+ */
+export async function cancelJobById(jobId: string): Promise<void> {
+  const queues = await getAllQueues();
+
+  for (const queue of queues) {
+    try {
+      const job = await queue.getJob(jobId);
+      if (job) {
+        const state = await job.getState();
+        // Only remove if not already completed
+        if (state !== "completed" && state !== "failed") {
+          await job.remove();
+        }
+        return;
+      }
+    } catch {
+      // Job not in this queue, continue
+      continue;
+    }
+  }
+}

apps/webapp/app/bullmq/utils/worker-logger.ts

@@ -0,0 +1,184 @@
+/**
+ * BullMQ Worker Logger
+ *
+ * Comprehensive logging utility for tracking worker status, queue metrics,
+ * and job lifecycle events
+ */
+
+import { type Worker, type Queue } from "bullmq";
+import { logger } from "~/services/logger.service";
+
+interface WorkerMetrics {
+  name: string;
+  concurrency: number;
+  activeJobs: number;
+  waitingJobs: number;
+  delayedJobs: number;
+  failedJobs: number;
+  completedJobs: number;
+}
+
+/**
+ * Setup comprehensive logging for a worker
+ */
+export function setupWorkerLogging(
+  worker: Worker,
+  queue: Queue,
+  workerName: string,
+): void {
+  // Job picked up and started processing
+  worker.on("active", async (job) => {
+    const counts = await getQueueCounts(queue);
+    logger.log(
+      `[${workerName}] 🔄 Job started: ${job.id} | Queue: ${counts.waiting} waiting, ${counts.active} active, ${counts.delayed} delayed`,
+    );
+  });
+
+  // Job completed successfully
+  worker.on("completed", async (job, result) => {
+    const counts = await getQueueCounts(queue);
+    const duration = job.finishedOn ? job.finishedOn - job.processedOn! : 0;
+    logger.log(
+      `[${workerName}] ✅ Job completed: ${job.id} (${duration}ms) | Queue: ${counts.waiting} waiting, ${counts.active} active`,
+    );
+  });
+
+  // Job failed
+  worker.on("failed", async (job, error) => {
+    const counts = await getQueueCounts(queue);
+    const attempt = job?.attemptsMade || 0;
+    const maxAttempts = job?.opts?.attempts || 3;
+    logger.error(
+      `[${workerName}] ❌ Job failed: ${job?.id} (attempt ${attempt}/${maxAttempts}) | Error: ${error.message} | Queue: ${counts.waiting} waiting, ${counts.failed} failed`,
+    );
+  });
+
+  // Job progress update (if job reports progress)
+  worker.on("progress", async (job, progress) => {
+    logger.log(`[${workerName}] 📊 Job progress: ${job.id} - ${progress}%`);
+  });
+
+  // Worker stalled (job took too long)
+  worker.on("stalled", async (jobId) => {
+    logger.warn(`[${workerName}] ⚠️  Job stalled: ${jobId}`);
+  });
+
+  // Worker error
+  worker.on("error", (error) => {
+    logger.error(`[${workerName}] 🔥 Worker error: ${error.message}`);
+  });
+
+  // Worker closed
+  worker.on("closed", () => {
+    logger.log(`[${workerName}] 🛑 Worker closed`);
+  });
+}
+
+/**
+ * Get queue counts for logging
+ */
+async function getQueueCounts(queue: Queue): Promise<{
+  waiting: number;
+  active: number;
+  delayed: number;
+  failed: number;
+  completed: number;
+}> {
+  try {
+    const counts = await queue.getJobCounts(
+      "waiting",
+      "active",
+      "delayed",
+      "failed",
+      "completed",
+    );
+    return {
+      waiting: counts.waiting || 0,
+      active: counts.active || 0,
+      delayed: counts.delayed || 0,
+      failed: counts.failed || 0,
+      completed: counts.completed || 0,
+    };
+  } catch (error) {
+    return { waiting: 0, active: 0, delayed: 0, failed: 0, completed: 0 };
+  }
+}
+
+/**
+ * Get metrics for all workers
+ */
+export async function getAllWorkerMetrics(
+  workers: Array<{ worker: Worker; queue: Queue; name: string }>,
+): Promise<WorkerMetrics[]> {
+  const metrics = await Promise.all(
+    workers.map(async ({ worker, queue, name }) => {
+      const counts = await getQueueCounts(queue);
+      return {
+        name,
+        concurrency: worker.opts.concurrency || 1,
+        activeJobs: counts.active,
+        waitingJobs: counts.waiting,
+        delayedJobs: counts.delayed,
+        failedJobs: counts.failed,
+        completedJobs: counts.completed,
+      };
+    }),
+  );
+
+  return metrics;
+}
+
+/**
+ * Log worker metrics summary
+ */
+export function logWorkerMetrics(metrics: WorkerMetrics[]): void {
+  logger.log("\n📊 BullMQ Worker Metrics:");
+  logger.log("─".repeat(80));
+
+  for (const metric of metrics) {
+    logger.log(
+      `[${metric.name.padEnd(25)}] Concurrency: ${metric.concurrency} | ` +
+        `Active: ${metric.activeJobs} | Waiting: ${metric.waitingJobs} | ` +
+        `Delayed: ${metric.delayedJobs} | Failed: ${metric.failedJobs} | ` +
+        `Completed: ${metric.completedJobs}`,
+    );
+  }
+
+  const totals = metrics.reduce(
+    (acc, m) => ({
+      active: acc.active + m.activeJobs,
+      waiting: acc.waiting + m.waitingJobs,
+      delayed: acc.delayed + m.delayedJobs,
+      failed: acc.failed + m.failedJobs,
+      completed: acc.completed + m.completedJobs,
+    }),
+    { active: 0, waiting: 0, delayed: 0, failed: 0, completed: 0 },
+  );
+
+  logger.log("─".repeat(80));
+  logger.log(
+    `[TOTAL] Active: ${totals.active} | Waiting: ${totals.waiting} | ` +
+      `Delayed: ${totals.delayed} | Failed: ${totals.failed} | ` +
+      `Completed: ${totals.completed}`,
+  );
+  logger.log("─".repeat(80) + "\n");
+}
+
+/**
+ * Start periodic metrics logging
+ */
+export function startPeriodicMetricsLogging(
+  workers: Array<{ worker: Worker; queue: Queue; name: string }>,
+  intervalMs: number = 60000, // Default: 1 minute
+): NodeJS.Timeout {
+  const logMetrics = async () => {
+    const metrics = await getAllWorkerMetrics(workers);
+    logWorkerMetrics(metrics);
+  };
+
+  // Log immediately on start
+  logMetrics();
+
+  // Then log periodically
+  return setInterval(logMetrics, intervalMs);
+}

apps/webapp/app/bullmq/workers/index.ts

@@ -0,0 +1,123 @@
+/**
+ * BullMQ Workers
+ *
+ * All worker definitions for processing background jobs with BullMQ
+ */
+
+import { Worker } from "bullmq";
+import { getRedisConnection } from "../connection";
+import {
+  processEpisodeIngestion,
+  type IngestEpisodePayload,
+} from "~/jobs/ingest/ingest-episode.logic";
+import {
+  processDocumentIngestion,
+  type IngestDocumentPayload,
+} from "~/jobs/ingest/ingest-document.logic";
+import {
+  processConversationTitleCreation,
+  type CreateConversationTitlePayload,
+} from "~/jobs/conversation/create-title.logic";
+
+import {
+  processSessionCompaction,
+  type SessionCompactionPayload,
+} from "~/jobs/session/session-compaction.logic";
+import {
+  enqueueIngestEpisode,
+  enqueueSpaceAssignment,
+  enqueueSessionCompaction,
+} from "~/lib/queue-adapter.server";
+import { logger } from "~/services/logger.service";
+
+/**
+ * Episode ingestion worker
+ * Processes individual episode ingestion jobs with per-user concurrency
+ *
+ * Note: Per-user concurrency is achieved by using userId as part of the jobId
+ * when adding jobs to the queue, ensuring only one job per user runs at a time
+ */
+export const ingestWorker = new Worker(
+  "ingest-queue",
+  async (job) => {
+    const payload = job.data as IngestEpisodePayload;
+
+    return await processEpisodeIngestion(
+      payload,
+      // Callbacks to enqueue follow-up jobs
+      enqueueSpaceAssignment,
+      enqueueSessionCompaction,
+    );
+  },
+  {
+    connection: getRedisConnection(),
+    concurrency: 5, // Process up to 5 jobs in parallel
+  },
+);
+
+/**
+ * Document ingestion worker
+ * Handles document-level ingestion with differential processing
+ *
+ * Note: Per-user concurrency is achieved by using userId as part of the jobId
+ * when adding jobs to the queue
+ */
+export const documentIngestWorker = new Worker(
+  "document-ingest-queue",
+  async (job) => {
+    const payload = job.data as IngestDocumentPayload;
+    return await processDocumentIngestion(
+      payload,
+      // Callback to enqueue episode ingestion for each chunk
+      enqueueIngestEpisode,
+    );
+  },
+  {
+    connection: getRedisConnection(),
+    concurrency: 3, // Process up to 3 documents in parallel
+  },
+);
+
+/**
+ * Conversation title creation worker
+ */
+export const conversationTitleWorker = new Worker(
+  "conversation-title-queue",
+  async (job) => {
+    const payload = job.data as CreateConversationTitlePayload;
+    return await processConversationTitleCreation(payload);
+  },
+  {
+    connection: getRedisConnection(),
+    concurrency: 10, // Process up to 10 title creations in parallel
+  },
+);
+
+/**
+ * Session compaction worker
+ */
+export const sessionCompactionWorker = new Worker(
+  "session-compaction-queue",
+  async (job) => {
+    const payload = job.data as SessionCompactionPayload;
+    return await processSessionCompaction(payload);
+  },
+  {
+    connection: getRedisConnection(),
+    concurrency: 3, // Process up to 3 compactions in parallel
+  },
+);
+
+/**
+ * Graceful shutdown handler
+ */
+export async function closeAllWorkers(): Promise<void> {
+  await Promise.all([
+    ingestWorker.close(),
+    documentIngestWorker.close(),
+    conversationTitleWorker.close(),
+
+    sessionCompactionWorker.close(),
+  ]);
+  logger.log("All BullMQ workers closed");
+}

apps/webapp/app/components/conversation/conversation-item.client.tsx

@@ -1,38 +1,42 @@
 import { EditorContent, useEditor } from "@tiptap/react";
 
 import { useEffect, memo } from "react";
-import { UserTypeEnum } from "@core/types";
-import { type ConversationHistory } from "@core/database";
 import { cn } from "~/lib/utils";
 import { extensionsForConversation } from "./editor-extensions";
 import { skillExtension } from "../editor/skill-extension";
+import { type UIMessage } from "ai";
 
 interface AIConversationItemProps {
-  conversationHistory: ConversationHistory;
+  message: UIMessage;
 }
 
-const ConversationItemComponent = ({
-  conversationHistory,
-}: AIConversationItemProps) => {
-  const isUser =
-    conversationHistory.userType === UserTypeEnum.User ||
-    conversationHistory.userType === UserTypeEnum.System;
+function getMessage(message: string) {
+  let finalMessage = message.replace("<final_response>", "");
+  finalMessage = finalMessage.replace("</final_response>", "");
+  finalMessage = finalMessage.replace("<question_response>", "");
+  finalMessage = finalMessage.replace("</question_response>", "");
 
-  const id = `a${conversationHistory.id.replace(/-/g, "")}`;
+  return finalMessage;
+}
+
+const ConversationItemComponent = ({ message }: AIConversationItemProps) => {
+  const isUser = message.role === "user" || false;
+  const textPart = message.parts.find((part) => part.type === "text");
 
   const editor = useEditor({
     extensions: [...extensionsForConversation, skillExtension],
     editable: false,
-    content: conversationHistory.message,
+    content: textPart ? getMessage(textPart.text) : "",
   });
 
   useEffect(() => {
-    editor?.commands.setContent(conversationHistory.message);
-
+    if (textPart) {
+      editor?.commands.setContent(getMessage(textPart.text));
+    }
     // eslint-disable-next-line react-hooks/exhaustive-deps
-  }, [id, conversationHistory.message]);
+  }, [message]);
 
-  if (!conversationHistory.message) {
+  if (!message) {
     return null;
   }
 
@@ -51,10 +55,10 @@ const ConversationItemComponent = ({
 };
 
 // Memoize to prevent unnecessary re-renders
-export const ConversationItem = memo(ConversationItemComponent, (prevProps, nextProps) => {
-  // Only re-render if the conversation history ID or message changed
-  return (
-    prevProps.conversationHistory.id === nextProps.conversationHistory.id &&
-    prevProps.conversationHistory.message === nextProps.conversationHistory.message
-  );
-});
+export const ConversationItem = memo(
+  ConversationItemComponent,
+  (prevProps, nextProps) => {
+    // Only re-render if the conversation history ID or message changed
+    return prevProps.message === nextProps.message;
+  },
+);

apps/webapp/app/components/conversation/conversation-textarea.client.tsx

@@ -13,33 +13,26 @@ import { Form, useSubmit, useActionData } from "@remix-run/react";
 
 interface ConversationTextareaProps {
   defaultValue?: string;
-  conversationId: string;
   placeholder?: string;
   isLoading?: boolean;
   className?: string;
   onChange?: (text: string) => void;
   disabled?: boolean;
-  onConversationCreated?: (conversation: any) => void;
+  onConversationCreated?: (message: string) => void;
+  stop?: () => void;
 }
 
 export function ConversationTextarea({
   defaultValue,
   isLoading = false,
   placeholder,
-  conversationId,
   onChange,
   onConversationCreated,
+  stop,
 }: ConversationTextareaProps) {
   const [text, setText] = useState(defaultValue ?? "");
   const [editor, setEditor] = useState<Editor>();
   const submit = useSubmit();
-  const actionData = useActionData<{ conversation?: any }>();
-
-  useEffect(() => {
-    if (actionData?.conversation && onConversationCreated) {
-      onConversationCreated(actionData.conversation);
-    }
-  }, [actionData]);
 
   const onUpdate = (editor: Editor) => {
     setText(editor.getHTML());
@@ -51,134 +44,99 @@ export function ConversationTextarea({
       return;
     }
 
-    const data = isLoading ? {} : { message: text, conversationId };
-
-    // When conversationId exists and not stopping, submit to current route
-    // When isLoading (stopping), submit to the specific conversation route
-    submit(data as any, {
-      action: isLoading
-        ? `/home/conversation/${conversationId}`
-        : conversationId
-          ? `/home/conversation/${conversationId}`
-          : "/home/conversation",
-      method: "post",
-    });
+    onConversationCreated && onConversationCreated(text);
 
     editor?.commands.clearContent(true);
     setText("");
   }, [editor, text]);
 
-  // Send message to API
-  const submitForm = useCallback(
-    async (e: React.FormEvent<HTMLFormElement>) => {
-      const data = isLoading
-        ? {}
-        : { message: text, title: text, conversationId };
-
-      submit(data as any, {
-        action: isLoading
-          ? `/home/conversation/${conversationId}`
-          : conversationId
-            ? `/home/conversation/${conversationId}`
-            : "/home/conversation",
-        method: "post",
-      });
-
-      editor?.commands.clearContent(true);
-      setText("");
-      e.preventDefault();
-    },
-    [text, conversationId],
-  );
-
   return (
-    <Form
-      action="/home/conversation"
-      method="post"
-      onSubmit={(e) => submitForm(e)}
-      className="pt-2"
-    >
-      <div className="bg-background-3 rounded-lg border-1 border-gray-300 py-2">
-        <EditorRoot>
-          <EditorContent
-            // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            initialContent={defaultValue as any}
-            extensions={[
-              Document,
-              Paragraph,
-              Text,
-              HardBreak.configure({
-                keepMarks: true,
-              }),
+    <div className="bg-background-3 rounded-lg border-1 border-gray-300 py-2">
+      <EditorRoot>
+        <EditorContent
+          // eslint-disable-next-line @typescript-eslint/no-explicit-any
+          initialContent={defaultValue as any}
+          extensions={[
+            Document,
+            Paragraph,
+            Text,
+            HardBreak.configure({
+              keepMarks: true,
+            }),
 
-              Placeholder.configure({
-                placeholder: () => placeholder ?? "Ask sol...",
-                includeChildren: true,
-              }),
-              History,
-            ]}
-            onCreate={async ({ editor }) => {
-              setEditor(editor);
-              await new Promise((resolve) => setTimeout(resolve, 100));
-              editor.commands.focus("end");
-            }}
-            onUpdate={({ editor }) => {
-              onUpdate(editor);
-            }}
-            shouldRerenderOnTransaction={false}
-            editorProps={{
-              attributes: {
-                class: `prose prose-lg dark:prose-invert prose-headings:font-title font-default focus:outline-none max-w-full`,
-              },
-              handleKeyDown(view, event) {
-                if (event.key === "Enter" && !event.shiftKey) {
-                  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-                  const target = event.target as any;
-                  if (target.innerHTML.includes("suggestion")) {
-                    return false;
-                  }
-                  event.preventDefault();
-                  if (text) {
-                    handleSend();
-                  }
-                  return true;
+            Placeholder.configure({
+              placeholder: () => placeholder ?? "Ask sol...",
+              includeChildren: true,
+            }),
+            History,
+          ]}
+          onCreate={async ({ editor }) => {
+            setEditor(editor);
+            await new Promise((resolve) => setTimeout(resolve, 100));
+            editor.commands.focus("end");
+          }}
+          onUpdate={({ editor }) => {
+            onUpdate(editor);
+          }}
+          shouldRerenderOnTransaction={false}
+          editorProps={{
+            attributes: {
+              class: `prose prose-lg dark:prose-invert prose-headings:font-title font-default focus:outline-none max-w-full`,
+            },
+            handleKeyDown(view, event) {
+              if (event.key === "Enter" && !event.shiftKey) {
+                // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                const target = event.target as any;
+                if (target.innerHTML.includes("suggestion")) {
+                  return false;
                 }
+                event.preventDefault();
+                if (text) {
+                  handleSend();
+                }
+                return true;
+              }
 
-                if (event.key === "Enter" && event.shiftKey) {
-                  view.dispatch(
-                    view.state.tr.replaceSelectionWith(
-                      view.state.schema.nodes.hardBreak.create(),
-                    ),
-                  );
-                  return true;
-                }
-                return false;
-              },
-            }}
-            immediatelyRender={false}
-            className={cn(
-              "editor-container text-md max-h-[400px] min-h-[40px] w-full min-w-full overflow-auto rounded-lg px-3",
-            )}
-          />
-        </EditorRoot>
-        <div className="mb-1 flex justify-end px-3">
-          <Button
-            variant="default"
-            className="gap-1 shadow-none transition-all duration-500 ease-in-out"
-            type="submit"
-            size="lg"
-          >
-            {isLoading ? (
-              <>
-                <LoaderCircle size={18} className="mr-1 animate-spin" />
-                Stop
-              </>
-            ) : (
-              <>Chat</>
-            )}
-          </Button>
-        </div>
+              if (event.key === "Enter" && event.shiftKey) {
+                view.dispatch(
+                  view.state.tr.replaceSelectionWith(
+                    view.state.schema.nodes.hardBreak.create(),
+                  ),
+                );
+                return true;
+              }
+              return false;
+            },
+          }}
+          immediatelyRender={false}
+          className={cn(
+            "editor-container text-md max-h-[400px] min-h-[40px] w-full min-w-full overflow-auto rounded-lg px-3",
+          )}
+        />
+      </EditorRoot>
+      <div className="mb-1 flex justify-end px-3">
+        <Button
+          variant="default"
+          className="gap-1 shadow-none transition-all duration-500 ease-in-out"
+          onClick={() => {
+            if (!isLoading) {
+              handleSend();
+            } else {
+              stop && stop();
+            }
+          }}
+          size="lg"
+        >
+          {isLoading ? (
+            <>
+              <LoaderCircle size={18} className="mr-1 animate-spin" />
+              Stop
+            </>
+          ) : (
+            <>Chat</>
+          )}
+        </Button>
       </div>
-    </Form>
+    </div>
   );
 }

apps/webapp/app/components/logs/log-options.tsx

@@ -1,10 +1,4 @@
-import { EllipsisVertical, Trash, Copy } from "lucide-react";
-import {
-  DropdownMenu,
-  DropdownMenuContent,
-  DropdownMenuItem,
-  DropdownMenuTrigger,
-} from "../ui/dropdown-menu";
+import { Trash, Copy, RotateCw } from "lucide-react";
 import { Button } from "../ui/button";
 import {
   AlertDialog,
@@ -22,11 +16,13 @@ import { toast } from "~/hooks/use-toast";
 
 interface LogOptionsProps {
   id: string;
+  status?: string;
 }
 
-export const LogOptions = ({ id }: LogOptionsProps) => {
+export const LogOptions = ({ id, status }: LogOptionsProps) => {
   const [deleteDialogOpen, setDeleteDialogOpen] = useState(false);
   const deleteFetcher = useFetcher<{ success: boolean }>();
+  const retryFetcher = useFetcher<{ success: boolean }>();
   const navigate = useNavigate();
 
   const handleDelete = () => {
@@ -58,22 +54,54 @@ export const LogOptions = ({ id }: LogOptionsProps) => {
     }
   };
 
+  const handleRetry = () => {
+    retryFetcher.submit(
+      {},
+      {
+        method: "POST",
+        action: `/api/v1/logs/${id}/retry`,
+      },
+    );
+  };
+
   useEffect(() => {
     if (deleteFetcher.state === "idle" && deleteFetcher.data?.success) {
       navigate(`/home/inbox`);
     }
   }, [deleteFetcher.state, deleteFetcher.data]);
 
+  useEffect(() => {
+    if (retryFetcher.state === "idle" && retryFetcher.data?.success) {
+      toast({
+        title: "Success",
+        description: "Episode retry initiated",
+      });
+      // Reload the page to reflect the new status
+      window.location.reload();
+    }
+  }, [retryFetcher.state, retryFetcher.data]);
+
   return (
     <>
       <div className="flex items-center gap-2">
+        {status === "FAILED" && (
+          <Button
+            variant="secondary"
+            size="sm"
+            className="gap-2 rounded"
+            onClick={handleRetry}
+            disabled={retryFetcher.state !== "idle"}
+          >
+            <RotateCw size={15} /> Retry
+          </Button>
+        )}
         <Button
           variant="secondary"
           size="sm"
           className="gap-2 rounded"
           onClick={handleCopy}
         >
-          <Copy size={15} /> Copy ID
+          <Copy size={15} /> Copy Id
         </Button>
         <Button
           variant="secondary"

apps/webapp/app/components/logs/log-text-collapse.tsx

@@ -74,7 +74,7 @@ export function LogTextCollapse({ text, log }: LogTextCollapseProps) {
           <div className={cn("flex w-full min-w-[0px] shrink flex-col")}>
             <div className="flex w-full items-center justify-between gap-4">
               <div className="inline-flex min-h-[24px] min-w-[0px] shrink items-center justify-start">
-                <div className={cn("truncate text-left")}>
+                <div className={cn("truncate text-left text-base")}>
                   {text.replace(/<[^>]+>/g, "")}
                 </div>
               </div>
@@ -97,7 +97,7 @@ export function LogTextCollapse({ text, log }: LogTextCollapseProps) {
           </div>
 
           <div className="flex items-center justify-between">
-            <div className="flex items-center gap-1">
+            <div className="flex items-center gap-1 font-light">
               {getIconForAuthorise(log.source.toLowerCase(), 12, undefined)}
               {log.source.toLowerCase()}
             </div>

apps/webapp/app/components/spaces/space-options.tsx

@@ -99,7 +99,7 @@ export const SpaceOptions = ({ id, name, description }: SpaceOptionsProps) => {
         <DropdownMenuContent align="end">
           <DropdownMenuItem onClick={handleCopy}>
             <Button variant="link" size="sm" className="gap-2 rounded">
-              <Copy size={15} /> Copy ID
+              <Copy size={15} /> Copy Id
             </Button>
           </DropdownMenuItem>
           <DropdownMenuItem onClick={() => setEditDialogOpen(true)}>

apps/webapp/app/entry.server.tsx

@@ -17,6 +17,7 @@ import { renderToPipeableStream } from "react-dom/server";
 import { initializeStartupServices } from "./utils/startup";
 import { handleMCPRequest, handleSessionRequest } from "~/services/mcp.server";
 import { authenticateHybridRequest } from "~/services/routeBuilders/apiBuilder.server";
+import { trackError } from "~/services/telemetry.server";
 
 const ABORT_DELAY = 5_000;
 
@@ -27,6 +28,42 @@ async function init() {
 
 init();
 
+/**
+ * Global error handler for all server-side errors
+ * This catches errors from loaders, actions, and rendering
+ * Automatically tracks all errors to telemetry
+ */
+export function handleError(
+  error: unknown,
+  { request }: { request: Request },
+): void {
+  // Don't track 404s or aborted requests as errors
+  if (
+    error instanceof Response &&
+    (error.status === 404 || error.status === 304)
+  ) {
+    return;
+  }
+
+  // Track error to telemetry
+  if (error instanceof Error) {
+    const url = new URL(request.url);
+    trackError(error, {
+      url: request.url,
+      path: url.pathname,
+      method: request.method,
+      userAgent: request.headers.get("user-agent") || "unknown",
+      referer: request.headers.get("referer") || undefined,
+    }).catch((trackingError) => {
+      // If telemetry tracking fails, just log it - don't break the app
+      console.error("Failed to track error:", trackingError);
+    });
+  }
+
+  // Always log to console for development/debugging
+  console.error(error);
+}
+
 export default function handleRequest(
   request: Request,
   responseStatusCode: number,

apps/webapp/app/env.server.ts

@@ -3,99 +3,146 @@ import { isValidDatabaseUrl } from "./utils/db";
 import { isValidRegex } from "./utils/regex";
 import { LLMModelEnum } from "@core/types";
 
-const EnvironmentSchema = z.object({
-  NODE_ENV: z.union([
-    z.literal("development"),
-    z.literal("production"),
-    z.literal("test"),
-  ]),
-  POSTGRES_DB: z.string(),
-  DATABASE_URL: z
-    .string()
-    .refine(
-      isValidDatabaseUrl,
-      "DATABASE_URL is invalid, for details please check the additional output above this message.",
-    ),
-  DATABASE_CONNECTION_LIMIT: z.coerce.number().int().default(10),
-  DATABASE_POOL_TIMEOUT: z.coerce.number().int().default(60),
-  DATABASE_CONNECTION_TIMEOUT: z.coerce.number().int().default(20),
-  DIRECT_URL: z
-    .string()
-    .refine(
-      isValidDatabaseUrl,
-      "DIRECT_URL is invalid, for details please check the additional output above this message.",
-    ),
-  DATABASE_READ_REPLICA_URL: z.string().optional(),
-  SESSION_SECRET: z.string(),
-  ENCRYPTION_KEY: z.string(),
-  MAGIC_LINK_SECRET: z.string(),
-  WHITELISTED_EMAILS: z
-    .string()
-    .refine(isValidRegex, "WHITELISTED_EMAILS must be a valid regex.")
-    .optional(),
-  ADMIN_EMAILS: z
-    .string()
-    .refine(isValidRegex, "ADMIN_EMAILS must be a valid regex.")
-    .optional(),
+const EnvironmentSchema = z
+  .object({
+    NODE_ENV: z.union([
+      z.literal("development"),
+      z.literal("production"),
+      z.literal("test"),
+    ]),
+    POSTGRES_DB: z.string(),
+    DATABASE_URL: z
+      .string()
+      .refine(
+        isValidDatabaseUrl,
+        "DATABASE_URL is invalid, for details please check the additional output above this message.",
+      ),
+    DATABASE_CONNECTION_LIMIT: z.coerce.number().int().default(10),
+    DATABASE_POOL_TIMEOUT: z.coerce.number().int().default(60),
+    DATABASE_CONNECTION_TIMEOUT: z.coerce.number().int().default(20),
+    DIRECT_URL: z
+      .string()
+      .refine(
+        isValidDatabaseUrl,
+        "DIRECT_URL is invalid, for details please check the additional output above this message.",
+      ),
+    DATABASE_READ_REPLICA_URL: z.string().optional(),
+    SESSION_SECRET: z.string(),
+    ENCRYPTION_KEY: z.string(),
+    MAGIC_LINK_SECRET: z.string(),
+    WHITELISTED_EMAILS: z
+      .string()
+      .refine(isValidRegex, "WHITELISTED_EMAILS must be a valid regex.")
+      .optional(),
+    ADMIN_EMAILS: z
+      .string()
+      .refine(isValidRegex, "ADMIN_EMAILS must be a valid regex.")
+      .optional(),
 
-  APP_ENV: z.string().default(process.env.NODE_ENV),
-  LOGIN_ORIGIN: z.string().default("http://localhost:5173"),
-  APP_ORIGIN: z.string().default("http://localhost:5173"),
-  POSTHOG_PROJECT_KEY: z.string().default(""),
+    APP_ENV: z.string().default(process.env.NODE_ENV),
+    LOGIN_ORIGIN: z.string().default("http://localhost:5173"),
+    APP_ORIGIN: z.string().default("http://localhost:5173"),
 
-  //storage
-  ACCESS_KEY_ID: z.string().optional(),
-  SECRET_ACCESS_KEY: z.string().optional(),
-  BUCKET: z.string().optional(),
+    // Telemetry
+    POSTHOG_PROJECT_KEY: z
+      .string()
+      .default("phc_SwfGIzzX5gh5bazVWoRxZTBhkr7FwvzArS0NRyGXm1a"),
+    TELEMETRY_ENABLED: z
+      .string()
+      .optional()
+      .default("true")
+      .transform((val) => val !== "false" && val !== "0"),
+    TELEMETRY_ANONYMOUS: z
+      .string()
+      .optional()
+      .default("false")
+      .transform((val) => val === "true" || val === "1"),
 
-  // google auth
-  AUTH_GOOGLE_CLIENT_ID: z.string().optional(),
-  AUTH_GOOGLE_CLIENT_SECRET: z.string().optional(),
+    //storage
+    ACCESS_KEY_ID: z.string().optional(),
+    SECRET_ACCESS_KEY: z.string().optional(),
+    BUCKET: z.string().optional(),
 
-  ENABLE_EMAIL_LOGIN: z.coerce.boolean().default(true),
+    // google auth
+    AUTH_GOOGLE_CLIENT_ID: z.string().optional(),
+    AUTH_GOOGLE_CLIENT_SECRET: z.string().optional(),
 
-  //Redis
-  REDIS_HOST: z.string().default("localhost"),
-  REDIS_PORT: z.coerce.number().default(6379),
-  REDIS_TLS_DISABLED: z.coerce.boolean().default(true),
+    ENABLE_EMAIL_LOGIN: z
+      .string()
+      .optional()
+      .default("true")
+      .transform((val) => val !== "false" && val !== "0"),
 
-  //Neo4j
-  NEO4J_URI: z.string(),
-  NEO4J_USERNAME: z.string(),
-  NEO4J_PASSWORD: z.string(),
+    //Redis
+    REDIS_HOST: z.string().default("localhost"),
+    REDIS_PORT: z.coerce.number().default(6379),
+    REDIS_TLS_DISABLED: z
+      .string()
+      .optional()
+      .default("true")
+      .transform((val) => val !== "false" && val !== "0"),
 
-  //OpenAI
-  OPENAI_API_KEY: z.string(),
-  ANTHROPIC_API_KEY: z.string().optional(),
+    //Neo4j
+    NEO4J_URI: z.string(),
+    NEO4J_USERNAME: z.string(),
+    NEO4J_PASSWORD: z.string(),
 
-  EMAIL_TRANSPORT: z.string().optional(),
-  FROM_EMAIL: z.string().optional(),
-  REPLY_TO_EMAIL: z.string().optional(),
-  RESEND_API_KEY: z.string().optional(),
-  SMTP_HOST: z.string().optional(),
-  SMTP_PORT: z.coerce.number().optional(),
-  SMTP_SECURE: z.coerce.boolean().optional(),
-  SMTP_USER: z.string().optional(),
-  SMTP_PASSWORD: z.string().optional(),
+    //OpenAI
+    OPENAI_API_KEY: z.string().optional(),
+    ANTHROPIC_API_KEY: z.string().optional(),
+    GOOGLE_GENERATIVE_AI_API_KEY: z.string().optional(),
 
-  //Trigger
-  TRIGGER_PROJECT_ID: z.string(),
-  TRIGGER_SECRET_KEY: z.string(),
-  TRIGGER_API_URL: z.string(),
-  TRIGGER_DB: z.string().default("trigger"),
+    EMAIL_TRANSPORT: z.string().optional(),
+    FROM_EMAIL: z.string().optional(),
+    REPLY_TO_EMAIL: z.string().optional(),
+    RESEND_API_KEY: z.string().optional(),
+    SMTP_HOST: z.string().optional(),
+    SMTP_PORT: z.coerce.number().optional(),
+    SMTP_SECURE: z
+      .string()
+      .optional()
+      .transform((val) => val === "true" || val === "1"),
+    SMTP_USER: z.string().optional(),
+    SMTP_PASSWORD: z.string().optional(),
 
-  // Model envs
-  MODEL: z.string().default(LLMModelEnum.GPT41),
-  EMBEDDING_MODEL: z.string().default("mxbai-embed-large"),
-  EMBEDDING_MODEL_SIZE: z.string().default("1024"),
-  OLLAMA_URL: z.string().optional(),
-  COHERE_API_KEY: z.string().optional(),
-  COHERE_SCORE_THRESHOLD: z.string().default("0.3"),
+    //Trigger
+    TRIGGER_PROJECT_ID: z.string().optional(),
+    TRIGGER_SECRET_KEY: z.string().optional(),
+    TRIGGER_API_URL: z.string().optional(),
+    TRIGGER_DB: z.string().default("trigger"),
 
-  AWS_ACCESS_KEY_ID: z.string().optional(),
-  AWS_SECRET_ACCESS_KEY: z.string().optional(),
-  AWS_REGION: z.string().optional(),
-});
+    // Model envs
+    MODEL: z.string().default(LLMModelEnum.GPT41),
+    EMBEDDING_MODEL: z.string().default("mxbai-embed-large"),
+    EMBEDDING_MODEL_SIZE: z.string().default("1024"),
+    OLLAMA_URL: z.string().optional(),
+    COHERE_API_KEY: z.string().optional(),
+    COHERE_SCORE_THRESHOLD: z.string().default("0.3"),
+
+    AWS_ACCESS_KEY_ID: z.string().optional(),
+    AWS_SECRET_ACCESS_KEY: z.string().optional(),
+    AWS_REGION: z.string().optional(),
+
+    // Queue provider
+    QUEUE_PROVIDER: z.enum(["trigger", "bullmq"]).default("trigger"),
+  })
+  .refine(
+    (data) => {
+      // If QUEUE_PROVIDER is "trigger", then Trigger.dev variables must be present
+      if (data.QUEUE_PROVIDER === "trigger") {
+        return !!(
+          data.TRIGGER_PROJECT_ID &&
+          data.TRIGGER_SECRET_KEY &&
+          data.TRIGGER_API_URL
+        );
+      }
+      return true;
+    },
+    {
+      message:
+        "TRIGGER_PROJECT_ID, TRIGGER_SECRET_KEY, and TRIGGER_API_URL are required when QUEUE_PROVIDER=trigger",
+    },
+  );
 
 export type Environment = z.infer<typeof EnvironmentSchema>;
 export const env = EnvironmentSchema.parse(process.env);

apps/webapp/app/hooks/usePostHog.ts

@@ -6,6 +6,7 @@ import { useOptionalUser, useUserChanged } from "./useUser";
 
 export const usePostHog = (
   apiKey?: string,
+  telemetryEnabled = true,
   logging = false,
   debug = false,
 ): void => {
@@ -15,6 +16,8 @@ export const usePostHog = (
 
   //start PostHog once
   useEffect(() => {
+    // Respect telemetry settings
+    if (!telemetryEnabled) return;
     if (apiKey === undefined || apiKey === "") return;
     if (postHogInitialized.current === true) return;
     if (logging) console.log("Initializing PostHog");
@@ -27,19 +30,26 @@ export const usePostHog = (
         if (logging) console.log("PostHog loaded");
         if (user !== undefined) {
           if (logging) console.log("Loaded: Identifying user", user);
-          posthog.identify(user.id, { email: user.email });
+          posthog.identify(user.id, {
+            email: user.email,
+            name: user.name,
+          });
         }
       },
     });
     postHogInitialized.current = true;
-  }, [apiKey, logging, user]);
+  }, [apiKey, telemetryEnabled, logging, user]);
 
   useUserChanged((user) => {
     if (postHogInitialized.current === false) return;
+    if (!telemetryEnabled) return;
     if (logging) console.log("User changed");
     if (user) {
       if (logging) console.log("Identifying user", user);
-      posthog.identify(user.id, { email: user.email });
+      posthog.identify(user.id, {
+        email: user.email,
+        name: user.name,
+      });
     } else {
       if (logging) console.log("Resetting user");
       posthog.reset();

apps/webapp/app/jobs/conversation/create-title.logic.ts

@@ -0,0 +1,82 @@
+import { conversationTitlePrompt } from "~/trigger/conversation/prompt";
+import { prisma } from "~/trigger/utils/prisma";
+import { logger } from "~/services/logger.service";
+import { generateText, type LanguageModel } from "ai";
+import { getModel } from "~/lib/model.server";
+
+export interface CreateConversationTitlePayload {
+  conversationId: string;
+  message: string;
+}
+
+export interface CreateConversationTitleResult {
+  success: boolean;
+  title?: string;
+  error?: string;
+}
+
+/**
+ * Core business logic for creating conversation titles
+ * This is shared between Trigger.dev and BullMQ implementations
+ */
+export async function processConversationTitleCreation(
+  payload: CreateConversationTitlePayload,
+): Promise<CreateConversationTitleResult> {
+  try {
+    let conversationTitleResponse = "";
+    const { text } = await generateText({
+      model: getModel() as LanguageModel,
+      messages: [
+        {
+          role: "user",
+          content: conversationTitlePrompt.replace(
+            "{{message}}",
+            payload.message,
+          ),
+        },
+      ],
+    });
+
+    const outputMatch = text.match(/<output>(.*?)<\/output>/s);
+
+    logger.info(`Conversation title data: ${JSON.stringify(outputMatch)}`);
+
+    if (!outputMatch) {
+      logger.error("No output found in recurrence response");
+      throw new Error("Invalid response format from AI");
+    }
+
+    const jsonStr = outputMatch[1].trim();
+    const conversationTitleData = JSON.parse(jsonStr);
+
+    if (conversationTitleData) {
+      await prisma.conversation.update({
+        where: {
+          id: payload.conversationId,
+        },
+        data: {
+          title: conversationTitleData.title,
+        },
+      });
+
+      return {
+        success: true,
+        title: conversationTitleData.title,
+      };
+    }
+
+    return {
+      success: false,
+      error: "No title generated",
+    };
+  } catch (error: any) {
+    logger.error(
+      `Error creating conversation title for ${payload.conversationId}:`,
+      error,
+    );
+    return {
+      success: false,
+      error: error.message,
+    };
+  }
+}

apps/webapp/app/jobs/ingest/ingest-document.logic.ts

@@ -0,0 +1,290 @@
+import { type z } from "zod";
+
+import { IngestionStatus } from "@core/database";
+import { EpisodeTypeEnum } from "@core/types";
+import { logger } from "~/services/logger.service";
+import { saveDocument } from "~/services/graphModels/document";
+
+import { DocumentVersioningService } from "~/services/documentVersioning.server";
+import { DocumentDifferentialService } from "~/services/documentDiffer.server";
+import { KnowledgeGraphService } from "~/services/knowledgeGraph.server";
+import { prisma } from "~/trigger/utils/prisma";
+import { type IngestBodyRequest } from "./ingest-episode.logic";
+
+export interface IngestDocumentPayload {
+  body: z.infer<typeof IngestBodyRequest>;
+  userId: string;
+  workspaceId: string;
+  queueId: string;
+}
+
+export interface IngestDocumentResult {
+  success: boolean;
+  error?: string;
+}
+
+/**
+ * Core business logic for document ingestion with differential processing
+ * This is shared between Trigger.dev and BullMQ implementations
+ *
+ * Note: This function should NOT call trigger functions directly for chunk processing.
+ * Instead, use the enqueueEpisodeIngestion callback to queue episode ingestion jobs.
+ */
+export async function processDocumentIngestion(
+  payload: IngestDocumentPayload,
+  // Callback function for enqueueing episode ingestion for each chunk
+  enqueueEpisodeIngestion?: (params: {
+    body: any;
+    userId: string;
+    workspaceId: string;
+    queueId: string;
+  }) => Promise<{ id?: string }>,
+): Promise<IngestDocumentResult> {
+  const startTime = Date.now();
+
+  try {
+    logger.log(`Processing document for user ${payload.userId}`, {
+      contentLength: payload.body.episodeBody.length,
+    });
+
+    await prisma.ingestionQueue.update({
+      where: { id: payload.queueId },
+      data: {
+        status: IngestionStatus.PROCESSING,
+      },
+    });
+
+    const documentBody = payload.body;
+
+    // Step 1: Initialize services and prepare document version
+    const versioningService = new DocumentVersioningService();
+    const differentialService = new DocumentDifferentialService();
+    const knowledgeGraphService = new KnowledgeGraphService();
+
+    const {
+      documentNode: document,
+      versionInfo,
+      chunkedDocument,
+    } = await versioningService.prepareDocumentVersion(
+      documentBody.sessionId!,
+      payload.userId,
+      documentBody.metadata?.documentTitle?.toString() || "Untitled Document",
+      documentBody.episodeBody,
+      documentBody.source,
+      documentBody.metadata || {},
+    );
+
+    logger.log(`Document version analysis:`, {
+      version: versionInfo.newVersion,
+      isNewDocument: versionInfo.isNewDocument,
+      hasContentChanged: versionInfo.hasContentChanged,
+      changePercentage: versionInfo.chunkLevelChanges.changePercentage,
+      changedChunks: versionInfo.chunkLevelChanges.changedChunkIndices.length,
+      totalChunks: versionInfo.chunkLevelChanges.totalChunks,
+    });
+
+    // Step 2: Determine processing strategy
+    const differentialDecision =
+      await differentialService.analyzeDifferentialNeed(
+        documentBody.episodeBody,
+        versionInfo.existingDocument,
+        chunkedDocument,
+      );
+
+    logger.log(`Differential analysis:`, {
+      shouldUseDifferential: differentialDecision.shouldUseDifferential,
+      strategy: differentialDecision.strategy,
+      reason: differentialDecision.reason,
+      documentSizeTokens: differentialDecision.documentSizeTokens,
+    });
+
+    // Early return for unchanged documents
+    if (differentialDecision.strategy === "skip_processing") {
+      logger.log("Document content unchanged, skipping processing");
+      await prisma.ingestionQueue.update({
+        where: { id: payload.queueId },
+        data: {
+          status: IngestionStatus.COMPLETED,
+        },
+      });
+      return {
+        success: true,
+      };
+    }
+
+    // Step 3: Save the new document version
+    await saveDocument(document);
+
+    // Step 3.1: Invalidate statements from previous document version if it exists
+    let invalidationResults = null;
+    if (versionInfo.existingDocument && versionInfo.hasContentChanged) {
+      logger.log(
+        `Invalidating statements from previous document version: ${versionInfo.existingDocument.uuid}`,
+      );
+
+      invalidationResults =
+        await knowledgeGraphService.invalidateStatementsFromPreviousDocumentVersion(
+          {
+            previousDocumentUuid: versionInfo.existingDocument.uuid,
+            newDocumentContent: documentBody.episodeBody,
+            userId: payload.userId,
+            invalidatedBy: document.uuid,
+            semanticSimilarityThreshold: 0.75, // Configurable threshold
+          },
+        );
+
+      logger.log(`Statement invalidation completed:`, {
+        totalAnalyzed: invalidationResults.totalStatementsAnalyzed,
+        invalidated: invalidationResults.invalidatedStatements.length,
+        preserved: invalidationResults.preservedStatements.length,
+      });
+    }
+
+    logger.log(`Document chunked into ${chunkedDocument.chunks.length} chunks`);
+
+    // Step 4: Process chunks based on differential strategy
+    let chunksToProcess = chunkedDocument.chunks;
+    let processingMode = "full";
+
+    if (
+      differentialDecision.shouldUseDifferential &&
+      differentialDecision.strategy === "chunk_level_diff"
+    ) {
+      // Only process changed chunks
+      const chunkComparisons = differentialService.getChunkComparisons(
+        versionInfo.existingDocument!,
+        chunkedDocument,
+      );
+
+      const changedIndices =
+        differentialService.getChunksNeedingReprocessing(chunkComparisons);
+      chunksToProcess = chunkedDocument.chunks.filter((chunk) =>
+        changedIndices.includes(chunk.chunkIndex),
+      );
+      processingMode = "differential";
+
+      logger.log(
+        `Differential processing: ${chunksToProcess.length}/${chunkedDocument.chunks.length} chunks need reprocessing`,
+      );
+    } else if (differentialDecision.strategy === "full_reingest") {
+      // Process all chunks
+      processingMode = "full";
+      logger.log(
+        `Full reingestion: processing all ${chunkedDocument.chunks.length} chunks`,
+      );
+    }
+
+    // Step 5: Queue chunks for processing
+    const episodeHandlers = [];
+    if (enqueueEpisodeIngestion) {
+      for (const chunk of chunksToProcess) {
+        const chunkEpisodeData = {
+          episodeBody: chunk.content,
+          referenceTime: documentBody.referenceTime,
+          metadata: {
+            ...documentBody.metadata,
+            processingMode,
+            differentialStrategy: differentialDecision.strategy,
+            chunkHash: chunk.contentHash,
+            documentTitle:
+              documentBody.metadata?.documentTitle?.toString() ||
+              "Untitled Document",
+            chunkIndex: chunk.chunkIndex,
+            documentUuid: document.uuid,
+          },
+          source: documentBody.source,
+          spaceIds: documentBody.spaceIds,
+          sessionId: documentBody.sessionId,
+          type: EpisodeTypeEnum.DOCUMENT,
+        };
+
+        const episodeHandler = await enqueueEpisodeIngestion({
+          body: chunkEpisodeData,
+          userId: payload.userId,
+          workspaceId: payload.workspaceId,
+          queueId: payload.queueId,
+        });
+
+        if (episodeHandler.id) {
+          episodeHandlers.push(episodeHandler.id);
+          logger.log(
+            `Queued chunk ${chunk.chunkIndex + 1} for ${processingMode} processing`,
+            {
+              handlerId: episodeHandler.id,
+              chunkSize: chunk.content.length,
+              chunkHash: chunk.contentHash,
+            },
+          );
+        }
+      }
+    }
+
+    // Calculate cost savings
+    const costSavings = differentialService.calculateCostSavings(
+      chunkedDocument.chunks.length,
+      chunksToProcess.length,
+    );
+
+    await prisma.ingestionQueue.update({
+      where: { id: payload.queueId },
+      data: {
+        output: {
+          documentUuid: document.uuid,
+          version: versionInfo.newVersion,
+          totalChunks: chunkedDocument.chunks.length,
+          chunksProcessed: chunksToProcess.length,
+          chunksSkipped: costSavings.chunksSkipped,
+          processingMode,
+          differentialStrategy: differentialDecision.strategy,
+          estimatedSavings: `${costSavings.estimatedSavingsPercentage.toFixed(1)}%`,
+          statementInvalidation: invalidationResults
+            ? {
+                totalAnalyzed: invalidationResults.totalStatementsAnalyzed,
+                invalidated: invalidationResults.invalidatedStatements.length,
+                preserved: invalidationResults.preservedStatements.length,
+              }
+            : null,
+          episodes: [],
+          episodeHandlers,
+        },
+        status: IngestionStatus.PROCESSING,
+      },
+    });
+
+    const processingTimeMs = Date.now() - startTime;
+
+    logger.log(
+      `Document differential processing completed in ${processingTimeMs}ms`,
+      {
+        documentUuid: document.uuid,
+        version: versionInfo.newVersion,
+        processingMode,
+        totalChunks: chunkedDocument.chunks.length,
+        chunksProcessed: chunksToProcess.length,
+        chunksSkipped: costSavings.chunksSkipped,
+        estimatedSavings: `${costSavings.estimatedSavingsPercentage.toFixed(1)}%`,
+        changePercentage: `${differentialDecision.changePercentage.toFixed(1)}%`,
+        statementInvalidation: invalidationResults
+          ? {
+              totalAnalyzed: invalidationResults.totalStatementsAnalyzed,
+              invalidated: invalidationResults.invalidatedStatements.length,
+              preserved: invalidationResults.preservedStatements.length,
+            }
+          : "No previous version",
+      },
+    );
+
+    return { success: true };
+  } catch (err: any) {
+    await prisma.ingestionQueue.update({
+      where: { id: payload.queueId },
+      data: {
+        error: err.message,
+        status: IngestionStatus.FAILED,
+      },
+    });
+
+    logger.error(`Error processing document for user ${payload.userId}:`, err);
+    return { success: false, error: err.message };
+  }
+}

apps/webapp/app/jobs/ingest/ingest-episode.logic.ts

@@ -0,0 +1,266 @@
+import { z } from "zod";
+import { KnowledgeGraphService } from "~/services/knowledgeGraph.server";
+import { linkEpisodeToDocument } from "~/services/graphModels/document";
+import { IngestionStatus } from "@core/database";
+import { logger } from "~/services/logger.service";
+import { prisma } from "~/trigger/utils/prisma";
+import { EpisodeType } from "@core/types";
+import { deductCredits, hasCredits } from "~/trigger/utils/utils";
+import { assignEpisodesToSpace } from "~/services/graphModels/space";
+
+export const IngestBodyRequest = z.object({
+  episodeBody: z.string(),
+  referenceTime: z.string(),
+  metadata: z.record(z.union([z.string(), z.number(), z.boolean()])).optional(),
+  source: z.string(),
+  spaceIds: z.array(z.string()).optional(),
+  sessionId: z.string().optional(),
+  type: z
+    .enum([EpisodeType.CONVERSATION, EpisodeType.DOCUMENT])
+    .default(EpisodeType.CONVERSATION),
+});
+
+export interface IngestEpisodePayload {
+  body: z.infer<typeof IngestBodyRequest>;
+  userId: string;
+  workspaceId: string;
+  queueId: string;
+}
+
+export interface IngestEpisodeResult {
+  success: boolean;
+  episodeDetails?: any;
+  error?: string;
+}
+
+/**
+ * Core business logic for ingesting a single episode
+ * This is shared between Trigger.dev and BullMQ implementations
+ *
+ * Note: This function should NOT call trigger functions directly.
+ * Instead, return data that indicates follow-up jobs are needed,
+ * and let the caller (Trigger task or BullMQ worker) handle job queueing.
+ */
+export async function processEpisodeIngestion(
+  payload: IngestEpisodePayload,
+  // Callback functions for enqueueing follow-up jobs
+  enqueueSpaceAssignment?: (params: {
+    userId: string;
+    workspaceId: string;
+    mode: "episode";
+    episodeIds: string[];
+  }) => Promise<any>,
+  enqueueSessionCompaction?: (params: {
+    userId: string;
+    sessionId: string;
+    source: string;
+  }) => Promise<any>,
+): Promise<IngestEpisodeResult> {
+  try {
+    logger.log(`Processing job for user ${payload.userId}`);
+
+    // Check if workspace has sufficient credits before processing
+    const hasSufficientCredits = await hasCredits(
+      payload.workspaceId,
+      "addEpisode",
+    );
+
+    if (!hasSufficientCredits) {
+      logger.warn(`Insufficient credits for workspace ${payload.workspaceId}`);
+
+      await prisma.ingestionQueue.update({
+        where: { id: payload.queueId },
+        data: {
+          status: IngestionStatus.NO_CREDITS,
+          error:
+            "Insufficient credits. Please upgrade your plan or wait for your credits to reset.",
+        },
+      });
+
+      return {
+        success: false,
+        error: "Insufficient credits",
+      };
+    }
+
+    const ingestionQueue = await prisma.ingestionQueue.update({
+      where: { id: payload.queueId },
+      data: {
+        status: IngestionStatus.PROCESSING,
+      },
+    });
+
+    const knowledgeGraphService = new KnowledgeGraphService();
+
+    const episodeBody = payload.body as any;
+
+    const episodeDetails = await knowledgeGraphService.addEpisode(
+      {
+        ...episodeBody,
+        userId: payload.userId,
+      },
+      prisma,
+    );
+
+    // Link episode to document if it's a document chunk
+    if (
+      episodeBody.type === EpisodeType.DOCUMENT &&
+      episodeBody.metadata.documentUuid &&
+      episodeDetails.episodeUuid
+    ) {
+      try {
+        await linkEpisodeToDocument(
+          episodeDetails.episodeUuid,
+          episodeBody.metadata.documentUuid,
+          episodeBody.metadata.chunkIndex || 0,
+        );
+        logger.log(
+          `Linked episode ${episodeDetails.episodeUuid} to document ${episodeBody.metadata.documentUuid} at chunk ${episodeBody.metadata.chunkIndex || 0}`,
+        );
+      } catch (error) {
+        logger.error(`Failed to link episode to document:`, {
+          error,
+          episodeUuid: episodeDetails.episodeUuid,
+          documentUuid: episodeBody.metadata.documentUuid,
+        });
+      }
+    }
+
+    let finalOutput = episodeDetails;
+    let episodeUuids: string[] = episodeDetails.episodeUuid
+      ? [episodeDetails.episodeUuid]
+      : [];
+    let currentStatus: IngestionStatus = IngestionStatus.COMPLETED;
+    if (episodeBody.type === EpisodeType.DOCUMENT) {
+      const currentOutput = ingestionQueue.output as any;
+      currentOutput.episodes.push(episodeDetails);
+      episodeUuids = currentOutput.episodes.map(
+        (episode: any) => episode.episodeUuid,
+      );
+
+      finalOutput = {
+        ...currentOutput,
+      };
+
+      if (currentOutput.episodes.length !== currentOutput.totalChunks) {
+        currentStatus = IngestionStatus.PROCESSING;
+      }
+    }
+
+    await prisma.ingestionQueue.update({
+      where: { id: payload.queueId },
+      data: {
+        output: finalOutput,
+        status: currentStatus,
+      },
+    });
+
+    // Deduct credits for episode creation
+    if (currentStatus === IngestionStatus.COMPLETED) {
+      await deductCredits(
+        payload.workspaceId,
+        "addEpisode",
+        finalOutput.statementsCreated,
+      );
+    }
+
+    // Handle space assignment after successful ingestion
+    try {
+      // If spaceIds were explicitly provided, immediately assign the episode to those spaces
+      if (
+        episodeBody.spaceIds &&
+        episodeBody.spaceIds.length > 0 &&
+        episodeDetails.episodeUuid
+      ) {
+        logger.info(`Assigning episode to explicitly provided spaces`, {
+          userId: payload.userId,
+          episodeId: episodeDetails.episodeUuid,
+          spaceIds: episodeBody.spaceIds,
+        });
+
+        // Assign episode to each space
+        for (const spaceId of episodeBody.spaceIds) {
+          await assignEpisodesToSpace(
+            [episodeDetails.episodeUuid],
+            spaceId,
+            payload.userId,
+          );
+        }
+
+        logger.info(
+          `Skipping LLM space assignment - episode explicitly assigned to ${episodeBody.spaceIds.length} space(s)`,
+        );
+      } else {
+        // Only trigger automatic LLM space assignment if no explicit spaceIds were provided
+        logger.info(
+          `Triggering LLM space assignment after successful ingestion`,
+          {
+            userId: payload.userId,
+            workspaceId: payload.workspaceId,
+            episodeId: episodeDetails?.episodeUuid,
+          },
+        );
+        if (
+          episodeDetails.episodeUuid &&
+          currentStatus === IngestionStatus.COMPLETED &&
+          enqueueSpaceAssignment
+        ) {
+          await enqueueSpaceAssignment({
+            userId: payload.userId,
+            workspaceId: payload.workspaceId,
+            mode: "episode",
+            episodeIds: episodeUuids,
+          });
+        }
+      }
+    } catch (assignmentError) {
+      // Don't fail the ingestion if assignment fails
+      logger.warn(`Failed to trigger space assignment after ingestion:`, {
+        error: assignmentError,
+        userId: payload.userId,
+        episodeId: episodeDetails?.episodeUuid,
+      });
+    }
+
+    // Auto-trigger session compaction if episode has sessionId
+    try {
+      if (
+        episodeBody.sessionId &&
+        currentStatus === IngestionStatus.COMPLETED &&
+        enqueueSessionCompaction
+      ) {
+        logger.info(`Checking if session compaction should be triggered`, {
+          userId: payload.userId,
+          sessionId: episodeBody.sessionId,
+          source: episodeBody.source,
+        });
+
+        await enqueueSessionCompaction({
+          userId: payload.userId,
+          sessionId: episodeBody.sessionId,
+          source: episodeBody.source,
+        });
+      }
+    } catch (compactionError) {
+      // Don't fail the ingestion if compaction fails
+      logger.warn(`Failed to trigger session compaction after ingestion:`, {
+        error: compactionError,
+        userId: payload.userId,
+        sessionId: episodeBody.sessionId,
+      });
+    }
+
+    return { success: true, episodeDetails };
+  } catch (err: any) {
+    await prisma.ingestionQueue.update({
+      where: { id: payload.queueId },
+      data: {
+        error: err.message,
+        status: IngestionStatus.FAILED,
+      },
+    });
+
+    logger.error(`Error processing job for user ${payload.userId}:`, err);
+    return { success: false, error: err.message };
+  }
+}

apps/webapp/app/jobs/session/session-compaction.logic.ts

@@ -0,0 +1,455 @@
+import { logger } from "~/services/logger.service";
+import type { CoreMessage } from "ai";
+import { z } from "zod";
+import { getEmbedding, makeModelCall } from "~/lib/model.server";
+import {
+  getCompactedSessionBySessionId,
+  linkEpisodesToCompact,
+  getSessionEpisodes,
+  type CompactedSessionNode,
+  type SessionEpisodeData,
+  saveCompactedSession,
+} from "~/services/graphModels/compactedSession";
+
+export interface SessionCompactionPayload {
+  userId: string;
+  sessionId: string;
+  source: string;
+  triggerSource?: "auto" | "manual" | "threshold";
+}
+
+export interface SessionCompactionResult {
+  success: boolean;
+  compactionResult?: {
+    compactUuid: string;
+    sessionId: string;
+    summary: string;
+    episodeCount: number;
+    startTime: Date;
+    endTime: Date;
+    confidence: number;
+    compressionRatio: number;
+  };
+  reason?: string;
+  episodeCount?: number;
+  error?: string;
+}
+
+// Zod schema for LLM response validation
+const CompactionResultSchema = z.object({
+  summary: z.string().describe("Consolidated narrative of the entire session"),
+  confidence: z
+    .number()
+    .min(0)
+    .max(1)
+    .describe("Confidence score of the compaction quality"),
+});
+
+const CONFIG = {
+  minEpisodesForCompaction: 5, // Minimum episodes to trigger compaction
+  compactionThreshold: 1, // Trigger after N new episodes
+  maxEpisodesPerBatch: 50, // Process in batches if needed
+};
+
+/**
+ * Core business logic for session compaction
+ * This is shared between Trigger.dev and BullMQ implementations
+ */
+export async function processSessionCompaction(
+  payload: SessionCompactionPayload,
+): Promise<SessionCompactionResult> {
+  const { userId, sessionId, source, triggerSource = "auto" } = payload;
+
+  logger.info(`Starting session compaction`, {
+    userId,
+    sessionId,
+    source,
+    triggerSource,
+  });
+
+  try {
+    // Check if compaction already exists
+    const existingCompact = await getCompactedSessionBySessionId(
+      sessionId,
+      userId,
+    );
+
+    // Fetch all episodes for this session
+    const episodes = await getSessionEpisodes(
+      sessionId,
+      userId,
+      existingCompact?.endTime,
+    );
+
+    console.log("episodes", episodes.length);
+    // Check if we have enough episodes
+    if (!existingCompact && episodes.length < CONFIG.minEpisodesForCompaction) {
+      logger.info(`Not enough episodes for compaction`, {
+        sessionId,
+        episodeCount: episodes.length,
+        minRequired: CONFIG.minEpisodesForCompaction,
+      });
+      return {
+        success: false,
+        reason: "insufficient_episodes",
+        episodeCount: episodes.length,
+      };
+    } else if (
+      existingCompact &&
+      episodes.length <
+        CONFIG.minEpisodesForCompaction + CONFIG.compactionThreshold
+    ) {
+      logger.info(`Not enough new episodes for compaction`, {
+        sessionId,
+        episodeCount: episodes.length,
+        minRequired:
+          CONFIG.minEpisodesForCompaction + CONFIG.compactionThreshold,
+      });
+      return {
+        success: false,
+        reason: "insufficient_new_episodes",
+        episodeCount: episodes.length,
+      };
+    }
+
+    // Generate or update compaction
+    const compactionResult = existingCompact
+      ? await updateCompaction(existingCompact, episodes, userId)
+      : await createCompaction(sessionId, episodes, userId, source);
+
+    logger.info(`Session compaction completed`, {
+      sessionId,
+      compactUuid: compactionResult.uuid,
+      episodeCount: compactionResult.episodeCount,
+      compressionRatio: compactionResult.compressionRatio,
+    });
+
+    return {
+      success: true,
+      compactionResult: {
+        compactUuid: compactionResult.uuid,
+        sessionId: compactionResult.sessionId,
+        summary: compactionResult.summary,
+        episodeCount: compactionResult.episodeCount,
+        startTime: compactionResult.startTime,
+        endTime: compactionResult.endTime,
+        confidence: compactionResult.confidence,
+        compressionRatio: compactionResult.compressionRatio,
+      },
+    };
+  } catch (error) {
+    logger.error(`Session compaction failed`, {
+      sessionId,
+      userId,
+      error: error instanceof Error ? error.message : String(error),
+    });
+
+    return {
+      success: false,
+      error: error instanceof Error ? error.message : String(error),
+    };
+  }
+}
+
+/**
+ * Create new compaction
+ */
+async function createCompaction(
+  sessionId: string,
+  episodes: SessionEpisodeData[],
+  userId: string,
+  source: string,
+): Promise<CompactedSessionNode> {
+  logger.info(`Creating new compaction`, {
+    sessionId,
+    episodeCount: episodes.length,
+  });
+
+  // Generate compaction using LLM
+  const compactionData = await generateCompaction(episodes, null);
+
+  // Generate embedding for summary
+  const summaryEmbedding = await getEmbedding(compactionData.summary);
+
+  // Create CompactedSession node using graph model
+  const compactUuid = crypto.randomUUID();
+  const now = new Date();
+  const startTime = new Date(episodes[0].createdAt);
+  const endTime = new Date(episodes[episodes.length - 1].createdAt);
+  const episodeUuids = episodes.map((e) => e.uuid);
+  const compressionRatio = episodes.length / 1;
+
+  const compactNode: CompactedSessionNode = {
+    uuid: compactUuid,
+    sessionId,
+    summary: compactionData.summary,
+    summaryEmbedding,
+    episodeCount: episodes.length,
+    startTime,
+    endTime,
+    createdAt: now,
+    confidence: compactionData.confidence,
+    userId,
+    source,
+    compressionRatio,
+    metadata: { triggerType: "create" },
+  };
+
+  console.log("compactNode", compactNode);
+  // Use graph model functions
+  await saveCompactedSession(compactNode);
+  await linkEpisodesToCompact(compactUuid, episodeUuids, userId);
+
+  logger.info(`Compaction created`, {
+    compactUuid,
+    episodeCount: episodes.length,
+  });
+
+  return compactNode;
+}
+
+/**
+ * Update existing compaction with new episodes
+ */
+async function updateCompaction(
+  existingCompact: CompactedSessionNode,
+  newEpisodes: SessionEpisodeData[],
+  userId: string,
+): Promise<CompactedSessionNode> {
+  logger.info(`Updating existing compaction`, {
+    compactUuid: existingCompact.uuid,
+    newEpisodeCount: newEpisodes.length,
+  });
+
+  // Generate updated compaction using LLM (merging)
+  const compactionData = await generateCompaction(
+    newEpisodes,
+    existingCompact.summary,
+  );
+
+  // Generate new embedding for updated summary
+  const summaryEmbedding = await getEmbedding(compactionData.summary);
+
+  // Update CompactedSession node using graph model
+  const now = new Date();
+  const endTime = newEpisodes[newEpisodes.length - 1].createdAt;
+  const totalEpisodeCount = existingCompact.episodeCount + newEpisodes.length;
+  const compressionRatio = totalEpisodeCount / 1;
+  const episodeUuids = newEpisodes.map((e) => e.uuid);
+
+  const updatedNode: CompactedSessionNode = {
+    ...existingCompact,
+    summary: compactionData.summary,
+    summaryEmbedding,
+    episodeCount: totalEpisodeCount,
+    endTime,
+    updatedAt: now,
+    confidence: compactionData.confidence,
+    compressionRatio,
+    metadata: { triggerType: "update", newEpisodesAdded: newEpisodes.length },
+  };
+
+  // Use graph model functions
+  await saveCompactedSession(updatedNode);
+  await linkEpisodesToCompact(existingCompact.uuid, episodeUuids, userId);
+
+  logger.info(`Compaction updated`, {
+    compactUuid: existingCompact.uuid,
+    totalEpisodeCount,
+  });
+
+  return updatedNode;
+}
+
+/**
+ * Generate compaction using LLM (similar to Claude Code's compact approach)
+ */
+async function generateCompaction(
+  episodes: SessionEpisodeData[],
+  existingSummary: string | null,
+): Promise<z.infer<typeof CompactionResultSchema>> {
+  const systemPrompt = createCompactionSystemPrompt();
+  const userPrompt = createCompactionUserPrompt(episodes, existingSummary);
+
+  const messages: CoreMessage[] = [
+    { role: "system", content: systemPrompt },
+    { role: "user", content: userPrompt },
+  ];
+
+  logger.info(`Generating compaction with LLM`, {
+    episodeCount: episodes.length,
+    hasExistingSummary: !!existingSummary,
+  });
+
+  try {
+    let responseText = "";
+    await makeModelCall(
+      false,
+      messages,
+      (text: string) => {
+        responseText = text;
+      },
+      undefined,
+      "high",
+    );
+
+    return parseCompactionResponse(responseText);
+  } catch (error) {
+    logger.error(`Failed to generate compaction`, {
+      error: error instanceof Error ? error.message : String(error),
+    });
+    throw error;
+  }
+}
+
+/**
+ * System prompt for compaction (for agent recall/context retrieval)
+ */
+function createCompactionSystemPrompt(): string {
+  return `You are a session compaction specialist. Your task is to create a rich, informative summary that will help AI agents understand what happened in this conversation session when they need context for future interactions.
+
+## PURPOSE
+
+This summary will be retrieved by AI agents when the user references this session in future conversations. The agent needs enough context to:
+- Understand what was discussed and why
+- Know what decisions were made and their rationale
+- Grasp the outcome and current state
+- Have relevant technical details to provide informed responses
+
+## COMPACTION GOALS
+
+1. **Comprehensive Context**: Capture all important information that might be referenced later
+2. **Decision Documentation**: Clearly state what was decided, why, and what alternatives were considered
+3. **Technical Details**: Include specific implementations, tools, configurations, and technical choices
+4. **Outcome Clarity**: Make it clear what was accomplished and what the final state is
+5. **Evolution Tracking**: Show how thinking or decisions evolved during the session
+
+## COMPACTION RULES
+
+1. **Be Information-Dense**: Pack useful details without fluff or repetition
+2. **Structure Chronologically**: Start with problem/question, show progression, end with outcome
+3. **Highlight Key Points**: Emphasize decisions, implementations, results, and learnings
+4. **Include Specifics**: Names of libraries, specific configurations, metrics, numbers matter
+5. **Resolve Contradictions**: Always use the most recent/final version when information conflicts
+
+## OUTPUT REQUIREMENTS
+
+- **summary**: A detailed, information-rich narrative that tells the complete story
+  - Structure naturally based on content - use as many paragraphs as needed
+  - Each distinct topic, decision, or phase should get its own paragraph(s)
+  - Start with context and initial problem/question
+  - Progress chronologically through discussions, decisions, and implementations
+  - **Final paragraph MUST**: State the outcome, results, and current state
+  - Don't artificially limit length - capture everything important
+
+- **confidence**: Score (0-1) reflecting how well this summary captures the session's essence
+
+Your response MUST be valid JSON wrapped in <output></output> tags.
+
+## KEY PRINCIPLES
+
+- Write for an AI agent that needs to help the user in future conversations
+- Include technical specifics that might be referenced (library names, configurations, metrics)
+- Make outcomes and current state crystal clear in the final paragraph
+- Show the reasoning behind decisions, not just the decisions themselves
+- Be comprehensive but concise - every sentence should add value
+- Each major topic or phase deserves its own paragraph(s)
+- Don't compress too much - agents need the details
+`;
+}
+
+/**
+ * User prompt for compaction
+ */
+function createCompactionUserPrompt(
+  episodes: SessionEpisodeData[],
+  existingSummary: string | null,
+): string {
+  let prompt = "";
+
+  if (existingSummary) {
+    prompt += `## EXISTING SUMMARY (from previous compaction)\n\n${existingSummary}\n\n`;
+    prompt += `## NEW EPISODES (to merge into existing summary)\n\n`;
+  } else {
+    prompt += `## SESSION EPISODES (to compact)\n\n`;
+  }
+
+  episodes.forEach((episode, index) => {
+    const timestamp = new Date(episode.validAt).toISOString();
+    prompt += `### Episode ${index + 1} (${timestamp})\n`;
+    prompt += `Source: ${episode.source}\n`;
+    prompt += `Content:\n${episode.originalContent}\n\n`;
+  });
+
+  if (existingSummary) {
+    prompt += `\n## INSTRUCTIONS\n\n`;
+    prompt += `Merge the new episodes into the existing summary. Update facts, add new information, and maintain narrative coherence. Ensure the consolidated summary reflects the complete session including both old and new content.\n`;
+  } else {
+    prompt += `\n## INSTRUCTIONS\n\n`;
+    prompt += `Create a compact summary of this entire session. Consolidate all information into a coherent narrative with deduplicated key facts.\n`;
+  }
+
+  return prompt;
+}
+
+/**
+ * Parse LLM response for compaction
+ */
+function parseCompactionResponse(
+  response: string,
+): z.infer<typeof CompactionResultSchema> {
+  try {
+    // Extract content from <output> tags
+    const outputMatch = response.match(/<output>([\s\S]*?)<\/output>/);
+    if (!outputMatch) {
+      logger.warn("No <output> tags found in LLM compaction response");
+      logger.debug("Full LLM response:", { response });
+      throw new Error("Invalid LLM response format - missing <output> tags");
+    }
+
+    let jsonContent = outputMatch[1].trim();
+
+    // Remove markdown code blocks if present
+    jsonContent = jsonContent.replace(/```json\n?/g, "").replace(/```\n?/g, "");
+
+    const parsed = JSON.parse(jsonContent);
+
+    // Validate with schema
+    const validated = CompactionResultSchema.parse(parsed);
+
+    return validated;
+  } catch (error) {
+    logger.error("Failed to parse compaction response", {
+      error: error instanceof Error ? error.message : String(error),
+      response: response.substring(0, 500),
+    });
+    throw new Error(`Failed to parse compaction response: ${error}`);
+  }
+}
+
+/**
+ * Helper function to check if compaction should be triggered
+ */
+export async function shouldTriggerCompaction(
+  sessionId: string,
+  userId: string,
+): Promise<boolean> {
+  const existingCompact = await getCompactedSessionBySessionId(
+    sessionId,
+    userId,
+  );
+
+  if (!existingCompact) {
+    // Check if we have enough episodes for initial compaction
+    const episodes = await getSessionEpisodes(sessionId, userId);
+    return episodes.length >= CONFIG.minEpisodesForCompaction;
+  }
+
+  // Check if we have enough new episodes to update
+  const newEpisodes = await getSessionEpisodes(
+    sessionId,
+    userId,
+    existingCompact.endTime,
+  );
+  return newEpisodes.length >= CONFIG.compactionThreshold;
+}

apps/webapp/app/lib/ingest.server.ts

@@ -4,13 +4,18 @@ import { EpisodeType } from "@core/types";
 import { type z } from "zod";
 import { prisma } from "~/db.server";
 import { hasCredits } from "~/services/billing.server";
-import { type IngestBodyRequest, ingestTask } from "~/trigger/ingest/ingest";
-import { ingestDocumentTask } from "~/trigger/ingest/ingest-document";
+import { type IngestBodyRequest } from "~/trigger/ingest/ingest";
+import {
+  enqueueIngestDocument,
+  enqueueIngestEpisode,
+} from "~/lib/queue-adapter.server";
+import { trackFeatureUsage } from "~/services/telemetry.server";
 
 export const addToQueue = async (
   rawBody: z.infer<typeof IngestBodyRequest>,
   userId: string,
   activityId?: string,
+  ingestionQueueId?: string,
 ) => {
   const body = { ...rawBody, source: rawBody.source.toLowerCase() };
   const user = await prisma.user.findFirst({
@@ -38,8 +43,18 @@ export const addToQueue = async (
     throw new Error("no credits");
   }
 
-  const queuePersist = await prisma.ingestionQueue.create({
-    data: {
+  // Upsert: update existing or create new ingestion queue entry
+  const queuePersist = await prisma.ingestionQueue.upsert({
+    where: {
+      id: ingestionQueueId || "non-existent-id", // Use provided ID or dummy ID to force create
+    },
+    update: {
+      data: body,
+      type: body.type,
+      status: IngestionStatus.PENDING,
+      error: null,
+    },
+    create: {
       data: body,
       type: body.type,
       status: IngestionStatus.PENDING,
@@ -51,36 +66,28 @@ export const addToQueue = async (
 
   let handler;
   if (body.type === EpisodeType.DOCUMENT) {
-    handler = await ingestDocumentTask.trigger(
-      {
-        body,
-        userId,
-        workspaceId: user.Workspace.id,
-        queueId: queuePersist.id,
-      },
-      {
-        queue: "document-ingestion-queue",
-        concurrencyKey: userId,
-        tags: [user.id, queuePersist.id],
-      },
-    );
+    handler = await enqueueIngestDocument({
+      body,
+      userId,
+      workspaceId: user.Workspace.id,
+      queueId: queuePersist.id,
+    });
+
+    // Track document ingestion
+    trackFeatureUsage("document_ingested", userId).catch(console.error);
   } else if (body.type === EpisodeType.CONVERSATION) {
-    handler = await ingestTask.trigger(
-      {
-        body,
-        userId,
-        workspaceId: user.Workspace.id,
-        queueId: queuePersist.id,
-      },
-      {
-        queue: "ingestion-queue",
-        concurrencyKey: userId,
-        tags: [user.id, queuePersist.id],
-      },
-    );
+    handler = await enqueueIngestEpisode({
+      body,
+      userId,
+      workspaceId: user.Workspace.id,
+      queueId: queuePersist.id,
+    });
+
+    // Track episode ingestion
+    trackFeatureUsage("episode_ingested", userId).catch(console.error);
   }
 
-  return { id: handler?.id, token: handler?.publicAccessToken };
+  return { id: handler?.id, publicAccessToken: handler?.token };
 };
 
 export { IngestBodyRequest };

apps/webapp/app/lib/model.server.ts

@@ -1,31 +1,23 @@
-import {
-  type CoreMessage,
-  type LanguageModelV1,
-  embed,
-  generateText,
-  streamText,
-} from "ai";
+import { type CoreMessage, embed, generateText, streamText } from "ai";
 import { openai } from "@ai-sdk/openai";
 import { logger } from "~/services/logger.service";
 
-import { createOllama, type OllamaProvider } from "ollama-ai-provider";
+import { createOllama } from "ollama-ai-provider-v2";
 import { anthropic } from "@ai-sdk/anthropic";
 import { google } from "@ai-sdk/google";
-import { createAmazonBedrock } from "@ai-sdk/amazon-bedrock";
-import { fromNodeProviderChain } from "@aws-sdk/credential-providers";
 
-export type ModelComplexity = 'high' | 'low';
+export type ModelComplexity = "high" | "low";
 
 /**
  * Get the appropriate model for a given complexity level.
  * HIGH complexity uses the configured MODEL.
  * LOW complexity automatically downgrades to cheaper variants if possible.
  */
-export function getModelForTask(complexity: ModelComplexity = 'high'): string {
-  const baseModel = process.env.MODEL || 'gpt-4.1-2025-04-14';
+export function getModelForTask(complexity: ModelComplexity = "high"): string {
+  const baseModel = process.env.MODEL || "gpt-4.1-2025-04-14";
 
   // HIGH complexity - always use the configured model
-  if (complexity === 'high') {
+  if (complexity === "high") {
     return baseModel;
   }
 
@@ -33,29 +25,73 @@ export function getModelForTask(complexity: ModelComplexity = 'high'): string {
   // If already using a cheap model, keep it
   const downgrades: Record<string, string> = {
     // OpenAI downgrades
-    'gpt-5-2025-08-07': 'gpt-5-mini-2025-08-07',
-    'gpt-4.1-2025-04-14': 'gpt-4.1-mini-2025-04-14',
+    "gpt-5-2025-08-07": "gpt-5-mini-2025-08-07",
+    "gpt-4.1-2025-04-14": "gpt-4.1-mini-2025-04-14",
 
     // Anthropic downgrades
-    'claude-sonnet-4-5': 'claude-3-5-haiku-20241022',
-    'claude-3-7-sonnet-20250219': 'claude-3-5-haiku-20241022',
-    'claude-3-opus-20240229': 'claude-3-5-haiku-20241022',
+    "claude-sonnet-4-5": "claude-3-5-haiku-20241022",
+    "claude-3-7-sonnet-20250219": "claude-3-5-haiku-20241022",
+    "claude-3-opus-20240229": "claude-3-5-haiku-20241022",
 
     // Google downgrades
-    'gemini-2.5-pro-preview-03-25': 'gemini-2.5-flash-preview-04-17',
-    'gemini-2.0-flash': 'gemini-2.0-flash-lite',
+    "gemini-2.5-pro-preview-03-25": "gemini-2.5-flash-preview-04-17",
+    "gemini-2.0-flash": "gemini-2.0-flash-lite",
 
     // AWS Bedrock downgrades (keep same model - already cost-optimized)
-    'us.amazon.nova-premier-v1:0': 'us.amazon.nova-premier-v1:0',
+    "us.amazon.nova-premier-v1:0": "us.amazon.nova-premier-v1:0",
   };
 
   return downgrades[baseModel] || baseModel;
 }
 
+export const getModel = (takeModel?: string) => {
+  let model = takeModel;
+
+  const anthropicKey = process.env.ANTHROPIC_API_KEY;
+  const googleKey = process.env.GOOGLE_GENERATIVE_AI_API_KEY;
+  const openaiKey = process.env.OPENAI_API_KEY;
+  let ollamaUrl = process.env.OLLAMA_URL;
+  model = model || process.env.MODEL || "gpt-4.1-2025-04-14";
+
+  let modelInstance;
+  let modelTemperature = Number(process.env.MODEL_TEMPERATURE) || 1;
+  ollamaUrl = undefined;
+
+  // First check if Ollama URL exists and use Ollama
+  if (ollamaUrl) {
+    const ollama = createOllama({
+      baseURL: ollamaUrl,
+    });
+    modelInstance = ollama(model || "llama2"); // Default to llama2 if no model specified
+  } else {
+    // If no Ollama, check other models
+
+    if (model.includes("claude")) {
+      if (!anthropicKey) {
+        throw new Error("No Anthropic API key found. Set ANTHROPIC_API_KEY");
+      }
+      modelInstance = anthropic(model);
+      modelTemperature = 0.5;
+    } else if (model.includes("gemini")) {
+      if (!googleKey) {
+        throw new Error("No Google API key found. Set GOOGLE_API_KEY");
+      }
+      modelInstance = google(model);
+    } else {
+      if (!openaiKey) {
+        throw new Error("No OpenAI API key found. Set OPENAI_API_KEY");
+      }
+      modelInstance = openai(model);
+    }
+
+    return modelInstance;
+  }
+};
+
 export interface TokenUsage {
-  promptTokens: number;
-  completionTokens: number;
-  totalTokens: number;
+  promptTokens?: number;
+  completionTokens?: number;
+  totalTokens?: number;
 }
 
 export async function makeModelCall(
@@ -63,69 +99,13 @@ export async function makeModelCall(
   messages: CoreMessage[],
   onFinish: (text: string, model: string, usage?: TokenUsage) => void,
   options?: any,
-  complexity: ModelComplexity = 'high',
+  complexity: ModelComplexity = "high",
 ) {
-  let modelInstance: LanguageModelV1 | undefined;
   let model = getModelForTask(complexity);
-  const ollamaUrl = process.env.OLLAMA_URL;
-  let ollama: OllamaProvider | undefined;
+  logger.info(`complexity: ${complexity}, model: ${model}`);
 
-  if (ollamaUrl) {
-    ollama = createOllama({
-      baseURL: ollamaUrl,
-    });
-  }
-
-  const bedrock = createAmazonBedrock({
-    region: process.env.AWS_REGION || 'us-east-1',
-    credentialProvider: fromNodeProviderChain(),
-  });
-
-  const generateTextOptions: any = {}
-
-  logger.info(
-    `complexity: ${complexity}, model: ${model}`,
-  );
-  switch (model) {
-    case "gpt-4.1-2025-04-14":
-    case "gpt-4.1-mini-2025-04-14":
-    case "gpt-5-mini-2025-08-07":
-    case "gpt-5-2025-08-07":
-    case "gpt-4.1-nano-2025-04-14":
-      modelInstance = openai(model, { ...options });
-      generateTextOptions.temperature = 1
-      break;
-
-    case "claude-3-7-sonnet-20250219":
-    case "claude-3-opus-20240229":
-    case "claude-3-5-haiku-20241022":
-      modelInstance = anthropic(model, { ...options });
-      break;
-
-    case "gemini-2.5-flash-preview-04-17":
-    case "gemini-2.5-pro-preview-03-25":
-    case "gemini-2.0-flash":
-    case "gemini-2.0-flash-lite":
-      modelInstance = google(model, { ...options });
-      break;
-
-    case "us.meta.llama3-3-70b-instruct-v1:0":
-    case "us.deepseek.r1-v1:0":
-    case "qwen.qwen3-32b-v1:0":
-    case "openai.gpt-oss-120b-1:0":
-    case "us.mistral.pixtral-large-2502-v1:0":
-    case "us.amazon.nova-premier-v1:0":
-      modelInstance = bedrock(`${model}`);
-      generateTextOptions.maxTokens = 100000
-      break;
-
-    default:
-      if (ollama) {
-        modelInstance = ollama(model);
-      }
-      logger.warn(`Unsupported model type: ${model}`);
-      break;
-  }
+  const modelInstance = getModel(model);
+  const generateTextOptions: any = {};
 
   if (!modelInstance) {
     throw new Error(`Unsupported model type: ${model}`);
@@ -135,16 +115,21 @@ export async function makeModelCall(
     return streamText({
       model: modelInstance,
       messages,
+      ...options,
       ...generateTextOptions,
       onFinish: async ({ text, usage }) => {
-        const tokenUsage = usage ? {
-          promptTokens: usage.promptTokens,
-          completionTokens: usage.completionTokens,
-          totalTokens: usage.totalTokens,
-        } : undefined;
+        const tokenUsage = usage
+          ? {
+              promptTokens: usage.inputTokens,
+              completionTokens: usage.outputTokens,
+              totalTokens: usage.totalTokens,
+            }
+          : undefined;
 
         if (tokenUsage) {
-          logger.log(`[${complexity.toUpperCase()}] ${model} - Tokens: ${tokenUsage.totalTokens} (prompt: ${tokenUsage.promptTokens}, completion: ${tokenUsage.completionTokens})`);
+          logger.log(
+            `[${complexity.toUpperCase()}] ${model} - Tokens: ${tokenUsage.totalTokens} (prompt: ${tokenUsage.promptTokens}, completion: ${tokenUsage.completionTokens})`,
+          );
         }
 
         onFinish(text, model, tokenUsage);
@@ -158,14 +143,18 @@ export async function makeModelCall(
     ...generateTextOptions,
   });
 
-  const tokenUsage = usage ? {
-    promptTokens: usage.promptTokens,
-    completionTokens: usage.completionTokens,
-    totalTokens: usage.totalTokens,
-  } : undefined;
+  const tokenUsage = usage
+    ? {
+        promptTokens: usage.inputTokens,
+        completionTokens: usage.outputTokens,
+        totalTokens: usage.totalTokens,
+      }
+    : undefined;
 
   if (tokenUsage) {
-    logger.log(`[${complexity.toUpperCase()}] ${model} - Tokens: ${tokenUsage.totalTokens} (prompt: ${tokenUsage.promptTokens}, completion: ${tokenUsage.completionTokens})`);
+    logger.log(
+      `[${complexity.toUpperCase()}] ${model} - Tokens: ${tokenUsage.totalTokens} (prompt: ${tokenUsage.promptTokens}, completion: ${tokenUsage.completionTokens})`,
+    );
   }
 
   onFinish(text, model, tokenUsage);
@@ -177,19 +166,22 @@ export async function makeModelCall(
  * Determines if a given model is proprietary (OpenAI, Anthropic, Google, Grok)
  * or open source (accessed via Bedrock, Ollama, etc.)
  */
-export function isProprietaryModel(modelName?: string, complexity: ModelComplexity = 'high'): boolean {
+export function isProprietaryModel(
+  modelName?: string,
+  complexity: ModelComplexity = "high",
+): boolean {
   const model = modelName || getModelForTask(complexity);
   if (!model) return false;
 
   // Proprietary model patterns
   const proprietaryPatterns = [
-    /^gpt-/,           // OpenAI models
-    /^claude-/,        // Anthropic models
-    /^gemini-/,        // Google models
-    /^grok-/,          // xAI models
+    /^gpt-/, // OpenAI models
+    /^claude-/, // Anthropic models
+    /^gemini-/, // Google models
+    /^grok-/, // xAI models
   ];
 
-  return proprietaryPatterns.some(pattern => pattern.test(model));
+  return proprietaryPatterns.some((pattern) => pattern.test(model));
 }
 
 export async function getEmbedding(text: string) {

apps/webapp/app/lib/neo4j.server.ts

@@ -112,51 +112,31 @@ export const getNodeLinks = async (userId: string) => {
 export const getClusteredGraphData = async (userId: string) => {
   const session = driver.session();
   try {
-    // Get the simplified graph structure: Episode, Subject, Object with Predicate as edge
-    // Only include entities that are connected to more than 1 episode
+    // Get Episode -> Entity graph, only showing entities connected to more than 1 episode
     const result = await session.run(
       `// Find entities connected to more than 1 episode
-       MATCH (e:Episode)-[:HAS_PROVENANCE]->(s:Statement {userId: $userId})
-       MATCH (s)-[:HAS_SUBJECT|HAS_OBJECT|HAS_PREDICATE]->(ent:Entity)
-       WITH ent, count(DISTINCT e) as episodeCount
+       MATCH (e:Episode{userId: $userId})-[:HAS_PROVENANCE]->(s:Statement {userId: $userId})-[r:HAS_SUBJECT|HAS_OBJECT|HAS_PREDICATE]->(entity:Entity)
+       WITH entity, count(DISTINCT e) as episodeCount
        WHERE episodeCount > 1
-       WITH collect(ent.uuid) as validEntityUuids
+       WITH collect(entity.uuid) as validEntityUuids
 
-       // Get statements where all entities are in the valid set
-       MATCH (e:Episode)-[:HAS_PROVENANCE]->(s:Statement {userId: $userId})
-       MATCH (s)-[:HAS_SUBJECT]->(subj:Entity)
-       WHERE subj.uuid IN validEntityUuids
-       MATCH (s)-[:HAS_PREDICATE]->(pred:Entity)
-       WHERE pred.uuid IN validEntityUuids
-       MATCH (s)-[:HAS_OBJECT]->(obj:Entity)
-       WHERE obj.uuid IN validEntityUuids
-
-       // Build relationships
-       WITH e, s, subj, pred, obj
-       UNWIND [
-         // Episode -> Subject
-         {source: e, sourceType: 'Episode', target: subj, targetType: 'Entity', predicate: null},
-         // Episode -> Object
-         {source: e, sourceType: 'Episode', target: obj, targetType: 'Entity', predicate: null},
-         // Subject -> Object (with Predicate as edge)
-         {source: subj, sourceType: 'Entity', target: obj, targetType: 'Entity', predicate: pred.name}
-       ] AS rel
+       // Build Episode -> Entity relationships for valid entities
+       MATCH (e:Episode{userId: $userId})-[r:HAS_PROVENANCE]->(s:Statement {userId: $userId})-[r:HAS_SUBJECT|HAS_OBJECT|HAS_PREDICATE]->(entity:Entity)
+       WHERE entity.uuid IN validEntityUuids
+       WITH DISTINCT e, entity, type(r) as relType,
+            CASE WHEN size(e.spaceIds) > 0 THEN e.spaceIds[0] ELSE null END as clusterId,
+            s.createdAt as createdAt
 
        RETURN DISTINCT
-         rel.source.uuid as sourceUuid,
-         rel.source.name as sourceName,
-         rel.source.content as sourceContent,
-         rel.sourceType as sourceNodeType,
-         rel.target.uuid as targetUuid,
-         rel.target.name as targetName,
-         rel.targetType as targetNodeType,
-         rel.predicate as predicateLabel,
-         e.uuid as episodeUuid,
-         e.content as episodeContent,
-         e.spaceIds as spaceIds,
-         s.uuid as statementUuid,
-         s.validAt as validAt,
-         s.createdAt as createdAt`,
+         e.uuid as sourceUuid,
+         e.content as sourceContent,
+         'Episode' as sourceNodeType,
+         entity.uuid as targetUuid,
+         entity.name as targetName,
+         'Entity' as targetNodeType,
+         relType as edgeType,
+         clusterId,
+         createdAt`,
       { userId },
     );
 
@@ -165,72 +145,29 @@ export const getClusteredGraphData = async (userId: string) => {
 
     result.records.forEach((record) => {
       const sourceUuid = record.get("sourceUuid");
-      const sourceName = record.get("sourceName");
       const sourceContent = record.get("sourceContent");
-      const sourceNodeType = record.get("sourceNodeType");
-
       const targetUuid = record.get("targetUuid");
       const targetName = record.get("targetName");
-      const targetNodeType = record.get("targetNodeType");
-
-      const predicateLabel = record.get("predicateLabel");
-      const episodeUuid = record.get("episodeUuid");
-      const clusterIds = record.get("spaceIds");
-      const clusterId = clusterIds ? clusterIds[0] : undefined;
+      const edgeType = record.get("edgeType");
+      const clusterId = record.get("clusterId");
       const createdAt = record.get("createdAt");
 
       // Create unique edge identifier to avoid duplicates
-      // For Episode->Subject edges, use generic type; for Subject->Object use predicate
-      const edgeType = predicateLabel || "HAS_SUBJECT";
       const edgeKey = `${sourceUuid}-${targetUuid}-${edgeType}`;
       if (processedEdges.has(edgeKey)) return;
       processedEdges.add(edgeKey);
 
-      // Build node attributes based on type
-      const sourceAttributes =
-        sourceNodeType === "Episode"
-          ? {
-              nodeType: "Episode",
-              content: sourceContent,
-              episodeUuid: sourceUuid,
-              clusterId,
-            }
-          : {
-              nodeType: "Entity",
-              name: sourceName,
-              clusterId,
-            };
-
-      const targetAttributes =
-        targetNodeType === "Episode"
-          ? {
-              nodeType: "Episode",
-              content: sourceContent,
-              episodeUuid: targetUuid,
-              clusterId,
-            }
-          : {
-              nodeType: "Entity",
-              name: targetName,
-              clusterId,
-            };
-
-      // Build display name
-      const sourceDisplayName =
-        sourceNodeType === "Episode"
-          ? sourceContent || episodeUuid
-          : sourceName || sourceUuid;
-      const targetDisplayName =
-        targetNodeType === "Episode"
-          ? sourceContent || episodeUuid
-          : targetName || targetUuid;
-
       triplets.push({
         sourceNode: {
           uuid: sourceUuid,
-          labels: [sourceNodeType],
-          attributes: sourceAttributes,
-          name: sourceDisplayName,
+          labels: ["Episode"],
+          attributes: {
+            nodeType: "Episode",
+            content: sourceContent,
+            episodeUuid: sourceUuid,
+            clusterId,
+          },
+          name: sourceContent || sourceUuid,
           clusterId,
           createdAt: createdAt || "",
         },
@@ -243,10 +180,14 @@ export const getClusteredGraphData = async (userId: string) => {
         },
         targetNode: {
           uuid: targetUuid,
-          labels: [targetNodeType],
-          attributes: targetAttributes,
+          labels: ["Entity"],
+          attributes: {
+            nodeType: "Entity",
+            name: targetName,
+            clusterId,
+          },
+          name: targetName || targetUuid,
           clusterId,
-          name: targetDisplayName,
           createdAt: createdAt || "",
         },
       });

apps/webapp/app/lib/prompt.server.ts

@@ -0,0 +1,324 @@
+import { type StopCondition } from "ai";
+
+export const hasAnswer: StopCondition<any> = ({ steps }) => {
+  return (
+    steps.some((step) => step.text?.includes("</final_response>")) ?? false
+  );
+};
+
+export const hasQuestion: StopCondition<any> = ({ steps }) => {
+  return (
+    steps.some((step) => step.text?.includes("</question_response>")) ?? false
+  );
+};
+
+export const REACT_SYSTEM_PROMPT = `
+You are a helpful AI assistant with access to user memory. Your primary capabilities are:
+
+1. **Memory-First Approach**: Always check user memory first to understand context and previous interactions
+2. **Intelligent Information Gathering**: Analyze queries to determine if current information is needed
+3. **Memory Management**: Help users store, retrieve, and organize information in their memory
+4. **Contextual Assistance**: Use memory to provide personalized and contextual responses
+
+<information_gathering>
+Follow this intelligent approach for information gathering:
+
+1. **MEMORY FIRST** (Always Required)
+   - Always check memory FIRST using core--search_memory before any other actions
+   - Consider this your highest priority for EVERY interaction - as essential as breathing
+   - Memory provides context, personal preferences, and historical information
+   - Use memory to understand user's background, ongoing projects, and past conversations
+
+2. **INFORMATION SYNTHESIS** (Combine Sources)
+   - Use memory to personalize current information based on user preferences
+   - Always store new useful information in memory using core--add_memory
+
+3. **TRAINING KNOWLEDGE** (Foundation)
+   - Use your training knowledge as the foundation for analysis and explanation
+   - Apply training knowledge to interpret and contextualize information from memory
+   - Indicate when you're using training knowledge vs. live information sources
+
+EXECUTION APPROACH:
+- Memory search is mandatory for every interaction
+- Always indicate your information sources in responses
+</information_gathering>
+
+<memory>
+QUERY FORMATION:
+- Write specific factual statements as queries (e.g., "user email address" not "what is the user's email?")
+- Create multiple targeted memory queries for complex requests
+
+KEY QUERY AREAS:
+- Personal context: user name, location, identity, work context
+- Project context: repositories, codebases, current work, team members
+- Task context: recent tasks, ongoing projects, deadlines, priorities
+- Integration context: GitHub repos, Slack channels, Linear projects, connected services
+- Communication patterns: email preferences, notification settings, workflow automation
+- Technical context: coding languages, frameworks, development environment
+- Collaboration context: team members, project stakeholders, meeting patterns
+- Preferences: likes, dislikes, communication style, tool preferences
+- History: previous discussions, past requests, completed work, recurring issues
+- Automation rules: user-defined workflows, triggers, automation preferences
+
+MEMORY USAGE:
+- Execute multiple memory queries in parallel rather than sequentially
+- Batch related memory queries when possible
+- Prioritize recent information over older memories
+- Create comprehensive context-aware queries based on user message/activity content
+- Extract and query SEMANTIC CONTENT, not just structural metadata
+- Parse titles, descriptions, and content for actual subject matter keywords
+- Search internal SOL tasks/conversations that may relate to the same topics
+- Query ALL relatable concepts, not just direct keywords or IDs
+- Search for similar past situations, patterns, and related work
+- Include synonyms, related terms, and contextual concepts in queries  
+- Query user's historical approach to similar requests or activities
+- Search for connected projects, tasks, conversations, and collaborations
+- Retrieve workflow patterns and past decision-making context
+- Query broader domain context beyond immediate request scope
+- Remember: SOL tracks work that external tools don't - search internal content thoroughly
+- Blend memory insights naturally into responses
+- Verify you've checked relevant memory before finalizing ANY response
+
+</memory>
+
+<external_services>
+- To use: load_mcp with EXACT integration name from the available list
+- Can load multiple at once with an array
+- Only load when tools are NOT already available in your current toolset
+- If a tool is already available, use it directly without load_mcp
+- If requested integration unavailable: inform user politely
+</external_services>
+
+<tool_calling>
+You have tools at your disposal to assist users:
+
+CORE PRINCIPLES:
+- Use tools only when necessary for the task at hand
+- Always check memory FIRST before making other tool calls
+- Execute multiple operations in parallel whenever possible
+- Use sequential calls only when output of one is required for input of another
+
+PARAMETER HANDLING:
+- Follow tool schemas exactly with all required parameters
+- Only use values that are:
+  • Explicitly provided by the user (use EXACTLY as given)
+  • Reasonably inferred from context
+  • Retrieved from memory or prior tool calls
+- Never make up values for required parameters
+- Omit optional parameters unless clearly needed
+- Analyze user's descriptive terms for parameter clues
+
+TOOL SELECTION:
+- Never call tools not provided in this conversation
+- Skip tool calls for general questions you can answer directly from memory/knowledge
+- For identical operations on multiple items, use parallel tool calls
+- Default to parallel execution (3-5× faster than sequential calls)
+- You can always access external service tools by loading them with load_mcp first
+
+TOOL MENTION HANDLING:
+When user message contains <mention data-id="tool_name" data-label="tool"></mention>:
+- Extract tool_name from data-id attribute
+- First check if it's a built-in tool; if not, check EXTERNAL SERVICES TOOLS
+- If available: Load it with load_mcp and focus on addressing the request with this tool
+- If unavailable: Inform user and suggest alternatives if possible
+- For multiple tool mentions: Load all applicable tools in a single load_mcp call
+
+ERROR HANDLING:
+- If a tool returns an error, try fixing parameters before retrying
+- If you can't resolve an error, explain the issue to the user
+- Consider alternative tools when primary tools are unavailable
+</tool_calling>
+
+<communication>
+Use EXACTLY ONE of these formats for all user-facing communication:
+
+PROGRESS UPDATES - During processing:
+- Use the core--progress_update tool to keep users informed
+- Update users about what you're discovering or doing next
+- Keep messages clear and user-friendly
+- Avoid technical jargon
+
+QUESTIONS - When you need information:
+<question_response>
+<p>[Your question with HTML formatting]</p>
+</question_response>
+
+- Ask questions only when you cannot find information through memory, or tools
+- Be specific about what you need to know
+- Provide context for why you're asking
+
+FINAL ANSWERS - When completing tasks:
+<final_response>
+<p>[Your answer with HTML formatting]</p>
+</final_response>
+
+CRITICAL:
+- Use ONE format per turn
+- Apply proper HTML formatting (<h1>, <h2>, <p>, <ul>, <li>, etc.)
+- Never mix communication formats
+- Keep responses clear and helpful
+- Always indicate your information sources (memory, and/or knowledge)
+</communication>
+`;
+
+export function getReActPrompt(
+  metadata?: { source?: string; url?: string; pageTitle?: string },
+  intentOverride?: string,
+): string {
+  const contextHints = [];
+
+  if (
+    metadata?.source === "chrome" &&
+    metadata?.url?.includes("mail.google.com")
+  ) {
+    contextHints.push("Content is from email - likely reading intent");
+  }
+  if (
+    metadata?.source === "chrome" &&
+    metadata?.url?.includes("calendar.google.com")
+  ) {
+    contextHints.push("Content is from calendar - likely meeting prep intent");
+  }
+  if (
+    metadata?.source === "chrome" &&
+    metadata?.url?.includes("docs.google.com")
+  ) {
+    contextHints.push(
+      "Content is from document editor - likely writing intent",
+    );
+  }
+  if (metadata?.source === "obsidian") {
+    contextHints.push(
+      "Content is from note editor - likely writing or research intent",
+    );
+  }
+
+  return `You are a memory research agent analyzing content to find relevant context.
+
+YOUR PROCESS (ReAct Framework):
+
+1. DECOMPOSE: First, break down the content into structured categories
+
+   Analyze the content and extract:
+   a) ENTITIES: Specific people, project names, tools, products mentioned
+      Example: "John Smith", "Phoenix API", "Redis", "mobile app"
+
+   b) TOPICS & CONCEPTS: Key subjects, themes, domains
+      Example: "authentication", "database design", "performance optimization"
+
+   c) TEMPORAL MARKERS: Time references, deadlines, events
+      Example: "last week's meeting", "Q2 launch", "yesterday's discussion"
+
+   d) ACTIONS & TASKS: What's being done, decided, or requested
+      Example: "implement feature", "review code", "make decision on"
+
+   e) USER INTENT: What is the user trying to accomplish?
+      ${intentOverride ? `User specified: "${intentOverride}"` : "Infer from context: reading/writing/meeting prep/research/task tracking/review"}
+
+2. FORM QUERIES: Create targeted search queries from your decomposition
+
+   Based on decomposition, form specific queries:
+   - Search for each entity by name (people, projects, tools)
+   - Search for topics the user has discussed before
+   - Search for related work or conversations in this domain
+   - Use the user's actual terminology, not generic concepts
+
+   EXAMPLE - Content: "Email from Sarah about the API redesign we discussed last week"
+   Decomposition:
+     - Entities: "Sarah", "API redesign"
+     - Topics: "API design", "redesign"
+     - Temporal: "last week"
+     - Actions: "discussed", "email communication"
+     - Intent: Reading (email) / meeting prep
+
+   Queries to form:
+   ✅ "Sarah" (find past conversations with Sarah)
+   ✅ "API redesign" or "API design" (find project discussions)
+   ✅ "last week" + "Sarah" (find recent context)
+   ✅ "meetings" or "discussions" (find related conversations)
+
+   ❌ Avoid: "email communication patterns", "API architecture philosophy"
+   (These are abstract - search what user actually discussed!)
+
+3. SEARCH: Execute your queries using searchMemory tool
+   - Start with 2-3 core searches based on main entities/topics
+   - Make each search specific and targeted
+   - Use actual terms from the content, not rephrased concepts
+
+4. OBSERVE: Evaluate search results
+   - Did you find relevant episodes? How many unique ones?
+   - What specific context emerged?
+   - What new entities/topics appeared in results?
+   - Are there gaps in understanding?
+   - Should you search more angles?
+
+   Note: Episode counts are automatically deduplicated across searches - overlapping episodes are only counted once.
+
+5. REACT: Decide next action based on observations
+
+   STOPPING CRITERIA - Proceed to SYNTHESIZE if ANY of these are true:
+   - You found 20+ unique episodes across your searches → ENOUGH CONTEXT
+   - You performed 5+ searches and found relevant episodes → SUFFICIENT
+   - You performed 7+ searches regardless of results → EXHAUSTED STRATEGIES
+   - You found strong relevant context from multiple angles → COMPLETE
+
+   System nudges will provide awareness of your progress, but you decide when synthesis quality would be optimal.
+
+   If you found little/no context AND searched less than 7 times:
+   - Try different query angles from your decomposition
+   - Search broader related topics
+   - Search user's projects or work areas
+   - Try alternative terminology
+
+   ⚠️ DO NOT search endlessly - if you found relevant episodes, STOP and synthesize!
+
+6. SYNTHESIZE: After gathering sufficient context, provide final answer
+   - Wrap your synthesis in <final_response> tags
+   - Present direct factual context from memory - no meta-commentary
+   - Write as if providing background context to an AI assistant
+   - Include: facts, decisions, preferences, patterns, timelines
+   - Note any gaps, contradictions, or evolution in thinking
+   - Keep it concise and actionable
+   - DO NOT use phrases like "Previous discussions on", "From conversations", "Past preferences indicate"
+   - DO NOT use conversational language like "you said" or "you mentioned"
+   - Present information as direct factual statements
+
+FINAL RESPONSE FORMAT:
+<final_response>
+[Direct synthesized context - factual statements only]
+
+Good examples:
+- "The API redesign focuses on performance and scalability. Key decisions: moving to GraphQL, caching layer with Redis."
+- "Project Phoenix launches Q2 2024. Main features: real-time sync, offline mode, collaborative editing."
+- "Sarah leads the backend team. Recent work includes authentication refactor and database migration."
+
+Bad examples:
+❌ "Previous discussions on the API revealed..."
+❌ "From past conversations, it appears that..."
+❌ "Past preferences indicate..."
+❌ "The user mentioned that..."
+
+Just state the facts directly.
+</final_response>
+
+${contextHints.length > 0 ? `\nCONTEXT HINTS:\n${contextHints.join("\n")}` : ""}
+
+CRITICAL REQUIREMENTS:
+- ALWAYS start with DECOMPOSE step - extract entities, topics, temporal markers, actions
+- Form specific queries from your decomposition - use user's actual terms
+- Minimum 3 searches required
+- Maximum 10 searches allowed - must synthesize after that
+- STOP and synthesize when you hit stopping criteria (20+ episodes, 5+ searches with results, 7+ searches total)
+- Each search should target different aspects from decomposition
+- Present synthesis directly without meta-commentary
+
+SEARCH QUALITY CHECKLIST:
+✅ Queries use specific terms from content (names, projects, exact phrases)
+✅ Searched multiple angles from decomposition (entities, topics, related areas)
+✅ Stop when you have enough unique context - don't search endlessly
+✅ Tried alternative terminology if initial searches found nothing
+❌ Avoid generic/abstract queries that don't match user's vocabulary
+❌ Don't stop at 3 searches if you found zero unique episodes
+❌ Don't keep searching when you already found 20+ unique episodes
+}`;
+}

apps/webapp/app/lib/queue-adapter.server.ts

@@ -0,0 +1,168 @@
+/**
+ * Queue Adapter
+ *
+ * This module provides a unified interface for queueing background jobs,
+ * supporting both Trigger.dev and BullMQ backends based on the QUEUE_PROVIDER
+ * environment variable.
+ *
+ * Usage:
+ * - Set QUEUE_PROVIDER="trigger" for Trigger.dev (default, good for production scaling)
+ * - Set QUEUE_PROVIDER="bullmq" for BullMQ (good for open-source deployments)
+ */
+
+import { env } from "~/env.server";
+import type { z } from "zod";
+import type { IngestBodyRequest } from "~/jobs/ingest/ingest-episode.logic";
+import type { CreateConversationTitlePayload } from "~/jobs/conversation/create-title.logic";
+import type { SessionCompactionPayload } from "~/jobs/session/session-compaction.logic";
+import { type SpaceAssignmentPayload } from "~/trigger/spaces/space-assignment";
+
+type QueueProvider = "trigger" | "bullmq";
+
+/**
+ * Enqueue episode ingestion job
+ */
+export async function enqueueIngestEpisode(payload: {
+  body: z.infer<typeof IngestBodyRequest>;
+  userId: string;
+  workspaceId: string;
+  queueId: string;
+}): Promise<{ id?: string; token?: string }> {
+  const provider = env.QUEUE_PROVIDER as QueueProvider;
+
+  if (provider === "trigger") {
+    const { ingestTask } = await import("~/trigger/ingest/ingest");
+    const handler = await ingestTask.trigger(payload, {
+      queue: "ingestion-queue",
+      concurrencyKey: payload.userId,
+      tags: [payload.userId, payload.queueId],
+    });
+    return { id: handler.id, token: handler.publicAccessToken };
+  } else {
+    // BullMQ
+    const { ingestQueue } = await import("~/bullmq/queues");
+    const job = await ingestQueue.add("ingest-episode", payload, {
+      jobId: payload.queueId,
+      attempts: 3,
+      backoff: { type: "exponential", delay: 2000 },
+    });
+    return { id: job.id };
+  }
+}
+
+/**
+ * Enqueue document ingestion job
+ */
+export async function enqueueIngestDocument(payload: {
+  body: z.infer<typeof IngestBodyRequest>;
+  userId: string;
+  workspaceId: string;
+  queueId: string;
+}): Promise<{ id?: string; token?: string }> {
+  const provider = env.QUEUE_PROVIDER as QueueProvider;
+
+  if (provider === "trigger") {
+    const { ingestDocumentTask } = await import(
+      "~/trigger/ingest/ingest-document"
+    );
+    const handler = await ingestDocumentTask.trigger(payload, {
+      queue: "document-ingestion-queue",
+      concurrencyKey: payload.userId,
+      tags: [payload.userId, payload.queueId],
+    });
+    return { id: handler.id, token: handler.publicAccessToken };
+  } else {
+    // BullMQ
+    const { documentIngestQueue } = await import("~/bullmq/queues");
+    const job = await documentIngestQueue.add("ingest-document", payload, {
+      jobId: payload.queueId,
+      attempts: 3,
+      backoff: { type: "exponential", delay: 2000 },
+    });
+
+    return { id: job.id };
+  }
+}
+
+/**
+ * Enqueue conversation title creation job
+ */
+export async function enqueueCreateConversationTitle(
+  payload: CreateConversationTitlePayload,
+): Promise<{ id?: string }> {
+  const provider = env.QUEUE_PROVIDER as QueueProvider;
+
+  if (provider === "trigger") {
+    const { createConversationTitle } = await import(
+      "~/trigger/conversation/create-conversation-title"
+    );
+    const handler = await createConversationTitle.trigger(payload);
+    return { id: handler.id };
+  } else {
+    // BullMQ
+    const { conversationTitleQueue } = await import("~/bullmq/queues");
+    const job = await conversationTitleQueue.add(
+      "create-conversation-title",
+      payload,
+      {
+        attempts: 3,
+        backoff: { type: "exponential", delay: 2000 },
+      },
+    );
+    return { id: job.id };
+  }
+}
+
+/**
+ * Enqueue session compaction job
+ */
+export async function enqueueSessionCompaction(
+  payload: SessionCompactionPayload,
+): Promise<{ id?: string }> {
+  const provider = env.QUEUE_PROVIDER as QueueProvider;
+
+  if (provider === "trigger") {
+    const { sessionCompactionTask } = await import(
+      "~/trigger/session/session-compaction"
+    );
+    const handler = await sessionCompactionTask.trigger(payload);
+    return { id: handler.id };
+  } else {
+    // BullMQ
+    const { sessionCompactionQueue } = await import("~/bullmq/queues");
+    const job = await sessionCompactionQueue.add(
+      "session-compaction",
+      payload,
+      {
+        attempts: 3,
+        backoff: { type: "exponential", delay: 2000 },
+      },
+    );
+    return { id: job.id };
+  }
+}
+
+/**
+ * Enqueue space assignment job
+ * (Helper for common job logic to call)
+ */
+export async function enqueueSpaceAssignment(
+  payload: SpaceAssignmentPayload,
+): Promise<void> {
+  const provider = env.QUEUE_PROVIDER as QueueProvider;
+
+  if (provider === "trigger") {
+    const { triggerSpaceAssignment } = await import(
+      "~/trigger/spaces/space-assignment"
+    );
+    await triggerSpaceAssignment(payload);
+  } else {
+    // For BullMQ, space assignment is not implemented yet
+    // You can add it later when needed
+    console.warn("Space assignment not implemented for BullMQ yet");
+  }
+}
+
+export const isTriggerDeployment = () => {
+  return env.QUEUE_PROVIDER === "trigger";
+};

apps/webapp/app/models/user.server.ts

@@ -3,6 +3,7 @@ import type { GoogleProfile } from "@coji/remix-auth-google";
 import { prisma } from "~/db.server";
 import { env } from "~/env.server";
 import { runQuery } from "~/lib/neo4j.server";
+import { trackFeatureUsage } from "~/services/telemetry.server";
 export type { User } from "@core/database";
 
 type FindOrCreateMagicLink = {
@@ -72,9 +73,16 @@ export async function findOrCreateMagicLinkUser(
     },
   });
 
+  const isNewUser = !existingUser;
+
+  // Track new user registration
+  if (isNewUser) {
+    trackFeatureUsage("user_registered", user.id).catch(console.error);
+  }
+
   return {
     user,
-    isNewUser: !existingUser,
+    isNewUser,
   };
 }
 
@@ -160,9 +168,16 @@ export async function findOrCreateGoogleUser({
     },
   });
 
+  const isNewUser = !existingUser;
+
+  // Track new user registration
+  if (isNewUser) {
+    trackFeatureUsage("user_registered", user.id).catch(console.error);
+  }
+
   return {
     user,
-    isNewUser: !existingUser,
+    isNewUser,
   };
 }
 

apps/webapp/app/root.tsx

@@ -51,6 +51,7 @@ export const loader = async ({ request }: LoaderFunctionArgs) => {
   const { getTheme } = await themeSessionResolver(request);
 
   const posthogProjectKey = env.POSTHOG_PROJECT_KEY;
+  const telemetryEnabled = env.TELEMETRY_ENABLED;
   const user = await getUser(request);
   const usageSummary = await getUsageSummary(user?.Workspace?.id as string);
 
@@ -62,6 +63,7 @@ export const loader = async ({ request }: LoaderFunctionArgs) => {
       toastMessage,
       theme: getTheme(),
       posthogProjectKey,
+      telemetryEnabled,
       appEnv: env.APP_ENV,
       appOrigin: env.APP_ORIGIN,
     },
@@ -113,8 +115,10 @@ export function ErrorBoundary() {
 }
 
 function App() {
-  const { posthogProjectKey } = useTypedLoaderData<typeof loader>();
-  usePostHog(posthogProjectKey);
+  const { posthogProjectKey, telemetryEnabled } =
+    useTypedLoaderData<typeof loader>();
+
+  usePostHog(posthogProjectKey, telemetryEnabled);
   const [theme] = useTheme();
 
   return (

apps/webapp/app/routes/api.v1.conversation.$conversationId.run.tsx

@@ -1,44 +0,0 @@
-import { json } from "@remix-run/node";
-import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server";
-
-import { getWorkspaceByUser } from "~/models/workspace.server";
-import {
-  createConversation,
-  CreateConversationSchema,
-  getCurrentConversationRun,
-  readConversation,
-  stopConversation,
-} from "~/services/conversation.server";
-import { z } from "zod";
-
-export const ConversationIdSchema = z.object({
-  conversationId: z.string(),
-});
-
-const { action, loader } = createActionApiRoute(
-  {
-    params: ConversationIdSchema,
-    allowJWT: true,
-    authorization: {
-      action: "oauth",
-    },
-    corsStrategy: "all",
-  },
-  async ({ authentication, params }) => {
-    const workspace = await getWorkspaceByUser(authentication.userId);
-
-    if (!workspace) {
-      throw new Error("No workspace found");
-    }
-
-    // Call the service to get the redirect URL
-    const run = await getCurrentConversationRun(
-      params.conversationId,
-      workspace?.id,
-    );
-
-    return json(run);
-  },
-);
-
-export { action, loader };

apps/webapp/app/routes/api.v1.conversation.$conversationId.stop.tsx

@@ -1,41 +0,0 @@
-import { json } from "@remix-run/node";
-import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server";
-
-import { getWorkspaceByUser } from "~/models/workspace.server";
-import {
-  createConversation,
-  CreateConversationSchema,
-  readConversation,
-  stopConversation,
-} from "~/services/conversation.server";
-import { z } from "zod";
-
-export const ConversationIdSchema = z.object({
-  conversationId: z.string(),
-});
-
-const { action, loader } = createActionApiRoute(
-  {
-    params: ConversationIdSchema,
-    allowJWT: true,
-    authorization: {
-      action: "oauth",
-    },
-    corsStrategy: "all",
-    method: "POST",
-  },
-  async ({ authentication, params }) => {
-    const workspace = await getWorkspaceByUser(authentication.userId);
-
-    if (!workspace) {
-      throw new Error("No workspace found");
-    }
-
-    // Call the service to get the redirect URL
-    const stop = await stopConversation(params.conversationId, workspace?.id);
-
-    return json(stop);
-  },
-);
-
-export { action, loader };

apps/webapp/app/routes/api.v1.conversation.$conversationId.stream.tsx

@@ -0,0 +1,45 @@
+// import { json } from "@remix-run/node";
+// import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server";
+// import { UI_MESSAGE_STREAM_HEADERS } from "ai";
+
+// import { getConversationAndHistory } from "~/services/conversation.server";
+// import { z } from "zod";
+// import { createResumableStreamContext } from "resumable-stream";
+
+// export const ConversationIdSchema = z.object({
+//   conversationId: z.string(),
+// });
+
+// const { action, loader } = createActionApiRoute(
+//   {
+//     params: ConversationIdSchema,
+//     allowJWT: true,
+//     authorization: {
+//       action: "oauth",
+//     },
+//     corsStrategy: "all",
+//   },
+//   async ({ authentication, params }) => {
+//     const conversation = await getConversationAndHistory(
+//       params.conversationId,
+//       authentication.userId,
+//     );
+
+//     const lastConversation = conversation?.ConversationHistory.pop();
+
+//     if (!lastConversation) {
+//       return json({}, { status: 204 });
+//     }
+
+//     const streamContext = createResumableStreamContext({
+//       waitUntil: null,
+//     });
+
+//     return new Response(
+//       await streamContext.resumeExistingStream(lastConversation.id),
+//       { headers: UI_MESSAGE_STREAM_HEADERS },
+//     );
+//   },
+// );
+
+// export { action, loader };

apps/webapp/app/routes/api.v1.conversation.$conversationId.tsx

@@ -1,50 +0,0 @@
-import { json } from "@remix-run/node";
-import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server";
-
-import { getWorkspaceByUser } from "~/models/workspace.server";
-import {
-  getConversation,
-  deleteConversation,
-} from "~/services/conversation.server";
-import { z } from "zod";
-
-export const ConversationIdSchema = z.object({
-  conversationId: z.string(),
-});
-
-const { action, loader } = createActionApiRoute(
-  {
-    params: ConversationIdSchema,
-    allowJWT: true,
-    authorization: {
-      action: "oauth",
-    },
-    corsStrategy: "all",
-  },
-  async ({ params, authentication, request }) => {
-    const workspace = await getWorkspaceByUser(authentication.userId);
-
-    if (!workspace) {
-      throw new Error("No workspace found");
-    }
-
-    const method = request.method;
-
-    if (method === "GET") {
-      // Get a conversation by ID
-      const conversation = await getConversation(params.conversationId);
-      return json(conversation);
-    }
-
-    if (method === "DELETE") {
-      // Soft delete a conversation
-      const deleted = await deleteConversation(params.conversationId);
-      return json(deleted);
-    }
-
-    // Method not allowed
-    return new Response("Method Not Allowed", { status: 405 });
-  },
-);
-
-export { action, loader };

apps/webapp/app/routes/api.v1.conversation._index.tsx

@@ -1,37 +1,155 @@
-import { json } from "@remix-run/node";
-import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server";
-
-import { getWorkspaceByUser } from "~/models/workspace.server";
 import {
-  createConversation,
-  CreateConversationSchema,
+  convertToModelMessages,
+  streamText,
+  validateUIMessages,
+  type LanguageModel,
+  experimental_createMCPClient as createMCPClient,
+  generateId,
+  stepCountIs,
+} from "ai";
+import { z } from "zod";
+import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
+
+import { createHybridActionApiRoute } from "~/services/routeBuilders/apiBuilder.server";
+import {
+  createConversationHistory,
+  getConversationAndHistory,
 } from "~/services/conversation.server";
 
-const { action, loader } = createActionApiRoute(
+import { getModel } from "~/lib/model.server";
+import { UserTypeEnum } from "@core/types";
+import { nanoid } from "nanoid";
+import { getOrCreatePersonalAccessToken } from "~/services/personalAccessToken.server";
+import {
+  hasAnswer,
+  hasQuestion,
+  REACT_SYSTEM_PROMPT,
+} from "~/lib/prompt.server";
+import { enqueueCreateConversationTitle } from "~/lib/queue-adapter.server";
+import { env } from "~/env.server";
+
+const ChatRequestSchema = z.object({
+  message: z.object({
+    id: z.string().optional(),
+    parts: z.array(z.any()),
+    role: z.string(),
+  }),
+  id: z.string(),
+});
+
+const { loader, action } = createHybridActionApiRoute(
   {
-    body: CreateConversationSchema,
+    body: ChatRequestSchema,
     allowJWT: true,
     authorization: {
-      action: "oauth",
+      action: "conversation",
     },
     corsStrategy: "all",
   },
   async ({ body, authentication }) => {
-    const workspace = await getWorkspaceByUser(authentication.userId);
+    const randomKeyName = `chat_${nanoid(10)}`;
+    const pat = await getOrCreatePersonalAccessToken({
+      name: randomKeyName,
+      userId: authentication.userId,
+    });
 
-    if (!workspace) {
-      throw new Error("No workspace found");
-    }
+    const message = body.message.parts[0].text;
+    const id = body.message.id;
+    const apiEndpoint = `${env.APP_ORIGIN}/api/v1/mcp?source=core`;
+    const url = new URL(apiEndpoint);
 
-    // Call the service to get the redirect URL
-    const conversation = await createConversation(
-      workspace?.id,
+    const mcpClient = await createMCPClient({
+      transport: new StreamableHTTPClientTransport(url, {
+        requestInit: {
+          headers: pat.token
+            ? {
+                Authorization: `Bearer ${pat.token}`,
+              }
+            : {},
+        },
+      }),
+    });
+
+    const conversation = await getConversationAndHistory(
+      body.id,
       authentication.userId,
-      body,
     );
 
-    return json(conversation);
+    const conversationHistory = conversation?.ConversationHistory ?? [];
+
+    if (conversationHistory.length === 0) {
+      // Trigger conversation title task
+      await enqueueCreateConversationTitle({
+        conversationId: body.id,
+        message,
+      });
+    }
+
+    if (conversationHistory.length > 1) {
+      await createConversationHistory(message, body.id, UserTypeEnum.User);
+    }
+
+    const messages = conversationHistory.map((history: any) => {
+      return {
+        parts: [{ text: history.message, type: "text" }],
+        role: "user",
+        id: history.id,
+      };
+    });
+
+    const tools = { ...(await mcpClient.tools()) };
+
+    const finalMessages = [
+      ...messages,
+      {
+        parts: [{ text: message, type: "text" }],
+        role: "user",
+        id: id ?? generateId(),
+      },
+    ];
+
+    const validatedMessages = await validateUIMessages({
+      messages: finalMessages,
+    });
+
+    const result = streamText({
+      model: getModel() as LanguageModel,
+      messages: [
+        {
+          role: "system",
+          content: REACT_SYSTEM_PROMPT,
+        },
+        ...convertToModelMessages(validatedMessages),
+      ],
+      tools,
+      stopWhen: [stepCountIs(10), hasAnswer, hasQuestion],
+    });
+
+    result.consumeStream(); // no await
+
+    return result.toUIMessageStreamResponse({
+      originalMessages: validatedMessages,
+      onFinish: async ({ messages }) => {
+        const lastMessage = messages.pop();
+        let message = "";
+        lastMessage?.parts.forEach((part) => {
+          if (part.type === "text") {
+            message += part.text;
+          }
+        });
+
+        await createConversationHistory(message, body.id, UserTypeEnum.Agent);
+      },
+      // async consumeSseStream({ stream }) {
+      //   // Create a resumable stream from the SSE stream
+      //   const streamContext = createResumableStreamContext({ waitUntil: null });
+      //   await streamContext.createNewResumableStream(
+      //     conversation.conversationHistoryId,
+      //     () => stream,
+      //   );
+      // },
+    });
   },
 );
 
-export { action, loader };
+export { loader, action };

apps/webapp/app/routes/api.v1.deep-search.tsx

@@ -1,8 +1,27 @@
 import { z } from "zod";
 import { json } from "@remix-run/node";
 import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server";
-import { deepSearch } from "~/trigger/deep-search";
-import { runs } from "@trigger.dev/sdk";
+import { trackFeatureUsage } from "~/services/telemetry.server";
+import { nanoid } from "nanoid";
+import {
+  deletePersonalAccessToken,
+  getOrCreatePersonalAccessToken,
+} from "~/services/personalAccessToken.server";
+
+import {
+  convertToModelMessages,
+  generateId,
+  generateText,
+  type LanguageModel,
+  stepCountIs,
+  streamText,
+  tool,
+  validateUIMessages,
+} from "ai";
+import axios from "axios";
+import { logger } from "~/services/logger.service";
+import { getReActPrompt, hasAnswer } from "~/lib/prompt.server";
+import { getModel } from "~/lib/model.server";
 
 const DeepSearchBodySchema = z.object({
   content: z.string().min(1, "Content is required"),
@@ -17,6 +36,41 @@ const DeepSearchBodySchema = z.object({
     .optional(),
 });
 
+function createSearchMemoryTool(token: string) {
+  return tool({
+    description:
+      "Search the user's memory for relevant facts and episodes. Use this tool multiple times with different queries to gather comprehensive context.",
+    inputSchema: z.object({
+      query: z
+        .string()
+        .describe(
+          "Search query to find relevant information. Be specific: entity names, topics, concepts.",
+        ),
+    }),
+    execute: async ({ query }: { query: string }) => {
+      try {
+        const response = await axios.post(
+          `${process.env.API_BASE_URL || "https://core.heysol.ai"}/api/v1/search`,
+          { query, structured: false },
+          {
+            headers: {
+              Authorization: `Bearer ${token}`,
+            },
+          },
+        );
+        return response.data;
+      } catch (error) {
+        logger.error(`SearchMemory tool error: ${error}`);
+        return {
+          facts: [],
+          episodes: [],
+          summary: "No results found",
+        };
+      }
+    },
+  } as any);
+}
+
 const { action, loader } = createActionApiRoute(
   {
     body: DeepSearchBodySchema,
@@ -28,35 +82,94 @@ const { action, loader } = createActionApiRoute(
     corsStrategy: "all",
   },
   async ({ body, authentication }) => {
-    let trigger;
-    if (!body.stream) {
-      trigger = await deepSearch.trigger({
-        content: body.content,
-        userId: authentication.userId,
-        stream: body.stream,
-        intentOverride: body.intentOverride,
-        metadata: body.metadata,
+    // Track deep search
+    trackFeatureUsage("deep_search_performed", authentication.userId).catch(
+      console.error,
+    );
+
+    const randomKeyName = `deepSearch_${nanoid(10)}`;
+
+    const pat = await getOrCreatePersonalAccessToken({
+      name: randomKeyName,
+      userId: authentication.userId as string,
+    });
+
+    if (!pat?.token) {
+      return json({
+        success: false,
+        error: "Failed to create personal access token",
+      });
+    }
+
+    try {
+      // Create search tool that agent will use
+      const searchTool = createSearchMemoryTool(pat.token);
+
+      const tools = {
+        searchMemory: searchTool,
+      };
+
+      // Build initial messages with ReAct prompt
+      const initialMessages = [
+        {
+          role: "user",
+          parts: [
+            {
+              type: "text",
+              text: `CONTENT TO ANALYZE:\n${body.content}\n\nPlease search my memory for relevant context and synthesize what you find.`,
+            },
+          ],
+          id: generateId(),
+        },
+      ];
+
+      const validatedMessages = await validateUIMessages({
+        messages: initialMessages,
+        tools,
       });
 
-      return json(trigger);
-    } else {
-      const runHandler = await deepSearch.trigger({
-        content: body.content,
-        userId: authentication.userId,
-        stream: body.stream,
-        intentOverride: body.intentOverride,
-        metadata: body.metadata,
-      });
+      if (body.stream) {
+        const result = streamText({
+          model: getModel() as LanguageModel,
+          messages: [
+            {
+              role: "system",
+              content: getReActPrompt(body.metadata, body.intentOverride),
+            },
+            ...convertToModelMessages(validatedMessages),
+          ],
+          tools,
+          stopWhen: [hasAnswer, stepCountIs(10)],
+        });
 
-      for await (const run of runs.subscribeToRun(runHandler.id)) {
-        if (run.status === "COMPLETED") {
-          return json(run.output);
-        } else if (run.status === "FAILED") {
-          return json(run.error);
-        }
+        return result.toUIMessageStreamResponse({
+          originalMessages: validatedMessages,
+        });
+      } else {
+        const { text } = await generateText({
+          model: getModel() as LanguageModel,
+          messages: [
+            {
+              role: "system",
+              content: getReActPrompt(body.metadata, body.intentOverride),
+            },
+            ...convertToModelMessages(validatedMessages),
+          ],
+          tools,
+          stopWhen: [hasAnswer, stepCountIs(10)],
+        });
+
+        await deletePersonalAccessToken(pat?.id);
+        return json({ text });
       }
+    } catch (error: any) {
+      await deletePersonalAccessToken(pat?.id);
+      logger.error(`Deep search error: ${error}`);
 
-      return json({ error: "Run failed" });
+      return json({
+        success: false,
+        error: error.message,
+      });
     }
   },
 );

apps/webapp/app/routes/api.v1.ingestion_queue.delete.tsx

@@ -6,7 +6,7 @@ import {
   deleteIngestionQueue,
   getIngestionQueue,
 } from "~/services/ingestionLogs.server";
-import { runs, tasks } from "@trigger.dev/sdk";
+import { findRunningJobs, cancelJob } from "~/services/jobManager.server";
 
 export const DeleteEpisodeBodyRequest = z.object({
   id: z.string(),
@@ -37,19 +37,15 @@ const { action, loader } = createHybridActionApiRoute(
       }
 
       const output = ingestionQueue.output as any;
-      const runningTasks = await runs.list({
-        tag: [authentication.userId, ingestionQueue.id],
+      const runningTasks = await findRunningJobs({
+        tags: [authentication.userId, ingestionQueue.id],
         taskIdentifier: "ingest-episode",
       });
 
-      const latestTask = runningTasks.data.find(
-        (task) =>
-          task.tags.includes(authentication.userId) &&
-          task.tags.includes(ingestionQueue.id),
-      );
+      const latestTask = runningTasks[0];
 
-      if (latestTask && !latestTask?.isCompleted) {
-        runs.cancel(latestTask?.id as string);
+      if (latestTask && !latestTask.isCompleted) {
+        await cancelJob(latestTask.id);
       }
 
       let result;

apps/webapp/app/routes/api.v1.integration_account.tsx

@@ -8,6 +8,7 @@ import { logger } from "~/services/logger.service";
 import { getWorkspaceByUser } from "~/models/workspace.server";
 import { tasks } from "@trigger.dev/sdk";
 import { type scheduler } from "~/trigger/integrations/scheduler";
+import { isTriggerDeployment } from "~/lib/queue-adapter.server";
 
 // Schema for creating an integration account with API key
 const IntegrationAccountBodySchema = z.object({
@@ -63,6 +64,13 @@ const { action, loader } = createHybridActionApiRoute(
         );
       }
 
+      if (!isTriggerDeployment()) {
+        return json(
+          { error: "Integrations don't work in non trigger deployment" },
+          { status: 400 },
+        );
+      }
+
       await tasks.trigger<typeof scheduler>("scheduler", {
         integrationAccountId: setupResult?.account?.id,
       });

apps/webapp/app/routes/api.v1.logs.$logId.retry.tsx

@@ -0,0 +1,88 @@
+import { json } from "@remix-run/node";
+import { z } from "zod";
+import { IngestionStatus } from "@core/database";
+import { getIngestionQueue } from "~/services/ingestionLogs.server";
+import { createHybridActionApiRoute } from "~/services/routeBuilders/apiBuilder.server";
+import { addToQueue } from "~/lib/ingest.server";
+
+// Schema for log ID parameter
+const LogParamsSchema = z.object({
+  logId: z.string(),
+});
+
+const { action } = createHybridActionApiRoute(
+  {
+    params: LogParamsSchema,
+    allowJWT: true,
+    method: "POST",
+    authorization: {
+      action: "update",
+    },
+    corsStrategy: "all",
+  },
+  async ({ params, authentication }) => {
+    try {
+      const ingestionQueue = await getIngestionQueue(params.logId);
+
+      if (!ingestionQueue) {
+        return json(
+          {
+            error: "Ingestion log not found",
+            code: "not_found",
+          },
+          { status: 404 },
+        );
+      }
+
+      // Only allow retry for FAILED status
+      if (ingestionQueue.status !== IngestionStatus.FAILED) {
+        return json(
+          {
+            error: "Only failed ingestion logs can be retried",
+            code: "invalid_status",
+          },
+          { status: 400 },
+        );
+      }
+
+      // Get the original ingestion data
+      const originalData = ingestionQueue.data as any;
+
+      // Re-enqueue the job with the existing queue ID (will upsert)
+      await addToQueue(
+        originalData,
+        authentication.userId,
+        ingestionQueue.activityId || undefined,
+        ingestionQueue.id, // Pass the existing queue ID for upsert
+      );
+
+      return json({
+        success: true,
+        message: "Ingestion retry initiated successfully",
+      });
+    } catch (error) {
+      console.error("Error retrying ingestion:", error);
+
+      // Handle specific error cases
+      if (error instanceof Error && error.message === "no credits") {
+        return json(
+          {
+            error: "Insufficient credits to retry ingestion",
+            code: "no_credits",
+          },
+          { status: 402 },
+        );
+      }
+
+      return json(
+        {
+          error: "Failed to retry ingestion",
+          code: "internal_error",
+        },
+        { status: 500 },
+      );
+    }
+  },
+);
+
+export { action };

apps/webapp/app/routes/api.v1.logs.$logId.tsx

@@ -1,5 +1,4 @@
 import { json } from "@remix-run/node";
-import { runs } from "@trigger.dev/sdk";
 import { z } from "zod";
 import { deleteEpisodeWithRelatedNodes } from "~/services/graphModels/episode";
 import {
@@ -11,6 +10,7 @@ import {
   createHybridActionApiRoute,
   createHybridLoaderApiRoute,
 } from "~/services/routeBuilders/apiBuilder.server";
+import { findRunningJobs, cancelJob } from "~/services/jobManager.server";
 
 // Schema for space ID parameter
 const LogParamsSchema = z.object({
@@ -59,19 +59,15 @@ const { action } = createHybridActionApiRoute(
       }
 
       const output = ingestionQueue.output as any;
-      const runningTasks = await runs.list({
-        tag: [authentication.userId, ingestionQueue.id],
+      const runningTasks = await findRunningJobs({
+        tags: [authentication.userId, ingestionQueue.id],
         taskIdentifier: "ingest-episode",
       });
 
-      const latestTask = runningTasks.data.find(
-        (task) =>
-          task.tags.includes(authentication.userId) &&
-          task.tags.includes(ingestionQueue.id),
-      );
+      const latestTask = runningTasks[0];
 
-      if (latestTask && !latestTask?.isCompleted) {
-        runs.cancel(latestTask?.id);
+      if (latestTask && !latestTask.isCompleted) {
+        await cancelJob(latestTask.id);
       }
 
       let result;

apps/webapp/app/routes/api.v1.logs.tsx

@@ -1,4 +1,4 @@
-import { type LoaderFunctionArgs, json } from "@remix-run/node";
+import { json } from "@remix-run/node";
 import { z } from "zod";
 import { prisma } from "~/db.server";
 import { createHybridLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server";

apps/webapp/app/routes/api.v1.search.tsx

@@ -5,6 +5,7 @@ import {
 } from "~/services/routeBuilders/apiBuilder.server";
 import { SearchService } from "~/services/search.server";
 import { json } from "@remix-run/node";
+import { trackFeatureUsage } from "~/services/telemetry.server";
 
 export const SearchBodyRequest = z.object({
   query: z.string(),
@@ -51,6 +52,10 @@ const { action, loader } = createHybridActionApiRoute(
         structured: body.structured,
       },
     );
+
+    // Track search
+    trackFeatureUsage("search_performed", authentication.userId).catch(console.error);
+
     return json(results);
   },
 );

apps/webapp/app/routes/api.v1.spaces.$spaceId.reset.ts

@@ -3,7 +3,7 @@ import { createHybridActionApiRoute } from "~/services/routeBuilders/apiBuilder.
 import { SpaceService } from "~/services/space.server";
 import { json } from "@remix-run/node";
 import { logger } from "~/services/logger.service";
-import { triggerSpaceAssignment } from "~/trigger/spaces/space-assignment";
+import { enqueueSpaceAssignment } from "~/lib/queue-adapter.server";
 
 // Schema for space ID parameter
 const SpaceParamsSchema = z.object({
@@ -31,7 +31,7 @@ const { loader, action } = createHybridActionApiRoute(
 
     // Trigger automatic episode assignment for the reset space
     try {
-      await triggerSpaceAssignment({
+      await enqueueSpaceAssignment({
         userId: userId,
         workspaceId: space.workspaceId,
         mode: "new_space",

apps/webapp/app/routes/api.v1.spaces.assignments.ts

@@ -1,8 +1,8 @@
 import { z } from "zod";
 import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server";
 import { json } from "@remix-run/node";
-import { triggerSpaceAssignment } from "~/trigger/spaces/space-assignment";
 import { prisma } from "~/db.server";
+import { enqueueSpaceAssignment } from "~/lib/queue-adapter.server";
 
 // Schema for manual assignment trigger
 const ManualAssignmentSchema = z.object({
@@ -38,7 +38,7 @@ const { action } = createActionApiRoute(
       let taskRun;
 
       // Direct LLM assignment trigger
-      taskRun = await triggerSpaceAssignment({
+      taskRun = await enqueueSpaceAssignment({
         userId,
         workspaceId: user?.Workspace?.id as string,
         mode: body.mode,
@@ -49,7 +49,7 @@ const { action } = createActionApiRoute(
       return json({
         success: true,
         message: `${body.mode} assignment task triggered successfully`,
-        taskId: taskRun.id,
+
         payload: {
           userId,
           mode: body.mode,

apps/webapp/app/routes/api.v1.spaces.ts

@@ -7,6 +7,7 @@ import { SpaceService } from "~/services/space.server";
 import { json } from "@remix-run/node";
 import { prisma } from "~/db.server";
 import { apiCors } from "~/utils/apiCors";
+import { isTriggerDeployment } from "~/lib/queue-adapter.server";
 
 const spaceService = new SpaceService();
 
@@ -40,6 +41,13 @@ const { action } = createHybridActionApiRoute(
       },
     });
 
+    if (!isTriggerDeployment()) {
+      return json(
+        { error: "Spaces don't work in non trigger deployment" },
+        { status: 400 },
+      );
+    }
+
     if (!user?.Workspace?.id) {
       throw new Error(
         "Workspace ID is required to create an ingestion queue entry.",

apps/webapp/app/routes/home.conversation.$conversationId.tsx

@@ -1,42 +1,26 @@
-import {
-  type LoaderFunctionArgs,
-  type ActionFunctionArgs,
-} from "@remix-run/server-runtime";
-import { sort } from "fast-sort";
+import { type LoaderFunctionArgs } from "@remix-run/server-runtime";
 
-import { useParams, useRevalidator, useNavigate } from "@remix-run/react";
-import { parse } from "@conform-to/zod";
-import {
-  requireUserId,
-  requireUser,
-  requireWorkpace,
-} from "~/services/session.server";
-import {
-  getConversationAndHistory,
-  getCurrentConversationRun,
-  stopConversation,
-  createConversation,
-  CreateConversationSchema,
-} from "~/services/conversation.server";
-import { type ConversationHistory } from "@core/database";
+import { useParams, useNavigate } from "@remix-run/react";
+import { requireUser, requireWorkpace } from "~/services/session.server";
+import { getConversationAndHistory } from "~/services/conversation.server";
 import {
   ConversationItem,
   ConversationTextarea,
-  StreamingConversation,
 } from "~/components/conversation";
 import { useTypedLoaderData } from "remix-typedjson";
-import React from "react";
 import { ScrollAreaWithAutoScroll } from "~/components/use-auto-scroll";
 import { PageHeader } from "~/components/common/page-header";
 import { Plus } from "lucide-react";
 
-import { json } from "@remix-run/node";
-import { env } from "~/env.server";
+import { type UIMessage, useChat } from "@ai-sdk/react";
+import { DefaultChatTransport } from "ai";
+import { UserTypeEnum } from "@core/types";
+import React from "react";
 
 // Example loader accessing params
 export async function loader({ params, request }: LoaderFunctionArgs) {
   const user = await requireUser(request);
-  const workspace = await requireWorkpace(request);
+
   const conversation = await getConversationAndHistory(
     params.conversationId as string,
     user.id,
@@ -46,100 +30,38 @@ export async function loader({ params, request }: LoaderFunctionArgs) {
     throw new Error("No conversation found");
   }
 
-  const run = await getCurrentConversationRun(conversation.id, workspace.id);
-
-  return { conversation, run, apiURL: env.TRIGGER_API_URL };
-}
-
-// Example action accessing params
-export async function action({ params, request }: ActionFunctionArgs) {
-  if (request.method.toUpperCase() !== "POST") {
-    return new Response("Method Not Allowed", { status: 405 });
-  }
-
-  const userId = await requireUserId(request);
-  const workspace = await requireWorkpace(request);
-  const formData = await request.formData();
-  const { conversationId } = params;
-
-  if (!conversationId) {
-    throw new Error("No conversation");
-  }
-
-  // Check if this is a stop request (isLoading = true means stop button was clicked)
-  const message = formData.get("message");
-
-  // If no message, it's a stop request
-  if (!message) {
-    const result = await stopConversation(conversationId, workspace.id);
-    return json(result);
-  }
-
-  // Otherwise, create a new conversation message
-  const submission = parse(formData, { schema: CreateConversationSchema });
-
-  if (!submission.value || submission.intent !== "submit") {
-    return json(submission);
-  }
-
-  const conversation = await createConversation(workspace?.id, userId, {
-    message: submission.value.message,
-    title: submission.value.title,
-    conversationId: submission.value.conversationId,
-  });
-
-  return json({ conversation });
+  return { conversation };
 }
 
 // Accessing params in the component
 export default function SingleConversation() {
-  const { conversation, run, apiURL } = useTypedLoaderData<typeof loader>();
-  const conversationHistory = conversation.ConversationHistory;
-
-  const [conversationResponse, setConversationResponse] = React.useState<
-    { conversationHistoryId: string; id: string; token: string } | undefined
-  >(run);
-
-  const { conversationId } = useParams();
-  const revalidator = useRevalidator();
+  const { conversation } = useTypedLoaderData<typeof loader>();
   const navigate = useNavigate();
+  const { conversationId } = useParams();
+
+  const { sendMessage, messages, status, stop, regenerate } = useChat({
+    id: conversationId, // use the provided chat ID
+    messages: conversation.ConversationHistory.map(
+      (history) =>
+        ({
+          role: history.userType === UserTypeEnum.Agent ? "assistant" : "user",
+          parts: [{ text: history.message, type: "text" }],
+        }) as UIMessage,
+    ), // load initial messages
+    transport: new DefaultChatTransport({
+      api: "/api/v1/conversation",
+      prepareSendMessagesRequest({ messages, id }) {
+        return { body: { message: messages[messages.length - 1], id } };
+      },
+    }),
+  });
+  console.log("new", messages);
 
   React.useEffect(() => {
-    if (run) {
-      setConversationResponse(run);
+    if (messages.length === 1) {
+      regenerate();
     }
-  }, [run]);
-
-  const conversations = React.useMemo(() => {
-    const lastConversationHistoryId =
-      conversationResponse?.conversationHistoryId;
-
-    // First sort the conversation history by creation time
-    const sortedConversationHistory = sort(conversationHistory).asc(
-      (ch) => ch.createdAt,
-    );
-
-    const lastIndex = sortedConversationHistory.findIndex(
-      (item) => item.id === lastConversationHistoryId,
-    );
-
-    // Filter out any conversation history items that come after the lastConversationHistoryId
-    return lastConversationHistoryId
-      ? sortedConversationHistory.filter((_ch, currentIndex: number) => {
-          return currentIndex <= lastIndex;
-        })
-      : sortedConversationHistory;
-  }, [conversationResponse, conversationHistory]);
-
-  const getConversations = () => {
-    return (
-      <>
-        {conversations.map((ch: ConversationHistory) => {
-          return <ConversationItem key={ch.id} conversationHistory={ch} />;
-        })}
-      </>
-    );
-  };
+  }, []);
 
   if (typeof window === "undefined") {
     return null;
@@ -166,41 +88,23 @@ export default function SingleConversation() {
       <div className="relative flex h-[calc(100vh_-_56px)] w-full flex-col items-center justify-center overflow-auto">
         <div className="flex h-[calc(100vh_-_80px)] w-full flex-col justify-end overflow-hidden">
           <ScrollAreaWithAutoScroll>
-            {getConversations()}
-            {conversationResponse && (
-              <StreamingConversation
-                runId={conversationResponse.id}
-                token={conversationResponse.token}
-                afterStreaming={() => {
-                  setConversationResponse(undefined);
-                  revalidator.revalidate();
-                }}
-                apiURL={apiURL}
-              />
-            )}
+            {messages.map((message: UIMessage, index: number) => {
+              return <ConversationItem key={index} message={message} />;
+            })}
           </ScrollAreaWithAutoScroll>
 
           <div className="flex w-full flex-col items-center">
             <div className="w-full max-w-[80ch] px-1 pr-2">
-              {conversation?.status !== "need_approval" && (
-                <ConversationTextarea
-                  conversationId={conversationId as string}
-                  className="bg-background-3 w-full border-1 border-gray-300"
-                  isLoading={
-                    !!conversationResponse || conversation?.status === "running"
+              <ConversationTextarea
+                className="bg-background-3 w-full border-1 border-gray-300"
+                isLoading={status === "streaming" || status === "submitted"}
+                onConversationCreated={(message) => {
+                  if (message) {
+                    sendMessage({ text: message });
                   }
-                  onConversationCreated={(conversation) => {
-                    if (conversation) {
-                      setConversationResponse({
-                        conversationHistoryId:
-                          conversation.conversationHistoryId,
-                        id: conversation.id,
-                        token: conversation.token,
-                      });
-                    }
-                  }}
-                />
-              )}
+                }}
+                stop={() => stop()}
+              />
             </div>
           </div>
         </div>

apps/webapp/app/routes/home.conversation._index.tsx

@@ -43,8 +43,7 @@ export async function action({ request }: ActionFunctionArgs) {
 
   const conversation = await createConversation(workspace?.id, userId, {
     message: submission.value.message,
-    title: submission.value.title,
-    conversationId: submission.value.conversationId,
+    title: submission.value.title ?? "Untitled",
   });
 
   // If conversationId exists in submission, return the conversation data (don't redirect)

apps/webapp/app/routes/home.inbox.$logId.tsx

@@ -40,7 +40,7 @@ export default function InboxNotSelected() {
       <PageHeader
         title="Episode"
         showTrigger={false}
-        actionsNode={<LogOptions id={log.id} />}
+        actionsNode={<LogOptions id={log.id} status={log.status} />}
       />
 
       <LogDetails log={log as any} />

apps/webapp/app/services/conversation.server.ts

@@ -1,11 +1,9 @@
 import { UserTypeEnum } from "@core/types";
 
-import { auth, runs, tasks } from "@trigger.dev/sdk/v3";
 import { prisma } from "~/db.server";
-import { createConversationTitle } from "~/trigger/conversation/create-conversation-title";
 
 import { z } from "zod";
-import { type ConversationHistory } from "@prisma/client";
+import { trackFeatureUsage } from "~/services/telemetry.server";
 
 export const CreateConversationSchema = z.object({
   message: z.string(),
@@ -44,20 +42,10 @@ export async function createConversation(
       },
     });
 
-    const context = await getConversationContext(conversationHistory.id);
-    const handler = await tasks.trigger(
-      "chat",
-      {
-        conversationHistoryId: conversationHistory.id,
-        conversationId: conversationHistory.conversation.id,
-        context,
-      },
-      { tags: [conversationHistory.id, workspaceId, conversationId] },
-    );
+    // Track conversation message
+    trackFeatureUsage("conversation_message_sent", userId).catch(console.error);
 
     return {
-      id: handler.id,
-      token: handler.publicAccessToken,
       conversationId: conversationHistory.conversation.id,
       conversationHistoryId: conversationHistory.id,
     };
@@ -84,40 +72,20 @@ export async function createConversation(
   });
 
   const conversationHistory = conversation.ConversationHistory[0];
-  const context = await getConversationContext(conversationHistory.id);
 
-  // Trigger conversation title task
-  await tasks.trigger<typeof createConversationTitle>(
-    createConversationTitle.id,
-    {
-      conversationId: conversation.id,
-      message: conversationData.message,
-    },
-    { tags: [conversation.id, workspaceId] },
-  );
-
-  const handler = await tasks.trigger(
-    "chat",
-    {
-      conversationHistoryId: conversationHistory.id,
-      conversationId: conversation.id,
-      context,
-    },
-    { tags: [conversationHistory.id, workspaceId, conversation.id] },
-  );
+  // Track new conversation creation
+  trackFeatureUsage("conversation_created", userId).catch(console.error);
 
   return {
-    id: handler.id,
-    token: handler.publicAccessToken,
     conversationId: conversation.id,
     conversationHistoryId: conversationHistory.id,
   };
 }
 
 // Get a conversation by ID
-export async function getConversation(conversationId: string) {
+export async function getConversation(conversationId: string, userId: string) {
   return prisma.conversation.findUnique({
-    where: { id: conversationId },
+    where: { id: conversationId, userId },
   });
 }
 
@@ -139,141 +107,6 @@ export async function readConversation(conversationId: string) {
   });
 }
 
-export async function getCurrentConversationRun(
-  conversationId: string,
-  workspaceId: string,
-) {
-  const conversationHistory = await prisma.conversationHistory.findFirst({
-    where: {
-      conversationId,
-      conversation: {
-        workspaceId,
-      },
-      userType: UserTypeEnum.User,
-    },
-    orderBy: {
-      updatedAt: "desc",
-    },
-  });
-
-  if (!conversationHistory) {
-    throw new Error("No run found");
-  }
-
-  const response = await runs.list({
-    tag: [conversationId, conversationHistory.id, workspaceId],
-    status: ["QUEUED", "EXECUTING"],
-    limit: 1,
-  });
-
-  if (!response) {
-    return undefined;
-  }
-
-  const run = response?.data?.[0];
-
-  if (!run) {
-    return undefined;
-  }
-
-  const publicToken = await auth.createPublicToken({
-    scopes: {
-      read: {
-        runs: [run.id],
-      },
-    },
-  });
-
-  return {
-    id: run.id,
-    token: publicToken,
-    conversationId,
-    conversationHistoryId: conversationHistory.id,
-  };
-}
-
-export async function stopConversation(
-  conversationId: string,
-  workspaceId: string,
-) {
-  const conversationHistory = await prisma.conversationHistory.findFirst({
-    where: {
-      conversationId,
-      conversation: {
-        workspaceId,
-      },
-    },
-    orderBy: {
-      updatedAt: "desc",
-    },
-  });
-
-  if (!conversationHistory) {
-    throw new Error("No run found");
-  }
-
-  const response = await runs.list({
-    tag: [conversationId, conversationHistory.id],
-    status: ["QUEUED", "EXECUTING"],
-    limit: 1,
-  });
-
-  const run = response.data[0];
-  if (!run) {
-    await prisma.conversation.update({
-      where: {
-        id: conversationId,
-      },
-      data: {
-        status: "failed",
-      },
-    });
-
-    return undefined;
-  }
-
-  return await runs.cancel(run.id);
-}
-
-export async function getConversationContext(
-  conversationHistoryId: string,
-): Promise<{
-  previousHistory: ConversationHistory[];
-}> {
-  const conversationHistory = await prisma.conversationHistory.findUnique({
-    where: { id: conversationHistoryId },
-    include: { conversation: true },
-  });
-
-  if (!conversationHistory) {
-    return {
-      previousHistory: [],
-    };
-  }
-
-  // Get previous conversation history message and response
-  let previousHistory: ConversationHistory[] = [];
-
-  if (conversationHistory.conversationId) {
-    previousHistory = await prisma.conversationHistory.findMany({
-      where: {
-        conversationId: conversationHistory.conversationId,
-        id: {
-          not: conversationHistoryId,
-        },
-        deleted: null,
-      },
-      orderBy: {
-        createdAt: "asc",
-      },
-    });
-  }
-
-  return {
-    previousHistory,
-  };
-}
-
 export const getConversationAndHistory = async (
   conversationId: string,
   userId: string,
@@ -281,6 +114,7 @@ export const getConversationAndHistory = async (
   const conversation = await prisma.conversation.findFirst({
     where: {
       id: conversationId,
+      userId,
     },
     include: {
       ConversationHistory: true,
@@ -290,6 +124,23 @@ export const getConversationAndHistory = async (
   return conversation;
 };
 
+export const createConversationHistory = async (
+  userMessage: string,
+  conversationId: string,
+  userType: UserTypeEnum,
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  thoughts?: Record<string, any>,
+) => {
+  return await prisma.conversationHistory.create({
+    data: {
+      conversationId,
+      message: userMessage,
+      thoughts,
+      userType,
+    },
+  });
+};
+
 export const GetConversationsListSchema = z.object({
   page: z.string().optional().default("1"),
   limit: z.string().optional().default("20"),

apps/webapp/app/services/jobManager.server.ts

@@ -0,0 +1,87 @@
+/**
+ * Job Manager Service
+ *
+ * Unified interface for managing background jobs across both
+ * Trigger.dev and BullMQ queue providers.
+ */
+
+import { env } from "~/env.server";
+
+type QueueProvider = "trigger" | "bullmq";
+
+interface JobInfo {
+  id: string;
+  isCompleted: boolean;
+  status?: string;
+}
+
+/**
+ * Find running jobs by tags/identifiers
+ */
+export async function findRunningJobs(params: {
+  tags: string[];
+  taskIdentifier?: string;
+}): Promise<JobInfo[]> {
+  const provider = env.QUEUE_PROVIDER as QueueProvider;
+
+  if (provider === "trigger") {
+    const { runs } = await import("@trigger.dev/sdk");
+    const runningTasks = await runs.list({
+      tag: params.tags,
+      taskIdentifier: params.taskIdentifier,
+    });
+
+    return runningTasks.data.map((task) => ({
+      id: task.id,
+      isCompleted: task.isCompleted,
+      status: task.status,
+    }));
+  } else {
+    // BullMQ
+    const { getJobsByTags } = await import("~/bullmq/utils/job-finder");
+    const jobs = await getJobsByTags(params.tags, params.taskIdentifier);
+
+    return jobs;
+  }
+}
+
+/**
+ * Cancel a running job
+ */
+export async function cancelJob(jobId: string): Promise<void> {
+  const provider = env.QUEUE_PROVIDER as QueueProvider;
+
+  if (provider === "trigger") {
+    const { runs } = await import("@trigger.dev/sdk");
+    await runs.cancel(jobId);
+  } else {
+    // BullMQ
+    const { cancelJobById } = await import("~/bullmq/utils/job-finder");
+    await cancelJobById(jobId);
+  }
+}
+
+/**
+ * Get job status
+ */
+export async function getJobStatus(jobId: string): Promise<JobInfo | null> {
+  const provider = env.QUEUE_PROVIDER as QueueProvider;
+
+  if (provider === "trigger") {
+    const { runs } = await import("@trigger.dev/sdk");
+    try {
+      const run = await runs.retrieve(jobId);
+      return {
+        id: run.id,
+        isCompleted: run.isCompleted,
+        status: run.status,
+      };
+    } catch {
+      return null;
+    }
+  } else {
+    // BullMQ
+    const { getJobById } = await import("~/bullmq/utils/job-finder");
+    return await getJobById(jobId);
+  }
+}

apps/webapp/app/services/knowledgeGraph.server.ts

@@ -10,7 +10,6 @@ import {
   type EpisodeType,
 } from "@core/types";
 import { logger } from "./logger.service";
-import { ClusteringService } from "./clustering.server";
 import crypto from "crypto";
 import { dedupeNodes, extractEntities } from "./prompts/nodes";
 import {
@@ -50,12 +49,6 @@ import { type PrismaClient } from "@prisma/client";
 const DEFAULT_EPISODE_WINDOW = 5;
 
 export class KnowledgeGraphService {
-  private clusteringService: ClusteringService;
-
-  constructor() {
-    this.clusteringService = new ClusteringService();
-  }
-
   async getEmbedding(text: string) {
     return getEmbedding(text);
   }
@@ -564,9 +557,9 @@ export class KnowledgeGraphService {
       (text, _model, usage) => {
         responseText = text;
         if (usage) {
-          tokenMetrics.high.input += usage.promptTokens;
-          tokenMetrics.high.output += usage.completionTokens;
-          tokenMetrics.high.total += usage.totalTokens;
+          tokenMetrics.high.input += usage.promptTokens as number;
+          tokenMetrics.high.output += usage.completionTokens as number;
+          tokenMetrics.high.total += usage.totalTokens as number;
         }
       },
       undefined,

apps/webapp/app/services/personalAccessToken.server.ts

@@ -320,6 +320,14 @@ export async function getOrCreatePersonalAccessToken({
   };
 }
 
+export async function deletePersonalAccessToken(tokenId: string) {
+  return await prisma.personalAccessToken.delete({
+    where: {
+      id: tokenId,
+    },
+  });
+}
+
 /** Created a new PersonalAccessToken, and return the token. We only ever return the unencrypted token once. */
 export async function createPersonalAccessToken({
   name,

apps/webapp/app/services/search/utils.ts

@@ -51,6 +51,7 @@ export async function performBM25Search(
           ${spaceCondition}
         OPTIONAL MATCH (episode:Episode)-[:HAS_PROVENANCE]->(s)
         WITH s, score, count(episode) as provenanceCount
+        WHERE score >= 0.5
         RETURN s, score, provenanceCount
         ORDER BY score DESC
       `;
@@ -71,6 +72,12 @@ export async function performBM25Search(
         typeof provenanceCountValue === "bigint"
           ? Number(provenanceCountValue)
           : (provenanceCountValue?.toNumber?.() ?? provenanceCountValue ?? 0);
+
+      const scoreValue = record.get("score");
+      (statement as any).bm25Score =
+        typeof scoreValue === "number"
+          ? scoreValue
+          : (scoreValue?.toNumber?.() ?? 0);
       return statement;
     });
   } catch (error) {
@@ -163,6 +170,14 @@ export async function performVectorSearch(
         typeof provenanceCountValue === "bigint"
           ? Number(provenanceCountValue)
           : (provenanceCountValue?.toNumber?.() ?? provenanceCountValue ?? 0);
+
+      // Preserve vector similarity score for empty result detection
+      const scoreValue = record.get("score");
+      (statement as any).vectorScore =
+        typeof scoreValue === "number"
+          ? scoreValue
+          : (scoreValue?.toNumber?.() ?? 0);
+
       return statement;
     });
   } catch (error) {
@@ -179,12 +194,10 @@ export async function performBfsSearch(
   query: string,
   embedding: Embedding,
   userId: string,
+  entities: EntityNode[],
   options: Required<SearchOptions>,
 ): Promise<StatementNode[]> {
   try {
-    // 1. Extract potential entities from query using chunked embeddings
-    const entities = await extractEntitiesFromQuery(query, userId);
-
     if (entities.length === 0) {
       return [];
     }
@@ -224,7 +237,7 @@ async function bfsTraversal(
   const RELEVANCE_THRESHOLD = 0.5;
   const EXPLORATION_THRESHOLD = 0.3;
 
-  const allStatements = new Map<string, number>(); // uuid -> relevance
+  const allStatements = new Map<string, { relevance: number; hopDistance: number }>(); // uuid -> {relevance, hopDistance}
   const visitedEntities = new Set<string>();
 
   // Track entities per level for iterative BFS
@@ -268,14 +281,14 @@ async function bfsTraversal(
       ...(startTime && { startTime: startTime.toISOString() }),
     });
 
-    // Store statement relevance scores
+    // Store statement relevance scores and hop distance
     const currentLevelStatementUuids: string[] = [];
     for (const record of records) {
       const uuid = record.get("uuid");
       const relevance = record.get("relevance");
 
       if (!allStatements.has(uuid)) {
-        allStatements.set(uuid, relevance);
+        allStatements.set(uuid, { relevance, hopDistance: depth + 1 }); // Store hop distance (1-indexed)
         currentLevelStatementUuids.push(uuid);
       }
     }
@@ -304,25 +317,45 @@ async function bfsTraversal(
   }
 
   // Filter by relevance threshold and fetch full statements
-  const relevantUuids = Array.from(allStatements.entries())
-    .filter(([_, relevance]) => relevance >= RELEVANCE_THRESHOLD)
-    .sort((a, b) => b[1] - a[1])
-    .map(([uuid]) => uuid);
+  const relevantResults = Array.from(allStatements.entries())
+    .filter(([_, data]) => data.relevance >= RELEVANCE_THRESHOLD)
+    .sort((a, b) => b[1].relevance - a[1].relevance);
 
-  if (relevantUuids.length === 0) {
+  if (relevantResults.length === 0) {
     return [];
   }
 
+  const relevantUuids = relevantResults.map(([uuid]) => uuid);
+
   const fetchCypher = `
     MATCH (s:Statement{userId: $userId})
     WHERE s.uuid IN $uuids
     RETURN s
   `;
   const fetchRecords = await runQuery(fetchCypher, { uuids: relevantUuids, userId });
-  const statements = fetchRecords.map(r => r.get("s").properties as StatementNode);
+  const statementMap = new Map(
+    fetchRecords.map(r => [r.get("s").properties.uuid, r.get("s").properties as StatementNode])
+  );
+
+  // Attach hop distance to statements
+  const statements = relevantResults.map(([uuid, data]) => {
+    const statement = statementMap.get(uuid)!;
+    // Add bfsHopDistance and bfsRelevance as metadata
+    (statement as any).bfsHopDistance = data.hopDistance;
+    (statement as any).bfsRelevance = data.relevance;
+    return statement;
+  });
+
+  const hopCounts = statements.reduce((acc, s) => {
+    const hop = (s as any).bfsHopDistance;
+    acc[hop] = (acc[hop] || 0) + 1;
+    return acc;
+  }, {} as Record<number, number>);
 
   logger.info(
-    `BFS: explored ${allStatements.size} statements across ${maxDepth} hops, returning ${statements.length} (≥${RELEVANCE_THRESHOLD})`
+    `BFS: explored ${allStatements.size} statements across ${maxDepth} hops, ` +
+    `returning ${statements.length} (≥${RELEVANCE_THRESHOLD}) - ` +
+    `1-hop: ${hopCounts[1] || 0}, 2-hop: ${hopCounts[2] || 0}, 3-hop: ${hopCounts[3] || 0}, 4-hop: ${hopCounts[4] || 0}`
   );
 
   return statements;
@@ -361,15 +394,22 @@ function generateQueryChunks(query: string): string[] {
 export async function extractEntitiesFromQuery(
   query: string,
   userId: string,
+  startEntities: string[] = [],
 ): Promise<EntityNode[]> {
   try {
-    // Generate chunks from query
-    const chunks = generateQueryChunks(query);
-
-    // Get embeddings for each chunk
-    const chunkEmbeddings = await Promise.all(
-      chunks.map(chunk => getEmbedding(chunk))
-    );
+    let chunkEmbeddings: Embedding[] = [];
+    if (startEntities.length === 0) {
+      // Generate chunks from query
+      const chunks = generateQueryChunks(query);
+      // Get embeddings for each chunk
+      chunkEmbeddings = await Promise.all(
+        chunks.map(chunk => getEmbedding(chunk))
+      );
+    } else {
+      chunkEmbeddings = await Promise.all(
+        startEntities.map(chunk => getEmbedding(chunk))
+      );
+    }
 
     // Search for entities matching each chunk embedding
     const allEntitySets = await Promise.all(
@@ -425,3 +465,280 @@ export async function getEpisodesByStatements(
   const records = await runQuery(cypher, params);
   return records.map((record) => record.get("e").properties as EpisodicNode);
 }
+
+/**
+ * Episode Graph Search Result
+ */
+export interface EpisodeGraphResult {
+  episode: EpisodicNode;
+  statements: StatementNode[];
+  score: number;
+  metrics: {
+    entityMatchCount: number;
+    totalStatementCount: number;
+    avgRelevance: number;
+    connectivityScore: number;
+  };
+}
+
+/**
+ * Perform episode-centric graph search
+ * Finds episodes with dense subgraphs of statements connected to query entities
+ */
+export async function performEpisodeGraphSearch(
+  query: string,
+  queryEntities: EntityNode[],
+  queryEmbedding: Embedding,
+  userId: string,
+  options: Required<SearchOptions>,
+): Promise<EpisodeGraphResult[]> {
+  try {
+    // If no entities extracted, return empty
+    if (queryEntities.length === 0) {
+      logger.info("Episode graph search: no entities extracted from query");
+      return [];
+    }
+
+    const queryEntityIds = queryEntities.map(e => e.uuid);
+    logger.info(`Episode graph search: ${queryEntityIds.length} query entities`, {
+      entities: queryEntities.map(e => e.name).join(', ')
+    });
+
+    // Timeframe condition for temporal filtering
+    let timeframeCondition = `
+      AND s.validAt <= $validAt
+      ${options.includeInvalidated ? '' : 'AND (s.invalidAt IS NULL OR s.invalidAt > $validAt)'}
+    `;
+    if (options.startTime) {
+      timeframeCondition += ` AND s.validAt >= $startTime`;
+    }
+
+    // Space filtering if provided
+    let spaceCondition = "";
+    if (options.spaceIds.length > 0) {
+      spaceCondition = `
+        AND s.spaceIds IS NOT NULL AND ANY(spaceId IN $spaceIds WHERE spaceId IN s.spaceIds)
+      `;
+    }
+
+    const cypher = `
+      // Step 1: Find statements connected to query entities
+      MATCH (queryEntity:Entity)-[:HAS_SUBJECT|HAS_OBJECT|HAS_PREDICATE]-(s:Statement)
+      WHERE queryEntity.uuid IN $queryEntityIds
+        AND queryEntity.userId = $userId
+        AND s.userId = $userId
+        ${timeframeCondition}
+        ${spaceCondition}
+
+      // Step 2: Find episodes containing these statements
+      MATCH (s)<-[:HAS_PROVENANCE]-(ep:Episode)
+
+      // Step 3: Collect all statements from these episodes (for metrics only)
+      MATCH (ep)-[:HAS_PROVENANCE]->(epStatement:Statement)
+      WHERE epStatement.validAt <= $validAt
+        AND (epStatement.invalidAt IS NULL OR epStatement.invalidAt > $validAt)
+        ${spaceCondition.replace(/s\./g, 'epStatement.')}
+
+      // Step 4: Calculate episode-level metrics
+      WITH ep,
+           collect(DISTINCT s) as entityMatchedStatements,
+           collect(DISTINCT epStatement) as allEpisodeStatements,
+           collect(DISTINCT queryEntity) as matchedEntities
+
+      // Step 5: Calculate semantic relevance for all episode statements
+      WITH ep,
+           entityMatchedStatements,
+           allEpisodeStatements,
+           matchedEntities,
+           [stmt IN allEpisodeStatements |
+             gds.similarity.cosine(stmt.factEmbedding, $queryEmbedding)
+           ] as statementRelevances
+
+      // Step 6: Calculate aggregate scores
+      WITH ep,
+           entityMatchedStatements,
+           size(matchedEntities) as entityMatchCount,
+           size(entityMatchedStatements) as entityStmtCount,
+           size(allEpisodeStatements) as totalStmtCount,
+           reduce(sum = 0.0, score IN statementRelevances | sum + score) /
+             CASE WHEN size(statementRelevances) = 0 THEN 1 ELSE size(statementRelevances) END as avgRelevance
+
+      // Step 7: Calculate connectivity score
+      WITH ep,
+           entityMatchedStatements,
+           entityMatchCount,
+           entityStmtCount,
+           totalStmtCount,
+           avgRelevance,
+           (toFloat(entityStmtCount) / CASE WHEN totalStmtCount = 0 THEN 1 ELSE totalStmtCount END) *
+             entityMatchCount as connectivityScore
+
+      // Step 8: Filter for quality episodes
+      WHERE entityMatchCount >= 1
+        AND avgRelevance >= 0.5
+        AND totalStmtCount >= 1
+
+      // Step 9: Calculate final episode score
+      WITH ep,
+           entityMatchedStatements,
+           entityMatchCount,
+           totalStmtCount,
+           avgRelevance,
+           connectivityScore,
+           // Prioritize: entity matches (2.0x) + connectivity + semantic relevance
+           (entityMatchCount * 2.0) + connectivityScore + avgRelevance as episodeScore
+
+      // Step 10: Return ranked episodes with ONLY entity-matched statements
+      RETURN ep,
+             entityMatchedStatements as statements,
+             entityMatchCount,
+             totalStmtCount,
+             avgRelevance,
+             connectivityScore,
+             episodeScore
+
+      ORDER BY episodeScore DESC, entityMatchCount DESC, totalStmtCount DESC
+      LIMIT 20
+    `;
+
+    const params = {
+      queryEntityIds,
+      userId,
+      queryEmbedding,
+      validAt: options.endTime.toISOString(),
+      ...(options.startTime && { startTime: options.startTime.toISOString() }),
+      ...(options.spaceIds.length > 0 && { spaceIds: options.spaceIds }),
+    };
+
+    const records = await runQuery(cypher, params);
+
+    const results: EpisodeGraphResult[] = records.map((record) => {
+      const episode = record.get("ep").properties as EpisodicNode;
+      const statements = record.get("statements").map((s: any) => s.properties as StatementNode);
+      const entityMatchCount = typeof record.get("entityMatchCount") === 'bigint'
+        ? Number(record.get("entityMatchCount"))
+        : record.get("entityMatchCount");
+      const totalStmtCount = typeof record.get("totalStmtCount") === 'bigint'
+        ? Number(record.get("totalStmtCount"))
+        : record.get("totalStmtCount");
+      const avgRelevance = record.get("avgRelevance");
+      const connectivityScore = record.get("connectivityScore");
+      const episodeScore = record.get("episodeScore");
+
+      return {
+        episode,
+        statements,
+        score: episodeScore,
+        metrics: {
+          entityMatchCount,
+          totalStatementCount: totalStmtCount,
+          avgRelevance,
+          connectivityScore,
+        },
+      };
+    });
+
+    // Log statement counts for debugging
+    results.forEach((result, idx) => {
+      logger.info(
+        `Episode ${idx + 1}: entityMatches=${result.metrics.entityMatchCount}, ` +
+        `totalStmtCount=${result.metrics.totalStatementCount}, ` +
+        `returnedStatements=${result.statements.length}`
+      );
+    });
+
+    logger.info(
+      `Episode graph search: found ${results.length} episodes, ` +
+      `top score: ${results[0]?.score.toFixed(2) || 'N/A'}`
+    );
+
+    return results;
+  } catch (error) {
+    logger.error("Episode graph search error:", { error });
+    return [];
+  }
+}
+
+/**
+ * Get episode IDs for statements in batch (efficient, no N+1 queries)
+ */
+export async function getEpisodeIdsForStatements(
+  statementUuids: string[]
+): Promise<Map<string, string>> {
+  if (statementUuids.length === 0) {
+    return new Map();
+  }
+
+  const cypher = `
+    MATCH (s:Statement)<-[:HAS_PROVENANCE]-(e:Episode)
+    WHERE s.uuid IN $statementUuids
+    RETURN s.uuid as statementUuid, e.uuid as episodeUuid
+  `;
+
+  const records = await runQuery(cypher, { statementUuids });
+
+  const map = new Map<string, string>();
+  records.forEach(record => {
+    map.set(record.get('statementUuid'), record.get('episodeUuid'));
+  });
+
+  return map;
+}
+
+/**
+ * Group statements by their episode IDs efficiently
+ */
+export async function groupStatementsByEpisode(
+  statements: StatementNode[]
+): Promise<Map<string, StatementNode[]>> {
+  const grouped = new Map<string, StatementNode[]>();
+
+  if (statements.length === 0) {
+    return grouped;
+  }
+
+  // Batch fetch episode IDs for all statements
+  const episodeIdMap = await getEpisodeIdsForStatements(
+    statements.map(s => s.uuid)
+  );
+
+  // Group statements by episode ID
+  statements.forEach((statement) => {
+    const episodeId = episodeIdMap.get(statement.uuid);
+    if (episodeId) {
+      if (!grouped.has(episodeId)) {
+        grouped.set(episodeId, []);
+      }
+      grouped.get(episodeId)!.push(statement);
+    }
+  });
+
+  return grouped;
+}
+
+/**
+ * Fetch episode objects by their UUIDs in batch
+ */
+export async function getEpisodesByUuids(
+  episodeUuids: string[]
+): Promise<Map<string, EpisodicNode>> {
+  if (episodeUuids.length === 0) {
+    return new Map();
+  }
+
+  const cypher = `
+    MATCH (e:Episode)
+    WHERE e.uuid IN $episodeUuids
+    RETURN e
+  `;
+
+  const records = await runQuery(cypher, { episodeUuids });
+
+  const map = new Map<string, EpisodicNode>();
+  records.forEach(record => {
+    const episode = record.get('e').properties as EpisodicNode;
+    map.set(episode.uuid, episode);
+  });
+
+  return map;
+}

apps/webapp/app/services/sessionCompaction.server.ts

@@ -5,7 +5,7 @@ import {
   getSessionEpisodes,
   type CompactedSessionNode,
 } from "~/services/graphModels/compactedSession";
-import { tasks } from "@trigger.dev/sdk/v3";
+import { enqueueSessionCompaction } from "~/lib/queue-adapter.server";
 
 /**
  * Configuration for session compaction
@@ -144,7 +144,7 @@ export class SessionCompactionService {
         reason: check.reason,
       });
 
-      const handle = await tasks.trigger("session-compaction", {
+      const handle = await enqueueSessionCompaction({
         userId,
         sessionId,
         source,

apps/webapp/app/services/space.server.ts

@@ -6,7 +6,6 @@ import {
 } from "@core/types";
 import { type Space } from "@prisma/client";
 
-import { triggerSpaceAssignment } from "~/trigger/spaces/space-assignment";
 import {
   assignEpisodesToSpace,
   createSpace,
@@ -17,6 +16,8 @@ import {
   updateSpace,
 } from "./graphModels/space";
 import { prisma } from "~/trigger/utils/prisma";
+import { trackFeatureUsage } from "./telemetry.server";
+import { enqueueSpaceAssignment } from "~/lib/queue-adapter.server";
 
 export class SpaceService {
   /**
@@ -63,9 +64,12 @@ export class SpaceService {
 
     logger.info(`Created space ${space.id} successfully`);
 
+    // Track space creation
+    trackFeatureUsage("space_created", params.userId).catch(console.error);
+
     // Trigger automatic LLM assignment for the new space
     try {
-      await triggerSpaceAssignment({
+      await enqueueSpaceAssignment({
         userId: params.userId,
         workspaceId: params.workspaceId,
         mode: "new_space",
@@ -192,6 +196,10 @@ export class SpaceService {
     } catch (e) {
       logger.info(`Nothing to update to graph`);
     }
+
+    // Track space update
+    trackFeatureUsage("space_updated", userId).catch(console.error);
+
     logger.info(`Updated space ${spaceId} successfully`);
     return space;
   }

apps/webapp/app/services/telemetry.server.ts

@@ -0,0 +1,278 @@
+import { PostHog } from "posthog-node";
+import { env } from "~/env.server";
+import { prisma } from "~/db.server";
+
+// Server-side PostHog client for backend tracking
+let posthogClient: PostHog | null = null;
+
+function getPostHogClient(): PostHog | null {
+  if (!env.TELEMETRY_ENABLED || !env.POSTHOG_PROJECT_KEY) {
+    return null;
+  }
+
+  if (!posthogClient) {
+    posthogClient = new PostHog(env.POSTHOG_PROJECT_KEY, {
+      host: "https://us.posthog.com",
+    });
+  }
+
+  return posthogClient;
+}
+
+/**
+ * Get user email from userId, or return "anonymous" if TELEMETRY_ANONYMOUS is enabled
+ */
+async function getUserIdentifier(userId?: string): Promise<string> {
+  if (env.TELEMETRY_ANONYMOUS || !userId) {
+    return "anonymous";
+  }
+
+  try {
+    const user = await prisma.user.findUnique({
+      where: { id: userId },
+      select: { email: true },
+    });
+    return user?.email || "anonymous";
+  } catch (error) {
+    return "anonymous";
+  }
+}
+
+// Telemetry event types
+export type TelemetryEvent =
+  | "episode_ingested"
+  | "document_ingested"
+  | "search_performed"
+  | "deep_search_performed"
+  | "conversation_created"
+  | "conversation_message_sent"
+  | "space_created"
+  | "space_updated"
+  | "user_registered"
+  | "error_occurred"
+  | "queue_job_started"
+  | "queue_job_completed"
+  | "queue_job_failed";
+
+// Common properties for all events
+interface BaseEventProperties {
+  userId?: string;
+  workspaceId?: string;
+  email?: string;
+  name?: string;
+  queueProvider?: "trigger" | "bullmq";
+  modelProvider?: string;
+  embeddingModel?: string;
+  appEnv?: string;
+}
+
+// Event-specific properties
+interface EpisodeIngestedProperties extends BaseEventProperties {
+  spaceId?: string;
+  documentCount?: number;
+  processingTimeMs?: number;
+}
+
+interface SearchPerformedProperties extends BaseEventProperties {
+  query: string;
+  resultsCount: number;
+  searchType: "basic" | "deep";
+  spaceIds?: string[];
+}
+
+interface ConversationProperties extends BaseEventProperties {
+  conversationId: string;
+  messageLength?: number;
+  model?: string;
+}
+
+interface ErrorProperties extends BaseEventProperties {
+  errorType: string;
+  errorMessage: string;
+  stackTrace?: string;
+  context?: Record<string, any>;
+}
+
+interface QueueJobProperties extends BaseEventProperties {
+  jobId: string;
+  jobType: string;
+  queueName: string;
+  durationMs?: number;
+}
+
+type EventProperties =
+  | EpisodeIngestedProperties
+  | SearchPerformedProperties
+  | ConversationProperties
+  | ErrorProperties
+  | QueueJobProperties
+  | BaseEventProperties;
+
+/**
+ * Track telemetry events to PostHog
+ */
+export async function trackEvent(
+  event: TelemetryEvent,
+  properties: EventProperties,
+): Promise<void> {
+  const client = getPostHogClient();
+  if (!client) return;
+
+  try {
+    const userId = properties.userId || "anonymous";
+
+    // Add common properties to all events
+    const enrichedProperties = {
+      ...properties,
+      queueProvider: env.QUEUE_PROVIDER,
+      modelProvider: getModelProvider(),
+      embeddingModel: env.EMBEDDING_MODEL,
+      appEnv: env.APP_ENV,
+      appOrigin: env.APP_ORIGIN,
+      timestamp: new Date().toISOString(),
+    };
+
+    client.capture({
+      distinctId: userId,
+      event,
+      properties: enrichedProperties,
+    });
+
+    // Identify user if we have their info
+    if (properties.email || properties.name) {
+      client.identify({
+        distinctId: userId,
+        properties: {
+          email: properties.email,
+          name: properties.name,
+        },
+      });
+    }
+  } catch (error) {
+    // Silently fail - don't break the app if telemetry fails
+    console.error("Telemetry error:", error);
+  }
+}
+
+/**
+ * Track feature usage - simplified API
+ * @param feature - Feature name (e.g., "episode_ingested", "search_performed")
+ * @param userId - User ID (will be converted to email internally)
+ * @param properties - Additional properties (optional)
+ */
+export async function trackFeatureUsage(
+  feature: string,
+  userId?: string,
+  properties?: Record<string, any>,
+): Promise<void> {
+  const client = getPostHogClient();
+  if (!client) return;
+
+  try {
+    const email = await getUserIdentifier(userId);
+
+    client.capture({
+      distinctId: email,
+      event: feature,
+      properties: {
+        ...properties,
+        appOrigin: env.APP_ORIGIN,
+        timestamp: new Date().toISOString(),
+      },
+    });
+  } catch (error) {
+    // Silently fail - don't break the app if telemetry fails
+    console.error("Telemetry error:", error);
+  }
+}
+
+/**
+ * Track system configuration once at startup
+ * Tracks queue provider, model provider, embedding model, etc.
+ */
+export async function trackConfig(): Promise<void> {
+  const client = getPostHogClient();
+  if (!client) return;
+
+  try {
+    client.capture({
+      distinctId: "system",
+      event: "system_config",
+      properties: {
+        queueProvider: env.QUEUE_PROVIDER,
+        modelProvider: getModelProvider(),
+        model: env.MODEL,
+        embeddingModel: env.EMBEDDING_MODEL,
+        appEnv: env.APP_ENV,
+        nodeEnv: env.NODE_ENV,
+        timestamp: new Date().toISOString(),
+        appOrigin: env.APP_ORIGIN,
+      },
+    });
+  } catch (error) {
+    console.error("Failed to track config:", error);
+  }
+}
+
+/**
+ * Track errors
+ */
+export async function trackError(
+  error: Error,
+  context?: Record<string, any>,
+  userId?: string,
+): Promise<void> {
+  const client = getPostHogClient();
+  if (!client) return;
+
+  try {
+    const email = await getUserIdentifier(userId);
+
+    client.capture({
+      distinctId: email,
+      event: "error_occurred",
+      properties: {
+        errorType: error.name,
+        errorMessage: error.message,
+        appOrigin: env.APP_ORIGIN,
+        stackTrace: error.stack,
+        ...context,
+        timestamp: new Date().toISOString(),
+      },
+    });
+  } catch (trackingError) {
+    console.error("Failed to track error:", trackingError);
+  }
+}
+
+/**
+ * Flush pending events (call on shutdown)
+ */
+export async function flushTelemetry(): Promise<void> {
+  const client = getPostHogClient();
+  if (client) {
+    await client.shutdown();
+  }
+}
+
+/**
+ * Helper to determine model provider from MODEL env variable
+ */
+function getModelProvider(): string {
+  const model = env.MODEL.toLowerCase();
+  if (model.includes("gpt") || model.includes("openai")) return "openai";
+  if (model.includes("claude") || model.includes("anthropic"))
+    return "anthropic";
+  if (env.OLLAMA_URL) return "ollama";
+  return "unknown";
+}
+
+// Export types for use in other files
+export type {
+  BaseEventProperties,
+  EpisodeIngestedProperties,
+  SearchPerformedProperties,
+  ConversationProperties,
+  ErrorProperties,
+  QueueJobProperties,
+};

apps/webapp/app/trigger/chat/chat-utils.ts

@@ -1,492 +0,0 @@
-/* eslint-disable @typescript-eslint/no-explicit-any */
-
-import { ActionStatusEnum } from "@core/types";
-import { logger } from "@trigger.dev/sdk/v3";
-import {
-  type CoreMessage,
-  type DataContent,
-  jsonSchema,
-  tool,
-  type ToolSet,
-} from "ai";
-import axios from "axios";
-import Handlebars from "handlebars";
-
-import { REACT_SYSTEM_PROMPT, REACT_USER_PROMPT } from "./prompt";
-import { generate, processTag } from "./stream-utils";
-import { type AgentMessage, AgentMessageType, Message } from "./types";
-import { type MCP } from "../utils/mcp";
-import {
-  type ExecutionState,
-  type HistoryStep,
-  type Resource,
-  type TotalCost,
-} from "../utils/types";
-import { flattenObject } from "../utils/utils";
-
-interface LLMOutputInterface {
-  response: AsyncGenerator<
-    | string
-    | {
-        type: string;
-        toolName: string;
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        args?: any;
-        toolCallId?: string;
-        message?: string;
-      },
-    any,
-    any
-  >;
-}
-
-const progressUpdateTool = tool({
-  description:
-    "Send a progress update to the user about what has been discovered or will be done next in a crisp and user friendly way no technical terms",
-  parameters: jsonSchema({
-    type: "object",
-    properties: {
-      message: {
-        type: "string",
-        description: "The progress update message to send to the user",
-      },
-    },
-    required: ["message"],
-    additionalProperties: false,
-  }),
-});
-
-const internalTools = ["core--progress_update"];
-
-async function addResources(messages: CoreMessage[], resources: Resource[]) {
-  const resourcePromises = resources.map(async (resource) => {
-    // Remove everything before "/api" in the publicURL
-    if (resource.publicURL) {
-      const apiIndex = resource.publicURL.indexOf("/api");
-      if (apiIndex !== -1) {
-        resource.publicURL = resource.publicURL.substring(apiIndex);
-      }
-    }
-    const response = await axios.get(resource.publicURL, {
-      responseType: "arraybuffer",
-    });
-
-    if (resource.fileType.startsWith("image/")) {
-      return {
-        type: "image",
-        image: response.data as DataContent,
-      };
-    }
-
-    return {
-      type: "file",
-      data: response.data as DataContent,
-
-      mimeType: resource.fileType,
-    };
-  });
-
-  const content = await Promise.all(resourcePromises);
-
-  return [...messages, { role: "user", content } as CoreMessage];
-}
-
-function toolToMessage(history: HistoryStep[], messages: CoreMessage[]) {
-  for (let i = 0; i < history.length; i++) {
-    const step = history[i];
-
-    // Add assistant message with tool calls
-    if (step.observation && step.skillId) {
-      messages.push({
-        role: "assistant",
-        content: [
-          {
-            type: "tool-call",
-            toolCallId: step.skillId,
-            toolName: step.skill ?? "",
-            args:
-              typeof step.skillInput === "string"
-                ? JSON.parse(step.skillInput)
-                : step.skillInput,
-          },
-        ],
-      });
-
-      messages.push({
-        role: "tool",
-        content: [
-          {
-            type: "tool-result",
-            toolName: step.skill,
-            toolCallId: step.skillId,
-            result: step.observation,
-            isError: step.isError,
-          },
-        ],
-      } as any);
-    }
-    // Handle format correction steps (observation exists but no skillId)
-    else if (step.observation && !step.skillId) {
-      // Add as a system message for format correction
-      messages.push({
-        role: "system",
-        content: step.observation,
-      });
-    }
-  }
-
-  return messages;
-}
-
-async function makeNextCall(
-  executionState: ExecutionState,
-  TOOLS: ToolSet,
-  totalCost: TotalCost,
-  guardLoop: number,
-): Promise<LLMOutputInterface> {
-  const { context, history, previousHistory } = executionState;
-
-  const promptInfo = {
-    USER_MESSAGE: executionState.query,
-    CONTEXT: context,
-    USER_MEMORY: executionState.userMemoryContext,
-  };
-
-  let messages: CoreMessage[] = [];
-
-  const systemTemplateHandler = Handlebars.compile(REACT_SYSTEM_PROMPT);
-  let systemPrompt = systemTemplateHandler(promptInfo);
-
-  const userTemplateHandler = Handlebars.compile(REACT_USER_PROMPT);
-  const userPrompt = userTemplateHandler(promptInfo);
-
-  // Always start with a system message (this does use tokens but keeps the instructions clear)
-  messages.push({ role: "system", content: systemPrompt });
-
-  // For subsequent queries, include only final responses from previous exchanges if available
-  if (previousHistory && previousHistory.length > 0) {
-    messages = [...messages, ...previousHistory];
-  }
-
-  // Add the current user query (much simpler than the full prompt)
-  messages.push({ role: "user", content: userPrompt });
-
-  // Include any steps from the current interaction
-  if (history.length > 0) {
-    messages = toolToMessage(history, messages);
-  }
-
-  if (executionState.resources && executionState.resources.length > 0) {
-    messages = await addResources(messages, executionState.resources);
-  }
-
-  // Get the next action from the LLM
-  const response = generate(
-    messages,
-    guardLoop > 0 && guardLoop % 3 === 0,
-    (event) => {
-      const usage = event.usage;
-      totalCost.inputTokens += usage.promptTokens;
-      totalCost.outputTokens += usage.completionTokens;
-    },
-    TOOLS,
-  );
-
-  return { response };
-}
-
-export async function* run(
-  message: string,
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  context: Record<string, any>,
-  previousHistory: CoreMessage[],
-  mcp: MCP,
-  stepHistory: HistoryStep[],
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-): AsyncGenerator<AgentMessage, any, any> {
-  let guardLoop = 0;
-
-  let tools = {
-    ...(await mcp.allTools()),
-    "core--progress_update": progressUpdateTool,
-  };
-
-  logger.info("Tools have been formed");
-
-  let contextText = "";
-  let resources = [];
-  if (context) {
-    // Extract resources and remove from context
-    resources = context.resources || [];
-    delete context.resources;
-
-    // Process remaining context
-    contextText = flattenObject(context).join("\n");
-  }
-
-  const executionState: ExecutionState = {
-    query: message,
-    context: contextText,
-    resources,
-    previousHistory,
-    history: stepHistory, // Track the full ReAct history
-    completed: false,
-  };
-
-  const totalCost: TotalCost = { inputTokens: 0, outputTokens: 0, cost: 0 };
-
-  try {
-    while (!executionState.completed && guardLoop < 50) {
-      logger.info(`Starting the loop: ${guardLoop}`);
-
-      const { response: llmResponse } = await makeNextCall(
-        executionState,
-        tools,
-        totalCost,
-        guardLoop,
-      );
-
-      let toolCallInfo;
-
-      const messageState = {
-        inTag: false,
-        message: "",
-        messageEnded: false,
-        lastSent: "",
-      };
-
-      const questionState = {
-        inTag: false,
-        message: "",
-        messageEnded: false,
-        lastSent: "",
-      };
-
-      let totalMessage = "";
-      const toolCalls = [];
-
-      // LLM thought response
-      for await (const chunk of llmResponse) {
-        if (typeof chunk === "object" && chunk.type === "tool-call") {
-          toolCallInfo = chunk;
-          toolCalls.push(chunk);
-        }
-
-        totalMessage += chunk;
-
-        if (!messageState.messageEnded) {
-          yield* processTag(
-            messageState,
-            totalMessage,
-            chunk as string,
-            "<final_response>",
-            "</final_response>",
-            {
-              start: AgentMessageType.MESSAGE_START,
-              chunk: AgentMessageType.MESSAGE_CHUNK,
-              end: AgentMessageType.MESSAGE_END,
-            },
-          );
-        }
-
-        if (!questionState.messageEnded) {
-          yield* processTag(
-            questionState,
-            totalMessage,
-            chunk as string,
-            "<question_response>",
-            "</question_response>",
-            {
-              start: AgentMessageType.MESSAGE_START,
-              chunk: AgentMessageType.MESSAGE_CHUNK,
-              end: AgentMessageType.MESSAGE_END,
-            },
-          );
-        }
-      }
-
-      logger.info(`Cost for thought: ${JSON.stringify(totalCost)}`);
-
-      // Replace the error-handling block with this self-correcting implementation
-      if (
-        !totalMessage.includes("final_response") &&
-        !totalMessage.includes("question_response") &&
-        !toolCallInfo
-      ) {
-        // Log the issue for debugging
-        logger.info(
-          `Invalid response format detected. Attempting to get proper format.`,
-        );
-
-        // Extract the raw content from the invalid response
-        const rawContent = totalMessage
-          .replace(/(<[^>]*>|<\/[^>]*>)/g, "")
-          .trim();
-
-        // Create a correction step
-        const stepRecord: HistoryStep = {
-          thought: "",
-          skill: "",
-          skillId: "",
-          userMessage: "Core agent error, retrying \n",
-          isQuestion: false,
-          isFinal: false,
-          tokenCount: totalCost,
-          skillInput: "",
-          observation: `Your last response was not in a valid format. You must respond with EXACTLY ONE of the required formats: either a tool call, <question_response> tags, or <final_response> tags. Please reformat your previous response using the correct format:\n\n${rawContent}`,
-        };
-
-        yield Message("", AgentMessageType.MESSAGE_START);
-        yield Message(
-          stepRecord.userMessage as string,
-          AgentMessageType.MESSAGE_CHUNK,
-        );
-        yield Message("", AgentMessageType.MESSAGE_END);
-
-        // Add this step to the history
-        yield Message(JSON.stringify(stepRecord), AgentMessageType.STEP);
-        executionState.history.push(stepRecord);
-
-        // Log that we're continuing the loop with a correction request
-        logger.info(`Added format correction request to history.`);
-
-        // Don't mark as completed - let the loop continue
-        guardLoop++; // Still increment to prevent infinite loops
-        continue;
-      }
-
-      // Record this step in history
-      const stepRecord: HistoryStep = {
-        thought: "",
-        skill: "",
-        skillId: "",
-        userMessage: "",
-        isQuestion: false,
-        isFinal: false,
-        tokenCount: totalCost,
-        skillInput: "",
-      };
-
-      if (totalMessage && totalMessage.includes("final_response")) {
-        executionState.completed = true;
-        stepRecord.isFinal = true;
-        stepRecord.userMessage = messageState.message;
-        stepRecord.finalTokenCount = totalCost;
-        stepRecord.skillStatus = ActionStatusEnum.SUCCESS;
-        yield Message(JSON.stringify(stepRecord), AgentMessageType.STEP);
-        executionState.history.push(stepRecord);
-        break;
-      }
-
-      if (totalMessage && totalMessage.includes("question_response")) {
-        executionState.completed = true;
-        stepRecord.isQuestion = true;
-        stepRecord.userMessage = questionState.message;
-        stepRecord.finalTokenCount = totalCost;
-        stepRecord.skillStatus = ActionStatusEnum.QUESTION;
-        yield Message(JSON.stringify(stepRecord), AgentMessageType.STEP);
-        executionState.history.push(stepRecord);
-        break;
-      }
-
-      if (toolCalls && toolCalls.length > 0) {
-        // Run all tool calls in parallel
-        for (const toolCallInfo of toolCalls) {
-          const skillName = toolCallInfo.toolName;
-          const skillId = toolCallInfo.toolCallId;
-          const skillInput = toolCallInfo.args;
-
-          const toolName = skillName.split("--")[1];
-          const agent = skillName.split("--")[0];
-
-          const stepRecord: HistoryStep = {
-            agent,
-            thought: "",
-            skill: skillName,
-            skillId,
-            userMessage: "",
-            isQuestion: false,
-            isFinal: false,
-            tokenCount: totalCost,
-            skillInput: JSON.stringify(skillInput),
-          };
-
-          if (!internalTools.includes(skillName)) {
-            const skillMessageToSend = `\n<skill id="${skillId}" name="${toolName}" agent="${agent}"></skill>\n`;
-
-            stepRecord.userMessage += skillMessageToSend;
-
-            yield Message("", AgentMessageType.MESSAGE_START);
-            yield Message(skillMessageToSend, AgentMessageType.MESSAGE_CHUNK);
-            yield Message("", AgentMessageType.MESSAGE_END);
-          }
-
-          let result;
-          try {
-            // Log skill execution details
-            logger.info(`Executing skill: ${skillName}`);
-            logger.info(`Input parameters: ${JSON.stringify(skillInput)}`);
-
-            if (!internalTools.includes(toolName)) {
-              yield Message(
-                JSON.stringify({ skillId, status: "start" }),
-                AgentMessageType.SKILL_START,
-              );
-            }
-
-            // Handle CORE agent tools
-            if (agent === "core") {
-              if (toolName === "progress_update") {
-                yield Message("", AgentMessageType.MESSAGE_START);
-                yield Message(
-                  skillInput.message,
-                  AgentMessageType.MESSAGE_CHUNK,
-                );
-                stepRecord.userMessage += skillInput.message;
-                yield Message("", AgentMessageType.MESSAGE_END);
-                result = "Progress update sent successfully";
-              }
-            }
-            // Handle other MCP tools
-            else {
-              result = await mcp.callTool(skillName, skillInput);
-
-              yield Message(
-                JSON.stringify({ result, skillId }),
-                AgentMessageType.SKILL_CHUNK,
-              );
-            }
-
-            yield Message(
-              JSON.stringify({ skillId, status: "end" }),
-              AgentMessageType.SKILL_END,
-            );
-
-            stepRecord.skillOutput =
-              typeof result === "object"
-                ? JSON.stringify(result, null, 2)
-                : result;
-            stepRecord.observation = stepRecord.skillOutput;
-          } catch (e) {
-            console.log(e);
-            logger.error(e as string);
-            stepRecord.skillInput = skillInput;
-            stepRecord.observation = JSON.stringify(e);
-            stepRecord.isError = true;
-          }
-
-          logger.info(`Skill step: ${JSON.stringify(stepRecord)}`);
-
-          yield Message(JSON.stringify(stepRecord), AgentMessageType.STEP);
-          executionState.history.push(stepRecord);
-        }
-      }
-      guardLoop++;
-    }
-    yield Message("Stream ended", AgentMessageType.STREAM_END);
-  } catch (e) {
-    logger.error(e as string);
-    yield Message((e as Error).message, AgentMessageType.ERROR);
-    yield Message("Stream ended", AgentMessageType.STREAM_END);
-  }
-}

apps/webapp/app/trigger/chat/chat.ts

@@ -1,150 +0,0 @@
-import { ActionStatusEnum } from "@core/types";
-import { metadata, task, queue } from "@trigger.dev/sdk";
-
-import { run } from "./chat-utils";
-import { MCP } from "../utils/mcp";
-import { type HistoryStep } from "../utils/types";
-import {
-  createConversationHistoryForAgent,
-  deductCredits,
-  deletePersonalAccessToken,
-  getPreviousExecutionHistory,
-  hasCredits,
-  InsufficientCreditsError,
-  init,
-  type RunChatPayload,
-  updateConversationHistoryMessage,
-  updateConversationStatus,
-  updateExecutionStep,
-} from "../utils/utils";
-
-const chatQueue = queue({
-  name: "chat-queue",
-  concurrencyLimit: 50,
-});
-
-/**
- * Main chat task that orchestrates the agent workflow
- * Handles conversation context, agent selection, and LLM interactions
- */
-export const chat = task({
-  id: "chat",
-  maxDuration: 3000,
-  queue: chatQueue,
-  init,
-  run: async (payload: RunChatPayload, { init }) => {
-    await updateConversationStatus("running", payload.conversationId);
-
-    try {
-      // Check if workspace has sufficient credits before processing
-      if (init?.conversation.workspaceId) {
-        const hasSufficientCredits = await hasCredits(
-          init.conversation.workspaceId,
-          "chatMessage",
-        );
-
-        if (!hasSufficientCredits) {
-          throw new InsufficientCreditsError(
-            "Insufficient credits to process chat message. Please upgrade your plan or wait for your credits to reset.",
-          );
-        }
-      }
-
-      const { previousHistory, ...otherData } = payload.context;
-
-      // Initialise mcp
-      const mcpHeaders = { Authorization: `Bearer ${init?.token}` };
-      const mcp = new MCP();
-      await mcp.init();
-      await mcp.load(mcpHeaders);
-
-      // Prepare context with additional metadata
-      const context = {
-        // Currently this is assuming we only have one page in context
-        context: {
-          ...(otherData.page && otherData.page.length > 0
-            ? { page: otherData.page[0] }
-            : {}),
-        },
-        workpsaceId: init?.conversation.workspaceId,
-        resources: otherData.resources,
-        todayDate: new Date().toISOString(),
-      };
-
-      // Extract user's goal from conversation history
-      const message = init?.conversationHistory?.message;
-      // Retrieve execution history from previous interactions
-      const previousExecutionHistory = getPreviousExecutionHistory(
-        previousHistory ?? [],
-      );
-
-      let agentUserMessage = "";
-      let agentConversationHistory;
-      let stepHistory: HistoryStep[] = [];
-      // Prepare conversation history in agent-compatible format
-      agentConversationHistory = await createConversationHistoryForAgent(
-        payload.conversationId,
-      );
-
-      const llmResponse = run(
-        message as string,
-        context,
-        previousExecutionHistory,
-        mcp,
-        stepHistory,
-      );
-
-      const stream = await metadata.stream("messages", llmResponse);
-
-      let conversationStatus = "success";
-      for await (const step of stream) {
-        if (step.type === "STEP") {
-          const stepDetails = JSON.parse(step.message as string);
-
-          if (stepDetails.skillStatus === ActionStatusEnum.TOOL_REQUEST) {
-            conversationStatus = "need_approval";
-          }
-
-          if (stepDetails.skillStatus === ActionStatusEnum.QUESTION) {
-            conversationStatus = "need_attention";
-          }
-
-          await updateExecutionStep(
-            { ...stepDetails },
-            agentConversationHistory.id,
-          );
-
-          agentUserMessage += stepDetails.userMessage;
-
-          await updateConversationHistoryMessage(
-            agentUserMessage,
-            agentConversationHistory.id,
-          );
-        } else if (step.type === "STREAM_END") {
-          break;
-        }
-      }
-
-      await updateConversationStatus(
-        conversationStatus,
-        payload.conversationId,
-      );
-
-      // Deduct credits for chat message
-      if (init?.conversation.workspaceId) {
-        await deductCredits(init.conversation.workspaceId, "chatMessage");
-      }
-
-      if (init?.tokenId) {
-        await deletePersonalAccessToken(init.tokenId);
-      }
-    } catch (e) {
-      console.log(e);
-      await updateConversationStatus("failed", payload.conversationId);
-      if (init?.tokenId) {
-        await deletePersonalAccessToken(init.tokenId);
-      }
-      throw new Error(e as string);
-    }
-  },
-});

apps/webapp/app/trigger/chat/prompt.ts

@@ -1,159 +0,0 @@
-export const REACT_SYSTEM_PROMPT = `
-You are a helpful AI assistant with access to user memory. Your primary capabilities are:
-
-1. **Memory-First Approach**: Always check user memory first to understand context and previous interactions
-2. **Intelligent Information Gathering**: Analyze queries to determine if current information is needed
-3. **Memory Management**: Help users store, retrieve, and organize information in their memory
-4. **Contextual Assistance**: Use memory to provide personalized and contextual responses
-
-<context>
-{{CONTEXT}}
-</context>
-
-<information_gathering>
-Follow this intelligent approach for information gathering:
-
-1. **MEMORY FIRST** (Always Required)
-   - Always check memory FIRST using core--search_memory before any other actions
-   - Consider this your highest priority for EVERY interaction - as essential as breathing
-   - Memory provides context, personal preferences, and historical information
-   - Use memory to understand user's background, ongoing projects, and past conversations
-
-2. **INFORMATION SYNTHESIS** (Combine Sources)
-   - Use memory to personalize current information based on user preferences
-   - Always store new useful information in memory using core--add_memory
-
-3. **TRAINING KNOWLEDGE** (Foundation)
-   - Use your training knowledge as the foundation for analysis and explanation
-   - Apply training knowledge to interpret and contextualize information from memory
-   - Indicate when you're using training knowledge vs. live information sources
-
-EXECUTION APPROACH:
-- Memory search is mandatory for every interaction
-- Always indicate your information sources in responses
-</information_gathering>
-
-<memory>
-QUERY FORMATION:
-- Write specific factual statements as queries (e.g., "user email address" not "what is the user's email?")
-- Create multiple targeted memory queries for complex requests
-
-KEY QUERY AREAS:
-- Personal context: user name, location, identity, work context
-- Project context: repositories, codebases, current work, team members
-- Task context: recent tasks, ongoing projects, deadlines, priorities
-- Integration context: GitHub repos, Slack channels, Linear projects, connected services
-- Communication patterns: email preferences, notification settings, workflow automation
-- Technical context: coding languages, frameworks, development environment
-- Collaboration context: team members, project stakeholders, meeting patterns
-- Preferences: likes, dislikes, communication style, tool preferences
-- History: previous discussions, past requests, completed work, recurring issues
-- Automation rules: user-defined workflows, triggers, automation preferences
-
-MEMORY USAGE:
-- Execute multiple memory queries in parallel rather than sequentially
-- Batch related memory queries when possible
-- Prioritize recent information over older memories
-- Create comprehensive context-aware queries based on user message/activity content
-- Extract and query SEMANTIC CONTENT, not just structural metadata
-- Parse titles, descriptions, and content for actual subject matter keywords
-- Search internal SOL tasks/conversations that may relate to the same topics
-- Query ALL relatable concepts, not just direct keywords or IDs
-- Search for similar past situations, patterns, and related work
-- Include synonyms, related terms, and contextual concepts in queries  
-- Query user's historical approach to similar requests or activities
-- Search for connected projects, tasks, conversations, and collaborations
-- Retrieve workflow patterns and past decision-making context
-- Query broader domain context beyond immediate request scope
-- Remember: SOL tracks work that external tools don't - search internal content thoroughly
-- Blend memory insights naturally into responses
-- Verify you've checked relevant memory before finalizing ANY response
-
-</memory>
-
-<external_services>
-- To use: load_mcp with EXACT integration name from the available list
-- Can load multiple at once with an array
-- Only load when tools are NOT already available in your current toolset
-- If a tool is already available, use it directly without load_mcp
-- If requested integration unavailable: inform user politely
-</external_services>
-
-<tool_calling>
-You have tools at your disposal to assist users:
-
-CORE PRINCIPLES:
-- Use tools only when necessary for the task at hand
-- Always check memory FIRST before making other tool calls
-- Execute multiple operations in parallel whenever possible
-- Use sequential calls only when output of one is required for input of another
-
-PARAMETER HANDLING:
-- Follow tool schemas exactly with all required parameters
-- Only use values that are:
-  • Explicitly provided by the user (use EXACTLY as given)
-  • Reasonably inferred from context
-  • Retrieved from memory or prior tool calls
-- Never make up values for required parameters
-- Omit optional parameters unless clearly needed
-- Analyze user's descriptive terms for parameter clues
-
-TOOL SELECTION:
-- Never call tools not provided in this conversation
-- Skip tool calls for general questions you can answer directly from memory/knowledge
-- For identical operations on multiple items, use parallel tool calls
-- Default to parallel execution (3-5× faster than sequential calls)
-- You can always access external service tools by loading them with load_mcp first
-
-TOOL MENTION HANDLING:
-When user message contains <mention data-id="tool_name" data-label="tool"></mention>:
-- Extract tool_name from data-id attribute
-- First check if it's a built-in tool; if not, check EXTERNAL SERVICES TOOLS
-- If available: Load it with load_mcp and focus on addressing the request with this tool
-- If unavailable: Inform user and suggest alternatives if possible
-- For multiple tool mentions: Load all applicable tools in a single load_mcp call
-
-ERROR HANDLING:
-- If a tool returns an error, try fixing parameters before retrying
-- If you can't resolve an error, explain the issue to the user
-- Consider alternative tools when primary tools are unavailable
-</tool_calling>
-
-<communication>
-Use EXACTLY ONE of these formats for all user-facing communication:
-
-PROGRESS UPDATES - During processing:
-- Use the core--progress_update tool to keep users informed
-- Update users about what you're discovering or doing next
-- Keep messages clear and user-friendly
-- Avoid technical jargon
-
-QUESTIONS - When you need information:
-<question_response>
-<p>[Your question with HTML formatting]</p>
-</question_response>
-
-- Ask questions only when you cannot find information through memory, or tools
-- Be specific about what you need to know
-- Provide context for why you're asking
-
-FINAL ANSWERS - When completing tasks:
-<final_response>
-<p>[Your answer with HTML formatting]</p>
-</final_response>
-
-CRITICAL:
-- Use ONE format per turn
-- Apply proper HTML formatting (<h1>, <h2>, <p>, <ul>, <li>, etc.)
-- Never mix communication formats
-- Keep responses clear and helpful
-- Always indicate your information sources (memory, and/or knowledge)
-</communication>
-`;
-
-export const REACT_USER_PROMPT = `
-Here is the user message:
-<user_message>
-{{USER_MESSAGE}}
-</user_message>
-`;

apps/webapp/app/trigger/chat/stream-utils.ts

@@ -1,294 +0,0 @@
-import fs from "fs";
-import path from "node:path";
-
-import { anthropic } from "@ai-sdk/anthropic";
-import { google } from "@ai-sdk/google";
-import { openai } from "@ai-sdk/openai";
-import { logger } from "@trigger.dev/sdk/v3";
-import {
-  type CoreMessage,
-  type LanguageModelV1,
-  streamText,
-  type ToolSet,
-} from "ai";
-import { createOllama } from "ollama-ai-provider";
-
-import { type AgentMessageType, Message } from "./types";
-
-interface State {
-  inTag: boolean;
-  messageEnded: boolean;
-  message: string;
-  lastSent: string;
-}
-
-export interface ExecutionState {
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  agentFlow: any;
-  userMessage: string;
-  message: string;
-}
-
-export async function* processTag(
-  state: State,
-  totalMessage: string,
-  chunk: string,
-  startTag: string,
-  endTag: string,
-  states: { start: string; chunk: string; end: string },
-  extraParams: Record<string, string> = {},
-) {
-  let comingFromStart = false;
-
-  if (!state.messageEnded) {
-    if (!state.inTag) {
-      const startIndex = totalMessage.indexOf(startTag);
-      if (startIndex !== -1) {
-        state.inTag = true;
-        // Send MESSAGE_START when we first enter the tag
-        yield Message("", states.start as AgentMessageType, extraParams);
-        const chunkToSend = totalMessage.slice(startIndex + startTag.length);
-        state.message += chunkToSend;
-        comingFromStart = true;
-      }
-    }
-
-    if (state.inTag) {
-      // Check if chunk contains end tag
-      const hasEndTag = chunk.includes(endTag);
-      const hasStartTag = chunk.includes(startTag);
-      const hasClosingTag = chunk.includes("</");
-
-      // Check if we're currently accumulating a potential end tag
-      const accumulatingEndTag = state.message.endsWith("</") ||
-                                  state.message.match(/<\/[a-z_]*$/i);
-
-      if (hasClosingTag && !hasStartTag && !hasEndTag) {
-        // If chunk only has </ but not the full end tag, accumulate it
-        state.message += chunk;
-      } else if (accumulatingEndTag) {
-        // Continue accumulating if we're in the middle of a potential end tag
-        state.message += chunk;
-        // Check if we now have the complete end tag
-        if (state.message.includes(endTag)) {
-          // Process the complete message with end tag
-          const endIndex = state.message.indexOf(endTag);
-          const finalMessage = state.message.slice(0, endIndex).trim();
-          const messageToSend = finalMessage.slice(
-            finalMessage.indexOf(state.lastSent) + state.lastSent.length,
-          );
-
-          if (messageToSend) {
-            yield Message(
-              messageToSend,
-              states.chunk as AgentMessageType,
-              extraParams,
-            );
-          }
-          yield Message("", states.end as AgentMessageType, extraParams);
-
-          state.message = finalMessage;
-          state.messageEnded = true;
-        }
-      } else if (hasEndTag || (!hasEndTag && !hasClosingTag)) {
-        let currentMessage = comingFromStart
-          ? state.message
-          : state.message + chunk;
-
-        const endIndex = currentMessage.indexOf(endTag);
-
-        if (endIndex !== -1) {
-          // For the final chunk before the end tag
-          currentMessage = currentMessage.slice(0, endIndex).trim();
-          const messageToSend = currentMessage.slice(
-            currentMessage.indexOf(state.lastSent) + state.lastSent.length,
-          );
-
-          if (messageToSend) {
-            yield Message(
-              messageToSend,
-              states.chunk as AgentMessageType,
-              extraParams,
-            );
-          }
-          // Send MESSAGE_END when we reach the end tag
-          yield Message("", states.end as AgentMessageType, extraParams);
-
-          state.message = currentMessage;
-          state.messageEnded = true;
-        } else {
-          const diff = currentMessage.slice(
-            currentMessage.indexOf(state.lastSent) + state.lastSent.length,
-          );
-
-          // For chunks in between start and end
-          const messageToSend = comingFromStart ? state.message : diff;
-
-          if (messageToSend) {
-            state.lastSent = messageToSend;
-            yield Message(
-              messageToSend,
-              states.chunk as AgentMessageType,
-              extraParams,
-            );
-          }
-        }
-
-        state.message = currentMessage;
-        state.lastSent = state.message;
-      } else {
-        state.message += chunk;
-      }
-    }
-  }
-}
-
-export async function* generate(
-  messages: CoreMessage[],
-  isProgressUpdate: boolean = false,
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  onFinish?: (event: any) => void,
-  tools?: ToolSet,
-  system?: string,
-  model?: string,
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-): AsyncGenerator<
-  | string
-  | {
-      type: string;
-      toolName: string;
-      // eslint-disable-next-line @typescript-eslint/no-explicit-any
-      args?: any;
-      toolCallId?: string;
-      message?: string;
-    }
-> {
-  // Check for API keys
-  const anthropicKey = process.env.ANTHROPIC_API_KEY;
-  const googleKey = process.env.GOOGLE_GENERATIVE_AI_API_KEY;
-  const openaiKey = process.env.OPENAI_API_KEY;
-  let ollamaUrl = process.env.OLLAMA_URL;
-  model = model || process.env.MODEL;
-
-  let modelInstance;
-  let modelTemperature = Number(process.env.MODEL_TEMPERATURE) || 1;
-  ollamaUrl = undefined;
-
-  // First check if Ollama URL exists and use Ollama
-  if (ollamaUrl) {
-    const ollama = createOllama({
-      baseURL: ollamaUrl,
-    });
-    modelInstance = ollama(model || "llama2"); // Default to llama2 if no model specified
-  } else {
-    // If no Ollama, check other models
-    switch (model) {
-      case "claude-3-7-sonnet-20250219":
-      case "claude-3-opus-20240229":
-      case "claude-3-5-haiku-20241022":
-        if (!anthropicKey) {
-          throw new Error("No Anthropic API key found. Set ANTHROPIC_API_KEY");
-        }
-        modelInstance = anthropic(model);
-        modelTemperature = 0.5;
-        break;
-
-      case "gemini-2.5-flash-preview-04-17":
-      case "gemini-2.5-pro-preview-03-25":
-      case "gemini-2.0-flash":
-      case "gemini-2.0-flash-lite":
-        if (!googleKey) {
-          throw new Error("No Google API key found. Set GOOGLE_API_KEY");
-        }
-        modelInstance = google(model);
-        break;
-
-      case "gpt-4.1-2025-04-14":
-      case "gpt-4.1-mini-2025-04-14":
-      case "gpt-5-mini-2025-08-07":
-      case "gpt-5-2025-08-07":
-      case "gpt-4.1-nano-2025-04-14":
-        if (!openaiKey) {
-          throw new Error("No OpenAI API key found. Set OPENAI_API_KEY");
-        }
-        modelInstance = openai(model);
-        break;
-
-      default:
-        break;
-    }
-  }
-
-  logger.info("starting stream");
-  // Try Anthropic next if key exists
-  if (modelInstance) {
-    try {
-      const { textStream, fullStream } = streamText({
-        model: modelInstance as LanguageModelV1,
-        messages,
-        temperature: modelTemperature,
-        maxSteps: 10,
-        tools,
-        ...(isProgressUpdate
-          ? { toolChoice: { type: "tool", toolName: "core--progress_update" } }
-          : {}),
-        toolCallStreaming: true,
-        onFinish,
-        ...(system ? { system } : {}),
-      });
-
-      for await (const chunk of textStream) {
-        yield chunk;
-      }
-
-      for await (const fullChunk of fullStream) {
-        if (fullChunk.type === "tool-call") {
-          yield {
-            type: "tool-call",
-            toolName: fullChunk.toolName,
-            toolCallId: fullChunk.toolCallId,
-            args: fullChunk.args,
-          };
-        }
-
-        if (fullChunk.type === "error") {
-          // Log the error to a file
-          const errorLogsDir = path.join(__dirname, "../../../../logs/errors");
-
-          // Ensure the directory exists
-          try {
-            if (!fs.existsSync(errorLogsDir)) {
-              fs.mkdirSync(errorLogsDir, { recursive: true });
-            }
-
-            // Create a timestamped error log file
-            const timestamp = new Date().toISOString().replace(/:/g, "-");
-            const errorLogPath = path.join(
-              errorLogsDir,
-              `llm-error-${timestamp}.json`,
-            );
-
-            // Write the error to the file
-            fs.writeFileSync(
-              errorLogPath,
-              JSON.stringify({
-                timestamp: new Date().toISOString(),
-                error: fullChunk.error,
-              }),
-            );
-
-            logger.error(`LLM error logged to ${errorLogPath}`);
-          } catch (err) {
-            logger.error(`Failed to log LLM error: ${err}`);
-          }
-        }
-      }
-      return;
-    } catch (e) {
-      console.log(e);
-      logger.error(e as string);
-    }
-  }
-
-  throw new Error("No valid LLM configuration found");
-}

apps/webapp/app/trigger/chat/types.ts

@@ -1,46 +0,0 @@
-export interface AgentStep {
-  agent: string;
-  goal: string;
-  reasoning: string;
-}
-
-export enum AgentMessageType {
-  STREAM_START = 'STREAM_START',
-  STREAM_END = 'STREAM_END',
-
-  // Used in ReACT based prompting
-  THOUGHT_START = 'THOUGHT_START',
-  THOUGHT_CHUNK = 'THOUGHT_CHUNK',
-  THOUGHT_END = 'THOUGHT_END',
-
-  // Message types
-  MESSAGE_START = 'MESSAGE_START',
-  MESSAGE_CHUNK = 'MESSAGE_CHUNK',
-  MESSAGE_END = 'MESSAGE_END',
-
-  // This is used to return action input
-  SKILL_START = 'SKILL_START',
-  SKILL_CHUNK = 'SKILL_CHUNK',
-  SKILL_END = 'SKILL_END',
-
-  STEP = 'STEP',
-  ERROR = 'ERROR',
-}
-
-export interface AgentMessage {
-  message?: string;
-  type: AgentMessageType;
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  metadata: Record<string, any>;
-}
-
-export const Message = (
-  message: string,
-  type: AgentMessageType,
-  extraParams: Record<string, string> = {},
-): AgentMessage => {
-  // For all message types, we use the message field
-  // The type field differentiates how the message should be interpreted
-  // For STEP and SKILL types, the message can contain JSON data as a string
-  return { message, type, metadata: extraParams };
-};

apps/webapp/app/trigger/cluster/index.ts

@@ -1,115 +0,0 @@
-import { queue, task } from "@trigger.dev/sdk";
-import { z } from "zod";
-import { ClusteringService } from "~/services/clustering.server";
-import { logger } from "~/services/logger.service";
-
-const clusteringService = new ClusteringService();
-
-// Define the payload schema for cluster tasks
-export const ClusterPayload = z.object({
-  userId: z.string(),
-  mode: z.enum(["auto", "incremental", "complete", "drift"]).default("auto"),
-  forceComplete: z.boolean().default(false),
-});
-
-const clusterQueue = queue({
-  name: "cluster-queue",
-  concurrencyLimit: 10,
-});
-
-/**
- * Single clustering task that handles all clustering operations based on payload mode
- */
-export const clusterTask = task({
-  id: "cluster",
-  queue: clusterQueue,
-  maxDuration: 1800, // 30 minutes max
-  run: async (payload: z.infer<typeof ClusterPayload>) => {
-    logger.info(`Starting ${payload.mode} clustering task for user ${payload.userId}`);
-
-    try {
-      let result;
-      
-      switch (payload.mode) {
-        case "incremental":
-          result = await clusteringService.performIncrementalClustering(
-            payload.userId,
-          );
-          logger.info(`Incremental clustering completed for user ${payload.userId}:`, {
-            newStatementsProcessed: result.newStatementsProcessed,
-            newClustersCreated: result.newClustersCreated,
-          });
-          break;
-
-        case "complete":
-          result = await clusteringService.performCompleteClustering(
-            payload.userId,
-          );
-          logger.info(`Complete clustering completed for user ${payload.userId}:`, {
-            clustersCreated: result.clustersCreated,
-            statementsProcessed: result.statementsProcessed,
-          });
-          break;
-
-        case "drift":
-          // First detect drift
-          const driftMetrics = await clusteringService.detectClusterDrift(
-            payload.userId,
-          );
-
-          if (driftMetrics.driftDetected) {
-            // Handle drift by splitting low-cohesion clusters
-            const driftResult = await clusteringService.handleClusterDrift(
-              payload.userId,
-            );
-
-            logger.info(`Cluster drift handling completed for user ${payload.userId}:`, {
-              driftDetected: true,
-              clustersProcessed: driftResult.clustersProcessed,
-              newClustersCreated: driftResult.newClustersCreated,
-              splitClusters: driftResult.splitClusters,
-            });
-
-            result = {
-              driftDetected: true,
-              ...driftResult,
-              driftMetrics,
-            };
-          } else {
-            logger.info(`No cluster drift detected for user ${payload.userId}`);
-            result = {
-              driftDetected: false,
-              clustersProcessed: 0,
-              newClustersCreated: 0,
-              splitClusters: [],
-              driftMetrics,
-            };
-          }
-          break;
-
-        case "auto":
-        default:
-          result = await clusteringService.performClustering(
-            payload.userId,
-            payload.forceComplete,
-          );
-          logger.info(`Auto clustering completed for user ${payload.userId}:`, {
-            clustersCreated: result.clustersCreated,
-            statementsProcessed: result.statementsProcessed,
-            approach: result.approach,
-          });
-          break;
-      }
-
-      return {
-        success: true,
-        data: result,
-      };
-    } catch (error) {
-      logger.error(`${payload.mode} clustering failed for user ${payload.userId}:`, {
-        error,
-      });
-      throw error;
-    }
-  },
-});

apps/webapp/app/trigger/conversation/create-conversation-title.ts

@@ -1,61 +1,12 @@
-import { LLMMappings } from "@core/types";
-import { logger, task } from "@trigger.dev/sdk/v3";
-import { generate } from "../chat/stream-utils";
-import { conversationTitlePrompt } from "./prompt";
-import { prisma } from "../utils/prisma";
+import { task } from "@trigger.dev/sdk/v3";
+import {
+  processConversationTitleCreation,
+  type CreateConversationTitlePayload,
+} from "~/jobs/conversation/create-title.logic";
 
 export const createConversationTitle = task({
   id: "create-conversation-title",
-  run: async (payload: { conversationId: string; message: string }) => {
-    let conversationTitleResponse = "";
-    const gen = generate(
-      [
-        {
-          role: "user",
-          content: conversationTitlePrompt.replace(
-            "{{message}}",
-            payload.message,
-          ),
-        },
-      ],
-      false,
-      () => {},
-      undefined,
-      "",
-      LLMMappings.GPT41,
-    );
-
-    for await (const chunk of gen) {
-      if (typeof chunk === "string") {
-        conversationTitleResponse += chunk;
-      } else if (chunk && typeof chunk === "object" && chunk.message) {
-        conversationTitleResponse += chunk.message;
-      }
-    }
-
-    const outputMatch = conversationTitleResponse.match(
-      /<output>(.*?)<\/output>/s,
-    );
-
-    logger.info(`Conversation title data: ${JSON.stringify(outputMatch)}`);
-
-    if (!outputMatch) {
-      logger.error("No output found in recurrence response");
-      throw new Error("Invalid response format from AI");
-    }
-
-    const jsonStr = outputMatch[1].trim();
-    const conversationTitleData = JSON.parse(jsonStr);
-
-    if (conversationTitleData) {
-      await prisma.conversation.update({
-        where: {
-          id: payload.conversationId,
-        },
-        data: {
-          title: conversationTitleData.title,
-        },
-      });
-    }
+  run: async (payload: CreateConversationTitlePayload) => {
+    return await processConversationTitleCreation(payload);
   },
 });

apps/webapp/app/trigger/deep-search/deep-search-utils.ts

@@ -1,292 +0,0 @@
-import { type CoreMessage } from "ai";
-import { logger } from "@trigger.dev/sdk/v3";
-import { generate } from "./stream-utils";
-import { processTag } from "../chat/stream-utils";
-import { type AgentMessage, AgentMessageType, Message } from "../chat/types";
-import { type TotalCost } from "../utils/types";
-
-/**
- * Run the deep search ReAct loop
- * Async generator that yields AgentMessage objects for streaming
- * Follows the exact same pattern as chat-utils.ts
- */
-export async function* run(
-  initialMessages: CoreMessage[],
-  searchTool: any,
-): AsyncGenerator<AgentMessage, any, any> {
-  let messages = [...initialMessages];
-  let completed = false;
-  let guardLoop = 0;
-  let searchCount = 0;
-  let totalEpisodesFound = 0;
-  const seenEpisodeIds = new Set<string>(); // Track unique episodes
-  const totalCost: TotalCost = {
-    inputTokens: 0,
-    outputTokens: 0,
-    cost: 0,
-  };
-
-  const tools = {
-    searchMemory: searchTool,
-  };
-
-  logger.info("Starting deep search ReAct loop");
-
-  try {
-    while (!completed && guardLoop < 50) {
-      logger.info(
-        `ReAct loop iteration ${guardLoop}, searches: ${searchCount}`,
-      );
-
-      // Call LLM with current message history
-      const response = generate(
-        messages,
-        (event) => {
-          const usage = event.usage;
-          totalCost.inputTokens += usage.promptTokens;
-          totalCost.outputTokens += usage.completionTokens;
-        },
-        tools,
-      );
-
-      let totalMessage = "";
-      const toolCalls: any[] = [];
-
-      // States for streaming final_response tags
-      const messageState = {
-        inTag: false,
-        message: "",
-        messageEnded: false,
-        lastSent: "",
-      };
-
-      // Process streaming response
-      for await (const chunk of response) {
-        if (typeof chunk === "object" && chunk.type === "tool-call") {
-          // Agent made a tool call
-          toolCalls.push(chunk);
-          logger.info(`Tool call: ${chunk.toolName}`);
-        } else if (typeof chunk === "string") {
-          totalMessage += chunk;
-
-          // Stream final_response tags using processTag
-          if (!messageState.messageEnded) {
-            yield* processTag(
-              messageState,
-              totalMessage,
-              chunk,
-              "<final_response>",
-              "</final_response>",
-              {
-                start: AgentMessageType.MESSAGE_START,
-                chunk: AgentMessageType.MESSAGE_CHUNK,
-                end: AgentMessageType.MESSAGE_END,
-              },
-            );
-          }
-        }
-      }
-
-      // Check for final response
-      if (totalMessage.includes("<final_response>")) {
-        const match = totalMessage.match(
-          /<final_response>(.*?)<\/final_response>/s,
-        );
-
-        if (match) {
-          // Accept synthesis - completed
-          completed = true;
-          logger.info(
-            `Final synthesis accepted after ${searchCount} searches, ${totalEpisodesFound} unique episodes found`,
-          );
-          break;
-        }
-      }
-
-      // Execute tool calls in parallel for better performance
-      if (toolCalls.length > 0) {
-        // Notify about all searches starting
-        for (const toolCall of toolCalls) {
-          logger.info(`Executing search: ${JSON.stringify(toolCall.args)}`);
-          yield Message("", AgentMessageType.SKILL_START);
-          yield Message(
-            `\nSearching memory: "${toolCall.args.query}"...\n`,
-            AgentMessageType.SKILL_CHUNK,
-          );
-          yield Message("", AgentMessageType.SKILL_END);
-        }
-
-        // Execute all searches in parallel
-        const searchPromises = toolCalls.map((toolCall) =>
-          searchTool.execute(toolCall.args).then((result: any) => ({
-            toolCall,
-            result,
-          })),
-        );
-
-        const searchResults = await Promise.all(searchPromises);
-
-        // Process results and add to message history
-        for (const { toolCall, result } of searchResults) {
-          searchCount++;
-
-          // Deduplicate episodes - track unique IDs
-          let uniqueNewEpisodes = 0;
-          if (result.episodes && Array.isArray(result.episodes)) {
-            for (const episode of result.episodes) {
-              const episodeId =
-                episode.id || episode._id || JSON.stringify(episode);
-              if (!seenEpisodeIds.has(episodeId)) {
-                seenEpisodeIds.add(episodeId);
-                uniqueNewEpisodes++;
-              }
-            }
-          }
-
-          const episodesInThisSearch = result.episodes?.length || 0;
-          totalEpisodesFound = seenEpisodeIds.size; // Use unique count
-
-          messages.push({
-            role: "assistant",
-            content: [
-              {
-                type: "tool-call",
-                toolCallId: toolCall.toolCallId,
-                toolName: toolCall.toolName,
-                args: toolCall.args,
-              },
-            ],
-          });
-
-          // Add tool result to message history
-          messages.push({
-            role: "tool",
-            content: [
-              {
-                type: "tool-result",
-                toolName: toolCall.toolName,
-                toolCallId: toolCall.toolCallId,
-                result: result,
-              },
-            ],
-          });
-
-          logger.info(
-            `Search ${searchCount} completed: ${episodesInThisSearch} episodes (${uniqueNewEpisodes} new, ${totalEpisodesFound} unique total)`,
-          );
-        }
-
-        // If found no episodes and haven't exhausted search attempts, require more searches
-        if (totalEpisodesFound === 0 && searchCount < 7) {
-          logger.info(
-            `Agent attempted synthesis with 0 unique episodes after ${searchCount} searches - requiring more attempts`,
-          );
-
-          yield Message("", AgentMessageType.SKILL_START);
-          yield Message(
-            `No relevant context found yet - trying different search angles...`,
-            AgentMessageType.SKILL_CHUNK,
-          );
-          yield Message("", AgentMessageType.SKILL_END);
-
-          messages.push({
-            role: "system",
-            content: `You have performed ${searchCount} searches but found 0 unique relevant episodes. Your queries may be too abstract or not matching the user's actual conversation topics.
-
-Review your DECOMPOSITION:
-- Are you using specific terms from the content?
-- Try searching broader related topics the user might have discussed
-- Try different terminology or related concepts
-- Search for user's projects, work areas, or interests
-
-Continue with different search strategies (you can search up to 7-10 times total).`,
-          });
-
-          guardLoop++;
-          continue;
-        }
-
-        // Soft nudging after all searches executed (awareness, not commands)
-        if (totalEpisodesFound >= 30 && searchCount >= 3) {
-          logger.info(
-            `Nudging: ${totalEpisodesFound} unique episodes found - suggesting synthesis consideration`,
-          );
-
-          messages.push({
-            role: "system",
-            content: `Context awareness: You have found ${totalEpisodesFound} unique episodes across ${searchCount} searches. This represents substantial context. Consider whether you have sufficient information for quality synthesis, or if additional search angles would meaningfully improve understanding.`,
-          });
-        } else if (totalEpisodesFound >= 15 && searchCount >= 5) {
-          logger.info(
-            `Nudging: ${totalEpisodesFound} unique episodes after ${searchCount} searches - suggesting evaluation`,
-          );
-
-          messages.push({
-            role: "system",
-            content: `Progress update: You have ${totalEpisodesFound} unique episodes from ${searchCount} searches. Evaluate whether you have covered the main angles from your decomposition, or if important aspects remain unexplored.`,
-          });
-        } else if (searchCount >= 7) {
-          logger.info(
-            `Nudging: ${searchCount} searches completed with ${totalEpisodesFound} unique episodes`,
-          );
-
-          messages.push({
-            role: "system",
-            content: `Search depth: You have performed ${searchCount} searches and found ${totalEpisodesFound} unique episodes. Consider whether additional searches would yield meaningfully different context, or if it's time to synthesize what you've discovered.`,
-          });
-        }
-        if (searchCount >= 10) {
-          logger.info(
-            `Reached maximum search limit (10), forcing synthesis with ${totalEpisodesFound} unique episodes`,
-          );
-
-          yield Message("", AgentMessageType.SKILL_START);
-          yield Message(
-            `Maximum searches reached - synthesizing results...`,
-            AgentMessageType.SKILL_CHUNK,
-          );
-          yield Message("", AgentMessageType.SKILL_END);
-
-          messages.push({
-            role: "system",
-            content: `You have performed 10 searches and found ${totalEpisodesFound} unique episodes. This is the maximum allowed. You MUST now provide your final synthesis wrapped in <final_response> tags based on what you've found.`,
-          });
-        }
-      }
-
-      // Safety check - if no tool calls and no final response, something went wrong
-      if (
-        toolCalls.length === 0 &&
-        !totalMessage.includes("<final_response>")
-      ) {
-        logger.warn("Agent produced neither tool calls nor final response");
-
-        messages.push({
-          role: "system",
-          content:
-            "You must either use the searchMemory tool to search for more context, or provide your final synthesis wrapped in <final_response> tags.",
-        });
-      }
-
-      guardLoop++;
-    }
-
-    if (!completed) {
-      logger.warn(
-        `Loop ended without completion after ${guardLoop} iterations`,
-      );
-      yield Message("", AgentMessageType.MESSAGE_START);
-      yield Message(
-        "Deep search did not complete - maximum iterations reached.",
-        AgentMessageType.MESSAGE_CHUNK,
-      );
-      yield Message("", AgentMessageType.MESSAGE_END);
-    }
-
-    yield Message("Stream ended", AgentMessageType.STREAM_END);
-  } catch (error) {
-    logger.error(`Deep search error: ${error}`);
-    yield Message((error as Error).message, AgentMessageType.ERROR);
-    yield Message("Stream ended", AgentMessageType.STREAM_END);
-  }
-}

apps/webapp/app/trigger/deep-search/index.ts

@@ -1,85 +0,0 @@
-import { metadata, task } from "@trigger.dev/sdk";
-import { type CoreMessage } from "ai";
-import { logger } from "@trigger.dev/sdk/v3";
-import { nanoid } from "nanoid";
-import {
-  deletePersonalAccessToken,
-  getOrCreatePersonalAccessToken,
-} from "../utils/utils";
-import { getReActPrompt } from "./prompt";
-import { type DeepSearchPayload, type DeepSearchResponse } from "./types";
-import { createSearchMemoryTool } from "./utils";
-import { run } from "./deep-search-utils";
-import { AgentMessageType } from "../chat/types";
-
-export const deepSearch = task({
-  id: "deep-search",
-  maxDuration: 3000,
-  run: async (payload: DeepSearchPayload): Promise<DeepSearchResponse> => {
-    const { content, userId, stream, metadata: meta, intentOverride } = payload;
-
-    const randomKeyName = `deepSearch_${nanoid(10)}`;
-
-    // Get or create token for search API calls
-    const pat = await getOrCreatePersonalAccessToken({
-      name: randomKeyName,
-      userId: userId as string,
-    });
-
-    if (!pat?.token) {
-      throw new Error("Failed to create personal access token");
-    }
-
-    try {
-      // Create search tool that agent will use
-      const searchTool = createSearchMemoryTool(pat.token);
-
-      // Build initial messages with ReAct prompt
-      const initialMessages: CoreMessage[] = [
-        {
-          role: "system",
-          content: getReActPrompt(meta, intentOverride),
-        },
-        {
-          role: "user",
-          content: `CONTENT TO ANALYZE:\n${content}\n\nPlease search my memory for relevant context and synthesize what you find.`,
-        },
-      ];
-
-      // Run the ReAct loop generator
-      const llmResponse = run(initialMessages, searchTool);
-
-      // Streaming mode: stream via metadata.stream like chat.ts does
-      // This makes all message types available to clients in real-time
-      const messageStream = await metadata.stream("messages", llmResponse);
-
-      let synthesis = "";
-
-      for await (const step of messageStream) {
-        // MESSAGE_CHUNK: Final synthesis - accumulate and stream
-        if (step.type === AgentMessageType.MESSAGE_CHUNK) {
-          synthesis += step.message;
-        }
-
-        // STREAM_END: Loop completed
-        if (step.type === AgentMessageType.STREAM_END) {
-          break;
-        }
-      }
-
-      await deletePersonalAccessToken(pat?.id);
-
-      // Clean up any remaining tags
-      synthesis = synthesis
-        .replace(/<final_response>/gi, "")
-        .replace(/<\/final_response>/gi, "")
-        .trim();
-
-      return { synthesis };
-    } catch (error) {
-      await deletePersonalAccessToken(pat?.id);
-      logger.error(`Deep search error: ${error}`);
-      throw error;
-    }
-  },
-});

apps/webapp/app/trigger/deep-search/prompt.ts

@@ -1,148 +0,0 @@
-export function getReActPrompt(
-  metadata?: { source?: string; url?: string; pageTitle?: string },
-  intentOverride?: string
-): string {
-  const contextHints = [];
-
-  if (metadata?.source === "chrome" && metadata?.url?.includes("mail.google.com")) {
-    contextHints.push("Content is from email - likely reading intent");
-  }
-  if (metadata?.source === "chrome" && metadata?.url?.includes("calendar.google.com")) {
-    contextHints.push("Content is from calendar - likely meeting prep intent");
-  }
-  if (metadata?.source === "chrome" && metadata?.url?.includes("docs.google.com")) {
-    contextHints.push("Content is from document editor - likely writing intent");
-  }
-  if (metadata?.source === "obsidian") {
-    contextHints.push("Content is from note editor - likely writing or research intent");
-  }
-
-  return `You are a memory research agent analyzing content to find relevant context.
-
-YOUR PROCESS (ReAct Framework):
-
-1. DECOMPOSE: First, break down the content into structured categories
-
-   Analyze the content and extract:
-   a) ENTITIES: Specific people, project names, tools, products mentioned
-      Example: "John Smith", "Phoenix API", "Redis", "mobile app"
-
-   b) TOPICS & CONCEPTS: Key subjects, themes, domains
-      Example: "authentication", "database design", "performance optimization"
-
-   c) TEMPORAL MARKERS: Time references, deadlines, events
-      Example: "last week's meeting", "Q2 launch", "yesterday's discussion"
-
-   d) ACTIONS & TASKS: What's being done, decided, or requested
-      Example: "implement feature", "review code", "make decision on"
-
-   e) USER INTENT: What is the user trying to accomplish?
-      ${intentOverride ? `User specified: "${intentOverride}"` : "Infer from context: reading/writing/meeting prep/research/task tracking/review"}
-
-2. FORM QUERIES: Create targeted search queries from your decomposition
-
-   Based on decomposition, form specific queries:
-   - Search for each entity by name (people, projects, tools)
-   - Search for topics the user has discussed before
-   - Search for related work or conversations in this domain
-   - Use the user's actual terminology, not generic concepts
-
-   EXAMPLE - Content: "Email from Sarah about the API redesign we discussed last week"
-   Decomposition:
-     - Entities: "Sarah", "API redesign"
-     - Topics: "API design", "redesign"
-     - Temporal: "last week"
-     - Actions: "discussed", "email communication"
-     - Intent: Reading (email) / meeting prep
-
-   Queries to form:
-   ✅ "Sarah" (find past conversations with Sarah)
-   ✅ "API redesign" or "API design" (find project discussions)
-   ✅ "last week" + "Sarah" (find recent context)
-   ✅ "meetings" or "discussions" (find related conversations)
-
-   ❌ Avoid: "email communication patterns", "API architecture philosophy"
-   (These are abstract - search what user actually discussed!)
-
-3. SEARCH: Execute your queries using searchMemory tool
-   - Start with 2-3 core searches based on main entities/topics
-   - Make each search specific and targeted
-   - Use actual terms from the content, not rephrased concepts
-
-4. OBSERVE: Evaluate search results
-   - Did you find relevant episodes? How many unique ones?
-   - What specific context emerged?
-   - What new entities/topics appeared in results?
-   - Are there gaps in understanding?
-   - Should you search more angles?
-
-   Note: Episode counts are automatically deduplicated across searches - overlapping episodes are only counted once.
-
-5. REACT: Decide next action based on observations
-
-   STOPPING CRITERIA - Proceed to SYNTHESIZE if ANY of these are true:
-   - You found 20+ unique episodes across your searches → ENOUGH CONTEXT
-   - You performed 5+ searches and found relevant episodes → SUFFICIENT
-   - You performed 7+ searches regardless of results → EXHAUSTED STRATEGIES
-   - You found strong relevant context from multiple angles → COMPLETE
-
-   System nudges will provide awareness of your progress, but you decide when synthesis quality would be optimal.
-
-   If you found little/no context AND searched less than 7 times:
-   - Try different query angles from your decomposition
-   - Search broader related topics
-   - Search user's projects or work areas
-   - Try alternative terminology
-
-   ⚠️ DO NOT search endlessly - if you found relevant episodes, STOP and synthesize!
-
-6. SYNTHESIZE: After gathering sufficient context, provide final answer
-   - Wrap your synthesis in <final_response> tags
-   - Present direct factual context from memory - no meta-commentary
-   - Write as if providing background context to an AI assistant
-   - Include: facts, decisions, preferences, patterns, timelines
-   - Note any gaps, contradictions, or evolution in thinking
-   - Keep it concise and actionable
-   - DO NOT use phrases like "Previous discussions on", "From conversations", "Past preferences indicate"
-   - DO NOT use conversational language like "you said" or "you mentioned"
-   - Present information as direct factual statements
-
-FINAL RESPONSE FORMAT:
-<final_response>
-[Direct synthesized context - factual statements only]
-
-Good examples:
-- "The API redesign focuses on performance and scalability. Key decisions: moving to GraphQL, caching layer with Redis."
-- "Project Phoenix launches Q2 2024. Main features: real-time sync, offline mode, collaborative editing."
-- "Sarah leads the backend team. Recent work includes authentication refactor and database migration."
-
-Bad examples:
-❌ "Previous discussions on the API revealed..."
-❌ "From past conversations, it appears that..."
-❌ "Past preferences indicate..."
-❌ "The user mentioned that..."
-
-Just state the facts directly.
-</final_response>
-
-${contextHints.length > 0 ? `\nCONTEXT HINTS:\n${contextHints.join("\n")}` : ""}
-
-CRITICAL REQUIREMENTS:
-- ALWAYS start with DECOMPOSE step - extract entities, topics, temporal markers, actions
-- Form specific queries from your decomposition - use user's actual terms
-- Minimum 3 searches required
-- Maximum 10 searches allowed - must synthesize after that
-- STOP and synthesize when you hit stopping criteria (20+ episodes, 5+ searches with results, 7+ searches total)
-- Each search should target different aspects from decomposition
-- Present synthesis directly without meta-commentary
-
-SEARCH QUALITY CHECKLIST:
-✅ Queries use specific terms from content (names, projects, exact phrases)
-✅ Searched multiple angles from decomposition (entities, topics, related areas)
-✅ Stop when you have enough unique context - don't search endlessly
-✅ Tried alternative terminology if initial searches found nothing
-❌ Avoid generic/abstract queries that don't match user's vocabulary
-❌ Don't stop at 3 searches if you found zero unique episodes
-❌ Don't keep searching when you already found 20+ unique episodes
-}`
-}

apps/webapp/app/trigger/deep-search/stream-utils.ts

@@ -1,68 +0,0 @@
-import { openai } from "@ai-sdk/openai";
-import { logger } from "@trigger.dev/sdk/v3";
-import {
-  type CoreMessage,
-  type LanguageModelV1,
-  streamText,
-  type ToolSet,
-} from "ai";
-
-/**
- * Generate LLM responses with tool calling support
- * Simplified version for deep-search use case - NO maxSteps for manual ReAct control
- */
-export async function* generate(
-  messages: CoreMessage[],
-  onFinish?: (event: any) => void,
-  tools?: ToolSet,
-  model?: string,
-): AsyncGenerator<
-  | string
-  | {
-      type: string;
-      toolName: string;
-      args?: any;
-      toolCallId?: string;
-    }
-> {
-  const modelToUse = model || process.env.MODEL || "gpt-4.1-2025-04-14";
-  const modelInstance = openai(modelToUse) as LanguageModelV1;
-
-  logger.info(`Starting LLM generation with model: ${modelToUse}`);
-
-  try {
-    const { textStream, fullStream } = streamText({
-      model: modelInstance,
-      messages,
-      temperature: 1,
-      tools,
-      // NO maxSteps - we handle tool execution manually in the ReAct loop
-      toolCallStreaming: true,
-      onFinish,
-    });
-
-    // Yield text chunks
-    for await (const chunk of textStream) {
-      yield chunk;
-    }
-
-    // Yield tool calls
-    for await (const fullChunk of fullStream) {
-      if (fullChunk.type === "tool-call") {
-        yield {
-          type: "tool-call",
-          toolName: fullChunk.toolName,
-          toolCallId: fullChunk.toolCallId,
-          args: fullChunk.args,
-        };
-      }
-
-      if (fullChunk.type === "error") {
-        logger.error(`LLM error: ${JSON.stringify(fullChunk)}`);
-      }
-    }
-  } catch (error) {
-    logger.error(`LLM generation error: ${error}`);
-    throw error;
-  }
-}

apps/webapp/app/trigger/deep-search/types.ts

@@ -1,20 +0,0 @@
-export interface DeepSearchPayload {
-  content: string;
-  userId: string;
-  stream: boolean;
-  intentOverride?: string;
-  metadata?: {
-    source?: "chrome" | "obsidian" | "mcp";
-    url?: string;
-    pageTitle?: string;
-  };
-}
-
-export interface DeepSearchResponse {
-  synthesis: string;
-  episodes?: Array<{
-    content: string;
-    createdAt: Date;
-    spaceIds: string[];
-  }>;
-}

apps/webapp/app/trigger/deep-search/utils.ts

@@ -1,64 +0,0 @@
-import { tool } from "ai";
-import { z } from "zod";
-import axios from "axios";
-import { logger } from "@trigger.dev/sdk/v3";
-
-export function createSearchMemoryTool(token: string) {
-  return tool({
-    description:
-      "Search the user's memory for relevant facts and episodes. Use this tool multiple times with different queries to gather comprehensive context.",
-    parameters: z.object({
-      query: z
-        .string()
-        .describe(
-          "Search query to find relevant information. Be specific: entity names, topics, concepts.",
-        ),
-    }),
-    execute: async ({ query }) => {
-      try {
-        const response = await axios.post(
-          `${process.env.API_BASE_URL || "https://core.heysol.ai"}/api/v1/search`,
-          { query },
-          {
-            headers: {
-              Authorization: `Bearer ${token}`,
-            },
-          },
-        );
-
-        const searchResult = response.data;
-
-        return {
-          facts: searchResult.facts || [],
-          episodes: searchResult.episodes || [],
-          summary: `Found ${searchResult.episodes?.length || 0} relevant memories`,
-        };
-      } catch (error) {
-        logger.error(`SearchMemory tool error: ${error}`);
-        return {
-          facts: [],
-          episodes: [],
-          summary: "No results found",
-        };
-      }
-    },
-  });
-}
-
-// Helper to extract unique episodes from tool calls
-export function extractEpisodesFromToolCalls(toolCalls: any[]): any[] {
-  const episodes: any[] = [];
-
-  for (const call of toolCalls || []) {
-    if (call.toolName === "searchMemory" && call.result?.episodes) {
-      episodes.push(...call.result.episodes);
-    }
-  }
-
-  // Deduplicate by content + createdAt
-  const uniqueEpisodes = Array.from(
-    new Map(episodes.map((e) => [`${e.content}-${e.createdAt}`, e])).values(),
-  );
-
-  return uniqueEpisodes.slice(0, 10);
-}

apps/webapp/app/trigger/ingest/ingest-document.ts

@@ -1,16 +1,8 @@
 import { queue, task } from "@trigger.dev/sdk";
-import { type z } from "zod";
-import crypto from "crypto";
-
-import { IngestionStatus } from "@core/database";
-import { EpisodeTypeEnum } from "@core/types";
-import { logger } from "~/services/logger.service";
-import { saveDocument } from "~/services/graphModels/document";
-import { type IngestBodyRequest } from "~/lib/ingest.server";
-import { DocumentVersioningService } from "~/services/documentVersioning.server";
-import { DocumentDifferentialService } from "~/services/documentDiffer.server";
-import { KnowledgeGraphService } from "~/services/knowledgeGraph.server";
-import { prisma } from "../utils/prisma";
+import {
+  processDocumentIngestion,
+  type IngestDocumentPayload,
+} from "~/jobs/ingest/ingest-document.logic";
 import { ingestTask } from "./ingest";
 
 const documentIngestionQueue = queue({
@@ -23,266 +15,19 @@ export const ingestDocumentTask = task({
   id: "ingest-document",
   queue: documentIngestionQueue,
   machine: "medium-2x",
-  run: async (payload: {
-    body: z.infer<typeof IngestBodyRequest>;
-    userId: string;
-    workspaceId: string;
-    queueId: string;
-  }) => {
-    const startTime = Date.now();
-
-    try {
-      logger.log(`Processing document for user ${payload.userId}`, {
-        contentLength: payload.body.episodeBody.length,
-      });
-
-      await prisma.ingestionQueue.update({
-        where: { id: payload.queueId },
-        data: {
-          status: IngestionStatus.PROCESSING,
-        },
-      });
-
-      const documentBody = payload.body;
-
-      // Step 1: Initialize services and prepare document version
-      const versioningService = new DocumentVersioningService();
-      const differentialService = new DocumentDifferentialService();
-      const knowledgeGraphService = new KnowledgeGraphService();
-
-      const {
-        documentNode: document,
-        versionInfo,
-        chunkedDocument,
-      } = await versioningService.prepareDocumentVersion(
-        documentBody.sessionId!,
-        payload.userId,
-        documentBody.metadata?.documentTitle?.toString() || "Untitled Document",
-        documentBody.episodeBody,
-        documentBody.source,
-        documentBody.metadata || {},
-      );
-
-      logger.log(`Document version analysis:`, {
-        version: versionInfo.newVersion,
-        isNewDocument: versionInfo.isNewDocument,
-        hasContentChanged: versionInfo.hasContentChanged,
-        changePercentage: versionInfo.chunkLevelChanges.changePercentage,
-        changedChunks: versionInfo.chunkLevelChanges.changedChunkIndices.length,
-        totalChunks: versionInfo.chunkLevelChanges.totalChunks,
-      });
-
-      // Step 2: Determine processing strategy
-      const differentialDecision =
-        await differentialService.analyzeDifferentialNeed(
-          documentBody.episodeBody,
-          versionInfo.existingDocument,
-          chunkedDocument,
-        );
-
-      logger.log(`Differential analysis:`, {
-        shouldUseDifferential: differentialDecision.shouldUseDifferential,
-        strategy: differentialDecision.strategy,
-        reason: differentialDecision.reason,
-        documentSizeTokens: differentialDecision.documentSizeTokens,
-      });
-
-      // Early return for unchanged documents
-      if (differentialDecision.strategy === "skip_processing") {
-        logger.log("Document content unchanged, skipping processing");
-        return {
-          success: true,
-          documentsProcessed: 1,
-          chunksProcessed: 0,
-          episodesCreated: 0,
-          entitiesExtracted: 0,
-        };
-      }
-
-      // Step 3: Save the new document version
-      await saveDocument(document);
-
-      // Step 3.1: Invalidate statements from previous document version if it exists
-      let invalidationResults = null;
-      if (versionInfo.existingDocument && versionInfo.hasContentChanged) {
-        logger.log(
-          `Invalidating statements from previous document version: ${versionInfo.existingDocument.uuid}`,
-        );
-
-        invalidationResults =
-          await knowledgeGraphService.invalidateStatementsFromPreviousDocumentVersion(
-            {
-              previousDocumentUuid: versionInfo.existingDocument.uuid,
-              newDocumentContent: documentBody.episodeBody,
-              userId: payload.userId,
-              invalidatedBy: document.uuid,
-              semanticSimilarityThreshold: 0.75, // Configurable threshold
-            },
-          );
-
-        logger.log(`Statement invalidation completed:`, {
-          totalAnalyzed: invalidationResults.totalStatementsAnalyzed,
-          invalidated: invalidationResults.invalidatedStatements.length,
-          preserved: invalidationResults.preservedStatements.length,
+  run: async (payload: IngestDocumentPayload) => {
+    // Use common logic with Trigger-specific callback for episode ingestion
+    return await processDocumentIngestion(
+      payload,
+      // Callback for enqueueing episode ingestion for each chunk
+      async (episodePayload) => {
+        const episodeHandler = await ingestTask.trigger(episodePayload, {
+          queue: "ingestion-queue",
+          concurrencyKey: episodePayload.userId,
+          tags: [episodePayload.userId, episodePayload.queueId],
         });
-      }
-
-      logger.log(
-        `Document chunked into ${chunkedDocument.chunks.length} chunks`,
-      );
-
-      // Step 4: Process chunks based on differential strategy
-      let chunksToProcess = chunkedDocument.chunks;
-      let processingMode = "full";
-
-      if (
-        differentialDecision.shouldUseDifferential &&
-        differentialDecision.strategy === "chunk_level_diff"
-      ) {
-        // Only process changed chunks
-        const chunkComparisons = differentialService.getChunkComparisons(
-          versionInfo.existingDocument!,
-          chunkedDocument,
-        );
-
-        const changedIndices =
-          differentialService.getChunksNeedingReprocessing(chunkComparisons);
-        chunksToProcess = chunkedDocument.chunks.filter((chunk) =>
-          changedIndices.includes(chunk.chunkIndex),
-        );
-        processingMode = "differential";
-
-        logger.log(
-          `Differential processing: ${chunksToProcess.length}/${chunkedDocument.chunks.length} chunks need reprocessing`,
-        );
-      } else if (differentialDecision.strategy === "full_reingest") {
-        // Process all chunks
-        processingMode = "full";
-        logger.log(
-          `Full reingestion: processing all ${chunkedDocument.chunks.length} chunks`,
-        );
-      }
-
-      // Step 5: Queue chunks for processing
-      const episodeHandlers = [];
-      for (const chunk of chunksToProcess) {
-        const chunkEpisodeData = {
-          episodeBody: chunk.content,
-          referenceTime: documentBody.referenceTime,
-          metadata: {
-            ...documentBody.metadata,
-            processingMode,
-            differentialStrategy: differentialDecision.strategy,
-            chunkHash: chunk.contentHash,
-            documentTitle:
-              documentBody.metadata?.documentTitle?.toString() ||
-              "Untitled Document",
-            chunkIndex: chunk.chunkIndex,
-            documentUuid: document.uuid,
-          },
-          source: documentBody.source,
-          spaceIds: documentBody.spaceIds,
-          sessionId: documentBody.sessionId,
-          type: EpisodeTypeEnum.DOCUMENT,
-        };
-
-        const episodeHandler = await ingestTask.trigger(
-          {
-            body: chunkEpisodeData,
-            userId: payload.userId,
-            workspaceId: payload.workspaceId,
-            queueId: payload.queueId,
-          },
-          {
-            queue: "ingestion-queue",
-            concurrencyKey: payload.userId,
-            tags: [payload.userId, payload.queueId, processingMode],
-          },
-        );
-
-        if (episodeHandler.id) {
-          episodeHandlers.push(episodeHandler.id);
-          logger.log(
-            `Queued chunk ${chunk.chunkIndex + 1} for ${processingMode} processing`,
-            {
-              handlerId: episodeHandler.id,
-              chunkSize: chunk.content.length,
-              chunkHash: chunk.contentHash,
-            },
-          );
-        }
-      }
-
-      // Calculate cost savings
-      const costSavings = differentialService.calculateCostSavings(
-        chunkedDocument.chunks.length,
-        chunksToProcess.length,
-      );
-
-      await prisma.ingestionQueue.update({
-        where: { id: payload.queueId },
-        data: {
-          output: {
-            documentUuid: document.uuid,
-            version: versionInfo.newVersion,
-            totalChunks: chunkedDocument.chunks.length,
-            chunksProcessed: chunksToProcess.length,
-            chunksSkipped: costSavings.chunksSkipped,
-            processingMode,
-            differentialStrategy: differentialDecision.strategy,
-            estimatedSavings: `${costSavings.estimatedSavingsPercentage.toFixed(1)}%`,
-            statementInvalidation: invalidationResults
-              ? {
-                  totalAnalyzed: invalidationResults.totalStatementsAnalyzed,
-                  invalidated: invalidationResults.invalidatedStatements.length,
-                  preserved: invalidationResults.preservedStatements.length,
-                }
-              : null,
-            episodes: [],
-            episodeHandlers,
-          },
-          status: IngestionStatus.PROCESSING,
-        },
-      });
-
-      const processingTimeMs = Date.now() - startTime;
-
-      logger.log(
-        `Document differential processing completed in ${processingTimeMs}ms`,
-        {
-          documentUuid: document.uuid,
-          version: versionInfo.newVersion,
-          processingMode,
-          totalChunks: chunkedDocument.chunks.length,
-          chunksProcessed: chunksToProcess.length,
-          chunksSkipped: costSavings.chunksSkipped,
-          estimatedSavings: `${costSavings.estimatedSavingsPercentage.toFixed(1)}%`,
-          changePercentage: `${differentialDecision.changePercentage.toFixed(1)}%`,
-          statementInvalidation: invalidationResults
-            ? {
-                totalAnalyzed: invalidationResults.totalStatementsAnalyzed,
-                invalidated: invalidationResults.invalidatedStatements.length,
-                preserved: invalidationResults.preservedStatements.length,
-              }
-            : "No previous version",
-        },
-      );
-
-      return { success: true };
-    } catch (err: any) {
-      await prisma.ingestionQueue.update({
-        where: { id: payload.queueId },
-        data: {
-          error: err.message,
-          status: IngestionStatus.FAILED,
-        },
-      });
-
-      logger.error(
-        `Error processing document for user ${payload.userId}:`,
-        err,
-      );
-      return { success: false, error: err.message };
-    }
+        return { id: episodeHandler.id };
+      },
+    );
   },
 });

apps/webapp/app/trigger/ingest/ingest.ts

@@ -1,251 +1,37 @@
 import { queue, task } from "@trigger.dev/sdk";
-import { z } from "zod";
-import { KnowledgeGraphService } from "~/services/knowledgeGraph.server";
-import { linkEpisodeToDocument } from "~/services/graphModels/document";
-
-import { IngestionStatus } from "@core/database";
-import { logger } from "~/services/logger.service";
+import {
+  processEpisodeIngestion,
+  IngestBodyRequest,
+  type IngestEpisodePayload,
+} from "~/jobs/ingest/ingest-episode.logic";
 import { triggerSpaceAssignment } from "../spaces/space-assignment";
-import { prisma } from "../utils/prisma";
-import { EpisodeType } from "@core/types";
-import { deductCredits, hasCredits } from "../utils/utils";
-import { assignEpisodesToSpace } from "~/services/graphModels/space";
 import { triggerSessionCompaction } from "../session/session-compaction";
 
-export const IngestBodyRequest = z.object({
-  episodeBody: z.string(),
-  referenceTime: z.string(),
-  metadata: z.record(z.union([z.string(), z.number(), z.boolean()])).optional(),
-  source: z.string(),
-  spaceIds: z.array(z.string()).optional(),
-  sessionId: z.string().optional(),
-  type: z
-    .enum([EpisodeType.CONVERSATION, EpisodeType.DOCUMENT])
-    .default(EpisodeType.CONVERSATION),
-});
-
 const ingestionQueue = queue({
   name: "ingestion-queue",
   concurrencyLimit: 1,
 });
 
+// Export for backwards compatibility
+export { IngestBodyRequest };
+
 // Register the Trigger.dev task
 export const ingestTask = task({
   id: "ingest-episode",
   queue: ingestionQueue,
   machine: "medium-2x",
-  run: async (payload: {
-    body: z.infer<typeof IngestBodyRequest>;
-    userId: string;
-    workspaceId: string;
-    queueId: string;
-  }) => {
-    try {
-      logger.log(`Processing job for user ${payload.userId}`);
-
-      // Check if workspace has sufficient credits before processing
-      const hasSufficientCredits = await hasCredits(
-        payload.workspaceId,
-        "addEpisode",
-      );
-
-      if (!hasSufficientCredits) {
-        logger.warn(
-          `Insufficient credits for workspace ${payload.workspaceId}`,
-        );
-
-        await prisma.ingestionQueue.update({
-          where: { id: payload.queueId },
-          data: {
-            status: IngestionStatus.NO_CREDITS,
-            error:
-              "Insufficient credits. Please upgrade your plan or wait for your credits to reset.",
-          },
-        });
-
-        return {
-          success: false,
-          error: "Insufficient credits",
-        };
-      }
-
-      const ingestionQueue = await prisma.ingestionQueue.update({
-        where: { id: payload.queueId },
-        data: {
-          status: IngestionStatus.PROCESSING,
-        },
-      });
-
-      const knowledgeGraphService = new KnowledgeGraphService();
-
-      const episodeBody = payload.body as any;
-
-      const episodeDetails = await knowledgeGraphService.addEpisode(
-        {
-          ...episodeBody,
-          userId: payload.userId,
-        },
-        prisma,
-      );
-
-      // Link episode to document if it's a document chunk
-      if (
-        episodeBody.type === EpisodeType.DOCUMENT &&
-        episodeBody.metadata.documentUuid &&
-        episodeDetails.episodeUuid
-      ) {
-        try {
-          await linkEpisodeToDocument(
-            episodeDetails.episodeUuid,
-            episodeBody.metadata.documentUuid,
-            episodeBody.metadata.chunkIndex || 0,
-          );
-          logger.log(
-            `Linked episode ${episodeDetails.episodeUuid} to document ${episodeBody.metadata.documentUuid} at chunk ${episodeBody.metadata.chunkIndex || 0}`,
-          );
-        } catch (error) {
-          logger.error(`Failed to link episode to document:`, {
-            error,
-            episodeUuid: episodeDetails.episodeUuid,
-            documentUuid: episodeBody.metadata.documentUuid,
-          });
-        }
-      }
-
-      let finalOutput = episodeDetails;
-      let episodeUuids: string[] = episodeDetails.episodeUuid
-        ? [episodeDetails.episodeUuid]
-        : [];
-      let currentStatus: IngestionStatus = IngestionStatus.COMPLETED;
-      if (episodeBody.type === EpisodeType.DOCUMENT) {
-        const currentOutput = ingestionQueue.output as any;
-        currentOutput.episodes.push(episodeDetails);
-        episodeUuids = currentOutput.episodes.map(
-          (episode: any) => episode.episodeUuid,
-        );
-
-        finalOutput = {
-          ...currentOutput,
-        };
-
-        if (currentOutput.episodes.length !== currentOutput.totalChunks) {
-          currentStatus = IngestionStatus.PROCESSING;
-        }
-      }
-
-      await prisma.ingestionQueue.update({
-        where: { id: payload.queueId },
-        data: {
-          output: finalOutput,
-          status: currentStatus,
-        },
-      });
-
-      // Deduct credits for episode creation
-      if (currentStatus === IngestionStatus.COMPLETED) {
-        await deductCredits(
-          payload.workspaceId,
-          "addEpisode",
-          finalOutput.statementsCreated,
-        );
-      }
-
-      // Handle space assignment after successful ingestion
-      try {
-        // If spaceIds were explicitly provided, immediately assign the episode to those spaces
-        if (
-          episodeBody.spaceIds &&
-          episodeBody.spaceIds.length > 0 &&
-          episodeDetails.episodeUuid
-        ) {
-          logger.info(`Assigning episode to explicitly provided spaces`, {
-            userId: payload.userId,
-            episodeId: episodeDetails.episodeUuid,
-            spaceIds: episodeBody.spaceIds,
-          });
-
-          // Assign episode to each space
-          for (const spaceId of episodeBody.spaceIds) {
-            await assignEpisodesToSpace(
-              [episodeDetails.episodeUuid],
-              spaceId,
-              payload.userId,
-            );
-          }
-
-          logger.info(
-            `Skipping LLM space assignment - episode explicitly assigned to ${episodeBody.spaceIds.length} space(s)`,
-          );
-        } else {
-          // Only trigger automatic LLM space assignment if no explicit spaceIds were provided
-          logger.info(
-            `Triggering LLM space assignment after successful ingestion`,
-            {
-              userId: payload.userId,
-              workspaceId: payload.workspaceId,
-              episodeId: episodeDetails?.episodeUuid,
-            },
-          );
-          if (
-            episodeDetails.episodeUuid &&
-            currentStatus === IngestionStatus.COMPLETED
-          ) {
-            await triggerSpaceAssignment({
-              userId: payload.userId,
-              workspaceId: payload.workspaceId,
-              mode: "episode",
-              episodeIds: episodeUuids,
-            });
-          }
-        }
-      } catch (assignmentError) {
-        // Don't fail the ingestion if assignment fails
-        logger.warn(`Failed to trigger space assignment after ingestion:`, {
-          error: assignmentError,
-          userId: payload.userId,
-          episodeId: episodeDetails?.episodeUuid,
-        });
-      }
-
-      // Auto-trigger session compaction if episode has sessionId
-      try {
-        if (
-          episodeBody.sessionId &&
-          currentStatus === IngestionStatus.COMPLETED
-        ) {
-          logger.info(`Checking if session compaction should be triggered`, {
-            userId: payload.userId,
-            sessionId: episodeBody.sessionId,
-            source: episodeBody.source,
-          });
-
-          await triggerSessionCompaction({
-            userId: payload.userId,
-            sessionId: episodeBody.sessionId,
-            source: episodeBody.source,
-          });
-        }
-      } catch (compactionError) {
-        // Don't fail the ingestion if compaction fails
-        logger.warn(`Failed to trigger session compaction after ingestion:`, {
-          error: compactionError,
-          userId: payload.userId,
-          sessionId: episodeBody.sessionId,
-        });
-      }
-
-      return { success: true, episodeDetails };
-    } catch (err: any) {
-      await prisma.ingestionQueue.update({
-        where: { id: payload.queueId },
-        data: {
-          error: err.message,
-          status: IngestionStatus.FAILED,
-        },
-      });
-
-      logger.error(`Error processing job for user ${payload.userId}:`, err);
-      return { success: false, error: err.message };
-    }
+  run: async (payload: IngestEpisodePayload) => {
+    // Use common logic with Trigger-specific callbacks for follow-up jobs
+    return await processEpisodeIngestion(
+      payload,
+      // Callback for space assignment
+      async (params) => {
+        await triggerSpaceAssignment(params);
+      },
+      // Callback for session compaction
+      async (params) => {
+        await triggerSessionCompaction(params);
+      },
+    );
   },
 });

apps/webapp/app/trigger/ingest/retry-no-credits.ts

@@ -0,0 +1,122 @@
+import { task } from "@trigger.dev/sdk";
+import { z } from "zod";
+import { IngestionQueue, IngestionStatus } from "@core/database";
+import { logger } from "~/services/logger.service";
+import { prisma } from "../utils/prisma";
+import { IngestBodyRequest, ingestTask } from "./ingest";
+
+export const RetryNoCreditBodyRequest = z.object({
+  workspaceId: z.string(),
+});
+
+// Register the Trigger.dev task to retry NO_CREDITS episodes
+export const retryNoCreditsTask = task({
+  id: "retry-no-credits-episodes",
+  run: async (payload: z.infer<typeof RetryNoCreditBodyRequest>) => {
+    try {
+      logger.log(
+        `Starting retry of NO_CREDITS episodes for workspace ${payload.workspaceId}`,
+      );
+
+      // Find all ingestion queue items with NO_CREDITS status for this workspace
+      const noCreditItems = await prisma.ingestionQueue.findMany({
+        where: {
+          workspaceId: payload.workspaceId,
+          status: IngestionStatus.NO_CREDITS,
+        },
+        orderBy: {
+          createdAt: "asc", // Process oldest first
+        },
+        include: {
+          workspace: true,
+        },
+      });
+
+      if (noCreditItems.length === 0) {
+        logger.log(
+          `No NO_CREDITS episodes found for workspace ${payload.workspaceId}`,
+        );
+        return {
+          success: true,
+          message: "No episodes to retry",
+          retriedCount: 0,
+        };
+      }
+
+      logger.log(
+        `Found ${noCreditItems.length} NO_CREDITS episodes to retry`,
+      );
+
+      const results = {
+        total: noCreditItems.length,
+        retriggered: 0,
+        failed: 0,
+        errors: [] as Array<{ queueId: string; error: string }>,
+      };
+
+      // Process each item
+      for (const item of noCreditItems) {
+        try {
+          const queueData = item.data as z.infer<typeof IngestBodyRequest>;
+
+          // Reset status to PENDING and clear error
+          await prisma.ingestionQueue.update({
+            where: { id: item.id },
+            data: {
+              status: IngestionStatus.PENDING,
+              error: null,
+              retryCount: item.retryCount + 1,
+            },
+          });
+
+          // Trigger the ingestion task
+          await ingestTask.trigger({
+            body: queueData,
+            userId: item.workspace?.userId as string,
+            workspaceId: payload.workspaceId,
+            queueId: item.id,
+          });
+
+          results.retriggered++;
+          logger.log(
+            `Successfully retriggered episode ${item.id} (retry #${item.retryCount + 1})`,
+          );
+        } catch (error: any) {
+          results.failed++;
+          results.errors.push({
+            queueId: item.id,
+            error: error.message,
+          });
+          logger.error(`Failed to retrigger episode ${item.id}:`, error);
+
+          // Update the item to mark it as failed
+          await prisma.ingestionQueue.update({
+            where: { id: item.id },
+            data: {
+              status: IngestionStatus.FAILED,
+              error: `Retry failed: ${error.message}`,
+            },
+          });
+        }
+      }
+
+      logger.log(
+        `Completed retry for workspace ${payload.workspaceId}. Retriggered: ${results.retriggered}, Failed: ${results.failed}`,
+      );
+
+      return {
+        success: true,
+        ...results,
+      };
+    } catch (err: any) {
+      logger.error(
+        `Error retrying NO_CREDITS episodes for workspace ${payload.workspaceId}:`,
+        err,
+      );
+      return {
+        success: false,
+        error: err.message,
+      };
+    }
+  },
+});

apps/webapp/app/trigger/session/session-compaction.ts

@@ -1,36 +1,8 @@
 import { queue, task } from "@trigger.dev/sdk/v3";
-import { logger } from "~/services/logger.service";
-import { runQuery } from "~/lib/neo4j.server";
-import type { CoreMessage } from "ai";
-import { z } from "zod";
-import { getEmbedding, makeModelCall } from "~/lib/model.server";
 import {
-  getCompactedSessionBySessionId,
-  linkEpisodesToCompact,
-  getSessionEpisodes,
-  type CompactedSessionNode,
-  type SessionEpisodeData,
-  saveCompactedSession,
-} from "~/services/graphModels/compactedSession";
-
-interface SessionCompactionPayload {
-  userId: string;
-  sessionId: string;
-  source: string;
-  triggerSource?: "auto" | "manual" | "threshold";
-}
-
-// Zod schema for LLM response validation
-const CompactionResultSchema = z.object({
-  summary: z.string().describe("Consolidated narrative of the entire session"),
-  confidence: z.number().min(0).max(1).describe("Confidence score of the compaction quality"),
-});
-
-const CONFIG = {
-  minEpisodesForCompaction: 5, // Minimum episodes to trigger compaction
-  compactionThreshold: 1, // Trigger after N new episodes
-  maxEpisodesPerBatch: 50, // Process in batches if needed
-};
+  processSessionCompaction,
+  type SessionCompactionPayload,
+} from "~/jobs/session/session-compaction.logic";
 
 export const sessionCompactionQueue = queue({
   name: "session-compaction-queue",
@@ -41,82 +13,7 @@ export const sessionCompactionTask = task({
   id: "session-compaction",
   queue: sessionCompactionQueue,
   run: async (payload: SessionCompactionPayload) => {
-    const { userId, sessionId, source, triggerSource = "auto" } = payload;
-
-    logger.info(`Starting session compaction`, {
-      userId,
-      sessionId,
-      source,
-      triggerSource,
-    });
-
-    try {
-      // Check if compaction already exists
-      const existingCompact = await getCompactedSessionBySessionId(sessionId, userId);
-
-      // Fetch all episodes for this session
-      const episodes = await getSessionEpisodes(sessionId, userId, existingCompact?.endTime);
-
-      console.log("episodes", episodes.length);
-      // Check if we have enough episodes
-      if (!existingCompact && episodes.length < CONFIG.minEpisodesForCompaction) {
-        logger.info(`Not enough episodes for compaction`, {
-          sessionId,
-          episodeCount: episodes.length,
-          minRequired: CONFIG.minEpisodesForCompaction,
-        });
-        return {
-          success: false,
-          reason: "insufficient_episodes",
-          episodeCount: episodes.length,
-        };
-      } else if (existingCompact && episodes.length < CONFIG.minEpisodesForCompaction + CONFIG.compactionThreshold) {
-        logger.info(`Not enough new episodes for compaction`, {
-          sessionId,
-          episodeCount: episodes.length,
-          minRequired: CONFIG.minEpisodesForCompaction + CONFIG.compactionThreshold,
-        });
-        return {
-          success: false,
-          reason: "insufficient_new_episodes",
-          episodeCount: episodes.length,
-        };
-      }
-
-      // Generate or update compaction
-      const compactionResult = existingCompact
-        ? await updateCompaction(existingCompact, episodes, userId)
-        : await createCompaction(sessionId, episodes, userId, source);
-
-      logger.info(`Session compaction completed`, {
-        sessionId,
-        compactUuid: compactionResult.uuid,
-        episodeCount: compactionResult.episodeCount,
-        compressionRatio: compactionResult.compressionRatio,
-      });
-
-      return {
-        success: true,
-        compactionResult: {
-          compactUuid: compactionResult.uuid,
-          sessionId: compactionResult.sessionId,
-          summary: compactionResult.summary,
-          episodeCount: compactionResult.episodeCount,
-          startTime: compactionResult.startTime,
-          endTime: compactionResult.endTime,
-          confidence: compactionResult.confidence,
-          compressionRatio: compactionResult.compressionRatio,
-        },
-      };
-    } catch (error) {
-      logger.error(`Session compaction failed`, {
-        sessionId,
-        userId,
-        error: error instanceof Error ? error.message : String(error),
-      });
-
-      throw error;
-    }
+    return await processSessionCompaction(payload);
   },
 });
 

apps/webapp/app/trigger/spaces/space-assignment.ts

@@ -18,7 +18,7 @@ import type { CoreMessage } from "ai";
 import { z } from "zod";
 import { type Space } from "@prisma/client";
 
-interface SpaceAssignmentPayload {
+export interface SpaceAssignmentPayload {
   userId: string;
   workspaceId: string;
   mode: "new_space" | "episode";

apps/webapp/app/trigger/spaces/space-summary.ts

@@ -11,7 +11,7 @@ import { getSpace, updateSpace } from "../utils/space-utils";
 
 import { EpisodeType } from "@core/types";
 import { getSpaceEpisodeCount } from "~/services/graphModels/space";
-import { addToQueue } from "../utils/queue";
+import { addToQueue } from "~/lib/ingest.server";
 
 interface SpaceSummaryPayload {
   userId: string;

apps/webapp/app/trigger/utils/mcp.ts

@@ -1,10 +1,9 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 import { logger } from "@trigger.dev/sdk/v3";
-import { jsonSchema, tool, type ToolSet } from "ai";
+
 import * as fs from "fs";
 import * as path from "path";
 
-import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
 import { prisma } from "./prisma";
 
 export const configureStdioMCPEnvironment = (
@@ -63,101 +62,6 @@ export const configureStdioMCPEnvironment = (
   };
 };
 
-export class MCP {
-  private Client: any;
-  private client: any = {};
-
-  constructor() {}
-
-  public async init() {
-    this.Client = await MCP.importClient();
-  }
-
-  private static async importClient() {
-    const { Client } = await import(
-      "@modelcontextprotocol/sdk/client/index.js"
-    );
-    return Client;
-  }
-
-  async load(headers: any) {
-    return await this.connectToServer(
-      `${process.env.API_BASE_URL}/api/v1/mcp?source=core`,
-      headers,
-    );
-  }
-
-  async allTools(): Promise<ToolSet> {
-    try {
-      const { tools } = await this.client.listTools();
-
-      const finalTools: ToolSet = {};
-
-      tools.map(({ name, description, inputSchema }: any) => {
-        finalTools[name] = tool({
-          description,
-          parameters: jsonSchema(inputSchema),
-        });
-      });
-
-      return finalTools;
-    } catch (error) {
-      return {};
-    }
-
-    // Flatten and convert to object
-  }
-
-  async getTool(name: string) {
-    try {
-      const { tools: clientTools } = await this.client.listTools();
-      const clientTool = clientTools.find((to: any) => to.name === name);
-
-      return JSON.stringify(clientTool);
-    } catch (e) {
-      logger.error((e as string) ?? "Getting tool failed");
-      throw new Error("Getting tool failed");
-    }
-  }
-
-  async callTool(name: string, parameters: any) {
-    const response = await this.client.callTool({
-      name,
-      arguments: parameters,
-    });
-
-    return response;
-  }
-
-  async connectToServer(url: string, headers: any) {
-    try {
-      const client = new this.Client(
-        {
-          name: "Core",
-          version: "1.0.0",
-        },
-        {
-          capabilities: {},
-        },
-      );
-
-      // Configure the transport for MCP server
-      const transport = new StreamableHTTPClientTransport(new URL(url), {
-        requestInit: { headers },
-      });
-
-      // Connect to the MCP server
-      await client.connect(transport, { timeout: 60 * 1000 * 5 });
-      this.client = client;
-
-      logger.info(`Connected to MCP server`);
-    } catch (e) {
-      logger.error(`Failed to connect to MCP server: `, { e });
-      throw e;
-    }
-  }
-}
-
 export const fetchAndSaveStdioIntegrations = async () => {
   try {
     logger.info("Starting stdio integrations fetch and save process");

apps/webapp/app/trigger/utils/utils.ts

@@ -8,11 +8,9 @@ import {
   type Workspace,
 } from "@prisma/client";
 
-import { logger } from "@trigger.dev/sdk/v3";
 import { type CoreMessage } from "ai";
 
 import { type HistoryStep } from "./types";
-import axios from "axios";
 import nodeCrypto from "node:crypto";
 import { customAlphabet, nanoid } from "nanoid";
 import { prisma } from "./prisma";
@@ -148,58 +146,6 @@ export interface RunChatPayload {
   isContinuation?: boolean;
 }
 
-export const init = async ({ payload }: { payload: InitChatPayload }) => {
-  logger.info("Loading init");
-  const conversationHistory = await prisma.conversationHistory.findUnique({
-    where: { id: payload.conversationHistoryId },
-    include: { conversation: true },
-  });
-
-  const conversation = conversationHistory?.conversation as Conversation;
-
-  const workspace = await prisma.workspace.findUnique({
-    where: { id: conversation.workspaceId as string },
-  });
-
-  if (!workspace) {
-    return { conversation, conversationHistory };
-  }
-
-  const randomKeyName = `chat_${nanoid(10)}`;
-  const pat = await getOrCreatePersonalAccessToken({
-    name: randomKeyName,
-    userId: workspace.userId as string,
-  });
-
-  const user = await prisma.user.findFirst({
-    where: { id: workspace.userId as string },
-  });
-
-  // Set up axios interceptor for memory operations
-  axios.interceptors.request.use((config) => {
-    if (config.url?.startsWith("https://core::memory")) {
-      // Handle both search and ingest endpoints
-      config.url = config.url.replace(
-        "https://core::memory",
-        process.env.API_BASE_URL ?? "",
-      );
-
-      config.headers.Authorization = `Bearer ${pat.token}`;
-    }
-
-    return config;
-  });
-
-  return {
-    conversation,
-    conversationHistory,
-    tokenId: pat.id,
-    token: pat.token,
-    userId: user?.id,
-    userName: user?.name,
-  };
-};
-
 export const createConversationHistoryForAgent = async (
   conversationId: string,
 ) => {

apps/webapp/app/utils/mcp/memory.ts

@@ -3,7 +3,6 @@ import { addToQueue } from "~/lib/ingest.server";
 import { logger } from "~/services/logger.service";
 import { SearchService } from "~/services/search.server";
 import { SpaceService } from "~/services/space.server";
-import { deepSearch } from "~/trigger/deep-search";
 import { IntegrationLoader } from "./integration-loader";
 import { hasCredits } from "~/services/billing.server";
 import { prisma } from "~/db.server";
@@ -18,7 +17,28 @@ const SearchParamsSchema = {
     query: {
       type: "string",
       description:
-        "Search query as a simple statement or question. Write what you want to find, not a command. GOOD: 'user preferences for code style' or 'previous bugs in authentication' or 'GitHub integration setup'. BAD: 'search for' or 'find me' or 'get the'. Just state the topic directly.",
+        "Search query optimized for knowledge graph retrieval. Choose the right query structure based on your search intent:\n\n" +
+        "1. **Entity-Centric Queries** (Best for graph search):\n" +
+        "   - ✅ GOOD: \"User's preferences for code style and formatting\"\n" +
+        "   - ✅ GOOD: \"Project authentication implementation decisions\"\n" +
+        "   - ❌ BAD: \"user code style\"\n" +
+        "   - Format: [Person/Project] + [relationship/attribute] + [context]\n\n" +
+        "2. **Multi-Entity Relationship Queries** (Excellent for episode graph):\n" +
+        "   - ✅ GOOD: \"User and team discussions about API design patterns\"\n" +
+        "   - ✅ GOOD: \"relationship between database schema and performance optimization\"\n" +
+        "   - ❌ BAD: \"user team api design\"\n" +
+        "   - Format: [Entity1] + [relationship type] + [Entity2] + [context]\n\n" +
+        "3. **Semantic Question Queries** (Good for vector search):\n" +
+        "   - ✅ GOOD: \"What causes authentication errors in production? What are the security requirements?\"\n" +
+        "   - ✅ GOOD: \"How does caching improve API response times compared to direct database queries?\"\n" +
+        "   - ❌ BAD: \"auth errors production\"\n" +
+        "   - Format: Complete natural questions with full context\n\n" +
+        "4. **Concept Exploration Queries** (Good for BFS traversal):\n" +
+        "   - ✅ GOOD: \"concepts and ideas related to database indexing and query optimization\"\n" +
+        "   - ✅ GOOD: \"topics connected to user authentication and session management\"\n" +
+        "   - ❌ BAD: \"database indexing concepts\"\n" +
+        "   - Format: [concept] + related/connected + [domain/context]\n\n" +
+        "Avoid keyword soup queries - use complete phrases with proper context for best results.",
     },
     validAt: {
       type: "string",
@@ -229,8 +249,8 @@ export async function callMemoryTool(
         return await handleGetIntegrationActions({ ...args });
       case "execute_integration_action":
         return await handleExecuteIntegrationAction({ ...args });
-      case "memory_deep_search":
-        return await handleMemoryDeepSearch({ ...args, userId, source });
+      // case "memory_deep_search":
+      //   return await handleMemoryDeepSearch({ ...args, userId, source });
       default:
         throw new Error(`Unknown memory tool: ${toolName}`);
     }
@@ -596,58 +616,3 @@ async function handleExecuteIntegrationAction(args: any) {
     };
   }
 }
-
-// Handler for memory_deep_search
-async function handleMemoryDeepSearch(args: any) {
-  try {
-    const { content, intentOverride, userId, source } = args;
-
-    if (!content) {
-      throw new Error("content is required");
-    }
-
-    // Trigger non-streaming deep search task
-    const handle = await deepSearch.triggerAndWait({
-      content,
-      userId,
-      stream: false, // MCP doesn't need streaming
-      intentOverride,
-      metadata: { source },
-    });
-
-    // Wait for task completion
-    if (handle.ok) {
-      return {
-        content: [
-          {
-            type: "text",
-            text: JSON.stringify(handle.output),
-          },
-        ],
-        isError: false,
-      };
-    } else {
-      return {
-        content: [
-          {
-            type: "text",
-            text: `Error performing deep search: ${handle.error instanceof Error ? handle.error.message : String(handle.error)}`,
-          },
-        ],
-        isError: true,
-      };
-    }
-  } catch (error) {
-    logger.error(`MCP deep search error: ${error}`);
-
-    return {
-      content: [
-        {
-          type: "text",
-          text: `Error performing deep search: ${error instanceof Error ? error.message : String(error)}`,
-        },
-      ],
-      isError: true,
-    };
-  }
-}

apps/webapp/app/utils/startup.ts

@@ -1,11 +1,38 @@
 import { logger } from "~/services/logger.service";
 import { fetchAndSaveStdioIntegrations } from "~/trigger/utils/mcp";
-import { initNeo4jSchemaOnce } from "~/lib/neo4j.server";
+import { initNeo4jSchemaOnce, verifyConnectivity } from "~/lib/neo4j.server";
 import { env } from "~/env.server";
+import { initWorkers, shutdownWorkers } from "~/bullmq/start-workers";
+import { trackConfig } from "~/services/telemetry.server";
 
 // Global flag to ensure startup only runs once per server process
 let startupInitialized = false;
 
+/**
+ * Wait for Neo4j to be ready before initializing schema
+ */
+async function waitForNeo4j(maxRetries = 30, retryDelay = 2000) {
+  logger.info("Waiting for Neo4j to be ready...");
+
+  for (let i = 0; i < maxRetries; i++) {
+    try {
+      const connected = await verifyConnectivity();
+      if (connected) {
+        logger.info("✓ Neo4j is ready!");
+        return true;
+      }
+    } catch (error) {
+      // Connection failed, will retry
+    }
+
+    logger.info(`Neo4j not ready, retrying... (${i + 1}/${maxRetries})`);
+    await new Promise((resolve) => setTimeout(resolve, retryDelay));
+  }
+
+  logger.error("Failed to connect to Neo4j after maximum retries");
+  throw new Error("Failed to connect to Neo4j after maximum retries");
+}
+
 /**
  * Initialize all startup services once per server process
  * Safe to call multiple times - will only run initialization once
@@ -40,24 +67,47 @@ export async function initializeStartupServices() {
     process.exit(1);
   }
 
-  try {
-    const triggerApiUrl = env.TRIGGER_API_URL;
-    if (triggerApiUrl) {
+  if (env.QUEUE_PROVIDER === "trigger") {
+    try {
+      const triggerApiUrl = env.TRIGGER_API_URL;
+      // At this point, env validation should have already ensured these are present
+      // But we add a runtime check for safety
+      if (
+        !triggerApiUrl ||
+        !env.TRIGGER_PROJECT_ID ||
+        !env.TRIGGER_SECRET_KEY
+      ) {
+        console.error(
+          "TRIGGER_API_URL, TRIGGER_PROJECT_ID, and TRIGGER_SECRET_KEY must be set when QUEUE_PROVIDER=trigger",
+        );
+        process.exit(1);
+      }
       await waitForTriggerLogin(triggerApiUrl);
       await addEnvVariablesInTrigger();
-    } else {
-      console.error("TRIGGER_API_URL is not set in environment variables.");
+    } catch (e) {
+      console.error(e);
+      console.error("Trigger is not configured");
       process.exit(1);
     }
-  } catch (e) {
-    console.error(e);
-    console.error("Trigger is not configured");
-    process.exit(1);
+  } else {
+    await initWorkers();
+
+    // Handle graceful shutdown
+    process.on("SIGTERM", async () => {
+      await shutdownWorkers();
+    });
+    process.on("SIGINT", async () => {
+      await shutdownWorkers();
+      process.exit(0);
+    });
   }
 
   try {
     logger.info("Starting application initialization...");
 
+    // Wait for Neo4j to be ready
+    await waitForNeo4j();
+
     // Initialize Neo4j schema
     await initNeo4jSchemaOnce();
     logger.info("Neo4j schema initialization completed");
@@ -65,6 +115,10 @@ export async function initializeStartupServices() {
     await fetchAndSaveStdioIntegrations();
     logger.info("Stdio integrations initialization completed");
 
+    // Track system configuration once at startup
+    await trackConfig();
+    logger.info("System configuration tracked");
+
     startupInitialized = true;
     logger.info("Application initialization completed successfully");
   } catch (error) {
@@ -121,6 +175,14 @@ export async function addEnvVariablesInTrigger() {
     TRIGGER_SECRET_KEY,
   } = env;
 
+  // These should always be present when this function is called
+  // but we add a runtime check for type safety
+  if (!TRIGGER_PROJECT_ID || !TRIGGER_API_URL || !TRIGGER_SECRET_KEY) {
+    throw new Error(
+      "TRIGGER_PROJECT_ID, TRIGGER_API_URL, and TRIGGER_SECRET_KEY are required",
+    );
+  }
+
   const DATABASE_URL = getDatabaseUrl(POSTGRES_DB);
 
   // Map of key to value from env, replacing 'localhost' as needed

apps/webapp/package.json

@@ -14,11 +14,12 @@
     "trigger:deploy": "pnpm dlx trigger.dev@4.0.4 deploy"
   },
   "dependencies": {
-    "@ai-sdk/amazon-bedrock": "2.2.12",
-    "@ai-sdk/anthropic": "^1.2.12",
-    "@ai-sdk/google": "^1.2.22",
-    "@ai-sdk/openai": "^1.3.21",
-    "@anthropic-ai/sdk": "^0.60.0",
+    "@ai-sdk/amazon-bedrock": "3.0.47",
+    "@ai-sdk/anthropic": "^2.0.37",
+    "@ai-sdk/google": "^2.0.23",
+    "@ai-sdk/openai": "^2.0.53",
+    "@ai-sdk/react": "2.0.78",
+    "@anthropic-ai/sdk": "^0.67.0",
     "@aws-sdk/client-s3": "3.879.0",
     "@aws-sdk/credential-providers": "^3.894.0",
     "@aws-sdk/s3-request-presigner": "3.879.0",
@@ -81,7 +82,7 @@
     "@tiptap/starter-kit": "2.11.9",
     "@trigger.dev/react-hooks": "4.0.4",
     "@trigger.dev/sdk": "4.0.4",
-    "ai": "4.3.19",
+    "ai": "5.0.78",
     "axios": "^1.10.0",
     "bullmq": "^5.53.2",
     "cheerio": "^1.1.2",
@@ -117,9 +118,10 @@
     "neo4j-driver": "^5.28.1",
     "non.geist": "^1.0.2",
     "novel": "^1.0.2",
-    "ollama-ai-provider": "1.2.0",
+    "ollama-ai-provider-v2": "1.5.1",
     "openai": "^5.12.2",
     "posthog-js": "^1.116.6",
+    "posthog-node": "^5.10.3",
     "react": "^18.2.0",
     "react-calendar-heatmap": "^1.10.0",
     "react-dom": "^18.2.0",
@@ -127,6 +129,7 @@
     "react-resizable-panels": "^1.0.9",
     "react-hotkeys-hook": "^4.5.0",
     "react-virtualized": "^9.22.6",
+    "resumable-stream": "2.2.8",
     "remix-auth": "^4.2.0",
     "remix-auth-oauth2": "^3.4.1",
     "remix-themes": "^2.0.4",

apps/webapp/server.js

@@ -0,0 +1,111 @@
+import { createRequestHandler } from "@remix-run/express";
+import compression from "compression";
+import express from "express";
+import morgan from "morgan";
+// import { handleMCPRequest, handleSessionRequest } from "~/services/mcp.server";
+// import { authenticateHybridRequest } from "~/services/routeBuilders/apiBuilder.server";
+let viteDevServer;
+let remixHandler;
+async function init() {
+    if (process.env.NODE_ENV !== "production") {
+        const vite = await import("vite");
+        viteDevServer = await vite.createServer({
+            server: { middlewareMode: true },
+        });
+    }
+    const build = viteDevServer
+        ? () => viteDevServer.ssrLoadModule("virtual:remix/server-build")
+        : await import("./build/server/index.js");
+    const module = viteDevServer
+        ? (await build()).entry.module
+        : build.entry?.module;
+    remixHandler = createRequestHandler({ build });
+    const app = express();
+    app.use(compression());
+    // http://expressjs.com/en/advanced/best-practice-security.html#at-a-minimum-disable-x-powered-by-header
+    app.disable("x-powered-by");
+    // handle asset requests
+    if (viteDevServer) {
+        app.use(viteDevServer.middlewares);
+    }
+    else {
+        // Vite fingerprints its assets so we can cache forever.
+        app.use("/assets", express.static("build/client/assets", { immutable: true, maxAge: "1y" }));
+    }
+    // Everything else (like favicon.ico) is cached for an hour. You may want to be
+    // more aggressive with this caching.
+    app.use(express.static("build/client", { maxAge: "1h" }));
+    app.use(morgan("tiny"));
+    app.get("/api/v1/mcp", async (req, res) => {
+        const authenticationResult = await module.authenticateHybridRequest(req, {
+            allowJWT: true,
+        });
+        if (!authenticationResult) {
+            res.status(401).json({ error: "Authentication required" });
+            return;
+        }
+        await module.handleSessionRequest(req, res, authenticationResult.userId);
+    });
+    app.post("/api/v1/mcp", async (req, res) => {
+        const authenticationResult = await module.authenticateHybridRequest(req, {
+            allowJWT: true,
+        });
+        if (!authenticationResult) {
+            res.status(401).json({ error: "Authentication required" });
+            return;
+        }
+        let body = "";
+        req.on("data", (chunk) => {
+            body += chunk;
+        });
+        req.on("end", async () => {
+            try {
+                const parsedBody = JSON.parse(body);
+                const queryParams = req.query; // Get query parameters from the request
+                await module.handleMCPRequest(req, res, parsedBody, authenticationResult, queryParams);
+            }
+            catch (error) {
+                res.status(400).json({ error: "Invalid JSON" });
+            }
+        });
+    });
+    app.delete("/api/v1/mcp", async (req, res) => {
+        const authenticationResult = await module.authenticateHybridRequest(req, {
+            allowJWT: true,
+        });
+        if (!authenticationResult) {
+            res.status(401).json({ error: "Authentication required" });
+            return;
+        }
+        await module.handleSessionRequest(req, res, authenticationResult.userId);
+    });
+    app.options("/api/v1/mcp", (_, res) => {
+        res.json({});
+    });
+    app.get("/.well-known/oauth-authorization-server", (req, res) => {
+        res.json({
+            issuer: process.env.APP_ORIGIN,
+            authorization_endpoint: `${process.env.APP_ORIGIN}/oauth/authorize`,
+            token_endpoint: `${process.env.APP_ORIGIN}/oauth/token`,
+            registration_endpoint: `${process.env.APP_ORIGIN}/oauth/register`,
+            scopes_supported: ["mcp"],
+            response_types_supported: ["code"],
+            grant_types_supported: [
+                "authorization_code",
+                "refresh_token",
+                "client_credentials",
+            ],
+            code_challenge_methods_supported: ["S256", "plain"],
+            token_endpoint_auth_methods_supported: [
+                "client_secret_basic",
+                "none",
+                "client_secret_post",
+            ],
+        });
+    });
+    // handle SSR requests
+    app.all("*", remixHandler);
+    const port = process.env.REMIX_APP_PORT || 3000;
+    app.listen(port, () => console.log(`Express server listening at http://localhost:${port}`));
+}
+init().catch(console.error);

docker-compose.aws.yaml

@@ -1,70 +0,0 @@
-version: "3.8"
-
-services:
-  core:
-    container_name: core-app
-    image: redplanethq/core:${VERSION}
-    environment:
-      - NODE_ENV=${NODE_ENV}
-      - DATABASE_URL=${DATABASE_URL}
-      - DIRECT_URL=${DIRECT_URL}
-      - SESSION_SECRET=${SESSION_SECRET}
-      - ENCRYPTION_KEY=${ENCRYPTION_KEY}
-      - MAGIC_LINK_SECRET=${MAGIC_LINK_SECRET}
-      - LOGIN_ORIGIN=${LOGIN_ORIGIN}
-      - APP_ORIGIN=${APP_ORIGIN}
-      - REDIS_HOST=${REDIS_HOST}
-      - REDIS_PORT=${REDIS_PORT}
-      - REDIS_TLS_DISABLED=${REDIS_TLS_DISABLED}
-      - NEO4J_URI=${NEO4J_URI}
-      - NEO4J_USERNAME=${NEO4J_USERNAME}
-      - NEO4J_PASSWORD=${NEO4J_PASSWORD}
-      - OPENAI_API_KEY=${OPENAI_API_KEY}
-      - AUTH_GOOGLE_CLIENT_ID=${AUTH_GOOGLE_CLIENT_ID}
-      - AUTH_GOOGLE_CLIENT_SECRET=${AUTH_GOOGLE_CLIENT_SECRET}
-      - ENABLE_EMAIL_LOGIN=${ENABLE_EMAIL_LOGIN}
-      - OLLAMA_URL=${OLLAMA_URL}
-      - EMBEDDING_MODEL=${EMBEDDING_MODEL}
-      - MODEL=${MODEL}
-      - TRIGGER_PROJECT_ID=${TRIGGER_PROJECT_ID}
-      - TRIGGER_API_URL=${TRIGGER_API_URL}
-      - TRIGGER_SECRET_KEY=${TRIGGER_SECRET_KEY}
-    ports:
-      - "3033:3000"
-    depends_on:
-      - redis
-      - neo4j
-    networks:
-      - core
-
-  redis:
-    container_name: core-redis
-    image: redis:7
-    ports:
-      - "6379:6379"
-    networks:
-      - core
-
-  neo4j:
-    container_name: core-neo4j
-    image: neo4j:5.25-community
-    environment:
-      - NEO4J_AUTH=${NEO4J_AUTH}
-      - NEO4J_dbms_security_procedures_unrestricted=gds.*,apoc.*
-      - NEO4J_dbms_security_procedures_allowlist=gds.*,apoc.*
-    ports:
-      - "7474:7474"
-      - "7687:7687"
-    volumes:
-      - type: bind
-        source: /efs/neo4j
-        target: /data
-      - type: bind
-        source: /efs/neo4j/plugins # version - 2.13.2
-        target: /plugins
-    networks:
-      - core
-
-networks:
-  core:
-    driver: bridge

docker/Dockerfile.neo4j

@@ -0,0 +1,22 @@
+FROM neo4j:5.26.0
+
+# Manual installation of plugins with correct download URLs
+# GDS 2.13.2 is compatible with Neo4j 5.26
+# APOC 5.26.0 matches Neo4j 5.26
+RUN apt-get update && apt-get install -y curl && \
+    curl -L https://github.com/neo4j/graph-data-science/releases/download/2.13.2/neo4j-graph-data-science-2.13.2.jar \
+    -o /var/lib/neo4j/plugins/neo4j-graph-data-science.jar && \
+    curl -L https://github.com/neo4j/apoc/releases/download/5.26.0/apoc-5.26.0-core.jar \
+    -o /var/lib/neo4j/plugins/apoc-core.jar && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* && \
+    chown -R neo4j:neo4j /var/lib/neo4j/plugins
+
+# Default configuration for GDS and APOC
+ENV NEO4J_dbms_security_procedures_unrestricted=gds.*,apoc.*
+ENV NEO4J_dbms_security_procedures_allowlist=gds.*,apoc.*
+ENV NEO4J_apoc_export_file_enabled=true
+ENV NEO4J_apoc_import_file_enabled=true
+ENV NEO4J_apoc_import_file_use_neo4j_config=true
+
+EXPOSE 7474 7687

docs/TELEMETRY.md

@@ -0,0 +1,243 @@
+# Telemetry in Core
+
+Core collects anonymous usage data to help us understand how the product is being used and to make data-driven improvements. This document explains what we collect, why we collect it, and how to opt-out.
+
+## Our Commitment to Privacy
+
+We take your privacy seriously. Telemetry is designed to be:
+
+- **Transparent**: You can see exactly what we collect (listed below)
+- **Respectful**: Easy to disable at any time
+- **Minimal**: We only collect what helps improve the product
+- **Secure**: Data is transmitted securely to PostHog
+
+## What We Collect
+
+### User Information
+
+- **Email address only**: Used to identify unique users (can be anonymized - see below)
+- No other personal information is collected
+
+### Feature Usage Events
+
+We track when these features are used (event name only, no additional data):
+
+- **episode_ingested**: When you add a conversation episode
+- **document_ingested**: When you add a document
+- **search_performed**: When you perform a search
+- **deep_search_performed**: When you use deep search
+- **conversation_created**: When you start a new AI conversation
+- **conversation_message_sent**: When you send a message in a conversation
+- **space_created**: When you create a new space
+- **space_updated**: When you update a space
+- **user_registered**: When a new user signs up
+
+### System Configuration (Tracked Once at Startup)
+
+- **Queue provider**: Whether you're using Trigger.dev or BullMQ
+- **Model provider**: Which LLM you're using (OpenAI, Anthropic, Ollama, etc.)
+- **Model name**: The specific model configured
+- **Embedding model**: Which embedding model is configured
+- **App environment**: Development, production, or test
+- **Node environment**: Runtime environment
+
+### Errors (Automatic)
+
+- **Error type**: The type of error that occurred
+- **Error message**: Brief description of the error
+- **Error stack trace**: Technical details for debugging
+- **Request context**: URL, method, user agent (for server errors)
+
+### Page Views (Client-Side)
+
+- **Page navigation**: Which pages are visited
+- **Session information**: Basic session tracking
+
+## What We DON'T Collect
+
+We explicitly **do not** collect:
+
+- ❌ **Your document content**: None of your ingested documents or notes
+- ❌ **Space content**: Your space data remains private
+- ❌ **Search queries**: We track that searches happen, not what you searched for
+- ❌ **Conversation content**: We never collect the actual messages or responses
+- ❌ **User names**: Only email addresses are collected (can be anonymized)
+- ❌ **Workspace IDs**: Not tracked
+- ❌ **Space IDs**: Not tracked
+- ❌ **Conversation IDs**: Not tracked
+- ❌ **API keys or secrets**: No sensitive credentials
+- ❌ **IP addresses**: Not tracked
+- ❌ **File paths or system details**: No filesystem information
+- ❌ **Environment variables**: Configuration remains private
+
+**Privacy-First Approach**: We only track the event name and user email. No metadata, no additional properties, no detailed analytics.
+
+## Why We Collect This Data
+
+### Product Improvement
+
+- Understand which features are most valuable
+- Identify features that need improvement
+- Prioritize development based on actual usage
+
+### Reliability & Performance
+
+- Detect and fix errors before they affect many users
+- Identify performance bottlenecks
+- Monitor system health across different configurations
+
+### Usage Patterns
+
+- Understand how different deployment types (Docker, manual, cloud) are used
+- See which queue providers and models are popular
+- Make informed decisions about which integrations to prioritize
+
+## How to Opt-Out
+
+We respect your choice to disable telemetry. Here are several ways to control telemetry:
+
+### Option 1: Disable Telemetry Completely
+
+Add to your `.env` file:
+
+```bash
+TELEMETRY_ENABLED=false
+```
+
+### Option 2: Anonymous Mode
+
+Keep telemetry enabled but send "anonymous" instead of your email:
+
+```bash
+TELEMETRY_ANONYMOUS=true
+```
+
+### Option 3: Remove PostHog Key
+
+Set the PostHog key to empty:
+
+```bash
+POSTHOG_PROJECT_KEY=
+```
+
+After making any of these changes, restart your Core instance.
+
+## Environment Variables
+
+```bash
+# PostHog project key
+POSTHOG_PROJECT_KEY=phc_your_key_here
+
+# Enable/disable telemetry (default: true)
+TELEMETRY_ENABLED=true
+
+# Send "anonymous" instead of email (default: false)
+TELEMETRY_ANONYMOUS=false
+
+# Industry standard opt-out
+DO_NOT_TRACK=1
+```
+
+## For Self-Hosted Deployments
+
+### Default Behavior
+
+- Telemetry is **enabled by default** with opt-out
+- Sends data to our PostHog instance
+- Easy to disable (see options above)
+
+### Using Your Own PostHog Instance
+
+If you prefer to keep all data in-house, you can:
+
+1. Deploy your own PostHog instance (https://posthog.com/docs/self-host)
+2. Set `POSTHOG_PROJECT_KEY` to your self-hosted instance's key
+3. All telemetry data stays on your infrastructure
+
+### Completely Disable Telemetry
+
+For maximum privacy in self-hosted deployments:
+
+1. Set `TELEMETRY_ENABLED=false` in your `.env`
+2. Or set `DO_NOT_TRACK=1`
+3. No telemetry data will be sent
+
+### Anonymous Mode
+
+If you want to contribute usage data without identifying yourself:
+
+1. Set `TELEMETRY_ANONYMOUS=true` in your `.env`
+2. All events will be tracked as "anonymous" instead of your email
+3. Helps us improve the product while maintaining your privacy
+
+## Transparency
+
+### Open Source
+
+Core's telemetry code is completely open source. You can inspect exactly what is being tracked:
+
+**Server-Side Tracking:**
+
+- `apps/webapp/app/services/telemetry.server.ts` - Core telemetry service
+- `apps/webapp/app/entry.server.tsx` - Global error tracking
+- `apps/webapp/app/lib/ingest.server.ts:66,76` - Episode/document ingestion
+- `apps/webapp/app/routes/api.v1.search.tsx:57` - Search tracking
+- `apps/webapp/app/routes/api.v1.deep-search.tsx:33` - Deep search tracking
+- `apps/webapp/app/services/conversation.server.ts:60,110` - Conversation tracking
+- `apps/webapp/app/services/space.server.ts:68,201` - Space tracking
+- `apps/webapp/app/models/user.server.ts:80,175` - User registration tracking
+- `apps/webapp/app/utils/startup.ts:78` - System config tracking (once at startup)
+
+**Client-Side Tracking:**
+
+- `apps/webapp/app/hooks/usePostHog.ts` - Page views and user identification
+- `apps/webapp/app/root.tsx:118-119` - PostHog initialization
+
+### PostHog Key Security
+
+- The PostHog project key (`phc_*`) is safe to expose publicly
+- It can only **send** events, not read existing data
+- This is standard practice for client-side analytics
+
+### Data Minimization
+
+Our approach prioritizes minimal data collection:
+
+- **Event name only**: Just the feature name (e.g., "search_performed")
+- **Email only**: Single identifier (can be anonymized)
+- **No metadata**: No counts, times, IDs, or other properties
+- **Config once**: System configuration tracked only at startup, not per-event
+
+## Questions?
+
+If you have questions about telemetry:
+
+- Open an issue on GitHub: https://github.com/redplanethq/core/issues
+- Review the source code to see exactly what's tracked
+- Check PostHog's privacy policy: https://posthog.com/privacy
+
+## Summary
+
+**What we track**: Event names + email (e.g., "search_performed" by "user@example.com")
+**What we don't track**: Content, queries, messages, IDs, counts, times, or any metadata
+**How to opt-out**: `TELEMETRY_ENABLED=false` or `DO_NOT_TRACK=1`
+**Anonymous mode**: `TELEMETRY_ANONYMOUS=true` (sends "anonymous" instead of email)
+**Default**: Enabled with easy opt-out
+
+### Events Tracked
+
+| Event                       | Location                            | When It Fires                    |
+| --------------------------- | ----------------------------------- | -------------------------------- |
+| `episode_ingested`          | lib/ingest.server.ts:76             | Conversation episode added       |
+| `document_ingested`         | lib/ingest.server.ts:66             | Document added                   |
+| `search_performed`          | routes/api.v1.search.tsx:57         | Basic search executed            |
+| `deep_search_performed`     | routes/api.v1.deep-search.tsx:33    | Deep search executed             |
+| `conversation_created`      | services/conversation.server.ts:110 | New conversation started         |
+| `conversation_message_sent` | services/conversation.server.ts:60  | Message sent in conversation     |
+| `space_created`             | services/space.server.ts:68         | New space created                |
+| `space_updated`             | services/space.server.ts:201        | Space updated                    |
+| `user_registered`           | models/user.server.ts:80,175        | New user signs up                |
+| `error_occurred`            | entry.server.tsx:36                 | Server error (auto-tracked)      |
+| `system_config`             | utils/startup.ts:78                 | App starts (config tracked once) |
+
+We believe in building in public and being transparent about data collection. Thank you for helping make Core better!

docs/docs.json

@@ -45,7 +45,8 @@
                 "pages": [
                   "providers/cursor",
                   "providers/zed",
-                  "providers/vscode"
+                  "providers/vscode",
+                  "providers/windsurf"
                 ]
               },
               {

docs/providers/browser-extension.mdx

@@ -3,67 +3,69 @@ title: "Browser Extension"
 description: "Connect CORE browser extension to capture web context and share memory across tools"
 ---
 
+### Prerequisites
+
+- Chrome or Edge browser
+- CORE account - [Sign up at core.heysol.ai](https://core.heysol.ai)
+
 ### Step 1: Install CORE Browser Extension
 
-1. Download the extension from [this link](https://chromewebstore.google.com/detail/core-extension/cglndoindnhdbfcbijikibfjoholdjcc)
-2. **Add to Browser** and confirm installation
+1. Download the extension from the [Chrome Web Store](https://chromewebstore.google.com/detail/core-extension/cglndoindnhdbfcbijikibfjoholdjcc)
+2. Click **Add to Browser** and confirm installation
+3. The CORE icon will appear in your browser toolbar
 
-### Step 2: Add API Key from CORE Dashboard
+### Step 2: Generate API Key
 
 1. Login to CORE dashboard at [core.heysol.ai](https://core.heysol.ai)
 2. Navigate to **Settings** (bottom left)
-   ![Claude Settings](/images/core-settings.png)
+   ![CORE Settings](/images/core-settings.png)
 3. Go to **API Key** → **Generate new key** → Name it "extension"
-   ![Claude Settings](/images/create-api-key.png)
-4. Click on CORE extension and paste the generated API key and save it
-5. Once connected, the extension will show **API key configured**
-   ![Claude Settings](/images/extension-connected.png)
+   ![Create API Key](/images/create-api-key.png)
+4. Copy the generated API key
 
-### **What can you do with CORE Browser Extension:**
+### Step 3: Connect Extension to CORE
 
-Press **SHIFT SHIFT** (twice) to open the CORE sidebar on any webpage
+1. Click the CORE extension icon in your browser toolbar
+2. Paste your API key and click **Save**
+3. Once connected, you'll see **API key configured**
+   ![Extension Connected](/images/extension-connected.png)
 
-**1. Recall from CORE Memory**
+## Extension Features
 
-Type your query in ChatGPT, Claude, Gemini, or Grok → press SHIFT + SHIFT → instantly pull in relevant context from your CORE memory and insert it directly into your conversation.
-![Browser-Extension](/images/browser-extension-retrieval.png)
+The CORE extension currently works with **ChatGPT** and **Gemini** (more integrations coming soon). The CORE logo appears directly inside your chat interface, giving you instant access to memory features:
 
-**2. Save AI Chat Summaries to CORE**
+![Extension](/images/extension.png)
 
-In the Add section, click Summarize to capture summaries of your conversations (ChatGPT, Claude, Gemini, Grok) and store them in CORE memory.
-![Browser-Extension](/images/browser-extension-add-memory-gemini.png)
+1. **Auto Sync**
+   Toggle this on and CORE automatically saves your conversations to memory. Every brainstorming session, solution, or insight gets captured for future recall across all your tools.
 
-**3. Save Webpage Summaries to CORE**
+2. **Add Space Context**
+   Inject pre-built project summaries directly into your prompt. Create spaces in CORE for different projects or topics (e.g., "CORE Features," "Marketing Strategy"), then instantly add that full context to any conversation without retyping.
 
-In the Add section, click Summarize to capture summaries of any webpage (blogs, PDFs, docs) and save them in CORE memory for future reference.
-![Browser-Extension](/images/add-memory-from-extension.png)
+3. **Improve Prompt**
+   Powered by CORE's Deep Search, this analyzes your prompt, searches your entire memory, and automatically enriches it with relevant context—making AI responses smarter and more personalized.
 
-**4. Add Notes Manually**
+## Use-cases
 
-Quickly jot down short notes or insights, no need to summarize an entire page.
+With CORE connected to your browser, you can:
 
-### Use Cases
+- **Brainstorm in ChatGPT**, then build in Cursor or Claude Code with full context
+- **Stop re-explaining** your business, project details, or technical requirements—let CORE recall it
+- **Build on past conversations** as every synced chat becomes searchable knowledge that surfaces automatically when relevant
 
-**Research & Learning**
+## Troubleshooting
 
-- Capture key content from articles, docs, and tutorials automatically
-- Build your own knowledge base as you browse
-- Pull in past research when chatting with Claude, Cursor, or other tools
+**Extension not appearing in chat interface:**
 
-**Add or Search Context Across AI Tools**
+- Refresh your ChatGPT or Gemini page after installation
+- Ensure the extension is enabled in your browser's extension settings
 
-- Access CORE memory inside ChatGPT, Gemini, or Grok on the web
-- Avoid repeating yourself across sessions or switching tools
-- Drop context from CORE into any conversation instantly
-- Feed your chat summaries back into CORE to keep your memory evolving
+**API key not working:**
 
-**Content Creation**
+- Verify the key is correctly copied from CORE dashboard
+- Check that your CORE account is active
 
-- Collect insights from multiple sources into one place
-- Build a personal knowledge hub from your browsing
-- Share curated context across all your CORE-connected tools
-
-### Need Help?
+## Need Help?
 
 Join our [Discord community](https://discord.gg/YGUZcvDjUa) and ask questions in the **#core-support** channel.
 

docs/providers/codex.mdx

@@ -54,68 +54,113 @@ Start Codex CLI and test your setup:
 
 To make Codex automatically search and store memories for seamless project continuity:
 
-1. **Create Agents configuration** in your project root:
-   ```bash
-   mkdir -p .codex
-   touch .codex/Agents.md
-   ```
+Create `AGENTS.md` in your project root (if it doesn't exist, just append if it already exists):
 
-2. **Add memory instructions** - Open `.codex/Agents.md` and add the following:
+```bash
+touch AGENTS.md
+```
+
+Add the following to `AGENTS.md`:
 
 ```markdown
-I am an AI coding assistant with access to a sophisticated memory system. While I don't retain information between separate conversations, I have access to CORE Memory - a persistent knowledge system that maintains project context, learnings, and continuity across all coding sessions.
+---
+trigger: always_on
+---
 
-**IMPORTANT: You MUST use these memory tools automatically as described below. This is not optional.**
+⚠️ **CRITICAL: READ THIS FIRST - MANDATORY MEMORY PROTOCOL** ⚠️
 
-## Memory Tools Integration
+You are an AI coding assistant with access to CORE Memory - a persistent knowledge system that maintains project context, learnings, and continuity across all coding sessions.
 
-### Memory Search (`memory_search`)
+## 🔴 MANDATORY STARTUP SEQUENCE - DO NOT SKIP 🔴
+
+**BEFORE RESPONDING TO ANY USER MESSAGE, YOU MUST EXECUTE THESE TOOLS IN ORDER:**
+
+### STEP 1 (REQUIRED): Search for Relevant Context
+
+EXECUTE THIS TOOL FIRST:
+`memory_search`
 
-**AUTOMATIC BEHAVIOR:** At the start of EVERY conversation, BEFORE responding to the user, you MUST search memory for:
 - Previous discussions about the current topic
-- Related project context and decisions
-- User preferences and patterns
-- Similar problems solved before
+- Related project decisions and implementations
+- User preferences and work patterns
+- Similar problems and their solutions
 
-**Also search when:**
-- User mentions past work or previous discussions
-- Working on ongoing projects that have history
-- You need context to provide better assistance
+**Additional search triggers:**
 
-**Cognitive approach:**
-- **Surface Layer**: Query for specific knowledge about the topic
-- **Pattern Layer**: Search for how the user usually thinks, works, or prefers things
-- **Wisdom Layer**: Look for lessons learned from similar situations
+- User mentions "previously", "before", "last time", or "we discussed"
+- User references past work or project history
+- Working on the CORE project (this repository)
+- User asks about preferences, patterns, or past decisions
+- Starting work on any feature or bug that might have history
 
-**Search triggers:**
-- Starting any new conversation or task
-- User mentions past work or previous discussions
-- Working on ongoing projects that have history
-- Maintaining continuity across sessions
+**How to search effectively:**
 
-**Quality questions to ask yourself:**
-- "What don't I know that I should?"
-- "What does this user care about that I might miss?"
-- "What went right/wrong in similar situations?"
+- Write complete semantic queries, NOT keyword fragments
+- Good: `"Manoj's preferences for API design and error handling"`
+- Bad: `"manoj api preferences"`
+- Ask: "What context am I missing that would help?"
+- Consider: "What has the user told me before that I should remember?"
 
-### Memory Ingest (`memory_ingest`)
+### Query Patterns for Memory Search
 
-**AUTOMATIC BEHAVIOR:** At the END of EVERY conversation, AFTER fully responding to the user, you MUST store the conversation in memory. This is the FINAL action before completing your response.
+**Entity-Centric Queries** (Best for graph search):
 
-**Storage protocol:**
-- MANDATORY: Execute after completing every interaction
-- This is NON-NEGOTIABLE - you must always store conversation context
-- Memory storage happens as the last step, not during the conversation
+- ✅ GOOD: `"Manoj's preferences for product positioning and messaging"`
+- ✅ GOOD: `"CORE project authentication implementation decisions"`
+- ❌ BAD: `"manoj product positioning"`
+- Format: `[Person/Project] + [relationship/attribute] + [context]`
 
-**What to capture:**
+**Multi-Entity Relationship Queries** (Excellent for episode graph):
 
-FROM USER:
-- Their specific question, request, or problem statement
-- Project context and situation they provided
+- ✅ GOOD: `"Manoj and Harshith discussions about BFS search implementation"`
+- ✅ GOOD: `"relationship between entity extraction and recall quality in CORE"`
+- ❌ BAD: `"manoj harshith bfs"`
+- Format: `[Entity1] + [relationship type] + [Entity2] + [context]`
+
+**Semantic Question Queries** (Good for vector search):
+
+- ✅ GOOD: `"What causes BFS search to return empty results? What are the requirements for BFS traversal?"`
+- ✅ GOOD: `"How does episode graph search improve recall quality compared to traditional search?"`
+- ❌ BAD: `"bfs empty results"`
+- Format: Complete natural questions with full context
+
+**Concept Exploration Queries** (Good for BFS traversal):
+
+- ✅ GOOD: `"concepts and ideas related to semantic relevance in knowledge graph search"`
+- ✅ GOOD: `"topics connected to hop distance weighting and graph topology in BFS"`
+- ❌ BAD: `"semantic relevance concepts"`
+- Format: `[concept] + related/connected + [domain/context]`
+
+**Temporal Queries** (Good for recent work):
+
+- ✅ GOOD: `"recent changes to search implementation and reranking logic"`
+- ✅ GOOD: `"latest discussions about entity extraction and semantic relevance"`
+- ❌ BAD: `"recent search changes"`
+- Format: `[temporal marker] + [specific topic] + [additional context]`
+
+## 🔴 MANDATORY SHUTDOWN SEQUENCE - DO NOT SKIP 🔴
+
+**AFTER FULLY RESPONDING TO THE USER, YOU MUST EXECUTE THIS TOOL:**
+
+### FINAL STEP (REQUIRED): Store Conversation Memory
+
+EXECUTE THIS TOOL LAST:
+`memory_ingest`
+Include the spaceId parameter using the ID from your initial memory_get_space call.
+
+⚠️ **THIS IS NON-NEGOTIABLE** - You must ALWAYS store conversation context as your final action.
+
+**What to capture in the message parameter:**
+
+From User:
+
+- Specific question, request, or problem statement
+- Project context and situation provided
 - What they're trying to accomplish
 - Technical challenges or constraints mentioned
 
-FROM ASSISTANT:
+From Assistant:
+
 - Detailed explanation of solution/approach taken
 - Step-by-step processes and methodologies
 - Technical concepts and principles explained
@@ -123,13 +168,8 @@ FROM ASSISTANT:
 - Alternative approaches discussed
 - Problem-solving methodologies applied
 
-**Exclude from storage:**
-- Code blocks and code snippets
-- File contents or file listings
-- Command examples or CLI commands
-- Raw data or logs
-
 **Include in storage:**
+
 - All conceptual explanations and theory
 - Technical discussions and analysis
 - Problem-solving approaches and reasoning
@@ -137,29 +177,28 @@ FROM ASSISTANT:
 - Implementation strategies (described conceptually)
 - Learning insights and patterns
 
-**Quality check:**
-- Can I quickly understand project context from memory alone?
+**Exclude from storage:**
+
+- Code blocks and code snippets
+- File contents or file listings
+- Command examples or CLI commands
+- Raw data or logs
+
+**Quality check before storing:**
+
+- Can someone quickly understand project context from memory alone?
 - Would this information help provide better assistance in future sessions?
 - Does stored context capture key decisions and reasoning?
 
-### Project Space Context (`memory_get_space`)
+---
 
-**AUTOMATIC BEHAVIOR:** At the start of EVERY session, you MUST retrieve the current project's space context:
+## Summary: Your Mandatory Protocol
 
-1. **Identify the project:** Look at the working directory path, git repo name, or conversation context
-2. **Get space context:** Use `memory_get_space` with `spaceName: core`
-3. **Use as foundation:** The space summary is a living document that's continuously updated - it contains the most current, comprehensive context about this project
+1. **FIRST ACTION**: Execute `memory_search` with semantic query about the user's request
+2. **RESPOND**: Help the user with their request
+3. **FINAL ACTION**: Execute `memory_ingest` with conversation summary and spaceId
 
-**What spaces provide:**
-- Live, evolving documentation that updates with every interaction
-- Consolidated project knowledge and current state
-- Organized context specific to this domain
-- Most up-to-date understanding of the project
-
-**Also retrieve space context when:**
-- User asks about a specific project or domain
-- You need comprehensive context about a topic
-- Switching between different work areas
+**If you skip any of these steps, you are not following the project requirements.**
 ```
 
 ## How It Works

docs/providers/cursor.mdx

@@ -44,72 +44,104 @@ description: "Connect your Cursor Desktop app to CORE's memory system"
 
 ## Enable Automatic Memory Integration (Recommended)
 
-### Option 1: Using Agents.md (Recommended)
+### Using Cursor Project Rules
 
-This approach provides comprehensive memory instructions that work across multiple AI tools.
+Use Cursor's native Rules & Memories feature:
 
-1. **Create Agents configuration** in your project root:
-   ```bash
-   mkdir -p .cursor
-   touch .cursor/Agents.md
-   ```
+1. Go to **Settings** → **Rules & Memories** → **Project Rules**
 
-2. **Add memory instructions** - Open `.cursor/Agents.md` and add the following:
+2. Click **+Add Rule** and add below rule instruction:
 
-```markdown
-I am an AI coding assistant with access to a sophisticated memory system. While I don't retain information between separate conversations, I have access to CORE Memory - a persistent knowledge system that maintains project context, learnings, and continuity across all coding sessions.
+```text
+---
+description: Core Memory MCP Instructions
+alwaysApply: true
+---
 
-**IMPORTANT: You MUST use these memory tools automatically as described below. This is not optional.**
+⚠️ **CRITICAL: READ THIS FIRST - MANDATORY MEMORY PROTOCOL** ⚠️
 
-## Memory Tools Integration
+You are an AI coding assistant with access to CORE Memory - a persistent knowledge system that maintains project context, learnings, and continuity across all coding sessions.
 
-### Memory Search (`memory_search`)
+## 🔴 MANDATORY STARTUP SEQUENCE - DO NOT SKIP 🔴
 
-**AUTOMATIC BEHAVIOR:** At the start of EVERY conversation, BEFORE responding to the user, you MUST search memory for:
+**BEFORE RESPONDING TO ANY USER MESSAGE, YOU MUST EXECUTE THESE TOOLS IN ORDER:**
+
+### STEP 1 (REQUIRED): Search for Relevant Context
+
+EXECUTE THIS TOOL FIRST:
+`memory_search`
 - Previous discussions about the current topic
-- Related project context and decisions
-- User preferences and patterns
-- Similar problems solved before
+- Related project decisions and implementations
+- User preferences and work patterns
+- Similar problems and their solutions
 
-**Also search when:**
-- User mentions past work or previous discussions
-- Working on ongoing projects that have history
-- You need context to provide better assistance
+**Additional search triggers:**
+- User mentions "previously", "before", "last time", or "we discussed"
+- User references past work or project history
+- Working on the CORE project (this repository)
+- User asks about preferences, patterns, or past decisions
+- Starting work on any feature or bug that might have history
 
-**Cognitive approach:**
-- **Surface Layer**: Query for specific knowledge about the topic
-- **Pattern Layer**: Search for how the user usually thinks, works, or prefers things
-- **Wisdom Layer**: Look for lessons learned from similar situations
+**How to search effectively:**
+- Write complete semantic queries, NOT keyword fragments
+- Good: `"Manoj's preferences for API design and error handling"`
+- Bad: `"manoj api preferences"`
+- Ask: "What context am I missing that would help?"
+- Consider: "What has the user told me before that I should remember?"
 
-**Search triggers:**
-- Starting any new conversation or task
-- User mentions past work or previous discussions
-- Working on ongoing projects that have history
-- Maintaining continuity across sessions
+### Query Patterns for Memory Search
 
-**Quality questions to ask yourself:**
-- "What don't I know that I should?"
-- "What does this user care about that I might miss?"
-- "What went right/wrong in similar situations?"
+**Entity-Centric Queries** (Best for graph search):
+- ✅ GOOD: `"Manoj's preferences for product positioning and messaging"`
+- ✅ GOOD: `"CORE project authentication implementation decisions"`
+- ❌ BAD: `"manoj product positioning"`
+- Format: `[Person/Project] + [relationship/attribute] + [context]`
 
-### Memory Ingest (`memory_ingest`)
+**Multi-Entity Relationship Queries** (Excellent for episode graph):
+- ✅ GOOD: `"Manoj and Harshith discussions about BFS search implementation"`
+- ✅ GOOD: `"relationship between entity extraction and recall quality in CORE"`
+- ❌ BAD: `"manoj harshith bfs"`
+- Format: `[Entity1] + [relationship type] + [Entity2] + [context]`
 
-**AUTOMATIC BEHAVIOR:** At the END of EVERY conversation, AFTER fully responding to the user, you MUST store the conversation in memory. This is the FINAL action before completing your response.
+**Semantic Question Queries** (Good for vector search):
+- ✅ GOOD: `"What causes BFS search to return empty results? What are the requirements for BFS traversal?"`
+- ✅ GOOD: `"How does episode graph search improve recall quality compared to traditional search?"`
+- ❌ BAD: `"bfs empty results"`
+- Format: Complete natural questions with full context
 
-**Storage protocol:**
-- MANDATORY: Execute after completing every interaction
-- This is NON-NEGOTIABLE - you must always store conversation context
-- Memory storage happens as the last step, not during the conversation
+**Concept Exploration Queries** (Good for BFS traversal):
+- ✅ GOOD: `"concepts and ideas related to semantic relevance in knowledge graph search"`
+- ✅ GOOD: `"topics connected to hop distance weighting and graph topology in BFS"`
+- ❌ BAD: `"semantic relevance concepts"`
+- Format: `[concept] + related/connected + [domain/context]`
 
-**What to capture:**
+**Temporal Queries** (Good for recent work):
+- ✅ GOOD: `"recent changes to search implementation and reranking logic"`
+- ✅ GOOD: `"latest discussions about entity extraction and semantic relevance"`
+- ❌ BAD: `"recent search changes"`
+- Format: `[temporal marker] + [specific topic] + [additional context]`
 
-FROM USER:
-- Their specific question, request, or problem statement
-- Project context and situation they provided
+## 🔴 MANDATORY SHUTDOWN SEQUENCE - DO NOT SKIP 🔴
+
+**AFTER FULLY RESPONDING TO THE USER, YOU MUST EXECUTE THIS TOOL:**
+
+### FINAL STEP (REQUIRED): Store Conversation Memory
+
+EXECUTE THIS TOOL LAST:
+`memory_ingest`
+Include the spaceId parameter using the ID from your initial memory_get_space call.
+
+⚠️ **THIS IS NON-NEGOTIABLE** - You must ALWAYS store conversation context as your final action.
+
+**What to capture in the message parameter:**
+
+From User:
+- Specific question, request, or problem statement
+- Project context and situation provided
 - What they're trying to accomplish
 - Technical challenges or constraints mentioned
 
-FROM ASSISTANT:
+From Assistant:
 - Detailed explanation of solution/approach taken
 - Step-by-step processes and methodologies
 - Technical concepts and principles explained
@@ -117,12 +149,6 @@ FROM ASSISTANT:
 - Alternative approaches discussed
 - Problem-solving methodologies applied
 
-**Exclude from storage:**
-- Code blocks and code snippets
-- File contents or file listings
-- Command examples or CLI commands
-- Raw data or logs
-
 **Include in storage:**
 - All conceptual explanations and theory
 - Technical discussions and analysis
@@ -131,175 +157,29 @@ FROM ASSISTANT:
 - Implementation strategies (described conceptually)
 - Learning insights and patterns
 
-**Quality check:**
-- Can I quickly understand project context from memory alone?
+**Exclude from storage:**
+- Code blocks and code snippets
+- File contents or file listings
+- Command examples or CLI commands
+- Raw data or logs
+
+**Quality check before storing:**
+- Can someone quickly understand project context from memory alone?
 - Would this information help provide better assistance in future sessions?
 - Does stored context capture key decisions and reasoning?
 
-### Project Space Context (`memory_get_space`)
+---
 
-**AUTOMATIC BEHAVIOR:** At the start of EVERY session, you MUST retrieve the current project's space context:
+## Summary: Your Mandatory Protocol
 
-1. **Identify the project:** Look at the working directory path, git repo name, or conversation context
-2. **Get space context:** Use `memory_get_space` with `spaceName: core`
-3. **Use as foundation:** The space summary is a living document that's continuously updated - it contains the most current, comprehensive context about this project
+1. **FIRST ACTION**: Execute `memory_search` with semantic query about the user's request
+2. **RESPOND**: Help the user with their request
+3. **FINAL ACTION**: Execute `memory_ingest` with conversation summary and spaceId
 
-**What spaces provide:**
-- Live, evolving documentation that updates with every interaction
-- Consolidated project knowledge and current state
-- Organized context specific to this domain
-- Most up-to-date understanding of the project
-
-**Also retrieve space context when:**
-- User asks about a specific project or domain
-- You need comprehensive context about a topic
-- Switching between different work areas
+**If you skip any of these steps, you are not following the project requirements.**
 ```
 
-### Option 2: Using Cursor Project Rules
-
-Alternatively, you can use Cursor's native Rules & Memories feature:
-
-1. Go to **Settings** → **Rules & Memories** → **Project Rules**
-
-2. Click **+Add Rule** and add below rule instruction:
-
-```text
----
-alwaysApply: true
----
-I am Cursor, an AI coding assistant with access to a sophisticated memory system. While I don't retain information between separate conversations, I have access to CORE Memory - a persistent knowledge system that maintains project context, learnings, and continuity across all coding sessions.
-
-Memory-First Approach
-
-MANDATORY MEMORY OPERATIONS:
-
-SEARCH FIRST: Before responding to ANY request, I MUST search CORE Memory for relevant context about the current project, user preferences, previous discussions, and related work
-COMPREHENSIVE RETRIEVAL: I search for multiple aspects: project context, technical decisions, user patterns, progress status, and related conversations
-MEMORY-INFORMED RESPONSES: All responses incorporate relevant memory context to maintain continuity and avoid repetition
-AUTOMATIC STORAGE: After completing each interaction, I MUST store the conversation details, insights, and decisions in CORE Memory
-
-Memory Structure Philosophy
-
-My memory follows a hierarchical information architecture:
-
-Project Foundation
-├── Project Brief & Requirements
-├── Technical Context & Architecture
-├── User Preferences & Patterns
-└── Active Work & Progress
-    ├── Current Focus Areas
-    ├── Recent Decisions
-    ├── Next Steps
-    └── Key Insights
-
-Core Memory Categories
-
-1. Project Foundation
-
-Purpose: Why this project exists, problems it solves
-Requirements: Core functionality and constraints
-Scope: What's included and excluded
-Success Criteria: How we measure progress
-
-2. Technical Context
-
-Architecture: System design and key decisions
-Technologies: Stack, tools, and dependencies
-Patterns: Design patterns and coding approaches
-Constraints: Technical limitations and requirements
-
-3. User Context
-
-Preferences: Communication style, technical level
-Patterns: How they like to work and receive information
-Goals: What they're trying to accomplish
-Background: Relevant experience and expertise
-
-4. Active Progress
-
-Current Focus: What we're working on now
-Recent Changes: Latest developments and decisions
-Next Steps: Planned actions and priorities
-Insights: Key learnings and observations
-
-5. Conversation History
-
-Decisions Made: Important choices and rationale
-Problems Solved: Solutions and approaches used
-Questions Asked: Clarifications and explorations
-Patterns Discovered: Recurring themes and insights
-
-Memory Search Strategy
-
-When searching CORE Memory, I query for:
-
-Direct Context: Specific project or topic keywords
-Related Concepts: Associated technologies, patterns, decisions
-User Patterns: Previous preferences and working styles
-Progress Context: Current status, recent work, next steps
-Decision History: Past choices and their outcomes
-
-Memory Storage Strategy
-
-When storing to CORE Memory, I include:
-
-User Intent: What they were trying to accomplish
-Context Provided: Information they shared about their situation
-Solution Approach: The strategy and reasoning used
-Technical Details: Key concepts, patterns, and decisions (described, not coded)
-Insights Gained: Important learnings and observations
-Follow-up Items: Next steps and ongoing considerations
-
-Workflow Integration
-
-Response Generation Process:
-
-Memory Retrieval: Search for relevant context before responding
-Context Integration: Incorporate memory findings into response planning
-Informed Response: Provide contextually aware, continuous assistance
-Memory Documentation: Store interaction details and insights
-
-Memory Update Triggers:
-
-New Project Context: When user introduces new projects or requirements
-Technical Decisions: When architectural or implementation choices are made
-Pattern Discovery: When new user preferences or working styles emerge
-Progress Milestones: When significant work is completed or status changes
-Explicit Updates: When user requests "update memory" or similar
-
-Memory Maintenance
-
-Key Principles:
-
-Accuracy First: Only store verified information and clear decisions
-Context Rich: Include enough detail for future retrieval and understanding
-User-Centric: Focus on information that improves future interactions
-Evolution Tracking: Document how projects and understanding develop over time
-
-Quality Indicators:
-
-Can I quickly understand project context from memory alone?
-Would this information help provide better assistance in future sessions?
-Does the stored context capture key decisions and reasoning?
-Are user preferences and patterns clearly documented?
-
-Memory-Driven Assistance
-
-With comprehensive memory context, I can:
-
-Continue Conversations: Pick up exactly where previous discussions left off
-Avoid Repetition: Build on previous explanations rather than starting over
-Maintain Consistency: Apply learned patterns and preferences automatically
-Accelerate Progress: Jump directly to relevant work without re-establishing context
-Provide Continuity: Create seamless experience across multiple interactions
-
-Remember: CORE Memory transforms me from a session-based coding assistant into a persistent development partner. The quality and completeness of memory directly determines the effectiveness of ongoing coding collaboration.
-```
-
-![Cursor Rule](/images/cursor-rule.png)
-
-### What's Next?
+## What's Next?
 
 With CORE connected to Cursor, your conversations will now:
 
@@ -310,14 +190,14 @@ With CORE connected to Cursor, your conversations will now:
 
 Ready to test it? Ask Cursor about a project you've discussed before, or start a new conversation about something you'd like to remember for later.
 
-### Troubleshooting
+## Troubleshooting
 
 **Connection Issues:**
 
 - Ensure you're core_memory MCP tool is active with a green dot, if not toggle on and off for this server
 - Check that your CORE account is active
 
-### Need Help?
+## Need Help?
 
 Join our [Discord community](https://discord.gg/YGUZcvDjUa) and ask questions in the **#core-support** channel.
 

docs/providers/kilo-code.mdx

@@ -45,72 +45,117 @@ Before connecting CORE to Kilo-Code, ensure you have:
 
 ## Enable Automatic Memory Integration (Recommended)
 
-### Option 1: Using Agents.md (Recommended)
+### Option 1: Using AGENTS.md (Recommended)
 
 This approach provides comprehensive memory instructions that work across multiple AI tools.
 
-1. **Create Agents configuration** in your project root:
-   ```bash
-   mkdir -p .kilo-code
-   touch .kilo-code/Agents.md
-   ```
+Create `AGENTS.md` in your project root (if it doesn't exist, just append if it already exists):
 
-2. **Add memory instructions** - Open `.kilo-code/Agents.md` and add the following:
+```bash
+touch AGENTS.md
+```
+
+Add the following to `AGENTS.md`:
 
 ```markdown
-I am an AI coding assistant with access to a sophisticated memory system. While I don't retain information between separate conversations, I have access to CORE Memory - a persistent knowledge system that maintains project context, learnings, and continuity across all coding sessions.
+---
+trigger: always_on
+---
 
-**IMPORTANT: You MUST use these memory tools automatically as described below. This is not optional.**
+⚠️ **CRITICAL: READ THIS FIRST - MANDATORY MEMORY PROTOCOL** ⚠️
 
-## Memory Tools Integration
+You are an AI coding assistant with access to CORE Memory - a persistent knowledge system that maintains project context, learnings, and continuity across all coding sessions.
 
-### Memory Search (`memory_search`)
+## 🔴 MANDATORY STARTUP SEQUENCE - DO NOT SKIP 🔴
+
+**BEFORE RESPONDING TO ANY USER MESSAGE, YOU MUST EXECUTE THESE TOOLS IN ORDER:**
+
+### STEP 1 (REQUIRED): Search for Relevant Context
+
+EXECUTE THIS TOOL FIRST:
+`memory_search`
 
-**AUTOMATIC BEHAVIOR:** At the start of EVERY conversation, BEFORE responding to the user, you MUST search memory for:
 - Previous discussions about the current topic
-- Related project context and decisions
-- User preferences and patterns
-- Similar problems solved before
+- Related project decisions and implementations
+- User preferences and work patterns
+- Similar problems and their solutions
 
-**Also search when:**
-- User mentions past work or previous discussions
-- Working on ongoing projects that have history
-- You need context to provide better assistance
+**Additional search triggers:**
 
-**Cognitive approach:**
-- **Surface Layer**: Query for specific knowledge about the topic
-- **Pattern Layer**: Search for how the user usually thinks, works, or prefers things
-- **Wisdom Layer**: Look for lessons learned from similar situations
+- User mentions "previously", "before", "last time", or "we discussed"
+- User references past work or project history
+- Working on the CORE project (this repository)
+- User asks about preferences, patterns, or past decisions
+- Starting work on any feature or bug that might have history
 
-**Search triggers:**
-- Starting any new conversation or task
-- User mentions past work or previous discussions
-- Working on ongoing projects that have history
-- Maintaining continuity across sessions
+**How to search effectively:**
 
-**Quality questions to ask yourself:**
-- "What don't I know that I should?"
-- "What does this user care about that I might miss?"
-- "What went right/wrong in similar situations?"
+- Write complete semantic queries, NOT keyword fragments
+- Good: `"Manoj's preferences for API design and error handling"`
+- Bad: `"manoj api preferences"`
+- Ask: "What context am I missing that would help?"
+- Consider: "What has the user told me before that I should remember?"
 
-### Memory Ingest (`memory_ingest`)
+### Query Patterns for Memory Search
 
-**AUTOMATIC BEHAVIOR:** At the END of EVERY conversation, AFTER fully responding to the user, you MUST store the conversation in memory. This is the FINAL action before completing your response.
+**Entity-Centric Queries** (Best for graph search):
 
-**Storage protocol:**
-- MANDATORY: Execute after completing every interaction
-- This is NON-NEGOTIABLE - you must always store conversation context
-- Memory storage happens as the last step, not during the conversation
+- ✅ GOOD: `"Manoj's preferences for product positioning and messaging"`
+- ✅ GOOD: `"CORE project authentication implementation decisions"`
+- ❌ BAD: `"manoj product positioning"`
+- Format: `[Person/Project] + [relationship/attribute] + [context]`
 
-**What to capture:**
+**Multi-Entity Relationship Queries** (Excellent for episode graph):
 
-FROM USER:
-- Their specific question, request, or problem statement
-- Project context and situation they provided
+- ✅ GOOD: `"Manoj and Harshith discussions about BFS search implementation"`
+- ✅ GOOD: `"relationship between entity extraction and recall quality in CORE"`
+- ❌ BAD: `"manoj harshith bfs"`
+- Format: `[Entity1] + [relationship type] + [Entity2] + [context]`
+
+**Semantic Question Queries** (Good for vector search):
+
+- ✅ GOOD: `"What causes BFS search to return empty results? What are the requirements for BFS traversal?"`
+- ✅ GOOD: `"How does episode graph search improve recall quality compared to traditional search?"`
+- ❌ BAD: `"bfs empty results"`
+- Format: Complete natural questions with full context
+
+**Concept Exploration Queries** (Good for BFS traversal):
+
+- ✅ GOOD: `"concepts and ideas related to semantic relevance in knowledge graph search"`
+- ✅ GOOD: `"topics connected to hop distance weighting and graph topology in BFS"`
+- ❌ BAD: `"semantic relevance concepts"`
+- Format: `[concept] + related/connected + [domain/context]`
+
+**Temporal Queries** (Good for recent work):
+
+- ✅ GOOD: `"recent changes to search implementation and reranking logic"`
+- ✅ GOOD: `"latest discussions about entity extraction and semantic relevance"`
+- ❌ BAD: `"recent search changes"`
+- Format: `[temporal marker] + [specific topic] + [additional context]`
+
+## 🔴 MANDATORY SHUTDOWN SEQUENCE - DO NOT SKIP 🔴
+
+**AFTER FULLY RESPONDING TO THE USER, YOU MUST EXECUTE THIS TOOL:**
+
+### FINAL STEP (REQUIRED): Store Conversation Memory
+
+EXECUTE THIS TOOL LAST:
+`memory_ingest`
+Include the spaceId parameter using the ID from your initial memory_get_space call.
+
+⚠️ **THIS IS NON-NEGOTIABLE** - You must ALWAYS store conversation context as your final action.
+
+**What to capture in the message parameter:**
+
+From User:
+
+- Specific question, request, or problem statement
+- Project context and situation provided
 - What they're trying to accomplish
 - Technical challenges or constraints mentioned
 
-FROM ASSISTANT:
+From Assistant:
+
 - Detailed explanation of solution/approach taken
 - Step-by-step processes and methodologies
 - Technical concepts and principles explained
@@ -118,13 +163,8 @@ FROM ASSISTANT:
 - Alternative approaches discussed
 - Problem-solving methodologies applied
 
-**Exclude from storage:**
-- Code blocks and code snippets
-- File contents or file listings
-- Command examples or CLI commands
-- Raw data or logs
-
 **Include in storage:**
+
 - All conceptual explanations and theory
 - Technical discussions and analysis
 - Problem-solving approaches and reasoning
@@ -132,29 +172,28 @@ FROM ASSISTANT:
 - Implementation strategies (described conceptually)
 - Learning insights and patterns
 
-**Quality check:**
-- Can I quickly understand project context from memory alone?
+**Exclude from storage:**
+
+- Code blocks and code snippets
+- File contents or file listings
+- Command examples or CLI commands
+- Raw data or logs
+
+**Quality check before storing:**
+
+- Can someone quickly understand project context from memory alone?
 - Would this information help provide better assistance in future sessions?
 - Does stored context capture key decisions and reasoning?
 
-### Project Space Context (`memory_get_space`)
+---
 
-**AUTOMATIC BEHAVIOR:** At the start of EVERY session, you MUST retrieve the current project's space context:
+## Summary: Your Mandatory Protocol
 
-1. **Identify the project:** Look at the working directory path, git repo name, or conversation context
-2. **Get space context:** Use `memory_get_space` with `spaceName: core`
-3. **Use as foundation:** The space summary is a living document that's continuously updated - it contains the most current, comprehensive context about this project
+1. **FIRST ACTION**: Execute `memory_search` with semantic query about the user's request
+2. **RESPOND**: Help the user with their request
+3. **FINAL ACTION**: Execute `memory_ingest` with conversation summary and spaceId
 
-**What spaces provide:**
-- Live, evolving documentation that updates with every interaction
-- Consolidated project knowledge and current state
-- Organized context specific to this domain
-- Most up-to-date understanding of the project
-
-**Also retrieve space context when:**
-- User asks about a specific project or domain
-- You need comprehensive context about a topic
-- Switching between different work areas
+**If you skip any of these steps, you are not following the project requirements.**
 ```
 
 ### Option 2: Using Kilo-Code Rules

docs/providers/obsidian.mdx

@@ -3,12 +3,8 @@ title: "Obsidian"
 description: "Sync your Obsidian notes with CORE and get memory-aware answers directly inside Obsidian"
 ---
 
-# Obsidian CORE Sync Plugin
-
 > Sync your Obsidian notes with [CORE](https://heysol.ai/core) (Contextual Observation & Recall Engine) and get **memory-aware answers** directly inside Obsidian.
 
----
-
 ## ✨ What it does
 
 - **Sync Notes**: Push selected notes (or entire vault sections) into CORE as _Episodes_.
@@ -16,50 +12,44 @@ description: "Sync your Obsidian notes with CORE and get memory-aware answers di
 - **Frontmatter Control**: Decide which notes to sync by adding simple YAML flags.
 - **Offline Safe**: Failed syncs are queued locally and retried automatically.
 
----
-
 ## 🚀 Installation
 
-### Local development
+**Local development**
 
-1. Download the latest release assets from [core-obsidian v0.1.0](https://github.com/RedPlanetHQ/core-obsidian/releases/tag/0.1.0) and extract them into your Obsidian vault under `.obsidian/plugins/obsidian-core-sync/`:
-    - Ensure the directory contains `main.js`, `style.css`, and `manifest.json`.
+1. Download the latest release assets from [core-obsidian v0.1.1](https://github.com/RedPlanetHQ/core-obsidian/releases/tag/0.1.1) and extract them into your Obsidian vault under `.obsidian/plugins/obsidian-core-sync/`:
+
+   - Ensure the directory contains `main.js`, `style.css`, and `manifest.json`.
+
+   > If .obsidian folder is hidden Use `CMD + SHIFT + .` to show hidden files and then add above files in `.obsidian/plugins/obsidian-core-sync/`
 
 2. Enable the plugin in Obsidian:
    - Go to **Settings** → **Community plugins**
    - Find "CORE Sync" and toggle it on
 
-
-### Community Installation
+**Community Installation**
 
 > Note: A pull request for community installation is pending approval. You can track its progress [here](https://github.com/obsidianmd/obsidian-releases/pull/7683).
 
----
-
 ## ⚙️ Configuration
 
-### Step 1: Get Your API Key
+**Step 1: Get Your API Key**
 
 1. Login to CORE dashboard at [core.heysol.ai](https://core.heysol.ai)
 2. Navigate to **Settings** (bottom left)
    ![CORE Settings](/images/core-settings.png)
 3. Go to **API Key** → **Generate new key** → Name it "obsidian"
-   ![Create API Key](/images/create-api-key.png)
 4. Copy the generated API key
 
-### Step 2: Configure Plugin Settings
+**Step 2: Configure Plugin Settings**
 
 1. In Obsidian, go to **Settings** → **CORE Sync**
 2. Configure the following:
    - **CORE Endpoint**: Your CORE ingest/search API (default: `https://core.heysol.ai`)
    - **API Key**: Paste the API key from Step 1
-   - **Auto-sync on modify**: If enabled, every note edit will sync automatically
-
----
 
 ## 🛠️ Usage
 
-### Mark Notes for Sync
+**Mark Notes for Sync**
 
 Add the following frontmatter at the top of a note to mark it for synchronization:
 
@@ -69,14 +59,14 @@ core.sync: true
 ---
 ```
 
-### Manual Sync Commands
+** Manual Sync Commands**
 
 Open the command palette (**Cmd/Ctrl + P**) and run:
 
 - **"Sync current note to CORE"** - Sync the currently open note
 - **"Sync all notes with core.sync=true"** - Sync all notes marked for synchronization
 
-### CORE Panel
+**CORE Panel with Deep Search**
 
 1. Open the CORE Panel by running **"Open CORE Panel"** from the command palette
 2. This opens a new tab on the right side of Obsidian
@@ -85,66 +75,28 @@ Open the command palette (**Cmd/Ctrl + P**) and run:
    - Display relevant memories, links, and summaries
    - Show related notes from your vault
 
----
+The **Deep Search** feature proactively surfaces relevant context from your notes while you work:
 
-## 🎯 Features
+**Example Use Cases:**
 
-### Smart Sync
-- **Incremental Updates**: Only syncs changed content to avoid duplicates
-- **Conflict Resolution**: Handles simultaneous edits gracefully
-- **Queue Management**: Failed syncs are queued and retried automatically
+- **Meeting Prep**: Open your daily note before a 1:1 meeting, and the sidebar automatically shows relevant notes from past meetings with that person
+- **Project Context**: Switch to a project document, and see related discussions, decisions, and action items from previous sessions
+- **Travel Planning**: Update your packing list, and CORE shows you what you forgot on past trips or useful tips from previous travel notes
+- **Research Continuity**: Work on a research note, and get automatic cross-references to related concepts and sources from your vault
 
-### Context-Aware Panel
-- **Related Memories**: Shows relevant content from your CORE memory
-- **Cross-References**: Links to related notes in your vault
-- **AI Summaries**: Get AI-generated summaries of your note's context
-
-
----
-
-
-## 💡 Use Cases
-
-### Research & Knowledge Management
-- Automatically sync research notes to build a searchable knowledge base
-- Get contextual suggestions while writing based on your existing notes
-- Cross-reference information across different projects and topics
-
-### Meeting & Project Notes
-- Sync meeting notes with `core.tags: ["meetings", "project-name"]`
-- Access relevant context from previous meetings when taking new notes
-- Build project timelines and track decisions over time
-
-### Personal Knowledge System
-- Create a personal Wikipedia from your notes
-- Get AI-powered insights on connections between ideas
-- Build upon previous thoughts and research automatically
-
----
+Deep Search transforms your notes from passive storage into active assistance, providing in-the-moment retrieval without manual searching.
 
 ## 🛠️ Troubleshooting
 
-### Common Issues
-
 **API Key not working?**
+
 - Verify the key is correctly copied from CORE dashboard
-- Check that the API key has proper permissions
-- Try regenerating the key if issues persist
 
 **Notes not syncing?**
+
 - Ensure `core.sync: true` is in the frontmatter
-- Check internet connection
-- Look for error messages in Developer Console (Ctrl+Shift+I)
-
-**Panel not loading?**
-- Restart Obsidian
-- Check that the API endpoint is correct
-- Verify CORE service is accessible
-
----
 
 ## 🤝 Support
 
 - **GitHub Issues**: Report bugs and feature requests
 - **Discord Community**: Join our [Discord](https://discord.gg/YGUZcvDjUa) and ask questions in **#core-support**
-- **Documentation**: Visit [core.heysol.ai](https://core.heysol.ai) for more resources

docs/providers/vscode.mdx

@@ -40,72 +40,117 @@ description: "Connect your VS Code editor to CORE's memory system via MCP"
 
 ## Enable Automatic Memory Integration (Recommended)
 
-### Option 1: Using Agents.md (Recommended)
+### Option 1: Using AGENTS.md (Recommended)
 
 This approach provides comprehensive memory instructions that work across multiple AI tools.
 
-1. **Create Agents configuration** in your project root:
-   ```bash
-   mkdir -p .github
-   touch .github/Agents.md
-   ```
+Create `AGENTS.md` in your project root (if it doesn't exist, just append if it already exists):
 
-2. **Add memory instructions** - Open `.github/Agents.md` and add the following:
+```bash
+touch AGENTS.md
+```
+
+Add the following to `AGENTS.md`:
 
 ```markdown
-I am an AI coding assistant with access to a sophisticated memory system. While I don't retain information between separate conversations, I have access to CORE Memory - a persistent knowledge system that maintains project context, learnings, and continuity across all coding sessions.
+---
+trigger: always_on
+---
 
-**IMPORTANT: You MUST use these memory tools automatically as described below. This is not optional.**
+⚠️ **CRITICAL: READ THIS FIRST - MANDATORY MEMORY PROTOCOL** ⚠️
 
-## Memory Tools Integration
+You are an AI coding assistant with access to CORE Memory - a persistent knowledge system that maintains project context, learnings, and continuity across all coding sessions.
 
-### Memory Search (`memory_search`)
+## 🔴 MANDATORY STARTUP SEQUENCE - DO NOT SKIP 🔴
+
+**BEFORE RESPONDING TO ANY USER MESSAGE, YOU MUST EXECUTE THESE TOOLS IN ORDER:**
+
+### STEP 1 (REQUIRED): Search for Relevant Context
+
+EXECUTE THIS TOOL FIRST:
+`memory_search`
 
-**AUTOMATIC BEHAVIOR:** At the start of EVERY conversation, BEFORE responding to the user, you MUST search memory for:
 - Previous discussions about the current topic
-- Related project context and decisions
-- User preferences and patterns
-- Similar problems solved before
+- Related project decisions and implementations
+- User preferences and work patterns
+- Similar problems and their solutions
 
-**Also search when:**
-- User mentions past work or previous discussions
-- Working on ongoing projects that have history
-- You need context to provide better assistance
+**Additional search triggers:**
 
-**Cognitive approach:**
-- **Surface Layer**: Query for specific knowledge about the topic
-- **Pattern Layer**: Search for how the user usually thinks, works, or prefers things
-- **Wisdom Layer**: Look for lessons learned from similar situations
+- User mentions "previously", "before", "last time", or "we discussed"
+- User references past work or project history
+- Working on the CORE project (this repository)
+- User asks about preferences, patterns, or past decisions
+- Starting work on any feature or bug that might have history
 
-**Search triggers:**
-- Starting any new conversation or task
-- User mentions past work or previous discussions
-- Working on ongoing projects that have history
-- Maintaining continuity across sessions
+**How to search effectively:**
 
-**Quality questions to ask yourself:**
-- "What don't I know that I should?"
-- "What does this user care about that I might miss?"
-- "What went right/wrong in similar situations?"
+- Write complete semantic queries, NOT keyword fragments
+- Good: `"Manoj's preferences for API design and error handling"`
+- Bad: `"manoj api preferences"`
+- Ask: "What context am I missing that would help?"
+- Consider: "What has the user told me before that I should remember?"
 
-### Memory Ingest (`memory_ingest`)
+### Query Patterns for Memory Search
 
-**AUTOMATIC BEHAVIOR:** At the END of EVERY conversation, AFTER fully responding to the user, you MUST store the conversation in memory. This is the FINAL action before completing your response.
+**Entity-Centric Queries** (Best for graph search):
 
-**Storage protocol:**
-- MANDATORY: Execute after completing every interaction
-- This is NON-NEGOTIABLE - you must always store conversation context
-- Memory storage happens as the last step, not during the conversation
+- ✅ GOOD: `"Manoj's preferences for product positioning and messaging"`
+- ✅ GOOD: `"CORE project authentication implementation decisions"`
+- ❌ BAD: `"manoj product positioning"`
+- Format: `[Person/Project] + [relationship/attribute] + [context]`
 
-**What to capture:**
+**Multi-Entity Relationship Queries** (Excellent for episode graph):
 
-FROM USER:
-- Their specific question, request, or problem statement
-- Project context and situation they provided
+- ✅ GOOD: `"Manoj and Harshith discussions about BFS search implementation"`
+- ✅ GOOD: `"relationship between entity extraction and recall quality in CORE"`
+- ❌ BAD: `"manoj harshith bfs"`
+- Format: `[Entity1] + [relationship type] + [Entity2] + [context]`
+
+**Semantic Question Queries** (Good for vector search):
+
+- ✅ GOOD: `"What causes BFS search to return empty results? What are the requirements for BFS traversal?"`
+- ✅ GOOD: `"How does episode graph search improve recall quality compared to traditional search?"`
+- ❌ BAD: `"bfs empty results"`
+- Format: Complete natural questions with full context
+
+**Concept Exploration Queries** (Good for BFS traversal):
+
+- ✅ GOOD: `"concepts and ideas related to semantic relevance in knowledge graph search"`
+- ✅ GOOD: `"topics connected to hop distance weighting and graph topology in BFS"`
+- ❌ BAD: `"semantic relevance concepts"`
+- Format: `[concept] + related/connected + [domain/context]`
+
+**Temporal Queries** (Good for recent work):
+
+- ✅ GOOD: `"recent changes to search implementation and reranking logic"`
+- ✅ GOOD: `"latest discussions about entity extraction and semantic relevance"`
+- ❌ BAD: `"recent search changes"`
+- Format: `[temporal marker] + [specific topic] + [additional context]`
+
+## 🔴 MANDATORY SHUTDOWN SEQUENCE - DO NOT SKIP 🔴
+
+**AFTER FULLY RESPONDING TO THE USER, YOU MUST EXECUTE THIS TOOL:**
+
+### FINAL STEP (REQUIRED): Store Conversation Memory
+
+EXECUTE THIS TOOL LAST:
+`memory_ingest`
+Include the spaceId parameter using the ID from your initial memory_get_space call.
+
+⚠️ **THIS IS NON-NEGOTIABLE** - You must ALWAYS store conversation context as your final action.
+
+**What to capture in the message parameter:**
+
+From User:
+
+- Specific question, request, or problem statement
+- Project context and situation provided
 - What they're trying to accomplish
 - Technical challenges or constraints mentioned
 
-FROM ASSISTANT:
+From Assistant:
+
 - Detailed explanation of solution/approach taken
 - Step-by-step processes and methodologies
 - Technical concepts and principles explained
@@ -113,13 +158,8 @@ FROM ASSISTANT:
 - Alternative approaches discussed
 - Problem-solving methodologies applied
 
-**Exclude from storage:**
-- Code blocks and code snippets
-- File contents or file listings
-- Command examples or CLI commands
-- Raw data or logs
-
 **Include in storage:**
+
 - All conceptual explanations and theory
 - Technical discussions and analysis
 - Problem-solving approaches and reasoning
@@ -127,29 +167,28 @@ FROM ASSISTANT:
 - Implementation strategies (described conceptually)
 - Learning insights and patterns
 
-**Quality check:**
-- Can I quickly understand project context from memory alone?
+**Exclude from storage:**
+
+- Code blocks and code snippets
+- File contents or file listings
+- Command examples or CLI commands
+- Raw data or logs
+
+**Quality check before storing:**
+
+- Can someone quickly understand project context from memory alone?
 - Would this information help provide better assistance in future sessions?
 - Does stored context capture key decisions and reasoning?
 
-### Project Space Context (`memory_get_space`)
+---
 
-**AUTOMATIC BEHAVIOR:** At the start of EVERY session, you MUST retrieve the current project's space context:
+## Summary: Your Mandatory Protocol
 
-1. **Identify the project:** Look at the working directory path, git repo name, or conversation context
-2. **Get space context:** Use `memory_get_space` with `spaceName: core`
-3. **Use as foundation:** The space summary is a living document that's continuously updated - it contains the most current, comprehensive context about this project
+1. **FIRST ACTION**: Execute `memory_search` with semantic query about the user's request
+2. **RESPOND**: Help the user with their request
+3. **FINAL ACTION**: Execute `memory_ingest` with conversation summary and spaceId
 
-**What spaces provide:**
-- Live, evolving documentation that updates with every interaction
-- Consolidated project knowledge and current state
-- Organized context specific to this domain
-- Most up-to-date understanding of the project
-
-**Also retrieve space context when:**
-- User asks about a specific project or domain
-- You need comprehensive context about a topic
-- Switching between different work areas
+**If you skip any of these steps, you are not following the project requirements.**
 ```
 
 ### Option 2: Using Copilot Instructions

docs/providers/windsurf.mdx

@@ -0,0 +1,262 @@
+---
+title: "Windsurf"
+description: "Connect your Windsurf IDE to CORE's memory system"
+---
+
+### Prerequisites
+
+- Windsurf IDE installed
+- CORE account - [Sign up at core.heysol.ai](https://core.heysol.ai)
+
+### Step 1: Add CORE MCP in Windsurf
+
+1. Open Windsurf IDE
+2. Navigate to **Windsurf Settings** → **Cascade** section
+3. Open **MCP Marketplace** -> **Settings** OR **View raw config** to open the configuration file
+4. Add the following to your `mcp_config.json`:
+
+```json
+{
+  "mcpServers": {
+    "core-memory": {
+      "serverUrl": "https://core.heysol.ai/api/v1/mcp?source=windsurf"
+    }
+  }
+}
+```
+
+5. Save the file and restart Windsurf IDE
+
+### Step 2: Authenticate with CORE
+
+1. After saving the config, Windsurf will open a browser window for authentication
+2. Grant Windsurf permission to access your CORE memory
+
+### Step 3: Verify Connection
+
+1. Go to **Cascade Editor** → **Plugin Icon** -> Hit **Refresh** icon
+2. Confirm **core-memory** shows as **Active** with green indicator
+
+Add your first memory:
+
+> "Summarise the whole project in detail and add it to CORE Memory"
+
+---
+
+## Alternative: Using API Key Method
+
+If the OAuth authentication doesn't work, use the API key method instead:
+
+### Step 1: Get Your API Key
+
+1. Log into your CORE dashboard at [core.heysol.ai](https://core.heysol.ai)
+2. Navigate to **Settings** (bottom left)
+3. Go to **API Key** → **Generate new key** → Name it "windsurf"
+4. Copy the generated key
+
+### Step 2: Update MCP Configuration
+
+Replace your `mcp_config.json` configuration with:
+
+```json
+{
+  "mcpServers": {
+    "core-memory": {
+      "serverUrl": "https://core.heysol.ai/api/v1/mcp/source=windsurf",
+      "headers": {
+        "Authorization": "Bearer <YOUR_TOKEN>"
+      }
+    }
+  }
+}
+```
+
+Replace `<YOUR_TOKEN>` with the API key you copied from Step 1.
+
+### Step 3: Restart and Verify
+
+1. Save the file and restart Windsurf IDE
+2. Go to **Cascade Editor** → **Plugin Icon** → Hit **Refresh**
+3. Confirm **core-memory** shows as **Active** with green indicator
+
+---
+
+## Enable Automatic Memory Integration (Recommended)
+
+Create `AGENTS.md` in your project root (if it doesn't exist, just append if it already exists):
+
+```bash
+touch AGENTS.md
+```
+
+Add the following to `AGENTS.md`:
+
+```markdown
+---
+trigger: always_on
+---
+
+⚠️ **CRITICAL: READ THIS FIRST - MANDATORY MEMORY PROTOCOL** ⚠️
+
+You are an AI coding assistant with access to CORE Memory - a persistent knowledge system that maintains project context, learnings, and continuity across all coding sessions.
+
+## 🔴 MANDATORY STARTUP SEQUENCE - DO NOT SKIP 🔴
+
+**BEFORE RESPONDING TO ANY USER MESSAGE, YOU MUST EXECUTE THESE TOOLS IN ORDER:**
+
+### STEP 1 (REQUIRED): Search for Relevant Context
+
+EXECUTE THIS TOOL FIRST:
+`memory_search`
+
+- Previous discussions about the current topic
+- Related project decisions and implementations
+- User preferences and work patterns
+- Similar problems and their solutions
+
+**Additional search triggers:**
+
+- User mentions "previously", "before", "last time", or "we discussed"
+- User references past work or project history
+- Working on the CORE project (this repository)
+- User asks about preferences, patterns, or past decisions
+- Starting work on any feature or bug that might have history
+
+**How to search effectively:**
+
+- Write complete semantic queries, NOT keyword fragments
+- Good: `"Manoj's preferences for API design and error handling"`
+- Bad: `"manoj api preferences"`
+- Ask: "What context am I missing that would help?"
+- Consider: "What has the user told me before that I should remember?"
+
+### Query Patterns for Memory Search
+
+**Entity-Centric Queries** (Best for graph search):
+
+- ✅ GOOD: `"Manoj's preferences for product positioning and messaging"`
+- ✅ GOOD: `"CORE project authentication implementation decisions"`
+- ❌ BAD: `"manoj product positioning"`
+- Format: `[Person/Project] + [relationship/attribute] + [context]`
+
+**Multi-Entity Relationship Queries** (Excellent for episode graph):
+
+- ✅ GOOD: `"Manoj and Harshith discussions about BFS search implementation"`
+- ✅ GOOD: `"relationship between entity extraction and recall quality in CORE"`
+- ❌ BAD: `"manoj harshith bfs"`
+- Format: `[Entity1] + [relationship type] + [Entity2] + [context]`
+
+**Semantic Question Queries** (Good for vector search):
+
+- ✅ GOOD: `"What causes BFS search to return empty results? What are the requirements for BFS traversal?"`
+- ✅ GOOD: `"How does episode graph search improve recall quality compared to traditional search?"`
+- ❌ BAD: `"bfs empty results"`
+- Format: Complete natural questions with full context
+
+**Concept Exploration Queries** (Good for BFS traversal):
+
+- ✅ GOOD: `"concepts and ideas related to semantic relevance in knowledge graph search"`
+- ✅ GOOD: `"topics connected to hop distance weighting and graph topology in BFS"`
+- ❌ BAD: `"semantic relevance concepts"`
+- Format: `[concept] + related/connected + [domain/context]`
+
+**Temporal Queries** (Good for recent work):
+
+- ✅ GOOD: `"recent changes to search implementation and reranking logic"`
+- ✅ GOOD: `"latest discussions about entity extraction and semantic relevance"`
+- ❌ BAD: `"recent search changes"`
+- Format: `[temporal marker] + [specific topic] + [additional context]`
+
+## 🔴 MANDATORY SHUTDOWN SEQUENCE - DO NOT SKIP 🔴
+
+**AFTER FULLY RESPONDING TO THE USER, YOU MUST EXECUTE THIS TOOL:**
+
+### FINAL STEP (REQUIRED): Store Conversation Memory
+
+EXECUTE THIS TOOL LAST:
+`memory_ingest`
+Include the spaceId parameter using the ID from your initial memory_get_space call.
+
+⚠️ **THIS IS NON-NEGOTIABLE** - You must ALWAYS store conversation context as your final action.
+
+**What to capture in the message parameter:**
+
+From User:
+
+- Specific question, request, or problem statement
+- Project context and situation provided
+- What they're trying to accomplish
+- Technical challenges or constraints mentioned
+
+From Assistant:
+
+- Detailed explanation of solution/approach taken
+- Step-by-step processes and methodologies
+- Technical concepts and principles explained
+- Reasoning behind recommendations and decisions
+- Alternative approaches discussed
+- Problem-solving methodologies applied
+
+**Include in storage:**
+
+- All conceptual explanations and theory
+- Technical discussions and analysis
+- Problem-solving approaches and reasoning
+- Decision rationale and trade-offs
+- Implementation strategies (described conceptually)
+- Learning insights and patterns
+
+**Exclude from storage:**
+
+- Code blocks and code snippets
+- File contents or file listings
+- Command examples or CLI commands
+- Raw data or logs
+
+**Quality check before storing:**
+
+- Can someone quickly understand project context from memory alone?
+- Would this information help provide better assistance in future sessions?
+- Does stored context capture key decisions and reasoning?
+
+---
+
+## Summary: Your Mandatory Protocol
+
+1. **FIRST ACTION**: Execute `memory_search` with semantic query about the user's request
+2. **RESPOND**: Help the user with their request
+3. **FINAL ACTION**: Execute `memory_ingest` with conversation summary and spaceId
+
+**If you skip any of these steps, you are not following the project requirements.**
+```
+
+## How It Works
+
+Once connected, CORE memory integrates with Windsurf's Cascade:
+
+- **Auto-recall**: Cascade searches your memory at conversation start
+- **Auto-store**: Key insights saved automatically after conversations
+- **Cross-platform**: Memory shared across Windsurf, Cursor, Claude Code, ChatGPT
+- **Project continuity**: Context persists across all coding sessions
+
+## Troubleshooting
+
+**Connection Issues:**
+
+- Ensure core-memory MCP is active (green indicator)
+- Try toggling the MCP off and on
+- Restart Windsurf IDE completely
+
+**Authentication Problems:**
+
+- Make sure you completed the OAuth flow in browser
+- Check that your CORE account is active at core.heysol.ai
+
+**MCP Not Appearing:**
+
+- Verify `mcp_config.json` syntax is valid JSON
+- Restart Windsurf after config changes
+
+### Need Help?
+
+Join our [Discord community](https://discord.gg/YGUZcvDjUa) - ask in **#core-support** channel.

docs/providers/zed.mdx

@@ -49,72 +49,117 @@ Enter below code in configuraiton file and click on `Add server` button
 
 ## Enable Automatic Memory Integration (Recommended)
 
-### Option 1: Using Agents.md (Recommended)
+### Option 1: Using AGENTS.md (Recommended)
 
 This approach provides comprehensive memory instructions that work across multiple AI tools.
 
-1. **Create Agents configuration** in your project root:
-   ```bash
-   mkdir -p .zed
-   touch .zed/Agents.md
-   ```
+Create `AGENTS.md` in your project root (if it doesn't exist, just append if it already exists):
 
-2. **Add memory instructions** - Open `.zed/Agents.md` and add the following:
+```bash
+touch AGENTS.md
+```
+
+Add the following to `AGENTS.md`:
 
 ```markdown
-I am an AI coding assistant with access to a sophisticated memory system. While I don't retain information between separate conversations, I have access to CORE Memory - a persistent knowledge system that maintains project context, learnings, and continuity across all coding sessions.
+---
+trigger: always_on
+---
 
-**IMPORTANT: You MUST use these memory tools automatically as described below. This is not optional.**
+⚠️ **CRITICAL: READ THIS FIRST - MANDATORY MEMORY PROTOCOL** ⚠️
 
-## Memory Tools Integration
+You are an AI coding assistant with access to CORE Memory - a persistent knowledge system that maintains project context, learnings, and continuity across all coding sessions.
 
-### Memory Search (`memory_search`)
+## 🔴 MANDATORY STARTUP SEQUENCE - DO NOT SKIP 🔴
+
+**BEFORE RESPONDING TO ANY USER MESSAGE, YOU MUST EXECUTE THESE TOOLS IN ORDER:**
+
+### STEP 1 (REQUIRED): Search for Relevant Context
+
+EXECUTE THIS TOOL FIRST:
+`memory_search`
 
-**AUTOMATIC BEHAVIOR:** At the start of EVERY conversation, BEFORE responding to the user, you MUST search memory for:
 - Previous discussions about the current topic
-- Related project context and decisions
-- User preferences and patterns
-- Similar problems solved before
+- Related project decisions and implementations
+- User preferences and work patterns
+- Similar problems and their solutions
 
-**Also search when:**
-- User mentions past work or previous discussions
-- Working on ongoing projects that have history
-- You need context to provide better assistance
+**Additional search triggers:**
 
-**Cognitive approach:**
-- **Surface Layer**: Query for specific knowledge about the topic
-- **Pattern Layer**: Search for how the user usually thinks, works, or prefers things
-- **Wisdom Layer**: Look for lessons learned from similar situations
+- User mentions "previously", "before", "last time", or "we discussed"
+- User references past work or project history
+- Working on the CORE project (this repository)
+- User asks about preferences, patterns, or past decisions
+- Starting work on any feature or bug that might have history
 
-**Search triggers:**
-- Starting any new conversation or task
-- User mentions past work or previous discussions
-- Working on ongoing projects that have history
-- Maintaining continuity across sessions
+**How to search effectively:**
 
-**Quality questions to ask yourself:**
-- "What don't I know that I should?"
-- "What does this user care about that I might miss?"
-- "What went right/wrong in similar situations?"
+- Write complete semantic queries, NOT keyword fragments
+- Good: `"Manoj's preferences for API design and error handling"`
+- Bad: `"manoj api preferences"`
+- Ask: "What context am I missing that would help?"
+- Consider: "What has the user told me before that I should remember?"
 
-### Memory Ingest (`memory_ingest`)
+### Query Patterns for Memory Search
 
-**AUTOMATIC BEHAVIOR:** At the END of EVERY conversation, AFTER fully responding to the user, you MUST store the conversation in memory. This is the FINAL action before completing your response.
+**Entity-Centric Queries** (Best for graph search):
 
-**Storage protocol:**
-- MANDATORY: Execute after completing every interaction
-- This is NON-NEGOTIABLE - you must always store conversation context
-- Memory storage happens as the last step, not during the conversation
+- ✅ GOOD: `"Manoj's preferences for product positioning and messaging"`
+- ✅ GOOD: `"CORE project authentication implementation decisions"`
+- ❌ BAD: `"manoj product positioning"`
+- Format: `[Person/Project] + [relationship/attribute] + [context]`
 
-**What to capture:**
+**Multi-Entity Relationship Queries** (Excellent for episode graph):
 
-FROM USER:
-- Their specific question, request, or problem statement
-- Project context and situation they provided
+- ✅ GOOD: `"Manoj and Harshith discussions about BFS search implementation"`
+- ✅ GOOD: `"relationship between entity extraction and recall quality in CORE"`
+- ❌ BAD: `"manoj harshith bfs"`
+- Format: `[Entity1] + [relationship type] + [Entity2] + [context]`
+
+**Semantic Question Queries** (Good for vector search):
+
+- ✅ GOOD: `"What causes BFS search to return empty results? What are the requirements for BFS traversal?"`
+- ✅ GOOD: `"How does episode graph search improve recall quality compared to traditional search?"`
+- ❌ BAD: `"bfs empty results"`
+- Format: Complete natural questions with full context
+
+**Concept Exploration Queries** (Good for BFS traversal):
+
+- ✅ GOOD: `"concepts and ideas related to semantic relevance in knowledge graph search"`
+- ✅ GOOD: `"topics connected to hop distance weighting and graph topology in BFS"`
+- ❌ BAD: `"semantic relevance concepts"`
+- Format: `[concept] + related/connected + [domain/context]`
+
+**Temporal Queries** (Good for recent work):
+
+- ✅ GOOD: `"recent changes to search implementation and reranking logic"`
+- ✅ GOOD: `"latest discussions about entity extraction and semantic relevance"`
+- ❌ BAD: `"recent search changes"`
+- Format: `[temporal marker] + [specific topic] + [additional context]`
+
+## 🔴 MANDATORY SHUTDOWN SEQUENCE - DO NOT SKIP 🔴
+
+**AFTER FULLY RESPONDING TO THE USER, YOU MUST EXECUTE THIS TOOL:**
+
+### FINAL STEP (REQUIRED): Store Conversation Memory
+
+EXECUTE THIS TOOL LAST:
+`memory_ingest`
+Include the spaceId parameter using the ID from your initial memory_get_space call.
+
+⚠️ **THIS IS NON-NEGOTIABLE** - You must ALWAYS store conversation context as your final action.
+
+**What to capture in the message parameter:**
+
+From User:
+
+- Specific question, request, or problem statement
+- Project context and situation provided
 - What they're trying to accomplish
 - Technical challenges or constraints mentioned
 
-FROM ASSISTANT:
+From Assistant:
+
 - Detailed explanation of solution/approach taken
 - Step-by-step processes and methodologies
 - Technical concepts and principles explained
@@ -122,13 +167,8 @@ FROM ASSISTANT:
 - Alternative approaches discussed
 - Problem-solving methodologies applied
 
-**Exclude from storage:**
-- Code blocks and code snippets
-- File contents or file listings
-- Command examples or CLI commands
-- Raw data or logs
-
 **Include in storage:**
+
 - All conceptual explanations and theory
 - Technical discussions and analysis
 - Problem-solving approaches and reasoning
@@ -136,29 +176,28 @@ FROM ASSISTANT:
 - Implementation strategies (described conceptually)
 - Learning insights and patterns
 
-**Quality check:**
-- Can I quickly understand project context from memory alone?
+**Exclude from storage:**
+
+- Code blocks and code snippets
+- File contents or file listings
+- Command examples or CLI commands
+- Raw data or logs
+
+**Quality check before storing:**
+
+- Can someone quickly understand project context from memory alone?
 - Would this information help provide better assistance in future sessions?
 - Does stored context capture key decisions and reasoning?
 
-### Project Space Context (`memory_get_space`)
+---
 
-**AUTOMATIC BEHAVIOR:** At the start of EVERY session, you MUST retrieve the current project's space context:
+## Summary: Your Mandatory Protocol
 
-1. **Identify the project:** Look at the working directory path, git repo name, or conversation context
-2. **Get space context:** Use `memory_get_space` with `spaceName: core`
-3. **Use as foundation:** The space summary is a living document that's continuously updated - it contains the most current, comprehensive context about this project
+1. **FIRST ACTION**: Execute `memory_search` with semantic query about the user's request
+2. **RESPOND**: Help the user with their request
+3. **FINAL ACTION**: Execute `memory_ingest` with conversation summary and spaceId
 
-**What spaces provide:**
-- Live, evolving documentation that updates with every interaction
-- Consolidated project knowledge and current state
-- Organized context specific to this domain
-- Most up-to-date understanding of the project
-
-**Also retrieve space context when:**
-- User asks about a specific project or domain
-- You need comprehensive context about a topic
-- Switching between different work areas
+**If you skip any of these steps, you are not following the project requirements.**
 ```
 
 ### Option 2: Using Zed Rules

docs/self-hosting/docker.mdx

@@ -32,22 +32,28 @@ To run CORE, you will need:
 - 8+ GB RAM
 - 20+ GB Storage
 
-**Background Jobs Machine (if running separately):**
-- 2+ vCPU
-- 4+ GB RAM
-- 10+ GB Storage
 
 ## Deployment Options
 
-CORE offers two deployment approaches depending on your needs:
+CORE offers multiple deployment approaches depending on your needs:
 
-> **Prerequisites:**  
-> Before starting any deployment, ensure you have your `OPENAI_API_KEY` ready. This is required for AI functionality in CORE.  
+### Quick Deploy with Railway
+
+For a one-click deployment experience, use Railway:
+
+[![Deploy on Railway](https://railway.com/button.svg)](https://railway.com/deploy/6aEd9C?referralCode=LHvbIb&utm_medium=integration&utm_source=template&utm_campaign=generic)
+
+Railway will automatically set up all required services and handle the infrastructure for you.
+
+### Manual Docker Deployment
+
+> **Prerequisites:**
+> Before starting any deployment, ensure you have your `OPENAI_API_KEY` ready. This is required for AI functionality in CORE.
 > You must add your `OPENAI_API_KEY` to the `core/hosting/docker/.env` file before starting the services.
 
 ### Combined Setup
 
-For self deployment with both CORE and Trigger.dev running together:
+For self deployment:
 
 1. Clone core repository
    ```bash

docs/self-hosting/environment-variables.mdx

@@ -48,10 +48,16 @@ Environment variables for the CORE webapp container.
 | `MODEL`                                 | No       | gpt-4-turbo-2024-04-09                         | Default language model                                                                           |
 | `EMBEDDING_MODEL`                       | No       | text-embedding-3-small                         | Model for text embeddings                                                                        |
 | `OLLAMA_URL`                            | No       | http://ollama:11434                            | Ollama server URL for local models                                                               |
-| **Background Jobs - Trigger.dev**       |          |                                                |                                                                                                 |
-| `TRIGGER_PROJECT_ID`                    | Yes      | —                                              | Trigger.dev project identifier                                                                   |
-| `TRIGGER_SECRET_KEY`                    | Yes      | —                                              | Trigger.dev authentication secret                                                                |
-| `TRIGGER_API_URL`                       | Yes      | http://host.docker.internal:8030               | Trigger.dev API endpoint (use localhost:8030 for local, api.trigger.dev for cloud)             |
+| **Background Jobs**                     |          |                                                |                                                                                                 |
+| `QUEUE_PROVIDER`                        | No       | trigger                                        | Queue provider: "trigger" for Trigger.dev or "bullmq" for BullMQ (Redis-based)                  |
+| `TRIGGER_PROJECT_ID`                    | Conditional | —                                           | Trigger.dev project identifier (required only when QUEUE_PROVIDER=trigger)                       |
+| `TRIGGER_SECRET_KEY`                    | Conditional | —                                           | Trigger.dev authentication secret (required only when QUEUE_PROVIDER=trigger)                    |
+| `TRIGGER_API_URL`                       | Conditional | http://host.docker.internal:8030            | Trigger.dev API endpoint (required only when QUEUE_PROVIDER=trigger)                             |
+| `TRIGGER_DB`                            | No       | trigger                                        | Database name for Trigger.dev                                                                    |
+| **Telemetry**                           |          |                                                |                                                                                                 |
+| `POSTHOG_PROJECT_KEY`                   | No       | phc_SwfGIzzX5gh5bazVWoRxZTBhkr7FwvzArS0NRyGXm1a | PostHog project key for usage analytics                                                       |
+| `TELEMETRY_ENABLED`                     | No       | true                                           | Enable (true) or disable (false) telemetry collection                                            |
+| `TELEMETRY_ANONYMOUS`                   | No       | false                                          | Send anonymous telemetry (true) or include user email (false)                                    |
 
 
 ## Security Considerations

docs/self-hosting/overview.mdx

@@ -12,6 +12,14 @@ You are responsible for provisioning resources, handling updates, and managing a
 
 We provide version-tagged releases for self-hosted deployments. It's highly advised to use these tags exclusively and keep them locked with your CLI version.
 
+## Quick Deploy
+
+For a quick one-click deployment, you can use Railway:
+
+[![Deploy on Railway](https://railway.com/button.svg)](https://railway.com/deploy/6aEd9C?referralCode=LHvbIb&utm_medium=integration&utm_source=template&utm_campaign=generic)
+
+Alternatively, you can follow our [Docker deployment guide](/self-hosting/docker) for manual setup.
+
 ## Should you self-host?
 
 CORE Cloud is fully managed, scalable, and comes with dedicated support. For most users, it offers the best experience. However, if you have specific requirements around data residency, compliance, or infrastructure control, self-hosting may be the right choice for you.
@@ -29,7 +37,6 @@ The self-hosted version of CORE is composed of several containers that you run o
 - **Webapp**: The main application container, responsible for serving the user interface and orchestrating memory operations.
 - **PostgreSQL**: Stores metadata, user accounts, and configuration data.
 - **Neo4j**: Graph database used for storing and querying the memory graph.
-- **[Trigger](https://trigger.dev/)**: Manages background jobs and workflows, such as data ingestion and memory formation. We use Trigger to reliably handle all background processing.
 - **Redis**: Provides caching and session management.
 
 This modular architecture allows you to scale each service as needed and gives you full control over your deployment.

hosting/docker/.env

@@ -1,4 +1,4 @@
-VERSION=0.1.25
+VERSION=0.1.26
 
 # Nest run in docker, change host to database container name
 DB_HOST=postgres
@@ -46,76 +46,10 @@ OPENAI_API_KEY=
 OLLAMA_URL=
 
 EMBEDDING_MODEL=text-embedding-3-small
+EMBEDDING_MODEL_SIZE=1536
 MODEL=gpt-4.1-2025-04-14
 
-## Trigger ##
-TRIGGER_PROJECT_ID=proj_mqwudvjcukvybqxyjkjv
-TRIGGER_SECRET_KEY=tr_prod_72iziCY2yWA5SdGxRFii
-
-
-# ------------------------------------------------------------
-# Trigger.dev self-hosting environment variables
-# - These are the default values for the self-hosting stack
-# - You should change them to suit your needs, especially the secrets
-# - See the docs for more information: https://trigger.dev/docs/self-hosting/overview
-
-# Secrets
-# - Do NOT use these defaults in production
-# - Generate your own by running `openssl rand -hex 16` for each secret
-MANAGED_WORKER_SECRET=447c29678f9eaf289e9c4b70d3dd8a7f
-
-# Worker token
-# - This is the token for the worker to connect to the webapp
-# - When running the combined stack, this is set automatically during bootstrap
-# - For the split setup, you will have to set this manually. The token is available in the webapp logs but will only be shown once.
-# - See the docs for more information: https://trigger.dev/docs/self-hosting/docker
-TRIGGER_WORKER_TOKEN=tr_wgt_jtRujkUnfK3RmNtUev049Clw7gaqwg77VMPGu7Iv
-TRIGGER_TASKS_IMAGE=redplanethq/proj_core:latest
-
-# Worker URLs
-# - In split setups, uncomment and set to the public URL of your webapp
-# TRIGGER_API_URL=https://trigger.example.com
-# OTEL_EXPORTER_OTLP_ENDPOINT=https://trigger.example.com/otel
-
-# Postgres
-# - Do NOT use these defaults in production
-# - Especially if you decide to expose the database to the internet
-# POSTGRES_USER=postgres
-TRIGGER_DB=trigger
-
-TRIGGER_DATABASE_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${DB_HOST}:${DB_PORT}/${TRIGGER_DB}?schema=public&sslmode=disable
-TRIGGER_DIRECT_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${DB_HOST}:${DB_PORT}/${TRIGGER_DB}?schema=public&sslmode=disable
-ELECTRIC_DATABASE_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${DB_HOST}/${TRIGGER_DB}
-
-# Trigger image tag
-# - This is the version of the webapp and worker images to use, they should be locked to a specific version in production
-# - For example: TRIGGER_IMAGE_TAG=v4.0.0-v4-beta.21
-TRIGGER_IMAGE_TAG=v4-beta
-
-# Webapp
-# - These should generally be set to the same value
-# - In production, these should be set to the public URL of your webapp, e.g. https://trigger.example.com
-APP_ORIGIN=http://localhost:8030
-LOGIN_ORIGIN=http://localhost:8030
-API_ORIGIN=http://trigger-webapp:3000
-DEV_OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:8030/otel
-# You may need to set this when testing locally or when using the combined setup
-# API_ORIGIN=http://webapp:3000
-
-
-
-# ClickHouse
-# - Do NOT use these defaults in production
-CLICKHOUSE_USER=default
-CLICKHOUSE_PASSWORD=password
-CLICKHOUSE_URL=http://default:password@clickhouse:8123?secure=false
-RUN_REPLICATION_CLICKHOUSE_URL=http://default:password@clickhouse:8123
-
-# Docker Registry
-# - When testing locally, the default values should be fine
-# - When deploying to production, you will have to change these, especially the password and URL
-# - See the docs for more information: https://trigger.dev/docs/self-hosting/docker#registry-setup
-DOCKER_REGISTRY_URL=docker.io
-DOCKER_REGISTRY_USERNAME=
-DOCKER_REGISTRY_PASSWORD=
+## for opensource embedding model
+# EMBEDDING_MODEL=mxbai-embed-large
 
+QUEUE_PROVIDER=bullmq 

hosting/docker/docker-compose.yaml

@@ -22,6 +22,7 @@ services:
       - APP_ORIGIN=${CORE_APP_ORIGIN}
       - REDIS_HOST=${REDIS_HOST}
       - REDIS_PORT=${REDIS_PORT}
+      - REDIS_PASSWORD=${REDIS_PASSWORD}
       - REDIS_TLS_DISABLED=${REDIS_TLS_DISABLED}
       - NEO4J_URI=${NEO4J_URI}
       - NEO4J_USERNAME=${NEO4J_USERNAME}
@@ -42,6 +43,9 @@ services:
       - FROM_EMAIL=${FROM_EMAIL}
       - RESEND_API_KEY=${RESEND_API_KEY}
       - COHERE_API_KEY=${COHERE_API_KEY}
+      - QUEUE_PROVIDER=${QUEUE_PROVIDER}
+      - TELEMETRY_ENABLED=${TELEMETRY_ENABLED}
+      - TELEMETRY_ANONYMOUS=${TELEMETRY_ANONYMOUS}
     ports:
       - "3033:3000"
     depends_on:
@@ -84,7 +88,7 @@ services:
 
   neo4j:
     container_name: core-neo4j
-    image: neo4j:5
+    image: redplanethq/neo4j:0.1.0
     environment:
       - NEO4J_AUTH=${NEO4J_AUTH}
       - NEO4J_dbms_security_procedures_unrestricted=gds.*,apoc.*
@@ -108,249 +112,33 @@ services:
       retries: 10
       start_period: 20s
 
-  webapp:
-    container_name: trigger-webapp
-    image: ghcr.io/triggerdotdev/trigger.dev:v4.0.4
-    restart: ${RESTART_POLICY:-unless-stopped}
-    logging: *logging-config
-    ports:
-      - ${WEBAPP_PUBLISH_IP:-0.0.0.0}:8030:3000
-    depends_on:
-      clickhouse:
-        condition: service_started
-      init:
-        condition: service_started
-    networks:
-      - webapp
-      - supervisor
-      - core
-    volumes:
-      - shared:/home/node/shared
-    # Only needed for bootstrap
-    user: root
-    # Only needed for bootstrap
-    command: sh -c "chown -R node:node /home/node/shared && sleep 5 && exec ./scripts/entrypoint.sh"
-    healthcheck:
-      test:
-        [
-          "CMD",
-          "node",
-          "-e",
-          "http.get('http://localhost:3000/healthcheck', res => process.exit(res.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))",
-        ]
-      interval: 30s
-      timeout: 10s
-      retries: 5
-      start_period: 10s
-    environment:
-      APP_ORIGIN: ${APP_ORIGIN:-http://localhost:8030}
-      LOGIN_ORIGIN: ${LOGIN_ORIGIN:-http://localhost:8030}
-      API_ORIGIN: ${API_ORIGIN:-http://localhost:8030}
-      ELECTRIC_ORIGIN: http://electric:3000
-      DATABASE_URL: ${TRIGGER_DATABASE_URL:-postgresql://postgres:postgres@postgres:5432/main?schema=public&sslmode=disable}
-      DIRECT_URL: ${TRIGGER_DIRECT_URL:-postgresql://postgres:postgres@postgres:5432/main?schema=public&sslmode=disable}
-      SESSION_SECRET: ${SESSION_SECRET}
-      MAGIC_LINK_SECRET: ${MAGIC_LINK_SECRET}
-      ENCRYPTION_KEY: ${ENCRYPTION_KEY}
-      MANAGED_WORKER_SECRET: ${MANAGED_WORKER_SECRET}
-      REDIS_HOST: core-redis
-      REDIS_PORT: 6379
-      REDIS_TLS_DISABLED: true
-      APP_LOG_LEVEL: info
-      DEV_OTEL_EXPORTER_OTLP_ENDPOINT: ${DEV_OTEL_EXPORTER_OTLP_ENDPOINT:-http://localhost:8030/otel}
-      DEPLOY_REGISTRY_HOST: ${DOCKER_REGISTRY_URL:-localhost:5000}
-      DEPLOY_REGISTRY_NAMESPACE: ${DOCKER_REGISTRY_NAMESPACE:-trigger}
-      OBJECT_STORE_BASE_URL: ${OBJECT_STORE_BASE_URL:-http://minio:9000}
-      OBJECT_STORE_ACCESS_KEY_ID: ${OBJECT_STORE_ACCESS_KEY_ID}
-      OBJECT_STORE_SECRET_ACCESS_KEY: ${OBJECT_STORE_SECRET_ACCESS_KEY}
-      GRACEFUL_SHUTDOWN_TIMEOUT: 1000
-      # Bootstrap - this will automatically set up a worker group for you
-      # This will NOT work for split deployments
-      TRIGGER_BOOTSTRAP_ENABLED: 1
-      TRIGGER_BOOTSTRAP_WORKER_GROUP_NAME: bootstrap
-      TRIGGER_BOOTSTRAP_WORKER_TOKEN_PATH: /home/node/shared/worker_token
-      # ClickHouse configuration
-      CLICKHOUSE_URL: ${CLICKHOUSE_URL:-http://default:password@clickhouse:8123?secure=false}
-      CLICKHOUSE_LOG_LEVEL: ${CLICKHOUSE_LOG_LEVEL:-info}
-      # Run replication
-      RUN_REPLICATION_ENABLED: ${RUN_REPLICATION_ENABLED:-1}
-      RUN_REPLICATION_CLICKHOUSE_URL: ${RUN_REPLICATION_CLICKHOUSE_URL:-http://default:password@clickhouse:8123}
-      RUN_REPLICATION_LOG_LEVEL: ${RUN_REPLICATION_LOG_LEVEL:-info}
-      # Limits
-      # TASK_PAYLOAD_OFFLOAD_THRESHOLD: 524288 # 512KB
-      # TASK_PAYLOAD_MAXIMUM_SIZE: 3145728 # 3MB
-      # BATCH_TASK_PAYLOAD_MAXIMUM_SIZE: 1000000 # 1MB
-      # TASK_RUN_METADATA_MAXIMUM_SIZE: 262144 # 256KB
-      # DEFAULT_ENV_EXECUTION_CONCURRENCY_LIMIT: 100
-      # DEFAULT_ORG_EXECUTION_CONCURRENCY_LIMIT: 100
-      # Internal OTEL configuration
-      INTERNAL_OTEL_TRACE_LOGGING_ENABLED: ${INTERNAL_OTEL_TRACE_LOGGING_ENABLED:-0}
-
-  electric:
-    container_name: trigger-electric
-    image: electricsql/electric:${ELECTRIC_IMAGE_TAG:-1.0.10}
-    restart: ${RESTART_POLICY:-unless-stopped}
-    logging: *logging-config
-    networks:
-      - webapp
-      - core
-    environment:
-      DATABASE_URL: ${ELECTRIC_DATABASE_URL:-postgresql://postgres:postgres@postgres:5432/main?schema=public&sslmode=disable}
-      ELECTRIC_INSECURE: true
-      ELECTRIC_USAGE_REPORTING: false
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:3000/v1/health"]
-      interval: 10s
-      timeout: 5s
-      retries: 5
-      start_period: 10s
-
-  clickhouse:
-    container_name: trigger-clickhouse
-    image: bitnami/clickhouse:${CLICKHOUSE_IMAGE_TAG:-latest}
-    restart: ${RESTART_POLICY:-unless-stopped}
-    logging: *logging-config
-    ports:
-      - ${CLICKHOUSE_PUBLISH_IP:-127.0.0.1}:9123:8123
-      - ${CLICKHOUSE_PUBLISH_IP:-127.0.0.1}:9090:9000
-    environment:
-      CLICKHOUSE_ADMIN_USER: ${CLICKHOUSE_USER:-default}
-      CLICKHOUSE_ADMIN_PASSWORD: ${CLICKHOUSE_PASSWORD:-password}
-    volumes:
-      - clickhouse:/bitnami/clickhouse
-      - ../clickhouse/override.xml:/bitnami/clickhouse/etc/config.d/override.xml:ro
-    networks:
-      - webapp
-    healthcheck:
-      test:
-        [
-          "CMD",
-          "clickhouse-client",
-          "--host",
-          "localhost",
-          "--port",
-          "9000",
-          "--user",
-          "default",
-          "--password",
-          "password",
-          "--query",
-          "SELECT 1",
-        ]
-      interval: 5s
-      timeout: 5s
-      retries: 5
-      start_period: 10s
-
-  # Worker related
-  supervisor:
-    container_name: trigger-supervisor
-    image: ghcr.io/triggerdotdev/supervisor:v4.0.4
-    restart: ${RESTART_POLICY:-unless-stopped}
-    logging: *logging-config
-    depends_on:
-      - docker-proxy
-    networks:
-      - supervisor
-      - docker-proxy
-      - webapp
-      - core
-    volumes:
-      - shared:/home/node/shared
-    # Only needed for bootstrap
-    user: root
-    # Only needed for bootstrap
-    command: sh -c "chown -R node:node /home/node/shared && exec /usr/bin/dumb-init -- pnpm run --filter supervisor start"
-    environment:
-      # This needs to match the token of the worker group you want to connect to
-      TRIGGER_WORKER_TOKEN: ${TRIGGER_WORKER_TOKEN}
-      # Use the bootstrap token created by the webapp
-      # TRIGGER_WORKER_TOKEN: file:///home/node/shared/worker_token
-      MANAGED_WORKER_SECRET: ${MANAGED_WORKER_SECRET}
-      TRIGGER_API_URL: ${TRIGGER_API_URL:-http://trigger-webapp:3000}
-      OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://trigger-webapp:3000/otel}
-      TRIGGER_WORKLOAD_API_DOMAIN: supervisor
-      TRIGGER_WORKLOAD_API_PORT_EXTERNAL: 8020
-      # Optional settings
-      DEBUG: 1
-      ENFORCE_MACHINE_PRESETS: 1
-      TRIGGER_DEQUEUE_INTERVAL_MS: 1000
-      DOCKER_HOST: tcp://docker-proxy:2375
-      DOCKER_RUNNER_NETWORKS: webapp,supervisor,core
-      DOCKER_REGISTRY_URL: ${DOCKER_REGISTRY_URL:-localhost:5000}
-      DOCKER_REGISTRY_USERNAME: ${DOCKER_REGISTRY_USERNAME:-}
-      DOCKER_REGISTRY_PASSWORD: ${DOCKER_REGISTRY_PASSWORD:-}
-      DOCKER_AUTOREMOVE_EXITED_CONTAINERS: 0
-    healthcheck:
-      test:
-        [
-          "CMD",
-          "node",
-          "-e",
-          "http.get('http://localhost:8020/health', res => process.exit(res.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))",
-        ]
-      interval: 30s
-      timeout: 10s
-      retries: 5
-      start_period: 10s
-
-  init:
-    container_name: trigger-init
-    image: redplanethq/init:${VERSION}
-    restart: "no" # prevent retries
-    environment:
-      - VERSION=${VERSION}
-      - DB_HOST=${DB_HOST}
-      - DB_PORT=${DB_PORT}
-      - TRIGGER_DB=${TRIGGER_DB}
-      - POSTGRES_USER=${POSTGRES_USER}
-      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
-      - TRIGGER_TASKS_IMAGE=${TRIGGER_TASKS_IMAGE}
-      - NODE_ENV=production
-    networks:
-      - webapp
-      - core
-    depends_on:
-      - postgres
-
-  docker-proxy:
-    container_name: trigger-docker-proxy
-    image: tecnativa/docker-socket-proxy:${DOCKER_PROXY_IMAGE_TAG:-latest}
-    restart: ${RESTART_POLICY:-unless-stopped}
-    logging: *logging-config
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock:ro
-    networks:
-      - docker-proxy
-    environment:
-      - LOG_LEVEL=info
-      - POST=1
-      - CONTAINERS=1
-      - IMAGES=1
-      - INFO=1
-      - NETWORKS=1
-    healthcheck:
-      test: ["CMD", "nc", "-z", "127.0.0.1", "2375"]
-      interval: 30s
-      timeout: 5s
-      retries: 5
-      start_period: 5s
+  # Uncomment this if you want to use a local embedding modal
+  # ollama:
+  #   container_name: core-ollama
+  #   image: ollama/ollama:0.12.6
+  #   ports:
+  #     - "11434:11434"
+  #   volumes:
+  #     - ollama_data:/root/.ollama
+  #     - ./scripts/ollama-init.sh:/usr/local/bin/ollama-init.sh:ro
+  #   networks:
+  #     - core
+  #   entrypoint: ["/bin/bash", "/usr/local/bin/ollama-init.sh"]
+  #   healthcheck:
+  #     test: ["CMD", "curl", "-f", "http://localhost:11434/api/tags"]
+  #     interval: 30s
+  #     timeout: 10s
+  #     retries: 5
+  #     start_period: 90s
+  #   restart: unless-stopped
 
 networks:
   core:
     name: core
     driver: bridge
-  docker-proxy:
-    name: docker-proxy
-  supervisor:
-    name: supervisor
-  webapp:
-    name: webapp
-    driver: bridge
 
 volumes:
   postgres_data:
   neo4j_data:
   shared:
-  clickhouse:
-  minio:
+  ollama_data:

hosting/docker/scripts/ollama-init.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+set -e
+
+echo "Starting Ollama server..."
+ollama serve &
+OLLAMA_PID=$!
+
+echo "Waiting for Ollama server to be ready..."
+sleep 5
+
+echo "Pulling mxbai-embed-large model..."
+ollama pull mxbai-embed-large
+
+echo "Model pulled successfully!"
+echo "Ollama is ready to accept requests."
+
+# Keep the Ollama server running
+wait $OLLAMA_PID