bump: version 0.1.27

* feat: space v3

* feat: connected space creation

* fix:

* fix: session_id for memory ingestion

* chore: simplify gitignore patterns for agent directories

---------

Co-authored-by: Manoj <saimanoj58@gmail.com>

.env.example

@@ -1,4 +1,4 @@
-VERSION=0.1.25
+VERSION=0.1.27
 
 # Nest run in docker, change host to database container name
 DB_HOST=localhost
@@ -41,10 +41,7 @@ NEO4J_USERNAME=neo4j
 NEO4J_PASSWORD=27192e6432564f4788d55c15131bd5ac
 OPENAI_API_KEY=
 
-
 MAGIC_LINK_SECRET=27192e6432564f4788d55c15131bd5ac
-
-
 NEO4J_AUTH=neo4j/27192e6432564f4788d55c15131bd5ac
 OLLAMA_URL=http://ollama:11434
 
@@ -56,7 +53,5 @@ AWS_ACCESS_KEY_ID=
 AWS_SECRET_ACCESS_KEY=
 AWS_REGION=us-east-1
 
-## Trigger ##
-TRIGGER_PROJECT_ID=
-TRIGGER_SECRET_KEY=
-TRIGGER_API_URL=http://host.docker.internal:8030
+QUEUE_PROVIDER=bullmq
+

.github/workflows/build-docker-image.yml

@@ -7,32 +7,6 @@ on:
   workflow_dispatch:
 
 jobs:
-  build-init:
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          ref: main
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-
-      - name: Login to Docker Registry
-        run: echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u "${{ secrets.DOCKER_USERNAME }}" --password-stdin
-
-      - name: Build and Push Frontend Docker Image
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          file: ./apps/init/Dockerfile
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: redplanethq/init:${{ github.ref_name }}
-
   build-webapp:
     runs-on: ubuntu-latest
 

.gitignore

@@ -46,13 +46,14 @@ registry/
 
 .cursor
 CLAUDE.md
+AGENTS.md
 
 .claude
-.clinerules/byterover-rules.md
-.kilocode/rules/byterover-rules.md
-.roo/rules/byterover-rules.md
-.windsurf/rules/byterover-rules.md
-.cursor/rules/byterover-rules.mdc
-.kiro/steering/byterover-rules.md
-.qoder/rules/byterover-rules.md
-.augment/rules/byterover-rules.md
+.clinerules
+.kilocode
+.roo
+.windsurf
+.cursor
+.kiro
+.qoder
+.augment

.vscode/settings.json

@@ -1,7 +0,0 @@
-{
-  "eslint.workingDirectories": [
-    {
-      "mode": "auto"
-    }
-  ]
-}

README.md

@@ -55,7 +55,7 @@ CORE memory achieves **88.24%** average accuracy in Locomo dataset across all re
 
 ## Overview
 
-**Problem** 
+**Problem**
 
 Developers waste time re-explaining context to AI tools. Hit token limits in Claude? Start fresh and lose everything. Switch from ChatGPT/Claude to Cursor? Explain your context again. Your conversations, decisions, and insights vanish between sessions. With every new AI tool, the cost of context switching grows.
 
@@ -64,8 +64,13 @@ Developers waste time re-explaining context to AI tools. Hit token limits in Cla
 CORE is an open-source unified, persistent memory layer for all your AI tools. Your context follows you from Cursor to Claude to ChatGPT to Claude Code. One knowledge graph remembers who said what, when, and why. Connect once, remember everywhere. Stop managing context and start building.
 
 ## 🚀 CORE Self-Hosting
+
 Want to run CORE on your own infrastructure? Self-hosting gives you complete control over your data and deployment.
 
+**Quick Deploy Options:**
+
+[![Deploy on Railway](https://railway.com/button.svg)](https://railway.com/deploy/core?referralCode=LHvbIb&utm_medium=integration&utm_source=template&utm_campaign=generic)
+
 **Prerequisites**:
 
 - Docker (20.10.0+) and Docker Compose (2.20.0+) installed
@@ -76,15 +81,20 @@ Want to run CORE on your own infrastructure? Self-hosting gives you complete con
 ### Setup
 
 1. Clone the repository:
+
 ```
 git clone https://github.com/RedPlanetHQ/core.git
 cd core
 ```
+
 2. Configure environment variables in `core/.env`:
+
 ```
 OPENAI_API_KEY=your_openai_api_key
 ```
+
 3. Start the service
+
 ```
 docker-compose up -d
 ```
@@ -96,6 +106,7 @@ Once deployed, you can configure your AI providers (OpenAI, Anthropic) and start
 Note: We tried open-source models like Ollama or GPT OSS but facts generation were not good, we are still figuring out how to improve on that and then will also support OSS models.
 
 ## 🚀 CORE Cloud
+
 **Build your unified memory graph in 5 minutes:**
 
 Don't want to manage infrastructure? CORE Cloud lets you build your personal memory system instantly - no setup, no servers, just memory that works.
@@ -111,24 +122,24 @@ Don't want to manage infrastructure? CORE Cloud lets you build your personal mem
 
 ## 🧩 Key Features
 
-### 🧠 **Unified, Portable Memory**: 
+### 🧠 **Unified, Portable Memory**:
+
 Add and recall your memory across **Cursor, Windsurf, Claude Desktop, Claude Code, Gemini CLI, AWS's Kiro, VS Code, and Roo Code** via MCP
 
 ![core-claude](https://github.com/user-attachments/assets/56c98288-ee87-4cd0-8b02-860aca1c7f9a)
 
-
-### 🕸️ **Temporal + Reified Knowledge Graph**: 
+### 🕸️ **Temporal + Reified Knowledge Graph**:
 
 Remember the story behind every fact—track who said what, when, and why with rich relationships and full provenance, not just flat storage
 
 ![core-memory-graph](https://github.com/user-attachments/assets/5d1ee659-d519-4624-85d1-e0497cbdd60a)
 
-
-### 🌐 **Browser Extension**: 
+### 🌐 **Browser Extension**:
 
 Save conversations and content from ChatGPT, Grok, Gemini, Twitter, YouTube, blog posts, and any webpage directly into your CORE memory.
 
 **How to Use Extension**
+
 1. [Download the Extension](https://chromewebstore.google.com/detail/core-extension/cglndoindnhdbfcbijikibfjoholdjcc) from the Chrome Web Store.
 2. Login to [CORE dashboard](https://core.heysol.ai)
    - Navigate to Settings (bottom left)
@@ -137,13 +148,12 @@ Save conversations and content from ChatGPT, Grok, Gemini, Twitter, YouTube, blo
 
 https://github.com/user-attachments/assets/6e629834-1b9d-4fe6-ae58-a9068986036a
 
+### 💬 **Chat with Memory**:
 
-### 💬 **Chat with Memory**: 
 Ask questions like "What are my writing preferences?" with instant insights from your connected knowledge
 
 ![chat-with-memory](https://github.com/user-attachments/assets/d798802f-bd51-4daf-b2b5-46de7d206f66)
 
-
 ### ⚡ **Auto-Sync from Apps**:
 
 Automatically capture relevant context from Linear, Slack, Notion, GitHub and other connected apps into your CORE memory
@@ -152,16 +162,12 @@ Automatically capture relevant context from Linear, Slack, Notion, GitHub and ot
 
 ![core-slack](https://github.com/user-attachments/assets/d5fefe38-221e-4076-8a44-8ed673960f03)
 
-
-### 🔗 **MCP Integration Hub**: 
+### 🔗 **MCP Integration Hub**:
 
 Connect Linear, Slack, GitHub, Notion once to CORE—then use all their tools in Claude, Cursor, or any MCP client with a single URL
 
-
 ![core-linear-claude](https://github.com/user-attachments/assets/7d59d92b-8c56-4745-a7ab-9a3c0341aa32)
 
-
-
 ## How CORE create memory
 
 <img width="12885" height="3048" alt="memory-ingest-diagram" src="https://github.com/user-attachments/assets/c51679de-8260-4bee-bebf-aff32c6b8e13" />
@@ -175,7 +181,6 @@ CORE’s ingestion pipeline has four phases designed to capture evolving context
 
 The Result: Instead of a flat database, CORE gives you a memory that grows and changes with you - preserving context, evolution, and ownership so agents can actually use it.
 
-
 ![memory-ingest-eg](https://github.com/user-attachments/assets/1d0a8007-153a-4842-9586-f6f4de43e647)
 
 ## How CORE recalls from memory
@@ -200,7 +205,7 @@ Explore our documentation to get the most out of CORE
 - [Connect Core MCP with Claude](https://docs.heysol.ai/providers/claude)
 - [Connect Core MCP with Cursor](https://docs.heysol.ai/providers/cursor)
 - [Connect Core MCP with Claude Code](https://docs.heysol.ai/providers/claude-code)
-- [Connect Core MCP with Codex](https://docs.heysol.ai/providers/codex) 
+- [Connect Core MCP with Codex](https://docs.heysol.ai/providers/codex)
 
 - [Basic Concepts](https://docs.heysol.ai/overview)
 - [API Reference](https://docs.heysol.ai/api-reference/get-user-profile)
@@ -245,16 +250,11 @@ Have questions or feedback? We're here to help:
 <a href="https://github.com/RedPlanetHQ/core/graphs/contributors">
   <img src="https://contrib.rocks/image?repo=RedPlanetHQ/core" />
 </a>
+<<<<<<< Updated upstream
 
+<<<<<<< HEAD
 
+# =======
 
-
-
-
-
-
-
-
-
-
-
+> > > > > > > Stashed changes
+> > > > > > > 62db6c1 (feat: automatic space identification)

apps/init/.gitignore

@@ -1,51 +0,0 @@
-# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
-
-# Dependencies
-node_modules
-.pnp
-.pnp.js
-
-# Local env files
-.env
-.env.local
-.env.development.local
-.env.test.local
-.env.production.local
-
-# Testing
-coverage
-
-# Turbo
-.turbo
-
-# Vercel
-.vercel
-
-# Build Outputs
-.next/
-out/
-build
-dist
-.tshy/
-.tshy-build/
-
-# Debug
-npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
-
-# Misc
-.DS_Store
-*.pem
-
-docker-compose.dev.yaml
-
-clickhouse/
-.vscode/
-registry/
-
-.cursor
-CLAUDE.md
-
-.claude
-

apps/init/Dockerfile

@@ -1,70 +0,0 @@
-ARG NODE_IMAGE=node:20.11.1-bullseye-slim@sha256:5a5a92b3a8d392691c983719dbdc65d9f30085d6dcd65376e7a32e6fe9bf4cbe
-
-FROM ${NODE_IMAGE} AS pruner
-
-WORKDIR /core
-
-COPY --chown=node:node . .
-RUN npx -q turbo@2.5.3 prune --scope=@redplanethq/init --docker
-RUN find . -name "node_modules" -type d -prune -exec rm -rf '{}' +
-
-# Base strategy to have layer caching
-FROM ${NODE_IMAGE} AS base
-RUN apt-get update && apt-get install -y openssl dumb-init postgresql-client
-WORKDIR /core
-COPY --chown=node:node .gitignore .gitignore
-COPY --from=pruner --chown=node:node /core/out/json/ .
-COPY --from=pruner --chown=node:node /core/out/pnpm-lock.yaml ./pnpm-lock.yaml
-COPY --from=pruner --chown=node:node /core/out/pnpm-workspace.yaml ./pnpm-workspace.yaml
-
-## Dev deps
-FROM base AS dev-deps
-WORKDIR /core
-# Corepack is used to install pnpm
-RUN corepack enable
-ENV NODE_ENV development
-RUN pnpm install --ignore-scripts --no-frozen-lockfile
-
-## Production deps
-FROM base AS production-deps
-WORKDIR /core
-# Corepack is used to install pnpm
-RUN corepack enable
-ENV NODE_ENV production
-RUN pnpm install --prod --no-frozen-lockfile
-
-## Builder (builds the init CLI)
-FROM base AS builder
-WORKDIR /core
-# Corepack is used to install pnpm
-RUN corepack enable
-
-COPY --from=pruner --chown=node:node /core/out/full/ .
-COPY --from=dev-deps --chown=node:node /core/ .
-COPY --chown=node:node turbo.json turbo.json
-COPY --chown=node:node .configs/tsconfig.base.json .configs/tsconfig.base.json
-RUN pnpm run build --filter=@redplanethq/init...
-
-# Runner
-FROM ${NODE_IMAGE} AS runner
-RUN apt-get update && apt-get install -y openssl postgresql-client ca-certificates
-WORKDIR /core
-RUN corepack enable
-ENV NODE_ENV production
-
-COPY --from=base /usr/bin/dumb-init /usr/bin/dumb-init
-COPY --from=pruner --chown=node:node /core/out/full/ .
-COPY --from=production-deps --chown=node:node /core .
-COPY --from=builder --chown=node:node /core/apps/init/dist ./apps/init/dist
-
-# Copy the trigger dump file
-COPY --chown=node:node apps/init/trigger.dump ./apps/init/trigger.dump
-
-# Copy and set up entrypoint script
-COPY --chown=node:node apps/init/entrypoint.sh ./apps/init/entrypoint.sh
-RUN chmod +x ./apps/init/entrypoint.sh
-
-USER node
-WORKDIR /core/apps/init
-ENTRYPOINT ["dumb-init", "--"]
-CMD ["./entrypoint.sh"]

apps/init/README.md

@@ -1,197 +0,0 @@
-# Core CLI
-
-🧠 **CORE - Contextual Observation & Recall Engine**
-
-A Command-Line Interface for setting up and managing the Core development environment.
-
-## Installation
-
-```bash
-npm install -g @redplanethq/core
-```
-
-## Commands
-
-### `core init`
-
-**One-time setup command** - Initializes the Core development environment with full configuration.
-
-### `core start`
-
-**Daily usage command** - Starts all Core services (Docker containers).
-
-### `core stop`
-
-**Daily usage command** - Stops all Core services (Docker containers).
-
-## Getting Started
-
-### Prerequisites
-
-- **Node.js** (v18.20.0 or higher)
-- **Docker** and **Docker Compose**
-- **Git**
-- **pnpm** package manager
-
-### Initial Setup
-
-1. **Clone the Core repository:**
-   ```bash
-   git clone https://github.com/redplanethq/core.git
-   cd core
-   ```
-
-2. **Run the initialization command:**
-   ```bash
-   core init
-   ```
-
-3. **The CLI will guide you through the complete setup process:**
-
-#### Step 1: Prerequisites Check
-- The CLI shows a checklist of required tools
-- Confirms you're in the Core repository directory
-- Exits with instructions if prerequisites aren't met
-
-#### Step 2: Environment Configuration
-
-- Copies `.env.example` to `.env` in the root directory
-- Copies `trigger/.env.example` to `trigger/.env`
-- Skips copying if `.env` files already exist
-
-#### Step 3: Docker Services Startup
-
-- Starts main Core services: `docker compose up -d`
-- Starts Trigger.dev services: `docker compose up -d` (in trigger/ directory)
-- Shows real-time output with progress indicators
-
-#### Step 4: Database Health Check
-
-- Verifies PostgreSQL is running on `localhost:5432`
-- Retries for up to 60 seconds if needed
-
-#### Step 5: Trigger.dev Setup (Interactive)
-
-- **If Trigger.dev is not configured:**
-
-  1. Prompts you to open http://localhost:8030
-  2. Asks you to login to Trigger.dev
-  3. Guides you to create an organization and project
-  4. Collects your Project ID and Secret Key
-  5. Updates `.env` with your Trigger.dev configuration
-  6. Restarts Core services with new configuration
-
-- **If Trigger.dev is already configured:**
-  - Skips setup and shows "Configuration already exists" message
-
-#### Step 6: Docker Registry Login
-
-- Displays docker login command with credentials from `.env`
-- Waits for you to complete the login process
-
-#### Step 7: Trigger.dev Task Deployment
-
-- Automatically runs: `npx trigger.dev@v4-beta login -a http://localhost:8030`
-- Deploys tasks with: `pnpm trigger:deploy`
-- Shows manual deployment instructions if automatic deployment fails
-
-#### Step 8: Setup Complete!
-
-- Confirms all services are running
-- Shows service URLs and connection information
-
-## Daily Usage
-
-After initial setup, use these commands for daily development:
-
-### Start Services
-
-```bash
-core start
-```
-
-Starts all Docker containers for Core development.
-
-### Stop Services
-
-```bash
-core stop
-```
-
-Stops all Docker containers.
-
-## Service URLs
-
-After setup, these services will be available:
-
-- **Core Application**: http://localhost:3033
-- **Trigger.dev**: http://localhost:8030
-- **PostgreSQL**: localhost:5432
-
-## Troubleshooting
-
-### Repository Not Found
-
-If you run commands outside the Core repository:
-
-- The CLI will ask you to confirm you're in the Core repository
-- If not, it provides instructions to clone the repository
-- Navigate to the Core repository directory before running commands again
-
-### Docker Issues
-
-- Ensure Docker is running
-- Check Docker Compose is installed
-- Verify you have sufficient system resources
-
-### Trigger.dev Setup Issues
-
-- Check container logs: `docker logs trigger-webapp --tail 50`
-- Ensure you can access http://localhost:8030
-- Verify your network allows connections to localhost
-
-### Environment Variables
-
-The CLI automatically manages these environment variables:
-
-- `TRIGGER_PROJECT_ID` - Your Trigger.dev project ID
-- `TRIGGER_SECRET_KEY` - Your Trigger.dev secret key
-- Docker registry credentials for deployment
-
-### Manual Trigger.dev Deployment
-
-If automatic deployment fails, run manually:
-
-```bash
-npx trigger.dev@v4-beta login -a http://localhost:8030
-pnpm trigger:deploy
-```
-
-## Development Workflow
-
-1. **First time setup:** `core init`
-2. **Daily development:**
-   - `core start` - Start your development environment
-   - Do your development work
-   - `core stop` - Stop services when done
-
-## Support
-
-For issues and questions:
-
-- Check the main Core repository: https://github.com/redplanethq/core
-- Review Docker container logs for troubleshooting
-- Ensure all prerequisites are properly installed
-
-## Features
-
-- 🚀 **One-command setup** - Complete environment initialization
-- 🔄 **Smart configuration** - Skips already configured components
-- 📱 **Real-time feedback** - Live progress indicators and output
-- 🐳 **Docker integration** - Full container lifecycle management
-- 🔧 **Interactive setup** - Guided configuration process
-- 🎯 **Error handling** - Graceful failure with recovery instructions
-
----
-
-**Happy coding with Core!** 🎉

apps/init/entrypoint.sh

@@ -1,22 +0,0 @@
-#!/bin/sh
-
-# Exit on any error
-set -e
-
-echo "Starting init CLI..."
-
-# Wait for database to be ready
-echo "Waiting for database connection..."
-until pg_isready -h "${DB_HOST:-localhost}" -p "${DB_PORT:-5432}" -U "${POSTGRES_USER:-docker}"; do
-  echo "Database is unavailable - sleeping"
-  sleep 2
-done
-
-echo "Database is ready!"
-
-# Run the init command
-echo "Running init command..."
-node ./dist/esm/index.js init
-
-echo "Init completed successfully!"
-exit 0

apps/init/package.json

@@ -1,145 +0,0 @@
-{
-  "name": "@redplanethq/init",
-  "version": "0.1.0",
-  "description": "A init service to create trigger instance",
-  "type": "module",
-  "license": "MIT",
-  "repository": {
-    "type": "git",
-    "url": "https://github.com/redplanethq/core",
-    "directory": "apps/init"
-  },
-  "publishConfig": {
-    "access": "public"
-  },
-  "keywords": [
-    "typescript"
-  ],
-  "files": [
-    "dist",
-    "trigger.dump"
-  ],
-  "bin": {
-    "core": "./dist/esm/index.js"
-  },
-  "tshy": {
-    "selfLink": false,
-    "main": false,
-    "module": false,
-    "dialects": [
-      "esm"
-    ],
-    "project": "./tsconfig.json",
-    "exclude": [
-      "**/*.test.ts"
-    ],
-    "exports": {
-      "./package.json": "./package.json",
-      ".": "./src/index.ts"
-    }
-  },
-  "devDependencies": {
-    "@epic-web/test-server": "^0.1.0",
-    "@types/gradient-string": "^1.1.2",
-    "@types/ini": "^4.1.1",
-    "@types/object-hash": "3.0.6",
-    "@types/polka": "^0.5.7",
-    "@types/react": "^18.2.48",
-    "@types/resolve": "^1.20.6",
-    "@types/rimraf": "^4.0.5",
-    "@types/semver": "^7.5.0",
-    "@types/source-map-support": "0.5.10",
-    "@types/ws": "^8.5.3",
-    "cpy-cli": "^5.0.0",
-    "execa": "^8.0.1",
-    "find-up": "^7.0.0",
-    "rimraf": "^5.0.7",
-    "ts-essentials": "10.0.1",
-    "tshy": "^3.0.2",
-    "tsx": "4.17.0"
-  },
-  "scripts": {
-    "clean": "rimraf dist .tshy .tshy-build .turbo",
-    "typecheck": "tsc -p tsconfig.src.json --noEmit",
-    "build": "tshy",
-    "test": "vitest",
-    "test:e2e": "vitest --run -c ./e2e/vitest.config.ts"
-  },
-  "dependencies": {
-    "@clack/prompts": "^0.10.0",
-    "@depot/cli": "0.0.1-cli.2.80.0",
-    "@opentelemetry/api": "1.9.0",
-    "@opentelemetry/api-logs": "0.52.1",
-    "@opentelemetry/exporter-logs-otlp-http": "0.52.1",
-    "@opentelemetry/exporter-trace-otlp-http": "0.52.1",
-    "@opentelemetry/instrumentation": "0.52.1",
-    "@opentelemetry/instrumentation-fetch": "0.52.1",
-    "@opentelemetry/resources": "1.25.1",
-    "@opentelemetry/sdk-logs": "0.52.1",
-    "@opentelemetry/sdk-node": "0.52.1",
-    "@opentelemetry/sdk-trace-base": "1.25.1",
-    "@opentelemetry/sdk-trace-node": "1.25.1",
-    "@opentelemetry/semantic-conventions": "1.25.1",
-    "ansi-escapes": "^7.0.0",
-    "braces": "^3.0.3",
-    "c12": "^1.11.1",
-    "chalk": "^5.2.0",
-    "chokidar": "^3.6.0",
-    "cli-table3": "^0.6.3",
-    "commander": "^9.4.1",
-    "defu": "^6.1.4",
-    "dotenv": "^16.4.5",
-    "dotenv-expand": "^12.0.2",
-    "esbuild": "^0.23.0",
-    "eventsource": "^3.0.2",
-    "evt": "^2.4.13",
-    "fast-npm-meta": "^0.2.2",
-    "git-last-commit": "^1.0.1",
-    "gradient-string": "^2.0.2",
-    "has-flag": "^5.0.1",
-    "import-in-the-middle": "1.11.0",
-    "import-meta-resolve": "^4.1.0",
-    "ini": "^5.0.0",
-    "jsonc-parser": "3.2.1",
-    "magicast": "^0.3.4",
-    "minimatch": "^10.0.1",
-    "mlly": "^1.7.1",
-    "nypm": "^0.5.4",
-    "nanoid": "3.3.8",
-    "object-hash": "^3.0.0",
-    "open": "^10.0.3",
-    "knex": "3.1.0",
-    "p-limit": "^6.2.0",
-    "p-retry": "^6.1.0",
-    "partysocket": "^1.0.2",
-    "pkg-types": "^1.1.3",
-    "polka": "^0.5.2",
-    "pg": "8.16.3",
-    "resolve": "^1.22.8",
-    "semver": "^7.5.0",
-    "signal-exit": "^4.1.0",
-    "source-map-support": "0.5.21",
-    "std-env": "^3.7.0",
-    "supports-color": "^10.0.0",
-    "tiny-invariant": "^1.2.0",
-    "tinyexec": "^0.3.1",
-    "tinyglobby": "^0.2.10",
-    "uuid": "11.1.0",
-    "ws": "^8.18.0",
-    "xdg-app-paths": "^8.3.0",
-    "zod": "3.23.8",
-    "zod-validation-error": "^1.5.0"
-  },
-  "engines": {
-    "node": ">=18.20.0"
-  },
-  "exports": {
-    "./package.json": "./package.json",
-    ".": {
-      "import": {
-        "types": "./dist/esm/index.d.ts",
-        "default": "./dist/esm/index.js"
-      }
-    }
-  }
-}

apps/init/src/cli/index.ts

@@ -1,14 +0,0 @@
-import { Command } from "commander";
-import { initCommand } from "../commands/init.js";
-import { VERSION } from "./version.js";
-
-const program = new Command();
-
-program.name("core").description("Core CLI - A Command-Line Interface for Core").version(VERSION);
-
-program
-  .command("init")
-  .description("Initialize Core development environment (run once)")
-  .action(initCommand);
-
-program.parse(process.argv);

apps/init/src/cli/version.ts

@@ -1,3 +0,0 @@
-import { env } from "../utils/env.js";
-
-export const VERSION = env.VERSION;

apps/init/src/commands/init.ts

@@ -1,36 +0,0 @@
-import { intro, outro, note } from "@clack/prompts";
-import { printCoreBrainLogo } from "../utils/ascii.js";
-import { initTriggerDatabase, updateWorkerImage } from "../utils/trigger.js";
-
-export async function initCommand() {
-  // Display the CORE brain logo
-  printCoreBrainLogo();
-
-  intro("🚀 Core Development Environment Setup");
-
-  try {
-    await initTriggerDatabase();
-    await updateWorkerImage();
-
-    note(
-      [
-        "Your services will start running:",
-        "",
-        "• Core Application: http://localhost:3033",
-        "• Trigger.dev: http://localhost:8030",
-        "• PostgreSQL: localhost:5432",
-        "",
-        "You can now start developing with Core!",
-        "",
-        "ℹ️  When logging in to the Core Application, you can find the login URL in the Docker container logs:",
-        "    docker logs core-app --tail 50",
-      ].join("\n"),
-      "🚀 Services Running"
-    );
-    outro("🎉 Setup Complete!");
-    process.exit(0);
-  } catch (error: any) {
-    outro(`❌ Setup failed: ${error.message}`);
-    process.exit(1);
-  }
-}

apps/init/src/index.ts

@@ -1,3 +0,0 @@
-#!/usr/bin/env node
-
-import "./cli/index.js";

apps/init/src/utils/ascii.ts

@@ -1,29 +0,0 @@
-import chalk from "chalk";
-import { VERSION } from "../cli/version.js";
-
-export function printCoreBrainLogo(): void {
-  const brain = `
-    ██████╗ ██████╗ ██████╗ ███████╗
-   ██╔════╝██╔═══██╗██╔══██╗██╔════╝
-   ██║     ██║   ██║██████╔╝█████╗  
-   ██║     ██║   ██║██╔══██╗██╔══╝  
-   ╚██████╗╚██████╔╝██║  ██║███████╗
-    ╚═════╝ ╚═════╝ ╚═╝  ╚═╝╚══════╝
-                                    
-         o     o     o
-       o   o---o---o   o
-      o---o   o   o---o---o
-     o   o---o---o---o   o
-      o---o   o   o---o---o
-       o   o---o---o   o
-         o     o     o
-
-  `;
-
-  console.log(chalk.cyan(brain));
-  console.log(
-    chalk.bold.white(
-      `    🧠 CORE - Contextual Observation & Recall Engine ${VERSION ? chalk.gray(`(${VERSION})`) : ""}\n`
-    )
-  );
-}

apps/init/src/utils/env.ts

@@ -1,24 +0,0 @@
-import { z } from "zod";
-
-const EnvironmentSchema = z.object({
-  // Version
-  VERSION: z.string().default("0.1.24"),
-
-  // Database
-  DB_HOST: z.string().default("localhost"),
-  DB_PORT: z.string().default("5432"),
-  TRIGGER_DB: z.string().default("trigger"),
-  POSTGRES_USER: z.string().default("docker"),
-  POSTGRES_PASSWORD: z.string().default("docker"),
-
-  // Trigger database
-  TRIGGER_TASKS_IMAGE: z.string().default("redplanethq/proj_core:latest"),
-
-  // Node environment
-  NODE_ENV: z
-    .union([z.literal("development"), z.literal("production"), z.literal("test")])
-    .default("development"),
-});
-
-export type Environment = z.infer<typeof EnvironmentSchema>;
-export const env = EnvironmentSchema.parse(process.env);

apps/init/src/utils/trigger.ts

@@ -1,182 +0,0 @@
-import Knex from "knex";
-import path from "path";
-import { fileURLToPath } from "url";
-import { env } from "./env.js";
-import { spinner, note, log } from "@clack/prompts";
-
-const __filename = fileURLToPath(import.meta.url);
-const __dirname = path.dirname(__filename);
-
-/**
- * Returns a PostgreSQL database URL for the given database name.
- * Throws if required environment variables are missing.
- */
-export function getDatabaseUrl(dbName: string): string {
-  const { POSTGRES_USER, POSTGRES_PASSWORD, DB_HOST, DB_PORT } = env;
-
-  if (!POSTGRES_USER || !POSTGRES_PASSWORD || !DB_HOST || !DB_PORT || !dbName) {
-    throw new Error(
-      "One or more required environment variables are missing: POSTGRES_USER, POSTGRES_PASSWORD, DB_HOST, DB_PORT, dbName"
-    );
-  }
-
-  return `postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${DB_HOST}:${DB_PORT}/${dbName}`;
-}
-
-/**
- * Checks if the database specified by TRIGGER_DB exists, and creates it if it does not.
- * Returns { exists: boolean, created: boolean } - exists indicates success, created indicates if database was newly created.
- */
-export async function ensureDatabaseExists(): Promise<{ exists: boolean; created: boolean }> {
-  const { TRIGGER_DB } = env;
-
-  if (!TRIGGER_DB) {
-    throw new Error("TRIGGER_DB environment variable is missing");
-  }
-
-  // Build a connection string to the default 'postgres' database
-  const adminDbUrl = getDatabaseUrl("postgres");
-
-  // Create a Knex instance for the admin connection
-  const adminKnex = Knex({
-    client: "pg",
-    connection: adminDbUrl,
-  });
-
-  const s = spinner();
-  s.start("Checking for Trigger.dev database...");
-
-  try {
-    // Check if the database exists
-    const result = await adminKnex.select(1).from("pg_database").where("datname", TRIGGER_DB);
-
-    if (result.length === 0) {
-      s.message("Database not found. Creating...");
-      // Database does not exist, create it
-      await adminKnex.raw(`CREATE DATABASE "${TRIGGER_DB}"`);
-      s.stop("Database created.");
-      return { exists: true, created: true };
-    } else {
-      s.stop("Database exists.");
-      return { exists: true, created: false };
-    }
-  } catch (err) {
-    s.stop("Failed to ensure database exists.");
-    log.warning("Failed to ensure database exists: " + (err as Error).message);
-    return { exists: false, created: false };
-  } finally {
-    await adminKnex.destroy();
-  }
-}
-
-// Main initialization function
-export async function initTriggerDatabase() {
-  const { TRIGGER_DB } = env;
-
-  if (!TRIGGER_DB) {
-    throw new Error("TRIGGER_DB environment variable is missing");
-  }
-
-  // Ensure the database exists
-  const { exists, created } = await ensureDatabaseExists();
-  if (!exists) {
-    throw new Error("Failed to create or verify database exists");
-  }
-
-  // Only run pg_restore if the database was newly created
-  if (!created) {
-    note("Database already exists, skipping restore from trigger.dump");
-    return;
-  }
-
-  // Run pg_restore with the trigger.dump file
-  const dumpFilePath = path.join(__dirname, "../../../trigger.dump");
-  const connectionString = getDatabaseUrl(TRIGGER_DB);
-
-  const s = spinner();
-  s.start("Restoring database from trigger.dump...");
-
-  try {
-    // Use execSync and capture stdout/stderr, send to spinner.log
-    const { spawn } = await import("child_process");
-    await new Promise<void>((resolve, reject) => {
-      const child = spawn(
-        "pg_restore",
-        ["--verbose", "--no-acl", "--no-owner", "-d", connectionString, dumpFilePath],
-        { stdio: ["ignore", "pipe", "pipe"] }
-      );
-
-      child.stdout.on("data", (data) => {
-        s.message(data.toString());
-      });
-
-      child.stderr.on("data", (data) => {
-        s.message(data.toString());
-      });
-
-      child.on("close", (code) => {
-        if (code === 0) {
-          s.stop("Database restored successfully from trigger.dump");
-          resolve();
-        } else {
-          s.stop("Failed to restore database.");
-          log.warning(`Failed to restore database: pg_restore exited with code ${code}`);
-          reject(new Error(`Database restore failed: pg_restore exited with code ${code}`));
-        }
-      });
-
-      child.on("error", (err) => {
-        s.stop("Failed to restore database.");
-        log.warning("Failed to restore database: " + err.message);
-        reject(new Error(`Database restore failed: ${err.message}`));
-      });
-    });
-  } catch (error: any) {
-    s.stop("Failed to restore database.");
-    log.warning("Failed to restore database: " + error.message);
-    throw new Error(`Database restore failed: ${error.message}`);
-  }
-}
-
-export async function updateWorkerImage() {
-  const { TRIGGER_DB, TRIGGER_TASKS_IMAGE } = env;
-
-  if (!TRIGGER_DB) {
-    throw new Error("TRIGGER_DB environment variable is missing");
-  }
-
-  const connectionString = getDatabaseUrl(TRIGGER_DB);
-
-  const knex = Knex({
-    client: "pg",
-    connection: connectionString,
-  });
-
-  const s = spinner();
-  s.start("Updating worker image reference...");
-
-  try {
-    // Get the first record from WorkerDeployment table
-    const firstWorkerDeployment = await knex("WorkerDeployment").select("id").first();
-
-    if (!firstWorkerDeployment) {
-      s.stop("No WorkerDeployment records found, skipping image update");
-      note("No WorkerDeployment records found, skipping image update");
-      return;
-    }
-
-    // Update the imageReference column with the TRIGGER_TASKS_IMAGE value
-    await knex("WorkerDeployment").where("id", firstWorkerDeployment.id).update({
-      imageReference: TRIGGER_TASKS_IMAGE,
-      updatedAt: new Date(),
-    });
-
-    s.stop(`Successfully updated worker image reference to: ${TRIGGER_TASKS_IMAGE}`);
-  } catch (error: any) {
-    s.stop("Failed to update worker image.");
-    log.warning("Failed to update worker image: " + error.message);
-    throw new Error(`Worker image update failed: ${error.message}`);
-  } finally {
-    await knex.destroy();
-  }
-}

apps/init/tsconfig.json

@@ -1,40 +0,0 @@
-{
-  "include": ["./src/**/*.ts"],
-  "exclude": ["./src/**/*.test.ts"],
-  "compilerOptions": {
-    "target": "es2022",
-    "lib": ["ES2022", "DOM", "DOM.Iterable", "DOM.AsyncIterable"],
-    "module": "NodeNext",
-    "moduleResolution": "NodeNext",
-    "moduleDetection": "force",
-    "verbatimModuleSyntax": false,
-    "jsx": "react",
-
-    "strict": true,
-    "alwaysStrict": true,
-    "strictPropertyInitialization": true,
-    "skipLibCheck": true,
-    "forceConsistentCasingInFileNames": true,
-    "noUnusedLocals": false,
-    "noUnusedParameters": false,
-    "noImplicitAny": true,
-    "noImplicitReturns": true,
-    "noImplicitThis": true,
-
-    "noFallthroughCasesInSwitch": true,
-    "resolveJsonModule": true,
-
-    "removeComments": false,
-    "esModuleInterop": true,
-    "emitDecoratorMetadata": false,
-    "experimentalDecorators": false,
-    "downlevelIteration": true,
-    "isolatedModules": true,
-    "noUncheckedIndexedAccess": true,
-
-    "pretty": true,
-    "isolatedDeclarations": false,
-    "composite": true,
-    "sourceMap": true
-  }
-}

apps/init/vite.config.ts

@@ -1,8 +0,0 @@
-import { configDefaults, defineConfig } from "vitest/config";
-
-export default defineConfig({
-  test: {
-    globals: true,
-    exclude: [...configDefaults.exclude, "e2e/**/*"],
-  },
-});

apps/webapp/app/bullmq/connection.ts

@@ -0,0 +1,50 @@
+import Redis, { type RedisOptions } from "ioredis";
+
+let redisConnection: Redis | null = null;
+
+/**
+ * Get or create a Redis connection for BullMQ
+ * This connection is shared across all queues and workers
+ */
+export function getRedisConnection() {
+  if (redisConnection) {
+    return redisConnection;
+  }
+
+  // Dynamically import ioredis only when needed
+
+  const redisConfig: RedisOptions = {
+    host: process.env.REDIS_HOST,
+    port: parseInt(process.env.REDIS_PORT as string),
+    password: process.env.REDIS_PASSWORD,
+    maxRetriesPerRequest: null, // Required for BullMQ
+    enableReadyCheck: false, // Required for BullMQ
+  };
+
+  // Add TLS configuration if not disabled
+  if (!process.env.REDIS_TLS_DISABLED) {
+    redisConfig.tls = {};
+  }
+
+  redisConnection = new Redis(redisConfig);
+
+  redisConnection.on("error", (error) => {
+    console.error("Redis connection error:", error);
+  });
+
+  redisConnection.on("connect", () => {
+    console.log("Redis connected successfully");
+  });
+
+  return redisConnection;
+}
+
+/**
+ * Close the Redis connection (useful for graceful shutdown)
+ */
+export async function closeRedisConnection(): Promise<void> {
+  if (redisConnection) {
+    await redisConnection.quit();
+    redisConnection = null;
+  }
+}

apps/webapp/app/bullmq/queues/index.ts

@@ -0,0 +1,160 @@
+/**
+ * BullMQ Queues
+ *
+ * All queue definitions for the BullMQ implementation
+ */
+
+import { Queue } from "bullmq";
+import { getRedisConnection } from "../connection";
+
+/**
+ * Episode ingestion queue
+ * Handles individual episode ingestion (including document chunks)
+ */
+export const ingestQueue = new Queue("ingest-queue", {
+  connection: getRedisConnection(),
+  defaultJobOptions: {
+    attempts: 3,
+    backoff: {
+      type: "exponential",
+      delay: 2000,
+    },
+    removeOnComplete: {
+      age: 3600, // Keep completed jobs for 1 hour
+      count: 1000, // Keep last 1000 completed jobs
+    },
+    removeOnFail: {
+      age: 86400, // Keep failed jobs for 24 hours
+    },
+  },
+});
+
+/**
+ * Document ingestion queue
+ * Handles document-level ingestion with differential processing
+ */
+export const documentIngestQueue = new Queue("document-ingest-queue", {
+  connection: getRedisConnection(),
+  defaultJobOptions: {
+    attempts: 3,
+    backoff: {
+      type: "exponential",
+      delay: 2000,
+    },
+    removeOnComplete: {
+      age: 3600,
+      count: 1000,
+    },
+    removeOnFail: {
+      age: 86400,
+    },
+  },
+});
+
+/**
+ * Conversation title creation queue
+ */
+export const conversationTitleQueue = new Queue("conversation-title-queue", {
+  connection: getRedisConnection(),
+  defaultJobOptions: {
+    attempts: 3,
+    backoff: {
+      type: "exponential",
+      delay: 2000,
+    },
+    removeOnComplete: {
+      age: 3600,
+      count: 1000,
+    },
+    removeOnFail: {
+      age: 86400,
+    },
+  },
+});
+
+/**
+ * Session compaction queue
+ */
+export const sessionCompactionQueue = new Queue("session-compaction-queue", {
+  connection: getRedisConnection(),
+  defaultJobOptions: {
+    attempts: 3,
+    backoff: {
+      type: "exponential",
+      delay: 2000,
+    },
+    removeOnComplete: {
+      age: 3600,
+      count: 1000,
+    },
+    removeOnFail: {
+      age: 86400,
+    },
+  },
+});
+
+/**
+ * BERT topic analysis queue
+ * Handles CPU-intensive topic modeling on user episodes
+ */
+export const bertTopicQueue = new Queue("bert-topic-queue", {
+  connection: getRedisConnection(),
+  defaultJobOptions: {
+    attempts: 2, // Only 2 attempts due to long runtime
+    backoff: {
+      type: "exponential",
+      delay: 5000,
+    },
+    removeOnComplete: {
+      age: 7200, // Keep completed jobs for 2 hours
+      count: 100,
+    },
+    removeOnFail: {
+      age: 172800, // Keep failed jobs for 48 hours (for debugging)
+    },
+  },
+});
+
+/**
+ * Space assignment queue
+ * Handles assigning episodes to spaces based on semantic matching
+ */
+export const spaceAssignmentQueue = new Queue("space-assignment-queue", {
+  connection: getRedisConnection(),
+  defaultJobOptions: {
+    attempts: 3,
+    backoff: {
+      type: "exponential",
+      delay: 2000,
+    },
+    removeOnComplete: {
+      age: 3600,
+      count: 1000,
+    },
+    removeOnFail: {
+      age: 86400,
+    },
+  },
+});
+
+/**
+ * Space summary queue
+ * Handles generating summaries for spaces
+ */
+export const spaceSummaryQueue = new Queue("space-summary-queue", {
+  connection: getRedisConnection(),
+  defaultJobOptions: {
+    attempts: 3,
+    backoff: {
+      type: "exponential",
+      delay: 2000,
+    },
+    removeOnComplete: {
+      age: 3600,
+      count: 1000,
+    },
+    removeOnFail: {
+      age: 86400,
+    },
+  },
+});

apps/webapp/app/bullmq/start-workers.ts

@@ -0,0 +1,154 @@
+/**
+ * BullMQ Worker Startup Script
+ *
+ * This script starts all BullMQ workers for processing background jobs.
+ * Run this as a separate process alongside your main application.
+ *
+ * Usage:
+ *   tsx apps/webapp/app/bullmq/start-workers.ts
+ */
+
+import { logger } from "~/services/logger.service";
+import {
+  ingestWorker,
+  documentIngestWorker,
+  conversationTitleWorker,
+  sessionCompactionWorker,
+  closeAllWorkers,
+  bertTopicWorker,
+  spaceAssignmentWorker,
+  spaceSummaryWorker,
+} from "./workers";
+import {
+  ingestQueue,
+  documentIngestQueue,
+  conversationTitleQueue,
+  sessionCompactionQueue,
+  bertTopicQueue,
+  spaceAssignmentQueue,
+  spaceSummaryQueue,
+} from "./queues";
+import {
+  setupWorkerLogging,
+  startPeriodicMetricsLogging,
+} from "./utils/worker-logger";
+
+let metricsInterval: NodeJS.Timeout | null = null;
+
+/**
+ * Initialize and start all BullMQ workers with comprehensive logging
+ */
+export async function initWorkers(): Promise<void> {
+  // Setup comprehensive logging for all workers
+  setupWorkerLogging(ingestWorker, ingestQueue, "ingest-episode");
+  setupWorkerLogging(
+    documentIngestWorker,
+    documentIngestQueue,
+    "ingest-document",
+  );
+  setupWorkerLogging(
+    conversationTitleWorker,
+    conversationTitleQueue,
+    "conversation-title",
+  );
+
+  setupWorkerLogging(
+    sessionCompactionWorker,
+    sessionCompactionQueue,
+    "session-compaction",
+  );
+
+  setupWorkerLogging(bertTopicWorker, bertTopicQueue, "bert-topic");
+
+  setupWorkerLogging(
+    spaceAssignmentWorker,
+    spaceAssignmentQueue,
+    "space-assignment",
+  );
+
+  setupWorkerLogging(spaceSummaryWorker, spaceSummaryQueue, "space-summary");
+
+  // Start periodic metrics logging (every 60 seconds)
+  metricsInterval = startPeriodicMetricsLogging(
+    [
+      { worker: ingestWorker, queue: ingestQueue, name: "ingest-episode" },
+      {
+        worker: documentIngestWorker,
+        queue: documentIngestQueue,
+        name: "ingest-document",
+      },
+      {
+        worker: conversationTitleWorker,
+        queue: conversationTitleQueue,
+        name: "conversation-title",
+      },
+      {
+        worker: sessionCompactionWorker,
+        queue: sessionCompactionQueue,
+        name: "session-compaction",
+      },
+
+      {
+        worker: bertTopicWorker,
+        queue: bertTopicQueue,
+        name: "bert-topic",
+      },
+
+      {
+        worker: spaceAssignmentWorker,
+        queue: spaceAssignmentQueue,
+        name: "space-assignment",
+      },
+
+      {
+        worker: spaceSummaryWorker,
+        queue: spaceAssignmentQueue,
+        name: "space-summary",
+      },
+    ],
+    60000, // Log metrics every 60 seconds
+  );
+
+  // Log worker startup
+  logger.log("\n🚀 Starting BullMQ workers...");
+  logger.log("─".repeat(80));
+  logger.log(`✓ Ingest worker: ${ingestWorker.name} (concurrency: 5)`);
+  logger.log(
+    `✓ Document ingest worker: ${documentIngestWorker.name} (concurrency: 3)`,
+  );
+  logger.log(
+    `✓ Conversation title worker: ${conversationTitleWorker.name} (concurrency: 10)`,
+  );
+
+  logger.log(
+    `✓ Session compaction worker: ${sessionCompactionWorker.name} (concurrency: 3)`,
+  );
+  logger.log("─".repeat(80));
+  logger.log("✅ All BullMQ workers started and listening for jobs");
+  logger.log("📊 Metrics will be logged every 60 seconds\n");
+}
+
+/**
+ * Shutdown all workers gracefully
+ */
+export async function shutdownWorkers(): Promise<void> {
+  logger.log("Shutdown signal received, closing workers gracefully...");
+  if (metricsInterval) {
+    clearInterval(metricsInterval);
+  }
+  await closeAllWorkers();
+}
+
+// If running as standalone script, initialize workers
+if (import.meta.url === `file://${process.argv[1]}`) {
+  initWorkers();
+
+  // Handle graceful shutdown
+  const shutdown = async () => {
+    await shutdownWorkers();
+    process.exit(0);
+  };
+
+  process.on("SIGTERM", shutdown);
+  process.on("SIGINT", shutdown);
+}

apps/webapp/app/bullmq/utils/job-finder.ts

@@ -0,0 +1,132 @@
+/**
+ * BullMQ Job Finder Utilities
+ *
+ * Helper functions to find, retrieve, and cancel BullMQ jobs
+ */
+
+interface JobInfo {
+  id: string;
+  isCompleted: boolean;
+  status?: string;
+}
+
+/**
+ * Get all active queues
+ */
+async function getAllQueues() {
+  const {
+    ingestQueue,
+    documentIngestQueue,
+    conversationTitleQueue,
+    sessionCompactionQueue,
+  } = await import("../queues");
+
+  return [
+    ingestQueue,
+    documentIngestQueue,
+    conversationTitleQueue,
+    sessionCompactionQueue,
+  ];
+}
+
+/**
+ * Find jobs by tags (metadata stored in job data)
+ * Since BullMQ doesn't have native tag support like Trigger.dev,
+ * we search through jobs and check if their data contains the required identifiers
+ */
+export async function getJobsByTags(
+  tags: string[],
+  taskIdentifier?: string,
+): Promise<JobInfo[]> {
+  const queues = await getAllQueues();
+  const matchingJobs: JobInfo[] = [];
+
+  for (const queue of queues) {
+    // Skip if taskIdentifier is specified and doesn't match queue name
+    if (taskIdentifier && !queue.name.includes(taskIdentifier)) {
+      continue;
+    }
+
+    // Get all active and waiting jobs
+    const [active, waiting, delayed] = await Promise.all([
+      queue.getActive(),
+      queue.getWaiting(),
+      queue.getDelayed(),
+    ]);
+
+    const allJobs = [...active, ...waiting, ...delayed];
+
+    for (const job of allJobs) {
+      // Check if job data contains all required tags
+      const jobData = job.data as any;
+      const matchesTags = tags.every(
+        (tag) =>
+          job.id?.includes(tag) ||
+          jobData.userId === tag ||
+          jobData.workspaceId === tag ||
+          jobData.queueId === tag,
+      );
+
+      if (matchesTags) {
+        const state = await job.getState();
+        matchingJobs.push({
+          id: job.id!,
+          isCompleted: state === "completed" || state === "failed",
+          status: state,
+        });
+      }
+    }
+  }
+
+  return matchingJobs;
+}
+
+/**
+ * Get a specific job by ID across all queues
+ */
+export async function getJobById(jobId: string): Promise<JobInfo | null> {
+  const queues = await getAllQueues();
+
+  for (const queue of queues) {
+    try {
+      const job = await queue.getJob(jobId);
+      if (job) {
+        const state = await job.getState();
+        return {
+          id: job.id!,
+          isCompleted: state === "completed" || state === "failed",
+          status: state,
+        };
+      }
+    } catch {
+      // Job not in this queue, continue
+      continue;
+    }
+  }
+
+  return null;
+}
+
+/**
+ * Cancel a job by ID
+ */
+export async function cancelJobById(jobId: string): Promise<void> {
+  const queues = await getAllQueues();
+
+  for (const queue of queues) {
+    try {
+      const job = await queue.getJob(jobId);
+      if (job) {
+        const state = await job.getState();
+        // Only remove if not already completed
+        if (state !== "completed" && state !== "failed") {
+          await job.remove();
+        }
+        return;
+      }
+    } catch {
+      // Job not in this queue, continue
+      continue;
+    }
+  }
+}

apps/webapp/app/bullmq/utils/worker-logger.ts

@@ -0,0 +1,184 @@
+/**
+ * BullMQ Worker Logger
+ *
+ * Comprehensive logging utility for tracking worker status, queue metrics,
+ * and job lifecycle events
+ */
+
+import { type Worker, type Queue } from "bullmq";
+import { logger } from "~/services/logger.service";
+
+interface WorkerMetrics {
+  name: string;
+  concurrency: number;
+  activeJobs: number;
+  waitingJobs: number;
+  delayedJobs: number;
+  failedJobs: number;
+  completedJobs: number;
+}
+
+/**
+ * Setup comprehensive logging for a worker
+ */
+export function setupWorkerLogging(
+  worker: Worker,
+  queue: Queue,
+  workerName: string,
+): void {
+  // Job picked up and started processing
+  worker.on("active", async (job) => {
+    const counts = await getQueueCounts(queue);
+    logger.log(
+      `[${workerName}] 🔄 Job started: ${job.id} | Queue: ${counts.waiting} waiting, ${counts.active} active, ${counts.delayed} delayed`,
+    );
+  });
+
+  // Job completed successfully
+  worker.on("completed", async (job, result) => {
+    const counts = await getQueueCounts(queue);
+    const duration = job.finishedOn ? job.finishedOn - job.processedOn! : 0;
+    logger.log(
+      `[${workerName}] ✅ Job completed: ${job.id} (${duration}ms) | Queue: ${counts.waiting} waiting, ${counts.active} active`,
+    );
+  });
+
+  // Job failed
+  worker.on("failed", async (job, error) => {
+    const counts = await getQueueCounts(queue);
+    const attempt = job?.attemptsMade || 0;
+    const maxAttempts = job?.opts?.attempts || 3;
+    logger.error(
+      `[${workerName}] ❌ Job failed: ${job?.id} (attempt ${attempt}/${maxAttempts}) | Error: ${error.message} | Queue: ${counts.waiting} waiting, ${counts.failed} failed`,
+    );
+  });
+
+  // Job progress update (if job reports progress)
+  worker.on("progress", async (job, progress) => {
+    logger.log(`[${workerName}] 📊 Job progress: ${job.id} - ${progress}%`);
+  });
+
+  // Worker stalled (job took too long)
+  worker.on("stalled", async (jobId) => {
+    logger.warn(`[${workerName}] ⚠️  Job stalled: ${jobId}`);
+  });
+
+  // Worker error
+  worker.on("error", (error) => {
+    logger.error(`[${workerName}] 🔥 Worker error: ${error.message}`);
+  });
+
+  // Worker closed
+  worker.on("closed", () => {
+    logger.log(`[${workerName}] 🛑 Worker closed`);
+  });
+}
+
+/**
+ * Get queue counts for logging
+ */
+async function getQueueCounts(queue: Queue): Promise<{
+  waiting: number;
+  active: number;
+  delayed: number;
+  failed: number;
+  completed: number;
+}> {
+  try {
+    const counts = await queue.getJobCounts(
+      "waiting",
+      "active",
+      "delayed",
+      "failed",
+      "completed",
+    );
+    return {
+      waiting: counts.waiting || 0,
+      active: counts.active || 0,
+      delayed: counts.delayed || 0,
+      failed: counts.failed || 0,
+      completed: counts.completed || 0,
+    };
+  } catch (error) {
+    return { waiting: 0, active: 0, delayed: 0, failed: 0, completed: 0 };
+  }
+}
+
+/**
+ * Get metrics for all workers
+ */
+export async function getAllWorkerMetrics(
+  workers: Array<{ worker: Worker; queue: Queue; name: string }>,
+): Promise<WorkerMetrics[]> {
+  const metrics = await Promise.all(
+    workers.map(async ({ worker, queue, name }) => {
+      const counts = await getQueueCounts(queue);
+      return {
+        name,
+        concurrency: worker.opts.concurrency || 1,
+        activeJobs: counts.active,
+        waitingJobs: counts.waiting,
+        delayedJobs: counts.delayed,
+        failedJobs: counts.failed,
+        completedJobs: counts.completed,
+      };
+    }),
+  );
+
+  return metrics;
+}
+
+/**
+ * Log worker metrics summary
+ */
+export function logWorkerMetrics(metrics: WorkerMetrics[]): void {
+  logger.log("\n📊 BullMQ Worker Metrics:");
+  logger.log("─".repeat(80));
+
+  for (const metric of metrics) {
+    logger.log(
+      `[${metric.name.padEnd(25)}] Concurrency: ${metric.concurrency} | ` +
+        `Active: ${metric.activeJobs} | Waiting: ${metric.waitingJobs} | ` +
+        `Delayed: ${metric.delayedJobs} | Failed: ${metric.failedJobs} | ` +
+        `Completed: ${metric.completedJobs}`,
+    );
+  }
+
+  const totals = metrics.reduce(
+    (acc, m) => ({
+      active: acc.active + m.activeJobs,
+      waiting: acc.waiting + m.waitingJobs,
+      delayed: acc.delayed + m.delayedJobs,
+      failed: acc.failed + m.failedJobs,
+      completed: acc.completed + m.completedJobs,
+    }),
+    { active: 0, waiting: 0, delayed: 0, failed: 0, completed: 0 },
+  );
+
+  logger.log("─".repeat(80));
+  logger.log(
+    `[TOTAL] Active: ${totals.active} | Waiting: ${totals.waiting} | ` +
+      `Delayed: ${totals.delayed} | Failed: ${totals.failed} | ` +
+      `Completed: ${totals.completed}`,
+  );
+  logger.log("─".repeat(80) + "\n");
+}
+
+/**
+ * Start periodic metrics logging
+ */
+export function startPeriodicMetricsLogging(
+  workers: Array<{ worker: Worker; queue: Queue; name: string }>,
+  intervalMs: number = 60000, // Default: 1 minute
+): NodeJS.Timeout {
+  const logMetrics = async () => {
+    const metrics = await getAllWorkerMetrics(workers);
+    logWorkerMetrics(metrics);
+  };
+
+  // Log immediately on start
+  logMetrics();
+
+  // Then log periodically
+  return setInterval(logMetrics, intervalMs);
+}

apps/webapp/app/bullmq/workers/index.ts

@@ -0,0 +1,200 @@
+/**
+ * BullMQ Workers
+ *
+ * All worker definitions for processing background jobs with BullMQ
+ */
+
+import { Worker } from "bullmq";
+import { getRedisConnection } from "../connection";
+import {
+  processEpisodeIngestion,
+  type IngestEpisodePayload,
+} from "~/jobs/ingest/ingest-episode.logic";
+import {
+  processDocumentIngestion,
+  type IngestDocumentPayload,
+} from "~/jobs/ingest/ingest-document.logic";
+import {
+  processConversationTitleCreation,
+  type CreateConversationTitlePayload,
+} from "~/jobs/conversation/create-title.logic";
+import {
+  processSessionCompaction,
+  type SessionCompactionPayload,
+} from "~/jobs/session/session-compaction.logic";
+import {
+  processTopicAnalysis,
+  type TopicAnalysisPayload,
+} from "~/jobs/bert/topic-analysis.logic";
+
+import {
+  enqueueIngestEpisode,
+  enqueueSpaceAssignment,
+  enqueueSessionCompaction,
+  enqueueBertTopicAnalysis,
+  enqueueSpaceSummary,
+} from "~/lib/queue-adapter.server";
+import { logger } from "~/services/logger.service";
+import {
+  processSpaceAssignment,
+  type SpaceAssignmentPayload,
+} from "~/jobs/spaces/space-assignment.logic";
+import {
+  processSpaceSummary,
+  type SpaceSummaryPayload,
+} from "~/jobs/spaces/space-summary.logic";
+
+/**
+ * Episode ingestion worker
+ * Processes individual episode ingestion jobs with global concurrency
+ *
+ * Note: BullMQ uses global concurrency limit (5 jobs max).
+ * Trigger.dev uses per-user concurrency via concurrencyKey.
+ * For most open-source deployments, global concurrency is sufficient.
+ */
+export const ingestWorker = new Worker(
+  "ingest-queue",
+  async (job) => {
+    const payload = job.data as IngestEpisodePayload;
+
+    return await processEpisodeIngestion(
+      payload,
+      // Callbacks to enqueue follow-up jobs
+      enqueueSpaceAssignment,
+      enqueueSessionCompaction,
+      enqueueBertTopicAnalysis,
+    );
+  },
+  {
+    connection: getRedisConnection(),
+    concurrency: 1, // Global limit: process up to 1 jobs in parallel
+  },
+);
+
+/**
+ * Document ingestion worker
+ * Handles document-level ingestion with differential processing
+ *
+ * Note: Per-user concurrency is achieved by using userId as part of the jobId
+ * when adding jobs to the queue
+ */
+export const documentIngestWorker = new Worker(
+  "document-ingest-queue",
+  async (job) => {
+    const payload = job.data as IngestDocumentPayload;
+    return await processDocumentIngestion(
+      payload,
+      // Callback to enqueue episode ingestion for each chunk
+      enqueueIngestEpisode,
+    );
+  },
+  {
+    connection: getRedisConnection(),
+    concurrency: 3, // Process up to 3 documents in parallel
+  },
+);
+
+/**
+ * Conversation title creation worker
+ */
+export const conversationTitleWorker = new Worker(
+  "conversation-title-queue",
+  async (job) => {
+    const payload = job.data as CreateConversationTitlePayload;
+    return await processConversationTitleCreation(payload);
+  },
+  {
+    connection: getRedisConnection(),
+    concurrency: 10, // Process up to 10 title creations in parallel
+  },
+);
+
+/**
+ * Session compaction worker
+ */
+export const sessionCompactionWorker = new Worker(
+  "session-compaction-queue",
+  async (job) => {
+    const payload = job.data as SessionCompactionPayload;
+    return await processSessionCompaction(payload);
+  },
+  {
+    connection: getRedisConnection(),
+    concurrency: 3, // Process up to 3 compactions in parallel
+  },
+);
+
+/**
+ * BERT topic analysis worker
+ * Handles CPU-intensive topic modeling
+ */
+export const bertTopicWorker = new Worker(
+  "bert-topic-queue",
+  async (job) => {
+    const payload = job.data as TopicAnalysisPayload;
+    return await processTopicAnalysis(
+      payload,
+      // Callback to enqueue space summary
+      enqueueSpaceSummary,
+    );
+  },
+  {
+    connection: getRedisConnection(),
+    concurrency: 2, // Process up to 2 analyses in parallel (CPU-intensive)
+  },
+);
+
+/**
+ * Space assignment worker
+ * Handles assigning episodes to spaces based on semantic matching
+ *
+ * Note: Global concurrency of 1 ensures sequential processing.
+ * Trigger.dev uses per-user concurrency via concurrencyKey.
+ */
+export const spaceAssignmentWorker = new Worker(
+  "space-assignment-queue",
+  async (job) => {
+    const payload = job.data as SpaceAssignmentPayload;
+    return await processSpaceAssignment(
+      payload,
+      // Callback to enqueue space summary
+      enqueueSpaceSummary,
+    );
+  },
+  {
+    connection: getRedisConnection(),
+    concurrency: 1, // Global limit: process one job at a time
+  },
+);
+
+/**
+ * Space summary worker
+ * Handles generating summaries for spaces
+ */
+export const spaceSummaryWorker = new Worker(
+  "space-summary-queue",
+  async (job) => {
+    const payload = job.data as SpaceSummaryPayload;
+    return await processSpaceSummary(payload);
+  },
+  {
+    connection: getRedisConnection(),
+    concurrency: 1, // Process one space summary at a time
+  },
+);
+
+/**
+ * Graceful shutdown handler
+ */
+export async function closeAllWorkers(): Promise<void> {
+  await Promise.all([
+    ingestWorker.close(),
+    documentIngestWorker.close(),
+    conversationTitleWorker.close(),
+    sessionCompactionWorker.close(),
+    bertTopicWorker.close(),
+    spaceSummaryWorker.close(),
+    spaceAssignmentWorker.close(),
+  ]);
+  logger.log("All BullMQ workers closed");
+}

apps/webapp/app/components/conversation/conversation-item.client.tsx

@@ -1,38 +1,42 @@
 import { EditorContent, useEditor } from "@tiptap/react";
 
 import { useEffect, memo } from "react";
-import { UserTypeEnum } from "@core/types";
-import { type ConversationHistory } from "@core/database";
 import { cn } from "~/lib/utils";
 import { extensionsForConversation } from "./editor-extensions";
 import { skillExtension } from "../editor/skill-extension";
+import { type UIMessage } from "ai";
 
 interface AIConversationItemProps {
-  conversationHistory: ConversationHistory;
+  message: UIMessage;
 }
 
-const ConversationItemComponent = ({
-  conversationHistory,
-}: AIConversationItemProps) => {
-  const isUser =
-    conversationHistory.userType === UserTypeEnum.User ||
-    conversationHistory.userType === UserTypeEnum.System;
+function getMessage(message: string) {
+  let finalMessage = message.replace("<final_response>", "");
+  finalMessage = finalMessage.replace("</final_response>", "");
+  finalMessage = finalMessage.replace("<question_response>", "");
+  finalMessage = finalMessage.replace("</question_response>", "");
 
-  const id = `a${conversationHistory.id.replace(/-/g, "")}`;
+  return finalMessage;
+}
+
+const ConversationItemComponent = ({ message }: AIConversationItemProps) => {
+  const isUser = message.role === "user" || false;
+  const textPart = message.parts.find((part) => part.type === "text");
 
   const editor = useEditor({
     extensions: [...extensionsForConversation, skillExtension],
     editable: false,
-    content: conversationHistory.message,
+    content: textPart ? getMessage(textPart.text) : "",
   });
 
   useEffect(() => {
-    editor?.commands.setContent(conversationHistory.message);
-
+    if (textPart) {
+      editor?.commands.setContent(getMessage(textPart.text));
+    }
     // eslint-disable-next-line react-hooks/exhaustive-deps
-  }, [id, conversationHistory.message]);
+  }, [message]);
 
-  if (!conversationHistory.message) {
+  if (!message) {
     return null;
   }
 
@@ -51,10 +55,10 @@ const ConversationItemComponent = ({
 };
 
 // Memoize to prevent unnecessary re-renders
-export const ConversationItem = memo(ConversationItemComponent, (prevProps, nextProps) => {
-  // Only re-render if the conversation history ID or message changed
-  return (
-    prevProps.conversationHistory.id === nextProps.conversationHistory.id &&
-    prevProps.conversationHistory.message === nextProps.conversationHistory.message
-  );
-});
+export const ConversationItem = memo(
+  ConversationItemComponent,
+  (prevProps, nextProps) => {
+    // Only re-render if the conversation history ID or message changed
+    return prevProps.message === nextProps.message;
+  },
+);

apps/webapp/app/components/conversation/conversation-textarea.client.tsx

@@ -13,33 +13,26 @@ import { Form, useSubmit, useActionData } from "@remix-run/react";
 
 interface ConversationTextareaProps {
   defaultValue?: string;
-  conversationId: string;
   placeholder?: string;
   isLoading?: boolean;
   className?: string;
   onChange?: (text: string) => void;
   disabled?: boolean;
-  onConversationCreated?: (conversation: any) => void;
+  onConversationCreated?: (message: string) => void;
+  stop?: () => void;
 }
 
 export function ConversationTextarea({
   defaultValue,
   isLoading = false,
   placeholder,
-  conversationId,
   onChange,
   onConversationCreated,
+  stop,
 }: ConversationTextareaProps) {
   const [text, setText] = useState(defaultValue ?? "");
   const [editor, setEditor] = useState<Editor>();
   const submit = useSubmit();
-  const actionData = useActionData<{ conversation?: any }>();
-
-  useEffect(() => {
-    if (actionData?.conversation && onConversationCreated) {
-      onConversationCreated(actionData.conversation);
-    }
-  }, [actionData]);
 
   const onUpdate = (editor: Editor) => {
     setText(editor.getHTML());
@@ -51,134 +44,99 @@ export function ConversationTextarea({
       return;
     }
 
-    const data = isLoading ? {} : { message: text, conversationId };
-
-    // When conversationId exists and not stopping, submit to current route
-    // When isLoading (stopping), submit to the specific conversation route
-    submit(data as any, {
-      action: isLoading
-        ? `/home/conversation/${conversationId}`
-        : conversationId
-          ? `/home/conversation/${conversationId}`
-          : "/home/conversation",
-      method: "post",
-    });
+    onConversationCreated && onConversationCreated(text);
 
     editor?.commands.clearContent(true);
     setText("");
   }, [editor, text]);
 
-  // Send message to API
-  const submitForm = useCallback(
-    async (e: React.FormEvent<HTMLFormElement>) => {
-      const data = isLoading
-        ? {}
-        : { message: text, title: text, conversationId };
-
-      submit(data as any, {
-        action: isLoading
-          ? `/home/conversation/${conversationId}`
-          : conversationId
-            ? `/home/conversation/${conversationId}`
-            : "/home/conversation",
-        method: "post",
-      });
-
-      editor?.commands.clearContent(true);
-      setText("");
-      e.preventDefault();
-    },
-    [text, conversationId],
-  );
-
   return (
-    <Form
-      action="/home/conversation"
-      method="post"
-      onSubmit={(e) => submitForm(e)}
-      className="pt-2"
-    >
-      <div className="bg-background-3 rounded-lg border-1 border-gray-300 py-2">
-        <EditorRoot>
-          <EditorContent
-            // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            initialContent={defaultValue as any}
-            extensions={[
-              Document,
-              Paragraph,
-              Text,
-              HardBreak.configure({
-                keepMarks: true,
-              }),
+    <div className="bg-background-3 rounded-lg border-1 border-gray-300 py-2">
+      <EditorRoot>
+        <EditorContent
+          // eslint-disable-next-line @typescript-eslint/no-explicit-any
+          initialContent={defaultValue as any}
+          extensions={[
+            Document,
+            Paragraph,
+            Text,
+            HardBreak.configure({
+              keepMarks: true,
+            }),
 
-              Placeholder.configure({
-                placeholder: () => placeholder ?? "Ask sol...",
-                includeChildren: true,
-              }),
-              History,
-            ]}
-            onCreate={async ({ editor }) => {
-              setEditor(editor);
-              await new Promise((resolve) => setTimeout(resolve, 100));
-              editor.commands.focus("end");
-            }}
-            onUpdate={({ editor }) => {
-              onUpdate(editor);
-            }}
-            shouldRerenderOnTransaction={false}
-            editorProps={{
-              attributes: {
-                class: `prose prose-lg dark:prose-invert prose-headings:font-title font-default focus:outline-none max-w-full`,
-              },
-              handleKeyDown(view, event) {
-                if (event.key === "Enter" && !event.shiftKey) {
-                  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-                  const target = event.target as any;
-                  if (target.innerHTML.includes("suggestion")) {
-                    return false;
-                  }
-                  event.preventDefault();
-                  if (text) {
-                    handleSend();
-                  }
-                  return true;
+            Placeholder.configure({
+              placeholder: () => placeholder ?? "Ask sol...",
+              includeChildren: true,
+            }),
+            History,
+          ]}
+          onCreate={async ({ editor }) => {
+            setEditor(editor);
+            await new Promise((resolve) => setTimeout(resolve, 100));
+            editor.commands.focus("end");
+          }}
+          onUpdate={({ editor }) => {
+            onUpdate(editor);
+          }}
+          shouldRerenderOnTransaction={false}
+          editorProps={{
+            attributes: {
+              class: `prose prose-lg dark:prose-invert prose-headings:font-title font-default focus:outline-none max-w-full`,
+            },
+            handleKeyDown(view, event) {
+              if (event.key === "Enter" && !event.shiftKey) {
+                // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                const target = event.target as any;
+                if (target.innerHTML.includes("suggestion")) {
+                  return false;
                 }
+                event.preventDefault();
+                if (text) {
+                  handleSend();
+                }
+                return true;
+              }
 
-                if (event.key === "Enter" && event.shiftKey) {
-                  view.dispatch(
-                    view.state.tr.replaceSelectionWith(
-                      view.state.schema.nodes.hardBreak.create(),
-                    ),
-                  );
-                  return true;
-                }
-                return false;
-              },
-            }}
-            immediatelyRender={false}
-            className={cn(
-              "editor-container text-md max-h-[400px] min-h-[40px] w-full min-w-full overflow-auto rounded-lg px-3",
-            )}
-          />
-        </EditorRoot>
-        <div className="mb-1 flex justify-end px-3">
-          <Button
-            variant="default"
-            className="gap-1 shadow-none transition-all duration-500 ease-in-out"
-            type="submit"
-            size="lg"
-          >
-            {isLoading ? (
-              <>
-                <LoaderCircle size={18} className="mr-1 animate-spin" />
-                Stop
-              </>
-            ) : (
-              <>Chat</>
-            )}
-          </Button>
-        </div>
+              if (event.key === "Enter" && event.shiftKey) {
+                view.dispatch(
+                  view.state.tr.replaceSelectionWith(
+                    view.state.schema.nodes.hardBreak.create(),
+                  ),
+                );
+                return true;
+              }
+              return false;
+            },
+          }}
+          immediatelyRender={false}
+          className={cn(
+            "editor-container text-md max-h-[400px] min-h-[40px] w-full min-w-full overflow-auto rounded-lg px-3",
+          )}
+        />
+      </EditorRoot>
+      <div className="mb-1 flex justify-end px-3">
+        <Button
+          variant="default"
+          className="gap-1 shadow-none transition-all duration-500 ease-in-out"
+          onClick={() => {
+            if (!isLoading) {
+              handleSend();
+            } else {
+              stop && stop();
+            }
+          }}
+          size="lg"
+        >
+          {isLoading ? (
+            <>
+              <LoaderCircle size={18} className="mr-1 animate-spin" />
+              Stop
+            </>
+          ) : (
+            <>Chat</>
+          )}
+        </Button>
       </div>
-    </Form>
+    </div>
   );
 }

apps/webapp/app/components/logs/log-options.tsx

@@ -1,10 +1,4 @@
-import { EllipsisVertical, Trash, Copy } from "lucide-react";
-import {
-  DropdownMenu,
-  DropdownMenuContent,
-  DropdownMenuItem,
-  DropdownMenuTrigger,
-} from "../ui/dropdown-menu";
+import { Trash, Copy, RotateCw } from "lucide-react";
 import { Button } from "../ui/button";
 import {
   AlertDialog,
@@ -22,11 +16,13 @@ import { toast } from "~/hooks/use-toast";
 
 interface LogOptionsProps {
   id: string;
+  status?: string;
 }
 
-export const LogOptions = ({ id }: LogOptionsProps) => {
+export const LogOptions = ({ id, status }: LogOptionsProps) => {
   const [deleteDialogOpen, setDeleteDialogOpen] = useState(false);
   const deleteFetcher = useFetcher<{ success: boolean }>();
+  const retryFetcher = useFetcher<{ success: boolean }>();
   const navigate = useNavigate();
 
   const handleDelete = () => {
@@ -58,22 +54,54 @@ export const LogOptions = ({ id }: LogOptionsProps) => {
     }
   };
 
+  const handleRetry = () => {
+    retryFetcher.submit(
+      {},
+      {
+        method: "POST",
+        action: `/api/v1/logs/${id}/retry`,
+      },
+    );
+  };
+
   useEffect(() => {
     if (deleteFetcher.state === "idle" && deleteFetcher.data?.success) {
       navigate(`/home/inbox`);
     }
   }, [deleteFetcher.state, deleteFetcher.data]);
 
+  useEffect(() => {
+    if (retryFetcher.state === "idle" && retryFetcher.data?.success) {
+      toast({
+        title: "Success",
+        description: "Episode retry initiated",
+      });
+      // Reload the page to reflect the new status
+      window.location.reload();
+    }
+  }, [retryFetcher.state, retryFetcher.data]);
+
   return (
     <>
       <div className="flex items-center gap-2">
+        {status === "FAILED" && (
+          <Button
+            variant="secondary"
+            size="sm"
+            className="gap-2 rounded"
+            onClick={handleRetry}
+            disabled={retryFetcher.state !== "idle"}
+          >
+            <RotateCw size={15} /> Retry
+          </Button>
+        )}
         <Button
           variant="secondary"
           size="sm"
           className="gap-2 rounded"
           onClick={handleCopy}
         >
-          <Copy size={15} /> Copy ID
+          <Copy size={15} /> Copy Id
         </Button>
         <Button
           variant="secondary"

apps/webapp/app/components/logs/log-text-collapse.tsx

@@ -74,7 +74,7 @@ export function LogTextCollapse({ text, log }: LogTextCollapseProps) {
           <div className={cn("flex w-full min-w-[0px] shrink flex-col")}>
             <div className="flex w-full items-center justify-between gap-4">
               <div className="inline-flex min-h-[24px] min-w-[0px] shrink items-center justify-start">
-                <div className={cn("truncate text-left")}>
+                <div className={cn("truncate text-left text-base")}>
                   {text.replace(/<[^>]+>/g, "")}
                 </div>
               </div>
@@ -97,7 +97,7 @@ export function LogTextCollapse({ text, log }: LogTextCollapseProps) {
           </div>
 
           <div className="flex items-center justify-between">
-            <div className="flex items-center gap-1">
+            <div className="flex items-center gap-1 font-light">
               {getIconForAuthorise(log.source.toLowerCase(), 12, undefined)}
               {log.source.toLowerCase()}
             </div>

apps/webapp/app/components/spaces/space-options.tsx

@@ -99,7 +99,7 @@ export const SpaceOptions = ({ id, name, description }: SpaceOptionsProps) => {
         <DropdownMenuContent align="end">
           <DropdownMenuItem onClick={handleCopy}>
             <Button variant="link" size="sm" className="gap-2 rounded">
-              <Copy size={15} /> Copy ID
+              <Copy size={15} /> Copy Id
             </Button>
           </DropdownMenuItem>
           <DropdownMenuItem onClick={() => setEditDialogOpen(true)}>

apps/webapp/app/entry.server.tsx

@@ -17,6 +17,7 @@ import { renderToPipeableStream } from "react-dom/server";
 import { initializeStartupServices } from "./utils/startup";
 import { handleMCPRequest, handleSessionRequest } from "~/services/mcp.server";
 import { authenticateHybridRequest } from "~/services/routeBuilders/apiBuilder.server";
+import { trackError } from "~/services/telemetry.server";
 
 const ABORT_DELAY = 5_000;
 
@@ -27,6 +28,42 @@ async function init() {
 
 init();
 
+/**
+ * Global error handler for all server-side errors
+ * This catches errors from loaders, actions, and rendering
+ * Automatically tracks all errors to telemetry
+ */
+export function handleError(
+  error: unknown,
+  { request }: { request: Request },
+): void {
+  // Don't track 404s or aborted requests as errors
+  if (
+    error instanceof Response &&
+    (error.status === 404 || error.status === 304)
+  ) {
+    return;
+  }
+
+  // Track error to telemetry
+  if (error instanceof Error) {
+    const url = new URL(request.url);
+    trackError(error, {
+      url: request.url,
+      path: url.pathname,
+      method: request.method,
+      userAgent: request.headers.get("user-agent") || "unknown",
+      referer: request.headers.get("referer") || undefined,
+    }).catch((trackingError) => {
+      // If telemetry tracking fails, just log it - don't break the app
+      console.error("Failed to track error:", trackingError);
+    });
+  }
+
+  // Always log to console for development/debugging
+  console.error(error);
+}
+
 export default function handleRequest(
   request: Request,
   responseStatusCode: number,

apps/webapp/app/env.server.ts

@@ -3,99 +3,146 @@ import { isValidDatabaseUrl } from "./utils/db";
 import { isValidRegex } from "./utils/regex";
 import { LLMModelEnum } from "@core/types";
 
-const EnvironmentSchema = z.object({
-  NODE_ENV: z.union([
-    z.literal("development"),
-    z.literal("production"),
-    z.literal("test"),
-  ]),
-  POSTGRES_DB: z.string(),
-  DATABASE_URL: z
-    .string()
-    .refine(
-      isValidDatabaseUrl,
-      "DATABASE_URL is invalid, for details please check the additional output above this message.",
-    ),
-  DATABASE_CONNECTION_LIMIT: z.coerce.number().int().default(10),
-  DATABASE_POOL_TIMEOUT: z.coerce.number().int().default(60),
-  DATABASE_CONNECTION_TIMEOUT: z.coerce.number().int().default(20),
-  DIRECT_URL: z
-    .string()
-    .refine(
-      isValidDatabaseUrl,
-      "DIRECT_URL is invalid, for details please check the additional output above this message.",
-    ),
-  DATABASE_READ_REPLICA_URL: z.string().optional(),
-  SESSION_SECRET: z.string(),
-  ENCRYPTION_KEY: z.string(),
-  MAGIC_LINK_SECRET: z.string(),
-  WHITELISTED_EMAILS: z
-    .string()
-    .refine(isValidRegex, "WHITELISTED_EMAILS must be a valid regex.")
-    .optional(),
-  ADMIN_EMAILS: z
-    .string()
-    .refine(isValidRegex, "ADMIN_EMAILS must be a valid regex.")
-    .optional(),
+const EnvironmentSchema = z
+  .object({
+    NODE_ENV: z.union([
+      z.literal("development"),
+      z.literal("production"),
+      z.literal("test"),
+    ]),
+    POSTGRES_DB: z.string(),
+    DATABASE_URL: z
+      .string()
+      .refine(
+        isValidDatabaseUrl,
+        "DATABASE_URL is invalid, for details please check the additional output above this message.",
+      ),
+    DATABASE_CONNECTION_LIMIT: z.coerce.number().int().default(10),
+    DATABASE_POOL_TIMEOUT: z.coerce.number().int().default(60),
+    DATABASE_CONNECTION_TIMEOUT: z.coerce.number().int().default(20),
+    DIRECT_URL: z
+      .string()
+      .refine(
+        isValidDatabaseUrl,
+        "DIRECT_URL is invalid, for details please check the additional output above this message.",
+      ),
+    DATABASE_READ_REPLICA_URL: z.string().optional(),
+    SESSION_SECRET: z.string(),
+    ENCRYPTION_KEY: z.string(),
+    MAGIC_LINK_SECRET: z.string(),
+    WHITELISTED_EMAILS: z
+      .string()
+      .refine(isValidRegex, "WHITELISTED_EMAILS must be a valid regex.")
+      .optional(),
+    ADMIN_EMAILS: z
+      .string()
+      .refine(isValidRegex, "ADMIN_EMAILS must be a valid regex.")
+      .optional(),
 
-  APP_ENV: z.string().default(process.env.NODE_ENV),
-  LOGIN_ORIGIN: z.string().default("http://localhost:5173"),
-  APP_ORIGIN: z.string().default("http://localhost:5173"),
-  POSTHOG_PROJECT_KEY: z.string().default(""),
+    APP_ENV: z.string().default(process.env.NODE_ENV),
+    LOGIN_ORIGIN: z.string().default("http://localhost:5173"),
+    APP_ORIGIN: z.string().default("http://localhost:5173"),
 
-  //storage
-  ACCESS_KEY_ID: z.string().optional(),
-  SECRET_ACCESS_KEY: z.string().optional(),
-  BUCKET: z.string().optional(),
+    // Telemetry
+    POSTHOG_PROJECT_KEY: z
+      .string()
+      .default("phc_SwfGIzzX5gh5bazVWoRxZTBhkr7FwvzArS0NRyGXm1a"),
+    TELEMETRY_ENABLED: z
+      .string()
+      .optional()
+      .default("true")
+      .transform((val) => val !== "false" && val !== "0"),
+    TELEMETRY_ANONYMOUS: z
+      .string()
+      .optional()
+      .default("false")
+      .transform((val) => val === "true" || val === "1"),
 
-  // google auth
-  AUTH_GOOGLE_CLIENT_ID: z.string().optional(),
-  AUTH_GOOGLE_CLIENT_SECRET: z.string().optional(),
+    //storage
+    ACCESS_KEY_ID: z.string().optional(),
+    SECRET_ACCESS_KEY: z.string().optional(),
+    BUCKET: z.string().optional(),
 
-  ENABLE_EMAIL_LOGIN: z.coerce.boolean().default(true),
+    // google auth
+    AUTH_GOOGLE_CLIENT_ID: z.string().optional(),
+    AUTH_GOOGLE_CLIENT_SECRET: z.string().optional(),
 
-  //Redis
-  REDIS_HOST: z.string().default("localhost"),
-  REDIS_PORT: z.coerce.number().default(6379),
-  REDIS_TLS_DISABLED: z.coerce.boolean().default(true),
+    ENABLE_EMAIL_LOGIN: z
+      .string()
+      .optional()
+      .default("true")
+      .transform((val) => val !== "false" && val !== "0"),
 
-  //Neo4j
-  NEO4J_URI: z.string(),
-  NEO4J_USERNAME: z.string(),
-  NEO4J_PASSWORD: z.string(),
+    //Redis
+    REDIS_HOST: z.string().default("localhost"),
+    REDIS_PORT: z.coerce.number().default(6379),
+    REDIS_TLS_DISABLED: z
+      .string()
+      .optional()
+      .default("true")
+      .transform((val) => val !== "false" && val !== "0"),
 
-  //OpenAI
-  OPENAI_API_KEY: z.string(),
-  ANTHROPIC_API_KEY: z.string().optional(),
+    //Neo4j
+    NEO4J_URI: z.string(),
+    NEO4J_USERNAME: z.string(),
+    NEO4J_PASSWORD: z.string(),
 
-  EMAIL_TRANSPORT: z.string().optional(),
-  FROM_EMAIL: z.string().optional(),
-  REPLY_TO_EMAIL: z.string().optional(),
-  RESEND_API_KEY: z.string().optional(),
-  SMTP_HOST: z.string().optional(),
-  SMTP_PORT: z.coerce.number().optional(),
-  SMTP_SECURE: z.coerce.boolean().optional(),
-  SMTP_USER: z.string().optional(),
-  SMTP_PASSWORD: z.string().optional(),
+    //OpenAI
+    OPENAI_API_KEY: z.string().optional(),
+    ANTHROPIC_API_KEY: z.string().optional(),
+    GOOGLE_GENERATIVE_AI_API_KEY: z.string().optional(),
 
-  //Trigger
-  TRIGGER_PROJECT_ID: z.string(),
-  TRIGGER_SECRET_KEY: z.string(),
-  TRIGGER_API_URL: z.string(),
-  TRIGGER_DB: z.string().default("trigger"),
+    EMAIL_TRANSPORT: z.string().optional(),
+    FROM_EMAIL: z.string().optional(),
+    REPLY_TO_EMAIL: z.string().optional(),
+    RESEND_API_KEY: z.string().optional(),
+    SMTP_HOST: z.string().optional(),
+    SMTP_PORT: z.coerce.number().optional(),
+    SMTP_SECURE: z
+      .string()
+      .optional()
+      .transform((val) => val === "true" || val === "1"),
+    SMTP_USER: z.string().optional(),
+    SMTP_PASSWORD: z.string().optional(),
 
-  // Model envs
-  MODEL: z.string().default(LLMModelEnum.GPT41),
-  EMBEDDING_MODEL: z.string().default("mxbai-embed-large"),
-  EMBEDDING_MODEL_SIZE: z.string().default("1024"),
-  OLLAMA_URL: z.string().optional(),
-  COHERE_API_KEY: z.string().optional(),
-  COHERE_SCORE_THRESHOLD: z.string().default("0.3"),
+    //Trigger
+    TRIGGER_PROJECT_ID: z.string().optional(),
+    TRIGGER_SECRET_KEY: z.string().optional(),
+    TRIGGER_API_URL: z.string().optional(),
+    TRIGGER_DB: z.string().default("trigger"),
 
-  AWS_ACCESS_KEY_ID: z.string().optional(),
-  AWS_SECRET_ACCESS_KEY: z.string().optional(),
-  AWS_REGION: z.string().optional(),
-});
+    // Model envs
+    MODEL: z.string().default(LLMModelEnum.GPT41),
+    EMBEDDING_MODEL: z.string().default("mxbai-embed-large"),
+    EMBEDDING_MODEL_SIZE: z.string().default("1024"),
+    OLLAMA_URL: z.string().optional(),
+    COHERE_API_KEY: z.string().optional(),
+    COHERE_SCORE_THRESHOLD: z.string().default("0.3"),
+
+    AWS_ACCESS_KEY_ID: z.string().optional(),
+    AWS_SECRET_ACCESS_KEY: z.string().optional(),
+    AWS_REGION: z.string().optional(),
+
+    // Queue provider
+    QUEUE_PROVIDER: z.enum(["trigger", "bullmq"]).default("trigger"),
+  })
+  .refine(
+    (data) => {
+      // If QUEUE_PROVIDER is "trigger", then Trigger.dev variables must be present
+      if (data.QUEUE_PROVIDER === "trigger") {
+        return !!(
+          data.TRIGGER_PROJECT_ID &&
+          data.TRIGGER_SECRET_KEY &&
+          data.TRIGGER_API_URL
+        );
+      }
+      return true;
+    },
+    {
+      message:
+        "TRIGGER_PROJECT_ID, TRIGGER_SECRET_KEY, and TRIGGER_API_URL are required when QUEUE_PROVIDER=trigger",
+    },
+  );
 
 export type Environment = z.infer<typeof EnvironmentSchema>;
 export const env = EnvironmentSchema.parse(process.env);

apps/webapp/app/hooks/usePostHog.ts

@@ -6,6 +6,7 @@ import { useOptionalUser, useUserChanged } from "./useUser";
 
 export const usePostHog = (
   apiKey?: string,
+  telemetryEnabled = true,
   logging = false,
   debug = false,
 ): void => {
@@ -15,6 +16,8 @@ export const usePostHog = (
 
   //start PostHog once
   useEffect(() => {
+    // Respect telemetry settings
+    if (!telemetryEnabled) return;
     if (apiKey === undefined || apiKey === "") return;
     if (postHogInitialized.current === true) return;
     if (logging) console.log("Initializing PostHog");
@@ -27,19 +30,26 @@ export const usePostHog = (
         if (logging) console.log("PostHog loaded");
         if (user !== undefined) {
           if (logging) console.log("Loaded: Identifying user", user);
-          posthog.identify(user.id, { email: user.email });
+          posthog.identify(user.id, {
+            email: user.email,
+            name: user.name,
+          });
         }
       },
     });
     postHogInitialized.current = true;
-  }, [apiKey, logging, user]);
+  }, [apiKey, telemetryEnabled, logging, user]);
 
   useUserChanged((user) => {
     if (postHogInitialized.current === false) return;
+    if (!telemetryEnabled) return;
     if (logging) console.log("User changed");
     if (user) {
       if (logging) console.log("Identifying user", user);
-      posthog.identify(user.id, { email: user.email });
+      posthog.identify(user.id, {
+        email: user.email,
+        name: user.name,
+      });
     } else {
       if (logging) console.log("Resetting user");
       posthog.reset();

apps/webapp/app/jobs/bert/topic-analysis.logic.ts

@@ -0,0 +1,250 @@
+import { exec } from "child_process";
+import { promisify } from "util";
+import { identifySpacesForTopics } from "~/jobs/spaces/space-identification.logic";
+import { assignEpisodesToSpace } from "~/services/graphModels/space";
+import { logger } from "~/services/logger.service";
+import { SpaceService } from "~/services/space.server";
+import { prisma } from "~/trigger/utils/prisma";
+
+const execAsync = promisify(exec);
+
+export interface TopicAnalysisPayload {
+  userId: string;
+  workspaceId: string;
+  minTopicSize?: number;
+  nrTopics?: number;
+}
+
+export interface TopicAnalysisResult {
+  topics: {
+    [topicId: string]: {
+      keywords: string[];
+      episodeIds: string[];
+    };
+  };
+}
+
+/**
+ * Run BERT analysis using exec (for BullMQ/Docker)
+ */
+async function runBertWithExec(
+  userId: string,
+  minTopicSize: number,
+  nrTopics?: number,
+): Promise<string> {
+  let command = `python3 /core/apps/webapp/python/main.py ${userId} --json`;
+
+  if (minTopicSize) {
+    command += ` --min-topic-size ${minTopicSize}`;
+  }
+
+  if (nrTopics) {
+    command += ` --nr-topics ${nrTopics}`;
+  }
+
+  console.log(`[BERT Topic Analysis] Executing: ${command}`);
+
+  const { stdout, stderr } = await execAsync(command, {
+    timeout: 300000, // 5 minutes
+    maxBuffer: 10 * 1024 * 1024, // 10MB buffer for large outputs
+  });
+
+  if (stderr) {
+    console.warn(`[BERT Topic Analysis] Warnings:`, stderr);
+  }
+
+  return stdout;
+}
+
+/**
+ * Process BERT topic analysis on user's episodes
+ * This is the common logic shared between Trigger.dev and BullMQ
+ *
+ * NOTE: This function does NOT update workspace.metadata.lastTopicAnalysisAt
+ * That should be done by the caller BEFORE enqueueing this job to prevent
+ * duplicate analyses from racing conditions.
+ */
+export async function processTopicAnalysis(
+  payload: TopicAnalysisPayload,
+  enqueueSpaceSummary?: (params: {
+    spaceId: string;
+    userId: string;
+  }) => Promise<any>,
+  pythonRunner?: (
+    userId: string,
+    minTopicSize: number,
+    nrTopics?: number,
+  ) => Promise<string>,
+): Promise<TopicAnalysisResult> {
+  const { userId, workspaceId, minTopicSize = 10, nrTopics } = payload;
+
+  console.log(`[BERT Topic Analysis] Starting analysis for user: ${userId}`);
+  console.log(
+    `[BERT Topic Analysis] Parameters: minTopicSize=${minTopicSize}, nrTopics=${nrTopics || "auto"}`,
+  );
+
+  try {
+    const startTime = Date.now();
+
+    // Run BERT analysis using provided runner or default exec
+    const runner = pythonRunner || runBertWithExec;
+    const stdout = await runner(userId, minTopicSize, nrTopics);
+
+    const duration = Date.now() - startTime;
+    console.log(`[BERT Topic Analysis] Completed in ${duration}ms`);
+
+    // Parse the JSON output
+    const result: TopicAnalysisResult = JSON.parse(stdout);
+
+    // Log summary
+    const topicCount = Object.keys(result.topics).length;
+    const totalEpisodes = Object.values(result.topics).reduce(
+      (sum, topic) => sum + topic.episodeIds.length,
+      0,
+    );
+
+    console.log(
+      `[BERT Topic Analysis] Found ${topicCount} topics covering ${totalEpisodes} episodes`,
+    );
+
+    // Step 2: Identify spaces for topics using LLM
+    try {
+      logger.info("[BERT Topic Analysis] Starting space identification", {
+        userId,
+        topicCount,
+      });
+
+      const spaceProposals = await identifySpacesForTopics({
+        userId,
+        topics: result.topics,
+      });
+
+      logger.info("[BERT Topic Analysis] Space identification completed", {
+        userId,
+        proposalCount: spaceProposals.length,
+      });
+
+      // Step 3: Create or find spaces and assign episodes
+      // Get existing spaces from PostgreSQL
+      const existingSpacesFromDb = await prisma.space.findMany({
+        where: { workspaceId },
+      });
+      const existingSpacesByName = new Map(
+        existingSpacesFromDb.map((s) => [s.name.toLowerCase(), s]),
+      );
+
+      for (const proposal of spaceProposals) {
+        try {
+          // Check if space already exists (case-insensitive match)
+          let spaceId: string;
+          const existingSpace = existingSpacesByName.get(
+            proposal.name.toLowerCase(),
+          );
+
+          if (existingSpace) {
+            // Use existing space
+            spaceId = existingSpace.id;
+            logger.info("[BERT Topic Analysis] Using existing space", {
+              spaceName: proposal.name,
+              spaceId,
+            });
+          } else {
+            // Create new space (creates in both PostgreSQL and Neo4j)
+            // Skip automatic space assignment since we're manually assigning from BERT topics
+            const spaceService = new SpaceService();
+            const newSpace = await spaceService.createSpace({
+              name: proposal.name,
+              description: proposal.intent,
+              userId,
+              workspaceId,
+            });
+            spaceId = newSpace.id;
+            logger.info("[BERT Topic Analysis] Created new space", {
+              spaceName: proposal.name,
+              spaceId,
+              intent: proposal.intent,
+            });
+          }
+
+          // Collect all episode IDs from the topics in this proposal
+          const episodeIds: string[] = [];
+          for (const topicId of proposal.topics) {
+            const topic = result.topics[topicId];
+            if (topic) {
+              episodeIds.push(...topic.episodeIds);
+            }
+          }
+
+          // Assign all episodes from these topics to the space
+          if (episodeIds.length > 0) {
+            await assignEpisodesToSpace(episodeIds, spaceId, userId);
+            logger.info("[BERT Topic Analysis] Assigned episodes to space", {
+              spaceName: proposal.name,
+              spaceId,
+              episodeCount: episodeIds.length,
+              topics: proposal.topics,
+            });
+
+            // Step 4: Trigger space summary if callback provided
+            if (enqueueSpaceSummary) {
+              await enqueueSpaceSummary({ spaceId, userId });
+              logger.info("[BERT Topic Analysis] Triggered space summary", {
+                spaceName: proposal.name,
+                spaceId,
+              });
+            }
+          }
+        } catch (spaceError) {
+          logger.error(
+            "[BERT Topic Analysis] Failed to process space proposal",
+            {
+              proposal,
+              error: spaceError,
+            },
+          );
+          // Continue with other proposals
+        }
+      }
+    } catch (spaceIdentificationError) {
+      logger.error(
+        "[BERT Topic Analysis] Space identification failed, returning topics only",
+        {
+          error: spaceIdentificationError,
+        },
+      );
+      // Return topics even if space identification fails
+    }
+
+    return result;
+  } catch (error) {
+    console.error(`[BERT Topic Analysis] Error:`, error);
+
+    if (error instanceof Error) {
+      // Check for timeout
+      if (error.message.includes("ETIMEDOUT")) {
+        throw new Error(
+          `Topic analysis timed out after 5 minutes. User may have too many episodes.`,
+        );
+      }
+
+      // Check for Python errors
+      if (error.message.includes("python3: not found")) {
+        throw new Error(`Python 3 is not installed or not available in PATH.`);
+      }
+
+      // Check for Neo4j connection errors
+      if (error.message.includes("Failed to connect to Neo4j")) {
+        throw new Error(
+          `Could not connect to Neo4j. Check NEO4J_URI and credentials.`,
+        );
+      }
+
+      // Check for no episodes
+      if (error.message.includes("No episodes found")) {
+        throw new Error(`No episodes found for userId: ${userId}`);
+      }
+    }
+
+    throw error;
+  }
+}

apps/webapp/app/jobs/conversation/create-title.logic.ts

@@ -0,0 +1,82 @@
+import { conversationTitlePrompt } from "~/trigger/conversation/prompt";
+import { prisma } from "~/trigger/utils/prisma";
+import { logger } from "~/services/logger.service";
+import { generateText, type LanguageModel } from "ai";
+import { getModel } from "~/lib/model.server";
+
+export interface CreateConversationTitlePayload {
+  conversationId: string;
+  message: string;
+}
+
+export interface CreateConversationTitleResult {
+  success: boolean;
+  title?: string;
+  error?: string;
+}
+
+/**
+ * Core business logic for creating conversation titles
+ * This is shared between Trigger.dev and BullMQ implementations
+ */
+export async function processConversationTitleCreation(
+  payload: CreateConversationTitlePayload,
+): Promise<CreateConversationTitleResult> {
+  try {
+    let conversationTitleResponse = "";
+    const { text } = await generateText({
+      model: getModel() as LanguageModel,
+      messages: [
+        {
+          role: "user",
+          content: conversationTitlePrompt.replace(
+            "{{message}}",
+            payload.message,
+          ),
+        },
+      ],
+    });
+
+    const outputMatch = text.match(/<output>(.*?)<\/output>/s);
+
+    logger.info(`Conversation title data: ${JSON.stringify(outputMatch)}`);
+
+    if (!outputMatch) {
+      logger.error("No output found in recurrence response");
+      throw new Error("Invalid response format from AI");
+    }
+
+    const jsonStr = outputMatch[1].trim();
+    const conversationTitleData = JSON.parse(jsonStr);
+
+    if (conversationTitleData) {
+      await prisma.conversation.update({
+        where: {
+          id: payload.conversationId,
+        },
+        data: {
+          title: conversationTitleData.title,
+        },
+      });
+
+      return {
+        success: true,
+        title: conversationTitleData.title,
+      };
+    }
+
+    return {
+      success: false,
+      error: "No title generated",
+    };
+  } catch (error: any) {
+    logger.error(
+      `Error creating conversation title for ${payload.conversationId}:`,
+      error,
+    );
+    return {
+      success: false,
+      error: error.message,
+    };
+  }
+}

apps/webapp/app/jobs/ingest/ingest-document.logic.ts

@@ -0,0 +1,290 @@
+import { type z } from "zod";
+
+import { IngestionStatus } from "@core/database";
+import { EpisodeTypeEnum } from "@core/types";
+import { logger } from "~/services/logger.service";
+import { saveDocument } from "~/services/graphModels/document";
+
+import { DocumentVersioningService } from "~/services/documentVersioning.server";
+import { DocumentDifferentialService } from "~/services/documentDiffer.server";
+import { KnowledgeGraphService } from "~/services/knowledgeGraph.server";
+import { prisma } from "~/trigger/utils/prisma";
+import { type IngestBodyRequest } from "./ingest-episode.logic";
+
+export interface IngestDocumentPayload {
+  body: z.infer<typeof IngestBodyRequest>;
+  userId: string;
+  workspaceId: string;
+  queueId: string;
+}
+
+export interface IngestDocumentResult {
+  success: boolean;
+  error?: string;
+}
+
+/**
+ * Core business logic for document ingestion with differential processing
+ * This is shared between Trigger.dev and BullMQ implementations
+ *
+ * Note: This function should NOT call trigger functions directly for chunk processing.
+ * Instead, use the enqueueEpisodeIngestion callback to queue episode ingestion jobs.
+ */
+export async function processDocumentIngestion(
+  payload: IngestDocumentPayload,
+  // Callback function for enqueueing episode ingestion for each chunk
+  enqueueEpisodeIngestion?: (params: {
+    body: any;
+    userId: string;
+    workspaceId: string;
+    queueId: string;
+  }) => Promise<{ id?: string }>,
+): Promise<IngestDocumentResult> {
+  const startTime = Date.now();
+
+  try {
+    logger.log(`Processing document for user ${payload.userId}`, {
+      contentLength: payload.body.episodeBody.length,
+    });
+
+    await prisma.ingestionQueue.update({
+      where: { id: payload.queueId },
+      data: {
+        status: IngestionStatus.PROCESSING,
+      },
+    });
+
+    const documentBody = payload.body;
+
+    // Step 1: Initialize services and prepare document version
+    const versioningService = new DocumentVersioningService();
+    const differentialService = new DocumentDifferentialService();
+    const knowledgeGraphService = new KnowledgeGraphService();
+
+    const {
+      documentNode: document,
+      versionInfo,
+      chunkedDocument,
+    } = await versioningService.prepareDocumentVersion(
+      documentBody.sessionId!,
+      payload.userId,
+      documentBody.metadata?.documentTitle?.toString() || "Untitled Document",
+      documentBody.episodeBody,
+      documentBody.source,
+      documentBody.metadata || {},
+    );
+
+    logger.log(`Document version analysis:`, {
+      version: versionInfo.newVersion,
+      isNewDocument: versionInfo.isNewDocument,
+      hasContentChanged: versionInfo.hasContentChanged,
+      changePercentage: versionInfo.chunkLevelChanges.changePercentage,
+      changedChunks: versionInfo.chunkLevelChanges.changedChunkIndices.length,
+      totalChunks: versionInfo.chunkLevelChanges.totalChunks,
+    });
+
+    // Step 2: Determine processing strategy
+    const differentialDecision =
+      await differentialService.analyzeDifferentialNeed(
+        documentBody.episodeBody,
+        versionInfo.existingDocument,
+        chunkedDocument,
+      );
+
+    logger.log(`Differential analysis:`, {
+      shouldUseDifferential: differentialDecision.shouldUseDifferential,
+      strategy: differentialDecision.strategy,
+      reason: differentialDecision.reason,
+      documentSizeTokens: differentialDecision.documentSizeTokens,
+    });
+
+    // Early return for unchanged documents
+    if (differentialDecision.strategy === "skip_processing") {
+      logger.log("Document content unchanged, skipping processing");
+      await prisma.ingestionQueue.update({
+        where: { id: payload.queueId },
+        data: {
+          status: IngestionStatus.COMPLETED,
+        },
+      });
+      return {
+        success: true,
+      };
+    }
+
+    // Step 3: Save the new document version
+    await saveDocument(document);
+
+    // Step 3.1: Invalidate statements from previous document version if it exists
+    let invalidationResults = null;
+    if (versionInfo.existingDocument && versionInfo.hasContentChanged) {
+      logger.log(
+        `Invalidating statements from previous document version: ${versionInfo.existingDocument.uuid}`,
+      );
+
+      invalidationResults =
+        await knowledgeGraphService.invalidateStatementsFromPreviousDocumentVersion(
+          {
+            previousDocumentUuid: versionInfo.existingDocument.uuid,
+            newDocumentContent: documentBody.episodeBody,
+            userId: payload.userId,
+            invalidatedBy: document.uuid,
+            semanticSimilarityThreshold: 0.75, // Configurable threshold
+          },
+        );
+
+      logger.log(`Statement invalidation completed:`, {
+        totalAnalyzed: invalidationResults.totalStatementsAnalyzed,
+        invalidated: invalidationResults.invalidatedStatements.length,
+        preserved: invalidationResults.preservedStatements.length,
+      });
+    }
+
+    logger.log(`Document chunked into ${chunkedDocument.chunks.length} chunks`);
+
+    // Step 4: Process chunks based on differential strategy
+    let chunksToProcess = chunkedDocument.chunks;
+    let processingMode = "full";
+
+    if (
+      differentialDecision.shouldUseDifferential &&
+      differentialDecision.strategy === "chunk_level_diff"
+    ) {
+      // Only process changed chunks
+      const chunkComparisons = differentialService.getChunkComparisons(
+        versionInfo.existingDocument!,
+        chunkedDocument,
+      );
+
+      const changedIndices =
+        differentialService.getChunksNeedingReprocessing(chunkComparisons);
+      chunksToProcess = chunkedDocument.chunks.filter((chunk) =>
+        changedIndices.includes(chunk.chunkIndex),
+      );
+      processingMode = "differential";
+
+      logger.log(
+        `Differential processing: ${chunksToProcess.length}/${chunkedDocument.chunks.length} chunks need reprocessing`,
+      );
+    } else if (differentialDecision.strategy === "full_reingest") {
+      // Process all chunks
+      processingMode = "full";
+      logger.log(
+        `Full reingestion: processing all ${chunkedDocument.chunks.length} chunks`,
+      );
+    }
+
+    // Step 5: Queue chunks for processing
+    const episodeHandlers = [];
+    if (enqueueEpisodeIngestion) {
+      for (const chunk of chunksToProcess) {
+        const chunkEpisodeData = {
+          episodeBody: chunk.content,
+          referenceTime: documentBody.referenceTime,
+          metadata: {
+            ...documentBody.metadata,
+            processingMode,
+            differentialStrategy: differentialDecision.strategy,
+            chunkHash: chunk.contentHash,
+            documentTitle:
+              documentBody.metadata?.documentTitle?.toString() ||
+              "Untitled Document",
+            chunkIndex: chunk.chunkIndex,
+            documentUuid: document.uuid,
+          },
+          source: documentBody.source,
+          spaceIds: documentBody.spaceIds,
+          sessionId: documentBody.sessionId,
+          type: EpisodeTypeEnum.DOCUMENT,
+        };
+
+        const episodeHandler = await enqueueEpisodeIngestion({
+          body: chunkEpisodeData,
+          userId: payload.userId,
+          workspaceId: payload.workspaceId,
+          queueId: payload.queueId,
+        });
+
+        if (episodeHandler.id) {
+          episodeHandlers.push(episodeHandler.id);
+          logger.log(
+            `Queued chunk ${chunk.chunkIndex + 1} for ${processingMode} processing`,
+            {
+              handlerId: episodeHandler.id,
+              chunkSize: chunk.content.length,
+              chunkHash: chunk.contentHash,
+            },
+          );
+        }
+      }
+    }
+
+    // Calculate cost savings
+    const costSavings = differentialService.calculateCostSavings(
+      chunkedDocument.chunks.length,
+      chunksToProcess.length,
+    );
+
+    await prisma.ingestionQueue.update({
+      where: { id: payload.queueId },
+      data: {
+        output: {
+          documentUuid: document.uuid,
+          version: versionInfo.newVersion,
+          totalChunks: chunkedDocument.chunks.length,
+          chunksProcessed: chunksToProcess.length,
+          chunksSkipped: costSavings.chunksSkipped,
+          processingMode,
+          differentialStrategy: differentialDecision.strategy,
+          estimatedSavings: `${costSavings.estimatedSavingsPercentage.toFixed(1)}%`,
+          statementInvalidation: invalidationResults
+            ? {
+                totalAnalyzed: invalidationResults.totalStatementsAnalyzed,
+                invalidated: invalidationResults.invalidatedStatements.length,
+                preserved: invalidationResults.preservedStatements.length,
+              }
+            : null,
+          episodes: [],
+          episodeHandlers,
+        },
+        status: IngestionStatus.PROCESSING,
+      },
+    });
+
+    const processingTimeMs = Date.now() - startTime;
+
+    logger.log(
+      `Document differential processing completed in ${processingTimeMs}ms`,
+      {
+        documentUuid: document.uuid,
+        version: versionInfo.newVersion,
+        processingMode,
+        totalChunks: chunkedDocument.chunks.length,
+        chunksProcessed: chunksToProcess.length,
+        chunksSkipped: costSavings.chunksSkipped,
+        estimatedSavings: `${costSavings.estimatedSavingsPercentage.toFixed(1)}%`,
+        changePercentage: `${differentialDecision.changePercentage.toFixed(1)}%`,
+        statementInvalidation: invalidationResults
+          ? {
+              totalAnalyzed: invalidationResults.totalStatementsAnalyzed,
+              invalidated: invalidationResults.invalidatedStatements.length,
+              preserved: invalidationResults.preservedStatements.length,
+            }
+          : "No previous version",
+      },
+    );
+
+    return { success: true };
+  } catch (err: any) {
+    await prisma.ingestionQueue.update({
+      where: { id: payload.queueId },
+      data: {
+        error: err.message,
+        status: IngestionStatus.FAILED,
+      },
+    });
+
+    logger.error(`Error processing document for user ${payload.userId}:`, err);
+    return { success: false, error: err.message };
+  }
+}

apps/webapp/app/jobs/ingest/ingest-episode.logic.ts

@@ -0,0 +1,314 @@
+import { z } from "zod";
+import { KnowledgeGraphService } from "~/services/knowledgeGraph.server";
+import { linkEpisodeToDocument } from "~/services/graphModels/document";
+import { IngestionStatus } from "@core/database";
+import { logger } from "~/services/logger.service";
+import { prisma } from "~/trigger/utils/prisma";
+import { EpisodeType } from "@core/types";
+import { deductCredits, hasCredits } from "~/trigger/utils/utils";
+import { assignEpisodesToSpace } from "~/services/graphModels/space";
+import {
+  shouldTriggerTopicAnalysis,
+  updateLastTopicAnalysisTime,
+} from "~/services/bertTopicAnalysis.server";
+
+export const IngestBodyRequest = z.object({
+  episodeBody: z.string(),
+  referenceTime: z.string(),
+  metadata: z.record(z.union([z.string(), z.number(), z.boolean()])).optional(),
+  source: z.string(),
+  spaceIds: z.array(z.string()).optional(),
+  sessionId: z.string().optional(),
+  type: z
+    .enum([EpisodeType.CONVERSATION, EpisodeType.DOCUMENT])
+    .default(EpisodeType.CONVERSATION),
+});
+
+export interface IngestEpisodePayload {
+  body: z.infer<typeof IngestBodyRequest>;
+  userId: string;
+  workspaceId: string;
+  queueId: string;
+}
+
+export interface IngestEpisodeResult {
+  success: boolean;
+  episodeDetails?: any;
+  error?: string;
+}
+
+/**
+ * Core business logic for ingesting a single episode
+ * This is shared between Trigger.dev and BullMQ implementations
+ *
+ * Note: This function should NOT call trigger functions directly.
+ * Instead, return data that indicates follow-up jobs are needed,
+ * and let the caller (Trigger task or BullMQ worker) handle job queueing.
+ */
+export async function processEpisodeIngestion(
+  payload: IngestEpisodePayload,
+  // Callback functions for enqueueing follow-up jobs
+  enqueueSpaceAssignment?: (params: {
+    userId: string;
+    workspaceId: string;
+    mode: "episode";
+    episodeIds: string[];
+  }) => Promise<any>,
+  enqueueSessionCompaction?: (params: {
+    userId: string;
+    sessionId: string;
+    source: string;
+  }) => Promise<any>,
+  enqueueBertTopicAnalysis?: (params: {
+    userId: string;
+    workspaceId: string;
+    minTopicSize?: number;
+    nrTopics?: number;
+  }) => Promise<any>,
+): Promise<IngestEpisodeResult> {
+  try {
+    logger.log(`Processing job for user ${payload.userId}`);
+
+    // Check if workspace has sufficient credits before processing
+    const hasSufficientCredits = await hasCredits(
+      payload.workspaceId,
+      "addEpisode",
+    );
+
+    if (!hasSufficientCredits) {
+      logger.warn(`Insufficient credits for workspace ${payload.workspaceId}`);
+
+      await prisma.ingestionQueue.update({
+        where: { id: payload.queueId },
+        data: {
+          status: IngestionStatus.NO_CREDITS,
+          error:
+            "Insufficient credits. Please upgrade your plan or wait for your credits to reset.",
+        },
+      });
+
+      return {
+        success: false,
+        error: "Insufficient credits",
+      };
+    }
+
+    const ingestionQueue = await prisma.ingestionQueue.update({
+      where: { id: payload.queueId },
+      data: {
+        status: IngestionStatus.PROCESSING,
+      },
+    });
+
+    const knowledgeGraphService = new KnowledgeGraphService();
+
+    const episodeBody = payload.body as any;
+
+    const episodeDetails = await knowledgeGraphService.addEpisode(
+      {
+        ...episodeBody,
+        userId: payload.userId,
+      },
+      prisma,
+    );
+
+    // Link episode to document if it's a document chunk
+    if (
+      episodeBody.type === EpisodeType.DOCUMENT &&
+      episodeBody.metadata.documentUuid &&
+      episodeDetails.episodeUuid
+    ) {
+      try {
+        await linkEpisodeToDocument(
+          episodeDetails.episodeUuid,
+          episodeBody.metadata.documentUuid,
+          episodeBody.metadata.chunkIndex || 0,
+        );
+        logger.log(
+          `Linked episode ${episodeDetails.episodeUuid} to document ${episodeBody.metadata.documentUuid} at chunk ${episodeBody.metadata.chunkIndex || 0}`,
+        );
+      } catch (error) {
+        logger.error(`Failed to link episode to document:`, {
+          error,
+          episodeUuid: episodeDetails.episodeUuid,
+          documentUuid: episodeBody.metadata.documentUuid,
+        });
+      }
+    }
+
+    let finalOutput = episodeDetails;
+    let episodeUuids: string[] = episodeDetails.episodeUuid
+      ? [episodeDetails.episodeUuid]
+      : [];
+    let currentStatus: IngestionStatus = IngestionStatus.COMPLETED;
+    if (episodeBody.type === EpisodeType.DOCUMENT) {
+      const currentOutput = ingestionQueue.output as any;
+      currentOutput.episodes.push(episodeDetails);
+      episodeUuids = currentOutput.episodes.map(
+        (episode: any) => episode.episodeUuid,
+      );
+
+      finalOutput = {
+        ...currentOutput,
+      };
+
+      if (currentOutput.episodes.length !== currentOutput.totalChunks) {
+        currentStatus = IngestionStatus.PROCESSING;
+      }
+    }
+
+    await prisma.ingestionQueue.update({
+      where: { id: payload.queueId },
+      data: {
+        output: finalOutput,
+        status: currentStatus,
+      },
+    });
+
+    // Deduct credits for episode creation
+    if (currentStatus === IngestionStatus.COMPLETED) {
+      await deductCredits(
+        payload.workspaceId,
+        "addEpisode",
+        finalOutput.statementsCreated,
+      );
+    }
+
+    // Handle space assignment after successful ingestion
+    try {
+      // If spaceIds were explicitly provided, immediately assign the episode to those spaces
+      if (
+        episodeBody.spaceIds &&
+        episodeBody.spaceIds.length > 0 &&
+        episodeDetails.episodeUuid
+      ) {
+        logger.info(`Assigning episode to explicitly provided spaces`, {
+          userId: payload.userId,
+          episodeId: episodeDetails.episodeUuid,
+          spaceIds: episodeBody.spaceIds,
+        });
+
+        // Assign episode to each space
+        for (const spaceId of episodeBody.spaceIds) {
+          await assignEpisodesToSpace(
+            [episodeDetails.episodeUuid],
+            spaceId,
+            payload.userId,
+          );
+        }
+
+        logger.info(
+          `Skipping LLM space assignment - episode explicitly assigned to ${episodeBody.spaceIds.length} space(s)`,
+        );
+      } else {
+        // Only trigger automatic LLM space assignment if no explicit spaceIds were provided
+        logger.info(
+          `Triggering LLM space assignment after successful ingestion`,
+          {
+            userId: payload.userId,
+            workspaceId: payload.workspaceId,
+            episodeId: episodeDetails?.episodeUuid,
+          },
+        );
+        if (
+          episodeDetails.episodeUuid &&
+          currentStatus === IngestionStatus.COMPLETED &&
+          enqueueSpaceAssignment
+        ) {
+          await enqueueSpaceAssignment({
+            userId: payload.userId,
+            workspaceId: payload.workspaceId,
+            mode: "episode",
+            episodeIds: episodeUuids,
+          });
+        }
+      }
+    } catch (assignmentError) {
+      // Don't fail the ingestion if assignment fails
+      logger.warn(`Failed to trigger space assignment after ingestion:`, {
+        error: assignmentError,
+        userId: payload.userId,
+        episodeId: episodeDetails?.episodeUuid,
+      });
+    }
+
+    // Auto-trigger session compaction if episode has sessionId
+    try {
+      if (
+        episodeBody.sessionId &&
+        currentStatus === IngestionStatus.COMPLETED &&
+        enqueueSessionCompaction
+      ) {
+        logger.info(`Checking if session compaction should be triggered`, {
+          userId: payload.userId,
+          sessionId: episodeBody.sessionId,
+          source: episodeBody.source,
+        });
+
+        await enqueueSessionCompaction({
+          userId: payload.userId,
+          sessionId: episodeBody.sessionId,
+          source: episodeBody.source,
+        });
+      }
+    } catch (compactionError) {
+      // Don't fail the ingestion if compaction fails
+      logger.warn(`Failed to trigger session compaction after ingestion:`, {
+        error: compactionError,
+        userId: payload.userId,
+        sessionId: episodeBody.sessionId,
+      });
+    }
+
+    // Auto-trigger BERT topic analysis if threshold met (20+ new episodes)
+    try {
+      if (
+        currentStatus === IngestionStatus.COMPLETED &&
+        enqueueBertTopicAnalysis
+      ) {
+        const shouldTrigger = await shouldTriggerTopicAnalysis(
+          payload.userId,
+          payload.workspaceId,
+        );
+
+        if (shouldTrigger) {
+          logger.info(
+            `Triggering BERT topic analysis after reaching 20+ new episodes`,
+            {
+              userId: payload.userId,
+              workspaceId: payload.workspaceId,
+            },
+          );
+
+          await enqueueBertTopicAnalysis({
+            userId: payload.userId,
+            workspaceId: payload.workspaceId,
+            minTopicSize: 10,
+          });
+
+          // Update the last analysis timestamp
+          await updateLastTopicAnalysisTime(payload.workspaceId);
+        }
+      }
+    } catch (topicAnalysisError) {
+      // Don't fail the ingestion if topic analysis fails
+      logger.warn(`Failed to trigger topic analysis after ingestion:`, {
+        error: topicAnalysisError,
+        userId: payload.userId,
+      });
+    }
+
+    return { success: true, episodeDetails };
+  } catch (err: any) {
+    await prisma.ingestionQueue.update({
+      where: { id: payload.queueId },
+      data: {
+        error: err.message,
+        status: IngestionStatus.FAILED,
+      },
+    });
+
+    logger.error(`Error processing job for user ${payload.userId}:`, err);
+    return { success: false, error: err.message };
+  }
+}

apps/webapp/app/jobs/session/session-compaction.logic.ts

@@ -0,0 +1,455 @@
+import { logger } from "~/services/logger.service";
+import type { CoreMessage } from "ai";
+import { z } from "zod";
+import { getEmbedding, makeModelCall } from "~/lib/model.server";
+import {
+  getCompactedSessionBySessionId,
+  linkEpisodesToCompact,
+  getSessionEpisodes,
+  type CompactedSessionNode,
+  type SessionEpisodeData,
+  saveCompactedSession,
+} from "~/services/graphModels/compactedSession";
+
+export interface SessionCompactionPayload {
+  userId: string;
+  sessionId: string;
+  source: string;
+  triggerSource?: "auto" | "manual" | "threshold";
+}
+
+export interface SessionCompactionResult {
+  success: boolean;
+  compactionResult?: {
+    compactUuid: string;
+    sessionId: string;
+    summary: string;
+    episodeCount: number;
+    startTime: Date;
+    endTime: Date;
+    confidence: number;
+    compressionRatio: number;
+  };
+  reason?: string;
+  episodeCount?: number;
+  error?: string;
+}
+
+// Zod schema for LLM response validation
+export const CompactionResultSchema = z.object({
+  summary: z.string().describe("Consolidated narrative of the entire session"),
+  confidence: z
+    .number()
+    .min(0)
+    .max(1)
+    .describe("Confidence score of the compaction quality"),
+});
+
+export const CONFIG = {
+  minEpisodesForCompaction: 5, // Minimum episodes to trigger compaction
+  compactionThreshold: 1, // Trigger after N new episodes
+  maxEpisodesPerBatch: 50, // Process in batches if needed
+};
+
+/**
+ * Core business logic for session compaction
+ * This is shared between Trigger.dev and BullMQ implementations
+ */
+export async function processSessionCompaction(
+  payload: SessionCompactionPayload,
+): Promise<SessionCompactionResult> {
+  const { userId, sessionId, source, triggerSource = "auto" } = payload;
+
+  logger.info(`Starting session compaction`, {
+    userId,
+    sessionId,
+    source,
+    triggerSource,
+  });
+
+  try {
+    // Check if compaction already exists
+    const existingCompact = await getCompactedSessionBySessionId(
+      sessionId,
+      userId,
+    );
+
+    // Fetch all episodes for this session
+    const episodes = await getSessionEpisodes(
+      sessionId,
+      userId,
+      existingCompact?.endTime,
+    );
+
+    console.log("episodes", episodes.length);
+    // Check if we have enough episodes
+    if (!existingCompact && episodes.length < CONFIG.minEpisodesForCompaction) {
+      logger.info(`Not enough episodes for compaction`, {
+        sessionId,
+        episodeCount: episodes.length,
+        minRequired: CONFIG.minEpisodesForCompaction,
+      });
+      return {
+        success: false,
+        reason: "insufficient_episodes",
+        episodeCount: episodes.length,
+      };
+    } else if (
+      existingCompact &&
+      episodes.length <
+        CONFIG.minEpisodesForCompaction + CONFIG.compactionThreshold
+    ) {
+      logger.info(`Not enough new episodes for compaction`, {
+        sessionId,
+        episodeCount: episodes.length,
+        minRequired:
+          CONFIG.minEpisodesForCompaction + CONFIG.compactionThreshold,
+      });
+      return {
+        success: false,
+        reason: "insufficient_new_episodes",
+        episodeCount: episodes.length,
+      };
+    }
+
+    // Generate or update compaction
+    const compactionResult = existingCompact
+      ? await updateCompaction(existingCompact, episodes, userId)
+      : await createCompaction(sessionId, episodes, userId, source);
+
+    logger.info(`Session compaction completed`, {
+      sessionId,
+      compactUuid: compactionResult.uuid,
+      episodeCount: compactionResult.episodeCount,
+      compressionRatio: compactionResult.compressionRatio,
+    });
+
+    return {
+      success: true,
+      compactionResult: {
+        compactUuid: compactionResult.uuid,
+        sessionId: compactionResult.sessionId,
+        summary: compactionResult.summary,
+        episodeCount: compactionResult.episodeCount,
+        startTime: compactionResult.startTime,
+        endTime: compactionResult.endTime,
+        confidence: compactionResult.confidence,
+        compressionRatio: compactionResult.compressionRatio,
+      },
+    };
+  } catch (error) {
+    logger.error(`Session compaction failed`, {
+      sessionId,
+      userId,
+      error: error instanceof Error ? error.message : String(error),
+    });
+
+    return {
+      success: false,
+      error: error instanceof Error ? error.message : String(error),
+    };
+  }
+}
+
+/**
+ * Create new compaction
+ */
+async function createCompaction(
+  sessionId: string,
+  episodes: SessionEpisodeData[],
+  userId: string,
+  source: string,
+): Promise<CompactedSessionNode> {
+  logger.info(`Creating new compaction`, {
+    sessionId,
+    episodeCount: episodes.length,
+  });
+
+  // Generate compaction using LLM
+  const compactionData = await generateCompaction(episodes, null);
+
+  // Generate embedding for summary
+  const summaryEmbedding = await getEmbedding(compactionData.summary);
+
+  // Create CompactedSession node using graph model
+  const compactUuid = crypto.randomUUID();
+  const now = new Date();
+  const startTime = new Date(episodes[0].createdAt);
+  const endTime = new Date(episodes[episodes.length - 1].createdAt);
+  const episodeUuids = episodes.map((e) => e.uuid);
+  const compressionRatio = episodes.length / 1;
+
+  const compactNode: CompactedSessionNode = {
+    uuid: compactUuid,
+    sessionId,
+    summary: compactionData.summary,
+    summaryEmbedding,
+    episodeCount: episodes.length,
+    startTime,
+    endTime,
+    createdAt: now,
+    confidence: compactionData.confidence,
+    userId,
+    source,
+    compressionRatio,
+    metadata: { triggerType: "create" },
+  };
+
+  console.log("compactNode", compactNode);
+  // Use graph model functions
+  await saveCompactedSession(compactNode);
+  await linkEpisodesToCompact(compactUuid, episodeUuids, userId);
+
+  logger.info(`Compaction created`, {
+    compactUuid,
+    episodeCount: episodes.length,
+  });
+
+  return compactNode;
+}
+
+/**
+ * Update existing compaction with new episodes
+ */
+async function updateCompaction(
+  existingCompact: CompactedSessionNode,
+  newEpisodes: SessionEpisodeData[],
+  userId: string,
+): Promise<CompactedSessionNode> {
+  logger.info(`Updating existing compaction`, {
+    compactUuid: existingCompact.uuid,
+    newEpisodeCount: newEpisodes.length,
+  });
+
+  // Generate updated compaction using LLM (merging)
+  const compactionData = await generateCompaction(
+    newEpisodes,
+    existingCompact.summary,
+  );
+
+  // Generate new embedding for updated summary
+  const summaryEmbedding = await getEmbedding(compactionData.summary);
+
+  // Update CompactedSession node using graph model
+  const now = new Date();
+  const endTime = newEpisodes[newEpisodes.length - 1].createdAt;
+  const totalEpisodeCount = existingCompact.episodeCount + newEpisodes.length;
+  const compressionRatio = totalEpisodeCount / 1;
+  const episodeUuids = newEpisodes.map((e) => e.uuid);
+
+  const updatedNode: CompactedSessionNode = {
+    ...existingCompact,
+    summary: compactionData.summary,
+    summaryEmbedding,
+    episodeCount: totalEpisodeCount,
+    endTime,
+    updatedAt: now,
+    confidence: compactionData.confidence,
+    compressionRatio,
+    metadata: { triggerType: "update", newEpisodesAdded: newEpisodes.length },
+  };
+
+  // Use graph model functions
+  await saveCompactedSession(updatedNode);
+  await linkEpisodesToCompact(existingCompact.uuid, episodeUuids, userId);
+
+  logger.info(`Compaction updated`, {
+    compactUuid: existingCompact.uuid,
+    totalEpisodeCount,
+  });
+
+  return updatedNode;
+}
+
+/**
+ * Generate compaction using LLM (similar to Claude Code's compact approach)
+ */
+async function generateCompaction(
+  episodes: SessionEpisodeData[],
+  existingSummary: string | null,
+): Promise<z.infer<typeof CompactionResultSchema>> {
+  const systemPrompt = createCompactionSystemPrompt();
+  const userPrompt = createCompactionUserPrompt(episodes, existingSummary);
+
+  const messages: CoreMessage[] = [
+    { role: "system", content: systemPrompt },
+    { role: "user", content: userPrompt },
+  ];
+
+  logger.info(`Generating compaction with LLM`, {
+    episodeCount: episodes.length,
+    hasExistingSummary: !!existingSummary,
+  });
+
+  try {
+    let responseText = "";
+    await makeModelCall(
+      false,
+      messages,
+      (text: string) => {
+        responseText = text;
+      },
+      undefined,
+      "high",
+    );
+
+    return parseCompactionResponse(responseText);
+  } catch (error) {
+    logger.error(`Failed to generate compaction`, {
+      error: error instanceof Error ? error.message : String(error),
+    });
+    throw error;
+  }
+}
+
+/**
+ * System prompt for compaction (for agent recall/context retrieval)
+ */
+function createCompactionSystemPrompt(): string {
+  return `You are a session compaction specialist. Your task is to create a rich, informative summary that will help AI agents understand what happened in this conversation session when they need context for future interactions.
+
+## PURPOSE
+
+This summary will be retrieved by AI agents when the user references this session in future conversations. The agent needs enough context to:
+- Understand what was discussed and why
+- Know what decisions were made and their rationale
+- Grasp the outcome and current state
+- Have relevant technical details to provide informed responses
+
+## COMPACTION GOALS
+
+1. **Comprehensive Context**: Capture all important information that might be referenced later
+2. **Decision Documentation**: Clearly state what was decided, why, and what alternatives were considered
+3. **Technical Details**: Include specific implementations, tools, configurations, and technical choices
+4. **Outcome Clarity**: Make it clear what was accomplished and what the final state is
+5. **Evolution Tracking**: Show how thinking or decisions evolved during the session
+
+## COMPACTION RULES
+
+1. **Be Information-Dense**: Pack useful details without fluff or repetition
+2. **Structure Chronologically**: Start with problem/question, show progression, end with outcome
+3. **Highlight Key Points**: Emphasize decisions, implementations, results, and learnings
+4. **Include Specifics**: Names of libraries, specific configurations, metrics, numbers matter
+5. **Resolve Contradictions**: Always use the most recent/final version when information conflicts
+
+## OUTPUT REQUIREMENTS
+
+- **summary**: A detailed, information-rich narrative that tells the complete story
+  - Structure naturally based on content - use as many paragraphs as needed
+  - Each distinct topic, decision, or phase should get its own paragraph(s)
+  - Start with context and initial problem/question
+  - Progress chronologically through discussions, decisions, and implementations
+  - **Final paragraph MUST**: State the outcome, results, and current state
+  - Don't artificially limit length - capture everything important
+
+- **confidence**: Score (0-1) reflecting how well this summary captures the session's essence
+
+Your response MUST be valid JSON wrapped in <output></output> tags.
+
+## KEY PRINCIPLES
+
+- Write for an AI agent that needs to help the user in future conversations
+- Include technical specifics that might be referenced (library names, configurations, metrics)
+- Make outcomes and current state crystal clear in the final paragraph
+- Show the reasoning behind decisions, not just the decisions themselves
+- Be comprehensive but concise - every sentence should add value
+- Each major topic or phase deserves its own paragraph(s)
+- Don't compress too much - agents need the details
+`;
+}
+
+/**
+ * User prompt for compaction
+ */
+function createCompactionUserPrompt(
+  episodes: SessionEpisodeData[],
+  existingSummary: string | null,
+): string {
+  let prompt = "";
+
+  if (existingSummary) {
+    prompt += `## EXISTING SUMMARY (from previous compaction)\n\n${existingSummary}\n\n`;
+    prompt += `## NEW EPISODES (to merge into existing summary)\n\n`;
+  } else {
+    prompt += `## SESSION EPISODES (to compact)\n\n`;
+  }
+
+  episodes.forEach((episode, index) => {
+    const timestamp = new Date(episode.validAt).toISOString();
+    prompt += `### Episode ${index + 1} (${timestamp})\n`;
+    prompt += `Source: ${episode.source}\n`;
+    prompt += `Content:\n${episode.originalContent}\n\n`;
+  });
+
+  if (existingSummary) {
+    prompt += `\n## INSTRUCTIONS\n\n`;
+    prompt += `Merge the new episodes into the existing summary. Update facts, add new information, and maintain narrative coherence. Ensure the consolidated summary reflects the complete session including both old and new content.\n`;
+  } else {
+    prompt += `\n## INSTRUCTIONS\n\n`;
+    prompt += `Create a compact summary of this entire session. Consolidate all information into a coherent narrative with deduplicated key facts.\n`;
+  }
+
+  return prompt;
+}
+
+/**
+ * Parse LLM response for compaction
+ */
+function parseCompactionResponse(
+  response: string,
+): z.infer<typeof CompactionResultSchema> {
+  try {
+    // Extract content from <output> tags
+    const outputMatch = response.match(/<output>([\s\S]*?)<\/output>/);
+    if (!outputMatch) {
+      logger.warn("No <output> tags found in LLM compaction response");
+      logger.debug("Full LLM response:", { response });
+      throw new Error("Invalid LLM response format - missing <output> tags");
+    }
+
+    let jsonContent = outputMatch[1].trim();
+
+    // Remove markdown code blocks if present
+    jsonContent = jsonContent.replace(/```json\n?/g, "").replace(/```\n?/g, "");
+
+    const parsed = JSON.parse(jsonContent);
+
+    // Validate with schema
+    const validated = CompactionResultSchema.parse(parsed);
+
+    return validated;
+  } catch (error) {
+    logger.error("Failed to parse compaction response", {
+      error: error instanceof Error ? error.message : String(error),
+      response: response.substring(0, 500),
+    });
+    throw new Error(`Failed to parse compaction response: ${error}`);
+  }
+}
+
+/**
+ * Helper function to check if compaction should be triggered
+ */
+export async function shouldTriggerCompaction(
+  sessionId: string,
+  userId: string,
+): Promise<boolean> {
+  const existingCompact = await getCompactedSessionBySessionId(
+    sessionId,
+    userId,
+  );
+
+  if (!existingCompact) {
+    // Check if we have enough episodes for initial compaction
+    const episodes = await getSessionEpisodes(sessionId, userId);
+    return episodes.length >= CONFIG.minEpisodesForCompaction;
+  }
+
+  // Check if we have enough new episodes to update
+  const newEpisodes = await getSessionEpisodes(
+    sessionId,
+    userId,
+    existingCompact.endTime,
+  );
+  return newEpisodes.length >= CONFIG.compactionThreshold;
+}

apps/webapp/app/jobs/spaces/space-identification.logic.ts

@@ -0,0 +1,229 @@
+/**
+ * Space Identification Logic
+ *
+ * Uses LLM to identify appropriate spaces for topics discovered by BERT analysis
+ */
+
+import { makeModelCall } from "~/lib/model.server";
+import { getAllSpacesForUser } from "~/services/graphModels/space";
+import { getEpisode } from "~/services/graphModels/episode";
+import { logger } from "~/services/logger.service";
+import type { SpaceNode } from "@core/types";
+
+export interface TopicData {
+  keywords: string[];
+  episodeIds: string[];
+}
+
+export interface SpaceProposal {
+  name: string;
+  intent: string;
+  confidence: number;
+  reason: string;
+  topics: string[]; // Array of topic IDs
+}
+
+interface IdentifySpacesParams {
+  userId: string;
+  topics: Record<string, TopicData>;
+}
+
+/**
+ * Identify spaces for topics using LLM analysis
+ * Takes top 10 keywords and top 5 episodes per topic
+ */
+export async function identifySpacesForTopics(
+  params: IdentifySpacesParams,
+): Promise<SpaceProposal[]> {
+  const { userId, topics } = params;
+
+  // Get existing spaces for the user
+  const existingSpaces = await getAllSpacesForUser(userId);
+
+  // Prepare topic data with top 10 keywords and top 5 episodes
+  const topicsForAnalysis = await Promise.all(
+    Object.entries(topics).map(async ([topicId, topicData]) => {
+      // Take top 10 keywords
+      const topKeywords = topicData.keywords.slice(0, 10);
+
+      // Take top 5 episodes and fetch their content
+      const topEpisodeIds = topicData.episodeIds.slice(0, 5);
+      const episodes = await Promise.all(
+        topEpisodeIds.map((id) => getEpisode(id)),
+      );
+
+      return {
+        topicId,
+        keywords: topKeywords,
+        episodes: episodes
+          .filter((e) => e !== null)
+          .map((e) => ({
+            content: e!.content.substring(0, 500), // Limit to 500 chars per episode
+          })),
+        episodeCount: topicData.episodeIds.length,
+      };
+    }),
+  );
+
+  // Build the prompt
+  const prompt = buildSpaceIdentificationPrompt(
+    existingSpaces,
+    topicsForAnalysis,
+  );
+
+  logger.info("Identifying spaces for topics", {
+    userId,
+    topicCount: Object.keys(topics).length,
+    existingSpaceCount: existingSpaces.length,
+  });
+
+  // Call LLM with structured output
+  let responseText = "";
+  await makeModelCall(
+    false, // not streaming
+    [{ role: "user", content: prompt }],
+    (text) => {
+      responseText = text;
+    },
+    {
+      temperature: 0.7,
+    },
+    "high", // Use high complexity for space identification
+  );
+
+  // Parse the response
+  const proposals = parseSpaceProposals(responseText);
+
+  logger.info("Space identification completed", {
+    userId,
+    proposalCount: proposals.length,
+  });
+
+  return proposals;
+}
+
+/**
+ * Build the prompt for space identification
+ */
+function buildSpaceIdentificationPrompt(
+  existingSpaces: SpaceNode[],
+  topics: Array<{
+    topicId: string;
+    keywords: string[];
+    episodes: Array<{ content: string }>;
+    episodeCount: number;
+  }>,
+): string {
+  const existingSpacesSection =
+    existingSpaces.length > 0
+      ? `## Existing Spaces
+
+The user currently has these spaces:
+${existingSpaces.map((s) => `- **${s.name}**: ${s.description || "No description"} (${s.contextCount || 0} episodes)`).join("\n")}
+
+When identifying new spaces, consider if topics fit into existing spaces or if new spaces are needed.`
+      : `## Existing Spaces
+
+The user currently has no spaces defined. This is a fresh start for space organization.`;
+
+  const topicsSection = `## Topics Discovered
+
+BERT topic modeling has identified ${topics.length} distinct topics from the user's episodes. Each topic represents a cluster of semantically related content.
+
+${topics
+  .map(
+    (t, idx) => `### Topic ${idx + 1} (ID: ${t.topicId})
+**Episode Count**: ${t.episodeCount}
+**Top Keywords**: ${t.keywords.join(", ")}
+
+**Sample Episodes** (showing ${t.episodes.length} of ${t.episodeCount}):
+${t.episodes.map((e, i) => `${i + 1}. ${e.content}`).join("\n")}
+`,
+  )
+  .join("\n")}`;
+
+  return `You are a knowledge organization expert. Your task is to analyze discovered topics and identify appropriate "spaces" (thematic containers) for organizing episodic memories.
+
+${existingSpacesSection}
+
+${topicsSection}
+
+## Task
+
+Analyze the topics above and identify spaces that would help organize this content meaningfully. For each space:
+
+1. **Consider existing spaces first**: If topics clearly belong to existing spaces, assign them there
+2. **Create new spaces when needed**: If topics represent distinct themes not covered by existing spaces
+3. **Group related topics**: Multiple topics can be assigned to the same space if they share a theme
+4. **Aim for 20-50 episodes per space**: This is the sweet spot for space cohesion
+5. **Focus on user intent**: What would help the user find and understand this content later?
+
+## Output Format
+
+Return your analysis as a JSON array of space proposals. Each proposal should have:
+
+\`\`\`json
+[
+  {
+    "name": "Space name (use existing space name if assigning to existing space)",
+    "intent": "Clear description of what this space represents",
+    "confidence": 0.85,
+    "reason": "Brief explanation of why these topics belong together",
+    "topics": ["topic-id-1", "topic-id-2"]
+  }
+]
+\`\`\`
+
+**Important Guidelines**:
+- **confidence**: 0.0-1.0 scale indicating how confident you are this is a good grouping
+- **topics**: Array of topic IDs (use the exact IDs from above like "0", "1", "-1", etc.)
+- **name**: For existing spaces, use the EXACT name. For new spaces, create a clear, concise name
+- Only propose spaces with confidence >= 0.6
+- Each topic should only appear in ONE space proposal
+- Topic "-1" is the outlier topic (noise) - only include if it genuinely fits a theme
+
+Return ONLY the JSON array, no additional text.`;
+}
+
+/**
+ * Parse space proposals from LLM response
+ */
+function parseSpaceProposals(responseText: string): SpaceProposal[] {
+  try {
+    // Extract JSON from markdown code blocks if present
+    const jsonMatch = responseText.match(/```(?:json)?\s*(\[[\s\S]*?\])\s*```/);
+    const jsonText = jsonMatch ? jsonMatch[1] : responseText;
+
+    const proposals = JSON.parse(jsonText.trim());
+
+    if (!Array.isArray(proposals)) {
+      throw new Error("Response is not an array");
+    }
+
+    // Validate and filter proposals
+    return proposals
+      .filter((p) => {
+        return (
+          p.name &&
+          p.intent &&
+          typeof p.confidence === "number" &&
+          p.confidence >= 0.6 &&
+          Array.isArray(p.topics) &&
+          p.topics.length > 0
+        );
+      })
+      .map((p) => ({
+        name: p.name.trim(),
+        intent: p.intent.trim(),
+        confidence: p.confidence,
+        reason: (p.reason || "").trim(),
+        topics: p.topics.map((t: any) => String(t)),
+      }));
+  } catch (error) {
+    logger.error("Failed to parse space proposals", {
+      error,
+      responseText: responseText.substring(0, 500),
+    });
+    return [];
+  }
+}

apps/webapp/app/jobs/spaces/space-summary.logic.ts

@@ -0,0 +1,721 @@
+import { logger } from "~/services/logger.service";
+import { SpaceService } from "~/services/space.server";
+import { makeModelCall } from "~/lib/model.server";
+import { runQuery } from "~/lib/neo4j.server";
+import { updateSpaceStatus, SPACE_STATUS } from "~/trigger/utils/space-status";
+import type { CoreMessage } from "ai";
+import { z } from "zod";
+import { getSpace, updateSpace } from "~/trigger/utils/space-utils";
+import { getSpaceEpisodeCount } from "~/services/graphModels/space";
+
+export interface SpaceSummaryPayload {
+  userId: string;
+  spaceId: string; // Single space only
+  triggerSource?: "assignment" | "manual" | "scheduled";
+}
+
+interface SpaceEpisodeData {
+  uuid: string;
+  content: string;
+  originalContent: string;
+  source: string;
+  createdAt: Date;
+  validAt: Date;
+  metadata: any;
+  sessionId: string | null;
+}
+
+interface SpaceSummaryData {
+  spaceId: string;
+  spaceName: string;
+  spaceDescription?: string;
+  contextCount: number;
+  summary: string;
+  keyEntities: string[];
+  themes: string[];
+  confidence: number;
+  lastUpdated: Date;
+  isIncremental: boolean;
+}
+
+// Zod schema for LLM response validation
+const SummaryResultSchema = z.object({
+  summary: z.string(),
+  keyEntities: z.array(z.string()),
+  themes: z.array(z.string()),
+  confidence: z.number().min(0).max(1),
+});
+
+const CONFIG = {
+  maxEpisodesForSummary: 20, // Limit episodes for performance
+  minEpisodesForSummary: 1, // Minimum episodes to generate summary
+  summaryEpisodeThreshold: 5, // Minimum new episodes required to trigger summary (configurable)
+};
+
+export interface SpaceSummaryResult {
+  success: boolean;
+  spaceId: string;
+  triggerSource: string;
+  summary?: {
+    statementCount: number;
+    confidence: number;
+    themesCount: number;
+  } | null;
+  reason?: string;
+}
+
+/**
+ * Core business logic for space summary generation
+ * This is shared between Trigger.dev and BullMQ implementations
+ */
+export async function processSpaceSummary(
+  payload: SpaceSummaryPayload,
+): Promise<SpaceSummaryResult> {
+  const { userId, spaceId, triggerSource = "manual" } = payload;
+
+  logger.info(`Starting space summary generation`, {
+    userId,
+    spaceId,
+    triggerSource,
+  });
+
+  try {
+    // Update status to processing
+    await updateSpaceStatus(spaceId, SPACE_STATUS.PROCESSING, {
+      userId,
+      operation: "space-summary",
+      metadata: { triggerSource, phase: "start_summary" },
+    });
+
+    // Generate summary for the single space
+    const summaryResult = await generateSpaceSummary(
+      spaceId,
+      userId,
+      triggerSource,
+    );
+
+    if (summaryResult) {
+      // Store the summary
+      await storeSummary(summaryResult);
+
+      // Update status to ready after successful completion
+      await updateSpaceStatus(spaceId, SPACE_STATUS.READY, {
+        userId,
+        operation: "space-summary",
+        metadata: {
+          triggerSource,
+          phase: "completed_summary",
+          contextCount: summaryResult.contextCount,
+          confidence: summaryResult.confidence,
+        },
+      });
+
+      logger.info(`Generated summary for space ${spaceId}`, {
+        statementCount: summaryResult.contextCount,
+        confidence: summaryResult.confidence,
+        themes: summaryResult.themes.length,
+        triggerSource,
+      });
+
+      return {
+        success: true,
+        spaceId,
+        triggerSource,
+        summary: {
+          statementCount: summaryResult.contextCount,
+          confidence: summaryResult.confidence,
+          themesCount: summaryResult.themes.length,
+        },
+      };
+    } else {
+      // No summary generated - this could be due to insufficient episodes or no new episodes
+      // This is not an error state, so update status to ready
+      await updateSpaceStatus(spaceId, SPACE_STATUS.READY, {
+        userId,
+        operation: "space-summary",
+        metadata: {
+          triggerSource,
+          phase: "no_summary_needed",
+          reason: "Insufficient episodes or no new episodes to summarize",
+        },
+      });
+
+      logger.info(
+        `No summary generated for space ${spaceId} - insufficient or no new episodes`,
+      );
+      return {
+        success: true,
+        spaceId,
+        triggerSource,
+        summary: null,
+        reason: "No episodes to summarize",
+      };
+    }
+  } catch (error) {
+    // Update status to error on exception
+    try {
+      await updateSpaceStatus(spaceId, SPACE_STATUS.ERROR, {
+        userId,
+        operation: "space-summary",
+        metadata: {
+          triggerSource,
+          phase: "exception",
+          error: error instanceof Error ? error.message : "Unknown error",
+        },
+      });
+    } catch (statusError) {
+      logger.warn(`Failed to update status to error for space ${spaceId}`, {
+        statusError,
+      });
+    }
+
+    logger.error(
+      `Error in space summary generation for space ${spaceId}:`,
+      error as Record<string, unknown>,
+    );
+    throw error;
+  }
+}
+
+async function generateSpaceSummary(
+  spaceId: string,
+  userId: string,
+  triggerSource?: "assignment" | "manual" | "scheduled",
+): Promise<SpaceSummaryData | null> {
+  try {
+    // 1. Get space details
+    const spaceService = new SpaceService();
+    const space = await spaceService.getSpace(spaceId, userId);
+
+    if (!space) {
+      logger.warn(`Space ${spaceId} not found for user ${userId}`);
+      return null;
+    }
+
+    // 2. Check episode count threshold (skip for manual triggers)
+    if (triggerSource !== "manual") {
+      const currentEpisodeCount = await getSpaceEpisodeCount(spaceId, userId);
+      const lastSummaryEpisodeCount = space.contextCount || 0;
+      const episodeDifference = currentEpisodeCount - lastSummaryEpisodeCount;
+
+      if (
+        episodeDifference < CONFIG.summaryEpisodeThreshold ||
+        lastSummaryEpisodeCount !== 0
+      ) {
+        logger.info(
+          `Skipping summary generation for space ${spaceId}: only ${episodeDifference} new episodes (threshold: ${CONFIG.summaryEpisodeThreshold})`,
+          {
+            currentEpisodeCount,
+            lastSummaryEpisodeCount,
+            episodeDifference,
+            threshold: CONFIG.summaryEpisodeThreshold,
+          },
+        );
+        return null;
+      }
+
+      logger.info(
+        `Proceeding with summary generation for space ${spaceId}: ${episodeDifference} new episodes (threshold: ${CONFIG.summaryEpisodeThreshold})`,
+        {
+          currentEpisodeCount,
+          lastSummaryEpisodeCount,
+          episodeDifference,
+        },
+      );
+    }
+
+    // 2. Check for existing summary
+    const existingSummary = await getExistingSummary(spaceId);
+    const isIncremental = existingSummary !== null;
+
+    // 3. Get episodes (all or new ones based on existing summary)
+    const episodes = await getSpaceEpisodes(
+      spaceId,
+      userId,
+      isIncremental ? existingSummary?.lastUpdated : undefined,
+    );
+
+    // Handle case where no new episodes exist for incremental update
+    if (isIncremental && episodes.length === 0) {
+      logger.info(
+        `No new episodes found for space ${spaceId}, skipping summary update`,
+      );
+      return null;
+    }
+
+    // Check minimum episode requirement for new summaries only
+    if (!isIncremental && episodes.length < CONFIG.minEpisodesForSummary) {
+      logger.info(
+        `Space ${spaceId} has insufficient episodes (${episodes.length}) for new summary`,
+      );
+      return null;
+    }
+
+    // 4. Process episodes using unified approach
+    let summaryResult;
+
+    if (episodes.length > CONFIG.maxEpisodesForSummary) {
+      logger.info(
+        `Large space detected (${episodes.length} episodes). Processing in batches.`,
+      );
+
+      // Process in batches, each building on previous result
+      const batches: SpaceEpisodeData[][] = [];
+      for (let i = 0; i < episodes.length; i += CONFIG.maxEpisodesForSummary) {
+        batches.push(episodes.slice(i, i + CONFIG.maxEpisodesForSummary));
+      }
+
+      let currentSummary = existingSummary?.summary || null;
+      let currentThemes = existingSummary?.themes || [];
+      let cumulativeConfidence = 0;
+
+      for (const [batchIndex, batch] of batches.entries()) {
+        logger.info(
+          `Processing batch ${batchIndex + 1}/${batches.length} with ${batch.length} episodes`,
+        );
+
+        const batchResult = await generateUnifiedSummary(
+          space.name,
+          space.description as string,
+          batch,
+          currentSummary,
+          currentThemes,
+        );
+
+        if (batchResult) {
+          currentSummary = batchResult.summary;
+          currentThemes = batchResult.themes;
+          cumulativeConfidence += batchResult.confidence;
+        } else {
+          logger.warn(`Failed to process batch ${batchIndex + 1}`);
+        }
+
+        // Small delay between batches
+        if (batchIndex < batches.length - 1) {
+          await new Promise((resolve) => setTimeout(resolve, 500));
+        }
+      }
+
+      summaryResult = currentSummary
+        ? {
+            summary: currentSummary,
+            themes: currentThemes,
+            confidence: Math.min(cumulativeConfidence / batches.length, 1.0),
+          }
+        : null;
+    } else {
+      logger.info(
+        `Processing ${episodes.length} episodes with unified approach`,
+      );
+
+      // Use unified approach for smaller spaces
+      summaryResult = await generateUnifiedSummary(
+        space.name,
+        space.description as string,
+        episodes,
+        existingSummary?.summary || null,
+        existingSummary?.themes || [],
+      );
+    }
+
+    if (!summaryResult) {
+      logger.warn(`Failed to generate LLM summary for space ${spaceId}`);
+      return null;
+    }
+
+    // Get the actual current counts from Neo4j
+    const currentEpisodeCount = await getSpaceEpisodeCount(spaceId, userId);
+
+    return {
+      spaceId: space.uuid,
+      spaceName: space.name,
+      spaceDescription: space.description as string,
+      contextCount: currentEpisodeCount,
+      summary: summaryResult.summary,
+      keyEntities: summaryResult.keyEntities || [],
+      themes: summaryResult.themes,
+      confidence: summaryResult.confidence,
+      lastUpdated: new Date(),
+      isIncremental,
+    };
+  } catch (error) {
+    logger.error(
+      `Error generating summary for space ${spaceId}:`,
+      error as Record<string, unknown>,
+    );
+    return null;
+  }
+}
+
+async function generateUnifiedSummary(
+  spaceName: string,
+  spaceDescription: string | undefined,
+  episodes: SpaceEpisodeData[],
+  previousSummary: string | null = null,
+  previousThemes: string[] = [],
+): Promise<{
+  summary: string;
+  themes: string[];
+  confidence: number;
+  keyEntities?: string[];
+} | null> {
+  try {
+    const prompt = createUnifiedSummaryPrompt(
+      spaceName,
+      spaceDescription,
+      episodes,
+      previousSummary,
+      previousThemes,
+    );
+
+    // Space summary generation requires HIGH complexity (creative synthesis, narrative generation)
+    let responseText = "";
+    await makeModelCall(
+      false,
+      prompt,
+      (text: string) => {
+        responseText = text;
+      },
+      undefined,
+      "high",
+    );
+
+    return parseSummaryResponse(responseText);
+  } catch (error) {
+    logger.error(
+      "Error generating unified summary:",
+      error as Record<string, unknown>,
+    );
+    return null;
+  }
+}
+
+function createUnifiedSummaryPrompt(
+  spaceName: string,
+  spaceDescription: string | undefined,
+  episodes: SpaceEpisodeData[],
+  previousSummary: string | null,
+  previousThemes: string[],
+): CoreMessage[] {
+  // If there are no episodes and no previous summary, we cannot generate a meaningful summary
+  if (episodes.length === 0 && previousSummary === null) {
+    throw new Error(
+      "Cannot generate summary without episodes or existing summary",
+    );
+  }
+
+  const episodesText = episodes
+    .map(
+      (episode) =>
+        `- ${episode.content} (Source: ${episode.source}, Session: ${episode.sessionId || "N/A"})`,
+    )
+    .join("\n");
+
+  // Extract key entities and themes from episode content
+  const contentWords = episodes
+    .map((ep) => ep.content.toLowerCase())
+    .join(" ")
+    .split(/\s+/)
+    .filter((word) => word.length > 3);
+
+  const wordFrequency = new Map<string, number>();
+  contentWords.forEach((word) => {
+    wordFrequency.set(word, (wordFrequency.get(word) || 0) + 1);
+  });
+
+  const topEntities = Array.from(wordFrequency.entries())
+    .sort(([, a], [, b]) => b - a)
+    .slice(0, 10)
+    .map(([word]) => word);
+
+  const isUpdate = previousSummary !== null;
+
+  return [
+    {
+      role: "system",
+      content: `You are an expert at analyzing and summarizing episodes within semantic spaces based on the space's intent and purpose. Your task is to ${isUpdate ? "update an existing summary by integrating new episodes" : "create a comprehensive summary of episodes"}.
+
+CRITICAL RULES:
+1. Base your summary ONLY on insights derived from the actual content/episodes provided
+2. Use the space's INTENT/PURPOSE (from description) to guide what to summarize and how to organize it
+3. Write in a factual, neutral tone - avoid promotional language ("pivotal", "invaluable", "cutting-edge")
+4. Be specific and concrete - reference actual content, patterns, and insights found in the episodes
+5. If episodes are insufficient for meaningful insights, state that more data is needed
+
+INTENT-DRIVEN SUMMARIZATION:
+Your summary should SERVE the space's intended purpose. Examples:
+- "Learning React" → Summarize React concepts, patterns, techniques learned
+- "Project X Updates" → Summarize progress, decisions, blockers, next steps
+- "Health Tracking" → Summarize metrics, trends, observations, insights
+- "Guidelines for React" → Extract actionable patterns, best practices, rules
+- "Evolution of design thinking" → Track how thinking changed over time, decision points
+The intent defines WHY this space exists - organize content to serve that purpose.
+
+INSTRUCTIONS:
+${
+  isUpdate
+    ? `1. Review the existing summary and themes carefully
+2. Analyze the new episodes for patterns and insights that align with the space's intent
+3. Identify connecting points between existing knowledge and new episodes
+4. Update the summary to seamlessly integrate new information while preserving valuable existing insights
+5. Evolve themes by adding new ones or refining existing ones based on the space's purpose
+6. Organize the summary to serve the space's intended use case`
+    : `1. Analyze the semantic content and relationships within the episodes
+2. Identify topics/sections that align with the space's INTENT and PURPOSE
+3. Create a coherent summary that serves the space's intended use case
+4. Organize the summary based on the space's purpose (not generic frequency-based themes)`
+}
+${isUpdate ? "7" : "5"}. Assess your confidence in the ${isUpdate ? "updated" : ""} summary quality (0.0-1.0)
+
+INTENT-ALIGNED ORGANIZATION:
+- Organize sections based on what serves the space's purpose
+- Topics don't need minimum episode counts - relevance to intent matters most
+- Each section should provide value aligned with the space's intended use
+- For "guidelines" spaces: focus on actionable patterns
+- For "tracking" spaces: focus on temporal patterns and changes
+- For "learning" spaces: focus on concepts and insights gained
+- Let the space's intent drive the structure, not rigid rules
+
+${
+  isUpdate
+    ? `CONNECTION FOCUS:
+- Entity relationships that span across batches/time
+- Theme evolution and expansion
+- Temporal patterns and progressions
+- Contradictions or confirmations of existing insights
+- New insights that complement existing knowledge`
+    : ""
+}
+
+RESPONSE FORMAT:
+Provide your response inside <output></output> tags with valid JSON. Include both HTML summary and markdown format.
+
+<output>
+{
+  "summary": "${isUpdate ? "Updated HTML summary that integrates new insights with existing knowledge. Write factually about what the statements reveal - mention specific entities, relationships, and patterns found in the data. Avoid marketing language. Use HTML tags for structure." : "Factual HTML summary based on patterns found in the statements. Report what the data actually shows - specific entities, relationships, frequencies, and concrete insights. Avoid promotional language. Use HTML tags like <p>, <strong>, <ul>, <li> for structure. Keep it concise and evidence-based."}",
+  "keyEntities": ["entity1", "entity2", "entity3"],
+  "themes": ["${isUpdate ? 'updated_theme1", "new_theme2", "evolved_theme3' : 'theme1", "theme2", "theme3'}"],
+  "confidence": 0.85
+}
+</output>
+
+JSON FORMATTING RULES:
+- HTML content in summary field is allowed and encouraged
+- Escape quotes within strings as \"
+- Escape HTML angle brackets if needed: &lt; and &gt;
+- Use proper HTML tags for structure: <p>, <strong>, <em>, <ul>, <li>, <h3>, etc.
+- HTML content should be well-formed and semantic
+
+GUIDELINES:
+${
+  isUpdate
+    ? `- Preserve valuable insights from existing summary
+- Integrate new information by highlighting connections
+- Themes should evolve naturally, don't replace wholesale
+- The updated summary should read as a coherent whole
+- Make the summary user-friendly and explain what value this space provides`
+    : `- Report only what the episodes actually reveal - be specific and concrete
+- Cite actual content and patterns found in the episodes
+- Avoid generic descriptions that could apply to any space
+- Use neutral, factual language - no "comprehensive", "robust", "cutting-edge" etc.
+- Themes must be backed by at least 3 supporting episodes with clear evidence
+- Better to have fewer, well-supported themes than many weak ones
+- Confidence should reflect actual data quality and coverage, not aspirational goals`
+}`,
+    },
+    {
+      role: "user",
+      content: `SPACE INFORMATION:
+Name: "${spaceName}"
+Intent/Purpose: ${spaceDescription || "No specific intent provided - organize naturally based on content"}
+
+${
+  isUpdate
+    ? `EXISTING SUMMARY:
+${previousSummary}
+
+EXISTING THEMES:
+${previousThemes.join(", ")}
+
+NEW EPISODES TO INTEGRATE (${episodes.length} episodes):`
+    : `EPISODES IN THIS SPACE (${episodes.length} episodes):`
+}
+${episodesText}
+
+${
+  episodes.length > 0
+    ? `TOP WORDS BY FREQUENCY:
+${topEntities.join(", ")}`
+    : ""
+}
+
+${
+  isUpdate
+    ? "Please identify connections between the existing summary and new episodes, then update the summary to integrate the new insights coherently. Organize the summary to SERVE the space's intent/purpose. Remember: only summarize insights from the actual episode content."
+    : "Please analyze the episodes and provide a comprehensive summary that SERVES the space's intent/purpose. Organize sections based on what would be most valuable for this space's intended use case. If the intent is unclear, organize naturally based on content patterns. Only summarize insights from actual episode content."
+}`,
+    },
+  ];
+}
+
+async function getExistingSummary(spaceId: string): Promise<{
+  summary: string;
+  themes: string[];
+  lastUpdated: Date;
+  contextCount: number;
+} | null> {
+  try {
+    const existingSummary = await getSpace(spaceId);
+
+    if (existingSummary?.summary) {
+      return {
+        summary: existingSummary.summary,
+        themes: existingSummary.themes,
+        lastUpdated: existingSummary.summaryGeneratedAt || new Date(),
+        contextCount: existingSummary.contextCount || 0,
+      };
+    }
+
+    return null;
+  } catch (error) {
+    logger.warn(`Failed to get existing summary for space ${spaceId}:`, {
+      error,
+    });
+    return null;
+  }
+}
+
+async function getSpaceEpisodes(
+  spaceId: string,
+  userId: string,
+  sinceDate?: Date,
+): Promise<SpaceEpisodeData[]> {
+  // Query episodes directly using Space-[:HAS_EPISODE]->Episode relationships
+  const params: any = { spaceId, userId };
+
+  let dateCondition = "";
+  if (sinceDate) {
+    dateCondition = "AND e.createdAt > $sinceDate";
+    params.sinceDate = sinceDate.toISOString();
+  }
+
+  const query = `
+    MATCH (space:Space {uuid: $spaceId, userId: $userId})-[:HAS_EPISODE]->(e:Episode {userId: $userId})
+    WHERE e IS NOT NULL ${dateCondition}
+    RETURN DISTINCT e
+    ORDER BY e.createdAt DESC
+  `;
+
+  const result = await runQuery(query, params);
+
+  return result.map((record) => {
+    const episode = record.get("e").properties;
+    return {
+      uuid: episode.uuid,
+      content: episode.content,
+      originalContent: episode.originalContent,
+      source: episode.source,
+      createdAt: new Date(episode.createdAt),
+      validAt: new Date(episode.validAt),
+      metadata: JSON.parse(episode.metadata || "{}"),
+      sessionId: episode.sessionId,
+    };
+  });
+}
+
+function parseSummaryResponse(response: string): {
+  summary: string;
+  themes: string[];
+  confidence: number;
+  keyEntities?: string[];
+} | null {
+  try {
+    // Extract content from <output> tags
+    const outputMatch = response.match(/<output>([\s\S]*?)<\/output>/);
+    if (!outputMatch) {
+      logger.warn("No <output> tags found in LLM summary response");
+      logger.debug("Full LLM response:", { response });
+      return null;
+    }
+
+    let jsonContent = outputMatch[1].trim();
+
+    let parsed;
+    try {
+      parsed = JSON.parse(jsonContent);
+    } catch (jsonError) {
+      logger.warn("JSON parsing failed, attempting cleanup and retry", {
+        originalError: jsonError,
+        jsonContent: jsonContent.substring(0, 500) + "...", // Log first 500 chars
+      });
+
+      // More aggressive cleanup for malformed JSON
+      jsonContent = jsonContent
+        .replace(/([^\\])"/g, '$1\\"') // Escape unescaped quotes
+        .replace(/^"/g, '\\"') // Escape quotes at start
+        .replace(/\\\\"/g, '\\"'); // Fix double-escaped quotes
+
+      parsed = JSON.parse(jsonContent);
+    }
+
+    // Validate the response structure
+    const validationResult = SummaryResultSchema.safeParse(parsed);
+    if (!validationResult.success) {
+      logger.warn("Invalid LLM summary response format:", {
+        error: validationResult.error,
+        parsedData: parsed,
+      });
+      return null;
+    }
+
+    return validationResult.data;
+  } catch (error) {
+    logger.error(
+      "Error parsing LLM summary response:",
+      error as Record<string, unknown>,
+    );
+    logger.debug("Failed response content:", { response });
+    return null;
+  }
+}
+
+async function storeSummary(summaryData: SpaceSummaryData): Promise<void> {
+  try {
+    // Store in PostgreSQL for API access and persistence
+    await updateSpace(summaryData);
+
+    // Also store in Neo4j for graph-based queries
+    const query = `
+      MATCH (space:Space {uuid: $spaceId})
+      SET space.summary = $summary,
+          space.keyEntities = $keyEntities,
+          space.themes = $themes,
+          space.summaryConfidence = $confidence,
+          space.summaryContextCount = $contextCount,
+          space.summaryLastUpdated = datetime($lastUpdated)
+      RETURN space
+    `;
+
+    await runQuery(query, {
+      spaceId: summaryData.spaceId,
+      summary: summaryData.summary,
+      keyEntities: summaryData.keyEntities,
+      themes: summaryData.themes,
+      confidence: summaryData.confidence,
+      contextCount: summaryData.contextCount,
+      lastUpdated: summaryData.lastUpdated.toISOString(),
+    });
+
+    logger.info(`Stored summary for space ${summaryData.spaceId}`, {
+      themes: summaryData.themes.length,
+      keyEntities: summaryData.keyEntities.length,
+      confidence: summaryData.confidence,
+    });
+  } catch (error) {
+    logger.error(
+      `Error storing summary for space ${summaryData.spaceId}:`,
+      error as Record<string, unknown>,
+    );
+    throw error;
+  }
+}

apps/webapp/app/lib/ingest.server.ts

@@ -4,13 +4,18 @@ import { EpisodeType } from "@core/types";
 import { type z } from "zod";
 import { prisma } from "~/db.server";
 import { hasCredits } from "~/services/billing.server";
-import { type IngestBodyRequest, ingestTask } from "~/trigger/ingest/ingest";
-import { ingestDocumentTask } from "~/trigger/ingest/ingest-document";
+import { type IngestBodyRequest } from "~/trigger/ingest/ingest";
+import {
+  enqueueIngestDocument,
+  enqueueIngestEpisode,
+} from "~/lib/queue-adapter.server";
+import { trackFeatureUsage } from "~/services/telemetry.server";
 
 export const addToQueue = async (
   rawBody: z.infer<typeof IngestBodyRequest>,
   userId: string,
   activityId?: string,
+  ingestionQueueId?: string,
 ) => {
   const body = { ...rawBody, source: rawBody.source.toLowerCase() };
   const user = await prisma.user.findFirst({
@@ -38,8 +43,18 @@ export const addToQueue = async (
     throw new Error("no credits");
   }
 
-  const queuePersist = await prisma.ingestionQueue.create({
-    data: {
+  // Upsert: update existing or create new ingestion queue entry
+  const queuePersist = await prisma.ingestionQueue.upsert({
+    where: {
+      id: ingestionQueueId || "non-existent-id", // Use provided ID or dummy ID to force create
+    },
+    update: {
+      data: body,
+      type: body.type,
+      status: IngestionStatus.PENDING,
+      error: null,
+    },
+    create: {
       data: body,
       type: body.type,
       status: IngestionStatus.PENDING,
@@ -51,36 +66,28 @@ export const addToQueue = async (
 
   let handler;
   if (body.type === EpisodeType.DOCUMENT) {
-    handler = await ingestDocumentTask.trigger(
-      {
-        body,
-        userId,
-        workspaceId: user.Workspace.id,
-        queueId: queuePersist.id,
-      },
-      {
-        queue: "document-ingestion-queue",
-        concurrencyKey: userId,
-        tags: [user.id, queuePersist.id],
-      },
-    );
+    handler = await enqueueIngestDocument({
+      body,
+      userId,
+      workspaceId: user.Workspace.id,
+      queueId: queuePersist.id,
+    });
+
+    // Track document ingestion
+    trackFeatureUsage("document_ingested", userId).catch(console.error);
   } else if (body.type === EpisodeType.CONVERSATION) {
-    handler = await ingestTask.trigger(
-      {
-        body,
-        userId,
-        workspaceId: user.Workspace.id,
-        queueId: queuePersist.id,
-      },
-      {
-        queue: "ingestion-queue",
-        concurrencyKey: userId,
-        tags: [user.id, queuePersist.id],
-      },
-    );
+    handler = await enqueueIngestEpisode({
+      body,
+      userId,
+      workspaceId: user.Workspace.id,
+      queueId: queuePersist.id,
+    });
+
+    // Track episode ingestion
+    trackFeatureUsage("episode_ingested", userId).catch(console.error);
   }
 
-  return { id: handler?.id, token: handler?.publicAccessToken };
+  return { id: handler?.id, publicAccessToken: handler?.token };
 };
 
 export { IngestBodyRequest };

apps/webapp/app/lib/model.server.ts

@@ -1,31 +1,23 @@
-import {
-  type CoreMessage,
-  type LanguageModelV1,
-  embed,
-  generateText,
-  streamText,
-} from "ai";
+import { type CoreMessage, embed, generateText, streamText } from "ai";
 import { openai } from "@ai-sdk/openai";
 import { logger } from "~/services/logger.service";
 
-import { createOllama, type OllamaProvider } from "ollama-ai-provider";
+import { createOllama } from "ollama-ai-provider-v2";
 import { anthropic } from "@ai-sdk/anthropic";
 import { google } from "@ai-sdk/google";
-import { createAmazonBedrock } from "@ai-sdk/amazon-bedrock";
-import { fromNodeProviderChain } from "@aws-sdk/credential-providers";
 
-export type ModelComplexity = 'high' | 'low';
+export type ModelComplexity = "high" | "low";
 
 /**
  * Get the appropriate model for a given complexity level.
  * HIGH complexity uses the configured MODEL.
  * LOW complexity automatically downgrades to cheaper variants if possible.
  */
-export function getModelForTask(complexity: ModelComplexity = 'high'): string {
-  const baseModel = process.env.MODEL || 'gpt-4.1-2025-04-14';
+export function getModelForTask(complexity: ModelComplexity = "high"): string {
+  const baseModel = process.env.MODEL || "gpt-4.1-2025-04-14";
 
   // HIGH complexity - always use the configured model
-  if (complexity === 'high') {
+  if (complexity === "high") {
     return baseModel;
   }
 
@@ -33,29 +25,73 @@ export function getModelForTask(complexity: ModelComplexity = 'high'): string {
   // If already using a cheap model, keep it
   const downgrades: Record<string, string> = {
     // OpenAI downgrades
-    'gpt-5-2025-08-07': 'gpt-5-mini-2025-08-07',
-    'gpt-4.1-2025-04-14': 'gpt-4.1-mini-2025-04-14',
+    "gpt-5-2025-08-07": "gpt-5-mini-2025-08-07",
+    "gpt-4.1-2025-04-14": "gpt-4.1-mini-2025-04-14",
 
     // Anthropic downgrades
-    'claude-sonnet-4-5': 'claude-3-5-haiku-20241022',
-    'claude-3-7-sonnet-20250219': 'claude-3-5-haiku-20241022',
-    'claude-3-opus-20240229': 'claude-3-5-haiku-20241022',
+    "claude-sonnet-4-5": "claude-3-5-haiku-20241022",
+    "claude-3-7-sonnet-20250219": "claude-3-5-haiku-20241022",
+    "claude-3-opus-20240229": "claude-3-5-haiku-20241022",
 
     // Google downgrades
-    'gemini-2.5-pro-preview-03-25': 'gemini-2.5-flash-preview-04-17',
-    'gemini-2.0-flash': 'gemini-2.0-flash-lite',
+    "gemini-2.5-pro-preview-03-25": "gemini-2.5-flash-preview-04-17",
+    "gemini-2.0-flash": "gemini-2.0-flash-lite",
 
     // AWS Bedrock downgrades (keep same model - already cost-optimized)
-    'us.amazon.nova-premier-v1:0': 'us.amazon.nova-premier-v1:0',
+    "us.amazon.nova-premier-v1:0": "us.amazon.nova-premier-v1:0",
   };
 
   return downgrades[baseModel] || baseModel;
 }
 
+export const getModel = (takeModel?: string) => {
+  let model = takeModel;
+
+  const anthropicKey = process.env.ANTHROPIC_API_KEY;
+  const googleKey = process.env.GOOGLE_GENERATIVE_AI_API_KEY;
+  const openaiKey = process.env.OPENAI_API_KEY;
+  let ollamaUrl = process.env.OLLAMA_URL;
+  model = model || process.env.MODEL || "gpt-4.1-2025-04-14";
+
+  let modelInstance;
+  let modelTemperature = Number(process.env.MODEL_TEMPERATURE) || 1;
+  ollamaUrl = undefined;
+
+  // First check if Ollama URL exists and use Ollama
+  if (ollamaUrl) {
+    const ollama = createOllama({
+      baseURL: ollamaUrl,
+    });
+    modelInstance = ollama(model || "llama2"); // Default to llama2 if no model specified
+  } else {
+    // If no Ollama, check other models
+
+    if (model.includes("claude")) {
+      if (!anthropicKey) {
+        throw new Error("No Anthropic API key found. Set ANTHROPIC_API_KEY");
+      }
+      modelInstance = anthropic(model);
+      modelTemperature = 0.5;
+    } else if (model.includes("gemini")) {
+      if (!googleKey) {
+        throw new Error("No Google API key found. Set GOOGLE_API_KEY");
+      }
+      modelInstance = google(model);
+    } else {
+      if (!openaiKey) {
+        throw new Error("No OpenAI API key found. Set OPENAI_API_KEY");
+      }
+      modelInstance = openai(model);
+    }
+
+    return modelInstance;
+  }
+};
+
 export interface TokenUsage {
-  promptTokens: number;
-  completionTokens: number;
-  totalTokens: number;
+  promptTokens?: number;
+  completionTokens?: number;
+  totalTokens?: number;
 }
 
 export async function makeModelCall(
@@ -63,69 +99,13 @@ export async function makeModelCall(
   messages: CoreMessage[],
   onFinish: (text: string, model: string, usage?: TokenUsage) => void,
   options?: any,
-  complexity: ModelComplexity = 'high',
+  complexity: ModelComplexity = "high",
 ) {
-  let modelInstance: LanguageModelV1 | undefined;
   let model = getModelForTask(complexity);
-  const ollamaUrl = process.env.OLLAMA_URL;
-  let ollama: OllamaProvider | undefined;
+  logger.info(`complexity: ${complexity}, model: ${model}`);
 
-  if (ollamaUrl) {
-    ollama = createOllama({
-      baseURL: ollamaUrl,
-    });
-  }
-
-  const bedrock = createAmazonBedrock({
-    region: process.env.AWS_REGION || 'us-east-1',
-    credentialProvider: fromNodeProviderChain(),
-  });
-
-  const generateTextOptions: any = {}
-
-  logger.info(
-    `complexity: ${complexity}, model: ${model}`,
-  );
-  switch (model) {
-    case "gpt-4.1-2025-04-14":
-    case "gpt-4.1-mini-2025-04-14":
-    case "gpt-5-mini-2025-08-07":
-    case "gpt-5-2025-08-07":
-    case "gpt-4.1-nano-2025-04-14":
-      modelInstance = openai(model, { ...options });
-      generateTextOptions.temperature = 1
-      break;
-
-    case "claude-3-7-sonnet-20250219":
-    case "claude-3-opus-20240229":
-    case "claude-3-5-haiku-20241022":
-      modelInstance = anthropic(model, { ...options });
-      break;
-
-    case "gemini-2.5-flash-preview-04-17":
-    case "gemini-2.5-pro-preview-03-25":
-    case "gemini-2.0-flash":
-    case "gemini-2.0-flash-lite":
-      modelInstance = google(model, { ...options });
-      break;
-
-    case "us.meta.llama3-3-70b-instruct-v1:0":
-    case "us.deepseek.r1-v1:0":
-    case "qwen.qwen3-32b-v1:0":
-    case "openai.gpt-oss-120b-1:0":
-    case "us.mistral.pixtral-large-2502-v1:0":
-    case "us.amazon.nova-premier-v1:0":
-      modelInstance = bedrock(`${model}`);
-      generateTextOptions.maxTokens = 100000
-      break;
-
-    default:
-      if (ollama) {
-        modelInstance = ollama(model);
-      }
-      logger.warn(`Unsupported model type: ${model}`);
-      break;
-  }
+  const modelInstance = getModel(model);
+  const generateTextOptions: any = {};
 
   if (!modelInstance) {
     throw new Error(`Unsupported model type: ${model}`);
@@ -135,16 +115,21 @@ export async function makeModelCall(
     return streamText({
       model: modelInstance,
       messages,
+      ...options,
       ...generateTextOptions,
       onFinish: async ({ text, usage }) => {
-        const tokenUsage = usage ? {
-          promptTokens: usage.promptTokens,
-          completionTokens: usage.completionTokens,
-          totalTokens: usage.totalTokens,
-        } : undefined;
+        const tokenUsage = usage
+          ? {
+              promptTokens: usage.inputTokens,
+              completionTokens: usage.outputTokens,
+              totalTokens: usage.totalTokens,
+            }
+          : undefined;
 
         if (tokenUsage) {
-          logger.log(`[${complexity.toUpperCase()}] ${model} - Tokens: ${tokenUsage.totalTokens} (prompt: ${tokenUsage.promptTokens}, completion: ${tokenUsage.completionTokens})`);
+          logger.log(
+            `[${complexity.toUpperCase()}] ${model} - Tokens: ${tokenUsage.totalTokens} (prompt: ${tokenUsage.promptTokens}, completion: ${tokenUsage.completionTokens})`,
+          );
         }
 
         onFinish(text, model, tokenUsage);
@@ -158,14 +143,18 @@ export async function makeModelCall(
     ...generateTextOptions,
   });
 
-  const tokenUsage = usage ? {
-    promptTokens: usage.promptTokens,
-    completionTokens: usage.completionTokens,
-    totalTokens: usage.totalTokens,
-  } : undefined;
+  const tokenUsage = usage
+    ? {
+        promptTokens: usage.inputTokens,
+        completionTokens: usage.outputTokens,
+        totalTokens: usage.totalTokens,
+      }
+    : undefined;
 
   if (tokenUsage) {
-    logger.log(`[${complexity.toUpperCase()}] ${model} - Tokens: ${tokenUsage.totalTokens} (prompt: ${tokenUsage.promptTokens}, completion: ${tokenUsage.completionTokens})`);
+    logger.log(
+      `[${complexity.toUpperCase()}] ${model} - Tokens: ${tokenUsage.totalTokens} (prompt: ${tokenUsage.promptTokens}, completion: ${tokenUsage.completionTokens})`,
+    );
   }
 
   onFinish(text, model, tokenUsage);
@@ -177,19 +166,22 @@ export async function makeModelCall(
  * Determines if a given model is proprietary (OpenAI, Anthropic, Google, Grok)
  * or open source (accessed via Bedrock, Ollama, etc.)
  */
-export function isProprietaryModel(modelName?: string, complexity: ModelComplexity = 'high'): boolean {
+export function isProprietaryModel(
+  modelName?: string,
+  complexity: ModelComplexity = "high",
+): boolean {
   const model = modelName || getModelForTask(complexity);
   if (!model) return false;
 
   // Proprietary model patterns
   const proprietaryPatterns = [
-    /^gpt-/,           // OpenAI models
-    /^claude-/,        // Anthropic models
-    /^gemini-/,        // Google models
-    /^grok-/,          // xAI models
+    /^gpt-/, // OpenAI models
+    /^claude-/, // Anthropic models
+    /^gemini-/, // Google models
+    /^grok-/, // xAI models
   ];
 
-  return proprietaryPatterns.some(pattern => pattern.test(model));
+  return proprietaryPatterns.some((pattern) => pattern.test(model));
 }
 
 export async function getEmbedding(text: string) {

apps/webapp/app/lib/neo4j.server.ts

@@ -112,51 +112,31 @@ export const getNodeLinks = async (userId: string) => {
 export const getClusteredGraphData = async (userId: string) => {
   const session = driver.session();
   try {
-    // Get the simplified graph structure: Episode, Subject, Object with Predicate as edge
-    // Only include entities that are connected to more than 1 episode
+    // Get Episode -> Entity graph, only showing entities connected to more than 1 episode
     const result = await session.run(
       `// Find entities connected to more than 1 episode
-       MATCH (e:Episode)-[:HAS_PROVENANCE]->(s:Statement {userId: $userId})
-       MATCH (s)-[:HAS_SUBJECT|HAS_OBJECT|HAS_PREDICATE]->(ent:Entity)
-       WITH ent, count(DISTINCT e) as episodeCount
+       MATCH (e:Episode{userId: $userId})-[:HAS_PROVENANCE]->(s:Statement {userId: $userId})-[r:HAS_SUBJECT|HAS_OBJECT|HAS_PREDICATE]->(entity:Entity)
+       WITH entity, count(DISTINCT e) as episodeCount
        WHERE episodeCount > 1
-       WITH collect(ent.uuid) as validEntityUuids
+       WITH collect(entity.uuid) as validEntityUuids
 
-       // Get statements where all entities are in the valid set
-       MATCH (e:Episode)-[:HAS_PROVENANCE]->(s:Statement {userId: $userId})
-       MATCH (s)-[:HAS_SUBJECT]->(subj:Entity)
-       WHERE subj.uuid IN validEntityUuids
-       MATCH (s)-[:HAS_PREDICATE]->(pred:Entity)
-       WHERE pred.uuid IN validEntityUuids
-       MATCH (s)-[:HAS_OBJECT]->(obj:Entity)
-       WHERE obj.uuid IN validEntityUuids
-
-       // Build relationships
-       WITH e, s, subj, pred, obj
-       UNWIND [
-         // Episode -> Subject
-         {source: e, sourceType: 'Episode', target: subj, targetType: 'Entity', predicate: null},
-         // Episode -> Object
-         {source: e, sourceType: 'Episode', target: obj, targetType: 'Entity', predicate: null},
-         // Subject -> Object (with Predicate as edge)
-         {source: subj, sourceType: 'Entity', target: obj, targetType: 'Entity', predicate: pred.name}
-       ] AS rel
+       // Build Episode -> Entity relationships for valid entities
+       MATCH (e:Episode{userId: $userId})-[r:HAS_PROVENANCE]->(s:Statement {userId: $userId})-[r:HAS_SUBJECT|HAS_OBJECT|HAS_PREDICATE]->(entity:Entity)
+       WHERE entity.uuid IN validEntityUuids
+       WITH DISTINCT e, entity, type(r) as relType,
+            CASE WHEN size(e.spaceIds) > 0 THEN e.spaceIds[0] ELSE null END as clusterId,
+            s.createdAt as createdAt
 
        RETURN DISTINCT
-         rel.source.uuid as sourceUuid,
-         rel.source.name as sourceName,
-         rel.source.content as sourceContent,
-         rel.sourceType as sourceNodeType,
-         rel.target.uuid as targetUuid,
-         rel.target.name as targetName,
-         rel.targetType as targetNodeType,
-         rel.predicate as predicateLabel,
-         e.uuid as episodeUuid,
-         e.content as episodeContent,
-         e.spaceIds as spaceIds,
-         s.uuid as statementUuid,
-         s.validAt as validAt,
-         s.createdAt as createdAt`,
+         e.uuid as sourceUuid,
+         e.content as sourceContent,
+         'Episode' as sourceNodeType,
+         entity.uuid as targetUuid,
+         entity.name as targetName,
+         'Entity' as targetNodeType,
+         relType as edgeType,
+         clusterId,
+         createdAt`,
       { userId },
     );
 
@@ -165,72 +145,29 @@ export const getClusteredGraphData = async (userId: string) => {
 
     result.records.forEach((record) => {
       const sourceUuid = record.get("sourceUuid");
-      const sourceName = record.get("sourceName");
       const sourceContent = record.get("sourceContent");
-      const sourceNodeType = record.get("sourceNodeType");
-
       const targetUuid = record.get("targetUuid");
       const targetName = record.get("targetName");
-      const targetNodeType = record.get("targetNodeType");
-
-      const predicateLabel = record.get("predicateLabel");
-      const episodeUuid = record.get("episodeUuid");
-      const clusterIds = record.get("spaceIds");
-      const clusterId = clusterIds ? clusterIds[0] : undefined;
+      const edgeType = record.get("edgeType");
+      const clusterId = record.get("clusterId");
       const createdAt = record.get("createdAt");
 
       // Create unique edge identifier to avoid duplicates
-      // For Episode->Subject edges, use generic type; for Subject->Object use predicate
-      const edgeType = predicateLabel || "HAS_SUBJECT";
       const edgeKey = `${sourceUuid}-${targetUuid}-${edgeType}`;
       if (processedEdges.has(edgeKey)) return;
       processedEdges.add(edgeKey);
 
-      // Build node attributes based on type
-      const sourceAttributes =
-        sourceNodeType === "Episode"
-          ? {
-              nodeType: "Episode",
-              content: sourceContent,
-              episodeUuid: sourceUuid,
-              clusterId,
-            }
-          : {
-              nodeType: "Entity",
-              name: sourceName,
-              clusterId,
-            };
-
-      const targetAttributes =
-        targetNodeType === "Episode"
-          ? {
-              nodeType: "Episode",
-              content: sourceContent,
-              episodeUuid: targetUuid,
-              clusterId,
-            }
-          : {
-              nodeType: "Entity",
-              name: targetName,
-              clusterId,
-            };
-
-      // Build display name
-      const sourceDisplayName =
-        sourceNodeType === "Episode"
-          ? sourceContent || episodeUuid
-          : sourceName || sourceUuid;
-      const targetDisplayName =
-        targetNodeType === "Episode"
-          ? sourceContent || episodeUuid
-          : targetName || targetUuid;
-
       triplets.push({
         sourceNode: {
           uuid: sourceUuid,
-          labels: [sourceNodeType],
-          attributes: sourceAttributes,
-          name: sourceDisplayName,
+          labels: ["Episode"],
+          attributes: {
+            nodeType: "Episode",
+            content: sourceContent,
+            episodeUuid: sourceUuid,
+            clusterId,
+          },
+          name: sourceContent || sourceUuid,
           clusterId,
           createdAt: createdAt || "",
         },
@@ -243,10 +180,14 @@ export const getClusteredGraphData = async (userId: string) => {
         },
         targetNode: {
           uuid: targetUuid,
-          labels: [targetNodeType],
-          attributes: targetAttributes,
+          labels: ["Entity"],
+          attributes: {
+            nodeType: "Entity",
+            name: targetName,
+            clusterId,
+          },
+          name: targetName || targetUuid,
           clusterId,
-          name: targetDisplayName,
           createdAt: createdAt || "",
         },
       });

apps/webapp/app/lib/prompt.server.ts

@@ -0,0 +1,324 @@
+import { type StopCondition } from "ai";
+
+export const hasAnswer: StopCondition<any> = ({ steps }) => {
+  return (
+    steps.some((step) => step.text?.includes("</final_response>")) ?? false
+  );
+};
+
+export const hasQuestion: StopCondition<any> = ({ steps }) => {
+  return (
+    steps.some((step) => step.text?.includes("</question_response>")) ?? false
+  );
+};
+
+export const REACT_SYSTEM_PROMPT = `
+You are a helpful AI assistant with access to user memory. Your primary capabilities are:
+
+1. **Memory-First Approach**: Always check user memory first to understand context and previous interactions
+2. **Intelligent Information Gathering**: Analyze queries to determine if current information is needed
+3. **Memory Management**: Help users store, retrieve, and organize information in their memory
+4. **Contextual Assistance**: Use memory to provide personalized and contextual responses
+
+<information_gathering>
+Follow this intelligent approach for information gathering:
+
+1. **MEMORY FIRST** (Always Required)
+   - Always check memory FIRST using core--search_memory before any other actions
+   - Consider this your highest priority for EVERY interaction - as essential as breathing
+   - Memory provides context, personal preferences, and historical information
+   - Use memory to understand user's background, ongoing projects, and past conversations
+
+2. **INFORMATION SYNTHESIS** (Combine Sources)
+   - Use memory to personalize current information based on user preferences
+   - Always store new useful information in memory using core--add_memory
+
+3. **TRAINING KNOWLEDGE** (Foundation)
+   - Use your training knowledge as the foundation for analysis and explanation
+   - Apply training knowledge to interpret and contextualize information from memory
+   - Indicate when you're using training knowledge vs. live information sources
+
+EXECUTION APPROACH:
+- Memory search is mandatory for every interaction
+- Always indicate your information sources in responses
+</information_gathering>
+
+<memory>
+QUERY FORMATION:
+- Write specific factual statements as queries (e.g., "user email address" not "what is the user's email?")
+- Create multiple targeted memory queries for complex requests
+
+KEY QUERY AREAS:
+- Personal context: user name, location, identity, work context
+- Project context: repositories, codebases, current work, team members
+- Task context: recent tasks, ongoing projects, deadlines, priorities
+- Integration context: GitHub repos, Slack channels, Linear projects, connected services
+- Communication patterns: email preferences, notification settings, workflow automation
+- Technical context: coding languages, frameworks, development environment
+- Collaboration context: team members, project stakeholders, meeting patterns
+- Preferences: likes, dislikes, communication style, tool preferences
+- History: previous discussions, past requests, completed work, recurring issues
+- Automation rules: user-defined workflows, triggers, automation preferences
+
+MEMORY USAGE:
+- Execute multiple memory queries in parallel rather than sequentially
+- Batch related memory queries when possible
+- Prioritize recent information over older memories
+- Create comprehensive context-aware queries based on user message/activity content
+- Extract and query SEMANTIC CONTENT, not just structural metadata
+- Parse titles, descriptions, and content for actual subject matter keywords
+- Search internal SOL tasks/conversations that may relate to the same topics
+- Query ALL relatable concepts, not just direct keywords or IDs
+- Search for similar past situations, patterns, and related work
+- Include synonyms, related terms, and contextual concepts in queries  
+- Query user's historical approach to similar requests or activities
+- Search for connected projects, tasks, conversations, and collaborations
+- Retrieve workflow patterns and past decision-making context
+- Query broader domain context beyond immediate request scope
+- Remember: SOL tracks work that external tools don't - search internal content thoroughly
+- Blend memory insights naturally into responses
+- Verify you've checked relevant memory before finalizing ANY response
+
+</memory>
+
+<external_services>
+- To use: load_mcp with EXACT integration name from the available list
+- Can load multiple at once with an array
+- Only load when tools are NOT already available in your current toolset
+- If a tool is already available, use it directly without load_mcp
+- If requested integration unavailable: inform user politely
+</external_services>
+
+<tool_calling>
+You have tools at your disposal to assist users:
+
+CORE PRINCIPLES:
+- Use tools only when necessary for the task at hand
+- Always check memory FIRST before making other tool calls
+- Execute multiple operations in parallel whenever possible
+- Use sequential calls only when output of one is required for input of another
+
+PARAMETER HANDLING:
+- Follow tool schemas exactly with all required parameters
+- Only use values that are:
+  • Explicitly provided by the user (use EXACTLY as given)
+  • Reasonably inferred from context
+  • Retrieved from memory or prior tool calls
+- Never make up values for required parameters
+- Omit optional parameters unless clearly needed
+- Analyze user's descriptive terms for parameter clues
+
+TOOL SELECTION:
+- Never call tools not provided in this conversation
+- Skip tool calls for general questions you can answer directly from memory/knowledge
+- For identical operations on multiple items, use parallel tool calls
+- Default to parallel execution (3-5× faster than sequential calls)
+- You can always access external service tools by loading them with load_mcp first
+
+TOOL MENTION HANDLING:
+When user message contains <mention data-id="tool_name" data-label="tool"></mention>:
+- Extract tool_name from data-id attribute
+- First check if it's a built-in tool; if not, check EXTERNAL SERVICES TOOLS
+- If available: Load it with load_mcp and focus on addressing the request with this tool
+- If unavailable: Inform user and suggest alternatives if possible
+- For multiple tool mentions: Load all applicable tools in a single load_mcp call
+
+ERROR HANDLING:
+- If a tool returns an error, try fixing parameters before retrying
+- If you can't resolve an error, explain the issue to the user
+- Consider alternative tools when primary tools are unavailable
+</tool_calling>
+
+<communication>
+Use EXACTLY ONE of these formats for all user-facing communication:
+
+PROGRESS UPDATES - During processing:
+- Use the core--progress_update tool to keep users informed
+- Update users about what you're discovering or doing next
+- Keep messages clear and user-friendly
+- Avoid technical jargon
+
+QUESTIONS - When you need information:
+<question_response>
+<p>[Your question with HTML formatting]</p>
+</question_response>
+
+- Ask questions only when you cannot find information through memory, or tools
+- Be specific about what you need to know
+- Provide context for why you're asking
+
+FINAL ANSWERS - When completing tasks:
+<final_response>
+<p>[Your answer with HTML formatting]</p>
+</final_response>
+
+CRITICAL:
+- Use ONE format per turn
+- Apply proper HTML formatting (<h1>, <h2>, <p>, <ul>, <li>, etc.)
+- Never mix communication formats
+- Keep responses clear and helpful
+- Always indicate your information sources (memory, and/or knowledge)
+</communication>
+`;
+
+export function getReActPrompt(
+  metadata?: { source?: string; url?: string; pageTitle?: string },
+  intentOverride?: string,
+): string {
+  const contextHints = [];
+
+  if (
+    metadata?.source === "chrome" &&
+    metadata?.url?.includes("mail.google.com")
+  ) {
+    contextHints.push("Content is from email - likely reading intent");
+  }
+  if (
+    metadata?.source === "chrome" &&
+    metadata?.url?.includes("calendar.google.com")
+  ) {
+    contextHints.push("Content is from calendar - likely meeting prep intent");
+  }
+  if (
+    metadata?.source === "chrome" &&
+    metadata?.url?.includes("docs.google.com")
+  ) {
+    contextHints.push(
+      "Content is from document editor - likely writing intent",
+    );
+  }
+  if (metadata?.source === "obsidian") {
+    contextHints.push(
+      "Content is from note editor - likely writing or research intent",
+    );
+  }
+
+  return `You are a memory research agent analyzing content to find relevant context.
+
+YOUR PROCESS (ReAct Framework):
+
+1. DECOMPOSE: First, break down the content into structured categories
+
+   Analyze the content and extract:
+   a) ENTITIES: Specific people, project names, tools, products mentioned
+      Example: "John Smith", "Phoenix API", "Redis", "mobile app"
+
+   b) TOPICS & CONCEPTS: Key subjects, themes, domains
+      Example: "authentication", "database design", "performance optimization"
+
+   c) TEMPORAL MARKERS: Time references, deadlines, events
+      Example: "last week's meeting", "Q2 launch", "yesterday's discussion"
+
+   d) ACTIONS & TASKS: What's being done, decided, or requested
+      Example: "implement feature", "review code", "make decision on"
+
+   e) USER INTENT: What is the user trying to accomplish?
+      ${intentOverride ? `User specified: "${intentOverride}"` : "Infer from context: reading/writing/meeting prep/research/task tracking/review"}
+
+2. FORM QUERIES: Create targeted search queries from your decomposition
+
+   Based on decomposition, form specific queries:
+   - Search for each entity by name (people, projects, tools)
+   - Search for topics the user has discussed before
+   - Search for related work or conversations in this domain
+   - Use the user's actual terminology, not generic concepts
+
+   EXAMPLE - Content: "Email from Sarah about the API redesign we discussed last week"
+   Decomposition:
+     - Entities: "Sarah", "API redesign"
+     - Topics: "API design", "redesign"
+     - Temporal: "last week"
+     - Actions: "discussed", "email communication"
+     - Intent: Reading (email) / meeting prep
+
+   Queries to form:
+   ✅ "Sarah" (find past conversations with Sarah)
+   ✅ "API redesign" or "API design" (find project discussions)
+   ✅ "last week" + "Sarah" (find recent context)
+   ✅ "meetings" or "discussions" (find related conversations)
+
+   ❌ Avoid: "email communication patterns", "API architecture philosophy"
+   (These are abstract - search what user actually discussed!)
+
+3. SEARCH: Execute your queries using searchMemory tool
+   - Start with 2-3 core searches based on main entities/topics
+   - Make each search specific and targeted
+   - Use actual terms from the content, not rephrased concepts
+
+4. OBSERVE: Evaluate search results
+   - Did you find relevant episodes? How many unique ones?
+   - What specific context emerged?
+   - What new entities/topics appeared in results?
+   - Are there gaps in understanding?
+   - Should you search more angles?
+
+   Note: Episode counts are automatically deduplicated across searches - overlapping episodes are only counted once.
+
+5. REACT: Decide next action based on observations
+
+   STOPPING CRITERIA - Proceed to SYNTHESIZE if ANY of these are true:
+   - You found 20+ unique episodes across your searches → ENOUGH CONTEXT
+   - You performed 5+ searches and found relevant episodes → SUFFICIENT
+   - You performed 7+ searches regardless of results → EXHAUSTED STRATEGIES
+   - You found strong relevant context from multiple angles → COMPLETE
+
+   System nudges will provide awareness of your progress, but you decide when synthesis quality would be optimal.
+
+   If you found little/no context AND searched less than 7 times:
+   - Try different query angles from your decomposition
+   - Search broader related topics
+   - Search user's projects or work areas
+   - Try alternative terminology
+
+   ⚠️ DO NOT search endlessly - if you found relevant episodes, STOP and synthesize!
+
+6. SYNTHESIZE: After gathering sufficient context, provide final answer
+   - Wrap your synthesis in <final_response> tags
+   - Present direct factual context from memory - no meta-commentary
+   - Write as if providing background context to an AI assistant
+   - Include: facts, decisions, preferences, patterns, timelines
+   - Note any gaps, contradictions, or evolution in thinking
+   - Keep it concise and actionable
+   - DO NOT use phrases like "Previous discussions on", "From conversations", "Past preferences indicate"
+   - DO NOT use conversational language like "you said" or "you mentioned"
+   - Present information as direct factual statements
+
+FINAL RESPONSE FORMAT:
+<final_response>
+[Direct synthesized context - factual statements only]
+
+Good examples:
+- "The API redesign focuses on performance and scalability. Key decisions: moving to GraphQL, caching layer with Redis."
+- "Project Phoenix launches Q2 2024. Main features: real-time sync, offline mode, collaborative editing."
+- "Sarah leads the backend team. Recent work includes authentication refactor and database migration."
+
+Bad examples:
+❌ "Previous discussions on the API revealed..."
+❌ "From past conversations, it appears that..."
+❌ "Past preferences indicate..."
+❌ "The user mentioned that..."
+
+Just state the facts directly.
+</final_response>
+
+${contextHints.length > 0 ? `\nCONTEXT HINTS:\n${contextHints.join("\n")}` : ""}
+
+CRITICAL REQUIREMENTS:
+- ALWAYS start with DECOMPOSE step - extract entities, topics, temporal markers, actions
+- Form specific queries from your decomposition - use user's actual terms
+- Minimum 3 searches required
+- Maximum 10 searches allowed - must synthesize after that
+- STOP and synthesize when you hit stopping criteria (20+ episodes, 5+ searches with results, 7+ searches total)
+- Each search should target different aspects from decomposition
+- Present synthesis directly without meta-commentary
+
+SEARCH QUALITY CHECKLIST:
+✅ Queries use specific terms from content (names, projects, exact phrases)
+✅ Searched multiple angles from decomposition (entities, topics, related areas)
+✅ Stop when you have enough unique context - don't search endlessly
+✅ Tried alternative terminology if initial searches found nothing
+❌ Avoid generic/abstract queries that don't match user's vocabulary
+❌ Don't stop at 3 searches if you found zero unique episodes
+❌ Don't keep searching when you already found 20+ unique episodes
+}`;
+}

apps/webapp/app/lib/queue-adapter.server.ts

@@ -0,0 +1,233 @@
+/**
+ * Queue Adapter
+ *
+ * This module provides a unified interface for queueing background jobs,
+ * supporting both Trigger.dev and BullMQ backends based on the QUEUE_PROVIDER
+ * environment variable.
+ *
+ * Usage:
+ * - Set QUEUE_PROVIDER="trigger" for Trigger.dev (default, good for production scaling)
+ * - Set QUEUE_PROVIDER="bullmq" for BullMQ (good for open-source deployments)
+ */
+
+import { env } from "~/env.server";
+import type { z } from "zod";
+import type { IngestBodyRequest } from "~/jobs/ingest/ingest-episode.logic";
+import type { CreateConversationTitlePayload } from "~/jobs/conversation/create-title.logic";
+import type { SessionCompactionPayload } from "~/jobs/session/session-compaction.logic";
+import type { SpaceAssignmentPayload } from "~/jobs/spaces/space-assignment.logic";
+import type { SpaceSummaryPayload } from "~/jobs/spaces/space-summary.logic";
+
+type QueueProvider = "trigger" | "bullmq";
+
+/**
+ * Enqueue episode ingestion job
+ */
+export async function enqueueIngestEpisode(payload: {
+  body: z.infer<typeof IngestBodyRequest>;
+  userId: string;
+  workspaceId: string;
+  queueId: string;
+}): Promise<{ id?: string; token?: string }> {
+  const provider = env.QUEUE_PROVIDER as QueueProvider;
+
+  if (provider === "trigger") {
+    const { ingestTask } = await import("~/trigger/ingest/ingest");
+    const handler = await ingestTask.trigger(payload, {
+      queue: "ingestion-queue",
+      concurrencyKey: payload.userId,
+      tags: [payload.userId, payload.queueId],
+    });
+    return { id: handler.id, token: handler.publicAccessToken };
+  } else {
+    // BullMQ
+    const { ingestQueue } = await import("~/bullmq/queues");
+    const job = await ingestQueue.add("ingest-episode", payload, {
+      jobId: payload.queueId,
+      attempts: 3,
+      backoff: { type: "exponential", delay: 2000 },
+    });
+    return { id: job.id };
+  }
+}
+
+/**
+ * Enqueue document ingestion job
+ */
+export async function enqueueIngestDocument(payload: {
+  body: z.infer<typeof IngestBodyRequest>;
+  userId: string;
+  workspaceId: string;
+  queueId: string;
+}): Promise<{ id?: string; token?: string }> {
+  const provider = env.QUEUE_PROVIDER as QueueProvider;
+
+  if (provider === "trigger") {
+    const { ingestDocumentTask } = await import(
+      "~/trigger/ingest/ingest-document"
+    );
+    const handler = await ingestDocumentTask.trigger(payload, {
+      queue: "document-ingestion-queue",
+      concurrencyKey: payload.userId,
+      tags: [payload.userId, payload.queueId],
+    });
+    return { id: handler.id, token: handler.publicAccessToken };
+  } else {
+    // BullMQ
+    const { documentIngestQueue } = await import("~/bullmq/queues");
+    const job = await documentIngestQueue.add("ingest-document", payload, {
+      jobId: payload.queueId,
+      attempts: 3,
+      backoff: { type: "exponential", delay: 2000 },
+    });
+
+    return { id: job.id };
+  }
+}
+
+/**
+ * Enqueue conversation title creation job
+ */
+export async function enqueueCreateConversationTitle(
+  payload: CreateConversationTitlePayload,
+): Promise<{ id?: string }> {
+  const provider = env.QUEUE_PROVIDER as QueueProvider;
+
+  if (provider === "trigger") {
+    const { createConversationTitle } = await import(
+      "~/trigger/conversation/create-conversation-title"
+    );
+    const handler = await createConversationTitle.trigger(payload);
+    return { id: handler.id };
+  } else {
+    // BullMQ
+    const { conversationTitleQueue } = await import("~/bullmq/queues");
+    const job = await conversationTitleQueue.add(
+      "create-conversation-title",
+      payload,
+      {
+        attempts: 3,
+        backoff: { type: "exponential", delay: 2000 },
+      },
+    );
+    return { id: job.id };
+  }
+}
+
+/**
+ * Enqueue session compaction job
+ */
+export async function enqueueSessionCompaction(
+  payload: SessionCompactionPayload,
+): Promise<{ id?: string }> {
+  const provider = env.QUEUE_PROVIDER as QueueProvider;
+
+  if (provider === "trigger") {
+    const { sessionCompactionTask } = await import(
+      "~/trigger/session/session-compaction"
+    );
+    const handler = await sessionCompactionTask.trigger(payload);
+    return { id: handler.id };
+  } else {
+    // BullMQ
+    const { sessionCompactionQueue } = await import("~/bullmq/queues");
+    const job = await sessionCompactionQueue.add(
+      "session-compaction",
+      payload,
+      {
+        attempts: 3,
+        backoff: { type: "exponential", delay: 2000 },
+      },
+    );
+    return { id: job.id };
+  }
+}
+
+/**
+ * Enqueue space assignment job
+ */
+export async function enqueueSpaceAssignment(
+  payload: SpaceAssignmentPayload,
+): Promise<{ id?: string }> {
+  const provider = env.QUEUE_PROVIDER as QueueProvider;
+
+  if (provider === "trigger") {
+    const { triggerSpaceAssignment } = await import(
+      "~/trigger/spaces/space-assignment"
+    );
+    const handler = await triggerSpaceAssignment(payload);
+    return { id: handler.id };
+  } else {
+    // BullMQ
+    const { spaceAssignmentQueue } = await import("~/bullmq/queues");
+    const job = await spaceAssignmentQueue.add("space-assignment", payload, {
+      jobId: `space-assignment-${payload.userId}-${payload.mode}-${Date.now()}`,
+      attempts: 3,
+      backoff: { type: "exponential", delay: 2000 },
+    });
+    return { id: job.id };
+  }
+}
+
+/**
+ * Enqueue space summary job
+ */
+export async function enqueueSpaceSummary(
+  payload: SpaceSummaryPayload,
+): Promise<{ id?: string }> {
+  const provider = env.QUEUE_PROVIDER as QueueProvider;
+
+  if (provider === "trigger") {
+    const { triggerSpaceSummary } = await import(
+      "~/trigger/spaces/space-summary"
+    );
+    const handler = await triggerSpaceSummary(payload);
+    return { id: handler.id };
+  } else {
+    // BullMQ
+    const { spaceSummaryQueue } = await import("~/bullmq/queues");
+    const job = await spaceSummaryQueue.add("space-summary", payload, {
+      jobId: `space-summary-${payload.spaceId}-${Date.now()}`,
+      attempts: 3,
+      backoff: { type: "exponential", delay: 2000 },
+    });
+    return { id: job.id };
+  }
+}
+
+/**
+ * Enqueue BERT topic analysis job
+ */
+export async function enqueueBertTopicAnalysis(payload: {
+  userId: string;
+  workspaceId: string;
+  minTopicSize?: number;
+  nrTopics?: number;
+}): Promise<{ id?: string }> {
+  const provider = env.QUEUE_PROVIDER as QueueProvider;
+
+  if (provider === "trigger") {
+    const { bertTopicAnalysisTask } = await import(
+      "~/trigger/bert/topic-analysis"
+    );
+    const handler = await bertTopicAnalysisTask.trigger(payload, {
+      queue: "bert-topic-analysis",
+      concurrencyKey: payload.userId,
+      tags: [payload.userId, "bert-analysis"],
+    });
+    return { id: handler.id };
+  } else {
+    // BullMQ
+    const { bertTopicQueue } = await import("~/bullmq/queues");
+    const job = await bertTopicQueue.add("topic-analysis", payload, {
+      jobId: `bert-${payload.userId}-${Date.now()}`,
+      attempts: 2, // Only 2 attempts for expensive operations
+      backoff: { type: "exponential", delay: 5000 },
+    });
+    return { id: job.id };
+  }
+}
+
+export const isTriggerDeployment = () => {
+  return env.QUEUE_PROVIDER === "trigger";
+};

apps/webapp/app/models/user.server.ts

@@ -3,6 +3,7 @@ import type { GoogleProfile } from "@coji/remix-auth-google";
 import { prisma } from "~/db.server";
 import { env } from "~/env.server";
 import { runQuery } from "~/lib/neo4j.server";
+import { trackFeatureUsage } from "~/services/telemetry.server";
 export type { User } from "@core/database";
 
 type FindOrCreateMagicLink = {
@@ -72,9 +73,16 @@ export async function findOrCreateMagicLinkUser(
     },
   });
 
+  const isNewUser = !existingUser;
+
+  // Track new user registration
+  if (isNewUser) {
+    trackFeatureUsage("user_registered", user.id).catch(console.error);
+  }
+
   return {
     user,
-    isNewUser: !existingUser,
+    isNewUser,
   };
 }
 
@@ -160,9 +168,16 @@ export async function findOrCreateGoogleUser({
     },
   });
 
+  const isNewUser = !existingUser;
+
+  // Track new user registration
+  if (isNewUser) {
+    trackFeatureUsage("user_registered", user.id).catch(console.error);
+  }
+
   return {
     user,
-    isNewUser: !existingUser,
+    isNewUser,
   };
 }
 

apps/webapp/app/models/workspace.server.ts

@@ -29,12 +29,6 @@ Exclude:
 • Anything not explicitly consented to share
 don't store anything the user did not explicitly consent to share.`;
 
-const githubDescription = `Everything related to my GitHub work - repos I'm working on, projects I contribute to, code I'm writing, PRs I'm reviewing. Basically my coding life on GitHub.`;
-
-const healthDescription = `My health and wellness stuff - how I'm feeling, what I'm learning about my body, experiments I'm trying, patterns I notice. Whatever matters to me about staying healthy.`;
-
-const fitnessDescription = `My workouts and training - what I'm doing at the gym, runs I'm going on, progress I'm making, goals I'm chasing. Anything related to physical exercise and getting stronger.`;
-
 export async function createWorkspace(
   input: CreateWorkspaceDto,
 ): Promise<Workspace> {
@@ -56,32 +50,7 @@ export async function createWorkspace(
   await ensureBillingInitialized(workspace.id);
 
   // Create default spaces
-  await Promise.all([
-    spaceService.createSpace({
-      name: "Profile",
-      description: profileRule,
-      userId: input.userId,
-      workspaceId: workspace.id,
-    }),
-    spaceService.createSpace({
-      name: "GitHub",
-      description: githubDescription,
-      userId: input.userId,
-      workspaceId: workspace.id,
-    }),
-    spaceService.createSpace({
-      name: "Health",
-      description: healthDescription,
-      userId: input.userId,
-      workspaceId: workspace.id,
-    }),
-    spaceService.createSpace({
-      name: "Fitness",
-      description: fitnessDescription,
-      userId: input.userId,
-      workspaceId: workspace.id,
-    }),
-  ]);
+  await Promise.all([]);
 
   try {
     const response = await sendEmail({ email: "welcome", to: user.email });

apps/webapp/app/root.tsx

@@ -51,6 +51,7 @@ export const loader = async ({ request }: LoaderFunctionArgs) => {
   const { getTheme } = await themeSessionResolver(request);
 
   const posthogProjectKey = env.POSTHOG_PROJECT_KEY;
+  const telemetryEnabled = env.TELEMETRY_ENABLED;
   const user = await getUser(request);
   const usageSummary = await getUsageSummary(user?.Workspace?.id as string);
 
@@ -62,6 +63,7 @@ export const loader = async ({ request }: LoaderFunctionArgs) => {
       toastMessage,
       theme: getTheme(),
       posthogProjectKey,
+      telemetryEnabled,
       appEnv: env.APP_ENV,
       appOrigin: env.APP_ORIGIN,
     },
@@ -113,8 +115,10 @@ export function ErrorBoundary() {
 }
 
 function App() {
-  const { posthogProjectKey } = useTypedLoaderData<typeof loader>();
-  usePostHog(posthogProjectKey);
+  const { posthogProjectKey, telemetryEnabled } =
+    useTypedLoaderData<typeof loader>();
+
+  usePostHog(posthogProjectKey, telemetryEnabled);
   const [theme] = useTheme();
 
   return (

apps/webapp/app/routes/api.v1.conversation.$conversationId.run.tsx

@@ -1,44 +0,0 @@
-import { json } from "@remix-run/node";
-import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server";
-
-import { getWorkspaceByUser } from "~/models/workspace.server";
-import {
-  createConversation,
-  CreateConversationSchema,
-  getCurrentConversationRun,
-  readConversation,
-  stopConversation,
-} from "~/services/conversation.server";
-import { z } from "zod";
-
-export const ConversationIdSchema = z.object({
-  conversationId: z.string(),
-});
-
-const { action, loader } = createActionApiRoute(
-  {
-    params: ConversationIdSchema,
-    allowJWT: true,
-    authorization: {
-      action: "oauth",
-    },
-    corsStrategy: "all",
-  },
-  async ({ authentication, params }) => {
-    const workspace = await getWorkspaceByUser(authentication.userId);
-
-    if (!workspace) {
-      throw new Error("No workspace found");
-    }
-
-    // Call the service to get the redirect URL
-    const run = await getCurrentConversationRun(
-      params.conversationId,
-      workspace?.id,
-    );
-
-    return json(run);
-  },
-);
-
-export { action, loader };

apps/webapp/app/routes/api.v1.conversation.$conversationId.stop.tsx

@@ -1,41 +0,0 @@
-import { json } from "@remix-run/node";
-import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server";
-
-import { getWorkspaceByUser } from "~/models/workspace.server";
-import {
-  createConversation,
-  CreateConversationSchema,
-  readConversation,
-  stopConversation,
-} from "~/services/conversation.server";
-import { z } from "zod";
-
-export const ConversationIdSchema = z.object({
-  conversationId: z.string(),
-});
-
-const { action, loader } = createActionApiRoute(
-  {
-    params: ConversationIdSchema,
-    allowJWT: true,
-    authorization: {
-      action: "oauth",
-    },
-    corsStrategy: "all",
-    method: "POST",
-  },
-  async ({ authentication, params }) => {
-    const workspace = await getWorkspaceByUser(authentication.userId);
-
-    if (!workspace) {
-      throw new Error("No workspace found");
-    }
-
-    // Call the service to get the redirect URL
-    const stop = await stopConversation(params.conversationId, workspace?.id);
-
-    return json(stop);
-  },
-);
-
-export { action, loader };

apps/webapp/app/routes/api.v1.conversation.$conversationId.stream.tsx

@@ -0,0 +1,45 @@
+// import { json } from "@remix-run/node";
+// import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server";
+// import { UI_MESSAGE_STREAM_HEADERS } from "ai";
+
+// import { getConversationAndHistory } from "~/services/conversation.server";
+// import { z } from "zod";
+// import { createResumableStreamContext } from "resumable-stream";
+
+// export const ConversationIdSchema = z.object({
+//   conversationId: z.string(),
+// });
+
+// const { action, loader } = createActionApiRoute(
+//   {
+//     params: ConversationIdSchema,
+//     allowJWT: true,
+//     authorization: {
+//       action: "oauth",
+//     },
+//     corsStrategy: "all",
+//   },
+//   async ({ authentication, params }) => {
+//     const conversation = await getConversationAndHistory(
+//       params.conversationId,
+//       authentication.userId,
+//     );
+
+//     const lastConversation = conversation?.ConversationHistory.pop();
+
+//     if (!lastConversation) {
+//       return json({}, { status: 204 });
+//     }
+
+//     const streamContext = createResumableStreamContext({
+//       waitUntil: null,
+//     });
+
+//     return new Response(
+//       await streamContext.resumeExistingStream(lastConversation.id),
+//       { headers: UI_MESSAGE_STREAM_HEADERS },
+//     );
+//   },
+// );
+
+// export { action, loader };

apps/webapp/app/routes/api.v1.conversation.$conversationId.tsx

@@ -1,50 +0,0 @@
-import { json } from "@remix-run/node";
-import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server";
-
-import { getWorkspaceByUser } from "~/models/workspace.server";
-import {
-  getConversation,
-  deleteConversation,
-} from "~/services/conversation.server";
-import { z } from "zod";
-
-export const ConversationIdSchema = z.object({
-  conversationId: z.string(),
-});
-
-const { action, loader } = createActionApiRoute(
-  {
-    params: ConversationIdSchema,
-    allowJWT: true,
-    authorization: {
-      action: "oauth",
-    },
-    corsStrategy: "all",
-  },
-  async ({ params, authentication, request }) => {
-    const workspace = await getWorkspaceByUser(authentication.userId);
-
-    if (!workspace) {
-      throw new Error("No workspace found");
-    }
-
-    const method = request.method;
-
-    if (method === "GET") {
-      // Get a conversation by ID
-      const conversation = await getConversation(params.conversationId);
-      return json(conversation);
-    }
-
-    if (method === "DELETE") {
-      // Soft delete a conversation
-      const deleted = await deleteConversation(params.conversationId);
-      return json(deleted);
-    }
-
-    // Method not allowed
-    return new Response("Method Not Allowed", { status: 405 });
-  },
-);
-
-export { action, loader };

apps/webapp/app/routes/api.v1.conversation._index.tsx

@@ -1,37 +1,159 @@
-import { json } from "@remix-run/node";
-import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server";
-
-import { getWorkspaceByUser } from "~/models/workspace.server";
 import {
-  createConversation,
-  CreateConversationSchema,
+  convertToModelMessages,
+  streamText,
+  validateUIMessages,
+  type LanguageModel,
+  experimental_createMCPClient as createMCPClient,
+  generateId,
+  stepCountIs,
+} from "ai";
+import { z } from "zod";
+import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
+
+import { createHybridActionApiRoute } from "~/services/routeBuilders/apiBuilder.server";
+import {
+  createConversationHistory,
+  getConversationAndHistory,
 } from "~/services/conversation.server";
 
-const { action, loader } = createActionApiRoute(
+import { getModel } from "~/lib/model.server";
+import { UserTypeEnum } from "@core/types";
+import { nanoid } from "nanoid";
+import {
+  deletePersonalAccessToken,
+  getOrCreatePersonalAccessToken,
+} from "~/services/personalAccessToken.server";
+import {
+  hasAnswer,
+  hasQuestion,
+  REACT_SYSTEM_PROMPT,
+} from "~/lib/prompt.server";
+import { enqueueCreateConversationTitle } from "~/lib/queue-adapter.server";
+import { env } from "~/env.server";
+
+const ChatRequestSchema = z.object({
+  message: z.object({
+    id: z.string().optional(),
+    parts: z.array(z.any()),
+    role: z.string(),
+  }),
+  id: z.string(),
+});
+
+const { loader, action } = createHybridActionApiRoute(
   {
-    body: CreateConversationSchema,
+    body: ChatRequestSchema,
     allowJWT: true,
     authorization: {
-      action: "oauth",
+      action: "conversation",
     },
     corsStrategy: "all",
   },
   async ({ body, authentication }) => {
-    const workspace = await getWorkspaceByUser(authentication.userId);
+    const randomKeyName = `chat_${nanoid(10)}`;
+    const pat = await getOrCreatePersonalAccessToken({
+      name: randomKeyName,
+      userId: authentication.userId,
+    });
 
-    if (!workspace) {
-      throw new Error("No workspace found");
-    }
+    const message = body.message.parts[0].text;
+    const id = body.message.id;
+    const apiEndpoint = `${env.APP_ORIGIN}/api/v1/mcp?source=core`;
+    const url = new URL(apiEndpoint);
 
-    // Call the service to get the redirect URL
-    const conversation = await createConversation(
-      workspace?.id,
+    const mcpClient = await createMCPClient({
+      transport: new StreamableHTTPClientTransport(url, {
+        requestInit: {
+          headers: pat.token
+            ? {
+                Authorization: `Bearer ${pat.token}`,
+              }
+            : {},
+        },
+      }),
+    });
+
+    const conversation = await getConversationAndHistory(
+      body.id,
       authentication.userId,
-      body,
     );
 
-    return json(conversation);
+    const conversationHistory = conversation?.ConversationHistory ?? [];
+
+    if (conversationHistory.length === 0) {
+      // Trigger conversation title task
+      await enqueueCreateConversationTitle({
+        conversationId: body.id,
+        message,
+      });
+    }
+
+    if (conversationHistory.length > 1) {
+      await createConversationHistory(message, body.id, UserTypeEnum.User);
+    }
+
+    const messages = conversationHistory.map((history: any) => {
+      return {
+        parts: [{ text: history.message, type: "text" }],
+        role: "user",
+        id: history.id,
+      };
+    });
+
+    const tools = { ...(await mcpClient.tools()) };
+
+    const finalMessages = [
+      ...messages,
+      {
+        parts: [{ text: message, type: "text" }],
+        role: "user",
+        id: id ?? generateId(),
+      },
+    ];
+
+    const validatedMessages = await validateUIMessages({
+      messages: finalMessages,
+    });
+
+    const result = streamText({
+      model: getModel() as LanguageModel,
+      messages: [
+        {
+          role: "system",
+          content: REACT_SYSTEM_PROMPT,
+        },
+        ...convertToModelMessages(validatedMessages),
+      ],
+      tools,
+      stopWhen: [stepCountIs(10), hasAnswer, hasQuestion],
+    });
+
+    result.consumeStream(); // no await
+    await deletePersonalAccessToken(pat?.id);
+
+    return result.toUIMessageStreamResponse({
+      originalMessages: validatedMessages,
+      onFinish: async ({ messages }) => {
+        const lastMessage = messages.pop();
+        let message = "";
+        lastMessage?.parts.forEach((part) => {
+          if (part.type === "text") {
+            message += part.text;
+          }
+        });
+
+        await createConversationHistory(message, body.id, UserTypeEnum.Agent);
+      },
+      // async consumeSseStream({ stream }) {
+      //   // Create a resumable stream from the SSE stream
+      //   const streamContext = createResumableStreamContext({ waitUntil: null });
+      //   await streamContext.createNewResumableStream(
+      //     conversation.conversationHistoryId,
+      //     () => stream,
+      //   );
+      // },
+    });
   },
 );
 
-export { action, loader };
+export { loader, action };

apps/webapp/app/routes/api.v1.deep-search.tsx

@@ -1,8 +1,27 @@
 import { z } from "zod";
 import { json } from "@remix-run/node";
 import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server";
-import { deepSearch } from "~/trigger/deep-search";
-import { runs } from "@trigger.dev/sdk";
+import { trackFeatureUsage } from "~/services/telemetry.server";
+import { nanoid } from "nanoid";
+import {
+  deletePersonalAccessToken,
+  getOrCreatePersonalAccessToken,
+} from "~/services/personalAccessToken.server";
+
+import {
+  convertToModelMessages,
+  generateId,
+  generateText,
+  type LanguageModel,
+  stepCountIs,
+  streamText,
+  tool,
+  validateUIMessages,
+} from "ai";
+import axios from "axios";
+import { logger } from "~/services/logger.service";
+import { getReActPrompt, hasAnswer } from "~/lib/prompt.server";
+import { getModel } from "~/lib/model.server";
 
 const DeepSearchBodySchema = z.object({
   content: z.string().min(1, "Content is required"),
@@ -17,6 +36,41 @@ const DeepSearchBodySchema = z.object({
     .optional(),
 });
 
+function createSearchMemoryTool(token: string) {
+  return tool({
+    description:
+      "Search the user's memory for relevant facts and episodes. Use this tool multiple times with different queries to gather comprehensive context.",
+    inputSchema: z.object({
+      query: z
+        .string()
+        .describe(
+          "Search query to find relevant information. Be specific: entity names, topics, concepts.",
+        ),
+    }),
+    execute: async ({ query }: { query: string }) => {
+      try {
+        const response = await axios.post(
+          `${process.env.API_BASE_URL || "https://core.heysol.ai"}/api/v1/search`,
+          { query, structured: false },
+          {
+            headers: {
+              Authorization: `Bearer ${token}`,
+            },
+          },
+        );
+        return response.data;
+      } catch (error) {
+        logger.error(`SearchMemory tool error: ${error}`);
+        return {
+          facts: [],
+          episodes: [],
+          summary: "No results found",
+        };
+      }
+    },
+  } as any);
+}
+
 const { action, loader } = createActionApiRoute(
   {
     body: DeepSearchBodySchema,
@@ -28,35 +82,94 @@ const { action, loader } = createActionApiRoute(
     corsStrategy: "all",
   },
   async ({ body, authentication }) => {
-    let trigger;
-    if (!body.stream) {
-      trigger = await deepSearch.trigger({
-        content: body.content,
-        userId: authentication.userId,
-        stream: body.stream,
-        intentOverride: body.intentOverride,
-        metadata: body.metadata,
+    // Track deep search
+    trackFeatureUsage("deep_search_performed", authentication.userId).catch(
+      console.error,
+    );
+
+    const randomKeyName = `deepSearch_${nanoid(10)}`;
+
+    const pat = await getOrCreatePersonalAccessToken({
+      name: randomKeyName,
+      userId: authentication.userId as string,
+    });
+
+    if (!pat?.token) {
+      return json({
+        success: false,
+        error: "Failed to create personal access token",
+      });
+    }
+
+    try {
+      // Create search tool that agent will use
+      const searchTool = createSearchMemoryTool(pat.token);
+
+      const tools = {
+        searchMemory: searchTool,
+      };
+
+      // Build initial messages with ReAct prompt
+      const initialMessages = [
+        {
+          role: "user",
+          parts: [
+            {
+              type: "text",
+              text: `CONTENT TO ANALYZE:\n${body.content}\n\nPlease search my memory for relevant context and synthesize what you find.`,
+            },
+          ],
+          id: generateId(),
+        },
+      ];
+
+      const validatedMessages = await validateUIMessages({
+        messages: initialMessages,
+        tools,
       });
 
-      return json(trigger);
-    } else {
-      const runHandler = await deepSearch.trigger({
-        content: body.content,
-        userId: authentication.userId,
-        stream: body.stream,
-        intentOverride: body.intentOverride,
-        metadata: body.metadata,
-      });
+      if (body.stream) {
+        const result = streamText({
+          model: getModel() as LanguageModel,
+          messages: [
+            {
+              role: "system",
+              content: getReActPrompt(body.metadata, body.intentOverride),
+            },
+            ...convertToModelMessages(validatedMessages),
+          ],
+          tools,
+          stopWhen: [hasAnswer, stepCountIs(10)],
+        });
 
-      for await (const run of runs.subscribeToRun(runHandler.id)) {
-        if (run.status === "COMPLETED") {
-          return json(run.output);
-        } else if (run.status === "FAILED") {
-          return json(run.error);
-        }
+        return result.toUIMessageStreamResponse({
+          originalMessages: validatedMessages,
+        });
+      } else {
+        const { text } = await generateText({
+          model: getModel() as LanguageModel,
+          messages: [
+            {
+              role: "system",
+              content: getReActPrompt(body.metadata, body.intentOverride),
+            },
+            ...convertToModelMessages(validatedMessages),
+          ],
+          tools,
+          stopWhen: [hasAnswer, stepCountIs(10)],
+        });
+
+        await deletePersonalAccessToken(pat?.id);
+        return json({ text });
       }
+    } catch (error: any) {
+      await deletePersonalAccessToken(pat?.id);
+      logger.error(`Deep search error: ${error}`);
 
-      return json({ error: "Run failed" });
+      return json({
+        success: false,
+        error: error.message,
+      });
     }
   },
 );

apps/webapp/app/routes/api.v1.ingestion_queue.delete.tsx

@@ -6,7 +6,7 @@ import {
   deleteIngestionQueue,
   getIngestionQueue,
 } from "~/services/ingestionLogs.server";
-import { runs, tasks } from "@trigger.dev/sdk";
+import { findRunningJobs, cancelJob } from "~/services/jobManager.server";
 
 export const DeleteEpisodeBodyRequest = z.object({
   id: z.string(),
@@ -37,19 +37,15 @@ const { action, loader } = createHybridActionApiRoute(
       }
 
       const output = ingestionQueue.output as any;
-      const runningTasks = await runs.list({
-        tag: [authentication.userId, ingestionQueue.id],
+      const runningTasks = await findRunningJobs({
+        tags: [authentication.userId, ingestionQueue.id],
         taskIdentifier: "ingest-episode",
       });
 
-      const latestTask = runningTasks.data.find(
-        (task) =>
-          task.tags.includes(authentication.userId) &&
-          task.tags.includes(ingestionQueue.id),
-      );
+      const latestTask = runningTasks[0];
 
-      if (latestTask && !latestTask?.isCompleted) {
-        runs.cancel(latestTask?.id as string);
+      if (latestTask && !latestTask.isCompleted) {
+        await cancelJob(latestTask.id);
       }
 
       let result;

apps/webapp/app/routes/api.v1.integration_account.tsx

@@ -8,6 +8,7 @@ import { logger } from "~/services/logger.service";
 import { getWorkspaceByUser } from "~/models/workspace.server";
 import { tasks } from "@trigger.dev/sdk";
 import { type scheduler } from "~/trigger/integrations/scheduler";
+import { isTriggerDeployment } from "~/lib/queue-adapter.server";
 
 // Schema for creating an integration account with API key
 const IntegrationAccountBodySchema = z.object({
@@ -63,6 +64,13 @@ const { action, loader } = createHybridActionApiRoute(
         );
       }
 
+      if (!isTriggerDeployment()) {
+        return json(
+          { error: "Integrations don't work in non trigger deployment" },
+          { status: 400 },
+        );
+      }
+
       await tasks.trigger<typeof scheduler>("scheduler", {
         integrationAccountId: setupResult?.account?.id,
       });

apps/webapp/app/routes/api.v1.logs.$logId.retry.tsx

@@ -0,0 +1,88 @@
+import { json } from "@remix-run/node";
+import { z } from "zod";
+import { IngestionStatus } from "@core/database";
+import { getIngestionQueue } from "~/services/ingestionLogs.server";
+import { createHybridActionApiRoute } from "~/services/routeBuilders/apiBuilder.server";
+import { addToQueue } from "~/lib/ingest.server";
+
+// Schema for log ID parameter
+const LogParamsSchema = z.object({
+  logId: z.string(),
+});
+
+const { action } = createHybridActionApiRoute(
+  {
+    params: LogParamsSchema,
+    allowJWT: true,
+    method: "POST",
+    authorization: {
+      action: "update",
+    },
+    corsStrategy: "all",
+  },
+  async ({ params, authentication }) => {
+    try {
+      const ingestionQueue = await getIngestionQueue(params.logId);
+
+      if (!ingestionQueue) {
+        return json(
+          {
+            error: "Ingestion log not found",
+            code: "not_found",
+          },
+          { status: 404 },
+        );
+      }
+
+      // Only allow retry for FAILED status
+      if (ingestionQueue.status !== IngestionStatus.FAILED) {
+        return json(
+          {
+            error: "Only failed ingestion logs can be retried",
+            code: "invalid_status",
+          },
+          { status: 400 },
+        );
+      }
+
+      // Get the original ingestion data
+      const originalData = ingestionQueue.data as any;
+
+      // Re-enqueue the job with the existing queue ID (will upsert)
+      await addToQueue(
+        originalData,
+        authentication.userId,
+        ingestionQueue.activityId || undefined,
+        ingestionQueue.id, // Pass the existing queue ID for upsert
+      );
+
+      return json({
+        success: true,
+        message: "Ingestion retry initiated successfully",
+      });
+    } catch (error) {
+      console.error("Error retrying ingestion:", error);
+
+      // Handle specific error cases
+      if (error instanceof Error && error.message === "no credits") {
+        return json(
+          {
+            error: "Insufficient credits to retry ingestion",
+            code: "no_credits",
+          },
+          { status: 402 },
+        );
+      }
+
+      return json(
+        {
+          error: "Failed to retry ingestion",
+          code: "internal_error",
+        },
+        { status: 500 },
+      );
+    }
+  },
+);
+
+export { action };

apps/webapp/app/routes/api.v1.logs.$logId.tsx

@@ -1,5 +1,4 @@
 import { json } from "@remix-run/node";
-import { runs } from "@trigger.dev/sdk";
 import { z } from "zod";
 import { deleteEpisodeWithRelatedNodes } from "~/services/graphModels/episode";
 import {
@@ -11,6 +10,7 @@ import {
   createHybridActionApiRoute,
   createHybridLoaderApiRoute,
 } from "~/services/routeBuilders/apiBuilder.server";
+import { findRunningJobs, cancelJob } from "~/services/jobManager.server";
 
 // Schema for space ID parameter
 const LogParamsSchema = z.object({
@@ -59,19 +59,15 @@ const { action } = createHybridActionApiRoute(
       }
 
       const output = ingestionQueue.output as any;
-      const runningTasks = await runs.list({
-        tag: [authentication.userId, ingestionQueue.id],
+      const runningTasks = await findRunningJobs({
+        tags: [authentication.userId, ingestionQueue.id],
         taskIdentifier: "ingest-episode",
       });
 
-      const latestTask = runningTasks.data.find(
-        (task) =>
-          task.tags.includes(authentication.userId) &&
-          task.tags.includes(ingestionQueue.id),
-      );
+      const latestTask = runningTasks[0];
 
-      if (latestTask && !latestTask?.isCompleted) {
-        runs.cancel(latestTask?.id);
+      if (latestTask && !latestTask.isCompleted) {
+        await cancelJob(latestTask.id);
       }
 
       let result;

apps/webapp/app/routes/api.v1.logs.tsx

@@ -1,6 +1,7 @@
-import { type LoaderFunctionArgs, json } from "@remix-run/node";
+import { json } from "@remix-run/node";
 import { z } from "zod";
 import { prisma } from "~/db.server";
+
 import { createHybridLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server";
 
 // Schema for logs search parameters

apps/webapp/app/routes/api.v1.search.tsx

@@ -5,6 +5,7 @@ import {
 } from "~/services/routeBuilders/apiBuilder.server";
 import { SearchService } from "~/services/search.server";
 import { json } from "@remix-run/node";
+import { trackFeatureUsage } from "~/services/telemetry.server";
 
 export const SearchBodyRequest = z.object({
   query: z.string(),
@@ -51,6 +52,10 @@ const { action, loader } = createHybridActionApiRoute(
         structured: body.structured,
       },
     );
+
+    // Track search
+    trackFeatureUsage("search_performed", authentication.userId).catch(console.error);
+
     return json(results);
   },
 );

apps/webapp/app/routes/api.v1.spaces.$spaceId.reset.ts

@@ -3,7 +3,7 @@ import { createHybridActionApiRoute } from "~/services/routeBuilders/apiBuilder.
 import { SpaceService } from "~/services/space.server";
 import { json } from "@remix-run/node";
 import { logger } from "~/services/logger.service";
-import { triggerSpaceAssignment } from "~/trigger/spaces/space-assignment";
+import { enqueueSpaceAssignment } from "~/lib/queue-adapter.server";
 
 // Schema for space ID parameter
 const SpaceParamsSchema = z.object({
@@ -31,7 +31,7 @@ const { loader, action } = createHybridActionApiRoute(
 
     // Trigger automatic episode assignment for the reset space
     try {
-      await triggerSpaceAssignment({
+      await enqueueSpaceAssignment({
         userId: userId,
         workspaceId: space.workspaceId,
         mode: "new_space",

apps/webapp/app/routes/api.v1.spaces.assignments.ts

@@ -1,8 +1,8 @@
 import { z } from "zod";
 import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server";
 import { json } from "@remix-run/node";
-import { triggerSpaceAssignment } from "~/trigger/spaces/space-assignment";
 import { prisma } from "~/db.server";
+import { enqueueSpaceAssignment } from "~/lib/queue-adapter.server";
 
 // Schema for manual assignment trigger
 const ManualAssignmentSchema = z.object({
@@ -38,7 +38,7 @@ const { action } = createActionApiRoute(
       let taskRun;
 
       // Direct LLM assignment trigger
-      taskRun = await triggerSpaceAssignment({
+      taskRun = await enqueueSpaceAssignment({
         userId,
         workspaceId: user?.Workspace?.id as string,
         mode: body.mode,
@@ -49,7 +49,7 @@ const { action } = createActionApiRoute(
       return json({
         success: true,
         message: `${body.mode} assignment task triggered successfully`,
-        taskId: taskRun.id,
+
         payload: {
           userId,
           mode: body.mode,

apps/webapp/app/routes/api.v1.spaces.ts

@@ -7,6 +7,10 @@ import { SpaceService } from "~/services/space.server";
 import { json } from "@remix-run/node";
 import { prisma } from "~/db.server";
 import { apiCors } from "~/utils/apiCors";
+import {
+  enqueueSpaceAssignment,
+  isTriggerDeployment,
+} from "~/lib/queue-adapter.server";
 
 const spaceService = new SpaceService();
 
@@ -40,6 +44,13 @@ const { action } = createHybridActionApiRoute(
       },
     });
 
+    if (!isTriggerDeployment()) {
+      return json(
+        { error: "Spaces don't work in non trigger deployment" },
+        { status: 400 },
+      );
+    }
+
     if (!user?.Workspace?.id) {
       throw new Error(
         "Workspace ID is required to create an ingestion queue entry.",
@@ -66,6 +77,14 @@ const { action } = createHybridActionApiRoute(
         workspaceId: user.Workspace.id,
       });
 
+      await enqueueSpaceAssignment({
+        userId: user.id,
+        workspaceId: user.Workspace.id,
+        mode: "new_space",
+        newSpaceId: space.id,
+        batchSize: 25, // Analyze recent statements for the new space
+      });
+
       return json({ space, success: true });
     }
 

apps/webapp/app/routes/home.conversation.$conversationId.tsx

@@ -1,42 +1,26 @@
-import {
-  type LoaderFunctionArgs,
-  type ActionFunctionArgs,
-} from "@remix-run/server-runtime";
-import { sort } from "fast-sort";
+import { type LoaderFunctionArgs } from "@remix-run/server-runtime";
 
-import { useParams, useRevalidator, useNavigate } from "@remix-run/react";
-import { parse } from "@conform-to/zod";
-import {
-  requireUserId,
-  requireUser,
-  requireWorkpace,
-} from "~/services/session.server";
-import {
-  getConversationAndHistory,
-  getCurrentConversationRun,
-  stopConversation,
-  createConversation,
-  CreateConversationSchema,
-} from "~/services/conversation.server";
-import { type ConversationHistory } from "@core/database";
+import { useParams, useNavigate } from "@remix-run/react";
+import { requireUser, requireWorkpace } from "~/services/session.server";
+import { getConversationAndHistory } from "~/services/conversation.server";
 import {
   ConversationItem,
   ConversationTextarea,
-  StreamingConversation,
 } from "~/components/conversation";
 import { useTypedLoaderData } from "remix-typedjson";
-import React from "react";
 import { ScrollAreaWithAutoScroll } from "~/components/use-auto-scroll";
 import { PageHeader } from "~/components/common/page-header";
 import { Plus } from "lucide-react";
 
-import { json } from "@remix-run/node";
-import { env } from "~/env.server";
+import { type UIMessage, useChat } from "@ai-sdk/react";
+import { DefaultChatTransport } from "ai";
+import { UserTypeEnum } from "@core/types";
+import React from "react";
 
 // Example loader accessing params
 export async function loader({ params, request }: LoaderFunctionArgs) {
   const user = await requireUser(request);
-  const workspace = await requireWorkpace(request);
+
   const conversation = await getConversationAndHistory(
     params.conversationId as string,
     user.id,
@@ -46,100 +30,38 @@ export async function loader({ params, request }: LoaderFunctionArgs) {
     throw new Error("No conversation found");
   }
 
-  const run = await getCurrentConversationRun(conversation.id, workspace.id);
-
-  return { conversation, run, apiURL: env.TRIGGER_API_URL };
-}
-
-// Example action accessing params
-export async function action({ params, request }: ActionFunctionArgs) {
-  if (request.method.toUpperCase() !== "POST") {
-    return new Response("Method Not Allowed", { status: 405 });
-  }
-
-  const userId = await requireUserId(request);
-  const workspace = await requireWorkpace(request);
-  const formData = await request.formData();
-  const { conversationId } = params;
-
-  if (!conversationId) {
-    throw new Error("No conversation");
-  }
-
-  // Check if this is a stop request (isLoading = true means stop button was clicked)
-  const message = formData.get("message");
-
-  // If no message, it's a stop request
-  if (!message) {
-    const result = await stopConversation(conversationId, workspace.id);
-    return json(result);
-  }
-
-  // Otherwise, create a new conversation message
-  const submission = parse(formData, { schema: CreateConversationSchema });
-
-  if (!submission.value || submission.intent !== "submit") {
-    return json(submission);
-  }
-
-  const conversation = await createConversation(workspace?.id, userId, {
-    message: submission.value.message,
-    title: submission.value.title,
-    conversationId: submission.value.conversationId,
-  });
-
-  return json({ conversation });
+  return { conversation };
 }
 
 // Accessing params in the component
 export default function SingleConversation() {
-  const { conversation, run, apiURL } = useTypedLoaderData<typeof loader>();
-  const conversationHistory = conversation.ConversationHistory;
-
-  const [conversationResponse, setConversationResponse] = React.useState<
-    { conversationHistoryId: string; id: string; token: string } | undefined
-  >(run);
-
-  const { conversationId } = useParams();
-  const revalidator = useRevalidator();
+  const { conversation } = useTypedLoaderData<typeof loader>();
   const navigate = useNavigate();
+  const { conversationId } = useParams();
+
+  const { sendMessage, messages, status, stop, regenerate } = useChat({
+    id: conversationId, // use the provided chat ID
+    messages: conversation.ConversationHistory.map(
+      (history) =>
+        ({
+          role: history.userType === UserTypeEnum.Agent ? "assistant" : "user",
+          parts: [{ text: history.message, type: "text" }],
+        }) as UIMessage,
+    ), // load initial messages
+    transport: new DefaultChatTransport({
+      api: "/api/v1/conversation",
+      prepareSendMessagesRequest({ messages, id }) {
+        return { body: { message: messages[messages.length - 1], id } };
+      },
+    }),
+  });
+  console.log("new", messages);
 
   React.useEffect(() => {
-    if (run) {
-      setConversationResponse(run);
+    if (messages.length === 1) {
+      regenerate();
     }
-  }, [run]);
-
-  const conversations = React.useMemo(() => {
-    const lastConversationHistoryId =
-      conversationResponse?.conversationHistoryId;
-
-    // First sort the conversation history by creation time
-    const sortedConversationHistory = sort(conversationHistory).asc(
-      (ch) => ch.createdAt,
-    );
-
-    const lastIndex = sortedConversationHistory.findIndex(
-      (item) => item.id === lastConversationHistoryId,
-    );
-
-    // Filter out any conversation history items that come after the lastConversationHistoryId
-    return lastConversationHistoryId
-      ? sortedConversationHistory.filter((_ch, currentIndex: number) => {
-          return currentIndex <= lastIndex;
-        })
-      : sortedConversationHistory;
-  }, [conversationResponse, conversationHistory]);
-
-  const getConversations = () => {
-    return (
-      <>
-        {conversations.map((ch: ConversationHistory) => {
-          return <ConversationItem key={ch.id} conversationHistory={ch} />;
-        })}
-      </>
-    );
-  };
+  }, []);
 
   if (typeof window === "undefined") {
     return null;
@@ -166,41 +88,23 @@ export default function SingleConversation() {
       <div className="relative flex h-[calc(100vh_-_56px)] w-full flex-col items-center justify-center overflow-auto">
         <div className="flex h-[calc(100vh_-_80px)] w-full flex-col justify-end overflow-hidden">
           <ScrollAreaWithAutoScroll>
-            {getConversations()}
-            {conversationResponse && (
-              <StreamingConversation
-                runId={conversationResponse.id}
-                token={conversationResponse.token}
-                afterStreaming={() => {
-                  setConversationResponse(undefined);
-                  revalidator.revalidate();
-                }}
-                apiURL={apiURL}
-              />
-            )}
+            {messages.map((message: UIMessage, index: number) => {
+              return <ConversationItem key={index} message={message} />;
+            })}
           </ScrollAreaWithAutoScroll>
 
           <div className="flex w-full flex-col items-center">
             <div className="w-full max-w-[80ch] px-1 pr-2">
-              {conversation?.status !== "need_approval" && (
-                <ConversationTextarea
-                  conversationId={conversationId as string}
-                  className="bg-background-3 w-full border-1 border-gray-300"
-                  isLoading={
-                    !!conversationResponse || conversation?.status === "running"
+              <ConversationTextarea
+                className="bg-background-3 w-full border-1 border-gray-300"
+                isLoading={status === "streaming" || status === "submitted"}
+                onConversationCreated={(message) => {
+                  if (message) {
+                    sendMessage({ text: message });
                   }
-                  onConversationCreated={(conversation) => {
-                    if (conversation) {
-                      setConversationResponse({
-                        conversationHistoryId:
-                          conversation.conversationHistoryId,
-                        id: conversation.id,
-                        token: conversation.token,
-                      });
-                    }
-                  }}
-                />
-              )}
+                }}
+                stop={() => stop()}
+              />
             </div>
           </div>
         </div>

apps/webapp/app/routes/home.conversation._index.tsx

@@ -43,8 +43,7 @@ export async function action({ request }: ActionFunctionArgs) {
 
   const conversation = await createConversation(workspace?.id, userId, {
     message: submission.value.message,
-    title: submission.value.title,
-    conversationId: submission.value.conversationId,
+    title: submission.value.title ?? "Untitled",
   });
 
   // If conversationId exists in submission, return the conversation data (don't redirect)

apps/webapp/app/routes/home.inbox.$logId.tsx

@@ -40,7 +40,7 @@ export default function InboxNotSelected() {
       <PageHeader
         title="Episode"
         showTrigger={false}
-        actionsNode={<LogOptions id={log.id} />}
+        actionsNode={<LogOptions id={log.id} status={log.status} />}
       />
 
       <LogDetails log={log as any} />

apps/webapp/app/services/bertTopicAnalysis.server.ts

@@ -0,0 +1,107 @@
+import { prisma } from "~/trigger/utils/prisma";
+import { logger } from "~/services/logger.service";
+import { runQuery } from "~/lib/neo4j.server";
+
+interface WorkspaceMetadata {
+  lastTopicAnalysisAt?: string;
+  [key: string]: any;
+}
+
+/**
+ * Check if we should trigger a BERT topic analysis for this workspace
+ * Criteria: 20+ new episodes since last analysis (or no previous analysis)
+ */
+export async function shouldTriggerTopicAnalysis(
+  userId: string,
+  workspaceId: string,
+): Promise<boolean> {
+  try {
+    // Get workspace metadata
+    const workspace = await prisma.workspace.findUnique({
+      where: { id: workspaceId },
+      select: { metadata: true },
+    });
+
+    if (!workspace) {
+      logger.warn(`Workspace not found: ${workspaceId}`);
+      return false;
+    }
+
+    const metadata = (workspace.metadata || {}) as WorkspaceMetadata;
+    const lastAnalysisAt = metadata.lastTopicAnalysisAt;
+
+    // Count episodes since last analysis
+    const query = lastAnalysisAt
+      ? `
+        MATCH (e:Episode {userId: $userId})
+        WHERE e.createdAt > datetime($lastAnalysisAt)
+        RETURN count(e) as newEpisodeCount
+      `
+      : `
+        MATCH (e:Episode {userId: $userId})
+        RETURN count(e) as totalEpisodeCount
+      `;
+
+    const result = await runQuery(query, {
+      userId,
+      lastAnalysisAt,
+    });
+
+    const episodeCount = lastAnalysisAt
+      ? result[0]?.get("newEpisodeCount")?.toNumber() || 0
+      : result[0]?.get("totalEpisodeCount")?.toNumber() || 0;
+
+    logger.info(
+      `[Topic Analysis Check] User: ${userId}, New episodes: ${episodeCount}, Last analysis: ${lastAnalysisAt || "never"}`,
+    );
+
+    // Trigger if 20+ new episodes
+    return episodeCount >= 20;
+  } catch (error) {
+    logger.error(
+      `[Topic Analysis Check] Error checking episode count:`,
+      error,
+    );
+    return false;
+  }
+}
+
+/**
+ * Update workspace metadata with last topic analysis timestamp
+ */
+export async function updateLastTopicAnalysisTime(
+  workspaceId: string,
+): Promise<void> {
+  try {
+    const workspace = await prisma.workspace.findUnique({
+      where: { id: workspaceId },
+      select: { metadata: true },
+    });
+
+    if (!workspace) {
+      logger.warn(`Workspace not found: ${workspaceId}`);
+      return;
+    }
+
+    const metadata = (workspace.metadata || {}) as WorkspaceMetadata;
+
+    await prisma.workspace.update({
+      where: { id: workspaceId },
+      data: {
+        metadata: {
+          ...metadata,
+          lastTopicAnalysisAt: new Date().toISOString(),
+        },
+      },
+    });
+
+    logger.info(
+      `[Topic Analysis] Updated last analysis timestamp for workspace: ${workspaceId}`,
+    );
+  } catch (error) {
+    logger.error(
+      `[Topic Analysis] Error updating last analysis timestamp:`,
+      error,
+    );
+  }
+}

apps/webapp/app/services/conversation.server.ts

@@ -1,11 +1,9 @@
 import { UserTypeEnum } from "@core/types";
 
-import { auth, runs, tasks } from "@trigger.dev/sdk/v3";
 import { prisma } from "~/db.server";
-import { createConversationTitle } from "~/trigger/conversation/create-conversation-title";
 
 import { z } from "zod";
-import { type ConversationHistory } from "@prisma/client";
+import { trackFeatureUsage } from "~/services/telemetry.server";
 
 export const CreateConversationSchema = z.object({
   message: z.string(),
@@ -44,20 +42,10 @@ export async function createConversation(
       },
     });
 
-    const context = await getConversationContext(conversationHistory.id);
-    const handler = await tasks.trigger(
-      "chat",
-      {
-        conversationHistoryId: conversationHistory.id,
-        conversationId: conversationHistory.conversation.id,
-        context,
-      },
-      { tags: [conversationHistory.id, workspaceId, conversationId] },
-    );
+    // Track conversation message
+    trackFeatureUsage("conversation_message_sent", userId).catch(console.error);
 
     return {
-      id: handler.id,
-      token: handler.publicAccessToken,
       conversationId: conversationHistory.conversation.id,
       conversationHistoryId: conversationHistory.id,
     };
@@ -84,40 +72,20 @@ export async function createConversation(
   });
 
   const conversationHistory = conversation.ConversationHistory[0];
-  const context = await getConversationContext(conversationHistory.id);
 
-  // Trigger conversation title task
-  await tasks.trigger<typeof createConversationTitle>(
-    createConversationTitle.id,
-    {
-      conversationId: conversation.id,
-      message: conversationData.message,
-    },
-    { tags: [conversation.id, workspaceId] },
-  );
-
-  const handler = await tasks.trigger(
-    "chat",
-    {
-      conversationHistoryId: conversationHistory.id,
-      conversationId: conversation.id,
-      context,
-    },
-    { tags: [conversationHistory.id, workspaceId, conversation.id] },
-  );
+  // Track new conversation creation
+  trackFeatureUsage("conversation_created", userId).catch(console.error);
 
   return {
-    id: handler.id,
-    token: handler.publicAccessToken,
     conversationId: conversation.id,
     conversationHistoryId: conversationHistory.id,
   };
 }
 
 // Get a conversation by ID
-export async function getConversation(conversationId: string) {
+export async function getConversation(conversationId: string, userId: string) {
   return prisma.conversation.findUnique({
-    where: { id: conversationId },
+    where: { id: conversationId, userId },
   });
 }
 
@@ -139,141 +107,6 @@ export async function readConversation(conversationId: string) {
   });
 }
 
-export async function getCurrentConversationRun(
-  conversationId: string,
-  workspaceId: string,
-) {
-  const conversationHistory = await prisma.conversationHistory.findFirst({
-    where: {
-      conversationId,
-      conversation: {
-        workspaceId,
-      },
-      userType: UserTypeEnum.User,
-    },
-    orderBy: {
-      updatedAt: "desc",
-    },
-  });
-
-  if (!conversationHistory) {
-    throw new Error("No run found");
-  }
-
-  const response = await runs.list({
-    tag: [conversationId, conversationHistory.id, workspaceId],
-    status: ["QUEUED", "EXECUTING"],
-    limit: 1,
-  });
-
-  if (!response) {
-    return undefined;
-  }
-
-  const run = response?.data?.[0];
-
-  if (!run) {
-    return undefined;
-  }
-
-  const publicToken = await auth.createPublicToken({
-    scopes: {
-      read: {
-        runs: [run.id],
-      },
-    },
-  });
-
-  return {
-    id: run.id,
-    token: publicToken,
-    conversationId,
-    conversationHistoryId: conversationHistory.id,
-  };
-}
-
-export async function stopConversation(
-  conversationId: string,
-  workspaceId: string,
-) {
-  const conversationHistory = await prisma.conversationHistory.findFirst({
-    where: {
-      conversationId,
-      conversation: {
-        workspaceId,
-      },
-    },
-    orderBy: {
-      updatedAt: "desc",
-    },
-  });
-
-  if (!conversationHistory) {
-    throw new Error("No run found");
-  }
-
-  const response = await runs.list({
-    tag: [conversationId, conversationHistory.id],
-    status: ["QUEUED", "EXECUTING"],
-    limit: 1,
-  });
-
-  const run = response.data[0];
-  if (!run) {
-    await prisma.conversation.update({
-      where: {
-        id: conversationId,
-      },
-      data: {
-        status: "failed",
-      },
-    });
-
-    return undefined;
-  }
-
-  return await runs.cancel(run.id);
-}
-
-export async function getConversationContext(
-  conversationHistoryId: string,
-): Promise<{
-  previousHistory: ConversationHistory[];
-}> {
-  const conversationHistory = await prisma.conversationHistory.findUnique({
-    where: { id: conversationHistoryId },
-    include: { conversation: true },
-  });
-
-  if (!conversationHistory) {
-    return {
-      previousHistory: [],
-    };
-  }
-
-  // Get previous conversation history message and response
-  let previousHistory: ConversationHistory[] = [];
-
-  if (conversationHistory.conversationId) {
-    previousHistory = await prisma.conversationHistory.findMany({
-      where: {
-        conversationId: conversationHistory.conversationId,
-        id: {
-          not: conversationHistoryId,
-        },
-        deleted: null,
-      },
-      orderBy: {
-        createdAt: "asc",
-      },
-    });
-  }
-
-  return {
-    previousHistory,
-  };
-}
-
 export const getConversationAndHistory = async (
   conversationId: string,
   userId: string,
@@ -281,6 +114,7 @@ export const getConversationAndHistory = async (
   const conversation = await prisma.conversation.findFirst({
     where: {
       id: conversationId,
+      userId,
     },
     include: {
       ConversationHistory: true,
@@ -290,6 +124,23 @@ export const getConversationAndHistory = async (
   return conversation;
 };
 
+export const createConversationHistory = async (
+  userMessage: string,
+  conversationId: string,
+  userType: UserTypeEnum,
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  thoughts?: Record<string, any>,
+) => {
+  return await prisma.conversationHistory.create({
+    data: {
+      conversationId,
+      message: userMessage,
+      thoughts,
+      userType,
+    },
+  });
+};
+
 export const GetConversationsListSchema = z.object({
   page: z.string().optional().default("1"),
   limit: z.string().optional().default("20"),

apps/webapp/app/services/graphModels/space.ts

@@ -45,6 +45,43 @@ export async function createSpace(
   };
 }
 
+/**
+ * Get all active spaces for a user
+ */
+export async function getAllSpacesForUser(
+  userId: string,
+): Promise<SpaceNode[]> {
+  const query = `
+    MATCH (s:Space {userId: $userId})
+    WHERE s.isActive = true
+
+    // Count episodes assigned to each space
+    OPTIONAL MATCH (s)-[:HAS_EPISODE]->(e:Episode {userId: $userId})
+
+    WITH s, count(e) as episodeCount
+    RETURN s, episodeCount
+    ORDER BY s.createdAt DESC
+  `;
+
+  const result = await runQuery(query, { userId });
+
+  return result.map((record) => {
+    const spaceData = record.get("s").properties;
+    const episodeCount = record.get("episodeCount") || 0;
+
+    return {
+      uuid: spaceData.uuid,
+      name: spaceData.name,
+      description: spaceData.description,
+      userId: spaceData.userId,
+      createdAt: new Date(spaceData.createdAt),
+      updatedAt: new Date(spaceData.updatedAt),
+      isActive: spaceData.isActive,
+      contextCount: Number(episodeCount),
+    };
+  });
+}
+
 /**
  * Get a specific space by ID
  */

apps/webapp/app/services/jobManager.server.ts

@@ -0,0 +1,87 @@
+/**
+ * Job Manager Service
+ *
+ * Unified interface for managing background jobs across both
+ * Trigger.dev and BullMQ queue providers.
+ */
+
+import { env } from "~/env.server";
+
+type QueueProvider = "trigger" | "bullmq";
+
+interface JobInfo {
+  id: string;
+  isCompleted: boolean;
+  status?: string;
+}
+
+/**
+ * Find running jobs by tags/identifiers
+ */
+export async function findRunningJobs(params: {
+  tags: string[];
+  taskIdentifier?: string;
+}): Promise<JobInfo[]> {
+  const provider = env.QUEUE_PROVIDER as QueueProvider;
+
+  if (provider === "trigger") {
+    const { runs } = await import("@trigger.dev/sdk");
+    const runningTasks = await runs.list({
+      tag: params.tags,
+      taskIdentifier: params.taskIdentifier,
+    });
+
+    return runningTasks.data.map((task) => ({
+      id: task.id,
+      isCompleted: task.isCompleted,
+      status: task.status,
+    }));
+  } else {
+    // BullMQ
+    const { getJobsByTags } = await import("~/bullmq/utils/job-finder");
+    const jobs = await getJobsByTags(params.tags, params.taskIdentifier);
+
+    return jobs;
+  }
+}
+
+/**
+ * Cancel a running job
+ */
+export async function cancelJob(jobId: string): Promise<void> {
+  const provider = env.QUEUE_PROVIDER as QueueProvider;
+
+  if (provider === "trigger") {
+    const { runs } = await import("@trigger.dev/sdk");
+    await runs.cancel(jobId);
+  } else {
+    // BullMQ
+    const { cancelJobById } = await import("~/bullmq/utils/job-finder");
+    await cancelJobById(jobId);
+  }
+}
+
+/**
+ * Get job status
+ */
+export async function getJobStatus(jobId: string): Promise<JobInfo | null> {
+  const provider = env.QUEUE_PROVIDER as QueueProvider;
+
+  if (provider === "trigger") {
+    const { runs } = await import("@trigger.dev/sdk");
+    try {
+      const run = await runs.retrieve(jobId);
+      return {
+        id: run.id,
+        isCompleted: run.isCompleted,
+        status: run.status,
+      };
+    } catch {
+      return null;
+    }
+  } else {
+    // BullMQ
+    const { getJobById } = await import("~/bullmq/utils/job-finder");
+    return await getJobById(jobId);
+  }
+}

apps/webapp/app/services/knowledgeGraph.server.ts

@@ -10,7 +10,6 @@ import {
   type EpisodeType,
 } from "@core/types";
 import { logger } from "./logger.service";
-import { ClusteringService } from "./clustering.server";
 import crypto from "crypto";
 import { dedupeNodes, extractEntities } from "./prompts/nodes";
 import {
@@ -50,12 +49,6 @@ import { type PrismaClient } from "@prisma/client";
 const DEFAULT_EPISODE_WINDOW = 5;
 
 export class KnowledgeGraphService {
-  private clusteringService: ClusteringService;
-
-  constructor() {
-    this.clusteringService = new ClusteringService();
-  }
-
   async getEmbedding(text: string) {
     return getEmbedding(text);
   }
@@ -564,9 +557,9 @@ export class KnowledgeGraphService {
       (text, _model, usage) => {
         responseText = text;
         if (usage) {
-          tokenMetrics.high.input += usage.promptTokens;
-          tokenMetrics.high.output += usage.completionTokens;
-          tokenMetrics.high.total += usage.totalTokens;
+          tokenMetrics.high.input += usage.promptTokens as number;
+          tokenMetrics.high.output += usage.completionTokens as number;
+          tokenMetrics.high.total += usage.totalTokens as number;
         }
       },
       undefined,

apps/webapp/app/services/mcp.server.ts

@@ -58,6 +58,7 @@ async function createMcpServer(
     // Handle memory tools and integration meta-tools
     if (
       name.startsWith("memory_") ||
+      name === "get_session_id" ||
       name === "get_integrations" ||
       name === "get_integration_actions" ||
       name === "execute_integration_action"

apps/webapp/app/services/personalAccessToken.server.ts

@@ -320,6 +320,14 @@ export async function getOrCreatePersonalAccessToken({
   };
 }
 
+export async function deletePersonalAccessToken(tokenId: string) {
+  return await prisma.personalAccessToken.delete({
+    where: {
+      id: tokenId,
+    },
+  });
+}
+
 /** Created a new PersonalAccessToken, and return the token. We only ever return the unencrypted token once. */
 export async function createPersonalAccessToken({
   name,

apps/webapp/app/services/search/utils.ts

@@ -51,6 +51,7 @@ export async function performBM25Search(
           ${spaceCondition}
         OPTIONAL MATCH (episode:Episode)-[:HAS_PROVENANCE]->(s)
         WITH s, score, count(episode) as provenanceCount
+        WHERE score >= 0.5
         RETURN s, score, provenanceCount
         ORDER BY score DESC
       `;
@@ -71,6 +72,12 @@ export async function performBM25Search(
         typeof provenanceCountValue === "bigint"
           ? Number(provenanceCountValue)
           : (provenanceCountValue?.toNumber?.() ?? provenanceCountValue ?? 0);
+
+      const scoreValue = record.get("score");
+      (statement as any).bm25Score =
+        typeof scoreValue === "number"
+          ? scoreValue
+          : (scoreValue?.toNumber?.() ?? 0);
       return statement;
     });
   } catch (error) {
@@ -163,6 +170,14 @@ export async function performVectorSearch(
         typeof provenanceCountValue === "bigint"
           ? Number(provenanceCountValue)
           : (provenanceCountValue?.toNumber?.() ?? provenanceCountValue ?? 0);
+
+      // Preserve vector similarity score for empty result detection
+      const scoreValue = record.get("score");
+      (statement as any).vectorScore =
+        typeof scoreValue === "number"
+          ? scoreValue
+          : (scoreValue?.toNumber?.() ?? 0);
+
       return statement;
     });
   } catch (error) {
@@ -179,12 +194,10 @@ export async function performBfsSearch(
   query: string,
   embedding: Embedding,
   userId: string,
+  entities: EntityNode[],
   options: Required<SearchOptions>,
 ): Promise<StatementNode[]> {
   try {
-    // 1. Extract potential entities from query using chunked embeddings
-    const entities = await extractEntitiesFromQuery(query, userId);
-
     if (entities.length === 0) {
       return [];
     }
@@ -224,7 +237,7 @@ async function bfsTraversal(
   const RELEVANCE_THRESHOLD = 0.5;
   const EXPLORATION_THRESHOLD = 0.3;
 
-  const allStatements = new Map<string, number>(); // uuid -> relevance
+  const allStatements = new Map<string, { relevance: number; hopDistance: number }>(); // uuid -> {relevance, hopDistance}
   const visitedEntities = new Set<string>();
 
   // Track entities per level for iterative BFS
@@ -268,14 +281,14 @@ async function bfsTraversal(
       ...(startTime && { startTime: startTime.toISOString() }),
     });
 
-    // Store statement relevance scores
+    // Store statement relevance scores and hop distance
     const currentLevelStatementUuids: string[] = [];
     for (const record of records) {
       const uuid = record.get("uuid");
       const relevance = record.get("relevance");
 
       if (!allStatements.has(uuid)) {
-        allStatements.set(uuid, relevance);
+        allStatements.set(uuid, { relevance, hopDistance: depth + 1 }); // Store hop distance (1-indexed)
         currentLevelStatementUuids.push(uuid);
       }
     }
@@ -304,25 +317,45 @@ async function bfsTraversal(
   }
 
   // Filter by relevance threshold and fetch full statements
-  const relevantUuids = Array.from(allStatements.entries())
-    .filter(([_, relevance]) => relevance >= RELEVANCE_THRESHOLD)
-    .sort((a, b) => b[1] - a[1])
-    .map(([uuid]) => uuid);
+  const relevantResults = Array.from(allStatements.entries())
+    .filter(([_, data]) => data.relevance >= RELEVANCE_THRESHOLD)
+    .sort((a, b) => b[1].relevance - a[1].relevance);
 
-  if (relevantUuids.length === 0) {
+  if (relevantResults.length === 0) {
     return [];
   }
 
+  const relevantUuids = relevantResults.map(([uuid]) => uuid);
+
   const fetchCypher = `
     MATCH (s:Statement{userId: $userId})
     WHERE s.uuid IN $uuids
     RETURN s
   `;
   const fetchRecords = await runQuery(fetchCypher, { uuids: relevantUuids, userId });
-  const statements = fetchRecords.map(r => r.get("s").properties as StatementNode);
+  const statementMap = new Map(
+    fetchRecords.map(r => [r.get("s").properties.uuid, r.get("s").properties as StatementNode])
+  );
+
+  // Attach hop distance to statements
+  const statements = relevantResults.map(([uuid, data]) => {
+    const statement = statementMap.get(uuid)!;
+    // Add bfsHopDistance and bfsRelevance as metadata
+    (statement as any).bfsHopDistance = data.hopDistance;
+    (statement as any).bfsRelevance = data.relevance;
+    return statement;
+  });
+
+  const hopCounts = statements.reduce((acc, s) => {
+    const hop = (s as any).bfsHopDistance;
+    acc[hop] = (acc[hop] || 0) + 1;
+    return acc;
+  }, {} as Record<number, number>);
 
   logger.info(
-    `BFS: explored ${allStatements.size} statements across ${maxDepth} hops, returning ${statements.length} (≥${RELEVANCE_THRESHOLD})`
+    `BFS: explored ${allStatements.size} statements across ${maxDepth} hops, ` +
+    `returning ${statements.length} (≥${RELEVANCE_THRESHOLD}) - ` +
+    `1-hop: ${hopCounts[1] || 0}, 2-hop: ${hopCounts[2] || 0}, 3-hop: ${hopCounts[3] || 0}, 4-hop: ${hopCounts[4] || 0}`
   );
 
   return statements;
@@ -361,15 +394,22 @@ function generateQueryChunks(query: string): string[] {
 export async function extractEntitiesFromQuery(
   query: string,
   userId: string,
+  startEntities: string[] = [],
 ): Promise<EntityNode[]> {
   try {
-    // Generate chunks from query
-    const chunks = generateQueryChunks(query);
-
-    // Get embeddings for each chunk
-    const chunkEmbeddings = await Promise.all(
-      chunks.map(chunk => getEmbedding(chunk))
-    );
+    let chunkEmbeddings: Embedding[] = [];
+    if (startEntities.length === 0) {
+      // Generate chunks from query
+      const chunks = generateQueryChunks(query);
+      // Get embeddings for each chunk
+      chunkEmbeddings = await Promise.all(
+        chunks.map(chunk => getEmbedding(chunk))
+      );
+    } else {
+      chunkEmbeddings = await Promise.all(
+        startEntities.map(chunk => getEmbedding(chunk))
+      );
+    }
 
     // Search for entities matching each chunk embedding
     const allEntitySets = await Promise.all(
@@ -425,3 +465,280 @@ export async function getEpisodesByStatements(
   const records = await runQuery(cypher, params);
   return records.map((record) => record.get("e").properties as EpisodicNode);
 }
+
+/**
+ * Episode Graph Search Result
+ */
+export interface EpisodeGraphResult {
+  episode: EpisodicNode;
+  statements: StatementNode[];
+  score: number;
+  metrics: {
+    entityMatchCount: number;
+    totalStatementCount: number;
+    avgRelevance: number;
+    connectivityScore: number;
+  };
+}
+
+/**
+ * Perform episode-centric graph search
+ * Finds episodes with dense subgraphs of statements connected to query entities
+ */
+export async function performEpisodeGraphSearch(
+  query: string,
+  queryEntities: EntityNode[],
+  queryEmbedding: Embedding,
+  userId: string,
+  options: Required<SearchOptions>,
+): Promise<EpisodeGraphResult[]> {
+  try {
+    // If no entities extracted, return empty
+    if (queryEntities.length === 0) {
+      logger.info("Episode graph search: no entities extracted from query");
+      return [];
+    }
+
+    const queryEntityIds = queryEntities.map(e => e.uuid);
+    logger.info(`Episode graph search: ${queryEntityIds.length} query entities`, {
+      entities: queryEntities.map(e => e.name).join(', ')
+    });
+
+    // Timeframe condition for temporal filtering
+    let timeframeCondition = `
+      AND s.validAt <= $validAt
+      ${options.includeInvalidated ? '' : 'AND (s.invalidAt IS NULL OR s.invalidAt > $validAt)'}
+    `;
+    if (options.startTime) {
+      timeframeCondition += ` AND s.validAt >= $startTime`;
+    }
+
+    // Space filtering if provided
+    let spaceCondition = "";
+    if (options.spaceIds.length > 0) {
+      spaceCondition = `
+        AND s.spaceIds IS NOT NULL AND ANY(spaceId IN $spaceIds WHERE spaceId IN s.spaceIds)
+      `;
+    }
+
+    const cypher = `
+      // Step 1: Find statements connected to query entities
+      MATCH (queryEntity:Entity)-[:HAS_SUBJECT|HAS_OBJECT|HAS_PREDICATE]-(s:Statement)
+      WHERE queryEntity.uuid IN $queryEntityIds
+        AND queryEntity.userId = $userId
+        AND s.userId = $userId
+        ${timeframeCondition}
+        ${spaceCondition}
+
+      // Step 2: Find episodes containing these statements
+      MATCH (s)<-[:HAS_PROVENANCE]-(ep:Episode)
+
+      // Step 3: Collect all statements from these episodes (for metrics only)
+      MATCH (ep)-[:HAS_PROVENANCE]->(epStatement:Statement)
+      WHERE epStatement.validAt <= $validAt
+        AND (epStatement.invalidAt IS NULL OR epStatement.invalidAt > $validAt)
+        ${spaceCondition.replace(/s\./g, 'epStatement.')}
+
+      // Step 4: Calculate episode-level metrics
+      WITH ep,
+           collect(DISTINCT s) as entityMatchedStatements,
+           collect(DISTINCT epStatement) as allEpisodeStatements,
+           collect(DISTINCT queryEntity) as matchedEntities
+
+      // Step 5: Calculate semantic relevance for all episode statements
+      WITH ep,
+           entityMatchedStatements,
+           allEpisodeStatements,
+           matchedEntities,
+           [stmt IN allEpisodeStatements |
+             gds.similarity.cosine(stmt.factEmbedding, $queryEmbedding)
+           ] as statementRelevances
+
+      // Step 6: Calculate aggregate scores
+      WITH ep,
+           entityMatchedStatements,
+           size(matchedEntities) as entityMatchCount,
+           size(entityMatchedStatements) as entityStmtCount,
+           size(allEpisodeStatements) as totalStmtCount,
+           reduce(sum = 0.0, score IN statementRelevances | sum + score) /
+             CASE WHEN size(statementRelevances) = 0 THEN 1 ELSE size(statementRelevances) END as avgRelevance
+
+      // Step 7: Calculate connectivity score
+      WITH ep,
+           entityMatchedStatements,
+           entityMatchCount,
+           entityStmtCount,
+           totalStmtCount,
+           avgRelevance,
+           (toFloat(entityStmtCount) / CASE WHEN totalStmtCount = 0 THEN 1 ELSE totalStmtCount END) *
+             entityMatchCount as connectivityScore
+
+      // Step 8: Filter for quality episodes
+      WHERE entityMatchCount >= 1
+        AND avgRelevance >= 0.5
+        AND totalStmtCount >= 1
+
+      // Step 9: Calculate final episode score
+      WITH ep,
+           entityMatchedStatements,
+           entityMatchCount,
+           totalStmtCount,
+           avgRelevance,
+           connectivityScore,
+           // Prioritize: entity matches (2.0x) + connectivity + semantic relevance
+           (entityMatchCount * 2.0) + connectivityScore + avgRelevance as episodeScore
+
+      // Step 10: Return ranked episodes with ONLY entity-matched statements
+      RETURN ep,
+             entityMatchedStatements as statements,
+             entityMatchCount,
+             totalStmtCount,
+             avgRelevance,
+             connectivityScore,
+             episodeScore
+
+      ORDER BY episodeScore DESC, entityMatchCount DESC, totalStmtCount DESC
+      LIMIT 20
+    `;
+
+    const params = {
+      queryEntityIds,
+      userId,
+      queryEmbedding,
+      validAt: options.endTime.toISOString(),
+      ...(options.startTime && { startTime: options.startTime.toISOString() }),
+      ...(options.spaceIds.length > 0 && { spaceIds: options.spaceIds }),
+    };
+
+    const records = await runQuery(cypher, params);
+
+    const results: EpisodeGraphResult[] = records.map((record) => {
+      const episode = record.get("ep").properties as EpisodicNode;
+      const statements = record.get("statements").map((s: any) => s.properties as StatementNode);
+      const entityMatchCount = typeof record.get("entityMatchCount") === 'bigint'
+        ? Number(record.get("entityMatchCount"))
+        : record.get("entityMatchCount");
+      const totalStmtCount = typeof record.get("totalStmtCount") === 'bigint'
+        ? Number(record.get("totalStmtCount"))
+        : record.get("totalStmtCount");
+      const avgRelevance = record.get("avgRelevance");
+      const connectivityScore = record.get("connectivityScore");
+      const episodeScore = record.get("episodeScore");
+
+      return {
+        episode,
+        statements,
+        score: episodeScore,
+        metrics: {
+          entityMatchCount,
+          totalStatementCount: totalStmtCount,
+          avgRelevance,
+          connectivityScore,
+        },
+      };
+    });
+
+    // Log statement counts for debugging
+    results.forEach((result, idx) => {
+      logger.info(
+        `Episode ${idx + 1}: entityMatches=${result.metrics.entityMatchCount}, ` +
+        `totalStmtCount=${result.metrics.totalStatementCount}, ` +
+        `returnedStatements=${result.statements.length}`
+      );
+    });
+
+    logger.info(
+      `Episode graph search: found ${results.length} episodes, ` +
+      `top score: ${results[0]?.score.toFixed(2) || 'N/A'}`
+    );
+
+    return results;
+  } catch (error) {
+    logger.error("Episode graph search error:", { error });
+    return [];
+  }
+}
+
+/**
+ * Get episode IDs for statements in batch (efficient, no N+1 queries)
+ */
+export async function getEpisodeIdsForStatements(
+  statementUuids: string[]
+): Promise<Map<string, string>> {
+  if (statementUuids.length === 0) {
+    return new Map();
+  }
+
+  const cypher = `
+    MATCH (s:Statement)<-[:HAS_PROVENANCE]-(e:Episode)
+    WHERE s.uuid IN $statementUuids
+    RETURN s.uuid as statementUuid, e.uuid as episodeUuid
+  `;
+
+  const records = await runQuery(cypher, { statementUuids });
+
+  const map = new Map<string, string>();
+  records.forEach(record => {
+    map.set(record.get('statementUuid'), record.get('episodeUuid'));
+  });
+
+  return map;
+}
+
+/**
+ * Group statements by their episode IDs efficiently
+ */
+export async function groupStatementsByEpisode(
+  statements: StatementNode[]
+): Promise<Map<string, StatementNode[]>> {
+  const grouped = new Map<string, StatementNode[]>();
+
+  if (statements.length === 0) {
+    return grouped;
+  }
+
+  // Batch fetch episode IDs for all statements
+  const episodeIdMap = await getEpisodeIdsForStatements(
+    statements.map(s => s.uuid)
+  );
+
+  // Group statements by episode ID
+  statements.forEach((statement) => {
+    const episodeId = episodeIdMap.get(statement.uuid);
+    if (episodeId) {
+      if (!grouped.has(episodeId)) {
+        grouped.set(episodeId, []);
+      }
+      grouped.get(episodeId)!.push(statement);
+    }
+  });
+
+  return grouped;
+}
+
+/**
+ * Fetch episode objects by their UUIDs in batch
+ */
+export async function getEpisodesByUuids(
+  episodeUuids: string[]
+): Promise<Map<string, EpisodicNode>> {
+  if (episodeUuids.length === 0) {
+    return new Map();
+  }
+
+  const cypher = `
+    MATCH (e:Episode)
+    WHERE e.uuid IN $episodeUuids
+    RETURN e
+  `;
+
+  const records = await runQuery(cypher, { episodeUuids });
+
+  const map = new Map<string, EpisodicNode>();
+  records.forEach(record => {
+    const episode = record.get('e').properties as EpisodicNode;
+    map.set(episode.uuid, episode);
+  });
+
+  return map;
+}

apps/webapp/app/services/sessionCompaction.server.ts

@@ -1,262 +0,0 @@
-import { logger } from "~/services/logger.service";
-import {
-  getCompactedSessionBySessionId,
-  getCompactionStats,
-  getSessionEpisodes,
-  type CompactedSessionNode,
-} from "~/services/graphModels/compactedSession";
-import { tasks } from "@trigger.dev/sdk/v3";
-
-/**
- * Configuration for session compaction
- */
-export const COMPACTION_CONFIG = {
-  minEpisodesForCompaction: 5, // Minimum episodes to trigger initial compaction
-  compactionThreshold: 1, // Trigger update after N new episodes
-  autoCompactionEnabled: true, // Enable automatic compaction
-};
-
-/**
- * SessionCompactionService - Manages session compaction lifecycle
- */
-export class SessionCompactionService {
-  /**
-   * Check if a session should be compacted
-   */
-  async shouldCompact(sessionId: string, userId: string): Promise<{
-    shouldCompact: boolean;
-    reason: string;
-    episodeCount?: number;
-    newEpisodeCount?: number;
-  }> {
-    try {
-      // Get existing compact
-      const existingCompact = await getCompactedSessionBySessionId(sessionId, userId);
-
-      if (!existingCompact) {
-        // No compact exists, check if we have enough episodes
-        const episodeCount = await this.getSessionEpisodeCount(sessionId, userId);
-
-        if (episodeCount >= COMPACTION_CONFIG.minEpisodesForCompaction) {
-          return {
-            shouldCompact: true,
-            reason: "initial_compaction",
-            episodeCount,
-          };
-        }
-
-        return {
-          shouldCompact: false,
-          reason: "insufficient_episodes",
-          episodeCount,
-        };
-      }
-
-      // Compact exists, check if we have enough new episodes
-      const newEpisodeCount = await this.getNewEpisodeCount(
-        sessionId,
-        userId,
-        existingCompact.endTime
-      );
-
-      if (newEpisodeCount >= COMPACTION_CONFIG.compactionThreshold) {
-        return {
-          shouldCompact: true,
-          reason: "update_compaction",
-          newEpisodeCount,
-        };
-      }
-
-      return {
-        shouldCompact: false,
-        reason: "insufficient_new_episodes",
-        newEpisodeCount,
-      };
-    } catch (error) {
-      logger.error(`Error checking if session should compact`, {
-        sessionId,
-        userId,
-        error: error instanceof Error ? error.message : String(error),
-      });
-
-      return {
-        shouldCompact: false,
-        reason: "error",
-      };
-    }
-  }
-
-  /**
-   * Get total episode count for a session
-   */
-  private async getSessionEpisodeCount(
-    sessionId: string,
-    userId: string
-  ): Promise<number> {
-    const episodes = await getSessionEpisodes(sessionId, userId);
-    return episodes.length;
-  }
-
-  /**
-   * Get count of new episodes since last compaction
-   */
-  private async getNewEpisodeCount(
-    sessionId: string,
-    userId: string,
-    afterTime: Date
-  ): Promise<number> {
-    const episodes = await getSessionEpisodes(sessionId, userId, afterTime);
-    return episodes.length;
-  }
-
-  /**
-   * Trigger compaction for a session
-   */
-  async triggerCompaction(
-    sessionId: string,
-    userId: string,
-    source: string,
-    triggerSource: "auto" | "manual" | "threshold" = "auto"
-  ): Promise<{ success: boolean; taskId?: string; error?: string }> {
-    try {
-      // Check if compaction should be triggered
-      const check = await this.shouldCompact(sessionId, userId);
-
-      if (!check.shouldCompact) {
-        logger.info(`Compaction not needed`, {
-          sessionId,
-          userId,
-          reason: check.reason,
-        });
-
-        return {
-          success: false,
-          error: `Compaction not needed: ${check.reason}`,
-        };
-      }
-
-      // Trigger the compaction task
-      logger.info(`Triggering session compaction`, {
-        sessionId,
-        userId,
-        source,
-        triggerSource,
-        reason: check.reason,
-      });
-
-      const handle = await tasks.trigger("session-compaction", {
-        userId,
-        sessionId,
-        source,
-        triggerSource,
-      });
-
-      logger.info(`Session compaction triggered`, {
-        sessionId,
-        userId,
-        taskId: handle.id,
-      });
-
-      return {
-        success: true,
-        taskId: handle.id,
-      };
-    } catch (error) {
-      logger.error(`Failed to trigger compaction`, {
-        sessionId,
-        userId,
-        error: error instanceof Error ? error.message : String(error),
-      });
-
-      return {
-        success: false,
-        error: error instanceof Error ? error.message : "Unknown error",
-      };
-    }
-  }
-
-  /**
-   * Get compacted session for recall
-   */
-  async getCompactForRecall(
-    sessionId: string,
-    userId: string
-  ): Promise<CompactedSessionNode | null> {
-    try {
-      return await getCompactedSessionBySessionId(sessionId, userId);
-    } catch (error) {
-      logger.error(`Error fetching compact for recall`, {
-        sessionId,
-        userId,
-        error: error instanceof Error ? error.message : String(error),
-      });
-      return null;
-    }
-  }
-
-  /**
-   * Get compaction statistics for a user
-   */
-  async getStats(userId: string): Promise<{
-    totalCompacts: number;
-    totalEpisodes: number;
-    averageCompressionRatio: number;
-    mostRecentCompaction: Date | null;
-  }> {
-    try {
-      return await getCompactionStats(userId);
-    } catch (error) {
-      logger.error(`Error fetching compaction stats`, {
-        userId,
-        error: error instanceof Error ? error.message : String(error),
-      });
-
-      return {
-        totalCompacts: 0,
-        totalEpisodes: 0,
-        averageCompressionRatio: 0,
-        mostRecentCompaction: null,
-      };
-    }
-  }
-
-  /**
-   * Auto-trigger compaction after episode ingestion
-   * Called from ingestion pipeline
-   */
-  async autoTriggerAfterIngestion(
-    sessionId: string | null | undefined,
-    userId: string,
-    source: string
-  ): Promise<void> {
-    // Skip if no sessionId or auto-compaction disabled
-    if (!sessionId || !COMPACTION_CONFIG.autoCompactionEnabled) {
-      return;
-    }
-
-    try {
-      const check = await this.shouldCompact(sessionId, userId);
-
-      if (check.shouldCompact) {
-        logger.info(`Auto-triggering compaction after ingestion`, {
-          sessionId,
-          userId,
-          reason: check.reason,
-        });
-
-        // Trigger compaction asynchronously (don't wait)
-        await this.triggerCompaction(sessionId, userId, source, "auto");
-      }
-    } catch (error) {
-      // Log error but don't fail ingestion
-      logger.error(`Error in auto-trigger compaction`, {
-        sessionId,
-        userId,
-        error: error instanceof Error ? error.message : String(error),
-      });
-    }
-  }
-}
-
-// Singleton instance
-export const sessionCompactionService = new SessionCompactionService();

apps/webapp/app/services/space.server.ts

@@ -6,7 +6,6 @@ import {
 } from "@core/types";
 import { type Space } from "@prisma/client";
 
-import { triggerSpaceAssignment } from "~/trigger/spaces/space-assignment";
 import {
   assignEpisodesToSpace,
   createSpace,
@@ -63,24 +62,8 @@ export class SpaceService {
 
     logger.info(`Created space ${space.id} successfully`);
 
-    // Trigger automatic LLM assignment for the new space
-    try {
-      await triggerSpaceAssignment({
-        userId: params.userId,
-        workspaceId: params.workspaceId,
-        mode: "new_space",
-        newSpaceId: space.id,
-        batchSize: 25, // Analyze recent statements for the new space
-      });
-
-      logger.info(`Triggered LLM space assignment for new space ${space.id}`);
-    } catch (error) {
-      // Don't fail space creation if LLM assignment fails
-      logger.warn(
-        `Failed to trigger LLM assignment for space ${space.id}:`,
-        error as Record<string, unknown>,
-      );
-    }
+    // Track space creation
+    // trackFeatureUsage("space_created", params.userId).catch(console.error);
 
     return space;
   }
@@ -192,6 +175,7 @@ export class SpaceService {
     } catch (e) {
       logger.info(`Nothing to update to graph`);
     }
+
     logger.info(`Updated space ${spaceId} successfully`);
     return space;
   }

apps/webapp/app/services/telemetry.server.ts

@@ -0,0 +1,278 @@
+import { PostHog } from "posthog-node";
+import { env } from "~/env.server";
+import { prisma } from "~/db.server";
+
+// Server-side PostHog client for backend tracking
+let posthogClient: PostHog | null = null;
+
+function getPostHogClient(): PostHog | null {
+  if (!env.TELEMETRY_ENABLED || !env.POSTHOG_PROJECT_KEY) {
+    return null;
+  }
+
+  if (!posthogClient) {
+    posthogClient = new PostHog(env.POSTHOG_PROJECT_KEY, {
+      host: "https://us.posthog.com",
+    });
+  }
+
+  return posthogClient;
+}
+
+/**
+ * Get user email from userId, or return "anonymous" if TELEMETRY_ANONYMOUS is enabled
+ */
+async function getUserIdentifier(userId?: string): Promise<string> {
+  if (env.TELEMETRY_ANONYMOUS || !userId) {
+    return "anonymous";
+  }
+
+  try {
+    const user = await prisma.user.findUnique({
+      where: { id: userId },
+      select: { email: true },
+    });
+    return user?.email || "anonymous";
+  } catch (error) {
+    return "anonymous";
+  }
+}
+
+// Telemetry event types
+export type TelemetryEvent =
+  | "episode_ingested"
+  | "document_ingested"
+  | "search_performed"
+  | "deep_search_performed"
+  | "conversation_created"
+  | "conversation_message_sent"
+  | "space_created"
+  | "space_updated"
+  | "user_registered"
+  | "error_occurred"
+  | "queue_job_started"
+  | "queue_job_completed"
+  | "queue_job_failed";
+
+// Common properties for all events
+interface BaseEventProperties {
+  userId?: string;
+  workspaceId?: string;
+  email?: string;
+  name?: string;
+  queueProvider?: "trigger" | "bullmq";
+  modelProvider?: string;
+  embeddingModel?: string;
+  appEnv?: string;
+}
+
+// Event-specific properties
+interface EpisodeIngestedProperties extends BaseEventProperties {
+  spaceId?: string;
+  documentCount?: number;
+  processingTimeMs?: number;
+}
+
+interface SearchPerformedProperties extends BaseEventProperties {
+  query: string;
+  resultsCount: number;
+  searchType: "basic" | "deep";
+  spaceIds?: string[];
+}
+
+interface ConversationProperties extends BaseEventProperties {
+  conversationId: string;
+  messageLength?: number;
+  model?: string;
+}
+
+interface ErrorProperties extends BaseEventProperties {
+  errorType: string;
+  errorMessage: string;
+  stackTrace?: string;
+  context?: Record<string, any>;
+}
+
+interface QueueJobProperties extends BaseEventProperties {
+  jobId: string;
+  jobType: string;
+  queueName: string;
+  durationMs?: number;
+}
+
+type EventProperties =
+  | EpisodeIngestedProperties
+  | SearchPerformedProperties
+  | ConversationProperties
+  | ErrorProperties
+  | QueueJobProperties
+  | BaseEventProperties;
+
+/**
+ * Track telemetry events to PostHog
+ */
+export async function trackEvent(
+  event: TelemetryEvent,
+  properties: EventProperties,
+): Promise<void> {
+  const client = getPostHogClient();
+  if (!client) return;
+
+  try {
+    const userId = properties.userId || "anonymous";
+
+    // Add common properties to all events
+    const enrichedProperties = {
+      ...properties,
+      queueProvider: env.QUEUE_PROVIDER,
+      modelProvider: getModelProvider(),
+      embeddingModel: env.EMBEDDING_MODEL,
+      appEnv: env.APP_ENV,
+      appOrigin: env.APP_ORIGIN,
+      timestamp: new Date().toISOString(),
+    };
+
+    client.capture({
+      distinctId: userId,
+      event,
+      properties: enrichedProperties,
+    });
+
+    // Identify user if we have their info
+    if (properties.email || properties.name) {
+      client.identify({
+        distinctId: userId,
+        properties: {
+          email: properties.email,
+          name: properties.name,
+        },
+      });
+    }
+  } catch (error) {
+    // Silently fail - don't break the app if telemetry fails
+    console.error("Telemetry error:", error);
+  }
+}
+
+/**
+ * Track feature usage - simplified API
+ * @param feature - Feature name (e.g., "episode_ingested", "search_performed")
+ * @param userId - User ID (will be converted to email internally)
+ * @param properties - Additional properties (optional)
+ */
+export async function trackFeatureUsage(
+  feature: string,
+  userId?: string,
+  properties?: Record<string, any>,
+): Promise<void> {
+  const client = getPostHogClient();
+  if (!client) return;
+
+  try {
+    const email = await getUserIdentifier(userId);
+
+    client.capture({
+      distinctId: email,
+      event: feature,
+      properties: {
+        ...properties,
+        appOrigin: env.APP_ORIGIN,
+        timestamp: new Date().toISOString(),
+      },
+    });
+  } catch (error) {
+    // Silently fail - don't break the app if telemetry fails
+    console.error("Telemetry error:", error);
+  }
+}
+
+/**
+ * Track system configuration once at startup
+ * Tracks queue provider, model provider, embedding model, etc.
+ */
+export async function trackConfig(): Promise<void> {
+  const client = getPostHogClient();
+  if (!client) return;
+
+  try {
+    client.capture({
+      distinctId: "system",
+      event: "system_config",
+      properties: {
+        queueProvider: env.QUEUE_PROVIDER,
+        modelProvider: getModelProvider(),
+        model: env.MODEL,
+        embeddingModel: env.EMBEDDING_MODEL,
+        appEnv: env.APP_ENV,
+        nodeEnv: env.NODE_ENV,
+        timestamp: new Date().toISOString(),
+        appOrigin: env.APP_ORIGIN,
+      },
+    });
+  } catch (error) {
+    console.error("Failed to track config:", error);
+  }
+}
+
+/**
+ * Track errors
+ */
+export async function trackError(
+  error: Error,
+  context?: Record<string, any>,
+  userId?: string,
+): Promise<void> {
+  const client = getPostHogClient();
+  if (!client) return;
+
+  try {
+    const email = await getUserIdentifier(userId);
+
+    client.capture({
+      distinctId: email,
+      event: "error_occurred",
+      properties: {
+        errorType: error.name,
+        errorMessage: error.message,
+        appOrigin: env.APP_ORIGIN,
+        stackTrace: error.stack,
+        ...context,
+        timestamp: new Date().toISOString(),
+      },
+    });
+  } catch (trackingError) {
+    console.error("Failed to track error:", trackingError);
+  }
+}
+
+/**
+ * Flush pending events (call on shutdown)
+ */
+export async function flushTelemetry(): Promise<void> {
+  const client = getPostHogClient();
+  if (client) {
+    await client.shutdown();
+  }
+}
+
+/**
+ * Helper to determine model provider from MODEL env variable
+ */
+function getModelProvider(): string {
+  const model = env.MODEL.toLowerCase();
+  if (model.includes("gpt") || model.includes("openai")) return "openai";
+  if (model.includes("claude") || model.includes("anthropic"))
+    return "anthropic";
+  if (env.OLLAMA_URL) return "ollama";
+  return "unknown";
+}
+
+// Export types for use in other files
+export type {
+  BaseEventProperties,
+  EpisodeIngestedProperties,
+  SearchPerformedProperties,
+  ConversationProperties,
+  ErrorProperties,
+  QueueJobProperties,
+};

apps/webapp/app/trigger/bert/topic-analysis.ts

@@ -0,0 +1,53 @@
+import { task } from "@trigger.dev/sdk/v3";
+import { python } from "@trigger.dev/python";
+import {
+  processTopicAnalysis,
+  type TopicAnalysisPayload,
+} from "~/jobs/bert/topic-analysis.logic";
+import { spaceSummaryTask } from "~/trigger/spaces/space-summary";
+
+/**
+ * Python runner for Trigger.dev using python.runScript
+ */
+async function runBertWithTriggerPython(
+  userId: string,
+  minTopicSize: number,
+  nrTopics?: number,
+): Promise<string> {
+  const args = [userId, "--json"];
+
+  if (nrTopics) {
+    args.push("--nr-topics", String(nrTopics));
+  }
+
+  console.log(
+    `[BERT Topic Analysis] Running with Trigger.dev Python: args=${args.join(" ")}`,
+  );
+
+  const result = await python.runScript("./python/main.py", args);
+  return result.stdout;
+}
+
+/**
+ * Trigger.dev task for BERT topic analysis
+ *
+ * This is a thin wrapper around the common logic in jobs/bert/topic-analysis.logic.ts
+ */
+export const bertTopicAnalysisTask = task({
+  id: "bert-topic-analysis",
+  queue: {
+    name: "bert-topic-analysis",
+    concurrencyLimit: 3, // Max 3 parallel analyses to avoid CPU overload
+  },
+  run: async (payload: TopicAnalysisPayload) => {
+    return await processTopicAnalysis(
+      payload,
+      // Callback to enqueue space summary
+      async (params) => {
+        await spaceSummaryTask.trigger(params);
+      },
+      // Python runner for Trigger.dev
+      runBertWithTriggerPython,
+    );
+  },
+});

apps/webapp/app/trigger/chat/chat-utils.ts

@@ -1,492 +0,0 @@
-/* eslint-disable @typescript-eslint/no-explicit-any */
-
-import { ActionStatusEnum } from "@core/types";
-import { logger } from "@trigger.dev/sdk/v3";
-import {
-  type CoreMessage,
-  type DataContent,
-  jsonSchema,
-  tool,
-  type ToolSet,
-} from "ai";
-import axios from "axios";
-import Handlebars from "handlebars";
-
-import { REACT_SYSTEM_PROMPT, REACT_USER_PROMPT } from "./prompt";
-import { generate, processTag } from "./stream-utils";
-import { type AgentMessage, AgentMessageType, Message } from "./types";
-import { type MCP } from "../utils/mcp";
-import {
-  type ExecutionState,
-  type HistoryStep,
-  type Resource,
-  type TotalCost,
-} from "../utils/types";
-import { flattenObject } from "../utils/utils";
-
-interface LLMOutputInterface {
-  response: AsyncGenerator<
-    | string
-    | {
-        type: string;
-        toolName: string;
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        args?: any;
-        toolCallId?: string;
-        message?: string;
-      },
-    any,
-    any
-  >;
-}
-
-const progressUpdateTool = tool({
-  description:
-    "Send a progress update to the user about what has been discovered or will be done next in a crisp and user friendly way no technical terms",
-  parameters: jsonSchema({
-    type: "object",
-    properties: {
-      message: {
-        type: "string",
-        description: "The progress update message to send to the user",
-      },
-    },
-    required: ["message"],
-    additionalProperties: false,
-  }),
-});
-
-const internalTools = ["core--progress_update"];
-
-async function addResources(messages: CoreMessage[], resources: Resource[]) {
-  const resourcePromises = resources.map(async (resource) => {
-    // Remove everything before "/api" in the publicURL
-    if (resource.publicURL) {
-      const apiIndex = resource.publicURL.indexOf("/api");
-      if (apiIndex !== -1) {
-        resource.publicURL = resource.publicURL.substring(apiIndex);
-      }
-    }
-    const response = await axios.get(resource.publicURL, {
-      responseType: "arraybuffer",
-    });
-
-    if (resource.fileType.startsWith("image/")) {
-      return {
-        type: "image",
-        image: response.data as DataContent,
-      };
-    }
-
-    return {
-      type: "file",
-      data: response.data as DataContent,
-
-      mimeType: resource.fileType,
-    };
-  });
-
-  const content = await Promise.all(resourcePromises);
-
-  return [...messages, { role: "user", content } as CoreMessage];
-}
-
-function toolToMessage(history: HistoryStep[], messages: CoreMessage[]) {
-  for (let i = 0; i < history.length; i++) {
-    const step = history[i];
-
-    // Add assistant message with tool calls
-    if (step.observation && step.skillId) {
-      messages.push({
-        role: "assistant",
-        content: [
-          {
-            type: "tool-call",
-            toolCallId: step.skillId,
-            toolName: step.skill ?? "",
-            args:
-              typeof step.skillInput === "string"
-                ? JSON.parse(step.skillInput)
-                : step.skillInput,
-          },
-        ],
-      });
-
-      messages.push({
-        role: "tool",
-        content: [
-          {
-            type: "tool-result",
-            toolName: step.skill,
-            toolCallId: step.skillId,
-            result: step.observation,
-            isError: step.isError,
-          },
-        ],
-      } as any);
-    }
-    // Handle format correction steps (observation exists but no skillId)
-    else if (step.observation && !step.skillId) {
-      // Add as a system message for format correction
-      messages.push({
-        role: "system",
-        content: step.observation,
-      });
-    }
-  }
-
-  return messages;
-}
-
-async function makeNextCall(
-  executionState: ExecutionState,
-  TOOLS: ToolSet,
-  totalCost: TotalCost,
-  guardLoop: number,
-): Promise<LLMOutputInterface> {
-  const { context, history, previousHistory } = executionState;
-
-  const promptInfo = {
-    USER_MESSAGE: executionState.query,
-    CONTEXT: context,
-    USER_MEMORY: executionState.userMemoryContext,
-  };
-
-  let messages: CoreMessage[] = [];
-
-  const systemTemplateHandler = Handlebars.compile(REACT_SYSTEM_PROMPT);
-  let systemPrompt = systemTemplateHandler(promptInfo);
-
-  const userTemplateHandler = Handlebars.compile(REACT_USER_PROMPT);
-  const userPrompt = userTemplateHandler(promptInfo);
-
-  // Always start with a system message (this does use tokens but keeps the instructions clear)
-  messages.push({ role: "system", content: systemPrompt });
-
-  // For subsequent queries, include only final responses from previous exchanges if available
-  if (previousHistory && previousHistory.length > 0) {
-    messages = [...messages, ...previousHistory];
-  }
-
-  // Add the current user query (much simpler than the full prompt)
-  messages.push({ role: "user", content: userPrompt });
-
-  // Include any steps from the current interaction
-  if (history.length > 0) {
-    messages = toolToMessage(history, messages);
-  }
-
-  if (executionState.resources && executionState.resources.length > 0) {
-    messages = await addResources(messages, executionState.resources);
-  }
-
-  // Get the next action from the LLM
-  const response = generate(
-    messages,
-    guardLoop > 0 && guardLoop % 3 === 0,
-    (event) => {
-      const usage = event.usage;
-      totalCost.inputTokens += usage.promptTokens;
-      totalCost.outputTokens += usage.completionTokens;
-    },
-    TOOLS,
-  );
-
-  return { response };
-}
-
-export async function* run(
-  message: string,
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  context: Record<string, any>,
-  previousHistory: CoreMessage[],
-  mcp: MCP,
-  stepHistory: HistoryStep[],
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-): AsyncGenerator<AgentMessage, any, any> {
-  let guardLoop = 0;
-
-  let tools = {
-    ...(await mcp.allTools()),
-    "core--progress_update": progressUpdateTool,
-  };
-
-  logger.info("Tools have been formed");
-
-  let contextText = "";
-  let resources = [];
-  if (context) {
-    // Extract resources and remove from context
-    resources = context.resources || [];
-    delete context.resources;
-
-    // Process remaining context
-    contextText = flattenObject(context).join("\n");
-  }
-
-  const executionState: ExecutionState = {
-    query: message,
-    context: contextText,
-    resources,
-    previousHistory,
-    history: stepHistory, // Track the full ReAct history
-    completed: false,
-  };
-
-  const totalCost: TotalCost = { inputTokens: 0, outputTokens: 0, cost: 0 };
-
-  try {
-    while (!executionState.completed && guardLoop < 50) {
-      logger.info(`Starting the loop: ${guardLoop}`);
-
-      const { response: llmResponse } = await makeNextCall(
-        executionState,
-        tools,
-        totalCost,
-        guardLoop,
-      );
-
-      let toolCallInfo;
-
-      const messageState = {
-        inTag: false,
-        message: "",
-        messageEnded: false,
-        lastSent: "",
-      };
-
-      const questionState = {
-        inTag: false,
-        message: "",
-        messageEnded: false,
-        lastSent: "",
-      };
-
-      let totalMessage = "";
-      const toolCalls = [];
-
-      // LLM thought response
-      for await (const chunk of llmResponse) {
-        if (typeof chunk === "object" && chunk.type === "tool-call") {
-          toolCallInfo = chunk;
-          toolCalls.push(chunk);
-        }
-
-        totalMessage += chunk;
-
-        if (!messageState.messageEnded) {
-          yield* processTag(
-            messageState,
-            totalMessage,
-            chunk as string,
-            "<final_response>",
-            "</final_response>",
-            {
-              start: AgentMessageType.MESSAGE_START,
-              chunk: AgentMessageType.MESSAGE_CHUNK,
-              end: AgentMessageType.MESSAGE_END,
-            },
-          );
-        }
-
-        if (!questionState.messageEnded) {
-          yield* processTag(
-            questionState,
-            totalMessage,
-            chunk as string,
-            "<question_response>",
-            "</question_response>",
-            {
-              start: AgentMessageType.MESSAGE_START,
-              chunk: AgentMessageType.MESSAGE_CHUNK,
-              end: AgentMessageType.MESSAGE_END,
-            },
-          );
-        }
-      }
-
-      logger.info(`Cost for thought: ${JSON.stringify(totalCost)}`);
-
-      // Replace the error-handling block with this self-correcting implementation
-      if (
-        !totalMessage.includes("final_response") &&
-        !totalMessage.includes("question_response") &&
-        !toolCallInfo
-      ) {
-        // Log the issue for debugging
-        logger.info(
-          `Invalid response format detected. Attempting to get proper format.`,
-        );
-
-        // Extract the raw content from the invalid response
-        const rawContent = totalMessage
-          .replace(/(<[^>]*>|<\/[^>]*>)/g, "")
-          .trim();
-
-        // Create a correction step
-        const stepRecord: HistoryStep = {
-          thought: "",
-          skill: "",
-          skillId: "",
-          userMessage: "Core agent error, retrying \n",
-          isQuestion: false,
-          isFinal: false,
-          tokenCount: totalCost,
-          skillInput: "",
-          observation: `Your last response was not in a valid format. You must respond with EXACTLY ONE of the required formats: either a tool call, <question_response> tags, or <final_response> tags. Please reformat your previous response using the correct format:\n\n${rawContent}`,
-        };
-
-        yield Message("", AgentMessageType.MESSAGE_START);
-        yield Message(
-          stepRecord.userMessage as string,
-          AgentMessageType.MESSAGE_CHUNK,
-        );
-        yield Message("", AgentMessageType.MESSAGE_END);
-
-        // Add this step to the history
-        yield Message(JSON.stringify(stepRecord), AgentMessageType.STEP);
-        executionState.history.push(stepRecord);
-
-        // Log that we're continuing the loop with a correction request
-        logger.info(`Added format correction request to history.`);
-
-        // Don't mark as completed - let the loop continue
-        guardLoop++; // Still increment to prevent infinite loops
-        continue;
-      }
-
-      // Record this step in history
-      const stepRecord: HistoryStep = {
-        thought: "",
-        skill: "",
-        skillId: "",
-        userMessage: "",
-        isQuestion: false,
-        isFinal: false,
-        tokenCount: totalCost,
-        skillInput: "",
-      };
-
-      if (totalMessage && totalMessage.includes("final_response")) {
-        executionState.completed = true;
-        stepRecord.isFinal = true;
-        stepRecord.userMessage = messageState.message;
-        stepRecord.finalTokenCount = totalCost;
-        stepRecord.skillStatus = ActionStatusEnum.SUCCESS;
-        yield Message(JSON.stringify(stepRecord), AgentMessageType.STEP);
-        executionState.history.push(stepRecord);
-        break;
-      }
-
-      if (totalMessage && totalMessage.includes("question_response")) {
-        executionState.completed = true;
-        stepRecord.isQuestion = true;
-        stepRecord.userMessage = questionState.message;
-        stepRecord.finalTokenCount = totalCost;
-        stepRecord.skillStatus = ActionStatusEnum.QUESTION;
-        yield Message(JSON.stringify(stepRecord), AgentMessageType.STEP);
-        executionState.history.push(stepRecord);
-        break;
-      }
-
-      if (toolCalls && toolCalls.length > 0) {
-        // Run all tool calls in parallel
-        for (const toolCallInfo of toolCalls) {
-          const skillName = toolCallInfo.toolName;
-          const skillId = toolCallInfo.toolCallId;
-          const skillInput = toolCallInfo.args;
-
-          const toolName = skillName.split("--")[1];
-          const agent = skillName.split("--")[0];
-
-          const stepRecord: HistoryStep = {
-            agent,
-            thought: "",
-            skill: skillName,
-            skillId,
-            userMessage: "",
-            isQuestion: false,
-            isFinal: false,
-            tokenCount: totalCost,
-            skillInput: JSON.stringify(skillInput),
-          };
-
-          if (!internalTools.includes(skillName)) {
-            const skillMessageToSend = `\n<skill id="${skillId}" name="${toolName}" agent="${agent}"></skill>\n`;
-
-            stepRecord.userMessage += skillMessageToSend;
-
-            yield Message("", AgentMessageType.MESSAGE_START);
-            yield Message(skillMessageToSend, AgentMessageType.MESSAGE_CHUNK);
-            yield Message("", AgentMessageType.MESSAGE_END);
-          }
-
-          let result;
-          try {
-            // Log skill execution details
-            logger.info(`Executing skill: ${skillName}`);
-            logger.info(`Input parameters: ${JSON.stringify(skillInput)}`);
-
-            if (!internalTools.includes(toolName)) {
-              yield Message(
-                JSON.stringify({ skillId, status: "start" }),
-                AgentMessageType.SKILL_START,
-              );
-            }
-
-            // Handle CORE agent tools
-            if (agent === "core") {
-              if (toolName === "progress_update") {
-                yield Message("", AgentMessageType.MESSAGE_START);
-                yield Message(
-                  skillInput.message,
-                  AgentMessageType.MESSAGE_CHUNK,
-                );
-                stepRecord.userMessage += skillInput.message;
-                yield Message("", AgentMessageType.MESSAGE_END);
-                result = "Progress update sent successfully";
-              }
-            }
-            // Handle other MCP tools
-            else {
-              result = await mcp.callTool(skillName, skillInput);
-
-              yield Message(
-                JSON.stringify({ result, skillId }),
-                AgentMessageType.SKILL_CHUNK,
-              );
-            }
-
-            yield Message(
-              JSON.stringify({ skillId, status: "end" }),
-              AgentMessageType.SKILL_END,
-            );
-
-            stepRecord.skillOutput =
-              typeof result === "object"
-                ? JSON.stringify(result, null, 2)
-                : result;
-            stepRecord.observation = stepRecord.skillOutput;
-          } catch (e) {
-            console.log(e);
-            logger.error(e as string);
-            stepRecord.skillInput = skillInput;
-            stepRecord.observation = JSON.stringify(e);
-            stepRecord.isError = true;
-          }
-
-          logger.info(`Skill step: ${JSON.stringify(stepRecord)}`);
-
-          yield Message(JSON.stringify(stepRecord), AgentMessageType.STEP);
-          executionState.history.push(stepRecord);
-        }
-      }
-      guardLoop++;
-    }
-    yield Message("Stream ended", AgentMessageType.STREAM_END);
-  } catch (e) {
-    logger.error(e as string);
-    yield Message((e as Error).message, AgentMessageType.ERROR);
-    yield Message("Stream ended", AgentMessageType.STREAM_END);
-  }
-}

apps/webapp/app/trigger/chat/chat.ts

@@ -1,150 +0,0 @@
-import { ActionStatusEnum } from "@core/types";
-import { metadata, task, queue } from "@trigger.dev/sdk";
-
-import { run } from "./chat-utils";
-import { MCP } from "../utils/mcp";
-import { type HistoryStep } from "../utils/types";
-import {
-  createConversationHistoryForAgent,
-  deductCredits,
-  deletePersonalAccessToken,
-  getPreviousExecutionHistory,
-  hasCredits,
-  InsufficientCreditsError,
-  init,
-  type RunChatPayload,
-  updateConversationHistoryMessage,
-  updateConversationStatus,
-  updateExecutionStep,
-} from "../utils/utils";
-
-const chatQueue = queue({
-  name: "chat-queue",
-  concurrencyLimit: 50,
-});
-
-/**
- * Main chat task that orchestrates the agent workflow
- * Handles conversation context, agent selection, and LLM interactions
- */
-export const chat = task({
-  id: "chat",
-  maxDuration: 3000,
-  queue: chatQueue,
-  init,
-  run: async (payload: RunChatPayload, { init }) => {
-    await updateConversationStatus("running", payload.conversationId);
-
-    try {
-      // Check if workspace has sufficient credits before processing
-      if (init?.conversation.workspaceId) {
-        const hasSufficientCredits = await hasCredits(
-          init.conversation.workspaceId,
-          "chatMessage",
-        );
-
-        if (!hasSufficientCredits) {
-          throw new InsufficientCreditsError(
-            "Insufficient credits to process chat message. Please upgrade your plan or wait for your credits to reset.",
-          );
-        }
-      }
-
-      const { previousHistory, ...otherData } = payload.context;
-
-      // Initialise mcp
-      const mcpHeaders = { Authorization: `Bearer ${init?.token}` };
-      const mcp = new MCP();
-      await mcp.init();
-      await mcp.load(mcpHeaders);
-
-      // Prepare context with additional metadata
-      const context = {
-        // Currently this is assuming we only have one page in context
-        context: {
-          ...(otherData.page && otherData.page.length > 0
-            ? { page: otherData.page[0] }
-            : {}),
-        },
-        workpsaceId: init?.conversation.workspaceId,
-        resources: otherData.resources,
-        todayDate: new Date().toISOString(),
-      };
-
-      // Extract user's goal from conversation history
-      const message = init?.conversationHistory?.message;
-      // Retrieve execution history from previous interactions
-      const previousExecutionHistory = getPreviousExecutionHistory(
-        previousHistory ?? [],
-      );
-
-      let agentUserMessage = "";
-      let agentConversationHistory;
-      let stepHistory: HistoryStep[] = [];
-      // Prepare conversation history in agent-compatible format
-      agentConversationHistory = await createConversationHistoryForAgent(
-        payload.conversationId,
-      );
-
-      const llmResponse = run(
-        message as string,
-        context,
-        previousExecutionHistory,
-        mcp,
-        stepHistory,
-      );
-
-      const stream = await metadata.stream("messages", llmResponse);
-
-      let conversationStatus = "success";
-      for await (const step of stream) {
-        if (step.type === "STEP") {
-          const stepDetails = JSON.parse(step.message as string);
-
-          if (stepDetails.skillStatus === ActionStatusEnum.TOOL_REQUEST) {
-            conversationStatus = "need_approval";
-          }
-
-          if (stepDetails.skillStatus === ActionStatusEnum.QUESTION) {
-            conversationStatus = "need_attention";
-          }
-
-          await updateExecutionStep(
-            { ...stepDetails },
-            agentConversationHistory.id,
-          );
-
-          agentUserMessage += stepDetails.userMessage;
-
-          await updateConversationHistoryMessage(
-            agentUserMessage,
-            agentConversationHistory.id,
-          );
-        } else if (step.type === "STREAM_END") {
-          break;
-        }
-      }
-
-      await updateConversationStatus(
-        conversationStatus,
-        payload.conversationId,
-      );
-
-      // Deduct credits for chat message
-      if (init?.conversation.workspaceId) {
-        await deductCredits(init.conversation.workspaceId, "chatMessage");
-      }
-
-      if (init?.tokenId) {
-        await deletePersonalAccessToken(init.tokenId);
-      }
-    } catch (e) {
-      console.log(e);
-      await updateConversationStatus("failed", payload.conversationId);
-      if (init?.tokenId) {
-        await deletePersonalAccessToken(init.tokenId);
-      }
-      throw new Error(e as string);
-    }
-  },
-});

apps/webapp/app/trigger/chat/prompt.ts

@@ -1,159 +0,0 @@
-export const REACT_SYSTEM_PROMPT = `
-You are a helpful AI assistant with access to user memory. Your primary capabilities are:
-
-1. **Memory-First Approach**: Always check user memory first to understand context and previous interactions
-2. **Intelligent Information Gathering**: Analyze queries to determine if current information is needed
-3. **Memory Management**: Help users store, retrieve, and organize information in their memory
-4. **Contextual Assistance**: Use memory to provide personalized and contextual responses
-
-<context>
-{{CONTEXT}}
-</context>
-
-<information_gathering>
-Follow this intelligent approach for information gathering:
-
-1. **MEMORY FIRST** (Always Required)
-   - Always check memory FIRST using core--search_memory before any other actions
-   - Consider this your highest priority for EVERY interaction - as essential as breathing
-   - Memory provides context, personal preferences, and historical information
-   - Use memory to understand user's background, ongoing projects, and past conversations
-
-2. **INFORMATION SYNTHESIS** (Combine Sources)
-   - Use memory to personalize current information based on user preferences
-   - Always store new useful information in memory using core--add_memory
-
-3. **TRAINING KNOWLEDGE** (Foundation)
-   - Use your training knowledge as the foundation for analysis and explanation
-   - Apply training knowledge to interpret and contextualize information from memory
-   - Indicate when you're using training knowledge vs. live information sources
-
-EXECUTION APPROACH:
-- Memory search is mandatory for every interaction
-- Always indicate your information sources in responses
-</information_gathering>
-
-<memory>
-QUERY FORMATION:
-- Write specific factual statements as queries (e.g., "user email address" not "what is the user's email?")
-- Create multiple targeted memory queries for complex requests
-
-KEY QUERY AREAS:
-- Personal context: user name, location, identity, work context
-- Project context: repositories, codebases, current work, team members
-- Task context: recent tasks, ongoing projects, deadlines, priorities
-- Integration context: GitHub repos, Slack channels, Linear projects, connected services
-- Communication patterns: email preferences, notification settings, workflow automation
-- Technical context: coding languages, frameworks, development environment
-- Collaboration context: team members, project stakeholders, meeting patterns
-- Preferences: likes, dislikes, communication style, tool preferences
-- History: previous discussions, past requests, completed work, recurring issues
-- Automation rules: user-defined workflows, triggers, automation preferences
-
-MEMORY USAGE:
-- Execute multiple memory queries in parallel rather than sequentially
-- Batch related memory queries when possible
-- Prioritize recent information over older memories
-- Create comprehensive context-aware queries based on user message/activity content
-- Extract and query SEMANTIC CONTENT, not just structural metadata
-- Parse titles, descriptions, and content for actual subject matter keywords
-- Search internal SOL tasks/conversations that may relate to the same topics
-- Query ALL relatable concepts, not just direct keywords or IDs
-- Search for similar past situations, patterns, and related work
-- Include synonyms, related terms, and contextual concepts in queries  
-- Query user's historical approach to similar requests or activities
-- Search for connected projects, tasks, conversations, and collaborations
-- Retrieve workflow patterns and past decision-making context
-- Query broader domain context beyond immediate request scope
-- Remember: SOL tracks work that external tools don't - search internal content thoroughly
-- Blend memory insights naturally into responses
-- Verify you've checked relevant memory before finalizing ANY response
-
-</memory>
-
-<external_services>
-- To use: load_mcp with EXACT integration name from the available list
-- Can load multiple at once with an array
-- Only load when tools are NOT already available in your current toolset
-- If a tool is already available, use it directly without load_mcp
-- If requested integration unavailable: inform user politely
-</external_services>
-
-<tool_calling>
-You have tools at your disposal to assist users:
-
-CORE PRINCIPLES:
-- Use tools only when necessary for the task at hand
-- Always check memory FIRST before making other tool calls
-- Execute multiple operations in parallel whenever possible
-- Use sequential calls only when output of one is required for input of another
-
-PARAMETER HANDLING:
-- Follow tool schemas exactly with all required parameters
-- Only use values that are:
-  • Explicitly provided by the user (use EXACTLY as given)
-  • Reasonably inferred from context
-  • Retrieved from memory or prior tool calls
-- Never make up values for required parameters
-- Omit optional parameters unless clearly needed
-- Analyze user's descriptive terms for parameter clues
-
-TOOL SELECTION:
-- Never call tools not provided in this conversation
-- Skip tool calls for general questions you can answer directly from memory/knowledge
-- For identical operations on multiple items, use parallel tool calls
-- Default to parallel execution (3-5× faster than sequential calls)
-- You can always access external service tools by loading them with load_mcp first
-
-TOOL MENTION HANDLING:
-When user message contains <mention data-id="tool_name" data-label="tool"></mention>:
-- Extract tool_name from data-id attribute
-- First check if it's a built-in tool; if not, check EXTERNAL SERVICES TOOLS
-- If available: Load it with load_mcp and focus on addressing the request with this tool
-- If unavailable: Inform user and suggest alternatives if possible
-- For multiple tool mentions: Load all applicable tools in a single load_mcp call
-
-ERROR HANDLING:
-- If a tool returns an error, try fixing parameters before retrying
-- If you can't resolve an error, explain the issue to the user
-- Consider alternative tools when primary tools are unavailable
-</tool_calling>
-
-<communication>
-Use EXACTLY ONE of these formats for all user-facing communication:
-
-PROGRESS UPDATES - During processing:
-- Use the core--progress_update tool to keep users informed
-- Update users about what you're discovering or doing next
-- Keep messages clear and user-friendly
-- Avoid technical jargon
-
-QUESTIONS - When you need information:
-<question_response>
-<p>[Your question with HTML formatting]</p>
-</question_response>
-
-- Ask questions only when you cannot find information through memory, or tools
-- Be specific about what you need to know
-- Provide context for why you're asking
-
-FINAL ANSWERS - When completing tasks:
-<final_response>
-<p>[Your answer with HTML formatting]</p>
-</final_response>
-
-CRITICAL:
-- Use ONE format per turn
-- Apply proper HTML formatting (<h1>, <h2>, <p>, <ul>, <li>, etc.)
-- Never mix communication formats
-- Keep responses clear and helpful
-- Always indicate your information sources (memory, and/or knowledge)
-</communication>
-`;
-
-export const REACT_USER_PROMPT = `
-Here is the user message:
-<user_message>
-{{USER_MESSAGE}}
-</user_message>
-`;

apps/webapp/app/trigger/chat/stream-utils.ts

@@ -1,294 +0,0 @@
-import fs from "fs";
-import path from "node:path";
-
-import { anthropic } from "@ai-sdk/anthropic";
-import { google } from "@ai-sdk/google";
-import { openai } from "@ai-sdk/openai";
-import { logger } from "@trigger.dev/sdk/v3";
-import {
-  type CoreMessage,
-  type LanguageModelV1,
-  streamText,
-  type ToolSet,
-} from "ai";
-import { createOllama } from "ollama-ai-provider";
-
-import { type AgentMessageType, Message } from "./types";
-
-interface State {
-  inTag: boolean;
-  messageEnded: boolean;
-  message: string;
-  lastSent: string;
-}
-
-export interface ExecutionState {
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  agentFlow: any;
-  userMessage: string;
-  message: string;
-}
-
-export async function* processTag(
-  state: State,
-  totalMessage: string,
-  chunk: string,
-  startTag: string,
-  endTag: string,
-  states: { start: string; chunk: string; end: string },
-  extraParams: Record<string, string> = {},
-) {
-  let comingFromStart = false;
-
-  if (!state.messageEnded) {
-    if (!state.inTag) {
-      const startIndex = totalMessage.indexOf(startTag);
-      if (startIndex !== -1) {
-        state.inTag = true;
-        // Send MESSAGE_START when we first enter the tag
-        yield Message("", states.start as AgentMessageType, extraParams);
-        const chunkToSend = totalMessage.slice(startIndex + startTag.length);
-        state.message += chunkToSend;
-        comingFromStart = true;
-      }
-    }
-
-    if (state.inTag) {
-      // Check if chunk contains end tag
-      const hasEndTag = chunk.includes(endTag);
-      const hasStartTag = chunk.includes(startTag);
-      const hasClosingTag = chunk.includes("</");
-
-      // Check if we're currently accumulating a potential end tag
-      const accumulatingEndTag = state.message.endsWith("</") ||
-                                  state.message.match(/<\/[a-z_]*$/i);
-
-      if (hasClosingTag && !hasStartTag && !hasEndTag) {
-        // If chunk only has </ but not the full end tag, accumulate it
-        state.message += chunk;
-      } else if (accumulatingEndTag) {
-        // Continue accumulating if we're in the middle of a potential end tag
-        state.message += chunk;
-        // Check if we now have the complete end tag
-        if (state.message.includes(endTag)) {
-          // Process the complete message with end tag
-          const endIndex = state.message.indexOf(endTag);
-          const finalMessage = state.message.slice(0, endIndex).trim();
-          const messageToSend = finalMessage.slice(
-            finalMessage.indexOf(state.lastSent) + state.lastSent.length,
-          );
-
-          if (messageToSend) {
-            yield Message(
-              messageToSend,
-              states.chunk as AgentMessageType,
-              extraParams,
-            );
-          }
-          yield Message("", states.end as AgentMessageType, extraParams);
-
-          state.message = finalMessage;
-          state.messageEnded = true;
-        }
-      } else if (hasEndTag || (!hasEndTag && !hasClosingTag)) {
-        let currentMessage = comingFromStart
-          ? state.message
-          : state.message + chunk;
-
-        const endIndex = currentMessage.indexOf(endTag);
-
-        if (endIndex !== -1) {
-          // For the final chunk before the end tag
-          currentMessage = currentMessage.slice(0, endIndex).trim();
-          const messageToSend = currentMessage.slice(
-            currentMessage.indexOf(state.lastSent) + state.lastSent.length,
-          );
-
-          if (messageToSend) {
-            yield Message(
-              messageToSend,
-              states.chunk as AgentMessageType,
-              extraParams,
-            );
-          }
-          // Send MESSAGE_END when we reach the end tag
-          yield Message("", states.end as AgentMessageType, extraParams);
-
-          state.message = currentMessage;
-          state.messageEnded = true;
-        } else {
-          const diff = currentMessage.slice(
-            currentMessage.indexOf(state.lastSent) + state.lastSent.length,
-          );
-
-          // For chunks in between start and end
-          const messageToSend = comingFromStart ? state.message : diff;
-
-          if (messageToSend) {
-            state.lastSent = messageToSend;
-            yield Message(
-              messageToSend,
-              states.chunk as AgentMessageType,
-              extraParams,
-            );
-          }
-        }
-
-        state.message = currentMessage;
-        state.lastSent = state.message;
-      } else {
-        state.message += chunk;
-      }
-    }
-  }
-}
-
-export async function* generate(
-  messages: CoreMessage[],
-  isProgressUpdate: boolean = false,
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  onFinish?: (event: any) => void,
-  tools?: ToolSet,
-  system?: string,
-  model?: string,
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-): AsyncGenerator<
-  | string
-  | {
-      type: string;
-      toolName: string;
-      // eslint-disable-next-line @typescript-eslint/no-explicit-any
-      args?: any;
-      toolCallId?: string;
-      message?: string;
-    }
-> {
-  // Check for API keys
-  const anthropicKey = process.env.ANTHROPIC_API_KEY;
-  const googleKey = process.env.GOOGLE_GENERATIVE_AI_API_KEY;
-  const openaiKey = process.env.OPENAI_API_KEY;
-  let ollamaUrl = process.env.OLLAMA_URL;
-  model = model || process.env.MODEL;
-
-  let modelInstance;
-  let modelTemperature = Number(process.env.MODEL_TEMPERATURE) || 1;
-  ollamaUrl = undefined;
-
-  // First check if Ollama URL exists and use Ollama
-  if (ollamaUrl) {
-    const ollama = createOllama({
-      baseURL: ollamaUrl,
-    });
-    modelInstance = ollama(model || "llama2"); // Default to llama2 if no model specified
-  } else {
-    // If no Ollama, check other models
-    switch (model) {
-      case "claude-3-7-sonnet-20250219":
-      case "claude-3-opus-20240229":
-      case "claude-3-5-haiku-20241022":
-        if (!anthropicKey) {
-          throw new Error("No Anthropic API key found. Set ANTHROPIC_API_KEY");
-        }
-        modelInstance = anthropic(model);
-        modelTemperature = 0.5;
-        break;
-
-      case "gemini-2.5-flash-preview-04-17":
-      case "gemini-2.5-pro-preview-03-25":
-      case "gemini-2.0-flash":
-      case "gemini-2.0-flash-lite":
-        if (!googleKey) {
-          throw new Error("No Google API key found. Set GOOGLE_API_KEY");
-        }
-        modelInstance = google(model);
-        break;
-
-      case "gpt-4.1-2025-04-14":
-      case "gpt-4.1-mini-2025-04-14":
-      case "gpt-5-mini-2025-08-07":
-      case "gpt-5-2025-08-07":
-      case "gpt-4.1-nano-2025-04-14":
-        if (!openaiKey) {
-          throw new Error("No OpenAI API key found. Set OPENAI_API_KEY");
-        }
-        modelInstance = openai(model);
-        break;
-
-      default:
-        break;
-    }
-  }
-
-  logger.info("starting stream");
-  // Try Anthropic next if key exists
-  if (modelInstance) {
-    try {
-      const { textStream, fullStream } = streamText({
-        model: modelInstance as LanguageModelV1,
-        messages,
-        temperature: modelTemperature,
-        maxSteps: 10,
-        tools,
-        ...(isProgressUpdate
-          ? { toolChoice: { type: "tool", toolName: "core--progress_update" } }
-          : {}),
-        toolCallStreaming: true,
-        onFinish,
-        ...(system ? { system } : {}),
-      });
-
-      for await (const chunk of textStream) {
-        yield chunk;
-      }
-
-      for await (const fullChunk of fullStream) {
-        if (fullChunk.type === "tool-call") {
-          yield {
-            type: "tool-call",
-            toolName: fullChunk.toolName,
-            toolCallId: fullChunk.toolCallId,
-            args: fullChunk.args,
-          };
-        }
-
-        if (fullChunk.type === "error") {
-          // Log the error to a file
-          const errorLogsDir = path.join(__dirname, "../../../../logs/errors");
-
-          // Ensure the directory exists
-          try {
-            if (!fs.existsSync(errorLogsDir)) {
-              fs.mkdirSync(errorLogsDir, { recursive: true });
-            }
-
-            // Create a timestamped error log file
-            const timestamp = new Date().toISOString().replace(/:/g, "-");
-            const errorLogPath = path.join(
-              errorLogsDir,
-              `llm-error-${timestamp}.json`,
-            );
-
-            // Write the error to the file
-            fs.writeFileSync(
-              errorLogPath,
-              JSON.stringify({
-                timestamp: new Date().toISOString(),
-                error: fullChunk.error,
-              }),
-            );
-
-            logger.error(`LLM error logged to ${errorLogPath}`);
-          } catch (err) {
-            logger.error(`Failed to log LLM error: ${err}`);
-          }
-        }
-      }
-      return;
-    } catch (e) {
-      console.log(e);
-      logger.error(e as string);
-    }
-  }
-
-  throw new Error("No valid LLM configuration found");
-}

apps/webapp/app/trigger/chat/types.ts

@@ -1,46 +0,0 @@
-export interface AgentStep {
-  agent: string;
-  goal: string;
-  reasoning: string;
-}
-
-export enum AgentMessageType {
-  STREAM_START = 'STREAM_START',
-  STREAM_END = 'STREAM_END',
-
-  // Used in ReACT based prompting
-  THOUGHT_START = 'THOUGHT_START',
-  THOUGHT_CHUNK = 'THOUGHT_CHUNK',
-  THOUGHT_END = 'THOUGHT_END',
-
-  // Message types
-  MESSAGE_START = 'MESSAGE_START',
-  MESSAGE_CHUNK = 'MESSAGE_CHUNK',
-  MESSAGE_END = 'MESSAGE_END',
-
-  // This is used to return action input
-  SKILL_START = 'SKILL_START',
-  SKILL_CHUNK = 'SKILL_CHUNK',
-  SKILL_END = 'SKILL_END',
-
-  STEP = 'STEP',
-  ERROR = 'ERROR',
-}
-
-export interface AgentMessage {
-  message?: string;
-  type: AgentMessageType;
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  metadata: Record<string, any>;
-}
-
-export const Message = (
-  message: string,
-  type: AgentMessageType,
-  extraParams: Record<string, string> = {},
-): AgentMessage => {
-  // For all message types, we use the message field
-  // The type field differentiates how the message should be interpreted
-  // For STEP and SKILL types, the message can contain JSON data as a string
-  return { message, type, metadata: extraParams };
-};

apps/webapp/app/trigger/cluster/index.ts

@@ -1,115 +0,0 @@
-import { queue, task } from "@trigger.dev/sdk";
-import { z } from "zod";
-import { ClusteringService } from "~/services/clustering.server";
-import { logger } from "~/services/logger.service";
-
-const clusteringService = new ClusteringService();
-
-// Define the payload schema for cluster tasks
-export const ClusterPayload = z.object({
-  userId: z.string(),
-  mode: z.enum(["auto", "incremental", "complete", "drift"]).default("auto"),
-  forceComplete: z.boolean().default(false),
-});
-
-const clusterQueue = queue({
-  name: "cluster-queue",
-  concurrencyLimit: 10,
-});
-
-/**
- * Single clustering task that handles all clustering operations based on payload mode
- */
-export const clusterTask = task({
-  id: "cluster",
-  queue: clusterQueue,
-  maxDuration: 1800, // 30 minutes max
-  run: async (payload: z.infer<typeof ClusterPayload>) => {
-    logger.info(`Starting ${payload.mode} clustering task for user ${payload.userId}`);
-
-    try {
-      let result;
-      
-      switch (payload.mode) {
-        case "incremental":
-          result = await clusteringService.performIncrementalClustering(
-            payload.userId,
-          );
-          logger.info(`Incremental clustering completed for user ${payload.userId}:`, {
-            newStatementsProcessed: result.newStatementsProcessed,
-            newClustersCreated: result.newClustersCreated,
-          });
-          break;
-
-        case "complete":
-          result = await clusteringService.performCompleteClustering(
-            payload.userId,
-          );
-          logger.info(`Complete clustering completed for user ${payload.userId}:`, {
-            clustersCreated: result.clustersCreated,
-            statementsProcessed: result.statementsProcessed,
-          });
-          break;
-
-        case "drift":
-          // First detect drift
-          const driftMetrics = await clusteringService.detectClusterDrift(
-            payload.userId,
-          );
-
-          if (driftMetrics.driftDetected) {
-            // Handle drift by splitting low-cohesion clusters
-            const driftResult = await clusteringService.handleClusterDrift(
-              payload.userId,
-            );
-
-            logger.info(`Cluster drift handling completed for user ${payload.userId}:`, {
-              driftDetected: true,
-              clustersProcessed: driftResult.clustersProcessed,
-              newClustersCreated: driftResult.newClustersCreated,
-              splitClusters: driftResult.splitClusters,
-            });
-
-            result = {
-              driftDetected: true,
-              ...driftResult,
-              driftMetrics,
-            };
-          } else {
-            logger.info(`No cluster drift detected for user ${payload.userId}`);
-            result = {
-              driftDetected: false,
-              clustersProcessed: 0,
-              newClustersCreated: 0,
-              splitClusters: [],
-              driftMetrics,
-            };
-          }
-          break;
-
-        case "auto":
-        default:
-          result = await clusteringService.performClustering(
-            payload.userId,
-            payload.forceComplete,
-          );
-          logger.info(`Auto clustering completed for user ${payload.userId}:`, {
-            clustersCreated: result.clustersCreated,
-            statementsProcessed: result.statementsProcessed,
-            approach: result.approach,
-          });
-          break;
-      }
-
-      return {
-        success: true,
-        data: result,
-      };
-    } catch (error) {
-      logger.error(`${payload.mode} clustering failed for user ${payload.userId}:`, {
-        error,
-      });
-      throw error;
-    }
-  },
-});

apps/webapp/app/trigger/conversation/create-conversation-title.ts

@@ -1,61 +1,12 @@
-import { LLMMappings } from "@core/types";
-import { logger, task } from "@trigger.dev/sdk/v3";
-import { generate } from "../chat/stream-utils";
-import { conversationTitlePrompt } from "./prompt";
-import { prisma } from "../utils/prisma";
+import { task } from "@trigger.dev/sdk/v3";
+import {
+  processConversationTitleCreation,
+  type CreateConversationTitlePayload,
+} from "~/jobs/conversation/create-title.logic";
 
 export const createConversationTitle = task({
   id: "create-conversation-title",
-  run: async (payload: { conversationId: string; message: string }) => {
-    let conversationTitleResponse = "";
-    const gen = generate(
-      [
-        {
-          role: "user",
-          content: conversationTitlePrompt.replace(
-            "{{message}}",
-            payload.message,
-          ),
-        },
-      ],
-      false,
-      () => {},
-      undefined,
-      "",
-      LLMMappings.GPT41,
-    );
-
-    for await (const chunk of gen) {
-      if (typeof chunk === "string") {
-        conversationTitleResponse += chunk;
-      } else if (chunk && typeof chunk === "object" && chunk.message) {
-        conversationTitleResponse += chunk.message;
-      }
-    }
-
-    const outputMatch = conversationTitleResponse.match(
-      /<output>(.*?)<\/output>/s,
-    );
-
-    logger.info(`Conversation title data: ${JSON.stringify(outputMatch)}`);
-
-    if (!outputMatch) {
-      logger.error("No output found in recurrence response");
-      throw new Error("Invalid response format from AI");
-    }
-
-    const jsonStr = outputMatch[1].trim();
-    const conversationTitleData = JSON.parse(jsonStr);
-
-    if (conversationTitleData) {
-      await prisma.conversation.update({
-        where: {
-          id: payload.conversationId,
-        },
-        data: {
-          title: conversationTitleData.title,
-        },
-      });
-    }
+  run: async (payload: CreateConversationTitlePayload) => {
+    return await processConversationTitleCreation(payload);
   },
 });

apps/webapp/app/trigger/deep-search/deep-search-utils.ts

@@ -1,292 +0,0 @@
-import { type CoreMessage } from "ai";
-import { logger } from "@trigger.dev/sdk/v3";
-import { generate } from "./stream-utils";
-import { processTag } from "../chat/stream-utils";
-import { type AgentMessage, AgentMessageType, Message } from "../chat/types";
-import { type TotalCost } from "../utils/types";
-
-/**
- * Run the deep search ReAct loop
- * Async generator that yields AgentMessage objects for streaming
- * Follows the exact same pattern as chat-utils.ts
- */
-export async function* run(
-  initialMessages: CoreMessage[],
-  searchTool: any,
-): AsyncGenerator<AgentMessage, any, any> {
-  let messages = [...initialMessages];
-  let completed = false;
-  let guardLoop = 0;
-  let searchCount = 0;
-  let totalEpisodesFound = 0;
-  const seenEpisodeIds = new Set<string>(); // Track unique episodes
-  const totalCost: TotalCost = {
-    inputTokens: 0,
-    outputTokens: 0,
-    cost: 0,
-  };
-
-  const tools = {
-    searchMemory: searchTool,
-  };
-
-  logger.info("Starting deep search ReAct loop");
-
-  try {
-    while (!completed && guardLoop < 50) {
-      logger.info(
-        `ReAct loop iteration ${guardLoop}, searches: ${searchCount}`,
-      );
-
-      // Call LLM with current message history
-      const response = generate(
-        messages,
-        (event) => {
-          const usage = event.usage;
-          totalCost.inputTokens += usage.promptTokens;
-          totalCost.outputTokens += usage.completionTokens;
-        },
-        tools,
-      );
-
-      let totalMessage = "";
-      const toolCalls: any[] = [];
-
-      // States for streaming final_response tags
-      const messageState = {
-        inTag: false,
-        message: "",
-        messageEnded: false,
-        lastSent: "",
-      };
-
-      // Process streaming response
-      for await (const chunk of response) {
-        if (typeof chunk === "object" && chunk.type === "tool-call") {
-          // Agent made a tool call
-          toolCalls.push(chunk);
-          logger.info(`Tool call: ${chunk.toolName}`);
-        } else if (typeof chunk === "string") {
-          totalMessage += chunk;
-
-          // Stream final_response tags using processTag
-          if (!messageState.messageEnded) {
-            yield* processTag(
-              messageState,
-              totalMessage,
-              chunk,
-              "<final_response>",
-              "</final_response>",
-              {
-                start: AgentMessageType.MESSAGE_START,
-                chunk: AgentMessageType.MESSAGE_CHUNK,
-                end: AgentMessageType.MESSAGE_END,
-              },
-            );
-          }
-        }
-      }
-
-      // Check for final response
-      if (totalMessage.includes("<final_response>")) {
-        const match = totalMessage.match(
-          /<final_response>(.*?)<\/final_response>/s,
-        );
-
-        if (match) {
-          // Accept synthesis - completed
-          completed = true;
-          logger.info(
-            `Final synthesis accepted after ${searchCount} searches, ${totalEpisodesFound} unique episodes found`,
-          );
-          break;
-        }
-      }
-
-      // Execute tool calls in parallel for better performance
-      if (toolCalls.length > 0) {
-        // Notify about all searches starting
-        for (const toolCall of toolCalls) {
-          logger.info(`Executing search: ${JSON.stringify(toolCall.args)}`);
-          yield Message("", AgentMessageType.SKILL_START);
-          yield Message(
-            `\nSearching memory: "${toolCall.args.query}"...\n`,
-            AgentMessageType.SKILL_CHUNK,
-          );
-          yield Message("", AgentMessageType.SKILL_END);
-        }
-
-        // Execute all searches in parallel
-        const searchPromises = toolCalls.map((toolCall) =>
-          searchTool.execute(toolCall.args).then((result: any) => ({
-            toolCall,
-            result,
-          })),
-        );
-
-        const searchResults = await Promise.all(searchPromises);
-
-        // Process results and add to message history
-        for (const { toolCall, result } of searchResults) {
-          searchCount++;
-
-          // Deduplicate episodes - track unique IDs
-          let uniqueNewEpisodes = 0;
-          if (result.episodes && Array.isArray(result.episodes)) {
-            for (const episode of result.episodes) {
-              const episodeId =
-                episode.id || episode._id || JSON.stringify(episode);
-              if (!seenEpisodeIds.has(episodeId)) {
-                seenEpisodeIds.add(episodeId);
-                uniqueNewEpisodes++;
-              }
-            }
-          }
-
-          const episodesInThisSearch = result.episodes?.length || 0;
-          totalEpisodesFound = seenEpisodeIds.size; // Use unique count
-
-          messages.push({
-            role: "assistant",
-            content: [
-              {
-                type: "tool-call",
-                toolCallId: toolCall.toolCallId,
-                toolName: toolCall.toolName,
-                args: toolCall.args,
-              },
-            ],
-          });
-
-          // Add tool result to message history
-          messages.push({
-            role: "tool",
-            content: [
-              {
-                type: "tool-result",
-                toolName: toolCall.toolName,
-                toolCallId: toolCall.toolCallId,
-                result: result,
-              },
-            ],
-          });
-
-          logger.info(
-            `Search ${searchCount} completed: ${episodesInThisSearch} episodes (${uniqueNewEpisodes} new, ${totalEpisodesFound} unique total)`,
-          );
-        }
-
-        // If found no episodes and haven't exhausted search attempts, require more searches
-        if (totalEpisodesFound === 0 && searchCount < 7) {
-          logger.info(
-            `Agent attempted synthesis with 0 unique episodes after ${searchCount} searches - requiring more attempts`,
-          );
-
-          yield Message("", AgentMessageType.SKILL_START);
-          yield Message(
-            `No relevant context found yet - trying different search angles...`,
-            AgentMessageType.SKILL_CHUNK,
-          );
-          yield Message("", AgentMessageType.SKILL_END);
-
-          messages.push({
-            role: "system",
-            content: `You have performed ${searchCount} searches but found 0 unique relevant episodes. Your queries may be too abstract or not matching the user's actual conversation topics.
-
-Review your DECOMPOSITION:
-- Are you using specific terms from the content?
-- Try searching broader related topics the user might have discussed
-- Try different terminology or related concepts
-- Search for user's projects, work areas, or interests
-
-Continue with different search strategies (you can search up to 7-10 times total).`,
-          });
-
-          guardLoop++;
-          continue;
-        }
-
-        // Soft nudging after all searches executed (awareness, not commands)
-        if (totalEpisodesFound >= 30 && searchCount >= 3) {
-          logger.info(
-            `Nudging: ${totalEpisodesFound} unique episodes found - suggesting synthesis consideration`,
-          );
-
-          messages.push({
-            role: "system",
-            content: `Context awareness: You have found ${totalEpisodesFound} unique episodes across ${searchCount} searches. This represents substantial context. Consider whether you have sufficient information for quality synthesis, or if additional search angles would meaningfully improve understanding.`,
-          });
-        } else if (totalEpisodesFound >= 15 && searchCount >= 5) {
-          logger.info(
-            `Nudging: ${totalEpisodesFound} unique episodes after ${searchCount} searches - suggesting evaluation`,
-          );
-
-          messages.push({
-            role: "system",
-            content: `Progress update: You have ${totalEpisodesFound} unique episodes from ${searchCount} searches. Evaluate whether you have covered the main angles from your decomposition, or if important aspects remain unexplored.`,
-          });
-        } else if (searchCount >= 7) {
-          logger.info(
-            `Nudging: ${searchCount} searches completed with ${totalEpisodesFound} unique episodes`,
-          );
-
-          messages.push({
-            role: "system",
-            content: `Search depth: You have performed ${searchCount} searches and found ${totalEpisodesFound} unique episodes. Consider whether additional searches would yield meaningfully different context, or if it's time to synthesize what you've discovered.`,
-          });
-        }
-        if (searchCount >= 10) {
-          logger.info(
-            `Reached maximum search limit (10), forcing synthesis with ${totalEpisodesFound} unique episodes`,
-          );
-
-          yield Message("", AgentMessageType.SKILL_START);
-          yield Message(
-            `Maximum searches reached - synthesizing results...`,
-            AgentMessageType.SKILL_CHUNK,
-          );
-          yield Message("", AgentMessageType.SKILL_END);
-
-          messages.push({
-            role: "system",
-            content: `You have performed 10 searches and found ${totalEpisodesFound} unique episodes. This is the maximum allowed. You MUST now provide your final synthesis wrapped in <final_response> tags based on what you've found.`,
-          });
-        }
-      }
-
-      // Safety check - if no tool calls and no final response, something went wrong
-      if (
-        toolCalls.length === 0 &&
-        !totalMessage.includes("<final_response>")
-      ) {
-        logger.warn("Agent produced neither tool calls nor final response");
-
-        messages.push({
-          role: "system",
-          content:
-            "You must either use the searchMemory tool to search for more context, or provide your final synthesis wrapped in <final_response> tags.",
-        });
-      }
-
-      guardLoop++;
-    }
-
-    if (!completed) {
-      logger.warn(
-        `Loop ended without completion after ${guardLoop} iterations`,
-      );
-      yield Message("", AgentMessageType.MESSAGE_START);
-      yield Message(
-        "Deep search did not complete - maximum iterations reached.",
-        AgentMessageType.MESSAGE_CHUNK,
-      );
-      yield Message("", AgentMessageType.MESSAGE_END);
-    }
-
-    yield Message("Stream ended", AgentMessageType.STREAM_END);
-  } catch (error) {
-    logger.error(`Deep search error: ${error}`);
-    yield Message((error as Error).message, AgentMessageType.ERROR);
-    yield Message("Stream ended", AgentMessageType.STREAM_END);
-  }
-}

apps/webapp/app/trigger/deep-search/index.ts

@@ -1,85 +0,0 @@
-import { metadata, task } from "@trigger.dev/sdk";
-import { type CoreMessage } from "ai";
-import { logger } from "@trigger.dev/sdk/v3";
-import { nanoid } from "nanoid";
-import {
-  deletePersonalAccessToken,
-  getOrCreatePersonalAccessToken,
-} from "../utils/utils";
-import { getReActPrompt } from "./prompt";
-import { type DeepSearchPayload, type DeepSearchResponse } from "./types";
-import { createSearchMemoryTool } from "./utils";
-import { run } from "./deep-search-utils";
-import { AgentMessageType } from "../chat/types";
-
-export const deepSearch = task({
-  id: "deep-search",
-  maxDuration: 3000,
-  run: async (payload: DeepSearchPayload): Promise<DeepSearchResponse> => {
-    const { content, userId, stream, metadata: meta, intentOverride } = payload;
-
-    const randomKeyName = `deepSearch_${nanoid(10)}`;
-
-    // Get or create token for search API calls
-    const pat = await getOrCreatePersonalAccessToken({
-      name: randomKeyName,
-      userId: userId as string,
-    });
-
-    if (!pat?.token) {
-      throw new Error("Failed to create personal access token");
-    }
-
-    try {
-      // Create search tool that agent will use
-      const searchTool = createSearchMemoryTool(pat.token);
-
-      // Build initial messages with ReAct prompt
-      const initialMessages: CoreMessage[] = [
-        {
-          role: "system",
-          content: getReActPrompt(meta, intentOverride),
-        },
-        {
-          role: "user",
-          content: `CONTENT TO ANALYZE:\n${content}\n\nPlease search my memory for relevant context and synthesize what you find.`,
-        },
-      ];
-
-      // Run the ReAct loop generator
-      const llmResponse = run(initialMessages, searchTool);
-
-      // Streaming mode: stream via metadata.stream like chat.ts does
-      // This makes all message types available to clients in real-time
-      const messageStream = await metadata.stream("messages", llmResponse);
-
-      let synthesis = "";
-
-      for await (const step of messageStream) {
-        // MESSAGE_CHUNK: Final synthesis - accumulate and stream
-        if (step.type === AgentMessageType.MESSAGE_CHUNK) {
-          synthesis += step.message;
-        }
-
-        // STREAM_END: Loop completed
-        if (step.type === AgentMessageType.STREAM_END) {
-          break;
-        }
-      }
-
-      await deletePersonalAccessToken(pat?.id);
-
-      // Clean up any remaining tags
-      synthesis = synthesis
-        .replace(/<final_response>/gi, "")
-        .replace(/<\/final_response>/gi, "")
-        .trim();
-
-      return { synthesis };
-    } catch (error) {
-      await deletePersonalAccessToken(pat?.id);
-      logger.error(`Deep search error: ${error}`);
-      throw error;
-    }
-  },
-});

apps/webapp/app/trigger/deep-search/prompt.ts

@@ -1,148 +0,0 @@
-export function getReActPrompt(
-  metadata?: { source?: string; url?: string; pageTitle?: string },
-  intentOverride?: string
-): string {
-  const contextHints = [];
-
-  if (metadata?.source === "chrome" && metadata?.url?.includes("mail.google.com")) {
-    contextHints.push("Content is from email - likely reading intent");
-  }
-  if (metadata?.source === "chrome" && metadata?.url?.includes("calendar.google.com")) {
-    contextHints.push("Content is from calendar - likely meeting prep intent");
-  }
-  if (metadata?.source === "chrome" && metadata?.url?.includes("docs.google.com")) {
-    contextHints.push("Content is from document editor - likely writing intent");
-  }
-  if (metadata?.source === "obsidian") {
-    contextHints.push("Content is from note editor - likely writing or research intent");
-  }
-
-  return `You are a memory research agent analyzing content to find relevant context.
-
-YOUR PROCESS (ReAct Framework):
-
-1. DECOMPOSE: First, break down the content into structured categories
-
-   Analyze the content and extract:
-   a) ENTITIES: Specific people, project names, tools, products mentioned
-      Example: "John Smith", "Phoenix API", "Redis", "mobile app"
-
-   b) TOPICS & CONCEPTS: Key subjects, themes, domains
-      Example: "authentication", "database design", "performance optimization"
-
-   c) TEMPORAL MARKERS: Time references, deadlines, events
-      Example: "last week's meeting", "Q2 launch", "yesterday's discussion"
-
-   d) ACTIONS & TASKS: What's being done, decided, or requested
-      Example: "implement feature", "review code", "make decision on"
-
-   e) USER INTENT: What is the user trying to accomplish?
-      ${intentOverride ? `User specified: "${intentOverride}"` : "Infer from context: reading/writing/meeting prep/research/task tracking/review"}
-
-2. FORM QUERIES: Create targeted search queries from your decomposition
-
-   Based on decomposition, form specific queries:
-   - Search for each entity by name (people, projects, tools)
-   - Search for topics the user has discussed before
-   - Search for related work or conversations in this domain
-   - Use the user's actual terminology, not generic concepts
-
-   EXAMPLE - Content: "Email from Sarah about the API redesign we discussed last week"
-   Decomposition:
-     - Entities: "Sarah", "API redesign"
-     - Topics: "API design", "redesign"
-     - Temporal: "last week"
-     - Actions: "discussed", "email communication"
-     - Intent: Reading (email) / meeting prep
-
-   Queries to form:
-   ✅ "Sarah" (find past conversations with Sarah)
-   ✅ "API redesign" or "API design" (find project discussions)
-   ✅ "last week" + "Sarah" (find recent context)
-   ✅ "meetings" or "discussions" (find related conversations)
-
-   ❌ Avoid: "email communication patterns", "API architecture philosophy"
-   (These are abstract - search what user actually discussed!)
-
-3. SEARCH: Execute your queries using searchMemory tool
-   - Start with 2-3 core searches based on main entities/topics
-   - Make each search specific and targeted
-   - Use actual terms from the content, not rephrased concepts
-
-4. OBSERVE: Evaluate search results
-   - Did you find relevant episodes? How many unique ones?
-   - What specific context emerged?
-   - What new entities/topics appeared in results?
-   - Are there gaps in understanding?
-   - Should you search more angles?
-
-   Note: Episode counts are automatically deduplicated across searches - overlapping episodes are only counted once.
-
-5. REACT: Decide next action based on observations
-
-   STOPPING CRITERIA - Proceed to SYNTHESIZE if ANY of these are true:
-   - You found 20+ unique episodes across your searches → ENOUGH CONTEXT
-   - You performed 5+ searches and found relevant episodes → SUFFICIENT
-   - You performed 7+ searches regardless of results → EXHAUSTED STRATEGIES
-   - You found strong relevant context from multiple angles → COMPLETE
-
-   System nudges will provide awareness of your progress, but you decide when synthesis quality would be optimal.
-
-   If you found little/no context AND searched less than 7 times:
-   - Try different query angles from your decomposition
-   - Search broader related topics
-   - Search user's projects or work areas
-   - Try alternative terminology
-
-   ⚠️ DO NOT search endlessly - if you found relevant episodes, STOP and synthesize!
-
-6. SYNTHESIZE: After gathering sufficient context, provide final answer
-   - Wrap your synthesis in <final_response> tags
-   - Present direct factual context from memory - no meta-commentary
-   - Write as if providing background context to an AI assistant
-   - Include: facts, decisions, preferences, patterns, timelines
-   - Note any gaps, contradictions, or evolution in thinking
-   - Keep it concise and actionable
-   - DO NOT use phrases like "Previous discussions on", "From conversations", "Past preferences indicate"
-   - DO NOT use conversational language like "you said" or "you mentioned"
-   - Present information as direct factual statements
-
-FINAL RESPONSE FORMAT:
-<final_response>
-[Direct synthesized context - factual statements only]
-
-Good examples:
-- "The API redesign focuses on performance and scalability. Key decisions: moving to GraphQL, caching layer with Redis."
-- "Project Phoenix launches Q2 2024. Main features: real-time sync, offline mode, collaborative editing."
-- "Sarah leads the backend team. Recent work includes authentication refactor and database migration."
-
-Bad examples:
-❌ "Previous discussions on the API revealed..."
-❌ "From past conversations, it appears that..."
-❌ "Past preferences indicate..."
-❌ "The user mentioned that..."
-
-Just state the facts directly.
-</final_response>
-
-${contextHints.length > 0 ? `\nCONTEXT HINTS:\n${contextHints.join("\n")}` : ""}
-
-CRITICAL REQUIREMENTS:
-- ALWAYS start with DECOMPOSE step - extract entities, topics, temporal markers, actions
-- Form specific queries from your decomposition - use user's actual terms
-- Minimum 3 searches required
-- Maximum 10 searches allowed - must synthesize after that
-- STOP and synthesize when you hit stopping criteria (20+ episodes, 5+ searches with results, 7+ searches total)
-- Each search should target different aspects from decomposition
-- Present synthesis directly without meta-commentary
-
-SEARCH QUALITY CHECKLIST:
-✅ Queries use specific terms from content (names, projects, exact phrases)
-✅ Searched multiple angles from decomposition (entities, topics, related areas)
-✅ Stop when you have enough unique context - don't search endlessly
-✅ Tried alternative terminology if initial searches found nothing
-❌ Avoid generic/abstract queries that don't match user's vocabulary
-❌ Don't stop at 3 searches if you found zero unique episodes
-❌ Don't keep searching when you already found 20+ unique episodes
-}`
-}

apps/webapp/app/trigger/deep-search/stream-utils.ts

@@ -1,68 +0,0 @@
-import { openai } from "@ai-sdk/openai";
-import { logger } from "@trigger.dev/sdk/v3";
-import {
-  type CoreMessage,
-  type LanguageModelV1,
-  streamText,
-  type ToolSet,
-} from "ai";
-
-/**
- * Generate LLM responses with tool calling support
- * Simplified version for deep-search use case - NO maxSteps for manual ReAct control
- */
-export async function* generate(
-  messages: CoreMessage[],
-  onFinish?: (event: any) => void,
-  tools?: ToolSet,
-  model?: string,
-): AsyncGenerator<
-  | string
-  | {
-      type: string;
-      toolName: string;
-      args?: any;
-      toolCallId?: string;
-    }
-> {
-  const modelToUse = model || process.env.MODEL || "gpt-4.1-2025-04-14";
-  const modelInstance = openai(modelToUse) as LanguageModelV1;
-
-  logger.info(`Starting LLM generation with model: ${modelToUse}`);
-
-  try {
-    const { textStream, fullStream } = streamText({
-      model: modelInstance,
-      messages,
-      temperature: 1,
-      tools,
-      // NO maxSteps - we handle tool execution manually in the ReAct loop
-      toolCallStreaming: true,
-      onFinish,
-    });
-
-    // Yield text chunks
-    for await (const chunk of textStream) {
-      yield chunk;
-    }
-
-    // Yield tool calls
-    for await (const fullChunk of fullStream) {
-      if (fullChunk.type === "tool-call") {
-        yield {
-          type: "tool-call",
-          toolName: fullChunk.toolName,
-          toolCallId: fullChunk.toolCallId,
-          args: fullChunk.args,
-        };
-      }
-
-      if (fullChunk.type === "error") {
-        logger.error(`LLM error: ${JSON.stringify(fullChunk)}`);
-      }
-    }
-  } catch (error) {
-    logger.error(`LLM generation error: ${error}`);
-    throw error;
-  }
-}

apps/webapp/app/trigger/deep-search/types.ts

@@ -1,20 +0,0 @@
-export interface DeepSearchPayload {
-  content: string;
-  userId: string;
-  stream: boolean;
-  intentOverride?: string;
-  metadata?: {
-    source?: "chrome" | "obsidian" | "mcp";
-    url?: string;
-    pageTitle?: string;
-  };
-}
-
-export interface DeepSearchResponse {
-  synthesis: string;
-  episodes?: Array<{
-    content: string;
-    createdAt: Date;
-    spaceIds: string[];
-  }>;
-}

apps/webapp/app/trigger/deep-search/utils.ts

@@ -1,64 +0,0 @@
-import { tool } from "ai";
-import { z } from "zod";
-import axios from "axios";
-import { logger } from "@trigger.dev/sdk/v3";
-
-export function createSearchMemoryTool(token: string) {
-  return tool({
-    description:
-      "Search the user's memory for relevant facts and episodes. Use this tool multiple times with different queries to gather comprehensive context.",
-    parameters: z.object({
-      query: z
-        .string()
-        .describe(
-          "Search query to find relevant information. Be specific: entity names, topics, concepts.",
-        ),
-    }),
-    execute: async ({ query }) => {
-      try {
-        const response = await axios.post(
-          `${process.env.API_BASE_URL || "https://core.heysol.ai"}/api/v1/search`,
-          { query },
-          {
-            headers: {
-              Authorization: `Bearer ${token}`,
-            },
-          },
-        );
-
-        const searchResult = response.data;
-
-        return {
-          facts: searchResult.facts || [],
-          episodes: searchResult.episodes || [],
-          summary: `Found ${searchResult.episodes?.length || 0} relevant memories`,
-        };
-      } catch (error) {
-        logger.error(`SearchMemory tool error: ${error}`);
-        return {
-          facts: [],
-          episodes: [],
-          summary: "No results found",
-        };
-      }
-    },
-  });
-}
-
-// Helper to extract unique episodes from tool calls
-export function extractEpisodesFromToolCalls(toolCalls: any[]): any[] {
-  const episodes: any[] = [];
-
-  for (const call of toolCalls || []) {
-    if (call.toolName === "searchMemory" && call.result?.episodes) {
-      episodes.push(...call.result.episodes);
-    }
-  }
-
-  // Deduplicate by content + createdAt
-  const uniqueEpisodes = Array.from(
-    new Map(episodes.map((e) => [`${e.content}-${e.createdAt}`, e])).values(),
-  );
-
-  return uniqueEpisodes.slice(0, 10);
-}

apps/webapp/app/trigger/ingest/ingest-document.ts

@@ -1,16 +1,8 @@
 import { queue, task } from "@trigger.dev/sdk";
-import { type z } from "zod";
-import crypto from "crypto";
-
-import { IngestionStatus } from "@core/database";
-import { EpisodeTypeEnum } from "@core/types";
-import { logger } from "~/services/logger.service";
-import { saveDocument } from "~/services/graphModels/document";
-import { type IngestBodyRequest } from "~/lib/ingest.server";
-import { DocumentVersioningService } from "~/services/documentVersioning.server";
-import { DocumentDifferentialService } from "~/services/documentDiffer.server";
-import { KnowledgeGraphService } from "~/services/knowledgeGraph.server";
-import { prisma } from "../utils/prisma";
+import {
+  processDocumentIngestion,
+  type IngestDocumentPayload,
+} from "~/jobs/ingest/ingest-document.logic";
 import { ingestTask } from "./ingest";
 
 const documentIngestionQueue = queue({
@@ -23,266 +15,19 @@ export const ingestDocumentTask = task({
   id: "ingest-document",
   queue: documentIngestionQueue,
   machine: "medium-2x",
-  run: async (payload: {
-    body: z.infer<typeof IngestBodyRequest>;
-    userId: string;
-    workspaceId: string;
-    queueId: string;
-  }) => {
-    const startTime = Date.now();
-
-    try {
-      logger.log(`Processing document for user ${payload.userId}`, {
-        contentLength: payload.body.episodeBody.length,
-      });
-
-      await prisma.ingestionQueue.update({
-        where: { id: payload.queueId },
-        data: {
-          status: IngestionStatus.PROCESSING,
-        },
-      });
-
-      const documentBody = payload.body;
-
-      // Step 1: Initialize services and prepare document version
-      const versioningService = new DocumentVersioningService();
-      const differentialService = new DocumentDifferentialService();
-      const knowledgeGraphService = new KnowledgeGraphService();
-
-      const {
-        documentNode: document,
-        versionInfo,
-        chunkedDocument,
-      } = await versioningService.prepareDocumentVersion(
-        documentBody.sessionId!,
-        payload.userId,
-        documentBody.metadata?.documentTitle?.toString() || "Untitled Document",
-        documentBody.episodeBody,
-        documentBody.source,
-        documentBody.metadata || {},
-      );
-
-      logger.log(`Document version analysis:`, {
-        version: versionInfo.newVersion,
-        isNewDocument: versionInfo.isNewDocument,
-        hasContentChanged: versionInfo.hasContentChanged,
-        changePercentage: versionInfo.chunkLevelChanges.changePercentage,
-        changedChunks: versionInfo.chunkLevelChanges.changedChunkIndices.length,
-        totalChunks: versionInfo.chunkLevelChanges.totalChunks,
-      });
-
-      // Step 2: Determine processing strategy
-      const differentialDecision =
-        await differentialService.analyzeDifferentialNeed(
-          documentBody.episodeBody,
-          versionInfo.existingDocument,
-          chunkedDocument,
-        );
-
-      logger.log(`Differential analysis:`, {
-        shouldUseDifferential: differentialDecision.shouldUseDifferential,
-        strategy: differentialDecision.strategy,
-        reason: differentialDecision.reason,
-        documentSizeTokens: differentialDecision.documentSizeTokens,
-      });
-
-      // Early return for unchanged documents
-      if (differentialDecision.strategy === "skip_processing") {
-        logger.log("Document content unchanged, skipping processing");
-        return {
-          success: true,
-          documentsProcessed: 1,
-          chunksProcessed: 0,
-          episodesCreated: 0,
-          entitiesExtracted: 0,
-        };
-      }
-
-      // Step 3: Save the new document version
-      await saveDocument(document);
-
-      // Step 3.1: Invalidate statements from previous document version if it exists
-      let invalidationResults = null;
-      if (versionInfo.existingDocument && versionInfo.hasContentChanged) {
-        logger.log(
-          `Invalidating statements from previous document version: ${versionInfo.existingDocument.uuid}`,
-        );
-
-        invalidationResults =
-          await knowledgeGraphService.invalidateStatementsFromPreviousDocumentVersion(
-            {
-              previousDocumentUuid: versionInfo.existingDocument.uuid,
-              newDocumentContent: documentBody.episodeBody,
-              userId: payload.userId,
-              invalidatedBy: document.uuid,
-              semanticSimilarityThreshold: 0.75, // Configurable threshold
-            },
-          );
-
-        logger.log(`Statement invalidation completed:`, {
-          totalAnalyzed: invalidationResults.totalStatementsAnalyzed,
-          invalidated: invalidationResults.invalidatedStatements.length,
-          preserved: invalidationResults.preservedStatements.length,
+  run: async (payload: IngestDocumentPayload) => {
+    // Use common logic with Trigger-specific callback for episode ingestion
+    return await processDocumentIngestion(
+      payload,
+      // Callback for enqueueing episode ingestion for each chunk
+      async (episodePayload) => {
+        const episodeHandler = await ingestTask.trigger(episodePayload, {
+          queue: "ingestion-queue",
+          concurrencyKey: episodePayload.userId,
+          tags: [episodePayload.userId, episodePayload.queueId],
         });
-      }
-
-      logger.log(
-        `Document chunked into ${chunkedDocument.chunks.length} chunks`,
-      );
-
-      // Step 4: Process chunks based on differential strategy
-      let chunksToProcess = chunkedDocument.chunks;
-      let processingMode = "full";
-
-      if (
-        differentialDecision.shouldUseDifferential &&
-        differentialDecision.strategy === "chunk_level_diff"
-      ) {
-        // Only process changed chunks
-        const chunkComparisons = differentialService.getChunkComparisons(
-          versionInfo.existingDocument!,
-          chunkedDocument,
-        );
-
-        const changedIndices =
-          differentialService.getChunksNeedingReprocessing(chunkComparisons);
-        chunksToProcess = chunkedDocument.chunks.filter((chunk) =>
-          changedIndices.includes(chunk.chunkIndex),
-        );
-        processingMode = "differential";
-
-        logger.log(
-          `Differential processing: ${chunksToProcess.length}/${chunkedDocument.chunks.length} chunks need reprocessing`,
-        );
-      } else if (differentialDecision.strategy === "full_reingest") {
-        // Process all chunks
-        processingMode = "full";
-        logger.log(
-          `Full reingestion: processing all ${chunkedDocument.chunks.length} chunks`,
-        );
-      }
-
-      // Step 5: Queue chunks for processing
-      const episodeHandlers = [];
-      for (const chunk of chunksToProcess) {
-        const chunkEpisodeData = {
-          episodeBody: chunk.content,
-          referenceTime: documentBody.referenceTime,
-          metadata: {
-            ...documentBody.metadata,
-            processingMode,
-            differentialStrategy: differentialDecision.strategy,
-            chunkHash: chunk.contentHash,
-            documentTitle:
-              documentBody.metadata?.documentTitle?.toString() ||
-              "Untitled Document",
-            chunkIndex: chunk.chunkIndex,
-            documentUuid: document.uuid,
-          },
-          source: documentBody.source,
-          spaceIds: documentBody.spaceIds,
-          sessionId: documentBody.sessionId,
-          type: EpisodeTypeEnum.DOCUMENT,
-        };
-
-        const episodeHandler = await ingestTask.trigger(
-          {
-            body: chunkEpisodeData,
-            userId: payload.userId,
-            workspaceId: payload.workspaceId,
-            queueId: payload.queueId,
-          },
-          {
-            queue: "ingestion-queue",
-            concurrencyKey: payload.userId,
-            tags: [payload.userId, payload.queueId, processingMode],
-          },
-        );
-
-        if (episodeHandler.id) {
-          episodeHandlers.push(episodeHandler.id);
-          logger.log(
-            `Queued chunk ${chunk.chunkIndex + 1} for ${processingMode} processing`,
-            {
-              handlerId: episodeHandler.id,
-              chunkSize: chunk.content.length,
-              chunkHash: chunk.contentHash,
-            },
-          );
-        }
-      }
-
-      // Calculate cost savings
-      const costSavings = differentialService.calculateCostSavings(
-        chunkedDocument.chunks.length,
-        chunksToProcess.length,
-      );
-
-      await prisma.ingestionQueue.update({
-        where: { id: payload.queueId },
-        data: {
-          output: {
-            documentUuid: document.uuid,
-            version: versionInfo.newVersion,
-            totalChunks: chunkedDocument.chunks.length,
-            chunksProcessed: chunksToProcess.length,
-            chunksSkipped: costSavings.chunksSkipped,
-            processingMode,
-            differentialStrategy: differentialDecision.strategy,
-            estimatedSavings: `${costSavings.estimatedSavingsPercentage.toFixed(1)}%`,
-            statementInvalidation: invalidationResults
-              ? {
-                  totalAnalyzed: invalidationResults.totalStatementsAnalyzed,
-                  invalidated: invalidationResults.invalidatedStatements.length,
-                  preserved: invalidationResults.preservedStatements.length,
-                }
-              : null,
-            episodes: [],
-            episodeHandlers,
-          },
-          status: IngestionStatus.PROCESSING,
-        },
-      });
-
-      const processingTimeMs = Date.now() - startTime;
-
-      logger.log(
-        `Document differential processing completed in ${processingTimeMs}ms`,
-        {
-          documentUuid: document.uuid,
-          version: versionInfo.newVersion,
-          processingMode,
-          totalChunks: chunkedDocument.chunks.length,
-          chunksProcessed: chunksToProcess.length,
-          chunksSkipped: costSavings.chunksSkipped,
-          estimatedSavings: `${costSavings.estimatedSavingsPercentage.toFixed(1)}%`,
-          changePercentage: `${differentialDecision.changePercentage.toFixed(1)}%`,
-          statementInvalidation: invalidationResults
-            ? {
-                totalAnalyzed: invalidationResults.totalStatementsAnalyzed,
-                invalidated: invalidationResults.invalidatedStatements.length,
-                preserved: invalidationResults.preservedStatements.length,
-              }
-            : "No previous version",
-        },
-      );
-
-      return { success: true };
-    } catch (err: any) {
-      await prisma.ingestionQueue.update({
-        where: { id: payload.queueId },
-        data: {
-          error: err.message,
-          status: IngestionStatus.FAILED,
-        },
-      });
-
-      logger.error(
-        `Error processing document for user ${payload.userId}:`,
-        err,
-      );
-      return { success: false, error: err.message };
-    }
+        return { id: episodeHandler.id };
+      },
+    );
   },
 });

apps/webapp/app/trigger/ingest/ingest.ts

@@ -1,251 +1,46 @@
 import { queue, task } from "@trigger.dev/sdk";
-import { z } from "zod";
-import { KnowledgeGraphService } from "~/services/knowledgeGraph.server";
-import { linkEpisodeToDocument } from "~/services/graphModels/document";
-
-import { IngestionStatus } from "@core/database";
-import { logger } from "~/services/logger.service";
+import {
+  processEpisodeIngestion,
+  IngestBodyRequest,
+  type IngestEpisodePayload,
+} from "~/jobs/ingest/ingest-episode.logic";
 import { triggerSpaceAssignment } from "../spaces/space-assignment";
-import { prisma } from "../utils/prisma";
-import { EpisodeType } from "@core/types";
-import { deductCredits, hasCredits } from "../utils/utils";
-import { assignEpisodesToSpace } from "~/services/graphModels/space";
 import { triggerSessionCompaction } from "../session/session-compaction";
-
-export const IngestBodyRequest = z.object({
-  episodeBody: z.string(),
-  referenceTime: z.string(),
-  metadata: z.record(z.union([z.string(), z.number(), z.boolean()])).optional(),
-  source: z.string(),
-  spaceIds: z.array(z.string()).optional(),
-  sessionId: z.string().optional(),
-  type: z
-    .enum([EpisodeType.CONVERSATION, EpisodeType.DOCUMENT])
-    .default(EpisodeType.CONVERSATION),
-});
+import { bertTopicAnalysisTask } from "../bert/topic-analysis";
 
 const ingestionQueue = queue({
   name: "ingestion-queue",
   concurrencyLimit: 1,
 });
 
+// Export for backwards compatibility
+export { IngestBodyRequest };
+
 // Register the Trigger.dev task
 export const ingestTask = task({
   id: "ingest-episode",
   queue: ingestionQueue,
   machine: "medium-2x",
-  run: async (payload: {
-    body: z.infer<typeof IngestBodyRequest>;
-    userId: string;
-    workspaceId: string;
-    queueId: string;
-  }) => {
-    try {
-      logger.log(`Processing job for user ${payload.userId}`);
-
-      // Check if workspace has sufficient credits before processing
-      const hasSufficientCredits = await hasCredits(
-        payload.workspaceId,
-        "addEpisode",
-      );
-
-      if (!hasSufficientCredits) {
-        logger.warn(
-          `Insufficient credits for workspace ${payload.workspaceId}`,
-        );
-
-        await prisma.ingestionQueue.update({
-          where: { id: payload.queueId },
-          data: {
-            status: IngestionStatus.NO_CREDITS,
-            error:
-              "Insufficient credits. Please upgrade your plan or wait for your credits to reset.",
-          },
+  run: async (payload: IngestEpisodePayload) => {
+    // Use common logic with Trigger-specific callbacks for follow-up jobs
+    return await processEpisodeIngestion(
+      payload,
+      // Callback for space assignment
+      async (params) => {
+        await triggerSpaceAssignment(params);
+      },
+      // Callback for session compaction
+      async (params) => {
+        await triggerSessionCompaction(params);
+      },
+      // Callback for BERT topic analysis
+      async (params) => {
+        await bertTopicAnalysisTask.trigger(params, {
+          queue: "bert-topic-analysis",
+          concurrencyKey: params.userId,
+          tags: [params.userId, "bert-analysis"],
         });
-
-        return {
-          success: false,
-          error: "Insufficient credits",
-        };
-      }
-
-      const ingestionQueue = await prisma.ingestionQueue.update({
-        where: { id: payload.queueId },
-        data: {
-          status: IngestionStatus.PROCESSING,
-        },
-      });
-
-      const knowledgeGraphService = new KnowledgeGraphService();
-
-      const episodeBody = payload.body as any;
-
-      const episodeDetails = await knowledgeGraphService.addEpisode(
-        {
-          ...episodeBody,
-          userId: payload.userId,
-        },
-        prisma,
-      );
-
-      // Link episode to document if it's a document chunk
-      if (
-        episodeBody.type === EpisodeType.DOCUMENT &&
-        episodeBody.metadata.documentUuid &&
-        episodeDetails.episodeUuid
-      ) {
-        try {
-          await linkEpisodeToDocument(
-            episodeDetails.episodeUuid,
-            episodeBody.metadata.documentUuid,
-            episodeBody.metadata.chunkIndex || 0,
-          );
-          logger.log(
-            `Linked episode ${episodeDetails.episodeUuid} to document ${episodeBody.metadata.documentUuid} at chunk ${episodeBody.metadata.chunkIndex || 0}`,
-          );
-        } catch (error) {
-          logger.error(`Failed to link episode to document:`, {
-            error,
-            episodeUuid: episodeDetails.episodeUuid,
-            documentUuid: episodeBody.metadata.documentUuid,
-          });
-        }
-      }
-
-      let finalOutput = episodeDetails;
-      let episodeUuids: string[] = episodeDetails.episodeUuid
-        ? [episodeDetails.episodeUuid]
-        : [];
-      let currentStatus: IngestionStatus = IngestionStatus.COMPLETED;
-      if (episodeBody.type === EpisodeType.DOCUMENT) {
-        const currentOutput = ingestionQueue.output as any;
-        currentOutput.episodes.push(episodeDetails);
-        episodeUuids = currentOutput.episodes.map(
-          (episode: any) => episode.episodeUuid,
-        );
-
-        finalOutput = {
-          ...currentOutput,
-        };
-
-        if (currentOutput.episodes.length !== currentOutput.totalChunks) {
-          currentStatus = IngestionStatus.PROCESSING;
-        }
-      }
-
-      await prisma.ingestionQueue.update({
-        where: { id: payload.queueId },
-        data: {
-          output: finalOutput,
-          status: currentStatus,
-        },
-      });
-
-      // Deduct credits for episode creation
-      if (currentStatus === IngestionStatus.COMPLETED) {
-        await deductCredits(
-          payload.workspaceId,
-          "addEpisode",
-          finalOutput.statementsCreated,
-        );
-      }
-
-      // Handle space assignment after successful ingestion
-      try {
-        // If spaceIds were explicitly provided, immediately assign the episode to those spaces
-        if (
-          episodeBody.spaceIds &&
-          episodeBody.spaceIds.length > 0 &&
-          episodeDetails.episodeUuid
-        ) {
-          logger.info(`Assigning episode to explicitly provided spaces`, {
-            userId: payload.userId,
-            episodeId: episodeDetails.episodeUuid,
-            spaceIds: episodeBody.spaceIds,
-          });
-
-          // Assign episode to each space
-          for (const spaceId of episodeBody.spaceIds) {
-            await assignEpisodesToSpace(
-              [episodeDetails.episodeUuid],
-              spaceId,
-              payload.userId,
-            );
-          }
-
-          logger.info(
-            `Skipping LLM space assignment - episode explicitly assigned to ${episodeBody.spaceIds.length} space(s)`,
-          );
-        } else {
-          // Only trigger automatic LLM space assignment if no explicit spaceIds were provided
-          logger.info(
-            `Triggering LLM space assignment after successful ingestion`,
-            {
-              userId: payload.userId,
-              workspaceId: payload.workspaceId,
-              episodeId: episodeDetails?.episodeUuid,
-            },
-          );
-          if (
-            episodeDetails.episodeUuid &&
-            currentStatus === IngestionStatus.COMPLETED
-          ) {
-            await triggerSpaceAssignment({
-              userId: payload.userId,
-              workspaceId: payload.workspaceId,
-              mode: "episode",
-              episodeIds: episodeUuids,
-            });
-          }
-        }
-      } catch (assignmentError) {
-        // Don't fail the ingestion if assignment fails
-        logger.warn(`Failed to trigger space assignment after ingestion:`, {
-          error: assignmentError,
-          userId: payload.userId,
-          episodeId: episodeDetails?.episodeUuid,
-        });
-      }
-
-      // Auto-trigger session compaction if episode has sessionId
-      try {
-        if (
-          episodeBody.sessionId &&
-          currentStatus === IngestionStatus.COMPLETED
-        ) {
-          logger.info(`Checking if session compaction should be triggered`, {
-            userId: payload.userId,
-            sessionId: episodeBody.sessionId,
-            source: episodeBody.source,
-          });
-
-          await triggerSessionCompaction({
-            userId: payload.userId,
-            sessionId: episodeBody.sessionId,
-            source: episodeBody.source,
-          });
-        }
-      } catch (compactionError) {
-        // Don't fail the ingestion if compaction fails
-        logger.warn(`Failed to trigger session compaction after ingestion:`, {
-          error: compactionError,
-          userId: payload.userId,
-          sessionId: episodeBody.sessionId,
-        });
-      }
-
-      return { success: true, episodeDetails };
-    } catch (err: any) {
-      await prisma.ingestionQueue.update({
-        where: { id: payload.queueId },
-        data: {
-          error: err.message,
-          status: IngestionStatus.FAILED,
-        },
-      });
-
-      logger.error(`Error processing job for user ${payload.userId}:`, err);
-      return { success: false, error: err.message };
-    }
+      },
+    );
   },
 });

apps/webapp/app/trigger/ingest/retry-no-credits.ts

@@ -0,0 +1,120 @@
+import { task } from "@trigger.dev/sdk";
+import { z } from "zod";
+import { IngestionStatus } from "@core/database";
+import { logger } from "~/services/logger.service";
+import { prisma } from "../utils/prisma";
+import { type IngestBodyRequest, ingestTask } from "./ingest";
+
+export const RetryNoCreditBodyRequest = z.object({
+  workspaceId: z.string(),
+});
+
+// Register the Trigger.dev task to retry NO_CREDITS episodes
+export const retryNoCreditsTask = task({
+  id: "retry-no-credits-episodes",
+  run: async (payload: z.infer<typeof RetryNoCreditBodyRequest>) => {
+    try {
+      logger.log(
+        `Starting retry of NO_CREDITS episodes for workspace ${payload.workspaceId}`,
+      );
+
+      // Find all ingestion queue items with NO_CREDITS status for this workspace
+      const noCreditItems = await prisma.ingestionQueue.findMany({
+        where: {
+          workspaceId: payload.workspaceId,
+          status: IngestionStatus.NO_CREDITS,
+        },
+        orderBy: {
+          createdAt: "asc", // Process oldest first
+        },
+        include: {
+          workspace: true,
+        },
+      });
+
+      if (noCreditItems.length === 0) {
+        logger.log(
+          `No NO_CREDITS episodes found for workspace ${payload.workspaceId}`,
+        );
+        return {
+          success: true,
+          message: "No episodes to retry",
+          retriedCount: 0,
+        };
+      }
+
+      logger.log(`Found ${noCreditItems.length} NO_CREDITS episodes to retry`);
+
+      const results = {
+        total: noCreditItems.length,
+        retriggered: 0,
+        failed: 0,
+        errors: [] as Array<{ queueId: string; error: string }>,
+      };
+
+      // Process each item
+      for (const item of noCreditItems) {
+        try {
+          const queueData = item.data as z.infer<typeof IngestBodyRequest>;
+
+          // Reset status to PENDING and clear error
+          await prisma.ingestionQueue.update({
+            where: { id: item.id },
+            data: {
+              status: IngestionStatus.PENDING,
+              error: null,
+              retryCount: item.retryCount + 1,
+            },
+          });
+
+          // Trigger the ingestion task
+          await ingestTask.trigger({
+            body: queueData,
+            userId: item.workspace?.userId as string,
+            workspaceId: payload.workspaceId,
+            queueId: item.id,
+          });
+
+          results.retriggered++;
+          logger.log(
+            `Successfully retriggered episode ${item.id} (retry #${item.retryCount + 1})`,
+          );
+        } catch (error: any) {
+          results.failed++;
+          results.errors.push({
+            queueId: item.id,
+            error: error.message,
+          });
+          logger.error(`Failed to retrigger episode ${item.id}:`, error);
+
+          // Update the item to mark it as failed
+          await prisma.ingestionQueue.update({
+            where: { id: item.id },
+            data: {
+              status: IngestionStatus.FAILED,
+              error: `Retry failed: ${error.message}`,
+            },
+          });
+        }
+      }
+
+      logger.log(
+        `Completed retry for workspace ${payload.workspaceId}. Retriggered: ${results.retriggered}, Failed: ${results.failed}`,
+      );
+
+      return {
+        success: true,
+        ...results,
+      };
+    } catch (err: any) {
+      logger.error(
+        `Error retrying NO_CREDITS episodes for workspace ${payload.workspaceId}:`,
+        err,
+      );
+      return {
+        success: false,
+        error: err.message,
+      };
+    }
+  },
+});

apps/webapp/app/trigger/session/session-compaction.ts

@@ -1,36 +1,8 @@
 import { queue, task } from "@trigger.dev/sdk/v3";
-import { logger } from "~/services/logger.service";
-import { runQuery } from "~/lib/neo4j.server";
-import type { CoreMessage } from "ai";
-import { z } from "zod";
-import { getEmbedding, makeModelCall } from "~/lib/model.server";
 import {
-  getCompactedSessionBySessionId,
-  linkEpisodesToCompact,
-  getSessionEpisodes,
-  type CompactedSessionNode,
-  type SessionEpisodeData,
-  saveCompactedSession,
-} from "~/services/graphModels/compactedSession";
-
-interface SessionCompactionPayload {
-  userId: string;
-  sessionId: string;
-  source: string;
-  triggerSource?: "auto" | "manual" | "threshold";
-}
-
-// Zod schema for LLM response validation
-const CompactionResultSchema = z.object({
-  summary: z.string().describe("Consolidated narrative of the entire session"),
-  confidence: z.number().min(0).max(1).describe("Confidence score of the compaction quality"),
-});
-
-const CONFIG = {
-  minEpisodesForCompaction: 5, // Minimum episodes to trigger compaction
-  compactionThreshold: 1, // Trigger after N new episodes
-  maxEpisodesPerBatch: 50, // Process in batches if needed
-};
+  processSessionCompaction,
+  type SessionCompactionPayload,
+} from "~/jobs/session/session-compaction.logic";
 
 export const sessionCompactionQueue = queue({
   name: "session-compaction-queue",
@@ -41,82 +13,7 @@ export const sessionCompactionTask = task({
   id: "session-compaction",
   queue: sessionCompactionQueue,
   run: async (payload: SessionCompactionPayload) => {
-    const { userId, sessionId, source, triggerSource = "auto" } = payload;
-
-    logger.info(`Starting session compaction`, {
-      userId,
-      sessionId,
-      source,
-      triggerSource,
-    });
-
-    try {
-      // Check if compaction already exists
-      const existingCompact = await getCompactedSessionBySessionId(sessionId, userId);
-
-      // Fetch all episodes for this session
-      const episodes = await getSessionEpisodes(sessionId, userId, existingCompact?.endTime);
-
-      console.log("episodes", episodes.length);
-      // Check if we have enough episodes
-      if (!existingCompact && episodes.length < CONFIG.minEpisodesForCompaction) {
-        logger.info(`Not enough episodes for compaction`, {
-          sessionId,
-          episodeCount: episodes.length,
-          minRequired: CONFIG.minEpisodesForCompaction,
-        });
-        return {
-          success: false,
-          reason: "insufficient_episodes",
-          episodeCount: episodes.length,
-        };
-      } else if (existingCompact && episodes.length < CONFIG.minEpisodesForCompaction + CONFIG.compactionThreshold) {
-        logger.info(`Not enough new episodes for compaction`, {
-          sessionId,
-          episodeCount: episodes.length,
-          minRequired: CONFIG.minEpisodesForCompaction + CONFIG.compactionThreshold,
-        });
-        return {
-          success: false,
-          reason: "insufficient_new_episodes",
-          episodeCount: episodes.length,
-        };
-      }
-
-      // Generate or update compaction
-      const compactionResult = existingCompact
-        ? await updateCompaction(existingCompact, episodes, userId)
-        : await createCompaction(sessionId, episodes, userId, source);
-
-      logger.info(`Session compaction completed`, {
-        sessionId,
-        compactUuid: compactionResult.uuid,
-        episodeCount: compactionResult.episodeCount,
-        compressionRatio: compactionResult.compressionRatio,
-      });
-
-      return {
-        success: true,
-        compactionResult: {
-          compactUuid: compactionResult.uuid,
-          sessionId: compactionResult.sessionId,
-          summary: compactionResult.summary,
-          episodeCount: compactionResult.episodeCount,
-          startTime: compactionResult.startTime,
-          endTime: compactionResult.endTime,
-          confidence: compactionResult.confidence,
-          compressionRatio: compactionResult.compressionRatio,
-        },
-      };
-    } catch (error) {
-      logger.error(`Session compaction failed`, {
-        sessionId,
-        userId,
-        error: error instanceof Error ? error.message : String(error),
-      });
-
-      throw error;
-    }
+    return await processSessionCompaction(payload);
   },
 });