From 7b28781efd235fc46829e3337ab2b172b981b6ef Mon Sep 17 00:00:00 2001 From: Harshith Mullapudi Date: Wed, 4 Jun 2025 17:06:03 +0530 Subject: [PATCH] Fix: added queue --- apps/webapp/app/env.server.ts | 5 + apps/webapp/app/lib/ingest.queue.ts | 45 ++++ apps/webapp/app/routes/ingest.tsx | 58 +++++ .../workspaces.$workspaceSlug.ingest.tsx | 37 --- .../app/services/knowledgeGraph.server.ts | 107 ++++---- apps/webapp/package.json | 2 + apps/webapp/vite.config.ts | 3 + helixdb-cfg/queries.hx | 231 +----------------- helixdb-cfg/schema.hx | 38 +-- .../migration.sql | 8 + packages/database/prisma/schema.prisma | 4 +- packages/types/src/graph/graph.entity.ts | 5 +- pnpm-lock.yaml | 193 ++++++++++++++- turbo.json | 5 +- 14 files changed, 398 insertions(+), 343 deletions(-) create mode 100644 apps/webapp/app/lib/ingest.queue.ts create mode 100644 apps/webapp/app/routes/ingest.tsx delete mode 100644 apps/webapp/app/routes/workspaces.$workspaceSlug.ingest.tsx create mode 100644 packages/database/prisma/migrations/20250604101446_space_optional_in_queue/migration.sql diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index f8137fd..08afc4a 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -34,6 +34,11 @@ const EnvironmentSchema = z.object({ // google auth AUTH_GOOGLE_CLIENT_ID: z.string().optional(), AUTH_GOOGLE_CLIENT_SECRET: z.string().optional(), + + //Redis + REDIS_HOST: z.string().default("localhost"), + REDIS_PORT: z.coerce.number().default(6379), + REDIS_TLS_DISABLED: z.coerce.boolean().default(true), }); export type Environment = z.infer; diff --git a/apps/webapp/app/lib/ingest.queue.ts b/apps/webapp/app/lib/ingest.queue.ts new file mode 100644 index 0000000..43e8ef5 --- /dev/null +++ b/apps/webapp/app/lib/ingest.queue.ts @@ -0,0 +1,45 @@ +// lib/ingest.queue.ts +import { Queue, Worker } from "bullmq"; +import IORedis from "ioredis"; +import { env } from "~/env.server"; +import { KnowledgeGraphService } from "../services/knowledgeGraph.server"; + +const connection = new IORedis({ + port: env.REDIS_PORT, + host: env.REDIS_HOST, + maxRetriesPerRequest: null, + enableReadyCheck: false, +}); + +const userQueues = new Map(); +const userWorkers = new Map(); + +async function processUserJob(userId: string, job: any) { + try { + console.log(job); + console.log(`Processing job for user ${userId}`); + const knowledgeGraphService = new KnowledgeGraphService(); + + knowledgeGraphService.addEpisode({ ...job.data.body, userId }); + + // your processing logic + } catch (err) { + console.error(`Error processing job for user ${userId}:`, err); + } +} + +export function getUserQueue(userId: string) { + if (!userQueues.has(userId)) { + const queueName = `ingest-${userId}`; + const queue = new Queue(queueName, { connection }); + userQueues.set(userId, queue); + + const worker = new Worker(queueName, (job) => processUserJob(userId, job), { + connection, + concurrency: 1, + }); + userWorkers.set(userId, worker); + } + + return userQueues.get(userId)!; +} diff --git a/apps/webapp/app/routes/ingest.tsx b/apps/webapp/app/routes/ingest.tsx new file mode 100644 index 0000000..84d1aa7 --- /dev/null +++ b/apps/webapp/app/routes/ingest.tsx @@ -0,0 +1,58 @@ +import { EpisodeType } from "@recall/types"; +import { json, LoaderFunctionArgs } from "@remix-run/node"; +import { z } from "zod"; +import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server"; +import { KnowledgeGraphService } from "../services/knowledgeGraph.server"; +import { getUserQueue } from "~/lib/ingest.queue"; +import { prisma } from "~/db.server"; +import { IngestionStatus } from "@recall/database"; + +export const IngestBodyRequest = z.object({ + name: z.string(), + episodeBody: z.string(), + referenceTime: z.string(), + type: z.enum([EpisodeType.Conversation, EpisodeType.Text]), // Assuming these are the EpisodeType values + source: z.string(), + spaceId: z.string().optional(), + sessionId: z.string().optional(), +}); + +const { action, loader } = createActionApiRoute( + { + body: IngestBodyRequest, + allowJWT: true, + authorization: { + action: "ingest", + }, + corsStrategy: "all", + }, + async ({ body, headers, params, authentication }) => { + const queuePersist = await prisma.ingestionQueue.create({ + data: { + spaceId: body.spaceId, + data: body, + status: IngestionStatus.PENDING, + priority: 1, + }, + }); + + const ingestionQueue = getUserQueue(authentication.userId); + + await ingestionQueue.add( + `ingest-user-${authentication.userId}`, // 👈 unique name per user + { + queueId: queuePersist.id, + spaceId: body.spaceId, + userId: authentication.userId, + body, + }, + { + jobId: `${authentication.userId}-${Date.now()}`, // unique per job but grouped under user + }, + ); + + return json({}); + }, +); + +export { action, loader }; diff --git a/apps/webapp/app/routes/workspaces.$workspaceSlug.ingest.tsx b/apps/webapp/app/routes/workspaces.$workspaceSlug.ingest.tsx deleted file mode 100644 index 73a0f7b..0000000 --- a/apps/webapp/app/routes/workspaces.$workspaceSlug.ingest.tsx +++ /dev/null @@ -1,37 +0,0 @@ -import { json, LoaderFunctionArgs } from "@remix-run/node"; -import { z } from "zod"; -import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server"; - -const ParamsSchema = z.object({ - workspaceSlug: z.string(), -}); - -export const IngestBodyRequest = z.object({ - name: z.string(), - episodeBody: z.string(), - referenceTime: z.string(), - type: z.enum(["CONVERSATION", "TEXT"]), // Assuming these are the EpisodeType values - source: z.string(), - userId: z.string(), - spaceId: z.string().optional(), - sessionId: z.string().optional(), -}); - -const { action, loader } = createActionApiRoute( - { - params: ParamsSchema, - body: IngestBodyRequest, - allowJWT: true, - authorization: { - action: "ingest", - }, - corsStrategy: "all", - }, - async ({ body, headers, params, authentication }) => { - console.log(body, headers, params, authentication); - - return json({}); - }, -); - -export { action, loader }; diff --git a/apps/webapp/app/services/knowledgeGraph.server.ts b/apps/webapp/app/services/knowledgeGraph.server.ts index f020bff..270ef5f 100644 --- a/apps/webapp/app/services/knowledgeGraph.server.ts +++ b/apps/webapp/app/services/knowledgeGraph.server.ts @@ -1,4 +1,3 @@ -import HelixDB from "helix-ts"; import { openai } from "@ai-sdk/openai"; import { type CoreMessage, @@ -7,16 +6,13 @@ import { type LanguageModelV1, streamText, } from "ai"; -import { LLMMappings, LLMModelEnum } from "@recall/types"; +import { EpisodeType, LLMMappings, LLMModelEnum } from "@recall/types"; import { logger } from "./logger.service"; import crypto from "crypto"; import { dedupeNodes, extract_message, extract_text } from "./prompts/nodes"; import { extract_statements } from "./prompts/statements"; -export enum EpisodeType { - Conversation = "CONVERSATION", - Text = "TEXT", -} +const HelixDB = await import("helix-ts").then((m) => m.default); /** * Interface for episodic node in the reified knowledge graph @@ -173,6 +169,8 @@ export class KnowledgeGraphService { const startTime = Date.now(); const now = new Date(); + console.log(params); + try { // Step 1: Context Retrieval - Get previous episodes for context const previousEpisodes = await this.retrieveEpisodes( @@ -204,53 +202,53 @@ export class KnowledgeGraphService { ); // Step 4: Entity Resolution - Resolve extracted nodes to existing nodes or create new ones - const { resolvedNodes, uuidMap } = await this.resolveExtractedNodes( - extractedNodes, - episode, - previousEpisodes, - ); + // const { resolvedNodes, uuidMap } = await this.resolveExtractedNodes( + // extractedNodes, + // episode, + // previousEpisodes, + // ); - // Step 5: Statement Extraction - Extract statements (triples) instead of direct edges - const extractedStatements = await this.extractStatements( - episode, - resolvedNodes, - previousEpisodes, - ); + // // Step 5: Statement Extraction - Extract statements (triples) instead of direct edges + // const extractedStatements = await this.extractStatements( + // episode, + // resolvedNodes, + // previousEpisodes, + // ); - // Step 6: Statement Resolution - Resolve statements and detect contradictions - const { resolvedStatements, invalidatedStatements } = - await this.resolveStatements( - extractedStatements, - episode, - resolvedNodes, - ); + // // Step 6: Statement Resolution - Resolve statements and detect contradictions + // const { resolvedStatements, invalidatedStatements } = + // await this.resolveStatements( + // extractedStatements, + // episode, + // resolvedNodes, + // ); - // Step 7: Role Assignment & Attribute Extraction - Extract additional attributes for nodes - const hydratedNodes = await this.extractAttributesFromNodes( - resolvedNodes, - episode, - previousEpisodes, - ); + // // Step 7: Role Assignment & Attribute Extraction - Extract additional attributes for nodes + // const hydratedNodes = await this.extractAttributesFromNodes( + // resolvedNodes, + // episode, + // previousEpisodes, + // ); // Step 8: Generate embeddings for semantic search // Note: In this implementation, embeddings are generated during extraction // but could be moved to a separate step for clarity // Step 10: Save everything to HelixDB using the reified + temporal structure - await this.saveToHelixDB( - episode, - hydratedNodes, - resolvedStatements, - invalidatedStatements, - ); + // await this.saveToHelixDB( + // episode, + // hydratedNodes, + // resolvedStatements, + // invalidatedStatements, + // ); const endTime = Date.now(); const processingTimeMs = endTime - startTime; return { episodeUuid: episode.uuid, - nodesCreated: hydratedNodes.length, - statementsCreated: resolvedStatements.length, + // nodesCreated: hydratedNodes.length, + // statementsCreated: resolvedStatements.length, processingTimeMs, }; } catch (error) { @@ -326,7 +324,7 @@ export class KnowledgeGraphService { extractedNodes: EntityNode[], episode: EpisodicNode, previousEpisodes: EpisodicNode[], - ): Promise<{ resolvedNodes: EntityNode[]; uuidMap: Map }> { + ) { const uuidMap = new Map(); const existingNodesLists = await Promise.all( @@ -655,19 +653,14 @@ export class KnowledgeGraphService { // Get the detect_contradiction prompt from the prompt library // The prompt should be updated to handle reified statements specifically - const messages = - promptLibrary.detectContradiction.detect_json.call(promptContext); + + // promptLibrary.detectContradiction.detect_json.call(promptContext); let responseText = ""; - await this.makeModelCall( - false, - LLMModelEnum.GPT41, - messages as CoreMessage[], - (text) => { - responseText = text; - }, - ); + await this.makeModelCall(false, LLMModelEnum.GPT41, [], (text) => { + responseText = text; + }); try { const result = JSON.parse(responseText); @@ -728,7 +721,7 @@ export class KnowledgeGraphService { name: episode.name, content: episode.content, source: episode.source, - sourceDescription: episode.sourceDescription, + // sourceDescription: episode.sourceDescription, userId: episode.userId || null, labels: episode.labels || [], createdAt: episode.createdAt.toISOString(), @@ -755,15 +748,15 @@ export class KnowledgeGraphService { await helixClient.query("saveStatement", { uuid: triple.statement.uuid, fact: triple.statement.fact, - groupId: triple.statement.groupId, + // groupId: triple.statement.groupId, userId: triple.statement.userId || null, createdAt: triple.statement.createdAt.toISOString(), validAt: triple.statement.validAt.toISOString(), invalidAt: triple.statement.invalidAt ? triple.statement.invalidAt.toISOString() : null, - attributesJson: triple.statement.attributesJson, - embedding: triple.statement.embedding || [], + // attributesJson: triple.statement.attributesJson, + // embedding: triple.statement.embedding || [], }); // Create HasSubject edge @@ -804,13 +797,13 @@ export class KnowledgeGraphService { await helixClient.query("saveStatement", { uuid: triple.statement.uuid, fact: triple.statement.fact, - groupId: triple.statement.groupId, + // groupId: triple.statement.groupId, userId: triple.statement.userId || null, createdAt: triple.statement.createdAt.toISOString(), validAt: triple.statement.validAt.toISOString(), - invalidAt: triple.statement.invalidAt.toISOString(), // This will be the episode.validAt timestamp - attributesJson: triple.statement.attributesJson, - embedding: triple.statement.embedding || [], + // invalidAt: triple.statement.invalidAt.toISOString(), // This will be the episode.validAt timestamp + // attributesJson: triple.statement.attributesJson, + // embedding: triple.statement.embedding || [], }); } } catch (error) { diff --git a/apps/webapp/package.json b/apps/webapp/package.json index f1776a2..32cc196 100644 --- a/apps/webapp/package.json +++ b/apps/webapp/package.json @@ -37,6 +37,8 @@ "morgan": "^1.10.0", "nanoid": "3.3.8", "jose": "^5.2.3", + "bullmq": "^5.53.2", + "ioredis": "^5.6.1", "non.geist": "^1.0.2", "posthog-js": "^1.116.6", "react": "^18.2.0", diff --git a/apps/webapp/vite.config.ts b/apps/webapp/vite.config.ts index 77527a2..274903a 100644 --- a/apps/webapp/vite.config.ts +++ b/apps/webapp/vite.config.ts @@ -27,4 +27,7 @@ export default defineConfig({ allowedHosts: true, port: 3033, }, + ssr: { + noExternal: ["helix-ts"], + }, }); diff --git a/helixdb-cfg/queries.hx b/helixdb-cfg/queries.hx index 87bf430..58c5464 100644 --- a/helixdb-cfg/queries.hx +++ b/helixdb-cfg/queries.hx @@ -1,230 +1,17 @@ // Save an episode to the database -QUERY saveEpisode(uuid: String, name: String, content: String, source: String, - sourceDescription: String, userId: String, labels: [String], - createdAt: String, validAt: String, embedding: [F32]) => - episode <- AddV({ - uuid: uuid, +QUERY saveEpisode(name: String, content: String, source: String, + userId: String, + createdAt: I64, space: String, episodeType: String, + sessionId: String, validAt: I64, embedding: [F64]) => + episode <- AddV(embedding, { name: name, content: content, source: source, - sourceDescription: sourceDescription, userId: userId, - labels: labels, createdAt: createdAt, - validAt: validAt, - embedding: embedding + space: space, + sessionId: sessionId, + episodeType: episodeType, + validAt: validAt }) RETURN episode - -// Get a specific episode by UUID -QUERY getEpisode(uuid: String) => - episode <- V(uuid) - RETURN episode - -// Get recent episodes with optional filters -QUERY getRecentEpisodes(referenceTime: String, limit: I32, userId: String, source: String) => - episodes <- V::WHERE(_::{validAt}::LTE(referenceTime)) - // Apply filters if provided - episodes <- IF userId != NULL THEN episodes::WHERE(_::{userId}::EQ(userId)) ELSE episodes - episodes <- IF source != NULL THEN episodes::WHERE(_::{source}::EQ(source)) ELSE episodes - // Sort and limit - episodes <- episodes::Sort({validAt: -1})::Limit(limit) - RETURN episodes - -// Save an entity node -QUERY saveEntity(uuid: String, name: String, summary: String, - userId: String, createdAt: String, attributesJson: String, embedding: [F32]) => - entity <- AddV({ - uuid: uuid, - name: name, - summary: summary, - userId: userId, - createdAt: createdAt, - attributesJson: attributesJson, - embedding: embedding - }) - RETURN entity - -// Get an entity by UUID -QUERY getEntity(uuid: String) => - entity <- V(uuid) - RETURN entity - -// Save a statement with temporal information -QUERY saveStatement(uuid: String, fact: String, groupId: String, userId: String, - createdAt: String, validAt: String, invalidAt: String, - attributesJson: String, embedding: [F32]) => - statement <- AddV({ - uuid: uuid, - fact: fact, - groupId: groupId, - userId: userId, - createdAt: createdAt, - validAt: validAt, - invalidAt: invalidAt, - attributesJson: attributesJson, - embedding: embedding - }) - RETURN statement - -// Create HasSubject edge -QUERY createHasSubjectEdge(uuid: String, statementId: String, entityId: String, createdAt: String) => - statement <- V(statementId) - entity <- V(entityId) - edge <- AddE::From(statement)::To(entity)({ - uuid: uuid, - createdAt: createdAt - }) - RETURN edge - -// Create HasObject edge -QUERY createHasObjectEdge(uuid: String, statementId: String, entityId: String, createdAt: String) => - statement <- V(statementId) - entity <- V(entityId) - edge <- AddE::From(statement)::To(entity)({ - uuid: uuid, - createdAt: createdAt - }) - RETURN edge - -// Create HasPredicate edge -QUERY createHasPredicateEdge(uuid: String, statementId: String, entityId: String, createdAt: String) => - statement <- V(statementId) - entity <- V(entityId) - edge <- AddE::From(statement)::To(entity)({ - uuid: uuid, - createdAt: createdAt - }) - RETURN edge - -// Create HasProvenance edge -QUERY createHasProvenanceEdge(uuid: String, statementId: String, episodeId: String, createdAt: String) => - statement <- V(statementId) - episode <- V(episodeId) - edge <- AddE::From(statement)::To(episode)({ - uuid: uuid, - createdAt: createdAt - }) - RETURN edge - -// Get all statements for a subject entity -QUERY getStatementsForSubject(entityId: String) => - entity <- V(entityId) - statements <- entity::In - RETURN statements - -// Get all statements for an object entity -QUERY getStatementsForObject(entityId: String) => - entity <- V(entityId) - statements <- entity::In - RETURN statements - -// Get all statements with a specific predicate -QUERY getStatementsForPredicate(predicateId: String) => - predicate <- V(predicateId) - statements <- predicate::In - RETURN statements - -// Get all statements from an episode -QUERY getStatementsFromEpisode(episodeId: String) => - episode <- V(episodeId) - statements <- episode::In - RETURN statements - -// Get the complete subject-predicate-object triples for a statement -QUERY getTripleForStatement(statementId: String) => - statement <- V(statementId) - subject <- statement::Out - predicate <- statement::Out - object <- statement::Out - RETURN { - statement: statement, - subject: subject, - predicate: predicate, - object: object - } - -// Find all statements valid at a specific time -QUERY getStatementsValidAtTime(timestamp: String, userId: String) => - statements <- V::WHERE( - AND( - _::{validAt}::LTE(timestamp), - OR( - _::{invalidAt}::GT(timestamp), - _::{invalidAt}::EQ(NULL) - ) - ) - ) - // Filter by userId if provided - statements <- IF userId != NULL THEN - statements::WHERE(_::{userId}::EQ(userId)) - ELSE - statements - RETURN statements - -// Find contradictory statements (same subject and predicate but different objects) -QUERY findContradictoryStatements(subjectId: String, predicateId: String) => - subject <- V(subjectId) - predicate <- V(predicateId) - - // Get all statements that have this subject - statements <- subject::In - - // Filter to those with the specified predicate - statements <- statements::WHERE( - _::Out::ID()::EQ(predicateId) - ) - - // Get all valid statements - valid_statements <- statements::WHERE( - OR( - _::{invalidAt}::EQ(NULL), - _::{invalidAt}::GT(NOW()) - ) - ) - - RETURN valid_statements - -// Find semantically similar entities using vector embeddings -QUERY findSimilarEntities(queryEmbedding: [F32], limit: I32, threshold: F32) => - entities <- V::Neighbor(queryEmbedding, threshold)::Limit(limit) - RETURN entities - -// Find semantically similar statements using vector embeddings -QUERY findSimilarStatements(queryEmbedding: [F32], limit: I32, threshold: F32) => - statements <- V::Neighbor(queryEmbedding, threshold)::Limit(limit) - RETURN statements - -// Retrieve a complete knowledge triple (subject, predicate, object) with temporal information -QUERY getTemporalTriple(statementId: String) => - statement <- V(statementId) - subject <- statement::Out - predicate <- statement::Out - object <- statement::Out - episode <- statement::Out - - RETURN { - statement: { - id: statement::{uuid}, - fact: statement::{fact}, - validAt: statement::{validAt}, - invalidAt: statement::{invalidAt}, - attributesJson: statement::{attributesJson} - }, - subject: { - id: subject::{uuid}, - name: subject::{name} - }, - predicate: { - id: predicate::{uuid}, - name: predicate::{name} - }, - object: { - id: object::{uuid}, - name: object::{name} - }, - provenance: { - id: episode::{uuid}, - name: episode::{name} - } - } diff --git a/helixdb-cfg/schema.hx b/helixdb-cfg/schema.hx index eaedea5..dec5c6c 100644 --- a/helixdb-cfg/schema.hx +++ b/helixdb-cfg/schema.hx @@ -10,11 +10,11 @@ V::Episode { name: String, content: String, source: String, - type: String, + episodeType: String, userId: String, - createdAt: DateTime, - validAt: DateTime, - labels: Array, + createdAt: I64, + validAt: I64, + labels: [String], space: String, sessionId: String } @@ -22,9 +22,9 @@ V::Episode { V::Entity { name: String, summary: String, - type: String, - createdAt: DateTime, - attributes: String + entityType: String, + createdAt: Date, + attributes: String, userId: String, space: String } @@ -33,46 +33,46 @@ V::Entity { // This allows tracking validity periods, provenance, and treating facts as objects themselves V::Statement { fact: String, - createdAt: DateTime, - validAt: DateTime, - invalidAt: DateTime, - attributes: String + createdAt: Date, + validAt: Date, + invalidAt: Date, + attributes: String, userId: String, space: String } // Subject of the statement (the entity the statement is about) E::HasSubject { - To: Entity, From: Statement, + To: Entity, Properties: { - createdAt: DateTime + createdAt: Date } } // Object of the statement (the entity that receives the action or is related to) E::HasObject { - To: Entity, From: Statement, + To: Entity, Properties: { - createdAt: DateTime + createdAt: Date } } // Predicate of the statement (the relationship type or verb) E::HasPredicate { - To: Entity, From: Statement, + To: Entity, Properties: { - createdAt: DateTime + createdAt: Date } } // Provenance connection - links a statement to its source episode E::HasProvenance { - To: Episode, From: Statement, + To: Episode, Properties: { - createdAt: DateTime + createdAt: Date } } \ No newline at end of file diff --git a/packages/database/prisma/migrations/20250604101446_space_optional_in_queue/migration.sql b/packages/database/prisma/migrations/20250604101446_space_optional_in_queue/migration.sql new file mode 100644 index 0000000..de5ad87 --- /dev/null +++ b/packages/database/prisma/migrations/20250604101446_space_optional_in_queue/migration.sql @@ -0,0 +1,8 @@ +-- DropForeignKey +ALTER TABLE "IngestionQueue" DROP CONSTRAINT "IngestionQueue_spaceId_fkey"; + +-- AlterTable +ALTER TABLE "IngestionQueue" ALTER COLUMN "spaceId" DROP NOT NULL; + +-- AddForeignKey +ALTER TABLE "IngestionQueue" ADD CONSTRAINT "IngestionQueue_spaceId_fkey" FOREIGN KEY ("spaceId") REFERENCES "Space"("id") ON DELETE SET NULL ON UPDATE CASCADE; diff --git a/packages/database/prisma/schema.prisma b/packages/database/prisma/schema.prisma index 60a102c..9a3f085 100644 --- a/packages/database/prisma/schema.prisma +++ b/packages/database/prisma/schema.prisma @@ -167,8 +167,8 @@ model IngestionQueue { id String @id @default(cuid()) // Relations - space Space @relation(fields: [spaceId], references: [id]) - spaceId String + space Space? @relation(fields: [spaceId], references: [id]) + spaceId String? // Queue metadata data Json // The actual data to be processed diff --git a/packages/types/src/graph/graph.entity.ts b/packages/types/src/graph/graph.entity.ts index f857b82..055fbf2 100644 --- a/packages/types/src/graph/graph.entity.ts +++ b/packages/types/src/graph/graph.entity.ts @@ -1,7 +1,6 @@ export enum EpisodeType { - Message = "message", - Code = "code", - Documentation = "documentation", + Conversation = "CONVERSATION", + Text = "TEXT", } export interface AddEpisodeParams { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0898371..180d644 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -81,6 +81,9 @@ importers: ai: specifier: 4.3.14 version: 4.3.14(react@18.3.1)(zod@3.23.8) + bullmq: + specifier: ^5.53.2 + version: 5.53.2 class-variance-authority: specifier: ^0.7.1 version: 0.7.1 @@ -99,6 +102,9 @@ importers: helix-ts: specifier: ^1.0.4 version: 1.0.4 + ioredis: + specifier: ^5.6.1 + version: 5.6.1 isbot: specifier: ^4.1.0 version: 4.4.0 @@ -1061,6 +1067,9 @@ packages: resolution: {integrity: sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA==} deprecated: Use @eslint/object-schema instead + '@ioredis/commands@1.2.0': + resolution: {integrity: sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==} + '@isaacs/cliui@8.0.2': resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} engines: {node: '>=12'} @@ -1102,6 +1111,36 @@ packages: '@mjackson/headers@0.10.0': resolution: {integrity: sha512-U1Eu1gF979k7ZoIBsJyD+T5l9MjtPONsZfoXfktsQHPJD0s7SokBGx+tLKDLsOY+gzVYAWS0yRFDNY8cgbQzWQ==} + '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3': + resolution: {integrity: sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==} + cpu: [arm64] + os: [darwin] + + '@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.3': + resolution: {integrity: sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw==} + cpu: [x64] + os: [darwin] + + '@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.3': + resolution: {integrity: sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg==} + cpu: [arm64] + os: [linux] + + '@msgpackr-extract/msgpackr-extract-linux-arm@3.0.3': + resolution: {integrity: sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw==} + cpu: [arm] + os: [linux] + + '@msgpackr-extract/msgpackr-extract-linux-x64@3.0.3': + resolution: {integrity: sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg==} + cpu: [x64] + os: [linux] + + '@msgpackr-extract/msgpackr-extract-win32-x64@3.0.3': + resolution: {integrity: sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ==} + cpu: [x64] + os: [win32] + '@napi-rs/wasm-runtime@0.2.10': resolution: {integrity: sha512-bCsCyeZEwVErsGmyPNSzwfwFn4OdxBj0mmv6hOFucB/k81Ojdu68RbZdxYsRQUPc9l6SU5F/cG+bXgWs3oUgsQ==} @@ -2126,6 +2165,9 @@ packages: buffer@5.7.1: resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==} + bullmq@5.53.2: + resolution: {integrity: sha512-xHgxrP/yNJHD7VCw1h+eRBh+2TCPBCM39uC9gCyksYc6ufcJP+HTZ/A2lzB2x7qMFWrvsX7tM40AT2BmdkYL/Q==} + bytes@3.1.2: resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} engines: {node: '>= 0.8'} @@ -2248,6 +2290,10 @@ packages: resolution: {integrity: sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==} engines: {node: '>=6'} + cluster-key-slot@1.1.2: + resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==} + engines: {node: '>=0.10.0'} + color-convert@1.9.3: resolution: {integrity: sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==} @@ -2336,6 +2382,10 @@ packages: typescript: optional: true + cron-parser@4.9.0: + resolution: {integrity: sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==} + engines: {node: '>=12.0.0'} + cross-env@7.0.3: resolution: {integrity: sha512-+/HKd6EgcQCJGh2PSjZuUitQBQynKor4wrFbRg4DtAgS1aWO+gU52xpH7M9ScGgXSYmAVS9bIJ8EzuaGw0oNAw==} engines: {node: '>=10.14', npm: '>=6', yarn: '>=1'} @@ -2482,6 +2532,10 @@ packages: defined@1.0.1: resolution: {integrity: sha512-hsBd2qSVCRE+5PmNdHt1uzyrFu5d3RwmFDKzyNZMFq/EwDNJF7Ee5+D5oEKF0hU6LhtoUF1macFvOe4AskQC1Q==} + denque@2.1.0: + resolution: {integrity: sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==} + engines: {node: '>=0.10'} + depd@2.0.0: resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==} engines: {node: '>= 0.8'} @@ -3361,6 +3415,10 @@ packages: resolution: {integrity: sha512-4gd7VpWNQNB4UKKCFFVcp1AVv+FMOgs9NKzjHKusc8jTMhd5eL1NqQqOpE0KzMds804/yHlglp3uxgluOqAPLw==} engines: {node: '>= 0.4'} + ioredis@5.6.1: + resolution: {integrity: sha512-UxC0Yv1Y4WRJiGQxQkP0hfdL0/5/6YvdfOOClRgJ0qppSarkhneSa6UvkMkms0AkdGimSH3Ikqm+6mkMmX7vGA==} + engines: {node: '>=12.22.0'} + ipaddr.js@1.9.1: resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==} engines: {node: '>= 0.10'} @@ -3767,6 +3825,12 @@ packages: lodash.debounce@4.0.8: resolution: {integrity: sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==} + lodash.defaults@4.2.0: + resolution: {integrity: sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==} + + lodash.isarguments@3.1.0: + resolution: {integrity: sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==} + lodash.isplainobject@4.0.6: resolution: {integrity: sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==} @@ -3812,6 +3876,10 @@ packages: peerDependencies: react: ^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0 + luxon@3.6.1: + resolution: {integrity: sha512-tJLxrKJhO2ukZ5z0gyjY1zPh3Rh88Ej9P7jNrZiHMUXHae1yvI2imgOZtL1TO8TW6biMMKfTtAOoEJANgtWBMQ==} + engines: {node: '>=12'} + lz-string@1.5.0: resolution: {integrity: sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ==} hasBin: true @@ -4108,6 +4176,13 @@ packages: ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + msgpackr-extract@3.0.3: + resolution: {integrity: sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA==} + hasBin: true + + msgpackr@1.11.4: + resolution: {integrity: sha512-uaff7RG9VIC4jacFW9xzL3jc0iM32DNHe4jYVycBcjUePT/Klnfj7pqtWJt9khvDFizmjN2TlYniYmSS2LIaZg==} + nanoid@3.3.8: resolution: {integrity: sha512-WNLf5Sd8oZxOm+TzppcYk8gVOgP+l58xNy58D0nbUnOxOWRWvlcCV4kUF7ltmI6PsrLl/BgKEyS4mqsGChFN0w==} engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} @@ -4132,6 +4207,9 @@ packages: resolution: {integrity: sha512-myRT3DiWPHqho5PrJaIRyaMv2kgYf0mUVgBNOYMuCH5Ki1yEiQaf/ZJuQ62nvpc44wL5WDbTX7yGJi1Neevw8w==} engines: {node: '>= 0.6'} + node-abort-controller@3.1.1: + resolution: {integrity: sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==} + node-emoji@1.11.0: resolution: {integrity: sha512-wo2DpQkQp7Sjm2A0cq+sN7EHKO6Sl0ctXeBdFZrL9T9+UywORbufTcTZxom8YqpLQt/FqNMUkOpkZrJVYSKD3A==} @@ -4144,6 +4222,10 @@ packages: encoding: optional: true + node-gyp-build-optional-packages@5.2.2: + resolution: {integrity: sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==} + hasBin: true + node-releases@2.0.19: resolution: {integrity: sha512-xxOWJsBKtzAq7DY0J+DTzuz58K8e7sJbdgwkbMWQe8UYB6ekmsQ45q0M/tJDsGaZmbC+l7n57UV8Hl5tHxO9uw==} @@ -4763,6 +4845,14 @@ packages: resolution: {integrity: sha512-6tDA8g98We0zd0GvVeMT9arEOnTw9qM03L9cJXaCjrip1OO764RDBLBfrB4cwzNGDj5OA5ioymC9GkizgWJDUg==} engines: {node: '>=8'} + redis-errors@1.2.0: + resolution: {integrity: sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==} + engines: {node: '>=4'} + + redis-parser@3.0.0: + resolution: {integrity: sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==} + engines: {node: '>=4'} + reduce-css-calc@2.1.8: resolution: {integrity: sha512-8liAVezDmUcH+tdzoEGrhfbGcP7nOV4NkGE3a74+qqvE7nt9i4sKLGBuZNOnpI4WiGksiNPklZxva80061QiPg==} @@ -5085,6 +5175,9 @@ packages: stable-hash@0.0.5: resolution: {integrity: sha512-+L3ccpzibovGXFK+Ap/f8LOS0ahMrHTf3xu7mMLSpEGU0EO9ucaysSylKo9eRDFNhWve/y275iPmIZ4z39a9iA==} + standard-as-callback@2.1.0: + resolution: {integrity: sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==} + statuses@2.0.1: resolution: {integrity: sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==} engines: {node: '>= 0.8'} @@ -5510,6 +5603,10 @@ packages: resolution: {integrity: sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==} engines: {node: '>= 0.4.0'} + uuid@9.0.1: + resolution: {integrity: sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==} + hasBin: true + uvu@0.5.6: resolution: {integrity: sha512-+g8ENReyr8YsOc6fv/NVJs2vFdHBnBNdfE49rshrTzDWOlUx4Gq7KOS2GD8eqhy2j+Ejq29+SbKH8yjkAqXqoA==} engines: {node: '>=8'} @@ -6509,6 +6606,8 @@ snapshots: '@humanwhocodes/object-schema@2.0.3': {} + '@ioredis/commands@1.2.0': {} + '@isaacs/cliui@8.0.2': dependencies: string-width: 5.1.2 @@ -6581,6 +6680,24 @@ snapshots: '@mjackson/headers@0.10.0': {} + '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3': + optional: true + + '@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.3': + optional: true + + '@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.3': + optional: true + + '@msgpackr-extract/msgpackr-extract-linux-arm@3.0.3': + optional: true + + '@msgpackr-extract/msgpackr-extract-linux-x64@3.0.3': + optional: true + + '@msgpackr-extract/msgpackr-extract-win32-x64@3.0.3': + optional: true + '@napi-rs/wasm-runtime@0.2.10': dependencies: '@emnapi/core': 1.4.3 @@ -7788,6 +7905,18 @@ snapshots: base64-js: 1.5.1 ieee754: 1.2.1 + bullmq@5.53.2: + dependencies: + cron-parser: 4.9.0 + ioredis: 5.6.1 + msgpackr: 1.11.4 + node-abort-controller: 3.1.1 + semver: 7.7.2 + tslib: 2.8.1 + uuid: 9.0.1 + transitivePeerDependencies: + - supports-color + bytes@3.1.2: {} cac@6.7.14: {} @@ -7911,6 +8040,8 @@ snapshots: clsx@2.1.1: {} + cluster-key-slot@1.1.2: {} + color-convert@1.9.3: dependencies: color-name: 1.1.3 @@ -7990,6 +8121,10 @@ snapshots: optionalDependencies: typescript: 5.8.3 + cron-parser@4.9.0: + dependencies: + luxon: 3.6.1 + cross-env@7.0.3: dependencies: cross-spawn: 7.0.6 @@ -8132,6 +8267,8 @@ snapshots: defined@1.0.1: {} + denque@2.1.0: {} + depd@2.0.0: {} dequal@2.0.3: {} @@ -9271,6 +9408,20 @@ snapshots: hasown: 2.0.2 side-channel: 1.1.0 + ioredis@5.6.1: + dependencies: + '@ioredis/commands': 1.2.0 + cluster-key-slot: 1.1.2 + debug: 4.4.1 + denque: 2.1.0 + lodash.defaults: 4.2.0 + lodash.isarguments: 3.1.0 + redis-errors: 1.2.0 + redis-parser: 3.0.0 + standard-as-callback: 2.1.0 + transitivePeerDependencies: + - supports-color + ipaddr.js@1.9.1: {} is-alphabetical@2.0.1: {} @@ -9628,6 +9779,10 @@ snapshots: lodash.debounce@4.0.8: {} + lodash.defaults@4.2.0: {} + + lodash.isarguments@3.1.0: {} + lodash.isplainobject@4.0.6: {} lodash.merge@4.6.2: {} @@ -9666,6 +9821,8 @@ snapshots: dependencies: react: 18.3.1 + luxon@3.6.1: {} + lz-string@1.5.0: {} magic-string@0.30.17: @@ -10137,6 +10294,22 @@ snapshots: ms@2.1.3: {} + msgpackr-extract@3.0.3: + dependencies: + node-gyp-build-optional-packages: 5.2.2 + optionalDependencies: + '@msgpackr-extract/msgpackr-extract-darwin-arm64': 3.0.3 + '@msgpackr-extract/msgpackr-extract-darwin-x64': 3.0.3 + '@msgpackr-extract/msgpackr-extract-linux-arm': 3.0.3 + '@msgpackr-extract/msgpackr-extract-linux-arm64': 3.0.3 + '@msgpackr-extract/msgpackr-extract-linux-x64': 3.0.3 + '@msgpackr-extract/msgpackr-extract-win32-x64': 3.0.3 + optional: true + + msgpackr@1.11.4: + optionalDependencies: + msgpackr-extract: 3.0.3 + nanoid@3.3.8: {} napi-postinstall@0.2.4: {} @@ -10149,6 +10322,8 @@ snapshots: negotiator@0.6.4: {} + node-abort-controller@3.1.1: {} + node-emoji@1.11.0: dependencies: lodash: 4.17.21 @@ -10157,6 +10332,11 @@ snapshots: dependencies: whatwg-url: 5.0.0 + node-gyp-build-optional-packages@5.2.2: + dependencies: + detect-libc: 2.0.4 + optional: true + node-releases@2.0.19: {} non.geist@1.0.4: {} @@ -10748,6 +10928,12 @@ snapshots: indent-string: 4.0.0 strip-indent: 3.0.0 + redis-errors@1.2.0: {} + + redis-parser@3.0.0: + dependencies: + redis-errors: 1.2.0 + reduce-css-calc@2.1.8: dependencies: css-unit-converter: 1.1.2 @@ -11112,6 +11298,8 @@ snapshots: stable-hash@0.0.5: {} + standard-as-callback@2.1.0: {} + statuses@2.0.1: {} stop-iteration-iterator@1.1.0: @@ -11390,8 +11578,7 @@ snapshots: tslib@1.14.1: {} - tslib@2.8.1: - optional: true + tslib@2.8.1: {} tsutils@3.21.0(typescript@5.8.3): dependencies: @@ -11612,6 +11799,8 @@ snapshots: utils-merge@1.0.1: {} + uuid@9.0.1: {} + uvu@0.5.6: dependencies: dequal: 2.0.3 diff --git a/turbo.json b/turbo.json index b4c1954..8b16422 100644 --- a/turbo.json +++ b/turbo.json @@ -58,6 +58,9 @@ "AUTH_GOOGLE_CLIENT_SECRET", "APP_ENV", "APP_LOG_LEVEL", - "ENCRYPTION_KEY" + "ENCRYPTION_KEY", + "REDIS_HOST", + "REDIS_PORT", + "REDIS_TLS_DISABLED" ] }