Improve GitHub notification messages and add README for integrations

This commit is contained in:
Manoj 2025-09-22 19:58:21 +05:30 committed by GitHub
parent 59620151f2
commit 812d7dea51
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 685 additions and 182 deletions

View File

@ -115,10 +115,12 @@ Ask questions like "What are my writing preferences?" with instant insights from
![chat-with-memory](https://github.com/user-attachments/assets/d798802f-bd51-4daf-b2b5-46de7d206f66)
### ⚡ **Auto-Sync from Apps**:
### ⚡ **Auto-Sync from Apps**:
Automatically capture relevant context from Linear, Slack, Notion, GitHub and other connected apps into your CORE memory
📖 **[View All Integrations](./integrations/README.md)** - Complete list of supported services and their features
![core-slack](https://github.com/user-attachments/assets/d5fefe38-221e-4076-8a44-8ed673960f03)

View File

@ -47,7 +47,7 @@ export function useLogs({ endpoint, source, status, type }: UseLogsOptions) {
(pageNum: number) => {
const params = new URLSearchParams();
params.set("page", pageNum.toString());
params.set("limit", "5");
params.set("limit", "50");
if (source) params.set("source", source);
if (status) params.set("status", status);
if (type) params.set("type", type);

View File

@ -1,150 +1,157 @@
import { type LoaderFunctionArgs, json } from "@remix-run/node";
import { z } from "zod";
import { prisma } from "~/db.server";
import { requireUserId } from "~/services/session.server";
import { createHybridLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server";
/**
* Optimizations:
* - Use `findMany` with `select` instead of `include` to fetch only required fields.
* - Use `count` with the same where clause, but only after fetching logs (to avoid unnecessary count if no logs).
* - Use a single query for availableSources with minimal fields.
* - Avoid unnecessary object spreading and type casting.
* - Minimize nested object traversal in mapping.
*/
export async function loader({ request }: LoaderFunctionArgs) {
const userId = await requireUserId(request);
const url = new URL(request.url);
// Schema for logs search parameters
const LogsSearchParams = z.object({
page: z.string().optional(),
limit: z.string().optional(),
source: z.string().optional(),
status: z.string().optional(),
type: z.string().optional(),
});
const page = parseInt(url.searchParams.get("page") || "1");
const limit = parseInt(url.searchParams.get("limit") || "100");
const source = url.searchParams.get("source");
const status = url.searchParams.get("status");
const type = url.searchParams.get("type");
const skip = (page - 1) * limit;
export const loader = createHybridLoaderApiRoute(
{
allowJWT: true,
searchParams: LogsSearchParams,
corsStrategy: "all",
findResource: async () => 1,
},
async ({ authentication, searchParams }) => {
const page = parseInt(searchParams.page || "1");
const limit = parseInt(searchParams.limit || "100");
const source = searchParams.source;
const status = searchParams.status;
const type = searchParams.type;
const skip = (page - 1) * limit;
// Get user and workspace in one query
const user = await prisma.user.findUnique({
where: { id: userId },
select: { Workspace: { select: { id: true } } },
});
// Get user and workspace in one query
const user = await prisma.user.findUnique({
where: { id: authentication.userId },
select: { Workspace: { select: { id: true } } },
});
if (!user?.Workspace) {
throw new Response("Workspace not found", { status: 404 });
}
if (!user?.Workspace) {
throw new Response("Workspace not found", { status: 404 });
}
// Build where clause for filtering
const whereClause: any = {
workspaceId: user.Workspace.id,
};
if (status) {
whereClause.status = status;
}
if (type) {
whereClause.data = {
path: ["type"],
equals: type,
// Build where clause for filtering
const whereClause: any = {
workspaceId: user.Workspace.id,
};
}
// If source filter is provided, filter by integration source
if (source) {
whereClause.activity = {
integrationAccount: {
integrationDefinition: {
slug: source,
if (status) {
whereClause.status = status;
}
if (type) {
whereClause.data = {
path: ["type"],
equals: type,
};
}
// If source filter is provided, filter by integration source
if (source) {
whereClause.activity = {
integrationAccount: {
integrationDefinition: {
slug: source,
},
},
},
};
}
};
}
// Use select to fetch only required fields for logs
const [logs, totalCount, availableSources] = await Promise.all([
prisma.ingestionQueue.findMany({
where: whereClause,
select: {
id: true,
createdAt: true,
processedAt: true,
status: true,
error: true,
type: true,
output: true,
data: true,
activity: {
select: {
text: true,
sourceURL: true,
integrationAccount: {
select: {
integrationDefinition: {
select: {
name: true,
slug: true,
// Use select to fetch only required fields for logs
const [logs, totalCount, availableSources] = await Promise.all([
prisma.ingestionQueue.findMany({
where: whereClause,
select: {
id: true,
createdAt: true,
processedAt: true,
status: true,
error: true,
type: true,
output: true,
data: true,
activity: {
select: {
text: true,
sourceURL: true,
integrationAccount: {
select: {
integrationDefinition: {
select: {
name: true,
slug: true,
},
},
},
},
},
},
},
},
orderBy: {
createdAt: "desc",
},
skip,
take: limit,
}),
orderBy: {
createdAt: "desc",
},
skip,
take: limit,
}),
prisma.ingestionQueue.count({
where: whereClause,
}),
prisma.ingestionQueue.count({
where: whereClause,
}),
prisma.integrationDefinitionV2.findMany({
where: {
IntegrationAccount: {
some: {
workspaceId: user.Workspace.id,
prisma.integrationDefinitionV2.findMany({
where: {
IntegrationAccount: {
some: {
workspaceId: user.Workspace.id,
},
},
},
},
select: {
name: true,
slug: true,
},
}),
]);
select: {
name: true,
slug: true,
},
}),
]);
// Format the response
const formattedLogs = logs.map((log) => {
const integrationDef =
log.activity?.integrationAccount?.integrationDefinition;
const logData = log.data as any;
// Format the response
const formattedLogs = logs.map((log) => {
const integrationDef =
log.activity?.integrationAccount?.integrationDefinition;
const logData = log.data as any;
return {
id: log.id,
source: integrationDef?.name || logData?.source || "Unknown",
ingestText:
log.activity?.text ||
logData?.episodeBody ||
logData?.text ||
"No content",
time: log.createdAt,
processedAt: log.processedAt,
episodeUUID: (log.output as any)?.episodeUuid,
status: log.status,
error: log.error,
sourceURL: log.activity?.sourceURL,
integrationSlug: integrationDef?.slug,
data: log.data,
};
});
return {
id: log.id,
source: integrationDef?.name || logData?.source || "Unknown",
ingestText:
log.activity?.text ||
logData?.episodeBody ||
logData?.text ||
"No content",
time: log.createdAt,
processedAt: log.processedAt,
episodeUUID: (log.output as any)?.episodeUuid,
status: log.status,
error: log.error,
sourceURL: log.activity?.sourceURL,
integrationSlug: integrationDef?.slug,
data: log.data,
};
});
return json({
logs: formattedLogs,
totalCount,
page,
limit,
hasMore: skip + logs.length < totalCount,
availableSources,
});
}
return json({
logs: formattedLogs,
totalCount,
page,
limit,
hasMore: skip + logs.length < totalCount,
availableSources,
});
},
);

View File

@ -50,7 +50,7 @@ export const extensionSearch = task({
{ query },
{
headers: {
Authorization: `Bearer rc_pat_v41311t6trhr3c8sc7ap4hsbhp6pwsstzyunaazq`,
Authorization: `Bearer ${pat.token}`,
},
},
);

135
integrations/README.md Normal file
View File

@ -0,0 +1,135 @@
# CORE Integrations
Integrations connect external services to CORE's knowledge graph, automatically capturing activities and context to build your persistent memory layer.
## Available Integrations
### 🐙 [GitHub](./github/README.md)
Tracks your GitHub activities and notifications, including PRs, issues, comments, and repository events.
**Features:**
- Pull request creation, comments, and reviews
- Issue tracking and assignments
- Notification processing (mentions, reviews, assignments)
- Repository watching and subscriptions
- Team mentions and state changes
### 📐 [Linear](./linear/README.md)
Project management and issue tracking integration.
**Features:**
- Issue creation and updates
- Project milestone tracking
- Team assignments and workflows
### 💬 [Slack](./slack/README.md)
Workspace communication and activity tracking.
**Features:**
- Channel message monitoring
- Direct message tracking
- Thread participation
## How Integrations Work
1. **Authentication**: OAuth2 setup with service-specific scopes
2. **Data Collection**: Either scheduled sync (every 5 minutes) or real-time webhooks when supported
3. **Event Processing**: Converting activities into structured events
4. **Entity Extraction**: Identifying users, projects, repositories, etc.
5. **Knowledge Graph Ingestion**: Creating episodes and relationships in CORE
## Common Features
### 🔄 Data Collection Methods
- **Scheduled Sync**: Periodic API polling (every 5 minutes) for services like GitHub and Linear
- **Real-time Webhooks**: Instant event delivery for services that support personal webhooks (like Slack)
- **Incremental Updates**: Only fetch new activities since last sync
- **Deduplication**: Prevent duplicate events from being processed
- **Rate Limiting**: Respect API limits with intelligent backoff
- **Error Handling**: Graceful degradation on service outages
### 📊 Activity Tracking
- **User Actions**: What you created, commented, or modified
- **Mentions**: When others reference you in discussions
- **Assignments**: Tasks or issues assigned to you
- **State Changes**: Status updates on projects you follow
### 🧠 Knowledge Graph Integration
- **Entities**: People, projects, repositories, issues, organizations
- **Relationships**: Created, commented, assigned, mentioned, collaborated
- **Temporal Context**: When events occurred and their sequence
- **Cross-Integration Links**: Connections between different services
## Event Format
All integrations generate events in a consistent format for knowledge graph ingestion:
```typescript
{
text: "{actor} {action} {object} in {context}: {details}",
sourceURL: "https://service.com/link/to/event",
timestamp: "2025-01-20T10:30:00Z",
integration: "github" | "linear" | "slack"
}
```
### Example Events
```
john_doe created PR #123 in facebook/react: Fix memory leak in hooks
alice_smith mentioned manoj_k in linear/project issue #456: Can you review?
team mentioned manoj_k's team in slack/engineering: Weekly standup reminder
```
## Configuration
Each integration requires:
1. **OAuth Setup**: Service-specific authentication
2. **Scope Configuration**: Permissions for data access
3. **Sync Schedule**: How frequently to check for updates
4. **Filtering Rules**: What events to include/exclude
## Development Guide
### Adding New Integrations
1. **Create Integration Directory**
```bash
mkdir integrations/{service-name}
cd integrations/{service-name}
```
2. **Required Files**
```
src/
├── index.ts # Main entry point
├── schedule.ts # Sync logic and event processing
├── utils.ts # API utilities
├── account-create.ts # Authentication setup
└── README.md # Integration documentation
```
3. **Core Implementation**
- Extend `IntegrationCLI` class
- Implement OAuth2 authentication
- Define sync schedule and event processing
- Handle API rate limits and errors
4. **Event Processing**
- Convert service events to standard format
- Extract entities and relationships
- Ensure consistent naming and structure
- Add deduplication logic

View File

@ -0,0 +1,72 @@
# GitHub Integration
Automatic GitHub activity tracking and notification processing for CORE memory system.
## Overview
The GitHub integration captures your GitHub activities and notifications, processes them into structured events, and ingests them into CORE's knowledge graph for intelligent memory and context building.
## Features
### 📊 Activity Tracking
- **Pull Requests**: Created, commented, reviewed
- **Issues**: Created, assigned, commented, self-assigned
- **Comments**: PR comments, issue comments
- **Repository Events**: Watching, subscribing, state changes
### 🔔 Notification Processing
- **Assignments**: Issues and PRs assigned to you
- **Reviews**: PR review requests
- **Mentions**: @mentions in discussions
- **Comments**: New comments on your content
- **State Changes**: PR/issue open/close/merge events
- **Subscriptions**: Updates on watched repositories
- **Team Mentions**: Team @mentions
### 🔗 MCP Integration
- Uses GitHub Copilot MCP server for extended functionality
- Provides seamless integration with GitHub's AI tools
## Authentication
Uses OAuth2 with the following scopes:
- `user` - Access user profile information
- `public_repo` - Access public repositories
- `repo` - Access private repositories
- `notifications` - Read notifications
- `gist` - Access gists
- `read:org` - Read organization membership
- `repo_hooks` - Manage repository webhooks
## Configuration
### Schedule
- **Frequency**: Every 5 minutes (`*/5 * * * *`)
- **Sync Window**: 24 hours (configurable)
- **Rate Limiting**: Built-in GitHub API rate limit handling
### Data Processing
- **Deduplication**: Filters out duplicate events using timestamps
- **Entity Extraction**: Extracts users, repositories, PR/issue numbers
- **Relationship Mapping**: Creates connections between entities
- **Temporal Tracking**: Maintains event chronology
## Event Types
### User Activities
```
{username} created PR #{number} in {repo}: {title}
{username} created issue #{number} in {repo}: {title}
{username} commented on PR #{number} in {repo}: {title}
{username} commented on issue #{number} in {repo}: {title}
{username} assigned themselves to issue #{number} in {repo}: {title}
```
### Notifications
```
Issue #{number} assigned to {username} in {repo}: {title}
{actor} commented on {username}'s PR #{number} in {repo}: {body}
{actor} mentioned {username} in {repo} issue #{number}: {body}
{username} requested to review PR #{number} in {repo}: {title}
{actor} changed PR #{number} state to {state} in {repo}: {title}
```

View File

@ -1,6 +1,6 @@
{
"name": "@core/github",
"version": "0.1.0",
"version": "0.1.1",
"description": "github extension for CORE",
"main": "./bin/index.js",
"module": "./bin/index.mjs",

View File

@ -9,8 +9,8 @@ importers:
.:
dependencies:
'@redplanethq/sdk':
specifier: 0.1.1
version: 0.1.1
specifier: 0.1.2
version: 0.1.2
axios:
specifier: ^1.7.9
version: 1.11.0
@ -466,8 +466,8 @@ packages:
resolution: {integrity: sha512-QNqXyfVS2wm9hweSYD2O7F0G06uurj9kZ96TRQE5Y9hU7+tgdZwIkbAKc5Ocy1HxEY2kuDQa6cQ1WRs/O5LFKA==}
engines: {node: ^12.20.0 || ^14.18.0 || >=16.0.0}
'@redplanethq/sdk@0.1.1':
resolution: {integrity: sha512-tfR1c9p7vNeCL5jsF9QlEZcRFLsihaHe/ZQWVKZYXzAZ6GugoIFBaayGfVvjNjyEnN3nlrl3usKBX+hhaKzg0g==}
'@redplanethq/sdk@0.1.2':
resolution: {integrity: sha512-Si+ae2OV0UNy7yvLECjJ1Y/6HHhRO0yU9svIFvBzMlaR/57SDP+KJZLyfWx/PJX+hOuWipyoYTLA9aYULt6z2w==}
engines: {node: '>=18.0.0'}
'@rollup/rollup-android-arm-eabi@4.46.2':
@ -2455,7 +2455,7 @@ snapshots:
'@pkgr/core@0.2.9': {}
'@redplanethq/sdk@0.1.1':
'@redplanethq/sdk@0.1.2':
dependencies:
commander: 14.0.0

View File

@ -46,7 +46,7 @@ async function fetchUserInfo(accessToken: string) {
/**
* Processes GitHub notifications into activity messages
*/
async function processNotifications(accessToken: string, lastSyncTime: string): Promise<any[]> {
async function processNotifications(accessToken: string, lastSyncTime: string, username: string): Promise<any[]> {
const activities = [];
const allowedReasons = [
'assign',
@ -117,35 +117,36 @@ async function processNotifications(accessToken: string, lastSyncTime: string):
switch (notification.reason) {
case 'assign':
title = `${isIssue ? 'Issue' : 'PR'} assigned to you: #${githubData.number} - ${githubData.title}`;
title = `${isIssue ? 'Issue' : 'PR'} #${githubData.number} assigned to ${username} in ${repository.full_name}: ${githubData.title}`;
break;
case 'author':
if (isComment) {
title = `New comment on your ${isIssue ? 'issue' : 'PR'} by ${githubData.user?.login}: ${githubData.body}`;
title = `${githubData.user?.login} commented on ${username}'s ${isIssue ? 'issue' : 'PR'} #${githubData.number} in ${repository.full_name}: ${githubData.body}`;
} else {
title = `You created this ${isIssue ? 'issue' : 'PR'}: #${githubData.number} - ${githubData.title}`;
title = `${username} created ${isIssue ? 'issue' : 'PR'} #${githubData.number} in ${repository.full_name}: ${githubData.title}`;
}
break;
case 'comment':
title = `New comment by ${githubData.user?.login} in ${repository.full_name}: ${githubData.body}`;
title = `${githubData.user?.login} commented on ${isIssue ? 'issue' : 'PR'} #${githubData.number} in ${repository.full_name}: ${githubData.body}`;
break;
case 'manual':
title = `You subscribed to: #${githubData.number} - ${githubData.title}`;
title = `${username} subscribed to ${isIssue ? 'issue' : 'PR'} #${githubData.number} in ${repository.full_name}: ${githubData.title}`;
break;
case 'mention':
title = `@mentioned by ${githubData.user?.login} in ${repository.full_name}: ${githubData.body}`;
title = `${githubData.user?.login} mentioned ${username} in ${repository.full_name} ${isIssue ? 'issue' : 'PR'} #${githubData.number}: ${githubData.body}`;
break;
case 'review_requested':
title = `PR review requested in ${repository.full_name}: #${githubData.number} - ${githubData.title}`;
title = `${username} requested to review PR #${githubData.number} in ${repository.full_name}: ${githubData.title}`;
break;
case 'state_change': {
let stateInfo = '';
let actor = githubData.user?.login || 'someone';
if (githubData.state) {
stateInfo = `to ${githubData.state}`;
} else if (githubData.merged) {
@ -153,28 +154,28 @@ async function processNotifications(accessToken: string, lastSyncTime: string):
} else if (githubData.closed_at) {
stateInfo = 'to closed';
}
title = `State changed ${stateInfo} in ${repository.full_name}: #${githubData.number} - ${githubData.title}`;
title = `${actor} changed ${isIssue ? 'issue' : 'PR'} #${githubData.number} state ${stateInfo} in ${repository.full_name}: ${githubData.title}`;
break;
}
case 'subscribed':
if (isComment) {
title = `New comment on watched ${isIssue ? 'issue' : 'PR'} in ${repository.full_name} by ${githubData.user?.login}: ${githubData.body}`;
title = `${githubData.user?.login} commented on watched ${isIssue ? 'issue' : 'PR'} #${githubData.number} in ${repository.full_name}: ${githubData.body}`;
} else if (isPullRequest) {
title = `New PR created in watched repo ${repository.full_name}: #${githubData.number} - ${githubData.title}`;
title = `New PR #${githubData.number} created in watched repo ${repository.full_name}: ${githubData.title}`;
} else if (isIssue) {
title = `New issue created in watched repo ${repository.full_name}: #${githubData.number} - ${githubData.title}`;
title = `New issue #${githubData.number} created in watched repo ${repository.full_name}: ${githubData.title}`;
} else {
title = `Update in watched repo ${repository.full_name}: #${githubData.number} - ${githubData.title}`;
title = `Update on watched ${isIssue ? 'issue' : 'PR'} #${githubData.number} in ${repository.full_name}: ${githubData.title}`;
}
break;
case 'team_mention':
title = `Your team was mentioned in ${repository.full_name}`;
title = `${username}'s team mentioned in ${repository.full_name} ${isIssue ? 'issue' : 'PR'} #${githubData.number}: ${githubData.title}`;
break;
default:
title = `GitHub notification: ${repository.full_name}`;
title = `GitHub notification for ${username} in ${repository.full_name}`;
break;
}
@ -236,19 +237,19 @@ async function processUserEvents(
switch (event.type) {
case 'pr':
title = `You created PR #${event.number}: ${event.title}`;
title = `${username} created PR #${event.number}: ${event.title}`;
break;
case 'issue':
title = `You created issue #${event.number}: ${event.title}`;
title = `${username} created issue #${event.number}: ${event.title}`;
break;
case 'pr_comment':
title = `You commented on PR #${event.number}: ${event.title}`;
title = `${username} commented on PR #${event.number}: ${event.title}`;
break;
case 'issue_comment':
title = `You commented on issue #${event.number}: ${event.title}`;
title = `${username} commented on issue #${event.number}: ${event.title}`;
break;
case 'self_assigned_issue':
title = `You assigned yourself to issue #${event.number}: ${event.title}`;
title = `${username} assigned themselves to issue #${event.number}: ${event.title}`;
break;
default:
title = `GitHub activity: ${event.title || 'Unknown'}`;
@ -319,6 +320,7 @@ export async function handleSchedule(config: any, state: any) {
const notificationActivities = await processNotifications(
integrationConfiguration.access_token,
lastSyncTime,
settings.username || 'user',
);
messages.push(...notificationActivities);
} catch (error) {

View File

@ -33,22 +33,22 @@ export async function getUserEvents(
] = await Promise.all([
// Search for PRs created by user
getGithubData(
`https://api.github.com/search/issues?q=author:${username}+type:pr+created:>${formattedDate}&sort=created&order=desc&page=${page}&per_page=10`,
`https://api.github.com/search/issues?q=author:${username}+type:pr+created:>=${formattedDate}&sort=created&order=desc&page=${page}&per_page=10`,
accessToken,
),
// Search for issues created by user
getGithubData(
`https://api.github.com/search/issues?q=author:${username}+type:issue+created:>${formattedDate}&sort=created&order=desc&page=${page}&per_page=10`,
`https://api.github.com/search/issues?q=author:${username}+type:issue+created:>=${formattedDate}&sort=created&order=desc&page=${page}&per_page=10`,
accessToken,
),
// Search for issues/PRs the user commented on
getGithubData(
`https://api.github.com/search/issues?q=commenter:${username}+updated:>${formattedDate}&sort=updated&order=desc&page=${page}&per_page=10`,
`https://api.github.com/search/issues?q=commenter:${username}+updated:>=${formattedDate}&sort=updated&order=desc&page=${page}&per_page=10`,
accessToken,
),
// Search for issues assigned to the user and authored by the user (self-assigned)
getGithubData(
`https://api.github.com/search/issues?q=assignee:${username}+author:${username}+type:issue+updated:>${formattedDate}&sort=updated&order=desc&page=${page}&per_page=10`,
`https://api.github.com/search/issues?q=assignee:${username}+author:${username}+type:issue+updated:>=${formattedDate}&sort=updated&order=desc&page=${page}&per_page=10`,
accessToken,
),
]);
@ -58,17 +58,26 @@ export async function getUserEvents(
console.log('Comments found:', commentsResponse?.items?.length || 0);
console.log('Self-assigned issues found:', assignedIssuesResponse?.items?.length || 0);
// Filter function to exclude items older than the exact since timestamp
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const filterBySince = (item: any) => {
if (!since) return true;
const itemDate = new Date(item.created_at || item.updated_at);
const sinceDate = new Date(since);
return itemDate >= sinceDate;
};
// Return simplified results - combine PRs, issues, commented items, and self-assigned issues
const results = [
// eslint-disable-next-line @typescript-eslint/no-explicit-any
...(prsResponse?.items || []).map((item: any) => ({ ...item, type: 'pr' })),
...(prsResponse?.items || []).filter(filterBySince).map((item: any) => ({ ...item, type: 'pr' })),
...(issuesResponse?.items || [])
// eslint-disable-next-line @typescript-eslint/no-explicit-any
.filter((item: any) => !item.pull_request)
.filter((item: any) => !item.pull_request && filterBySince(item))
// eslint-disable-next-line @typescript-eslint/no-explicit-any
.map((item: any) => ({ ...item, type: 'issue' })),
// eslint-disable-next-line @typescript-eslint/no-explicit-any
...(commentsResponse?.items || []).map((item: any) => ({
...(commentsResponse?.items || []).filter(filterBySince).map((item: any) => ({
...item,
type: item.pull_request ? 'pr_comment' : 'issue_comment',
})),
@ -76,9 +85,9 @@ export async function getUserEvents(
...(assignedIssuesResponse?.items || [])
// eslint-disable-next-line @typescript-eslint/no-explicit-any
.filter((item: any) => {
// Only include if not already in issuesResponse (by id)
// Only include if not already in issuesResponse (by id) and passes since filter
// eslint-disable-next-line @typescript-eslint/no-explicit-any
return !(issuesResponse?.items || []).some((issue: any) => issue.id === item.id);
return !(issuesResponse?.items || []).some((issue: any) => issue.id === item.id) && filterBySince(item);
})
// eslint-disable-next-line @typescript-eslint/no-explicit-any
.map((item: any) => ({

View File

@ -0,0 +1,107 @@
# Linear Integration
Automatic Linear activity tracking and issue management integration for CORE memory system.
## Overview
The Linear integration captures your Linear project activities, issue interactions, and team collaborations, processing them into structured events for CORE's knowledge graph to build intelligent context around your project management workflow.
## Features
### 📊 Issue Tracking
- **Issue Creation**: Track new issues you create across projects
- **Issue Updates**: Monitor changes to issue status, priority, and assignments
- **Comments**: Capture comments on issues you're involved with
- **Assignments**: Track issues assigned to you or by you
### 🎯 Project Management
- **Project Milestones**: Monitor progress on project goals and deadlines
- **Team Workflows**: Track team assignments and collaboration patterns
- **Status Changes**: Issue state transitions (todo, in progress, done, etc.)
- **Priority Updates**: Changes to issue priority and urgency
### 🔗 MCP Integration
- Direct Linear MCP server integration at `https://mcp.linear.app/mcp`
- Provides enhanced functionality for Linear operations
- Seamless integration with Linear's GraphQL API
## Authentication
Uses **API Key** authentication:
- Requires Linear Personal API Token
- Full access to your Linear workspace data
- Secure token-based authentication
## Configuration
### Schedule
- **Frequency**: Every 5 minutes (`*/5 * * * *`)
- **Sync Types**: Issues, comments, and user actions
- **Incremental Sync**: Tracks last sync timestamps for each data type
### Data Processing
- **GraphQL API**: Uses Linear's GraphQL endpoint for efficient data fetching
- **User Context**: Fetches your user information for proper attribution
- **Temporal Tracking**: Maintains chronological order of activities
- **Deduplication**: Prevents duplicate event processing
## Event Types
### Issue Management
```
{username} created issue in {project}: {title}
{username} updated issue #{number} in {project}: {title}
{username} commented on issue #{number} in {project}: {comment}
{username} assigned issue #{number} to {assignee} in {project}: {title}
Issue #{number} status changed to {status} in {project}: {title}
Issue #{number} priority changed to {priority} in {project}: {title}
```
### Project Activities
```
{username} created milestone in {project}: {milestone}
{username} completed milestone in {project}: {milestone}
Team assignment updated for issue #{number} in {project}: {title}
```
## Knowledge Graph Integration
Events are processed into CORE's knowledge graph with:
- **Entities**: Users, projects, issues, milestones, teams
- **Relationships**: Created, assigned, commented, completed, collaborated
- **Attributes**: Issue numbers, status, priority, timestamps
- **Context**: Project associations, team memberships, workflow stages
## Sync Management
The integration maintains separate sync timestamps for:
- **Issues**: `lastIssuesSync` - New and updated issues
- **Comments**: `lastCommentsSync` - Comment activity
- **User Actions**: `lastUserActionsSync` - Specific user-initiated activities
This ensures comprehensive coverage without missing activities across different Linear data types.
## Rate Limits & Performance
- **GraphQL Efficiency**: Single requests fetch related data
- **Pagination Support**: Handles large datasets with cursor-based pagination
- **API Rate Limits**: Respects Linear's API limits with intelligent backoff
- **Error Handling**: Graceful degradation on API failures
- **Incremental Updates**: Only fetches changes since last sync
## Usage
The integration runs automatically once configured with your Linear API key:
1. **Fetch User Context**: Get your Linear user information
2. **Query Recent Activities**: Fetch issues, comments, and actions since last sync
3. **Process Events**: Convert Linear data into standardized event format
4. **Extract Entities**: Identify users, projects, issues, and relationships
5. **Create Episodes**: Generate memory episodes for knowledge graph ingestion
6. **Update Sync State**: Save timestamps for next incremental sync
## API Reference
- **Linear GraphQL API**: https://developers.linear.app/docs/graphql/working-with-the-graphql-api
- **Authentication**: https://developers.linear.app/docs/graphql/working-with-the-graphql-api#authentication
- **Rate Limiting**: https://developers.linear.app/docs/graphql/working-with-the-graphql-api#rate-limiting

View File

@ -0,0 +1,144 @@
# Slack Integration
Real-time Slack workspace activity tracking via webhooks for CORE memory system.
## Overview
The Slack integration captures your workspace communications, reactions, and collaborations through real-time webhooks and API access, creating a comprehensive memory layer of your team interactions and project discussions.
## Features
### 💬 Message Tracking
- **Channel Messages**: Monitor messages in channels you're active in
- **Direct Messages**: Track DM conversations and threads
- **Thread Participation**: Capture threaded discussions and replies
- **Mentions**: Track when you're @mentioned in conversations
### 🎯 Team Collaboration
- **Reactions**: Track emoji reactions on messages
- **Stars**: Monitor starred messages and important content
- **Channel Management**: Track channel joins, leaves, and participation
- **Team Interactions**: Capture cross-team communication patterns
### 🔗 MCP Integration
- **Stdio MCP Server**: Uses dedicated Slack MCP server for enhanced functionality
- **Message Tools**: Supports message creation and interaction tools
- **Real-time Events**: Webhook-based instant event delivery
## Authentication
Uses **OAuth2** with comprehensive scopes:
- `stars:read` & `stars:write` - Access starred content
- `team:read` & `users:read` - Team and user information
- `channels:read` & `channels:history` - Public channel access
- `groups:read` - Private channel access (if member)
- `im:read` & `im:history` - Direct message access
- `mpim:read`, `mpim:write` & `mpim:history` - Multi-party DM access
- `chat:write` - Message sending capabilities
- `reactions:read` & `reactions:write` - Reaction access
- `users.profile:read` - User profile information
## Configuration
### Data Collection Method
- **Real-time Webhooks**: Instant event delivery (unlike scheduled sync)
- **Event Streaming**: Continuous monitoring of workspace activity
- **Context-aware**: Captures conversation context and threading
- **User-scoped**: Only accesses data you have permission to see
### MCP Server Setup
```json
{
"type": "stdio",
"url": "https://integrations.heysol.ai/slack/mcp/slack-mcp-server",
"env": {
"SLACK_MCP_XOXP_TOKEN": "${config:access_token}",
"SLACK_MCP_ADD_MESSAGE_TOOL": true
}
}
```
## Event Types
### Message Activities
```
{username} sent message in #{channel}: {message}
{username} replied to thread in #{channel}: {reply}
{username} sent DM to {recipient}: {message}
{username} mentioned {target} in #{channel}: {message}
```
### Interaction Events
```
{username} reacted with :{emoji}: to message in #{channel}
{username} starred message in #{channel}: {message}
{username} joined channel #{channel}
{username} left channel #{channel}
```
### Team Collaboration
```
{username} created channel #{channel}
{username} archived channel #{channel}
{username} updated channel #{channel} topic: {topic}
Team discussion started in #{channel} about {topic}
```
## Webhook Event Processing
The integration processes various Slack webhook events:
- **Message Events**: New messages, edits, deletions
- **Reaction Events**: Emoji reactions added/removed
- **Channel Events**: Channel creation, archiving, topic changes
- **User Events**: Joins, leaves, status changes
- **Thread Events**: Threaded conversation activity
### Event Processing Flow
1. **Webhook Receipt**: Real-time event from Slack
2. **Event Validation**: Verify event authenticity and permissions
3. **Context Enrichment**: Fetch additional message/user/channel context
4. **Activity Creation**: Generate structured activity event
5. **Knowledge Graph**: Send to CORE for entity extraction
## Knowledge Graph Integration
Events create rich relationships in CORE's knowledge graph:
- **Entities**: Users, channels, messages, teams, workspaces
- **Relationships**: Sent, replied, mentioned, reacted, joined, starred
- **Attributes**: Timestamps, message content, reaction types, channel topics
- **Context**: Workspace culture, team dynamics, project discussions
## Privacy & Security
### Data Access
- **User-scoped Permissions**: Only data you have access to
- **Workspace Boundaries**: Confined to connected workspace
- **Message Content**: Captures text for context (respects channel permissions)
- **Sensitive Data**: Follows Slack's data handling guidelines
### Security Measures
- **OAuth2 Flow**: Secure token-based authentication
- **Webhook Verification**: Validates event authenticity
- **Rate Limiting**: Respects Slack API limits
- **Error Handling**: Graceful handling of permission errors
## Usage
The integration operates through real-time webhooks:
1. **Webhook Setup**: Configure Slack webhook endpoints
2. **Event Reception**: Receive real-time workspace events
3. **Context Fetching**: Enrich events with additional API data
4. **Activity Generation**: Create structured activity messages
5. **Entity Processing**: Extract users, channels, topics for knowledge graph
6. **Memory Integration**: Store in CORE for intelligent recall
## API Reference
- **Slack Web API**: https://api.slack.com/web
- **Events API**: https://api.slack.com/events-api
- **OAuth2 Guide**: https://api.slack.com/authentication/oauth-v2
- **Webhook Events**: https://api.slack.com/events
- **Rate Limiting**: https://api.slack.com/docs/rate-limits

View File

@ -1,19 +1,21 @@
# Echo SDK
# Core SDK
The Echo SDK provides tools and utilities for building integrations with the Echo platform.
The Core SDK provides tools and utilities for building integrations with the Core platform.
## Integration System
The Echo integration system uses a CLI-based approach where each integration is a command-line tool that responds to specific events. This makes integrations portable, testable, and easy to debug.
The Core integration system uses a CLI-based approach where each integration is a command-line tool that responds to specific events. This makes integrations portable, testable, and easy to debug.
### Integration Event Types
Each integration CLI handles 5 core event types:
#### 1. `spec`
Returns the integration's metadata and configuration.
**Usage:**
```bash
my-integration spec
```
@ -21,9 +23,11 @@ my-integration spec
**Returns:** Integration specification including name, description, auth config, etc.
#### 2. `setup`
Processes authentication data and returns tokens/credentials to be saved.
**Usage:**
```bash
my-integration setup --event-body '{"code":"oauth_code","state":"state"}' --integration-definition '{}'
```
@ -31,9 +35,11 @@ my-integration setup --event-body '{"code":"oauth_code","state":"state"}' --inte
**Returns:** Configuration data (tokens, credentials) to be stored for the account.
#### 3. `identify`
Extracts accountId from webhook data to route webhooks to the correct account.
**Usage:**
```bash
my-integration identify --webhook-data '{"team_id":"T123","event":{}}'
```
@ -41,9 +47,11 @@ my-integration identify --webhook-data '{"team_id":"T123","event":{}}'
**Returns:** Account identifier for webhook routing.
#### 4. `process`
Handles webhook events and returns activity data.
**Usage:**
```bash
my-integration process --event-data '{"type":"reaction_added","reaction":"=M"}' --config '{"access_token":"token"}'
```
@ -51,9 +59,11 @@ my-integration process --event-data '{"type":"reaction_added","reaction":"=M"}'
**Returns:** Activity messages representing user actions.
#### 5. `sync`
Performs scheduled data synchronization for integrations that don't support webhooks.
**Usage:**
```bash
my-integration sync --config '{"access_token":"token","last_sync":"2023-01-01T00:00:00Z"}'
```
@ -72,20 +82,24 @@ All integration responses are wrapped in a `Message` object with a `type` field:
### Building an Integration
1. **Install the SDK:**
```bash
npm install @echo/core-sdk
npm install @Core/core-sdk
```
2. **Create your integration class:**
```typescript
import { IntegrationCLI } from '@echo/core-sdk';
import { IntegrationCLI } from '@Core/core-sdk';
class MyIntegration extends IntegrationCLI {
constructor() {
super('my-integration', '1.0.0');
}
protected async handleEvent(eventPayload: IntegrationEventPayload): Promise<any> {
protected async handleEvent(
eventPayload: IntegrationEventPayload,
): Promise<any> {
switch (eventPayload.event) {
case 'SETUP':
return this.handleSetup(eventPayload);
@ -110,24 +124,28 @@ class MyIntegration extends IntegrationCLI {
OAuth2: {
token_url: 'https://api.example.com/oauth/token',
authorization_url: 'https://api.example.com/oauth/authorize',
scopes: ['read', 'write']
}
}
scopes: ['read', 'write'],
},
},
};
}
private async handleSetup(eventPayload: IntegrationEventPayload): Promise<any> {
private async handleSetup(
eventPayload: IntegrationEventPayload,
): Promise<any> {
// Process OAuth response and return tokens to save
const { code } = eventPayload.eventBody;
// Exchange code for tokens...
return {
access_token: 'token',
refresh_token: 'refresh_token',
expires_at: Date.now() + 3600000
expires_at: Date.now() + 3600000,
};
}
private async handleProcess(eventPayload: IntegrationEventPayload): Promise<any> {
private async handleProcess(
eventPayload: IntegrationEventPayload,
): Promise<any> {
// Handle webhook events
const { eventData } = eventPayload.eventBody;
// Process event and return activity...
@ -135,23 +153,29 @@ class MyIntegration extends IntegrationCLI {
type: 'message',
user: 'user123',
content: 'Hello world',
timestamp: new Date()
timestamp: new Date(),
};
}
private async handleIdentify(eventPayload: IntegrationEventPayload): Promise<any> {
private async handleIdentify(
eventPayload: IntegrationEventPayload,
): Promise<any> {
// Extract account ID from webhook
const { team_id } = eventPayload.eventBody;
return { id: team_id };
}
private async handleSync(eventPayload: IntegrationEventPayload): Promise<any> {
private async handleSync(
eventPayload: IntegrationEventPayload,
): Promise<any> {
// Perform scheduled sync
const { config } = eventPayload;
// Fetch data since last sync...
return {
activities: [/* activity data */],
state: { last_sync: new Date().toISOString() }
activities: [
/* activity data */
],
state: { last_sync: new Date().toISOString() },
};
}
}
@ -162,6 +186,7 @@ integration.parse();
```
3. **Build and package your integration:**
```bash
npm run build
npm pack
@ -200,4 +225,4 @@ node dist/index.js process --event-data '{"type":"test"}' --config '{"token":"te
5. **Store minimal state** for sync operations
6. **Test all event types** thoroughly
For more examples, see the integrations in the `integrations/` directory.
For more examples, see the integrations in the `integrations/` directory.