Skip to content

Integrate slackbot with baml client and sage backend #2308

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: canary
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 51 additions & 10 deletions typescript/apps/sage-backend/app/actions/query.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,19 @@
'use server';

import type { QueryRequest, QueryResponse } from '@baml/sage-interface';
import { b } from '../../baml_client';
import { searchPinecone } from '../../lib/pinecone-api';

async function tryLoadBamlClient(): Promise<any | null> {
try {
// Avoid static analysis by Next/Webpack
// eslint-disable-next-line no-eval
const mod = await (0, eval)("import('../../' + 'baml_client')");
return mod;
} catch {
return null;
}
}

export async function submitQuery(request: QueryRequest): Promise<QueryResponse> {
const docs = await searchPinecone(request.message.text);
const pineconeRankedDocs = docs.map((doc) => ({
Expand All @@ -12,6 +22,32 @@ export async function submitQuery(request: QueryRequest): Promise<QueryResponse>
body: doc.body,
}));

const baml = await tryLoadBamlClient();

if (!baml) {
// Fallback: return top docs summary without LLM plan
const ranked: Array<{ title: string; url: string; relevance: 'very-relevant' | 'relevant' | 'not-relevant' }> =
pineconeRankedDocs.slice(0, 3).map((d) => ({
title: d.title,
url: d.url,
relevance: 'very-relevant',
}));

return {
session_id: request.session_id,
message: {
role: 'assistant',
message_id: `msg-${new Date().toISOString()}`,
text:
'BAML client unavailable at build time. Returning top related docs. Ask again once the backend is fully configured.',
ranked_docs: ranked,
suggested_messages: [],
},
};
}

const { b } = baml as { b: any };

const plan = await b.PlanQuery({
text: request.message.text,
language_preference: request.message.language_preference,
Expand All @@ -31,22 +67,27 @@ export async function submitQuery(request: QueryRequest): Promise<QueryResponse>
});

// Merge titles from rankedDocs into plan.ranked_docs
const relevantDocs = (plan.ranked_docs ?? []).map((planDoc) => {
const matchingRankedDoc = pineconeRankedDocs.find((rd) => rd.title === planDoc.title);
return {
title: planDoc.title,
url: matchingRankedDoc?.url ?? '',
relevance: planDoc.relevance,
};
});
const relevantDocs: Array<{ title: string; url: string; relevance: 'very-relevant' | 'relevant' | 'not-relevant' }> =
(plan.ranked_docs ?? []).map((planDoc: any) => {
const matchingRankedDoc = pineconeRankedDocs.find((rd) => rd.title === planDoc.title);
return {
title: planDoc.title,
url: matchingRankedDoc?.url ?? '',
relevance: planDoc.relevance,
} as { title: string; url: string; relevance: 'very-relevant' | 'relevant' | 'not-relevant' };
});

const dedupedRankedDocs: Array<{ title: string; url: string; relevance: 'very-relevant' | 'relevant' | 'not-relevant' }> = Array.from(
new Map(relevantDocs.map((doc) => [doc.url, doc])).values(),
);

return {
session_id: request.session_id,
message: {
role: 'assistant',
message_id: `msg-${new Date().toISOString()}`,
text: plan.answer,
ranked_docs: Array.from(new Map(relevantDocs.map((doc) => [doc.url, doc])).values()),
ranked_docs: dedupedRankedDocs,
suggested_messages: plan.refine_query?.suggested_queries,
},
};
Expand Down
3 changes: 1 addition & 2 deletions typescript/apps/sage-backend/app/api/ask-baml/chat/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,6 @@ import type { NextRequest } from 'next/server';
import { NextResponse } from 'next/server';
import { submitQuery } from '@/app/actions/query';

const notionLogger = new NotionLogger();

export async function POST(httpRequest: NextRequest) {
try {
const body = await httpRequest.json();
Expand All @@ -27,6 +25,7 @@ export async function POST(httpRequest: NextRequest) {

const result = await submitQuery(request);

const notionLogger = new NotionLogger();
notionLogger
.appendEntry({
session_id: request.session_id,
Expand Down
19 changes: 11 additions & 8 deletions typescript/apps/sage-backend/app/api/ask-baml/feedback/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,6 @@ import { SendFeedbackRequestSchema } from '@baml/sage-interface';
import type { NextRequest } from 'next/server';
import { after, NextResponse } from 'next/server';

const slack = new SlackFeedbackLogger();
const notionLogger = new NotionLogger();

export async function POST(request: NextRequest) {
try {
const body = await request.json();
Expand All @@ -28,11 +25,17 @@ export async function POST(request: NextRequest) {

// Deliberately do not await these, so that the request can return immediately.
after(async () => {
console.info('Feedback will be logged to Notion and Slack');
const { pageId: notionPageId } = await notionLogger.updateFeedback(feedbackData);
const notionLink = notionPageId ? notionLogger.toUrl({ pageId: notionPageId }) : undefined;
await slack.sendFeedback({ ...feedbackData, notionLink });
console.info('Feedback logged to Notion and Slack');
try {
const slack = new SlackFeedbackLogger();
const notionLogger = new NotionLogger();
console.info('Feedback will be logged to Notion and Slack');
const { pageId: notionPageId } = await notionLogger.updateFeedback(feedbackData);
const notionLink = notionPageId ? notionLogger.toUrl({ pageId: notionPageId }) : undefined;
await slack.sendFeedback({ ...feedbackData, notionLink });
console.info('Feedback logged to Notion and Slack');
} catch (error) {
console.error('Failed to send feedback to Notion/Slack:', error);
}
});

return NextResponse.json({
Expand Down
163 changes: 163 additions & 0 deletions typescript/apps/sage-backend/app/api/slack/events/route.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,163 @@
import type { NextRequest } from 'next/server';
import { NextResponse, after } from 'next/server';
import crypto from 'crypto';
import { WebClient } from '@slack/web-api';
import { submitQuery } from '@/app/actions/query';
import type { Message, QueryRequest } from '@baml/sage-interface';

export const runtime = 'nodejs';

function timingSafeEqual(a: string, b: string): boolean {
const aBuf = Buffer.from(a, 'utf8');
const bBuf = Buffer.from(b, 'utf8');
if (aBuf.length !== bBuf.length) return false;
return crypto.timingSafeEqual(aBuf, bBuf);
}

function verifySlackSignature({
rawBody,
timestamp,
signature,
secret,
}: {
rawBody: string;
timestamp: string | null;
signature: string | null;
secret: string;
}): boolean {
if (!timestamp || !signature) return false;
// Reject old timestamps (> 5 minutes)
const fiveMinutes = 60 * 5;
const tsNum = Number(timestamp);
if (!Number.isFinite(tsNum)) return false;
const now = Math.floor(Date.now() / 1000);
if (Math.abs(now - tsNum) > fiveMinutes) return false;

const sigBase = `v0:${timestamp}:${rawBody}`;
const hmac = crypto.createHmac('sha256', secret).update(sigBase).digest('hex');
const expected = `v0=${hmac}`;
return timingSafeEqual(expected, signature);
}

function stripMentionsAndFormatting(text: string): string {
return text
.replace(/<@[^>]+>/g, '') // remove mentions
Comment on lines +43 to +44

Check failure

Code scanning / CodeQL

Polynomial regular expression used on uncontrolled data High

This
regular expression
that depends on
a user-provided value
may run slow on strings starting with '<@' and with many repetitions of '<@'.
.replace(/<([^|>]+)\|[^>]+>/g, '$1') // <url|text> -> url
.replace(/\s+/g, ' ')
.trim();
}

export async function POST(req: NextRequest) {
const signingSecret = process.env.SLACK_SIGNING_SECRET;
const botToken = process.env.SLACK_BOUNDARY_BOT_TOKEN;
if (!signingSecret || !botToken) {
return NextResponse.json({ error: 'Slack environment not configured' }, { status: 500 });
}

const rawBody = await req.text();
const timestamp = req.headers.get('x-slack-request-timestamp');
const signature = req.headers.get('x-slack-signature');

if (!verifySlackSignature({ rawBody, timestamp, signature, secret: signingSecret })) {
return NextResponse.json({ error: 'Invalid signature' }, { status: 401 });
}

let payload: any;
try {
payload = JSON.parse(rawBody);
} catch (e) {
return NextResponse.json({ error: 'Invalid JSON' }, { status: 400 });
}

// URL verification challenge
if (payload.type === 'url_verification' && payload.challenge) {
return new NextResponse(payload.challenge, {
headers: { 'content-type': 'text/plain' },
});
}

// Acknowledge immediately; process asynchronously
after(async () => {
try {
const event = payload.event;
if (!event) return;

// Only handle app mentions and direct messages for now
if (event.type !== 'app_mention' && !(event.type === 'message' && event.channel_type === 'im')) {
return;
}

const slack = new WebClient(botToken);
const channel: string = event.channel;
const thread_ts: string = event.thread_ts || event.ts;
const text: string = typeof event.text === 'string' ? event.text : '';

const cleaned = stripMentionsAndFormatting(text);
if (!cleaned) return;

// Build prev_messages from thread history (last 10 messages)
let prev_messages: Message[] = [];
try {
const auth = await slack.auth.test();
const botUserId = auth.user_id;
const replies = await slack.conversations.replies({ channel, ts: thread_ts, inclusive: true, limit: 10 });
const msgs = (replies.messages || []) as Array<{ user?: string; bot_id?: string; text?: string; subtype?: string; ts: string }>;
prev_messages = msgs
.filter((m) => !m.subtype || m.subtype === 'bot_message')
.map((m) => {
const mText = stripMentionsAndFormatting(m.text || '');
const isAssistant = (m.user && m.user === botUserId) || Boolean(m.bot_id);
if (isAssistant) {
return {
role: 'assistant',
message_id: `slack-${m.ts}`,
text: mText,
ranked_docs: [],
} as Message;
}
return {
role: 'user',
text: mText,
} as Message;
})
.slice(0, -1); // exclude current event message
} catch {}

const sessionId = `${channel}:${thread_ts}`;
const request: QueryRequest = {
session_id: sessionId,
prev_messages,
message: {
role: 'user',
text: cleaned,
language_preference: 'en',
},
};

// Optional: send typing indicator (ephemeral)
try {
await slack.chat.postEphemeral({ channel, user: event.user, text: 'Thinking…' });
} catch {}

const result = await submitQuery(request);

const answer = result.message.text || 'I could not find an answer.';
const links = (result.message.ranked_docs || [])
.slice(0, 3)
.map((d) => `<${d.url}|${d.title}>`)
.join(' • ');

const finalText = links ? `${answer}\n\nSources: ${links}` : answer;

await slack.chat.postMessage({
channel,
text: finalText,
thread_ts,
});
} catch (err) {
console.error('Slack event handling failed:', err);
}
});

return NextResponse.json({ ok: true });
}
38 changes: 24 additions & 14 deletions typescript/apps/sage-backend/lib/pinecone-api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,25 @@ const PINECONE_INDEX_NAME =
: 'ask-baml-dev';
console.log('Using pinecone index:', PINECONE_INDEX_NAME);

const openaiClient = new OpenAI({
apiKey: process.env.OPENAI_API_KEY ?? '',
});
let openaiClient: OpenAI | null = null;
function getOpenAI(): OpenAI {
if (!openaiClient) {
openaiClient = new OpenAI({ apiKey: process.env.OPENAI_API_KEY ?? '' });
}
return openaiClient;
}

const pineconeClient = new Pinecone({
apiKey: process.env.PINECONE_API_KEY ?? '',
});
let pineconeClient: Pinecone | null = null;
function getPinecone(): Pinecone {
if (!pineconeClient) {
pineconeClient = new Pinecone({ apiKey: process.env.PINECONE_API_KEY ?? '' });
}
return pineconeClient;
}

const pineconeIndex = pineconeClient.Index(PINECONE_INDEX_NAME);
function getPineconeIndex() {
return getPinecone().Index(PINECONE_INDEX_NAME);
}

const CorpusDocumentSchema = z.object({
title: z.string(),
Expand Down Expand Up @@ -185,7 +195,7 @@ async function generateEmbeddingsForDocs(docs: CorpusDocument[]): Promise<Embedd
}
}

const embeddingResponse = await openaiClient.embeddings.create({
const embeddingResponse = await getOpenAI().embeddings.create({
model: 'text-embedding-3-large',
input: chunk,
});
Expand Down Expand Up @@ -223,7 +233,7 @@ async function upsertToPinecone(embeddingsWithMetadata: EmbeddingWithMetadata[])
// Execute all batches in parallel
await Promise.all(
batches.map(async (batch, index) => {
await pineconeIndex.upsert(batch);
await getPineconeIndex().upsert(batch);
console.log(`Upserted batch ${index + 1}/${batches.length} with ${batch.length} records`);
}),
);
Expand Down Expand Up @@ -297,13 +307,13 @@ export async function populatePinecone(docsYmlPath: string): Promise<void> {
]);
console.log(`Computed embeddings for ${embeddingsWithMetadata.length} chunks`);

const beforeStats = await pineconeIndex.describeIndexStats();
const beforeStats = await getPineconeIndex().describeIndexStats();
console.log('Before stats', beforeStats);
const deleted = await pineconeIndex.deleteAll();
const deleted = await getPineconeIndex().deleteAll();
console.log('Deleted old embeddings from Pinecone', deleted);
await upsertToPinecone(embeddingsWithMetadata);
console.log('Upserted new embeddings to Pinecone');
const afterStats = await pineconeIndex.describeIndexStats();
const afterStats = await getPineconeIndex().describeIndexStats();
console.log('After stats', afterStats);
console.log(`✅ Successfully populated Pinecone with ${embeddingsWithMetadata.length} chunks`);
}
Expand All @@ -312,8 +322,8 @@ export async function populatePinecone(docsYmlPath: string): Promise<void> {
* Search Pinecone for relevant documents using vector similarity
*/
export async function searchPinecone(query: string) {
const results = await pineconeIndex.query({
vector: await openaiClient.embeddings
const results = await getPineconeIndex().query({
vector: await getOpenAI().embeddings
.create({
model: EMBEDDING_MODEL,
input: query,
Expand Down
Loading