diff --git a/src/components/AgentActionDisplay.tsx b/src/components/AgentActionDisplay.tsx index c16138a..2bc3bec 100644 --- a/src/components/AgentActionDisplay.tsx +++ b/src/components/AgentActionDisplay.tsx @@ -2,7 +2,14 @@ import { useState } from 'react'; import { cn } from '@/lib/utils'; -import { ChevronDown, ChevronUp, Bot, Search, Zap, Microscope } from 'lucide-react'; +import { + ChevronDown, + ChevronUp, + Bot, + Search, + Zap, + Microscope, +} from 'lucide-react'; import { AgentActionEvent } from './ChatWindow'; interface AgentActionDisplayProps { @@ -48,7 +55,9 @@ const AgentActionDisplay = ({ events, messageId }: AgentActionDisplayProps) => {
{getActionIcon(latestEvent.action)} - {latestEvent.action === 'SYNTHESIZING_RESPONSE' ? 'Agent Log' : formatActionName(latestEvent.action)} + {latestEvent.action === 'SYNTHESIZING_RESPONSE' + ? 'Agent Log' + : formatActionName(latestEvent.action)}
{isExpanded ? ( @@ -83,7 +92,11 @@ const AgentActionDisplay = ({ events, messageId }: AgentActionDisplayProps) => { {event.details.sourceUrl && (
Source: - {event.details.sourceUrl} + + + {event.details.sourceUrl} + +
)} {event.details.skipReason && ( @@ -92,12 +105,15 @@ const AgentActionDisplay = ({ events, messageId }: AgentActionDisplayProps) => { {event.details.skipReason} )} - {event.details.searchQuery && event.details.searchQuery !== event.details.query && ( -
- Search Query: - "{event.details.searchQuery}" -
- )} + {event.details.searchQuery && + event.details.searchQuery !== event.details.query && ( +
+ Search Query: + + "{event.details.searchQuery}" + +
+ )} {event.details.sourcesFound !== undefined && (
Sources Found: @@ -131,7 +147,9 @@ const AgentActionDisplay = ({ events, messageId }: AgentActionDisplayProps) => { {event.details.processingType && (
Processing Type: - {event.details.processingType.replace('-', ' ')} + + {event.details.processingType.replace('-', ' ')} +
)} {event.details.insufficiencyReason && ( diff --git a/src/components/Chat.tsx b/src/components/Chat.tsx index f7c5045..2bffb1f 100644 --- a/src/components/Chat.tsx +++ b/src/components/Chat.tsx @@ -236,12 +236,11 @@ const Chat = ({ /> )} {/* Show empty agent action display if this is the last user message and we're loading */} - {loading && isLast && (!msg.agentActions || msg.agentActions.length === 0) && ( - - )} + {loading && + isLast && + (!msg.agentActions || msg.agentActions.length === 0) && ( + + )} )} {!isLast && msg.role === 'assistant' && ( diff --git a/src/components/ChatWindow.tsx b/src/components/ChatWindow.tsx index 1fdccf8..0da120b 100644 --- a/src/components/ChatWindow.tsx +++ b/src/components/ChatWindow.tsx @@ -442,7 +442,10 @@ const ChatWindow = ({ id }: { id?: string }) => { // Update the user message with agent actions setMessages((prev) => prev.map((message) => { - if (message.messageId === data.messageId && message.role === 'user') { + if ( + message.messageId === data.messageId && + message.role === 'user' + ) { const updatedActions = [ ...(message.agentActions || []), agentActionEvent, diff --git a/src/components/MessageSources.tsx b/src/components/MessageSources.tsx index 32d42e9..93458b5 100644 --- a/src/components/MessageSources.tsx +++ b/src/components/MessageSources.tsx @@ -40,12 +40,18 @@ const MessageSources = ({ sources }: { sources: Document[] }) => { {/* Processing type indicator */} {source.metadata.processingType === 'preview-only' && ( - + )} {source.metadata.processingType === 'full-content' && ( - + )}
diff --git a/src/lib/agents/analyzerAgent.ts b/src/lib/agents/analyzerAgent.ts index 9d028bf..ee2b9f8 100644 --- a/src/lib/agents/analyzerAgent.ts +++ b/src/lib/agents/analyzerAgent.ts @@ -32,13 +32,14 @@ export class AnalyzerAgent { type: 'agent_action', data: { action: 'ANALYZING_CONTEXT', - message: 'Analyzing the context to see if we have enough information to answer the query', + message: + 'Analyzing the context to see if we have enough information to answer the query', details: { documentCount: state.relevantDocuments.length, query: state.query, - searchIterations: state.searchInstructionHistory.length - } - } + searchIterations: state.searchInstructionHistory.length, + }, + }, }); console.log( @@ -94,7 +95,10 @@ Today's date is ${formatDateForLLM(new Date())} ).format({ systemInstructions: this.systemInstructions, context: state.relevantDocuments - .map((doc, index) => `${doc?.metadata?.title ? `${doc?.metadata?.title}` : ''}${doc.pageContent}`) + .map( + (doc, index) => + `${doc?.metadata?.title ? `${doc?.metadata?.title}` : ''}${doc.pageContent}`, + ) .join('\n\n'), }); @@ -115,9 +119,7 @@ Today's date is ${formatDateForLLM(new Date())} const moreInfoQuestion = await moreInfoOutputParser.parse( response.content as string, ); - const reason = await reasonOutputParser.parse( - response.content as string, - ); + const reason = await reasonOutputParser.parse(response.content as string); console.log('Analysis result:', analysisResult); console.log('More info question:', moreInfoQuestion); @@ -129,15 +131,16 @@ Today's date is ${formatDateForLLM(new Date())} type: 'agent_action', data: { action: 'MORE_DATA_NEEDED', - message: 'Current context is insufficient - gathering more information', + message: + 'Current context is insufficient - gathering more information', details: { reason: reason, nextSearchQuery: moreInfoQuestion, documentCount: state.relevantDocuments.length, searchIterations: state.searchInstructionHistory.length, - query: state.query - } - } + query: state.query, + }, + }, }); return new Command({ @@ -159,13 +162,14 @@ Today's date is ${formatDateForLLM(new Date())} type: 'agent_action', data: { action: 'INFORMATION_GATHERING_COMPLETE', - message: 'Sufficient information gathered - ready to synthesize response', + message: + 'Sufficient information gathered - ready to synthesize response', details: { documentCount: state.relevantDocuments.length, searchIterations: state.searchInstructionHistory.length, - query: state.query - } - } + query: state.query, + }, + }, }); return new Command({ diff --git a/src/lib/agents/synthesizerAgent.ts b/src/lib/agents/synthesizerAgent.ts index 71a29b1..77a036f 100644 --- a/src/lib/agents/synthesizerAgent.ts +++ b/src/lib/agents/synthesizerAgent.ts @@ -37,9 +37,9 @@ export class SynthesizerAgent { details: { query: state.query, documentCount: state.relevantDocuments.length, - searchIterations: state.searchInstructionHistory.length - } - } + searchIterations: state.searchInstructionHistory.length, + }, + }, }); const synthesisPrompt = `You are an expert information synthesizer. Based on the search results and analysis provided, create a comprehensive, well-structured answer to the user's query. diff --git a/src/lib/agents/webSearchAgent.ts b/src/lib/agents/webSearchAgent.ts index 585f6ad..d1efc9b 100644 --- a/src/lib/agents/webSearchAgent.ts +++ b/src/lib/agents/webSearchAgent.ts @@ -9,7 +9,10 @@ import { webSearchRetrieverAgentPrompt } from '../prompts/webSearch'; import { searchSearxng } from '../searxng'; import { formatDateForLLM } from '../utils'; import { summarizeWebContent } from '../utils/summarizeWebContent'; -import { analyzePreviewContent, PreviewContent } from '../utils/analyzePreviewContent'; +import { + analyzePreviewContent, + PreviewContent, +} from '../utils/analyzePreviewContent'; import { AgentState } from './agentState'; export class WebSearchAgent { @@ -44,9 +47,9 @@ export class WebSearchAgent { query: state.query, searchInstructions: state.searchInstructions || state.query, documentCount: state.relevantDocuments.length, - searchIterations: state.searchInstructionHistory.length - } - } + searchIterations: state.searchInstructionHistory.length, + }, + }, }); const template = PromptTemplate.fromTemplate(webSearchRetrieverAgentPrompt); @@ -70,7 +73,7 @@ export class WebSearchAgent { try { console.log(`Performing web search for query: "${searchQuery}"`); - + // Emit executing web search event this.emitter.emit('agent_action', { type: 'agent_action', @@ -81,9 +84,9 @@ export class WebSearchAgent { query: state.query, searchQuery: searchQuery, documentCount: state.relevantDocuments.length, - searchIterations: state.searchInstructionHistory.length - } - } + searchIterations: state.searchInstructionHistory.length, + }, + }, }); const searchResults = await searchSearxng(searchQuery, { @@ -102,30 +105,34 @@ export class WebSearchAgent { searchQuery: searchQuery, sourcesFound: searchResults.results.length, documentCount: state.relevantDocuments.length, - searchIterations: state.searchInstructionHistory.length - } - } + searchIterations: state.searchInstructionHistory.length, + }, + }, }); let bannedUrls = state.bannedUrls || []; - + // Extract preview content from top 8 search results for analysis const previewContents: PreviewContent[] = searchResults.results - .filter(result => !bannedUrls.includes(result.url)) // Filter out banned URLs first + .filter((result) => !bannedUrls.includes(result.url)) // Filter out banned URLs first .slice(0, 8) // Then take top 8 results - .map(result => ({ + .map((result) => ({ title: result.title || 'Untitled', snippet: result.content || '', - url: result.url + url: result.url, })); - console.log(`Extracted preview content from ${previewContents.length} search results for analysis`); + console.log( + `Extracted preview content from ${previewContents.length} search results for analysis`, + ); // Perform preview analysis to determine if full content retrieval is needed let previewAnalysisResult = null; if (previewContents.length > 0) { - console.log('Starting preview content analysis to determine if full processing is needed'); - + console.log( + 'Starting preview content analysis to determine if full processing is needed', + ); + // Emit preview analysis event this.emitter.emit('agent_action', { type: 'agent_action', @@ -136,9 +143,9 @@ export class WebSearchAgent { query: state.query, previewCount: previewContents.length, documentCount: state.relevantDocuments.length, - searchIterations: state.searchInstructionHistory.length - } - } + searchIterations: state.searchInstructionHistory.length, + }, + }, }); previewAnalysisResult = await analyzePreviewContent( @@ -147,10 +154,12 @@ export class WebSearchAgent { state.messages, this.llm, this.systemInstructions, - this.signal + this.signal, ); - console.log(`Preview analysis result: ${previewAnalysisResult.isSufficient ? 'SUFFICIENT' : 'INSUFFICIENT'}${previewAnalysisResult.reason ? ` - ${previewAnalysisResult.reason}` : ''}`); + console.log( + `Preview analysis result: ${previewAnalysisResult.isSufficient ? 'SUFFICIENT' : 'INSUFFICIENT'}${previewAnalysisResult.reason ? ` - ${previewAnalysisResult.reason}` : ''}`, + ); } let documents: Document[] = []; @@ -159,8 +168,10 @@ export class WebSearchAgent { // Conditional workflow based on preview analysis result if (previewAnalysisResult && previewAnalysisResult.isSufficient) { // Preview content is sufficient - create documents from preview content - console.log('Preview content determined sufficient - skipping full content retrieval'); - + console.log( + 'Preview content determined sufficient - skipping full content retrieval', + ); + // Emit preview processing event this.emitter.emit('agent_action', { type: 'agent_action', @@ -172,30 +183,38 @@ export class WebSearchAgent { previewCount: previewContents.length, documentCount: state.relevantDocuments.length, searchIterations: state.searchInstructionHistory.length, - processingType: 'preview-only' - } - } + processingType: 'preview-only', + }, + }, }); // Create documents from preview content - documents = previewContents.map((content, index) => new Document({ - pageContent: `# ${content.title}\n\n${content.snippet}`, - metadata: { - title: content.title, - url: content.url, - source: content.url, - processingType: 'preview-only', - snippet: content.snippet - } - })); + documents = previewContents.map( + (content, index) => + new Document({ + pageContent: `# ${content.title}\n\n${content.snippet}`, + metadata: { + title: content.title, + url: content.url, + source: content.url, + processingType: 'preview-only', + snippet: content.snippet, + }, + }), + ); - console.log(`Created ${documents.length} documents from preview content`); - + console.log( + `Created ${documents.length} documents from preview content`, + ); } else { // Preview content is insufficient - proceed with full content processing - const insufficiencyReason = previewAnalysisResult?.reason || 'Preview content not available or insufficient'; - console.log(`Preview content insufficient: ${insufficiencyReason} - proceeding with full content retrieval`); - + const insufficiencyReason = + previewAnalysisResult?.reason || + 'Preview content not available or insufficient'; + console.log( + `Preview content insufficient: ${insufficiencyReason} - proceeding with full content retrieval`, + ); + // Emit full processing event this.emitter.emit('agent_action', { type: 'agent_action', @@ -207,101 +226,106 @@ export class WebSearchAgent { insufficiencyReason: insufficiencyReason, documentCount: state.relevantDocuments.length, searchIterations: state.searchInstructionHistory.length, - processingType: 'full-content' - } - } + processingType: 'full-content', + }, + }, }); // Summarize the top 2 search results for (const result of searchResults.results) { - if (bannedUrls.includes(result.url)) { - console.log(`Skipping banned URL: ${result.url}`); - // Note: We don't emit an agent_action event for banned URLs as this is an internal - // optimization that should be transparent to the user - continue; // Skip banned URLs - } - if (attemptedUrlCount >= 5) { - console.warn( - 'Too many attempts to summarize URLs, stopping further attempts.', - ); - break; // Limit the number of attempts to summarize URLs - } - attemptedUrlCount++; - - bannedUrls.push(result.url); // Add to banned URLs to avoid duplicates - - if (documents.length >= 1) { - break; // Limit to top 1 document - } - - // Emit analyzing source event - this.emitter.emit('agent_action', { - type: 'agent_action', - data: { - action: 'ANALYZING_SOURCE', - message: `Analyzing content from: ${result.title || result.url}`, - details: { - query: state.query, - sourceUrl: result.url, - sourceTitle: result.title || 'Untitled', - documentCount: state.relevantDocuments.length, - searchIterations: state.searchInstructionHistory.length - } + if (bannedUrls.includes(result.url)) { + console.log(`Skipping banned URL: ${result.url}`); + // Note: We don't emit an agent_action event for banned URLs as this is an internal + // optimization that should be transparent to the user + continue; // Skip banned URLs } - }); + if (attemptedUrlCount >= 5) { + console.warn( + 'Too many attempts to summarize URLs, stopping further attempts.', + ); + break; // Limit the number of attempts to summarize URLs + } + attemptedUrlCount++; - const summaryResult = await summarizeWebContent( - result.url, - state.query, - this.llm, - this.systemInstructions, - this.signal, - ); - - if (summaryResult.document) { - documents.push(summaryResult.document); - - // Emit context updated event + bannedUrls.push(result.url); // Add to banned URLs to avoid duplicates + + if (documents.length >= 1) { + break; // Limit to top 1 document + } + + // Emit analyzing source event this.emitter.emit('agent_action', { type: 'agent_action', data: { - action: 'CONTEXT_UPDATED', - message: `Added information from ${summaryResult.document.metadata.title || result.url} to context`, - details: { - query: state.query, - sourceUrl: result.url, - sourceTitle: summaryResult.document.metadata.title || 'Untitled', - contentLength: summaryResult.document.pageContent.length, - documentCount: state.relevantDocuments.length + documents.length, - searchIterations: state.searchInstructionHistory.length - } - } - }); - - console.log( - `Summarized content from ${result.url} to ${summaryResult.document.pageContent.length} characters. Content: ${summaryResult.document.pageContent}`, - ); - } else { - console.warn(`No relevant content found for URL: ${result.url}`); - - // Emit skipping irrelevant source event for non-relevant content - this.emitter.emit('agent_action', { - type: 'agent_action', - data: { - action: 'SKIPPING_IRRELEVANT_SOURCE', - message: `Source ${result.title || result.url} was not relevant - trying next`, + action: 'ANALYZING_SOURCE', + message: `Analyzing content from: ${result.title || result.url}`, details: { query: state.query, sourceUrl: result.url, sourceTitle: result.title || 'Untitled', - skipReason: summaryResult.notRelevantReason || 'Content was not relevant to the query', - documentCount: state.relevantDocuments.length + documents.length, - searchIterations: state.searchInstructionHistory.length - } - } + documentCount: state.relevantDocuments.length, + searchIterations: state.searchInstructionHistory.length, + }, + }, }); + + const summaryResult = await summarizeWebContent( + result.url, + state.query, + this.llm, + this.systemInstructions, + this.signal, + ); + + if (summaryResult.document) { + documents.push(summaryResult.document); + + // Emit context updated event + this.emitter.emit('agent_action', { + type: 'agent_action', + data: { + action: 'CONTEXT_UPDATED', + message: `Added information from ${summaryResult.document.metadata.title || result.url} to context`, + details: { + query: state.query, + sourceUrl: result.url, + sourceTitle: + summaryResult.document.metadata.title || 'Untitled', + contentLength: summaryResult.document.pageContent.length, + documentCount: + state.relevantDocuments.length + documents.length, + searchIterations: state.searchInstructionHistory.length, + }, + }, + }); + + console.log( + `Summarized content from ${result.url} to ${summaryResult.document.pageContent.length} characters. Content: ${summaryResult.document.pageContent}`, + ); + } else { + console.warn(`No relevant content found for URL: ${result.url}`); + + // Emit skipping irrelevant source event for non-relevant content + this.emitter.emit('agent_action', { + type: 'agent_action', + data: { + action: 'SKIPPING_IRRELEVANT_SOURCE', + message: `Source ${result.title || result.url} was not relevant - trying next`, + details: { + query: state.query, + sourceUrl: result.url, + sourceTitle: result.title || 'Untitled', + skipReason: + summaryResult.notRelevantReason || + 'Content was not relevant to the query', + documentCount: + state.relevantDocuments.length + documents.length, + searchIterations: state.searchInstructionHistory.length, + }, + }, + }); + } } - } } // Close the else block for full content processing if (documents.length === 0) { diff --git a/src/lib/search/agentSearch.ts b/src/lib/search/agentSearch.ts index 6daeac8..defda63 100644 --- a/src/lib/search/agentSearch.ts +++ b/src/lib/search/agentSearch.ts @@ -5,18 +5,13 @@ import { HumanMessage, SystemMessage, } from '@langchain/core/messages'; -import { - END, - MemorySaver, - START, - StateGraph, -} from '@langchain/langgraph'; +import { END, MemorySaver, START, StateGraph } from '@langchain/langgraph'; import { EventEmitter } from 'events'; -import { +import { AgentState, WebSearchAgent, AnalyzerAgent, - SynthesizerAgent + SynthesizerAgent, } from '../agents'; /** @@ -49,19 +44,19 @@ export class AgentSearch { llm, emitter, systemInstructions, - signal + signal, ); this.analyzerAgent = new AnalyzerAgent( llm, emitter, systemInstructions, - signal + signal, ); this.synthesizerAgent = new SynthesizerAgent( llm, emitter, personaInstructions, - signal + signal, ); } @@ -70,15 +65,27 @@ export class AgentSearch { */ private createWorkflow() { const workflow = new StateGraph(AgentState) - .addNode('web_search', this.webSearchAgent.execute.bind(this.webSearchAgent), { - ends: ['analyzer'], - }) - .addNode('analyzer', this.analyzerAgent.execute.bind(this.analyzerAgent), { - ends: ['web_search', 'synthesizer'], - }) - .addNode('synthesizer', this.synthesizerAgent.execute.bind(this.synthesizerAgent), { - ends: [END], - }) + .addNode( + 'web_search', + this.webSearchAgent.execute.bind(this.webSearchAgent), + { + ends: ['analyzer'], + }, + ) + .addNode( + 'analyzer', + this.analyzerAgent.execute.bind(this.analyzerAgent), + { + ends: ['web_search', 'synthesizer'], + }, + ) + .addNode( + 'synthesizer', + this.synthesizerAgent.execute.bind(this.synthesizerAgent), + { + ends: [END], + }, + ) .addEdge(START, 'analyzer'); return workflow.compile({ checkpointer: this.checkpointer }); diff --git a/src/lib/search/metaSearchAgent.ts b/src/lib/search/metaSearchAgent.ts index 91c9271..38c1ecd 100644 --- a/src/lib/search/metaSearchAgent.ts +++ b/src/lib/search/metaSearchAgent.ts @@ -180,8 +180,8 @@ class MetaSearchAgent implements MetaSearchAgentType { ? `${systemInstructions}\n\n` : ''; - const res = - await llm.invoke(`${systemPrompt}You are a web search summarizer, tasked with summarizing a piece of text retrieved from a web search. Your job is to summarize the + const res = await llm.invoke( + `${systemPrompt}You are a web search summarizer, tasked with summarizing a piece of text retrieved from a web search. Your job is to summarize the text into a detailed, 2-4 paragraph explanation that captures the main ideas and provides a comprehensive answer to the query. If the query is \"summarize\", you should provide a detailed summary of the text. If the query is a specific question, you should answer it in the summary. @@ -239,7 +239,9 @@ class MetaSearchAgent implements MetaSearchAgentType { Make sure to answer the query in the summary. - `, { signal }); + `, + { signal }, + ); const document = new Document({ pageContent: res.content as string, diff --git a/src/lib/utils/analyzePreviewContent.ts b/src/lib/utils/analyzePreviewContent.ts index 2f1367a..95ab3b1 100644 --- a/src/lib/utils/analyzePreviewContent.ts +++ b/src/lib/utils/analyzePreviewContent.ts @@ -24,37 +24,37 @@ export const analyzePreviewContent = async ( ): Promise => { try { console.log(`Analyzing preview content for query: "${query}"`); - console.log(`Preview content being analyzed:`, previewContents.map(content => ({ - title: content.title, - snippet: content.snippet.substring(0, 100) + '...', - url: content.url - }))); + console.log( + `Preview content being analyzed:`, + previewContents.map((content) => ({ + title: content.title, + snippet: content.snippet.substring(0, 100) + '...', + url: content.url, + })), + ); // Format preview content for analysis const formattedPreviewContent = previewContents - .map((content, index) => - `Source ${index + 1}: + .map( + (content, index) => + `Source ${index + 1}: Title: ${content.title} Snippet: ${content.snippet} URL: ${content.url} ----` +---`, ) .join('\n\n'); // Format chat history for context const formattedChatHistory = chatHistory .slice(-10) // Only include last 10 messages for context - .map((message, index) => - `${message._getType()}: ${message.content}` - ) + .map((message, index) => `${message._getType()}: ${message.content}`) .join('\n'); - const systemPrompt = systemInstructions - ? `${systemInstructions}\n\n` - : ''; + const systemPrompt = systemInstructions ? `${systemInstructions}\n\n` : ''; console.log(`Invoking LLM for preview content analysis`); - + const analysisResponse = await llm.invoke( `${systemPrompt}You are a preview content analyzer, tasked with determining if search result snippets contain sufficient information to answer a user's query. @@ -85,9 +85,10 @@ ${formattedPreviewContent} if (!analysisResponse || !analysisResponse.content) { console.error('No analysis response returned from LLM'); - return { - isSufficient: false, - reason: 'No analysis response returned from LLM - falling back to full content processing' + return { + isSufficient: false, + reason: + 'No analysis response returned from LLM - falling back to full content processing', }; } @@ -99,30 +100,36 @@ ${formattedPreviewContent} console.log(`LLM decision response:`, decision); if (decision.toLowerCase().trim() === 'sufficient') { - console.log('Preview content determined to be sufficient for answering the query'); + console.log( + 'Preview content determined to be sufficient for answering the query', + ); return { isSufficient: true }; } else if (decision.toLowerCase().startsWith('not_needed')) { // Extract the reason from the "not_needed" response - const reason = decision.startsWith('not_needed') + const reason = decision.startsWith('not_needed') ? decision.substring('not_needed:'.length).trim() : 'Preview content insufficient for complete answer'; - - console.log(`Preview content determined to be insufficient. Reason: ${reason}`); + + console.log( + `Preview content determined to be insufficient. Reason: ${reason}`, + ); return { isSufficient: false, reason }; } else { // Default to not sufficient if unclear response - console.log(`Unclear LLM response, defaulting to insufficient: ${decision}`); - return { - isSufficient: false, - reason: 'Unclear analysis response - falling back to full content processing' + console.log( + `Unclear LLM response, defaulting to insufficient: ${decision}`, + ); + return { + isSufficient: false, + reason: + 'Unclear analysis response - falling back to full content processing', }; } - } catch (error) { console.error('Error analyzing preview content:', error); - return { - isSufficient: false, - reason: `Error during preview analysis: ${error instanceof Error ? error.message : 'Unknown error'} - falling back to full content processing` + return { + isSufficient: false, + reason: `Error during preview analysis: ${error instanceof Error ? error.message : 'Unknown error'} - falling back to full content processing`, }; } }; diff --git a/src/lib/utils/summarizeWebContent.ts b/src/lib/utils/summarizeWebContent.ts index 8128dd4..40746f1 100644 --- a/src/lib/utils/summarizeWebContent.ts +++ b/src/lib/utils/summarizeWebContent.ts @@ -68,7 +68,10 @@ ${i === 0 ? content.metadata.html : content.pageContent}, if (!summary || !summary.content) { console.error(`No summary content returned for URL: ${url}`); - return { document: null, notRelevantReason: 'No summary content returned from LLM' }; + return { + document: null, + notRelevantReason: 'No summary content returned from LLM', + }; } const summaryParser = new LineOutputParser({ key: 'summary' }); @@ -84,18 +87,18 @@ ${i === 0 ? content.metadata.html : content.pageContent}, `LLM response for URL "${url}" indicates it's not needed or is empty:`, summarizedContent, ); - + // Extract the reason from the "not_needed" response - const reason = summarizedContent.startsWith('not_needed') + const reason = summarizedContent.startsWith('not_needed') ? summarizedContent.substring('not_needed:'.length).trim() - : summarizedContent.trim().length === 0 + : summarizedContent.trim().length === 0 ? 'Source content was empty or could not be processed' : 'Source content was not relevant to the query'; - + return { document: null, notRelevantReason: reason }; } - return { + return { document: new Document({ pageContent: summarizedContent, metadata: { @@ -104,7 +107,7 @@ ${i === 0 ? content.metadata.html : content.pageContent}, processingType: 'full-content', }, }), - notRelevantReason: undefined + notRelevantReason: undefined, }; }; @@ -138,10 +141,16 @@ ${i === 0 ? content.metadata.html : content.pageContent}, return await summarizeContent(webContent); } else { console.log(`No valid content found for URL: ${url}`); - return { document: null, notRelevantReason: 'No valid content found at the URL' }; + return { + document: null, + notRelevantReason: 'No valid content found at the URL', + }; } } catch (error) { console.error(`Error processing URL ${url}:`, error); - return { document: null, notRelevantReason: `Error processing URL: ${error instanceof Error ? error.message : 'Unknown error'}` }; + return { + document: null, + notRelevantReason: `Error processing URL: ${error instanceof Error ? error.message : 'Unknown error'}`, + }; } };