sk/Perplexica
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

chore: Update dependencies and fix import paths

Browse source
This commit is contained in:
Jin Yucong 2024-07-05 15:49:43 +08:00
parent 3b737a078a
commit 81c5e30fda
46 changed files with 1626 additions and 371 deletions
Download patch file Download diff file Expand all files Collapse all files

32
src/agents/academicSearchAgent.ts
View file

@ -8,7 +8,7 @@ import type { StreamEvent } from "@langchain/core/tracers/log_stream";
import type { BaseChatModel } from "@langchain/core/language_models/chat_models";
import type { Embeddings } from "@langchain/core/embeddings";
import formatChatHistoryAsString from "../utils/formatHistory";
import eventEmitter from "events";
import eventEmitter from "node:events";
import computeSimilarity from "../utils/computeSimilarity";
import logger from "../utils/logger";
@ -55,7 +55,7 @@ const basicAcademicSearchResponsePrompt = `
Anything between the \`context\` is retrieved from a search engine and is not a part of the conversation with the user. Today's date is ${new Date().toISOString()}
`;
const strParser = new StringOutputParser();
const stringParser = new StringOutputParser();
const handleStream = async (stream: AsyncGenerator<StreamEvent, unknown, unknown>, emitter: eventEmitter) => {
for await (const event of stream) {
@ -80,7 +80,7 @@ const createBasicAcademicSearchRetrieverChain = (llm: BaseChatModel) => {
return RunnableSequence.from([
PromptTemplate.fromTemplate(basicAcademicSearchRetrieverPrompt),
llm,
strParser,
stringParser,
RunnableLambda.from(async (input: string) => {
if (input === "not_needed") {
return { query: "", docs: [] };
@ -108,30 +108,30 @@ const createBasicAcademicSearchRetrieverChain = (llm: BaseChatModel) => {
]);
};
const processDocs = async (docs: Document[]) => {
return docs.map((_, index) => `${index + 1}. ${docs[index].pageContent}`).join("\n");
};
const createBasicAcademicSearchAnsweringChain = (llm: BaseChatModel, embeddings: Embeddings) => {
const basicAcademicSearchRetrieverChain = createBasicAcademicSearchRetrieverChain(llm);
const processDocs = async (docs: Document[]) => {
return docs.map((_, index) => `${index + 1}. ${docs[index].pageContent}`).join("\n");
};
const rerankDocs = async ({ query, docs }: { query: string; docs: Document[] }) => {
if (docs.length === 0) {
return docs;
}
const docsWithContent = docs.filter(doc => doc.pageContent && doc.pageContent.length > 0);
const docsWithContent = docs.filter(document => document.pageContent && document.pageContent.length > 0);
const [docEmbeddings, queryEmbedding] = await Promise.all([
embeddings.embedDocuments(docsWithContent.map(doc => doc.pageContent)),
const [documentEmbeddings, queryEmbedding] = await Promise.all([
embeddings.embedDocuments(docsWithContent.map(document => document.pageContent)),
embeddings.embedQuery(query),
]);
const similarity = docEmbeddings.map((docEmbedding, i) => {
const sim = computeSimilarity(queryEmbedding, docEmbedding);
const similarity = documentEmbeddings.map((documentEmbedding, index) => {
const sim = computeSimilarity(queryEmbedding, documentEmbedding);
return {
index: i,
index: index,
similarity: sim,
};
});
@ -167,7 +167,7 @@ const createBasicAcademicSearchAnsweringChain = (llm: BaseChatModel, embeddings:
["user", "{query}"],
]),
llm,
strParser,
stringParser,
]).withConfig({
runName: "FinalResponseGenerator",
});
@ -190,9 +190,9 @@ const basicAcademicSearch = (query: string, history: BaseMessage[], llm: BaseCha
);
handleStream(stream, emitter);
} catch (err) {
} catch (error) {
emitter.emit("error", JSON.stringify({ data: "An error has occurred please try again later" }));
logger.error(`Error in academic search: ${err}`);
logger.error(`Error in academic search: ${error}`);
}
return emitter;

8
src/agents/imageSearchAgent.ts
View file

@ -32,7 +32,7 @@ type ImageSearchChainInput = {
query: string;
};
const strParser = new StringOutputParser();
const stringParser = new StringOutputParser();
const createImageSearchChain = (llm: BaseChatModel) => {
return RunnableSequence.from([
@ -46,7 +46,7 @@ const createImageSearchChain = (llm: BaseChatModel) => {
}),
PromptTemplate.fromTemplate(imageSearchChainPrompt),
llm,
strParser,
stringParser,
RunnableLambda.from(async (input: string) => {
const res = await searchSearxng(input, {
engines: ["bing images", "google images"],
@ -54,7 +54,7 @@ const createImageSearchChain = (llm: BaseChatModel) => {
const images = [];
res.results.forEach(result => {
for (const result of res.results) {
if (result.img_src && result.url && result.title) {
images.push({
img_src: result.img_src,
@ -62,7 +62,7 @@ const createImageSearchChain = (llm: BaseChatModel) => {
title: result.title,
});
}
});
}
return images.slice(0, 10);
}),

34
src/agents/redditSearchAgent.ts
View file

@ -8,7 +8,7 @@ import type { StreamEvent } from "@langchain/core/tracers/log_stream";
import type { BaseChatModel } from "@langchain/core/language_models/chat_models";
import type { Embeddings } from "@langchain/core/embeddings";
import formatChatHistoryAsString from "../utils/formatHistory";
import eventEmitter from "events";
import eventEmitter from "node:events";
import computeSimilarity from "../utils/computeSimilarity";
import logger from "../utils/logger";
@ -55,7 +55,7 @@ const basicRedditSearchResponsePrompt = `
Anything between the \`context\` is retrieved from Reddit and is not a part of the conversation with the user. Today's date is ${new Date().toISOString()}
`;
const strParser = new StringOutputParser();
const stringParser = new StringOutputParser();
const handleStream = async (stream: AsyncGenerator<StreamEvent, unknown, unknown>, emitter: eventEmitter) => {
for await (const event of stream) {
@ -80,7 +80,7 @@ const createBasicRedditSearchRetrieverChain = (llm: BaseChatModel) => {
return RunnableSequence.from([
PromptTemplate.fromTemplate(basicRedditSearchRetrieverPrompt),
llm,
strParser,
stringParser,
RunnableLambda.from(async (input: string) => {
if (input === "not_needed") {
return { query: "", docs: [] };
@ -94,7 +94,7 @@ const createBasicRedditSearchRetrieverChain = (llm: BaseChatModel) => {
const documents = res.results.map(
result =>
new Document({
pageContent: result.content ? result.content : result.title,
pageContent: result.content ?? result.title,
metadata: {
title: result.title,
url: result.url,
@ -108,30 +108,30 @@ const createBasicRedditSearchRetrieverChain = (llm: BaseChatModel) => {
]);
};
const processDocs = async (docs: Document[]) => {
return docs.map((_, index) => `${index + 1}. ${docs[index].pageContent}`).join("\n");
};
const createBasicRedditSearchAnsweringChain = (llm: BaseChatModel, embeddings: Embeddings) => {
const basicRedditSearchRetrieverChain = createBasicRedditSearchRetrieverChain(llm);
const processDocs = async (docs: Document[]) => {
return docs.map((_, index) => `${index + 1}. ${docs[index].pageContent}`).join("\n");
};
const rerankDocs = async ({ query, docs }: { query: string; docs: Document[] }) => {
if (docs.length === 0) {
return docs;
}
const docsWithContent = docs.filter(doc => doc.pageContent && doc.pageContent.length > 0);
const docsWithContent = docs.filter(document => document.pageContent && document.pageContent.length > 0);
const [docEmbeddings, queryEmbedding] = await Promise.all([
embeddings.embedDocuments(docsWithContent.map(doc => doc.pageContent)),
const [documentEmbeddings, queryEmbedding] = await Promise.all([
embeddings.embedDocuments(docsWithContent.map(document => document.pageContent)),
embeddings.embedQuery(query),
]);
const similarity = docEmbeddings.map((docEmbedding, i) => {
const sim = computeSimilarity(queryEmbedding, docEmbedding);
const similarity = documentEmbeddings.map((documentEmbedding, index) => {
const sim = computeSimilarity(queryEmbedding, documentEmbedding);
return {
index: i,
index: index,
similarity: sim,
};
});
@ -168,7 +168,7 @@ const createBasicRedditSearchAnsweringChain = (llm: BaseChatModel, embeddings: E
["user", "{query}"],
]),
llm,
strParser,
stringParser,
]).withConfig({
runName: "FinalResponseGenerator",
});
@ -190,9 +190,9 @@ const basicRedditSearch = (query: string, history: BaseMessage[], llm: BaseChatM
);
handleStream(stream, emitter);
} catch (err) {
} catch (error) {
emitter.emit("error", JSON.stringify({ data: "An error has occurred please try again later" }));
logger.error(`Error in RedditSearch: ${err}`);
logger.error(`Error in RedditSearch: ${error}`);
}
return emitter;

8
src/agents/videoSearchAgent.ts
View file

@ -32,7 +32,7 @@ type VideoSearchChainInput = {
query: string;
};
const strParser = new StringOutputParser();
const stringParser = new StringOutputParser();
const createVideoSearchChain = (llm: BaseChatModel) => {
return RunnableSequence.from([
@ -46,7 +46,7 @@ const createVideoSearchChain = (llm: BaseChatModel) => {
}),
PromptTemplate.fromTemplate(VideoSearchChainPrompt),
llm,
strParser,
stringParser,
RunnableLambda.from(async (input: string) => {
const res = await searchSearxng(input, {
engines: ["youtube"],
@ -54,7 +54,7 @@ const createVideoSearchChain = (llm: BaseChatModel) => {
const videos = [];
res.results.forEach(result => {
for (const result of res.results) {
if (result.thumbnail && result.url && result.title && result.iframe_src) {
videos.push({
img_src: result.thumbnail,
@ -63,7 +63,7 @@ const createVideoSearchChain = (llm: BaseChatModel) => {
iframe_src: result.iframe_src,
});
}
});
}
return videos.slice(0, 10);
}),

32
src/agents/webSearchAgent.ts
View file

@ -8,7 +8,7 @@ import type { StreamEvent } from "@langchain/core/tracers/log_stream";
import type { BaseChatModel } from "@langchain/core/language_models/chat_models";
import type { Embeddings } from "@langchain/core/embeddings";
import formatChatHistoryAsString from "../utils/formatHistory";
import eventEmitter from "events";
import eventEmitter from "node:events";
import computeSimilarity from "../utils/computeSimilarity";
import logger from "../utils/logger";
@ -55,7 +55,7 @@ const basicWebSearchResponsePrompt = `
Anything between the \`context\` is retrieved from a search engine and is not a part of the conversation with the user. Today's date is ${new Date().toISOString()}
`;
const strParser = new StringOutputParser();
const stringParser = new StringOutputParser();
const handleStream = async (stream: AsyncGenerator<StreamEvent, unknown, unknown>, emitter: eventEmitter) => {
for await (const event of stream) {
@ -80,7 +80,7 @@ const createBasicWebSearchRetrieverChain = (llm: BaseChatModel) => {
return RunnableSequence.from([
PromptTemplate.fromTemplate(basicSearchRetrieverPrompt),
llm,
strParser,
stringParser,
RunnableLambda.from(async (input: string) => {
if (input === "not_needed") {
return { query: "", docs: [] };
@ -107,30 +107,30 @@ const createBasicWebSearchRetrieverChain = (llm: BaseChatModel) => {
]);
};
const processDocs = async (docs: Document[]) => {
return docs.map((_, index) => `${index + 1}. ${docs[index].pageContent}`).join("\n");
};
const createBasicWebSearchAnsweringChain = (llm: BaseChatModel, embeddings: Embeddings) => {
const basicWebSearchRetrieverChain = createBasicWebSearchRetrieverChain(llm);
const processDocs = async (docs: Document[]) => {
return docs.map((_, index) => `${index + 1}. ${docs[index].pageContent}`).join("\n");
};
const rerankDocs = async ({ query, docs }: { query: string; docs: Document[] }) => {
if (docs.length === 0) {
return docs;
}
const docsWithContent = docs.filter(doc => doc.pageContent && doc.pageContent.length > 0);
const docsWithContent = docs.filter(document => document.pageContent && document.pageContent.length > 0);
const [docEmbeddings, queryEmbedding] = await Promise.all([
embeddings.embedDocuments(docsWithContent.map(doc => doc.pageContent)),
const [documentEmbeddings, queryEmbedding] = await Promise.all([
embeddings.embedDocuments(docsWithContent.map(document => document.pageContent)),
embeddings.embedQuery(query),
]);
const similarity = docEmbeddings.map((docEmbedding, i) => {
const sim = computeSimilarity(queryEmbedding, docEmbedding);
const similarity = documentEmbeddings.map((documentEmbedding, index) => {
const sim = computeSimilarity(queryEmbedding, documentEmbedding);
return {
index: i,
index: index,
similarity: sim,
};
});
@ -167,7 +167,7 @@ const createBasicWebSearchAnsweringChain = (llm: BaseChatModel, embeddings: Embe
["user", "{query}"],
]),
llm,
strParser,
stringParser,
]).withConfig({
runName: "FinalResponseGenerator",
});
@ -190,9 +190,9 @@ const basicWebSearch = (query: string, history: BaseMessage[], llm: BaseChatMode
);
handleStream(stream, emitter);
} catch (err) {
} catch (error) {
emitter.emit("error", JSON.stringify({ data: "An error has occurred please try again later" }));
logger.error(`Error in websearch: ${err}`);
logger.error(`Error in websearch: ${error}`);
}
return emitter;

20
src/agents/wolframAlphaSearchAgent.ts
View file

@ -8,7 +8,7 @@ import type { StreamEvent } from "@langchain/core/tracers/log_stream";
import type { BaseChatModel } from "@langchain/core/language_models/chat_models";
import type { Embeddings } from "@langchain/core/embeddings";
import formatChatHistoryAsString from "../utils/formatHistory";
import eventEmitter from "events";
import eventEmitter from "node:events";
import logger from "../utils/logger";
const basicWolframAlphaSearchRetrieverPrompt = `
@ -54,7 +54,7 @@ const basicWolframAlphaSearchResponsePrompt = `
Anything between the \`context\` is retrieved from Wolfram Alpha and is not a part of the conversation with the user. Today's date is ${new Date().toISOString()}
`;
const strParser = new StringOutputParser();
const stringParser = new StringOutputParser();
const handleStream = async (stream: AsyncGenerator<StreamEvent, unknown, unknown>, emitter: eventEmitter) => {
for await (const event of stream) {
@ -79,7 +79,7 @@ const createBasicWolframAlphaSearchRetrieverChain = (llm: BaseChatModel) => {
return RunnableSequence.from([
PromptTemplate.fromTemplate(basicWolframAlphaSearchRetrieverPrompt),
llm,
strParser,
stringParser,
RunnableLambda.from(async (input: string) => {
if (input === "not_needed") {
return { query: "", docs: [] };
@ -107,13 +107,13 @@ const createBasicWolframAlphaSearchRetrieverChain = (llm: BaseChatModel) => {
]);
};
const processDocs = (docs: Document[]) => {
return docs.map((_, index) => `${index + 1}. ${docs[index].pageContent}`).join("\n");
};
const createBasicWolframAlphaSearchAnsweringChain = (llm: BaseChatModel) => {
const basicWolframAlphaSearchRetrieverChain = createBasicWolframAlphaSearchRetrieverChain(llm);
const processDocs = (docs: Document[]) => {
return docs.map((_, index) => `${index + 1}. ${docs[index].pageContent}`).join("\n");
};
return RunnableSequence.from([
RunnableMap.from({
query: (input: BasicChainInput) => input.query,
@ -139,7 +139,7 @@ const createBasicWolframAlphaSearchAnsweringChain = (llm: BaseChatModel) => {
["user", "{query}"],
]),
llm,
strParser,
stringParser,
]).withConfig({
runName: "FinalResponseGenerator",
});
@ -161,9 +161,9 @@ const basicWolframAlphaSearch = (query: string, history: BaseMessage[], llm: Bas
);
handleStream(stream, emitter);
} catch (err) {
} catch (error) {
emitter.emit("error", JSON.stringify({ data: "An error has occurred please try again later" }));
logger.error(`Error in WolframAlphaSearch: ${err}`);
logger.error(`Error in WolframAlphaSearch: ${error}`);
}
return emitter;

10
src/agents/writingAssistant.ts
View file

@ -3,7 +3,7 @@ import { ChatPromptTemplate, MessagesPlaceholder } from "@langchain/core/prompts
import { RunnableSequence } from "@langchain/core/runnables";
import { StringOutputParser } from "@langchain/core/output_parsers";
import type { StreamEvent } from "@langchain/core/tracers/log_stream";
import eventEmitter from "events";
import eventEmitter from "node:events";
import type { BaseChatModel } from "@langchain/core/language_models/chat_models";
import type { Embeddings } from "@langchain/core/embeddings";
import logger from "../utils/logger";
@ -13,7 +13,7 @@ You are Perplexica, an AI model who is expert at searching the web and answering
Since you are a writing assistant, you would not perform web searches. If you think you lack information to answer the query, you can ask the user for more information or suggest them to switch to a different focus mode.
`;
const strParser = new StringOutputParser();
const stringParser = new StringOutputParser();
const handleStream = async (stream: AsyncGenerator<StreamEvent, unknown, unknown>, emitter: eventEmitter) => {
for await (const event of stream) {
@ -34,7 +34,7 @@ const createWritingAssistantChain = (llm: BaseChatModel) => {
["user", "{query}"],
]),
llm,
strParser,
stringParser,
]).withConfig({
runName: "FinalResponseGenerator",
});
@ -62,9 +62,9 @@ const handleWritingAssistant = (
);
handleStream(stream, emitter);
} catch (err) {
} catch (error) {
emitter.emit("error", JSON.stringify({ data: "An error has occurred please try again later" }));
logger.error(`Error in writing assistant: ${err}`);
logger.error(`Error in writing assistant: ${error}`);
}
return emitter;

34
src/agents/youtubeSearchAgent.ts
View file

@ -8,7 +8,7 @@ import type { StreamEvent } from "@langchain/core/tracers/log_stream";
import type { BaseChatModel } from "@langchain/core/language_models/chat_models";
import type { Embeddings } from "@langchain/core/embeddings";
import formatChatHistoryAsString from "../utils/formatHistory";
import eventEmitter from "events";
import eventEmitter from "node:events";
import computeSimilarity from "../utils/computeSimilarity";
import logger from "../utils/logger";
@ -55,7 +55,7 @@ const basicYoutubeSearchResponsePrompt = `
Anything between the \`context\` is retrieved from Youtube and is not a part of the conversation with the user. Today's date is ${new Date().toISOString()}
`;
const strParser = new StringOutputParser();
const stringParser = new StringOutputParser();
const handleStream = async (stream: AsyncGenerator<StreamEvent, unknown, unknown>, emitter: eventEmitter) => {
for await (const event of stream) {
@ -80,7 +80,7 @@ const createBasicYoutubeSearchRetrieverChain = (llm: BaseChatModel) => {
return RunnableSequence.from([
PromptTemplate.fromTemplate(basicYoutubeSearchRetrieverPrompt),
llm,
strParser,
stringParser,
RunnableLambda.from(async (input: string) => {
if (input === "not_needed") {
return { query: "", docs: [] };
@ -94,7 +94,7 @@ const createBasicYoutubeSearchRetrieverChain = (llm: BaseChatModel) => {
const documents = res.results.map(
result =>
new Document({
pageContent: result.content ? result.content : result.title,
pageContent: result.content ?? result.title,
metadata: {
title: result.title,
url: result.url,
@ -108,30 +108,30 @@ const createBasicYoutubeSearchRetrieverChain = (llm: BaseChatModel) => {
]);
};
const processDocs = async (docs: Document[]) => {
return docs.map((_, index) => `${index + 1}. ${docs[index].pageContent}`).join("\n");
};
const createBasicYoutubeSearchAnsweringChain = (llm: BaseChatModel, embeddings: Embeddings) => {
const basicYoutubeSearchRetrieverChain = createBasicYoutubeSearchRetrieverChain(llm);
const processDocs = async (docs: Document[]) => {
return docs.map((_, index) => `${index + 1}. ${docs[index].pageContent}`).join("\n");
};
const rerankDocs = async ({ query, docs }: { query: string; docs: Document[] }) => {
if (docs.length === 0) {
return docs;
}
const docsWithContent = docs.filter(doc => doc.pageContent && doc.pageContent.length > 0);
const docsWithContent = docs.filter(document => document.pageContent && document.pageContent.length > 0);
const [docEmbeddings, queryEmbedding] = await Promise.all([
embeddings.embedDocuments(docsWithContent.map(doc => doc.pageContent)),
const [documentEmbeddings, queryEmbedding] = await Promise.all([
embeddings.embedDocuments(docsWithContent.map(document => document.pageContent)),
embeddings.embedQuery(query),
]);
const similarity = docEmbeddings.map((docEmbedding, i) => {
const sim = computeSimilarity(queryEmbedding, docEmbedding);
const similarity = documentEmbeddings.map((documentEmbedding, index) => {
const sim = computeSimilarity(queryEmbedding, documentEmbedding);
return {
index: i,
index: index,
similarity: sim,
};
});
@ -168,7 +168,7 @@ const createBasicYoutubeSearchAnsweringChain = (llm: BaseChatModel, embeddings:
["user", "{query}"],
]),
llm,
strParser,
stringParser,
]).withConfig({
runName: "FinalResponseGenerator",
});
@ -191,9 +191,9 @@ const basicYoutubeSearch = (query: string, history: BaseMessage[], llm: BaseChat
);
handleStream(stream, emitter);
} catch (err) {
} catch (error) {
emitter.emit("error", JSON.stringify({ data: "An error has occurred please try again later" }));
logger.error(`Error in youtube search: ${err}`);
logger.error(`Error in youtube search: ${error}`);
}
return emitter;