From 88304d29c157db7b9f7d4156c669582da28c1ddb Mon Sep 17 00:00:00 2001 From: ItzCrazyKns Date: Wed, 17 Apr 2024 21:00:43 +0530 Subject: [PATCH 001/470] feat(readme): use detached mode for docker compose --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9d26fae..3f2f63d 100644 --- a/README.md +++ b/README.md @@ -64,12 +64,12 @@ There are mainly 2 ways of installing Perplexica - With Docker, Without Docker. 5. Ensure you are in the directory containing the `docker-compose.yaml` file and execute: ```bash - docker compose up + docker compose up -d ``` 6. Wait a few minutes for the setup to complete. You can access Perplexica at http://localhost:3000 in your web browser. -**Note**: Once the terminal is stopped, Perplexica will also stop. To restart it, you will need to open Docker Desktop and run Perplexica again. +**Note**: After the containers are built, you can start Perplexica directly from Docker without having to open a terminal. ### Non-Docker Installation From f9ab543bcf7319c64a14fbd089f9f5a5eb355c5f Mon Sep 17 00:00:00 2001 From: ItzCrazyKns Date: Thu, 18 Apr 2024 17:47:51 +0530 Subject: [PATCH 002/470] feat(navbar): Fix alignment --- ui/components/Navbar.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/components/Navbar.tsx b/ui/components/Navbar.tsx index 9e0b483..57dcf6c 100644 --- a/ui/components/Navbar.tsx +++ b/ui/components/Navbar.tsx @@ -43,7 +43,7 @@ const Navbar = ({ messages }: { messages: Message[] }) => { size={17} className="active:scale-95 transition duration-100 cursor-pointer lg:hidden" /> -
+

{timeAgo} ago

From dd1ce4e324d866cae27caffbf53c2069d1bce2f4 Mon Sep 17 00:00:00 2001 From: ItzCrazyKns Date: Thu, 18 Apr 2024 18:15:17 +0530 Subject: [PATCH 003/470] feat(agents): replace LLMs with chat LLMs --- src/agents/academicSearchAgent.ts | 11 +++-------- src/agents/imageSearchAgent.ts | 6 +++--- src/agents/redditSearchAgent.ts | 11 +++-------- src/agents/webSearchAgent.ts | 11 +++-------- src/agents/wolframAlphaSearchAgent.ts | 9 ++------- src/agents/writingAssistant.ts | 4 ++-- src/agents/youtubeSearchAgent.ts | 11 +++-------- 7 files changed, 19 insertions(+), 44 deletions(-) diff --git a/src/agents/academicSearchAgent.ts b/src/agents/academicSearchAgent.ts index 7c3d448..edb7a63 100644 --- a/src/agents/academicSearchAgent.ts +++ b/src/agents/academicSearchAgent.ts @@ -9,7 +9,7 @@ import { RunnableMap, RunnableLambda, } from '@langchain/core/runnables'; -import { ChatOpenAI, OpenAI, OpenAIEmbeddings } from '@langchain/openai'; +import { ChatOpenAI, OpenAIEmbeddings } from '@langchain/openai'; import { StringOutputParser } from '@langchain/core/output_parsers'; import { Document } from '@langchain/core/documents'; import { searchSearxng } from '../core/searxng'; @@ -18,16 +18,11 @@ import formatChatHistoryAsString from '../utils/formatHistory'; import eventEmitter from 'events'; import computeSimilarity from '../utils/computeSimilarity'; -const chatLLM = new ChatOpenAI({ +const llm = new ChatOpenAI({ modelName: process.env.MODEL_NAME, temperature: 0.7, }); -const llm = new OpenAI({ - temperature: 0, - modelName: process.env.MODEL_NAME, -}); - const embeddings = new OpenAIEmbeddings({ modelName: 'text-embedding-3-large', }); @@ -215,7 +210,7 @@ const basicAcademicSearchAnsweringChain = RunnableSequence.from([ new MessagesPlaceholder('chat_history'), ['user', '{query}'], ]), - chatLLM, + llm, strParser, ]).withConfig({ runName: 'FinalResponseGenerator', diff --git a/src/agents/imageSearchAgent.ts b/src/agents/imageSearchAgent.ts index 3a2c9db..bf49de0 100644 --- a/src/agents/imageSearchAgent.ts +++ b/src/agents/imageSearchAgent.ts @@ -4,15 +4,15 @@ import { RunnableLambda, } from '@langchain/core/runnables'; import { PromptTemplate } from '@langchain/core/prompts'; -import { OpenAI } from '@langchain/openai'; +import { ChatOpenAI } from '@langchain/openai'; import formatChatHistoryAsString from '../utils/formatHistory'; import { BaseMessage } from '@langchain/core/messages'; import { StringOutputParser } from '@langchain/core/output_parsers'; import { searchSearxng } from '../core/searxng'; -const llm = new OpenAI({ - temperature: 0, +const llm = new ChatOpenAI({ modelName: process.env.MODEL_NAME, + temperature: 0.7, }); const imageSearchChainPrompt = ` diff --git a/src/agents/redditSearchAgent.ts b/src/agents/redditSearchAgent.ts index 77f293e..3b6a274 100644 --- a/src/agents/redditSearchAgent.ts +++ b/src/agents/redditSearchAgent.ts @@ -9,7 +9,7 @@ import { RunnableMap, RunnableLambda, } from '@langchain/core/runnables'; -import { ChatOpenAI, OpenAI, OpenAIEmbeddings } from '@langchain/openai'; +import { ChatOpenAI, OpenAIEmbeddings } from '@langchain/openai'; import { StringOutputParser } from '@langchain/core/output_parsers'; import { Document } from '@langchain/core/documents'; import { searchSearxng } from '../core/searxng'; @@ -18,16 +18,11 @@ import formatChatHistoryAsString from '../utils/formatHistory'; import eventEmitter from 'events'; import computeSimilarity from '../utils/computeSimilarity'; -const chatLLM = new ChatOpenAI({ +const llm = new ChatOpenAI({ modelName: process.env.MODEL_NAME, temperature: 0.7, }); -const llm = new OpenAI({ - temperature: 0, - modelName: process.env.MODEL_NAME, -}); - const embeddings = new OpenAIEmbeddings({ modelName: 'text-embedding-3-large', }); @@ -211,7 +206,7 @@ const basicRedditSearchAnsweringChain = RunnableSequence.from([ new MessagesPlaceholder('chat_history'), ['user', '{query}'], ]), - chatLLM, + llm, strParser, ]).withConfig({ runName: 'FinalResponseGenerator', diff --git a/src/agents/webSearchAgent.ts b/src/agents/webSearchAgent.ts index f5799e3..047eb3d 100644 --- a/src/agents/webSearchAgent.ts +++ b/src/agents/webSearchAgent.ts @@ -9,7 +9,7 @@ import { RunnableMap, RunnableLambda, } from '@langchain/core/runnables'; -import { ChatOpenAI, OpenAI, OpenAIEmbeddings } from '@langchain/openai'; +import { ChatOpenAI, OpenAIEmbeddings } from '@langchain/openai'; import { StringOutputParser } from '@langchain/core/output_parsers'; import { Document } from '@langchain/core/documents'; import { searchSearxng } from '../core/searxng'; @@ -18,16 +18,11 @@ import formatChatHistoryAsString from '../utils/formatHistory'; import eventEmitter from 'events'; import computeSimilarity from '../utils/computeSimilarity'; -const chatLLM = new ChatOpenAI({ +const llm = new ChatOpenAI({ modelName: process.env.MODEL_NAME, temperature: 0.7, }); -const llm = new OpenAI({ - temperature: 0, - modelName: process.env.MODEL_NAME, -}); - const embeddings = new OpenAIEmbeddings({ modelName: 'text-embedding-3-large', }); @@ -210,7 +205,7 @@ const basicWebSearchAnsweringChain = RunnableSequence.from([ new MessagesPlaceholder('chat_history'), ['user', '{query}'], ]), - chatLLM, + llm, strParser, ]).withConfig({ runName: 'FinalResponseGenerator', diff --git a/src/agents/wolframAlphaSearchAgent.ts b/src/agents/wolframAlphaSearchAgent.ts index c071ef0..4ab1990 100644 --- a/src/agents/wolframAlphaSearchAgent.ts +++ b/src/agents/wolframAlphaSearchAgent.ts @@ -17,16 +17,11 @@ import type { StreamEvent } from '@langchain/core/tracers/log_stream'; import formatChatHistoryAsString from '../utils/formatHistory'; import eventEmitter from 'events'; -const chatLLM = new ChatOpenAI({ +const llm = new ChatOpenAI({ modelName: process.env.MODEL_NAME, temperature: 0.7, }); -const llm = new OpenAI({ - temperature: 0, - modelName: process.env.MODEL_NAME, -}); - const basicWolframAlphaSearchRetrieverPrompt = ` You will be given a conversation below and a follow up question. You need to rephrase the follow-up question if needed so it is a standalone question that can be used by the LLM to search the web for information. If it is a writing task or a simple hi, hello rather than a question, you need to return \`not_needed\` as the response. @@ -169,7 +164,7 @@ const basicWolframAlphaSearchAnsweringChain = RunnableSequence.from([ new MessagesPlaceholder('chat_history'), ['user', '{query}'], ]), - chatLLM, + llm, strParser, ]).withConfig({ runName: 'FinalResponseGenerator', diff --git a/src/agents/writingAssistant.ts b/src/agents/writingAssistant.ts index 2c8d66e..0fc5097 100644 --- a/src/agents/writingAssistant.ts +++ b/src/agents/writingAssistant.ts @@ -9,7 +9,7 @@ import { StringOutputParser } from '@langchain/core/output_parsers'; import type { StreamEvent } from '@langchain/core/tracers/log_stream'; import eventEmitter from 'events'; -const chatLLM = new ChatOpenAI({ +const llm = new ChatOpenAI({ modelName: process.env.MODEL_NAME, temperature: 0.7, }); @@ -50,7 +50,7 @@ const writingAssistantChain = RunnableSequence.from([ new MessagesPlaceholder('chat_history'), ['user', '{query}'], ]), - chatLLM, + llm, strParser, ]).withConfig({ runName: 'FinalResponseGenerator', diff --git a/src/agents/youtubeSearchAgent.ts b/src/agents/youtubeSearchAgent.ts index 9ab5ed8..7c1bcf5 100644 --- a/src/agents/youtubeSearchAgent.ts +++ b/src/agents/youtubeSearchAgent.ts @@ -9,7 +9,7 @@ import { RunnableMap, RunnableLambda, } from '@langchain/core/runnables'; -import { ChatOpenAI, OpenAI, OpenAIEmbeddings } from '@langchain/openai'; +import { ChatOpenAI, OpenAIEmbeddings } from '@langchain/openai'; import { StringOutputParser } from '@langchain/core/output_parsers'; import { Document } from '@langchain/core/documents'; import { searchSearxng } from '../core/searxng'; @@ -18,16 +18,11 @@ import formatChatHistoryAsString from '../utils/formatHistory'; import eventEmitter from 'events'; import computeSimilarity from '../utils/computeSimilarity'; -const chatLLM = new ChatOpenAI({ +const llm = new ChatOpenAI({ modelName: process.env.MODEL_NAME, temperature: 0.7, }); -const llm = new OpenAI({ - temperature: 0, - modelName: process.env.MODEL_NAME, -}); - const embeddings = new OpenAIEmbeddings({ modelName: 'text-embedding-3-large', }); @@ -211,7 +206,7 @@ const basicYoutubeSearchAnsweringChain = RunnableSequence.from([ new MessagesPlaceholder('chat_history'), ['user', '{query}'], ]), - chatLLM, + llm, strParser, ]).withConfig({ runName: 'FinalResponseGenerator', From c6a5790d3381765050843b1d2bfcb715d78ceda0 Mon Sep 17 00:00:00 2001 From: ItzCrazyKns Date: Sat, 20 Apr 2024 09:32:19 +0530 Subject: [PATCH 004/470] feat(config): Use toml instead of env --- .env.example | 5 - .github/ISSUE_TEMPLATE/bug_report.md | 2 +- .github/ISSUE_TEMPLATE/custom.md | 3 - .github/ISSUE_TEMPLATE/feature_request.md | 1 - .gitignore | 3 + .prettierignore | 38 ++++ README.md | 8 +- backend.dockerfile | 5 +- package.json | 5 +- sample.config.toml | 9 + src/agents/academicSearchAgent.ts | 243 ++++++++++++---------- src/agents/imageSearchAgent.ts | 79 +++---- src/agents/redditSearchAgent.ts | 232 +++++++++++---------- src/agents/webSearchAgent.ts | 232 +++++++++++---------- src/agents/wolframAlphaSearchAgent.ts | 154 +++++++------- src/agents/writingAssistant.ts | 40 ++-- src/agents/youtubeSearchAgent.ts | 233 +++++++++++---------- src/app.ts | 7 +- src/config.ts | 32 +++ src/core/searxng.ts | 5 +- src/routes/images.ts | 12 +- src/utils/computeSimilarity.ts | 7 +- src/websocket/connectionManager.ts | 15 +- src/websocket/messageHandler.ts | 16 +- src/websocket/websocketServer.ts | 4 +- yarn.lock | 5 + 26 files changed, 799 insertions(+), 596 deletions(-) delete mode 100644 .env.example create mode 100644 .prettierignore create mode 100644 sample.config.toml create mode 100644 src/config.ts diff --git a/.env.example b/.env.example deleted file mode 100644 index bc67919..0000000 --- a/.env.example +++ /dev/null @@ -1,5 +0,0 @@ -PORT=3001 -OPENAI_API_KEY= -SIMILARITY_MEASURE=cosine # cosine or dot -SEARXNG_API_URL= # no need to fill this if using docker -MODEL_NAME=gpt-3.5-turbo \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index e065bb4..1de1177 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -4,7 +4,6 @@ about: Create an issue to help us fix bugs title: '' labels: bug assignees: '' - --- **Describe the bug** @@ -12,6 +11,7 @@ A clear and concise description of what the bug is. **To Reproduce** Steps to reproduce the behavior: + 1. Go to '...' 2. Click on '....' 3. Scroll down to '....' diff --git a/.github/ISSUE_TEMPLATE/custom.md b/.github/ISSUE_TEMPLATE/custom.md index 48d5f81..96a4735 100644 --- a/.github/ISSUE_TEMPLATE/custom.md +++ b/.github/ISSUE_TEMPLATE/custom.md @@ -4,7 +4,4 @@ about: Describe this issue template's purpose here. title: '' labels: '' assignees: '' - --- - - diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 11fc491..5f0a04c 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -4,7 +4,6 @@ about: Suggest an idea for this project title: '' labels: enhancement assignees: '' - --- **Is your feature request related to a problem? Please describe.** diff --git a/.gitignore b/.gitignore index 0f857e0..d64d5cc 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,9 @@ yarn-error.log .env.test.local .env.production.local +# Config files +config.toml + # Log files logs/ *.log diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000..c184fdb --- /dev/null +++ b/.prettierignore @@ -0,0 +1,38 @@ +# Ignore all files in the node_modules directory +node_modules + +# Ignore all files in the .next directory (Next.js build output) +.next + +# Ignore all files in the .out directory (TypeScript build output) +.out + +# Ignore all files in the .cache directory (Prettier cache) +.cache + +# Ignore all files in the .vscode directory (Visual Studio Code settings) +.vscode + +# Ignore all files in the .idea directory (IntelliJ IDEA settings) +.idea + +# Ignore all files in the dist directory (build output) +dist + +# Ignore all files in the build directory (build output) +build + +# Ignore all files in the coverage directory (test coverage reports) +coverage + +# Ignore all files with the .log extension +*.log + +# Ignore all files with the .tmp extension +*.tmp + +# Ignore all files with the .swp extension +*.swp + +# Ignore all files with the .DS_Store extension (macOS specific) +.DS_Store \ No newline at end of file diff --git a/README.md b/README.md index 3f2f63d..942ba9e 100644 --- a/README.md +++ b/README.md @@ -56,10 +56,10 @@ There are mainly 2 ways of installing Perplexica - With Docker, Without Docker. 3. After cloning, navigate to the directory containing the project files. -4. Rename the `.env.example` file to `.env`. For Docker setups, you need only fill in the following fields: +4. Rename the `sample.config.toml` file to `config.toml`. For Docker setups, you need only fill in the following fields: - - `OPENAI_API_KEY` - - `SIMILARITY_MEASURE` (This is filled by default; you can leave it as is if you are unsure about it.) + - `OPENAI`: Your OpenAI API key. + - `SIMILARITY_MEASURE`: The similarity measure to use (This is filled by default; you can leave it as is if you are unsure about it.) 5. Ensure you are in the directory containing the `docker-compose.yaml` file and execute: @@ -75,7 +75,7 @@ There are mainly 2 ways of installing Perplexica - With Docker, Without Docker. For setups without Docker: -1. Follow the initial steps to clone the repository and rename the `.env.example` file to `.env` in the root directory. You will need to fill in all the fields in this file. +1. Follow the initial steps to clone the repository and rename the `sample.config.toml` file to `config.toml` in the root directory. You will need to fill in all the fields in this file. 2. Additionally, rename the `.env.example` file to `.env` in the `ui` folder and complete all fields. 3. The non-Docker setup requires manual configuration of both the backend and frontend. diff --git a/backend.dockerfile b/backend.dockerfile index 6cbd192..8bf34f0 100644 --- a/backend.dockerfile +++ b/backend.dockerfile @@ -1,16 +1,17 @@ FROM node:alpine ARG SEARXNG_API_URL -ENV SEARXNG_API_URL=${SEARXNG_API_URL} WORKDIR /home/perplexica COPY src /home/perplexica/src COPY tsconfig.json /home/perplexica/ -COPY .env /home/perplexica/ +COPY config.toml /home/perplexica/ COPY package.json /home/perplexica/ COPY yarn.lock /home/perplexica/ +RUN sed -i "s|SEARXNG = \".*\"|SEARXNG = \"${SEARXNG_API_URL}\"|g" /home/perplexica/config.toml + RUN yarn install RUN yarn build diff --git a/package.json b/package.json index c2f1aba..5006a93 100644 --- a/package.json +++ b/package.json @@ -4,9 +4,9 @@ "license": "MIT", "author": "ItzCrazyKns", "scripts": { - "start": "node --env-file=.env dist/app.js", + "start": "node dist/app.js", "build": "tsc", - "dev": "nodemon -r dotenv/config src/app.ts", + "dev": "nodemon src/app.ts", "format": "prettier . --check", "format:write": "prettier . --write" }, @@ -19,6 +19,7 @@ "typescript": "^5.4.3" }, "dependencies": { + "@iarna/toml": "^2.2.5", "@langchain/openai": "^0.0.25", "axios": "^1.6.8", "compute-cosine-similarity": "^1.1.0", diff --git a/sample.config.toml b/sample.config.toml new file mode 100644 index 0000000..1082184 --- /dev/null +++ b/sample.config.toml @@ -0,0 +1,9 @@ +[GENERAL] +PORT = 3001 # Port to run the server on +SIMILARITY_MEASURE = "cosine" # "cosine" or "dot" + +[API_KEYS] +OPENAI = "sk-1234567890abcdef1234567890abcdef" # OpenAI API key + +[API_ENDPOINTS] +SEARXNG = "http://localhost:32768" # SearxNG API ULR \ No newline at end of file diff --git a/src/agents/academicSearchAgent.ts b/src/agents/academicSearchAgent.ts index edb7a63..466088f 100644 --- a/src/agents/academicSearchAgent.ts +++ b/src/agents/academicSearchAgent.ts @@ -9,24 +9,16 @@ import { RunnableMap, RunnableLambda, } from '@langchain/core/runnables'; -import { ChatOpenAI, OpenAIEmbeddings } from '@langchain/openai'; import { StringOutputParser } from '@langchain/core/output_parsers'; import { Document } from '@langchain/core/documents'; import { searchSearxng } from '../core/searxng'; import type { StreamEvent } from '@langchain/core/tracers/log_stream'; +import type { BaseChatModel } from '@langchain/core/language_models/chat_models'; +import type { Embeddings } from '@langchain/core/embeddings'; import formatChatHistoryAsString from '../utils/formatHistory'; import eventEmitter from 'events'; import computeSimilarity from '../utils/computeSimilarity'; -const llm = new ChatOpenAI({ - modelName: process.env.MODEL_NAME, - temperature: 0.7, -}); - -const embeddings = new OpenAIEmbeddings({ - modelName: 'text-embedding-3-large', -}); - const basicAcademicSearchRetrieverPrompt = ` You will be given a conversation below and a follow up question. You need to rephrase the follow-up question if needed so it is a standalone question that can be used by the LLM to search the web for information. If it is a writing task or a simple hi, hello rather than a question, you need to return \`not_needed\` as the response. @@ -49,7 +41,7 @@ Rephrased question: `; const basicAcademicSearchResponsePrompt = ` - You are Perplexica, an AI model who is expert at searching the web and answering user's queries. You are set on focus mode 'Acadedemic', this means you will be searching for academic papers and articles on the web. + You are Perplexica, an AI model who is expert at searching the web and answering user's queries. You are set on focus mode 'Academic', this means you will be searching for academic papers and articles on the web. Generate a response that is informative and relevant to the user's query based on provided context (the context consits of search results containg a brief description of the content of that page). You must use this context to answer the user's query in the best way possible. Use an unbaised and journalistic tone in your response. Do not repeat the text. @@ -104,122 +96,140 @@ const handleStream = async ( } }; -const processDocs = async (docs: Document[]) => { - return docs - .map((_, index) => `${index + 1}. ${docs[index].pageContent}`) - .join('\n'); -}; - -const rerankDocs = async ({ - query, - docs, -}: { - query: string; - docs: Document[]; -}) => { - if (docs.length === 0) { - return docs; - } - - const docsWithContent = docs.filter( - (doc) => doc.pageContent && doc.pageContent.length > 0, - ); - - const docEmbeddings = await embeddings.embedDocuments( - docsWithContent.map((doc) => doc.pageContent), - ); - - const queryEmbedding = await embeddings.embedQuery(query); - - const similarity = docEmbeddings.map((docEmbedding, i) => { - const sim = computeSimilarity(queryEmbedding, docEmbedding); - - return { - index: i, - similarity: sim, - }; - }); - - const sortedDocs = similarity - .sort((a, b) => b.similarity - a.similarity) - .slice(0, 15) - .map((sim) => docsWithContent[sim.index]); - - return sortedDocs; -}; - type BasicChainInput = { chat_history: BaseMessage[]; query: string; }; -const basicAcademicSearchRetrieverChain = RunnableSequence.from([ - PromptTemplate.fromTemplate(basicAcademicSearchRetrieverPrompt), - llm, - strParser, - RunnableLambda.from(async (input: string) => { - if (input === 'not_needed') { - return { query: '', docs: [] }; +const createBasicAcademicSearchRetrieverChain = (llm: BaseChatModel) => { + return RunnableSequence.from([ + PromptTemplate.fromTemplate(basicAcademicSearchRetrieverPrompt), + llm, + strParser, + RunnableLambda.from(async (input: string) => { + if (input === 'not_needed') { + return { query: '', docs: [] }; + } + + const res = await searchSearxng(input, { + language: 'en', + engines: [ + 'arxiv', + 'google_scholar', + 'internet_archive_scholar', + 'pubmed', + ], + }); + + const documents = res.results.map( + (result) => + new Document({ + pageContent: result.content, + metadata: { + title: result.title, + url: result.url, + ...(result.img_src && { img_src: result.img_src }), + }, + }), + ); + + return { query: input, docs: documents }; + }), + ]); +}; + +const createBasicAcademicSearchAnsweringChain = ( + llm: BaseChatModel, + embeddings: Embeddings, +) => { + const basicAcademicSearchRetrieverChain = + createBasicAcademicSearchRetrieverChain(llm); + + const processDocs = async (docs: Document[]) => { + return docs + .map((_, index) => `${index + 1}. ${docs[index].pageContent}`) + .join('\n'); + }; + + const rerankDocs = async ({ + query, + docs, + }: { + query: string; + docs: Document[]; + }) => { + if (docs.length === 0) { + return docs; } - const res = await searchSearxng(input, { - language: 'en', - engines: [ - 'arxiv', - 'google_scholar', - 'internet_archive_scholar', - 'pubmed', - ], - }); - - const documents = res.results.map( - (result) => - new Document({ - pageContent: result.content, - metadata: { - title: result.title, - url: result.url, - ...(result.img_src && { img_src: result.img_src }), - }, - }), + const docsWithContent = docs.filter( + (doc) => doc.pageContent && doc.pageContent.length > 0, ); - return { query: input, docs: documents }; - }), -]); + const docEmbeddings = await embeddings.embedDocuments( + docsWithContent.map((doc) => doc.pageContent), + ); -const basicAcademicSearchAnsweringChain = RunnableSequence.from([ - RunnableMap.from({ - query: (input: BasicChainInput) => input.query, - chat_history: (input: BasicChainInput) => input.chat_history, - context: RunnableSequence.from([ - (input) => ({ - query: input.query, - chat_history: formatChatHistoryAsString(input.chat_history), - }), - basicAcademicSearchRetrieverChain - .pipe(rerankDocs) - .withConfig({ - runName: 'FinalSourceRetriever', - }) - .pipe(processDocs), + const queryEmbedding = await embeddings.embedQuery(query); + + const similarity = docEmbeddings.map((docEmbedding, i) => { + const sim = computeSimilarity(queryEmbedding, docEmbedding); + + return { + index: i, + similarity: sim, + }; + }); + + const sortedDocs = similarity + .sort((a, b) => b.similarity - a.similarity) + .slice(0, 15) + .map((sim) => docsWithContent[sim.index]); + + return sortedDocs; + }; + + return RunnableSequence.from([ + RunnableMap.from({ + query: (input: BasicChainInput) => input.query, + chat_history: (input: BasicChainInput) => input.chat_history, + context: RunnableSequence.from([ + (input) => ({ + query: input.query, + chat_history: formatChatHistoryAsString(input.chat_history), + }), + basicAcademicSearchRetrieverChain + .pipe(rerankDocs) + .withConfig({ + runName: 'FinalSourceRetriever', + }) + .pipe(processDocs), + ]), + }), + ChatPromptTemplate.fromMessages([ + ['system', basicAcademicSearchResponsePrompt], + new MessagesPlaceholder('chat_history'), + ['user', '{query}'], ]), - }), - ChatPromptTemplate.fromMessages([ - ['system', basicAcademicSearchResponsePrompt], - new MessagesPlaceholder('chat_history'), - ['user', '{query}'], - ]), - llm, - strParser, -]).withConfig({ - runName: 'FinalResponseGenerator', -}); + llm, + strParser, + ]).withConfig({ + runName: 'FinalResponseGenerator', + }); +}; -const basicAcademicSearch = (query: string, history: BaseMessage[]) => { +const basicAcademicSearch = ( + query: string, + history: BaseMessage[], + llm: BaseChatModel, + embeddings: Embeddings, +) => { const emitter = new eventEmitter(); try { + const basicAcademicSearchAnsweringChain = + createBasicAcademicSearchAnsweringChain(llm, embeddings); + const stream = basicAcademicSearchAnsweringChain.streamEvents( { chat_history: history, @@ -242,8 +252,13 @@ const basicAcademicSearch = (query: string, history: BaseMessage[]) => { return emitter; }; -const handleAcademicSearch = (message: string, history: BaseMessage[]) => { - const emitter = basicAcademicSearch(message, history); +const handleAcademicSearch = ( + message: string, + history: BaseMessage[], + llm: BaseChatModel, + embeddings: Embeddings, +) => { + const emitter = basicAcademicSearch(message, history, llm, embeddings); return emitter; }; diff --git a/src/agents/imageSearchAgent.ts b/src/agents/imageSearchAgent.ts index bf49de0..3adf631 100644 --- a/src/agents/imageSearchAgent.ts +++ b/src/agents/imageSearchAgent.ts @@ -4,16 +4,11 @@ import { RunnableLambda, } from '@langchain/core/runnables'; import { PromptTemplate } from '@langchain/core/prompts'; -import { ChatOpenAI } from '@langchain/openai'; import formatChatHistoryAsString from '../utils/formatHistory'; import { BaseMessage } from '@langchain/core/messages'; import { StringOutputParser } from '@langchain/core/output_parsers'; import { searchSearxng } from '../core/searxng'; - -const llm = new ChatOpenAI({ - modelName: process.env.MODEL_NAME, - temperature: 0.7, -}); +import type { BaseChatModel } from '@langchain/core/language_models/chat_models'; const imageSearchChainPrompt = ` You will be given a conversation below and a follow up question. You need to rephrase the follow-up question so it is a standalone question that can be used by the LLM to search the web for images. @@ -43,38 +38,48 @@ type ImageSearchChainInput = { const strParser = new StringOutputParser(); -const imageSearchChain = RunnableSequence.from([ - RunnableMap.from({ - chat_history: (input: ImageSearchChainInput) => { - return formatChatHistoryAsString(input.chat_history); - }, - query: (input: ImageSearchChainInput) => { - return input.query; - }, - }), - PromptTemplate.fromTemplate(imageSearchChainPrompt), - llm, - strParser, - RunnableLambda.from(async (input: string) => { - const res = await searchSearxng(input, { - categories: ['images'], - engines: ['bing_images', 'google_images'], - }); +const createImageSearchChain = (llm: BaseChatModel) => { + return RunnableSequence.from([ + RunnableMap.from({ + chat_history: (input: ImageSearchChainInput) => { + return formatChatHistoryAsString(input.chat_history); + }, + query: (input: ImageSearchChainInput) => { + return input.query; + }, + }), + PromptTemplate.fromTemplate(imageSearchChainPrompt), + llm, + strParser, + RunnableLambda.from(async (input: string) => { + const res = await searchSearxng(input, { + categories: ['images'], + engines: ['bing_images', 'google_images'], + }); - const images = []; + const images = []; - res.results.forEach((result) => { - if (result.img_src && result.url && result.title) { - images.push({ - img_src: result.img_src, - url: result.url, - title: result.title, - }); - } - }); + res.results.forEach((result) => { + if (result.img_src && result.url && result.title) { + images.push({ + img_src: result.img_src, + url: result.url, + title: result.title, + }); + } + }); - return images.slice(0, 10); - }), -]); + return images.slice(0, 10); + }), + ]); +}; -export default imageSearchChain; +const handleImageSearch = ( + input: ImageSearchChainInput, + llm: BaseChatModel, +) => { + const imageSearchChain = createImageSearchChain(llm); + return imageSearchChain.invoke(input); +}; + +export default handleImageSearch; diff --git a/src/agents/redditSearchAgent.ts b/src/agents/redditSearchAgent.ts index 3b6a274..dca3332 100644 --- a/src/agents/redditSearchAgent.ts +++ b/src/agents/redditSearchAgent.ts @@ -9,24 +9,16 @@ import { RunnableMap, RunnableLambda, } from '@langchain/core/runnables'; -import { ChatOpenAI, OpenAIEmbeddings } from '@langchain/openai'; import { StringOutputParser } from '@langchain/core/output_parsers'; import { Document } from '@langchain/core/documents'; import { searchSearxng } from '../core/searxng'; import type { StreamEvent } from '@langchain/core/tracers/log_stream'; +import type { BaseChatModel } from '@langchain/core/language_models/chat_models'; +import type { Embeddings } from '@langchain/core/embeddings'; import formatChatHistoryAsString from '../utils/formatHistory'; import eventEmitter from 'events'; import computeSimilarity from '../utils/computeSimilarity'; -const llm = new ChatOpenAI({ - modelName: process.env.MODEL_NAME, - temperature: 0.7, -}); - -const embeddings = new OpenAIEmbeddings({ - modelName: 'text-embedding-3-large', -}); - const basicRedditSearchRetrieverPrompt = ` You will be given a conversation below and a follow up question. You need to rephrase the follow-up question if needed so it is a standalone question that can be used by the LLM to search the web for information. If it is a writing task or a simple hi, hello rather than a question, you need to return \`not_needed\` as the response. @@ -104,118 +96,135 @@ const handleStream = async ( } }; -const processDocs = async (docs: Document[]) => { - return docs - .map((_, index) => `${index + 1}. ${docs[index].pageContent}`) - .join('\n'); -}; - -const rerankDocs = async ({ - query, - docs, -}: { - query: string; - docs: Document[]; -}) => { - if (docs.length === 0) { - return docs; - } - - const docsWithContent = docs.filter( - (doc) => doc.pageContent && doc.pageContent.length > 0, - ); - - const docEmbeddings = await embeddings.embedDocuments( - docsWithContent.map((doc) => doc.pageContent), - ); - - const queryEmbedding = await embeddings.embedQuery(query); - - const similarity = docEmbeddings.map((docEmbedding, i) => { - const sim = computeSimilarity(queryEmbedding, docEmbedding); - - return { - index: i, - similarity: sim, - }; - }); - - const sortedDocs = similarity - .sort((a, b) => b.similarity - a.similarity) - .slice(0, 15) - .filter((sim) => sim.similarity > 0.3) - .map((sim) => docsWithContent[sim.index]); - - return sortedDocs; -}; - type BasicChainInput = { chat_history: BaseMessage[]; query: string; }; -const basicRedditSearchRetrieverChain = RunnableSequence.from([ - PromptTemplate.fromTemplate(basicRedditSearchRetrieverPrompt), - llm, - strParser, - RunnableLambda.from(async (input: string) => { - if (input === 'not_needed') { - return { query: '', docs: [] }; +const createBasicRedditSearchRetrieverChain = (llm: BaseChatModel) => { + return RunnableSequence.from([ + PromptTemplate.fromTemplate(basicRedditSearchRetrieverPrompt), + llm, + strParser, + RunnableLambda.from(async (input: string) => { + if (input === 'not_needed') { + return { query: '', docs: [] }; + } + + const res = await searchSearxng(input, { + language: 'en', + engines: ['reddit'], + }); + + const documents = res.results.map( + (result) => + new Document({ + pageContent: result.content ? result.content : result.title, + metadata: { + title: result.title, + url: result.url, + ...(result.img_src && { img_src: result.img_src }), + }, + }), + ); + + return { query: input, docs: documents }; + }), + ]); +}; + +const createBasicRedditSearchAnsweringChain = ( + llm: BaseChatModel, + embeddings: Embeddings, +) => { + const basicRedditSearchRetrieverChain = + createBasicRedditSearchRetrieverChain(llm); + + const processDocs = async (docs: Document[]) => { + return docs + .map((_, index) => `${index + 1}. ${docs[index].pageContent}`) + .join('\n'); + }; + + const rerankDocs = async ({ + query, + docs, + }: { + query: string; + docs: Document[]; + }) => { + if (docs.length === 0) { + return docs; } - const res = await searchSearxng(input, { - language: 'en', - engines: ['reddit'], - }); - - const documents = res.results.map( - (result) => - new Document({ - pageContent: result.content ? result.content : result.title, - metadata: { - title: result.title, - url: result.url, - ...(result.img_src && { img_src: result.img_src }), - }, - }), + const docsWithContent = docs.filter( + (doc) => doc.pageContent && doc.pageContent.length > 0, ); - return { query: input, docs: documents }; - }), -]); + const docEmbeddings = await embeddings.embedDocuments( + docsWithContent.map((doc) => doc.pageContent), + ); -const basicRedditSearchAnsweringChain = RunnableSequence.from([ - RunnableMap.from({ - query: (input: BasicChainInput) => input.query, - chat_history: (input: BasicChainInput) => input.chat_history, - context: RunnableSequence.from([ - (input) => ({ - query: input.query, - chat_history: formatChatHistoryAsString(input.chat_history), - }), - basicRedditSearchRetrieverChain - .pipe(rerankDocs) - .withConfig({ - runName: 'FinalSourceRetriever', - }) - .pipe(processDocs), + const queryEmbedding = await embeddings.embedQuery(query); + + const similarity = docEmbeddings.map((docEmbedding, i) => { + const sim = computeSimilarity(queryEmbedding, docEmbedding); + + return { + index: i, + similarity: sim, + }; + }); + + const sortedDocs = similarity + .sort((a, b) => b.similarity - a.similarity) + .slice(0, 15) + .filter((sim) => sim.similarity > 0.3) + .map((sim) => docsWithContent[sim.index]); + + return sortedDocs; + }; + + return RunnableSequence.from([ + RunnableMap.from({ + query: (input: BasicChainInput) => input.query, + chat_history: (input: BasicChainInput) => input.chat_history, + context: RunnableSequence.from([ + (input) => ({ + query: input.query, + chat_history: formatChatHistoryAsString(input.chat_history), + }), + basicRedditSearchRetrieverChain + .pipe(rerankDocs) + .withConfig({ + runName: 'FinalSourceRetriever', + }) + .pipe(processDocs), + ]), + }), + ChatPromptTemplate.fromMessages([ + ['system', basicRedditSearchResponsePrompt], + new MessagesPlaceholder('chat_history'), + ['user', '{query}'], ]), - }), - ChatPromptTemplate.fromMessages([ - ['system', basicRedditSearchResponsePrompt], - new MessagesPlaceholder('chat_history'), - ['user', '{query}'], - ]), - llm, - strParser, -]).withConfig({ - runName: 'FinalResponseGenerator', -}); + llm, + strParser, + ]).withConfig({ + runName: 'FinalResponseGenerator', + }); +}; -const basicRedditSearch = (query: string, history: BaseMessage[]) => { +const basicRedditSearch = ( + query: string, + history: BaseMessage[], + llm: BaseChatModel, + embeddings: Embeddings, +) => { const emitter = new eventEmitter(); try { + const basicRedditSearchAnsweringChain = + createBasicRedditSearchAnsweringChain(llm, embeddings); const stream = basicRedditSearchAnsweringChain.streamEvents( { chat_history: history, @@ -238,8 +247,13 @@ const basicRedditSearch = (query: string, history: BaseMessage[]) => { return emitter; }; -const handleRedditSearch = (message: string, history: BaseMessage[]) => { - const emitter = basicRedditSearch(message, history); +const handleRedditSearch = ( + message: string, + history: BaseMessage[], + llm: BaseChatModel, + embeddings: Embeddings, +) => { + const emitter = basicRedditSearch(message, history, llm, embeddings); return emitter; }; diff --git a/src/agents/webSearchAgent.ts b/src/agents/webSearchAgent.ts index 047eb3d..66db2a1 100644 --- a/src/agents/webSearchAgent.ts +++ b/src/agents/webSearchAgent.ts @@ -9,24 +9,16 @@ import { RunnableMap, RunnableLambda, } from '@langchain/core/runnables'; -import { ChatOpenAI, OpenAIEmbeddings } from '@langchain/openai'; import { StringOutputParser } from '@langchain/core/output_parsers'; import { Document } from '@langchain/core/documents'; import { searchSearxng } from '../core/searxng'; import type { StreamEvent } from '@langchain/core/tracers/log_stream'; +import type { BaseChatModel } from '@langchain/core/language_models/chat_models'; +import type { Embeddings } from '@langchain/core/embeddings'; import formatChatHistoryAsString from '../utils/formatHistory'; import eventEmitter from 'events'; import computeSimilarity from '../utils/computeSimilarity'; -const llm = new ChatOpenAI({ - modelName: process.env.MODEL_NAME, - temperature: 0.7, -}); - -const embeddings = new OpenAIEmbeddings({ - modelName: 'text-embedding-3-large', -}); - const basicSearchRetrieverPrompt = ` You will be given a conversation below and a follow up question. You need to rephrase the follow-up question if needed so it is a standalone question that can be used by the LLM to search the web for information. If it is a writing task or a simple hi, hello rather than a question, you need to return \`not_needed\` as the response. @@ -104,117 +96,136 @@ const handleStream = async ( } }; -const processDocs = async (docs: Document[]) => { - return docs - .map((_, index) => `${index + 1}. ${docs[index].pageContent}`) - .join('\n'); -}; - -const rerankDocs = async ({ - query, - docs, -}: { - query: string; - docs: Document[]; -}) => { - if (docs.length === 0) { - return docs; - } - - const docsWithContent = docs.filter( - (doc) => doc.pageContent && doc.pageContent.length > 0, - ); - - const docEmbeddings = await embeddings.embedDocuments( - docsWithContent.map((doc) => doc.pageContent), - ); - - const queryEmbedding = await embeddings.embedQuery(query); - - const similarity = docEmbeddings.map((docEmbedding, i) => { - const sim = computeSimilarity(queryEmbedding, docEmbedding); - - return { - index: i, - similarity: sim, - }; - }); - - const sortedDocs = similarity - .sort((a, b) => b.similarity - a.similarity) - .filter((sim) => sim.similarity > 0.5) - .slice(0, 15) - .map((sim) => docsWithContent[sim.index]); - - return sortedDocs; -}; - type BasicChainInput = { chat_history: BaseMessage[]; query: string; }; -const basicWebSearchRetrieverChain = RunnableSequence.from([ - PromptTemplate.fromTemplate(basicSearchRetrieverPrompt), - llm, - strParser, - RunnableLambda.from(async (input: string) => { - if (input === 'not_needed') { - return { query: '', docs: [] }; +const createBasicWebSearchRetrieverChain = (llm: BaseChatModel) => { + return RunnableSequence.from([ + PromptTemplate.fromTemplate(basicSearchRetrieverPrompt), + llm, + strParser, + RunnableLambda.from(async (input: string) => { + if (input === 'not_needed') { + return { query: '', docs: [] }; + } + + const res = await searchSearxng(input, { + language: 'en', + }); + + const documents = res.results.map( + (result) => + new Document({ + pageContent: result.content, + metadata: { + title: result.title, + url: result.url, + ...(result.img_src && { img_src: result.img_src }), + }, + }), + ); + + return { query: input, docs: documents }; + }), + ]); +}; + +const createBasicWebSearchAnsweringChain = ( + llm: BaseChatModel, + embeddings: Embeddings, +) => { + const basicWebSearchRetrieverChain = createBasicWebSearchRetrieverChain(llm); + + const processDocs = async (docs: Document[]) => { + return docs + .map((_, index) => `${index + 1}. ${docs[index].pageContent}`) + .join('\n'); + }; + + const rerankDocs = async ({ + query, + docs, + }: { + query: string; + docs: Document[]; + }) => { + if (docs.length === 0) { + return docs; } - const res = await searchSearxng(input, { - language: 'en', - }); - - const documents = res.results.map( - (result) => - new Document({ - pageContent: result.content, - metadata: { - title: result.title, - url: result.url, - ...(result.img_src && { img_src: result.img_src }), - }, - }), + const docsWithContent = docs.filter( + (doc) => doc.pageContent && doc.pageContent.length > 0, ); - return { query: input, docs: documents }; - }), -]); + const docEmbeddings = await embeddings.embedDocuments( + docsWithContent.map((doc) => doc.pageContent), + ); -const basicWebSearchAnsweringChain = RunnableSequence.from([ - RunnableMap.from({ - query: (input: BasicChainInput) => input.query, - chat_history: (input: BasicChainInput) => input.chat_history, - context: RunnableSequence.from([ - (input) => ({ - query: input.query, - chat_history: formatChatHistoryAsString(input.chat_history), - }), - basicWebSearchRetrieverChain - .pipe(rerankDocs) - .withConfig({ - runName: 'FinalSourceRetriever', - }) - .pipe(processDocs), + const queryEmbedding = await embeddings.embedQuery(query); + + const similarity = docEmbeddings.map((docEmbedding, i) => { + const sim = computeSimilarity(queryEmbedding, docEmbedding); + + return { + index: i, + similarity: sim, + }; + }); + + const sortedDocs = similarity + .sort((a, b) => b.similarity - a.similarity) + .filter((sim) => sim.similarity > 0.5) + .slice(0, 15) + .map((sim) => docsWithContent[sim.index]); + + return sortedDocs; + }; + + return RunnableSequence.from([ + RunnableMap.from({ + query: (input: BasicChainInput) => input.query, + chat_history: (input: BasicChainInput) => input.chat_history, + context: RunnableSequence.from([ + (input) => ({ + query: input.query, + chat_history: formatChatHistoryAsString(input.chat_history), + }), + basicWebSearchRetrieverChain + .pipe(rerankDocs) + .withConfig({ + runName: 'FinalSourceRetriever', + }) + .pipe(processDocs), + ]), + }), + ChatPromptTemplate.fromMessages([ + ['system', basicWebSearchResponsePrompt], + new MessagesPlaceholder('chat_history'), + ['user', '{query}'], ]), - }), - ChatPromptTemplate.fromMessages([ - ['system', basicWebSearchResponsePrompt], - new MessagesPlaceholder('chat_history'), - ['user', '{query}'], - ]), - llm, - strParser, -]).withConfig({ - runName: 'FinalResponseGenerator', -}); + llm, + strParser, + ]).withConfig({ + runName: 'FinalResponseGenerator', + }); +}; -const basicWebSearch = (query: string, history: BaseMessage[]) => { +const basicWebSearch = ( + query: string, + history: BaseMessage[], + llm: BaseChatModel, + embeddings: Embeddings, +) => { const emitter = new eventEmitter(); try { + const basicWebSearchAnsweringChain = createBasicWebSearchAnsweringChain( + llm, + embeddings, + ); + const stream = basicWebSearchAnsweringChain.streamEvents( { chat_history: history, @@ -237,8 +248,13 @@ const basicWebSearch = (query: string, history: BaseMessage[]) => { return emitter; }; -const handleWebSearch = (message: string, history: BaseMessage[]) => { - const emitter = basicWebSearch(message, history); +const handleWebSearch = ( + message: string, + history: BaseMessage[], + llm: BaseChatModel, + embeddings: Embeddings, +) => { + const emitter = basicWebSearch(message, history, llm, embeddings); return emitter; }; diff --git a/src/agents/wolframAlphaSearchAgent.ts b/src/agents/wolframAlphaSearchAgent.ts index 4ab1990..a68110d 100644 --- a/src/agents/wolframAlphaSearchAgent.ts +++ b/src/agents/wolframAlphaSearchAgent.ts @@ -9,19 +9,15 @@ import { RunnableMap, RunnableLambda, } from '@langchain/core/runnables'; -import { ChatOpenAI, OpenAI } from '@langchain/openai'; import { StringOutputParser } from '@langchain/core/output_parsers'; import { Document } from '@langchain/core/documents'; import { searchSearxng } from '../core/searxng'; import type { StreamEvent } from '@langchain/core/tracers/log_stream'; +import type { BaseChatModel } from '@langchain/core/language_models/chat_models'; +import type { Embeddings } from '@langchain/core/embeddings'; import formatChatHistoryAsString from '../utils/formatHistory'; import eventEmitter from 'events'; -const llm = new ChatOpenAI({ - modelName: process.env.MODEL_NAME, - temperature: 0.7, -}); - const basicWolframAlphaSearchRetrieverPrompt = ` You will be given a conversation below and a follow up question. You need to rephrase the follow-up question if needed so it is a standalone question that can be used by the LLM to search the web for information. If it is a writing task or a simple hi, hello rather than a question, you need to return \`not_needed\` as the response. @@ -99,81 +95,94 @@ const handleStream = async ( } }; -const processDocs = async (docs: Document[]) => { - return docs - .map((_, index) => `${index + 1}. ${docs[index].pageContent}`) - .join('\n'); -}; - type BasicChainInput = { chat_history: BaseMessage[]; query: string; }; -const basicWolframAlphaSearchRetrieverChain = RunnableSequence.from([ - PromptTemplate.fromTemplate(basicWolframAlphaSearchRetrieverPrompt), - llm, - strParser, - RunnableLambda.from(async (input: string) => { - if (input === 'not_needed') { - return { query: '', docs: [] }; - } +const createBasicWolframAlphaSearchRetrieverChain = (llm: BaseChatModel) => { + return RunnableSequence.from([ + PromptTemplate.fromTemplate(basicWolframAlphaSearchRetrieverPrompt), + llm, + strParser, + RunnableLambda.from(async (input: string) => { + if (input === 'not_needed') { + return { query: '', docs: [] }; + } - const res = await searchSearxng(input, { - language: 'en', - engines: ['wolframalpha'], - }); + const res = await searchSearxng(input, { + language: 'en', + engines: ['wolframalpha'], + }); - const documents = res.results.map( - (result) => - new Document({ - pageContent: result.content, - metadata: { - title: result.title, - url: result.url, - ...(result.img_src && { img_src: result.img_src }), - }, + const documents = res.results.map( + (result) => + new Document({ + pageContent: result.content, + metadata: { + title: result.title, + url: result.url, + ...(result.img_src && { img_src: result.img_src }), + }, + }), + ); + + return { query: input, docs: documents }; + }), + ]); +}; + +const createBasicWolframAlphaSearchAnsweringChain = (llm: BaseChatModel) => { + const basicWolframAlphaSearchRetrieverChain = + createBasicWolframAlphaSearchRetrieverChain(llm); + + const processDocs = (docs: Document[]) => { + return docs + .map((_, index) => `${index + 1}. ${docs[index].pageContent}`) + .join('\n'); + }; + + return RunnableSequence.from([ + RunnableMap.from({ + query: (input: BasicChainInput) => input.query, + chat_history: (input: BasicChainInput) => input.chat_history, + context: RunnableSequence.from([ + (input) => ({ + query: input.query, + chat_history: formatChatHistoryAsString(input.chat_history), }), - ); - - return { query: input, docs: documents }; - }), -]); - -const basicWolframAlphaSearchAnsweringChain = RunnableSequence.from([ - RunnableMap.from({ - query: (input: BasicChainInput) => input.query, - chat_history: (input: BasicChainInput) => input.chat_history, - context: RunnableSequence.from([ - (input) => ({ - query: input.query, - chat_history: formatChatHistoryAsString(input.chat_history), - }), - basicWolframAlphaSearchRetrieverChain - .pipe(({ query, docs }) => { - return docs; - }) - .withConfig({ - runName: 'FinalSourceRetriever', - }) - .pipe(processDocs), + basicWolframAlphaSearchRetrieverChain + .pipe(({ query, docs }) => { + return docs; + }) + .withConfig({ + runName: 'FinalSourceRetriever', + }) + .pipe(processDocs), + ]), + }), + ChatPromptTemplate.fromMessages([ + ['system', basicWolframAlphaSearchResponsePrompt], + new MessagesPlaceholder('chat_history'), + ['user', '{query}'], ]), - }), - ChatPromptTemplate.fromMessages([ - ['system', basicWolframAlphaSearchResponsePrompt], - new MessagesPlaceholder('chat_history'), - ['user', '{query}'], - ]), - llm, - strParser, -]).withConfig({ - runName: 'FinalResponseGenerator', -}); + llm, + strParser, + ]).withConfig({ + runName: 'FinalResponseGenerator', + }); +}; -const basicWolframAlphaSearch = (query: string, history: BaseMessage[]) => { +const basicWolframAlphaSearch = ( + query: string, + history: BaseMessage[], + llm: BaseChatModel, +) => { const emitter = new eventEmitter(); try { + const basicWolframAlphaSearchAnsweringChain = + createBasicWolframAlphaSearchAnsweringChain(llm); const stream = basicWolframAlphaSearchAnsweringChain.streamEvents( { chat_history: history, @@ -196,8 +205,13 @@ const basicWolframAlphaSearch = (query: string, history: BaseMessage[]) => { return emitter; }; -const handleWolframAlphaSearch = (message: string, history: BaseMessage[]) => { - const emitter = basicWolframAlphaSearch(message, history); +const handleWolframAlphaSearch = ( + message: string, + history: BaseMessage[], + llm: BaseChatModel, + embeddings: Embeddings, +) => { + const emitter = basicWolframAlphaSearch(message, history, llm); return emitter; }; diff --git a/src/agents/writingAssistant.ts b/src/agents/writingAssistant.ts index 0fc5097..ff5365e 100644 --- a/src/agents/writingAssistant.ts +++ b/src/agents/writingAssistant.ts @@ -4,15 +4,11 @@ import { MessagesPlaceholder, } from '@langchain/core/prompts'; import { RunnableSequence } from '@langchain/core/runnables'; -import { ChatOpenAI } from '@langchain/openai'; import { StringOutputParser } from '@langchain/core/output_parsers'; import type { StreamEvent } from '@langchain/core/tracers/log_stream'; import eventEmitter from 'events'; - -const llm = new ChatOpenAI({ - modelName: process.env.MODEL_NAME, - temperature: 0.7, -}); +import type { BaseChatModel } from '@langchain/core/language_models/chat_models'; +import type { Embeddings } from '@langchain/core/embeddings'; const writingAssistantPrompt = ` You are Perplexica, an AI model who is expert at searching the web and answering user's queries. You are currently set on focus mode 'Writing Assistant', this means you will be helping the user write a response to a given query. @@ -44,22 +40,30 @@ const handleStream = async ( } }; -const writingAssistantChain = RunnableSequence.from([ - ChatPromptTemplate.fromMessages([ - ['system', writingAssistantPrompt], - new MessagesPlaceholder('chat_history'), - ['user', '{query}'], - ]), - llm, - strParser, -]).withConfig({ - runName: 'FinalResponseGenerator', -}); +const createWritingAssistantChain = (llm: BaseChatModel) => { + return RunnableSequence.from([ + ChatPromptTemplate.fromMessages([ + ['system', writingAssistantPrompt], + new MessagesPlaceholder('chat_history'), + ['user', '{query}'], + ]), + llm, + strParser, + ]).withConfig({ + runName: 'FinalResponseGenerator', + }); +}; -const handleWritingAssistant = (query: string, history: BaseMessage[]) => { +const handleWritingAssistant = ( + query: string, + history: BaseMessage[], + llm: BaseChatModel, + embeddings: Embeddings, +) => { const emitter = new eventEmitter(); try { + const writingAssistantChain = createWritingAssistantChain(llm); const stream = writingAssistantChain.streamEvents( { chat_history: history, diff --git a/src/agents/youtubeSearchAgent.ts b/src/agents/youtubeSearchAgent.ts index 7c1bcf5..4ed8b41 100644 --- a/src/agents/youtubeSearchAgent.ts +++ b/src/agents/youtubeSearchAgent.ts @@ -9,24 +9,16 @@ import { RunnableMap, RunnableLambda, } from '@langchain/core/runnables'; -import { ChatOpenAI, OpenAIEmbeddings } from '@langchain/openai'; import { StringOutputParser } from '@langchain/core/output_parsers'; import { Document } from '@langchain/core/documents'; import { searchSearxng } from '../core/searxng'; import type { StreamEvent } from '@langchain/core/tracers/log_stream'; +import type { BaseChatModel } from '@langchain/core/language_models/chat_models'; +import type { Embeddings } from '@langchain/core/embeddings'; import formatChatHistoryAsString from '../utils/formatHistory'; import eventEmitter from 'events'; import computeSimilarity from '../utils/computeSimilarity'; -const llm = new ChatOpenAI({ - modelName: process.env.MODEL_NAME, - temperature: 0.7, -}); - -const embeddings = new OpenAIEmbeddings({ - modelName: 'text-embedding-3-large', -}); - const basicYoutubeSearchRetrieverPrompt = ` You will be given a conversation below and a follow up question. You need to rephrase the follow-up question if needed so it is a standalone question that can be used by the LLM to search the web for information. If it is a writing task or a simple hi, hello rather than a question, you need to return \`not_needed\` as the response. @@ -104,118 +96,136 @@ const handleStream = async ( } }; -const processDocs = async (docs: Document[]) => { - return docs - .map((_, index) => `${index + 1}. ${docs[index].pageContent}`) - .join('\n'); -}; - -const rerankDocs = async ({ - query, - docs, -}: { - query: string; - docs: Document[]; -}) => { - if (docs.length === 0) { - return docs; - } - - const docsWithContent = docs.filter( - (doc) => doc.pageContent && doc.pageContent.length > 0, - ); - - const docEmbeddings = await embeddings.embedDocuments( - docsWithContent.map((doc) => doc.pageContent), - ); - - const queryEmbedding = await embeddings.embedQuery(query); - - const similarity = docEmbeddings.map((docEmbedding, i) => { - const sim = computeSimilarity(queryEmbedding, docEmbedding); - - return { - index: i, - similarity: sim, - }; - }); - - const sortedDocs = similarity - .sort((a, b) => b.similarity - a.similarity) - .slice(0, 15) - .filter((sim) => sim.similarity > 0.3) - .map((sim) => docsWithContent[sim.index]); - - return sortedDocs; -}; - type BasicChainInput = { chat_history: BaseMessage[]; query: string; }; -const basicYoutubeSearchRetrieverChain = RunnableSequence.from([ - PromptTemplate.fromTemplate(basicYoutubeSearchRetrieverPrompt), - llm, - strParser, - RunnableLambda.from(async (input: string) => { - if (input === 'not_needed') { - return { query: '', docs: [] }; +const createBasicYoutubeSearchRetrieverChain = (llm: BaseChatModel) => { + return RunnableSequence.from([ + PromptTemplate.fromTemplate(basicYoutubeSearchRetrieverPrompt), + llm, + strParser, + RunnableLambda.from(async (input: string) => { + if (input === 'not_needed') { + return { query: '', docs: [] }; + } + + const res = await searchSearxng(input, { + language: 'en', + engines: ['youtube'], + }); + + const documents = res.results.map( + (result) => + new Document({ + pageContent: result.content ? result.content : result.title, + metadata: { + title: result.title, + url: result.url, + ...(result.img_src && { img_src: result.img_src }), + }, + }), + ); + + return { query: input, docs: documents }; + }), + ]); +}; + +const createBasicYoutubeSearchAnsweringChain = ( + llm: BaseChatModel, + embeddings: Embeddings, +) => { + const basicYoutubeSearchRetrieverChain = + createBasicYoutubeSearchRetrieverChain(llm); + + const processDocs = async (docs: Document[]) => { + return docs + .map((_, index) => `${index + 1}. ${docs[index].pageContent}`) + .join('\n'); + }; + + const rerankDocs = async ({ + query, + docs, + }: { + query: string; + docs: Document[]; + }) => { + if (docs.length === 0) { + return docs; } - const res = await searchSearxng(input, { - language: 'en', - engines: ['youtube'], - }); - - const documents = res.results.map( - (result) => - new Document({ - pageContent: result.content ? result.content : result.title, - metadata: { - title: result.title, - url: result.url, - ...(result.img_src && { img_src: result.img_src }), - }, - }), + const docsWithContent = docs.filter( + (doc) => doc.pageContent && doc.pageContent.length > 0, ); - return { query: input, docs: documents }; - }), -]); + const docEmbeddings = await embeddings.embedDocuments( + docsWithContent.map((doc) => doc.pageContent), + ); -const basicYoutubeSearchAnsweringChain = RunnableSequence.from([ - RunnableMap.from({ - query: (input: BasicChainInput) => input.query, - chat_history: (input: BasicChainInput) => input.chat_history, - context: RunnableSequence.from([ - (input) => ({ - query: input.query, - chat_history: formatChatHistoryAsString(input.chat_history), - }), - basicYoutubeSearchRetrieverChain - .pipe(rerankDocs) - .withConfig({ - runName: 'FinalSourceRetriever', - }) - .pipe(processDocs), + const queryEmbedding = await embeddings.embedQuery(query); + + const similarity = docEmbeddings.map((docEmbedding, i) => { + const sim = computeSimilarity(queryEmbedding, docEmbedding); + + return { + index: i, + similarity: sim, + }; + }); + + const sortedDocs = similarity + .sort((a, b) => b.similarity - a.similarity) + .slice(0, 15) + .filter((sim) => sim.similarity > 0.3) + .map((sim) => docsWithContent[sim.index]); + + return sortedDocs; + }; + + return RunnableSequence.from([ + RunnableMap.from({ + query: (input: BasicChainInput) => input.query, + chat_history: (input: BasicChainInput) => input.chat_history, + context: RunnableSequence.from([ + (input) => ({ + query: input.query, + chat_history: formatChatHistoryAsString(input.chat_history), + }), + basicYoutubeSearchRetrieverChain + .pipe(rerankDocs) + .withConfig({ + runName: 'FinalSourceRetriever', + }) + .pipe(processDocs), + ]), + }), + ChatPromptTemplate.fromMessages([ + ['system', basicYoutubeSearchResponsePrompt], + new MessagesPlaceholder('chat_history'), + ['user', '{query}'], ]), - }), - ChatPromptTemplate.fromMessages([ - ['system', basicYoutubeSearchResponsePrompt], - new MessagesPlaceholder('chat_history'), - ['user', '{query}'], - ]), - llm, - strParser, -]).withConfig({ - runName: 'FinalResponseGenerator', -}); + llm, + strParser, + ]).withConfig({ + runName: 'FinalResponseGenerator', + }); +}; -const basicYoutubeSearch = (query: string, history: BaseMessage[]) => { +const basicYoutubeSearch = ( + query: string, + history: BaseMessage[], + llm: BaseChatModel, + embeddings: Embeddings, +) => { const emitter = new eventEmitter(); try { + const basicYoutubeSearchAnsweringChain = + createBasicYoutubeSearchAnsweringChain(llm, embeddings); + const stream = basicYoutubeSearchAnsweringChain.streamEvents( { chat_history: history, @@ -238,8 +248,13 @@ const basicYoutubeSearch = (query: string, history: BaseMessage[]) => { return emitter; }; -const handleYoutubeSearch = (message: string, history: BaseMessage[]) => { - const emitter = basicYoutubeSearch(message, history); +const handleYoutubeSearch = ( + message: string, + history: BaseMessage[], + llm: BaseChatModel, + embeddings: Embeddings, +) => { + const emitter = basicYoutubeSearch(message, history, llm, embeddings); return emitter; }; diff --git a/src/app.ts b/src/app.ts index 993cb23..19f95bc 100644 --- a/src/app.ts +++ b/src/app.ts @@ -3,6 +3,9 @@ import express from 'express'; import cors from 'cors'; import http from 'http'; import routes from './routes'; +import { getPort } from './config'; + +const port = getPort(); const app = express(); const server = http.createServer(app); @@ -19,8 +22,8 @@ app.get('/api', (_, res) => { res.status(200).json({ status: 'ok' }); }); -server.listen(process.env.PORT!, () => { - console.log(`API server started on port ${process.env.PORT}`); +server.listen(port, () => { + console.log(`API server started on port ${port}`); }); startWebSocketServer(server); diff --git a/src/config.ts b/src/config.ts new file mode 100644 index 0000000..c83522f --- /dev/null +++ b/src/config.ts @@ -0,0 +1,32 @@ +import fs from 'fs'; +import path from 'path'; +import toml, { JsonMap } from '@iarna/toml'; + +const configFileName = 'config.toml'; + +interface Config { + GENERAL: { + PORT: number; + SIMILARITY_MEASURE: string; + }; + API_KEYS: { + OPENAI: string; + }; + API_ENDPOINTS: { + SEARXNG: string; + }; +} + +const loadConfig = () => + toml.parse( + fs.readFileSync(path.join(process.cwd(), `${configFileName}`), 'utf-8'), + ) as any as Config; + +export const getPort = () => loadConfig().GENERAL.PORT; + +export const getSimilarityMeasure = () => + loadConfig().GENERAL.SIMILARITY_MEASURE; + +export const getOpenaiApiKey = () => loadConfig().API_KEYS.OPENAI; + +export const getSearxngApiEndpoint = () => loadConfig().API_ENDPOINTS.SEARXNG; diff --git a/src/core/searxng.ts b/src/core/searxng.ts index 3bb4a53..297e50f 100644 --- a/src/core/searxng.ts +++ b/src/core/searxng.ts @@ -1,4 +1,5 @@ import axios from 'axios'; +import { getSearxngApiEndpoint } from '../config'; interface SearxngSearchOptions { categories?: string[]; @@ -20,7 +21,9 @@ export const searchSearxng = async ( query: string, opts?: SearxngSearchOptions, ) => { - const url = new URL(`${process.env.SEARXNG_API_URL}/search?format=json`); + const searxngURL = getSearxngApiEndpoint(); + + const url = new URL(`${searxngURL}/search?format=json`); url.searchParams.append('q', query); if (opts) { diff --git a/src/routes/images.ts b/src/routes/images.ts index 5a33ac6..dd3925f 100644 --- a/src/routes/images.ts +++ b/src/routes/images.ts @@ -1,5 +1,7 @@ import express from 'express'; -import imageSearchChain from '../agents/imageSearchAgent'; +import handleImageSearch from '../agents/imageSearchAgent'; +import { ChatOpenAI } from '@langchain/openai'; +import { getOpenaiApiKey } from '../config'; const router = express.Router(); @@ -7,11 +9,13 @@ router.post('/', async (req, res) => { try { const { query, chat_history } = req.body; - const images = await imageSearchChain.invoke({ - query, - chat_history, + const llm = new ChatOpenAI({ + temperature: 0.7, + openAIApiKey: getOpenaiApiKey(), }); + const images = await handleImageSearch({ query, chat_history }, llm); + res.status(200).json({ images }); } catch (err) { res.status(500).json({ message: 'An error has occurred.' }); diff --git a/src/utils/computeSimilarity.ts b/src/utils/computeSimilarity.ts index 1b07cc7..6e36b75 100644 --- a/src/utils/computeSimilarity.ts +++ b/src/utils/computeSimilarity.ts @@ -1,10 +1,13 @@ import dot from 'compute-dot'; import cosineSimilarity from 'compute-cosine-similarity'; +import { getSimilarityMeasure } from '../config'; const computeSimilarity = (x: number[], y: number[]): number => { - if (process.env.SIMILARITY_MEASURE === 'cosine') { + const similarityMeasure = getSimilarityMeasure(); + + if (similarityMeasure === 'cosine') { return cosineSimilarity(x, y); - } else if (process.env.SIMILARITY_MEASURE === 'dot') { + } else if (similarityMeasure === 'dot') { return dot(x, y); } diff --git a/src/websocket/connectionManager.ts b/src/websocket/connectionManager.ts index a5746e4..2dc8d59 100644 --- a/src/websocket/connectionManager.ts +++ b/src/websocket/connectionManager.ts @@ -1,10 +1,23 @@ import { WebSocket } from 'ws'; import { handleMessage } from './messageHandler'; +import { ChatOpenAI, OpenAIEmbeddings } from '@langchain/openai'; +import { getOpenaiApiKey } from '../config'; export const handleConnection = (ws: WebSocket) => { + const llm = new ChatOpenAI({ + temperature: 0.7, + openAIApiKey: getOpenaiApiKey(), + }); + + const embeddings = new OpenAIEmbeddings({ + openAIApiKey: getOpenaiApiKey(), + modelName: 'text-embedding-3-large', + }); + ws.on( 'message', - async (message) => await handleMessage(message.toString(), ws), + async (message) => + await handleMessage(message.toString(), ws, llm, embeddings), ); ws.on('close', () => console.log('Connection closed')); diff --git a/src/websocket/messageHandler.ts b/src/websocket/messageHandler.ts index 83fa50d..48774bf 100644 --- a/src/websocket/messageHandler.ts +++ b/src/websocket/messageHandler.ts @@ -6,6 +6,8 @@ import handleWritingAssistant from '../agents/writingAssistant'; import handleWolframAlphaSearch from '../agents/wolframAlphaSearchAgent'; import handleYoutubeSearch from '../agents/youtubeSearchAgent'; import handleRedditSearch from '../agents/redditSearchAgent'; +import type { BaseChatModel } from '@langchain/core/language_models/chat_models'; +import type { Embeddings } from '@langchain/core/embeddings'; type Message = { type: string; @@ -58,7 +60,12 @@ const handleEmitterEvents = ( }); }; -export const handleMessage = async (message: string, ws: WebSocket) => { +export const handleMessage = async ( + message: string, + ws: WebSocket, + llm: BaseChatModel, + embeddings: Embeddings, +) => { try { const parsedMessage = JSON.parse(message) as Message; const id = Math.random().toString(36).substring(7); @@ -83,7 +90,12 @@ export const handleMessage = async (message: string, ws: WebSocket) => { if (parsedMessage.type === 'message') { const handler = searchHandlers[parsedMessage.focusMode]; if (handler) { - const emitter = handler(parsedMessage.content, history); + const emitter = handler( + parsedMessage.content, + history, + llm, + embeddings, + ); handleEmitterEvents(emitter, ws, id); } else { ws.send(JSON.stringify({ type: 'error', data: 'Invalid focus mode' })); diff --git a/src/websocket/websocketServer.ts b/src/websocket/websocketServer.ts index 8aca021..451f9f2 100644 --- a/src/websocket/websocketServer.ts +++ b/src/websocket/websocketServer.ts @@ -1,15 +1,17 @@ import { WebSocketServer } from 'ws'; import { handleConnection } from './connectionManager'; import http from 'http'; +import { getPort } from '../config'; export const initServer = ( server: http.Server, ) => { + const port = getPort(); const wss = new WebSocketServer({ server }); wss.on('connection', (ws) => { handleConnection(ws); }); - console.log(`WebSocket server started on port ${process.env.PORT}`); + console.log(`WebSocket server started on port ${port}`); }; diff --git a/yarn.lock b/yarn.lock index 8518bb2..080d4c0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -24,6 +24,11 @@ dependencies: "@jridgewell/trace-mapping" "0.3.9" +"@iarna/toml@^2.2.5": + version "2.2.5" + resolved "https://registry.yarnpkg.com/@iarna/toml/-/toml-2.2.5.tgz#b32366c89b43c6f8cefbdefac778b9c828e3ba8c" + integrity sha512-trnsAYxU3xnS1gPHPyU961coFyLkh4gAD/0zQ5mymY4yOZ+CYvsPqUbOFSw0aDM4y0tV7tiFxL/1XfXPNC6IPg== + "@jridgewell/resolve-uri@^3.0.3": version "3.1.2" resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz#7a0ee601f60f99a20c7c7c5ff0c80388c1189bd6" From 28a7175afc010e7df227e1c71a6b212dc430eacc Mon Sep 17 00:00:00 2001 From: ItzCrazyKns Date: Sat, 20 Apr 2024 10:23:56 +0530 Subject: [PATCH 005/470] feat(chat): Add loading for ws --- ui/components/ChatWindow.tsx | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/ui/components/ChatWindow.tsx b/ui/components/ChatWindow.tsx index 34102c6..94f4f00 100644 --- a/ui/components/ChatWindow.tsx +++ b/ui/components/ChatWindow.tsx @@ -152,7 +152,7 @@ const ChatWindow = () => { sendMessage(message.content); }; - return ( + return ws ? (
{messages.length > 0 ? ( <> @@ -173,7 +173,14 @@ const ChatWindow = () => { /> )}
- ); + ) : ( +
+ +
+ ) }; export default ChatWindow; From d37a1a80207dd5c6895d135654fd011b172897a3 Mon Sep 17 00:00:00 2001 From: ItzCrazyKns Date: Sat, 20 Apr 2024 11:18:52 +0530 Subject: [PATCH 006/470] feat(agents): support local LLMs --- CONTRIBUTING.md | 6 +-- README.md | 13 ++++- sample.config.toml | 5 +- src/agents/academicSearchAgent.ts | 2 +- src/agents/imageSearchAgent.ts | 2 +- src/agents/redditSearchAgent.ts | 2 +- src/agents/webSearchAgent.ts | 2 +- src/agents/wolframAlphaSearchAgent.ts | 2 +- src/agents/youtubeSearchAgent.ts | 2 +- src/config.ts | 14 +++++- src/core/agentPicker.ts | 69 --------------------------- src/lib/providers.ts | 58 ++++++++++++++++++++++ src/{core => lib}/searxng.ts | 0 src/websocket/connectionManager.ts | 36 +++++++++----- ui/components/ChatWindow.tsx | 22 +++++++-- 15 files changed, 135 insertions(+), 100 deletions(-) delete mode 100644 src/core/agentPicker.ts create mode 100644 src/lib/providers.ts rename src/{core => lib}/searxng.ts (100%) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index af43ae1..c779f91 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -9,16 +9,14 @@ Perplexica's design consists of two main domains: - **Frontend (`ui` directory)**: This is a Next.js application holding all user interface components. It's a self-contained environment that manages everything the user interacts with. - **Backend (root and `src` directory)**: The backend logic is situated in the `src` folder, but the root directory holds the main `package.json` for backend dependency management. -Both the root directory (for backend configurations outside `src`) and the `ui` folder come with an `.env.example` file. These are templates for environment variables that you need to set up manually for the application to run correctly. - ## Setting Up Your Environment Before diving into coding, setting up your local environment is key. Here's what you need to do: ### Backend -1. In the root directory, locate the `.env.example` file. -2. Rename it to `.env` and fill in the necessary environment variables specific to the backend. +1. In the root directory, locate the `sample.config.toml` file. +2. Rename it to `config.toml` and fill in the necessary configuration fields specific to the backend. 3. Run `npm install` to install dependencies. 4. Use `npm run dev` to start the backend in development mode. diff --git a/README.md b/README.md index 942ba9e..428ee30 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,8 @@ Perplexica is an open-source AI-powered searching tool or an AI-powered search e ## Features +- **Local LLMs**: You can make use local LLMs such as LLama2 and Mixtral using Ollama. + - **Two Main Modes:** - **Copilot Mode:** (In development) Boosts search by generating different queries to find more relevant internet sources. Like normal search instead of just using the context by SearxNG, it visits the top matches and tries to find relevant sources to the user's query directly from the page. - **Normal Mode:** Processes your query and performs a web search. @@ -58,7 +60,14 @@ There are mainly 2 ways of installing Perplexica - With Docker, Without Docker. 4. Rename the `sample.config.toml` file to `config.toml`. For Docker setups, you need only fill in the following fields: - - `OPENAI`: Your OpenAI API key. + - `CHAT_MODEL`: The name of the LLM to use. Example: `llama2` for Ollama users & `gpt-3.5-turbo` for OpenAI users. + - `CHAT_MODEL_PROVIDER`: The chat model provider, either `openai` or `ollama`. Depending upon which provider you use you would have to fill in the following fields: + + - `OPENAI`: Your OpenAI API key. **You only need to fill this if you wish to use OpenAI's models.** + - `OLLAMA`: Your Ollama API URL. **You need to fill this if you wish to use Ollama's models instead of OpenAI's.** + + **Note**: (In development) You can change these and use different models after running Perplexica as well from the settings page. + - `SIMILARITY_MEASURE`: The similarity measure to use (This is filled by default; you can leave it as is if you are unsure about it.) 5. Ensure you are in the directory containing the `docker-compose.yaml` file and execute: @@ -84,7 +93,7 @@ For setups without Docker: ## Upcoming Features - [ ] Finalizing Copilot Mode -- [ ] Adding support for multiple local LLMs and LLM providers such as Anthropic, Google, etc. +- [X] Adding support for local LLMs - [ ] Adding Discover and History Saving features - [x] Introducing various Focus Modes diff --git a/sample.config.toml b/sample.config.toml index 1082184..9f6d927 100644 --- a/sample.config.toml +++ b/sample.config.toml @@ -1,9 +1,12 @@ [GENERAL] PORT = 3001 # Port to run the server on SIMILARITY_MEASURE = "cosine" # "cosine" or "dot" +CHAT_MODEL_PROVIDER = "openai" # "openai" or "ollama" +CHAT_MODEL = "gpt-3.5-turbo" # Name of the model to use [API_KEYS] OPENAI = "sk-1234567890abcdef1234567890abcdef" # OpenAI API key [API_ENDPOINTS] -SEARXNG = "http://localhost:32768" # SearxNG API ULR \ No newline at end of file +SEARXNG = "http://localhost:32768" # SearxNG API ULR +OLLAMA = "http://localhost:11434" # Ollama API URL \ No newline at end of file diff --git a/src/agents/academicSearchAgent.ts b/src/agents/academicSearchAgent.ts index 466088f..e944946 100644 --- a/src/agents/academicSearchAgent.ts +++ b/src/agents/academicSearchAgent.ts @@ -11,7 +11,7 @@ import { } from '@langchain/core/runnables'; import { StringOutputParser } from '@langchain/core/output_parsers'; import { Document } from '@langchain/core/documents'; -import { searchSearxng } from '../core/searxng'; +import { searchSearxng } from '../lib/searxng'; import type { StreamEvent } from '@langchain/core/tracers/log_stream'; import type { BaseChatModel } from '@langchain/core/language_models/chat_models'; import type { Embeddings } from '@langchain/core/embeddings'; diff --git a/src/agents/imageSearchAgent.ts b/src/agents/imageSearchAgent.ts index 3adf631..3d8570e 100644 --- a/src/agents/imageSearchAgent.ts +++ b/src/agents/imageSearchAgent.ts @@ -7,7 +7,7 @@ import { PromptTemplate } from '@langchain/core/prompts'; import formatChatHistoryAsString from '../utils/formatHistory'; import { BaseMessage } from '@langchain/core/messages'; import { StringOutputParser } from '@langchain/core/output_parsers'; -import { searchSearxng } from '../core/searxng'; +import { searchSearxng } from '../lib/searxng'; import type { BaseChatModel } from '@langchain/core/language_models/chat_models'; const imageSearchChainPrompt = ` diff --git a/src/agents/redditSearchAgent.ts b/src/agents/redditSearchAgent.ts index dca3332..9b460da 100644 --- a/src/agents/redditSearchAgent.ts +++ b/src/agents/redditSearchAgent.ts @@ -11,7 +11,7 @@ import { } from '@langchain/core/runnables'; import { StringOutputParser } from '@langchain/core/output_parsers'; import { Document } from '@langchain/core/documents'; -import { searchSearxng } from '../core/searxng'; +import { searchSearxng } from '../lib/searxng'; import type { StreamEvent } from '@langchain/core/tracers/log_stream'; import type { BaseChatModel } from '@langchain/core/language_models/chat_models'; import type { Embeddings } from '@langchain/core/embeddings'; diff --git a/src/agents/webSearchAgent.ts b/src/agents/webSearchAgent.ts index 66db2a1..4141d0b 100644 --- a/src/agents/webSearchAgent.ts +++ b/src/agents/webSearchAgent.ts @@ -11,7 +11,7 @@ import { } from '@langchain/core/runnables'; import { StringOutputParser } from '@langchain/core/output_parsers'; import { Document } from '@langchain/core/documents'; -import { searchSearxng } from '../core/searxng'; +import { searchSearxng } from '../lib/searxng'; import type { StreamEvent } from '@langchain/core/tracers/log_stream'; import type { BaseChatModel } from '@langchain/core/language_models/chat_models'; import type { Embeddings } from '@langchain/core/embeddings'; diff --git a/src/agents/wolframAlphaSearchAgent.ts b/src/agents/wolframAlphaSearchAgent.ts index a68110d..cdcd222 100644 --- a/src/agents/wolframAlphaSearchAgent.ts +++ b/src/agents/wolframAlphaSearchAgent.ts @@ -11,7 +11,7 @@ import { } from '@langchain/core/runnables'; import { StringOutputParser } from '@langchain/core/output_parsers'; import { Document } from '@langchain/core/documents'; -import { searchSearxng } from '../core/searxng'; +import { searchSearxng } from '../lib/searxng'; import type { StreamEvent } from '@langchain/core/tracers/log_stream'; import type { BaseChatModel } from '@langchain/core/language_models/chat_models'; import type { Embeddings } from '@langchain/core/embeddings'; diff --git a/src/agents/youtubeSearchAgent.ts b/src/agents/youtubeSearchAgent.ts index 4ed8b41..9bb24ed 100644 --- a/src/agents/youtubeSearchAgent.ts +++ b/src/agents/youtubeSearchAgent.ts @@ -11,7 +11,7 @@ import { } from '@langchain/core/runnables'; import { StringOutputParser } from '@langchain/core/output_parsers'; import { Document } from '@langchain/core/documents'; -import { searchSearxng } from '../core/searxng'; +import { searchSearxng } from '../lib/searxng'; import type { StreamEvent } from '@langchain/core/tracers/log_stream'; import type { BaseChatModel } from '@langchain/core/language_models/chat_models'; import type { Embeddings } from '@langchain/core/embeddings'; diff --git a/src/config.ts b/src/config.ts index c83522f..055e37f 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,6 +1,6 @@ import fs from 'fs'; import path from 'path'; -import toml, { JsonMap } from '@iarna/toml'; +import toml from '@iarna/toml'; const configFileName = 'config.toml'; @@ -8,18 +8,21 @@ interface Config { GENERAL: { PORT: number; SIMILARITY_MEASURE: string; + CHAT_MODEL_PROVIDER: string; + CHAT_MODEL: string; }; API_KEYS: { OPENAI: string; }; API_ENDPOINTS: { SEARXNG: string; + OLLAMA: string; }; } const loadConfig = () => toml.parse( - fs.readFileSync(path.join(process.cwd(), `${configFileName}`), 'utf-8'), + fs.readFileSync(path.join(__dirname, `../${configFileName}`), 'utf-8'), ) as any as Config; export const getPort = () => loadConfig().GENERAL.PORT; @@ -27,6 +30,13 @@ export const getPort = () => loadConfig().GENERAL.PORT; export const getSimilarityMeasure = () => loadConfig().GENERAL.SIMILARITY_MEASURE; +export const getChatModelProvider = () => + loadConfig().GENERAL.CHAT_MODEL_PROVIDER; + +export const getChatModel = () => loadConfig().GENERAL.CHAT_MODEL; + export const getOpenaiApiKey = () => loadConfig().API_KEYS.OPENAI; export const getSearxngApiEndpoint = () => loadConfig().API_ENDPOINTS.SEARXNG; + +export const getOllamaApiEndpoint = () => loadConfig().API_ENDPOINTS.OLLAMA; diff --git a/src/core/agentPicker.ts b/src/core/agentPicker.ts deleted file mode 100644 index ff118da..0000000 --- a/src/core/agentPicker.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { z } from 'zod'; -import { OpenAI } from '@langchain/openai'; -import { RunnableSequence } from '@langchain/core/runnables'; -import { StructuredOutputParser } from 'langchain/output_parsers'; -import { PromptTemplate } from '@langchain/core/prompts'; - -const availableAgents = [ - { - name: 'webSearch', - description: - 'It is expert is searching the web for information and answer user queries', - }, - /* { - name: 'academicSearch', - description: - 'It is expert is searching the academic databases for information and answer user queries. It is particularly good at finding research papers and articles on topics like science, engineering, and technology. Use this instead of wolframAlphaSearch if the user query is not mathematical or scientific in nature', - }, - { - name: 'youtubeSearch', - description: - 'This model is expert at finding videos on youtube based on user queries', - }, - { - name: 'wolframAlphaSearch', - description: - 'This model is expert at finding answers to mathematical and scientific questions based on user queries.', - }, - { - name: 'redditSearch', - description: - 'This model is expert at finding posts and discussions on reddit based on user queries', - }, - { - name: 'writingAssistant', - description: - 'If there is no need for searching, this model is expert at generating text based on user queries', - }, */ -]; - -const parser = StructuredOutputParser.fromZodSchema( - z.object({ - agent: z.string().describe('The name of the selected agent'), - }), -); - -const prompt = ` - You are an AI model who is expert at finding suitable agents for user queries. The available agents are: - ${availableAgents.map((agent) => `- ${agent.name}: ${agent.description}`).join('\n')} - - Your task is to find the most suitable agent for the following query: {query} - - {format_instructions} -`; - -const chain = RunnableSequence.from([ - PromptTemplate.fromTemplate(prompt), - new OpenAI({ temperature: 0 }), - parser, -]); - -const pickSuitableAgent = async (query: string) => { - const res = await chain.invoke({ - query, - format_instructions: parser.getFormatInstructions(), - }); - return res.agent; -}; - -export default pickSuitableAgent; diff --git a/src/lib/providers.ts b/src/lib/providers.ts new file mode 100644 index 0000000..2dfde58 --- /dev/null +++ b/src/lib/providers.ts @@ -0,0 +1,58 @@ +import { ChatOpenAI, OpenAIEmbeddings } from '@langchain/openai'; +import { ChatOllama } from '@langchain/community/chat_models/ollama'; +import { OllamaEmbeddings } from '@langchain/community/embeddings/ollama'; +import { getOllamaApiEndpoint, getOpenaiApiKey } from '../config'; + +export const getAvailableProviders = async () => { + const openAIApiKey = getOpenaiApiKey(); + const ollamaEndpoint = getOllamaApiEndpoint(); + + const models = {}; + + if (openAIApiKey) { + models['openai'] = { + 'gpt-3.5-turbo': new ChatOpenAI({ + openAIApiKey, + modelName: 'gpt-3.5-turbo', + temperature: 0.7, + }), + 'gpt-4': new ChatOpenAI({ + openAIApiKey, + modelName: 'gpt-4', + temperature: 0.7, + }), + embeddings: new OpenAIEmbeddings({ + openAIApiKey, + modelName: 'text-embedding-3-large', + }), + }; + } + + if (ollamaEndpoint) { + try { + const response = await fetch(`${ollamaEndpoint}/api/tags`); + + const { models: ollamaModels } = (await response.json()) as any; + + models['ollama'] = ollamaModels.reduce((acc, model) => { + acc[model.model] = new ChatOllama({ + baseUrl: ollamaEndpoint, + model: model.model, + temperature: 0.7, + }); + return acc; + }, {}); + + if (Object.keys(models['ollama']).length > 0) { + models['ollama']['embeddings'] = new OllamaEmbeddings({ + baseUrl: ollamaEndpoint, + model: models['ollama'][Object.keys(models['ollama'])[0]].model, + }); + } + } catch (err) { + console.log(err); + } + } + + return models; +}; diff --git a/src/core/searxng.ts b/src/lib/searxng.ts similarity index 100% rename from src/core/searxng.ts rename to src/lib/searxng.ts diff --git a/src/websocket/connectionManager.ts b/src/websocket/connectionManager.ts index 2dc8d59..5b4e3f5 100644 --- a/src/websocket/connectionManager.ts +++ b/src/websocket/connectionManager.ts @@ -1,18 +1,32 @@ import { WebSocket } from 'ws'; import { handleMessage } from './messageHandler'; -import { ChatOpenAI, OpenAIEmbeddings } from '@langchain/openai'; -import { getOpenaiApiKey } from '../config'; +import { getChatModel, getChatModelProvider } from '../config'; +import { getAvailableProviders } from '../lib/providers'; +import { BaseChatModel } from '@langchain/core/language_models/chat_models'; +import type { Embeddings } from '@langchain/core/embeddings'; -export const handleConnection = (ws: WebSocket) => { - const llm = new ChatOpenAI({ - temperature: 0.7, - openAIApiKey: getOpenaiApiKey(), - }); +export const handleConnection = async (ws: WebSocket) => { + const models = await getAvailableProviders(); + const provider = getChatModelProvider(); + const chatModel = getChatModel(); - const embeddings = new OpenAIEmbeddings({ - openAIApiKey: getOpenaiApiKey(), - modelName: 'text-embedding-3-large', - }); + let llm: BaseChatModel | undefined; + let embeddings: Embeddings | undefined; + + if (models[provider] && models[provider][chatModel]) { + llm = models[provider][chatModel] as BaseChatModel | undefined; + embeddings = models[provider].embeddings as Embeddings | undefined; + } + + if (!llm || !embeddings) { + ws.send( + JSON.stringify({ + type: 'error', + data: 'Invalid LLM or embeddings model selected', + }), + ); + ws.close(); + } ws.on( 'message', diff --git a/ui/components/ChatWindow.tsx b/ui/components/ChatWindow.tsx index 94f4f00..4c138ff 100644 --- a/ui/components/ChatWindow.tsx +++ b/ui/components/ChatWindow.tsx @@ -174,13 +174,25 @@ const ChatWindow = () => { )}
) : ( -
-
+
- ) + ); }; export default ChatWindow; From e964ffcea5cffd8be6c3745c137fc095e15ebab7 Mon Sep 17 00:00:00 2001 From: ItzCrazyKns Date: Sat, 20 Apr 2024 11:22:39 +0530 Subject: [PATCH 007/470] feat(readme): remove excess space --- README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.md b/README.md index 428ee30..11335d1 100644 --- a/README.md +++ b/README.md @@ -26,19 +26,16 @@ Perplexica is an open-source AI-powered searching tool or an AI-powered search e ## Features - **Local LLMs**: You can make use local LLMs such as LLama2 and Mixtral using Ollama. - - **Two Main Modes:** - **Copilot Mode:** (In development) Boosts search by generating different queries to find more relevant internet sources. Like normal search instead of just using the context by SearxNG, it visits the top matches and tries to find relevant sources to the user's query directly from the page. - **Normal Mode:** Processes your query and performs a web search. - **Focus Modes:** Special modes to better answer specific types of questions. Perplexica currently has 6 focus modes: - 1. **All Mode:** Searches the entire web to find the best results. 2. **Writing Assistant Mode:** Helpful for writing tasks that does not require searching the web. 3. **Academic Search Mode:** Finds articles and papers, ideal for academic research. 4. **YouTube Search Mode:** Finds YouTube videos based on the search query. 5. **Wolfram Alpha Search Mode:** Answers queries that need calculations or data analysis using Wolfram Alpha. 6. **Reddit Search Mode:** Searches Reddit for discussions and opinions related to the query. - - **Current Information:** Some search tools might give you outdated info because they use data from crawling bots and convert them into embeddings and store them in a index (its like converting the web into embeddings which is quite expensive.). Unlike them, Perplexica uses SearxNG, a metasearch engine to get the results and rerank and get the most relevent source out of it, ensuring you always get the latest information without the overhead of daily data updates. It has many more features like image and video search. Some of the planned features are mentioned in [upcoming features](#upcoming-features). From 95461154d059f531ecca83e08301420b710410b0 Mon Sep 17 00:00:00 2001 From: ItzCrazyKns <95534749+ItzCrazyKns@users.noreply.github.com> Date: Sat, 20 Apr 2024 18:26:54 +0530 Subject: [PATCH 008/470] feat(sample-config): change `ULR` to `URL` --- sample.config.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sample.config.toml b/sample.config.toml index 9f6d927..26939bd 100644 --- a/sample.config.toml +++ b/sample.config.toml @@ -8,5 +8,5 @@ CHAT_MODEL = "gpt-3.5-turbo" # Name of the model to use OPENAI = "sk-1234567890abcdef1234567890abcdef" # OpenAI API key [API_ENDPOINTS] -SEARXNG = "http://localhost:32768" # SearxNG API ULR -OLLAMA = "http://localhost:11434" # Ollama API URL \ No newline at end of file +SEARXNG = "http://localhost:32768" # SearxNG API URL +OLLAMA = "http://localhost:11434" # Ollama API URL From 23b7feee0cd2ae1f9672cedc77853e6f6a1f69c7 Mon Sep 17 00:00:00 2001 From: ItzCrazyKns Date: Sat, 20 Apr 2024 20:46:16 +0530 Subject: [PATCH 009/470] feat(input-actions): fix popover mobile view --- README.md | 2 +- ui/components/MessageInputActions.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 11335d1..7bcb2b1 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ There are mainly 2 ways of installing Perplexica - With Docker, Without Docker. 4. Rename the `sample.config.toml` file to `config.toml`. For Docker setups, you need only fill in the following fields: - - `CHAT_MODEL`: The name of the LLM to use. Example: `llama2` for Ollama users & `gpt-3.5-turbo` for OpenAI users. + - `CHAT_MODEL`: The name of the LLM to use. Like `llama2` (using Ollama), `gpt-3.5-turbo` (using OpenAI), etc. - `CHAT_MODEL_PROVIDER`: The chat model provider, either `openai` or `ollama`. Depending upon which provider you use you would have to fill in the following fields: - `OPENAI`: Your OpenAI API key. **You only need to fill this if you wish to use OpenAI's models.** diff --git a/ui/components/MessageInputActions.tsx b/ui/components/MessageInputActions.tsx index 8d7deea..9c00c4d 100644 --- a/ui/components/MessageInputActions.tsx +++ b/ui/components/MessageInputActions.tsx @@ -109,7 +109,7 @@ export const Focus = ({ leaveTo="opacity-0 translate-y-1" > -
+
{focusModes.map((mode, i) => ( setFocusMode(mode.key)} From 5924690df297de67ee5c114b338e29bfd9a68bd4 Mon Sep 17 00:00:00 2001 From: ItzCrazyKns Date: Sat, 20 Apr 2024 22:12:07 +0530 Subject: [PATCH 010/470] feat(image-search): Use LLM from config --- src/routes/images.ts | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/src/routes/images.ts b/src/routes/images.ts index dd3925f..97f0b31 100644 --- a/src/routes/images.ts +++ b/src/routes/images.ts @@ -1,7 +1,8 @@ import express from 'express'; import handleImageSearch from '../agents/imageSearchAgent'; -import { ChatOpenAI } from '@langchain/openai'; -import { getOpenaiApiKey } from '../config'; +import { BaseChatModel } from '@langchain/core/language_models/chat_models'; +import { getAvailableProviders } from '../lib/providers'; +import { getChatModel, getChatModelProvider } from '../config'; const router = express.Router(); @@ -9,10 +10,20 @@ router.post('/', async (req, res) => { try { const { query, chat_history } = req.body; - const llm = new ChatOpenAI({ - temperature: 0.7, - openAIApiKey: getOpenaiApiKey(), - }); + const models = await getAvailableProviders(); + const provider = getChatModelProvider(); + const chatModel = getChatModel(); + + let llm: BaseChatModel | undefined; + + if (models[provider] && models[provider][chatModel]) { + llm = models[provider][chatModel] as BaseChatModel | undefined; + } + + if (!llm) { + res.status(500).json({ message: 'Invalid LLM model selected' }); + return; + } const images = await handleImageSearch({ query, chat_history }, llm); From 0ea2bec85de6c1258e94eb59386e51e70abbc8dd Mon Sep 17 00:00:00 2001 From: ItzCrazyKns Date: Sat, 20 Apr 2024 22:12:49 +0530 Subject: [PATCH 011/470] feat(config): Remove preassigned values --- sample.config.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sample.config.toml b/sample.config.toml index 26939bd..350ae03 100644 --- a/sample.config.toml +++ b/sample.config.toml @@ -5,8 +5,8 @@ CHAT_MODEL_PROVIDER = "openai" # "openai" or "ollama" CHAT_MODEL = "gpt-3.5-turbo" # Name of the model to use [API_KEYS] -OPENAI = "sk-1234567890abcdef1234567890abcdef" # OpenAI API key +OPENAI = "" # OpenAI API key - sk-1234567890abcdef1234567890abcdef [API_ENDPOINTS] SEARXNG = "http://localhost:32768" # SearxNG API URL -OLLAMA = "http://localhost:11434" # Ollama API URL +OLLAMA = "" # Ollama API URL - http://localhost:11434 From ec91289c0cd106169857d876da1ba2817dc9ec46 Mon Sep 17 00:00:00 2001 From: ItzCrazyKns Date: Sun, 21 Apr 2024 16:22:27 +0530 Subject: [PATCH 012/470] feat(messageSources): use arrow functions --- README.md | 4 ++-- ui/components/MessageSources.tsx | 11 +++++------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 7bcb2b1..9b646ee 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ There are mainly 2 ways of installing Perplexica - With Docker, Without Docker. - `OPENAI`: Your OpenAI API key. **You only need to fill this if you wish to use OpenAI's models.** - `OLLAMA`: Your Ollama API URL. **You need to fill this if you wish to use Ollama's models instead of OpenAI's.** - **Note**: (In development) You can change these and use different models after running Perplexica as well from the settings page. + **Note**: (In development) You can change these and use different models after running Perplexica as well from the settings page. - `SIMILARITY_MEASURE`: The similarity measure to use (This is filled by default; you can leave it as is if you are unsure about it.) @@ -90,7 +90,7 @@ For setups without Docker: ## Upcoming Features - [ ] Finalizing Copilot Mode -- [X] Adding support for local LLMs +- [x] Adding support for local LLMs - [ ] Adding Discover and History Saving features - [x] Introducing various Focus Modes diff --git a/ui/components/MessageSources.tsx b/ui/components/MessageSources.tsx index 8c78f2f..5816f8d 100644 --- a/ui/components/MessageSources.tsx +++ b/ui/components/MessageSources.tsx @@ -1,21 +1,20 @@ /* eslint-disable @next/next/no-img-element */ -import { cn } from '@/lib/utils'; import { Dialog, Transition } from '@headlessui/react'; import { Document } from '@langchain/core/documents'; -import Link from 'next/link'; import { Fragment, useState } from 'react'; const MessageSources = ({ sources }: { sources: Document[] }) => { const [isDialogOpen, setIsDialogOpen] = useState(false); - function closeModal() { + + const closeModal = () => { setIsDialogOpen(false); document.body.classList.remove('overflow-hidden-scrollable'); - } + }; - function openModal() { + const openModal = () => { setIsDialogOpen(true); document.body.classList.add('overflow-hidden-scrollable'); - } + }; return (
From fd65af53c3b9c4272b158a2bc320749711e09c36 Mon Sep 17 00:00:00 2001 From: ItzCrazyKns Date: Sun, 21 Apr 2024 20:52:47 +0530 Subject: [PATCH 013/470] feat(providers): add error handling --- src/lib/providers.ts | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/src/lib/providers.ts b/src/lib/providers.ts index 2dfde58..c730da8 100644 --- a/src/lib/providers.ts +++ b/src/lib/providers.ts @@ -10,22 +10,26 @@ export const getAvailableProviders = async () => { const models = {}; if (openAIApiKey) { - models['openai'] = { - 'gpt-3.5-turbo': new ChatOpenAI({ - openAIApiKey, - modelName: 'gpt-3.5-turbo', - temperature: 0.7, - }), - 'gpt-4': new ChatOpenAI({ - openAIApiKey, - modelName: 'gpt-4', - temperature: 0.7, - }), - embeddings: new OpenAIEmbeddings({ - openAIApiKey, - modelName: 'text-embedding-3-large', - }), - }; + try { + models['openai'] = { + 'gpt-3.5-turbo': new ChatOpenAI({ + openAIApiKey, + modelName: 'gpt-3.5-turbo', + temperature: 0.7, + }), + 'gpt-4': new ChatOpenAI({ + openAIApiKey, + modelName: 'gpt-4', + temperature: 0.7, + }), + embeddings: new OpenAIEmbeddings({ + openAIApiKey, + modelName: 'text-embedding-3-large', + }), + }; + } catch (err) { + console.log(`Error loading OpenAI models: ${err}`); + } } if (ollamaEndpoint) { @@ -50,7 +54,7 @@ export const getAvailableProviders = async () => { }); } } catch (err) { - console.log(err); + console.log(`Error loading Ollama models: ${err}`); } } From a86378e7268490056c9be8fe6b9c9d0ba1f9ef89 Mon Sep 17 00:00:00 2001 From: ItzCrazyKns Date: Tue, 23 Apr 2024 16:45:14 +0530 Subject: [PATCH 014/470] feat(config): add `updateConfig` method --- src/config.ts | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/config.ts b/src/config.ts index 055e37f..7929ba7 100644 --- a/src/config.ts +++ b/src/config.ts @@ -20,6 +20,10 @@ interface Config { }; } +type RecursivePartial = { + [P in keyof T]?: RecursivePartial; +}; + const loadConfig = () => toml.parse( fs.readFileSync(path.join(__dirname, `../${configFileName}`), 'utf-8'), @@ -40,3 +44,28 @@ export const getOpenaiApiKey = () => loadConfig().API_KEYS.OPENAI; export const getSearxngApiEndpoint = () => loadConfig().API_ENDPOINTS.SEARXNG; export const getOllamaApiEndpoint = () => loadConfig().API_ENDPOINTS.OLLAMA; + +export const updateConfig = (config: RecursivePartial) => { + const currentConfig = loadConfig(); + + for (const key in currentConfig) { + /* if (currentConfig[key] && !config[key]) { + config[key] = currentConfig[key]; + } */ + + if (currentConfig[key] && typeof currentConfig[key] === 'object') { + for (const nestedKey in currentConfig[key]) { + if (currentConfig[key][nestedKey] && !config[key][nestedKey]) { + config[key][nestedKey] = currentConfig[key][nestedKey]; + } + } + } else if (currentConfig[key] && !config[key]) { + config[key] = currentConfig[key]; + } + } + + fs.writeFileSync( + path.join(__dirname, `../${configFileName}`), + toml.stringify(config), + ); +}; From 3ffbddd237b581c64119ba76854d170582cc8cce Mon Sep 17 00:00:00 2001 From: ItzCrazyKns Date: Tue, 23 Apr 2024 16:46:14 +0530 Subject: [PATCH 015/470] feat(routes): add config route --- src/routes/config.ts | 58 ++++++++++++++++++++++++++++++++++++++++++++ src/routes/index.ts | 2 ++ 2 files changed, 60 insertions(+) create mode 100644 src/routes/config.ts diff --git a/src/routes/config.ts b/src/routes/config.ts new file mode 100644 index 0000000..ecdec17 --- /dev/null +++ b/src/routes/config.ts @@ -0,0 +1,58 @@ +import express from 'express'; +import { getAvailableProviders } from '../lib/providers'; +import { + getChatModel, + getChatModelProvider, + getOllamaApiEndpoint, + getOpenaiApiKey, + updateConfig, +} from '../config'; + +const router = express.Router(); + +router.get('/', async (_, res) => { + const config = {}; + + const providers = await getAvailableProviders(); + + for (const provider in providers) { + delete providers[provider]['embeddings']; + } + + config['providers'] = {}; + + for (const provider in providers) { + config['providers'][provider] = Object.keys(providers[provider]); + } + + config['selectedProvider'] = getChatModelProvider(); + config['selectedChatModel'] = getChatModel(); + + config['openeaiApiKey'] = getOpenaiApiKey(); + config['ollamaApiUrl'] = getOllamaApiEndpoint(); + + res.status(200).json(config); +}); + +router.post('/', async (req, res) => { + const config = req.body; + + const updatedConfig = { + GENERAL: { + CHAT_MODEL_PROVIDER: config.selectedProvider, + CHAT_MODEL: config.selectedChatModel, + }, + API_KEYS: { + OPENAI: config.openeaiApiKey, + }, + API_ENDPOINTS: { + OLLAMA: config.ollamaApiUrl, + }, + }; + + updateConfig(updatedConfig); + + res.status(200).json({ message: 'Config updated' }); +}); + +export default router; diff --git a/src/routes/index.ts b/src/routes/index.ts index f2800cf..a3262e4 100644 --- a/src/routes/index.ts +++ b/src/routes/index.ts @@ -1,8 +1,10 @@ import express from 'express'; import imagesRouter from './images'; +import configRouter from './config'; const router = express.Router(); router.use('/images', imagesRouter); +router.use('/config', configRouter); export default router; From b2b1d724eed5016a2b04166b7859d543bcee134a Mon Sep 17 00:00:00 2001 From: ItzCrazyKns Date: Tue, 23 Apr 2024 16:52:41 +0530 Subject: [PATCH 016/470] feat(ui): add settings page --- README.md | 1 + ui/components/SettingsDialog.tsx | 229 +++++++++++++++++++++++++++++++ ui/components/Sidebar.tsx | 27 ++-- 3 files changed, 244 insertions(+), 13 deletions(-) create mode 100644 ui/components/SettingsDialog.tsx diff --git a/README.md b/README.md index 9b646ee..c472369 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,7 @@ For setups without Docker: ## Upcoming Features - [ ] Finalizing Copilot Mode +- [x] Add settings page - [x] Adding support for local LLMs - [ ] Adding Discover and History Saving features - [x] Introducing various Focus Modes diff --git a/ui/components/SettingsDialog.tsx b/ui/components/SettingsDialog.tsx new file mode 100644 index 0000000..2487c71 --- /dev/null +++ b/ui/components/SettingsDialog.tsx @@ -0,0 +1,229 @@ +import { Dialog, Transition } from '@headlessui/react'; +import { CloudUpload, RefreshCcw, RefreshCw } from 'lucide-react'; +import React, { Fragment, useEffect, useState } from 'react'; + +interface SettingsType { + providers: { + [key: string]: string[]; + }; + selectedProvider: string; + selectedChatModel: string; + openeaiApiKey: string; + ollamaApiUrl: string; +} + +const SettingsDialog = ({ + isOpen, + setIsOpen, +}: { + isOpen: boolean; + setIsOpen: (isOpen: boolean) => void; +}) => { + const [config, setConfig] = useState(null); + const [isLoading, setIsLoading] = useState(false); + const [isUpdating, setIsUpdating] = useState(false); + + useEffect(() => { + if (isOpen) { + const fetchConfig = async () => { + setIsLoading(true); + const res = await fetch(`${process.env.NEXT_PUBLIC_API_URL}/config`); + const data = await res.json(); + setConfig(data); + setIsLoading(false); + }; + + fetchConfig(); + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [isOpen]); + + const handleSubmit = async () => { + setIsUpdating(true); + + try { + await fetch(`${process.env.NEXT_PUBLIC_API_URL}/config`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(config), + }); + } catch (err) { + console.log(err); + } finally { + setIsUpdating(false); + setIsOpen(false); + + window.location.reload(); + } + }; + + return ( + + setIsOpen(false)} + > + +
+ +
+
+ + + + Settings + + {config && !isLoading && ( +
+ {config.providers && ( +
+

LLM Provider

+ +
+ )} + {config.selectedProvider && ( +
+

Chat Model

+ +
+ )} + {config.selectedProvider === 'openai' && ( +
+

OpenAI API Key

+ + setConfig({ + ...config, + openeaiApiKey: e.target.value, + }) + } + className="bg-[#111111] px-3 py-2 flex items-center overflow-hidden border border-[#1C1C1C] text-white rounded-lg text-sm" + /> +
+ )} + {config.selectedProvider === 'ollama' && ( +
+

Ollama API URL

+ + setConfig({ + ...config, + ollamaApiUrl: e.target.value, + }) + } + className="bg-[#111111] px-3 py-2 flex items-center overflow-hidden border border-[#1C1C1C] text-white rounded-lg text-sm" + /> +
+ )} +
+ )} + {isLoading && ( +
+ +
+ )} +
+

+ We'll refresh the page after updating the settings. +

+ +
+ + +
+
+
+ + ); +}; + +export default SettingsDialog; diff --git a/ui/components/Sidebar.tsx b/ui/components/Sidebar.tsx index e562160..aef1153 100644 --- a/ui/components/Sidebar.tsx +++ b/ui/components/Sidebar.tsx @@ -1,16 +1,19 @@ 'use client'; import { cn } from '@/lib/utils'; -import { BookOpenText, Home, Search, SquarePen } from 'lucide-react'; -import { SiGithub } from '@icons-pack/react-simple-icons'; +import { BookOpenText, Home, Search, SquarePen, Settings } from 'lucide-react'; import Link from 'next/link'; import { useSelectedLayoutSegments } from 'next/navigation'; -import React from 'react'; +import React, { Fragment, useState } from 'react'; import Layout from './Layout'; +import { Dialog, Transition } from '@headlessui/react'; +import SettingsDialog from './SettingsDialog'; const Sidebar = ({ children }: { children: React.ReactNode }) => { const segments = useSelectedLayoutSegments(); + const [isSettingsOpen, setIsSettingsOpen] = useState(false); + const navLinks = [ { icon: Home, @@ -56,16 +59,14 @@ const Sidebar = ({ children }: { children: React.ReactNode }) => { ))}
- - - + setIsSettingsOpen(!isSettingsOpen)} + className="text-white cursor-pointer" + /> +
From 7653eaf1464e3ca350b79e2e6f8a5937ae017652 Mon Sep 17 00:00:00 2001 From: ItzCrazyKns Date: Tue, 23 Apr 2024 16:54:39 +0530 Subject: [PATCH 017/470] feat(config): avoid updating blank fields --- src/config.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/config.ts b/src/config.ts index 7929ba7..f373847 100644 --- a/src/config.ts +++ b/src/config.ts @@ -55,11 +55,15 @@ export const updateConfig = (config: RecursivePartial) => { if (currentConfig[key] && typeof currentConfig[key] === 'object') { for (const nestedKey in currentConfig[key]) { - if (currentConfig[key][nestedKey] && !config[key][nestedKey]) { + if ( + currentConfig[key][nestedKey] && + !config[key][nestedKey] && + config[key][nestedKey] !== '' + ) { config[key][nestedKey] = currentConfig[key][nestedKey]; } } - } else if (currentConfig[key] && !config[key]) { + } else if (currentConfig[key] && !config[key] && config[key] !== '') { config[key] = currentConfig[key]; } } From 6fe70a70ffcbd4b7a703dea86eae624dad4f037e Mon Sep 17 00:00:00 2001 From: ItzCrazyKns Date: Tue, 23 Apr 2024 17:06:44 +0530 Subject: [PATCH 018/470] feat(settings-dialog): enhance UI --- ui/components/SettingsDialog.tsx | 74 ++++++++++++++++---------------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/ui/components/SettingsDialog.tsx b/ui/components/SettingsDialog.tsx index 2487c71..671ec52 100644 --- a/ui/components/SettingsDialog.tsx +++ b/ui/components/SettingsDialog.tsx @@ -93,10 +93,12 @@ const SettingsDialog = ({ Settings {config && !isLoading && ( -
+
{config.providers && (
-

LLM Provider

+

+ Chat model Provider +

)} - {config.selectedProvider === 'openai' && ( -
-

OpenAI API Key

- - setConfig({ - ...config, - openeaiApiKey: e.target.value, - }) - } - className="bg-[#111111] px-3 py-2 flex items-center overflow-hidden border border-[#1C1C1C] text-white rounded-lg text-sm" - /> -
- )} - {config.selectedProvider === 'ollama' && ( -
-

Ollama API URL

- - setConfig({ - ...config, - ollamaApiUrl: e.target.value, - }) - } - className="bg-[#111111] px-3 py-2 flex items-center overflow-hidden border border-[#1C1C1C] text-white rounded-lg text-sm" - /> -
- )} +
+

OpenAI API Key

+ + setConfig({ + ...config, + openeaiApiKey: e.target.value, + }) + } + className="bg-[#111111] px-3 py-2 flex items-center overflow-hidden border border-[#1C1C1C] text-white rounded-lg text-sm" + /> +
+
+

Ollama API URL

+ + setConfig({ + ...config, + ollamaApiUrl: e.target.value, + }) + } + className="bg-[#111111] px-3 py-2 flex items-center overflow-hidden border border-[#1C1C1C] text-white rounded-lg text-sm" + /> +
)} {isLoading && ( @@ -211,9 +211,9 @@ const SettingsDialog = ({ disabled={isLoading || isUpdating} > {isUpdating ? ( - + ) : ( - + )}
From 8758fcbc13ddeced4d1d851ca7ef15458605edf2 Mon Sep 17 00:00:00 2001 From: ItzCrazyKns Date: Tue, 23 Apr 2024 17:15:07 +0530 Subject: [PATCH 019/470] feat(readme): update content --- README.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index c472369..fae0c7a 100644 --- a/README.md +++ b/README.md @@ -25,18 +25,18 @@ Perplexica is an open-source AI-powered searching tool or an AI-powered search e ## Features -- **Local LLMs**: You can make use local LLMs such as LLama2 and Mixtral using Ollama. +- **Local LLMs**: You can make use local LLMs such as Llama3 and Mixtral using Ollama. - **Two Main Modes:** - **Copilot Mode:** (In development) Boosts search by generating different queries to find more relevant internet sources. Like normal search instead of just using the context by SearxNG, it visits the top matches and tries to find relevant sources to the user's query directly from the page. - **Normal Mode:** Processes your query and performs a web search. - **Focus Modes:** Special modes to better answer specific types of questions. Perplexica currently has 6 focus modes: - 1. **All Mode:** Searches the entire web to find the best results. - 2. **Writing Assistant Mode:** Helpful for writing tasks that does not require searching the web. - 3. **Academic Search Mode:** Finds articles and papers, ideal for academic research. - 4. **YouTube Search Mode:** Finds YouTube videos based on the search query. - 5. **Wolfram Alpha Search Mode:** Answers queries that need calculations or data analysis using Wolfram Alpha. - 6. **Reddit Search Mode:** Searches Reddit for discussions and opinions related to the query. -- **Current Information:** Some search tools might give you outdated info because they use data from crawling bots and convert them into embeddings and store them in a index (its like converting the web into embeddings which is quite expensive.). Unlike them, Perplexica uses SearxNG, a metasearch engine to get the results and rerank and get the most relevent source out of it, ensuring you always get the latest information without the overhead of daily data updates. + - **All Mode:** Searches the entire web to find the best results. + - **Writing Assistant Mode:** Helpful for writing tasks that does not require searching the web. + - **Academic Search Mode:** Finds articles and papers, ideal for academic research. + - **YouTube Search Mode:** Finds YouTube videos based on the search query. + - **Wolfram Alpha Search Mode:** Answers queries that need calculations or data analysis using Wolfram Alpha. + - **Reddit Search Mode:** Searches Reddit for discussions and opinions related to the query. +- **Current Information:** Some search tools might give you outdated info because they use data from crawling bots and convert them into embeddings and store them in a index. Unlike them, Perplexica uses SearxNG, a metasearch engine to get the results and rerank and get the most relevent source out of it, ensuring you always get the latest information without the overhead of daily data updates. It has many more features like image and video search. Some of the planned features are mentioned in [upcoming features](#upcoming-features). @@ -63,7 +63,7 @@ There are mainly 2 ways of installing Perplexica - With Docker, Without Docker. - `OPENAI`: Your OpenAI API key. **You only need to fill this if you wish to use OpenAI's models.** - `OLLAMA`: Your Ollama API URL. **You need to fill this if you wish to use Ollama's models instead of OpenAI's.** - **Note**: (In development) You can change these and use different models after running Perplexica as well from the settings page. + **Note**: You can change these and use different models after running Perplexica as well from the settings page. - `SIMILARITY_MEASURE`: The similarity measure to use (This is filled by default; you can leave it as is if you are unsure about it.) From 7f8c73782cdd825c7ab651395d201e9dd5010e24 Mon Sep 17 00:00:00 2001 From: ItzCrazyKns Date: Tue, 23 Apr 2024 17:53:47 +0530 Subject: [PATCH 020/470] feat(settings-dialog): remove overflow --- ui/components/SettingsDialog.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/components/SettingsDialog.tsx b/ui/components/SettingsDialog.tsx index 671ec52..cd3e0c0 100644 --- a/ui/components/SettingsDialog.tsx +++ b/ui/components/SettingsDialog.tsx @@ -93,7 +93,7 @@ const SettingsDialog = ({ Settings {config && !isLoading && ( -
+
{config.providers && (

From 571cdc1b4ed4b22cb1da75493d72587e350a655c Mon Sep 17 00:00:00 2001 From: ItzCrazyKns Date: Tue, 23 Apr 2024 17:54:08 +0530 Subject: [PATCH 021/470] feat(settings-dialog): remove excess padding --- ui/components/SettingsDialog.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/components/SettingsDialog.tsx b/ui/components/SettingsDialog.tsx index cd3e0c0..e5ec51e 100644 --- a/ui/components/SettingsDialog.tsx +++ b/ui/components/SettingsDialog.tsx @@ -93,7 +93,7 @@ const SettingsDialog = ({ Settings {config && !isLoading && ( -

+
{config.providers && (

From 3b66808e7d210812b5a2253dc50343b1d485edf6 Mon Sep 17 00:00:00 2001 From: ItzCrazyKns Date: Wed, 24 Apr 2024 10:06:56 +0530 Subject: [PATCH 022/470] feat(message-input): prevent message when loading --- ui/components/Chat.tsx | 2 +- ui/components/MessageInput.tsx | 9 ++++++--- ui/components/SettingsDialog.tsx | 4 ++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/ui/components/Chat.tsx b/ui/components/Chat.tsx index 61ab3ab..294520b 100644 --- a/ui/components/Chat.tsx +++ b/ui/components/Chat.tsx @@ -77,7 +77,7 @@ const Chat = ({ className="bottom-24 lg:bottom-10 fixed z-40" style={{ width: dividerWidth }} > - +

)}
diff --git a/ui/components/MessageInput.tsx b/ui/components/MessageInput.tsx index 051afbc..baf6095 100644 --- a/ui/components/MessageInput.tsx +++ b/ui/components/MessageInput.tsx @@ -6,8 +6,10 @@ import { Attach, CopilotToggle } from './MessageInputActions'; const MessageInput = ({ sendMessage, + loading, }: { sendMessage: (message: string) => void; + loading: boolean; }) => { const [copilotEnabled, setCopilotEnabled] = useState(false); const [message, setMessage] = useState(''); @@ -25,12 +27,13 @@ const MessageInput = ({ return (
{ + if (loading) return; e.preventDefault(); sendMessage(message); setMessage(''); }} onKeyDown={(e) => { - if (e.key === 'Enter' && !e.shiftKey) { + if (e.key === 'Enter' && !e.shiftKey && !loading) { e.preventDefault(); sendMessage(message); setMessage(''); @@ -58,7 +61,7 @@ const MessageInput = ({ setCopilotEnabled={setCopilotEnabled} />
- +
diff --git a/ui/components/SearchImages.tsx b/ui/components/SearchImages.tsx index 022c90d..1f94963 100644 --- a/ui/components/SearchImages.tsx +++ b/ui/components/SearchImages.tsx @@ -3,6 +3,7 @@ import { ImagesIcon, PlusIcon } from 'lucide-react'; import { useState } from 'react'; import Lightbox from 'yet-another-react-lightbox'; import 'yet-another-react-lightbox/styles.css'; +import { Message } from './ChatWindow'; type Image = { url: string; @@ -10,7 +11,13 @@ type Image = { title: string; }; -const SearchImages = ({ query }: { query: string }) => { +const SearchImages = ({ + query, + chat_history, +}: { + query: string; + chat_history: Message[]; +}) => { const [images, setImages] = useState(null); const [loading, setLoading] = useState(false); const [open, setOpen] = useState(false); @@ -31,7 +38,7 @@ const SearchImages = ({ query }: { query: string }) => { }, body: JSON.stringify({ query: query, - chat_history: [], + chat_history: chat_history, }), }, ); From 66c5fcb4fae2abc62afbf5ca4d9dc0a51c068ff9 Mon Sep 17 00:00:00 2001 From: ItzCrazyKns Date: Sun, 28 Apr 2024 18:21:59 +0530 Subject: [PATCH 026/470] feat(navbar): use correct padding --- ui/components/Navbar.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/components/Navbar.tsx b/ui/components/Navbar.tsx index 57dcf6c..75c34a6 100644 --- a/ui/components/Navbar.tsx +++ b/ui/components/Navbar.tsx @@ -38,7 +38,7 @@ const Navbar = ({ messages }: { messages: Message[] }) => { }, []); return ( -
+
Date: Sun, 28 Apr 2024 18:34:56 +0530 Subject: [PATCH 027/470] feat(agents): fix engine names --- src/agents/academicSearchAgent.ts | 4 ++-- src/agents/imageSearchAgent.ts | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/agents/academicSearchAgent.ts b/src/agents/academicSearchAgent.ts index ac2aa8c..d08f282 100644 --- a/src/agents/academicSearchAgent.ts +++ b/src/agents/academicSearchAgent.ts @@ -115,8 +115,8 @@ const createBasicAcademicSearchRetrieverChain = (llm: BaseChatModel) => { language: 'en', engines: [ 'arxiv', - 'google_scholar', - 'internet_archive_scholar', + 'google scholar', + 'internetarchivescholar', 'pubmed', ], }); diff --git a/src/agents/imageSearchAgent.ts b/src/agents/imageSearchAgent.ts index 3d8570e..167019f 100644 --- a/src/agents/imageSearchAgent.ts +++ b/src/agents/imageSearchAgent.ts @@ -53,8 +53,7 @@ const createImageSearchChain = (llm: BaseChatModel) => { strParser, RunnableLambda.from(async (input: string) => { const res = await searchSearxng(input, { - categories: ['images'], - engines: ['bing_images', 'google_images'], + engines: ['bing images', 'google images'], }); const images = []; From f2c51420da7c4daae16621cc282860de0a42c227 Mon Sep 17 00:00:00 2001 From: ItzCrazyKns Date: Sun, 28 Apr 2024 19:14:02 +0530 Subject: [PATCH 028/470] feat(searxng-settings): drop unsupported engines --- searxng-settings.yml | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/searxng-settings.yml b/searxng-settings.yml index d4a8bad..da973c1 100644 --- a/searxng-settings.yml +++ b/searxng-settings.yml @@ -1071,25 +1071,6 @@ engines: require_api_key: false results: HTML - - name: azlyrics - shortcut: lyrics - engine: xpath - timeout: 4.0 - disabled: true - categories: [music, lyrics] - paging: true - search_url: https://search.azlyrics.com/search.php?q={query}&w=lyrics&p={pageno} - url_xpath: //td[@class="text-left visitedlyr"]/a/@href - title_xpath: //span/b/text() - content_xpath: //td[@class="text-left visitedlyr"]/a/small - about: - website: https://azlyrics.com - wikidata_id: Q66372542 - official_api_documentation: - use_official_api: false - require_api_key: false - results: HTML - - name: mastodon users engine: mastodon mastodon_type: accounts @@ -1569,11 +1550,6 @@ engines: shortcut: scc disabled: true - - name: framalibre - engine: framalibre - shortcut: frl - disabled: true - # - name: searx # engine: searx_engine # shortcut: se From c053af534c167457b4c8d8c1d4ea2e02e49c4a82 Mon Sep 17 00:00:00 2001 From: ItzCrazyKns Date: Sun, 28 Apr 2024 19:49:48 +0530 Subject: [PATCH 029/470] feat(readme): make installation steps more concise --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index a5e9b5c..8a3f923 100644 --- a/README.md +++ b/README.md @@ -59,11 +59,11 @@ There are mainly 2 ways of installing Perplexica - With Docker, Without Docker. 4. Rename the `sample.config.toml` file to `config.toml`. For Docker setups, you need only fill in the following fields: - - `CHAT_MODEL`: The name of the LLM to use. Like `llama2` (using Ollama), `gpt-3.5-turbo` (using OpenAI), etc. + - `CHAT_MODEL`: The name of the LLM to use. Like `llama3:latest` (using Ollama), `gpt-3.5-turbo` (using OpenAI), etc. - `CHAT_MODEL_PROVIDER`: The chat model provider, either `openai` or `ollama`. Depending upon which provider you use you would have to fill in the following fields: - - `OPENAI`: Your OpenAI API key. **You only need to fill this if you wish to use OpenAI's models.** - - `OLLAMA`: Your Ollama API URL. **You need to fill this if you wish to use Ollama's models instead of OpenAI's.** + - `OPENAI`: Your OpenAI API key. **You only need to fill this if you wish to use OpenAI's models**. + - `OLLAMA`: Your Ollama API URL. You should enter it as `http://host.docker.internal:PORT_NUMBER`. If you installed Ollama on port 11434, use `http://host.docker.internal:11434`. For other ports, adjust accordingly. **You need to fill this if you wish to use Ollama's models instead of OpenAI's**. **Note**: You can change these and use different models after running Perplexica as well from the settings page. From 9b5548e9f8dc667cb0f0fec3e896db668af2dd93 Mon Sep 17 00:00:00 2001 From: ItzCrazyKns Date: Sun, 28 Apr 2024 19:52:31 +0530 Subject: [PATCH 030/470] feat(sample-settings): update Ollama URL placeholder --- sample.config.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sample.config.toml b/sample.config.toml index 350ae03..2d09b4b 100644 --- a/sample.config.toml +++ b/sample.config.toml @@ -9,4 +9,4 @@ OPENAI = "" # OpenAI API key - sk-1234567890abcdef1234567890abcdef [API_ENDPOINTS] SEARXNG = "http://localhost:32768" # SearxNG API URL -OLLAMA = "" # Ollama API URL - http://localhost:11434 +OLLAMA = "" # Ollama API URL - http://host.docker.internal:11434 From 639129848a254c3b4843f30106258ea486d4bf58 Mon Sep 17 00:00:00 2001 From: SwiftyOS Date: Mon, 29 Apr 2024 10:49:15 +0200 Subject: [PATCH 031/470] feat(providers): add gpt-4-turbo provider --- src/lib/providers.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/lib/providers.ts b/src/lib/providers.ts index c730da8..f21f54e 100644 --- a/src/lib/providers.ts +++ b/src/lib/providers.ts @@ -22,6 +22,11 @@ export const getAvailableProviders = async () => { modelName: 'gpt-4', temperature: 0.7, }), + 'gpt-4-turbo': new ChatOpenAI({ + openAIApiKey, + modelName: 'gpt-4-turbo', + temperature: 0.7, + }), embeddings: new OpenAIEmbeddings({ openAIApiKey, modelName: 'text-embedding-3-large', From 7c84025f3c15f1c81e750db4719940f525a49891 Mon Sep 17 00:00:00 2001 From: ItzCrazyKns Date: Mon, 29 Apr 2024 21:22:33 +0530 Subject: [PATCH 032/470] feat(readme): add manual installation steps --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 8a3f923..38743e6 100644 --- a/README.md +++ b/README.md @@ -81,11 +81,11 @@ There are mainly 2 ways of installing Perplexica - With Docker, Without Docker. ### Non-Docker Installation -For setups without Docker: - -1. Follow the initial steps to clone the repository and rename the `sample.config.toml` file to `config.toml` in the root directory. You will need to fill in all the fields in this file. -2. Additionally, rename the `.env.example` file to `.env` in the `ui` folder and complete all fields. -3. The non-Docker setup requires manual configuration of both the backend and frontend. +1. Clone the repository and rename the `sample.config.toml` file to `config.toml` in the root directory. Ensure you complete all required fields in this file. +2. Rename the `.env.example` file to `.env` in the `ui` folder and fill in all necessary fields. +3. After populating the configuration and environment files, run `npm i` in both the `ui` folder and the root directory. +4. Install the dependencies and then execute `npm run build` in both the `ui` folder and the root directory. +5. Finally, start both the frontend and the backend by running `npm run start` in both the `ui` folder and the root directory. **Note**: Using Docker is recommended as it simplifies the setup process, especially for managing environment variables and dependencies. From aae85cd76781a5bad2cb4aa20638bf2b5edb274a Mon Sep 17 00:00:00 2001 From: ItzCrazyKns Date: Tue, 30 Apr 2024 12:18:18 +0530 Subject: [PATCH 033/470] feat(logging): add logger --- package.json | 1 + src/agents/academicSearchAgent.ts | 3 +- src/agents/redditSearchAgent.ts | 3 +- src/agents/webSearchAgent.ts | 3 +- src/agents/wolframAlphaSearchAgent.ts | 3 +- src/agents/writingAssistant.ts | 3 +- src/agents/youtubeSearchAgent.ts | 3 +- src/app.ts | 3 +- src/lib/providers.ts | 5 +- src/routes/images.ts | 3 +- src/utils/logger.ts | 22 +++ src/websocket/connectionManager.ts | 3 +- src/websocket/messageHandler.ts | 5 +- src/websocket/websocketServer.ts | 3 +- yarn.lock | 194 +++++++++++++++++++++++++- 15 files changed, 240 insertions(+), 17 deletions(-) create mode 100644 src/utils/logger.ts diff --git a/package.json b/package.json index 5006a93..a4b91e1 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "dotenv": "^16.4.5", "express": "^4.19.2", "langchain": "^0.1.30", + "winston": "^3.13.0", "ws": "^8.16.0", "zod": "^3.22.4" } diff --git a/src/agents/academicSearchAgent.ts b/src/agents/academicSearchAgent.ts index d08f282..5c11307 100644 --- a/src/agents/academicSearchAgent.ts +++ b/src/agents/academicSearchAgent.ts @@ -18,6 +18,7 @@ import type { Embeddings } from '@langchain/core/embeddings'; import formatChatHistoryAsString from '../utils/formatHistory'; import eventEmitter from 'events'; import computeSimilarity from '../utils/computeSimilarity'; +import logger from '../utils/logger'; const basicAcademicSearchRetrieverPrompt = ` You will be given a conversation below and a follow up question. You need to rephrase the follow-up question if needed so it is a standalone question that can be used by the LLM to search the web for information. @@ -245,7 +246,7 @@ const basicAcademicSearch = ( 'error', JSON.stringify({ data: 'An error has occurred please try again later' }), ); - console.error(err); + logger.error(`Error in academic search: ${err}`); } return emitter; diff --git a/src/agents/redditSearchAgent.ts b/src/agents/redditSearchAgent.ts index 4fee20a..34e9ec2 100644 --- a/src/agents/redditSearchAgent.ts +++ b/src/agents/redditSearchAgent.ts @@ -18,6 +18,7 @@ import type { Embeddings } from '@langchain/core/embeddings'; import formatChatHistoryAsString from '../utils/formatHistory'; import eventEmitter from 'events'; import computeSimilarity from '../utils/computeSimilarity'; +import logger from '../utils/logger'; const basicRedditSearchRetrieverPrompt = ` You will be given a conversation below and a follow up question. You need to rephrase the follow-up question if needed so it is a standalone question that can be used by the LLM to search the web for information. @@ -240,7 +241,7 @@ const basicRedditSearch = ( 'error', JSON.stringify({ data: 'An error has occurred please try again later' }), ); - console.error(err); + logger.error(`Error in RedditSearch: ${err}`); } return emitter; diff --git a/src/agents/webSearchAgent.ts b/src/agents/webSearchAgent.ts index 7b8e877..1364742 100644 --- a/src/agents/webSearchAgent.ts +++ b/src/agents/webSearchAgent.ts @@ -18,6 +18,7 @@ import type { Embeddings } from '@langchain/core/embeddings'; import formatChatHistoryAsString from '../utils/formatHistory'; import eventEmitter from 'events'; import computeSimilarity from '../utils/computeSimilarity'; +import logger from '../utils/logger'; const basicSearchRetrieverPrompt = ` You will be given a conversation below and a follow up question. You need to rephrase the follow-up question if needed so it is a standalone question that can be used by the LLM to search the web for information. @@ -241,7 +242,7 @@ const basicWebSearch = ( 'error', JSON.stringify({ data: 'An error has occurred please try again later' }), ); - console.error(err); + logger.error(`Error in websearch: ${err}`); } return emitter; diff --git a/src/agents/wolframAlphaSearchAgent.ts b/src/agents/wolframAlphaSearchAgent.ts index cdcd222..f810a1e 100644 --- a/src/agents/wolframAlphaSearchAgent.ts +++ b/src/agents/wolframAlphaSearchAgent.ts @@ -17,6 +17,7 @@ import type { BaseChatModel } from '@langchain/core/language_models/chat_models' import type { Embeddings } from '@langchain/core/embeddings'; import formatChatHistoryAsString from '../utils/formatHistory'; import eventEmitter from 'events'; +import logger from '../utils/logger'; const basicWolframAlphaSearchRetrieverPrompt = ` You will be given a conversation below and a follow up question. You need to rephrase the follow-up question if needed so it is a standalone question that can be used by the LLM to search the web for information. @@ -199,7 +200,7 @@ const basicWolframAlphaSearch = ( 'error', JSON.stringify({ data: 'An error has occurred please try again later' }), ); - console.error(err); + logger.error(`Error in WolframAlphaSearch: ${err}`); } return emitter; diff --git a/src/agents/writingAssistant.ts b/src/agents/writingAssistant.ts index ff5365e..7c2cb49 100644 --- a/src/agents/writingAssistant.ts +++ b/src/agents/writingAssistant.ts @@ -9,6 +9,7 @@ import type { StreamEvent } from '@langchain/core/tracers/log_stream'; import eventEmitter from 'events'; import type { BaseChatModel } from '@langchain/core/language_models/chat_models'; import type { Embeddings } from '@langchain/core/embeddings'; +import logger from '../utils/logger'; const writingAssistantPrompt = ` You are Perplexica, an AI model who is expert at searching the web and answering user's queries. You are currently set on focus mode 'Writing Assistant', this means you will be helping the user write a response to a given query. @@ -80,7 +81,7 @@ const handleWritingAssistant = ( 'error', JSON.stringify({ data: 'An error has occurred please try again later' }), ); - console.error(err); + logger.error(`Error in writing assistant: ${err}`); } return emitter; diff --git a/src/agents/youtubeSearchAgent.ts b/src/agents/youtubeSearchAgent.ts index 31ba660..4e82cc7 100644 --- a/src/agents/youtubeSearchAgent.ts +++ b/src/agents/youtubeSearchAgent.ts @@ -18,6 +18,7 @@ import type { Embeddings } from '@langchain/core/embeddings'; import formatChatHistoryAsString from '../utils/formatHistory'; import eventEmitter from 'events'; import computeSimilarity from '../utils/computeSimilarity'; +import logger from '../utils/logger'; const basicYoutubeSearchRetrieverPrompt = ` You will be given a conversation below and a follow up question. You need to rephrase the follow-up question if needed so it is a standalone question that can be used by the LLM to search the web for information. @@ -241,7 +242,7 @@ const basicYoutubeSearch = ( 'error', JSON.stringify({ data: 'An error has occurred please try again later' }), ); - console.error(err); + logger.error(`Error in youtube search: ${err}`); } return emitter; diff --git a/src/app.ts b/src/app.ts index 19f95bc..b8c2371 100644 --- a/src/app.ts +++ b/src/app.ts @@ -4,6 +4,7 @@ import cors from 'cors'; import http from 'http'; import routes from './routes'; import { getPort } from './config'; +import logger from './utils/logger'; const port = getPort(); @@ -23,7 +24,7 @@ app.get('/api', (_, res) => { }); server.listen(port, () => { - console.log(`API server started on port ${port}`); + logger.info(`Server is running on port ${port}`); }); startWebSocketServer(server); diff --git a/src/lib/providers.ts b/src/lib/providers.ts index f21f54e..71ed079 100644 --- a/src/lib/providers.ts +++ b/src/lib/providers.ts @@ -2,6 +2,7 @@ import { ChatOpenAI, OpenAIEmbeddings } from '@langchain/openai'; import { ChatOllama } from '@langchain/community/chat_models/ollama'; import { OllamaEmbeddings } from '@langchain/community/embeddings/ollama'; import { getOllamaApiEndpoint, getOpenaiApiKey } from '../config'; +import logger from '../utils/logger'; export const getAvailableProviders = async () => { const openAIApiKey = getOpenaiApiKey(); @@ -33,7 +34,7 @@ export const getAvailableProviders = async () => { }), }; } catch (err) { - console.log(`Error loading OpenAI models: ${err}`); + logger.error(`Error loading OpenAI models: ${err}`); } } @@ -59,7 +60,7 @@ export const getAvailableProviders = async () => { }); } } catch (err) { - console.log(`Error loading Ollama models: ${err}`); + logger.error(`Error loading Ollama models: ${err}`); } } diff --git a/src/routes/images.ts b/src/routes/images.ts index 45a4307..066a3ee 100644 --- a/src/routes/images.ts +++ b/src/routes/images.ts @@ -4,6 +4,7 @@ import { BaseChatModel } from '@langchain/core/language_models/chat_models'; import { getAvailableProviders } from '../lib/providers'; import { getChatModel, getChatModelProvider } from '../config'; import { HumanMessage, AIMessage } from '@langchain/core/messages'; +import logger from '../utils/logger'; const router = express.Router(); @@ -39,7 +40,7 @@ router.post('/', async (req, res) => { res.status(200).json({ images }); } catch (err) { res.status(500).json({ message: 'An error has occurred.' }); - console.log(err.message); + logger.error(`Error in image search: ${err.message}`); } }); diff --git a/src/utils/logger.ts b/src/utils/logger.ts new file mode 100644 index 0000000..1c81eb9 --- /dev/null +++ b/src/utils/logger.ts @@ -0,0 +1,22 @@ +import winston from 'winston'; + +const logger = winston.createLogger({ + level: 'info', + transports: [ + new winston.transports.Console({ + format: winston.format.combine( + winston.format.colorize(), + winston.format.simple(), + ), + }), + new winston.transports.File({ + filename: 'app.log', + format: winston.format.combine( + winston.format.timestamp(), + winston.format.json(), + ), + }), + ], +}); + +export default logger; diff --git a/src/websocket/connectionManager.ts b/src/websocket/connectionManager.ts index 5b4e3f5..afaaf44 100644 --- a/src/websocket/connectionManager.ts +++ b/src/websocket/connectionManager.ts @@ -4,6 +4,7 @@ import { getChatModel, getChatModelProvider } from '../config'; import { getAvailableProviders } from '../lib/providers'; import { BaseChatModel } from '@langchain/core/language_models/chat_models'; import type { Embeddings } from '@langchain/core/embeddings'; +import logger from '../utils/logger'; export const handleConnection = async (ws: WebSocket) => { const models = await getAvailableProviders(); @@ -34,5 +35,5 @@ export const handleConnection = async (ws: WebSocket) => { await handleMessage(message.toString(), ws, llm, embeddings), ); - ws.on('close', () => console.log('Connection closed')); + ws.on('close', () => logger.debug('Connection closed')); }; diff --git a/src/websocket/messageHandler.ts b/src/websocket/messageHandler.ts index 48774bf..537651f 100644 --- a/src/websocket/messageHandler.ts +++ b/src/websocket/messageHandler.ts @@ -8,6 +8,7 @@ import handleYoutubeSearch from '../agents/youtubeSearchAgent'; import handleRedditSearch from '../agents/redditSearchAgent'; import type { BaseChatModel } from '@langchain/core/language_models/chat_models'; import type { Embeddings } from '@langchain/core/embeddings'; +import logger from '../utils/logger'; type Message = { type: string; @@ -101,8 +102,8 @@ export const handleMessage = async ( ws.send(JSON.stringify({ type: 'error', data: 'Invalid focus mode' })); } } - } catch (error) { - console.error('Failed to handle message', error); + } catch (err) { ws.send(JSON.stringify({ type: 'error', data: 'Invalid message format' })); + logger.error(`Failed to handle message: ${err}`); } }; diff --git a/src/websocket/websocketServer.ts b/src/websocket/websocketServer.ts index 451f9f2..bc84f52 100644 --- a/src/websocket/websocketServer.ts +++ b/src/websocket/websocketServer.ts @@ -2,6 +2,7 @@ import { WebSocketServer } from 'ws'; import { handleConnection } from './connectionManager'; import http from 'http'; import { getPort } from '../config'; +import logger from '../utils/logger'; export const initServer = ( server: http.Server, @@ -13,5 +14,5 @@ export const initServer = ( handleConnection(ws); }); - console.log(`WebSocket server started on port ${port}`); + logger.info(`WebSocket server started on port ${port}`); }; diff --git a/yarn.lock b/yarn.lock index 080d4c0..6ce57fd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -17,6 +17,11 @@ node-fetch "^2.6.7" web-streams-polyfill "^3.2.1" +"@colors/colors@1.6.0", "@colors/colors@^1.6.0": + version "1.6.0" + resolved "https://registry.yarnpkg.com/@colors/colors/-/colors-1.6.0.tgz#ec6cd237440700bc23ca23087f513c75508958b0" + integrity sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA== + "@cspotcode/source-map-support@^0.8.0": version "0.8.1" resolved "https://registry.yarnpkg.com/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz#00629c35a688e05a88b1cda684fb9d5e73f000a1" @@ -24,6 +29,15 @@ dependencies: "@jridgewell/trace-mapping" "0.3.9" +"@dabh/diagnostics@^2.0.2": + version "2.0.3" + resolved "https://registry.yarnpkg.com/@dabh/diagnostics/-/diagnostics-2.0.3.tgz#7f7e97ee9a725dffc7808d93668cc984e1dc477a" + integrity sha512-hrlQOIi7hAfzsMqlGSFyVucrx38O+j6wiGOf//H2ecvIEqYN4ADBSS2iLMh5UFyDunCNniUIPk/q3riFv45xRA== + dependencies: + colorspace "1.1.x" + enabled "2.0.x" + kuler "^2.0.0" + "@iarna/toml@^2.2.5": version "2.2.5" resolved "https://registry.yarnpkg.com/@iarna/toml/-/toml-2.2.5.tgz#b32366c89b43c6f8cefbdefac778b9c828e3ba8c" @@ -222,6 +236,11 @@ "@types/node" "*" "@types/send" "*" +"@types/triple-beam@^1.3.2": + version "1.3.5" + resolved "https://registry.yarnpkg.com/@types/triple-beam/-/triple-beam-1.3.5.tgz#74fef9ffbaa198eb8b588be029f38b00299caa2c" + integrity sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw== + "@types/uuid@^9.0.1": version "9.0.8" resolved "https://registry.yarnpkg.com/@types/uuid/-/uuid-9.0.8.tgz#7545ba4fc3c003d6c756f651f3bf163d8f0f29ba" @@ -279,6 +298,11 @@ array-flatten@1.1.1: resolved "https://registry.yarnpkg.com/array-flatten/-/array-flatten-1.1.1.tgz#9a5f699051b1e7073328f2a008968b64ea2955d2" integrity sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg== +async@^3.2.3: + version "3.2.5" + resolved "https://registry.yarnpkg.com/async/-/async-3.2.5.tgz#ebd52a8fdaf7a2289a24df399f8d8485c8a46b66" + integrity sha512-baNZyqaaLhyLVKm/DlvdW051MSgO6b8eVfIezl9E5PqWxFgzLm/wQntEW4zOytVburDEr0JlALEpdOFwvErLsg== + asynckit@^0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" @@ -357,6 +381,47 @@ charenc@0.0.2: resolved "https://registry.yarnpkg.com/charenc/-/charenc-0.0.2.tgz#c0a1d2f3a7092e03774bfa83f14c0fc5790a8667" integrity sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA== +color-convert@^1.9.3: + version "1.9.3" + resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8" + integrity sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg== + dependencies: + color-name "1.1.3" + +color-name@1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.3.tgz#a7d0558bd89c42f795dd42328f740831ca53bc25" + integrity sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw== + +color-name@^1.0.0: + version "1.1.4" + resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2" + integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== + +color-string@^1.6.0: + version "1.9.1" + resolved "https://registry.yarnpkg.com/color-string/-/color-string-1.9.1.tgz#4467f9146f036f855b764dfb5bf8582bf342c7a4" + integrity sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg== + dependencies: + color-name "^1.0.0" + simple-swizzle "^0.2.2" + +color@^3.1.3: + version "3.2.1" + resolved "https://registry.yarnpkg.com/color/-/color-3.2.1.tgz#3544dc198caf4490c3ecc9a790b54fe9ff45e164" + integrity sha512-aBl7dZI9ENN6fUGC7mWpMTPNHmWUSNan9tuWN6ahh5ZLNk9baLJOnSMlrQkHcrfFgz2/RigjUVAjdx36VcemKA== + dependencies: + color-convert "^1.9.3" + color-string "^1.6.0" + +colorspace@1.1.x: + version "1.1.4" + resolved "https://registry.yarnpkg.com/colorspace/-/colorspace-1.1.4.tgz#8d442d1186152f60453bf8070cd66eb364e59243" + integrity sha512-BgvKJiuVu1igBUF2kEjRCZXol6wiiGbY5ipL/oVPwm0BL9sIpMIzM8IK7vwuxIIzOXMV3Ey5w+vxhm0rR/TN8w== + dependencies: + color "^3.1.3" + text-hex "1.0.x" + combined-stream@^1.0.8: version "1.0.8" resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f" @@ -494,6 +559,11 @@ ee-first@1.1.1: resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d" integrity sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow== +enabled@2.0.x: + version "2.0.0" + resolved "https://registry.yarnpkg.com/enabled/-/enabled-2.0.0.tgz#f9dd92ec2d6f4bbc0d5d1e64e21d61cd4665e7c2" + integrity sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ== + encodeurl@~1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59" @@ -573,6 +643,11 @@ express@^4.19.2: utils-merge "1.0.1" vary "~1.1.2" +fecha@^4.2.0: + version "4.2.3" + resolved "https://registry.yarnpkg.com/fecha/-/fecha-4.2.3.tgz#4d9ccdbc61e8629b259fdca67e65891448d569fd" + integrity sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw== + finalhandler@1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/finalhandler/-/finalhandler-1.2.0.tgz#7d23fe5731b207b4640e4fcd00aec1f9207a7b32" @@ -591,6 +666,11 @@ flat@^5.0.2: resolved "https://registry.yarnpkg.com/flat/-/flat-5.0.2.tgz#8ca6fe332069ffa9d324c327198c598259ceb241" integrity sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ== +fn.name@1.x.x: + version "1.1.0" + resolved "https://registry.yarnpkg.com/fn.name/-/fn.name-1.1.0.tgz#26cad8017967aea8731bc42961d04a3d5988accc" + integrity sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw== + follow-redirects@^1.15.6: version "1.15.6" resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.6.tgz#7f815c0cda4249c74ff09e95ef97c23b5fd0399b" @@ -700,7 +780,7 @@ iconv-lite@0.4.24: dependencies: safer-buffer ">= 2.1.2 < 3" -inherits@2.0.4: +inherits@2.0.4, inherits@^2.0.3: version "2.0.4" resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== @@ -715,11 +795,21 @@ is-any-array@^2.0.0: resolved "https://registry.yarnpkg.com/is-any-array/-/is-any-array-2.0.1.tgz#9233242a9c098220290aa2ec28f82ca7fa79899e" integrity sha512-UtilS7hLRu++wb/WBAw9bNuP1Eg04Ivn1vERJck8zJthEvXCBEBpGR/33u/xLKWEQf95803oalHrVDptcAvFdQ== +is-arrayish@^0.3.1: + version "0.3.2" + resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.3.2.tgz#4574a2ae56f7ab206896fb431eaeed066fdf8f03" + integrity sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ== + is-buffer@~1.1.6: version "1.1.6" resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be" integrity sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w== +is-stream@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-2.0.1.tgz#fac1e3d53b97ad5a9d0ae9cef2389f5810a5c077" + integrity sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg== + js-tiktoken@^1.0.7, js-tiktoken@^1.0.8: version "1.0.10" resolved "https://registry.yarnpkg.com/js-tiktoken/-/js-tiktoken-1.0.10.tgz#2b343ec169399dcee8f9ef9807dbd4fafd3b30dc" @@ -739,6 +829,11 @@ jsonpointer@^5.0.1: resolved "https://registry.yarnpkg.com/jsonpointer/-/jsonpointer-5.0.1.tgz#2110e0af0900fd37467b5907ecd13a7884a1b559" integrity sha512-p/nXbhSEcu3pZRdkW1OfJhpsVtW1gd4Wa1fnQc9YLiTfAjn0312eMKimbdIQzuZl9aa9xUGaRlP9T/CJE/ditQ== +kuler@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/kuler/-/kuler-2.0.0.tgz#e2c570a3800388fb44407e851531c1d670b061b3" + integrity sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A== + langchain@^0.1.30: version "0.1.30" resolved "https://registry.yarnpkg.com/langchain/-/langchain-0.1.30.tgz#e1adb3f1849fcd5c596c668300afd5dc8cb37a97" @@ -778,6 +873,18 @@ langsmith@~0.1.1, langsmith@~0.1.7: p-retry "4" uuid "^9.0.0" +logform@^2.3.2, logform@^2.4.0: + version "2.6.0" + resolved "https://registry.yarnpkg.com/logform/-/logform-2.6.0.tgz#8c82a983f05d6eaeb2d75e3decae7a768b2bf9b5" + integrity sha512-1ulHeNPp6k/LD8H91o7VYFBng5i1BDE7HoKxVbZiGFidS1Rj65qcywLxX+pVfAPoQJEjRdvKcusKwOupHCVOVQ== + dependencies: + "@colors/colors" "1.6.0" + "@types/triple-beam" "^1.3.2" + fecha "^4.2.0" + ms "^2.1.1" + safe-stable-stringify "^2.3.1" + triple-beam "^1.3.0" + make-error@^1.1.1: version "1.3.6" resolved "https://registry.yarnpkg.com/make-error/-/make-error-1.3.6.tgz#2eb2e37ea9b67c4891f684a1394799af484cf7a2" @@ -865,7 +972,7 @@ ms@2.0.0: resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" integrity sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A== -ms@2.1.3, ms@^2.0.0: +ms@2.1.3, ms@^2.0.0, ms@^2.1.1: version "2.1.3" resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== @@ -909,6 +1016,13 @@ on-finished@2.4.1: dependencies: ee-first "1.1.1" +one-time@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/one-time/-/one-time-1.0.0.tgz#e06bc174aed214ed58edede573b433bbf827cb45" + integrity sha512-5DXOiRKwuSEcQ/l0kGCF6Q3jcADFv5tSmRaJck/OqkVFcOzutB134KRSfF0xDrL39MNnqxbHBbUUcjZIhTgb2g== + dependencies: + fn.name "1.x.x" + openai@^4.26.0: version "4.31.0" resolved "https://registry.yarnpkg.com/openai/-/openai-4.31.0.tgz#5d96045c4eb244fa21f0fff0981043a2c9f09e8c" @@ -1007,12 +1121,21 @@ raw-body@2.5.2: iconv-lite "0.4.24" unpipe "1.0.0" +readable-stream@^3.4.0, readable-stream@^3.6.0: + version "3.6.2" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.2.tgz#56a9b36ea965c00c5a93ef31eb111a0f11056967" + integrity sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA== + dependencies: + inherits "^2.0.3" + string_decoder "^1.1.1" + util-deprecate "^1.0.1" + retry@^0.13.1: version "0.13.1" resolved "https://registry.yarnpkg.com/retry/-/retry-0.13.1.tgz#185b1587acf67919d63b357349e03537b2484658" integrity sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg== -safe-buffer@5.2.1: +safe-buffer@5.2.1, safe-buffer@~5.2.0: version "5.2.1" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6" integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== @@ -1022,6 +1145,11 @@ safe-buffer@~5.1.1: resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== +safe-stable-stringify@^2.3.1: + version "2.4.3" + resolved "https://registry.yarnpkg.com/safe-stable-stringify/-/safe-stable-stringify-2.4.3.tgz#138c84b6f6edb3db5f8ef3ef7115b8f55ccbf886" + integrity sha512-e2bDA2WJT0wxseVd4lsDP4+3ONX6HpMXQa1ZhFQ7SU+GjvORCmShbCMltrtIDfkYhVHrOcPtj+KhmDBdPdZD1g== + "safer-buffer@>= 2.1.2 < 3": version "2.1.2" resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" @@ -1083,11 +1211,35 @@ side-channel@^1.0.4: get-intrinsic "^1.2.4" object-inspect "^1.13.1" +simple-swizzle@^0.2.2: + version "0.2.2" + resolved "https://registry.yarnpkg.com/simple-swizzle/-/simple-swizzle-0.2.2.tgz#a4da6b635ffcccca33f70d17cb92592de95e557a" + integrity sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg== + dependencies: + is-arrayish "^0.3.1" + +stack-trace@0.0.x: + version "0.0.10" + resolved "https://registry.yarnpkg.com/stack-trace/-/stack-trace-0.0.10.tgz#547c70b347e8d32b4e108ea1a2a159e5fdde19c0" + integrity sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg== + statuses@2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/statuses/-/statuses-2.0.1.tgz#55cb000ccf1d48728bd23c685a063998cf1a1b63" integrity sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ== +string_decoder@^1.1.1: + version "1.3.0" + resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.3.0.tgz#42f114594a46cf1a8e30b0a84f56c78c3edac21e" + integrity sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA== + dependencies: + safe-buffer "~5.2.0" + +text-hex@1.0.x: + version "1.0.0" + resolved "https://registry.yarnpkg.com/text-hex/-/text-hex-1.0.0.tgz#69dc9c1b17446ee79a92bf5b884bb4b9127506f5" + integrity sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg== + toidentifier@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.1.tgz#3be34321a88a820ed1bd80dfaa33e479fbb8dd35" @@ -1098,6 +1250,11 @@ tr46@~0.0.3: resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" integrity sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw== +triple-beam@^1.3.0: + version "1.4.1" + resolved "https://registry.yarnpkg.com/triple-beam/-/triple-beam-1.4.1.tgz#6fde70271dc6e5d73ca0c3b24e2d92afb7441984" + integrity sha512-aZbgViZrg1QNcG+LULa7nhZpJTZSLm/mXnHXnbAbjmN5aSa0y7V+wvv6+4WaBtpISJzThKy+PIPxc1Nq1EJ9mg== + ts-node@^10.9.2: version "10.9.2" resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-10.9.2.tgz#70f021c9e185bccdca820e26dc413805c101c71f" @@ -1140,6 +1297,11 @@ unpipe@1.0.0, unpipe@~1.0.0: resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec" integrity sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ== +util-deprecate@^1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" + integrity sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw== + utils-merge@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713" @@ -1193,6 +1355,32 @@ whatwg-url@^5.0.0: tr46 "~0.0.3" webidl-conversions "^3.0.0" +winston-transport@^4.7.0: + version "4.7.0" + resolved "https://registry.yarnpkg.com/winston-transport/-/winston-transport-4.7.0.tgz#e302e6889e6ccb7f383b926df6936a5b781bd1f0" + integrity sha512-ajBj65K5I7denzer2IYW6+2bNIVqLGDHqDw3Ow8Ohh+vdW+rv4MZ6eiDvHoKhfJFZ2auyN8byXieDDJ96ViONg== + dependencies: + logform "^2.3.2" + readable-stream "^3.6.0" + triple-beam "^1.3.0" + +winston@^3.13.0: + version "3.13.0" + resolved "https://registry.yarnpkg.com/winston/-/winston-3.13.0.tgz#e76c0d722f78e04838158c61adc1287201de7ce3" + integrity sha512-rwidmA1w3SE4j0E5MuIufFhyJPBDG7Nu71RkZor1p2+qHvJSZ9GYDA81AyleQcZbh/+V6HjeBdfnTZJm9rSeQQ== + dependencies: + "@colors/colors" "^1.6.0" + "@dabh/diagnostics" "^2.0.2" + async "^3.2.3" + is-stream "^2.0.0" + logform "^2.4.0" + one-time "^1.0.0" + readable-stream "^3.4.0" + safe-stable-stringify "^2.3.1" + stack-trace "0.0.x" + triple-beam "^1.3.0" + winston-transport "^4.7.0" + ws@^8.16.0: version "8.16.0" resolved "https://registry.yarnpkg.com/ws/-/ws-8.16.0.tgz#d1cd774f36fbc07165066a60e40323eab6446fd4" From ee053cf31e2b2cdcd8c124c622e5a3b92ccc1f9b Mon Sep 17 00:00:00 2001 From: ItzCrazyKns Date: Tue, 30 Apr 2024 12:39:04 +0530 Subject: [PATCH 034/470] feat(search-image): add button animations --- ui/components/SearchImages.tsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ui/components/SearchImages.tsx b/ui/components/SearchImages.tsx index 1f94963..0ab5076 100644 --- a/ui/components/SearchImages.tsx +++ b/ui/components/SearchImages.tsx @@ -92,7 +92,7 @@ const SearchImages = ({ key={i} src={image.img_src} alt={image.title} - className="h-full w-full aspect-video object-cover rounded-lg transition duration-200 active:scale-95 cursor-pointer" + className="h-full w-full aspect-video object-cover rounded-lg transition duration-200 active:scale-95 hover:scale-[1.02] cursor-zoom-in" /> )) : images.map((image, i) => ( @@ -108,13 +108,13 @@ const SearchImages = ({ key={i} src={image.img_src} alt={image.title} - className="h-full w-full aspect-video object-cover rounded-lg transition duration-200 active:scale-95 cursor-pointer" + className="h-full w-full aspect-video object-cover rounded-lg transition duration-200 active:scale-95 hover:scale-[1.02] cursor-zoom-in" /> ))} {images.length > 4 && ( )} {loading && ( -
+
{[...Array(4)].map((_, i) => (

- View {images.slice(0, 2).length} more + View {images.length - 3} more

)} From 6e304e7051a295e727a68a0ad0535ebb06e9f1a0 Mon Sep 17 00:00:00 2001 From: ItzCrazyKns Date: Tue, 30 Apr 2024 14:31:32 +0530 Subject: [PATCH 036/470] feat(video-search): add video search --- src/agents/videoSearchAgent.ts | 90 ++++++++++++++++ src/lib/searxng.ts | 2 + src/routes/index.ts | 2 + src/routes/videos.ts | 47 ++++++++ ui/components/MessageBox.tsx | 12 +-- ui/components/SearchVideos.tsx | 190 +++++++++++++++++++++++++++++++++ 6 files changed, 336 insertions(+), 7 deletions(-) create mode 100644 src/agents/videoSearchAgent.ts create mode 100644 src/routes/videos.ts create mode 100644 ui/components/SearchVideos.tsx diff --git a/src/agents/videoSearchAgent.ts b/src/agents/videoSearchAgent.ts new file mode 100644 index 0000000..cdd1ac0 --- /dev/null +++ b/src/agents/videoSearchAgent.ts @@ -0,0 +1,90 @@ +import { + RunnableSequence, + RunnableMap, + RunnableLambda, +} from '@langchain/core/runnables'; +import { PromptTemplate } from '@langchain/core/prompts'; +import formatChatHistoryAsString from '../utils/formatHistory'; +import { BaseMessage } from '@langchain/core/messages'; +import { StringOutputParser } from '@langchain/core/output_parsers'; +import { searchSearxng } from '../lib/searxng'; +import type { BaseChatModel } from '@langchain/core/language_models/chat_models'; + +const VideoSearchChainPrompt = ` + You will be given a conversation below and a follow up question. You need to rephrase the follow-up question so it is a standalone question that can be used by the LLM to search Youtube for videos. + You need to make sure the rephrased question agrees with the conversation and is relevant to the conversation. + + Example: + 1. Follow up question: How does a car work? + Rephrased: How does a car work? + + 2. Follow up question: What is the theory of relativity? + Rephrased: What is theory of relativity + + 3. Follow up question: How does an AC work? + Rephrased: How does an AC work + + Conversation: + {chat_history} + + Follow up question: {query} + Rephrased question: + `; + +type VideoSearchChainInput = { + chat_history: BaseMessage[]; + query: string; +}; + +const strParser = new StringOutputParser(); + +const createVideoSearchChain = (llm: BaseChatModel) => { + return RunnableSequence.from([ + RunnableMap.from({ + chat_history: (input: VideoSearchChainInput) => { + return formatChatHistoryAsString(input.chat_history); + }, + query: (input: VideoSearchChainInput) => { + return input.query; + }, + }), + PromptTemplate.fromTemplate(VideoSearchChainPrompt), + llm, + strParser, + RunnableLambda.from(async (input: string) => { + const res = await searchSearxng(input, { + engines: ['youtube'], + }); + + const videos = []; + + res.results.forEach((result) => { + if ( + result.thumbnail && + result.url && + result.title && + result.iframe_src + ) { + videos.push({ + img_src: result.thumbnail, + url: result.url, + title: result.title, + iframe_src: result.iframe_src, + }); + } + }); + + return videos.slice(0, 10); + }), + ]); +}; + +const handleVideoSearch = ( + input: VideoSearchChainInput, + llm: BaseChatModel, +) => { + const VideoSearchChain = createVideoSearchChain(llm); + return VideoSearchChain.invoke(input); +}; + +export default handleVideoSearch; diff --git a/src/lib/searxng.ts b/src/lib/searxng.ts index 297e50f..da62457 100644 --- a/src/lib/searxng.ts +++ b/src/lib/searxng.ts @@ -13,8 +13,10 @@ interface SearxngSearchResult { url: string; img_src?: string; thumbnail_src?: string; + thumbnail?: string; content?: string; author?: string; + iframe_src?: string; } export const searchSearxng = async ( diff --git a/src/routes/index.ts b/src/routes/index.ts index a3262e4..bcfc3d3 100644 --- a/src/routes/index.ts +++ b/src/routes/index.ts @@ -1,10 +1,12 @@ import express from 'express'; import imagesRouter from './images'; +import videosRouter from './videos'; import configRouter from './config'; const router = express.Router(); router.use('/images', imagesRouter); +router.use('/videos', videosRouter); router.use('/config', configRouter); export default router; diff --git a/src/routes/videos.ts b/src/routes/videos.ts new file mode 100644 index 0000000..bfd5fa8 --- /dev/null +++ b/src/routes/videos.ts @@ -0,0 +1,47 @@ +import express from 'express'; +import { BaseChatModel } from '@langchain/core/language_models/chat_models'; +import { getAvailableProviders } from '../lib/providers'; +import { getChatModel, getChatModelProvider } from '../config'; +import { HumanMessage, AIMessage } from '@langchain/core/messages'; +import logger from '../utils/logger'; +import handleVideoSearch from '../agents/videoSearchAgent'; + +const router = express.Router(); + +router.post('/', async (req, res) => { + try { + let { query, chat_history } = req.body; + + chat_history = chat_history.map((msg: any) => { + if (msg.role === 'user') { + return new HumanMessage(msg.content); + } else if (msg.role === 'assistant') { + return new AIMessage(msg.content); + } + }); + + const models = await getAvailableProviders(); + const provider = getChatModelProvider(); + const chatModel = getChatModel(); + + let llm: BaseChatModel | undefined; + + if (models[provider] && models[provider][chatModel]) { + llm = models[provider][chatModel] as BaseChatModel | undefined; + } + + if (!llm) { + res.status(500).json({ message: 'Invalid LLM model selected' }); + return; + } + + const videos = await handleVideoSearch({ chat_history, query }, llm); + + res.status(200).json({ videos }); + } catch (err) { + res.status(500).json({ message: 'An error has occurred.' }); + logger.error(`Error in video search: ${err.message}`); + } +}); + +export default router; diff --git a/ui/components/MessageBox.tsx b/ui/components/MessageBox.tsx index cb9da14..3ccda13 100644 --- a/ui/components/MessageBox.tsx +++ b/ui/components/MessageBox.tsx @@ -16,6 +16,7 @@ import Copy from './MessageActions/Copy'; import Rewrite from './MessageActions/Rewrite'; import MessageSources from './MessageSources'; import SearchImages from './SearchImages'; +import SearchVideos from './SearchVideos'; const MessageBox = ({ message, @@ -120,13 +121,10 @@ const MessageBox = ({ query={history[messageIndex - 1].content} chat_history={history.slice(0, messageIndex - 1)} /> -
-
- -

Search videos

-
- -
+
)} diff --git a/ui/components/SearchVideos.tsx b/ui/components/SearchVideos.tsx new file mode 100644 index 0000000..335664e --- /dev/null +++ b/ui/components/SearchVideos.tsx @@ -0,0 +1,190 @@ +/* eslint-disable @next/next/no-img-element */ +import { PlayCircle, PlayIcon, PlusIcon, VideoIcon } from 'lucide-react'; +import { useState } from 'react'; +import Lightbox, { GenericSlide, VideoSlide } from 'yet-another-react-lightbox'; +import 'yet-another-react-lightbox/styles.css'; +import { Message } from './ChatWindow'; + +type Video = { + url: string; + img_src: string; + title: string; + iframe_src: string; +}; + +declare module 'yet-another-react-lightbox' { + export interface VideoSlide extends GenericSlide { + type: 'video-slide'; + src: string; + iframe_src: string; + } + + interface SlideTypes { + 'video-slide': VideoSlide; + } +} + +const Searchvideos = ({ + query, + chat_history, +}: { + query: string; + chat_history: Message[]; +}) => { + const [videos, setVideos] = useState(null); + const [loading, setLoading] = useState(false); + const [open, setOpen] = useState(false); + const [slides, setSlides] = useState([]); + + return ( + <> + {!loading && videos === null && ( + + )} + {loading && ( +
+ {[...Array(4)].map((_, i) => ( +
+ ))} +
+ )} + {videos !== null && videos.length > 0 && ( + <> +
+ {videos.length > 4 + ? videos.slice(0, 3).map((video, i) => ( +
{ + setOpen(true); + setSlides([ + slides[i], + ...slides.slice(0, i), + ...slides.slice(i + 1), + ]); + }} + className="relative transition duration-200 active:scale-95 hover:scale-[1.02] cursor-pointer" + key={i} + > + {video.title} +
+ +

Video

+
+
+ )) + : videos.map((video, i) => ( +
{ + setOpen(true); + setSlides([ + slides[i], + ...slides.slice(0, i), + ...slides.slice(i + 1), + ]); + }} + className="relative transition duration-200 active:scale-95 hover:scale-[1.02] cursor-pointer" + key={i} + > + {video.title} +
+ +

Video

+
+
+ ))} + {videos.length > 4 && ( + + )} +
+ setOpen(false)} + slides={slides} + render={{ + slide: ({ slide }) => + slide.type === 'video-slide' ? ( +
+