sk/Perplexica
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(parsers): Enhance parsers to deal with some thinking models better.

Browse source
This commit is contained in:
Willie Zutz 2025-05-08 00:20:31 -06:00
parent bdace57cfd
commit 3b547251c2
4 changed files with 32 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

15
src/lib/outputParsers/lineOutputParser.ts
View file

@ -21,6 +21,10 @@ class LineOutputParser extends BaseOutputParser<string> {
async parse(text: string): Promise<string> {
text = text.trim() || '';
// First, remove all <think>...</think> blocks to avoid parsing tags inside thinking content
// This might be a little aggressive. Prompt massaging might be all we need, but this is a guarantee and should rarely mess anything up.
text = this.removeThinkingBlocks(text);
const regex = /^(\s*(-|\*|\d+\.\s|\d+\)\s|\u2022)\s*)+/;
const startKeyIndex = text.indexOf(`<${this.key}>`);
const endKeyIndex = text.indexOf(`</${this.key}>`);
@ -40,6 +44,17 @@ class LineOutputParser extends BaseOutputParser<string> {
return line;
}
/**
* Removes all content within <think>...</think> blocks
* @param text The input text containing thinking blocks
* @returns The text with all thinking blocks removed
*/
private removeThinkingBlocks(text: string): string {
// Use regex to identify and remove all <think>...</think> blocks
// Using the 's' flag to make dot match newlines
return text.replace(/<think>[\s\S]*?<\/think>/g, '').trim();
}
getFormatInstructions(): string {
throw new Error('Not implemented.');
}

15
src/lib/outputParsers/listLineOutputParser.ts
View file

@ -21,6 +21,10 @@ class LineListOutputParser extends BaseOutputParser<string[]> {
async parse(text: string): Promise<string[]> {
text = text.trim() || '';
// First, remove all <think>...</think> blocks to avoid parsing tags inside thinking content
// This might be a little aggressive. Prompt massaging might be all we need, but this is a guarantee and should rarely mess anything up.
text = this.removeThinkingBlocks(text);
const regex = /^(\s*(-|\*|\d+\.\s|\d+\)\s|\u2022)\s*)+/;
const startKeyIndex = text.indexOf(`<${this.key}>`);
const endKeyIndex = text.indexOf(`</${this.key}>`);
@ -42,6 +46,17 @@ class LineListOutputParser extends BaseOutputParser<string[]> {
return lines;
}
/**
* Removes all content within <think>...</think> blocks
* @param text The input text containing thinking blocks
* @returns The text with all thinking blocks removed
*/
private removeThinkingBlocks(text: string): string {
// Use regex to identify and remove all <think>...</think> blocks
// Using [\s\S] pattern to match all characters including newlines
return text.replace(/<think>[\s\S]*?<\/think>/g, '').trim();
}
getFormatInstructions(): string {
throw new Error('Not implemented.');
}