feat: RAG Integration (#238)

* feat: add rag provider and retriever

* feat: retriever tool

* feat: add retriever tool to the researcher node

* feat: add rag http apis

* feat: new message input supports resource mentions

* feat: new message input component support resource mentions

* refactor: need_web_search to need_search

* chore: RAG integration docs

* chore: change example api host

* fix: user message color in dark mode

* fix: mentions style

* feat: add local_search_tool to researcher prompt

* chore: research prompt

* fix: ragflow page size and reporter with

* docs: ragflow integration and add acknowledgment projects

* chore: format
This commit is contained in:
JeffJiang
2025-05-28 14:13:46 +08:00
committed by GitHub
parent 0565ab6d27
commit 462752b462
43 changed files with 1172 additions and 181 deletions

View File

@@ -3,18 +3,15 @@
import { AnimatePresence, motion } from "framer-motion";
import { ArrowUp, X } from "lucide-react";
import {
type KeyboardEvent,
useCallback,
useEffect,
useRef,
useState,
} from "react";
import { useCallback, useRef } from "react";
import { Detective } from "~/components/deer-flow/icons/detective";
import MessageInput, {
type MessageInputRef,
} from "~/components/deer-flow/message-input";
import { Tooltip } from "~/components/deer-flow/tooltip";
import { Button } from "~/components/ui/button";
import type { Option } from "~/core/messages";
import type { Option, Resource } from "~/core/messages";
import {
setEnableBackgroundInvestigation,
useSettingsStore,
@@ -23,7 +20,6 @@ import { cn } from "~/lib/utils";
export function InputBox({
className,
size,
responding,
feedback,
onSend,
@@ -34,72 +30,52 @@ export function InputBox({
size?: "large" | "normal";
responding?: boolean;
feedback?: { option: Option } | null;
onSend?: (message: string, options?: { interruptFeedback?: string }) => void;
onSend?: (
message: string,
options?: {
interruptFeedback?: string;
resources?: Array<Resource>;
},
) => void;
onCancel?: () => void;
onRemoveFeedback?: () => void;
}) {
const [message, setMessage] = useState("");
const [imeStatus, setImeStatus] = useState<"active" | "inactive">("inactive");
const [indent, setIndent] = useState(0);
const backgroundInvestigation = useSettingsStore(
(state) => state.general.enableBackgroundInvestigation,
);
const textareaRef = useRef<HTMLTextAreaElement>(null);
const containerRef = useRef<HTMLDivElement>(null);
const inputRef = useRef<MessageInputRef>(null);
const feedbackRef = useRef<HTMLDivElement>(null);
useEffect(() => {
if (feedback) {
setMessage("");
setTimeout(() => {
if (feedbackRef.current) {
setIndent(feedbackRef.current.offsetWidth);
}
}, 200);
}
setTimeout(() => {
textareaRef.current?.focus();
}, 0);
}, [feedback]);
const handleSendMessage = useCallback(() => {
if (responding) {
onCancel?.();
} else {
if (message.trim() === "") {
return;
}
if (onSend) {
onSend(message, {
interruptFeedback: feedback?.option.value,
});
setMessage("");
onRemoveFeedback?.();
}
}
}, [responding, onCancel, message, onSend, feedback, onRemoveFeedback]);
const handleKeyDown = useCallback(
(event: KeyboardEvent<HTMLTextAreaElement>) => {
const handleSendMessage = useCallback(
(message: string, resources: Array<Resource>) => {
console.log(message, resources);
if (responding) {
return;
}
if (
event.key === "Enter" &&
!event.shiftKey &&
!event.metaKey &&
!event.ctrlKey &&
imeStatus === "inactive"
) {
event.preventDefault();
handleSendMessage();
onCancel?.();
} else {
if (message.trim() === "") {
return;
}
if (onSend) {
onSend(message, {
interruptFeedback: feedback?.option.value,
resources,
});
onRemoveFeedback?.();
}
}
},
[responding, imeStatus, handleSendMessage],
[responding, onCancel, onSend, feedback, onRemoveFeedback],
);
return (
<div className={cn("bg-card relative rounded-[24px] border", className)}>
<div
className={cn(
"bg-card relative flex h-full w-full flex-col rounded-[24px] border",
className,
)}
ref={containerRef}
>
<div className="w-full">
<AnimatePresence>
{feedback && (
@@ -122,25 +98,10 @@ export function InputBox({
</motion.div>
)}
</AnimatePresence>
<textarea
ref={textareaRef}
className={cn(
"m-0 w-full resize-none border-none px-4 py-3 text-lg",
size === "large" ? "min-h-32" : "min-h-4",
)}
style={{ textIndent: feedback ? `${indent}px` : 0 }}
placeholder={
feedback
? `Describe how you ${feedback.option.text.toLocaleLowerCase()}?`
: "What can I do for you?"
}
value={message}
onCompositionStart={() => setImeStatus("active")}
onCompositionEnd={() => setImeStatus("inactive")}
onKeyDown={handleKeyDown}
onChange={(event) => {
setMessage(event.target.value);
}}
<MessageInput
className={cn("h-24 px-4 pt-3")}
ref={inputRef}
onEnter={handleSendMessage}
/>
</div>
<div className="flex items-center px-4 py-2">
@@ -181,7 +142,7 @@ export function InputBox({
variant="outline"
size="icon"
className={cn("h-10 w-10 rounded-full")}
onClick={handleSendMessage}
onClick={() => inputRef.current?.submit()}
>
{responding ? (
<div className="flex h-10 w-10 items-center justify-center">

View File

@@ -174,7 +174,14 @@ function MessageListItem({
>
<MessageBubble message={message}>
<div className="flex w-full flex-col">
<Markdown>{message?.content}</Markdown>
<Markdown
className={cn(
message.role === "user" &&
"prose-invert not-dark:text-secondary dark:text-inherit",
)}
>
{message?.content}
</Markdown>
</div>
</MessageBubble>
</div>
@@ -214,9 +221,8 @@ function MessageBubble({
return (
<div
className={cn(
`flex w-fit max-w-[85%] flex-col rounded-2xl px-4 py-3 shadow`,
message.role === "user" &&
"text-primary-foreground bg-brand rounded-ee-none",
`group flex w-fit max-w-[85%] flex-col rounded-2xl px-4 py-3 text-nowrap shadow`,
message.role === "user" && "bg-brand rounded-ee-none",
message.role === "assistant" && "bg-card rounded-es-none",
className,
)}

View File

@@ -15,7 +15,7 @@ import {
} from "~/components/ui/card";
import { fastForwardReplay } from "~/core/api";
import { useReplayMetadata } from "~/core/api/hooks";
import type { Option } from "~/core/messages";
import type { Option, Resource } from "~/core/messages";
import { useReplay } from "~/core/replay";
import { sendMessage, useMessageIds, useStore } from "~/core/store";
import { env } from "~/env";
@@ -36,7 +36,13 @@ export function MessagesBlock({ className }: { className?: string }) {
const abortControllerRef = useRef<AbortController | null>(null);
const [feedback, setFeedback] = useState<{ option: Option } | null>(null);
const handleSend = useCallback(
async (message: string, options?: { interruptFeedback?: string }) => {
async (
message: string,
options?: {
interruptFeedback?: string;
resources?: Array<Resource>;
},
) => {
const abortController = new AbortController();
abortControllerRef.current = abortController;
try {
@@ -45,6 +51,7 @@ export function MessagesBlock({ className }: { className?: string }) {
{
interruptFeedback:
options?.interruptFeedback ?? feedback?.option.value,
resources: options?.resources,
},
{
abortSignal: abortController.signal,

View File

@@ -276,8 +276,8 @@ function CrawlToolCall({ toolCall }: { toolCall: ToolCallRuntime }) {
}
function PythonToolCall({ toolCall }: { toolCall: ToolCallRuntime }) {
const code = useMemo<string>(() => {
return (toolCall.args as { code: string }).code;
const code = useMemo<string | undefined>(() => {
return (toolCall.args as { code?: string }).code;
}, [toolCall.args]);
const { resolvedTheme } = useTheme();
return (
@@ -302,7 +302,7 @@ function PythonToolCall({ toolCall }: { toolCall: ToolCallRuntime }) {
boxShadow: "none",
}}
>
{code.trim()}
{code?.trim() ?? ""}
</SyntaxHighlighter>
</div>
</div>

View File

@@ -53,10 +53,7 @@ export function ResearchReportBlock({
// }, [isCompleted]);
return (
<div
ref={contentRef}
className={cn("relative flex flex-col pt-4 pb-8", className)}
>
<div ref={contentRef} className={cn("w-full pt-4 pb-8", className)}>
{!isReplay && isCompleted && editing ? (
<ReportEditor
content={message?.content}

View File

@@ -37,7 +37,7 @@ export const Link = ({
}, [credibleLinks, href, responding, checkLinkCredibility]);
return (
<span className="flex items-center gap-1.5">
<span className="inline-flex items-center gap-1.5">
<a href={href} target="_blank" rel="noopener noreferrer">
{children}
</a>

View File

@@ -0,0 +1,184 @@
// Copyright (c) 2025 Bytedance Ltd. and/or its affiliates
// SPDX-License-Identifier: MIT
"use client";
import Mention from "@tiptap/extension-mention";
import { Editor, Extension, type Content } from "@tiptap/react";
import {
EditorContent,
type EditorInstance,
EditorRoot,
type JSONContent,
StarterKit,
Placeholder,
} from "novel";
import { Markdown } from "tiptap-markdown";
import { useDebouncedCallback } from "use-debounce";
import { cx } from "class-variance-authority";
import "~/styles/prosemirror.css";
import { resourceSuggestion } from "./resource-suggestion";
import React, { forwardRef, useMemo, useRef } from "react";
import type { Resource } from "~/core/messages";
import { useRAGProvider } from "~/core/api/hooks";
export interface MessageInputRef {
focus: () => void;
submit: () => void;
}
export interface MessageInputProps {
className?: string;
placeholder?: string;
onChange?: (markdown: string) => void;
onEnter?: (message: string, resources: Array<Resource>) => void;
}
function formatMessage(content: JSONContent) {
if (content.content) {
const output: {
text: string;
resources: Array<Resource>;
} = {
text: "",
resources: [],
};
for (const node of content.content) {
const { text, resources } = formatMessage(node);
output.text += text;
output.resources.push(...resources);
}
return output;
} else {
return formatItem(content);
}
}
function formatItem(item: JSONContent): {
text: string;
resources: Array<Resource>;
} {
if (item.type === "text") {
return { text: item.text ?? "", resources: [] };
}
if (item.type === "mention") {
return {
text: `[${item.attrs?.label}](${item.attrs?.id})`,
resources: [
{ uri: item.attrs?.id ?? "", title: item.attrs?.label ?? "" },
],
};
}
return { text: "", resources: [] };
}
const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
({ className, onChange, onEnter }: MessageInputProps, ref) => {
const editorRef = useRef<Editor>(null);
const debouncedUpdates = useDebouncedCallback(
async (editor: EditorInstance) => {
if (onChange) {
const markdown = editor.storage.markdown.getMarkdown();
onChange(markdown);
}
},
200,
);
React.useImperativeHandle(ref, () => ({
focus: () => {
editorRef.current?.view.focus();
},
submit: () => {
if (onEnter) {
const { text, resources } = formatMessage(
editorRef.current?.getJSON() ?? [],
);
onEnter(text, resources);
}
},
}));
const { provider, loading } = useRAGProvider();
const extensions = useMemo(() => {
const extensions = [
StarterKit,
Markdown.configure({
html: true,
tightLists: true,
tightListClass: "tight",
bulletListMarker: "-",
linkify: false,
breaks: false,
transformPastedText: false,
transformCopiedText: false,
}),
Placeholder.configure({
placeholder: provider
? "What can I do for you? \nYou may refer to RAG resources by using @."
: "What can I do for you?",
emptyEditorClass: "placeholder",
}),
Extension.create({
name: "keyboardHandler",
addKeyboardShortcuts() {
return {
Enter: () => {
if (onEnter) {
const { text, resources } = formatMessage(
this.editor.getJSON() ?? [],
);
onEnter(text, resources);
}
return this.editor.commands.clearContent();
},
};
},
}),
];
if (provider) {
extensions.push(
Mention.configure({
HTMLAttributes: {
class: "mention",
},
suggestion: resourceSuggestion,
}) as Extension,
);
}
return extensions;
}, [onEnter, provider]);
if (loading) {
return null;
}
return (
<div className={className}>
<EditorRoot>
<EditorContent
immediatelyRender={false}
extensions={extensions}
className="border-muted h-full w-full overflow-auto"
editorProps={{
attributes: {
class:
"prose prose-base dark:prose-invert prose-headings:font-title font-default focus:outline-none max-w-full",
},
}}
onCreate={({ editor }) => {
editorRef.current = editor;
}}
onUpdate={({ editor }) => {
debouncedUpdates(editor);
}}
></EditorContent>
</EditorRoot>
</div>
);
},
);
export default MessageInput;

View File

@@ -0,0 +1,84 @@
import { forwardRef, useEffect, useImperativeHandle, useState } from "react";
import type { Resource } from "~/core/messages";
import { cn } from "~/lib/utils";
export interface ResourceMentionsProps {
items: Array<Resource>;
command: (item: { id: string; label: string }) => void;
}
export const ResourceMentions = forwardRef<
{ onKeyDown: (args: { event: KeyboardEvent }) => boolean },
ResourceMentionsProps
>((props, ref) => {
const [selectedIndex, setSelectedIndex] = useState(0);
const selectItem = (index: number) => {
const item = props.items[index];
if (item) {
props.command({ id: item.uri, label: item.title });
}
};
const upHandler = () => {
setSelectedIndex(
(selectedIndex + props.items.length - 1) % props.items.length,
);
};
const downHandler = () => {
setSelectedIndex((selectedIndex + 1) % props.items.length);
};
const enterHandler = () => {
selectItem(selectedIndex);
};
useEffect(() => setSelectedIndex(0), [props.items]);
useImperativeHandle(ref, () => ({
onKeyDown: ({ event }) => {
if (event.key === "ArrowUp") {
upHandler();
return true;
}
if (event.key === "ArrowDown") {
downHandler();
return true;
}
if (event.key === "Enter") {
enterHandler();
return true;
}
return false;
},
}));
return (
<div className="bg-card border-var(--border) relative flex flex-col gap-1 overflow-auto rounded-md border p-2 shadow">
{props.items.length ? (
props.items.map((item, index) => (
<button
className={cn(
"focus-visible:ring-ring hover:bg-accent hover:text-accent-foreground inline-flex h-9 w-full items-center justify-start gap-2 rounded-md px-4 py-2 text-sm whitespace-nowrap transition-colors focus-visible:ring-1 focus-visible:outline-none disabled:pointer-events-none disabled:opacity-50 [&_svg]:pointer-events-none [&_svg]:size-4 [&_svg]:shrink-0",
selectedIndex === index &&
"bg-secondary text-secondary-foreground",
)}
key={index}
onClick={() => selectItem(index)}
>
{item.title}
</button>
))
) : (
<div className="items-center justify-center text-gray-500">
No result
</div>
)}
</div>
);
});

View File

@@ -0,0 +1,83 @@
import type { MentionOptions } from "@tiptap/extension-mention";
import { ReactRenderer } from "@tiptap/react";
import {
ResourceMentions,
type ResourceMentionsProps,
} from "./resource-mentions";
import type { Instance, Props } from "tippy.js";
import tippy from "tippy.js";
import { resolveServiceURL } from "~/core/api/resolve-service-url";
import type { Resource } from "~/core/messages";
export const resourceSuggestion: MentionOptions["suggestion"] = {
items: ({ query }) => {
return fetch(resolveServiceURL(`rag/resources?query=${query}`), {
method: "GET",
})
.then((res) => res.json())
.then((res) => {
return res.resources as Array<Resource>;
})
.catch((err) => {
return [];
});
},
render: () => {
let reactRenderer: ReactRenderer<
{ onKeyDown: (args: { event: KeyboardEvent }) => boolean },
ResourceMentionsProps
>;
let popup: Instance<Props>[] | null = null;
return {
onStart: (props) => {
if (!props.clientRect) {
return;
}
reactRenderer = new ReactRenderer(ResourceMentions, {
props,
editor: props.editor,
});
popup = tippy("body", {
getReferenceClientRect: props.clientRect as any,
appendTo: () => document.body,
content: reactRenderer.element,
showOnCreate: true,
interactive: true,
trigger: "manual",
placement: "top-start",
});
},
onUpdate(props) {
reactRenderer.updateProps(props);
if (!props.clientRect) {
return;
}
popup?.[0]?.setProps({
getReferenceClientRect: props.clientRect as any,
});
},
onKeyDown(props) {
if (props.event.key === "Escape") {
popup?.[0]?.hide();
return true;
}
return reactRenderer.ref?.onKeyDown(props) ?? false;
},
onExit() {
popup?.[0]?.destroy();
reactRenderer.destroy();
},
};
},
};

View File

@@ -66,17 +66,6 @@ const ReportEditor = ({ content, onMarkdownChange }: ReportEditorProps) => {
const debouncedUpdates = useDebouncedCallback(
async (editor: EditorInstance) => {
// const json = editor.getJSON();
// // setCharsCount(editor.storage.characterCount.words());
// window.localStorage.setItem(
// "html-content",
// highlightCodeblocks(editor.getHTML()),
// );
// window.localStorage.setItem("novel-content", JSON.stringify(json));
// window.localStorage.setItem(
// "markdown",
// editor.storage.markdown.getMarkdown(),
// );
if (onMarkdownChange) {
const markdown = editor.storage.markdown.getMarkdown();
onMarkdownChange(markdown);
@@ -86,12 +75,6 @@ const ReportEditor = ({ content, onMarkdownChange }: ReportEditorProps) => {
500,
);
// useEffect(() => {
// const content = window.localStorage.getItem("novel-content");
// if (content) setInitialContent(JSON.parse(content));
// else setInitialContent(defaultEditorContent);
// }, []);
if (!initialContent) return null;
return (

View File

@@ -4,6 +4,7 @@
import { env } from "~/env";
import type { MCPServerMetadata } from "../mcp";
import type { Resource } from "../messages";
import { extractReplayIdFromSearchParams } from "../replay/get-replay-id";
import { fetchStream } from "../sse";
import { sleep } from "../utils";
@@ -15,6 +16,7 @@ export async function* chatStream(
userMessage: string,
params: {
thread_id: string;
resources?: Array<Resource>;
auto_accepted_plan: boolean;
max_plan_iterations: number;
max_step_num: number;

View File

@@ -3,9 +3,12 @@
import { useEffect, useRef, useState } from "react";
import { env } from "~/env";
import { useReplay } from "../replay";
import { fetchReplayTitle } from "./chat";
import { getRAGConfig } from "./rag";
export function useReplayMetadata() {
const { isReplay } = useReplay();
@@ -39,3 +42,26 @@ export function useReplayMetadata() {
}, [isLoading, isReplay, title]);
return { title, isLoading, hasError: error };
}
export function useRAGProvider() {
const [loading, setLoading] = useState(true);
const [provider, setProvider] = useState<string | null>(null);
useEffect(() => {
if (env.NEXT_PUBLIC_STATIC_WEBSITE_ONLY) {
setLoading(false);
return;
}
getRAGConfig()
.then(setProvider)
.catch((e) => {
setProvider(null);
console.error("Failed to get RAG provider", e);
})
.finally(() => {
setLoading(false);
});
}, []);
return { provider, loading };
}

24
web/src/core/api/rag.ts Normal file
View File

@@ -0,0 +1,24 @@
import type { Resource } from "../messages";
import { resolveServiceURL } from "./resolve-service-url";
export function queryRAGResources(query: string) {
return fetch(resolveServiceURL(`rag/resources?query=${query}`), {
method: "GET",
})
.then((res) => res.json())
.then((res) => {
return res.resources as Array<Resource>;
})
.catch((err) => {
return [];
});
}
export function getRAGConfig() {
return fetch(resolveServiceURL(`rag/config`), {
method: "GET",
})
.then((res) => res.json())
.then((res) => res.provider);
}

View File

@@ -21,6 +21,7 @@ export interface Message {
options?: Option[];
finishReason?: "stop" | "interrupt" | "tool_calls";
interruptFeedback?: string;
resources?: Array<Resource>;
}
export interface Option {
@@ -35,3 +36,8 @@ export interface ToolCallRuntime {
argsChunks?: string[];
result?: string;
}
export interface Resource {
uri: string;
title: string;
}

View File

@@ -7,7 +7,7 @@ import { create } from "zustand";
import { useShallow } from "zustand/react/shallow";
import { chatStream, generatePodcast } from "../api";
import type { Message } from "../messages";
import type { Message, Resource } from "../messages";
import { mergeMessage } from "../messages";
import { parseJSON } from "../utils";
@@ -78,8 +78,10 @@ export async function sendMessage(
content?: string,
{
interruptFeedback,
resources,
}: {
interruptFeedback?: string;
resources?: Array<Resource>;
} = {},
options: { abortSignal?: AbortSignal } = {},
) {
@@ -90,6 +92,7 @@ export async function sendMessage(
role: "user",
content: content,
contentChunks: [content],
resources,
});
}
@@ -99,6 +102,7 @@ export async function sendMessage(
{
thread_id: THREAD_ID,
interrupt_feedback: interruptFeedback,
resources,
auto_accepted_plan: settings.autoAcceptedPlan,
enable_background_investigation:
settings.enableBackgroundInvestigation ?? true,

View File

@@ -187,7 +187,7 @@
--sidebar-accent-foreground: oklch(0.985 0 0);
--sidebar-border: oklch(1 0 0 / 10%);
--sidebar-ring: oklch(0.556 0 0);
--brand: #5494f3;
--brand: rgb(17, 103, 234);
--novel-highlight-default: #000000;
--novel-highlight-purple: #3f2c4b;

View File

@@ -1,7 +1,9 @@
@import "./globals.css";
.prose {
color: inherit;
max-width: inherit;
}
.prose .placeholder {
opacity: 0.65;
}
.ProseMirror {
@@ -15,6 +17,7 @@
pointer-events: none;
height: 0;
}
.ProseMirror p.is-empty::before {
content: attr(data-placeholder);
float: left;
@@ -23,6 +26,14 @@
height: 0;
}
.ProseMirror .mention {
background-color: var(--purple-light);
border-radius: 0.4rem;
box-decoration-break: clone;
color: var(--brand);
padding: 0.1rem 0.3rem;
}
/* Custom image styles */
.ProseMirror img {