From b17c087174cc5999392fe6160ba2fe3692acefa1 Mon Sep 17 00:00:00 2001 From: JeffJiang Date: Thu, 5 Mar 2026 11:16:34 +0800 Subject: [PATCH] Implement optimistic UI for file uploads and enhance message handling (#967) * feat(upload): implement optimistic UI for file uploads and enhance message handling * feat(middleware): enhance file handling by collecting historical uploads from directory * feat(thread-title): update page title handling for new threads and improve loading state * feat(uploads-middleware): enhance file extraction by verifying file existence in uploads directory * feat(thread-stream): update file path reference to use virtual_path for uploads * feat(tests): add core behaviour tests for UploadsMiddleware * feat(tests): remove unused pytest import from test_uploads_middleware_core_logic.py * feat: enhance file upload handling and localization support - Update UploadsMiddleware to validate filenames more robustly. - Modify MessageListItem to parse uploaded files from raw content for backward compatibility. - Add localization for uploading messages in English and Chinese. - Introduce parseUploadedFiles utility to extract uploaded files from message content. --- .../agents/middlewares/uploads_middleware.py | 211 +++++------ .../test_uploads_middleware_core_logic.py | 334 ++++++++++++++++++ .../workspace/messages/message-list-item.tsx | 152 ++++++-- .../src/components/workspace/thread-title.tsx | 5 +- frontend/src/core/i18n/locales/en-US.ts | 11 +- frontend/src/core/i18n/locales/types.ts | 6 + frontend/src/core/i18n/locales/zh-CN.ts | 5 + frontend/src/core/messages/utils.ts | 37 +- frontend/src/core/threads/hooks.ts | 287 ++++++++++----- 9 files changed, 790 insertions(+), 258 deletions(-) create mode 100644 backend/tests/test_uploads_middleware_core_logic.py diff --git a/backend/src/agents/middlewares/uploads_middleware.py b/backend/src/agents/middlewares/uploads_middleware.py index 78402a0..5fd01f0 100644 --- a/backend/src/agents/middlewares/uploads_middleware.py +++ b/backend/src/agents/middlewares/uploads_middleware.py @@ -1,6 +1,6 @@ """Middleware to inject uploaded files information into agent context.""" -import re +import logging from pathlib import Path from typing import NotRequired, override @@ -11,6 +11,8 @@ from langgraph.runtime import Runtime from src.config.paths import Paths, get_paths +logger = logging.getLogger(__name__) + class UploadsMiddlewareState(AgentState): """State schema for uploads middleware.""" @@ -21,8 +23,9 @@ class UploadsMiddlewareState(AgentState): class UploadsMiddleware(AgentMiddleware[UploadsMiddlewareState]): """Middleware to inject uploaded files information into the agent context. - This middleware lists all files in the thread's uploads directory and - adds a system message with the file list before the agent processes the request. + Reads file metadata from the current message's additional_kwargs.files + (set by the frontend after upload) and prepends an block + to the last human message so the model knows which files are available. """ state_schema = UploadsMiddlewareState @@ -36,111 +39,91 @@ class UploadsMiddleware(AgentMiddleware[UploadsMiddlewareState]): super().__init__() self._paths = Paths(base_dir) if base_dir else get_paths() - def _get_uploads_dir(self, thread_id: str) -> Path: - """Get the uploads directory for a thread. - - Args: - thread_id: The thread ID. - - Returns: - Path to the uploads directory. - """ - return self._paths.sandbox_uploads_dir(thread_id) - - def _list_newly_uploaded_files(self, thread_id: str, last_message_files: set[str]) -> list[dict]: - """List only newly uploaded files that weren't in the last message. - - Args: - thread_id: The thread ID. - last_message_files: Set of filenames that were already shown in previous messages. - - Returns: - List of new file information dictionaries. - """ - uploads_dir = self._get_uploads_dir(thread_id) - - if not uploads_dir.exists(): - return [] - - files = [] - for file_path in sorted(uploads_dir.iterdir()): - if file_path.is_file() and file_path.name not in last_message_files: - stat = file_path.stat() - files.append( - { - "filename": file_path.name, - "size": stat.st_size, - "path": f"/mnt/user-data/uploads/{file_path.name}", - "extension": file_path.suffix, - } - ) - - return files - - def _create_files_message(self, files: list[dict]) -> str: + def _create_files_message(self, new_files: list[dict], historical_files: list[dict]) -> str: """Create a formatted message listing uploaded files. Args: - files: List of file information dictionaries. + new_files: Files uploaded in the current message. + historical_files: Files uploaded in previous messages. Returns: - Formatted string listing the files. + Formatted string inside tags. """ - if not files: - return "\nNo files have been uploaded yet.\n" + lines = [""] - lines = ["", "The following files have been uploaded and are available for use:", ""] - - for file in files: + lines.append("The following files were uploaded in this message:") + lines.append("") + for file in new_files: size_kb = file["size"] / 1024 - if size_kb < 1024: - size_str = f"{size_kb:.1f} KB" - else: - size_str = f"{size_kb / 1024:.1f} MB" - + size_str = f"{size_kb:.1f} KB" if size_kb < 1024 else f"{size_kb / 1024:.1f} MB" lines.append(f"- {file['filename']} ({size_str})") lines.append(f" Path: {file['path']}") lines.append("") + if historical_files: + lines.append("The following files were uploaded in previous messages and are still available:") + lines.append("") + for file in historical_files: + size_kb = file["size"] / 1024 + size_str = f"{size_kb:.1f} KB" if size_kb < 1024 else f"{size_kb / 1024:.1f} MB" + lines.append(f"- {file['filename']} ({size_str})") + lines.append(f" Path: {file['path']}") + lines.append("") + lines.append("You can read these files using the `read_file` tool with the paths shown above.") lines.append("") return "\n".join(lines) - def _extract_files_from_message(self, content: str) -> set[str]: - """Extract filenames from uploaded_files tag in message content. + def _files_from_kwargs(self, message: HumanMessage, uploads_dir: Path | None = None) -> list[dict] | None: + """Extract file info from message additional_kwargs.files. + + The frontend sends uploaded file metadata in additional_kwargs.files + after a successful upload. Each entry has: filename, size (bytes), + path (virtual path), status. Args: - content: Message content that may contain tag. + message: The human message to inspect. + uploads_dir: Physical uploads directory used to verify file existence. + When provided, entries whose files no longer exist are skipped. Returns: - Set of filenames mentioned in the tag. + List of file dicts with virtual paths, or None if the field is absent or empty. """ - # Match ... tag - match = re.search(r"([\s\S]*?)", content) - if not match: - return set() + kwargs_files = (message.additional_kwargs or {}).get("files") + if not isinstance(kwargs_files, list) or not kwargs_files: + return None - files_content = match.group(1) - - # Extract filenames from lines like "- filename.ext (size)" - # Need to capture everything before the opening parenthesis, including spaces - filenames = set() - for line in files_content.split("\n"): - # Match pattern: - filename with spaces.ext (size) - # Changed from [^\s(]+ to [^(]+ to allow spaces in filename - file_match = re.match(r"^-\s+(.+?)\s*\(", line.strip()) - if file_match: - filenames.add(file_match.group(1).strip()) - - return filenames + files = [] + for f in kwargs_files: + if not isinstance(f, dict): + continue + filename = f.get("filename") or "" + if not filename or Path(filename).name != filename: + continue + if uploads_dir is not None and not (uploads_dir / filename).is_file(): + continue + files.append( + { + "filename": filename, + "size": int(f.get("size") or 0), + "path": f"/mnt/user-data/uploads/{filename}", + "extension": Path(filename).suffix, + } + ) + return files if files else None @override def before_agent(self, state: UploadsMiddlewareState, runtime: Runtime) -> dict | None: """Inject uploaded files information before agent execution. - Only injects files that weren't already shown in previous messages. - Prepends file info to the last human message content. + New files come from the current message's additional_kwargs.files. + Historical files are scanned from the thread's uploads directory, + excluding the new ones. + + Prepends context to the last human message content. + The original additional_kwargs (including files metadata) is preserved + on the updated message so the frontend can read it from the stream. Args: state: Current agent state. @@ -149,72 +132,70 @@ class UploadsMiddleware(AgentMiddleware[UploadsMiddlewareState]): Returns: State updates including uploaded files list. """ - import logging - - logger = logging.getLogger(__name__) - - thread_id = runtime.context.get("thread_id") - if thread_id is None: - return None - messages = list(state.get("messages", [])) if not messages: return None - # Track all filenames that have been shown in previous messages (EXCEPT the last one) - shown_files: set[str] = set() - for msg in messages[:-1]: # Scan all messages except the last one - if isinstance(msg, HumanMessage): - content = msg.content if isinstance(msg.content, str) else "" - extracted = self._extract_files_from_message(content) - shown_files.update(extracted) - if extracted: - logger.info(f"Found previously shown files: {extracted}") - - logger.info(f"Total shown files from history: {shown_files}") - - # List only newly uploaded files - files = self._list_newly_uploaded_files(thread_id, shown_files) - logger.info(f"Newly uploaded files to inject: {[f['filename'] for f in files]}") - - if not files: - return None - - # Find the last human message and prepend file info to it last_message_index = len(messages) - 1 last_message = messages[last_message_index] if not isinstance(last_message, HumanMessage): return None + # Resolve uploads directory for existence checks + thread_id = runtime.context.get("thread_id") + uploads_dir = self._paths.sandbox_uploads_dir(thread_id) if thread_id else None + + # Get newly uploaded files from the current message's additional_kwargs.files + new_files = self._files_from_kwargs(last_message, uploads_dir) or [] + + # Collect historical files from the uploads directory (all except the new ones) + new_filenames = {f["filename"] for f in new_files} + historical_files: list[dict] = [] + if uploads_dir and uploads_dir.exists(): + for file_path in sorted(uploads_dir.iterdir()): + if file_path.is_file() and file_path.name not in new_filenames: + stat = file_path.stat() + historical_files.append( + { + "filename": file_path.name, + "size": stat.st_size, + "path": f"/mnt/user-data/uploads/{file_path.name}", + "extension": file_path.suffix, + } + ) + + if not new_files and not historical_files: + return None + + logger.debug(f"New files: {[f['filename'] for f in new_files]}, historical: {[f['filename'] for f in historical_files]}") + # Create files message and prepend to the last human message content - files_message = self._create_files_message(files) + files_message = self._create_files_message(new_files, historical_files) # Extract original content - handle both string and list formats original_content = "" if isinstance(last_message.content, str): original_content = last_message.content elif isinstance(last_message.content, list): - # Content is a list of content blocks (e.g., [{"type": "text", "text": "..."}]) text_parts = [] for block in last_message.content: if isinstance(block, dict) and block.get("type") == "text": text_parts.append(block.get("text", "")) original_content = "\n".join(text_parts) - logger.info(f"Original message content: {original_content[:100] if original_content else '(empty)'}") - - # Create new message with combined content + # Create new message with combined content. + # Preserve additional_kwargs (including files metadata) so the frontend + # can read structured file info from the streamed message. updated_message = HumanMessage( content=f"{files_message}\n\n{original_content}", id=last_message.id, additional_kwargs=last_message.additional_kwargs, ) - # Replace the last message messages[last_message_index] = updated_message return { - "uploaded_files": files, + "uploaded_files": new_files, "messages": messages, } diff --git a/backend/tests/test_uploads_middleware_core_logic.py b/backend/tests/test_uploads_middleware_core_logic.py new file mode 100644 index 0000000..864424a --- /dev/null +++ b/backend/tests/test_uploads_middleware_core_logic.py @@ -0,0 +1,334 @@ +"""Core behaviour tests for UploadsMiddleware. + +Covers: +- _files_from_kwargs: parsing, validation, existence check, virtual-path construction +- _create_files_message: output format with new-only and new+historical files +- before_agent: full injection pipeline (string & list content, preserved + additional_kwargs, historical files from uploads dir, edge-cases) +""" + +from pathlib import Path +from unittest.mock import MagicMock + +from langchain_core.messages import AIMessage, HumanMessage + +from src.agents.middlewares.uploads_middleware import UploadsMiddleware +from src.config.paths import Paths + +THREAD_ID = "thread-abc123" + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _middleware(tmp_path: Path) -> UploadsMiddleware: + return UploadsMiddleware(base_dir=str(tmp_path)) + + +def _runtime(thread_id: str | None = THREAD_ID) -> MagicMock: + rt = MagicMock() + rt.context = {"thread_id": thread_id} + return rt + + +def _uploads_dir(tmp_path: Path, thread_id: str = THREAD_ID) -> Path: + d = Paths(str(tmp_path)).sandbox_uploads_dir(thread_id) + d.mkdir(parents=True, exist_ok=True) + return d + + +def _human(content, files=None, **extra_kwargs): + additional_kwargs = dict(extra_kwargs) + if files is not None: + additional_kwargs["files"] = files + return HumanMessage(content=content, additional_kwargs=additional_kwargs) + + +# --------------------------------------------------------------------------- +# _files_from_kwargs +# --------------------------------------------------------------------------- + + +class TestFilesFromKwargs: + def test_returns_none_when_files_field_absent(self, tmp_path): + mw = _middleware(tmp_path) + msg = HumanMessage(content="hello") + assert mw._files_from_kwargs(msg) is None + + def test_returns_none_for_empty_files_list(self, tmp_path): + mw = _middleware(tmp_path) + msg = _human("hello", files=[]) + assert mw._files_from_kwargs(msg) is None + + def test_returns_none_for_non_list_files(self, tmp_path): + mw = _middleware(tmp_path) + msg = _human("hello", files="not-a-list") + assert mw._files_from_kwargs(msg) is None + + def test_skips_non_dict_entries(self, tmp_path): + mw = _middleware(tmp_path) + msg = _human("hi", files=["bad", 42, None]) + assert mw._files_from_kwargs(msg) is None + + def test_skips_entries_with_empty_filename(self, tmp_path): + mw = _middleware(tmp_path) + msg = _human("hi", files=[{"filename": "", "size": 100, "path": "/mnt/user-data/uploads/x"}]) + assert mw._files_from_kwargs(msg) is None + + def test_always_uses_virtual_path(self, tmp_path): + """path field must be /mnt/user-data/uploads/ regardless of what the frontend sent.""" + mw = _middleware(tmp_path) + msg = _human( + "hi", + files=[{"filename": "report.pdf", "size": 1024, "path": "/some/arbitrary/path/report.pdf"}], + ) + result = mw._files_from_kwargs(msg) + assert result is not None + assert result[0]["path"] == "/mnt/user-data/uploads/report.pdf" + + def test_skips_file_that_does_not_exist_on_disk(self, tmp_path): + mw = _middleware(tmp_path) + uploads_dir = _uploads_dir(tmp_path) + # file is NOT written to disk + msg = _human("hi", files=[{"filename": "missing.txt", "size": 50, "path": "/mnt/user-data/uploads/missing.txt"}]) + assert mw._files_from_kwargs(msg, uploads_dir) is None + + def test_accepts_file_that_exists_on_disk(self, tmp_path): + mw = _middleware(tmp_path) + uploads_dir = _uploads_dir(tmp_path) + (uploads_dir / "data.csv").write_text("a,b,c") + msg = _human("hi", files=[{"filename": "data.csv", "size": 5, "path": "/mnt/user-data/uploads/data.csv"}]) + result = mw._files_from_kwargs(msg, uploads_dir) + assert result is not None + assert len(result) == 1 + assert result[0]["filename"] == "data.csv" + assert result[0]["path"] == "/mnt/user-data/uploads/data.csv" + + def test_skips_nonexistent_but_accepts_existing_in_mixed_list(self, tmp_path): + mw = _middleware(tmp_path) + uploads_dir = _uploads_dir(tmp_path) + (uploads_dir / "present.txt").write_text("here") + msg = _human( + "hi", + files=[ + {"filename": "present.txt", "size": 4, "path": "/mnt/user-data/uploads/present.txt"}, + {"filename": "gone.txt", "size": 4, "path": "/mnt/user-data/uploads/gone.txt"}, + ], + ) + result = mw._files_from_kwargs(msg, uploads_dir) + assert result is not None + assert [f["filename"] for f in result] == ["present.txt"] + + def test_no_existence_check_when_uploads_dir_is_none(self, tmp_path): + """Without an uploads_dir argument the existence check is skipped entirely.""" + mw = _middleware(tmp_path) + msg = _human("hi", files=[{"filename": "phantom.txt", "size": 10, "path": "/mnt/user-data/uploads/phantom.txt"}]) + result = mw._files_from_kwargs(msg, uploads_dir=None) + assert result is not None + assert result[0]["filename"] == "phantom.txt" + + def test_size_is_coerced_to_int(self, tmp_path): + mw = _middleware(tmp_path) + msg = _human("hi", files=[{"filename": "f.txt", "size": "2048", "path": "/mnt/user-data/uploads/f.txt"}]) + result = mw._files_from_kwargs(msg) + assert result is not None + assert result[0]["size"] == 2048 + + def test_missing_size_defaults_to_zero(self, tmp_path): + mw = _middleware(tmp_path) + msg = _human("hi", files=[{"filename": "f.txt", "path": "/mnt/user-data/uploads/f.txt"}]) + result = mw._files_from_kwargs(msg) + assert result is not None + assert result[0]["size"] == 0 + + +# --------------------------------------------------------------------------- +# _create_files_message +# --------------------------------------------------------------------------- + + +class TestCreateFilesMessage: + def _new_file(self, filename="notes.txt", size=1024): + return {"filename": filename, "size": size, "path": f"/mnt/user-data/uploads/{filename}"} + + def test_new_files_section_always_present(self, tmp_path): + mw = _middleware(tmp_path) + msg = mw._create_files_message([self._new_file()], []) + assert "" in msg + assert "" in msg + assert "uploaded in this message" in msg + assert "notes.txt" in msg + assert "/mnt/user-data/uploads/notes.txt" in msg + + def test_historical_section_present_only_when_non_empty(self, tmp_path): + mw = _middleware(tmp_path) + + msg_no_hist = mw._create_files_message([self._new_file()], []) + assert "previous messages" not in msg_no_hist + + hist = self._new_file("old.txt") + msg_with_hist = mw._create_files_message([self._new_file()], [hist]) + assert "previous messages" in msg_with_hist + assert "old.txt" in msg_with_hist + + def test_size_formatting_kb(self, tmp_path): + mw = _middleware(tmp_path) + msg = mw._create_files_message([self._new_file(size=2048)], []) + assert "2.0 KB" in msg + + def test_size_formatting_mb(self, tmp_path): + mw = _middleware(tmp_path) + msg = mw._create_files_message([self._new_file(size=2 * 1024 * 1024)], []) + assert "2.0 MB" in msg + + def test_read_file_instruction_included(self, tmp_path): + mw = _middleware(tmp_path) + msg = mw._create_files_message([self._new_file()], []) + assert "read_file" in msg + + +# --------------------------------------------------------------------------- +# before_agent +# --------------------------------------------------------------------------- + + +class TestBeforeAgent: + def _state(self, *messages): + return {"messages": list(messages)} + + def test_returns_none_when_messages_empty(self, tmp_path): + mw = _middleware(tmp_path) + assert mw.before_agent({"messages": []}, _runtime()) is None + + def test_returns_none_when_last_message_is_not_human(self, tmp_path): + mw = _middleware(tmp_path) + state = self._state(HumanMessage(content="q"), AIMessage(content="a")) + assert mw.before_agent(state, _runtime()) is None + + def test_returns_none_when_no_files_in_kwargs(self, tmp_path): + mw = _middleware(tmp_path) + state = self._state(_human("plain message")) + assert mw.before_agent(state, _runtime()) is None + + def test_returns_none_when_all_files_missing_from_disk(self, tmp_path): + mw = _middleware(tmp_path) + _uploads_dir(tmp_path) # directory exists but is empty + msg = _human("hi", files=[{"filename": "ghost.txt", "size": 10, "path": "/mnt/user-data/uploads/ghost.txt"}]) + state = self._state(msg) + assert mw.before_agent(state, _runtime()) is None + + def test_injects_uploaded_files_tag_into_string_content(self, tmp_path): + mw = _middleware(tmp_path) + uploads_dir = _uploads_dir(tmp_path) + (uploads_dir / "report.pdf").write_bytes(b"pdf") + + msg = _human("please analyse", files=[{"filename": "report.pdf", "size": 3, "path": "/mnt/user-data/uploads/report.pdf"}]) + state = self._state(msg) + result = mw.before_agent(state, _runtime()) + + assert result is not None + updated_msg = result["messages"][-1] + assert isinstance(updated_msg.content, str) + assert "" in updated_msg.content + assert "report.pdf" in updated_msg.content + assert "please analyse" in updated_msg.content + + def test_injects_uploaded_files_tag_into_list_content(self, tmp_path): + mw = _middleware(tmp_path) + uploads_dir = _uploads_dir(tmp_path) + (uploads_dir / "data.csv").write_bytes(b"a,b") + + msg = _human( + [{"type": "text", "text": "analyse this"}], + files=[{"filename": "data.csv", "size": 3, "path": "/mnt/user-data/uploads/data.csv"}], + ) + state = self._state(msg) + result = mw.before_agent(state, _runtime()) + + assert result is not None + updated_msg = result["messages"][-1] + assert "" in updated_msg.content + assert "analyse this" in updated_msg.content + + def test_preserves_additional_kwargs_on_updated_message(self, tmp_path): + mw = _middleware(tmp_path) + uploads_dir = _uploads_dir(tmp_path) + (uploads_dir / "img.png").write_bytes(b"png") + + files_meta = [{"filename": "img.png", "size": 3, "path": "/mnt/user-data/uploads/img.png", "status": "uploaded"}] + msg = _human("check image", files=files_meta, element="task") + state = self._state(msg) + result = mw.before_agent(state, _runtime()) + + assert result is not None + updated_kwargs = result["messages"][-1].additional_kwargs + assert updated_kwargs.get("files") == files_meta + assert updated_kwargs.get("element") == "task" + + def test_uploaded_files_returned_in_state_update(self, tmp_path): + mw = _middleware(tmp_path) + uploads_dir = _uploads_dir(tmp_path) + (uploads_dir / "notes.txt").write_bytes(b"hello") + + msg = _human("review", files=[{"filename": "notes.txt", "size": 5, "path": "/mnt/user-data/uploads/notes.txt"}]) + result = mw.before_agent(self._state(msg), _runtime()) + + assert result is not None + assert result["uploaded_files"] == [ + { + "filename": "notes.txt", + "size": 5, + "path": "/mnt/user-data/uploads/notes.txt", + "extension": ".txt", + } + ] + + def test_historical_files_from_uploads_dir_excluding_new(self, tmp_path): + mw = _middleware(tmp_path) + uploads_dir = _uploads_dir(tmp_path) + (uploads_dir / "old.txt").write_bytes(b"old") + (uploads_dir / "new.txt").write_bytes(b"new") + + msg = _human("go", files=[{"filename": "new.txt", "size": 3, "path": "/mnt/user-data/uploads/new.txt"}]) + result = mw.before_agent(self._state(msg), _runtime()) + + assert result is not None + content = result["messages"][-1].content + assert "uploaded in this message" in content + assert "new.txt" in content + assert "previous messages" in content + assert "old.txt" in content + + def test_no_historical_section_when_upload_dir_is_empty(self, tmp_path): + mw = _middleware(tmp_path) + uploads_dir = _uploads_dir(tmp_path) + (uploads_dir / "only.txt").write_bytes(b"x") + + msg = _human("go", files=[{"filename": "only.txt", "size": 1, "path": "/mnt/user-data/uploads/only.txt"}]) + result = mw.before_agent(self._state(msg), _runtime()) + + content = result["messages"][-1].content + assert "previous messages" not in content + + def test_no_historical_scan_when_thread_id_is_none(self, tmp_path): + mw = _middleware(tmp_path) + msg = _human("go", files=[{"filename": "f.txt", "size": 1, "path": "/mnt/user-data/uploads/f.txt"}]) + # thread_id=None → _files_from_kwargs skips existence check, no dir scan + result = mw.before_agent(self._state(msg), _runtime(thread_id=None)) + # With no existence check, the file passes through and injection happens + assert result is not None + content = result["messages"][-1].content + assert "previous messages" not in content + + def test_message_id_preserved_on_updated_message(self, tmp_path): + mw = _middleware(tmp_path) + uploads_dir = _uploads_dir(tmp_path) + (uploads_dir / "f.txt").write_bytes(b"x") + + msg = _human("go", files=[{"filename": "f.txt", "size": 1, "path": "/mnt/user-data/uploads/f.txt"}]) + msg.id = "original-id-42" + result = mw.before_agent(self._state(msg), _runtime()) + + assert result["messages"][-1].id == "original-id-42" diff --git a/frontend/src/components/workspace/messages/message-list-item.tsx b/frontend/src/components/workspace/messages/message-list-item.tsx index 536e983..af137f2 100644 --- a/frontend/src/components/workspace/messages/message-list-item.tsx +++ b/frontend/src/components/workspace/messages/message-list-item.tsx @@ -1,22 +1,31 @@ import type { Message } from "@langchain/langgraph-sdk"; -import { FileIcon } from "lucide-react"; +import { FileIcon, Loader2Icon } from "lucide-react"; import { useParams } from "next/navigation"; import { memo, useMemo, type ImgHTMLAttributes } from "react"; import rehypeKatex from "rehype-katex"; +import { Loader } from "@/components/ai-elements/loader"; import { Message as AIElementMessage, MessageContent as AIElementMessageContent, MessageResponse as AIElementMessageResponse, MessageToolbar, } from "@/components/ai-elements/message"; +import { + Reasoning, + ReasoningContent, + ReasoningTrigger, +} from "@/components/ai-elements/reasoning"; +import { Task, TaskTrigger } from "@/components/ai-elements/task"; import { Badge } from "@/components/ui/badge"; import { resolveArtifactURL } from "@/core/artifacts/utils"; +import { useI18n } from "@/core/i18n/hooks"; import { extractContentFromMessage, extractReasoningContentFromMessage, parseUploadedFiles, - type UploadedFile, + stripUploadedFilesTag, + type FileInMessage, } from "@/core/messages/utils"; import { useRehypeSplitWordsIntoSpans } from "@/core/rehype"; import { humanMessagePlugins } from "@/core/streamdown"; @@ -121,37 +130,67 @@ function MessageContent_({ const rawContent = extractContentFromMessage(message); const reasoningContent = extractReasoningContentFromMessage(message); - const { contentToParse, uploadedFiles } = useMemo(() => { - if (!isLoading && reasoningContent && !rawContent) { - return { - contentToParse: reasoningContent, - uploadedFiles: [] as UploadedFile[], - }; + + const files = useMemo(() => { + const files = message.additional_kwargs?.files; + if (!Array.isArray(files) || files.length === 0) { + if (rawContent.includes("")) { + // If the content contains the tag, we return the parsed files from the content for backward compatibility. + return parseUploadedFiles(rawContent); + } + return null; } - if (isHuman && rawContent) { - const { files, cleanContent: contentWithoutFiles } = - parseUploadedFiles(rawContent); - return { contentToParse: contentWithoutFiles, uploadedFiles: files }; + return files as FileInMessage[]; + }, [message.additional_kwargs?.files, rawContent]); + + const contentToDisplay = useMemo(() => { + if (isHuman) { + return rawContent ? stripUploadedFilesTag(rawContent) : ""; } - return { - contentToParse: rawContent ?? "", - uploadedFiles: [] as UploadedFile[], - }; - }, [isLoading, rawContent, reasoningContent, isHuman]); + return rawContent ?? ""; + }, [rawContent, isHuman]); const filesList = - uploadedFiles.length > 0 && thread_id ? ( - + files && files.length > 0 && thread_id ? ( + ) : null; + // Uploading state: mock AI message shown while files upload + if (message.additional_kwargs?.element === "task") { + return ( + + + +
+ + {contentToDisplay} +
+ + + + ); + } + + // Reasoning-only AI message (no main response content yet) + if (!isHuman && reasoningContent && !rawContent) { + return ( + + + + {reasoningContent} + + + ); + } + if (isHuman) { - const messageResponse = contentToParse ? ( + const messageResponse = contentToDisplay ? ( - {contentToParse} + {contentToDisplay} ) : null; return ( @@ -170,7 +209,7 @@ function MessageContent_({ {filesList} {files.map((file, index) => ( - @@ -249,18 +297,48 @@ function UploadedFilesList({ } /** - * Single uploaded file card component + * Single file card that handles FileInMessage (supports uploading state) */ -function UploadedFileCard({ +function RichFileCard({ file, threadId, }: { - file: UploadedFile; + file: FileInMessage; threadId: string; }) { - if (!threadId) return null; - + const { t } = useI18n(); + const isUploading = file.status === "uploading"; const isImage = isImageFile(file.filename); + + if (isUploading) { + return ( +
+
+ + + {file.filename} + +
+
+ + {getFileTypeLabel(file.filename)} + + + {t.uploads.uploading} + +
+
+ ); + } + + if (!file.path) return null; + const fileUrl = resolveArtifactURL(file.path, threadId); if (isImage) { @@ -274,14 +352,14 @@ function UploadedFileCard({ {file.filename} ); } return ( -
+
{getFileTypeLabel(file.filename)} - {file.size} + + {formatBytes(file.size)} +
); diff --git a/frontend/src/components/workspace/thread-title.tsx b/frontend/src/components/workspace/thread-title.tsx index 77a5cb5..7e89761 100644 --- a/frontend/src/components/workspace/thread-title.tsx +++ b/frontend/src/components/workspace/thread-title.tsx @@ -4,6 +4,7 @@ import { useEffect } from "react"; import { useI18n } from "@/core/i18n/hooks"; import type { AgentThreadState } from "@/core/threads"; +import { useThreadChat } from "./chats"; import { FlipDisplay } from "./flip-display"; export function ThreadTitle({ @@ -15,8 +16,9 @@ export function ThreadTitle({ thread: BaseStream; }) { const { t } = useI18n(); + const { isNewThread } = useThreadChat(); useEffect(() => { - const pageTitle = !thread.values + const pageTitle = isNewThread ? t.pages.newChat : thread.values?.title && thread.values.title !== "Untitled" ? thread.values.title @@ -27,6 +29,7 @@ export function ThreadTitle({ document.title = `${pageTitle} - ${t.pages.appName}`; } }, [ + isNewThread, t.pages.newChat, t.pages.untitled, t.pages.appName, diff --git a/frontend/src/core/i18n/locales/en-US.ts b/frontend/src/core/i18n/locales/en-US.ts index dbb0008..ce1c90b 100644 --- a/frontend/src/core/i18n/locales/en-US.ts +++ b/frontend/src/core/i18n/locales/en-US.ts @@ -88,9 +88,11 @@ export const enUS: Translations = { reasoningEffortLow: "Low", reasoningEffortLowDescription: "Simple Logic Check + Shallow Deduction", reasoningEffortMedium: "Medium", - reasoningEffortMediumDescription: "Multi-layer Logic Analysis + Basic Verification", + reasoningEffortMediumDescription: + "Multi-layer Logic Analysis + Basic Verification", reasoningEffortHigh: "High", - reasoningEffortHighDescription: "Full-dimensional Logic Deduction + Multi-path Verification + Backward Check", + reasoningEffortHighDescription: + "Full-dimensional Logic Deduction + Multi-path Verification + Backward Check", searchModels: "Search models...", surpriseMe: "Surprise", surpriseMePrompt: "Surprise me", @@ -248,6 +250,11 @@ export const enUS: Translations = { }, // Subtasks + uploads: { + uploading: "Uploading...", + uploadingFiles: "Uploading files, please wait...", + }, + subtasks: { subtask: "Subtask", executing: (count: number) => diff --git a/frontend/src/core/i18n/locales/types.ts b/frontend/src/core/i18n/locales/types.ts index 639fa3b..06bb403 100644 --- a/frontend/src/core/i18n/locales/types.ts +++ b/frontend/src/core/i18n/locales/types.ts @@ -187,6 +187,12 @@ export interface Translations { skillInstallTooltip: string; }; + // Uploads + uploads: { + uploading: string; + uploadingFiles: string; + }; + // Subtasks subtasks: { subtask: string; diff --git a/frontend/src/core/i18n/locales/zh-CN.ts b/frontend/src/core/i18n/locales/zh-CN.ts index 630901e..693ea82 100644 --- a/frontend/src/core/i18n/locales/zh-CN.ts +++ b/frontend/src/core/i18n/locales/zh-CN.ts @@ -238,6 +238,11 @@ export const zhCN: Translations = { skillInstallTooltip: "安装技能并使其可在 DeerFlow 中使用", }, + uploads: { + uploading: "上传中...", + uploadingFiles: "文件上传中，请稍候...", + }, + subtasks: { subtask: "子任务", executing: (count: number) => diff --git a/frontend/src/core/messages/utils.ts b/frontend/src/core/messages/utils.ts index d73417d..c95ad49 100644 --- a/frontend/src/core/messages/utils.ts +++ b/frontend/src/core/messages/utils.ts @@ -263,57 +263,56 @@ export function findToolCallResult(toolCallId: string, messages: Message[]) { } /** - * Represents an uploaded file parsed from the tag + * Represents a file stored in message additional_kwargs.files. + * Used for optimistic UI (uploading state) and structured file metadata. */ -export interface UploadedFile { +export interface FileInMessage { filename: string; - size: string; - path: string; + size: number; // bytes + path?: string; // virtual path, may not be set during upload + status?: "uploading" | "uploaded"; } /** - * Result of parsing uploaded files from message content + * Strip tag from message content. + * Returns the content with the tag removed. */ -export interface ParsedUploadedFiles { - files: UploadedFile[]; - cleanContent: string; +export function stripUploadedFilesTag(content: string): string { + return content + .replace(/[\s\S]*?<\/uploaded_files>/g, "") + .trim(); } -/** - * Parse tag from message content and extract file information. - * Returns the list of uploaded files and the content with the tag removed. - */ -export function parseUploadedFiles(content: string): ParsedUploadedFiles { +export function parseUploadedFiles(content: string): FileInMessage[] { // Match ... tag const uploadedFilesRegex = /([\s\S]*?)<\/uploaded_files>/; // eslint-disable-next-line @typescript-eslint/prefer-regexp-exec const match = content.match(uploadedFilesRegex); if (!match) { - return { files: [], cleanContent: content }; + return []; } const uploadedFilesContent = match[1]; - const cleanContent = content.replace(uploadedFilesRegex, "").trim(); // Check if it's "No files have been uploaded yet." if (uploadedFilesContent?.includes("No files have been uploaded yet.")) { - return { files: [], cleanContent }; + return []; } // Parse file list // Format: - filename (size)\n Path: /path/to/file const fileRegex = /- ([^\n(]+)\s*\(([^)]+)\)\s*\n\s*Path:\s*([^\n]+)/g; - const files: UploadedFile[] = []; + const files: FileInMessage[] = []; let fileMatch; while ((fileMatch = fileRegex.exec(uploadedFilesContent ?? "")) !== null) { files.push({ filename: fileMatch[1].trim(), - size: fileMatch[2].trim(), + size: parseInt(fileMatch[2].trim(), 10) ?? 0, path: fileMatch[3].trim(), }); } - return { files, cleanContent }; + return files; } diff --git a/frontend/src/core/threads/hooks.ts b/frontend/src/core/threads/hooks.ts index b327aa0..73260ba 100644 --- a/frontend/src/core/threads/hooks.ts +++ b/frontend/src/core/threads/hooks.ts @@ -1,15 +1,18 @@ -import type { AIMessage } from "@langchain/langgraph-sdk"; +import type { AIMessage, Message } from "@langchain/langgraph-sdk"; import type { ThreadsClient } from "@langchain/langgraph-sdk/client"; import { useStream } from "@langchain/langgraph-sdk/react"; import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; -import { useCallback, useEffect, useState } from "react"; +import { useCallback, useEffect, useRef, useState } from "react"; import { toast } from "sonner"; import type { PromptInputMessage } from "@/components/ai-elements/prompt-input"; import { getAPIClient } from "../api"; +import { useI18n } from "../i18n/hooks"; +import type { FileInMessage } from "../messages/utils"; import type { LocalSettings } from "../settings"; import { useUpdateSubtask } from "../tasks/context"; +import type { UploadedFileInfo } from "../uploads"; import { uploadFiles } from "../uploads"; import type { AgentThread, AgentThreadState } from "./types"; @@ -36,11 +39,14 @@ export function useThreadStream({ onFinish, onToolEnd, }: ThreadStreamOptions) { + const { t } = useI18n(); const [_threadId, setThreadId] = useState(threadId ?? null); + const startedRef = useRef(false); useEffect(() => { if (_threadId && _threadId !== threadId) { setThreadId(threadId ?? null); + startedRef.current = false; // Reset for new thread } }, [threadId, _threadId]); @@ -54,7 +60,10 @@ export function useThreadStream({ fetchStateHistory: { limit: 1 }, onCreated(meta) { setThreadId(meta.thread_id); - onStart?.(meta.thread_id); + if (!startedRef.current) { + onStart?.(meta.thread_id); + startedRef.current = true; + } }, onLangChainEvent(event) { if (event.event === "on_tool_end") { @@ -85,6 +94,21 @@ export function useThreadStream({ }, }); + // Optimistic messages shown before the server stream responds + const [optimisticMessages, setOptimisticMessages] = useState([]); + // Track message count before sending so we know when server has responded + const prevMsgCountRef = useRef(thread.messages.length); + + // Clear optimistic when server messages arrive (count increases) + useEffect(() => { + if ( + optimisticMessages.length > 0 && + thread.messages.length > prevMsgCountRef.current + ) { + setOptimisticMessages([]); + } + }, [thread.messages.length, optimisticMessages.length]); + const sendMessage = useCallback( async ( threadId: string, @@ -93,98 +117,191 @@ export function useThreadStream({ ) => { const text = message.text.trim(); - // Upload files first if any - if (message.files && message.files.length > 0) { - try { - // Convert FileUIPart to File objects by fetching blob URLs - const filePromises = message.files.map(async (fileUIPart) => { - if (fileUIPart.url && fileUIPart.filename) { - try { - // Fetch the blob URL to get the file data - const response = await fetch(fileUIPart.url); - const blob = await response.blob(); + // Capture current count before showing optimistic messages + prevMsgCountRef.current = thread.messages.length; - // Create a File object from the blob - return new File([blob], fileUIPart.filename, { - type: fileUIPart.mediaType || blob.type, - }); - } catch (error) { - console.error( - `Failed to fetch file ${fileUIPart.filename}:`, - error, - ); - return null; - } - } - return null; - }); + // Build optimistic files list with uploading status + const optimisticFiles: FileInMessage[] = (message.files ?? []).map( + (f) => ({ + filename: f.filename ?? "", + size: 0, + status: "uploading" as const, + }), + ); - const conversionResults = await Promise.all(filePromises); - const files = conversionResults.filter( - (file): file is File => file !== null, - ); - const failedConversions = conversionResults.length - files.length; + // Create optimistic human message (shown immediately) + const optimisticHumanMsg: Message = { + type: "human", + id: `opt-human-${Date.now()}`, + content: text ? [{ type: "text", text }] : "", + additional_kwargs: + optimisticFiles.length > 0 ? { files: optimisticFiles } : {}, + }; - if (failedConversions > 0) { - throw new Error( - `Failed to prepare ${failedConversions} attachment(s) for upload. Please retry.`, - ); - } + const newOptimistic: Message[] = [optimisticHumanMsg]; + if (optimisticFiles.length > 0) { + // Mock AI message while files are being uploaded + newOptimistic.push({ + type: "ai", + id: `opt-ai-${Date.now()}`, + content: t.uploads.uploadingFiles, + additional_kwargs: { element: "task" }, + }); + } + setOptimisticMessages(newOptimistic); - if (!threadId) { - throw new Error("Thread is not ready for file upload."); - } - - if (files.length > 0) { - await uploadFiles(threadId, files); - } - } catch (error) { - console.error("Failed to upload files:", error); - const errorMessage = - error instanceof Error ? error.message : "Failed to upload files."; - toast.error(errorMessage); - throw error; - } + if (!startedRef.current) { + onStart?.(threadId); + startedRef.current = true; } - await thread.submit( - { - messages: [ - { - type: "human", - content: [ - { - type: "text", - text, - }, - ], + let uploadedFileInfo: UploadedFileInfo[] = []; + + try { + // Upload files first if any + if (message.files && message.files.length > 0) { + try { + // Convert FileUIPart to File objects by fetching blob URLs + const filePromises = message.files.map(async (fileUIPart) => { + if (fileUIPart.url && fileUIPart.filename) { + try { + // Fetch the blob URL to get the file data + const response = await fetch(fileUIPart.url); + const blob = await response.blob(); + + // Create a File object from the blob + return new File([blob], fileUIPart.filename, { + type: fileUIPart.mediaType || blob.type, + }); + } catch (error) { + console.error( + `Failed to fetch file ${fileUIPart.filename}:`, + error, + ); + return null; + } + } + return null; + }); + + const conversionResults = await Promise.all(filePromises); + const files = conversionResults.filter( + (file): file is File => file !== null, + ); + const failedConversions = conversionResults.length - files.length; + + if (failedConversions > 0) { + throw new Error( + `Failed to prepare ${failedConversions} attachment(s) for upload. Please retry.`, + ); + } + + if (!threadId) { + throw new Error("Thread is not ready for file upload."); + } + + if (files.length > 0) { + const uploadResponse = await uploadFiles(threadId, files); + uploadedFileInfo = uploadResponse.files; + + // Update optimistic human message with uploaded status + paths + const uploadedFiles: FileInMessage[] = uploadedFileInfo.map( + (info) => ({ + filename: info.filename, + size: info.size, + path: info.virtual_path, + status: "uploaded" as const, + }), + ); + setOptimisticMessages((messages) => { + if (messages.length > 1 && messages[0]) { + const humanMessage: Message = messages[0]; + return [ + { + ...humanMessage, + additional_kwargs: { files: uploadedFiles }, + }, + ...messages.slice(1), + ]; + } + return messages; + }); + } + } catch (error) { + console.error("Failed to upload files:", error); + const errorMessage = + error instanceof Error + ? error.message + : "Failed to upload files."; + toast.error(errorMessage); + setOptimisticMessages([]); + throw error; + } + } + + // Build files metadata for submission (included in additional_kwargs) + const filesForSubmit: FileInMessage[] = uploadedFileInfo.map( + (info) => ({ + filename: info.filename, + size: info.size, + path: info.virtual_path, + status: "uploaded" as const, + }), + ); + + await thread.submit( + { + messages: [ + { + type: "human", + content: [ + { + type: "text", + text, + }, + ], + additional_kwargs: + filesForSubmit.length > 0 ? { files: filesForSubmit } : {}, + }, + ], + }, + { + threadId: threadId, + streamSubgraphs: true, + streamResumable: true, + streamMode: ["values", "messages-tuple", "custom"], + config: { + recursion_limit: 1000, + }, + context: { + ...extraContext, + ...context, + thinking_enabled: context.mode !== "flash", + is_plan_mode: context.mode === "pro" || context.mode === "ultra", + subagent_enabled: context.mode === "ultra", + thread_id: threadId, }, - ], - }, - { - threadId: threadId, - streamSubgraphs: true, - streamResumable: true, - streamMode: ["values", "messages-tuple", "custom"], - config: { - recursion_limit: 1000, }, - context: { - ...extraContext, - ...context, - thinking_enabled: context.mode !== "flash", - is_plan_mode: context.mode === "pro" || context.mode === "ultra", - subagent_enabled: context.mode === "ultra", - thread_id: threadId, - }, - }, - ); - void queryClient.invalidateQueries({ queryKey: ["threads", "search"] }); - // afterSubmit?.(); + ); + void queryClient.invalidateQueries({ queryKey: ["threads", "search"] }); + } catch (error) { + setOptimisticMessages([]); + throw error; + } }, - [thread, context, queryClient], + [thread, t.uploads.uploadingFiles, onStart, context, queryClient], ); - return [thread, sendMessage] as const; + + // Merge thread with optimistic messages for display + const mergedThread = + optimisticMessages.length > 0 + ? ({ + ...thread, + messages: [...thread.messages, ...optimisticMessages], + } as typeof thread) + : thread; + + return [mergedThread, sendMessage] as const; } export function useThreads(