diff --git a/src/config/configuration.py b/src/config/configuration.py index 299d6cd..611618c 100644 --- a/src/config/configuration.py +++ b/src/config/configuration.py @@ -54,6 +54,9 @@ class Configuration: enforce_web_search: bool = ( False # Enforce at least one web search step in every plan ) + enforce_researcher_search: bool = ( + True # Enforce that researcher must use web search tool at least once + ) interrupt_before_tools: list[str] = field( default_factory=list ) # List of tool names to interrupt before execution diff --git a/src/graph/nodes.py b/src/graph/nodes.py index fa978bd..f85ab45 100644 --- a/src/graph/nodes.py +++ b/src/graph/nodes.py @@ -769,8 +769,51 @@ def research_team_node(state: State): pass +def validate_web_search_usage(messages: list, agent_name: str = "agent") -> bool: + """ + Validate if the agent has used the web search tool during execution. + + Args: + messages: List of messages from the agent execution + agent_name: Name of the agent (for logging purposes) + + Returns: + bool: True if web search tool was used, False otherwise + """ + web_search_used = False + + for message in messages: + # Check for ToolMessage instances indicating web search was used + if isinstance(message, ToolMessage) and message.name == "web_search": + web_search_used = True + logger.info(f"[VALIDATION] {agent_name} received ToolMessage from web_search tool") + break + + # Check for AIMessage content that mentions tool calls + if hasattr(message, 'tool_calls') and message.tool_calls: + for tool_call in message.tool_calls: + if tool_call.get('name') == "web_search": + web_search_used = True + logger.info(f"[VALIDATION] {agent_name} called web_search tool") + break + # break outer loop if web search was used + if web_search_used: + break + + # Check for message name attribute + if hasattr(message, 'name') and message.name == "web_search": + web_search_used = True + logger.info(f"[VALIDATION] {agent_name} used web_search tool") + break + + if not web_search_used: + logger.warning(f"[VALIDATION] {agent_name} did not use web_search tool") + + return web_search_used + + async def _execute_agent_step( - state: State, agent, agent_name: str + state: State, agent, agent_name: str, config: RunnableConfig = None ) -> Command[Literal["research_team"]]: """Helper function to execute a step using the specified agent.""" logger.debug(f"[_execute_agent_step] Starting execution for agent: {agent_name}") @@ -918,6 +961,27 @@ async def _execute_agent_step( logger.debug(f"{agent_name.capitalize()} full response: {response_content}") + # Validate web search usage for researcher agent if enforcement is enabled + web_search_validated = True + should_validate = agent_name == "researcher" + validation_info = "" + + if should_validate: + # Check if enforcement is enabled in configuration + configurable = Configuration.from_runnable_config(config) if config else Configuration() + if configurable.enforce_researcher_search: + web_search_validated = validate_web_search_usage(result["messages"], agent_name) + + # If web search was not used, add a warning to the response + if not web_search_validated: + logger.warning(f"[VALIDATION] Researcher did not use web_search tool. Adding reminder to response.") + # Add validation information to observations + validation_info = ( + "\n\n[WARNING] This research was completed without using the web_search tool. " + "Please verify that the information provided is accurate and up-to-date." + "\n\n[VALIDATION WARNING] Researcher did not use the web_search tool as recommended." + ) + # Update the step with the execution result current_step.execution_res = response_content logger.info(f"Step '{current_step.title}' execution completed by {agent_name}") @@ -930,7 +994,7 @@ async def _execute_agent_step( name=agent_name, ) ], - "observations": observations + [response_content], + "observations": observations + [response_content + validation_info], **preserve_state_meta_fields(state), }, goto="research_team", @@ -1000,7 +1064,7 @@ async def _setup_and_execute_agent_step( pre_model_hook, interrupt_before_tools=configurable.interrupt_before_tools, ) - return await _execute_agent_step(state, agent, agent_type) + return await _execute_agent_step(state, agent, agent_type, config) else: # Use default tools if no MCP servers are configured llm_token_limit = get_llm_token_limit_by_type(AGENT_LLM_MAP[agent_type]) @@ -1013,7 +1077,7 @@ async def _setup_and_execute_agent_step( pre_model_hook, interrupt_before_tools=configurable.interrupt_before_tools, ) - return await _execute_agent_step(state, agent, agent_type) + return await _execute_agent_step(state, agent, agent_type, config) async def researcher_node( @@ -1034,6 +1098,7 @@ async def researcher_node( logger.info(f"[researcher_node] Researcher tools count: {len(tools)}") logger.debug(f"[researcher_node] Researcher tools: {[tool.name if hasattr(tool, 'name') else str(tool) for tool in tools]}") + logger.info(f"[researcher_node] enforce_researcher_search is set to: {configurable.enforce_researcher_search}") return await _setup_and_execute_agent_step( state, diff --git a/src/prompts/researcher.md b/src/prompts/researcher.md index 2a261a5..7d49e8f 100644 --- a/src/prompts/researcher.md +++ b/src/prompts/researcher.md @@ -37,7 +37,8 @@ You have access to two types of tools: 3. **Plan the Solution**: Determine the best approach to solve the problem using the available tools. 4. **Execute the Solution**: - Forget your previous knowledge, so you **should leverage the tools** to retrieve the information. - - Use the {% if resources %}**local_search_tool** or{% endif %}**web_search** or other suitable search tool to perform a search with the provided keywords. + - **CRITICAL**: You MUST use the {% if resources %}**local_search_tool** or{% endif %}**web_search** tool to search for information. NEVER generate URLs on your own. All URLs must come from tool results. + - **MANDATORY**: Always perform at least one web search using the **web_search** tool at the beginning of your research. This is not optional. - When the task includes time range requirements: - Incorporate appropriate time-based search parameters in your queries (e.g., "after:2020", "before:2023", or specific date ranges) - Ensure search results respect the specified time constraints. @@ -71,6 +72,8 @@ You have access to two types of tools: # Notes +- **CRITICAL**: NEVER generate URLs on your own. All URLs must come from search tool results. This is a mandatory requirement. +- **MANDATORY**: Always start with a web search. Do not rely on your internal knowledge. - Always verify the relevance and credibility of the information gathered. - If no URL is provided, focus solely on the search results. - Never do any math or any file operations. diff --git a/src/prompts/researcher.zh_CN.md b/src/prompts/researcher.zh_CN.md index a63a9fa..2c998c0 100644 --- a/src/prompts/researcher.zh_CN.md +++ b/src/prompts/researcher.zh_CN.md @@ -37,7 +37,8 @@ CURRENT_TIME: {{ CURRENT_TIME }} 3. **规划解决方案**:确定使用可用工具解决问题的最佳方法。 4. **执行解决方案**: - 忘记你之前的知识,所以你**应该利用工具**来检索信息。 - - 使用{% if resources %}**local_search_tool**或{% endif %}**web_search**或其他合适的搜索工具以提供的关键词执行搜索。 + - **关键要求**:你必须使用{% if resources %}**local_search_tool**或{% endif %}**web_search**工具搜索信息。绝对不能自己生成URL。所有URL必须来自工具结果。 + - **强制要求**:在研究开始时必须使用**web_search**工具至少执行一次网络搜索。这不是可选项。 - 当任务包括时间范围要求时: - 在查询中纳入适当的基于时间的搜索参数(如"after:2020"、"before:2023"或特定日期范围) - 确保搜索结果尊重指定的时间约束。 @@ -66,6 +67,8 @@ CURRENT_TIME: {{ CURRENT_TIME }} # 注意 +- **关键要求**:绝对不能自己生成URL。所有URL必须来自搜索工具结果。这是强制要求。 +- **强制要求**:始终从网络搜索开始。不要依赖你的内部知识。 - 始终验证收集的信息的相关性和可信度。 - 如果未提供URL,仅关注搜索结果。 - 不要进行任何数学运算或文件操作。 diff --git a/src/rag/qdrant.py b/src/rag/qdrant.py index af1624f..fd01741 100644 --- a/src/rag/qdrant.py +++ b/src/rag/qdrant.py @@ -10,8 +10,7 @@ from typing import Any, Dict, List, Optional, Sequence, Set from langchain_openai import OpenAIEmbeddings from langchain_qdrant import QdrantVectorStore from openai import OpenAI -from qdrant_client import QdrantClient -from qdrant_client import grpc +from qdrant_client import QdrantClient, grpc from qdrant_client.models import ( Distance, FieldCondition, diff --git a/tests/integration/test_nodes.py b/tests/integration/test_nodes.py index e10e759..d788dc6 100644 --- a/tests/integration/test_nodes.py +++ b/tests/integration/test_nodes.py @@ -950,7 +950,7 @@ async def test_execute_agent_step_basic(mock_state_with_steps, mock_agent): assert "messages" in result.update assert "observations" in result.update # The new observation should be appended - assert result.update["observations"][-1] == "result content" + assert result.update["observations"][-1] == "result content" + "\n\n[WARNING] This research was completed without using the web_search tool. " + "Please verify that the information provided is accurate and up-to-date." + "\n\n[VALIDATION WARNING] Researcher did not use the web_search tool as recommended." # The step's execution_res should be updated assert ( mock_state_with_steps["current_plan"].steps[1].execution_res @@ -1004,7 +1004,7 @@ async def test_execute_agent_step_with_resources_and_researcher(mock_step): result = await _execute_agent_step(state, agent, "researcher") assert isinstance(result, Command) assert result.goto == "research_team" - assert result.update["observations"][-1] == "resource result" + assert result.update["observations"][-1] == "resource result" + "\n\n[WARNING] This research was completed without using the web_search tool. " + "Please verify that the information provided is accurate and up-to-date." + "\n\n[VALIDATION WARNING] Researcher did not use the web_search tool as recommended." @pytest.mark.asyncio @@ -1124,7 +1124,7 @@ def patch_create_agent(): @pytest.fixture def patch_execute_agent_step(): - async def fake_execute_agent_step(state, agent, agent_type): + async def fake_execute_agent_step(state, agent, agent_type, config=None): return "EXECUTED" with patch(