wanwu/deer-flow
1
0
Fork 0
mirror of https://gitee.com/wanwujie/deer-flow synced 2026-04-25 15:04:46 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(context): decrease token in web_search AIMessage (#827)

Browse Source 
This PR addresses token limit issues when web_search is enabled with include_raw_content by implementing a two-pronged approach: changing the default behavior to exclude raw content and adding compression logic for when raw content is included.
This commit is contained in:
Xun
2026-01-23 08:31:48 +08:00
committed by GitHub
parent 65cdc182d3
commit c0849af37e
5 changed files with 120 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/tools/crawl.py
View File

@@ -37,13 +37,13 @@ def crawl_tool(
"error": "PDF files cannot be crawled directly. Please download and view the PDF manually.",
"crawled_content": None,
"is_pdf": True
})
}, ensure_ascii=False)
return pdf_message
try:
crawler = Crawler()
article = crawler.crawl(url)
return json.dumps({"url": url, "crawled_content": article.to_markdown()[:1000]})
return json.dumps({"url": url, "crawled_content": article.to_markdown()[:1000]}, ensure_ascii=False)
except BaseException as e:
error_msg = f"Failed to crawl. Error: {repr(e)}"
logger.error(error_msg)

2
src/tools/search.py
View File

@@ -57,7 +57,7 @@ def get_web_search_tool(max_search_results: int):
exclude_domains: Optional[List[str]] = search_config.get("exclude_domains", [])
include_answer: bool = search_config.get("include_answer", False)
search_depth: str = search_config.get("search_depth", "advanced")
include_raw_content: bool = search_config.get("include_raw_content", True)
include_raw_content: bool = search_config.get("include_raw_content", False)
include_images: bool = search_config.get("include_images", True)
include_image_descriptions: bool = include_images and search_config.get(
"include_image_descriptions", True