mirror of
https://gitee.com/wanwujie/deer-flow
synced 2026-04-25 23:14:46 +08:00
7ec9e45702
* support infoquest * support html checker * support html checker * change line break format * change line break format * change line break format * change line break format * change line break format * change line break format * change line break format * change line break format * Fix several critical issues in the codebase - Resolve crawler panic by improving error handling - Fix plan validation to prevent invalid configurations - Correct InfoQuest crawler JSON conversion logic * add test for infoquest * add test for infoquest * Add InfoQuest introduction to the README * add test for infoquest * fix readme for infoquest * fix readme for infoquest * resolve the conflict * resolve the conflict * resolve the conflict * Fix formatting of INFOQUEST in SearchEngine enum * Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: Willem Jiang <143703838+willem-bd@users.noreply.github.com> Co-authored-by: Willem Jiang <willem.jiang@gmail.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
42 lines
1.5 KiB
Python
42 lines
1.5 KiB
Python
# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates
|
|
# SPDX-License-Identifier: MIT
|
|
|
|
import logging
|
|
import os
|
|
|
|
import requests
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class JinaClient:
|
|
def crawl(self, url: str, return_format: str = "html") -> str:
|
|
headers = {
|
|
"Content-Type": "application/json",
|
|
"X-Return-Format": return_format,
|
|
}
|
|
if os.getenv("JINA_API_KEY"):
|
|
headers["Authorization"] = f"Bearer {os.getenv('JINA_API_KEY')}"
|
|
else:
|
|
logger.warning(
|
|
"Jina API key is not set. Provide your own key to access a higher rate limit. See https://jina.ai/reader for more information."
|
|
)
|
|
data = {"url": url}
|
|
try:
|
|
response = requests.post("https://r.jina.ai/", headers=headers, json=data)
|
|
|
|
if response.status_code != 200:
|
|
error_message = f"Jina API returned status {response.status_code}: {response.text}"
|
|
logger.error(error_message)
|
|
return f"Error: {error_message}"
|
|
|
|
if not response.text or not response.text.strip():
|
|
error_message = "Jina API returned empty response"
|
|
logger.error(error_message)
|
|
return f"Error: {error_message}"
|
|
|
|
return response.text
|
|
except Exception as e:
|
|
error_message = f"Request to Jina API failed: {str(e)}"
|
|
logger.error(error_message)
|
|
return f"Error: {error_message}" |