mirror of
https://gitee.com/wanwujie/deer-flow
synced 2026-04-29 16:54:47 +08:00
24 lines
746 B
Python
24 lines
746 B
Python
|
import logging
|
||
|
|
import os
|
||
|
|
|
||
|
|
import requests
|
||
|
|
|
||
|
|
logger = logging.getLogger(__name__)
|
||
|
|
|
||
|
|
|
||
|
|
class JinaClient:
|
||
|
|
def crawl(self, url: str, return_format: str = "html") -> str:
|
||
|
|
headers = {
|
||
|
|
"Content-Type": "application/json",
|
||
|
|
"X-Return-Format": return_format,
|
||
|
|
}
|
||
|
|
if os.getenv("JINA_API_KEY"):
|
||
|
|
headers["Authorization"] = f"Bearer {os.getenv('JINA_API_KEY')}"
|
||
|
|
else:
|
||
|
|
logger.warning(
|
||
|
|
"Jina API key is not set. Provide your own key to access a higher rate limit. See https://jina.ai/reader for more information."
|
||
|
|
)
|
||
|
|
data = {"url": url}
|
||
|
|
response = requests.post("https://r.jina.ai/", headers=headers, json=data)
|
||
|
|
return response.text
|