mirror of
https://gitee.com/wanwujie/deer-flow
synced 2026-04-15 03:04:44 +08:00
975b344ca7
* fix: resolve issue #651 - crawl error with None content handling Fixed issue #651 by adding comprehensive null-safety checks and error handling to the crawl system. The fix prevents the ‘TypeError: Incoming markup is of an invalid type: None’ crash by: 1. Validating HTTP responses from Jina API 2. Handling None/empty content at extraction stage 3. Adding fallback handling in Article markdown/message conversion 4. Improving error diagnostics with detailed logging 5. Adding 16 new tests with 100% coverage for critical paths * Update src/crawler/readability_extractor.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update src/crawler/article.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
107 lines
3.1 KiB
Python
107 lines
3.1 KiB
Python
# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates
|
|
# SPDX-License-Identifier: MIT
|
|
|
|
import pytest
|
|
from unittest.mock import patch, Mock
|
|
from src.crawler.jina_client import JinaClient
|
|
|
|
|
|
class TestJinaClient:
|
|
@patch("src.crawler.jina_client.requests.post")
|
|
def test_crawl_success(self, mock_post):
|
|
# Arrange
|
|
mock_response = Mock()
|
|
mock_response.status_code = 200
|
|
mock_response.text = "<html><body>Test</body></html>"
|
|
mock_post.return_value = mock_response
|
|
|
|
client = JinaClient()
|
|
|
|
# Act
|
|
result = client.crawl("https://example.com")
|
|
|
|
# Assert
|
|
assert result == "<html><body>Test</body></html>"
|
|
mock_post.assert_called_once()
|
|
|
|
@patch("src.crawler.jina_client.requests.post")
|
|
def test_crawl_http_error(self, mock_post):
|
|
# Arrange
|
|
mock_response = Mock()
|
|
mock_response.status_code = 500
|
|
mock_response.text = "Internal Server Error"
|
|
mock_post.return_value = mock_response
|
|
|
|
client = JinaClient()
|
|
|
|
# Act & Assert
|
|
with pytest.raises(ValueError) as exc_info:
|
|
client.crawl("https://example.com")
|
|
|
|
assert "status 500" in str(exc_info.value)
|
|
|
|
@patch("src.crawler.jina_client.requests.post")
|
|
def test_crawl_empty_response(self, mock_post):
|
|
# Arrange
|
|
mock_response = Mock()
|
|
mock_response.status_code = 200
|
|
mock_response.text = ""
|
|
mock_post.return_value = mock_response
|
|
|
|
client = JinaClient()
|
|
|
|
# Act & Assert
|
|
with pytest.raises(ValueError) as exc_info:
|
|
client.crawl("https://example.com")
|
|
|
|
assert "empty response" in str(exc_info.value)
|
|
|
|
@patch("src.crawler.jina_client.requests.post")
|
|
def test_crawl_whitespace_only_response(self, mock_post):
|
|
# Arrange
|
|
mock_response = Mock()
|
|
mock_response.status_code = 200
|
|
mock_response.text = " \n \t "
|
|
mock_post.return_value = mock_response
|
|
|
|
client = JinaClient()
|
|
|
|
# Act & Assert
|
|
with pytest.raises(ValueError) as exc_info:
|
|
client.crawl("https://example.com")
|
|
|
|
assert "empty response" in str(exc_info.value)
|
|
|
|
@patch("src.crawler.jina_client.requests.post")
|
|
def test_crawl_not_found(self, mock_post):
|
|
# Arrange
|
|
mock_response = Mock()
|
|
mock_response.status_code = 404
|
|
mock_response.text = "Not Found"
|
|
mock_post.return_value = mock_response
|
|
|
|
client = JinaClient()
|
|
|
|
# Act & Assert
|
|
with pytest.raises(ValueError) as exc_info:
|
|
client.crawl("https://example.com")
|
|
|
|
assert "status 404" in str(exc_info.value)
|
|
|
|
@patch.dict("os.environ", {}, clear=True)
|
|
@patch("src.crawler.jina_client.requests.post")
|
|
def test_crawl_without_api_key_logs_warning(self, mock_post):
|
|
# Arrange
|
|
mock_response = Mock()
|
|
mock_response.status_code = 200
|
|
mock_response.text = "<html>Test</html>"
|
|
mock_post.return_value = mock_response
|
|
|
|
client = JinaClient()
|
|
|
|
# Act
|
|
result = client.crawl("https://example.com")
|
|
|
|
# Assert
|
|
assert result == "<html>Test</html>"
|