mirror of
https://gitee.com/wanwujie/deer-flow
synced 2026-04-05 15:10:20 +08:00
975b344ca7
* fix: resolve issue #651 - crawl error with None content handling Fixed issue #651 by adding comprehensive null-safety checks and error handling to the crawl system. The fix prevents the ‘TypeError: Incoming markup is of an invalid type: None’ crash by: 1. Validating HTTP responses from Jina API 2. Handling None/empty content at extraction stage 3. Adding fallback handling in Article markdown/message conversion 4. Improving error diagnostics with detailed logging 5. Adding 16 new tests with 100% coverage for critical paths * Update src/crawler/readability_extractor.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update src/crawler/article.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
104 lines
3.5 KiB
Python
104 lines
3.5 KiB
Python
# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates
|
|
# SPDX-License-Identifier: MIT
|
|
|
|
from unittest.mock import patch
|
|
from src.crawler.readability_extractor import ReadabilityExtractor
|
|
|
|
|
|
class TestReadabilityExtractor:
|
|
@patch("src.crawler.readability_extractor.simple_json_from_html_string")
|
|
def test_extract_article_with_valid_content(self, mock_simple_json):
|
|
# Arrange
|
|
mock_simple_json.return_value = {
|
|
"title": "Test Article",
|
|
"content": "<p>Article content</p>",
|
|
}
|
|
extractor = ReadabilityExtractor()
|
|
|
|
# Act
|
|
article = extractor.extract_article("<html>test</html>")
|
|
|
|
# Assert
|
|
assert article.title == "Test Article"
|
|
assert article.html_content == "<p>Article content</p>"
|
|
|
|
@patch("src.crawler.readability_extractor.simple_json_from_html_string")
|
|
def test_extract_article_with_none_content(self, mock_simple_json):
|
|
# Arrange
|
|
mock_simple_json.return_value = {
|
|
"title": "Test Article",
|
|
"content": None,
|
|
}
|
|
extractor = ReadabilityExtractor()
|
|
|
|
# Act
|
|
article = extractor.extract_article("<html>test</html>")
|
|
|
|
# Assert
|
|
assert article.title == "Test Article"
|
|
assert article.html_content == "<p>No content could be extracted from this page</p>"
|
|
|
|
@patch("src.crawler.readability_extractor.simple_json_from_html_string")
|
|
def test_extract_article_with_empty_content(self, mock_simple_json):
|
|
# Arrange
|
|
mock_simple_json.return_value = {
|
|
"title": "Test Article",
|
|
"content": "",
|
|
}
|
|
extractor = ReadabilityExtractor()
|
|
|
|
# Act
|
|
article = extractor.extract_article("<html>test</html>")
|
|
|
|
# Assert
|
|
assert article.title == "Test Article"
|
|
assert article.html_content == "<p>No content could be extracted from this page</p>"
|
|
|
|
@patch("src.crawler.readability_extractor.simple_json_from_html_string")
|
|
def test_extract_article_with_whitespace_only_content(self, mock_simple_json):
|
|
# Arrange
|
|
mock_simple_json.return_value = {
|
|
"title": "Test Article",
|
|
"content": " \n \t ",
|
|
}
|
|
extractor = ReadabilityExtractor()
|
|
|
|
# Act
|
|
article = extractor.extract_article("<html>test</html>")
|
|
|
|
# Assert
|
|
assert article.title == "Test Article"
|
|
assert article.html_content == "<p>No content could be extracted from this page</p>"
|
|
|
|
@patch("src.crawler.readability_extractor.simple_json_from_html_string")
|
|
def test_extract_article_with_none_title(self, mock_simple_json):
|
|
# Arrange
|
|
mock_simple_json.return_value = {
|
|
"title": None,
|
|
"content": "<p>Article content</p>",
|
|
}
|
|
extractor = ReadabilityExtractor()
|
|
|
|
# Act
|
|
article = extractor.extract_article("<html>test</html>")
|
|
|
|
# Assert
|
|
assert article.title == "Untitled"
|
|
assert article.html_content == "<p>Article content</p>"
|
|
|
|
@patch("src.crawler.readability_extractor.simple_json_from_html_string")
|
|
def test_extract_article_with_empty_title(self, mock_simple_json):
|
|
# Arrange
|
|
mock_simple_json.return_value = {
|
|
"title": "",
|
|
"content": "<p>Article content</p>",
|
|
}
|
|
extractor = ReadabilityExtractor()
|
|
|
|
# Act
|
|
article = extractor.extract_article("<html>test</html>")
|
|
|
|
# Assert
|
|
assert article.title == "Untitled"
|
|
assert article.html_content == "<p>Article content</p>"
|