fix: resolve issue #651 - crawl error with None content handling (#652)

* fix: resolve issue #651 - crawl error with None content handling Fixed issue #651 by adding comprehensive null-safety checks and error handling to the crawl system. The fix prevents the ‘TypeError: Incoming markup is of an invalid type: None’ crash by: 1. Validating HTTP responses from Jina API 2. Handling None/empty content at extraction stage 3. Adding fallback handling in Article markdown/message conversion 4. Improving error diagnostics with detailed logging 5. Adding 16 new tests with 100% coverage for critical paths * Update src/crawler/readability_extractor.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update src/crawler/article.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2026-04-23 14:14:46 +08:00 · 2025-10-24 17:06:54 +08:00
parent 2001a7c223
commit 975b344ca7
8 changed files with 328 additions and 9 deletions
--- a/tests/unit/crawler/test_jina_client.py
+++ b/tests/unit/crawler/test_jina_client.py
@@ -0,0 +1,106 @@
+# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates
+# SPDX-License-Identifier: MIT
+
+import pytest
+from unittest.mock import patch, Mock
+from src.crawler.jina_client import JinaClient
+
+
+class TestJinaClient:
+    @patch("src.crawler.jina_client.requests.post")
+    def test_crawl_success(self, mock_post):
+        # Arrange
+        mock_response = Mock()
+        mock_response.status_code = 200
+        mock_response.text = "<html><body>Test</body></html>"
+        mock_post.return_value = mock_response
+
+        client = JinaClient()
+
+        # Act
+        result = client.crawl("https://example.com")
+
+        # Assert
+        assert result == "<html><body>Test</body></html>"
+        mock_post.assert_called_once()
+
+    @patch("src.crawler.jina_client.requests.post")
+    def test_crawl_http_error(self, mock_post):
+        # Arrange
+        mock_response = Mock()
+        mock_response.status_code = 500
+        mock_response.text = "Internal Server Error"
+        mock_post.return_value = mock_response
+
+        client = JinaClient()
+
+        # Act & Assert
+        with pytest.raises(ValueError) as exc_info:
+            client.crawl("https://example.com")
+
+        assert "status 500" in str(exc_info.value)
+
+    @patch("src.crawler.jina_client.requests.post")
+    def test_crawl_empty_response(self, mock_post):
+        # Arrange
+        mock_response = Mock()
+        mock_response.status_code = 200
+        mock_response.text = ""
+        mock_post.return_value = mock_response
+
+        client = JinaClient()
+
+        # Act & Assert
+        with pytest.raises(ValueError) as exc_info:
+            client.crawl("https://example.com")
+
+        assert "empty response" in str(exc_info.value)
+
+    @patch("src.crawler.jina_client.requests.post")
+    def test_crawl_whitespace_only_response(self, mock_post):
+        # Arrange
+        mock_response = Mock()
+        mock_response.status_code = 200
+        mock_response.text = "   \n  \t  "
+        mock_post.return_value = mock_response
+
+        client = JinaClient()
+
+        # Act & Assert
+        with pytest.raises(ValueError) as exc_info:
+            client.crawl("https://example.com")
+
+        assert "empty response" in str(exc_info.value)
+
+    @patch("src.crawler.jina_client.requests.post")
+    def test_crawl_not_found(self, mock_post):
+        # Arrange
+        mock_response = Mock()
+        mock_response.status_code = 404
+        mock_response.text = "Not Found"
+        mock_post.return_value = mock_response
+
+        client = JinaClient()
+
+        # Act & Assert
+        with pytest.raises(ValueError) as exc_info:
+            client.crawl("https://example.com")
+
+        assert "status 404" in str(exc_info.value)
+
+    @patch.dict("os.environ", {}, clear=True)
+    @patch("src.crawler.jina_client.requests.post")
+    def test_crawl_without_api_key_logs_warning(self, mock_post):
+        # Arrange
+        mock_response = Mock()
+        mock_response.status_code = 200
+        mock_response.text = "<html>Test</html>"
+        mock_post.return_value = mock_response
+
+        client = JinaClient()
+
+        # Act
+        result = client.crawl("https://example.com")
+
+        # Assert
+        assert result == "<html>Test</html>"