fix: resolve issue #651 - crawl error with None content handling (#652)

* fix: resolve issue #651 - crawl error with None content handling
Fixed issue #651 by adding comprehensive null-safety checks and error handling to the crawl system.
The fix prevents the ‘TypeError: Incoming markup is of an invalid type: None’ crash by:
1. Validating HTTP responses from Jina API
2. Handling None/empty content at extraction stage
3. Adding fallback handling in Article markdown/message conversion
4. Improving error diagnostics with detailed logging
5. Adding 16 new tests with 100% coverage for critical paths

* Update src/crawler/readability_extractor.py

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update src/crawler/article.py

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

tests/unit/crawler/test_article.py

@@ -71,3 +71,43 @@ def test_to_message_handles_empty_html():
     result = article.to_message()
     assert isinstance(result, list)
     assert result[0]["type"] == "text"
+
+
+def test_to_markdown_handles_none_content():
+    article = Article("Test Title", None)
+    result = article.to_markdown(including_title=True)
+    assert "# Test Title" in result
+    assert "No content available" in result
+
+
+def test_to_markdown_handles_empty_string():
+    article = Article("Test Title", "")
+    result = article.to_markdown(including_title=True)
+    assert "# Test Title" in result
+    assert "No content available" in result
+
+
+def test_to_markdown_handles_whitespace_only():
+    article = Article("Test Title", "   \n  \t  ")
+    result = article.to_markdown(including_title=True)
+    assert "# Test Title" in result
+    assert "No content available" in result
+
+
+def test_to_message_handles_none_content():
+    article = Article("Title", None)
+    article.url = "http://test/"
+    result = article.to_message()
+    assert isinstance(result, list)
+    assert len(result) > 0
+    assert result[0]["type"] == "text"
+    assert "No content available" in result[0]["text"]
+
+
+def test_to_message_handles_whitespace_only_content():
+    article = Article("Title", "   \n  ")
+    article.url = "http://test/"
+    result = article.to_message()
+    assert isinstance(result, list)
+    assert result[0]["type"] == "text"
+    assert "No content available" in result[0]["text"]