mirror of
https://gitee.com/wanwujie/deer-flow
synced 2026-04-02 22:02:13 +08:00
b8bc80d89b
* refactor: extract shared skill installer and upload manager to harness Move duplicated business logic from Gateway routers and Client into shared harness modules, eliminating code duplication. New shared modules: - deerflow.skills.installer: 6 functions (zip security, extraction, install) - deerflow.uploads.manager: 7 functions (normalize, deduplicate, validate, list, delete, get_uploads_dir, ensure_uploads_dir) Key improvements: - SkillAlreadyExistsError replaces stringly-typed 409 status routing - normalize_filename rejects backslash-containing filenames - Read paths (list/delete) no longer mkdir via get_uploads_dir - Write paths use ensure_uploads_dir for explicit directory creation - list_files_in_dir does stat inside scandir context (no re-stat) - install_skill_from_archive uses single is_file() check (one syscall) - Fix agent config key not reset on update_mcp_config/update_skill Tests: 42 new (22 installer + 20 upload manager) + client hardening * refactor: centralize upload URL construction and clean up installer - Extract upload_virtual_path(), upload_artifact_url(), enrich_file_listing() into shared manager.py, eliminating 6 duplicated URL constructions across Gateway router and Client - Derive all upload URLs from VIRTUAL_PATH_PREFIX constant instead of hardcoded "mnt/user-data/uploads" strings - Eliminate TOCTOU pre-checks and double file read in installer — single ZipFile() open with exception handling replaces is_file() + is_zipfile() + ZipFile() sequence - Add missing re-exports: ensure_uploads_dir in uploads/__init__.py, SkillAlreadyExistsError in skills/__init__.py - Remove redundant .lower() on already-lowercase CONVERTIBLE_EXTENSIONS - Hoist sandbox_uploads_dir(thread_id) before loop in uploads router * fix: add input validation for thread_id and filename length - Reject thread_id containing unsafe filesystem characters (only allow alphanumeric, hyphens, underscores, dots) — prevents 500 on inputs like <script> or shell metacharacters - Reject filenames longer than 255 bytes (OS limit) in normalize_filename - Gateway upload router maps ValueError to 400 for invalid thread_id * fix: address PR review — symlink safety, input validation coverage, error ordering - list_files_in_dir: use follow_symlinks=False to prevent symlink metadata leakage; check is_dir() instead of exists() for non-directory paths - install_skill_from_archive: restore is_file() pre-check before extension validation so error messages match the documented exception contract - validate_thread_id: move from ensure_uploads_dir to get_uploads_dir so all entry points (upload/list/delete) are protected - delete_uploaded_file: catch ValueError from thread_id validation (was 500) - requires_llm marker: also skip when OPENAI_API_KEY is unset - e2e fixture: update TitleMiddleware exclusion comment (kept filtering — middleware triggers extra LLM calls that add non-determinism to tests) * chore: revert uv.lock to main — no dependency changes in this PR * fix: use monkeypatch for global config in e2e fixture to prevent test pollution The e2e_env fixture was calling set_title_config() and set_summarization_config() directly, which mutated global singletons without automatic cleanup. When pytest ran test_client_e2e.py before test_title_middleware_core_logic.py, the leaked enabled=False caused 5 title tests to fail in CI. Switched to monkeypatch.setattr on the module-level private variables so pytest restores the originals after each test. * fix: address code review — URL encoding, API consistency, test isolation - upload_artifact_url: percent-encode filename to handle spaces/#/? - deduplicate_filename: mutate seen set in place (caller no longer needs manual .add() — less error-prone API) - list_files_in_dir: document that size is int, enrich stringifies - e2e fixture: monkeypatch _app_config instead of set_app_config() to prevent global singleton pollution (same pattern as title/summarization fix) - _make_e2e_config: read LLM connection details from env vars so external contributors can override defaults - Update tests to match new deduplicate_filename contract * docs: rewrite RFC in English and add alternatives/breaking changes sections * fix: address code review feedback on PR #1202 - Rename deduplicate_filename to claim_unique_filename to make the in-place set mutation explicit in the function name - Replace PermissionError with PathTraversalError(ValueError) for path traversal detection — malformed input is 400, not 403 * fix: set _app_config_is_custom in e2e test fixture to prevent config.yaml lookup in CI --------- Co-authored-by: greatmengqi <chenmengqi.0376@bytedance.com> Co-authored-by: Willem Jiang <willem.jiang@gmail.com> Co-authored-by: DanielWalnut <45447813+hetaoBackend@users.noreply.github.com>
147 lines
5.0 KiB
Python
147 lines
5.0 KiB
Python
"""Tests for deerflow.uploads.manager — shared upload management logic."""
|
|
|
|
import pytest
|
|
|
|
from deerflow.uploads.manager import (
|
|
PathTraversalError,
|
|
claim_unique_filename,
|
|
delete_file_safe,
|
|
list_files_in_dir,
|
|
normalize_filename,
|
|
validate_path_traversal,
|
|
)
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# normalize_filename
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestNormalizeFilename:
|
|
def test_safe_filename(self):
|
|
assert normalize_filename("report.pdf") == "report.pdf"
|
|
|
|
def test_strips_path_components(self):
|
|
assert normalize_filename("../../etc/passwd") == "passwd"
|
|
|
|
def test_rejects_empty(self):
|
|
with pytest.raises(ValueError, match="empty"):
|
|
normalize_filename("")
|
|
|
|
def test_rejects_dot_dot(self):
|
|
with pytest.raises(ValueError, match="unsafe"):
|
|
normalize_filename("..")
|
|
|
|
def test_strips_separators(self):
|
|
assert normalize_filename("path/to/file.txt") == "file.txt"
|
|
|
|
def test_dot_only(self):
|
|
with pytest.raises(ValueError, match="unsafe"):
|
|
normalize_filename(".")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# claim_unique_filename
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestDeduplicateFilename:
|
|
def test_no_collision(self):
|
|
seen: set[str] = set()
|
|
assert claim_unique_filename("data.txt", seen) == "data.txt"
|
|
assert "data.txt" in seen
|
|
|
|
def test_single_collision(self):
|
|
seen = {"data.txt"}
|
|
assert claim_unique_filename("data.txt", seen) == "data_1.txt"
|
|
assert "data_1.txt" in seen
|
|
|
|
def test_triple_collision(self):
|
|
seen = {"data.txt", "data_1.txt", "data_2.txt"}
|
|
assert claim_unique_filename("data.txt", seen) == "data_3.txt"
|
|
assert "data_3.txt" in seen
|
|
|
|
def test_mutates_seen(self):
|
|
seen: set[str] = set()
|
|
claim_unique_filename("a.txt", seen)
|
|
claim_unique_filename("a.txt", seen)
|
|
assert seen == {"a.txt", "a_1.txt"}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# validate_path_traversal
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestValidatePathTraversal:
|
|
def test_inside_base_ok(self, tmp_path):
|
|
child = tmp_path / "file.txt"
|
|
child.touch()
|
|
validate_path_traversal(child, tmp_path) # no exception
|
|
|
|
def test_outside_base_raises(self, tmp_path):
|
|
outside = tmp_path / ".." / "evil.txt"
|
|
with pytest.raises(PathTraversalError, match="traversal"):
|
|
validate_path_traversal(outside, tmp_path)
|
|
|
|
def test_symlink_escape(self, tmp_path):
|
|
target = tmp_path.parent / "secret.txt"
|
|
target.touch()
|
|
link = tmp_path / "escape"
|
|
link.symlink_to(target)
|
|
with pytest.raises(PathTraversalError, match="traversal"):
|
|
validate_path_traversal(link, tmp_path)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# list_files_in_dir
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestListFilesInDir:
|
|
def test_empty_dir(self, tmp_path):
|
|
result = list_files_in_dir(tmp_path)
|
|
assert result == {"files": [], "count": 0}
|
|
|
|
def test_nonexistent_dir(self, tmp_path):
|
|
result = list_files_in_dir(tmp_path / "nope")
|
|
assert result == {"files": [], "count": 0}
|
|
|
|
def test_multiple_files_sorted(self, tmp_path):
|
|
(tmp_path / "b.txt").write_text("b")
|
|
(tmp_path / "a.txt").write_text("a")
|
|
result = list_files_in_dir(tmp_path)
|
|
assert result["count"] == 2
|
|
assert result["files"][0]["filename"] == "a.txt"
|
|
assert result["files"][1]["filename"] == "b.txt"
|
|
for f in result["files"]:
|
|
assert set(f.keys()) == {"filename", "size", "path", "extension", "modified"}
|
|
|
|
def test_ignores_subdirectories(self, tmp_path):
|
|
(tmp_path / "file.txt").write_text("data")
|
|
(tmp_path / "subdir").mkdir()
|
|
result = list_files_in_dir(tmp_path)
|
|
assert result["count"] == 1
|
|
assert result["files"][0]["filename"] == "file.txt"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# delete_file_safe
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestDeleteFileSafe:
|
|
def test_delete_existing_file(self, tmp_path):
|
|
f = tmp_path / "test.txt"
|
|
f.write_text("data")
|
|
result = delete_file_safe(tmp_path, "test.txt")
|
|
assert result["success"] is True
|
|
assert not f.exists()
|
|
|
|
def test_delete_nonexistent_raises(self, tmp_path):
|
|
with pytest.raises(FileNotFoundError):
|
|
delete_file_safe(tmp_path, "nope.txt")
|
|
|
|
def test_delete_traversal_raises(self, tmp_path):
|
|
with pytest.raises(PathTraversalError, match="traversal"):
|
|
delete_file_safe(tmp_path, "../outside.txt")
|