From 76803b826f30028e691ea981bf51da641c0be632 Mon Sep 17 00:00:00 2001 From: DanielWalnut <45447813+hetaoBackend@users.noreply.github.com> Date: Sat, 14 Mar 2026 22:55:52 +0800 Subject: [PATCH] refactor: split backend into harness (deerflow.*) and app (app.*) (#1131) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor: extract shared utils to break harness→app cross-layer imports Move _validate_skill_frontmatter to src/skills/validation.py and CONVERTIBLE_EXTENSIONS + convert_file_to_markdown to src/utils/file_conversion.py. This eliminates the two reverse dependencies from client.py (harness layer) into gateway/routers/ (app layer), preparing for the harness/app package split. Co-Authored-By: Claude Opus 4.6 * refactor: split backend/src into harness (deerflow.*) and app (app.*) Physically split the monolithic backend/src/ package into two layers: - **Harness** (`packages/harness/deerflow/`): publishable agent framework package with import prefix `deerflow.*`. Contains agents, sandbox, tools, models, MCP, skills, config, and all core infrastructure. - **App** (`app/`): unpublished application code with import prefix `app.*`. Contains gateway (FastAPI REST API) and channels (IM integrations). Key changes: - Move 13 harness modules to packages/harness/deerflow/ via git mv - Move gateway + channels to app/ via git mv - Rename all imports: src.* → deerflow.* (harness) / app.* (app layer) - Set up uv workspace with deerflow-harness as workspace member - Update langgraph.json, config.example.yaml, all scripts, Docker files - Add build-system (hatchling) to harness pyproject.toml - Add PYTHONPATH=. to gateway startup commands for app.* resolution - Update ruff.toml with known-first-party for import sorting - Update all documentation to reflect new directory structure Boundary rule enforced: harness code never imports from app. All 429 tests pass. Lint clean. Co-Authored-By: Claude Opus 4.6 * chore: add harness→app boundary check test and update docs Add test_harness_boundary.py that scans all Python files in packages/harness/deerflow/ and fails if any `from app.*` or `import app.*` statement is found. This enforces the architectural rule that the harness layer never depends on the app layer. Update CLAUDE.md to document the harness/app split architecture, import conventions, and the boundary enforcement test. Co-Authored-By: Claude Opus 4.6 * feat: add config versioning with auto-upgrade on startup When config.example.yaml schema changes, developers' local config.yaml files can silently become outdated. This adds a config_version field and auto-upgrade mechanism so breaking changes (like src.* → deerflow.* renames) are applied automatically before services start. - Add config_version: 1 to config.example.yaml - Add startup version check warning in AppConfig.from_file() - Add scripts/config-upgrade.sh with migration registry for value replacements - Add `make config-upgrade` target - Auto-run config-upgrade in serve.sh and start-daemon.sh before starting services - Add config error hints in service failure messages Co-Authored-By: Claude Opus 4.6 * fix comments * fix: update src.* import in test_sandbox_tools_security to deerflow.* Co-Authored-By: Claude Opus 4.6 * fix: handle empty config and search parent dirs for config.example.yaml Address Copilot review comments on PR #1131: - Guard against yaml.safe_load() returning None for empty config files - Search parent directories for config.example.yaml instead of only looking next to config.yaml, fixing detection in common setups Co-Authored-By: Claude Opus 4.6 * fix: correct skills root path depth and config_version type coercion - loader.py: fix get_skills_root_path() to use 5 parent levels (was 3) after harness split, file lives at packages/harness/deerflow/skills/ so parent×3 resolved to backend/packages/harness/ instead of backend/ - app_config.py: coerce config_version to int() before comparison in _check_config_version() to prevent TypeError when YAML stores value as string (e.g. config_version: "1") - tests: add regression tests for both fixes Co-Authored-By: Claude Sonnet 4.6 * fix: update test imports from src.* to deerflow.*/app.* after harness refactor Co-Authored-By: Claude Sonnet 4.6 --------- Co-authored-by: Claude Opus 4.6 --- .github/copilot-instructions.md | 12 +- .gitignore | 1 + Makefile | 8 +- README.md | 6 +- backend/CLAUDE.md | 122 ++++--- backend/CONTRIBUTING.md | 24 +- backend/Dockerfile | 2 +- backend/Makefile | 2 +- backend/{src => app}/__init__.py | 0 backend/{src => app}/channels/__init__.py | 4 +- backend/{src => app}/channels/base.py | 2 +- backend/{src => app}/channels/feishu.py | 4 +- backend/{src => app}/channels/manager.py | 31 +- backend/{src => app}/channels/message_bus.py | 0 backend/{src => app}/channels/service.py | 16 +- backend/{src => app}/channels/slack.py | 4 +- backend/{src => app}/channels/store.py | 2 +- backend/{src => app}/channels/telegram.py | 4 +- backend/{src => app}/gateway/__init__.py | 0 backend/{src => app}/gateway/app.py | 10 +- backend/{src => app}/gateway/config.py | 0 backend/{src => app}/gateway/path_utils.py | 2 +- .../{src => app}/gateway/routers/__init__.py | 0 .../{src => app}/gateway/routers/agents.py | 4 +- .../{src => app}/gateway/routers/artifacts.py | 2 +- .../{src => app}/gateway/routers/channels.py | 4 +- backend/{src => app}/gateway/routers/mcp.py | 2 +- .../{src => app}/gateway/routers/memory.py | 4 +- .../{src => app}/gateway/routers/models.py | 2 +- .../{src => app}/gateway/routers/skills.py | 105 +----- .../gateway/routers/suggestions.py | 2 +- .../{src => app}/gateway/routers/uploads.py | 42 +-- backend/debug.py | 17 +- backend/docs/APPLE_CONTAINER.md | 12 +- backend/docs/ARCHITECTURE.md | 18 +- backend/docs/AUTO_TITLE_GENERATION.md | 14 +- backend/docs/CONFIGURATION.md | 25 +- backend/docs/FILE_UPLOAD.md | 4 +- backend/docs/HARNESS_APP_SPLIT.md | 343 ++++++++++++++++++ backend/docs/MEMORY_IMPROVEMENTS_SUMMARY.md | 4 +- backend/docs/PATH_EXAMPLES.md | 2 +- backend/docs/SETUP.md | 6 +- .../docs/TITLE_GENERATION_IMPLEMENTATION.md | 12 +- backend/docs/TODO.md | 4 +- backend/docs/plan_mode_usage.md | 10 +- backend/docs/summarization.md | 4 +- backend/docs/task_tool_improvements.md | 4 +- backend/langgraph.json | 6 +- backend/packages/harness/deerflow/__init__.py | 0 .../harness/deerflow}/agents/__init__.py | 0 .../deerflow}/agents/checkpointer/__init__.py | 0 .../agents/checkpointer/async_provider.py | 8 +- .../deerflow}/agents/checkpointer/provider.py | 12 +- .../deerflow}/agents/lead_agent/__init__.py | 0 .../deerflow}/agents/lead_agent/agent.py | 32 +- .../deerflow}/agents/lead_agent/prompt.py | 10 +- .../deerflow}/agents/memory/__init__.py | 6 +- .../harness/deerflow}/agents/memory/prompt.py | 0 .../harness/deerflow}/agents/memory/queue.py | 4 +- .../deerflow}/agents/memory/updater.py | 8 +- .../middlewares/clarification_middleware.py | 0 .../dangling_tool_call_middleware.py | 0 .../middlewares/loop_detection_middleware.py | 0 .../agents/middlewares/memory_middleware.py | 4 +- .../middlewares/subagent_limit_middleware.py | 2 +- .../middlewares/thread_data_middleware.py | 4 +- .../agents/middlewares/title_middleware.py | 4 +- .../agents/middlewares/todo_middleware.py | 0 .../tool_error_handling_middleware.py | 13 +- .../agents/middlewares/uploads_middleware.py | 2 +- .../middlewares/view_image_middleware.py | 2 +- .../harness/deerflow}/agents/thread_state.py | 0 .../harness/deerflow}/client.py | 40 +- .../community/aio_sandbox/__init__.py | 0 .../community/aio_sandbox/aio_sandbox.py | 2 +- .../aio_sandbox/aio_sandbox_provider.py | 10 +- .../community/aio_sandbox/backend.py | 0 .../community/aio_sandbox/local_backend.py | 2 +- .../community/aio_sandbox/remote_backend.py | 2 +- .../community/aio_sandbox/sandbox_info.py | 0 .../deerflow}/community/firecrawl/tools.py | 2 +- .../community/image_search/__init__.py | 0 .../deerflow}/community/image_search/tools.py | 2 +- .../community/infoquest/infoquest_client.py | 0 .../deerflow}/community/infoquest/tools.py | 4 +- .../community/jina_ai/jina_client.py | 0 .../deerflow}/community/jina_ai/tools.py | 6 +- .../deerflow}/community/tavily/tools.py | 2 +- .../harness/deerflow}/config/__init__.py | 0 .../harness/deerflow}/config/agents_config.py | 2 +- .../harness/deerflow}/config/app_config.py | 75 +++- .../deerflow}/config/checkpointer_config.py | 0 .../deerflow}/config/extensions_config.py | 0 .../harness/deerflow}/config/memory_config.py | 0 .../harness/deerflow}/config/model_config.py | 0 .../harness/deerflow}/config/paths.py | 0 .../deerflow}/config/sandbox_config.py | 2 +- .../harness/deerflow}/config/skills_config.py | 2 +- .../deerflow}/config/subagents_config.py | 0 .../deerflow}/config/summarization_config.py | 0 .../harness/deerflow}/config/title_config.py | 0 .../harness/deerflow}/config/tool_config.py | 2 +- .../deerflow}/config/tracing_config.py | 0 .../harness/deerflow}/mcp/__init__.py | 0 .../harness/deerflow}/mcp/cache.py | 4 +- .../harness/deerflow}/mcp/client.py | 2 +- .../harness/deerflow}/mcp/oauth.py | 2 +- .../harness/deerflow}/mcp/tools.py | 6 +- .../harness/deerflow}/models/__init__.py | 0 .../harness/deerflow}/models/factory.py | 4 +- .../deerflow}/models/patched_deepseek.py | 0 .../harness/deerflow}/reflection/__init__.py | 0 .../harness/deerflow}/reflection/resolvers.py | 0 .../harness/deerflow}/sandbox/__init__.py | 0 .../harness/deerflow}/sandbox/exceptions.py | 0 .../deerflow}/sandbox/local/__init__.py | 0 .../deerflow}/sandbox/local/list_dir.py | 0 .../deerflow}/sandbox/local/local_sandbox.py | 4 +- .../sandbox/local/local_sandbox_provider.py | 8 +- .../harness/deerflow}/sandbox/middleware.py | 4 +- .../harness/deerflow}/sandbox/sandbox.py | 0 .../deerflow}/sandbox/sandbox_provider.py | 6 +- .../harness/deerflow}/sandbox/tools.py | 10 +- .../harness/deerflow/skills/__init__.py | 5 + .../harness/deerflow}/skills/loader.py | 8 +- .../harness/deerflow}/skills/parser.py | 0 .../harness/deerflow}/skills/types.py | 0 .../harness/deerflow/skills/validation.py | 85 +++++ .../harness/deerflow}/subagents/__init__.py | 0 .../deerflow}/subagents/builtins/__init__.py | 0 .../subagents/builtins/bash_agent.py | 2 +- .../subagents/builtins/general_purpose.py | 2 +- .../harness/deerflow}/subagents/config.py | 0 .../harness/deerflow}/subagents/executor.py | 8 +- .../harness/deerflow}/subagents/registry.py | 6 +- .../harness/deerflow}/tools/__init__.py | 0 .../deerflow}/tools/builtins/__init__.py | 0 .../tools/builtins/clarification_tool.py | 0 .../tools/builtins/present_file_tool.py | 4 +- .../tools/builtins/setup_agent_tool.py | 2 +- .../deerflow}/tools/builtins/task_tool.py | 10 +- .../tools/builtins/view_image_tool.py | 4 +- .../harness/deerflow}/tools/tools.py | 12 +- .../harness/deerflow/utils/file_conversion.py | 47 +++ .../harness/deerflow}/utils/network.py | 0 .../harness/deerflow}/utils/readability.py | 0 backend/packages/harness/pyproject.toml | 40 ++ backend/pyproject.toml | 31 +- backend/ruff.toml | 3 + backend/src/skills/__init__.py | 4 - backend/tests/conftest.py | 14 +- .../tests/test_channel_file_attachments.py | 42 +-- backend/tests/test_channels.py | 114 +++--- backend/tests/test_checkpointer.py | 32 +- backend/tests/test_checkpointer_none_fix.py | 8 +- backend/tests/test_client.py | 196 +++++----- backend/tests/test_client_live.py | 2 +- backend/tests/test_config_version.py | 125 +++++++ backend/tests/test_custom_agent.py | 92 ++--- .../test_docker_sandbox_mode_detection.py | 8 +- backend/tests/test_harness_boundary.py | 46 +++ backend/tests/test_infoquest_client.py | 22 +- .../tests/test_lead_agent_model_resolution.py | 12 +- .../tests/test_loop_detection_middleware.py | 2 +- backend/tests/test_mcp_client_config.py | 4 +- backend/tests/test_mcp_oauth.py | 4 +- backend/tests/test_memory_prompt_injection.py | 4 +- backend/tests/test_memory_upload_filtering.py | 4 +- backend/tests/test_model_factory.py | 12 +- .../test_present_file_tool_core_logic.py | 2 +- backend/tests/test_readability.py | 6 +- backend/tests/test_reflection_resolvers.py | 4 +- backend/tests/test_sandbox_tools_security.py | 2 +- backend/tests/test_skills_archive_root.py | 2 +- backend/tests/test_skills_loader.py | 11 +- backend/tests/test_skills_router.py | 4 +- backend/tests/test_subagent_executor.py | 38 +- backend/tests/test_subagent_timeout_config.py | 30 +- backend/tests/test_suggestions_router.py | 2 +- backend/tests/test_task_tool_core_logic.py | 22 +- backend/tests/test_title_generation.py | 4 +- .../tests/test_title_middleware_core_logic.py | 8 +- .../test_tool_error_handling_middleware.py | 2 +- backend/tests/test_tracing_config.py | 4 +- .../test_uploads_middleware_core_logic.py | 4 +- backend/tests/test_uploads_router.py | 2 +- backend/uv.lock | 81 +++-- config.example.yaml | 39 +- docker/docker-compose-dev.yaml | 2 +- docker/docker-compose.yaml | 2 +- docs/CODE_CHANGE_SUMMARY_BY_FILE.md | 8 +- docs/SKILL_NAME_CONFLICT_FIX.md | 12 +- scripts/config-upgrade.sh | 146 ++++++++ scripts/deploy.sh | 2 +- scripts/docker.sh | 4 +- scripts/serve.sh | 16 +- scripts/start-daemon.sh | 16 +- scripts/tool-error-degradation-detection.sh | 12 +- 198 files changed, 1786 insertions(+), 941 deletions(-) rename backend/{src => app}/__init__.py (100%) rename backend/{src => app}/channels/__init__.py (79%) rename backend/{src => app}/channels/base.py (98%) rename backend/{src => app}/channels/feishu.py (99%) rename backend/{src => app}/channels/manager.py (97%) rename backend/{src => app}/channels/message_bus.py (100%) rename backend/{src => app}/channels/service.py (93%) rename backend/{src => app}/channels/slack.py (98%) rename backend/{src => app}/channels/store.py (98%) rename backend/{src => app}/channels/telegram.py (99%) rename backend/{src => app}/gateway/__init__.py (100%) rename backend/{src => app}/gateway/app.py (95%) rename backend/{src => app}/gateway/config.py (100%) rename backend/{src => app}/gateway/path_utils.py (95%) rename backend/{src => app}/gateway/routers/__init__.py (100%) rename backend/{src => app}/gateway/routers/agents.py (98%) rename backend/{src => app}/gateway/routers/artifacts.py (99%) rename backend/{src => app}/gateway/routers/channels.py (93%) rename backend/{src => app}/gateway/routers/mcp.py (98%) rename backend/{src => app}/gateway/routers/memory.py (98%) rename backend/{src => app}/gateway/routers/models.py (98%) rename backend/{src => app}/gateway/routers/skills.py (79%) rename backend/{src => app}/gateway/routers/suggestions.py (99%) rename backend/{src => app}/gateway/routers/uploads.py (86%) create mode 100644 backend/docs/HARNESS_APP_SPLIT.md create mode 100644 backend/packages/harness/deerflow/__init__.py rename backend/{src => packages/harness/deerflow}/agents/__init__.py (100%) rename backend/{src => packages/harness/deerflow}/agents/checkpointer/__init__.py (100%) rename backend/{src => packages/harness/deerflow}/agents/checkpointer/async_provider.py (92%) rename backend/{src => packages/harness/deerflow}/agents/checkpointer/provider.py (94%) rename backend/{src => packages/harness/deerflow}/agents/lead_agent/__init__.py (100%) rename backend/{src => packages/harness/deerflow}/agents/lead_agent/agent.py (92%) rename backend/{src => packages/harness/deerflow}/agents/lead_agent/prompt.py (98%) rename backend/{src => packages/harness/deerflow}/agents/memory/__init__.py (88%) rename backend/{src => packages/harness/deerflow}/agents/memory/prompt.py (100%) rename backend/{src => packages/harness/deerflow}/agents/memory/queue.py (97%) rename backend/{src => packages/harness/deerflow}/agents/memory/updater.py (98%) rename backend/{src => packages/harness/deerflow}/agents/middlewares/clarification_middleware.py (100%) rename backend/{src => packages/harness/deerflow}/agents/middlewares/dangling_tool_call_middleware.py (100%) rename backend/{src => packages/harness/deerflow}/agents/middlewares/loop_detection_middleware.py (100%) rename backend/{src => packages/harness/deerflow}/agents/middlewares/memory_middleware.py (97%) rename backend/{src => packages/harness/deerflow}/agents/middlewares/subagent_limit_middleware.py (97%) rename backend/{src => packages/harness/deerflow}/agents/middlewares/thread_data_middleware.py (96%) rename backend/{src => packages/harness/deerflow}/agents/middlewares/title_middleware.py (97%) rename backend/{src => packages/harness/deerflow}/agents/middlewares/todo_middleware.py (100%) rename backend/{src => packages/harness/deerflow}/agents/middlewares/tool_error_handling_middleware.py (87%) rename backend/{src => packages/harness/deerflow}/agents/middlewares/uploads_middleware.py (99%) rename backend/{src => packages/harness/deerflow}/agents/middlewares/view_image_middleware.py (99%) rename backend/{src => packages/harness/deerflow}/agents/thread_state.py (100%) rename backend/{src => packages/harness/deerflow}/client.py (96%) rename backend/{src => packages/harness/deerflow}/community/aio_sandbox/__init__.py (100%) rename backend/{src => packages/harness/deerflow}/community/aio_sandbox/aio_sandbox.py (99%) rename backend/{src => packages/harness/deerflow}/community/aio_sandbox/aio_sandbox_provider.py (99%) rename backend/{src => packages/harness/deerflow}/community/aio_sandbox/backend.py (100%) rename backend/{src => packages/harness/deerflow}/community/aio_sandbox/local_backend.py (99%) rename backend/{src => packages/harness/deerflow}/community/aio_sandbox/remote_backend.py (98%) rename backend/{src => packages/harness/deerflow}/community/aio_sandbox/sandbox_info.py (100%) rename backend/{src => packages/harness/deerflow}/community/firecrawl/tools.py (98%) rename backend/{src => packages/harness/deerflow}/community/image_search/__init__.py (100%) rename backend/{src => packages/harness/deerflow}/community/image_search/tools.py (99%) rename backend/{src => packages/harness/deerflow}/community/infoquest/infoquest_client.py (100%) rename backend/{src => packages/harness/deerflow}/community/infoquest/tools.py (95%) rename backend/{src => packages/harness/deerflow}/community/jina_ai/jina_client.py (100%) rename backend/{src => packages/harness/deerflow}/community/jina_ai/tools.py (87%) rename backend/{src => packages/harness/deerflow}/community/tavily/tools.py (98%) rename backend/{src => packages/harness/deerflow}/config/__init__.py (100%) rename backend/{src => packages/harness/deerflow}/config/agents_config.py (98%) rename backend/{src => packages/harness/deerflow}/config/app_config.py (73%) rename backend/{src => packages/harness/deerflow}/config/checkpointer_config.py (100%) rename backend/{src => packages/harness/deerflow}/config/extensions_config.py (100%) rename backend/{src => packages/harness/deerflow}/config/memory_config.py (100%) rename backend/{src => packages/harness/deerflow}/config/model_config.py (100%) rename backend/{src => packages/harness/deerflow}/config/paths.py (100%) rename backend/{src => packages/harness/deerflow}/config/sandbox_config.py (95%) rename backend/{src => packages/harness/deerflow}/config/skills_config.py (95%) rename backend/{src => packages/harness/deerflow}/config/subagents_config.py (100%) rename backend/{src => packages/harness/deerflow}/config/summarization_config.py (100%) rename backend/{src => packages/harness/deerflow}/config/title_config.py (100%) rename backend/{src => packages/harness/deerflow}/config/tool_config.py (84%) rename backend/{src => packages/harness/deerflow}/config/tracing_config.py (100%) rename backend/{src => packages/harness/deerflow}/mcp/__init__.py (100%) rename backend/{src => packages/harness/deerflow}/mcp/cache.py (97%) rename backend/{src => packages/harness/deerflow}/mcp/client.py (96%) rename backend/{src => packages/harness/deerflow}/mcp/oauth.py (98%) rename backend/{src => packages/harness/deerflow}/mcp/tools.py (92%) rename backend/{src => packages/harness/deerflow}/models/__init__.py (100%) rename backend/{src => packages/harness/deerflow}/models/factory.py (96%) rename backend/{src => packages/harness/deerflow}/models/patched_deepseek.py (100%) rename backend/{src => packages/harness/deerflow}/reflection/__init__.py (100%) rename backend/{src => packages/harness/deerflow}/reflection/resolvers.py (100%) rename backend/{src => packages/harness/deerflow}/sandbox/__init__.py (100%) rename backend/{src => packages/harness/deerflow}/sandbox/exceptions.py (100%) rename backend/{src => packages/harness/deerflow}/sandbox/local/__init__.py (100%) rename backend/{src => packages/harness/deerflow}/sandbox/local/list_dir.py (100%) rename backend/{src => packages/harness/deerflow}/sandbox/local/local_sandbox.py (98%) rename backend/{src => packages/harness/deerflow}/sandbox/local/local_sandbox_provider.py (89%) rename backend/{src => packages/harness/deerflow}/sandbox/middleware.py (96%) rename backend/{src => packages/harness/deerflow}/sandbox/sandbox.py (100%) rename backend/{src => packages/harness/deerflow}/sandbox/sandbox_provider.py (95%) rename backend/{src => packages/harness/deerflow}/sandbox/tools.py (98%) create mode 100644 backend/packages/harness/deerflow/skills/__init__.py rename backend/{src => packages/harness/deerflow}/skills/loader.py (91%) rename backend/{src => packages/harness/deerflow}/skills/parser.py (100%) rename backend/{src => packages/harness/deerflow}/skills/types.py (100%) create mode 100644 backend/packages/harness/deerflow/skills/validation.py rename backend/{src => packages/harness/deerflow}/subagents/__init__.py (100%) rename backend/{src => packages/harness/deerflow}/subagents/builtins/__init__.py (100%) rename backend/{src => packages/harness/deerflow}/subagents/builtins/bash_agent.py (96%) rename backend/{src => packages/harness/deerflow}/subagents/builtins/general_purpose.py (97%) rename backend/{src => packages/harness/deerflow}/subagents/config.py (100%) rename backend/{src => packages/harness/deerflow}/subagents/executor.py (98%) rename backend/{src => packages/harness/deerflow}/subagents/registry.py (88%) rename backend/{src => packages/harness/deerflow}/tools/__init__.py (100%) rename backend/{src => packages/harness/deerflow}/tools/builtins/__init__.py (100%) rename backend/{src => packages/harness/deerflow}/tools/builtins/clarification_tool.py (100%) rename backend/{src => packages/harness/deerflow}/tools/builtins/present_file_tool.py (96%) rename backend/{src => packages/harness/deerflow}/tools/builtins/setup_agent_tool.py (97%) rename backend/{src => packages/harness/deerflow}/tools/builtins/task_tool.py (95%) rename backend/{src => packages/harness/deerflow}/tools/builtins/view_image_tool.py (96%) rename backend/{src => packages/harness/deerflow}/tools/tools.py (87%) create mode 100644 backend/packages/harness/deerflow/utils/file_conversion.py rename backend/{src => packages/harness/deerflow}/utils/network.py (100%) rename backend/{src => packages/harness/deerflow}/utils/readability.py (100%) create mode 100644 backend/packages/harness/pyproject.toml delete mode 100644 backend/src/skills/__init__.py create mode 100644 backend/tests/test_config_version.py create mode 100644 backend/tests/test_harness_boundary.py create mode 100755 scripts/config-upgrade.sh diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 63a424d..5c6f32f 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -143,12 +143,12 @@ Root-level orchestration and config: Backend core: -- `backend/src/agents/` - lead agent, middleware chain, memory -- `backend/src/gateway/` - FastAPI gateway API -- `backend/src/sandbox/` - sandbox provider + tool wrappers -- `backend/src/subagents/` - subagent registry/execution -- `backend/src/mcp/` - MCP integration -- `backend/langgraph.json` - graph entrypoint (`src.agents:make_lead_agent`) +- `backend/packages/harness/deerflow/agents/` - lead agent, middleware chain, memory +- `backend/app/gateway/` - FastAPI gateway API +- `backend/packages/harness/deerflow/sandbox/` - sandbox provider + tool wrappers +- `backend/packages/harness/deerflow/subagents/` - subagent registry/execution +- `backend/packages/harness/deerflow/mcp/` - MCP integration +- `backend/langgraph.json` - graph entrypoint (`deerflow.agents:make_lead_agent`) - `backend/pyproject.toml` - Python deps and `requires-python` - `backend/ruff.toml` - lint/format policy - `backend/tests/` - backend unit and integration-like tests diff --git a/.gitignore b/.gitignore index 4400f01..1f4be07 100644 --- a/.gitignore +++ b/.gitignore @@ -51,3 +51,4 @@ web/ # Deployment artifacts backend/Dockerfile.langgraph +config.yaml.bak diff --git a/Makefile b/Makefile index 9e390ee..4581652 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,13 @@ # DeerFlow - Unified Development Environment -.PHONY: help config check install dev dev-daemon start stop up down clean docker-init docker-start docker-stop docker-logs docker-logs-frontend docker-logs-gateway +.PHONY: help config config-upgrade check install dev dev-daemon start stop up down clean docker-init docker-start docker-stop docker-logs docker-logs-frontend docker-logs-gateway PYTHON ?= python help: @echo "DeerFlow Development Commands:" @echo " make config - Generate local config files (aborts if config already exists)" + @echo " make config-upgrade - Merge new fields from config.example.yaml into config.yaml" @echo " make check - Check if all required tools are installed" @echo " make install - Install all dependencies (frontend + backend)" @echo " make setup-sandbox - Pre-pull sandbox container image (recommended)" @@ -31,6 +32,9 @@ help: config: @$(PYTHON) ./scripts/configure.py +config-upgrade: + @./scripts/config-upgrade.sh + # Check required tools check: @$(PYTHON) ./scripts/check.py @@ -96,7 +100,7 @@ dev-daemon: stop: @echo "Stopping all services..." @-pkill -f "langgraph dev" 2>/dev/null || true - @-pkill -f "uvicorn src.gateway.app:app" 2>/dev/null || true + @-pkill -f "uvicorn app.gateway.app:app" 2>/dev/null || true @-pkill -f "next dev" 2>/dev/null || true @-pkill -f "next start" 2>/dev/null || true @-pkill -f "next-server" 2>/dev/null || true diff --git a/README.md b/README.md index e8aca44..a285285 100644 --- a/README.md +++ b/README.md @@ -145,7 +145,7 @@ make docker-init # Pull sandbox image (only once or when image updates) make docker-start # Start services (auto-detects sandbox mode from config.yaml) ``` -`make docker-start` starts `provisioner` only when `config.yaml` uses provisioner mode (`sandbox.use: src.community.aio_sandbox:AioSandboxProvider` with `provisioner_url`). +`make docker-start` starts `provisioner` only when `config.yaml` uses provisioner mode (`sandbox.use: deerflow.community.aio_sandbox:AioSandboxProvider` with `provisioner_url`). **Production** (builds images locally, mounts runtime config and data): @@ -437,7 +437,7 @@ DeerFlow is model-agnostic — it works with any LLM that implements the OpenAI- DeerFlow can be used as an embedded Python library without running the full HTTP services. The `DeerFlowClient` provides direct in-process access to all agent and Gateway capabilities, returning the same response schemas as the HTTP Gateway API: ```python -from src.client import DeerFlowClient +from deerflow.client import DeerFlowClient client = DeerFlowClient() @@ -456,7 +456,7 @@ client.update_skill("web-search", enabled=True) client.upload_files("thread-1", ["./report.pdf"]) # {"success": True, "files": [...]} ``` -All dict-returning methods are validated against Gateway Pydantic response models in CI (`TestGatewayConformance`), ensuring the embedded client stays in sync with the HTTP API schemas. See `backend/src/client.py` for full API documentation. +All dict-returning methods are validated against Gateway Pydantic response models in CI (`TestGatewayConformance`), ensuring the embedded client stays in sync with the HTTP API schemas. See `backend/packages/harness/deerflow/client.py` for full API documentation. ## Documentation diff --git a/backend/CLAUDE.md b/backend/CLAUDE.md index 5d5e000..fb38d44 100644 --- a/backend/CLAUDE.md +++ b/backend/CLAUDE.md @@ -22,33 +22,38 @@ deer-flow/ ├── backend/ # Backend application (this directory) │ ├── Makefile # Backend-only commands (dev, gateway, lint) │ ├── langgraph.json # LangGraph server configuration -│ ├── src/ -│ │ ├── agents/ # LangGraph agent system -│ │ │ ├── lead_agent/ # Main agent (factory + system prompt) -│ │ │ ├── middlewares/ # 10 middleware components -│ │ │ ├── memory/ # Memory extraction, queue, prompts -│ │ │ └── thread_state.py # ThreadState schema +│ ├── packages/ +│ │ └── harness/ # deerflow-harness package (import: deerflow.*) +│ │ ├── pyproject.toml +│ │ └── deerflow/ +│ │ ├── agents/ # LangGraph agent system +│ │ │ ├── lead_agent/ # Main agent (factory + system prompt) +│ │ │ ├── middlewares/ # 10 middleware components +│ │ │ ├── memory/ # Memory extraction, queue, prompts +│ │ │ └── thread_state.py # ThreadState schema +│ │ ├── sandbox/ # Sandbox execution system +│ │ │ ├── local/ # Local filesystem provider +│ │ │ ├── sandbox.py # Abstract Sandbox interface +│ │ │ ├── tools.py # bash, ls, read/write/str_replace +│ │ │ └── middleware.py # Sandbox lifecycle management +│ │ ├── subagents/ # Subagent delegation system +│ │ │ ├── builtins/ # general-purpose, bash agents +│ │ │ ├── executor.py # Background execution engine +│ │ │ └── registry.py # Agent registry +│ │ ├── tools/builtins/ # Built-in tools (present_files, ask_clarification, view_image) +│ │ ├── mcp/ # MCP integration (tools, cache, client) +│ │ ├── models/ # Model factory with thinking/vision support +│ │ ├── skills/ # Skills discovery, loading, parsing +│ │ ├── config/ # Configuration system (app, model, sandbox, tool, etc.) +│ │ ├── community/ # Community tools (tavily, jina_ai, firecrawl, image_search, aio_sandbox) +│ │ ├── reflection/ # Dynamic module loading (resolve_variable, resolve_class) +│ │ ├── utils/ # Utilities (network, readability) +│ │ └── client.py # Embedded Python client (DeerFlowClient) +│ ├── app/ # Application layer (import: app.*) │ │ ├── gateway/ # FastAPI Gateway API │ │ │ ├── app.py # FastAPI application │ │ │ └── routers/ # 6 route modules -│ │ ├── sandbox/ # Sandbox execution system -│ │ │ ├── local/ # Local filesystem provider -│ │ │ ├── sandbox.py # Abstract Sandbox interface -│ │ │ ├── tools.py # bash, ls, read/write/str_replace -│ │ │ └── middleware.py # Sandbox lifecycle management -│ │ ├── subagents/ # Subagent delegation system -│ │ │ ├── builtins/ # general-purpose, bash agents -│ │ │ ├── executor.py # Background execution engine -│ │ │ └── registry.py # Agent registry -│ │ ├── tools/builtins/ # Built-in tools (present_files, ask_clarification, view_image) -│ │ ├── mcp/ # MCP integration (tools, cache, client) -│ │ ├── models/ # Model factory with thinking/vision support -│ │ ├── skills/ # Skills discovery, loading, parsing -│ │ ├── config/ # Configuration system (app, model, sandbox, tool, etc.) -│ │ ├── community/ # Community tools (tavily, jina_ai, firecrawl, image_search, aio_sandbox) -│ │ ├── reflection/ # Dynamic module loading (resolve_variable, resolve_class) -│ │ ├── utils/ # Utilities (network, readability) -│ │ └── client.py # Embedded Python client (DeerFlowClient) +│ │ └── channels/ # IM platform integrations │ ├── tests/ # Test suite │ └── docs/ # Documentation ├── frontend/ # Next.js frontend application @@ -92,19 +97,48 @@ Regression tests related to Docker/provisioner behavior: - `tests/test_docker_sandbox_mode_detection.py` (mode detection from `config.yaml`) - `tests/test_provisioner_kubeconfig.py` (kubeconfig file/directory handling) +Boundary check (harness → app import firewall): +- `tests/test_harness_boundary.py` — ensures `packages/harness/deerflow/` never imports from `app.*` + CI runs these regression tests for every pull request via [.github/workflows/backend-unit-tests.yml](../.github/workflows/backend-unit-tests.yml). ## Architecture +### Harness / App Split + +The backend is split into two layers with a strict dependency direction: + +- **Harness** (`packages/harness/deerflow/`): Publishable agent framework package (`deerflow-harness`). Import prefix: `deerflow.*`. Contains agent orchestration, tools, sandbox, models, MCP, skills, config — everything needed to build and run agents. +- **App** (`app/`): Unpublished application code. Import prefix: `app.*`. Contains the FastAPI Gateway API and IM channel integrations (Feishu, Slack, Telegram). + +**Dependency rule**: App imports deerflow, but deerflow never imports app. This boundary is enforced by `tests/test_harness_boundary.py` which runs in CI. + +**Import conventions**: +```python +# Harness internal +from deerflow.agents import make_lead_agent +from deerflow.models import create_chat_model + +# App internal +from app.gateway.app import app +from app.channels.service import start_channel_service + +# App → Harness (allowed) +from deerflow.config import get_app_config + +# Harness → App (FORBIDDEN — enforced by test_harness_boundary.py) +# from app.gateway.routers.uploads import ... # ← will fail CI +``` + ### Agent System -**Lead Agent** (`src/agents/lead_agent/agent.py`): +**Lead Agent** (`packages/harness/deerflow/agents/lead_agent/agent.py`): - Entry point: `make_lead_agent(config: RunnableConfig)` registered in `langgraph.json` - Dynamic model selection via `create_chat_model()` with thinking/vision support - Tools loaded via `get_available_tools()` - combines sandbox, built-in, MCP, community, and subagent tools - System prompt generated by `apply_prompt_template()` with skills, memory, and subagent instructions -**ThreadState** (`src/agents/thread_state.py`): +**ThreadState** (`packages/harness/deerflow/agents/thread_state.py`): - Extends `AgentState` with: `sandbox`, `thread_data`, `title`, `artifacts`, `todos`, `uploaded_files`, `viewed_images` - Uses custom reducers: `merge_artifacts` (deduplicate), `merge_viewed_images` (merge/clear) @@ -116,7 +150,7 @@ CI runs these regression tests for every pull request via [.github/workflows/bac ### Middleware Chain -Middlewares execute in strict order in `src/agents/lead_agent/agent.py`: +Middlewares execute in strict order in `packages/harness/deerflow/agents/lead_agent/agent.py`: 1. **ThreadDataMiddleware** - Creates per-thread directories (`backend/.deer-flow/threads/{thread_id}/user-data/{workspace,uploads,outputs}`) 2. **UploadsMiddleware** - Tracks and injects newly uploaded files into conversation @@ -136,6 +170,8 @@ Middlewares execute in strict order in `src/agents/lead_agent/agent.py`: Setup: Copy `config.example.yaml` to `config.yaml` in the **project root** directory. +**Config Versioning**: `config.example.yaml` has a `config_version` field. On startup, `AppConfig.from_file()` compares user version vs example version and emits a warning if outdated. Missing `config_version` = version 0. Run `make config-upgrade` to auto-merge missing fields. When changing the config schema, bump `config_version` in `config.example.yaml`. + Configuration priority: 1. Explicit `config_path` argument 2. `DEER_FLOW_CONFIG_PATH` environment variable @@ -154,7 +190,7 @@ Configuration priority: 3. `extensions_config.json` in current directory (backend/) 4. `extensions_config.json` in parent directory (project root - **recommended location**) -### Gateway API (`src/gateway/`) +### Gateway API (`app/gateway/`) FastAPI application on port 8001 with health check at `GET /health`. @@ -172,13 +208,13 @@ FastAPI application on port 8001 with health check at `GET /health`. Proxied through nginx: `/api/langgraph/*` → LangGraph, all other `/api/*` → Gateway. -### Sandbox System (`src/sandbox/`) +### Sandbox System (`packages/harness/deerflow/sandbox/`) **Interface**: Abstract `Sandbox` with `execute_command`, `read_file`, `write_file`, `list_dir` **Provider Pattern**: `SandboxProvider` with `acquire`, `get`, `release` lifecycle **Implementations**: - `LocalSandboxProvider` - Singleton local filesystem execution with path mappings -- `AioSandboxProvider` (`src/community/`) - Docker-based isolation +- `AioSandboxProvider` (`packages/harness/deerflow/community/`) - Docker-based isolation **Virtual Path System**: - Agent sees: `/mnt/user-data/{workspace,uploads,outputs}`, `/mnt/skills` @@ -186,14 +222,14 @@ Proxied through nginx: `/api/langgraph/*` → LangGraph, all other `/api/*` → - Translation: `replace_virtual_path()` / `replace_virtual_paths_in_command()` - Detection: `is_local_sandbox()` checks `sandbox_id == "local"` -**Sandbox Tools** (in `src/sandbox/tools.py`): +**Sandbox Tools** (in `packages/harness/deerflow/sandbox/tools.py`): - `bash` - Execute commands with path translation and error handling - `ls` - Directory listing (tree format, max 2 levels) - `read_file` - Read file contents with optional line range - `write_file` - Write/append to files, creates directories - `str_replace` - Substring replacement (single or all occurrences) -### Subagent System (`src/subagents/`) +### Subagent System (`packages/harness/deerflow/subagents/`) **Built-in Agents**: `general-purpose` (all tools except `task`) and `bash` (command specialist) **Execution**: Dual thread pool - `_scheduler_pool` (3 workers) + `_execution_pool` (3 workers) @@ -201,7 +237,7 @@ Proxied through nginx: `/api/langgraph/*` → LangGraph, all other `/api/*` → **Flow**: `task()` tool → `SubagentExecutor` → background thread → poll 5s → SSE events → result **Events**: `task_started`, `task_running`, `task_completed`/`task_failed`/`task_timed_out` -### Tool System (`src/tools/`) +### Tool System (`packages/harness/deerflow/tools/`) `get_available_tools(groups, include_mcp, model_name, subagent_enabled)` assembles: 1. **Config-defined tools** - Resolved from `config.yaml` via `resolve_variable()` @@ -213,13 +249,13 @@ Proxied through nginx: `/api/langgraph/*` → LangGraph, all other `/api/*` → 4. **Subagent tool** (if enabled): - `task` - Delegate to subagent (description, prompt, subagent_type, max_turns) -**Community tools** (`src/community/`): +**Community tools** (`packages/harness/deerflow/community/`): - `tavily/` - Web search (5 results default) and web fetch (4KB limit) - `jina_ai/` - Web fetch via Jina reader API with readability extraction - `firecrawl/` - Web scraping via Firecrawl API - `image_search/` - Image search via DuckDuckGo -### MCP System (`src/mcp/`) +### MCP System (`packages/harness/deerflow/mcp/`) - Uses `langchain-mcp-adapters` `MultiServerMCPClient` for multi-server management - **Lazy initialization**: Tools loaded on first use via `get_cached_mcp_tools()` @@ -228,7 +264,7 @@ Proxied through nginx: `/api/langgraph/*` → LangGraph, all other `/api/*` → - **OAuth (HTTP/SSE)**: Supports token endpoint flows (`client_credentials`, `refresh_token`) with automatic token refresh + Authorization header injection - **Runtime updates**: Gateway API saves to extensions_config.json; LangGraph detects via mtime -### Skills System (`src/skills/`) +### Skills System (`packages/harness/deerflow/skills/`) - **Location**: `deer-flow/skills/{public,custom}/` - **Format**: Directory with `SKILL.md` (YAML frontmatter: name, description, license, allowed-tools) @@ -236,7 +272,7 @@ Proxied through nginx: `/api/langgraph/*` → LangGraph, all other `/api/*` → - **Injection**: Enabled skills listed in agent system prompt with container paths - **Installation**: `POST /api/skills/install` extracts .skill ZIP archive to custom/ directory -### Model Factory (`src/models/factory.py`) +### Model Factory (`packages/harness/deerflow/models/factory.py`) - `create_chat_model(name, thinking_enabled)` instantiates LLM from config via reflection - Supports `thinking_enabled` flag with per-model `when_thinking_enabled` overrides @@ -244,7 +280,7 @@ Proxied through nginx: `/api/langgraph/*` → LangGraph, all other `/api/*` → - Config values starting with `$` resolved as environment variables - Missing provider modules surface actionable install hints from reflection resolvers (for example `uv add langchain-google-genai`) -### IM Channels System (`src/channels/`) +### IM Channels System (`app/channels/`) Bridges external messaging platforms (Feishu, Slack, Telegram) to the DeerFlow agent via the LangGraph Server. @@ -273,7 +309,7 @@ Bridges external messaging platforms (Feishu, Slack, Telegram) to the DeerFlow a - `gateway_url` - Gateway API URL for auxiliary commands (default: `http://localhost:8001`) - Per-channel configs: `feishu` (app_id, app_secret), `slack` (bot_token, app_token), `telegram` (bot_token) -### Memory System (`src/agents/memory/`) +### Memory System (`packages/harness/deerflow/agents/memory/`) **Components**: - `updater.py` - LLM-based memory updates with fact extraction and atomic file I/O @@ -300,7 +336,7 @@ Bridges external messaging platforms (Feishu, Slack, Telegram) to the DeerFlow a - `max_facts` / `fact_confidence_threshold` - Fact storage limits (100 / 0.7) - `max_injection_tokens` - Token limit for prompt injection (2000) -### Reflection System (`src/reflection/`) +### Reflection System (`packages/harness/deerflow/reflection/`) - `resolve_variable(path)` - Import module and return variable (e.g., `module.path:variable_name`) - `resolve_class(path, base_class)` - Import and validate class against base class @@ -324,11 +360,11 @@ Bridges external messaging platforms (Feishu, Slack, Telegram) to the DeerFlow a Both can be modified at runtime via Gateway API endpoints or `DeerFlowClient` methods. -### Embedded Client (`src/client.py`) +### Embedded Client (`packages/harness/deerflow/client.py`) `DeerFlowClient` provides direct in-process access to all DeerFlow capabilities without HTTP services. All return types align with the Gateway API response schemas, so consumer code works identically in HTTP and embedded modes. -**Architecture**: Imports the same `src/` modules that LangGraph Server and Gateway API use. Shares the same config files and data directories. No FastAPI dependency. +**Architecture**: Imports the same `deerflow` modules that LangGraph Server and Gateway API use. Shares the same config files and data directories. No FastAPI dependency. **Agent Conversation** (replaces LangGraph Server): - `chat(message, thread_id)` — synchronous, returns final text @@ -367,7 +403,7 @@ Both can be modified at runtime via Gateway API endpoints or `DeerFlowClient` me - Run the full suite before and after your change: `make test` - Tests must pass before a feature is considered complete - For lightweight config/utility modules, prefer pure unit tests with no external dependencies -- If a module causes circular import issues in tests, add a `sys.modules` mock in `tests/conftest.py` (see existing example for `src.subagents.executor`) +- If a module causes circular import issues in tests, add a `sys.modules` mock in `tests/conftest.py` (see existing example for `deerflow.subagents.executor`) ```bash # Run all tests diff --git a/backend/CONTRIBUTING.md b/backend/CONTRIBUTING.md index 7ac0a0d..322710e 100644 --- a/backend/CONTRIBUTING.md +++ b/backend/CONTRIBUTING.md @@ -227,7 +227,7 @@ Example test: ```python import pytest -from src.models.factory import create_chat_model +from deerflow.models.factory import create_chat_model def test_create_chat_model_with_valid_name(): """Test that a valid model name creates a model instance.""" @@ -269,10 +269,10 @@ Include in your PR description: ### Adding New Tools -1. Create tool in `src/tools/builtins/` or `src/community/`: +1. Create tool in `packages/harness/deerflow/tools/builtins/` or `packages/harness/deerflow/community/`: ```python -# src/tools/builtins/my_tool.py +# packages/harness/deerflow/tools/builtins/my_tool.py from langchain_core.tools import tool @tool @@ -294,15 +294,15 @@ def my_tool(param: str) -> str: tools: - name: my_tool group: my_group - use: src.tools.builtins.my_tool:my_tool + use: deerflow.tools.builtins.my_tool:my_tool ``` ### Adding New Middleware -1. Create middleware in `src/agents/middlewares/`: +1. Create middleware in `packages/harness/deerflow/agents/middlewares/`: ```python -# src/agents/middlewares/my_middleware.py +# packages/harness/deerflow/agents/middlewares/my_middleware.py from langchain.agents.middleware import BaseMiddleware from langchain_core.runnables import RunnableConfig @@ -315,7 +315,7 @@ class MyMiddleware(BaseMiddleware): return state ``` -2. Register in `src/agents/lead_agent/agent.py`: +2. Register in `packages/harness/deerflow/agents/lead_agent/agent.py`: ```python middlewares = [ @@ -329,10 +329,10 @@ middlewares = [ ### Adding New API Endpoints -1. Create router in `src/gateway/routers/`: +1. Create router in `app/gateway/routers/`: ```python -# src/gateway/routers/my_router.py +# app/gateway/routers/my_router.py from fastapi import APIRouter router = APIRouter(prefix="/my-endpoint", tags=["my-endpoint"]) @@ -348,10 +348,10 @@ async def create_item(data: dict): return {"created": data} ``` -2. Register in `src/gateway/app.py`: +2. Register in `app/gateway/app.py`: ```python -from src.gateway.routers import my_router +from app.gateway.routers import my_router app.include_router(my_router.router) ``` @@ -360,7 +360,7 @@ app.include_router(my_router.router) When adding new configuration options: -1. Update `src/config/app_config.py` with new fields +1. Update `packages/harness/deerflow/config/app_config.py` with new fields 2. Add default values in `config.example.yaml` 3. Document in `docs/CONFIGURATION.md` diff --git a/backend/Dockerfile b/backend/Dockerfile index e3f2cfe..945aeb3 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -36,4 +36,4 @@ RUN --mount=type=cache,target=/root/.cache/uv \ EXPOSE 8001 2024 # Default command (can be overridden in docker-compose) -CMD ["sh", "-c", "uv run uvicorn src.gateway.app:app --host 0.0.0.0 --port 8001"] +CMD ["sh", "-c", "cd backend && PYTHONPATH=. uv run uvicorn app.gateway.app:app --host 0.0.0.0 --port 8001"] diff --git a/backend/Makefile b/backend/Makefile index 32dc4bf..adc8628 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -5,7 +5,7 @@ dev: uv run langgraph dev --no-browser --allow-blocking --no-reload gateway: - uv run uvicorn src.gateway.app:app --host 0.0.0.0 --port 8001 + PYTHONPATH=. uv run uvicorn app.gateway.app:app --host 0.0.0.0 --port 8001 test: PYTHONPATH=. uv run pytest tests/ -v diff --git a/backend/src/__init__.py b/backend/app/__init__.py similarity index 100% rename from backend/src/__init__.py rename to backend/app/__init__.py diff --git a/backend/src/channels/__init__.py b/backend/app/channels/__init__.py similarity index 79% rename from backend/src/channels/__init__.py rename to backend/app/channels/__init__.py index b7b2153..4a583c0 100644 --- a/backend/src/channels/__init__.py +++ b/backend/app/channels/__init__.py @@ -5,8 +5,8 @@ Provides a pluggable channel system that connects external messaging platforms which uses ``langgraph-sdk`` to communicate with the underlying LangGraph Server. """ -from src.channels.base import Channel -from src.channels.message_bus import InboundMessage, MessageBus, OutboundMessage +from app.channels.base import Channel +from app.channels.message_bus import InboundMessage, MessageBus, OutboundMessage __all__ = [ "Channel", diff --git a/backend/src/channels/base.py b/backend/app/channels/base.py similarity index 98% rename from backend/src/channels/base.py rename to backend/app/channels/base.py index 70111a9..d923653 100644 --- a/backend/src/channels/base.py +++ b/backend/app/channels/base.py @@ -6,7 +6,7 @@ import logging from abc import ABC, abstractmethod from typing import Any -from src.channels.message_bus import InboundMessage, InboundMessageType, MessageBus, OutboundMessage, ResolvedAttachment +from app.channels.message_bus import InboundMessage, InboundMessageType, MessageBus, OutboundMessage, ResolvedAttachment logger = logging.getLogger(__name__) diff --git a/backend/src/channels/feishu.py b/backend/app/channels/feishu.py similarity index 99% rename from backend/src/channels/feishu.py rename to backend/app/channels/feishu.py index 0e5b1a5..86aa46a 100644 --- a/backend/src/channels/feishu.py +++ b/backend/app/channels/feishu.py @@ -8,8 +8,8 @@ import logging import threading from typing import Any -from src.channels.base import Channel -from src.channels.message_bus import InboundMessageType, MessageBus, OutboundMessage, ResolvedAttachment +from app.channels.base import Channel +from app.channels.message_bus import InboundMessageType, MessageBus, OutboundMessage, ResolvedAttachment logger = logging.getLogger(__name__) diff --git a/backend/src/channels/manager.py b/backend/app/channels/manager.py similarity index 97% rename from backend/src/channels/manager.py rename to backend/app/channels/manager.py index 614a091..4146843 100644 --- a/backend/src/channels/manager.py +++ b/backend/app/channels/manager.py @@ -9,8 +9,8 @@ import time from collections.abc import Mapping from typing import Any -from src.channels.message_bus import InboundMessage, InboundMessageType, MessageBus, OutboundMessage, ResolvedAttachment -from src.channels.store import ChannelStore +from app.channels.message_bus import InboundMessage, InboundMessageType, MessageBus, OutboundMessage, ResolvedAttachment +from app.channels.store import ChannelStore logger = logging.getLogger(__name__) @@ -242,7 +242,7 @@ def _resolve_attachments(thread_id: str, artifacts: list[str]) -> list[ResolvedA Skips artifacts that cannot be resolved (missing files, invalid paths) and logs warnings for them. """ - from src.config.paths import get_paths + from deerflow.config.paths import get_paths attachments: list[ResolvedAttachment] = [] paths = get_paths() @@ -266,14 +266,16 @@ def _resolve_attachments(thread_id: str, artifacts: list[str]) -> list[ResolvedA continue mime, _ = mimetypes.guess_type(str(actual)) mime = mime or "application/octet-stream" - attachments.append(ResolvedAttachment( - virtual_path=virtual_path, - actual_path=actual, - filename=actual.name, - mime_type=mime, - size=actual.stat().st_size, - is_image=mime.startswith("image/"), - )) + attachments.append( + ResolvedAttachment( + virtual_path=virtual_path, + actual_path=actual, + filename=actual.name, + mime_type=mime, + size=actual.stat().st_size, + is_image=mime.startswith("image/"), + ) + ) except (ValueError, OSError) as exc: logger.warning("[Manager] failed to resolve artifact %s: %s", virtual_path, exc) return attachments @@ -348,12 +350,7 @@ class ChannelManager: def _resolve_run_params(self, msg: InboundMessage, thread_id: str) -> tuple[str, dict[str, Any], dict[str, Any]]: channel_layer, user_layer = self._resolve_session_layer(msg) - assistant_id = ( - user_layer.get("assistant_id") - or channel_layer.get("assistant_id") - or self._default_session.get("assistant_id") - or self._assistant_id - ) + assistant_id = user_layer.get("assistant_id") or channel_layer.get("assistant_id") or self._default_session.get("assistant_id") or self._assistant_id if not isinstance(assistant_id, str) or not assistant_id.strip(): assistant_id = self._assistant_id diff --git a/backend/src/channels/message_bus.py b/backend/app/channels/message_bus.py similarity index 100% rename from backend/src/channels/message_bus.py rename to backend/app/channels/message_bus.py diff --git a/backend/src/channels/service.py b/backend/app/channels/service.py similarity index 93% rename from backend/src/channels/service.py rename to backend/app/channels/service.py index 1ff1de6..1a07c21 100644 --- a/backend/src/channels/service.py +++ b/backend/app/channels/service.py @@ -5,17 +5,17 @@ from __future__ import annotations import logging from typing import Any -from src.channels.manager import ChannelManager -from src.channels.message_bus import MessageBus -from src.channels.store import ChannelStore +from app.channels.manager import ChannelManager +from app.channels.message_bus import MessageBus +from app.channels.store import ChannelStore logger = logging.getLogger(__name__) # Channel name → import path for lazy loading _CHANNEL_REGISTRY: dict[str, str] = { - "feishu": "src.channels.feishu:FeishuChannel", - "slack": "src.channels.slack:SlackChannel", - "telegram": "src.channels.telegram:TelegramChannel", + "feishu": "app.channels.feishu:FeishuChannel", + "slack": "app.channels.slack:SlackChannel", + "telegram": "app.channels.telegram:TelegramChannel", } @@ -49,7 +49,7 @@ class ChannelService: @classmethod def from_app_config(cls) -> ChannelService: """Create a ChannelService from the application config.""" - from src.config.app_config import get_app_config + from deerflow.config.app_config import get_app_config config = get_app_config() channels_config = {} @@ -116,7 +116,7 @@ class ChannelService: return False try: - from src.reflection import resolve_class + from deerflow.reflection import resolve_class channel_cls = resolve_class(import_path, base_class=None) except Exception: diff --git a/backend/src/channels/slack.py b/backend/app/channels/slack.py similarity index 98% rename from backend/src/channels/slack.py rename to backend/app/channels/slack.py index f28f2c1..7609110 100644 --- a/backend/src/channels/slack.py +++ b/backend/app/channels/slack.py @@ -8,8 +8,8 @@ from typing import Any from markdown_to_mrkdwn import SlackMarkdownConverter -from src.channels.base import Channel -from src.channels.message_bus import InboundMessageType, MessageBus, OutboundMessage, ResolvedAttachment +from app.channels.base import Channel +from app.channels.message_bus import InboundMessageType, MessageBus, OutboundMessage, ResolvedAttachment logger = logging.getLogger(__name__) diff --git a/backend/src/channels/store.py b/backend/app/channels/store.py similarity index 98% rename from backend/src/channels/store.py rename to backend/app/channels/store.py index 9c4cb7b..81f5d61 100644 --- a/backend/src/channels/store.py +++ b/backend/app/channels/store.py @@ -35,7 +35,7 @@ class ChannelStore: def __init__(self, path: str | Path | None = None) -> None: if path is None: - from src.config.paths import get_paths + from deerflow.config.paths import get_paths path = Path(get_paths().base_dir) / "channels" / "store.json" self._path = Path(path) diff --git a/backend/src/channels/telegram.py b/backend/app/channels/telegram.py similarity index 99% rename from backend/src/channels/telegram.py rename to backend/app/channels/telegram.py index 0abdb20..7153625 100644 --- a/backend/src/channels/telegram.py +++ b/backend/app/channels/telegram.py @@ -7,8 +7,8 @@ import logging import threading from typing import Any -from src.channels.base import Channel -from src.channels.message_bus import InboundMessageType, MessageBus, OutboundMessage, ResolvedAttachment +from app.channels.base import Channel +from app.channels.message_bus import InboundMessageType, MessageBus, OutboundMessage, ResolvedAttachment logger = logging.getLogger(__name__) diff --git a/backend/src/gateway/__init__.py b/backend/app/gateway/__init__.py similarity index 100% rename from backend/src/gateway/__init__.py rename to backend/app/gateway/__init__.py diff --git a/backend/src/gateway/app.py b/backend/app/gateway/app.py similarity index 95% rename from backend/src/gateway/app.py rename to backend/app/gateway/app.py index edcf6ad..4a59156 100644 --- a/backend/src/gateway/app.py +++ b/backend/app/gateway/app.py @@ -4,9 +4,8 @@ from contextlib import asynccontextmanager from fastapi import FastAPI -from src.config.app_config import get_app_config -from src.gateway.config import get_gateway_config -from src.gateway.routers import ( +from app.gateway.config import get_gateway_config +from app.gateway.routers import ( agents, artifacts, channels, @@ -17,6 +16,7 @@ from src.gateway.routers import ( suggestions, uploads, ) +from deerflow.config.app_config import get_app_config # Configure logging logging.basicConfig( @@ -50,7 +50,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: # Start IM channel service if any channels are configured try: - from src.channels.service import start_channel_service + from app.channels.service import start_channel_service channel_service = await start_channel_service() logger.info("Channel service started: %s", channel_service.get_status()) @@ -61,7 +61,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: # Stop channel service on shutdown try: - from src.channels.service import stop_channel_service + from app.channels.service import stop_channel_service await stop_channel_service() except Exception: diff --git a/backend/src/gateway/config.py b/backend/app/gateway/config.py similarity index 100% rename from backend/src/gateway/config.py rename to backend/app/gateway/config.py diff --git a/backend/src/gateway/path_utils.py b/backend/app/gateway/path_utils.py similarity index 95% rename from backend/src/gateway/path_utils.py rename to backend/app/gateway/path_utils.py index 77fc2e0..4869c94 100644 --- a/backend/src/gateway/path_utils.py +++ b/backend/app/gateway/path_utils.py @@ -4,7 +4,7 @@ from pathlib import Path from fastapi import HTTPException -from src.config.paths import get_paths +from deerflow.config.paths import get_paths def resolve_thread_virtual_path(thread_id: str, virtual_path: str) -> Path: diff --git a/backend/src/gateway/routers/__init__.py b/backend/app/gateway/routers/__init__.py similarity index 100% rename from backend/src/gateway/routers/__init__.py rename to backend/app/gateway/routers/__init__.py diff --git a/backend/src/gateway/routers/agents.py b/backend/app/gateway/routers/agents.py similarity index 98% rename from backend/src/gateway/routers/agents.py rename to backend/app/gateway/routers/agents.py index 0203240..00b3585 100644 --- a/backend/src/gateway/routers/agents.py +++ b/backend/app/gateway/routers/agents.py @@ -8,8 +8,8 @@ import yaml from fastapi import APIRouter, HTTPException from pydantic import BaseModel, Field -from src.config.agents_config import AgentConfig, list_custom_agents, load_agent_config, load_agent_soul -from src.config.paths import get_paths +from deerflow.config.agents_config import AgentConfig, list_custom_agents, load_agent_config, load_agent_soul +from deerflow.config.paths import get_paths logger = logging.getLogger(__name__) router = APIRouter(prefix="/api", tags=["agents"]) diff --git a/backend/src/gateway/routers/artifacts.py b/backend/app/gateway/routers/artifacts.py similarity index 99% rename from backend/src/gateway/routers/artifacts.py rename to backend/app/gateway/routers/artifacts.py index 18ca922..b2312bc 100644 --- a/backend/src/gateway/routers/artifacts.py +++ b/backend/app/gateway/routers/artifacts.py @@ -7,7 +7,7 @@ from urllib.parse import quote from fastapi import APIRouter, HTTPException, Request from fastapi.responses import FileResponse, HTMLResponse, PlainTextResponse, Response -from src.gateway.path_utils import resolve_thread_virtual_path +from app.gateway.path_utils import resolve_thread_virtual_path logger = logging.getLogger(__name__) diff --git a/backend/src/gateway/routers/channels.py b/backend/app/gateway/routers/channels.py similarity index 93% rename from backend/src/gateway/routers/channels.py rename to backend/app/gateway/routers/channels.py index abf9d09..abd8aa1 100644 --- a/backend/src/gateway/routers/channels.py +++ b/backend/app/gateway/routers/channels.py @@ -25,7 +25,7 @@ class ChannelRestartResponse(BaseModel): @router.get("/", response_model=ChannelStatusResponse) async def get_channels_status() -> ChannelStatusResponse: """Get the status of all IM channels.""" - from src.channels.service import get_channel_service + from app.channels.service import get_channel_service service = get_channel_service() if service is None: @@ -37,7 +37,7 @@ async def get_channels_status() -> ChannelStatusResponse: @router.post("/{name}/restart", response_model=ChannelRestartResponse) async def restart_channel(name: str) -> ChannelRestartResponse: """Restart a specific IM channel.""" - from src.channels.service import get_channel_service + from app.channels.service import get_channel_service service = get_channel_service() if service is None: diff --git a/backend/src/gateway/routers/mcp.py b/backend/app/gateway/routers/mcp.py similarity index 98% rename from backend/src/gateway/routers/mcp.py rename to backend/app/gateway/routers/mcp.py index 60efc2c..09133ea 100644 --- a/backend/src/gateway/routers/mcp.py +++ b/backend/app/gateway/routers/mcp.py @@ -6,7 +6,7 @@ from typing import Literal from fastapi import APIRouter, HTTPException from pydantic import BaseModel, Field -from src.config.extensions_config import ExtensionsConfig, get_extensions_config, reload_extensions_config +from deerflow.config.extensions_config import ExtensionsConfig, get_extensions_config, reload_extensions_config logger = logging.getLogger(__name__) router = APIRouter(prefix="/api", tags=["mcp"]) diff --git a/backend/src/gateway/routers/memory.py b/backend/app/gateway/routers/memory.py similarity index 98% rename from backend/src/gateway/routers/memory.py rename to backend/app/gateway/routers/memory.py index 6639feb..1a13424 100644 --- a/backend/src/gateway/routers/memory.py +++ b/backend/app/gateway/routers/memory.py @@ -3,8 +3,8 @@ from fastapi import APIRouter from pydantic import BaseModel, Field -from src.agents.memory.updater import get_memory_data, reload_memory_data -from src.config.memory_config import get_memory_config +from deerflow.agents.memory.updater import get_memory_data, reload_memory_data +from deerflow.config.memory_config import get_memory_config router = APIRouter(prefix="/api", tags=["memory"]) diff --git a/backend/src/gateway/routers/models.py b/backend/app/gateway/routers/models.py similarity index 98% rename from backend/src/gateway/routers/models.py rename to backend/app/gateway/routers/models.py index e158e7b..269b55c 100644 --- a/backend/src/gateway/routers/models.py +++ b/backend/app/gateway/routers/models.py @@ -1,7 +1,7 @@ from fastapi import APIRouter, HTTPException from pydantic import BaseModel, Field -from src.config import get_app_config +from deerflow.config import get_app_config router = APIRouter(prefix="/api", tags=["models"]) diff --git a/backend/src/gateway/routers/skills.py b/backend/app/gateway/routers/skills.py similarity index 79% rename from backend/src/gateway/routers/skills.py rename to backend/app/gateway/routers/skills.py index c281972..c208dba 100644 --- a/backend/src/gateway/routers/skills.py +++ b/backend/app/gateway/routers/skills.py @@ -1,22 +1,19 @@ import json import logging -import re import shutil import stat import tempfile import zipfile -from collections.abc import Mapping from pathlib import Path -from typing import cast -import yaml from fastapi import APIRouter, HTTPException from pydantic import BaseModel, Field -from src.config.extensions_config import ExtensionsConfig, SkillStateConfig, get_extensions_config, reload_extensions_config -from src.gateway.path_utils import resolve_thread_virtual_path -from src.skills import Skill, load_skills -from src.skills.loader import get_skills_root_path +from app.gateway.path_utils import resolve_thread_virtual_path +from deerflow.config.extensions_config import ExtensionsConfig, SkillStateConfig, get_extensions_config, reload_extensions_config +from deerflow.skills import Skill, load_skills +from deerflow.skills.loader import get_skills_root_path +from deerflow.skills.validation import _validate_skill_frontmatter logger = logging.getLogger(__name__) @@ -129,23 +126,6 @@ class SkillInstallResponse(BaseModel): message: str = Field(..., description="Installation result message") -# Allowed properties in SKILL.md frontmatter -ALLOWED_FRONTMATTER_PROPERTIES = { - "name", - "description", - "license", - "allowed-tools", - "metadata", - "compatibility", - "version", - "author", -} - - -def _safe_load_frontmatter(frontmatter_text: str) -> object: - return cast(object, yaml.safe_load(frontmatter_text)) - - def _should_ignore_archive_entry(path: Path) -> bool: return path.name.startswith(".") or path.name == "__MACOSX" @@ -159,81 +139,6 @@ def _resolve_skill_dir_from_archive_root(temp_path: Path) -> Path: return temp_path -def _validate_skill_frontmatter(skill_dir: Path) -> tuple[bool, str, str | None]: - """Validate a skill directory's SKILL.md frontmatter. - - Args: - skill_dir: Path to the skill directory containing SKILL.md. - - Returns: - Tuple of (is_valid, message, skill_name). - """ - skill_md = skill_dir / "SKILL.md" - if not skill_md.exists(): - return False, "SKILL.md not found", None - - content = skill_md.read_text() - if not content.startswith("---"): - return False, "No YAML frontmatter found", None - - # Extract frontmatter - match = re.match(r"^---\n(.*?)\n---", content, re.DOTALL) - if not match: - return False, "Invalid frontmatter format", None - - frontmatter_text = match.group(1) - - # Parse YAML frontmatter - try: - parsed_frontmatter = _safe_load_frontmatter(frontmatter_text) - if not isinstance(parsed_frontmatter, Mapping): - return False, "Frontmatter must be a YAML dictionary", None - parsed_frontmatter = cast(Mapping[object, object], parsed_frontmatter) - frontmatter: dict[str, object] = {str(key): value for key, value in parsed_frontmatter.items()} - except yaml.YAMLError as e: - return False, f"Invalid YAML in frontmatter: {e}", None - - # Check for unexpected properties - unexpected_keys = set(frontmatter.keys()) - ALLOWED_FRONTMATTER_PROPERTIES - if unexpected_keys: - return False, f"Unexpected key(s) in SKILL.md frontmatter: {', '.join(sorted(unexpected_keys))}", None - - # Check required fields - if "name" not in frontmatter: - return False, "Missing 'name' in frontmatter", None - if "description" not in frontmatter: - return False, "Missing 'description' in frontmatter", None - - # Validate name - name = frontmatter.get("name", "") - if not isinstance(name, str): - return False, f"Name must be a string, got {type(name).__name__}", None - name = name.strip() - if not name: - return False, "Name cannot be empty", None - - # Check naming convention (hyphen-case: lowercase with hyphens) - if not re.match(r"^[a-z0-9-]+$", name): - return False, f"Name '{name}' should be hyphen-case (lowercase letters, digits, and hyphens only)", None - if name.startswith("-") or name.endswith("-") or "--" in name: - return False, f"Name '{name}' cannot start/end with hyphen or contain consecutive hyphens", None - if len(name) > 64: - return False, f"Name is too long ({len(name)} characters). Maximum is 64 characters.", None - - # Validate description - description = frontmatter.get("description", "") - if not isinstance(description, str): - return False, f"Description must be a string, got {type(description).__name__}", None - description = description.strip() - if description: - if "<" in description or ">" in description: - return False, "Description cannot contain angle brackets (< or >)", None - if len(description) > 1024: - return False, f"Description is too long ({len(description)} characters). Maximum is 1024 characters.", None - - return True, "Skill is valid!", name - - def _skill_to_response(skill: Skill) -> SkillResponse: """Convert a Skill object to a SkillResponse.""" return SkillResponse( diff --git a/backend/src/gateway/routers/suggestions.py b/backend/app/gateway/routers/suggestions.py similarity index 99% rename from backend/src/gateway/routers/suggestions.py rename to backend/app/gateway/routers/suggestions.py index b5bb8df..dee985e 100644 --- a/backend/src/gateway/routers/suggestions.py +++ b/backend/app/gateway/routers/suggestions.py @@ -4,7 +4,7 @@ import logging from fastapi import APIRouter from pydantic import BaseModel, Field -from src.models import create_chat_model +from deerflow.models import create_chat_model logger = logging.getLogger(__name__) diff --git a/backend/src/gateway/routers/uploads.py b/backend/app/gateway/routers/uploads.py similarity index 86% rename from backend/src/gateway/routers/uploads.py rename to backend/app/gateway/routers/uploads.py index cf2a724..c43c820 100644 --- a/backend/src/gateway/routers/uploads.py +++ b/backend/app/gateway/routers/uploads.py @@ -6,24 +6,14 @@ from pathlib import Path from fastapi import APIRouter, File, HTTPException, UploadFile from pydantic import BaseModel -from src.config.paths import VIRTUAL_PATH_PREFIX, get_paths -from src.sandbox.sandbox_provider import get_sandbox_provider +from deerflow.config.paths import VIRTUAL_PATH_PREFIX, get_paths +from deerflow.sandbox.sandbox_provider import get_sandbox_provider +from deerflow.utils.file_conversion import CONVERTIBLE_EXTENSIONS, convert_file_to_markdown logger = logging.getLogger(__name__) router = APIRouter(prefix="/api/threads/{thread_id}/uploads", tags=["uploads"]) -# File extensions that should be converted to markdown -CONVERTIBLE_EXTENSIONS = { - ".pdf", - ".ppt", - ".pptx", - ".xls", - ".xlsx", - ".doc", - ".docx", -} - class UploadResponse(BaseModel): """Response model for file upload.""" @@ -47,32 +37,6 @@ def get_uploads_dir(thread_id: str) -> Path: return base_dir -async def convert_file_to_markdown(file_path: Path) -> Path | None: - """Convert a file to markdown using markitdown. - - Args: - file_path: Path to the file to convert. - - Returns: - Path to the markdown file if conversion was successful, None otherwise. - """ - try: - from markitdown import MarkItDown - - md = MarkItDown() - result = md.convert(str(file_path)) - - # Save as .md file with same name - md_path = file_path.with_suffix(".md") - md_path.write_text(result.text_content, encoding="utf-8") - - logger.info(f"Converted {file_path.name} to markdown: {md_path.name}") - return md_path - except Exception as e: - logger.error(f"Failed to convert {file_path.name} to markdown: {e}") - return None - - @router.post("", response_model=UploadResponse) async def upload_files( thread_id: str, diff --git a/backend/debug.py b/backend/debug.py index 851a2e4..f558d1d 100644 --- a/backend/debug.py +++ b/backend/debug.py @@ -3,6 +3,12 @@ Debug script for lead_agent. Run this file directly in VS Code with breakpoints. +Requirements: + Run with `uv run` from the backend/ directory so that the uv workspace + resolves deerflow-harness and app packages correctly: + + cd backend && PYTHONPATH=. uv run python debug.py + Usage: 1. Set breakpoints in agent.py or other files 2. Press F5 or use "Run and Debug" panel @@ -11,21 +17,14 @@ Usage: import asyncio import logging -import os -import sys -# Ensure we can import from src -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) - -# Load environment variables from dotenv import load_dotenv from langchain_core.messages import HumanMessage -from src.agents import make_lead_agent +from deerflow.agents import make_lead_agent load_dotenv() -# Configure logging logging.basicConfig( level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", @@ -36,7 +35,7 @@ logging.basicConfig( async def main(): # Initialize MCP tools at startup try: - from src.mcp import initialize_mcp_tools + from deerflow.mcp import initialize_mcp_tools await initialize_mcp_tools() except Exception as e: diff --git a/backend/docs/APPLE_CONTAINER.md b/backend/docs/APPLE_CONTAINER.md index 6ef82d0..bf2582f 100644 --- a/backend/docs/APPLE_CONTAINER.md +++ b/backend/docs/APPLE_CONTAINER.md @@ -80,7 +80,7 @@ docker stop # Auto-removes due to --rm ### Implementation Details -The implementation is in `backend/src/community/aio_sandbox/aio_sandbox_provider.py`: +The implementation is in `backend/packages/harness/deerflow/community/aio_sandbox/aio_sandbox_provider.py`: - `_detect_container_runtime()`: Detects available runtime at startup - `_start_container()`: Uses detected runtime, skips Docker-specific options for Apple Container @@ -93,14 +93,14 @@ No configuration changes are needed! The system works automatically. However, you can verify the runtime in use by checking the logs: ``` -INFO:src.community.aio_sandbox.aio_sandbox_provider:Detected Apple Container: container version 0.1.0 -INFO:src.community.aio_sandbox.aio_sandbox_provider:Starting sandbox container using container: ... +INFO:deerflow.community.aio_sandbox.aio_sandbox_provider:Detected Apple Container: container version 0.1.0 +INFO:deerflow.community.aio_sandbox.aio_sandbox_provider:Starting sandbox container using container: ... ``` Or for Docker: ``` -INFO:src.community.aio_sandbox.aio_sandbox_provider:Apple Container not available, falling back to Docker -INFO:src.community.aio_sandbox.aio_sandbox_provider:Starting sandbox container using docker: ... +INFO:deerflow.community.aio_sandbox.aio_sandbox_provider:Apple Container not available, falling back to Docker +INFO:deerflow.community.aio_sandbox.aio_sandbox_provider:Starting sandbox container using docker: ... ``` ## Container Images @@ -109,7 +109,7 @@ Both runtimes use OCI-compatible images. The default image works with both: ```yaml sandbox: - use: src.community.aio_sandbox:AioSandboxProvider + use: deerflow.community.aio_sandbox:AioSandboxProvider image: enterprise-public-cn-beijing.cr.volces.com/vefaas-public/all-in-one-sandbox:latest # Default image ``` diff --git a/backend/docs/ARCHITECTURE.md b/backend/docs/ARCHITECTURE.md index cf0285f..06fbea8 100644 --- a/backend/docs/ARCHITECTURE.md +++ b/backend/docs/ARCHITECTURE.md @@ -55,7 +55,7 @@ This document provides a comprehensive overview of the DeerFlow backend architec The LangGraph server is the core agent runtime, built on LangGraph for robust multi-agent workflow orchestration. -**Entry Point**: `src/agents/lead_agent/agent.py:make_lead_agent` +**Entry Point**: `packages/harness/deerflow/agents/lead_agent/agent.py:make_lead_agent` **Key Responsibilities**: - Agent creation and configuration @@ -70,7 +70,7 @@ The LangGraph server is the core agent runtime, built on LangGraph for robust mu { "agent": { "type": "agent", - "path": "src.agents:make_lead_agent" + "path": "deerflow.agents:make_lead_agent" } } ``` @@ -79,7 +79,7 @@ The LangGraph server is the core agent runtime, built on LangGraph for robust mu FastAPI application providing REST endpoints for non-agent operations. -**Entry Point**: `src/gateway/app.py` +**Entry Point**: `app/gateway/app.py` **Routers**: - `models.py` - `/api/models` - Model listing and details @@ -158,7 +158,7 @@ class ThreadState(AgentState): ▼ ▼ ┌─────────────────────────┐ ┌─────────────────────────┐ │ LocalSandboxProvider │ │ AioSandboxProvider │ -│ (src/sandbox/local.py) │ │ (src/community/) │ +│ (packages/harness/deerflow/sandbox/local.py) │ │ (packages/harness/deerflow/community/) │ │ │ │ │ │ - Singleton instance │ │ - Docker-based │ │ - Direct execution │ │ - Isolated containers │ @@ -192,7 +192,7 @@ class ThreadState(AgentState): ┌─────────────────────┐ ┌─────────────────────┐ ┌─────────────────────┐ │ Built-in Tools │ │ Configured Tools │ │ MCP Tools │ -│ (src/tools/) │ │ (config.yaml) │ │ (extensions.json) │ +│ (packages/harness/deerflow/tools/) │ │ (config.yaml) │ │ (extensions.json) │ ├─────────────────────┤ ├─────────────────────┤ ├─────────────────────┤ │ - present_file │ │ - web_search │ │ - github │ │ - ask_clarification │ │ - web_fetch │ │ - filesystem │ @@ -208,7 +208,7 @@ class ThreadState(AgentState): ▼ ┌─────────────────────────┐ │ get_available_tools() │ - │ (src/tools/__init__) │ + │ (packages/harness/deerflow/tools/__init__) │ └─────────────────────────┘ ``` @@ -217,7 +217,7 @@ class ThreadState(AgentState): ``` ┌─────────────────────────────────────────────────────────────────────────┐ │ Model Factory │ -│ (src/models/factory.py) │ +│ (packages/harness/deerflow/models/factory.py) │ └─────────────────────────────────────────────────────────────────────────┘ config.yaml: @@ -264,7 +264,7 @@ config.yaml: ``` ┌─────────────────────────────────────────────────────────────────────────┐ │ MCP Integration │ -│ (src/mcp/manager.py) │ +│ (packages/harness/deerflow/mcp/manager.py) │ └─────────────────────────────────────────────────────────────────────────┘ extensions_config.json: @@ -302,7 +302,7 @@ extensions_config.json: ``` ┌─────────────────────────────────────────────────────────────────────────┐ │ Skills System │ -│ (src/skills/loader.py) │ +│ (packages/harness/deerflow/skills/loader.py) │ └─────────────────────────────────────────────────────────────────────────┘ Directory Structure: diff --git a/backend/docs/AUTO_TITLE_GENERATION.md b/backend/docs/AUTO_TITLE_GENERATION.md index 855f2f9..024d4b4 100644 --- a/backend/docs/AUTO_TITLE_GENERATION.md +++ b/backend/docs/AUTO_TITLE_GENERATION.md @@ -50,7 +50,7 @@ checkpointer = PostgresSaver.from_conn_string( ```json { "graphs": { - "lead_agent": "src.agents:lead_agent" + "lead_agent": "deerflow.agents:lead_agent" }, "checkpointer": "checkpointer:checkpointer" } @@ -71,7 +71,7 @@ title: 或在代码中配置： ```python -from src.config.title_config import TitleConfig, set_title_config +from deerflow.config.title_config import TitleConfig, set_title_config set_title_config(TitleConfig( enabled=True, @@ -185,7 +185,7 @@ sequenceDiagram ```python # 测试 title 生成 import pytest -from src.agents.title_middleware import TitleMiddleware +from deerflow.agents.title_middleware import TitleMiddleware def test_title_generation(): # TODO: 添加单元测试 @@ -243,11 +243,11 @@ def after_agent(self, state: TitleMiddlewareState, runtime: Runtime) -> dict | N ## 相关文件 -- [`src/agents/thread_state.py`](../src/agents/thread_state.py) - ThreadState 定义 -- [`src/agents/title_middleware.py`](../src/agents/title_middleware.py) - TitleMiddleware 实现 -- [`src/config/title_config.py`](../src/config/title_config.py) - 配置管理 +- [`packages/harness/deerflow/agents/thread_state.py`](../packages/harness/deerflow/agents/thread_state.py) - ThreadState 定义 +- [`packages/harness/deerflow/agents/title_middleware.py`](../packages/harness/deerflow/agents/title_middleware.py) - TitleMiddleware 实现 +- [`packages/harness/deerflow/config/title_config.py`](../packages/harness/deerflow/config/title_config.py) - 配置管理 - [`config.yaml`](../config.yaml) - 配置文件 -- [`src/agents/lead_agent/agent.py`](../src/agents/lead_agent/agent.py) - Middleware 注册 +- [`packages/harness/deerflow/agents/lead_agent/agent.py`](../packages/harness/deerflow/agents/lead_agent/agent.py) - Middleware 注册 ## 参考资料 diff --git a/backend/docs/CONFIGURATION.md b/backend/docs/CONFIGURATION.md index 5ef319a..d461e54 100644 --- a/backend/docs/CONFIGURATION.md +++ b/backend/docs/CONFIGURATION.md @@ -2,6 +2,19 @@ This guide explains how to configure DeerFlow for your environment. +## Config Versioning + +`config.example.yaml` contains a `config_version` field that tracks schema changes. When the example version is higher than your local `config.yaml`, the application emits a startup warning: + +``` +WARNING - Your config.yaml (version 0) is outdated — the latest version is 1. +Run `make config-upgrade` to merge new fields into your config. +``` + +- **Missing `config_version`** in your config is treated as version 0. +- Run `make config-upgrade` to auto-merge missing fields (your existing values are preserved, a `.bak` backup is created). +- When changing the config schema, bump `config_version` in `config.example.yaml`. + ## Configuration Sections ### Models @@ -103,7 +116,7 @@ Configure specific tools available to the agent: tools: - name: web_search group: web - use: src.community.tavily.tools:web_search_tool + use: deerflow.community.tavily.tools:web_search_tool max_results: 5 # api_key: $TAVILY_API_KEY # Optional ``` @@ -124,13 +137,13 @@ DeerFlow supports multiple sandbox execution modes. Configure your preferred mod **Local Execution** (runs sandbox code directly on the host machine): ```yaml sandbox: - use: src.sandbox.local:LocalSandboxProvider # Local execution + use: deerflow.sandbox.local:LocalSandboxProvider # Local execution ``` **Docker Execution** (runs sandbox code in isolated Docker containers): ```yaml sandbox: - use: src.community.aio_sandbox:AioSandboxProvider # Docker-based sandbox + use: deerflow.community.aio_sandbox:AioSandboxProvider # Docker-based sandbox ``` **Docker Execution with Kubernetes** (runs sandbox code in Kubernetes pods via provisioner service): @@ -139,7 +152,7 @@ This mode runs each sandbox in an isolated Kubernetes Pod on your **host machine ```yaml sandbox: - use: src.community.aio_sandbox:AioSandboxProvider + use: deerflow.community.aio_sandbox:AioSandboxProvider provisioner_url: http://provisioner:8002 ``` @@ -152,13 +165,13 @@ Choose between local execution or Docker-based isolation: **Option 1: Local Sandbox** (default, simpler setup): ```yaml sandbox: - use: src.sandbox.local:LocalSandboxProvider + use: deerflow.sandbox.local:LocalSandboxProvider ``` **Option 2: Docker Sandbox** (isolated, more secure): ```yaml sandbox: - use: src.community.aio_sandbox:AioSandboxProvider + use: deerflow.community.aio_sandbox:AioSandboxProvider port: 8080 auto_start: true container_prefix: deer-flow-sandbox diff --git a/backend/docs/FILE_UPLOAD.md b/backend/docs/FILE_UPLOAD.md index b975e20..f19a9d7 100644 --- a/backend/docs/FILE_UPLOAD.md +++ b/backend/docs/FILE_UPLOAD.md @@ -212,11 +212,11 @@ backend/.deer-flow/threads/ ### 组件 -1. **Upload Router** (`src/gateway/routers/uploads.py`) +1. **Upload Router** (`app/gateway/routers/uploads.py`) - 处理文件上传、列表、删除请求 - 使用 markitdown 转换文档 -2. **Uploads Middleware** (`src/agents/middlewares/uploads_middleware.py`) +2. **Uploads Middleware** (`packages/harness/deerflow/agents/middlewares/uploads_middleware.py`) - 在每次 Agent 请求前注入文件列表 - 自动生成格式化的文件列表消息 diff --git a/backend/docs/HARNESS_APP_SPLIT.md b/backend/docs/HARNESS_APP_SPLIT.md new file mode 100644 index 0000000..cf0e26e --- /dev/null +++ b/backend/docs/HARNESS_APP_SPLIT.md @@ -0,0 +1,343 @@ +# DeerFlow 后端拆分设计文档：Harness + App + +> 状态：Draft +> 作者：DeerFlow Team +> 日期：2026-03-13 + +## 1. 背景与动机 + +DeerFlow 后端当前是一个单一 Python 包（`src.*`），包含了从底层 agent 编排到上层用户产品的所有代码。随着项目发展，这种结构带来了几个问题： + +- **复用困难**：其他产品（CLI 工具、Slack bot、第三方集成）想用 agent 能力，必须依赖整个后端，包括 FastAPI、IM SDK 等不需要的依赖 +- **职责模糊**：agent 编排逻辑和用户产品逻辑混在同一个 `src/` 下，边界不清晰 +- **依赖膨胀**：LangGraph Server 运行时不需要 FastAPI/uvicorn/Slack SDK，但当前必须安装全部依赖 + +本文档提出将后端拆分为两部分：**deerflow-harness**（可发布的 agent 框架包）和 **app**（不打包的用户产品代码）。 + +## 2. 核心概念 + +### 2.1 Harness（线束/框架层） + +Harness 是 agent 的构建与编排框架，回答 **"如何构建和运行 agent"** 的问题： + +- Agent 工厂与生命周期管理 +- Middleware pipeline +- 工具系统（内置工具 + MCP + 社区工具） +- 沙箱执行环境 +- 子 agent 委派 +- 记忆系统 +- 技能加载与注入 +- 模型工厂 +- 配置系统 + +**Harness 是一个可发布的 Python 包**（`deerflow-harness`），可以独立安装和使用。 + +**Harness 的设计原则**：对上层应用完全无感知。它不知道也不关心谁在调用它——可以是 Web App、CLI、Slack Bot、或者一个单元测试。 + +### 2.2 App（应用层） + +App 是面向用户的产品代码，回答 **"如何将 agent 呈现给用户"** 的问题： + +- Gateway API（FastAPI REST 接口） +- IM Channels（飞书、Slack、Telegram 集成） +- Custom Agent 的 CRUD 管理 +- 文件上传/下载的 HTTP 接口 + +**App 不打包、不发布**，它是 DeerFlow 项目内部的应用代码，直接运行。 + +**App 依赖 Harness，但 Harness 不依赖 App。** + +### 2.3 边界划分 + +| 模块 | 归属 | 说明 | +|------|------|------| +| `config/` | Harness | 配置系统是基础设施 | +| `reflection/` | Harness | 动态模块加载工具 | +| `utils/` | Harness | 通用工具函数 | +| `agents/` | Harness | Agent 工厂、middleware、state、memory | +| `subagents/` | Harness | 子 agent 委派系统 | +| `sandbox/` | Harness | 沙箱执行环境 | +| `tools/` | Harness | 工具注册与发现 | +| `mcp/` | Harness | MCP 协议集成 | +| `skills/` | Harness | 技能加载、解析、定义 schema | +| `models/` | Harness | LLM 模型工厂 | +| `community/` | Harness | 社区工具（tavily、jina 等） | +| `client.py` | Harness | 嵌入式 Python 客户端 | +| `gateway/` | App | FastAPI REST API | +| `channels/` | App | IM 平台集成 | + +**关于 Custom Agents**：agent 定义格式（`config.yaml` + `SOUL.md` schema）由 Harness 层的 `config/agents_config.py` 定义，但文件的存储、CRUD、发现机制由 App 层的 `gateway/routers/agents.py` 负责。 + +## 3. 目标架构 + +### 3.1 目录结构 + +``` +backend/ +├── packages/ +│ └── harness/ +│ ├── pyproject.toml # deerflow-harness 包定义 +│ └── deerflow/ # Python 包根（import 前缀: deerflow.*） +│ ├── __init__.py +│ ├── config/ +│ ├── reflection/ +│ ├── utils/ +│ ├── agents/ +│ │ ├── lead_agent/ +│ │ ├── middlewares/ +│ │ ├── memory/ +│ │ ├── checkpointer/ +│ │ └── thread_state.py +│ ├── subagents/ +│ ├── sandbox/ +│ ├── tools/ +│ ├── mcp/ +│ ├── skills/ +│ ├── models/ +│ ├── community/ +│ └── client.py +├── app/ # 不打包（import 前缀: app.*） +│ ├── __init__.py +│ ├── gateway/ +│ │ ├── __init__.py +│ │ ├── app.py +│ │ ├── config.py +│ │ ├── path_utils.py +│ │ └── routers/ +│ └── channels/ +│ ├── __init__.py +│ ├── base.py +│ ├── manager.py +│ ├── service.py +│ ├── store.py +│ ├── message_bus.py +│ ├── feishu.py +│ ├── slack.py +│ └── telegram.py +├── pyproject.toml # uv workspace root +├── langgraph.json +├── tests/ +├── docs/ +└── Makefile +``` + +### 3.2 Import 规则 + +两个层使用不同的 import 前缀，职责边界一目了然： + +```python +# --------------------------------------------------------------- +# Harness 内部互相引用（deerflow.* 前缀） +# --------------------------------------------------------------- +from deerflow.agents import make_lead_agent +from deerflow.models import create_chat_model +from deerflow.config import get_app_config +from deerflow.tools import get_available_tools + +# --------------------------------------------------------------- +# App 内部互相引用（app.* 前缀） +# --------------------------------------------------------------- +from app.gateway.app import app +from app.gateway.routers.uploads import upload_files +from app.channels.service import start_channel_service + +# --------------------------------------------------------------- +# App 调用 Harness（单向依赖，Harness 永远不 import app） +# --------------------------------------------------------------- +from deerflow.agents import make_lead_agent +from deerflow.models import create_chat_model +from deerflow.skills import load_skills +from deerflow.config.extensions_config import get_extensions_config +``` + +**App 调用 Harness 示例 — Gateway 中启动 agent**： + +```python +# app/gateway/routers/chat.py +from deerflow.agents.lead_agent.agent import make_lead_agent +from deerflow.models import create_chat_model +from deerflow.config import get_app_config + +async def create_chat_session(thread_id: str, model_name: str): + config = get_app_config() + model = create_chat_model(name=model_name) + agent = make_lead_agent(config=...) + # ... 使用 agent 处理用户消息 +``` + +**App 调用 Harness 示例 — Channel 中查询 skills**： + +```python +# app/channels/manager.py +from deerflow.skills import load_skills +from deerflow.agents.memory.updater import get_memory_data + +def handle_status_command(): + skills = load_skills(enabled_only=True) + memory = get_memory_data() + return f"Skills: {len(skills)}, Memory facts: {len(memory.get('facts', []))}" +``` + +**禁止方向**：Harness 代码中绝不能出现 `from app.` 或 `import app.`。 + +### 3.3 为什么 App 不打包 + +| 方面 | 打包（放 packages/ 下） | 不打包（放 backend/app/） | +|------|------------------------|--------------------------| +| 命名空间 | 需要 pkgutil `extend_path` 合并，或独立前缀 | 天然独立，`app.*` vs `deerflow.*` | +| 发布需求 | 没有——App 是项目内部代码 | 不需要 pyproject.toml | +| 复杂度 | 需要管理两个包的构建、版本、依赖声明 | 直接运行，零额外配置 | +| 运行方式 | `pip install deerflow-app` | `PYTHONPATH=. uvicorn app.gateway.app:app` | + +App 的唯一消费者是 DeerFlow 项目自身，没有独立发布的需求。放在 `backend/app/` 下作为普通 Python 包，通过 `PYTHONPATH` 或 editable install 让 Python 找到即可。 + +### 3.4 依赖关系 + +``` +┌─────────────────────────────────────┐ +│ app/ (不打包，直接运行) │ +│ ├── fastapi, uvicorn │ +│ ├── slack-sdk, lark-oapi, ... │ +│ └── import deerflow.* │ +└──────────────┬──────────────────────┘ + │ + ▼ +┌─────────────────────────────────────┐ +│ deerflow-harness (可发布的包) │ +│ ├── langgraph, langchain │ +│ ├── markitdown, pydantic, ... │ +│ └── 零 app 依赖 │ +└─────────────────────────────────────┘ +``` + +**依赖分类**： + +| 分类 | 依赖包 | +|------|--------| +| Harness only | agent-sandbox, langchain*, langgraph*, markdownify, markitdown, pydantic, pyyaml, readabilipy, tavily-python, firecrawl-py, tiktoken, ddgs, duckdb, httpx, kubernetes, dotenv | +| App only | fastapi, uvicorn, sse-starlette, python-multipart, lark-oapi, slack-sdk, python-telegram-bot, markdown-to-mrkdwn | +| Shared | langgraph-sdk（channels 用 HTTP client）, pydantic, httpx | + +### 3.5 Workspace 配置 + +`backend/pyproject.toml`（workspace root）： + +```toml +[project] +name = "deer-flow" +version = "0.1.0" +requires-python = ">=3.12" +dependencies = ["deerflow-harness"] + +[dependency-groups] +dev = ["pytest>=8.0.0", "ruff>=0.14.11"] +# App 的额外依赖（fastapi 等）也声明在 workspace root，因为 app 不打包 +app = ["fastapi", "uvicorn", "sse-starlette", "python-multipart"] +channels = ["lark-oapi", "slack-sdk", "python-telegram-bot"] + +[tool.uv.workspace] +members = ["packages/harness"] + +[tool.uv.sources] +deerflow-harness = { workspace = true } +``` + +## 4. 当前的跨层依赖问题 + +在拆分之前，需要先解决 `client.py` 中两处从 harness 到 app 的反向依赖： + +### 4.1 `_validate_skill_frontmatter` + +```python +# client.py — harness 导入了 app 层代码 +from src.gateway.routers.skills import _validate_skill_frontmatter +``` + +**解决方案**：将该函数提取到 `deerflow/skills/validation.py`。这是一个纯逻辑函数（解析 YAML frontmatter、校验字段），与 FastAPI 无关。 + +### 4.2 `CONVERTIBLE_EXTENSIONS` + `convert_file_to_markdown` + +```python +# client.py — harness 导入了 app 层代码 +from src.gateway.routers.uploads import CONVERTIBLE_EXTENSIONS, convert_file_to_markdown +``` + +**解决方案**：将它们提取到 `deerflow/utils/file_conversion.py`。仅依赖 `markitdown` + `pathlib`，是通用工具函数。 + +## 5. 基础设施变更 + +### 5.1 LangGraph Server + +LangGraph Server 只需要 harness 包。`langgraph.json` 更新： + +```json +{ + "dependencies": ["./packages/harness"], + "graphs": { + "lead_agent": "deerflow.agents:make_lead_agent" + }, + "checkpointer": { + "path": "./packages/harness/deerflow/agents/checkpointer/async_provider.py:make_checkpointer" + } +} +``` + +### 5.2 Gateway API + +```bash +# serve.sh / Makefile +# PYTHONPATH 包含 backend/ 根目录，使 app.* 和 deerflow.* 都能被找到 +PYTHONPATH=. uvicorn app.gateway.app:app --host 0.0.0.0 --port 8001 +``` + +### 5.3 Nginx + +无需变更（只做 URL 路由，不涉及 Python 模块路径）。 + +### 5.4 Docker + +Dockerfile 中的 module 引用从 `src.` 改为 `deerflow.` / `app.`，`COPY` 命令需覆盖 `packages/` 和 `app/` 目录。 + +## 6. 实施计划 + +分 3 个 PR 递进执行： + +### PR 1：提取共享工具函数（Low Risk） + +1. 创建 `src/skills/validation.py`，从 `gateway/routers/skills.py` 提取 `_validate_skill_frontmatter` +2. 创建 `src/utils/file_conversion.py`，从 `gateway/routers/uploads.py` 提取文件转换逻辑 +3. 更新 `client.py`、`gateway/routers/skills.py`、`gateway/routers/uploads.py` 的 import +4. 运行全部测试确认无回归 + +### PR 2：Rename + 物理拆分（High Risk，原子操作） + +1. 创建 `packages/harness/` 目录，创建 `pyproject.toml` +2. `git mv` 将 harness 相关模块从 `src/` 移入 `packages/harness/deerflow/` +3. `git mv` 将 app 相关模块从 `src/` 移入 `app/` +4. 全局替换 import： + - harness 模块：`src.*` → `deerflow.*`（所有 `.py` 文件、`langgraph.json`、测试、文档） + - app 模块：`src.gateway.*` → `app.gateway.*`、`src.channels.*` → `app.channels.*` +5. 更新 workspace root `pyproject.toml` +6. 更新 `langgraph.json`、`Makefile`、`Dockerfile` +7. `uv sync` + 全部测试 + 手动验证服务启动 + +### PR 3：边界检查 + 文档（Low Risk） + +1. 添加 lint 规则：检查 harness 不 import app 模块 +2. 更新 `CLAUDE.md`、`README.md` + +## 7. 风险与缓解 + +| 风险 | 影响 | 缓解措施 | +|------|------|----------| +| 全局 rename 误伤 | 字符串中的 `src` 被错误替换 | 正则精确匹配 `\bsrc\.`，review diff | +| LangGraph Server 找不到模块 | 服务启动失败 | `langgraph.json` 的 `dependencies` 指向正确的 harness 包路径 | +| App 的 `PYTHONPATH` 缺失 | Gateway/Channel 启动 import 报错 | Makefile/Docker 统一设置 `PYTHONPATH=.` | +| `config.yaml` 中的 `use` 字段引用旧路径 | 运行时模块解析失败 | `config.yaml` 中的 `use` 字段同步更新为 `deerflow.*` | +| 测试中 `sys.path` 混乱 | 测试失败 | 用 editable install（`uv sync`）确保 deerflow 可导入，`conftest.py` 中添加 `app/` 到 `sys.path` | + +## 8. 未来演进 + +- **独立发布**：harness 可以发布到内部 PyPI，让其他项目直接 `pip install deerflow-harness` +- **插件化 App**：不同的 app（web、CLI、bot）可以各自独立，都依赖同一个 harness +- **更细粒度拆分**：如果 harness 内部模块继续增长，可以进一步拆分（如 `deerflow-sandbox`、`deerflow-mcp`） diff --git a/backend/docs/MEMORY_IMPROVEMENTS_SUMMARY.md b/backend/docs/MEMORY_IMPROVEMENTS_SUMMARY.md index da2bcd8..3bb543a 100644 --- a/backend/docs/MEMORY_IMPROVEMENTS_SUMMARY.md +++ b/backend/docs/MEMORY_IMPROVEMENTS_SUMMARY.md @@ -33,6 +33,6 @@ No `current_context` argument is currently available in `main`. ## Verification Pointers -- Implementation: `backend/src/agents/memory/prompt.py` -- Prompt assembly: `backend/src/agents/lead_agent/prompt.py` +- Implementation: `packages/harness/deerflow/agents/memory/prompt.py` +- Prompt assembly: `packages/harness/deerflow/agents/lead_agent/prompt.py` - Regression tests: `backend/tests/test_memory_prompt_injection.py` diff --git a/backend/docs/PATH_EXAMPLES.md b/backend/docs/PATH_EXAMPLES.md index 0a3463f..f9d2b9f 100644 --- a/backend/docs/PATH_EXAMPLES.md +++ b/backend/docs/PATH_EXAMPLES.md @@ -144,7 +144,7 @@ async function uploadAndProcess(threadId: string, file: File) { ```python from pathlib import Path -from src.agents.middlewares.thread_data_middleware import THREAD_DATA_BASE_DIR +from deerflow.agents.middlewares.thread_data_middleware import THREAD_DATA_BASE_DIR def process_uploaded_file(thread_id: str, filename: str): # 使用实际路径 diff --git a/backend/docs/SETUP.md b/backend/docs/SETUP.md index 9e9214f..26f2f65 100644 --- a/backend/docs/SETUP.md +++ b/backend/docs/SETUP.md @@ -30,7 +30,7 @@ DeerFlow uses a YAML configuration file that should be placed in the **project r 4. **Verify configuration**: ```bash cd backend - python -c "from src.config import get_app_config; print('✓ Config loaded:', get_app_config().models[0].name)" + python -c "from deerflow.config import get_app_config; print('✓ Config loaded:', get_app_config().models[0].name)" ``` ## Important Notes @@ -51,7 +51,7 @@ The backend searches for `config.yaml` in this order: ## Sandbox Setup (Optional but Recommended) -If you plan to use Docker/Container-based sandbox (configured in `config.yaml` under `sandbox.use: src.community.aio_sandbox:AioSandboxProvider`), it's highly recommended to pre-pull the container image: +If you plan to use Docker/Container-based sandbox (configured in `config.yaml` under `sandbox.use: deerflow.community.aio_sandbox:AioSandboxProvider`), it's highly recommended to pre-pull the container image: ```bash # From project root @@ -72,7 +72,7 @@ If you skip this step, the image will be automatically pulled on first agent exe ```bash # Check where the backend is looking cd deer-flow/backend -python -c "from src.config.app_config import AppConfig; print(AppConfig.resolve_config_path())" +python -c "from deerflow.config.app_config import AppConfig; print(AppConfig.resolve_config_path())" ``` If it can't find the config: diff --git a/backend/docs/TITLE_GENERATION_IMPLEMENTATION.md b/backend/docs/TITLE_GENERATION_IMPLEMENTATION.md index 351d73c..501975e 100644 --- a/backend/docs/TITLE_GENERATION_IMPLEMENTATION.md +++ b/backend/docs/TITLE_GENERATION_IMPLEMENTATION.md @@ -4,27 +4,27 @@ ### 1. 核心实现文件 -#### [`src/agents/thread_state.py`](../src/agents/thread_state.py) +#### [`packages/harness/deerflow/agents/thread_state.py`](../packages/harness/deerflow/agents/thread_state.py) - ✅ 添加 `title: str | None = None` 字段到 `ThreadState` -#### [`src/config/title_config.py`](../src/config/title_config.py) (新建) +#### [`packages/harness/deerflow/config/title_config.py`](../packages/harness/deerflow/config/title_config.py) (新建) - ✅ 创建 `TitleConfig` 配置类 - ✅ 支持配置：enabled, max_words, max_chars, model_name, prompt_template - ✅ 提供 `get_title_config()` 和 `set_title_config()` 函数 - ✅ 提供 `load_title_config_from_dict()` 从配置文件加载 -#### [`src/agents/title_middleware.py`](../src/agents/title_middleware.py) (新建) +#### [`packages/harness/deerflow/agents/title_middleware.py`](../packages/harness/deerflow/agents/title_middleware.py) (新建) - ✅ 创建 `TitleMiddleware` 类 - ✅ 实现 `_should_generate_title()` 检查是否需要生成 - ✅ 实现 `_generate_title()` 调用 LLM 生成标题 - ✅ 实现 `after_agent()` 钩子，在首次对话后自动触发 - ✅ 包含 fallback 策略（LLM 失败时使用用户消息前几个词） -#### [`src/config/app_config.py`](../src/config/app_config.py) +#### [`packages/harness/deerflow/config/app_config.py`](../packages/harness/deerflow/config/app_config.py) - ✅ 导入 `load_title_config_from_dict` - ✅ 在 `from_file()` 中加载 title 配置 -#### [`src/agents/lead_agent/agent.py`](../src/agents/lead_agent/agent.py) +#### [`packages/harness/deerflow/agents/lead_agent/agent.py`](../packages/harness/deerflow/agents/lead_agent/agent.py) - ✅ 导入 `TitleMiddleware` - ✅ 注册到 `middleware` 列表：`[SandboxMiddleware(), TitleMiddleware()]` @@ -131,7 +131,7 @@ checkpointer = SqliteSaver.from_conn_string("checkpoints.db") // langgraph.json { "graphs": { - "lead_agent": "src.agents:lead_agent" + "lead_agent": "deerflow.agents:lead_agent" }, "checkpointer": "checkpointer:checkpointer" } diff --git a/backend/docs/TODO.md b/backend/docs/TODO.md index 49b6591..75385d7 100644 --- a/backend/docs/TODO.md +++ b/backend/docs/TODO.md @@ -21,8 +21,8 @@ - [ ] Support for more document formats in upload - [ ] Skill marketplace / remote skill installation - [ ] Optimize async concurrency in agent hot path (IM channels multi-task scenario) - - Replace `time.sleep(5)` with `asyncio.sleep()` in `src/tools/builtins/task_tool.py` (subagent polling) - - Replace `subprocess.run()` with `asyncio.create_subprocess_shell()` in `src/sandbox/local/local_sandbox.py` + - Replace `time.sleep(5)` with `asyncio.sleep()` in `packages/harness/deerflow/tools/builtins/task_tool.py` (subagent polling) + - Replace `subprocess.run()` with `asyncio.create_subprocess_shell()` in `packages/harness/deerflow/sandbox/local/local_sandbox.py` - Replace sync `requests` with `httpx.AsyncClient` in community tools (tavily, jina_ai, firecrawl, infoquest, image_search) - Replace sync `model.invoke()` with async `model.ainvoke()` in title_middleware and memory updater - Consider `asyncio.to_thread()` wrapper for remaining blocking file I/O diff --git a/backend/docs/plan_mode_usage.md b/backend/docs/plan_mode_usage.md index 2e4aedb..369f1bd 100644 --- a/backend/docs/plan_mode_usage.md +++ b/backend/docs/plan_mode_usage.md @@ -19,7 +19,7 @@ Plan mode is controlled via **runtime configuration** through the `is_plan_mode` ```python from langchain_core.runnables import RunnableConfig -from src.agents.lead_agent.agent import make_lead_agent +from deerflow.agents.lead_agent.agent import make_lead_agent # Enable plan mode via runtime configuration config = RunnableConfig( @@ -72,7 +72,7 @@ The agent will skip using the todo list for: ```python from langchain_core.runnables import RunnableConfig -from src.agents.lead_agent.agent import make_lead_agent +from deerflow.agents.lead_agent.agent import make_lead_agent # Create agent with plan mode ENABLED config_with_plan_mode = RunnableConfig( @@ -101,7 +101,7 @@ You can enable/disable plan mode dynamically for different conversations or task ```python from langchain_core.runnables import RunnableConfig -from src.agents.lead_agent.agent import make_lead_agent +from deerflow.agents.lead_agent.agent import make_lead_agent def create_agent_for_task(task_complexity: str): """Create agent with plan mode based on task complexity.""" @@ -154,7 +154,7 @@ make_lead_agent(config) ## Implementation Details ### Agent Module -- **Location**: `src/agents/lead_agent/agent.py` +- **Location**: `packages/harness/deerflow/agents/lead_agent/agent.py` - **Function**: `_create_todo_list_middleware(is_plan_mode: bool)` - Creates TodoListMiddleware if plan mode is enabled - **Function**: `_build_middlewares(config: RunnableConfig)` - Builds middleware chain based on runtime config - **Function**: `make_lead_agent(config: RunnableConfig)` - Creates agent with appropriate middlewares @@ -194,7 +194,7 @@ DeerFlow uses custom `system_prompt` and `tool_description` for the TodoListMidd - Comprehensive best practices section - Task completion requirements to prevent premature marking -The custom prompts are defined in `_create_todo_list_middleware()` in `/Users/hetao/workspace/deer-flow/backend/src/agents/lead_agent/agent.py:57`. +The custom prompts are defined in `_create_todo_list_middleware()` in `/Users/hetao/workspace/deer-flow/backend/packages/harness/deerflow/agents/lead_agent/agent.py:57`. ## Notes diff --git a/backend/docs/summarization.md b/backend/docs/summarization.md index d32c3d0..ca1e8dd 100644 --- a/backend/docs/summarization.md +++ b/backend/docs/summarization.md @@ -269,8 +269,8 @@ The middleware intelligently preserves message context: ### Code Structure -- **Configuration**: `src/config/summarization_config.py` -- **Integration**: `src/agents/lead_agent/agent.py` +- **Configuration**: `packages/harness/deerflow/config/summarization_config.py` +- **Integration**: `packages/harness/deerflow/agents/lead_agent/agent.py` - **Middleware**: Uses `langchain.agents.middleware.SummarizationMiddleware` ### Middleware Order diff --git a/backend/docs/task_tool_improvements.md b/backend/docs/task_tool_improvements.md index 3a20f98..7b04212 100644 --- a/backend/docs/task_tool_improvements.md +++ b/backend/docs/task_tool_improvements.md @@ -65,7 +65,7 @@ The `task_status_tool` is no longer exposed to the LLM. It's kept in the codebas ### Polling Logic -Located in `src/tools/builtins/task_tool.py`: +Located in `packages/harness/deerflow/tools/builtins/task_tool.py`: ```python # Start background execution @@ -93,7 +93,7 @@ while True: In addition to polling timeout, subagent execution now has a built-in timeout mechanism: -**Configuration** (`src/subagents/config.py`): +**Configuration** (`packages/harness/deerflow/subagents/config.py`): ```python @dataclass class SubagentConfig: diff --git a/backend/langgraph.json b/backend/langgraph.json index 980ba6b..74f5c69 100644 --- a/backend/langgraph.json +++ b/backend/langgraph.json @@ -6,9 +6,9 @@ ], "env": ".env", "graphs": { - "lead_agent": "src.agents:make_lead_agent" + "lead_agent": "deerflow.agents:make_lead_agent" }, "checkpointer": { - "path": "./src/agents/checkpointer/async_provider.py:make_checkpointer" + "path": "./packages/harness/deerflow/agents/checkpointer/async_provider.py:make_checkpointer" } -} \ No newline at end of file +} diff --git a/backend/packages/harness/deerflow/__init__.py b/backend/packages/harness/deerflow/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/src/agents/__init__.py b/backend/packages/harness/deerflow/agents/__init__.py similarity index 100% rename from backend/src/agents/__init__.py rename to backend/packages/harness/deerflow/agents/__init__.py diff --git a/backend/src/agents/checkpointer/__init__.py b/backend/packages/harness/deerflow/agents/checkpointer/__init__.py similarity index 100% rename from backend/src/agents/checkpointer/__init__.py rename to backend/packages/harness/deerflow/agents/checkpointer/__init__.py diff --git a/backend/src/agents/checkpointer/async_provider.py b/backend/packages/harness/deerflow/agents/checkpointer/async_provider.py similarity index 92% rename from backend/src/agents/checkpointer/async_provider.py rename to backend/packages/harness/deerflow/agents/checkpointer/async_provider.py index 028c306..abd802f 100644 --- a/backend/src/agents/checkpointer/async_provider.py +++ b/backend/packages/harness/deerflow/agents/checkpointer/async_provider.py @@ -7,12 +7,12 @@ Supported backends: memory, sqlite, postgres. Usage (e.g. FastAPI lifespan):: - from src.agents.checkpointer.async_provider import make_checkpointer + from deerflow.agents.checkpointer.async_provider import make_checkpointer async with make_checkpointer() as checkpointer: app.state.checkpointer = checkpointer # InMemorySaver if not configured -For sync usage see :mod:`src.agents.checkpointer.provider`. +For sync usage see :mod:`deerflow.agents.checkpointer.provider`. """ from __future__ import annotations @@ -23,13 +23,13 @@ from collections.abc import AsyncIterator from langgraph.types import Checkpointer -from src.agents.checkpointer.provider import ( +from deerflow.agents.checkpointer.provider import ( POSTGRES_CONN_REQUIRED, POSTGRES_INSTALL, SQLITE_INSTALL, _resolve_sqlite_conn_str, ) -from src.config.app_config import get_app_config +from deerflow.config.app_config import get_app_config logger = logging.getLogger(__name__) diff --git a/backend/src/agents/checkpointer/provider.py b/backend/packages/harness/deerflow/agents/checkpointer/provider.py similarity index 94% rename from backend/src/agents/checkpointer/provider.py rename to backend/packages/harness/deerflow/agents/checkpointer/provider.py index c2dc002..641830f 100644 --- a/backend/src/agents/checkpointer/provider.py +++ b/backend/packages/harness/deerflow/agents/checkpointer/provider.py @@ -7,7 +7,7 @@ Supported backends: memory, sqlite, postgres. Usage:: - from src.agents.checkpointer.provider import get_checkpointer, checkpointer_context + from deerflow.agents.checkpointer.provider import get_checkpointer, checkpointer_context # Singleton — reused across calls, closed on process exit cp = get_checkpointer() @@ -25,9 +25,9 @@ from collections.abc import Iterator from langgraph.types import Checkpointer -from src.config.app_config import get_app_config -from src.config.checkpointer_config import CheckpointerConfig -from src.config.paths import resolve_path +from deerflow.config.app_config import get_app_config +from deerflow.config.checkpointer_config import CheckpointerConfig +from deerflow.config.paths import resolve_path logger = logging.getLogger(__name__) @@ -128,8 +128,8 @@ def get_checkpointer() -> Checkpointer: # Ensure app config is loaded before checking checkpointer config # This prevents returning InMemorySaver when config.yaml actually has a checkpointer section # but hasn't been loaded yet - from src.config.app_config import _app_config - from src.config.checkpointer_config import get_checkpointer_config + from deerflow.config.app_config import _app_config + from deerflow.config.checkpointer_config import get_checkpointer_config if _app_config is None: # Only load config if it hasn't been initialized yet diff --git a/backend/src/agents/lead_agent/__init__.py b/backend/packages/harness/deerflow/agents/lead_agent/__init__.py similarity index 100% rename from backend/src/agents/lead_agent/__init__.py rename to backend/packages/harness/deerflow/agents/lead_agent/__init__.py diff --git a/backend/src/agents/lead_agent/agent.py b/backend/packages/harness/deerflow/agents/lead_agent/agent.py similarity index 92% rename from backend/src/agents/lead_agent/agent.py rename to backend/packages/harness/deerflow/agents/lead_agent/agent.py index de61cba..2fa588c 100644 --- a/backend/src/agents/lead_agent/agent.py +++ b/backend/packages/harness/deerflow/agents/lead_agent/agent.py @@ -4,20 +4,20 @@ from langchain.agents import create_agent from langchain.agents.middleware import SummarizationMiddleware from langchain_core.runnables import RunnableConfig -from src.agents.lead_agent.prompt import apply_prompt_template -from src.agents.middlewares.clarification_middleware import ClarificationMiddleware -from src.agents.middlewares.loop_detection_middleware import LoopDetectionMiddleware -from src.agents.middlewares.memory_middleware import MemoryMiddleware -from src.agents.middlewares.subagent_limit_middleware import SubagentLimitMiddleware -from src.agents.middlewares.title_middleware import TitleMiddleware -from src.agents.middlewares.todo_middleware import TodoMiddleware -from src.agents.middlewares.tool_error_handling_middleware import build_lead_runtime_middlewares -from src.agents.middlewares.view_image_middleware import ViewImageMiddleware -from src.agents.thread_state import ThreadState -from src.config.agents_config import load_agent_config -from src.config.app_config import get_app_config -from src.config.summarization_config import get_summarization_config -from src.models import create_chat_model +from deerflow.agents.lead_agent.prompt import apply_prompt_template +from deerflow.agents.middlewares.clarification_middleware import ClarificationMiddleware +from deerflow.agents.middlewares.loop_detection_middleware import LoopDetectionMiddleware +from deerflow.agents.middlewares.memory_middleware import MemoryMiddleware +from deerflow.agents.middlewares.subagent_limit_middleware import SubagentLimitMiddleware +from deerflow.agents.middlewares.title_middleware import TitleMiddleware +from deerflow.agents.middlewares.todo_middleware import TodoMiddleware +from deerflow.agents.middlewares.tool_error_handling_middleware import build_lead_runtime_middlewares +from deerflow.agents.middlewares.view_image_middleware import ViewImageMiddleware +from deerflow.agents.thread_state import ThreadState +from deerflow.config.agents_config import load_agent_config +from deerflow.config.app_config import get_app_config +from deerflow.config.summarization_config import get_summarization_config +from deerflow.models import create_chat_model logger = logging.getLogger(__name__) @@ -256,8 +256,8 @@ def _build_middlewares(config: RunnableConfig, model_name: str | None, agent_nam def make_lead_agent(config: RunnableConfig): # Lazy import to avoid circular dependency - from src.tools import get_available_tools - from src.tools.builtins import setup_agent + from deerflow.tools import get_available_tools + from deerflow.tools.builtins import setup_agent cfg = config.get("configurable", {}) diff --git a/backend/src/agents/lead_agent/prompt.py b/backend/packages/harness/deerflow/agents/lead_agent/prompt.py similarity index 98% rename from backend/src/agents/lead_agent/prompt.py rename to backend/packages/harness/deerflow/agents/lead_agent/prompt.py index 4988a7b..3591d79 100644 --- a/backend/src/agents/lead_agent/prompt.py +++ b/backend/packages/harness/deerflow/agents/lead_agent/prompt.py @@ -1,7 +1,7 @@ from datetime import datetime -from src.config.agents_config import load_agent_soul -from src.skills import load_skills +from deerflow.config.agents_config import load_agent_soul +from deerflow.skills import load_skills def _build_subagent_section(max_concurrent: int) -> str: @@ -292,8 +292,8 @@ def _get_memory_context(agent_name: str | None = None) -> str: Formatted memory context string wrapped in XML tags, or empty string if disabled. """ try: - from src.agents.memory import format_memory_for_injection, get_memory_data - from src.config.memory_config import get_memory_config + from deerflow.agents.memory import format_memory_for_injection, get_memory_data + from deerflow.config.memory_config import get_memory_config config = get_memory_config() if not config.enabled or not config.injection_enabled: @@ -323,7 +323,7 @@ def get_skills_prompt_section(available_skills: set[str] | None = None) -> str: skills = load_skills(enabled_only=True) try: - from src.config import get_app_config + from deerflow.config import get_app_config config = get_app_config() container_base_path = config.skills.container_path diff --git a/backend/src/agents/memory/__init__.py b/backend/packages/harness/deerflow/agents/memory/__init__.py similarity index 88% rename from backend/src/agents/memory/__init__.py rename to backend/packages/harness/deerflow/agents/memory/__init__.py index 849f9ae..6199964 100644 --- a/backend/src/agents/memory/__init__.py +++ b/backend/packages/harness/deerflow/agents/memory/__init__.py @@ -6,19 +6,19 @@ This module provides a global memory mechanism that: - Injects relevant memory into system prompts for personalized responses """ -from src.agents.memory.prompt import ( +from deerflow.agents.memory.prompt import ( FACT_EXTRACTION_PROMPT, MEMORY_UPDATE_PROMPT, format_conversation_for_update, format_memory_for_injection, ) -from src.agents.memory.queue import ( +from deerflow.agents.memory.queue import ( ConversationContext, MemoryUpdateQueue, get_memory_queue, reset_memory_queue, ) -from src.agents.memory.updater import ( +from deerflow.agents.memory.updater import ( MemoryUpdater, get_memory_data, reload_memory_data, diff --git a/backend/src/agents/memory/prompt.py b/backend/packages/harness/deerflow/agents/memory/prompt.py similarity index 100% rename from backend/src/agents/memory/prompt.py rename to backend/packages/harness/deerflow/agents/memory/prompt.py diff --git a/backend/src/agents/memory/queue.py b/backend/packages/harness/deerflow/agents/memory/queue.py similarity index 97% rename from backend/src/agents/memory/queue.py rename to backend/packages/harness/deerflow/agents/memory/queue.py index 9e4a757..4184a52 100644 --- a/backend/src/agents/memory/queue.py +++ b/backend/packages/harness/deerflow/agents/memory/queue.py @@ -6,7 +6,7 @@ from dataclasses import dataclass, field from datetime import datetime from typing import Any -from src.config.memory_config import get_memory_config +from deerflow.config.memory_config import get_memory_config @dataclass @@ -84,7 +84,7 @@ class MemoryUpdateQueue: def _process_queue(self) -> None: """Process all queued conversation contexts.""" # Import here to avoid circular dependency - from src.agents.memory.updater import MemoryUpdater + from deerflow.agents.memory.updater import MemoryUpdater with self._lock: if self._processing: diff --git a/backend/src/agents/memory/updater.py b/backend/packages/harness/deerflow/agents/memory/updater.py similarity index 98% rename from backend/src/agents/memory/updater.py rename to backend/packages/harness/deerflow/agents/memory/updater.py index 08a8860..19e3e20 100644 --- a/backend/src/agents/memory/updater.py +++ b/backend/packages/harness/deerflow/agents/memory/updater.py @@ -7,13 +7,13 @@ from datetime import datetime from pathlib import Path from typing import Any -from src.agents.memory.prompt import ( +from deerflow.agents.memory.prompt import ( MEMORY_UPDATE_PROMPT, format_conversation_for_update, ) -from src.config.memory_config import get_memory_config -from src.config.paths import get_paths -from src.models import create_chat_model +from deerflow.config.memory_config import get_memory_config +from deerflow.config.paths import get_paths +from deerflow.models import create_chat_model def _get_memory_file_path(agent_name: str | None = None) -> Path: diff --git a/backend/src/agents/middlewares/clarification_middleware.py b/backend/packages/harness/deerflow/agents/middlewares/clarification_middleware.py similarity index 100% rename from backend/src/agents/middlewares/clarification_middleware.py rename to backend/packages/harness/deerflow/agents/middlewares/clarification_middleware.py diff --git a/backend/src/agents/middlewares/dangling_tool_call_middleware.py b/backend/packages/harness/deerflow/agents/middlewares/dangling_tool_call_middleware.py similarity index 100% rename from backend/src/agents/middlewares/dangling_tool_call_middleware.py rename to backend/packages/harness/deerflow/agents/middlewares/dangling_tool_call_middleware.py diff --git a/backend/src/agents/middlewares/loop_detection_middleware.py b/backend/packages/harness/deerflow/agents/middlewares/loop_detection_middleware.py similarity index 100% rename from backend/src/agents/middlewares/loop_detection_middleware.py rename to backend/packages/harness/deerflow/agents/middlewares/loop_detection_middleware.py diff --git a/backend/src/agents/middlewares/memory_middleware.py b/backend/packages/harness/deerflow/agents/middlewares/memory_middleware.py similarity index 97% rename from backend/src/agents/middlewares/memory_middleware.py rename to backend/packages/harness/deerflow/agents/middlewares/memory_middleware.py index 5fa2e24..5076b0d 100644 --- a/backend/src/agents/middlewares/memory_middleware.py +++ b/backend/packages/harness/deerflow/agents/middlewares/memory_middleware.py @@ -7,8 +7,8 @@ from langchain.agents import AgentState from langchain.agents.middleware import AgentMiddleware from langgraph.runtime import Runtime -from src.agents.memory.queue import get_memory_queue -from src.config.memory_config import get_memory_config +from deerflow.agents.memory.queue import get_memory_queue +from deerflow.config.memory_config import get_memory_config class MemoryMiddlewareState(AgentState): diff --git a/backend/src/agents/middlewares/subagent_limit_middleware.py b/backend/packages/harness/deerflow/agents/middlewares/subagent_limit_middleware.py similarity index 97% rename from backend/src/agents/middlewares/subagent_limit_middleware.py rename to backend/packages/harness/deerflow/agents/middlewares/subagent_limit_middleware.py index f4778dc..11de513 100644 --- a/backend/src/agents/middlewares/subagent_limit_middleware.py +++ b/backend/packages/harness/deerflow/agents/middlewares/subagent_limit_middleware.py @@ -7,7 +7,7 @@ from langchain.agents import AgentState from langchain.agents.middleware import AgentMiddleware from langgraph.runtime import Runtime -from src.subagents.executor import MAX_CONCURRENT_SUBAGENTS +from deerflow.subagents.executor import MAX_CONCURRENT_SUBAGENTS logger = logging.getLogger(__name__) diff --git a/backend/src/agents/middlewares/thread_data_middleware.py b/backend/packages/harness/deerflow/agents/middlewares/thread_data_middleware.py similarity index 96% rename from backend/src/agents/middlewares/thread_data_middleware.py rename to backend/packages/harness/deerflow/agents/middlewares/thread_data_middleware.py index ec8f934..479c8c5 100644 --- a/backend/src/agents/middlewares/thread_data_middleware.py +++ b/backend/packages/harness/deerflow/agents/middlewares/thread_data_middleware.py @@ -4,8 +4,8 @@ from langchain.agents import AgentState from langchain.agents.middleware import AgentMiddleware from langgraph.runtime import Runtime -from src.agents.thread_state import ThreadDataState -from src.config.paths import Paths, get_paths +from deerflow.agents.thread_state import ThreadDataState +from deerflow.config.paths import Paths, get_paths class ThreadDataMiddlewareState(AgentState): diff --git a/backend/src/agents/middlewares/title_middleware.py b/backend/packages/harness/deerflow/agents/middlewares/title_middleware.py similarity index 97% rename from backend/src/agents/middlewares/title_middleware.py rename to backend/packages/harness/deerflow/agents/middlewares/title_middleware.py index 4650f60..3c74068 100644 --- a/backend/src/agents/middlewares/title_middleware.py +++ b/backend/packages/harness/deerflow/agents/middlewares/title_middleware.py @@ -6,8 +6,8 @@ from langchain.agents import AgentState from langchain.agents.middleware import AgentMiddleware from langgraph.runtime import Runtime -from src.config.title_config import get_title_config -from src.models import create_chat_model +from deerflow.config.title_config import get_title_config +from deerflow.models import create_chat_model class TitleMiddlewareState(AgentState): diff --git a/backend/src/agents/middlewares/todo_middleware.py b/backend/packages/harness/deerflow/agents/middlewares/todo_middleware.py similarity index 100% rename from backend/src/agents/middlewares/todo_middleware.py rename to backend/packages/harness/deerflow/agents/middlewares/todo_middleware.py diff --git a/backend/src/agents/middlewares/tool_error_handling_middleware.py b/backend/packages/harness/deerflow/agents/middlewares/tool_error_handling_middleware.py similarity index 87% rename from backend/src/agents/middlewares/tool_error_handling_middleware.py rename to backend/packages/harness/deerflow/agents/middlewares/tool_error_handling_middleware.py index 7a22fbc..8c9be44 100644 --- a/backend/src/agents/middlewares/tool_error_handling_middleware.py +++ b/backend/packages/harness/deerflow/agents/middlewares/tool_error_handling_middleware.py @@ -26,10 +26,7 @@ class ToolErrorHandlingMiddleware(AgentMiddleware[AgentState]): if len(detail) > 500: detail = detail[:497] + "..." - content = ( - f"Error: Tool '{tool_name}' failed with {exc.__class__.__name__}: {detail}. " - "Continue with available context, or choose an alternative tool." - ) + content = f"Error: Tool '{tool_name}' failed with {exc.__class__.__name__}: {detail}. Continue with available context, or choose an alternative tool." return ToolMessage( content=content, tool_call_id=tool_call_id, @@ -75,8 +72,8 @@ def _build_runtime_middlewares( lazy_init: bool = True, ) -> list[AgentMiddleware]: """Build shared base middlewares for agent execution.""" - from src.agents.middlewares.thread_data_middleware import ThreadDataMiddleware - from src.sandbox.middleware import SandboxMiddleware + from deerflow.agents.middlewares.thread_data_middleware import ThreadDataMiddleware + from deerflow.sandbox.middleware import SandboxMiddleware middlewares: list[AgentMiddleware] = [ ThreadDataMiddleware(lazy_init=lazy_init), @@ -84,12 +81,12 @@ def _build_runtime_middlewares( ] if include_uploads: - from src.agents.middlewares.uploads_middleware import UploadsMiddleware + from deerflow.agents.middlewares.uploads_middleware import UploadsMiddleware middlewares.insert(1, UploadsMiddleware()) if include_dangling_tool_call_patch: - from src.agents.middlewares.dangling_tool_call_middleware import DanglingToolCallMiddleware + from deerflow.agents.middlewares.dangling_tool_call_middleware import DanglingToolCallMiddleware middlewares.append(DanglingToolCallMiddleware()) diff --git a/backend/src/agents/middlewares/uploads_middleware.py b/backend/packages/harness/deerflow/agents/middlewares/uploads_middleware.py similarity index 99% rename from backend/src/agents/middlewares/uploads_middleware.py rename to backend/packages/harness/deerflow/agents/middlewares/uploads_middleware.py index 3703f3a..22a5448 100644 --- a/backend/src/agents/middlewares/uploads_middleware.py +++ b/backend/packages/harness/deerflow/agents/middlewares/uploads_middleware.py @@ -9,7 +9,7 @@ from langchain.agents.middleware import AgentMiddleware from langchain_core.messages import HumanMessage from langgraph.runtime import Runtime -from src.config.paths import Paths, get_paths +from deerflow.config.paths import Paths, get_paths logger = logging.getLogger(__name__) diff --git a/backend/src/agents/middlewares/view_image_middleware.py b/backend/packages/harness/deerflow/agents/middlewares/view_image_middleware.py similarity index 99% rename from backend/src/agents/middlewares/view_image_middleware.py rename to backend/packages/harness/deerflow/agents/middlewares/view_image_middleware.py index 404cf40..c4ec9ff 100644 --- a/backend/src/agents/middlewares/view_image_middleware.py +++ b/backend/packages/harness/deerflow/agents/middlewares/view_image_middleware.py @@ -7,7 +7,7 @@ from langchain.agents.middleware import AgentMiddleware from langchain_core.messages import AIMessage, HumanMessage, ToolMessage from langgraph.runtime import Runtime -from src.agents.thread_state import ViewedImageData +from deerflow.agents.thread_state import ViewedImageData class ViewImageMiddlewareState(AgentState): diff --git a/backend/src/agents/thread_state.py b/backend/packages/harness/deerflow/agents/thread_state.py similarity index 100% rename from backend/src/agents/thread_state.py rename to backend/packages/harness/deerflow/agents/thread_state.py diff --git a/backend/src/client.py b/backend/packages/harness/deerflow/client.py similarity index 96% rename from backend/src/client.py rename to backend/packages/harness/deerflow/client.py index 8f45e38..4dbd28e 100644 --- a/backend/src/client.py +++ b/backend/packages/harness/deerflow/client.py @@ -4,7 +4,7 @@ Provides direct programmatic access to DeerFlow's agent capabilities without requiring LangGraph Server or Gateway API processes. Usage: - from src.client import DeerFlowClient + from deerflow.client import DeerFlowClient client = DeerFlowClient() response = client.chat("Analyze this paper for me", thread_id="my-thread") @@ -34,13 +34,13 @@ from langchain.agents import create_agent from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolMessage from langchain_core.runnables import RunnableConfig -from src.agents.lead_agent.agent import _build_middlewares -from src.agents.lead_agent.prompt import apply_prompt_template -from src.agents.thread_state import ThreadState -from src.config.app_config import get_app_config, reload_app_config -from src.config.extensions_config import ExtensionsConfig, SkillStateConfig, get_extensions_config, reload_extensions_config -from src.config.paths import get_paths -from src.models import create_chat_model +from deerflow.agents.lead_agent.agent import _build_middlewares +from deerflow.agents.lead_agent.prompt import apply_prompt_template +from deerflow.agents.thread_state import ThreadState +from deerflow.config.app_config import get_app_config, reload_app_config +from deerflow.config.extensions_config import ExtensionsConfig, SkillStateConfig, get_extensions_config, reload_extensions_config +from deerflow.config.paths import get_paths +from deerflow.models import create_chat_model logger = logging.getLogger(__name__) @@ -81,7 +81,7 @@ class DeerFlowClient: Example:: - from src.client import DeerFlowClient + from deerflow.client import DeerFlowClient client = DeerFlowClient() @@ -211,7 +211,7 @@ class DeerFlowClient: } checkpointer = self._checkpointer if checkpointer is None: - from src.agents.checkpointer import get_checkpointer + from deerflow.agents.checkpointer import get_checkpointer checkpointer = get_checkpointer() if checkpointer is not None: @@ -224,7 +224,7 @@ class DeerFlowClient: @staticmethod def _get_tools(*, model_name: str | None, subagent_enabled: bool): """Lazy import to avoid circular dependency at module level.""" - from src.tools import get_available_tools + from deerflow.tools import get_available_tools return get_available_tools(model_name=model_name, subagent_enabled=subagent_enabled) @@ -422,7 +422,7 @@ class DeerFlowClient: Dict with "skills" key containing list of skill info dicts, matching the Gateway API ``SkillsListResponse`` schema. """ - from src.skills.loader import load_skills + from deerflow.skills.loader import load_skills return { "skills": [ @@ -443,7 +443,7 @@ class DeerFlowClient: Returns: Memory data dict (see src/agents/memory/updater.py for structure). """ - from src.agents.memory.updater import get_memory_data + from deerflow.agents.memory.updater import get_memory_data return get_memory_data() @@ -528,7 +528,7 @@ class DeerFlowClient: Returns: Skill info dict, or None if not found. """ - from src.skills.loader import load_skills + from deerflow.skills.loader import load_skills skill = next((s for s in load_skills(enabled_only=False) if s.name == name), None) if skill is None: @@ -555,7 +555,7 @@ class DeerFlowClient: ValueError: If the skill is not found. OSError: If the config file cannot be written. """ - from src.skills.loader import load_skills + from deerflow.skills.loader import load_skills skills = load_skills(enabled_only=False) skill = next((s for s in skills if s.name == name), None) @@ -603,8 +603,8 @@ class DeerFlowClient: FileNotFoundError: If the file does not exist. ValueError: If the file is invalid. """ - from src.gateway.routers.skills import _validate_skill_frontmatter - from src.skills.loader import get_skills_root_path + from deerflow.skills.loader import get_skills_root_path + from deerflow.skills.validation import _validate_skill_frontmatter path = Path(skill_path) if not path.exists(): @@ -664,7 +664,7 @@ class DeerFlowClient: Returns: The reloaded memory data dict. """ - from src.agents.memory.updater import reload_memory_data + from deerflow.agents.memory.updater import reload_memory_data return reload_memory_data() @@ -674,7 +674,7 @@ class DeerFlowClient: Returns: Memory config dict. """ - from src.config.memory_config import get_memory_config + from deerflow.config.memory_config import get_memory_config config = get_memory_config() return { @@ -726,7 +726,7 @@ class DeerFlowClient: FileNotFoundError: If any file does not exist. ValueError: If any supplied path exists but is not a regular file. """ - from src.gateway.routers.uploads import CONVERTIBLE_EXTENSIONS, convert_file_to_markdown + from deerflow.utils.file_conversion import CONVERTIBLE_EXTENSIONS, convert_file_to_markdown # Validate all files upfront to avoid partial uploads. resolved_files = [] diff --git a/backend/src/community/aio_sandbox/__init__.py b/backend/packages/harness/deerflow/community/aio_sandbox/__init__.py similarity index 100% rename from backend/src/community/aio_sandbox/__init__.py rename to backend/packages/harness/deerflow/community/aio_sandbox/__init__.py diff --git a/backend/src/community/aio_sandbox/aio_sandbox.py b/backend/packages/harness/deerflow/community/aio_sandbox/aio_sandbox.py similarity index 99% rename from backend/src/community/aio_sandbox/aio_sandbox.py rename to backend/packages/harness/deerflow/community/aio_sandbox/aio_sandbox.py index 1bf5383..27d05d3 100644 --- a/backend/src/community/aio_sandbox/aio_sandbox.py +++ b/backend/packages/harness/deerflow/community/aio_sandbox/aio_sandbox.py @@ -3,7 +3,7 @@ import logging from agent_sandbox import Sandbox as AioSandboxClient -from src.sandbox.sandbox import Sandbox +from deerflow.sandbox.sandbox import Sandbox logger = logging.getLogger(__name__) diff --git a/backend/src/community/aio_sandbox/aio_sandbox_provider.py b/backend/packages/harness/deerflow/community/aio_sandbox/aio_sandbox_provider.py similarity index 99% rename from backend/src/community/aio_sandbox/aio_sandbox_provider.py rename to backend/packages/harness/deerflow/community/aio_sandbox/aio_sandbox_provider.py index 343addd..0acb552 100644 --- a/backend/src/community/aio_sandbox/aio_sandbox_provider.py +++ b/backend/packages/harness/deerflow/community/aio_sandbox/aio_sandbox_provider.py @@ -20,10 +20,10 @@ import threading import time import uuid -from src.config import get_app_config -from src.config.paths import VIRTUAL_PATH_PREFIX, Paths, get_paths -from src.sandbox.sandbox import Sandbox -from src.sandbox.sandbox_provider import SandboxProvider +from deerflow.config import get_app_config +from deerflow.config.paths import VIRTUAL_PATH_PREFIX, Paths, get_paths +from deerflow.sandbox.sandbox import Sandbox +from deerflow.sandbox.sandbox_provider import SandboxProvider from .aio_sandbox import AioSandbox from .backend import SandboxBackend, wait_for_sandbox_ready @@ -51,7 +51,7 @@ class AioSandboxProvider(SandboxProvider): - Remote/K8s mode (connect to pre-existing sandbox URL) Configuration options in config.yaml under sandbox: - use: src.community.aio_sandbox:AioSandboxProvider + use: deerflow.community.aio_sandbox:AioSandboxProvider image: port: 8080 # Base port for local containers container_prefix: deer-flow-sandbox diff --git a/backend/src/community/aio_sandbox/backend.py b/backend/packages/harness/deerflow/community/aio_sandbox/backend.py similarity index 100% rename from backend/src/community/aio_sandbox/backend.py rename to backend/packages/harness/deerflow/community/aio_sandbox/backend.py diff --git a/backend/src/community/aio_sandbox/local_backend.py b/backend/packages/harness/deerflow/community/aio_sandbox/local_backend.py similarity index 99% rename from backend/src/community/aio_sandbox/local_backend.py rename to backend/packages/harness/deerflow/community/aio_sandbox/local_backend.py index 120ccf9..9dca58f 100644 --- a/backend/src/community/aio_sandbox/local_backend.py +++ b/backend/packages/harness/deerflow/community/aio_sandbox/local_backend.py @@ -10,7 +10,7 @@ import logging import os import subprocess -from src.utils.network import get_free_port, release_port +from deerflow.utils.network import get_free_port, release_port from .backend import SandboxBackend, wait_for_sandbox_ready from .sandbox_info import SandboxInfo diff --git a/backend/src/community/aio_sandbox/remote_backend.py b/backend/packages/harness/deerflow/community/aio_sandbox/remote_backend.py similarity index 98% rename from backend/src/community/aio_sandbox/remote_backend.py rename to backend/packages/harness/deerflow/community/aio_sandbox/remote_backend.py index fbec7af..458d9e6 100644 --- a/backend/src/community/aio_sandbox/remote_backend.py +++ b/backend/packages/harness/deerflow/community/aio_sandbox/remote_backend.py @@ -36,7 +36,7 @@ class RemoteSandboxBackend(SandboxBackend): Typical config.yaml:: sandbox: - use: src.community.aio_sandbox:AioSandboxProvider + use: deerflow.community.aio_sandbox:AioSandboxProvider provisioner_url: http://provisioner:8002 """ diff --git a/backend/src/community/aio_sandbox/sandbox_info.py b/backend/packages/harness/deerflow/community/aio_sandbox/sandbox_info.py similarity index 100% rename from backend/src/community/aio_sandbox/sandbox_info.py rename to backend/packages/harness/deerflow/community/aio_sandbox/sandbox_info.py diff --git a/backend/src/community/firecrawl/tools.py b/backend/packages/harness/deerflow/community/firecrawl/tools.py similarity index 98% rename from backend/src/community/firecrawl/tools.py rename to backend/packages/harness/deerflow/community/firecrawl/tools.py index 0bf46a6..495c60c 100644 --- a/backend/src/community/firecrawl/tools.py +++ b/backend/packages/harness/deerflow/community/firecrawl/tools.py @@ -3,7 +3,7 @@ import json from firecrawl import FirecrawlApp from langchain.tools import tool -from src.config import get_app_config +from deerflow.config import get_app_config def _get_firecrawl_client() -> FirecrawlApp: diff --git a/backend/src/community/image_search/__init__.py b/backend/packages/harness/deerflow/community/image_search/__init__.py similarity index 100% rename from backend/src/community/image_search/__init__.py rename to backend/packages/harness/deerflow/community/image_search/__init__.py diff --git a/backend/src/community/image_search/tools.py b/backend/packages/harness/deerflow/community/image_search/tools.py similarity index 99% rename from backend/src/community/image_search/tools.py rename to backend/packages/harness/deerflow/community/image_search/tools.py index 89ccf34..dc78a5a 100644 --- a/backend/src/community/image_search/tools.py +++ b/backend/packages/harness/deerflow/community/image_search/tools.py @@ -7,7 +7,7 @@ import logging from langchain.tools import tool -from src.config import get_app_config +from deerflow.config import get_app_config logger = logging.getLogger(__name__) diff --git a/backend/src/community/infoquest/infoquest_client.py b/backend/packages/harness/deerflow/community/infoquest/infoquest_client.py similarity index 100% rename from backend/src/community/infoquest/infoquest_client.py rename to backend/packages/harness/deerflow/community/infoquest/infoquest_client.py diff --git a/backend/src/community/infoquest/tools.py b/backend/packages/harness/deerflow/community/infoquest/tools.py similarity index 95% rename from backend/src/community/infoquest/tools.py rename to backend/packages/harness/deerflow/community/infoquest/tools.py index 555d1c7..bf1d77e 100644 --- a/backend/src/community/infoquest/tools.py +++ b/backend/packages/harness/deerflow/community/infoquest/tools.py @@ -1,7 +1,7 @@ from langchain.tools import tool -from src.config import get_app_config -from src.utils.readability import ReadabilityExtractor +from deerflow.config import get_app_config +from deerflow.utils.readability import ReadabilityExtractor from .infoquest_client import InfoQuestClient diff --git a/backend/src/community/jina_ai/jina_client.py b/backend/packages/harness/deerflow/community/jina_ai/jina_client.py similarity index 100% rename from backend/src/community/jina_ai/jina_client.py rename to backend/packages/harness/deerflow/community/jina_ai/jina_client.py diff --git a/backend/src/community/jina_ai/tools.py b/backend/packages/harness/deerflow/community/jina_ai/tools.py similarity index 87% rename from backend/src/community/jina_ai/tools.py rename to backend/packages/harness/deerflow/community/jina_ai/tools.py index 1a9cb41..0bde35a 100644 --- a/backend/src/community/jina_ai/tools.py +++ b/backend/packages/harness/deerflow/community/jina_ai/tools.py @@ -1,8 +1,8 @@ from langchain.tools import tool -from src.community.jina_ai.jina_client import JinaClient -from src.config import get_app_config -from src.utils.readability import ReadabilityExtractor +from deerflow.community.jina_ai.jina_client import JinaClient +from deerflow.config import get_app_config +from deerflow.utils.readability import ReadabilityExtractor readability_extractor = ReadabilityExtractor() diff --git a/backend/src/community/tavily/tools.py b/backend/packages/harness/deerflow/community/tavily/tools.py similarity index 98% rename from backend/src/community/tavily/tools.py rename to backend/packages/harness/deerflow/community/tavily/tools.py index d3741d9..de7996c 100644 --- a/backend/src/community/tavily/tools.py +++ b/backend/packages/harness/deerflow/community/tavily/tools.py @@ -3,7 +3,7 @@ import json from langchain.tools import tool from tavily import TavilyClient -from src.config import get_app_config +from deerflow.config import get_app_config def _get_tavily_client() -> TavilyClient: diff --git a/backend/src/config/__init__.py b/backend/packages/harness/deerflow/config/__init__.py similarity index 100% rename from backend/src/config/__init__.py rename to backend/packages/harness/deerflow/config/__init__.py diff --git a/backend/src/config/agents_config.py b/backend/packages/harness/deerflow/config/agents_config.py similarity index 98% rename from backend/src/config/agents_config.py rename to backend/packages/harness/deerflow/config/agents_config.py index 9d92a00..c35308d 100644 --- a/backend/src/config/agents_config.py +++ b/backend/packages/harness/deerflow/config/agents_config.py @@ -7,7 +7,7 @@ from typing import Any import yaml from pydantic import BaseModel -from src.config.paths import get_paths +from deerflow.config.paths import get_paths logger = logging.getLogger(__name__) diff --git a/backend/src/config/app_config.py b/backend/packages/harness/deerflow/config/app_config.py similarity index 73% rename from backend/src/config/app_config.py rename to backend/packages/harness/deerflow/config/app_config.py index 5b716e9..80ef5cc 100644 --- a/backend/src/config/app_config.py +++ b/backend/packages/harness/deerflow/config/app_config.py @@ -1,3 +1,4 @@ +import logging import os from pathlib import Path from typing import Any, Self @@ -6,19 +7,21 @@ import yaml from dotenv import load_dotenv from pydantic import BaseModel, ConfigDict, Field -from src.config.checkpointer_config import CheckpointerConfig, load_checkpointer_config_from_dict -from src.config.extensions_config import ExtensionsConfig -from src.config.memory_config import load_memory_config_from_dict -from src.config.model_config import ModelConfig -from src.config.sandbox_config import SandboxConfig -from src.config.skills_config import SkillsConfig -from src.config.subagents_config import load_subagents_config_from_dict -from src.config.summarization_config import load_summarization_config_from_dict -from src.config.title_config import load_title_config_from_dict -from src.config.tool_config import ToolConfig, ToolGroupConfig +from deerflow.config.checkpointer_config import CheckpointerConfig, load_checkpointer_config_from_dict +from deerflow.config.extensions_config import ExtensionsConfig +from deerflow.config.memory_config import load_memory_config_from_dict +from deerflow.config.model_config import ModelConfig +from deerflow.config.sandbox_config import SandboxConfig +from deerflow.config.skills_config import SkillsConfig +from deerflow.config.subagents_config import load_subagents_config_from_dict +from deerflow.config.summarization_config import load_summarization_config_from_dict +from deerflow.config.title_config import load_title_config_from_dict +from deerflow.config.tool_config import ToolConfig, ToolGroupConfig load_dotenv() +logger = logging.getLogger(__name__) + class AppConfig(BaseModel): """Config for the DeerFlow application""" @@ -75,7 +78,11 @@ class AppConfig(BaseModel): """ resolved_path = cls.resolve_config_path(config_path) with open(resolved_path, encoding="utf-8") as f: - config_data = yaml.safe_load(f) + config_data = yaml.safe_load(f) or {} + + # Check config version before processing + cls._check_config_version(config_data, resolved_path) + config_data = cls.resolve_env_variables(config_data) # Load title config if present @@ -105,6 +112,52 @@ class AppConfig(BaseModel): result = cls.model_validate(config_data) return result + @classmethod + def _check_config_version(cls, config_data: dict, config_path: Path) -> None: + """Check if the user's config.yaml is outdated compared to config.example.yaml. + + Emits a warning if the user's config_version is lower than the example's. + Missing config_version is treated as version 0 (pre-versioning). + """ + try: + user_version = int(config_data.get("config_version", 0)) + except (TypeError, ValueError): + user_version = 0 + + # Find config.example.yaml by searching config.yaml's directory and its parents + example_path = None + search_dir = config_path.parent + for _ in range(5): # search up to 5 levels + candidate = search_dir / "config.example.yaml" + if candidate.exists(): + example_path = candidate + break + parent = search_dir.parent + if parent == search_dir: + break + search_dir = parent + if example_path is None: + return + + try: + with open(example_path, encoding="utf-8") as f: + example_data = yaml.safe_load(f) + raw = example_data.get("config_version", 0) if example_data else 0 + try: + example_version = int(raw) + except (TypeError, ValueError): + example_version = 0 + except Exception: + return + + if user_version < example_version: + logger.warning( + "Your config.yaml (version %d) is outdated — the latest version is %d. " + "Run `make config-upgrade` to merge new fields into your config.", + user_version, + example_version, + ) + @classmethod def resolve_env_variables(cls, config: Any) -> Any: """Recursively resolve environment variables in the config. diff --git a/backend/src/config/checkpointer_config.py b/backend/packages/harness/deerflow/config/checkpointer_config.py similarity index 100% rename from backend/src/config/checkpointer_config.py rename to backend/packages/harness/deerflow/config/checkpointer_config.py diff --git a/backend/src/config/extensions_config.py b/backend/packages/harness/deerflow/config/extensions_config.py similarity index 100% rename from backend/src/config/extensions_config.py rename to backend/packages/harness/deerflow/config/extensions_config.py diff --git a/backend/src/config/memory_config.py b/backend/packages/harness/deerflow/config/memory_config.py similarity index 100% rename from backend/src/config/memory_config.py rename to backend/packages/harness/deerflow/config/memory_config.py diff --git a/backend/src/config/model_config.py b/backend/packages/harness/deerflow/config/model_config.py similarity index 100% rename from backend/src/config/model_config.py rename to backend/packages/harness/deerflow/config/model_config.py diff --git a/backend/src/config/paths.py b/backend/packages/harness/deerflow/config/paths.py similarity index 100% rename from backend/src/config/paths.py rename to backend/packages/harness/deerflow/config/paths.py diff --git a/backend/src/config/sandbox_config.py b/backend/packages/harness/deerflow/config/sandbox_config.py similarity index 95% rename from backend/src/config/sandbox_config.py rename to backend/packages/harness/deerflow/config/sandbox_config.py index 8cbafac..d025b44 100644 --- a/backend/src/config/sandbox_config.py +++ b/backend/packages/harness/deerflow/config/sandbox_config.py @@ -27,7 +27,7 @@ class SandboxConfig(BaseModel): use: str = Field( ..., - description="Class path of the sandbox provider (e.g. src.sandbox.local:LocalSandboxProvider)", + description="Class path of the sandbox provider (e.g. deerflow.sandbox.local:LocalSandboxProvider)", ) image: str | None = Field( default=None, diff --git a/backend/src/config/skills_config.py b/backend/packages/harness/deerflow/config/skills_config.py similarity index 95% rename from backend/src/config/skills_config.py rename to backend/packages/harness/deerflow/config/skills_config.py index 18876f7..225aecb 100644 --- a/backend/src/config/skills_config.py +++ b/backend/packages/harness/deerflow/config/skills_config.py @@ -31,7 +31,7 @@ class SkillsConfig(BaseModel): return path.resolve() else: # Default: ../skills relative to backend directory - from src.skills.loader import get_skills_root_path + from deerflow.skills.loader import get_skills_root_path return get_skills_root_path() diff --git a/backend/src/config/subagents_config.py b/backend/packages/harness/deerflow/config/subagents_config.py similarity index 100% rename from backend/src/config/subagents_config.py rename to backend/packages/harness/deerflow/config/subagents_config.py diff --git a/backend/src/config/summarization_config.py b/backend/packages/harness/deerflow/config/summarization_config.py similarity index 100% rename from backend/src/config/summarization_config.py rename to backend/packages/harness/deerflow/config/summarization_config.py diff --git a/backend/src/config/title_config.py b/backend/packages/harness/deerflow/config/title_config.py similarity index 100% rename from backend/src/config/title_config.py rename to backend/packages/harness/deerflow/config/title_config.py diff --git a/backend/src/config/tool_config.py b/backend/packages/harness/deerflow/config/tool_config.py similarity index 84% rename from backend/src/config/tool_config.py rename to backend/packages/harness/deerflow/config/tool_config.py index e267f0d..e9c0673 100644 --- a/backend/src/config/tool_config.py +++ b/backend/packages/harness/deerflow/config/tool_config.py @@ -15,6 +15,6 @@ class ToolConfig(BaseModel): group: str = Field(..., description="Group name for the tool") use: str = Field( ..., - description="Variable name of the tool provider(e.g. src.sandbox.tools:bash_tool)", + description="Variable name of the tool provider(e.g. deerflow.sandbox.tools:bash_tool)", ) model_config = ConfigDict(extra="allow") diff --git a/backend/src/config/tracing_config.py b/backend/packages/harness/deerflow/config/tracing_config.py similarity index 100% rename from backend/src/config/tracing_config.py rename to backend/packages/harness/deerflow/config/tracing_config.py diff --git a/backend/src/mcp/__init__.py b/backend/packages/harness/deerflow/mcp/__init__.py similarity index 100% rename from backend/src/mcp/__init__.py rename to backend/packages/harness/deerflow/mcp/__init__.py diff --git a/backend/src/mcp/cache.py b/backend/packages/harness/deerflow/mcp/cache.py similarity index 97% rename from backend/src/mcp/cache.py rename to backend/packages/harness/deerflow/mcp/cache.py index b019875..38750e1 100644 --- a/backend/src/mcp/cache.py +++ b/backend/packages/harness/deerflow/mcp/cache.py @@ -20,7 +20,7 @@ def _get_config_mtime() -> float | None: Returns: The modification time as a float, or None if the file doesn't exist. """ - from src.config.extensions_config import ExtensionsConfig + from deerflow.config.extensions_config import ExtensionsConfig config_path = ExtensionsConfig.resolve_config_path() if config_path and config_path.exists(): @@ -68,7 +68,7 @@ async def initialize_mcp_tools() -> list[BaseTool]: logger.info("MCP tools already initialized") return _mcp_tools_cache or [] - from src.mcp.tools import get_mcp_tools + from deerflow.mcp.tools import get_mcp_tools logger.info("Initializing MCP tools...") _mcp_tools_cache = await get_mcp_tools() diff --git a/backend/src/mcp/client.py b/backend/packages/harness/deerflow/mcp/client.py similarity index 96% rename from backend/src/mcp/client.py rename to backend/packages/harness/deerflow/mcp/client.py index 0e367c1..62bda9d 100644 --- a/backend/src/mcp/client.py +++ b/backend/packages/harness/deerflow/mcp/client.py @@ -3,7 +3,7 @@ import logging from typing import Any -from src.config.extensions_config import ExtensionsConfig, McpServerConfig +from deerflow.config.extensions_config import ExtensionsConfig, McpServerConfig logger = logging.getLogger(__name__) diff --git a/backend/src/mcp/oauth.py b/backend/packages/harness/deerflow/mcp/oauth.py similarity index 98% rename from backend/src/mcp/oauth.py rename to backend/packages/harness/deerflow/mcp/oauth.py index 44d5a04..b4cc1c1 100644 --- a/backend/src/mcp/oauth.py +++ b/backend/packages/harness/deerflow/mcp/oauth.py @@ -8,7 +8,7 @@ from dataclasses import dataclass from datetime import UTC, datetime, timedelta from typing import Any -from src.config.extensions_config import ExtensionsConfig, McpOAuthConfig +from deerflow.config.extensions_config import ExtensionsConfig, McpOAuthConfig logger = logging.getLogger(__name__) diff --git a/backend/src/mcp/tools.py b/backend/packages/harness/deerflow/mcp/tools.py similarity index 92% rename from backend/src/mcp/tools.py rename to backend/packages/harness/deerflow/mcp/tools.py index cb74029..78fea42 100644 --- a/backend/src/mcp/tools.py +++ b/backend/packages/harness/deerflow/mcp/tools.py @@ -4,9 +4,9 @@ import logging from langchain_core.tools import BaseTool -from src.config.extensions_config import ExtensionsConfig -from src.mcp.client import build_servers_config -from src.mcp.oauth import build_oauth_tool_interceptor, get_initial_oauth_headers +from deerflow.config.extensions_config import ExtensionsConfig +from deerflow.mcp.client import build_servers_config +from deerflow.mcp.oauth import build_oauth_tool_interceptor, get_initial_oauth_headers logger = logging.getLogger(__name__) diff --git a/backend/src/models/__init__.py b/backend/packages/harness/deerflow/models/__init__.py similarity index 100% rename from backend/src/models/__init__.py rename to backend/packages/harness/deerflow/models/__init__.py diff --git a/backend/src/models/factory.py b/backend/packages/harness/deerflow/models/factory.py similarity index 96% rename from backend/src/models/factory.py rename to backend/packages/harness/deerflow/models/factory.py index 80da587..ce8d22e 100644 --- a/backend/src/models/factory.py +++ b/backend/packages/harness/deerflow/models/factory.py @@ -2,8 +2,8 @@ import logging from langchain.chat_models import BaseChatModel -from src.config import get_app_config, get_tracing_config, is_tracing_enabled -from src.reflection import resolve_class +from deerflow.config import get_app_config, get_tracing_config, is_tracing_enabled +from deerflow.reflection import resolve_class logger = logging.getLogger(__name__) diff --git a/backend/src/models/patched_deepseek.py b/backend/packages/harness/deerflow/models/patched_deepseek.py similarity index 100% rename from backend/src/models/patched_deepseek.py rename to backend/packages/harness/deerflow/models/patched_deepseek.py diff --git a/backend/src/reflection/__init__.py b/backend/packages/harness/deerflow/reflection/__init__.py similarity index 100% rename from backend/src/reflection/__init__.py rename to backend/packages/harness/deerflow/reflection/__init__.py diff --git a/backend/src/reflection/resolvers.py b/backend/packages/harness/deerflow/reflection/resolvers.py similarity index 100% rename from backend/src/reflection/resolvers.py rename to backend/packages/harness/deerflow/reflection/resolvers.py diff --git a/backend/src/sandbox/__init__.py b/backend/packages/harness/deerflow/sandbox/__init__.py similarity index 100% rename from backend/src/sandbox/__init__.py rename to backend/packages/harness/deerflow/sandbox/__init__.py diff --git a/backend/src/sandbox/exceptions.py b/backend/packages/harness/deerflow/sandbox/exceptions.py similarity index 100% rename from backend/src/sandbox/exceptions.py rename to backend/packages/harness/deerflow/sandbox/exceptions.py diff --git a/backend/src/sandbox/local/__init__.py b/backend/packages/harness/deerflow/sandbox/local/__init__.py similarity index 100% rename from backend/src/sandbox/local/__init__.py rename to backend/packages/harness/deerflow/sandbox/local/__init__.py diff --git a/backend/src/sandbox/local/list_dir.py b/backend/packages/harness/deerflow/sandbox/local/list_dir.py similarity index 100% rename from backend/src/sandbox/local/list_dir.py rename to backend/packages/harness/deerflow/sandbox/local/list_dir.py diff --git a/backend/src/sandbox/local/local_sandbox.py b/backend/packages/harness/deerflow/sandbox/local/local_sandbox.py similarity index 98% rename from backend/src/sandbox/local/local_sandbox.py rename to backend/packages/harness/deerflow/sandbox/local/local_sandbox.py index b3cec11..70655c8 100644 --- a/backend/src/sandbox/local/local_sandbox.py +++ b/backend/packages/harness/deerflow/sandbox/local/local_sandbox.py @@ -3,8 +3,8 @@ import shutil import subprocess from pathlib import Path -from src.sandbox.local.list_dir import list_dir -from src.sandbox.sandbox import Sandbox +from deerflow.sandbox.local.list_dir import list_dir +from deerflow.sandbox.sandbox import Sandbox class LocalSandbox(Sandbox): diff --git a/backend/src/sandbox/local/local_sandbox_provider.py b/backend/packages/harness/deerflow/sandbox/local/local_sandbox_provider.py similarity index 89% rename from backend/src/sandbox/local/local_sandbox_provider.py rename to backend/packages/harness/deerflow/sandbox/local/local_sandbox_provider.py index 4fa3442..3274f9d 100644 --- a/backend/src/sandbox/local/local_sandbox_provider.py +++ b/backend/packages/harness/deerflow/sandbox/local/local_sandbox_provider.py @@ -1,6 +1,6 @@ -from src.sandbox.local.local_sandbox import LocalSandbox -from src.sandbox.sandbox import Sandbox -from src.sandbox.sandbox_provider import SandboxProvider +from deerflow.sandbox.local.local_sandbox import LocalSandbox +from deerflow.sandbox.sandbox import Sandbox +from deerflow.sandbox.sandbox_provider import SandboxProvider _singleton: LocalSandbox | None = None @@ -23,7 +23,7 @@ class LocalSandboxProvider(SandboxProvider): # Map skills container path to local skills directory try: - from src.config import get_app_config + from deerflow.config import get_app_config config = get_app_config() skills_path = config.skills.get_skills_path() diff --git a/backend/src/sandbox/middleware.py b/backend/packages/harness/deerflow/sandbox/middleware.py similarity index 96% rename from backend/src/sandbox/middleware.py rename to backend/packages/harness/deerflow/sandbox/middleware.py index a6c7e8d..61cfece 100644 --- a/backend/src/sandbox/middleware.py +++ b/backend/packages/harness/deerflow/sandbox/middleware.py @@ -5,8 +5,8 @@ from langchain.agents import AgentState from langchain.agents.middleware import AgentMiddleware from langgraph.runtime import Runtime -from src.agents.thread_state import SandboxState, ThreadDataState -from src.sandbox import get_sandbox_provider +from deerflow.agents.thread_state import SandboxState, ThreadDataState +from deerflow.sandbox import get_sandbox_provider logger = logging.getLogger(__name__) diff --git a/backend/src/sandbox/sandbox.py b/backend/packages/harness/deerflow/sandbox/sandbox.py similarity index 100% rename from backend/src/sandbox/sandbox.py rename to backend/packages/harness/deerflow/sandbox/sandbox.py diff --git a/backend/src/sandbox/sandbox_provider.py b/backend/packages/harness/deerflow/sandbox/sandbox_provider.py similarity index 95% rename from backend/src/sandbox/sandbox_provider.py rename to backend/packages/harness/deerflow/sandbox/sandbox_provider.py index 5440dd2..9051e60 100644 --- a/backend/src/sandbox/sandbox_provider.py +++ b/backend/packages/harness/deerflow/sandbox/sandbox_provider.py @@ -1,8 +1,8 @@ from abc import ABC, abstractmethod -from src.config import get_app_config -from src.reflection import resolve_class -from src.sandbox.sandbox import Sandbox +from deerflow.config import get_app_config +from deerflow.reflection import resolve_class +from deerflow.sandbox.sandbox import Sandbox class SandboxProvider(ABC): diff --git a/backend/src/sandbox/tools.py b/backend/packages/harness/deerflow/sandbox/tools.py similarity index 98% rename from backend/src/sandbox/tools.py rename to backend/packages/harness/deerflow/sandbox/tools.py index 7d492b5..ab1879c 100644 --- a/backend/src/sandbox/tools.py +++ b/backend/packages/harness/deerflow/sandbox/tools.py @@ -4,15 +4,15 @@ from pathlib import Path from langchain.tools import ToolRuntime, tool from langgraph.typing import ContextT -from src.agents.thread_state import ThreadDataState, ThreadState -from src.config.paths import VIRTUAL_PATH_PREFIX -from src.sandbox.exceptions import ( +from deerflow.agents.thread_state import ThreadDataState, ThreadState +from deerflow.config.paths import VIRTUAL_PATH_PREFIX +from deerflow.sandbox.exceptions import ( SandboxError, SandboxNotFoundError, SandboxRuntimeError, ) -from src.sandbox.sandbox import Sandbox -from src.sandbox.sandbox_provider import get_sandbox_provider +from deerflow.sandbox.sandbox import Sandbox +from deerflow.sandbox.sandbox_provider import get_sandbox_provider _ABSOLUTE_PATH_PATTERN = re.compile(r"(?()]+)") _LOCAL_BASH_SYSTEM_PATH_PREFIXES = ( diff --git a/backend/packages/harness/deerflow/skills/__init__.py b/backend/packages/harness/deerflow/skills/__init__.py new file mode 100644 index 0000000..d0ca62b --- /dev/null +++ b/backend/packages/harness/deerflow/skills/__init__.py @@ -0,0 +1,5 @@ +from .loader import get_skills_root_path, load_skills +from .types import Skill +from .validation import ALLOWED_FRONTMATTER_PROPERTIES, _validate_skill_frontmatter + +__all__ = ["load_skills", "get_skills_root_path", "Skill", "ALLOWED_FRONTMATTER_PROPERTIES", "_validate_skill_frontmatter"] diff --git a/backend/src/skills/loader.py b/backend/packages/harness/deerflow/skills/loader.py similarity index 91% rename from backend/src/skills/loader.py rename to backend/packages/harness/deerflow/skills/loader.py index 7d4c37f..d7ffa16 100644 --- a/backend/src/skills/loader.py +++ b/backend/packages/harness/deerflow/skills/loader.py @@ -12,8 +12,8 @@ def get_skills_root_path() -> Path: Returns: Path to the skills directory (deer-flow/skills) """ - # backend directory is current file's parent's parent's parent - backend_dir = Path(__file__).resolve().parent.parent.parent + # loader.py lives at packages/harness/deerflow/skills/loader.py — 5 parents up reaches backend/ + backend_dir = Path(__file__).resolve().parent.parent.parent.parent.parent # skills directory is sibling to backend directory skills_dir = backend_dir.parent / "skills" return skills_dir @@ -39,7 +39,7 @@ def load_skills(skills_path: Path | None = None, use_config: bool = True, enable if skills_path is None: if use_config: try: - from src.config import get_app_config + from deerflow.config import get_app_config config = get_app_config() skills_path = config.skills.get_skills_path() @@ -79,7 +79,7 @@ def load_skills(skills_path: Path | None = None, use_config: bool = True, enable # made through the Gateway API (which runs in a separate process) are immediately # reflected in the LangGraph Server when loading skills. try: - from src.config.extensions_config import ExtensionsConfig + from deerflow.config.extensions_config import ExtensionsConfig extensions_config = ExtensionsConfig.from_file() for skill in skills: diff --git a/backend/src/skills/parser.py b/backend/packages/harness/deerflow/skills/parser.py similarity index 100% rename from backend/src/skills/parser.py rename to backend/packages/harness/deerflow/skills/parser.py diff --git a/backend/src/skills/types.py b/backend/packages/harness/deerflow/skills/types.py similarity index 100% rename from backend/src/skills/types.py rename to backend/packages/harness/deerflow/skills/types.py diff --git a/backend/packages/harness/deerflow/skills/validation.py b/backend/packages/harness/deerflow/skills/validation.py new file mode 100644 index 0000000..648f2f6 --- /dev/null +++ b/backend/packages/harness/deerflow/skills/validation.py @@ -0,0 +1,85 @@ +"""Skill frontmatter validation utilities. + +Pure-logic validation of SKILL.md frontmatter — no FastAPI or HTTP dependencies. +""" + +import re +from pathlib import Path + +import yaml + +# Allowed properties in SKILL.md frontmatter +ALLOWED_FRONTMATTER_PROPERTIES = {"name", "description", "license", "allowed-tools", "metadata", "compatibility", "version", "author"} + + +def _validate_skill_frontmatter(skill_dir: Path) -> tuple[bool, str, str | None]: + """Validate a skill directory's SKILL.md frontmatter. + + Args: + skill_dir: Path to the skill directory containing SKILL.md. + + Returns: + Tuple of (is_valid, message, skill_name). + """ + skill_md = skill_dir / "SKILL.md" + if not skill_md.exists(): + return False, "SKILL.md not found", None + + content = skill_md.read_text() + if not content.startswith("---"): + return False, "No YAML frontmatter found", None + + # Extract frontmatter + match = re.match(r"^---\n(.*?)\n---", content, re.DOTALL) + if not match: + return False, "Invalid frontmatter format", None + + frontmatter_text = match.group(1) + + # Parse YAML frontmatter + try: + frontmatter = yaml.safe_load(frontmatter_text) + if not isinstance(frontmatter, dict): + return False, "Frontmatter must be a YAML dictionary", None + except yaml.YAMLError as e: + return False, f"Invalid YAML in frontmatter: {e}", None + + # Check for unexpected properties + unexpected_keys = set(frontmatter.keys()) - ALLOWED_FRONTMATTER_PROPERTIES + if unexpected_keys: + return False, f"Unexpected key(s) in SKILL.md frontmatter: {', '.join(sorted(unexpected_keys))}", None + + # Check required fields + if "name" not in frontmatter: + return False, "Missing 'name' in frontmatter", None + if "description" not in frontmatter: + return False, "Missing 'description' in frontmatter", None + + # Validate name + name = frontmatter.get("name", "") + if not isinstance(name, str): + return False, f"Name must be a string, got {type(name).__name__}", None + name = name.strip() + if not name: + return False, "Name cannot be empty", None + + # Check naming convention (hyphen-case: lowercase with hyphens) + if not re.match(r"^[a-z0-9-]+$", name): + return False, f"Name '{name}' should be hyphen-case (lowercase letters, digits, and hyphens only)", None + if name.startswith("-") or name.endswith("-") or "--" in name: + return False, f"Name '{name}' cannot start/end with hyphen or contain consecutive hyphens", None + if len(name) > 64: + return False, f"Name is too long ({len(name)} characters). Maximum is 64 characters.", None + + # Validate description + description = frontmatter.get("description", "") + if not isinstance(description, str): + return False, f"Description must be a string, got {type(description).__name__}", None + description = description.strip() + if description: + if "<" in description or ">" in description: + return False, "Description cannot contain angle brackets (< or >)", None + if len(description) > 1024: + return False, f"Description is too long ({len(description)} characters). Maximum is 1024 characters.", None + + return True, "Skill is valid!", name diff --git a/backend/src/subagents/__init__.py b/backend/packages/harness/deerflow/subagents/__init__.py similarity index 100% rename from backend/src/subagents/__init__.py rename to backend/packages/harness/deerflow/subagents/__init__.py diff --git a/backend/src/subagents/builtins/__init__.py b/backend/packages/harness/deerflow/subagents/builtins/__init__.py similarity index 100% rename from backend/src/subagents/builtins/__init__.py rename to backend/packages/harness/deerflow/subagents/builtins/__init__.py diff --git a/backend/src/subagents/builtins/bash_agent.py b/backend/packages/harness/deerflow/subagents/builtins/bash_agent.py similarity index 96% rename from backend/src/subagents/builtins/bash_agent.py rename to backend/packages/harness/deerflow/subagents/builtins/bash_agent.py index f3718b1..9188f09 100644 --- a/backend/src/subagents/builtins/bash_agent.py +++ b/backend/packages/harness/deerflow/subagents/builtins/bash_agent.py @@ -1,6 +1,6 @@ """Bash command execution subagent configuration.""" -from src.subagents.config import SubagentConfig +from deerflow.subagents.config import SubagentConfig BASH_AGENT_CONFIG = SubagentConfig( name="bash", diff --git a/backend/src/subagents/builtins/general_purpose.py b/backend/packages/harness/deerflow/subagents/builtins/general_purpose.py similarity index 97% rename from backend/src/subagents/builtins/general_purpose.py rename to backend/packages/harness/deerflow/subagents/builtins/general_purpose.py index 2ffad29..de48a2f 100644 --- a/backend/src/subagents/builtins/general_purpose.py +++ b/backend/packages/harness/deerflow/subagents/builtins/general_purpose.py @@ -1,6 +1,6 @@ """General-purpose subagent configuration.""" -from src.subagents.config import SubagentConfig +from deerflow.subagents.config import SubagentConfig GENERAL_PURPOSE_CONFIG = SubagentConfig( name="general-purpose", diff --git a/backend/src/subagents/config.py b/backend/packages/harness/deerflow/subagents/config.py similarity index 100% rename from backend/src/subagents/config.py rename to backend/packages/harness/deerflow/subagents/config.py diff --git a/backend/src/subagents/executor.py b/backend/packages/harness/deerflow/subagents/executor.py similarity index 98% rename from backend/src/subagents/executor.py rename to backend/packages/harness/deerflow/subagents/executor.py index b269dab..f6bd55b 100644 --- a/backend/src/subagents/executor.py +++ b/backend/packages/harness/deerflow/subagents/executor.py @@ -16,9 +16,9 @@ from langchain.tools import BaseTool from langchain_core.messages import AIMessage, HumanMessage from langchain_core.runnables import RunnableConfig -from src.agents.thread_state import SandboxState, ThreadDataState, ThreadState -from src.models import create_chat_model -from src.subagents.config import SubagentConfig +from deerflow.agents.thread_state import SandboxState, ThreadDataState, ThreadState +from deerflow.models import create_chat_model +from deerflow.subagents.config import SubagentConfig logger = logging.getLogger(__name__) @@ -166,7 +166,7 @@ class SubagentExecutor: model_name = _get_model_name(self.config, self.parent_model) model = create_chat_model(name=model_name, thinking_enabled=False) - from src.agents.middlewares.tool_error_handling_middleware import build_subagent_runtime_middlewares + from deerflow.agents.middlewares.tool_error_handling_middleware import build_subagent_runtime_middlewares # Reuse shared middleware composition with lead agent. middlewares = build_subagent_runtime_middlewares(lazy_init=True) diff --git a/backend/src/subagents/registry.py b/backend/packages/harness/deerflow/subagents/registry.py similarity index 88% rename from backend/src/subagents/registry.py rename to backend/packages/harness/deerflow/subagents/registry.py index e2a6b11..16afa2e 100644 --- a/backend/src/subagents/registry.py +++ b/backend/packages/harness/deerflow/subagents/registry.py @@ -3,8 +3,8 @@ import logging from dataclasses import replace -from src.subagents.builtins import BUILTIN_SUBAGENTS -from src.subagents.config import SubagentConfig +from deerflow.subagents.builtins import BUILTIN_SUBAGENTS +from deerflow.subagents.config import SubagentConfig logger = logging.getLogger(__name__) @@ -23,7 +23,7 @@ def get_subagent_config(name: str) -> SubagentConfig | None: return None # Apply timeout override from config.yaml (lazy import to avoid circular deps) - from src.config.subagents_config import get_subagents_app_config + from deerflow.config.subagents_config import get_subagents_app_config app_config = get_subagents_app_config() effective_timeout = app_config.get_timeout_for(name) diff --git a/backend/src/tools/__init__.py b/backend/packages/harness/deerflow/tools/__init__.py similarity index 100% rename from backend/src/tools/__init__.py rename to backend/packages/harness/deerflow/tools/__init__.py diff --git a/backend/src/tools/builtins/__init__.py b/backend/packages/harness/deerflow/tools/builtins/__init__.py similarity index 100% rename from backend/src/tools/builtins/__init__.py rename to backend/packages/harness/deerflow/tools/builtins/__init__.py diff --git a/backend/src/tools/builtins/clarification_tool.py b/backend/packages/harness/deerflow/tools/builtins/clarification_tool.py similarity index 100% rename from backend/src/tools/builtins/clarification_tool.py rename to backend/packages/harness/deerflow/tools/builtins/clarification_tool.py diff --git a/backend/src/tools/builtins/present_file_tool.py b/backend/packages/harness/deerflow/tools/builtins/present_file_tool.py similarity index 96% rename from backend/src/tools/builtins/present_file_tool.py rename to backend/packages/harness/deerflow/tools/builtins/present_file_tool.py index d3eb086..47a7648 100644 --- a/backend/src/tools/builtins/present_file_tool.py +++ b/backend/packages/harness/deerflow/tools/builtins/present_file_tool.py @@ -6,8 +6,8 @@ from langchain_core.messages import ToolMessage from langgraph.types import Command from langgraph.typing import ContextT -from src.agents.thread_state import ThreadState -from src.config.paths import VIRTUAL_PATH_PREFIX, get_paths +from deerflow.agents.thread_state import ThreadState +from deerflow.config.paths import VIRTUAL_PATH_PREFIX, get_paths OUTPUTS_VIRTUAL_PREFIX = f"{VIRTUAL_PATH_PREFIX}/outputs" diff --git a/backend/src/tools/builtins/setup_agent_tool.py b/backend/packages/harness/deerflow/tools/builtins/setup_agent_tool.py similarity index 97% rename from backend/src/tools/builtins/setup_agent_tool.py rename to backend/packages/harness/deerflow/tools/builtins/setup_agent_tool.py index e107e9f..26937fe 100644 --- a/backend/src/tools/builtins/setup_agent_tool.py +++ b/backend/packages/harness/deerflow/tools/builtins/setup_agent_tool.py @@ -6,7 +6,7 @@ from langchain_core.tools import tool from langgraph.prebuilt import ToolRuntime from langgraph.types import Command -from src.config.paths import get_paths +from deerflow.config.paths import get_paths logger = logging.getLogger(__name__) diff --git a/backend/src/tools/builtins/task_tool.py b/backend/packages/harness/deerflow/tools/builtins/task_tool.py similarity index 95% rename from backend/src/tools/builtins/task_tool.py rename to backend/packages/harness/deerflow/tools/builtins/task_tool.py index 0635047..902e1d7 100644 --- a/backend/src/tools/builtins/task_tool.py +++ b/backend/packages/harness/deerflow/tools/builtins/task_tool.py @@ -10,10 +10,10 @@ from langchain.tools import InjectedToolCallId, ToolRuntime, tool from langgraph.config import get_stream_writer from langgraph.typing import ContextT -from src.agents.lead_agent.prompt import get_skills_prompt_section -from src.agents.thread_state import ThreadState -from src.subagents import SubagentExecutor, get_subagent_config -from src.subagents.executor import SubagentStatus, cleanup_background_task, get_background_task_result +from deerflow.agents.lead_agent.prompt import get_skills_prompt_section +from deerflow.agents.thread_state import ThreadState +from deerflow.subagents import SubagentExecutor, get_subagent_config +from deerflow.subagents.executor import SubagentStatus, cleanup_background_task, get_background_task_result logger = logging.getLogger(__name__) @@ -96,7 +96,7 @@ def task_tool( # Get available tools (excluding task tool to prevent nesting) # Lazy import to avoid circular dependency - from src.tools import get_available_tools + from deerflow.tools import get_available_tools # Subagents should not have subagent tools enabled (prevent recursive nesting) tools = get_available_tools(model_name=parent_model, subagent_enabled=False) diff --git a/backend/src/tools/builtins/view_image_tool.py b/backend/packages/harness/deerflow/tools/builtins/view_image_tool.py similarity index 96% rename from backend/src/tools/builtins/view_image_tool.py rename to backend/packages/harness/deerflow/tools/builtins/view_image_tool.py index f979294..21d6623 100644 --- a/backend/src/tools/builtins/view_image_tool.py +++ b/backend/packages/harness/deerflow/tools/builtins/view_image_tool.py @@ -8,8 +8,8 @@ from langchain_core.messages import ToolMessage from langgraph.types import Command from langgraph.typing import ContextT -from src.agents.thread_state import ThreadState -from src.sandbox.tools import get_thread_data, replace_virtual_path +from deerflow.agents.thread_state import ThreadState +from deerflow.sandbox.tools import get_thread_data, replace_virtual_path @tool("view_image", parse_docstring=True) diff --git a/backend/src/tools/tools.py b/backend/packages/harness/deerflow/tools/tools.py similarity index 87% rename from backend/src/tools/tools.py rename to backend/packages/harness/deerflow/tools/tools.py index 2febdbc..5e7b560 100644 --- a/backend/src/tools/tools.py +++ b/backend/packages/harness/deerflow/tools/tools.py @@ -2,9 +2,9 @@ import logging from langchain.tools import BaseTool -from src.config import get_app_config -from src.reflection import resolve_variable -from src.tools.builtins import ask_clarification_tool, present_file_tool, task_tool, view_image_tool +from deerflow.config import get_app_config +from deerflow.reflection import resolve_variable +from deerflow.tools.builtins import ask_clarification_tool, present_file_tool, task_tool, view_image_tool logger = logging.getLogger(__name__) @@ -28,7 +28,7 @@ def get_available_tools( """Get all available tools from config. Note: MCP tools should be initialized at application startup using - `initialize_mcp_tools()` from src.mcp module. + `initialize_mcp_tools()` from deerflow.mcp module. Args: groups: Optional list of tool groups to filter by. @@ -50,8 +50,8 @@ def get_available_tools( mcp_tools = [] if include_mcp: try: - from src.config.extensions_config import ExtensionsConfig - from src.mcp.cache import get_cached_mcp_tools + from deerflow.config.extensions_config import ExtensionsConfig + from deerflow.mcp.cache import get_cached_mcp_tools extensions_config = ExtensionsConfig.from_file() if extensions_config.get_enabled_mcp_servers(): diff --git a/backend/packages/harness/deerflow/utils/file_conversion.py b/backend/packages/harness/deerflow/utils/file_conversion.py new file mode 100644 index 0000000..45cdf12 --- /dev/null +++ b/backend/packages/harness/deerflow/utils/file_conversion.py @@ -0,0 +1,47 @@ +"""File conversion utilities. + +Converts document files (PDF, PPT, Excel, Word) to Markdown using markitdown. +No FastAPI or HTTP dependencies — pure utility functions. +""" + +import logging +from pathlib import Path + +logger = logging.getLogger(__name__) + +# File extensions that should be converted to markdown +CONVERTIBLE_EXTENSIONS = { + ".pdf", + ".ppt", + ".pptx", + ".xls", + ".xlsx", + ".doc", + ".docx", +} + + +async def convert_file_to_markdown(file_path: Path) -> Path | None: + """Convert a file to markdown using markitdown. + + Args: + file_path: Path to the file to convert. + + Returns: + Path to the markdown file if conversion was successful, None otherwise. + """ + try: + from markitdown import MarkItDown + + md = MarkItDown() + result = md.convert(str(file_path)) + + # Save as .md file with same name + md_path = file_path.with_suffix(".md") + md_path.write_text(result.text_content, encoding="utf-8") + + logger.info(f"Converted {file_path.name} to markdown: {md_path.name}") + return md_path + except Exception as e: + logger.error(f"Failed to convert {file_path.name} to markdown: {e}") + return None diff --git a/backend/src/utils/network.py b/backend/packages/harness/deerflow/utils/network.py similarity index 100% rename from backend/src/utils/network.py rename to backend/packages/harness/deerflow/utils/network.py diff --git a/backend/src/utils/readability.py b/backend/packages/harness/deerflow/utils/readability.py similarity index 100% rename from backend/src/utils/readability.py rename to backend/packages/harness/deerflow/utils/readability.py diff --git a/backend/packages/harness/pyproject.toml b/backend/packages/harness/pyproject.toml new file mode 100644 index 0000000..f8c5bfc --- /dev/null +++ b/backend/packages/harness/pyproject.toml @@ -0,0 +1,40 @@ +[project] +name = "deerflow-harness" +version = "0.1.0" +description = "DeerFlow agent harness framework" +requires-python = ">=3.12" +dependencies = [ + "agent-sandbox>=0.0.19", + "dotenv>=0.9.9", + "httpx>=0.28.0", + "kubernetes>=30.0.0", + "langchain>=1.2.3", + "langchain-anthropic>=1.3.4", + "langchain-deepseek>=1.0.1", + "langchain-mcp-adapters>=0.1.0", + "langchain-openai>=1.1.7", + "langgraph>=1.0.6", + "langgraph-api>=0.7.0,<0.8.0", + "langgraph-cli>=0.4.14", + "langgraph-runtime-inmem>=0.22.1", + "markdownify>=1.2.2", + "markitdown[all,xlsx]>=0.0.1a2", + "pydantic>=2.12.5", + "pyyaml>=6.0.3", + "readabilipy>=0.3.0", + "tavily-python>=0.7.17", + "firecrawl-py>=1.15.0", + "tiktoken>=0.8.0", + "ddgs>=9.10.0", + "duckdb>=1.4.4", + "langchain-google-genai>=4.2.1", + "langgraph-checkpoint-sqlite>=3.0.3", + "langgraph-sdk>=0.1.51", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["deerflow"] diff --git a/backend/pyproject.toml b/backend/pyproject.toml index f2145b2..c94b468 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -5,35 +5,12 @@ description = "LangGraph-based AI agent system with sandbox execution capabiliti readme = "README.md" requires-python = ">=3.12" dependencies = [ - "agent-sandbox>=0.0.19", - "dotenv>=0.9.9", + "deerflow-harness", "fastapi>=0.115.0", "httpx>=0.28.0", - "kubernetes>=30.0.0", - "langchain>=1.2.3", - "langchain-anthropic>=1.3.4", - "langchain-deepseek>=1.0.1", - "langchain-mcp-adapters>=0.1.0", - "langchain-openai>=1.1.7", - "langgraph>=1.0.6", - "langgraph-api>=0.7.0,<0.8.0", - "langgraph-cli>=0.4.14", - "langgraph-runtime-inmem>=0.22.1", - "markdownify>=1.2.2", - "markitdown[all,xlsx]>=0.0.1a2", - "pydantic>=2.12.5", "python-multipart>=0.0.20", - "pyyaml>=6.0.3", - "readabilipy>=0.3.0", "sse-starlette>=2.1.0", - "tavily-python>=0.7.17", - "firecrawl-py>=1.15.0", - "tiktoken>=0.8.0", "uvicorn[standard]>=0.34.0", - "ddgs>=9.10.0", - "duckdb>=1.4.4", - "langchain-google-genai>=4.2.1", - "langgraph-checkpoint-sqlite>=3.0.3", "lark-oapi>=1.4.0", "slack-sdk>=3.33.0", "python-telegram-bot>=21.0", @@ -43,3 +20,9 @@ dependencies = [ [dependency-groups] dev = ["pytest>=8.0.0", "ruff>=0.14.11"] + +[tool.uv.workspace] +members = ["packages/harness"] + +[tool.uv.sources] +deerflow-harness = { workspace = true } diff --git a/backend/ruff.toml b/backend/ruff.toml index 6dbc56c..3514c05 100644 --- a/backend/ruff.toml +++ b/backend/ruff.toml @@ -5,6 +5,9 @@ target-version = "py312" select = ["E", "F", "I", "UP"] ignore = [] +[lint.isort] +known-first-party = ["deerflow", "app"] + [format] quote-style = "double" indent-style = "space" diff --git a/backend/src/skills/__init__.py b/backend/src/skills/__init__.py deleted file mode 100644 index f051298..0000000 --- a/backend/src/skills/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from .loader import get_skills_root_path, load_skills -from .types import Skill - -__all__ = ["load_skills", "get_skills_root_path", "Skill"] diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index f460ead..491961c 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -8,19 +8,19 @@ import sys from pathlib import Path from unittest.mock import MagicMock -# Make 'src' importable from any working directory +# Make 'app' and 'deerflow' importable from any working directory sys.path.insert(0, str(Path(__file__).parent.parent)) # Break the circular import chain that exists in production code: -# src.subagents.__init__ +# deerflow.subagents.__init__ # -> .executor (SubagentExecutor, SubagentResult) -# -> src.agents.thread_state -# -> src.agents.__init__ +# -> deerflow.agents.thread_state +# -> deerflow.agents.__init__ # -> lead_agent.agent # -> subagent_limit_middleware -# -> src.subagents.executor <-- circular! +# -> deerflow.subagents.executor <-- circular! # -# By injecting a mock for src.subagents.executor *before* any test module +# By injecting a mock for deerflow.subagents.executor *before* any test module # triggers the import, __init__.py's "from .executor import ..." succeeds # immediately without running the real executor module. _executor_mock = MagicMock() @@ -30,4 +30,4 @@ _executor_mock.SubagentStatus = MagicMock _executor_mock.MAX_CONCURRENT_SUBAGENTS = 3 _executor_mock.get_background_task_result = MagicMock() -sys.modules["src.subagents.executor"] = _executor_mock +sys.modules["deerflow.subagents.executor"] = _executor_mock diff --git a/backend/tests/test_channel_file_attachments.py b/backend/tests/test_channel_file_attachments.py index 1d1164b..60bc58e 100644 --- a/backend/tests/test_channel_file_attachments.py +++ b/backend/tests/test_channel_file_attachments.py @@ -6,8 +6,8 @@ import asyncio from pathlib import Path from unittest.mock import MagicMock, patch -from src.channels.base import Channel -from src.channels.message_bus import MessageBus, OutboundMessage, ResolvedAttachment +from app.channels.base import Channel +from app.channels.message_bus import MessageBus, OutboundMessage, ResolvedAttachment def _run(coro): @@ -102,7 +102,7 @@ class TestOutboundMessageAttachments: class TestResolveAttachments: def test_resolves_existing_file(self, tmp_path): """Successfully resolves a virtual path to an existing file.""" - from src.channels.manager import _resolve_attachments + from app.channels.manager import _resolve_attachments # Create the directory structure: threads/{thread_id}/user-data/outputs/ thread_id = "test-thread-123" @@ -115,7 +115,7 @@ class TestResolveAttachments: mock_paths.resolve_virtual_path.return_value = test_file mock_paths.sandbox_outputs_dir.return_value = outputs_dir - with patch("src.config.paths.get_paths", return_value=mock_paths): + with patch("deerflow.config.paths.get_paths", return_value=mock_paths): result = _resolve_attachments(thread_id, ["/mnt/user-data/outputs/report.pdf"]) assert len(result) == 1 @@ -126,7 +126,7 @@ class TestResolveAttachments: def test_resolves_image_file(self, tmp_path): """Images are detected by MIME type.""" - from src.channels.manager import _resolve_attachments + from app.channels.manager import _resolve_attachments thread_id = "test-thread" outputs_dir = tmp_path / "threads" / thread_id / "user-data" / "outputs" @@ -138,7 +138,7 @@ class TestResolveAttachments: mock_paths.resolve_virtual_path.return_value = img mock_paths.sandbox_outputs_dir.return_value = outputs_dir - with patch("src.config.paths.get_paths", return_value=mock_paths): + with patch("deerflow.config.paths.get_paths", return_value=mock_paths): result = _resolve_attachments(thread_id, ["/mnt/user-data/outputs/chart.png"]) assert len(result) == 1 @@ -147,7 +147,7 @@ class TestResolveAttachments: def test_skips_missing_file(self, tmp_path): """Missing files are skipped with a warning.""" - from src.channels.manager import _resolve_attachments + from app.channels.manager import _resolve_attachments outputs_dir = tmp_path / "outputs" outputs_dir.mkdir() @@ -156,30 +156,30 @@ class TestResolveAttachments: mock_paths.resolve_virtual_path.return_value = outputs_dir / "nonexistent.txt" mock_paths.sandbox_outputs_dir.return_value = outputs_dir - with patch("src.config.paths.get_paths", return_value=mock_paths): + with patch("deerflow.config.paths.get_paths", return_value=mock_paths): result = _resolve_attachments("t1", ["/mnt/user-data/outputs/nonexistent.txt"]) assert result == [] def test_skips_invalid_path(self): """Invalid paths (ValueError from resolve) are skipped.""" - from src.channels.manager import _resolve_attachments + from app.channels.manager import _resolve_attachments mock_paths = MagicMock() mock_paths.resolve_virtual_path.side_effect = ValueError("bad path") - with patch("src.config.paths.get_paths", return_value=mock_paths): + with patch("deerflow.config.paths.get_paths", return_value=mock_paths): result = _resolve_attachments("t1", ["/invalid/path"]) assert result == [] def test_rejects_uploads_path(self): """Paths under /mnt/user-data/uploads/ are rejected (security).""" - from src.channels.manager import _resolve_attachments + from app.channels.manager import _resolve_attachments mock_paths = MagicMock() - with patch("src.config.paths.get_paths", return_value=mock_paths): + with patch("deerflow.config.paths.get_paths", return_value=mock_paths): result = _resolve_attachments("t1", ["/mnt/user-data/uploads/secret.pdf"]) assert result == [] @@ -187,11 +187,11 @@ class TestResolveAttachments: def test_rejects_workspace_path(self): """Paths under /mnt/user-data/workspace/ are rejected (security).""" - from src.channels.manager import _resolve_attachments + from app.channels.manager import _resolve_attachments mock_paths = MagicMock() - with patch("src.config.paths.get_paths", return_value=mock_paths): + with patch("deerflow.config.paths.get_paths", return_value=mock_paths): result = _resolve_attachments("t1", ["/mnt/user-data/workspace/config.py"]) assert result == [] @@ -199,7 +199,7 @@ class TestResolveAttachments: def test_rejects_path_traversal_escape(self, tmp_path): """Paths that escape the outputs directory after resolution are rejected.""" - from src.channels.manager import _resolve_attachments + from app.channels.manager import _resolve_attachments thread_id = "t1" outputs_dir = tmp_path / "threads" / thread_id / "user-data" / "outputs" @@ -213,14 +213,14 @@ class TestResolveAttachments: mock_paths.resolve_virtual_path.return_value = escaped_file mock_paths.sandbox_outputs_dir.return_value = outputs_dir - with patch("src.config.paths.get_paths", return_value=mock_paths): + with patch("deerflow.config.paths.get_paths", return_value=mock_paths): result = _resolve_attachments(thread_id, ["/mnt/user-data/outputs/../uploads/stolen.txt"]) assert result == [] def test_multiple_artifacts_partial_resolution(self, tmp_path): """Mixed valid/invalid artifacts: only valid ones are returned.""" - from src.channels.manager import _resolve_attachments + from app.channels.manager import _resolve_attachments thread_id = "t1" outputs_dir = tmp_path / "outputs" @@ -238,7 +238,7 @@ class TestResolveAttachments: mock_paths.resolve_virtual_path.side_effect = resolve_side_effect - with patch("src.config.paths.get_paths", return_value=mock_paths): + with patch("deerflow.config.paths.get_paths", return_value=mock_paths): result = _resolve_attachments( thread_id, ["/mnt/user-data/outputs/data.csv", "/mnt/user-data/outputs/missing.txt"], @@ -417,17 +417,17 @@ class TestBaseChannelOnOutbound: class TestManagerArtifactResolution: def test_handle_chat_populates_attachments(self): """Verify _resolve_attachments is importable and works with the manager module.""" - from src.channels.manager import _resolve_attachments + from app.channels.manager import _resolve_attachments # Basic smoke test: empty artifacts returns empty list mock_paths = MagicMock() - with patch("src.config.paths.get_paths", return_value=mock_paths): + with patch("deerflow.config.paths.get_paths", return_value=mock_paths): result = _resolve_attachments("t1", []) assert result == [] def test_format_artifact_text_for_unresolved(self): """_format_artifact_text produces expected output.""" - from src.channels.manager import _format_artifact_text + from app.channels.manager import _format_artifact_text assert "report.pdf" in _format_artifact_text(["/mnt/user-data/outputs/report.pdf"]) result = _format_artifact_text(["/mnt/user-data/outputs/a.txt", "/mnt/user-data/outputs/b.txt"]) diff --git a/backend/tests/test_channels.py b/backend/tests/test_channels.py index b985932..9d5b71c 100644 --- a/backend/tests/test_channels.py +++ b/backend/tests/test_channels.py @@ -11,9 +11,9 @@ from unittest.mock import AsyncMock, MagicMock import pytest -from src.channels.base import Channel -from src.channels.message_bus import InboundMessage, InboundMessageType, MessageBus, OutboundMessage -from src.channels.store import ChannelStore +from app.channels.base import Channel +from app.channels.message_bus import InboundMessage, InboundMessageType, MessageBus, OutboundMessage +from app.channels.store import ChannelStore def _run(coro): @@ -277,19 +277,19 @@ class TestChannelBase: class TestExtractResponseText: def test_string_content(self): - from src.channels.manager import _extract_response_text + from app.channels.manager import _extract_response_text result = {"messages": [{"type": "ai", "content": "hello"}]} assert _extract_response_text(result) == "hello" def test_list_content_blocks(self): - from src.channels.manager import _extract_response_text + from app.channels.manager import _extract_response_text result = {"messages": [{"type": "ai", "content": [{"type": "text", "text": "hello"}, {"type": "text", "text": " world"}]}]} assert _extract_response_text(result) == "hello world" def test_picks_last_ai_message(self): - from src.channels.manager import _extract_response_text + from app.channels.manager import _extract_response_text result = { "messages": [ @@ -301,24 +301,24 @@ class TestExtractResponseText: assert _extract_response_text(result) == "second" def test_empty_messages(self): - from src.channels.manager import _extract_response_text + from app.channels.manager import _extract_response_text assert _extract_response_text({"messages": []}) == "" def test_no_ai_messages(self): - from src.channels.manager import _extract_response_text + from app.channels.manager import _extract_response_text result = {"messages": [{"type": "human", "content": "hi"}]} assert _extract_response_text(result) == "" def test_list_result(self): - from src.channels.manager import _extract_response_text + from app.channels.manager import _extract_response_text result = [{"type": "ai", "content": "from list"}] assert _extract_response_text(result) == "from list" def test_skips_empty_ai_content(self): - from src.channels.manager import _extract_response_text + from app.channels.manager import _extract_response_text result = { "messages": [ @@ -329,7 +329,7 @@ class TestExtractResponseText: assert _extract_response_text(result) == "actual response" def test_clarification_tool_message(self): - from src.channels.manager import _extract_response_text + from app.channels.manager import _extract_response_text result = { "messages": [ @@ -342,7 +342,7 @@ class TestExtractResponseText: def test_clarification_over_empty_ai(self): """When AI content is empty but ask_clarification tool message exists, use the tool message.""" - from src.channels.manager import _extract_response_text + from app.channels.manager import _extract_response_text result = { "messages": [ @@ -354,7 +354,7 @@ class TestExtractResponseText: def test_does_not_leak_previous_turn_text(self): """When current turn AI has no text (only tool calls), do not return previous turn's text.""" - from src.channels.manager import _extract_response_text + from app.channels.manager import _extract_response_text result = { "messages": [ @@ -415,7 +415,7 @@ def _make_async_iterator(items): class TestChannelManager: def test_handle_chat_creates_thread(self): - from src.channels.manager import ChannelManager + from app.channels.manager import ChannelManager async def go(): bus = MessageBus() @@ -459,7 +459,7 @@ class TestChannelManager: _run(go()) def test_handle_chat_uses_channel_session_overrides(self): - from src.channels.manager import ChannelManager + from app.channels.manager import ChannelManager async def go(): bus = MessageBus() @@ -506,7 +506,7 @@ class TestChannelManager: _run(go()) def test_handle_chat_uses_user_session_overrides(self): - from src.channels.manager import ChannelManager + from app.channels.manager import ChannelManager async def go(): bus = MessageBus() @@ -565,9 +565,9 @@ class TestChannelManager: _run(go()) def test_handle_feishu_chat_streams_multiple_outbound_updates(self, monkeypatch): - from src.channels.manager import ChannelManager + from app.channels.manager import ChannelManager - monkeypatch.setattr("src.channels.manager.STREAM_UPDATE_MIN_INTERVAL_SECONDS", 0.0) + monkeypatch.setattr("app.channels.manager.STREAM_UPDATE_MIN_INTERVAL_SECONDS", 0.0) async def go(): bus = MessageBus() @@ -634,9 +634,9 @@ class TestChannelManager: def test_handle_feishu_stream_error_still_sends_final(self, monkeypatch): """When the stream raises mid-way, a final outbound with is_final=True must still be published.""" - from src.channels.manager import ChannelManager + from app.channels.manager import ChannelManager - monkeypatch.setattr("src.channels.manager.STREAM_UPDATE_MIN_INTERVAL_SECONDS", 0.0) + monkeypatch.setattr("app.channels.manager.STREAM_UPDATE_MIN_INTERVAL_SECONDS", 0.0) async def go(): bus = MessageBus() @@ -685,7 +685,7 @@ class TestChannelManager: _run(go()) def test_handle_command_help(self): - from src.channels.manager import ChannelManager + from app.channels.manager import ChannelManager async def go(): bus = MessageBus() @@ -718,7 +718,7 @@ class TestChannelManager: _run(go()) def test_handle_command_new(self): - from src.channels.manager import ChannelManager + from app.channels.manager import ChannelManager async def go(): bus = MessageBus() @@ -761,7 +761,7 @@ class TestChannelManager: def test_each_topic_creates_new_thread(self): """Messages with distinct topic_ids should each create a new DeerFlow thread.""" - from src.channels.manager import ChannelManager + from app.channels.manager import ChannelManager async def go(): bus = MessageBus() @@ -813,7 +813,7 @@ class TestChannelManager: def test_same_topic_reuses_thread(self): """Messages with the same topic_id should reuse the same DeerFlow thread.""" - from src.channels.manager import ChannelManager + from app.channels.manager import ChannelManager async def go(): bus = MessageBus() @@ -857,7 +857,7 @@ class TestChannelManager: def test_none_topic_reuses_thread(self): """Messages with topic_id=None should reuse the same thread (e.g. Telegram private chat).""" - from src.channels.manager import ChannelManager + from app.channels.manager import ChannelManager async def go(): bus = MessageBus() @@ -901,7 +901,7 @@ class TestChannelManager: def test_different_topics_get_different_threads(self): """Messages with different topic_ids should create separate threads.""" - from src.channels.manager import ChannelManager + from app.channels.manager import ChannelManager async def go(): bus = MessageBus() @@ -951,7 +951,7 @@ class TestChannelManager: class TestExtractArtifacts: def test_extracts_from_present_files_tool_call(self): - from src.channels.manager import _extract_artifacts + from app.channels.manager import _extract_artifacts result = { "messages": [ @@ -969,7 +969,7 @@ class TestExtractArtifacts: assert _extract_artifacts(result) == ["/mnt/user-data/outputs/report.md"] def test_empty_when_no_present_files(self): - from src.channels.manager import _extract_artifacts + from app.channels.manager import _extract_artifacts result = { "messages": [ @@ -980,14 +980,14 @@ class TestExtractArtifacts: assert _extract_artifacts(result) == [] def test_empty_for_list_result_no_tool_calls(self): - from src.channels.manager import _extract_artifacts + from app.channels.manager import _extract_artifacts result = [{"type": "ai", "content": "hello"}] assert _extract_artifacts(result) == [] def test_only_extracts_after_last_human_message(self): """Artifacts from previous turns (before the last human message) should be ignored.""" - from src.channels.manager import _extract_artifacts + from app.channels.manager import _extract_artifacts result = { "messages": [ @@ -1015,7 +1015,7 @@ class TestExtractArtifacts: assert _extract_artifacts(result) == ["/mnt/user-data/outputs/chart.png"] def test_multiple_files_in_single_call(self): - from src.channels.manager import _extract_artifacts + from app.channels.manager import _extract_artifacts result = { "messages": [ @@ -1034,13 +1034,13 @@ class TestExtractArtifacts: class TestFormatArtifactText: def test_single_artifact(self): - from src.channels.manager import _format_artifact_text + from app.channels.manager import _format_artifact_text text = _format_artifact_text(["/mnt/user-data/outputs/report.md"]) assert text == "Created File: 📎 report.md" def test_multiple_artifacts(self): - from src.channels.manager import _format_artifact_text + from app.channels.manager import _format_artifact_text text = _format_artifact_text( ["/mnt/user-data/outputs/a.txt", "/mnt/user-data/outputs/b.csv"], @@ -1050,7 +1050,7 @@ class TestFormatArtifactText: class TestHandleChatWithArtifacts: def test_artifacts_appended_to_text(self): - from src.channels.manager import ChannelManager + from app.channels.manager import ChannelManager async def go(): bus = MessageBus() @@ -1097,7 +1097,7 @@ class TestHandleChatWithArtifacts: def test_artifacts_only_no_text(self): """When agent produces artifacts but no text, the artifacts should be the response.""" - from src.channels.manager import ChannelManager + from app.channels.manager import ChannelManager async def go(): bus = MessageBus() @@ -1145,7 +1145,7 @@ class TestHandleChatWithArtifacts: def test_only_last_turn_artifacts_returned(self): """Only artifacts from the current turn's present_files calls should be included.""" - from src.channels.manager import ChannelManager + from app.channels.manager import ChannelManager async def go(): bus = MessageBus() @@ -1228,7 +1228,7 @@ class TestHandleChatWithArtifacts: class TestFeishuChannel: def test_prepare_inbound_publishes_without_waiting_for_running_card(self): - from src.channels.feishu import FeishuChannel + from app.channels.feishu import FeishuChannel async def go(): bus = MessageBus() @@ -1270,7 +1270,7 @@ class TestFeishuChannel: _run(go()) def test_prepare_inbound_and_send_share_running_card_task(self): - from src.channels.feishu import FeishuChannel + from app.channels.feishu import FeishuChannel async def go(): bus = MessageBus() @@ -1339,7 +1339,7 @@ class TestFeishuChannel: ReplyMessageRequestBody, ) - from src.channels.feishu import FeishuChannel + from app.channels.feishu import FeishuChannel async def go(): bus = MessageBus() @@ -1402,7 +1402,7 @@ class TestFeishuChannel: class TestChannelService: def test_get_status_no_channels(self): - from src.channels.service import ChannelService + from app.channels.service import ChannelService async def go(): service = ChannelService(channels_config={}) @@ -1419,7 +1419,7 @@ class TestChannelService: _run(go()) def test_disabled_channels_are_skipped(self): - from src.channels.service import ChannelService + from app.channels.service import ChannelService async def go(): service = ChannelService( @@ -1434,7 +1434,7 @@ class TestChannelService: _run(go()) def test_session_config_is_forwarded_to_manager(self): - from src.channels.service import ChannelService + from app.channels.service import ChannelService service = ChannelService( channels_config={ @@ -1465,7 +1465,7 @@ class TestChannelService: class TestSlackSendRetry: def test_retries_on_failure_then_succeeds(self): - from src.channels.slack import SlackChannel + from app.channels.slack import SlackChannel async def go(): bus = MessageBus() @@ -1491,7 +1491,7 @@ class TestSlackSendRetry: _run(go()) def test_raises_after_all_retries_exhausted(self): - from src.channels.slack import SlackChannel + from app.channels.slack import SlackChannel async def go(): bus = MessageBus() @@ -1517,7 +1517,7 @@ class TestSlackSendRetry: class TestTelegramSendRetry: def test_retries_on_failure_then_succeeds(self): - from src.channels.telegram import TelegramChannel + from app.channels.telegram import TelegramChannel async def go(): bus = MessageBus() @@ -1547,7 +1547,7 @@ class TestTelegramSendRetry: _run(go()) def test_raises_after_all_retries_exhausted(self): - from src.channels.telegram import TelegramChannel + from app.channels.telegram import TelegramChannel async def go(): bus = MessageBus() @@ -1594,7 +1594,7 @@ class TestTelegramPrivateChatThread: """Verify that private chats use topic_id=None (single thread per chat).""" def test_private_chat_no_reply_uses_none_topic(self): - from src.channels.telegram import TelegramChannel + from app.channels.telegram import TelegramChannel async def go(): bus = MessageBus() @@ -1610,7 +1610,7 @@ class TestTelegramPrivateChatThread: _run(go()) def test_private_chat_with_reply_still_uses_none_topic(self): - from src.channels.telegram import TelegramChannel + from app.channels.telegram import TelegramChannel async def go(): bus = MessageBus() @@ -1626,7 +1626,7 @@ class TestTelegramPrivateChatThread: _run(go()) def test_group_chat_no_reply_uses_msg_id_as_topic(self): - from src.channels.telegram import TelegramChannel + from app.channels.telegram import TelegramChannel async def go(): bus = MessageBus() @@ -1642,7 +1642,7 @@ class TestTelegramPrivateChatThread: _run(go()) def test_group_chat_reply_uses_reply_msg_id_as_topic(self): - from src.channels.telegram import TelegramChannel + from app.channels.telegram import TelegramChannel async def go(): bus = MessageBus() @@ -1658,7 +1658,7 @@ class TestTelegramPrivateChatThread: _run(go()) def test_supergroup_chat_uses_msg_id_as_topic(self): - from src.channels.telegram import TelegramChannel + from app.channels.telegram import TelegramChannel async def go(): bus = MessageBus() @@ -1674,7 +1674,7 @@ class TestTelegramPrivateChatThread: _run(go()) def test_cmd_generic_private_chat_uses_none_topic(self): - from src.channels.telegram import TelegramChannel + from app.channels.telegram import TelegramChannel async def go(): bus = MessageBus() @@ -1691,7 +1691,7 @@ class TestTelegramPrivateChatThread: _run(go()) def test_cmd_generic_group_chat_uses_msg_id_as_topic(self): - from src.channels.telegram import TelegramChannel + from app.channels.telegram import TelegramChannel async def go(): bus = MessageBus() @@ -1708,7 +1708,7 @@ class TestTelegramPrivateChatThread: _run(go()) def test_cmd_generic_group_chat_reply_uses_reply_msg_id_as_topic(self): - from src.channels.telegram import TelegramChannel + from app.channels.telegram import TelegramChannel async def go(): bus = MessageBus() @@ -1734,20 +1734,20 @@ class TestSlackMarkdownConversion: """Verify that the SlackChannel.send() path applies mrkdwn conversion.""" def test_bold_converted(self): - from src.channels.slack import _slack_md_converter + from app.channels.slack import _slack_md_converter result = _slack_md_converter.convert("this is **bold** text") assert "*bold*" in result assert "**" not in result def test_link_converted(self): - from src.channels.slack import _slack_md_converter + from app.channels.slack import _slack_md_converter result = _slack_md_converter.convert("[click](https://example.com)") assert "" in result def test_heading_converted(self): - from src.channels.slack import _slack_md_converter + from app.channels.slack import _slack_md_converter result = _slack_md_converter.convert("# Title") assert "*Title*" in result diff --git a/backend/tests/test_checkpointer.py b/backend/tests/test_checkpointer.py index 59caf12..2e49ebf 100644 --- a/backend/tests/test_checkpointer.py +++ b/backend/tests/test_checkpointer.py @@ -5,8 +5,8 @@ from unittest.mock import MagicMock, patch import pytest -from src.agents.checkpointer import get_checkpointer, reset_checkpointer -from src.config.checkpointer_config import ( +from deerflow.agents.checkpointer import get_checkpointer, reset_checkpointer +from deerflow.config.checkpointer_config import ( CheckpointerConfig, get_checkpointer_config, load_checkpointer_config_from_dict, @@ -195,7 +195,7 @@ class TestClientCheckpointerFallback: """DeerFlowClient._ensure_agent falls back to get_checkpointer() when checkpointer=None.""" from langgraph.checkpoint.memory import InMemorySaver - from src.client import DeerFlowClient + from deerflow.client import DeerFlowClient load_checkpointer_config_from_dict({"type": "memory"}) @@ -212,12 +212,12 @@ class TestClientCheckpointerFallback: config_mock.checkpointer = None with ( - patch("src.client.get_app_config", return_value=config_mock), - patch("src.client.create_agent", side_effect=fake_create_agent), - patch("src.client.create_chat_model", return_value=MagicMock()), - patch("src.client._build_middlewares", return_value=[]), - patch("src.client.apply_prompt_template", return_value=""), - patch("src.client.DeerFlowClient._get_tools", return_value=[]), + patch("deerflow.client.get_app_config", return_value=config_mock), + patch("deerflow.client.create_agent", side_effect=fake_create_agent), + patch("deerflow.client.create_chat_model", return_value=MagicMock()), + patch("deerflow.client._build_middlewares", return_value=[]), + patch("deerflow.client.apply_prompt_template", return_value=""), + patch("deerflow.client.DeerFlowClient._get_tools", return_value=[]), ): client = DeerFlowClient(checkpointer=None) config = client._get_runnable_config("test-thread") @@ -228,7 +228,7 @@ class TestClientCheckpointerFallback: def test_client_explicit_checkpointer_takes_precedence(self): """An explicitly provided checkpointer is used even when config checkpointer is set.""" - from src.client import DeerFlowClient + from deerflow.client import DeerFlowClient load_checkpointer_config_from_dict({"type": "memory"}) @@ -246,12 +246,12 @@ class TestClientCheckpointerFallback: config_mock.checkpointer = None with ( - patch("src.client.get_app_config", return_value=config_mock), - patch("src.client.create_agent", side_effect=fake_create_agent), - patch("src.client.create_chat_model", return_value=MagicMock()), - patch("src.client._build_middlewares", return_value=[]), - patch("src.client.apply_prompt_template", return_value=""), - patch("src.client.DeerFlowClient._get_tools", return_value=[]), + patch("deerflow.client.get_app_config", return_value=config_mock), + patch("deerflow.client.create_agent", side_effect=fake_create_agent), + patch("deerflow.client.create_chat_model", return_value=MagicMock()), + patch("deerflow.client._build_middlewares", return_value=[]), + patch("deerflow.client.apply_prompt_template", return_value=""), + patch("deerflow.client.DeerFlowClient._get_tools", return_value=[]), ): client = DeerFlowClient(checkpointer=explicit_cp) config = client._get_runnable_config("test-thread") diff --git a/backend/tests/test_checkpointer_none_fix.py b/backend/tests/test_checkpointer_none_fix.py index 091aa31..4e128ad 100644 --- a/backend/tests/test_checkpointer_none_fix.py +++ b/backend/tests/test_checkpointer_none_fix.py @@ -12,13 +12,13 @@ class TestCheckpointerNoneFix: @pytest.mark.anyio async def test_async_make_checkpointer_returns_in_memory_saver_when_not_configured(self): """make_checkpointer should return InMemorySaver when config.checkpointer is None.""" - from src.agents.checkpointer.async_provider import make_checkpointer + from deerflow.agents.checkpointer.async_provider import make_checkpointer # Mock get_app_config to return a config with checkpointer=None mock_config = MagicMock() mock_config.checkpointer = None - with patch("src.agents.checkpointer.async_provider.get_app_config", return_value=mock_config): + with patch("deerflow.agents.checkpointer.async_provider.get_app_config", return_value=mock_config): async with make_checkpointer() as checkpointer: # Should return InMemorySaver, not None assert checkpointer is not None @@ -35,13 +35,13 @@ class TestCheckpointerNoneFix: def test_sync_checkpointer_context_returns_in_memory_saver_when_not_configured(self): """checkpointer_context should return InMemorySaver when config.checkpointer is None.""" - from src.agents.checkpointer.provider import checkpointer_context + from deerflow.agents.checkpointer.provider import checkpointer_context # Mock get_app_config to return a config with checkpointer=None mock_config = MagicMock() mock_config.checkpointer = None - with patch("src.agents.checkpointer.provider.get_app_config", return_value=mock_config): + with patch("deerflow.agents.checkpointer.provider.get_app_config", return_value=mock_config): with checkpointer_context() as checkpointer: # Should return InMemorySaver, not None assert checkpointer is not None diff --git a/backend/tests/test_client.py b/backend/tests/test_client.py index e5416e9..62742d5 100644 --- a/backend/tests/test_client.py +++ b/backend/tests/test_client.py @@ -11,12 +11,12 @@ from unittest.mock import MagicMock, patch import pytest from langchain_core.messages import AIMessage, HumanMessage, ToolMessage # noqa: F401 -from src.client import DeerFlowClient -from src.gateway.routers.mcp import McpConfigResponse -from src.gateway.routers.memory import MemoryConfigResponse, MemoryStatusResponse -from src.gateway.routers.models import ModelResponse, ModelsListResponse -from src.gateway.routers.skills import SkillInstallResponse, SkillResponse, SkillsListResponse -from src.gateway.routers.uploads import UploadResponse +from app.gateway.routers.mcp import McpConfigResponse +from app.gateway.routers.memory import MemoryConfigResponse, MemoryStatusResponse +from app.gateway.routers.models import ModelResponse, ModelsListResponse +from app.gateway.routers.skills import SkillInstallResponse, SkillResponse, SkillsListResponse +from app.gateway.routers.uploads import UploadResponse +from deerflow.client import DeerFlowClient # --------------------------------------------------------------------------- # Fixtures @@ -40,7 +40,7 @@ def mock_app_config(): @pytest.fixture def client(mock_app_config): """Create a DeerFlowClient with mocked config loading.""" - with patch("src.client.get_app_config", return_value=mock_app_config): + with patch("deerflow.client.get_app_config", return_value=mock_app_config): return DeerFlowClient() @@ -59,7 +59,7 @@ class TestClientInit: assert client._agent is None def test_custom_params(self, mock_app_config): - with patch("src.client.get_app_config", return_value=mock_app_config): + with patch("deerflow.client.get_app_config", return_value=mock_app_config): c = DeerFlowClient( model_name="gpt-4", thinking_enabled=False, @@ -73,15 +73,15 @@ class TestClientInit: def test_custom_config_path(self, mock_app_config): with ( - patch("src.client.reload_app_config") as mock_reload, - patch("src.client.get_app_config", return_value=mock_app_config), + patch("deerflow.client.reload_app_config") as mock_reload, + patch("deerflow.client.get_app_config", return_value=mock_app_config), ): DeerFlowClient(config_path="/tmp/custom.yaml") mock_reload.assert_called_once_with("/tmp/custom.yaml") def test_checkpointer_stored(self, mock_app_config): cp = MagicMock() - with patch("src.client.get_app_config", return_value=mock_app_config): + with patch("deerflow.client.get_app_config", return_value=mock_app_config): c = DeerFlowClient(checkpointer=cp) assert c._checkpointer is cp @@ -109,7 +109,7 @@ class TestConfigQueries: skill.category = "public" skill.enabled = True - with patch("src.skills.loader.load_skills", return_value=[skill]) as mock_load: + with patch("deerflow.skills.loader.load_skills", return_value=[skill]) as mock_load: result = client.list_skills() mock_load.assert_called_once_with(enabled_only=False) @@ -124,13 +124,13 @@ class TestConfigQueries: } def test_list_skills_enabled_only(self, client): - with patch("src.skills.loader.load_skills", return_value=[]) as mock_load: + with patch("deerflow.skills.loader.load_skills", return_value=[]) as mock_load: client.list_skills(enabled_only=True) mock_load.assert_called_once_with(enabled_only=True) def test_get_memory(self, client): memory = {"version": "1.0", "facts": []} - with patch("src.agents.memory.updater.get_memory_data", return_value=memory) as mock_mem: + with patch("deerflow.agents.memory.updater.get_memory_data", return_value=memory) as mock_mem: result = client.get_memory() mock_mem.assert_called_once() assert result == memory @@ -355,10 +355,10 @@ class TestEnsureAgent: config = client._get_runnable_config("t1") with ( - patch("src.client.create_chat_model"), - patch("src.client.create_agent", return_value=mock_agent), - patch("src.client._build_middlewares", return_value=[]), - patch("src.client.apply_prompt_template", return_value="prompt"), + patch("deerflow.client.create_chat_model"), + patch("deerflow.client.create_agent", return_value=mock_agent), + patch("deerflow.client._build_middlewares", return_value=[]), + patch("deerflow.client.apply_prompt_template", return_value="prompt"), patch.object(client, "_get_tools", return_value=[]), ): client._ensure_agent(config) @@ -371,12 +371,12 @@ class TestEnsureAgent: config = client._get_runnable_config("t1") with ( - patch("src.client.create_chat_model"), - patch("src.client.create_agent", return_value=mock_agent) as mock_create_agent, - patch("src.client._build_middlewares", return_value=[]), - patch("src.client.apply_prompt_template", return_value="prompt"), + patch("deerflow.client.create_chat_model"), + patch("deerflow.client.create_agent", return_value=mock_agent) as mock_create_agent, + patch("deerflow.client._build_middlewares", return_value=[]), + patch("deerflow.client.apply_prompt_template", return_value="prompt"), patch.object(client, "_get_tools", return_value=[]), - patch("src.agents.checkpointer.get_checkpointer", return_value=mock_checkpointer), + patch("deerflow.agents.checkpointer.get_checkpointer", return_value=mock_checkpointer), ): client._ensure_agent(config) @@ -387,12 +387,12 @@ class TestEnsureAgent: config = client._get_runnable_config("t1") with ( - patch("src.client.create_chat_model"), - patch("src.client.create_agent", return_value=mock_agent) as mock_create_agent, - patch("src.client._build_middlewares", return_value=[]), - patch("src.client.apply_prompt_template", return_value="prompt"), + patch("deerflow.client.create_chat_model"), + patch("deerflow.client.create_agent", return_value=mock_agent) as mock_create_agent, + patch("deerflow.client._build_middlewares", return_value=[]), + patch("deerflow.client.apply_prompt_template", return_value="prompt"), patch.object(client, "_get_tools", return_value=[]), - patch("src.agents.checkpointer.get_checkpointer", return_value=None), + patch("deerflow.agents.checkpointer.get_checkpointer", return_value=None), ): client._ensure_agent(config) @@ -452,7 +452,7 @@ class TestMcpConfig: ext_config = MagicMock() ext_config.mcp_servers = {"github": server} - with patch("src.client.get_extensions_config", return_value=ext_config): + with patch("deerflow.client.get_extensions_config", return_value=ext_config): result = client.get_mcp_config() assert "mcp_servers" in result @@ -478,9 +478,9 @@ class TestMcpConfig: client._agent = MagicMock() with ( - patch("src.client.ExtensionsConfig.resolve_config_path", return_value=tmp_path), - patch("src.client.get_extensions_config", return_value=current_config), - patch("src.client.reload_extensions_config", return_value=reloaded_config), + patch("deerflow.client.ExtensionsConfig.resolve_config_path", return_value=tmp_path), + patch("deerflow.client.get_extensions_config", return_value=current_config), + patch("deerflow.client.reload_extensions_config", return_value=reloaded_config), ): result = client.update_mcp_config({"new-server": {"enabled": True, "type": "sse"}}) @@ -513,13 +513,13 @@ class TestSkillsManagement: def test_get_skill_found(self, client): skill = self._make_skill() - with patch("src.skills.loader.load_skills", return_value=[skill]): + with patch("deerflow.skills.loader.load_skills", return_value=[skill]): result = client.get_skill("test-skill") assert result is not None assert result["name"] == "test-skill" def test_get_skill_not_found(self, client): - with patch("src.skills.loader.load_skills", return_value=[]): + with patch("deerflow.skills.loader.load_skills", return_value=[]): result = client.get_skill("nonexistent") assert result is None @@ -540,10 +540,10 @@ class TestSkillsManagement: client._agent = MagicMock() with ( - patch("src.skills.loader.load_skills", side_effect=[[skill], [updated_skill]]), - patch("src.client.ExtensionsConfig.resolve_config_path", return_value=tmp_path), - patch("src.client.get_extensions_config", return_value=ext_config), - patch("src.client.reload_extensions_config"), + patch("deerflow.skills.loader.load_skills", side_effect=[[skill], [updated_skill]]), + patch("deerflow.client.ExtensionsConfig.resolve_config_path", return_value=tmp_path), + patch("deerflow.client.get_extensions_config", return_value=ext_config), + patch("deerflow.client.reload_extensions_config"), ): result = client.update_skill("test-skill", enabled=False) assert result["enabled"] is False @@ -552,7 +552,7 @@ class TestSkillsManagement: tmp_path.unlink() def test_update_skill_not_found(self, client): - with patch("src.skills.loader.load_skills", return_value=[]): + with patch("deerflow.skills.loader.load_skills", return_value=[]): with pytest.raises(ValueError, match="not found"): client.update_skill("nonexistent", enabled=True) @@ -573,8 +573,8 @@ class TestSkillsManagement: (skills_root / "custom").mkdir(parents=True) with ( - patch("src.skills.loader.get_skills_root_path", return_value=skills_root), - patch("src.gateway.routers.skills._validate_skill_frontmatter", return_value=(True, "OK", "my-skill")), + patch("deerflow.skills.loader.get_skills_root_path", return_value=skills_root), + patch("deerflow.skills.validation._validate_skill_frontmatter", return_value=(True, "OK", "my-skill")), ): result = client.install_skill(archive_path) @@ -604,7 +604,7 @@ class TestSkillsManagement: class TestMemoryManagement: def test_reload_memory(self, client): data = {"version": "1.0", "facts": []} - with patch("src.agents.memory.updater.reload_memory_data", return_value=data): + with patch("deerflow.agents.memory.updater.reload_memory_data", return_value=data): result = client.reload_memory() assert result == data @@ -618,7 +618,7 @@ class TestMemoryManagement: config.injection_enabled = True config.max_injection_tokens = 2000 - with patch("src.config.memory_config.get_memory_config", return_value=config): + with patch("deerflow.config.memory_config.get_memory_config", return_value=config): result = client.get_memory_config() assert result["enabled"] is True @@ -637,8 +637,8 @@ class TestMemoryManagement: data = {"version": "1.0", "facts": []} with ( - patch("src.config.memory_config.get_memory_config", return_value=config), - patch("src.agents.memory.updater.get_memory_data", return_value=data), + patch("deerflow.config.memory_config.get_memory_config", return_value=config), + patch("deerflow.agents.memory.updater.get_memory_data", return_value=data), ): result = client.get_memory_status() @@ -720,8 +720,8 @@ class TestUploads: with ( patch.object(DeerFlowClient, "_get_uploads_dir", return_value=uploads_dir), - patch("src.gateway.routers.uploads.CONVERTIBLE_EXTENSIONS", {".pdf"}), - patch("src.gateway.routers.uploads.convert_file_to_markdown", side_effect=fake_convert), + patch("deerflow.utils.file_conversion.CONVERTIBLE_EXTENSIONS", {".pdf"}), + patch("deerflow.utils.file_conversion.convert_file_to_markdown", side_effect=fake_convert), patch("concurrent.futures.ThreadPoolExecutor", FakeExecutor), ): result = asyncio.run(call_upload()) @@ -793,7 +793,7 @@ class TestArtifacts: mock_paths = MagicMock() mock_paths.sandbox_user_data_dir.return_value = user_data_dir - with patch("src.client.get_paths", return_value=mock_paths): + with patch("deerflow.client.get_paths", return_value=mock_paths): content, mime = client.get_artifact("t1", "mnt/user-data/outputs/result.txt") assert content == b"artifact content" @@ -807,7 +807,7 @@ class TestArtifacts: mock_paths = MagicMock() mock_paths.sandbox_user_data_dir.return_value = user_data_dir - with patch("src.client.get_paths", return_value=mock_paths): + with patch("deerflow.client.get_paths", return_value=mock_paths): with pytest.raises(FileNotFoundError): client.get_artifact("t1", "mnt/user-data/outputs/nope.txt") @@ -823,7 +823,7 @@ class TestArtifacts: mock_paths = MagicMock() mock_paths.sandbox_user_data_dir.return_value = user_data_dir - with patch("src.client.get_paths", return_value=mock_paths): + with patch("deerflow.client.get_paths", return_value=mock_paths): with pytest.raises(PermissionError): client.get_artifact("t1", "mnt/user-data/../../../etc/passwd") @@ -1028,7 +1028,7 @@ class TestScenarioFileLifecycle: mock_paths = MagicMock() mock_paths.sandbox_user_data_dir.return_value = user_data_dir - with patch("src.client.get_paths", return_value=mock_paths): + with patch("deerflow.client.get_paths", return_value=mock_paths): content, mime = client.get_artifact("t-artifact", "mnt/user-data/outputs/analysis.json") assert json.loads(content) == {"result": "processed"} @@ -1064,12 +1064,12 @@ class TestScenarioConfigManagement: skill.category = "public" skill.enabled = True - with patch("src.skills.loader.load_skills", return_value=[skill]): + with patch("deerflow.skills.loader.load_skills", return_value=[skill]): skills_result = client.list_skills() assert len(skills_result["skills"]) == 1 # Get specific skill - with patch("src.skills.loader.load_skills", return_value=[skill]): + with patch("deerflow.skills.loader.load_skills", return_value=[skill]): detail = client.get_skill("web-search") assert detail is not None assert detail["enabled"] is True @@ -1091,9 +1091,9 @@ class TestScenarioConfigManagement: client._agent = MagicMock() # Simulate existing agent with ( - patch("src.client.ExtensionsConfig.resolve_config_path", return_value=config_file), - patch("src.client.get_extensions_config", return_value=current_config), - patch("src.client.reload_extensions_config", return_value=reloaded_config), + patch("deerflow.client.ExtensionsConfig.resolve_config_path", return_value=config_file), + patch("deerflow.client.get_extensions_config", return_value=current_config), + patch("deerflow.client.reload_extensions_config", return_value=reloaded_config), ): mcp_result = client.update_mcp_config({"my-mcp": {"enabled": True}}) assert "my-mcp" in mcp_result["mcp_servers"] @@ -1120,10 +1120,10 @@ class TestScenarioConfigManagement: client._agent = MagicMock() # Simulate re-created agent with ( - patch("src.skills.loader.load_skills", side_effect=[[skill], [toggled]]), - patch("src.client.ExtensionsConfig.resolve_config_path", return_value=config_file), - patch("src.client.get_extensions_config", return_value=ext_config), - patch("src.client.reload_extensions_config"), + patch("deerflow.skills.loader.load_skills", side_effect=[[skill], [toggled]]), + patch("deerflow.client.ExtensionsConfig.resolve_config_path", return_value=config_file), + patch("deerflow.client.get_extensions_config", return_value=ext_config), + patch("deerflow.client.reload_extensions_config"), ): skill_result = client.update_skill("code-gen", enabled=False) assert skill_result["enabled"] is False @@ -1146,10 +1146,10 @@ class TestScenarioAgentRecreation: config_b = client._get_runnable_config("t1", model_name="claude-3") with ( - patch("src.client.create_chat_model"), - patch("src.client.create_agent", side_effect=fake_create_agent), - patch("src.client._build_middlewares", return_value=[]), - patch("src.client.apply_prompt_template", return_value="prompt"), + patch("deerflow.client.create_chat_model"), + patch("deerflow.client.create_agent", side_effect=fake_create_agent), + patch("deerflow.client._build_middlewares", return_value=[]), + patch("deerflow.client.apply_prompt_template", return_value="prompt"), patch.object(client, "_get_tools", return_value=[]), ): client._ensure_agent(config_a) @@ -1173,10 +1173,10 @@ class TestScenarioAgentRecreation: config = client._get_runnable_config("t1", model_name="gpt-4") with ( - patch("src.client.create_chat_model"), - patch("src.client.create_agent", side_effect=fake_create_agent), - patch("src.client._build_middlewares", return_value=[]), - patch("src.client.apply_prompt_template", return_value="prompt"), + patch("deerflow.client.create_chat_model"), + patch("deerflow.client.create_agent", side_effect=fake_create_agent), + patch("deerflow.client._build_middlewares", return_value=[]), + patch("deerflow.client.apply_prompt_template", return_value="prompt"), patch.object(client, "_get_tools", return_value=[]), ): client._ensure_agent(config) @@ -1197,10 +1197,10 @@ class TestScenarioAgentRecreation: config = client._get_runnable_config("t1") with ( - patch("src.client.create_chat_model"), - patch("src.client.create_agent", side_effect=fake_create_agent), - patch("src.client._build_middlewares", return_value=[]), - patch("src.client.apply_prompt_template", return_value="prompt"), + patch("deerflow.client.create_chat_model"), + patch("deerflow.client.create_agent", side_effect=fake_create_agent), + patch("deerflow.client._build_middlewares", return_value=[]), + patch("deerflow.client.apply_prompt_template", return_value="prompt"), patch.object(client, "_get_tools", return_value=[]), ): client._ensure_agent(config) @@ -1271,7 +1271,7 @@ class TestScenarioThreadIsolation: mock_paths = MagicMock() mock_paths.sandbox_user_data_dir.side_effect = lambda tid: data_a if tid == "thread-a" else data_b - with patch("src.client.get_paths", return_value=mock_paths): + with patch("deerflow.client.get_paths", return_value=mock_paths): content, _ = client.get_artifact("thread-a", "mnt/user-data/outputs/result.txt") assert content == b"thread-a artifact" @@ -1302,17 +1302,17 @@ class TestScenarioMemoryWorkflow: config.injection_enabled = True config.max_injection_tokens = 2000 - with patch("src.agents.memory.updater.get_memory_data", return_value=initial_data): + with patch("deerflow.agents.memory.updater.get_memory_data", return_value=initial_data): mem = client.get_memory() assert len(mem["facts"]) == 1 - with patch("src.agents.memory.updater.reload_memory_data", return_value=updated_data): + with patch("deerflow.agents.memory.updater.reload_memory_data", return_value=updated_data): refreshed = client.reload_memory() assert len(refreshed["facts"]) == 2 with ( - patch("src.config.memory_config.get_memory_config", return_value=config), - patch("src.agents.memory.updater.get_memory_data", return_value=updated_data), + patch("deerflow.config.memory_config.get_memory_config", return_value=config), + patch("deerflow.agents.memory.updater.get_memory_data", return_value=updated_data), ): status = client.get_memory_status() assert status["config"]["enabled"] is True @@ -1340,8 +1340,8 @@ class TestScenarioSkillInstallAndUse: # Step 1: Install with ( - patch("src.skills.loader.get_skills_root_path", return_value=skills_root), - patch("src.gateway.routers.skills._validate_skill_frontmatter", return_value=(True, "OK", "my-analyzer")), + patch("deerflow.skills.loader.get_skills_root_path", return_value=skills_root), + patch("deerflow.skills.validation._validate_skill_frontmatter", return_value=(True, "OK", "my-analyzer")), ): result = client.install_skill(archive) assert result["success"] is True @@ -1355,7 +1355,7 @@ class TestScenarioSkillInstallAndUse: installed_skill.category = "custom" installed_skill.enabled = True - with patch("src.skills.loader.load_skills", return_value=[installed_skill]): + with patch("deerflow.skills.loader.load_skills", return_value=[installed_skill]): skills_result = client.list_skills() assert any(s["name"] == "my-analyzer" for s in skills_result["skills"]) @@ -1375,10 +1375,10 @@ class TestScenarioSkillInstallAndUse: config_file.write_text("{}") with ( - patch("src.skills.loader.load_skills", side_effect=[[installed_skill], [disabled_skill]]), - patch("src.client.ExtensionsConfig.resolve_config_path", return_value=config_file), - patch("src.client.get_extensions_config", return_value=ext_config), - patch("src.client.reload_extensions_config"), + patch("deerflow.skills.loader.load_skills", side_effect=[[installed_skill], [disabled_skill]]), + patch("deerflow.client.ExtensionsConfig.resolve_config_path", return_value=config_file), + patch("deerflow.client.get_extensions_config", return_value=ext_config), + patch("deerflow.client.reload_extensions_config"), ): toggled = client.update_skill("my-analyzer", enabled=False) assert toggled["enabled"] is False @@ -1475,8 +1475,8 @@ class TestScenarioEdgeCases: with ( patch.object(DeerFlowClient, "_get_uploads_dir", return_value=uploads_dir), - patch("src.gateway.routers.uploads.CONVERTIBLE_EXTENSIONS", {".pdf"}), - patch("src.gateway.routers.uploads.convert_file_to_markdown", side_effect=Exception("conversion failed")), + patch("deerflow.utils.file_conversion.CONVERTIBLE_EXTENSIONS", {".pdf"}), + patch("deerflow.utils.file_conversion.convert_file_to_markdown", side_effect=Exception("conversion failed")), ): result = client.upload_files("t-pdf-fail", [pdf_file]) @@ -1508,7 +1508,7 @@ class TestGatewayConformance: model.supports_thinking = False mock_app_config.models = [model] - with patch("src.client.get_app_config", return_value=mock_app_config): + with patch("deerflow.client.get_app_config", return_value=mock_app_config): client = DeerFlowClient() result = client.list_models() @@ -1525,7 +1525,7 @@ class TestGatewayConformance: mock_app_config.models = [model] mock_app_config.get_model_config.return_value = model - with patch("src.client.get_app_config", return_value=mock_app_config): + with patch("deerflow.client.get_app_config", return_value=mock_app_config): client = DeerFlowClient() result = client.get_model("test-model") @@ -1541,7 +1541,7 @@ class TestGatewayConformance: skill.category = "public" skill.enabled = True - with patch("src.skills.loader.load_skills", return_value=[skill]): + with patch("deerflow.skills.loader.load_skills", return_value=[skill]): result = client.list_skills() parsed = SkillsListResponse(**result) @@ -1556,7 +1556,7 @@ class TestGatewayConformance: skill.category = "public" skill.enabled = True - with patch("src.skills.loader.load_skills", return_value=[skill]): + with patch("deerflow.skills.loader.load_skills", return_value=[skill]): result = client.get_skill("web-search") assert result is not None @@ -1574,7 +1574,7 @@ class TestGatewayConformance: custom_dir = tmp_path / "custom" custom_dir.mkdir() - with patch("src.skills.loader.get_skills_root_path", return_value=tmp_path): + with patch("deerflow.skills.loader.get_skills_root_path", return_value=tmp_path): result = client.install_skill(archive) parsed = SkillInstallResponse(**result) @@ -1596,7 +1596,7 @@ class TestGatewayConformance: ext_config = MagicMock() ext_config.mcp_servers = {"test": server} - with patch("src.client.get_extensions_config", return_value=ext_config): + with patch("deerflow.client.get_extensions_config", return_value=ext_config): result = client.get_mcp_config() parsed = McpConfigResponse(**result) @@ -1622,9 +1622,9 @@ class TestGatewayConformance: config_file.write_text("{}") with ( - patch("src.client.get_extensions_config", return_value=ext_config), - patch("src.client.ExtensionsConfig.resolve_config_path", return_value=config_file), - patch("src.client.reload_extensions_config", return_value=ext_config), + patch("deerflow.client.get_extensions_config", return_value=ext_config), + patch("deerflow.client.ExtensionsConfig.resolve_config_path", return_value=config_file), + patch("deerflow.client.reload_extensions_config", return_value=ext_config), ): result = client.update_mcp_config({"srv": server.model_dump.return_value}) @@ -1655,7 +1655,7 @@ class TestGatewayConformance: mem_cfg.injection_enabled = True mem_cfg.max_injection_tokens = 2000 - with patch("src.config.memory_config.get_memory_config", return_value=mem_cfg): + with patch("deerflow.config.memory_config.get_memory_config", return_value=mem_cfg): result = client.get_memory_config() parsed = MemoryConfigResponse(**result) @@ -1689,8 +1689,8 @@ class TestGatewayConformance: } with ( - patch("src.config.memory_config.get_memory_config", return_value=mem_cfg), - patch("src.agents.memory.updater.get_memory_data", return_value=memory_data), + patch("deerflow.config.memory_config.get_memory_config", return_value=mem_cfg), + patch("deerflow.agents.memory.updater.get_memory_data", return_value=memory_data), ): result = client.get_memory_status() diff --git a/backend/tests/test_client_live.py b/backend/tests/test_client_live.py index f2caed3..3a5caad 100644 --- a/backend/tests/test_client_live.py +++ b/backend/tests/test_client_live.py @@ -12,7 +12,7 @@ from pathlib import Path import pytest -from src.client import DeerFlowClient, StreamEvent +from deerflow.client import DeerFlowClient, StreamEvent # Skip entire module in CI or when no config.yaml exists _skip_reason = None diff --git a/backend/tests/test_config_version.py b/backend/tests/test_config_version.py new file mode 100644 index 0000000..916b938 --- /dev/null +++ b/backend/tests/test_config_version.py @@ -0,0 +1,125 @@ +"""Tests for config version check and upgrade logic.""" + +from __future__ import annotations + +import logging +import tempfile +from pathlib import Path + +import yaml + +from deerflow.config.app_config import AppConfig + + +def _make_config_files(tmpdir: Path, user_config: dict, example_config: dict) -> Path: + """Write user config.yaml and config.example.yaml to a temp dir, return config path.""" + config_path = tmpdir / "config.yaml" + example_path = tmpdir / "config.example.yaml" + + # Minimal valid config needs sandbox + defaults = { + "sandbox": {"use": "deerflow.sandbox.local:LocalSandboxProvider"}, + } + for cfg in (user_config, example_config): + for k, v in defaults.items(): + cfg.setdefault(k, v) + + with open(config_path, "w", encoding="utf-8") as f: + yaml.dump(user_config, f) + with open(example_path, "w", encoding="utf-8") as f: + yaml.dump(example_config, f) + + return config_path + + +def test_missing_version_treated_as_zero(caplog): + """Config without config_version should be treated as version 0.""" + with tempfile.TemporaryDirectory() as tmpdir: + config_path = _make_config_files( + Path(tmpdir), + user_config={}, # no config_version + example_config={"config_version": 1}, + ) + with caplog.at_level(logging.WARNING, logger="deerflow.config.app_config"): + AppConfig._check_config_version( + {"sandbox": {"use": "deerflow.sandbox.local:LocalSandboxProvider"}}, + config_path, + ) + assert "outdated" in caplog.text + assert "version 0" in caplog.text + assert "version is 1" in caplog.text + + +def test_matching_version_no_warning(caplog): + """Config with matching version should not emit a warning.""" + with tempfile.TemporaryDirectory() as tmpdir: + config_path = _make_config_files( + Path(tmpdir), + user_config={"config_version": 1}, + example_config={"config_version": 1}, + ) + with caplog.at_level(logging.WARNING, logger="deerflow.config.app_config"): + AppConfig._check_config_version( + {"config_version": 1}, + config_path, + ) + assert "outdated" not in caplog.text + + +def test_outdated_version_emits_warning(caplog): + """Config with lower version should emit a warning.""" + with tempfile.TemporaryDirectory() as tmpdir: + config_path = _make_config_files( + Path(tmpdir), + user_config={"config_version": 1}, + example_config={"config_version": 2}, + ) + with caplog.at_level(logging.WARNING, logger="deerflow.config.app_config"): + AppConfig._check_config_version( + {"config_version": 1}, + config_path, + ) + assert "outdated" in caplog.text + assert "version 1" in caplog.text + assert "version is 2" in caplog.text + + +def test_no_example_file_no_warning(caplog): + """If config.example.yaml doesn't exist, no warning should be emitted.""" + with tempfile.TemporaryDirectory() as tmpdir: + config_path = Path(tmpdir) / "config.yaml" + with open(config_path, "w", encoding="utf-8") as f: + yaml.dump({"sandbox": {"use": "test"}}, f) + # No config.example.yaml created + + with caplog.at_level(logging.WARNING, logger="deerflow.config.app_config"): + AppConfig._check_config_version({}, config_path) + assert "outdated" not in caplog.text + + +def test_string_config_version_does_not_raise_type_error(caplog): + """config_version stored as a YAML string should not raise TypeError on comparison.""" + with tempfile.TemporaryDirectory() as tmpdir: + config_path = _make_config_files( + Path(tmpdir), + user_config={"config_version": "1"}, # string, as YAML can produce + example_config={"config_version": 2}, + ) + # Must not raise TypeError: '<' not supported between instances of 'str' and 'int' + AppConfig._check_config_version({"config_version": "1"}, config_path) + + +def test_newer_user_version_no_warning(caplog): + """If user has a newer version than example (edge case), no warning.""" + with tempfile.TemporaryDirectory() as tmpdir: + config_path = _make_config_files( + Path(tmpdir), + user_config={"config_version": 3}, + example_config={"config_version": 2}, + ) + with caplog.at_level(logging.WARNING, logger="deerflow.config.app_config"): + AppConfig._check_config_version( + {"config_version": 3}, + config_path, + ) + assert "outdated" not in caplog.text diff --git a/backend/tests/test_custom_agent.py b/backend/tests/test_custom_agent.py index cd45f44..6dfbffc 100644 --- a/backend/tests/test_custom_agent.py +++ b/backend/tests/test_custom_agent.py @@ -16,7 +16,7 @@ from fastapi.testclient import TestClient def _make_paths(base_dir: Path): """Return a Paths instance pointing to base_dir.""" - from src.config.paths import Paths + from deerflow.config.paths import Paths return Paths(base_dir=base_dir) @@ -72,7 +72,7 @@ class TestPaths: class TestAgentConfig: def test_minimal_config(self): - from src.config.agents_config import AgentConfig + from deerflow.config.agents_config import AgentConfig cfg = AgentConfig(name="my-agent") assert cfg.name == "my-agent" @@ -81,7 +81,7 @@ class TestAgentConfig: assert cfg.tool_groups is None def test_full_config(self): - from src.config.agents_config import AgentConfig + from deerflow.config.agents_config import AgentConfig cfg = AgentConfig( name="code-reviewer", @@ -94,7 +94,7 @@ class TestAgentConfig: assert cfg.tool_groups == ["file:read", "bash"] def test_config_from_dict(self): - from src.config.agents_config import AgentConfig + from deerflow.config.agents_config import AgentConfig data = {"name": "test-agent", "description": "A test", "model": "gpt-4"} cfg = AgentConfig(**data) @@ -113,8 +113,8 @@ class TestLoadAgentConfig: config_dict = {"name": "code-reviewer", "description": "Code review agent", "model": "deepseek-v3"} _write_agent(tmp_path, "code-reviewer", config_dict) - with patch("src.config.agents_config.get_paths", return_value=_make_paths(tmp_path)): - from src.config.agents_config import load_agent_config + with patch("deerflow.config.agents_config.get_paths", return_value=_make_paths(tmp_path)): + from deerflow.config.agents_config import load_agent_config cfg = load_agent_config("code-reviewer") @@ -123,8 +123,8 @@ class TestLoadAgentConfig: assert cfg.model == "deepseek-v3" def test_load_missing_agent_raises(self, tmp_path): - with patch("src.config.agents_config.get_paths", return_value=_make_paths(tmp_path)): - from src.config.agents_config import load_agent_config + with patch("deerflow.config.agents_config.get_paths", return_value=_make_paths(tmp_path)): + from deerflow.config.agents_config import load_agent_config with pytest.raises(FileNotFoundError): load_agent_config("nonexistent-agent") @@ -133,8 +133,8 @@ class TestLoadAgentConfig: # Create directory without config.yaml (tmp_path / "agents" / "broken-agent").mkdir(parents=True) - with patch("src.config.agents_config.get_paths", return_value=_make_paths(tmp_path)): - from src.config.agents_config import load_agent_config + with patch("deerflow.config.agents_config.get_paths", return_value=_make_paths(tmp_path)): + from deerflow.config.agents_config import load_agent_config with pytest.raises(FileNotFoundError): load_agent_config("broken-agent") @@ -146,8 +146,8 @@ class TestLoadAgentConfig: (agent_dir / "config.yaml").write_text("description: My agent\n") (agent_dir / "SOUL.md").write_text("Hello") - with patch("src.config.agents_config.get_paths", return_value=_make_paths(tmp_path)): - from src.config.agents_config import load_agent_config + with patch("deerflow.config.agents_config.get_paths", return_value=_make_paths(tmp_path)): + from deerflow.config.agents_config import load_agent_config cfg = load_agent_config("inferred-name") @@ -157,8 +157,8 @@ class TestLoadAgentConfig: config_dict = {"name": "restricted", "tool_groups": ["file:read", "file:write"]} _write_agent(tmp_path, "restricted", config_dict) - with patch("src.config.agents_config.get_paths", return_value=_make_paths(tmp_path)): - from src.config.agents_config import load_agent_config + with patch("deerflow.config.agents_config.get_paths", return_value=_make_paths(tmp_path)): + from deerflow.config.agents_config import load_agent_config cfg = load_agent_config("restricted") @@ -171,8 +171,8 @@ class TestLoadAgentConfig: (agent_dir / "config.yaml").write_text("name: legacy-agent\nprompt_file: system.md\n") (agent_dir / "SOUL.md").write_text("Soul content") - with patch("src.config.agents_config.get_paths", return_value=_make_paths(tmp_path)): - from src.config.agents_config import load_agent_config + with patch("deerflow.config.agents_config.get_paths", return_value=_make_paths(tmp_path)): + from deerflow.config.agents_config import load_agent_config cfg = load_agent_config("legacy-agent") @@ -189,8 +189,8 @@ class TestLoadAgentSoul: expected_soul = "You are a specialized code review expert." _write_agent(tmp_path, "code-reviewer", {"name": "code-reviewer"}, soul=expected_soul) - with patch("src.config.agents_config.get_paths", return_value=_make_paths(tmp_path)): - from src.config.agents_config import AgentConfig, load_agent_soul + with patch("deerflow.config.agents_config.get_paths", return_value=_make_paths(tmp_path)): + from deerflow.config.agents_config import AgentConfig, load_agent_soul cfg = AgentConfig(name="code-reviewer") soul = load_agent_soul(cfg.name) @@ -203,8 +203,8 @@ class TestLoadAgentSoul: (agent_dir / "config.yaml").write_text("name: no-soul\n") # No SOUL.md created - with patch("src.config.agents_config.get_paths", return_value=_make_paths(tmp_path)): - from src.config.agents_config import AgentConfig, load_agent_soul + with patch("deerflow.config.agents_config.get_paths", return_value=_make_paths(tmp_path)): + from deerflow.config.agents_config import AgentConfig, load_agent_soul cfg = AgentConfig(name="no-soul") soul = load_agent_soul(cfg.name) @@ -217,8 +217,8 @@ class TestLoadAgentSoul: (agent_dir / "config.yaml").write_text("name: empty-soul\n") (agent_dir / "SOUL.md").write_text(" \n ") - with patch("src.config.agents_config.get_paths", return_value=_make_paths(tmp_path)): - from src.config.agents_config import AgentConfig, load_agent_soul + with patch("deerflow.config.agents_config.get_paths", return_value=_make_paths(tmp_path)): + from deerflow.config.agents_config import AgentConfig, load_agent_soul cfg = AgentConfig(name="empty-soul") soul = load_agent_soul(cfg.name) @@ -233,8 +233,8 @@ class TestLoadAgentSoul: class TestListCustomAgents: def test_empty_when_no_agents_dir(self, tmp_path): - with patch("src.config.agents_config.get_paths", return_value=_make_paths(tmp_path)): - from src.config.agents_config import list_custom_agents + with patch("deerflow.config.agents_config.get_paths", return_value=_make_paths(tmp_path)): + from deerflow.config.agents_config import list_custom_agents agents = list_custom_agents() @@ -244,8 +244,8 @@ class TestListCustomAgents: _write_agent(tmp_path, "agent-a", {"name": "agent-a"}) _write_agent(tmp_path, "agent-b", {"name": "agent-b", "description": "B"}) - with patch("src.config.agents_config.get_paths", return_value=_make_paths(tmp_path)): - from src.config.agents_config import list_custom_agents + with patch("deerflow.config.agents_config.get_paths", return_value=_make_paths(tmp_path)): + from deerflow.config.agents_config import list_custom_agents agents = list_custom_agents() @@ -259,8 +259,8 @@ class TestListCustomAgents: # Invalid dir (no config.yaml) (tmp_path / "agents" / "invalid-dir").mkdir(parents=True) - with patch("src.config.agents_config.get_paths", return_value=_make_paths(tmp_path)): - from src.config.agents_config import list_custom_agents + with patch("deerflow.config.agents_config.get_paths", return_value=_make_paths(tmp_path)): + from deerflow.config.agents_config import list_custom_agents agents = list_custom_agents() @@ -274,8 +274,8 @@ class TestListCustomAgents: (agents_dir / "not-a-dir.txt").write_text("hello") _write_agent(tmp_path, "real-agent", {"name": "real-agent"}) - with patch("src.config.agents_config.get_paths", return_value=_make_paths(tmp_path)): - from src.config.agents_config import list_custom_agents + with patch("deerflow.config.agents_config.get_paths", return_value=_make_paths(tmp_path)): + from deerflow.config.agents_config import list_custom_agents agents = list_custom_agents() @@ -287,8 +287,8 @@ class TestListCustomAgents: _write_agent(tmp_path, "a-agent", {"name": "a-agent"}) _write_agent(tmp_path, "m-agent", {"name": "m-agent"}) - with patch("src.config.agents_config.get_paths", return_value=_make_paths(tmp_path)): - from src.config.agents_config import list_custom_agents + with patch("deerflow.config.agents_config.get_paths", return_value=_make_paths(tmp_path)): + from deerflow.config.agents_config import list_custom_agents agents = list_custom_agents() @@ -304,35 +304,35 @@ class TestListCustomAgents: class TestMemoryFilePath: def test_global_memory_path(self, tmp_path): """None agent_name should return global memory file.""" - import src.agents.memory.updater as updater_mod - from src.config.memory_config import MemoryConfig + import deerflow.agents.memory.updater as updater_mod + from deerflow.config.memory_config import MemoryConfig with ( - patch("src.agents.memory.updater.get_paths", return_value=_make_paths(tmp_path)), - patch("src.agents.memory.updater.get_memory_config", return_value=MemoryConfig(storage_path="")), + patch("deerflow.agents.memory.updater.get_paths", return_value=_make_paths(tmp_path)), + patch("deerflow.agents.memory.updater.get_memory_config", return_value=MemoryConfig(storage_path="")), ): path = updater_mod._get_memory_file_path(None) assert path == tmp_path / "memory.json" def test_agent_memory_path(self, tmp_path): """Providing agent_name should return per-agent memory file.""" - import src.agents.memory.updater as updater_mod - from src.config.memory_config import MemoryConfig + import deerflow.agents.memory.updater as updater_mod + from deerflow.config.memory_config import MemoryConfig with ( - patch("src.agents.memory.updater.get_paths", return_value=_make_paths(tmp_path)), - patch("src.agents.memory.updater.get_memory_config", return_value=MemoryConfig(storage_path="")), + patch("deerflow.agents.memory.updater.get_paths", return_value=_make_paths(tmp_path)), + patch("deerflow.agents.memory.updater.get_memory_config", return_value=MemoryConfig(storage_path="")), ): path = updater_mod._get_memory_file_path("code-reviewer") assert path == tmp_path / "agents" / "code-reviewer" / "memory.json" def test_different_paths_for_different_agents(self, tmp_path): - import src.agents.memory.updater as updater_mod - from src.config.memory_config import MemoryConfig + import deerflow.agents.memory.updater as updater_mod + from deerflow.config.memory_config import MemoryConfig with ( - patch("src.agents.memory.updater.get_paths", return_value=_make_paths(tmp_path)), - patch("src.agents.memory.updater.get_memory_config", return_value=MemoryConfig(storage_path="")), + patch("deerflow.agents.memory.updater.get_paths", return_value=_make_paths(tmp_path)), + patch("deerflow.agents.memory.updater.get_memory_config", return_value=MemoryConfig(storage_path="")), ): path_global = updater_mod._get_memory_file_path(None) path_a = updater_mod._get_memory_file_path("agent-a") @@ -352,7 +352,7 @@ def _make_test_app(tmp_path: Path): """Create a FastAPI app with the agents router, patching paths to tmp_path.""" from fastapi import FastAPI - from src.gateway.routers.agents import router + from app.gateway.routers.agents import router app = FastAPI() app.include_router(router) @@ -364,7 +364,7 @@ def agent_client(tmp_path): """TestClient with agents router, using tmp_path as base_dir.""" paths_instance = _make_paths(tmp_path) - with patch("src.config.agents_config.get_paths", return_value=paths_instance), patch("src.gateway.routers.agents.get_paths", return_value=paths_instance): + with patch("deerflow.config.agents_config.get_paths", return_value=paths_instance), patch("app.gateway.routers.agents.get_paths", return_value=paths_instance): app = _make_test_app(tmp_path) with TestClient(app) as client: client._tmp_path = tmp_path # type: ignore[attr-defined] diff --git a/backend/tests/test_docker_sandbox_mode_detection.py b/backend/tests/test_docker_sandbox_mode_detection.py index 7921e80..19c6751 100644 --- a/backend/tests/test_docker_sandbox_mode_detection.py +++ b/backend/tests/test_docker_sandbox_mode_detection.py @@ -39,7 +39,7 @@ def test_detect_mode_local_provider(): """Local sandbox provider should map to local mode.""" config = """ sandbox: - use: src.sandbox.local:LocalSandboxProvider + use: deerflow.sandbox.local:LocalSandboxProvider """.strip() assert _detect_mode_with_config(config) == "local" @@ -49,7 +49,7 @@ def test_detect_mode_aio_without_provisioner_url(): """AIO sandbox without provisioner_url should map to aio mode.""" config = """ sandbox: - use: src.community.aio_sandbox:AioSandboxProvider + use: deerflow.community.aio_sandbox:AioSandboxProvider """.strip() assert _detect_mode_with_config(config) == "aio" @@ -59,7 +59,7 @@ def test_detect_mode_provisioner_with_url(): """AIO sandbox with provisioner_url should map to provisioner mode.""" config = """ sandbox: - use: src.community.aio_sandbox:AioSandboxProvider + use: deerflow.community.aio_sandbox:AioSandboxProvider provisioner_url: http://provisioner:8002 """.strip() @@ -70,7 +70,7 @@ def test_detect_mode_ignores_commented_provisioner_url(): """Commented provisioner_url should not activate provisioner mode.""" config = """ sandbox: - use: src.community.aio_sandbox:AioSandboxProvider + use: deerflow.community.aio_sandbox:AioSandboxProvider # provisioner_url: http://provisioner:8002 """.strip() diff --git a/backend/tests/test_harness_boundary.py b/backend/tests/test_harness_boundary.py new file mode 100644 index 0000000..76e427d --- /dev/null +++ b/backend/tests/test_harness_boundary.py @@ -0,0 +1,46 @@ +"""Boundary check: harness layer must not import from app layer. + +The deerflow-harness package (packages/harness/deerflow/) is a standalone, +publishable agent framework. It must never depend on the app layer (app/). + +This test scans all Python files in the harness package and fails if any +``from app.`` or ``import app.`` statement is found. +""" + +import ast +from pathlib import Path + +HARNESS_ROOT = Path(__file__).parent.parent / "packages" / "harness" / "deerflow" + +BANNED_PREFIXES = ("app.",) + + +def _collect_imports(filepath: Path) -> list[tuple[int, str]]: + """Return (line_number, module_path) for every import in *filepath*.""" + source = filepath.read_text(encoding="utf-8") + try: + tree = ast.parse(source, filename=str(filepath)) + except SyntaxError: + return [] + + results: list[tuple[int, str]] = [] + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + results.append((node.lineno, alias.name)) + elif isinstance(node, ast.ImportFrom): + if node.module: + results.append((node.lineno, node.module)) + return results + + +def test_harness_does_not_import_app(): + violations: list[str] = [] + + for py_file in sorted(HARNESS_ROOT.rglob("*.py")): + for lineno, module in _collect_imports(py_file): + if any(module == prefix.rstrip(".") or module.startswith(prefix) for prefix in BANNED_PREFIXES): + rel = py_file.relative_to(HARNESS_ROOT.parent.parent.parent) + violations.append(f" {rel}:{lineno} imports {module}") + + assert not violations, "Harness layer must not import from app layer:\n" + "\n".join(violations) diff --git a/backend/tests/test_infoquest_client.py b/backend/tests/test_infoquest_client.py index 1e945af..190444d 100644 --- a/backend/tests/test_infoquest_client.py +++ b/backend/tests/test_infoquest_client.py @@ -3,8 +3,8 @@ import json from unittest.mock import MagicMock, patch -from src.community.infoquest import tools -from src.community.infoquest.infoquest_client import InfoQuestClient +from deerflow.community.infoquest import tools +from deerflow.community.infoquest.infoquest_client import InfoQuestClient class TestInfoQuestClient: @@ -24,7 +24,7 @@ class TestInfoQuestClient: assert client.fetch_navigation_timeout == 60 assert client.search_time_range == 24 - @patch("src.community.infoquest.infoquest_client.requests.post") + @patch("deerflow.community.infoquest.infoquest_client.requests.post") def test_fetch_success(self, mock_post): """Test successful fetch operation.""" mock_response = MagicMock() @@ -42,7 +42,7 @@ class TestInfoQuestClient: assert kwargs["json"]["url"] == "https://example.com" assert kwargs["json"]["format"] == "HTML" - @patch("src.community.infoquest.infoquest_client.requests.post") + @patch("deerflow.community.infoquest.infoquest_client.requests.post") def test_fetch_non_200_status(self, mock_post): """Test fetch operation with non-200 status code.""" mock_response = MagicMock() @@ -55,7 +55,7 @@ class TestInfoQuestClient: assert result == "Error: fetch API returned status 404: Not Found" - @patch("src.community.infoquest.infoquest_client.requests.post") + @patch("deerflow.community.infoquest.infoquest_client.requests.post") def test_fetch_empty_response(self, mock_post): """Test fetch operation with empty response.""" mock_response = MagicMock() @@ -68,7 +68,7 @@ class TestInfoQuestClient: assert result == "Error: no result found" - @patch("src.community.infoquest.infoquest_client.requests.post") + @patch("deerflow.community.infoquest.infoquest_client.requests.post") def test_web_search_raw_results_success(self, mock_post): """Test successful web_search_raw_results operation.""" mock_response = MagicMock() @@ -85,7 +85,7 @@ class TestInfoQuestClient: assert args[0] == "https://search.infoquest.bytepluses.com" assert kwargs["json"]["query"] == "test query" - @patch("src.community.infoquest.infoquest_client.requests.post") + @patch("deerflow.community.infoquest.infoquest_client.requests.post") def test_web_search_success(self, mock_post): """Test successful web_search operation.""" mock_response = MagicMock() @@ -133,7 +133,7 @@ class TestInfoQuestClient: assert cleaned[0]["thumbnail_url"] == "https://example.com/thumb1.jpg" assert cleaned[0]["url"] == "https://example.com/page1" - @patch("src.community.infoquest.tools._get_infoquest_client") + @patch("deerflow.community.infoquest.tools._get_infoquest_client") def test_web_search_tool(self, mock_get_client): """Test web_search_tool function.""" mock_client = MagicMock() @@ -146,7 +146,7 @@ class TestInfoQuestClient: mock_get_client.assert_called_once() mock_client.web_search.assert_called_once_with("test query") - @patch("src.community.infoquest.tools._get_infoquest_client") + @patch("deerflow.community.infoquest.tools._get_infoquest_client") def test_web_fetch_tool(self, mock_get_client): """Test web_fetch_tool function.""" mock_client = MagicMock() @@ -159,7 +159,7 @@ class TestInfoQuestClient: mock_get_client.assert_called_once() mock_client.fetch.assert_called_once_with("https://example.com") - @patch("src.community.infoquest.tools.get_app_config") + @patch("deerflow.community.infoquest.tools.get_app_config") def test_get_infoquest_client(self, mock_get_app_config): """Test _get_infoquest_client function with config.""" mock_config = MagicMock() @@ -173,7 +173,7 @@ class TestInfoQuestClient: assert client.fetch_timeout == 30 assert client.fetch_navigation_timeout == 60 - @patch("src.community.infoquest.infoquest_client.requests.post") + @patch("deerflow.community.infoquest.infoquest_client.requests.post") def test_web_search_api_error(self, mock_post): """Test web_search operation with API error.""" mock_post.side_effect = Exception("Connection error") diff --git a/backend/tests/test_lead_agent_model_resolution.py b/backend/tests/test_lead_agent_model_resolution.py index a1c979b..9498f04 100644 --- a/backend/tests/test_lead_agent_model_resolution.py +++ b/backend/tests/test_lead_agent_model_resolution.py @@ -4,16 +4,16 @@ from __future__ import annotations import pytest -from src.agents.lead_agent import agent as lead_agent_module -from src.config.app_config import AppConfig -from src.config.model_config import ModelConfig -from src.config.sandbox_config import SandboxConfig +from deerflow.agents.lead_agent import agent as lead_agent_module +from deerflow.config.app_config import AppConfig +from deerflow.config.model_config import ModelConfig +from deerflow.config.sandbox_config import SandboxConfig def _make_app_config(models: list[ModelConfig]) -> AppConfig: return AppConfig( models=models, - sandbox=SandboxConfig(use="src.sandbox.local:LocalSandboxProvider"), + sandbox=SandboxConfig(use="deerflow.sandbox.local:LocalSandboxProvider"), ) @@ -76,7 +76,7 @@ def test_resolve_model_name_raises_when_no_models_configured(monkeypatch): def test_make_lead_agent_disables_thinking_when_model_does_not_support_it(monkeypatch): app_config = _make_app_config([_make_model("safe-model", supports_thinking=False)]) - import src.tools as tools_module + import deerflow.tools as tools_module monkeypatch.setattr(lead_agent_module, "get_app_config", lambda: app_config) monkeypatch.setattr(tools_module, "get_available_tools", lambda **kwargs: []) diff --git a/backend/tests/test_loop_detection_middleware.py b/backend/tests/test_loop_detection_middleware.py index dfcfa41..b9aa2da 100644 --- a/backend/tests/test_loop_detection_middleware.py +++ b/backend/tests/test_loop_detection_middleware.py @@ -4,7 +4,7 @@ from unittest.mock import MagicMock from langchain_core.messages import AIMessage, SystemMessage -from src.agents.middlewares.loop_detection_middleware import ( +from deerflow.agents.middlewares.loop_detection_middleware import ( _HARD_STOP_MSG, LoopDetectionMiddleware, _hash_tool_calls, diff --git a/backend/tests/test_mcp_client_config.py b/backend/tests/test_mcp_client_config.py index 5d6dfe5..6d0083c 100644 --- a/backend/tests/test_mcp_client_config.py +++ b/backend/tests/test_mcp_client_config.py @@ -2,8 +2,8 @@ import pytest -from src.config.extensions_config import ExtensionsConfig, McpServerConfig -from src.mcp.client import build_server_params, build_servers_config +from deerflow.config.extensions_config import ExtensionsConfig, McpServerConfig +from deerflow.mcp.client import build_server_params, build_servers_config def test_build_server_params_stdio_success(): diff --git a/backend/tests/test_mcp_oauth.py b/backend/tests/test_mcp_oauth.py index b89ef03..27facd4 100644 --- a/backend/tests/test_mcp_oauth.py +++ b/backend/tests/test_mcp_oauth.py @@ -5,8 +5,8 @@ from __future__ import annotations import asyncio from typing import Any -from src.config.extensions_config import ExtensionsConfig -from src.mcp.oauth import OAuthTokenManager, build_oauth_tool_interceptor, get_initial_oauth_headers +from deerflow.config.extensions_config import ExtensionsConfig +from deerflow.mcp.oauth import OAuthTokenManager, build_oauth_tool_interceptor, get_initial_oauth_headers class _MockResponse: diff --git a/backend/tests/test_memory_prompt_injection.py b/backend/tests/test_memory_prompt_injection.py index d00bbd5..a9cf0d7 100644 --- a/backend/tests/test_memory_prompt_injection.py +++ b/backend/tests/test_memory_prompt_injection.py @@ -2,7 +2,7 @@ import math -from src.agents.memory.prompt import _coerce_confidence, format_memory_for_injection +from deerflow.agents.memory.prompt import _coerce_confidence, format_memory_for_injection def test_format_memory_includes_facts_section() -> None: @@ -39,7 +39,7 @@ def test_format_memory_sorts_facts_by_confidence_desc() -> None: def test_format_memory_respects_budget_when_adding_facts(monkeypatch) -> None: # Make token counting deterministic for this test by counting characters. - monkeypatch.setattr("src.agents.memory.prompt._count_tokens", lambda text, encoding_name="cl100k_base": len(text)) + monkeypatch.setattr("deerflow.agents.memory.prompt._count_tokens", lambda text, encoding_name="cl100k_base": len(text)) memory_data = { "user": {}, diff --git a/backend/tests/test_memory_upload_filtering.py b/backend/tests/test_memory_upload_filtering.py index 27d7366..45d0dbf 100644 --- a/backend/tests/test_memory_upload_filtering.py +++ b/backend/tests/test_memory_upload_filtering.py @@ -9,8 +9,8 @@ persisting in long-term memory: from langchain_core.messages import AIMessage, HumanMessage, ToolMessage -from src.agents.memory.updater import _strip_upload_mentions_from_memory -from src.agents.middlewares.memory_middleware import _filter_messages_for_memory +from deerflow.agents.memory.updater import _strip_upload_mentions_from_memory +from deerflow.agents.middlewares.memory_middleware import _filter_messages_for_memory # --------------------------------------------------------------------------- # Helpers diff --git a/backend/tests/test_model_factory.py b/backend/tests/test_model_factory.py index 21101bc..98027c1 100644 --- a/backend/tests/test_model_factory.py +++ b/backend/tests/test_model_factory.py @@ -1,14 +1,14 @@ -"""Tests for src.models.factory.create_chat_model.""" +"""Tests for deerflow.models.factory.create_chat_model.""" from __future__ import annotations import pytest from langchain.chat_models import BaseChatModel -from src.config.app_config import AppConfig -from src.config.model_config import ModelConfig -from src.config.sandbox_config import SandboxConfig -from src.models import factory as factory_module +from deerflow.config.app_config import AppConfig +from deerflow.config.model_config import ModelConfig +from deerflow.config.sandbox_config import SandboxConfig +from deerflow.models import factory as factory_module # --------------------------------------------------------------------------- # Helpers @@ -18,7 +18,7 @@ from src.models import factory as factory_module def _make_app_config(models: list[ModelConfig]) -> AppConfig: return AppConfig( models=models, - sandbox=SandboxConfig(use="src.sandbox.local:LocalSandboxProvider"), + sandbox=SandboxConfig(use="deerflow.sandbox.local:LocalSandboxProvider"), ) diff --git a/backend/tests/test_present_file_tool_core_logic.py b/backend/tests/test_present_file_tool_core_logic.py index 1f00850..3068ca5 100644 --- a/backend/tests/test_present_file_tool_core_logic.py +++ b/backend/tests/test_present_file_tool_core_logic.py @@ -3,7 +3,7 @@ import importlib from types import SimpleNamespace -present_file_tool_module = importlib.import_module("src.tools.builtins.present_file_tool") +present_file_tool_module = importlib.import_module("deerflow.tools.builtins.present_file_tool") def _make_runtime(outputs_path: str) -> SimpleNamespace: diff --git a/backend/tests/test_readability.py b/backend/tests/test_readability.py index 9545ee2..f6b6e41 100644 --- a/backend/tests/test_readability.py +++ b/backend/tests/test_readability.py @@ -4,7 +4,7 @@ import subprocess import pytest -from src.utils.readability import ReadabilityExtractor +from deerflow.utils.readability import ReadabilityExtractor def test_extract_article_falls_back_when_readability_js_fails(monkeypatch): @@ -23,7 +23,7 @@ def test_extract_article_falls_back_when_readability_js_fails(monkeypatch): return {"title": "Fallback Title", "content": "

Fallback Content

"} monkeypatch.setattr( - "src.utils.readability.simple_json_from_html_string", + "deerflow.utils.readability.simple_json_from_html_string", _fake_simple_json_from_html_string, ) @@ -46,7 +46,7 @@ def test_extract_article_re_raises_unexpected_exception(monkeypatch): return {"title": "Should Not Reach Fallback", "content": "

Fallback

"} monkeypatch.setattr( - "src.utils.readability.simple_json_from_html_string", + "deerflow.utils.readability.simple_json_from_html_string", _fake_simple_json_from_html_string, ) diff --git a/backend/tests/test_reflection_resolvers.py b/backend/tests/test_reflection_resolvers.py index 218b1d2..8ce0ea6 100644 --- a/backend/tests/test_reflection_resolvers.py +++ b/backend/tests/test_reflection_resolvers.py @@ -2,8 +2,8 @@ import pytest -from src.reflection import resolvers -from src.reflection.resolvers import resolve_variable +from deerflow.reflection import resolvers +from deerflow.reflection.resolvers import resolve_variable def test_resolve_variable_reports_install_hint_for_missing_google_provider(monkeypatch: pytest.MonkeyPatch): diff --git a/backend/tests/test_sandbox_tools_security.py b/backend/tests/test_sandbox_tools_security.py index 9acb0ca..b50e563 100644 --- a/backend/tests/test_sandbox_tools_security.py +++ b/backend/tests/test_sandbox_tools_security.py @@ -2,7 +2,7 @@ from pathlib import Path import pytest -from src.sandbox.tools import ( +from deerflow.sandbox.tools import ( VIRTUAL_PATH_PREFIX, mask_local_paths_in_output, replace_virtual_path, diff --git a/backend/tests/test_skills_archive_root.py b/backend/tests/test_skills_archive_root.py index 3e2e056..e90ae87 100644 --- a/backend/tests/test_skills_archive_root.py +++ b/backend/tests/test_skills_archive_root.py @@ -2,7 +2,7 @@ from pathlib import Path from fastapi import HTTPException -from src.gateway.routers.skills import _resolve_skill_dir_from_archive_root +from app.gateway.routers.skills import _resolve_skill_dir_from_archive_root def _write_skill(skill_dir: Path) -> None: diff --git a/backend/tests/test_skills_loader.py b/backend/tests/test_skills_loader.py index 6d8fdd7..1a43e97 100644 --- a/backend/tests/test_skills_loader.py +++ b/backend/tests/test_skills_loader.py @@ -2,7 +2,7 @@ from pathlib import Path -from src.skills.loader import load_skills +from deerflow.skills.loader import get_skills_root_path, load_skills def _write_skill(skill_dir: Path, name: str, description: str) -> None: @@ -12,6 +12,15 @@ def _write_skill(skill_dir: Path, name: str, description: str) -> None: (skill_dir / "SKILL.md").write_text(content, encoding="utf-8") +def test_get_skills_root_path_points_to_project_root_skills(): + """get_skills_root_path() should point to deer-flow/skills (sibling of backend/), not backend/packages/skills.""" + path = get_skills_root_path() + assert path.name == "skills", f"Expected 'skills', got '{path.name}'" + assert (path.parent / "backend").is_dir(), ( + f"Expected skills path's parent to be project root containing 'backend/', but got {path}" + ) + + def test_load_skills_discovers_nested_skills_and_sets_container_paths(tmp_path: Path): """Nested skills should be discovered recursively with correct container paths.""" skills_root = tmp_path / "skills" diff --git a/backend/tests/test_skills_router.py b/backend/tests/test_skills_router.py index 88dcbff..e4cf993 100644 --- a/backend/tests/test_skills_router.py +++ b/backend/tests/test_skills_router.py @@ -2,11 +2,11 @@ from collections.abc import Callable from pathlib import Path from typing import cast -import src.gateway.routers.skills as skills_router +from deerflow.skills.validation import _validate_skill_frontmatter VALIDATE_SKILL_FRONTMATTER = cast( Callable[[Path], tuple[bool, str, str | None]], - getattr(skills_router, "_validate_skill_frontmatter"), + _validate_skill_frontmatter, ) diff --git a/backend/tests/test_subagent_executor.py b/backend/tests/test_subagent_executor.py index d322a29..cbec7ac 100644 --- a/backend/tests/test_subagent_executor.py +++ b/backend/tests/test_subagent_executor.py @@ -8,7 +8,7 @@ Covers: - Async tool support (MCP tools) Note: Due to circular import issues in the main codebase, conftest.py mocks -src.subagents.executor. This test file uses delayed import via fixture to test +deerflow.subagents.executor. This test file uses delayed import via fixture to test the real implementation in isolation. """ @@ -21,13 +21,13 @@ import pytest # Module names that need to be mocked to break circular imports _MOCKED_MODULE_NAMES = [ - "src.agents", - "src.agents.thread_state", - "src.agents.middlewares", - "src.agents.middlewares.thread_data_middleware", - "src.sandbox", - "src.sandbox.middleware", - "src.models", + "deerflow.agents", + "deerflow.agents.thread_state", + "deerflow.agents.middlewares", + "deerflow.agents.middlewares.thread_data_middleware", + "deerflow.sandbox", + "deerflow.sandbox.middleware", + "deerflow.models", ] @@ -40,11 +40,11 @@ def _setup_executor_classes(): """ # Save original modules original_modules = {name: sys.modules.get(name) for name in _MOCKED_MODULE_NAMES} - original_executor = sys.modules.get("src.subagents.executor") + original_executor = sys.modules.get("deerflow.subagents.executor") # Remove mocked executor if exists (from conftest.py) - if "src.subagents.executor" in sys.modules: - del sys.modules["src.subagents.executor"] + if "deerflow.subagents.executor" in sys.modules: + del sys.modules["deerflow.subagents.executor"] # Set up mocks for name in _MOCKED_MODULE_NAMES: @@ -53,8 +53,8 @@ def _setup_executor_classes(): # Import real classes inside fixture from langchain_core.messages import AIMessage, HumanMessage - from src.subagents.config import SubagentConfig - from src.subagents.executor import ( + from deerflow.subagents.config import SubagentConfig + from deerflow.subagents.executor import ( SubagentExecutor, SubagentResult, SubagentStatus, @@ -81,9 +81,9 @@ def _setup_executor_classes(): # Restore executor module (conftest.py mock) if original_executor is not None: - sys.modules["src.subagents.executor"] = original_executor - elif "src.subagents.executor" in sys.modules: - del sys.modules["src.subagents.executor"] + sys.modules["deerflow.subagents.executor"] = original_executor + elif "deerflow.subagents.executor" in sys.modules: + del sys.modules["deerflow.subagents.executor"] # Helper classes that wrap real classes for testing @@ -641,7 +641,7 @@ class TestCleanupBackgroundTask: # Re-import to get the real module with cleanup_background_task import importlib - from src.subagents import executor + from deerflow.subagents import executor return importlib.reload(executor) @@ -749,9 +749,7 @@ class TestCleanupBackgroundTask: # Should not raise executor_module.cleanup_background_task("nonexistent-task") - def test_cleanup_removes_task_with_completed_at_even_if_running( - self, executor_module, classes - ): + def test_cleanup_removes_task_with_completed_at_even_if_running(self, executor_module, classes): """Test that cleanup removes task if completed_at is set, even if status is RUNNING. This is a safety net: if completed_at is set, the task is considered done diff --git a/backend/tests/test_subagent_timeout_config.py b/backend/tests/test_subagent_timeout_config.py index 0e6b655..9edd971 100644 --- a/backend/tests/test_subagent_timeout_config.py +++ b/backend/tests/test_subagent_timeout_config.py @@ -11,13 +11,13 @@ Covers: import pytest -from src.config.subagents_config import ( +from deerflow.config.subagents_config import ( SubagentOverrideConfig, SubagentsAppConfig, get_subagents_app_config, load_subagents_config_from_dict, ) -from src.subagents.config import SubagentConfig +from deerflow.subagents.config import SubagentConfig # --------------------------------------------------------------------------- # Helpers @@ -195,32 +195,32 @@ class TestRegistryGetSubagentConfig: _reset_subagents_config() def test_returns_none_for_unknown_agent(self): - from src.subagents.registry import get_subagent_config + from deerflow.subagents.registry import get_subagent_config assert get_subagent_config("nonexistent") is None def test_returns_config_for_builtin_agents(self): - from src.subagents.registry import get_subagent_config + from deerflow.subagents.registry import get_subagent_config assert get_subagent_config("general-purpose") is not None assert get_subagent_config("bash") is not None def test_default_timeout_preserved_when_no_config(self): - from src.subagents.registry import get_subagent_config + from deerflow.subagents.registry import get_subagent_config _reset_subagents_config(timeout_seconds=900) config = get_subagent_config("general-purpose") assert config.timeout_seconds == 900 def test_global_timeout_override_applied(self): - from src.subagents.registry import get_subagent_config + from deerflow.subagents.registry import get_subagent_config _reset_subagents_config(timeout_seconds=1800) config = get_subagent_config("general-purpose") assert config.timeout_seconds == 1800 def test_per_agent_timeout_override_applied(self): - from src.subagents.registry import get_subagent_config + from deerflow.subagents.registry import get_subagent_config load_subagents_config_from_dict( { @@ -232,7 +232,7 @@ class TestRegistryGetSubagentConfig: assert bash_config.timeout_seconds == 120 def test_per_agent_override_does_not_affect_other_agents(self): - from src.subagents.registry import get_subagent_config + from deerflow.subagents.registry import get_subagent_config load_subagents_config_from_dict( { @@ -245,8 +245,8 @@ class TestRegistryGetSubagentConfig: def test_builtin_config_object_is_not_mutated(self): """Registry must return a new object, leaving the builtin default intact.""" - from src.subagents.builtins import BUILTIN_SUBAGENTS - from src.subagents.registry import get_subagent_config + from deerflow.subagents.builtins import BUILTIN_SUBAGENTS + from deerflow.subagents.registry import get_subagent_config original_timeout = BUILTIN_SUBAGENTS["bash"].timeout_seconds load_subagents_config_from_dict({"timeout_seconds": 42}) @@ -257,8 +257,8 @@ class TestRegistryGetSubagentConfig: def test_config_preserves_other_fields(self): """Applying timeout override must not change other SubagentConfig fields.""" - from src.subagents.builtins import BUILTIN_SUBAGENTS - from src.subagents.registry import get_subagent_config + from deerflow.subagents.builtins import BUILTIN_SUBAGENTS + from deerflow.subagents.registry import get_subagent_config _reset_subagents_config(timeout_seconds=300) original = BUILTIN_SUBAGENTS["general-purpose"] @@ -282,21 +282,21 @@ class TestRegistryListSubagents: _reset_subagents_config() def test_lists_both_builtin_agents(self): - from src.subagents.registry import list_subagents + from deerflow.subagents.registry import list_subagents names = {cfg.name for cfg in list_subagents()} assert "general-purpose" in names assert "bash" in names def test_all_returned_configs_get_global_override(self): - from src.subagents.registry import list_subagents + from deerflow.subagents.registry import list_subagents _reset_subagents_config(timeout_seconds=123) for cfg in list_subagents(): assert cfg.timeout_seconds == 123, f"{cfg.name} has wrong timeout" def test_per_agent_overrides_reflected_in_list(self): - from src.subagents.registry import list_subagents + from deerflow.subagents.registry import list_subagents load_subagents_config_from_dict( { diff --git a/backend/tests/test_suggestions_router.py b/backend/tests/test_suggestions_router.py index 3ef8f15..4ff8e09 100644 --- a/backend/tests/test_suggestions_router.py +++ b/backend/tests/test_suggestions_router.py @@ -1,7 +1,7 @@ import asyncio from unittest.mock import MagicMock -from src.gateway.routers import suggestions +from app.gateway.routers import suggestions def test_strip_markdown_code_fence_removes_wrapping(): diff --git a/backend/tests/test_task_tool_core_logic.py b/backend/tests/test_task_tool_core_logic.py index 2bd3542..19d1b82 100644 --- a/backend/tests/test_task_tool_core_logic.py +++ b/backend/tests/test_task_tool_core_logic.py @@ -5,10 +5,10 @@ from enum import Enum from types import SimpleNamespace from unittest.mock import MagicMock -from src.subagents.config import SubagentConfig +from deerflow.subagents.config import SubagentConfig # Use module import so tests can patch the exact symbols referenced inside task_tool(). -task_tool_module = importlib.import_module("src.tools.builtins.task_tool") +task_tool_module = importlib.import_module("deerflow.tools.builtins.task_tool") class FakeSubagentStatus(Enum): @@ -110,8 +110,8 @@ def test_task_tool_emits_running_and_completed_events(monkeypatch): monkeypatch.setattr(task_tool_module, "get_background_task_result", lambda _: next(responses)) monkeypatch.setattr(task_tool_module, "get_stream_writer", lambda: events.append) monkeypatch.setattr(task_tool_module.time, "sleep", lambda _: None) - # task_tool lazily imports from src.tools at call time, so patch that module-level function. - monkeypatch.setattr("src.tools.get_available_tools", get_available_tools) + # task_tool lazily imports from deerflow.tools at call time, so patch that module-level function. + monkeypatch.setattr("deerflow.tools.get_available_tools", get_available_tools) output = task_tool_module.task_tool.func( runtime=runtime, @@ -156,7 +156,7 @@ def test_task_tool_returns_failed_message(monkeypatch): ) monkeypatch.setattr(task_tool_module, "get_stream_writer", lambda: events.append) monkeypatch.setattr(task_tool_module.time, "sleep", lambda _: None) - monkeypatch.setattr("src.tools.get_available_tools", lambda **kwargs: []) + monkeypatch.setattr("deerflow.tools.get_available_tools", lambda **kwargs: []) output = task_tool_module.task_tool.func( runtime=_make_runtime(), @@ -190,7 +190,7 @@ def test_task_tool_returns_timed_out_message(monkeypatch): ) monkeypatch.setattr(task_tool_module, "get_stream_writer", lambda: events.append) monkeypatch.setattr(task_tool_module.time, "sleep", lambda _: None) - monkeypatch.setattr("src.tools.get_available_tools", lambda **kwargs: []) + monkeypatch.setattr("deerflow.tools.get_available_tools", lambda **kwargs: []) output = task_tool_module.task_tool.func( runtime=_make_runtime(), @@ -226,7 +226,7 @@ def test_task_tool_polling_safety_timeout(monkeypatch): ) monkeypatch.setattr(task_tool_module, "get_stream_writer", lambda: events.append) monkeypatch.setattr(task_tool_module.time, "sleep", lambda _: None) - monkeypatch.setattr("src.tools.get_available_tools", lambda **kwargs: []) + monkeypatch.setattr("deerflow.tools.get_available_tools", lambda **kwargs: []) output = task_tool_module.task_tool.func( runtime=_make_runtime(), @@ -262,7 +262,7 @@ def test_cleanup_called_on_completed(monkeypatch): ) monkeypatch.setattr(task_tool_module, "get_stream_writer", lambda: events.append) monkeypatch.setattr(task_tool_module.time, "sleep", lambda _: None) - monkeypatch.setattr("src.tools.get_available_tools", lambda **kwargs: []) + monkeypatch.setattr("deerflow.tools.get_available_tools", lambda **kwargs: []) monkeypatch.setattr( task_tool_module, "cleanup_background_task", @@ -302,7 +302,7 @@ def test_cleanup_called_on_failed(monkeypatch): ) monkeypatch.setattr(task_tool_module, "get_stream_writer", lambda: events.append) monkeypatch.setattr(task_tool_module.time, "sleep", lambda _: None) - monkeypatch.setattr("src.tools.get_available_tools", lambda **kwargs: []) + monkeypatch.setattr("deerflow.tools.get_available_tools", lambda **kwargs: []) monkeypatch.setattr( task_tool_module, "cleanup_background_task", @@ -342,7 +342,7 @@ def test_cleanup_called_on_timed_out(monkeypatch): ) monkeypatch.setattr(task_tool_module, "get_stream_writer", lambda: events.append) monkeypatch.setattr(task_tool_module.time, "sleep", lambda _: None) - monkeypatch.setattr("src.tools.get_available_tools", lambda **kwargs: []) + monkeypatch.setattr("deerflow.tools.get_available_tools", lambda **kwargs: []) monkeypatch.setattr( task_tool_module, "cleanup_background_task", @@ -389,7 +389,7 @@ def test_cleanup_not_called_on_polling_safety_timeout(monkeypatch): ) monkeypatch.setattr(task_tool_module, "get_stream_writer", lambda: events.append) monkeypatch.setattr(task_tool_module.time, "sleep", lambda _: None) - monkeypatch.setattr("src.tools.get_available_tools", lambda **kwargs: []) + monkeypatch.setattr("deerflow.tools.get_available_tools", lambda **kwargs: []) monkeypatch.setattr( task_tool_module, "cleanup_background_task", diff --git a/backend/tests/test_title_generation.py b/backend/tests/test_title_generation.py index 13fd597..53b0a50 100644 --- a/backend/tests/test_title_generation.py +++ b/backend/tests/test_title_generation.py @@ -2,8 +2,8 @@ import pytest -from src.agents.middlewares.title_middleware import TitleMiddleware -from src.config.title_config import TitleConfig, get_title_config, set_title_config +from deerflow.agents.middlewares.title_middleware import TitleMiddleware +from deerflow.config.title_config import TitleConfig, get_title_config, set_title_config class TestTitleConfig: diff --git a/backend/tests/test_title_middleware_core_logic.py b/backend/tests/test_title_middleware_core_logic.py index 598e3df..8a60fd3 100644 --- a/backend/tests/test_title_middleware_core_logic.py +++ b/backend/tests/test_title_middleware_core_logic.py @@ -5,8 +5,8 @@ from unittest.mock import AsyncMock, MagicMock from langchain_core.messages import AIMessage, HumanMessage -from src.agents.middlewares.title_middleware import TitleMiddleware -from src.config.title_config import TitleConfig, get_title_config, set_title_config +from deerflow.agents.middlewares.title_middleware import TitleMiddleware +from deerflow.config.title_config import TitleConfig, get_title_config, set_title_config def _clone_title_config(config: TitleConfig) -> TitleConfig: @@ -78,7 +78,7 @@ class TestTitleMiddlewareCoreLogic: middleware = TitleMiddleware() fake_model = MagicMock() fake_model.ainvoke = AsyncMock(return_value=MagicMock(content='"A very long generated title"')) - monkeypatch.setattr("src.agents.middlewares.title_middleware.create_chat_model", lambda **kwargs: fake_model) + monkeypatch.setattr("deerflow.agents.middlewares.title_middleware.create_chat_model", lambda **kwargs: fake_model) state = { "messages": [ @@ -97,7 +97,7 @@ class TestTitleMiddlewareCoreLogic: middleware = TitleMiddleware() fake_model = MagicMock() fake_model.ainvoke = AsyncMock(side_effect=RuntimeError("LLM unavailable")) - monkeypatch.setattr("src.agents.middlewares.title_middleware.create_chat_model", lambda **kwargs: fake_model) + monkeypatch.setattr("deerflow.agents.middlewares.title_middleware.create_chat_model", lambda **kwargs: fake_model) state = { "messages": [ diff --git a/backend/tests/test_tool_error_handling_middleware.py b/backend/tests/test_tool_error_handling_middleware.py index 60e8981..698a0d8 100644 --- a/backend/tests/test_tool_error_handling_middleware.py +++ b/backend/tests/test_tool_error_handling_middleware.py @@ -4,7 +4,7 @@ import pytest from langchain_core.messages import ToolMessage from langgraph.errors import GraphInterrupt -from src.agents.middlewares.tool_error_handling_middleware import ToolErrorHandlingMiddleware +from deerflow.agents.middlewares.tool_error_handling_middleware import ToolErrorHandlingMiddleware def _request(name: str = "web_search", tool_call_id: str | None = "tc-1"): diff --git a/backend/tests/test_tracing_config.py b/backend/tests/test_tracing_config.py index c46e885..b638d39 100644 --- a/backend/tests/test_tracing_config.py +++ b/backend/tests/test_tracing_config.py @@ -1,8 +1,8 @@ -"""Tests for src.config.tracing_config.""" +"""Tests for deerflow.config.tracing_config.""" from __future__ import annotations -from src.config import tracing_config as tracing_module +from deerflow.config import tracing_config as tracing_module def _reset_tracing_cache() -> None: diff --git a/backend/tests/test_uploads_middleware_core_logic.py b/backend/tests/test_uploads_middleware_core_logic.py index 960a162..e69f809 100644 --- a/backend/tests/test_uploads_middleware_core_logic.py +++ b/backend/tests/test_uploads_middleware_core_logic.py @@ -12,8 +12,8 @@ from unittest.mock import MagicMock from langchain_core.messages import AIMessage, HumanMessage -from src.agents.middlewares.uploads_middleware import UploadsMiddleware -from src.config.paths import Paths +from deerflow.agents.middlewares.uploads_middleware import UploadsMiddleware +from deerflow.config.paths import Paths THREAD_ID = "thread-abc123" diff --git a/backend/tests/test_uploads_router.py b/backend/tests/test_uploads_router.py index db0d5f1..1e43ce5 100644 --- a/backend/tests/test_uploads_router.py +++ b/backend/tests/test_uploads_router.py @@ -5,7 +5,7 @@ from unittest.mock import AsyncMock, MagicMock, patch from fastapi import UploadFile -from src.gateway.routers import uploads +from app.gateway.routers import uploads def test_upload_files_writes_thread_storage_and_skips_local_sandbox_sync(tmp_path): diff --git a/backend/uv.lock b/backend/uv.lock index 33f87be..e08b5f1 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 2 requires-python = ">=3.12" resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", @@ -10,6 +10,12 @@ resolution-markers = [ "python_full_version < '3.13' and sys_platform != 'win32'", ] +[manifest] +members = [ + "deer-flow", + "deerflow-harness", +] + [[package]] name = "agent-sandbox" version = "0.0.19" @@ -642,12 +648,56 @@ wheels = [ name = "deer-flow" version = "0.1.0" source = { virtual = "." } +dependencies = [ + { name = "deerflow-harness" }, + { name = "fastapi" }, + { name = "httpx" }, + { name = "langgraph-sdk" }, + { name = "lark-oapi" }, + { name = "markdown-to-mrkdwn" }, + { name = "python-multipart" }, + { name = "python-telegram-bot" }, + { name = "slack-sdk" }, + { name = "sse-starlette" }, + { name = "uvicorn", extra = ["standard"] }, +] + +[package.dev-dependencies] +dev = [ + { name = "pytest" }, + { name = "ruff" }, +] + +[package.metadata] +requires-dist = [ + { name = "deerflow-harness", editable = "packages/harness" }, + { name = "fastapi", specifier = ">=0.115.0" }, + { name = "httpx", specifier = ">=0.28.0" }, + { name = "langgraph-sdk", specifier = ">=0.1.51" }, + { name = "lark-oapi", specifier = ">=1.4.0" }, + { name = "markdown-to-mrkdwn", specifier = ">=0.3.1" }, + { name = "python-multipart", specifier = ">=0.0.20" }, + { name = "python-telegram-bot", specifier = ">=21.0" }, + { name = "slack-sdk", specifier = ">=3.33.0" }, + { name = "sse-starlette", specifier = ">=2.1.0" }, + { name = "uvicorn", extras = ["standard"], specifier = ">=0.34.0" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "pytest", specifier = ">=8.0.0" }, + { name = "ruff", specifier = ">=0.14.11" }, +] + +[[package]] +name = "deerflow-harness" +version = "0.1.0" +source = { editable = "packages/harness" } dependencies = [ { name = "agent-sandbox" }, { name = "ddgs" }, { name = "dotenv" }, { name = "duckdb" }, - { name = "fastapi" }, { name = "firecrawl-py" }, { name = "httpx" }, { name = "kubernetes" }, @@ -663,26 +713,13 @@ dependencies = [ { name = "langgraph-cli" }, { name = "langgraph-runtime-inmem" }, { name = "langgraph-sdk" }, - { name = "lark-oapi" }, - { name = "markdown-to-mrkdwn" }, { name = "markdownify" }, { name = "markitdown", extra = ["all", "xlsx"] }, { name = "pydantic" }, - { name = "python-multipart" }, - { name = "python-telegram-bot" }, { name = "pyyaml" }, { name = "readabilipy" }, - { name = "slack-sdk" }, - { name = "sse-starlette" }, { name = "tavily-python" }, { name = "tiktoken" }, - { name = "uvicorn", extra = ["standard"] }, -] - -[package.dev-dependencies] -dev = [ - { name = "pytest" }, - { name = "ruff" }, ] [package.metadata] @@ -691,7 +728,6 @@ requires-dist = [ { name = "ddgs", specifier = ">=9.10.0" }, { name = "dotenv", specifier = ">=0.9.9" }, { name = "duckdb", specifier = ">=1.4.4" }, - { name = "fastapi", specifier = ">=0.115.0" }, { name = "firecrawl-py", specifier = ">=1.15.0" }, { name = "httpx", specifier = ">=0.28.0" }, { name = "kubernetes", specifier = ">=30.0.0" }, @@ -707,26 +743,13 @@ requires-dist = [ { name = "langgraph-cli", specifier = ">=0.4.14" }, { name = "langgraph-runtime-inmem", specifier = ">=0.22.1" }, { name = "langgraph-sdk", specifier = ">=0.1.51" }, - { name = "lark-oapi", specifier = ">=1.4.0" }, - { name = "markdown-to-mrkdwn", specifier = ">=0.3.1" }, { name = "markdownify", specifier = ">=1.2.2" }, { name = "markitdown", extras = ["all", "xlsx"], specifier = ">=0.0.1a2" }, { name = "pydantic", specifier = ">=2.12.5" }, - { name = "python-multipart", specifier = ">=0.0.20" }, - { name = "python-telegram-bot", specifier = ">=21.0" }, { name = "pyyaml", specifier = ">=6.0.3" }, { name = "readabilipy", specifier = ">=0.3.0" }, - { name = "slack-sdk", specifier = ">=3.33.0" }, - { name = "sse-starlette", specifier = ">=2.1.0" }, { name = "tavily-python", specifier = ">=0.7.17" }, { name = "tiktoken", specifier = ">=0.8.0" }, - { name = "uvicorn", extras = ["standard"], specifier = ">=0.34.0" }, -] - -[package.metadata.requires-dev] -dev = [ - { name = "pytest", specifier = ">=8.0.0" }, - { name = "ruff", specifier = ">=0.14.11" }, ] [[package]] diff --git a/config.example.yaml b/config.example.yaml index 3092b8b..6dcbf33 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -7,6 +7,13 @@ # - Environment variables are available for all field values. Example: `api_key: $OPENAI_API_KEY` # - The `use` path is a string that looks like "package_name.sub_package_name.module_name:class_name/variable_name". +# ============================================================================ +# Config Version (used to detect outdated config files) +# ============================================================================ +# Bump this number when the config schema changes. +# Run `make config-upgrade` to merge new fields into your local config.yaml. +config_version: 1 + # ============================================================================ # Models Configuration # ============================================================================ @@ -16,7 +23,7 @@ models: # Example: Volcengine (Doubao) model # - name: doubao-seed-1.8 # display_name: Doubao-Seed-1.8 - # use: src.models.patched_deepseek:PatchedChatDeepSeek + # use: deerflow.models.patched_deepseek:PatchedChatDeepSeek # model: doubao-seed-1-8-251228 # api_base: https://ark.cn-beijing.volces.com/api/v3 # api_key: $VOLCENGINE_API_KEY @@ -62,7 +69,7 @@ models: # Example: DeepSeek model (with thinking support) # - name: deepseek-v3 # display_name: DeepSeek V3 (Thinking) - # use: src.models.patched_deepseek:PatchedChatDeepSeek + # use: deerflow.models.patched_deepseek:PatchedChatDeepSeek # model: deepseek-reasoner # api_key: $DEEPSEEK_API_KEY # max_tokens: 16384 @@ -76,7 +83,7 @@ models: # Example: Kimi K2.5 model # - name: kimi-k2.5 # display_name: Kimi K2.5 - # use: src.models.patched_deepseek:PatchedChatDeepSeek + # use: deerflow.models.patched_deepseek:PatchedChatDeepSeek # model: kimi-k2.5 # api_base: https://api.moonshot.cn/v1 # api_key: $MOONSHOT_API_KEY @@ -160,27 +167,27 @@ tools: # Web search tool (requires Tavily API key) - name: web_search group: web - use: src.community.tavily.tools:web_search_tool + use: deerflow.community.tavily.tools:web_search_tool max_results: 5 # api_key: $TAVILY_API_KEY # Set if needed # Web search tool (requires InfoQuest API key) # - name: web_search # group: web - # use: src.community.infoquest.tools:web_search_tool + # use: deerflow.community.infoquest.tools:web_search_tool # # Used to limit the scope of search results, only returns content within the specified time range. Set to -1 to disable time filtering # search_time_range: 10 # Web fetch tool (uses Jina AI reader) - name: web_fetch group: web - use: src.community.jina_ai.tools:web_fetch_tool + use: deerflow.community.jina_ai.tools:web_fetch_tool timeout: 10 # Web fetch tool (uses InfoQuest AI reader) # - name: web_fetch # group: web - # use: src.community.infoquest.tools:web_fetch_tool + # use: deerflow.community.infoquest.tools:web_fetch_tool # # Overall timeout for the entire crawling process (in seconds). Set to positive value to enable, -1 to disable # timeout: 10 # # Waiting time after page loading (in seconds). Set to positive value to enable, -1 to disable @@ -192,30 +199,30 @@ tools: # Use this to find reference images before image generation - name: image_search group: web - use: src.community.image_search.tools:image_search_tool + use: deerflow.community.image_search.tools:image_search_tool max_results: 5 # File operations tools - name: ls group: file:read - use: src.sandbox.tools:ls_tool + use: deerflow.sandbox.tools:ls_tool - name: read_file group: file:read - use: src.sandbox.tools:read_file_tool + use: deerflow.sandbox.tools:read_file_tool - name: write_file group: file:write - use: src.sandbox.tools:write_file_tool + use: deerflow.sandbox.tools:write_file_tool - name: str_replace group: file:write - use: src.sandbox.tools:str_replace_tool + use: deerflow.sandbox.tools:str_replace_tool # Bash execution tool - name: bash group: bash - use: src.sandbox.tools:bash_tool + use: deerflow.sandbox.tools:bash_tool # ============================================================================ # Sandbox Configuration @@ -225,7 +232,7 @@ tools: # Option 1: Local Sandbox (Default) # Executes commands directly on the host machine sandbox: - use: src.sandbox.local:LocalSandboxProvider + use: deerflow.sandbox.local:LocalSandboxProvider # Option 2: Container-based AIO Sandbox # Executes commands in isolated containers (Docker or Apple Container) @@ -233,7 +240,7 @@ sandbox: # On other platforms: Uses Docker # Uncomment to use: # sandbox: -# use: src.community.aio_sandbox:AioSandboxProvider +# use: deerflow.community.aio_sandbox:AioSandboxProvider # # # Optional: Container image to use (works with both Docker and Apple Container) # # Default: enterprise-public-cn-beijing.cr.volces.com/vefaas-public/all-in-one-sandbox:latest @@ -271,7 +278,7 @@ sandbox: # Each sandbox_id gets a dedicated Pod in k3s, managed by the provisioner. # Recommended for production or advanced users who want better isolation and scalability.: # sandbox: -# use: src.community.aio_sandbox:AioSandboxProvider +# use: deerflow.community.aio_sandbox:AioSandboxProvider # provisioner_url: http://provisioner:8002 # ============================================================================ diff --git a/docker/docker-compose-dev.yaml b/docker/docker-compose-dev.yaml index aae3889..fb91dd1 100644 --- a/docker/docker-compose-dev.yaml +++ b/docker/docker-compose-dev.yaml @@ -110,7 +110,7 @@ services: dockerfile: backend/Dockerfile # cache_from disabled - requires manual setup: mkdir -p /tmp/docker-cache-gateway container_name: deer-flow-gateway - command: sh -c "cd backend && uv run uvicorn src.gateway.app:app --host 0.0.0.0 --port 8001 --reload --reload-include='*.yaml .env' > /app/logs/gateway.log 2>&1" + command: sh -c "cd backend && PYTHONPATH=. uv run uvicorn app.gateway.app:app --host 0.0.0.0 --port 8001 --reload --reload-include='*.yaml .env' > /app/logs/gateway.log 2>&1" volumes: - ../backend/:/app/backend/ # Preserve the .venv built during Docker image build — mounting the full backend/ diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index abedcdc..c352aa1 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -61,7 +61,7 @@ services: context: ../ dockerfile: backend/Dockerfile container_name: deer-flow-gateway - command: sh -c "cd backend && uv run uvicorn src.gateway.app:app --host 0.0.0.0 --port 8001 --workers 2" + command: sh -c "cd backend && PYTHONPATH=. uv run uvicorn app.gateway.app:app --host 0.0.0.0 --port 8001 --workers 2" volumes: - ${DEER_FLOW_CONFIG_PATH}:/app/backend/config.yaml:ro - ${DEER_FLOW_EXTENSIONS_CONFIG_PATH}:/app/backend/extensions_config.json:ro diff --git a/docs/CODE_CHANGE_SUMMARY_BY_FILE.md b/docs/CODE_CHANGE_SUMMARY_BY_FILE.md index 463dc79..3767555 100644 --- a/docs/CODE_CHANGE_SUMMARY_BY_FILE.md +++ b/docs/CODE_CHANGE_SUMMARY_BY_FILE.md @@ -23,7 +23,7 @@ --- -### 2. `backend/src/agents/lead_agent/prompt.py` +### 2. `backend/packages/harness/deerflow/agents/lead_agent/prompt.py` ```diff @@ -240,34 +240,8 @@ You have access to skills that provide optimized workflows for specific tasks. E @@ -76,7 +76,7 @@ --- -### 3. `backend/src/gateway/routers/artifacts.py` +### 3. `backend/app/gateway/routers/artifacts.py` ```diff @@ -1,12 +1,10 @@ @@ -92,7 +92,7 @@ +from fastapi import APIRouter, HTTPException, Request +from fastapi.responses import FileResponse, HTMLResponse, PlainTextResponse, Response - from src.gateway.path_utils import resolve_thread_virtual_path + from app.gateway.path_utils import resolve_thread_virtual_path ``` - **第 1 行**：删除 `import json`。 @@ -176,7 +176,7 @@ --- -### 4. `backend/src/subagents/builtins/general_purpose.py` +### 4. `backend/packages/harness/deerflow/subagents/builtins/general_purpose.py` ```diff @@ -24,21 +24,10 @@ Do NOT use for simple, single-step operations.""", diff --git a/docs/SKILL_NAME_CONFLICT_FIX.md b/docs/SKILL_NAME_CONFLICT_FIX.md index 2103401..b00caad 100644 --- a/docs/SKILL_NAME_CONFLICT_FIX.md +++ b/docs/SKILL_NAME_CONFLICT_FIX.md @@ -48,7 +48,7 @@ ## 详细代码改动 -### 一、后端配置层 (`backend/src/config/extensions_config.py`) +### 一、后端配置层 (`backend/packages/harness/deerflow/config/extensions_config.py`) #### 1.1 新增方法: `get_skill_key()` @@ -138,7 +138,7 @@ def is_skill_enabled(self, skill_name: str, skill_category: str) -> bool: --- -### 二、后端技能加载器 (`backend/src/skills/loader.py`) +### 二、后端技能加载器 (`backend/packages/harness/deerflow/skills/loader.py`) #### 2.1 添加重复检查逻辑 @@ -200,7 +200,7 @@ for category in ["public", "custom"]: --- -### 三、后端 API 路由 (`backend/src/gateway/routers/skills.py`) +### 三、后端 API 路由 (`backend/app/gateway/routers/skills.py`) #### 3.1 新增辅助函数: `_find_skill_by_name()` @@ -830,9 +830,9 @@ Body: { "enabled": false } ### 改动统计 - **后端文件**: 3 个文件修改 - - `backend/src/config/extensions_config.py`: +1 方法，修改 1 方法 - - `backend/src/skills/loader.py`: +重复检查逻辑 - - `backend/src/gateway/routers/skills.py`: +1 辅助函数，修改 3 个端点 + - `backend/packages/harness/deerflow/config/extensions_config.py`: +1 方法，修改 1 方法 + - `backend/packages/harness/deerflow/skills/loader.py`: +重复检查逻辑 + - `backend/app/gateway/routers/skills.py`: +1 辅助函数，修改 3 个端点 - **前端文件**: 3 个文件修改 - `frontend/src/core/skills/api.ts`: 修改 1 个函数 diff --git a/scripts/config-upgrade.sh b/scripts/config-upgrade.sh new file mode 100755 index 0000000..02d5926 --- /dev/null +++ b/scripts/config-upgrade.sh @@ -0,0 +1,146 @@ +#!/usr/bin/env bash +# +# config-upgrade.sh - Upgrade config.yaml to match config.example.yaml +# +# 1. Runs version-specific migrations (value replacements, renames, etc.) +# 2. Merges missing fields from the example into the user config +# 3. Backs up config.yaml to config.yaml.bak before modifying. + +set -e + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +EXAMPLE="$REPO_ROOT/config.example.yaml" + +# Resolve config.yaml location: env var > backend/ > repo root +if [ -n "$DEER_FLOW_CONFIG_PATH" ] && [ -f "$DEER_FLOW_CONFIG_PATH" ]; then + CONFIG="$DEER_FLOW_CONFIG_PATH" +elif [ -f "$REPO_ROOT/backend/config.yaml" ]; then + CONFIG="$REPO_ROOT/backend/config.yaml" +elif [ -f "$REPO_ROOT/config.yaml" ]; then + CONFIG="$REPO_ROOT/config.yaml" +else + CONFIG="" +fi + +if [ ! -f "$EXAMPLE" ]; then + echo "✗ config.example.yaml not found at $EXAMPLE" + exit 1 +fi + +if [ -z "$CONFIG" ]; then + echo "No config.yaml found — creating from example..." + cp "$EXAMPLE" "$REPO_ROOT/config.yaml" + echo "✓ config.yaml created. Please review and set your API keys." + exit 0 +fi + +# Use inline Python to do migrations + recursive merge with PyYAML +cd "$REPO_ROOT/backend" && uv run python3 -c " +import sys, shutil, copy, re +from pathlib import Path + +import yaml + +config_path = Path('$CONFIG') +example_path = Path('$EXAMPLE') + +with open(config_path, encoding='utf-8') as f: + raw_text = f.read() + user = yaml.safe_load(raw_text) or {} + +with open(example_path, encoding='utf-8') as f: + example = yaml.safe_load(f) or {} + +user_version = user.get('config_version', 0) +example_version = example.get('config_version', 0) + +if user_version >= example_version: + print(f'✓ config.yaml is already up to date (version {user_version}).') + sys.exit(0) + +print(f'Upgrading config.yaml: version {user_version} → {example_version}') +print() + +# ── Migrations ─────────────────────────────────────────────────────────── +# Each migration targets a specific version upgrade. +# 'replacements': list of (old_string, new_string) applied to the raw YAML text. +# This handles value changes that a dict merge cannot catch. + +MIGRATIONS = { + 1: { + 'description': 'Rename src.* module paths to deerflow.*', + 'replacements': [ + ('src.community.', 'deerflow.community.'), + ('src.sandbox.', 'deerflow.sandbox.'), + ('src.models.', 'deerflow.models.'), + ('src.tools.', 'deerflow.tools.'), + ], + }, + # Future migrations go here: + # 2: { + # 'description': '...', + # 'replacements': [('old', 'new')], + # }, +} + +# Apply migrations in order for versions (user_version, example_version] +migrated = [] +for version in range(user_version + 1, example_version + 1): + migration = MIGRATIONS.get(version) + if not migration: + continue + desc = migration.get('description', f'Migration to v{version}') + for old, new in migration.get('replacements', []): + if old in raw_text: + raw_text = raw_text.replace(old, new) + migrated.append(f'{old} → {new}') + +# Re-parse after text migrations +user = yaml.safe_load(raw_text) or {} + +if migrated: + print(f'Applied {len(migrated)} migration(s):') + for m in migrated: + print(f' ~ {m}') + print() + +# ── Merge missing fields ───────────────────────────────────────────────── + +added = [] + +def merge(target, source, path=''): + \"\"\"Recursively merge source into target, adding missing keys only.\"\"\" + for key, value in source.items(): + key_path = f'{path}.{key}' if path else key + if key not in target: + target[key] = copy.deepcopy(value) + added.append(key_path) + elif isinstance(value, dict) and isinstance(target[key], dict): + merge(target[key], value, key_path) + +merge(user, example) + +# Always update config_version +user['config_version'] = example_version + +# ── Write ───────────────────────────────────────────────────────────────── + +backup = config_path.with_suffix('.yaml.bak') +shutil.copy2(config_path, backup) +print(f'Backed up to {backup.name}') + +with open(config_path, 'w', encoding='utf-8') as f: + yaml.dump(user, f, default_flow_style=False, allow_unicode=True, sort_keys=False) + +if added: + print(f'Added {len(added)} new field(s):') + for a in added: + print(f' + {a}') + +if not migrated and not added: + print('No changes needed (version bumped only).') + +print() +print(f'✓ config.yaml upgraded to version {example_version}.') +print(' Please review the changes and set any new required values.') +" diff --git a/scripts/deploy.sh b/scripts/deploy.sh index 86b0710..66df240 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -124,7 +124,7 @@ detect_sandbox_mode() { } ' "$DEER_FLOW_CONFIG_PATH") - if [[ "$sandbox_use" == *"src.community.aio_sandbox:AioSandboxProvider"* ]]; then + if [[ "$sandbox_use" == *"deerflow.community.aio_sandbox:AioSandboxProvider"* ]]; then if [ -n "$provisioner_url" ]; then echo "provisioner" else diff --git a/scripts/docker.sh b/scripts/docker.sh index 312bc6e..c936f27 100755 --- a/scripts/docker.sh +++ b/scripts/docker.sh @@ -47,9 +47,9 @@ detect_sandbox_mode() { } ' "$config_file") - if [[ "$sandbox_use" == *"src.sandbox.local:LocalSandboxProvider"* ]]; then + if [[ "$sandbox_use" == *"deerflow.sandbox.local:LocalSandboxProvider"* ]]; then echo "local" - elif [[ "$sandbox_use" == *"src.community.aio_sandbox:AioSandboxProvider"* ]]; then + elif [[ "$sandbox_use" == *"deerflow.community.aio_sandbox:AioSandboxProvider"* ]]; then if [ -n "$provisioner_url" ]; then echo "provisioner" else diff --git a/scripts/serve.sh b/scripts/serve.sh index 72d1afb..6e16708 100755 --- a/scripts/serve.sh +++ b/scripts/serve.sh @@ -30,7 +30,7 @@ fi echo "Stopping existing services if any..." pkill -f "langgraph dev" 2>/dev/null || true -pkill -f "uvicorn src.gateway.app:app" 2>/dev/null || true +pkill -f "uvicorn app.gateway.app:app" 2>/dev/null || true pkill -f "next dev" 2>/dev/null || true pkill -f "next-server" 2>/dev/null || true nginx -c "$REPO_ROOT/docker/nginx/nginx.local.conf" -p "$REPO_ROOT" -s quit 2>/dev/null || true @@ -78,6 +78,10 @@ if ! { \ exit 1 fi +# ── Auto-upgrade config ────────────────────────────────────────────────── + +"$REPO_ROOT/scripts/config-upgrade.sh" + # ── Cleanup trap ───────────────────────────────────────────────────────────── cleanup() { @@ -85,7 +89,7 @@ cleanup() { echo "" echo "Shutting down services..." pkill -f "langgraph dev" 2>/dev/null || true - pkill -f "uvicorn src.gateway.app:app" 2>/dev/null || true + pkill -f "uvicorn app.gateway.app:app" 2>/dev/null || true pkill -f "next dev" 2>/dev/null || true pkill -f "next start" 2>/dev/null || true # Kill nginx using the captured PID first (most reliable), @@ -121,18 +125,24 @@ echo "Starting LangGraph server..." ./scripts/wait-for-port.sh 2024 60 "LangGraph" || { echo " See logs/langgraph.log for details" tail -20 logs/langgraph.log + if grep -qE "config_version|outdated|Environment variable .* not found|KeyError|ValidationError|config\.yaml" logs/langgraph.log 2>/dev/null; then + echo "" + echo " Hint: This may be a configuration issue. Try running 'make config-upgrade' to update your config.yaml." + fi cleanup } echo "✓ LangGraph server started on localhost:2024" echo "Starting Gateway API..." -(cd backend && uv run uvicorn src.gateway.app:app --host 0.0.0.0 --port 8001 $GATEWAY_EXTRA_FLAGS > ../logs/gateway.log 2>&1) & +(cd backend && PYTHONPATH=. uv run uvicorn app.gateway.app:app --host 0.0.0.0 --port 8001 $GATEWAY_EXTRA_FLAGS > ../logs/gateway.log 2>&1) & ./scripts/wait-for-port.sh 8001 30 "Gateway API" || { echo "✗ Gateway API failed to start. Last log output:" tail -60 logs/gateway.log echo "" echo "Likely configuration errors:" grep -E "Failed to load configuration|Environment variable .* not found|config\.yaml.*not found" logs/gateway.log | tail -5 || true + echo "" + echo " Hint: Try running 'make config-upgrade' to update your config.yaml with the latest fields." cleanup } echo "✓ Gateway API started on localhost:8001" diff --git a/scripts/start-daemon.sh b/scripts/start-daemon.sh index 16350c9..a60653d 100755 --- a/scripts/start-daemon.sh +++ b/scripts/start-daemon.sh @@ -16,7 +16,7 @@ cd "$REPO_ROOT" echo "Stopping existing services if any..." pkill -f "langgraph dev" 2>/dev/null || true -pkill -f "uvicorn src.gateway.app:app" 2>/dev/null || true +pkill -f "uvicorn app.gateway.app:app" 2>/dev/null || true pkill -f "next dev" 2>/dev/null || true nginx -c "$REPO_ROOT/docker/nginx/nginx.local.conf" -p "$REPO_ROOT" -s quit 2>/dev/null || true sleep 1 @@ -49,12 +49,16 @@ if ! { \ exit 1 fi +# ── Auto-upgrade config ────────────────────────────────────────────────── + +"$REPO_ROOT/scripts/config-upgrade.sh" + # ── Cleanup on failure ─────────────────────────────────────────────────────── cleanup_on_failure() { echo "Failed to start services, cleaning up..." pkill -f "langgraph dev" 2>/dev/null || true - pkill -f "uvicorn src.gateway.app:app" 2>/dev/null || true + pkill -f "uvicorn app.gateway.app:app" 2>/dev/null || true pkill -f "next dev" 2>/dev/null || true nginx -c "$REPO_ROOT/docker/nginx/nginx.local.conf" -p "$REPO_ROOT" -s quit 2>/dev/null || true sleep 1 @@ -73,16 +77,22 @@ nohup sh -c 'cd backend && NO_COLOR=1 uv run langgraph dev --no-browser --allow- ./scripts/wait-for-port.sh 2024 60 "LangGraph" || { echo "✗ LangGraph failed to start. Last log output:" tail -60 logs/langgraph.log + if grep -qE "config_version|outdated|Environment variable .* not found|KeyError|ValidationError|config\.yaml" logs/langgraph.log 2>/dev/null; then + echo "" + echo " Hint: This may be a configuration issue. Try running 'make config-upgrade' to update your config.yaml." + fi cleanup_on_failure exit 1 } echo "✓ LangGraph server started on localhost:2024" echo "Starting Gateway API..." -nohup sh -c 'cd backend && uv run uvicorn src.gateway.app:app --host 0.0.0.0 --port 8001 > ../logs/gateway.log 2>&1' & +nohup sh -c 'cd backend && PYTHONPATH=. uv run uvicorn app.gateway.app:app --host 0.0.0.0 --port 8001 > ../logs/gateway.log 2>&1' & ./scripts/wait-for-port.sh 8001 30 "Gateway API" || { echo "✗ Gateway API failed to start. Last log output:" tail -60 logs/gateway.log + echo "" + echo " Hint: Try running 'make config-upgrade' to update your config.yaml with the latest fields." cleanup_on_failure exit 1 } diff --git a/scripts/tool-error-degradation-detection.sh b/scripts/tool-error-degradation-detection.sh index 3bc8c9a..14b2da8 100755 --- a/scripts/tool-error-degradation-detection.sh +++ b/scripts/tool-error-degradation-detection.sh @@ -35,14 +35,14 @@ from requests.exceptions import SSLError from langchain.agents.middleware import AgentMiddleware from langchain_core.messages import ToolMessage -from src.agents.lead_agent.agent import _build_middlewares -from src.config import get_app_config -from src.sandbox.middleware import SandboxMiddleware +from deerflow.agents.lead_agent.agent import _build_middlewares +from deerflow.config import get_app_config +from deerflow.sandbox.middleware import SandboxMiddleware -from src.agents.middlewares.thread_data_middleware import ThreadDataMiddleware +from deerflow.agents.middlewares.thread_data_middleware import ThreadDataMiddleware HANDSHAKE_ERROR = "[SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1000)" -logging.getLogger("src.agents.middlewares.tool_error_handling_middleware").setLevel(logging.CRITICAL) +logging.getLogger("deerflow.agents.middlewares.tool_error_handling_middleware").setLevel(logging.CRITICAL) def _make_ssl_error(): @@ -150,7 +150,7 @@ def _validate_outputs(label, outputs): def _build_sub_middlewares(): try: - from src.agents.middlewares.tool_error_handling_middleware import build_subagent_runtime_middlewares + from deerflow.agents.middlewares.tool_error_handling_middleware import build_subagent_runtime_middlewares except Exception: return [ ThreadDataMiddleware(lazy_init=True),