From 50810c8212e122e3f5822ca23777e11747c3fa9d Mon Sep 17 00:00:00 2001 From: hetaoBackend Date: Tue, 20 Jan 2026 13:57:36 +0800 Subject: [PATCH] feat: add skills api --- .gitignore | 1 + MCP_SETUP.md | 261 ------------------------ backend/CLAUDE.md | 39 ++-- backend/src/agents/lead_agent/prompt.py | 4 +- backend/src/config/__init__.py | 4 +- backend/src/config/app_config.py | 10 +- backend/src/config/extensions_config.py | 221 ++++++++++++++++++++ backend/src/config/mcp_config.py | 186 ----------------- backend/src/gateway/app.py | 12 +- backend/src/gateway/routers/mcp.py | 22 +- backend/src/gateway/routers/skills.py | 227 +++++++++++++++++++++ backend/src/mcp/client.py | 8 +- backend/src/mcp/tools.py | 6 +- backend/src/skills/loader.py | 20 +- backend/src/skills/parser.py | 1 + backend/src/skills/types.py | 1 + backend/src/tools/tools.py | 2 +- extensions_config.example.json | 38 ++++ mcp_config.example.json | 54 ----- nginx.conf | 10 + skills/public/pdf-processing/SKILL.md | 2 +- 21 files changed, 586 insertions(+), 543 deletions(-) delete mode 100644 MCP_SETUP.md create mode 100644 backend/src/config/extensions_config.py delete mode 100644 backend/src/config/mcp_config.py create mode 100644 backend/src/gateway/routers/skills.py create mode 100644 extensions_config.example.json delete mode 100644 mcp_config.example.json diff --git a/.gitignore b/.gitignore index c1c7855..4d5fceb 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ venv/ # Configuration files config.yaml mcp_config.json +extensions_config.json # IDE .idea/ diff --git a/MCP_SETUP.md b/MCP_SETUP.md deleted file mode 100644 index 0f7f3b0..0000000 --- a/MCP_SETUP.md +++ /dev/null @@ -1,261 +0,0 @@ -# MCP (Model Context Protocol) Setup Guide - -This guide explains how to configure and use MCP servers with DeerFlow to extend your agent's capabilities. - -## What is MCP? - -MCP (Model Context Protocol) is a standardized protocol for integrating external tools and services with AI agents. It allows DeerFlow to connect to various MCP servers that provide additional capabilities like file system access, database queries, web browsing, and more. - -DeerFlow uses [langchain-mcp-adapters](https://github.com/langchain-ai/langchain-mcp-adapters) to seamlessly integrate MCP servers with the LangChain/LangGraph ecosystem. - -## Quick Start - -1. **Copy the example configuration:** - ```bash - cp mcp_config.example.json mcp_config.json - ``` - -2. **Enable desired MCP servers:** - Edit `mcp_config.json` and set `"enabled": true` for the servers you want to use. - -3. **Configure environment variables:** - Set any required API keys or credentials: - ```bash - export GITHUB_TOKEN="your_github_token" - export BRAVE_API_KEY="your_brave_api_key" - # etc. - ``` - -4. **Install MCP dependencies:** - ```bash - cd backend - make install - ``` - -5. **Restart the application:** - MCP tools will be automatically loaded and cached when first needed. - ```bash - cd backend - make dev - ``` - - **Note**: MCP tools use lazy initialization - they are automatically loaded on first use. - This works in both: - - **FastAPI server**: Eagerly initialized at startup for best performance - - **LangGraph Studio**: Automatically initialized on first agent creation - -## Configuration Format - -MCP servers are configured in `mcp_config.json`: - -```json -{ - "mcpServers": { - "server-name": { - "enabled": true, - "command": "npx", - "args": ["-y", "@modelcontextprotocol/server-package"], - "env": { - "API_KEY": "$ENV_VAR_NAME" - }, - "description": "What this server provides" - } - } -} -``` - -### Configuration Fields - -- **enabled** (boolean): Whether this MCP server is active -- **command** (string): Command to execute (e.g., "npx", "python", "node") -- **args** (array): Command arguments -- **env** (object): Environment variables (supports `$VAR_NAME` syntax) -- **description** (string): Human-readable description - -## Environment Variables - -Environment variables in the config use the `$VARIABLE_NAME` syntax and are resolved at runtime: - -```json -"env": { - "GITHUB_PERSONAL_ACCESS_TOKEN": "$GITHUB_TOKEN" -} -``` - -This will use the value of the `GITHUB_TOKEN` environment variable. - -## Popular MCP Servers - -### Filesystem Access -Provides read/write access to specified directories: -```json -"filesystem": { - "enabled": true, - "command": "npx", - "args": ["-y", "@modelcontextprotocol/server-filesystem", "/path/to/allowed/files"] -} -``` - -### PostgreSQL Database -Connect to PostgreSQL databases: -```json -"postgres": { - "enabled": true, - "command": "npx", - "args": ["-y", "@modelcontextprotocol/server-postgres", "postgresql://localhost/mydb"], - "env": { - "PGPASSWORD": "$POSTGRES_PASSWORD" - } -} -``` - -### GitHub Integration -Interact with GitHub repositories: -```json -"github": { - "enabled": true, - "command": "npx", - "args": ["-y", "@modelcontextprotocol/server-github"], - "env": { - "GITHUB_PERSONAL_ACCESS_TOKEN": "$GITHUB_TOKEN" - } -} -``` - -### Brave Search -Web search capabilities: -```json -"brave-search": { - "enabled": true, - "command": "npx", - "args": ["-y", "@modelcontextprotocol/server-brave-search"], - "env": { - "BRAVE_API_KEY": "$BRAVE_API_KEY" - } -} -``` - -### Puppeteer Browser Automation -Control headless browser: -```json -"puppeteer": { - "enabled": true, - "command": "npx", - "args": ["-y", "@modelcontextprotocol/server-puppeteer"] -} -``` - -## Custom MCP Servers - -You can also create your own MCP servers or use third-party implementations: - -```json -"my-custom-server": { - "enabled": true, - "command": "python", - "args": ["-m", "my_mcp_server_package"], - "env": { - "API_KEY": "$MY_API_KEY" - }, - "description": "My custom MCP server" -} -``` - -## Configuration File Location - -The MCP config file is loaded with the following priority: - -1. Explicit path via `DEER_FLOW_MCP_CONFIG_PATH` environment variable -2. `mcp_config.json` in current directory (backend/) -3. `mcp_config.json` in parent directory (project root - **recommended**) - -**Recommended location:** `/path/to/deer-flow/mcp_config.json` - -## Tool Naming Convention - -MCP tools are automatically named with the pattern: -``` -mcp_{server_name}_{tool_name} -``` - -For example, a tool named `read_file` from the `filesystem` server becomes: -``` -mcp_filesystem_read_file -``` - -## Custom Scripts and Initialization - -MCP tools are automatically initialized on first use, so you don't need to do anything special: - -```python -from src.agents import make_lead_agent - -# MCP tools will be automatically loaded when the agent is created -agent = make_lead_agent(config) -``` - -**Optional**: For better performance in long-running scripts, you can pre-initialize: - -```python -import asyncio -from src.mcp import initialize_mcp_tools -from src.agents import make_lead_agent - -async def main(): - # Optional: Pre-load MCP tools for faster first agent creation - await initialize_mcp_tools() - - # Create agent - MCP tools are already loaded - agent = make_lead_agent(config) - # ... rest of your code - -if __name__ == "__main__": - asyncio.run(main()) -``` - -## Troubleshooting - -### MCP tools not loading - -1. Check that `mcp` package is installed: - ```bash - cd backend - python -c "import mcp; print(mcp.__version__)" - ``` - -2. Verify your MCP config is valid JSON: - ```bash - python -m json.tool mcp_config.json - ``` - -3. Check application logs for MCP-related errors: - ```bash - # Look for lines containing "MCP" - grep -i mcp logs/app.log - ``` - -### Server fails to start - -1. Verify the command and arguments are correct -2. Check that required npm packages are installed globally or with npx -3. Ensure environment variables are set correctly - -### Tools not appearing - -1. Verify the server is enabled: `"enabled": true` -2. Check that the server starts successfully (see logs) -3. Ensure there are no permission issues with the command - -## Security Considerations - -- The `mcp_config.json` file may contain sensitive information and is excluded from git by default -- Only enable MCP servers from trusted sources -- Be cautious with filesystem and database access - restrict paths/permissions appropriately -- Review the capabilities of each MCP server before enabling it - -## Resources - -- MCP Specification: https://modelcontextprotocol.io -- Official MCP Servers: https://github.com/modelcontextprotocol/servers -- LangChain MCP Adapters: https://github.com/langchain-ai/langchain-mcp-adapters -- DeerFlow Documentation: See `CLAUDE.md` and `config.example.yaml` diff --git a/backend/CLAUDE.md b/backend/CLAUDE.md index 0648da9..785bd97 100644 --- a/backend/CLAUDE.md +++ b/backend/CLAUDE.md @@ -69,7 +69,6 @@ Config values starting with `$` are resolved as environment variables (e.g., `$O **MCP System** (`src/mcp/`) - Integrates with MCP servers to provide pluggable external tools using `langchain-mcp-adapters` -- Configuration in `mcp_config.json` in project root (separate from `config.yaml`) - Uses `MultiServerMCPClient` from langchain-mcp-adapters for multi-server management - **Automatic initialization**: Tools are loaded on first use with lazy initialization - Supports both eager loading (FastAPI startup) and lazy loading (LangGraph Studio) @@ -77,14 +76,7 @@ Config values starting with `$` are resolved as environment variables (e.g., `$O - `get_cached_mcp_tools()` automatically initializes tools if not already loaded - Works seamlessly in both FastAPI server and LangGraph Studio environments - Each server can be enabled/disabled independently via `enabled` flag -- Supports environment variable resolution (e.g., `$GITHUB_TOKEN`) -- Configuration priority: - 1. Explicit `config_path` argument - 2. `DEER_FLOW_MCP_CONFIG_PATH` environment variable - 3. `mcp_config.json` in current directory (backend/) - 4. `mcp_config.json` in parent directory (project root - **recommended location**) - Popular MCP servers: filesystem, postgres, github, brave-search, puppeteer -- See `mcp_config.example.json` for configuration examples - Built on top of langchain-ai/langchain-mcp-adapters for seamless integration **Reflection System** (`src/reflection/`) @@ -97,10 +89,11 @@ Config values starting with `$` are resolved as environment variables (e.g., `$O - Each skill has a `SKILL.md` file with YAML front matter (name, description, license) - Skills are automatically discovered and loaded at runtime - `load_skills()` scans directories and parses SKILL.md files -- Skills are injected into agent's system prompt with paths +- Skills are injected into agent's system prompt with paths (only enabled skills) - Path mapping system allows seamless access in both local and Docker sandbox: - Local sandbox: `/mnt/skills` → `/path/to/deer-flow/skills` - Docker sandbox: Automatically mounted as volume +- Each skill can be enabled/disabled independently via `enabled` flag in extensions config **Middleware System** - Custom middlewares in `src/agents/middlewares/`: Title generation, thread data, clarification, etc. @@ -124,13 +117,35 @@ Models, tools, sandbox providers, skills, and middleware settings are configured - `title`: Automatic thread title generation configuration - `summarization`: Automatic conversation summarization configuration -MCP servers are configured separately in `mcp_config.json`: -- `mcpServers`: Map of server name to configuration +**Extensions Configuration** (`extensions_config.json`) + +MCP servers and skills are configured together in `extensions_config.json` in project root: + +**Setup**: Copy `extensions_config.example.json` to `extensions_config.json` in the **project root** directory. + +```bash +# From project root (deer-flow/) +cp extensions_config.example.json extensions_config.json +``` + +Configuration priority: +1. Explicit `config_path` argument +2. `DEER_FLOW_EXTENSIONS_CONFIG_PATH` environment variable +3. `extensions_config.json` in current directory (backend/) +4. `extensions_config.json` in parent directory (project root - **recommended location**) +5. For backward compatibility: `mcp_config.json` (will be deprecated) + +Structure: +- `mcpServers`: Map of MCP server name to configuration - `enabled`: Whether the server is enabled (boolean) - `command`: Command to execute to start the server (e.g., "npx", "python") - `args`: Arguments to pass to the command (array) - - `env`: Environment variables (object with `$VAR` support) + - `env`: Environment variables (object with `$VAR` support for env variable resolution) - `description`: Human-readable description +- `skills`: Map of skill name to state configuration + - `enabled`: Whether the skill is enabled (boolean, default: true if not specified) + +Both MCP servers and skills can be modified at runtime via API endpoints. ## Code Style diff --git a/backend/src/agents/lead_agent/prompt.py b/backend/src/agents/lead_agent/prompt.py index 0edfea6..e845bc6 100644 --- a/backend/src/agents/lead_agent/prompt.py +++ b/backend/src/agents/lead_agent/prompt.py @@ -132,8 +132,8 @@ All temporary work happens in `/mnt/user-data/workspace`. Final deliverables mus def apply_prompt_template() -> str: - # Load all available skills - skills = load_skills() + # Load only enabled skills + skills = load_skills(enabled_only=True) # Get skills container path from config try: diff --git a/backend/src/config/__init__.py b/backend/src/config/__init__.py index f095265..b12e113 100644 --- a/backend/src/config/__init__.py +++ b/backend/src/config/__init__.py @@ -1,5 +1,5 @@ from .app_config import get_app_config -from .mcp_config import McpConfig, get_mcp_config +from .extensions_config import ExtensionsConfig, get_extensions_config from .skills_config import SkillsConfig -__all__ = ["get_app_config", "SkillsConfig", "McpConfig", "get_mcp_config"] +__all__ = ["get_app_config", "SkillsConfig", "ExtensionsConfig", "get_extensions_config"] diff --git a/backend/src/config/app_config.py b/backend/src/config/app_config.py index c6799d1..f99dbbf 100644 --- a/backend/src/config/app_config.py +++ b/backend/src/config/app_config.py @@ -6,7 +6,7 @@ import yaml from dotenv import load_dotenv from pydantic import BaseModel, ConfigDict, Field -from src.config.mcp_config import McpConfig +from src.config.extensions_config import ExtensionsConfig from src.config.model_config import ModelConfig from src.config.sandbox_config import SandboxConfig from src.config.skills_config import SkillsConfig @@ -25,7 +25,7 @@ class AppConfig(BaseModel): tools: list[ToolConfig] = Field(default_factory=list, description="Available tools") tool_groups: list[ToolGroupConfig] = Field(default_factory=list, description="Available tool groups") skills: SkillsConfig = Field(default_factory=SkillsConfig, description="Skills configuration") - mcp: McpConfig = Field(default_factory=McpConfig, description="MCP configuration") + extensions: ExtensionsConfig = Field(default_factory=ExtensionsConfig, description="Extensions configuration (MCP servers and skills state)") model_config = ConfigDict(extra="allow", frozen=False) @classmethod @@ -82,9 +82,9 @@ class AppConfig(BaseModel): if "summarization" in config_data: load_summarization_config_from_dict(config_data["summarization"]) - # Load MCP config separately (it's in a different file) - mcp_config = McpConfig.from_file() - config_data["mcp"] = mcp_config.model_dump() + # Load extensions config separately (it's in a different file) + extensions_config = ExtensionsConfig.from_file() + config_data["extensions"] = extensions_config.model_dump() result = cls.model_validate(config_data) return result diff --git a/backend/src/config/extensions_config.py b/backend/src/config/extensions_config.py new file mode 100644 index 0000000..68daf00 --- /dev/null +++ b/backend/src/config/extensions_config.py @@ -0,0 +1,221 @@ +"""Unified extensions configuration for MCP servers and skills.""" + +import json +import os +from pathlib import Path +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field + + +class McpServerConfig(BaseModel): + """Configuration for a single MCP server.""" + + enabled: bool = Field(default=True, description="Whether this MCP server is enabled") + command: str = Field(..., description="Command to execute to start the MCP server") + args: list[str] = Field(default_factory=list, description="Arguments to pass to the command") + env: dict[str, str] = Field(default_factory=dict, description="Environment variables for the MCP server") + description: str = Field(default="", description="Human-readable description of what this MCP server provides") + model_config = ConfigDict(extra="allow") + + +class SkillStateConfig(BaseModel): + """Configuration for a single skill's state.""" + + enabled: bool = Field(default=True, description="Whether this skill is enabled") + + +class ExtensionsConfig(BaseModel): + """Unified configuration for MCP servers and skills.""" + + mcp_servers: dict[str, McpServerConfig] = Field( + default_factory=dict, + description="Map of MCP server name to configuration", + alias="mcpServers", + ) + skills: dict[str, SkillStateConfig] = Field( + default_factory=dict, + description="Map of skill name to state configuration", + ) + model_config = ConfigDict(extra="allow", populate_by_name=True) + + @classmethod + def resolve_config_path(cls, config_path: str | None = None) -> Path | None: + """Resolve the extensions config file path. + + Priority: + 1. If provided `config_path` argument, use it. + 2. If provided `DEER_FLOW_EXTENSIONS_CONFIG_PATH` environment variable, use it. + 3. Otherwise, check for `extensions_config.json` in the current directory, then in the parent directory. + 4. For backward compatibility, also check for `mcp_config.json` if `extensions_config.json` is not found. + 5. If not found, return None (extensions are optional). + + Args: + config_path: Optional path to extensions config file. + + Returns: + Path to the extensions config file if found, otherwise None. + """ + if config_path: + path = Path(config_path) + if not path.exists(): + raise FileNotFoundError(f"Extensions config file specified by param `config_path` not found at {path}") + return path + elif os.getenv("DEER_FLOW_EXTENSIONS_CONFIG_PATH"): + path = Path(os.getenv("DEER_FLOW_EXTENSIONS_CONFIG_PATH")) + if not path.exists(): + raise FileNotFoundError(f"Extensions config file specified by environment variable `DEER_FLOW_EXTENSIONS_CONFIG_PATH` not found at {path}") + return path + else: + # Check if the extensions_config.json is in the current directory + path = Path(os.getcwd()) / "extensions_config.json" + if path.exists(): + return path + + # Check if the extensions_config.json is in the parent directory of CWD + path = Path(os.getcwd()).parent / "extensions_config.json" + if path.exists(): + return path + + # Backward compatibility: check for mcp_config.json + path = Path(os.getcwd()) / "mcp_config.json" + if path.exists(): + return path + + path = Path(os.getcwd()).parent / "mcp_config.json" + if path.exists(): + return path + + # Extensions are optional, so return None if not found + return None + + @classmethod + def from_file(cls, config_path: str | None = None) -> "ExtensionsConfig": + """Load extensions config from JSON file. + + See `resolve_config_path` for more details. + + Args: + config_path: Path to the extensions config file. + + Returns: + ExtensionsConfig: The loaded config, or empty config if file not found. + """ + resolved_path = cls.resolve_config_path(config_path) + if resolved_path is None: + # Return empty config if extensions config file is not found + return cls(mcp_servers={}, skills={}) + + with open(resolved_path) as f: + config_data = json.load(f) + + cls.resolve_env_variables(config_data) + return cls.model_validate(config_data) + + @classmethod + def resolve_env_variables(cls, config: dict[str, Any]) -> dict[str, Any]: + """Recursively resolve environment variables in the config. + + Environment variables are resolved using the `os.getenv` function. Example: $OPENAI_API_KEY + + Args: + config: The config to resolve environment variables in. + + Returns: + The config with environment variables resolved. + """ + for key, value in config.items(): + if isinstance(value, str): + if value.startswith("$"): + env_value = os.getenv(value[1:], None) + if env_value is not None: + config[key] = env_value + else: + config[key] = value + elif isinstance(value, dict): + config[key] = cls.resolve_env_variables(value) + elif isinstance(value, list): + config[key] = [cls.resolve_env_variables(item) if isinstance(item, dict) else item for item in value] + return config + + def get_enabled_mcp_servers(self) -> dict[str, McpServerConfig]: + """Get only the enabled MCP servers. + + Returns: + Dictionary of enabled MCP servers. + """ + return {name: config for name, config in self.mcp_servers.items() if config.enabled} + + def is_skill_enabled(self, skill_name: str) -> bool: + """Check if a skill is enabled. + + Args: + skill_name: Name of the skill + + Returns: + True if enabled, False otherwise (default if not in config) + """ + skill_config = self.skills.get(skill_name) + if skill_config is None: + # Default to disable if not in config + return False + return skill_config.enabled + + +_extensions_config: ExtensionsConfig | None = None + + +def get_extensions_config() -> ExtensionsConfig: + """Get the extensions config instance. + + Returns a cached singleton instance. Use `reload_extensions_config()` to reload + from file, or `reset_extensions_config()` to clear the cache. + + Returns: + The cached ExtensionsConfig instance. + """ + global _extensions_config + if _extensions_config is None: + _extensions_config = ExtensionsConfig.from_file() + return _extensions_config + + +def reload_extensions_config(config_path: str | None = None) -> ExtensionsConfig: + """Reload the extensions config from file and update the cached instance. + + This is useful when the config file has been modified and you want + to pick up the changes without restarting the application. + + Args: + config_path: Optional path to extensions config file. If not provided, + uses the default resolution strategy. + + Returns: + The newly loaded ExtensionsConfig instance. + """ + global _extensions_config + _extensions_config = ExtensionsConfig.from_file(config_path) + return _extensions_config + + +def reset_extensions_config() -> None: + """Reset the cached extensions config instance. + + This clears the singleton cache, causing the next call to + `get_extensions_config()` to reload from file. Useful for testing + or when switching between different configurations. + """ + global _extensions_config + _extensions_config = None + + +def set_extensions_config(config: ExtensionsConfig) -> None: + """Set a custom extensions config instance. + + This allows injecting a custom or mock config for testing purposes. + + Args: + config: The ExtensionsConfig instance to use. + """ + global _extensions_config + _extensions_config = config diff --git a/backend/src/config/mcp_config.py b/backend/src/config/mcp_config.py deleted file mode 100644 index 37b8e17..0000000 --- a/backend/src/config/mcp_config.py +++ /dev/null @@ -1,186 +0,0 @@ -"""MCP (Model Context Protocol) configuration.""" - -import json -import os -from pathlib import Path -from typing import Any - -from pydantic import BaseModel, ConfigDict, Field - - -class McpServerConfig(BaseModel): - """Configuration for a single MCP server.""" - - enabled: bool = Field(default=True, description="Whether this MCP server is enabled") - command: str = Field(..., description="Command to execute to start the MCP server") - args: list[str] = Field(default_factory=list, description="Arguments to pass to the command") - env: dict[str, str] = Field(default_factory=dict, description="Environment variables for the MCP server") - description: str = Field(default="", description="Human-readable description of what this MCP server provides") - model_config = ConfigDict(extra="allow") - - -class McpConfig(BaseModel): - """Configuration for all MCP servers.""" - - mcp_servers: dict[str, McpServerConfig] = Field( - default_factory=dict, - description="Map of MCP server name to configuration", - alias="mcpServers", - ) - model_config = ConfigDict(extra="allow", populate_by_name=True) - - @classmethod - def resolve_config_path(cls, config_path: str | None = None) -> Path | None: - """Resolve the MCP config file path. - - Priority: - 1. If provided `config_path` argument, use it. - 2. If provided `DEER_FLOW_MCP_CONFIG_PATH` environment variable, use it. - 3. Otherwise, check for `mcp_config.json` in the current directory, then in the parent directory. - 4. If not found, return None (MCP is optional). - - Args: - config_path: Optional path to MCP config file. - - Returns: - Path to the MCP config file if found, otherwise None. - """ - if config_path: - path = Path(config_path) - if not path.exists(): - raise FileNotFoundError(f"MCP config file specified by param `config_path` not found at {path}") - return path - elif os.getenv("DEER_FLOW_MCP_CONFIG_PATH"): - path = Path(os.getenv("DEER_FLOW_MCP_CONFIG_PATH")) - if not path.exists(): - raise FileNotFoundError(f"MCP config file specified by environment variable `DEER_FLOW_MCP_CONFIG_PATH` not found at {path}") - return path - else: - # Check if the mcp_config.json is in the current directory - path = Path(os.getcwd()) / "mcp_config.json" - if path.exists(): - return path - - # Check if the mcp_config.json is in the parent directory of CWD - path = Path(os.getcwd()).parent / "mcp_config.json" - if path.exists(): - return path - - # MCP is optional, so return None if not found - return None - - @classmethod - def from_file(cls, config_path: str | None = None) -> "McpConfig": - """Load MCP config from JSON file. - - See `resolve_config_path` for more details. - - Args: - config_path: Path to the MCP config file. - - Returns: - McpConfig: The loaded config, or empty config if file not found. - """ - resolved_path = cls.resolve_config_path(config_path) - if resolved_path is None: - # Return empty config if MCP config file is not found - return cls(mcp_servers={}) - - with open(resolved_path) as f: - config_data = json.load(f) - - cls.resolve_env_variables(config_data) - return cls.model_validate(config_data) - - @classmethod - def resolve_env_variables(cls, config: dict[str, Any]) -> dict[str, Any]: - """Recursively resolve environment variables in the config. - - Environment variables are resolved using the `os.getenv` function. Example: $OPENAI_API_KEY - - Args: - config: The config to resolve environment variables in. - - Returns: - The config with environment variables resolved. - """ - for key, value in config.items(): - if isinstance(value, str): - if value.startswith("$"): - env_value = os.getenv(value[1:], None) - if env_value is not None: - config[key] = env_value - else: - config[key] = value - elif isinstance(value, dict): - config[key] = cls.resolve_env_variables(value) - elif isinstance(value, list): - config[key] = [cls.resolve_env_variables(item) if isinstance(item, dict) else item for item in value] - return config - - def get_enabled_servers(self) -> dict[str, McpServerConfig]: - """Get only the enabled MCP servers. - - Returns: - Dictionary of enabled MCP servers. - """ - return {name: config for name, config in self.mcp_servers.items() if config.enabled} - - -_mcp_config: McpConfig | None = None - - -def get_mcp_config() -> McpConfig: - """Get the MCP config instance. - - Returns a cached singleton instance. Use `reload_mcp_config()` to reload - from file, or `reset_mcp_config()` to clear the cache. - - Returns: - The cached McpConfig instance. - """ - global _mcp_config - if _mcp_config is None: - _mcp_config = McpConfig.from_file() - return _mcp_config - - -def reload_mcp_config(config_path: str | None = None) -> McpConfig: - """Reload the MCP config from file and update the cached instance. - - This is useful when the config file has been modified and you want - to pick up the changes without restarting the application. - - Args: - config_path: Optional path to MCP config file. If not provided, - uses the default resolution strategy. - - Returns: - The newly loaded McpConfig instance. - """ - global _mcp_config - _mcp_config = McpConfig.from_file(config_path) - return _mcp_config - - -def reset_mcp_config() -> None: - """Reset the cached MCP config instance. - - This clears the singleton cache, causing the next call to - `get_mcp_config()` to reload from file. Useful for testing - or when switching between different configurations. - """ - global _mcp_config - _mcp_config = None - - -def set_mcp_config(config: McpConfig) -> None: - """Set a custom MCP config instance. - - This allows injecting a custom or mock config for testing purposes. - - Args: - config: The McpConfig instance to use. - """ - global _mcp_config - _mcp_config = config diff --git a/backend/src/gateway/app.py b/backend/src/gateway/app.py index 5a32dac..26f3c46 100644 --- a/backend/src/gateway/app.py +++ b/backend/src/gateway/app.py @@ -5,7 +5,7 @@ from contextlib import asynccontextmanager from fastapi import FastAPI from src.gateway.config import get_gateway_config -from src.gateway.routers import artifacts, mcp, models +from src.gateway.routers import artifacts, mcp, models, skills # Configure logging logging.basicConfig( @@ -53,13 +53,14 @@ API Gateway for DeerFlow - A LangGraph-based AI agent backend with sandbox execu - **Models Management**: Query and retrieve available AI models - **MCP Configuration**: Manage Model Context Protocol (MCP) server configurations +- **Skills Management**: Query and manage skills and their enabled status - **Artifacts**: Access thread artifacts and generated files - **Health Monitoring**: System health check endpoints ### Architecture LangGraph requests are handled by nginx reverse proxy. -This gateway provides custom endpoints for models, MCP configuration, and artifacts. +This gateway provides custom endpoints for models, MCP configuration, skills, and artifacts. """, version="0.1.0", lifespan=lifespan, @@ -75,6 +76,10 @@ This gateway provides custom endpoints for models, MCP configuration, and artifa "name": "mcp", "description": "Manage Model Context Protocol (MCP) server configurations", }, + { + "name": "skills", + "description": "Manage skills and their configurations", + }, { "name": "artifacts", "description": "Access and download thread artifacts and generated files", @@ -95,6 +100,9 @@ This gateway provides custom endpoints for models, MCP configuration, and artifa # MCP API is mounted at /api/mcp app.include_router(mcp.router) + # Skills API is mounted at /api/skills + app.include_router(skills.router) + # Artifacts API is mounted at /api/threads/{thread_id}/artifacts app.include_router(artifacts.router) diff --git a/backend/src/gateway/routers/mcp.py b/backend/src/gateway/routers/mcp.py index 254856b..ca9c73f 100644 --- a/backend/src/gateway/routers/mcp.py +++ b/backend/src/gateway/routers/mcp.py @@ -5,7 +5,7 @@ from pathlib import Path from fastapi import APIRouter, HTTPException from pydantic import BaseModel, Field -from src.config.mcp_config import McpConfig, get_mcp_config, reload_mcp_config +from src.config.extensions_config import ExtensionsConfig, get_extensions_config, reload_extensions_config from src.mcp.cache import reset_mcp_tools_cache logger = logging.getLogger(__name__) @@ -67,7 +67,7 @@ async def get_mcp_configuration() -> McpConfigResponse: } ``` """ - config = get_mcp_config() + config = get_extensions_config() return McpConfigResponse( mcp_servers={name: McpServerConfigResponse(**server.model_dump()) for name, server in config.mcp_servers.items()} @@ -114,15 +114,21 @@ async def update_mcp_configuration(request: McpConfigUpdateRequest) -> McpConfig """ try: # Get the current config path (or determine where to save it) - config_path = McpConfig.resolve_config_path() + config_path = ExtensionsConfig.resolve_config_path() # If no config file exists, create one in the parent directory (project root) if config_path is None: - config_path = Path.cwd().parent / "mcp_config.json" - logger.info(f"No existing MCP config found. Creating new config at: {config_path}") + config_path = Path.cwd().parent / "extensions_config.json" + logger.info(f"No existing extensions config found. Creating new config at: {config_path}") + + # Load current config to preserve skills configuration + current_config = get_extensions_config() # Convert request to dict format for JSON serialization - config_data = {"mcpServers": {name: server.model_dump() for name, server in request.mcp_servers.items()}} + config_data = { + "mcpServers": {name: server.model_dump() for name, server in request.mcp_servers.items()}, + "skills": {name: {"enabled": skill.enabled} for name, skill in current_config.skills.items()}, + } # Write the configuration to file with open(config_path, "w") as f: @@ -131,14 +137,14 @@ async def update_mcp_configuration(request: McpConfigUpdateRequest) -> McpConfig logger.info(f"MCP configuration updated and saved to: {config_path}") # Reload the configuration to update the cache - reload_mcp_config() + reload_extensions_config() # Reset MCP tools cache so they will be reinitialized with new config on next use reset_mcp_tools_cache() logger.info("MCP tools cache reset - tools will be reinitialized on next use") # Return the updated configuration - reloaded_config = get_mcp_config() + reloaded_config = get_extensions_config() return McpConfigResponse( mcp_servers={name: McpServerConfigResponse(**server.model_dump()) for name, server in reloaded_config.mcp_servers.items()} ) diff --git a/backend/src/gateway/routers/skills.py b/backend/src/gateway/routers/skills.py new file mode 100644 index 0000000..8afedff --- /dev/null +++ b/backend/src/gateway/routers/skills.py @@ -0,0 +1,227 @@ +import json +import logging +from pathlib import Path + +from fastapi import APIRouter, HTTPException +from pydantic import BaseModel, Field + +from src.config.extensions_config import ExtensionsConfig, SkillStateConfig, get_extensions_config, reload_extensions_config +from src.skills import Skill, load_skills + +logger = logging.getLogger(__name__) +router = APIRouter(prefix="/api", tags=["skills"]) + + +class SkillResponse(BaseModel): + """Response model for skill information.""" + + name: str = Field(..., description="Name of the skill") + description: str = Field(..., description="Description of what the skill does") + license: str | None = Field(None, description="License information") + category: str = Field(..., description="Category of the skill (public or custom)") + enabled: bool = Field(default=True, description="Whether this skill is enabled") + + +class SkillsListResponse(BaseModel): + """Response model for listing all skills.""" + + skills: list[SkillResponse] + + +class SkillUpdateRequest(BaseModel): + """Request model for updating a skill.""" + + enabled: bool = Field(..., description="Whether to enable or disable the skill") + + +def _skill_to_response(skill: Skill) -> SkillResponse: + """Convert a Skill object to a SkillResponse.""" + return SkillResponse( + name=skill.name, + description=skill.description, + license=skill.license, + category=skill.category, + enabled=skill.enabled, + ) + + +@router.get( + "/skills", + response_model=SkillsListResponse, + summary="List All Skills", + description="Retrieve a list of all available skills from both public and custom directories.", +) +async def list_skills() -> SkillsListResponse: + """List all available skills. + + Returns all skills regardless of their enabled status. + + Returns: + A list of all skills with their metadata. + + Example Response: + ```json + { + "skills": [ + { + "name": "PDF Processing", + "description": "Extract and analyze PDF content", + "license": "MIT", + "category": "public", + "enabled": true + }, + { + "name": "Frontend Design", + "description": "Generate frontend designs and components", + "license": null, + "category": "custom", + "enabled": false + } + ] + } + ``` + """ + try: + # Load all skills (including disabled ones) + skills = load_skills(enabled_only=False) + return SkillsListResponse(skills=[_skill_to_response(skill) for skill in skills]) + except Exception as e: + logger.error(f"Failed to load skills: {e}", exc_info=True) + raise HTTPException(status_code=500, detail=f"Failed to load skills: {str(e)}") + + +@router.get( + "/skills/{skill_name}", + response_model=SkillResponse, + summary="Get Skill Details", + description="Retrieve detailed information about a specific skill by its name.", +) +async def get_skill(skill_name: str) -> SkillResponse: + """Get a specific skill by name. + + Args: + skill_name: The name of the skill to retrieve. + + Returns: + Skill information if found. + + Raises: + HTTPException: 404 if skill not found. + + Example Response: + ```json + { + "name": "PDF Processing", + "description": "Extract and analyze PDF content", + "license": "MIT", + "category": "public", + "enabled": true + } + ``` + """ + try: + skills = load_skills(enabled_only=False) + skill = next((s for s in skills if s.name == skill_name), None) + + if skill is None: + raise HTTPException(status_code=404, detail=f"Skill '{skill_name}' not found") + + return _skill_to_response(skill) + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to get skill {skill_name}: {e}", exc_info=True) + raise HTTPException(status_code=500, detail=f"Failed to get skill: {str(e)}") + + +@router.put( + "/skills/{skill_name}", + response_model=SkillResponse, + summary="Update Skill", + description="Update a skill's enabled status by modifying the skills_state_config.json file.", +) +async def update_skill(skill_name: str, request: SkillUpdateRequest) -> SkillResponse: + """Update a skill's enabled status. + + This will modify the skills_state_config.json file to update the enabled state. + The SKILL.md file itself is not modified. + + Args: + skill_name: The name of the skill to update. + request: The update request containing the new enabled status. + + Returns: + The updated skill information. + + Raises: + HTTPException: 404 if skill not found, 500 if update fails. + + Example Request: + ```json + { + "enabled": false + } + ``` + + Example Response: + ```json + { + "name": "PDF Processing", + "description": "Extract and analyze PDF content", + "license": "MIT", + "category": "public", + "enabled": false + } + ``` + """ + try: + # Find the skill to verify it exists + skills = load_skills(enabled_only=False) + skill = next((s for s in skills if s.name == skill_name), None) + + if skill is None: + raise HTTPException(status_code=404, detail=f"Skill '{skill_name}' not found") + + # Get or create config path + config_path = ExtensionsConfig.resolve_config_path() + if config_path is None: + # Create new config file in parent directory (project root) + config_path = Path.cwd().parent / "extensions_config.json" + logger.info(f"No existing extensions config found. Creating new config at: {config_path}") + + # Load current configuration + extensions_config = get_extensions_config() + + # Update the skill's enabled status + extensions_config.skills[skill_name] = SkillStateConfig(enabled=request.enabled) + + # Convert to JSON format (preserve MCP servers config) + config_data = { + "mcpServers": {name: server.model_dump() for name, server in extensions_config.mcp_servers.items()}, + "skills": {name: {"enabled": skill_config.enabled} for name, skill_config in extensions_config.skills.items()}, + } + + # Write the configuration to file + with open(config_path, "w") as f: + json.dump(config_data, f, indent=2) + + logger.info(f"Skills configuration updated and saved to: {config_path}") + + # Reload the configuration to update the cache + reload_extensions_config() + + # Reload the skills to get the updated status + skills = load_skills(enabled_only=False) + updated_skill = next((s for s in skills if s.name == skill_name), None) + + if updated_skill is None: + raise HTTPException(status_code=500, detail=f"Failed to reload skill '{skill_name}' after update") + + logger.info(f"Skill '{skill_name}' enabled status updated to {request.enabled}") + return _skill_to_response(updated_skill) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to update skill {skill_name}: {e}", exc_info=True) + raise HTTPException(status_code=500, detail=f"Failed to update skill: {str(e)}") diff --git a/backend/src/mcp/client.py b/backend/src/mcp/client.py index 3af0d03..dc110a0 100644 --- a/backend/src/mcp/client.py +++ b/backend/src/mcp/client.py @@ -3,7 +3,7 @@ import logging from typing import Any -from src.config.mcp_config import McpConfig, McpServerConfig +from src.config.extensions_config import ExtensionsConfig, McpServerConfig logger = logging.getLogger(__name__) @@ -31,16 +31,16 @@ def build_server_params(server_name: str, config: McpServerConfig) -> dict[str, return params -def build_servers_config(mcp_config: McpConfig) -> dict[str, dict[str, Any]]: +def build_servers_config(extensions_config: ExtensionsConfig) -> dict[str, dict[str, Any]]: """Build servers configuration for MultiServerMCPClient. Args: - mcp_config: MCP configuration containing all servers. + extensions_config: Extensions configuration containing all MCP servers. Returns: Dictionary mapping server names to their parameters. """ - enabled_servers = mcp_config.get_enabled_servers() + enabled_servers = extensions_config.get_enabled_mcp_servers() if not enabled_servers: logger.info("No enabled MCP servers found") diff --git a/backend/src/mcp/tools.py b/backend/src/mcp/tools.py index daf90dc..92035cb 100644 --- a/backend/src/mcp/tools.py +++ b/backend/src/mcp/tools.py @@ -4,7 +4,7 @@ import logging from langchain_core.tools import BaseTool -from src.config.mcp_config import get_mcp_config +from src.config.extensions_config import get_extensions_config from src.mcp.client import build_servers_config logger = logging.getLogger(__name__) @@ -22,8 +22,8 @@ async def get_mcp_tools() -> list[BaseTool]: logger.warning("langchain-mcp-adapters not installed. Install it to enable MCP tools: pip install langchain-mcp-adapters") return [] - mcp_config = get_mcp_config() - servers_config = build_servers_config(mcp_config) + extensions_config = get_extensions_config() + servers_config = build_servers_config(extensions_config) if not servers_config: logger.info("No enabled MCP servers configured") diff --git a/backend/src/skills/loader.py b/backend/src/skills/loader.py index e6bd8bb..16ae721 100644 --- a/backend/src/skills/loader.py +++ b/backend/src/skills/loader.py @@ -18,18 +18,19 @@ def get_skills_root_path() -> Path: return skills_dir -def load_skills(skills_path: Path | None = None, use_config: bool = True) -> list[Skill]: +def load_skills(skills_path: Path | None = None, use_config: bool = True, enabled_only: bool = False) -> list[Skill]: """ Load all skills from the skills directory. Scans both public and custom skill directories, parsing SKILL.md files - to extract metadata. + to extract metadata. The enabled state is determined by the skills_state_config.json file. Args: skills_path: Optional custom path to skills directory. If not provided and use_config is True, uses path from config. Otherwise defaults to deer-flow/skills use_config: Whether to load skills path from config (default: True) + enabled_only: If True, only return enabled skills (default: False) Returns: List of Skill objects, sorted by name @@ -71,6 +72,21 @@ def load_skills(skills_path: Path | None = None, use_config: bool = True) -> lis if skill: skills.append(skill) + # Load skills state configuration and update enabled status + try: + from src.config.extensions_config import get_extensions_config + + extensions_config = get_extensions_config() + for skill in skills: + skill.enabled = extensions_config.is_skill_enabled(skill.name) + except Exception as e: + # If config loading fails, default to all enabled + print(f"Warning: Failed to load extensions config: {e}") + + # Filter by enabled status if requested + if enabled_only: + skills = [skill for skill in skills if skill.enabled] + # Sort by name for consistent ordering skills.sort(key=lambda s: s.name) diff --git a/backend/src/skills/parser.py b/backend/src/skills/parser.py index 069cdf9..eb96c2a 100644 --- a/backend/src/skills/parser.py +++ b/backend/src/skills/parser.py @@ -56,6 +56,7 @@ def parse_skill_file(skill_file: Path, category: str) -> Skill | None: skill_dir=skill_file.parent, skill_file=skill_file, category=category, + enabled=True, # Default to enabled, actual state comes from config file ) except Exception as e: diff --git a/backend/src/skills/types.py b/backend/src/skills/types.py index 9d46654..a1f161d 100644 --- a/backend/src/skills/types.py +++ b/backend/src/skills/types.py @@ -12,6 +12,7 @@ class Skill: skill_dir: Path skill_file: Path category: str # 'public' or 'custom' + enabled: bool = False # Whether this skill is enabled @property def skill_path(self) -> str: diff --git a/backend/src/tools/tools.py b/backend/src/tools/tools.py index 0a1b0b8..19b3d2a 100644 --- a/backend/src/tools/tools.py +++ b/backend/src/tools/tools.py @@ -32,7 +32,7 @@ def get_available_tools(groups: list[str] | None = None, include_mcp: bool = Tru # Get cached MCP tools if enabled mcp_tools = [] - if include_mcp and config.mcp and config.mcp.get_enabled_servers(): + if include_mcp and config.extensions and config.extensions.get_enabled_mcp_servers(): try: from src.mcp.cache import get_cached_mcp_tools diff --git a/extensions_config.example.json b/extensions_config.example.json new file mode 100644 index 0000000..d4b2cf3 --- /dev/null +++ b/extensions_config.example.json @@ -0,0 +1,38 @@ +{ + "mcpServers": { + "filesystem": { + "enabled": true, + "command": "npx", + "args": ["-y", "@modelcontextprotocol/server-filesystem", "/path/to/allowed/files"], + "env": {}, + "description": "Provides filesystem access within allowed directories" + }, + "github": { + "enabled": true, + "command": "npx", + "args": ["-y", "@modelcontextprotocol/server-github"], + "env": { + "GITHUB_TOKEN": "$GITHUB_TOKEN" + }, + "description": "GitHub MCP server for repository operations" + }, + "postgres": { + "enabled": false, + "command": "npx", + "args": ["-y", "@modelcontextprotocol/server-postgres", "postgresql://localhost/mydb"], + "env": {}, + "description": "PostgreSQL database access" + } + }, + "skills": { + "PDF Processing": { + "enabled": true + }, + "Frontend Design": { + "enabled": true + }, + "Data Analysis": { + "enabled": false + } + } +} diff --git a/mcp_config.example.json b/mcp_config.example.json deleted file mode 100644 index f66bfc5..0000000 --- a/mcp_config.example.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "mcpServers": { - "filesystem": { - "enabled": true, - "command": "npx", - "args": ["-y", "@modelcontextprotocol/server-filesystem", "/path/to/allowed/files"], - "env": {}, - "description": "Provides file system access within specified directory" - }, - "postgres": { - "enabled": false, - "command": "npx", - "args": ["-y", "@modelcontextprotocol/server-postgres", "postgresql://localhost/mydb"], - "env": { - "PGPASSWORD": "$POSTGRES_PASSWORD" - }, - "description": "PostgreSQL database access" - }, - "github": { - "enabled": false, - "command": "npx", - "args": ["-y", "@modelcontextprotocol/server-github"], - "env": { - "GITHUB_PERSONAL_ACCESS_TOKEN": "$GITHUB_TOKEN" - }, - "description": "GitHub repository operations" - }, - "brave-search": { - "enabled": false, - "command": "npx", - "args": ["-y", "@modelcontextprotocol/server-brave-search"], - "env": { - "BRAVE_API_KEY": "$BRAVE_API_KEY" - }, - "description": "Brave Search API integration" - }, - "puppeteer": { - "enabled": false, - "command": "npx", - "args": ["-y", "@modelcontextprotocol/server-puppeteer"], - "env": {}, - "description": "Browser automation with Puppeteer" - }, - "custom-server": { - "enabled": false, - "command": "python", - "args": ["-m", "my_custom_mcp_server"], - "env": { - "API_KEY": "$CUSTOM_API_KEY" - }, - "description": "Custom MCP server implementation" - } - } -} diff --git a/nginx.conf b/nginx.conf index d3867f3..432bc1c 100644 --- a/nginx.conf +++ b/nginx.conf @@ -63,6 +63,16 @@ http { proxy_set_header X-Forwarded-Proto $scheme; } + # Custom API: Skills configuration endpoint + location /api/skills { + proxy_pass http://gateway; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + # Custom API: Artifacts endpoint location ~ ^/api/threads/[^/]+/artifacts { proxy_pass http://gateway; diff --git a/skills/public/pdf-processing/SKILL.md b/skills/public/pdf-processing/SKILL.md index f6a22dd..9746c93 100644 --- a/skills/public/pdf-processing/SKILL.md +++ b/skills/public/pdf-processing/SKILL.md @@ -1,5 +1,5 @@ --- -name: pdf +name: pdf-processing description: Comprehensive PDF manipulation toolkit for extracting text and tables, creating new PDFs, merging/splitting documents, and handling forms. When Claude needs to fill in a PDF form or programmatically process, generate, or analyze PDF documents at scale. license: Proprietary. LICENSE.txt has complete terms ---