From 31bf49917c63cf5340db0598e1672360079ceac9 Mon Sep 17 00:00:00 2001 From: hetao Date: Thu, 22 Jan 2026 11:57:47 +0800 Subject: [PATCH] feat: add unified development environment with nginx proxy Add a root-level Makefile to manage frontend, backend, and nginx services: - `make check` validates required dependencies (Node.js 22+, pnpm, uv, nginx) - `make install` installs all project dependencies - `make dev` starts all services with unified port 2026 - `make stop` and `make clean` for cleanup Update nginx configuration: - Change port from 8000 to 2026 - Add frontend upstream and routing (port 3000) - Add /api/langgraph/* routing with path rewriting to LangGraph server - Keep other /api/* routes to Gateway API - Route non-API requests to frontend Update frontend configuration: - Use relative URLs through nginx proxy by default - Support environment variables for direct backend access - Construct full URL for LangGraph SDK compatibility Clean up backend Makefile by removing nginx and serve targets. Co-Authored-By: Claude Sonnet 4.5 --- .gitignore | 1 + Makefile | 179 ++++++++++++++++++++++++++++++ README.md | 64 +++++++---- backend/CLAUDE.md | 104 ++++++++++++++++- backend/Makefile | 27 ----- frontend/.env.example | 7 +- frontend/src/core/config/index.ts | 11 +- nginx.conf | 52 ++++++--- 8 files changed, 376 insertions(+), 69 deletions(-) create mode 100644 Makefile diff --git a/.gitignore b/.gitignore index af4a393..234a7eb 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,4 @@ coverage/ .deer-flow/ .claude/ skills/custom/* +logs/ diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..cb38824 --- /dev/null +++ b/Makefile @@ -0,0 +1,179 @@ +# DeerFlow - Unified Development Environment + +.PHONY: help check install dev stop clean + +help: + @echo "DeerFlow Development Commands:" + @echo " make check - Check if all required tools are installed" + @echo " make install - Install all dependencies (frontend + backend)" + @echo " make dev - Start all services (frontend + backend + nginx on localhost:2026)" + @echo " make stop - Stop all running services" + @echo " make clean - Clean up processes and temporary files" + +# Check required tools +check: + @echo "==========================================" + @echo " Checking Required Dependencies" + @echo "==========================================" + @echo "" + @FAILED=0; \ + echo "Checking Node.js..."; \ + if command -v node >/dev/null 2>&1; then \ + NODE_VERSION=$$(node -v | sed 's/v//'); \ + NODE_MAJOR=$$(echo $$NODE_VERSION | cut -d. -f1); \ + if [ $$NODE_MAJOR -ge 22 ]; then \ + echo " ✓ Node.js $$NODE_VERSION (>= 22 required)"; \ + else \ + echo " ✗ Node.js $$NODE_VERSION found, but version 22+ is required"; \ + echo " Install from: https://nodejs.org/"; \ + FAILED=1; \ + fi; \ + else \ + echo " ✗ Node.js not found (version 22+ required)"; \ + echo " Install from: https://nodejs.org/"; \ + FAILED=1; \ + fi; \ + echo ""; \ + echo "Checking pnpm..."; \ + if command -v pnpm >/dev/null 2>&1; then \ + PNPM_VERSION=$$(pnpm -v); \ + echo " ✓ pnpm $$PNPM_VERSION"; \ + else \ + echo " ✗ pnpm not found"; \ + echo " Install: npm install -g pnpm"; \ + echo " Or visit: https://pnpm.io/installation"; \ + FAILED=1; \ + fi; \ + echo ""; \ + echo "Checking uv..."; \ + if command -v uv >/dev/null 2>&1; then \ + UV_VERSION=$$(uv --version | awk '{print $$2}'); \ + echo " ✓ uv $$UV_VERSION"; \ + else \ + echo " ✗ uv not found"; \ + echo " Install: curl -LsSf https://astral.sh/uv/install.sh | sh"; \ + echo " Or visit: https://docs.astral.sh/uv/getting-started/installation/"; \ + FAILED=1; \ + fi; \ + echo ""; \ + echo "Checking nginx..."; \ + if command -v nginx >/dev/null 2>&1; then \ + NGINX_VERSION=$$(nginx -v 2>&1 | awk -F'/' '{print $$2}'); \ + echo " ✓ nginx $$NGINX_VERSION"; \ + else \ + echo " ✗ nginx not found"; \ + echo " macOS: brew install nginx"; \ + echo " Ubuntu: sudo apt install nginx"; \ + echo " Or visit: https://nginx.org/en/download.html"; \ + FAILED=1; \ + fi; \ + echo ""; \ + if [ $$FAILED -eq 0 ]; then \ + echo "=========================================="; \ + echo " ✓ All dependencies are installed!"; \ + echo "=========================================="; \ + echo ""; \ + echo "You can now run:"; \ + echo " make install - Install project dependencies"; \ + echo " make dev - Start development server"; \ + else \ + echo "=========================================="; \ + echo " ✗ Some dependencies are missing"; \ + echo "=========================================="; \ + echo ""; \ + echo "Please install the missing tools and run 'make check' again."; \ + exit 1; \ + fi + +# Install all dependencies +install: + @echo "Installing backend dependencies..." + @cd backend && uv sync + @echo "Installing frontend dependencies..." + @cd frontend && pnpm install + @echo "✓ All dependencies installed" + +# Start all services +dev: + @echo "Stopping existing services if any..." + @-pkill -f "langgraph dev" 2>/dev/null || true + @-pkill -f "uvicorn src.gateway.app:app" 2>/dev/null || true + @-pkill -f "next dev" 2>/dev/null || true + @-nginx -c $(PWD)/nginx.conf -p $(PWD) -s quit 2>/dev/null || true + @sleep 1 + @-pkill -9 nginx 2>/dev/null || true + @sleep 1 + @echo "" + @echo "==========================================" + @echo " Starting DeerFlow Development Server" + @echo "==========================================" + @echo "" + @echo "Services starting up..." + @echo " → Backend: LangGraph + Gateway" + @echo " → Frontend: Next.js" + @echo " → Nginx: Reverse Proxy" + @echo "" + @cleanup() { \ + echo ""; \ + echo "Shutting down services..."; \ + pkill -f "langgraph dev" 2>/dev/null || true; \ + pkill -f "uvicorn src.gateway.app:app" 2>/dev/null || true; \ + pkill -f "next dev" 2>/dev/null || true; \ + nginx -c $(PWD)/nginx.conf -p $(PWD) -s quit 2>/dev/null || true; \ + sleep 1; \ + pkill -9 nginx 2>/dev/null || true; \ + echo "✓ All services stopped"; \ + exit 0; \ + }; \ + trap cleanup INT TERM; \ + echo "Starting LangGraph server..."; \ + cd backend && uv run langgraph dev --no-browser --allow-blocking --no-reload > ../logs/langgraph.log 2>&1 & \ + sleep 3; \ + echo "✓ LangGraph server started on localhost:2024"; \ + echo "Starting Gateway API..."; \ + cd backend && uv run uvicorn src.gateway.app:app --host 0.0.0.0 --port 8001 > ../logs/gateway.log 2>&1 & \ + sleep 2; \ + echo "✓ Gateway API started on localhost:8001"; \ + echo "Starting Frontend..."; \ + cd frontend && pnpm run dev > ../logs/frontend.log 2>&1 & \ + sleep 3; \ + echo "✓ Frontend started on localhost:3000"; \ + echo "Starting Nginx reverse proxy..."; \ + mkdir -p logs && nginx -g 'daemon off;' -c $(PWD)/nginx.conf -p $(PWD) > logs/nginx.log 2>&1 & \ + sleep 2; \ + echo "✓ Nginx started on localhost:2026"; \ + echo ""; \ + echo "=========================================="; \ + echo " DeerFlow is ready!"; \ + echo "=========================================="; \ + echo ""; \ + echo " 🌐 Application: http://localhost:2026"; \ + echo " 📡 API Gateway: http://localhost:2026/api/*"; \ + echo " 🤖 LangGraph: http://localhost:2026/api/langgraph/*"; \ + echo ""; \ + echo " 📋 Logs:"; \ + echo " - LangGraph: logs/langgraph.log"; \ + echo " - Gateway: logs/gateway.log"; \ + echo " - Frontend: logs/frontend.log"; \ + echo " - Nginx: logs/nginx.log"; \ + echo ""; \ + echo "Press Ctrl+C to stop all services"; \ + echo ""; \ + wait + +# Stop all services +stop: + @echo "Stopping all services..." + @-pkill -f "langgraph dev" 2>/dev/null || true + @-pkill -f "uvicorn src.gateway.app:app" 2>/dev/null || true + @-pkill -f "next dev" 2>/dev/null || true + @-nginx -c $(PWD)/nginx.conf -p $(PWD) -s quit 2>/dev/null || true + @sleep 1 + @-pkill -9 nginx 2>/dev/null || true + @echo "✓ All services stopped" + +# Clean up +clean: stop + @echo "Cleaning up..." + @-rm -rf logs/*.log 2>/dev/null || true + @echo "✓ Cleanup complete" diff --git a/README.md b/README.md index e5d1dff..0c2e9d4 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,17 @@ A LangGraph-based AI agent backend with sandbox execution capabilities. ## Quick Start -1. **Configure the application**: +1. **Check system requirements**: + ```bash + make check + ``` + This will verify that you have all required tools installed: + - Node.js 22+ + - pnpm + - uv (Python package manager) + - nginx + +2. **Configure the application**: ```bash # Copy example configuration cp config.example.yaml config.yaml @@ -15,49 +25,58 @@ A LangGraph-based AI agent backend with sandbox execution capabilities. export OPENAI_API_KEY="your-key-here" # or edit config.yaml directly - # Optional: Enable MCP servers for additional tools - cp mcp_config.example.json mcp_config.json - # Edit mcp_config.json to enable desired servers + # Optional: Enable MCP servers and skills + cp extensions_config.example.json extensions_config.json + # Edit extensions_config.json to enable desired MCP servers and skills ``` -2. **Install dependencies**: +3. **Install dependencies**: ```bash - cd backend make install ``` -3. **Run development server**: +4. **Run development server** (starts frontend, backend, and nginx): ```bash make dev ``` -### Production Deployment +5. **Access the application**: + - Web Interface: http://localhost:2026 + - All API requests are automatically proxied through nginx -For production environments, use nginx as a reverse proxy to route traffic between the gateway and LangGraph services: +### Manual Deployment + +If you need to start services individually: 1. **Start backend services**: ```bash - # Terminal 1: Start Gateway API (port 8001) + # Terminal 1: Start LangGraph Server (port 2024) cd backend - python -m src.gateway.app + make dev - # Terminal 2: Start LangGraph Server (port 2024) + # Terminal 2: Start Gateway API (port 8001) cd backend - langgraph up + make gateway + + # Terminal 3: Start Frontend (port 3000) + cd frontend + pnpm dev ``` 2. **Start nginx**: ```bash - nginx -c $(pwd)/nginx.conf + make nginx + # or directly: nginx -c $(pwd)/nginx.conf -g 'daemon off;' ``` 3. **Access the application**: - - Main API: http://localhost:8000 + - Web Interface: http://localhost:2026 The nginx configuration provides: -- Unified entry point on port 8000 -- Routes `/api/models`, `/api/threads/*/artifacts`, and `/health` to Gateway (8001) -- Routes all other requests to LangGraph (2024) +- Unified entry point on port 2026 +- Routes `/api/langgraph/*` to LangGraph Server (2024) +- Routes other `/api/*` endpoints to Gateway API (8001) +- Routes non-API requests to Frontend (3000) - Centralized CORS handling - SSE/streaming support for real-time agent responses - Optimized timeouts for long-running operations @@ -82,11 +101,12 @@ deer-flow/ ### Architecture ``` -Client +Browser ↓ -Nginx (port 8000) ← Unified entry point - ├→ Gateway API (port 8001) ← /api/models, /api/threads/*/artifacts, /health - └→ LangGraph Server (port 2024) ← All other requests (agent interactions) +Nginx (port 2026) ← Unified entry point + ├→ Frontend (port 3000) ← / (non-API requests) + ├→ Gateway API (port 8001) ← /api/models, /api/mcp, /api/skills, /api/threads/*/artifacts + └→ LangGraph Server (port 2024) ← /api/langgraph/* (agent interactions) ``` ## Documentation diff --git a/backend/CLAUDE.md b/backend/CLAUDE.md index 785bd97..b46a034 100644 --- a/backend/CLAUDE.md +++ b/backend/CLAUDE.md @@ -4,17 +4,65 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## Project Overview -DeerFlow is a LangGraph-based AI agent backend that provides a "super agent" with sandbox execution capabilities. The agent can execute code, browse the web, and manage files in isolated sandbox environments. +DeerFlow is a LangGraph-based AI agent system with a full-stack architecture. The backend provides a "super agent" with sandbox execution capabilities that can execute code, browse the web, and manage files in isolated environments. + +**Architecture**: +- **LangGraph Server** (port 2024): Agent runtime and workflow execution +- **Gateway API** (port 8001): REST API for models, MCP, skills, and artifacts +- **Frontend** (port 3000): Next.js web interface +- **Nginx** (port 2026): Unified reverse proxy entry point + +**Project Structure**: +``` +deer-flow/ +├── Makefile # Root commands (check, install, dev, stop) +├── nginx.conf # Nginx reverse proxy configuration +├── config.yaml # Main application configuration +├── extensions_config.json # MCP servers and skills configuration +├── backend/ # Backend application (this directory) +│ ├── Makefile # Backend-only commands (dev, gateway, lint) +│ ├── src/ +│ │ ├── agents/ # LangGraph agents and workflows +│ │ ├── gateway/ # FastAPI Gateway API +│ │ ├── sandbox/ # Sandbox execution system +│ │ ├── tools/ # Agent tools +│ │ ├── mcp/ # MCP integration +│ │ └── skills/ # Skills loading and management +│ └── langgraph.json # LangGraph server configuration +├── frontend/ # Next.js frontend application +└── skills/ # Agent skills directory + ├── public/ # Public skills (committed) + └── custom/ # Custom skills (gitignored) +``` ## Commands +**Root directory** (for full application): ```bash -# Install dependencies +# Check system requirements +make check + +# Install all dependencies (frontend + backend) make install -# Run development server (LangGraph Studio) +# Start all services (LangGraph + Gateway + Frontend + Nginx) make dev +# Stop all services +make stop +``` + +**Backend directory** (for backend development only): +```bash +# Install backend dependencies +make install + +# Run LangGraph server only (port 2024) +make dev + +# Run Gateway API only (port 8001) +make gateway + # Lint make lint @@ -45,6 +93,16 @@ Config values starting with `$` are resolved as environment variables (e.g., `$O ### Core Components +**Gateway API** (`src/gateway/`) +- FastAPI application that provides REST endpoints for frontend integration +- Endpoints: + - `/api/models` - List available LLM models from configuration + - `/api/mcp` - Manage MCP server configurations (GET, POST) + - `/api/skills` - Manage skill configurations (GET, POST) + - `/api/threads/{thread_id}/artifacts/*` - Serve agent-generated artifacts (files, images, etc.) +- Works alongside LangGraph server, handling non-agent HTTP operations +- Proxied through nginx under `/api/*` routes (except `/api/langgraph/*`) + **Agent Graph** (`src/agents/`) - `lead_agent` is the main entry point registered in `langgraph.json` - Uses `ThreadState` which extends `AgentState` with sandbox state @@ -147,6 +205,46 @@ Structure: Both MCP servers and skills can be modified at runtime via API endpoints. +## Development Workflow + +### Running the Full Application + +From the **project root** directory: +```bash +make dev +``` + +This starts all services and makes the application available at `http://localhost:2026`. + +**Nginx routing**: +- `/api/langgraph/*` → LangGraph Server (2024) - Agent interactions, threads, streaming +- `/api/*` (other) → Gateway API (8001) - Models, MCP, skills, artifacts +- `/` (non-API) → Frontend (3000) - Web interface + +### Running Backend Services Separately + +For backend-only development, from the **backend** directory: + +```bash +# Terminal 1: LangGraph server +make dev + +# Terminal 2: Gateway API +make gateway +``` + +Direct access (without nginx): +- LangGraph: `http://localhost:2024` +- Gateway: `http://localhost:8001` + +### Frontend Configuration + +The frontend uses environment variables to connect to backend services: +- `NEXT_PUBLIC_LANGGRAPH_BASE_URL` - Defaults to `/api/langgraph` (through nginx) +- `NEXT_PUBLIC_BACKEND_BASE_URL` - Defaults to empty string (through nginx) + +When using `make dev` from root, the frontend automatically connects through nginx. + ## Code Style - Uses `ruff` for linting and formatting diff --git a/backend/Makefile b/backend/Makefile index 2cc0e80..768b9a3 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -7,33 +7,6 @@ dev: gateway: uv run uvicorn src.gateway.app:app --host 0.0.0.0 --port 8001 -nginx: - @echo "Stopping existing nginx if any..." - @-nginx -c $(PWD)/../nginx.conf -p $(PWD)/.. -s quit 2>/dev/null || true - @sleep 1 - @-pkill -9 nginx 2>/dev/null || true - @sleep 1 - @echo "Starting nginx..." - @cleanup() { kill 0 2>/dev/null; exit 0; }; \ - trap cleanup EXIT INT TERM; \ - nginx -g 'daemon off;' -c $(PWD)/../nginx.conf -p $(PWD)/.. - -serve: - @echo "Stopping existing services if any..." - @-pkill -f "langgraph dev" 2>/dev/null || true - @-pkill -f "uvicorn src.gateway.app:app" 2>/dev/null || true - @-nginx -c $(PWD)/../nginx.conf -p $(PWD)/.. -s quit 2>/dev/null || true - @sleep 1 - @-pkill -9 nginx 2>/dev/null || true - @sleep 1 - @echo "Starting services..." - @cleanup() { kill 0 2>/dev/null; sleep 0.5; kill -9 0 2>/dev/null; exit 0; }; \ - trap cleanup INT TERM; \ - uv run langgraph dev --no-browser --allow-blocking --no-reload & \ - sleep 3 && uv run uvicorn src.gateway.app:app --host 0.0.0.0 --port 8001 & \ - sleep 1 && nginx -g 'daemon off;' -c $(PWD)/../nginx.conf -p $(PWD)/.. & \ - wait - lint: uvx ruff check . diff --git a/frontend/.env.example b/frontend/.env.example index f1d671f..e30fce3 100644 --- a/frontend/.env.example +++ b/frontend/.env.example @@ -9,8 +9,11 @@ # When adding additional environment variables, the schema in "/src/env.js" # should be updated accordingly. -# The base URL of DeerFlow's backend service -NEXT_PUBLIC_BACKEND_BASE_URL="http://localhost:8000" +# Backend API URLs (optional) +# Leave these commented out to use the default nginx proxy (recommended for `make dev`) +# Only set these if you need to connect to backend services directly +# NEXT_PUBLIC_BACKEND_BASE_URL="http://localhost:8001" +# NEXT_PUBLIC_LANGGRAPH_BASE_URL="http://localhost:2024" # Better Auth ## Better Auth - Google OAuth diff --git a/frontend/src/core/config/index.ts b/frontend/src/core/config/index.ts index c2cf885..8806fb9 100644 --- a/frontend/src/core/config/index.ts +++ b/frontend/src/core/config/index.ts @@ -4,7 +4,9 @@ export function getBackendBaseURL() { if (env.NEXT_PUBLIC_BACKEND_BASE_URL) { return env.NEXT_PUBLIC_BACKEND_BASE_URL; } else { - return "http://localhost:8000"; + // Use empty string for same-origin requests through nginx + // Paths like /api/models will be handled by nginx proxy + return ""; } } @@ -12,6 +14,11 @@ export function getLangGraphBaseURL() { if (env.NEXT_PUBLIC_LANGGRAPH_BASE_URL) { return env.NEXT_PUBLIC_LANGGRAPH_BASE_URL; } else { - return getBackendBaseURL(); + // LangGraph SDK requires a full URL, construct it from current origin + if (typeof window !== "undefined") { + return `${window.location.origin}/api/langgraph`; + } + // Fallback for SSR + return "http://localhost:2026/api/langgraph"; } } diff --git a/nginx.conf b/nginx.conf index 432bc1c..de30282 100644 --- a/nginx.conf +++ b/nginx.conf @@ -23,8 +23,12 @@ http { server localhost:2024; } + upstream frontend { + server localhost:3000; + } + server { - listen 8000; + listen 2026; server_name _; # Hide CORS headers from upstream to prevent duplicates @@ -43,6 +47,34 @@ http { return 204; } + # LangGraph API routes + # Rewrites /api/langgraph/* to /* before proxying + location /api/langgraph/ { + rewrite ^/api/langgraph/(.*) /$1 break; + proxy_pass http://langgraph; + proxy_http_version 1.1; + + # Headers + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Connection ''; + + # SSE/Streaming support + proxy_buffering off; + proxy_cache off; + proxy_set_header X-Accel-Buffering no; + + # Timeouts for long-running requests + proxy_connect_timeout 600s; + proxy_send_timeout 600s; + proxy_read_timeout 600s; + + # Chunked transfer encoding + chunked_transfer_encoding on; + } + # Custom API: Models endpoint location /api/models { proxy_pass http://gateway; @@ -123,9 +155,9 @@ http { proxy_set_header X-Forwarded-Proto $scheme; } - # All other requests go to LangGraph + # All other requests go to frontend location / { - proxy_pass http://langgraph; + proxy_pass http://frontend; proxy_http_version 1.1; # Headers @@ -133,20 +165,14 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header Connection ''; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection 'upgrade'; + proxy_cache_bypass $http_upgrade; - # SSE/Streaming support - proxy_buffering off; - proxy_cache off; - proxy_set_header X-Accel-Buffering no; - - # Timeouts for long-running requests + # Timeouts proxy_connect_timeout 600s; proxy_send_timeout 600s; proxy_read_timeout 600s; - - # Chunked transfer encoding - chunked_transfer_encoding on; } } }