diff --git a/.gitignore b/.gitignore index abfa399..4400f01 100644 --- a/.gitignore +++ b/.gitignore @@ -48,3 +48,6 @@ sandbox_image_cache.tar # ignore the legacy `web` folder web/ + +# Deployment artifacts +backend/Dockerfile.langgraph diff --git a/Makefile b/Makefile index b0368fc..99f36e7 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ # DeerFlow - Unified Development Environment -.PHONY: help config check install dev dev-daemon start stop clean docker-init docker-start docker-stop docker-logs docker-logs-frontend docker-logs-gateway +.PHONY: help config check install dev dev-daemon start stop up down clean docker-init docker-start docker-stop docker-logs docker-logs-frontend docker-logs-gateway help: @echo "DeerFlow Development Commands:" @@ -14,6 +14,10 @@ help: @echo " make stop - Stop all running services" @echo " make clean - Clean up processes and temporary files" @echo "" + @echo "Docker Production Commands:" + @echo " make up - Build and start production Docker services (localhost:2026)" + @echo " make down - Stop and remove production Docker containers" + @echo "" @echo "Docker Development Commands:" @echo " make docker-init - Build the custom k3s image (with pre-cached sandbox image)" @echo " make docker-start - Start Docker services (mode-aware from config.yaml, localhost:2026)" @@ -141,3 +145,15 @@ docker-logs-frontend: @./scripts/docker.sh logs --frontend docker-logs-gateway: @./scripts/docker.sh logs --gateway + +# ========================================== +# Production Docker Commands +# ========================================== + +# Build and start production services +up: + @./scripts/deploy.sh + +# Stop and remove production containers +down: + @./scripts/deploy.sh down diff --git a/README.md b/README.md index 54904c3..0932b68 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ DeerFlow has newly integrated the intelligent search and crawling toolset indepe - [Advanced](#advanced) - [Sandbox Mode](#sandbox-mode) - [MCP Server](#mcp-server) - - [IM Channels](#im-channels) + - [IM Channels](#im-channels) - [From Deep Research to Super Agent Harness](#from-deep-research-to-super-agent-harness) - [Core Features](#core-features) - [Skills \& Tools](#skills--tools) @@ -129,17 +129,26 @@ DeerFlow has newly integrated the intelligent search and crawling toolset indepe #### Option 1: Docker (Recommended) -The fastest way to get started with a consistent environment: +**Development** (hot-reload, source mounts): -1. **Initialize and start**: - ```bash - make docker-init # Pull sandbox image (Only once or when image updates) - make docker-start # Start services (auto-detects sandbox mode from config.yaml) - ``` +```bash +make docker-init # Pull sandbox image (only once or when image updates) +make docker-start # Start services (auto-detects sandbox mode from config.yaml) +``` - `make docker-start` now starts `provisioner` only when `config.yaml` uses provisioner mode (`sandbox.use: src.community.aio_sandbox:AioSandboxProvider` with `provisioner_url`). +`make docker-start` starts `provisioner` only when `config.yaml` uses provisioner mode (`sandbox.use: src.community.aio_sandbox:AioSandboxProvider` with `provisioner_url`). -2. **Access**: http://localhost:2026 +**Production** (builds images locally, mounts runtime config and data): + +```bash +make up # Build images and start all production services +make down # Stop and remove containers +``` + +> [!NOTE] +> The LangGraph agent server currently runs via `langgraph dev` (the open-source CLI server). + +Access: http://localhost:2026 See [CONTRIBUTING.md](CONTRIBUTING.md) for detailed Docker development guide. diff --git a/backend/Dockerfile b/backend/Dockerfile index 6b79b23..e3f2cfe 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,10 +1,19 @@ # Backend Development Dockerfile FROM python:3.12-slim -# Install system dependencies +ARG NODE_MAJOR=22 + +# Install system dependencies + Node.js (provides npx for MCP servers) RUN apt-get update && apt-get install -y \ curl \ build-essential \ + gnupg \ + ca-certificates \ + && mkdir -p /etc/apt/keyrings \ + && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key -o /etc/apt/keyrings/nodesource.gpg \ + && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_MAJOR}.x nodistro main" > /etc/apt/sources.list.d/nodesource.list \ + && apt-get update \ + && apt-get install -y nodejs \ && rm -rf /var/lib/apt/lists/* # Install Docker CLI (for DooD: allows starting sandbox containers via host Docker socket) diff --git a/backend/langgraph.json b/backend/langgraph.json index e34d0b9..980ba6b 100644 --- a/backend/langgraph.json +++ b/backend/langgraph.json @@ -1,5 +1,6 @@ { "$schema": "https://langgra.ph/schema.json", + "python_version": "3.12", "dependencies": [ "." ], diff --git a/backend/uv.lock b/backend/uv.lock index 55a3553..c06f8c4 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -3,8 +3,8 @@ revision = 3 requires-python = ">=3.12" resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", - "python_full_version == '3.13.*' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform != 'win32'", + "python_full_version == '3.13.*' and sys_platform == 'win32'", "python_full_version == '3.13.*' and sys_platform != 'win32'", "python_full_version < '3.13' and sys_platform == 'win32'", "python_full_version < '3.13' and sys_platform != 'win32'", diff --git a/docker/docker-compose-dev.yaml b/docker/docker-compose-dev.yaml index d53a85a..aae3889 100644 --- a/docker/docker-compose-dev.yaml +++ b/docker/docker-compose-dev.yaml @@ -80,6 +80,7 @@ services: build: context: ../ dockerfile: frontend/Dockerfile + target: dev args: PNPM_STORE_PATH: ${PNPM_STORE_PATH:-/root/.local/share/pnpm/store} container_name: deer-flow-frontend diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml new file mode 100644 index 0000000..abedcdc --- /dev/null +++ b/docker/docker-compose.yaml @@ -0,0 +1,156 @@ +# DeerFlow Production Environment +# Usage: make up +# +# Services: +# - nginx: Reverse proxy (port 2026, configurable via PORT env var) +# - frontend: Next.js production server +# - gateway: FastAPI Gateway API +# - langgraph: LangGraph production server (Dockerfile generated by langgraph dockerfile) +# - provisioner: (optional) Sandbox provisioner for Kubernetes mode +# +# Key environment variables (set via environment/.env or scripts/deploy.sh): +# DEER_FLOW_HOME — runtime data dir, default $REPO_ROOT/backend/.deer-flow +# DEER_FLOW_CONFIG_PATH — path to config.yaml +# DEER_FLOW_EXTENSIONS_CONFIG_PATH — path to extensions_config.json +# DEER_FLOW_DOCKER_SOCKET — Docker socket path, default /var/run/docker.sock +# DEER_FLOW_REPO_ROOT — repo root (used for skills host path in DooD) +# BETTER_AUTH_SECRET — required for frontend auth/session security +# +# LangSmith tracing is disabled by default (LANGCHAIN_TRACING_V2=false). +# Set LANGCHAIN_TRACING_V2=true and LANGSMITH_API_KEY in .env to enable it. +# +# Access: http://localhost:${PORT:-2026} + +services: + # ── Reverse Proxy ────────────────────────────────────────────────────────── + nginx: + image: nginx:alpine + container_name: deer-flow-nginx + ports: + - "${PORT:-2026}:2026" + volumes: + - ./nginx/${NGINX_CONF:-nginx.conf}:/etc/nginx/nginx.conf:ro + depends_on: + - frontend + - gateway + - langgraph + networks: + - deer-flow + restart: unless-stopped + + # ── Frontend: Next.js Production ─────────────────────────────────────────── + frontend: + build: + context: ../ + dockerfile: frontend/Dockerfile + target: prod + args: + PNPM_STORE_PATH: ${PNPM_STORE_PATH:-/root/.local/share/pnpm/store} + container_name: deer-flow-frontend + environment: + - BETTER_AUTH_SECRET=${BETTER_AUTH_SECRET} + env_file: + - ../frontend/.env + networks: + - deer-flow + restart: unless-stopped + + # ── Gateway API ──────────────────────────────────────────────────────────── + gateway: + build: + context: ../ + dockerfile: backend/Dockerfile + container_name: deer-flow-gateway + command: sh -c "cd backend && uv run uvicorn src.gateway.app:app --host 0.0.0.0 --port 8001 --workers 2" + volumes: + - ${DEER_FLOW_CONFIG_PATH}:/app/backend/config.yaml:ro + - ${DEER_FLOW_EXTENSIONS_CONFIG_PATH}:/app/backend/extensions_config.json:ro + - ../skills:/app/skills:ro + - ${DEER_FLOW_HOME}:/app/backend/.deer-flow + # DooD: AioSandboxProvider starts sandbox containers via host Docker daemon + - ${DEER_FLOW_DOCKER_SOCKET}:/var/run/docker.sock + working_dir: /app + environment: + - CI=true + - DEER_FLOW_HOME=/app/backend/.deer-flow + # DooD path/network translation + - DEER_FLOW_HOST_BASE_DIR=${DEER_FLOW_HOME} + - DEER_FLOW_HOST_SKILLS_PATH=${DEER_FLOW_REPO_ROOT}/skills + - DEER_FLOW_SANDBOX_HOST=host.docker.internal + env_file: + - ../.env + extra_hosts: + - "host.docker.internal:host-gateway" + networks: + - deer-flow + restart: unless-stopped + + # ── LangGraph Server ─────────────────────────────────────────────────────── + # TODO: switch to langchain/langgraph-api (licensed) once a license key is available. + # For now, use `langgraph dev` (no license required) with the standard backend image. + langgraph: + build: + context: ../ + dockerfile: backend/Dockerfile + container_name: deer-flow-langgraph + command: sh -c "cd /app/backend && uv run langgraph dev --no-browser --allow-blocking --no-reload --host 0.0.0.0 --port 2024" + volumes: + - ${DEER_FLOW_CONFIG_PATH}:/app/config.yaml:ro + - ${DEER_FLOW_EXTENSIONS_CONFIG_PATH}:/app/extensions_config.json:ro + - ${DEER_FLOW_HOME}:/app/backend/.deer-flow + - ../skills:/app/skills:ro + - ../backend/.langgraph_api:/app/backend/.langgraph_api + # DooD: same as gateway + - ${DEER_FLOW_DOCKER_SOCKET}:/var/run/docker.sock + environment: + - CI=true + - DEER_FLOW_HOME=/app/backend/.deer-flow + - DEER_FLOW_CONFIG_PATH=/app/config.yaml + - DEER_FLOW_EXTENSIONS_CONFIG_PATH=/app/extensions_config.json + - DEER_FLOW_HOST_BASE_DIR=${DEER_FLOW_HOME} + - DEER_FLOW_HOST_SKILLS_PATH=${DEER_FLOW_REPO_ROOT}/skills + - DEER_FLOW_SANDBOX_HOST=host.docker.internal + # Disable LangSmith tracing — LANGSMITH_API_KEY is not required. + # Set LANGCHAIN_TRACING_V2=true and LANGSMITH_API_KEY in .env to enable. + - LANGCHAIN_TRACING_V2=${LANGCHAIN_TRACING_V2:-false} + env_file: + - ../.env + extra_hosts: + - "host.docker.internal:host-gateway" + networks: + - deer-flow + restart: unless-stopped + + # ── Sandbox Provisioner (optional, Kubernetes mode) ──────────────────────── + provisioner: + profiles: + - provisioner + build: + context: ./provisioner + dockerfile: Dockerfile + container_name: deer-flow-provisioner + volumes: + - ~/.kube/config:/root/.kube/config:ro + environment: + - K8S_NAMESPACE=deer-flow + - SANDBOX_IMAGE=enterprise-public-cn-beijing.cr.volces.com/vefaas-public/all-in-one-sandbox:latest + - SKILLS_HOST_PATH=${DEER_FLOW_REPO_ROOT}/skills + - THREADS_HOST_PATH=${DEER_FLOW_HOME}/threads + - KUBECONFIG_PATH=/root/.kube/config + - NODE_HOST=host.docker.internal + - K8S_API_SERVER=https://host.docker.internal:26443 + env_file: + - ../.env + extra_hosts: + - "host.docker.internal:host-gateway" + networks: + - deer-flow + restart: unless-stopped + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8002/health"] + interval: 10s + timeout: 5s + retries: 6 +networks: + deer-flow: + driver: bridge diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 941945f..d6cec9d 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -1,22 +1,35 @@ -# Frontend Development Dockerfile -FROM node:22-alpine +# Frontend Dockerfile +# Supports two targets: +# --target dev — install deps only, run `pnpm dev` at container start +# --target prod — full build baked in, run `pnpm start` at container start (default if no --target is specified) -# Accept build argument for pnpm store path ARG PNPM_STORE_PATH=/root/.local/share/pnpm/store -# Install pnpm at specific version (matching package.json) +# ── Base: shared setup ──────────────────────────────────────────────────────── +FROM node:22-alpine AS base +ARG PNPM_STORE_PATH RUN corepack enable && corepack install -g pnpm@10.26.2 - RUN pnpm config set store-dir ${PNPM_STORE_PATH} - -# Set working directory WORKDIR /app - -# Copy frontend source code COPY frontend ./frontend -# Install dependencies -RUN sh -c "cd /app/frontend && pnpm install --frozen-lockfile" - -# Expose Next.js dev server port +# ── Dev: install only, CMD is overridden by docker-compose ─────────────────── +FROM base AS dev +RUN cd /app/frontend && pnpm install --frozen-lockfile EXPOSE 3000 + +# ── Builder: install + compile Next.js ─────────────────────────────────────── +FROM base AS builder +RUN cd /app/frontend && pnpm install --frozen-lockfile +# Skip env validation — runtime vars are injected by nginx/container +RUN cd /app/frontend && SKIP_ENV_VALIDATION=1 pnpm build + +# ── Prod: minimal runtime with pre-built output ─────────────────────────────── +FROM node:22-alpine AS prod +ARG PNPM_STORE_PATH +RUN corepack enable && corepack install -g pnpm@10.26.2 +RUN pnpm config set store-dir ${PNPM_STORE_PATH} +WORKDIR /app +COPY --from=builder /app/frontend ./frontend +EXPOSE 3000 +CMD ["sh", "-c", "cd /app/frontend && pnpm start"] diff --git a/scripts/deploy.sh b/scripts/deploy.sh new file mode 100755 index 0000000..86b0710 --- /dev/null +++ b/scripts/deploy.sh @@ -0,0 +1,212 @@ +#!/usr/bin/env bash +# +# deploy.sh - Build and start (or stop) DeerFlow production services +# +# Usage: +# deploy.sh [up] — build images and start containers (default) +# deploy.sh down — stop and remove containers +# +# Must be run from the repo root directory. + +set -e + +CMD="${1:-up}" + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO_ROOT" + +DOCKER_DIR="$REPO_ROOT/docker" +COMPOSE_CMD=(docker compose -p deer-flow -f "$DOCKER_DIR/docker-compose.yaml") + +# ── Colors ──────────────────────────────────────────────────────────────────── + +GREEN='\033[0;32m' +BLUE='\033[0;34m' +YELLOW='\033[1;33m' +RED='\033[0;31m' +NC='\033[0m' + +# ── DEER_FLOW_HOME ──────────────────────────────────────────────────────────── + +if [ -z "$DEER_FLOW_HOME" ]; then + export DEER_FLOW_HOME="$REPO_ROOT/backend/.deer-flow" +fi +echo -e "${BLUE}DEER_FLOW_HOME=$DEER_FLOW_HOME${NC}" +mkdir -p "$DEER_FLOW_HOME" + +# ── DEER_FLOW_REPO_ROOT (for skills host path in DooD) ─────────────────────── + +export DEER_FLOW_REPO_ROOT="$REPO_ROOT" + +# ── config.yaml ─────────────────────────────────────────────────────────────── + +if [ -z "$DEER_FLOW_CONFIG_PATH" ]; then + export DEER_FLOW_CONFIG_PATH="$REPO_ROOT/config.yaml" +fi + +if [ ! -f "$DEER_FLOW_CONFIG_PATH" ]; then + # Try to seed from repo (config.example.yaml is the canonical template) + if [ -f "$REPO_ROOT/config.example.yaml" ]; then + cp "$REPO_ROOT/config.example.yaml" "$DEER_FLOW_CONFIG_PATH" + echo -e "${GREEN}✓ Seeded config.example.yaml → $DEER_FLOW_CONFIG_PATH${NC}" + echo -e "${YELLOW}⚠ config.yaml was seeded from the example template.${NC}" + echo " Edit $DEER_FLOW_CONFIG_PATH and set your model API keys before use." + else + echo -e "${RED}✗ No config.yaml found.${NC}" + echo " Run 'make config' from the repo root to generate one," + echo " then set the required model API keys." + exit 1 + fi +else + echo -e "${GREEN}✓ config.yaml: $DEER_FLOW_CONFIG_PATH${NC}" +fi + +# ── extensions_config.json ─────────────────────────────────────────────────── + +if [ -z "$DEER_FLOW_EXTENSIONS_CONFIG_PATH" ]; then + export DEER_FLOW_EXTENSIONS_CONFIG_PATH="$REPO_ROOT/extensions_config.json" +fi + +if [ ! -f "$DEER_FLOW_EXTENSIONS_CONFIG_PATH" ]; then + if [ -f "$REPO_ROOT/extensions_config.json" ]; then + cp "$REPO_ROOT/extensions_config.json" "$DEER_FLOW_EXTENSIONS_CONFIG_PATH" + echo -e "${GREEN}✓ Seeded extensions_config.json → $DEER_FLOW_EXTENSIONS_CONFIG_PATH${NC}" + else + # Create a minimal empty config so the gateway doesn't fail on startup + echo '{"mcpServers":{},"skills":{}}' > "$DEER_FLOW_EXTENSIONS_CONFIG_PATH" + echo -e "${YELLOW}⚠ extensions_config.json not found, created empty config at $DEER_FLOW_EXTENSIONS_CONFIG_PATH${NC}" + fi +else + echo -e "${GREEN}✓ extensions_config.json: $DEER_FLOW_EXTENSIONS_CONFIG_PATH${NC}" +fi + + +# ── BETTER_AUTH_SECRET ─────────────────────────────────────────────────────── +# Required by Next.js in production. Generated once and persisted so auth +# sessions survive container restarts. + +_secret_file="$DEER_FLOW_HOME/.better-auth-secret" +if [ -z "$BETTER_AUTH_SECRET" ]; then + if [ -f "$_secret_file" ]; then + export BETTER_AUTH_SECRET + BETTER_AUTH_SECRET="$(cat "$_secret_file")" + echo -e "${GREEN}✓ BETTER_AUTH_SECRET loaded from $_secret_file${NC}" + else + export BETTER_AUTH_SECRET + BETTER_AUTH_SECRET="$(python3 -c 'import secrets; print(secrets.token_hex(32))')" + echo "$BETTER_AUTH_SECRET" > "$_secret_file" + chmod 600 "$_secret_file" + echo -e "${GREEN}✓ BETTER_AUTH_SECRET generated → $_secret_file${NC}" + fi +fi + +# ── detect_sandbox_mode ─────────────────────────────────────────────────────── + +detect_sandbox_mode() { + local sandbox_use="" + local provisioner_url="" + + [ -f "$DEER_FLOW_CONFIG_PATH" ] || { echo "local"; return; } + + sandbox_use=$(awk ' + /^[[:space:]]*sandbox:[[:space:]]*$/ { in_sandbox=1; next } + in_sandbox && /^[^[:space:]#]/ { in_sandbox=0 } + in_sandbox && /^[[:space:]]*use:[[:space:]]*/ { + line=$0; sub(/^[[:space:]]*use:[[:space:]]*/, "", line); print line; exit + } + ' "$DEER_FLOW_CONFIG_PATH") + + provisioner_url=$(awk ' + /^[[:space:]]*sandbox:[[:space:]]*$/ { in_sandbox=1; next } + in_sandbox && /^[^[:space:]#]/ { in_sandbox=0 } + in_sandbox && /^[[:space:]]*provisioner_url:[[:space:]]*/ { + line=$0; sub(/^[[:space:]]*provisioner_url:[[:space:]]*/, "", line); print line; exit + } + ' "$DEER_FLOW_CONFIG_PATH") + + if [[ "$sandbox_use" == *"src.community.aio_sandbox:AioSandboxProvider"* ]]; then + if [ -n "$provisioner_url" ]; then + echo "provisioner" + else + echo "aio" + fi + else + echo "local" + fi +} + +# ── down ────────────────────────────────────────────────────────────────────── + +if [ "$CMD" = "down" ]; then + # Set minimal env var defaults so docker compose can parse the file without + # warning about unset variables that appear in volume specs. + export DEER_FLOW_HOME="${DEER_FLOW_HOME:-$REPO_ROOT/backend/.deer-flow}" + export DEER_FLOW_CONFIG_PATH="${DEER_FLOW_CONFIG_PATH:-$DEER_FLOW_HOME/config.yaml}" + export DEER_FLOW_EXTENSIONS_CONFIG_PATH="${DEER_FLOW_EXTENSIONS_CONFIG_PATH:-$DEER_FLOW_HOME/extensions_config.json}" + export DEER_FLOW_DOCKER_SOCKET="${DEER_FLOW_DOCKER_SOCKET:-/var/run/docker.sock}" + export DEER_FLOW_REPO_ROOT="${DEER_FLOW_REPO_ROOT:-$REPO_ROOT}" + export BETTER_AUTH_SECRET="${BETTER_AUTH_SECRET:-placeholder}" + "${COMPOSE_CMD[@]}" down + exit 0 +fi + +# ── Banner ──────────────────────────────────────────────────────────────────── + +echo "==========================================" +echo " DeerFlow Production Deployment" +echo "==========================================" +echo "" + +# ── Step 1: Detect sandbox mode ────────────────────────────────────────────── + +sandbox_mode="$(detect_sandbox_mode)" +echo -e "${BLUE}Sandbox mode: $sandbox_mode${NC}" + +if [ "$sandbox_mode" = "provisioner" ]; then + services="" + extra_args="--profile provisioner" +else + services="frontend gateway langgraph nginx" + extra_args="" +fi + + +# ── DEER_FLOW_DOCKER_SOCKET ─────────────────────────────────────────────────── + +if [ -z "$DEER_FLOW_DOCKER_SOCKET" ]; then + export DEER_FLOW_DOCKER_SOCKET="/var/run/docker.sock" +fi + +if [ "$sandbox_mode" != "local" ]; then + if [ ! -S "$DEER_FLOW_DOCKER_SOCKET" ]; then + echo -e "${RED}⚠ Docker socket not found at $DEER_FLOW_DOCKER_SOCKET${NC}" + echo " AioSandboxProvider (DooD) will not work." + exit 1 + else + echo -e "${GREEN}✓ Docker socket: $DEER_FLOW_DOCKER_SOCKET${NC}" + fi +fi + +echo "" + +# ── Step 2: Build and start ─────────────────────────────────────────────────── + +echo "Building images and starting containers..." +echo "" + +# shellcheck disable=SC2086 +"${COMPOSE_CMD[@]}" $extra_args up --build -d --remove-orphans $services + +echo "" +echo "==========================================" +echo " DeerFlow is running!" +echo "==========================================" +echo "" +echo " 🌐 Application: http://localhost:${PORT:-2026}" +echo " 📡 API Gateway: http://localhost:${PORT:-2026}/api/*" +echo " 🤖 LangGraph: http://localhost:${PORT:-2026}/api/langgraph/*" +echo "" +echo " Manage:" +echo " make down — stop and remove containers" +echo " make docker-logs — view logs" +echo ""