Compare commits
2 Commits
ef9a071aa1
...
cea46914fe
| Author | SHA1 | Date |
|---|---|---|
 Titan
|
cea46914fe | |
|
Titan
|
c41475b552 |
|
|
@ -46,3 +46,4 @@ sandbox_image_cache.tar
|
||||||
|
|
||||||
# ignore the legacy `web` folder
|
# ignore the legacy `web` folder
|
||||||
web/
|
web/
|
||||||
|
memo.md
|
||||||
|
|
|
||||||
41
Makefile
41
Makefile
|
|
@ -255,8 +255,47 @@ docker-stop:
|
||||||
docker-logs:
|
docker-logs:
|
||||||
@./scripts/docker.sh logs
|
@./scripts/docker.sh logs
|
||||||
|
|
||||||
# View Docker development logs
|
|
||||||
docker-logs-frontend:
|
docker-logs-frontend:
|
||||||
@./scripts/docker.sh logs --frontend
|
@./scripts/docker.sh logs --frontend
|
||||||
docker-logs-gateway:
|
docker-logs-gateway:
|
||||||
@./scripts/docker.sh logs --gateway
|
@./scripts/docker.sh logs --gateway
|
||||||
|
|
||||||
|
# ==========================================
|
||||||
|
# Docker Publish Command
|
||||||
|
# ==========================================
|
||||||
|
# Usage: make docker-publish VERSION=v220.20251202 SERVICE=frontend
|
||||||
|
# Example: make docker-publish VERSION=v220.20251202 SERVICE=frontend
|
||||||
|
docker-publish:
|
||||||
|
@if [ -z "$(VERSION)" ]; then \
|
||||||
|
echo "✗ VERSION is required (e.g. v220.20251202)"; \
|
||||||
|
exit 1; \
|
||||||
|
fi
|
||||||
|
@if [ -z "$(SERVICE)" ]; then \
|
||||||
|
echo "✗ SERVICE is required (frontend, gateway, langgraph)"; \
|
||||||
|
exit 1; \
|
||||||
|
fi
|
||||||
|
@echo "=========================================="
|
||||||
|
@echo " Building Docker image for $(SERVICE)"
|
||||||
|
@echo "=========================================="
|
||||||
|
@IMAGE=registry.xueai.art/deerflow/deerflow-$(SERVICE):$(VERSION); \
|
||||||
|
DOCKERFILE=$$(case "$(SERVICE)" in \
|
||||||
|
frontend) echo "frontend/Dockerfile";; \
|
||||||
|
gateway) echo "backend/Dockerfile";; \
|
||||||
|
langgraph) echo "backend/Dockerfile";; \
|
||||||
|
*) echo "";; \
|
||||||
|
esac); \
|
||||||
|
if [ -z "$$DOCKERFILE" ]; then \
|
||||||
|
echo "✗ Unknown SERVICE: $(SERVICE)"; \
|
||||||
|
exit 1; \
|
||||||
|
fi; \
|
||||||
|
docker build -f $$DOCKERFILE -t $$IMAGE .; \
|
||||||
|
if [ $$? -ne 0 ]; then \
|
||||||
|
echo "✗ Docker build failed"; \
|
||||||
|
exit 1; \
|
||||||
|
fi; \
|
||||||
|
docker push $$IMAGE; \
|
||||||
|
if [ $$? -ne 0 ]; then \
|
||||||
|
echo "✗ Docker push failed"; \
|
||||||
|
exit 1; \
|
||||||
|
fi; \
|
||||||
|
echo "✓ Docker image $$IMAGE built and pushed successfully"
|
||||||
|
|
@ -0,0 +1,96 @@
|
||||||
|
# DeerFlow Production Environment
|
||||||
|
# Usage: docker-compose -f docker-compose-prod.yaml up -d
|
||||||
|
#
|
||||||
|
# Services:
|
||||||
|
# - nginx: Reverse proxy (port 2026)
|
||||||
|
# - frontend: Frontend Next.js (production build)
|
||||||
|
# - gateway: Backend Gateway API
|
||||||
|
# - langgraph: LangGraph server
|
||||||
|
#
|
||||||
|
# All services use pre-built images from registry.xueai.art/deerflow
|
||||||
|
|
||||||
|
version: '3.8'
|
||||||
|
|
||||||
|
services:
|
||||||
|
nginx:
|
||||||
|
image: nginx:alpine
|
||||||
|
container_name: deer-flow-nginx
|
||||||
|
ports:
|
||||||
|
- "2026:2026"
|
||||||
|
volumes:
|
||||||
|
- ./nginx/nginx.local.conf:/etc/nginx/nginx.conf:ro
|
||||||
|
depends_on:
|
||||||
|
- frontend
|
||||||
|
- gateway
|
||||||
|
- langgraph
|
||||||
|
networks:
|
||||||
|
- deer-flow-dev
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
frontend:
|
||||||
|
image: registry.xueai.art/deerflow/deerflow-frontend:${VERSION}
|
||||||
|
container_name: deer-flow-frontend
|
||||||
|
environment:
|
||||||
|
- NODE_ENV=production
|
||||||
|
env_file:
|
||||||
|
- ../frontend/.env
|
||||||
|
networks:
|
||||||
|
- deer-flow-dev
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
gateway:
|
||||||
|
image: registry.xueai.art/deerflow/deerflow-backend:${VERSION}
|
||||||
|
container_name: deer-flow-gateway
|
||||||
|
volumes:
|
||||||
|
- ../config.yaml:/app/config.yaml
|
||||||
|
- ../skills:/app/skills
|
||||||
|
- ../logs:/app/logs
|
||||||
|
- ../backend/.deer-flow:/app/backend/.deer-flow
|
||||||
|
# Mount uv cache for faster dependency installation
|
||||||
|
- ~/.cache/uv:/root/.cache/uv
|
||||||
|
# Mount Docker socket for aio sandbox
|
||||||
|
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||||
|
environment:
|
||||||
|
- CI=true
|
||||||
|
- DOCKER_HOST=unix:///var/run/docker.sock
|
||||||
|
env_file:
|
||||||
|
- ../.env
|
||||||
|
extra_hosts:
|
||||||
|
# For Linux: map host.docker.internal to host gateway
|
||||||
|
- "host.docker.internal:host-gateway"
|
||||||
|
networks:
|
||||||
|
- deer-flow-dev
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
langgraph:
|
||||||
|
image: registry.xueai.art/deerflow/deerflow-langgraph:${VERSION}
|
||||||
|
container_name: deer-flow-langgraph
|
||||||
|
volumes:
|
||||||
|
# Persist LangGraph inmem runtime data (threads/checkpoints/store)
|
||||||
|
- ../backend/.langgraph_api:/app/backend/.langgraph_api
|
||||||
|
- ../config.yaml:/app/config.yaml
|
||||||
|
- ../skills:/app/skills
|
||||||
|
- ../logs:/app/logs
|
||||||
|
- ../backend/.deer-flow:/app/backend/.deer-flow
|
||||||
|
# Mount uv cache for faster dependency installation
|
||||||
|
- ~/.cache/uv:/root/.cache/uv
|
||||||
|
# Mount Docker socket for aio sandbox
|
||||||
|
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||||
|
environment:
|
||||||
|
- CI=true
|
||||||
|
- DOCKER_HOST=unix:///var/run/docker.sock
|
||||||
|
env_file:
|
||||||
|
- ../.env
|
||||||
|
extra_hosts:
|
||||||
|
# For Linux: map host.docker.internal to host gateway
|
||||||
|
- "host.docker.internal:host-gateway"
|
||||||
|
networks:
|
||||||
|
- deer-flow-dev
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
networks:
|
||||||
|
deer-flow-dev:
|
||||||
|
driver: bridge
|
||||||
|
ipam:
|
||||||
|
config:
|
||||||
|
- subnet: 192.168.200.0/24
|
||||||
105
memo.md
105
memo.md
|
|
@ -1,27 +1,90 @@
|
||||||
|
# 当前改动总结（2026-03-09）
|
||||||
|
|
||||||
1. 网络连接问题
|
## TODO备忘：AIO sandbox端口分配并发竞态
|
||||||
local_backend.py中使用localhost访问sandbox容器
|
- 问题：sandbox容器启动时端口分配（get_free_port + docker run）非原子操作，存在并发竞态。
|
||||||
但在Docker容器内部，localhost指向容器自身，而不是主机
|
- 多个会话并发检测到端口空闲，可能同时尝试分配同一端口，导致后一个容器启动失败（端口已被占用）。
|
||||||
需要改为host.docker.internal
|
- 建议：
|
||||||
1. 修改网络配置
|
- 增加端口分配锁（如文件锁、Redis等），保证端口分配与容器启动原子性。
|
||||||
文件: backend/src/community/aio_sandbox/local_backend.py
|
- 或在容器启动失败后自动重试分配新端口。
|
||||||
|
- 适用于高并发场景，低并发下概率极低。
|
||||||
|
# 当前改动总结（2026-03-09）
|
||||||
|
|
||||||
第116行: 添加sandbox_host = "host.docker.internal"
|
## 7) 近期补充变更（2026-03-13）
|
||||||
第119行: 将sandbox_url=f"http://localhost:{port}"改为sandbox_url=f"http://{sandbox_host}:{port}"
|
- langgraph 会话持久化：
|
||||||
第166-167行: 同样修改了discover方法中的URL构建
|
- 支持 langgraph API 层会话落盘，重启后历史线程/消息可恢复。
|
||||||
|
- 通过 .langgraph_api 挂载和主进程 exec 启动，保证 SIGTERM 优雅关闭和持久化。
|
||||||
|
- skill 扫描范围扩展：
|
||||||
|
- skill 扫描目录新增 /mnt/user-data/uploads 路径，支持用户上传 skill-yaml、skill 文件自动纳入扫描。
|
||||||
|
- 兼容 skill-package.yaml、skill.zip 等多种格式，自动生成 skill 目录。
|
||||||
|
- 外部创建 langgraph 会话：
|
||||||
|
- 支持通过 API/外部服务创建 langgraph 会话，thread_id 可由外部指定。
|
||||||
|
- 前端/第三方系统可直接初始化会话并绑定 skill_id，实现多入口集成。
|
||||||
|
# 当前改动总结（2026-03-09）
|
||||||
|
|
||||||
|
## 6) Skill YAML 自动导入与远程初始化（2026-03-13）
|
||||||
|
- 目标：支持上传 skill-yaml.yaml，一键导入为 skill 目录，并支持 skill_id/languageType 参数自动远程初始化。
|
||||||
|
- 前端：
|
||||||
|
- 新增 materializeSkillYaml/ bootstrapRemoteSkill API，支持 skill-yaml 文件解析与远程内容拉取。
|
||||||
|
- 在 chats/[thread_id]/page.tsx 页面加载时自动触发 skill 初始化（有 skill_id 参数时），并用 useEffect/Ref 保证只初始化一次。
|
||||||
|
- 提交时增加空消息 guard，避免页面初始化时误触发 submit。
|
||||||
|
- 上传文件卡片支持 YAML 文件解析（materializeSkillYaml），但按钮默认注释，后续可按需开放。
|
||||||
|
- 初始化期间禁用输入框，UI 显示“正在初始化 Skill 文件...”或失败提示。
|
||||||
|
- 后端：
|
||||||
|
- gateway/config.py 增加 skill_content_api_url 配置，支持环境变量覆盖。
|
||||||
|
- routers/skills.py 新增 /api/skills/materialize-yaml 与 /api/skills/bootstrap-remote 两个 POST 接口，分别支持本地 YAML 解析与远程内容拉取+目录生成。
|
||||||
|
- skill_yaml_importer.py 增强解析器，支持 package.structure、path/name/children、root sentinel、别名等多种 YAML schema，兼容复杂 skill-package.yaml。
|
||||||
|
- 解析异常时返回详细错误，前端可捕获并提示。
|
||||||
|
- 流程说明：
|
||||||
|
1. 用户上传 skill-yaml.yaml，可在文件卡片触发“导入为 Skill 目录”API（materializeSkillYaml）。（前端已注释掉相关代码）
|
||||||
|
2. 页面有 skill_id/languageType 参数时，自动调用 bootstrapRemoteSkill，拉取远程 YAML 并生成目录。
|
||||||
|
3. skill 初始化总在 skill 扫描前完成，且只触发一次，支持新/旧线程。
|
||||||
|
4. 提交时空消息 guard，避免页面加载时误触发 submit。
|
||||||
|
- 文件：
|
||||||
|
- frontend/src/app/workspace/chats/[thread_id]/page.tsx
|
||||||
|
- frontend/src/core/skills/api.ts
|
||||||
|
- frontend/src/components/workspace/messages/message-list-item.tsx
|
||||||
|
- frontend/src/core/threads/hooks.ts
|
||||||
|
- backend/src/gateway/config.py
|
||||||
|
- backend/src/gateway/routers/skills.py
|
||||||
|
- backend/src/gateway/skill_yaml_importer.py
|
||||||
|
- 效果：skill-yaml 自动导入、远程 skill 初始化、页面加载触发、解析器兼容多 schema，前后端 API 完整闭环。
|
||||||
|
# 当前改动总结（2026-03-09）
|
||||||
|
|
||||||
|
|
||||||
2. Docker socket挂载问题
|
## 1) AIO sandbox 网络访问修复（已完成）
|
||||||
gateway和langgraph容器需要访问Docker守护进程来启动sandbox容器
|
- 问题：容器内访问 `localhost` 实际指向容器自身，无法访问宿主机上的 sandbox 端口。
|
||||||
但容器没有挂载Docker socket
|
- 调整：将 sandbox 访问地址改为 `host.docker.internal`。
|
||||||
2. 添加Docker socket挂载
|
- 文件：`backend/src/community/aio_sandbox/local_backend.py`
|
||||||
文件: docker/docker-compose-dev.yaml
|
- 影响：`create()` / `discover()` 走宿主机映射端口可达。
|
||||||
|
|
||||||
为gateway和langgraph服务添加：
|
## 2) AIO sandbox Docker 权限与连通性修复（已完成）
|
||||||
volumes:
|
- 问题：`gateway` / `langgraph` 容器无法直接访问 Docker daemon，无法管理 AIO sandbox 容器。
|
||||||
# Mount Docker socket for aio sandbox
|
- 调整：为两个服务挂载 Docker socket，并设置 `DOCKER_HOST`。
|
||||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
- 文件：`docker/docker-compose-dev.yaml`
|
||||||
|
- 关键配置：
|
||||||
|
- volume: `/var/run/docker.sock:/var/run/docker.sock:ro`
|
||||||
|
- env: `DOCKER_HOST=unix:///var/run/docker.sock`
|
||||||
|
|
||||||
environment:
|
## 3) 会话持久化最终生效方案（已验证）
|
||||||
# Docker environment for aio sandbox
|
- 目标：`make docker-stop && make docker-start` 后 `/workspace/chats` 保留历史线程。
|
||||||
- DOCKER_HOST=unix:///var/run/docker.sock
|
- 最终生效改动（都在 `langgraph` 服务）：
|
||||||
|
1. 启动命令改为 `exec uv run langgraph dev ... --no-reload ...`
|
||||||
|
2. 挂载持久化目录：`../backend/.langgraph_api:/app/backend/.langgraph_api`
|
||||||
|
- 文件：`docker/docker-compose-dev.yaml`
|
||||||
|
|
||||||
|
### 原理说明
|
||||||
|
- `.langgraph_api` 挂载：把 inmem runtime 的落盘文件映射到宿主机，容器重建后仍保留。
|
||||||
|
- `exec`：让 LangGraph 主进程直接接收 `SIGTERM`，触发优雅关闭与 `PersistentDict` 写盘。
|
||||||
|
- `--no-reload`：避免热重载多进程导致的停机时序问题，保证落盘稳定。
|
||||||
|
|
||||||
|
## 5) Docker下目录权限修复（已完成）
|
||||||
|
- 目标：确保 sandbox 容器/Pod 挂载的 `/mnt/user-data` 及其子目录可被非 root 用户正常读写，避免上传/写入失败。
|
||||||
|
- 文件：
|
||||||
|
- `backend/src/community/aio_sandbox/aio_sandbox_provider.py`
|
||||||
|
- `backend/src/community/aio_sandbox/local_backend.py`
|
||||||
|
- `docker/provisioner/app.py`
|
||||||
|
- 关键措施：
|
||||||
|
1. 启动 sandbox 时自动创建 thread 挂载目录并 `chmod 777`，解析为 host 路径，保证权限生效。
|
||||||
|
2. 容器启动后用 `docker exec` 执行 `mkdir -p` 和 `chmod 777`，多次重试，确保 `/mnt/user-data/uploads/workspace/outputs` 可写。
|
||||||
|
3. K8s Pod spec 新增 init container，以 root 权限初始化 `/mnt/user-data` 并 `chmod -R 777`，安全上下文限制特权。
|
||||||
|
- 效果：sandbox 挂载目录无论本地还是 K8s，均可被非 root 用户正常写入，上传/输出/工作区权限问题彻底解决。
|
||||||
Loading…
Reference in New Issue