diff --git a/backend/docs/GUARDRAILS.md b/backend/docs/GUARDRAILS.md index 81fc4be4..86f0e0f2 100644 --- a/backend/docs/GUARDRAILS.md +++ b/backend/docs/GUARDRAILS.md @@ -112,6 +112,34 @@ guardrails: 3. Ask the agent: "Use bash to run echo hello" 4. The agent sees: `Guardrail denied: tool 'bash' was blocked (oap.tool_not_allowed)` +### Option 1.5: Built-in SensitiveDataProvider (Strict Secret Blocking) + +For secret-leak prevention, DeerFlow also ships `SensitiveDataProvider`, which +blocks tool calls targeting sensitive file patterns (for example `.env`, +`.env.*`, `*.pem`, `*.key`, `id_rsa*`, `secrets.*`, `credentials.*`). + +This provider is strict: it blocks matching access even when the user +explicitly asks the agent to reveal those files. + +**config.yaml:** +```yaml +guardrails: + enabled: true + fail_closed: true + provider: + use: deerflow.guardrails.builtin:SensitiveDataProvider + config: + protected_tools: ["read_file", "write_file", "str_replace", "ls", "glob", "grep", "bash"] + deny_basenames: [".env"] + deny_globs: [".env.*", "*.pem", "*.key", "id_rsa*", "secrets.*", "credentials.*"] + block_skills_env: true +``` + +**Behavior summary:** +- `read_file / ls / glob / grep / bash` attempting sensitive path access are denied +- `write_file / str_replace` touching sensitive targets are denied +- Denials are logged as structured audit events (tool/reason/thread/timestamp) + ### Option 2: OAP Passport Provider (Policy-Based) For policy enforcement based on the [Open Agent Passport (OAP)](https://github.com/aporthq/aport-spec) open standard. An OAP passport is a JSON document that declares an agent's identity, capabilities, and operational limits. Any provider that reads an OAP passport and returns OAP-compliant decisions works with DeerFlow. diff --git a/backend/packages/harness/deerflow/agents/middlewares/sensitive_output_redaction_middleware.py b/backend/packages/harness/deerflow/agents/middlewares/sensitive_output_redaction_middleware.py new file mode 100644 index 00000000..88069669 --- /dev/null +++ b/backend/packages/harness/deerflow/agents/middlewares/sensitive_output_redaction_middleware.py @@ -0,0 +1,64 @@ +"""Redact sensitive values from tool outputs before they re-enter the model context.""" + +from __future__ import annotations + +import re +from collections.abc import Awaitable, Callable +from typing import override + +from langchain.agents import AgentState +from langchain.agents.middleware import AgentMiddleware +from langchain_core.messages import ToolMessage +from langgraph.prebuilt.tool_node import ToolCallRequest +from langgraph.types import Command + +_SIMPLE_REDACTION = "[REDACTED]" + +_PATTERNS = [ + re.compile(r"(?im)\b([A-Z][A-Z0-9_]{1,64})\s*=\s*([^\s\"'`]{6,})"), + re.compile(r"(?i)\b(bearer\s+)[A-Za-z0-9._\-+/=]{8,}"), + re.compile(r"(?i)\b(sk-[A-Za-z0-9]{12,})\b"), + re.compile(r"(?i)\b(eyJ[A-Za-z0-9_\-]{10,}\.[A-Za-z0-9_\-]{10,}\.[A-Za-z0-9_\-]{10,})\b"), + re.compile(r"(?is)-----BEGIN [A-Z ]*PRIVATE KEY-----.*?-----END [A-Z ]*PRIVATE KEY-----"), +] + + +def _redact_text(text: str) -> str: + value = text + # Preserve var name for KEY=VALUE style output. + value = _PATTERNS[0].sub(lambda m: f"{m.group(1)}={_SIMPLE_REDACTION}", value) + value = _PATTERNS[1].sub(lambda m: f"{m.group(1)}{_SIMPLE_REDACTION}", value) + for pattern in _PATTERNS[2:]: + value = pattern.sub(_SIMPLE_REDACTION, value) + return value + + +class SensitiveOutputRedactionMiddleware(AgentMiddleware[AgentState]): + """Redact secrets from tool outputs.""" + + def _redact_tool_result(self, result: ToolMessage | Command) -> ToolMessage | Command: + if not isinstance(result, ToolMessage): + return result + if isinstance(result.content, str): + redacted = _redact_text(result.content) + if redacted != result.content: + return ToolMessage(content=redacted, tool_call_id=result.tool_call_id, name=result.name, status=result.status) + return result + return result + + @override + def wrap_tool_call( + self, + request: ToolCallRequest, + handler: Callable[[ToolCallRequest], ToolMessage | Command], + ) -> ToolMessage | Command: + return self._redact_tool_result(handler(request)) + + @override + async def awrap_tool_call( + self, + request: ToolCallRequest, + handler: Callable[[ToolCallRequest], Awaitable[ToolMessage | Command]], + ) -> ToolMessage | Command: + return self._redact_tool_result(await handler(request)) + diff --git a/backend/packages/harness/deerflow/agents/middlewares/tool_error_handling_middleware.py b/backend/packages/harness/deerflow/agents/middlewares/tool_error_handling_middleware.py index 096e6dac..66992204 100644 --- a/backend/packages/harness/deerflow/agents/middlewares/tool_error_handling_middleware.py +++ b/backend/packages/harness/deerflow/agents/middlewares/tool_error_handling_middleware.py @@ -127,8 +127,10 @@ def _build_runtime_middlewares( middlewares.append(GuardrailMiddleware(provider, fail_closed=guardrails_config.fail_closed, passport=guardrails_config.passport)) from deerflow.agents.middlewares.sandbox_audit_middleware import SandboxAuditMiddleware + from deerflow.agents.middlewares.sensitive_output_redaction_middleware import SensitiveOutputRedactionMiddleware middlewares.append(SandboxAuditMiddleware()) + middlewares.append(SensitiveOutputRedactionMiddleware()) middlewares.append(ToolErrorHandlingMiddleware()) return middlewares diff --git a/backend/packages/harness/deerflow/guardrails/__init__.py b/backend/packages/harness/deerflow/guardrails/__init__.py index 3c23cd09..de92d28f 100644 --- a/backend/packages/harness/deerflow/guardrails/__init__.py +++ b/backend/packages/harness/deerflow/guardrails/__init__.py @@ -1,11 +1,12 @@ """Pre-tool-call authorization middleware.""" -from deerflow.guardrails.builtin import AllowlistProvider +from deerflow.guardrails.builtin import AllowlistProvider, SensitiveDataProvider from deerflow.guardrails.middleware import GuardrailMiddleware from deerflow.guardrails.provider import GuardrailDecision, GuardrailProvider, GuardrailReason, GuardrailRequest __all__ = [ "AllowlistProvider", + "SensitiveDataProvider", "GuardrailDecision", "GuardrailMiddleware", "GuardrailProvider", diff --git a/backend/packages/harness/deerflow/guardrails/builtin.py b/backend/packages/harness/deerflow/guardrails/builtin.py index 53ce9f8d..6fceb4d4 100644 --- a/backend/packages/harness/deerflow/guardrails/builtin.py +++ b/backend/packages/harness/deerflow/guardrails/builtin.py @@ -1,7 +1,20 @@ """Built-in guardrail providers that ship with DeerFlow.""" +from __future__ import annotations + +import fnmatch +import json +import logging +import re +import shlex +from datetime import UTC, datetime +from pathlib import PurePosixPath +from typing import Any + from deerflow.guardrails.provider import GuardrailDecision, GuardrailReason, GuardrailRequest +logger = logging.getLogger(__name__) + class AllowlistProvider: """Simple allowlist/denylist provider. No external dependencies.""" @@ -21,3 +34,138 @@ class AllowlistProvider: async def aevaluate(self, request: GuardrailRequest) -> GuardrailDecision: return self.evaluate(request) + + +class SensitiveDataProvider: + """Block tool calls that may access sensitive files such as .env and keys.""" + + name = "sensitive-data" + + _DEFAULT_PROTECTED_TOOLS = { + "read_file", + "write_file", + "str_replace", + "ls", + "glob", + "grep", + "bash", + } + _DEFAULT_DENY_BASENAMES = {".env"} + _DEFAULT_DENY_GLOBS = { + ".env.*", + "*.pem", + "*.key", + "id_rsa*", + "secrets.*", + "credentials.*", + } + + def __init__( + self, + *, + protected_tools: list[str] | None = None, + deny_basenames: list[str] | None = None, + deny_globs: list[str] | None = None, + block_skills_env: bool = True, + **_: Any, + ): + self._protected_tools = {t.lower() for t in (protected_tools or list(self._DEFAULT_PROTECTED_TOOLS))} + self._deny_basenames = {n.lower() for n in (deny_basenames or list(self._DEFAULT_DENY_BASENAMES))} + self._deny_globs = {p.lower() for p in (deny_globs or list(self._DEFAULT_DENY_GLOBS))} + self._block_skills_env = block_skills_env + + def _normalize_candidate(self, raw: str | None) -> str: + if not raw: + return "" + return str(raw).strip().strip("\"'") + + def _looks_sensitive_path(self, raw_path: str) -> bool: + value = self._normalize_candidate(raw_path) + if not value: + return False + lowered = value.lower() + if self._block_skills_env and "/mnt/skills/" in lowered: + basename = PurePosixPath(lowered).name + if basename == ".env" or basename.startswith(".env."): + return True + basename = PurePosixPath(lowered).name + if basename in self._deny_basenames: + return True + return any(fnmatch.fnmatch(basename, pat) for pat in self._deny_globs) + + def _extract_bash_candidates(self, command: str) -> list[str]: + candidates: list[str] = [] + if not command: + return candidates + try: + tokens = shlex.split(command) + except ValueError: + tokens = command.split() + for token in tokens: + t = token.strip() + if not t: + continue + # Path-like tokens + if "/" in t or t.startswith("."): + candidates.append(t) + # file.env style arguments may not contain slash + if t.lower().startswith(".env"): + candidates.append(t) + return candidates + + def _collect_candidates(self, request: GuardrailRequest) -> list[str]: + args = request.tool_input if isinstance(request.tool_input, dict) else {} + tool = request.tool_name + candidates: list[str] = [] + if tool in {"read_file", "write_file", "str_replace", "ls"}: + path = args.get("path") + if isinstance(path, str): + candidates.append(path) + elif tool in {"glob", "grep"}: + path = args.get("path") + if isinstance(path, str): + candidates.append(path) + glob_pat = args.get("glob") + if isinstance(glob_pat, str): + candidates.append(glob_pat) + elif tool == "bash": + command = str(args.get("command") or "") + candidates.extend(self._extract_bash_candidates(command)) + # Fast-path for common secret exposure commands + if re.search(r"\b(printenv|env)\b", command, flags=re.IGNORECASE): + candidates.append(".env") + return candidates + + def _audit(self, request: GuardrailRequest, decision: GuardrailDecision) -> None: + if decision.allow: + return + code = decision.reasons[0].code if decision.reasons else "oap.blocked_pattern" + rec = { + "timestamp": datetime.now(UTC).isoformat(), + "provider": self.name, + "tool_name": request.tool_name, + "reason_code": code, + "thread_id": request.thread_id, + "agent_id": request.agent_id, + } + logger.warning("[SensitiveDataGuardrail] %s", json.dumps(rec, ensure_ascii=False)) + + def evaluate(self, request: GuardrailRequest) -> GuardrailDecision: + tool = (request.tool_name or "").lower() + if tool not in self._protected_tools: + return GuardrailDecision(allow=True, reasons=[GuardrailReason(code="oap.allowed")]) + + candidates = self._collect_candidates(request) + if any(self._looks_sensitive_path(c) for c in candidates): + decision = GuardrailDecision( + allow=False, + reasons=[GuardrailReason(code="oap.blocked_pattern", message="sensitive path access is blocked by policy")], + policy_id="sensitive-data.v1", + ) + self._audit(request, decision) + return decision + + return GuardrailDecision(allow=True, reasons=[GuardrailReason(code="oap.allowed")]) + + async def aevaluate(self, request: GuardrailRequest) -> GuardrailDecision: + return self.evaluate(request) diff --git a/backend/tests/test_guardrail_middleware.py b/backend/tests/test_guardrail_middleware.py index 5c021ba4..f204c56a 100644 --- a/backend/tests/test_guardrail_middleware.py +++ b/backend/tests/test_guardrail_middleware.py @@ -8,7 +8,7 @@ from unittest.mock import MagicMock import pytest from langgraph.errors import GraphBubbleUp -from deerflow.guardrails.builtin import AllowlistProvider +from deerflow.guardrails.builtin import AllowlistProvider, SensitiveDataProvider from deerflow.guardrails.middleware import GuardrailMiddleware from deerflow.guardrails.provider import GuardrailDecision, GuardrailReason, GuardrailRequest @@ -105,6 +105,46 @@ class TestAllowlistProvider: assert decision.allow is False +class TestSensitiveDataProvider: + def test_denies_reading_env_file(self): + provider = SensitiveDataProvider() + req = GuardrailRequest(tool_name="read_file", tool_input={"path": "/tmp/.env"}) + decision = provider.evaluate(req) + assert decision.allow is False + assert decision.reasons[0].code == "oap.blocked_pattern" + + def test_allows_normal_source_file(self): + provider = SensitiveDataProvider() + req = GuardrailRequest(tool_name="read_file", tool_input={"path": "/workspace/app/main.py"}) + decision = provider.evaluate(req) + assert decision.allow is True + + def test_denies_skills_env(self): + provider = SensitiveDataProvider() + req = GuardrailRequest(tool_name="read_file", tool_input={"path": "/mnt/skills/public/foo/.env.local"}) + decision = provider.evaluate(req) + assert decision.allow is False + + def test_denies_glob_or_grep_targeting_env(self): + provider = SensitiveDataProvider() + req = GuardrailRequest(tool_name="grep", tool_input={"path": "/workspace", "glob": "**/.env.*"}) + decision = provider.evaluate(req) + assert decision.allow is False + + def test_denies_bash_cat_env(self): + provider = SensitiveDataProvider() + req = GuardrailRequest(tool_name="bash", tool_input={"command": "cat /workspace/.env"}) + decision = provider.evaluate(req) + assert decision.allow is False + + def test_denies_case_variant_and_key_material(self): + provider = SensitiveDataProvider() + req1 = GuardrailRequest(tool_name="read_file", tool_input={"path": "/workspace/.ENV"}) + req2 = GuardrailRequest(tool_name="read_file", tool_input={"path": "/workspace/id_rsa.pub"}) + assert provider.evaluate(req1).allow is False + assert provider.evaluate(req2).allow is False + + # --- GuardrailMiddleware tests --- diff --git a/backend/tests/test_sensitive_output_redaction_middleware.py b/backend/tests/test_sensitive_output_redaction_middleware.py new file mode 100644 index 00000000..12eaa7d9 --- /dev/null +++ b/backend/tests/test_sensitive_output_redaction_middleware.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import asyncio +from unittest.mock import MagicMock + +from langchain_core.messages import ToolMessage + +from deerflow.agents.middlewares.sensitive_output_redaction_middleware import SensitiveOutputRedactionMiddleware + + +def _request(name: str = "bash", args: dict | None = None): + req = MagicMock() + req.tool_call = {"name": name, "args": args or {}, "id": "call_1"} + return req + + +def test_redacts_key_value_and_bearer(): + mw = SensitiveOutputRedactionMiddleware() + req = _request() + handler = MagicMock( + return_value=ToolMessage( + content="OPENAI_API_KEY=sk-abc123456789\nAuthorization: Bearer abcdefghijklmnop", + tool_call_id="call_1", + name="bash", + status="success", + ) + ) + result = mw.wrap_tool_call(req, handler) + assert "OPENAI_API_KEY=[REDACTED]" in result.content + assert "Bearer [REDACTED]" in result.content + + +def test_redacts_private_key_block(): + mw = SensitiveOutputRedactionMiddleware() + req = _request() + handler = MagicMock( + return_value=ToolMessage( + content="-----BEGIN PRIVATE KEY-----\nabc\n-----END PRIVATE KEY-----", + tool_call_id="call_1", + name="bash", + status="success", + ) + ) + result = mw.wrap_tool_call(req, handler) + assert result.content == "[REDACTED]" + + +def test_async_path_redacts_jwt(): + mw = SensitiveOutputRedactionMiddleware() + req = _request() + + async def handler(_): + return ToolMessage( + content="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.abcde12345.zyxwv98765", + tool_call_id="call_1", + name="bash", + status="success", + ) + + result = asyncio.run(mw.awrap_tool_call(req, handler)) + assert result.content == "[REDACTED]" + diff --git a/config.example.yaml b/config.example.yaml index be3cb7df..3512962f 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -832,3 +832,15 @@ checkpointer: # use: my_package:MyGuardrailProvider # config: # key: value + +# --- Option 4: SensitiveDataProvider (strict secret-file blocking) --- +# guardrails: +# enabled: true +# fail_closed: true +# provider: +# use: deerflow.guardrails.builtin:SensitiveDataProvider +# config: +# protected_tools: ["read_file", "write_file", "str_replace", "ls", "glob", "grep", "bash"] +# deny_basenames: [".env"] +# deny_globs: [".env.*", "*.pem", "*.key", "id_rsa*", "secrets.*", "credentials.*"] +# block_skills_env: true