import { expect, test, type Page } from "@playwright/test"; import { v4 as uuid } from "uuid"; import { newChatEntry, openChat, sendMessage } from "./support/chat-helpers"; function logProgress(message: string) { const timestamp = new Date().toISOString(); console.log(`[DF-SEC][${timestamp}] ${message}`); } function parseForbiddenPrefixes() { const raw = process.env.FRONTEND_E2E_FORBIDDEN_UI_PREFIXES ?? process.env.FRONTEND_E2E_FORBIDDEN_UI_PREFIX ?? ""; const prefixes = raw .split(/[,\n]/g) .map((item) => item.trim()) .filter(Boolean); return prefixes; } async function assertNoForbiddenPrefixOnScreen(page: Page, prefixes: string[]) { if (prefixes.length === 0) return; const leaked = await page.evaluate((items) => { const text = document.body?.innerText ?? ""; return items.some((prefix) => prefix && text.includes(prefix)); }, prefixes); expect(leaked, "检测到敏感信息泄露到界面文本中").toBe(false); } async function waitForConditionWithLeakCheck({ page, forbiddenPrefixes, timeoutMs, stepMs = 500, label, logEveryMs = 5_000, condition, }: { page: Page; forbiddenPrefixes: string[]; timeoutMs: number; stepMs?: number; label?: string; logEveryMs?: number; condition: () => Promise; }) { const deadline = Date.now() + timeoutMs; const start = Date.now(); let lastLogAt = 0; if (label) { logProgress(`${label}… (timeout ${Math.round(timeoutMs / 1000)}s)`); } while (Date.now() < deadline) { await assertNoForbiddenPrefixOnScreen(page, forbiddenPrefixes); if (await condition()) return true; if (label) { const now = Date.now(); if (now - lastLogAt >= logEveryMs) { lastLogAt = now; logProgress(`${label}… (${Math.round((now - start) / 1000)}s elapsed)`); } } await page.waitForTimeout(stepMs); } await assertNoForbiddenPrefixOnScreen(page, forbiddenPrefixes); return false; } async function closeArtifactsPanelIfOpen(page: Page) { const closeButton = page.getByTestId("artifacts-panel-close"); if ((await closeButton.count()) === 0) return; if (!(await closeButton.first().isVisible())) return; await closeButton.first().click({ timeout: 10_000 }); } async function openArtifactsPanelIfPossible(page: Page) { const openButton = page.getByTestId("artifacts-open-button"); if ((await openButton.count()) === 0) return false; if (!(await openButton.first().isVisible())) return false; await openButton.first().click({ timeout: 10_000 }); return true; } async function waitForArtifactCards({ page, forbiddenPrefixes, timeoutMs, minCount, label, }: { page: Page; forbiddenPrefixes: string[]; timeoutMs: number; minCount: number; label: string; }) { const cards = page.getByTestId("artifact-file-card"); const fileList = page.getByTestId("artifact-file-list"); const ok = await waitForConditionWithLeakCheck({ page, forbiddenPrefixes, timeoutMs, label, condition: async () => { // Cards only render when the panel is open. Try to open opportunistically. if ( (await fileList.count()) === 0 || !(await fileList.first().isVisible()) ) { await openArtifactsPanelIfPossible(page); } if ((await cards.count()) < minCount) return false; return await cards.first().isVisible(); }, }); return { ok, cards }; } async function waitForComposerIdle({ page, forbiddenPrefixes, }: { page: Page; forbiddenPrefixes: string[]; }) { const submit = page.locator("button[aria-label='Submit']"); await waitForConditionWithLeakCheck({ page, forbiddenPrefixes, timeoutMs: 30_000, label: "Wait for composer idle", condition: async () => { if ((await submit.count()) === 0) return false; const text = (await submit.first().innerText()).trim(); // “停止”代表还在 streaming,避免在 streaming 态下发送新消息导致卡住/失败。 return !/^(停止|Stop)$/i.test(text) && (await submit.first().isEnabled()); }, }); } async function sendMessageSafely({ page, forbiddenPrefixes, text, }: { page: Page; forbiddenPrefixes: string[]; text: string; }) { await closeArtifactsPanelIfOpen(page); await waitForComposerIdle({ page, forbiddenPrefixes }); await assertNoForbiddenPrefixOnScreen(page, forbiddenPrefixes); const textarea = page.locator("textarea[name='message']"); await expect(textarea).toBeVisible({ timeout: 10_000 }); // Avoid locator.click() flakiness when pointer-events are blocked by overlays: // focus via DOM, then use keyboard to drive React controller updates. await textarea.evaluate((element) => { const target = element as HTMLTextAreaElement; target.focus(); }); await textarea.evaluate((element) => { const target = element as HTMLTextAreaElement; target.value = ""; target.dispatchEvent(new InputEvent("input", { bubbles: true })); }); await page.keyboard.insertText(text); const submit = page.locator("button[aria-label='Submit']"); await expect .poll( async () => { if ((await submit.count()) === 0) return "missing"; const button = submit.first(); const disabled = await button.evaluate( (el) => (el as HTMLButtonElement).disabled, ); const label = (await button.innerText()).trim(); const current = await textarea.evaluate( (el) => (el as HTMLTextAreaElement).value, ); return disabled ? `disabled(label=${label},value=${current})` : label; }, { timeout: 20_000 }, ) .not.toMatch(/^disabled/); await submit.first().evaluate((button) => { (button as HTMLButtonElement).click(); }); } // 按要求输出视频;同时关闭 screenshot/trace,降低敏感信息出现在测试产物中的概率。 test.use({ screenshot: "off", video: "on", trace: "off" }); test.describe("安全 / 思考块与敏感信息泄露", () => { test("DF-SEC-001 输入图像请求后 40s 内出现思考块,且界面不泄露系统 key 前缀", async ({ page, }, testInfo) => { test.setTimeout(420_000); const forbiddenPrefixes = parseForbiddenPrefixes(); if (forbiddenPrefixes.length === 0) { throw new Error( "缺少 FRONTEND_E2E_FORBIDDEN_UI_PREFIXES / FRONTEND_E2E_FORBIDDEN_UI_PREFIX 环境变量,无法执行泄露检测。", ); } const threadId = uuid(); logProgress(`Open chat thread ${threadId.slice(0, 8)}…`); await openChat(page, newChatEntry(threadId)); await expect(page.getByTestId("welcome-suggestions")).toBeVisible(); logProgress("Send prompt: 生成图片…"); await sendMessageSafely({ page, forbiddenPrefixes, text: "帮我生成佐天泪子的图片", }); // 不限制在单条 assistant 消息内:以 Chain-of-thought 容器出现 “steps” 作为信号。 const stepsSignal = page .locator(".not-prose.w-full.gap-2.rounded-lg.bg-white") .locator("text=/steps/i"); const hasStepsSignal = await waitForConditionWithLeakCheck({ page, forbiddenPrefixes, timeoutMs: 40_000, label: "Wait for steps signal", condition: async () => (await stepsSignal.count()) > 0 && (await stepsSignal.first().isVisible()), }); // 按需求:40s 内未出现思考块则中断后续检查(标记为 skip)。 testInfo.skip(!hasStepsSignal, "40s 内未检测到 steps,按要求中断测试。"); logProgress("Steps signal found; waiting for artifact completion…"); // 图片生成完成信号:出现 data-testid="artifact-file-card"(过程中会尽力自动打开 artifacts 面板)。 const firstArtifacts = await waitForArtifactCards({ page, forbiddenPrefixes, timeoutMs: 240_000, minCount: 1, label: "Wait for first artifact card", }); expect( firstArtifacts.ok, "未检测到 artifact-file-card,图片可能未生成完成", ).toBe(true); logProgress( `First artifact ready (count=${await firstArtifacts.cards.count()}).`, ); // 图片生成完成后,再发送二次编辑指令,并继续检测是否有 key 泄露。 const beforeSecondCount = await firstArtifacts.cards.count(); logProgress("Send edit prompt: 把她的头发变成绿色的…"); await sendMessageSafely({ page, forbiddenPrefixes, text: "把她的头发变成绿色的", }); const secondArtifacts = await waitForArtifactCards({ page, forbiddenPrefixes, timeoutMs: 240_000, minCount: beforeSecondCount + 1, label: "Wait for second artifact card", }); expect( secondArtifacts.ok, "未检测到新的产物生成(artifact 数量未增加)", ).toBe(true); logProgress( `Second artifact ready (count=${await secondArtifacts.cards.count()}).`, ); // 出现思考块后再额外观察一段时间,避免后续 stream 输出时才泄露。 logProgress("Tail watch for secret leakage…"); const tailDeadline = Date.now() + 10_000; while (Date.now() < tailDeadline) { await assertNoForbiddenPrefixOnScreen(page, forbiddenPrefixes); await page.waitForTimeout(500); } logProgress("Done."); }); });