test:测试生成图片时禁止暴露apikey的用例
This commit is contained in:
parent
deac1537d0
commit
6a73d96778
|
|
@ -0,0 +1,296 @@
|
||||||
|
import { expect, test, type Page } from "@playwright/test";
|
||||||
|
import { v4 as uuid } from "uuid";
|
||||||
|
|
||||||
|
import { newChatEntry, openChat, sendMessage } from "./support/chat-helpers";
|
||||||
|
|
||||||
|
function logProgress(message: string) {
|
||||||
|
const timestamp = new Date().toISOString();
|
||||||
|
// eslint-disable-next-line no-console
|
||||||
|
console.log(`[DF-SEC][${timestamp}] ${message}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseForbiddenPrefixes() {
|
||||||
|
const raw =
|
||||||
|
process.env.FRONTEND_E2E_FORBIDDEN_UI_PREFIXES ??
|
||||||
|
process.env.FRONTEND_E2E_FORBIDDEN_UI_PREFIX ??
|
||||||
|
"";
|
||||||
|
const prefixes = raw
|
||||||
|
.split(/[,\n]/g)
|
||||||
|
.map((item) => item.trim())
|
||||||
|
.filter(Boolean);
|
||||||
|
return prefixes;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function assertNoForbiddenPrefixOnScreen(
|
||||||
|
page: Page,
|
||||||
|
prefixes: string[],
|
||||||
|
) {
|
||||||
|
if (prefixes.length === 0) return;
|
||||||
|
const leaked = await page.evaluate((items) => {
|
||||||
|
const text = document.body?.innerText ?? "";
|
||||||
|
return items.some((prefix) => prefix && text.includes(prefix));
|
||||||
|
}, prefixes);
|
||||||
|
|
||||||
|
expect(leaked, "检测到敏感信息泄露到界面文本中").toBe(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function waitForConditionWithLeakCheck({
|
||||||
|
page,
|
||||||
|
forbiddenPrefixes,
|
||||||
|
timeoutMs,
|
||||||
|
stepMs = 500,
|
||||||
|
label,
|
||||||
|
logEveryMs = 5_000,
|
||||||
|
condition,
|
||||||
|
}: {
|
||||||
|
page: Page;
|
||||||
|
forbiddenPrefixes: string[];
|
||||||
|
timeoutMs: number;
|
||||||
|
stepMs?: number;
|
||||||
|
label?: string;
|
||||||
|
logEveryMs?: number;
|
||||||
|
condition: () => Promise<boolean>;
|
||||||
|
}) {
|
||||||
|
const deadline = Date.now() + timeoutMs;
|
||||||
|
const start = Date.now();
|
||||||
|
let lastLogAt = 0;
|
||||||
|
if (label) {
|
||||||
|
logProgress(`${label}… (timeout ${Math.round(timeoutMs / 1000)}s)`);
|
||||||
|
}
|
||||||
|
while (Date.now() < deadline) {
|
||||||
|
await assertNoForbiddenPrefixOnScreen(page, forbiddenPrefixes);
|
||||||
|
if (await condition()) return true;
|
||||||
|
if (label) {
|
||||||
|
const now = Date.now();
|
||||||
|
if (now - lastLogAt >= logEveryMs) {
|
||||||
|
lastLogAt = now;
|
||||||
|
logProgress(
|
||||||
|
`${label}… (${Math.round((now - start) / 1000)}s elapsed)`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
await page.waitForTimeout(stepMs);
|
||||||
|
}
|
||||||
|
await assertNoForbiddenPrefixOnScreen(page, forbiddenPrefixes);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function closeArtifactsPanelIfOpen(page: Page) {
|
||||||
|
const closeButton = page.getByTestId("artifacts-panel-close");
|
||||||
|
if ((await closeButton.count()) === 0) return;
|
||||||
|
if (!(await closeButton.first().isVisible())) return;
|
||||||
|
await closeButton.first().click({ timeout: 10_000 });
|
||||||
|
}
|
||||||
|
|
||||||
|
async function openArtifactsPanelIfPossible(page: Page) {
|
||||||
|
const openButton = page.getByTestId("artifacts-open-button");
|
||||||
|
if ((await openButton.count()) === 0) return false;
|
||||||
|
if (!(await openButton.first().isVisible())) return false;
|
||||||
|
await openButton.first().click({ timeout: 10_000 });
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function waitForArtifactCards({
|
||||||
|
page,
|
||||||
|
forbiddenPrefixes,
|
||||||
|
timeoutMs,
|
||||||
|
minCount,
|
||||||
|
label,
|
||||||
|
}: {
|
||||||
|
page: Page;
|
||||||
|
forbiddenPrefixes: string[];
|
||||||
|
timeoutMs: number;
|
||||||
|
minCount: number;
|
||||||
|
label: string;
|
||||||
|
}) {
|
||||||
|
const cards = page.getByTestId("artifact-file-card");
|
||||||
|
const fileList = page.getByTestId("artifact-file-list");
|
||||||
|
|
||||||
|
const ok = await waitForConditionWithLeakCheck({
|
||||||
|
page,
|
||||||
|
forbiddenPrefixes,
|
||||||
|
timeoutMs,
|
||||||
|
label,
|
||||||
|
condition: async () => {
|
||||||
|
// Cards only render when the panel is open. Try to open opportunistically.
|
||||||
|
if ((await fileList.count()) === 0 || !(await fileList.first().isVisible())) {
|
||||||
|
await openArtifactsPanelIfPossible(page);
|
||||||
|
}
|
||||||
|
if ((await cards.count()) < minCount) return false;
|
||||||
|
return await cards.first().isVisible();
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
return { ok, cards };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function waitForComposerIdle({
|
||||||
|
page,
|
||||||
|
forbiddenPrefixes,
|
||||||
|
}: {
|
||||||
|
page: Page;
|
||||||
|
forbiddenPrefixes: string[];
|
||||||
|
}) {
|
||||||
|
const submit = page.locator("button[aria-label='Submit']");
|
||||||
|
await waitForConditionWithLeakCheck({
|
||||||
|
page,
|
||||||
|
forbiddenPrefixes,
|
||||||
|
timeoutMs: 30_000,
|
||||||
|
label: "Wait for composer idle",
|
||||||
|
condition: async () => {
|
||||||
|
if ((await submit.count()) === 0) return false;
|
||||||
|
const text = (await submit.first().innerText()).trim();
|
||||||
|
// “停止”代表还在 streaming，避免在 streaming 态下发送新消息导致卡住/失败。
|
||||||
|
return !/^(停止|Stop)$/i.test(text) && (await submit.first().isEnabled());
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function sendMessageSafely({
|
||||||
|
page,
|
||||||
|
forbiddenPrefixes,
|
||||||
|
text,
|
||||||
|
}: {
|
||||||
|
page: Page;
|
||||||
|
forbiddenPrefixes: string[];
|
||||||
|
text: string;
|
||||||
|
}) {
|
||||||
|
await closeArtifactsPanelIfOpen(page);
|
||||||
|
await waitForComposerIdle({ page, forbiddenPrefixes });
|
||||||
|
await assertNoForbiddenPrefixOnScreen(page, forbiddenPrefixes);
|
||||||
|
|
||||||
|
const textarea = page.locator("textarea[name='message']");
|
||||||
|
await expect(textarea).toBeVisible({ timeout: 10_000 });
|
||||||
|
// Avoid locator.click() flakiness when pointer-events are blocked by overlays:
|
||||||
|
// focus via DOM, then use keyboard to drive React controller updates.
|
||||||
|
await textarea.evaluate((element) => {
|
||||||
|
const target = element as HTMLTextAreaElement;
|
||||||
|
target.focus();
|
||||||
|
});
|
||||||
|
await textarea.evaluate((element) => {
|
||||||
|
const target = element as HTMLTextAreaElement;
|
||||||
|
const setter = Object.getOwnPropertyDescriptor(
|
||||||
|
HTMLTextAreaElement.prototype,
|
||||||
|
"value",
|
||||||
|
)?.set;
|
||||||
|
setter?.call(target, "");
|
||||||
|
target.dispatchEvent(new InputEvent("input", { bubbles: true }));
|
||||||
|
});
|
||||||
|
await page.keyboard.insertText(text);
|
||||||
|
|
||||||
|
const submit = page.locator("button[aria-label='Submit']");
|
||||||
|
await expect
|
||||||
|
.poll(
|
||||||
|
async () => {
|
||||||
|
if ((await submit.count()) === 0) return "missing";
|
||||||
|
const button = submit.first();
|
||||||
|
const disabled = await button.evaluate(
|
||||||
|
(el) => (el as HTMLButtonElement).disabled,
|
||||||
|
);
|
||||||
|
const label = (await button.innerText()).trim();
|
||||||
|
const current = await textarea.evaluate(
|
||||||
|
(el) => (el as HTMLTextAreaElement).value,
|
||||||
|
);
|
||||||
|
return disabled ? `disabled(label=${label},value=${current})` : label;
|
||||||
|
},
|
||||||
|
{ timeout: 20_000 },
|
||||||
|
)
|
||||||
|
.not.toMatch(/^disabled/);
|
||||||
|
|
||||||
|
await submit.first().evaluate((button) => {
|
||||||
|
(button as HTMLButtonElement).click();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// 按要求输出视频；同时关闭 screenshot/trace，降低敏感信息出现在测试产物中的概率。
|
||||||
|
test.use({ screenshot: "off", video: "on", trace: "off" });
|
||||||
|
|
||||||
|
test.describe("安全 / 思考块与敏感信息泄露", () => {
|
||||||
|
test("DF-SEC-001 输入图像请求后 40s 内出现思考块，且界面不泄露系统 key 前缀", async ({
|
||||||
|
page,
|
||||||
|
}, testInfo) => {
|
||||||
|
test.setTimeout(420_000);
|
||||||
|
|
||||||
|
const forbiddenPrefixes = parseForbiddenPrefixes();
|
||||||
|
if (forbiddenPrefixes.length === 0) {
|
||||||
|
throw new Error(
|
||||||
|
"缺少 FRONTEND_E2E_FORBIDDEN_UI_PREFIXES / FRONTEND_E2E_FORBIDDEN_UI_PREFIX 环境变量，无法执行泄露检测。",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const threadId = uuid();
|
||||||
|
logProgress(`Open chat thread ${threadId.slice(0, 8)}…`);
|
||||||
|
await openChat(page, newChatEntry(threadId));
|
||||||
|
await expect(page.getByTestId("welcome-suggestions")).toBeVisible();
|
||||||
|
|
||||||
|
logProgress("Send prompt: 生成图片…");
|
||||||
|
await sendMessageSafely({
|
||||||
|
page,
|
||||||
|
forbiddenPrefixes,
|
||||||
|
text: "帮我生成佐天泪子的图片",
|
||||||
|
});
|
||||||
|
|
||||||
|
// 不限制在单条 assistant 消息内：以 Chain-of-thought 容器出现 “steps” 作为信号。
|
||||||
|
const stepsSignal = page
|
||||||
|
.locator(".not-prose.w-full.gap-2.rounded-lg.bg-white")
|
||||||
|
.locator("text=/steps/i");
|
||||||
|
|
||||||
|
const hasStepsSignal = await waitForConditionWithLeakCheck({
|
||||||
|
page,
|
||||||
|
forbiddenPrefixes,
|
||||||
|
timeoutMs: 40_000,
|
||||||
|
label: "Wait for steps signal",
|
||||||
|
condition: async () =>
|
||||||
|
(await stepsSignal.count()) > 0 && (await stepsSignal.first().isVisible()),
|
||||||
|
});
|
||||||
|
|
||||||
|
// 按需求：40s 内未出现思考块则中断后续检查（标记为 skip）。
|
||||||
|
testInfo.skip(!hasStepsSignal, "40s 内未检测到 steps，按要求中断测试。");
|
||||||
|
|
||||||
|
logProgress("Steps signal found; waiting for artifact completion…");
|
||||||
|
// 图片生成完成信号：出现 data-testid="artifact-file-card"（过程中会尽力自动打开 artifacts 面板）。
|
||||||
|
const firstArtifacts = await waitForArtifactCards({
|
||||||
|
page,
|
||||||
|
forbiddenPrefixes,
|
||||||
|
timeoutMs: 240_000,
|
||||||
|
minCount: 1,
|
||||||
|
label: "Wait for first artifact card",
|
||||||
|
});
|
||||||
|
expect(firstArtifacts.ok, "未检测到 artifact-file-card，图片可能未生成完成").toBe(
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
logProgress(
|
||||||
|
`First artifact ready (count=${await firstArtifacts.cards.count()}).`,
|
||||||
|
);
|
||||||
|
|
||||||
|
// 图片生成完成后，再发送二次编辑指令，并继续检测是否有 key 泄露。
|
||||||
|
const beforeSecondCount = await firstArtifacts.cards.count();
|
||||||
|
logProgress("Send edit prompt: 把她的头发变成绿色的…");
|
||||||
|
await sendMessageSafely({
|
||||||
|
page,
|
||||||
|
forbiddenPrefixes,
|
||||||
|
text: "把她的头发变成绿色的",
|
||||||
|
});
|
||||||
|
|
||||||
|
const secondArtifacts = await waitForArtifactCards({
|
||||||
|
page,
|
||||||
|
forbiddenPrefixes,
|
||||||
|
timeoutMs: 240_000,
|
||||||
|
minCount: beforeSecondCount + 1,
|
||||||
|
label: "Wait for second artifact card",
|
||||||
|
});
|
||||||
|
expect(secondArtifacts.ok, "未检测到新的产物生成（artifact 数量未增加）").toBe(true);
|
||||||
|
logProgress(
|
||||||
|
`Second artifact ready (count=${await secondArtifacts.cards.count()}).`,
|
||||||
|
);
|
||||||
|
|
||||||
|
// 出现思考块后再额外观察一段时间，避免后续 stream 输出时才泄露。
|
||||||
|
logProgress("Tail watch for secret leakage…");
|
||||||
|
const tailDeadline = Date.now() + 10_000;
|
||||||
|
while (Date.now() < tailDeadline) {
|
||||||
|
await assertNoForbiddenPrefixOnScreen(page, forbiddenPrefixes);
|
||||||
|
await page.waitForTimeout(500);
|
||||||
|
}
|
||||||
|
logProgress("Done.");
|
||||||
|
});
|
||||||
|
});
|
||||||
Loading…
Reference in New Issue