From 444d164aea4dde9acbba8a30e6dfe53337bd8b36 Mon Sep 17 00:00:00 2001 From: yyh <92089059+lyzno1@users.noreply.github.com> Date: Tue, 30 Jun 2026 14:25:20 +0800 Subject: [PATCH] test(e2e): add agent v2 test infrastructure (#38191) --- e2e/AGENTS.md | 10 + e2e/features/agent-v2/configure-entry.feature | 9 + .../agent-v2/configure.steps.ts | 49 ++++ e2e/features/support/hooks.ts | 9 +- e2e/features/support/world.ts | 6 + e2e/scripts/common.ts | 2 + e2e/support/agent.ts | 229 ++++++++++++++++++ e2e/support/api.ts | 39 ++- e2e/support/process.ts | 22 +- eslint-suppressions.json | 18 -- 10 files changed, 354 insertions(+), 39 deletions(-) create mode 100644 e2e/features/agent-v2/configure-entry.feature create mode 100644 e2e/features/step-definitions/agent-v2/configure.steps.ts create mode 100644 e2e/support/agent.ts diff --git a/e2e/AGENTS.md b/e2e/AGENTS.md index c05b5105be6..017594bbe34 100644 --- a/e2e/AGENTS.md +++ b/e2e/AGENTS.md @@ -294,3 +294,13 @@ Or browse the step definition files directly: - `features/step-definitions/common/` — auth guards and navigation assertions shared by all features - `features/step-definitions//` — domain-specific steps scoped to a single feature area + +## Agent v2 scenarios + +Agent v2 scenarios live under `features/agent-v2/` and use the `@agent-v2` capability tag. + +The E2E web environment enables Agent v2 through `NEXT_PUBLIC_ENABLE_AGENT_V2=true` in `scripts/common.ts`, because `/roster` routes are guarded by that feature flag. + +Use `support/agent.ts` for Agent v2 API fixtures. It owns roster-shaped Agent IDs, configure/access route helpers, composer draft sync, build-draft helpers, publish, API access toggles, and Agent cleanup. Store created roster Agent IDs in `DifyWorld.createdAgentIds`; the shared `After` hook deletes them after each scenario. + +Keep Agent v2 step definitions under `features/step-definitions/agent-v2/`. Prefer API setup for prerequisite state, then use Playwright only for user-observable navigation, editing, and assertions. diff --git a/e2e/features/agent-v2/configure-entry.feature b/e2e/features/agent-v2/configure-entry.feature new file mode 100644 index 00000000000..5c15c84f24d --- /dev/null +++ b/e2e/features/agent-v2/configure-entry.feature @@ -0,0 +1,9 @@ +@agent-v2 @authenticated @infra +Feature: Agent v2 configure entry + Scenario: Open the configure page for an Agent v2 test agent + Given I am signed in as the default E2E admin + And an Agent v2 test agent has been created via API + And a minimal Agent v2 composer draft has been synced + When I open the Agent v2 configure page + Then I should be on the Agent v2 configure page + And I should see the Agent v2 configure workspace diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts new file mode 100644 index 00000000000..cbb49fef685 --- /dev/null +++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts @@ -0,0 +1,49 @@ +import type { DifyWorld } from '../../support/world' +import { Given, Then, When } from '@cucumber/cucumber' +import { expect } from '@playwright/test' +import { + createTestAgent, + getAgentConfigurePath, + saveAgentComposerDraft, +} from '../../../support/agent' + +Given('an Agent v2 test agent has been created via API', async function (this: DifyWorld) { + const agent = await createTestAgent() + this.createdAgentIds.push(agent.id) + this.lastCreatedAgentName = agent.name + this.lastCreatedAgentRole = agent.role +}) + +Given('a minimal Agent v2 composer draft has been synced', async function (this: DifyWorld) { + const agentId = this.createdAgentIds.at(-1) + if (!agentId) + throw new Error('No Agent v2 ID found. Create an Agent v2 test agent first.') + + await saveAgentComposerDraft(agentId) +}) + +When('I open the Agent v2 configure page', async function (this: DifyWorld) { + const agentId = this.createdAgentIds.at(-1) + if (!agentId) + throw new Error('No Agent v2 ID found. Create an Agent v2 test agent first.') + + await this.getPage().goto(getAgentConfigurePath(agentId)) +}) + +Then('I should be on the Agent v2 configure page', async function (this: DifyWorld) { + const agentId = this.createdAgentIds.at(-1) + if (!agentId) + throw new Error('No Agent v2 ID found. Create an Agent v2 test agent first.') + + await expect(this.getPage()).toHaveURL( + new RegExp(`/roster/agent/${agentId}/configure(?:\\?.*)?$`), + ) +}) + +Then('I should see the Agent v2 configure workspace', async function (this: DifyWorld) { + const page = this.getPage() + + await expect(page.getByRole('region', { name: 'Configure' })).toBeVisible({ timeout: 30_000 }) + await expect(page.getByRole('heading', { name: 'Configure' })).toBeVisible() + await expect(page.getByText(this.lastCreatedAgentName!)).toBeVisible() +}) diff --git a/e2e/features/support/hooks.ts b/e2e/features/support/hooks.ts index c1a535ee2c0..e67291a7ad2 100644 --- a/e2e/features/support/hooks.ts +++ b/e2e/features/support/hooks.ts @@ -1,4 +1,5 @@ import type { Browser } from '@playwright/test' +import type { Buffer } from 'node:buffer' import type { DifyWorld } from './world' import { mkdir, writeFile } from 'node:fs/promises' import path from 'node:path' @@ -6,6 +7,7 @@ import { fileURLToPath } from 'node:url' import { After, AfterAll, Before, BeforeAll, setDefaultTimeout, Status } from '@cucumber/cucumber' import { chromium } from '@playwright/test' import { AUTH_BOOTSTRAP_TIMEOUT_MS, ensureAuthenticatedState } from '../../fixtures/auth' +import { deleteTestAgent } from '../../support/agent' import { deleteTestApp } from '../../support/api' import { baseURL, cucumberHeadless, cucumberSlowMo } from '../../test-env' @@ -41,7 +43,7 @@ BeforeAll({ timeout: AUTH_BOOTSTRAP_TIMEOUT_MS }, async () => { slowMo: cucumberSlowMo, }) - console.log(`[e2e] session cache bootstrap against ${baseURL}`) + console.warn(`[e2e] session cache bootstrap against ${baseURL}`) await ensureAuthenticatedState(browser, baseURL) }) @@ -58,7 +60,7 @@ Before(async function (this: DifyWorld, { pickle }) { this.scenarioStartedAt = Date.now() const tags = pickle.tags.map(tag => tag.name).join(' ') - console.log(`[e2e] start ${pickle.name}${tags ? ` ${tags}` : ''}`) + console.warn(`[e2e] start ${pickle.name}${tags ? ` ${tags}` : ''}`) }) After(async function (this: DifyWorld, { pickle, result }) { @@ -85,10 +87,11 @@ After(async function (this: DifyWorld, { pickle, result }) { } const status = result?.status || 'UNKNOWN' - console.log( + console.warn( `[e2e] end ${pickle.name} status=${status}${elapsedMs ? ` durationMs=${elapsedMs}` : ''}`, ) + for (const id of this.createdAgentIds) await deleteTestAgent(id).catch(() => {}) for (const id of this.createdAppIds) await deleteTestApp(id).catch(() => {}) await this.closeSession() diff --git a/e2e/features/support/world.ts b/e2e/features/support/world.ts index b53087171f5..6faf740a7c9 100644 --- a/e2e/features/support/world.ts +++ b/e2e/features/support/world.ts @@ -13,7 +13,10 @@ export class DifyWorld extends World { scenarioStartedAt: number | undefined session: AuthSessionMetadata | undefined lastCreatedAppName: string | undefined + lastCreatedAgentName: string | undefined + lastCreatedAgentRole: string | undefined createdAppIds: string[] = [] + createdAgentIds: string[] = [] capturedDownloads: Download[] = [] shareURL: string | undefined @@ -26,7 +29,10 @@ export class DifyWorld extends World { this.consoleErrors = [] this.pageErrors = [] this.lastCreatedAppName = undefined + this.lastCreatedAgentName = undefined + this.lastCreatedAgentRole = undefined this.createdAppIds = [] + this.createdAgentIds = [] this.capturedDownloads = [] this.shareURL = undefined } diff --git a/e2e/scripts/common.ts b/e2e/scripts/common.ts index 2964892dd0b..20b9ab7fbd8 100644 --- a/e2e/scripts/common.ts +++ b/e2e/scripts/common.ts @@ -1,3 +1,4 @@ +import type { Buffer } from 'node:buffer' import type { ChildProcess } from 'node:child_process' import { spawn } from 'node:child_process' import { createHash } from 'node:crypto' @@ -42,6 +43,7 @@ export const webEnvExampleFile = path.join(webDir, '.env.example') export const apiEnvExampleFile = path.join(apiDir, 'tests', 'integration_tests', '.env.example') export const e2eWebEnvOverrides = { NEXT_PUBLIC_API_PREFIX: 'http://127.0.0.1:5001/console/api', + NEXT_PUBLIC_ENABLE_AGENT_V2: 'true', NEXT_PUBLIC_PUBLIC_API_PREFIX: 'http://127.0.0.1:5001/api', } satisfies Record diff --git a/e2e/support/agent.ts b/e2e/support/agent.ts new file mode 100644 index 00000000000..445c10b5cfc --- /dev/null +++ b/e2e/support/agent.ts @@ -0,0 +1,229 @@ +import { createApiContext, expectApiResponseOK, setAppSiteEnabled } from './api' + +export type AgentSeed = { + app_id?: string + backing_app_id?: string + description?: string + enable_site?: boolean + id: string + name: string + role?: string + site?: { + access_token?: string | null + app_base_url?: string | null + code?: string | null + } | null +} + +export type AgentSoulConfig = Record + +export type AgentComposerResponse = { + agent_soul?: AgentSoulConfig +} + +export type AgentBuildDraftResponse = { + agent_soul: AgentSoulConfig + draft: Record + variant: 'agent_app' +} + +export type AgentApiAccess = { + api_key_count: number + api_reference_url: string + endpoint: string + enabled: boolean + files_upload_endpoint: string +} + +export type AgentApiKey = { + id: string + token?: string +} + +export const defaultAgentSoulConfig: AgentSoulConfig = { + prompt: { + system_prompt: 'You are a Dify Agent E2E test assistant.', + }, +} + +export const getAgentConfigurePath = (agentId: string) => `/roster/agent/${agentId}/configure` +export const getAgentAccessPath = (agentId: string) => `/roster/agent/${agentId}/access` + +export async function createTestAgent({ + description = 'Created by Dify E2E.', + name = `E2E Agent ${Date.now()}`, + role = 'E2E test assistant', +}: { + description?: string + name?: string + role?: string +} = {}): Promise { + const ctx = await createApiContext() + try { + const response = await ctx.post('/console/api/agent', { + data: { + description, + icon: '🤖', + icon_background: '#FFEAD5', + icon_type: 'emoji', + name, + role, + }, + }) + await expectApiResponseOK(response, 'Create Agent v2 test agent') + return (await response.json()) as AgentSeed + } + finally { + await ctx.dispose() + } +} + +export async function getTestAgent(agentId: string): Promise { + const ctx = await createApiContext() + try { + const response = await ctx.get(`/console/api/agent/${agentId}`) + await expectApiResponseOK(response, `Get Agent v2 test agent ${agentId}`) + return (await response.json()) as AgentSeed + } + finally { + await ctx.dispose() + } +} + +export async function deleteTestAgent(agentId: string): Promise { + const ctx = await createApiContext() + try { + const response = await ctx.delete(`/console/api/agent/${agentId}`) + await expectApiResponseOK(response, `Delete Agent v2 test agent ${agentId}`) + } + finally { + await ctx.dispose() + } +} + +export async function saveAgentComposerDraft( + agentId: string, + agentSoul: AgentSoulConfig = defaultAgentSoulConfig, +): Promise { + const ctx = await createApiContext() + try { + const response = await ctx.put(`/console/api/agent/${agentId}/composer`, { + data: { + agent_soul: agentSoul, + save_strategy: 'save_to_current_version', + variant: 'agent_app', + }, + }) + await expectApiResponseOK(response, `Save Agent v2 composer draft for ${agentId}`) + return (await response.json()) as AgentComposerResponse + } + finally { + await ctx.dispose() + } +} + +export async function checkoutAgentBuildDraft(agentId: string): Promise { + const ctx = await createApiContext() + try { + const response = await ctx.post(`/console/api/agent/${agentId}/build-draft/checkout`, { + data: { force: true }, + }) + await expectApiResponseOK(response, `Checkout Agent v2 build draft for ${agentId}`) + return (await response.json()) as AgentBuildDraftResponse + } + finally { + await ctx.dispose() + } +} + +export async function discardAgentBuildDraft(agentId: string): Promise { + const ctx = await createApiContext() + try { + const response = await ctx.delete(`/console/api/agent/${agentId}/build-draft`) + await expectApiResponseOK(response, `Discard Agent v2 build draft for ${agentId}`) + } + finally { + await ctx.dispose() + } +} + +export async function publishAgent(agentId: string, versionNote = 'E2E publish'): Promise { + const ctx = await createApiContext() + try { + const response = await ctx.post(`/console/api/agent/${agentId}/publish`, { + data: { version_note: versionNote }, + }) + await expectApiResponseOK(response, `Publish Agent v2 test agent ${agentId}`) + } + finally { + await ctx.dispose() + } +} + +export async function enableAgentSiteAndGetURL(agentId: string): Promise { + const agent = await getTestAgent(agentId) + const appId = agent.app_id ?? agent.backing_app_id + if (!appId) + throw new Error(`Agent v2 ${agentId} does not expose a backing app ID.`) + + const appDetail = await setAppSiteEnabled(appId, true) + const token = agent.site?.access_token ?? agent.site?.code ?? appDetail.site.access_token + const baseURL = agent.site?.app_base_url ?? appDetail.site.app_base_url + + return `${baseURL.replace(/\/$/, '')}/agent/${token}` +} + +export async function getAgentApiAccess(agentId: string): Promise { + const ctx = await createApiContext() + try { + const response = await ctx.get(`/console/api/agent/${agentId}/api-access`) + await expectApiResponseOK(response, `Get Agent v2 API access for ${agentId}`) + return (await response.json()) as AgentApiAccess + } + finally { + await ctx.dispose() + } +} + +export async function setAgentApiAccess( + agentId: string, + enabled: boolean, +): Promise { + const ctx = await createApiContext() + try { + const response = await ctx.post(`/console/api/agent/${agentId}/api-enable`, { + data: { enable_api: enabled }, + }) + await expectApiResponseOK( + response, + `${enabled ? 'Enable' : 'Disable'} Agent v2 API access for ${agentId}`, + ) + return (await response.json()) as AgentApiAccess + } + finally { + await ctx.dispose() + } +} + +export async function createAgentApiKey(agentId: string): Promise { + const ctx = await createApiContext() + try { + const response = await ctx.post(`/console/api/agent/${agentId}/api-keys`) + await expectApiResponseOK(response, `Create Agent v2 API key for ${agentId}`) + return (await response.json()) as AgentApiKey + } + finally { + await ctx.dispose() + } +} + +export async function deleteAgentApiKey(agentId: string, apiKeyId: string): Promise { + const ctx = await createApiContext() + try { + const response = await ctx.delete(`/console/api/agent/${agentId}/api-keys/${apiKeyId}`) + await expectApiResponseOK(response, `Delete Agent v2 API key ${apiKeyId} for ${agentId}`) + } + finally { + await ctx.dispose() + } +} diff --git a/e2e/support/api.ts b/e2e/support/api.ts index 74c42d3e73f..81773dea595 100644 --- a/e2e/support/api.ts +++ b/e2e/support/api.ts @@ -1,3 +1,4 @@ +import type { APIResponse } from '@playwright/test' import { readFile } from 'node:fs/promises' import { request } from '@playwright/test' import { authStatePath } from '../fixtures/auth' @@ -7,7 +8,7 @@ type StorageState = { cookies: Array<{ name: string, value: string }> } -async function createApiContext() { +export async function createApiContext() { const state = JSON.parse(await readFile(authStatePath, 'utf8')) as StorageState const csrfToken = state.cookies.find(c => c.name.endsWith('csrf_token'))?.value ?? '' @@ -18,6 +19,14 @@ async function createApiContext() { }) } +export async function expectApiResponseOK(response: APIResponse, action: string): Promise { + if (response.ok()) + return + + const body = await response.text().catch(() => '') + throw new Error(`${action} failed with ${response.status()} ${response.statusText()}: ${body}`) +} + export type AppSeed = { id: string name: string @@ -141,20 +150,34 @@ export async function publishWorkflowApp(appId: string): Promise { } } -type AppDetailWithSite = { +export type AppDetailWithSite = { + mode?: string site: { access_token: string, app_base_url: string, enable_site: boolean } } +export function getAppSiteURL({ mode, site }: AppDetailWithSite): string { + const webAppMode = mode === 'completion' || mode === 'workflow' ? mode : 'chat' + return `${site.app_base_url}/${webAppMode}/${site.access_token}` +} + export async function enableAppSiteAndGetURL(appId: string): Promise { + return getAppSiteURL(await setAppSiteEnabled(appId, true)) +} + +export async function setAppSiteEnabled( + appId: string, + enabled: boolean, +): Promise { const ctx = await createApiContext() try { - await ctx.post(`/console/api/apps/${appId}/site-enable`, { - data: { enable_site: true }, + const enableResponse = await ctx.post(`/console/api/apps/${appId}/site-enable`, { + data: { enable_site: enabled }, }) - const res = await ctx.get(`/console/api/apps/${appId}`) - const body = (await res.json()) as AppDetailWithSite - const { app_base_url, access_token } = body.site - return `${app_base_url}/workflow/${access_token}` + await expectApiResponseOK(enableResponse, `${enabled ? 'Enable' : 'Disable'} app site ${appId}`) + + const detailResponse = await ctx.get(`/console/api/apps/${appId}`) + await expectApiResponseOK(detailResponse, `Get app site detail for ${appId}`) + return (await detailResponse.json()) as AppDetailWithSite } finally { await ctx.dispose() diff --git a/e2e/support/process.ts b/e2e/support/process.ts index 4de1161b08d..b54108e6855 100644 --- a/e2e/support/process.ts +++ b/e2e/support/process.ts @@ -122,21 +122,23 @@ const waitForProcessExit = (childProcess: ChildProcess, timeoutMs: number) => return } - const timeout = setTimeout(() => { - cleanup() - resolve() - }, timeoutMs) + let timeout: ReturnType - const onExit = () => { - cleanup() - resolve() - } - - const cleanup = () => { + function cleanup() { clearTimeout(timeout) childProcess.off('exit', onExit) } + function onExit() { + cleanup() + resolve() + } + + timeout = setTimeout(() => { + cleanup() + resolve() + }, timeoutMs) + childProcess.once('exit', onExit) }) diff --git a/eslint-suppressions.json b/eslint-suppressions.json index eb99f899a63..4aeb3f71088 100644 --- a/eslint-suppressions.json +++ b/eslint-suppressions.json @@ -1,22 +1,4 @@ { - "e2e/features/support/hooks.ts": { - "no-console": { - "count": 3 - }, - "node/prefer-global/buffer": { - "count": 1 - } - }, - "e2e/scripts/common.ts": { - "node/prefer-global/buffer": { - "count": 2 - } - }, - "e2e/support/process.ts": { - "ts/no-use-before-define": { - "count": 2 - } - }, "packages/migrate-no-unchecked-indexed-access/src/no-unchecked-indexed-access/migrate.ts": { "no-console": { "count": 11