test(e2e): add agent v2 test infrastructure (#38191)

e2e/AGENTS.md

@@ -294,3 +294,13 @@ Or browse the step definition files directly:
 
 - `features/step-definitions/common/` — auth guards and navigation assertions shared by all features
 - `features/step-definitions/<capability>/` — domain-specific steps scoped to a single feature area
+
+## Agent v2 scenarios
+
+Agent v2 scenarios live under `features/agent-v2/` and use the `@agent-v2` capability tag.
+
+The E2E web environment enables Agent v2 through `NEXT_PUBLIC_ENABLE_AGENT_V2=true` in `scripts/common.ts`, because `/roster` routes are guarded by that feature flag.
+
+Use `support/agent.ts` for Agent v2 API fixtures. It owns roster-shaped Agent IDs, configure/access route helpers, composer draft sync, build-draft helpers, publish, API access toggles, and Agent cleanup. Store created roster Agent IDs in `DifyWorld.createdAgentIds`; the shared `After` hook deletes them after each scenario.
+
+Keep Agent v2 step definitions under `features/step-definitions/agent-v2/`. Prefer API setup for prerequisite state, then use Playwright only for user-observable navigation, editing, and assertions.

e2e/features/agent-v2/configure-entry.feature

@@ -0,0 +1,9 @@
+@agent-v2 @authenticated @infra
+Feature: Agent v2 configure entry
+  Scenario: Open the configure page for an Agent v2 test agent
+    Given I am signed in as the default E2E admin
+    And an Agent v2 test agent has been created via API
+    And a minimal Agent v2 composer draft has been synced
+    When I open the Agent v2 configure page
+    Then I should be on the Agent v2 configure page
+    And I should see the Agent v2 configure workspace

e2e/features/step-definitions/agent-v2/configure.steps.ts

@@ -0,0 +1,49 @@
+import type { DifyWorld } from '../../support/world'
+import { Given, Then, When } from '@cucumber/cucumber'
+import { expect } from '@playwright/test'
+import {
+  createTestAgent,
+  getAgentConfigurePath,
+  saveAgentComposerDraft,
+} from '../../../support/agent'
+
+Given('an Agent v2 test agent has been created via API', async function (this: DifyWorld) {
+  const agent = await createTestAgent()
+  this.createdAgentIds.push(agent.id)
+  this.lastCreatedAgentName = agent.name
+  this.lastCreatedAgentRole = agent.role
+})
+
+Given('a minimal Agent v2 composer draft has been synced', async function (this: DifyWorld) {
+  const agentId = this.createdAgentIds.at(-1)
+  if (!agentId)
+    throw new Error('No Agent v2 ID found. Create an Agent v2 test agent first.')
+
+  await saveAgentComposerDraft(agentId)
+})
+
+When('I open the Agent v2 configure page', async function (this: DifyWorld) {
+  const agentId = this.createdAgentIds.at(-1)
+  if (!agentId)
+    throw new Error('No Agent v2 ID found. Create an Agent v2 test agent first.')
+
+  await this.getPage().goto(getAgentConfigurePath(agentId))
+})
+
+Then('I should be on the Agent v2 configure page', async function (this: DifyWorld) {
+  const agentId = this.createdAgentIds.at(-1)
+  if (!agentId)
+    throw new Error('No Agent v2 ID found. Create an Agent v2 test agent first.')
+
+  await expect(this.getPage()).toHaveURL(
+    new RegExp(`/roster/agent/${agentId}/configure(?:\\?.*)?$`),
+  )
+})
+
+Then('I should see the Agent v2 configure workspace', async function (this: DifyWorld) {
+  const page = this.getPage()
+
+  await expect(page.getByRole('region', { name: 'Configure' })).toBeVisible({ timeout: 30_000 })
+  await expect(page.getByRole('heading', { name: 'Configure' })).toBeVisible()
+  await expect(page.getByText(this.lastCreatedAgentName!)).toBeVisible()
+})

e2e/features/support/hooks.ts

@@ -1,4 +1,5 @@
 import type { Browser } from '@playwright/test'
+import type { Buffer } from 'node:buffer'
 import type { DifyWorld } from './world'
 import { mkdir, writeFile } from 'node:fs/promises'
 import path from 'node:path'
@@ -6,6 +7,7 @@ import { fileURLToPath } from 'node:url'
 import { After, AfterAll, Before, BeforeAll, setDefaultTimeout, Status } from '@cucumber/cucumber'
 import { chromium } from '@playwright/test'
 import { AUTH_BOOTSTRAP_TIMEOUT_MS, ensureAuthenticatedState } from '../../fixtures/auth'
+import { deleteTestAgent } from '../../support/agent'
 import { deleteTestApp } from '../../support/api'
 import { baseURL, cucumberHeadless, cucumberSlowMo } from '../../test-env'
 
@@ -41,7 +43,7 @@ BeforeAll({ timeout: AUTH_BOOTSTRAP_TIMEOUT_MS }, async () => {
     slowMo: cucumberSlowMo,
   })
 
-  console.log(`[e2e] session cache bootstrap against ${baseURL}`)
+  console.warn(`[e2e] session cache bootstrap against ${baseURL}`)
   await ensureAuthenticatedState(browser, baseURL)
 })
 
@@ -58,7 +60,7 @@ Before(async function (this: DifyWorld, { pickle }) {
   this.scenarioStartedAt = Date.now()
 
   const tags = pickle.tags.map(tag => tag.name).join(' ')
-  console.log(`[e2e] start ${pickle.name}${tags ? ` ${tags}` : ''}`)
+  console.warn(`[e2e] start ${pickle.name}${tags ? ` ${tags}` : ''}`)
 })
 
 After(async function (this: DifyWorld, { pickle, result }) {
@@ -85,10 +87,11 @@ After(async function (this: DifyWorld, { pickle, result }) {
   }
 
   const status = result?.status || 'UNKNOWN'
-  console.log(
+  console.warn(
     `[e2e] end ${pickle.name} status=${status}${elapsedMs ? ` durationMs=${elapsedMs}` : ''}`,
   )
 
+  for (const id of this.createdAgentIds) await deleteTestAgent(id).catch(() => {})
   for (const id of this.createdAppIds) await deleteTestApp(id).catch(() => {})
 
   await this.closeSession()

e2e/features/support/world.ts

@@ -13,7 +13,10 @@ export class DifyWorld extends World {
   scenarioStartedAt: number | undefined
   session: AuthSessionMetadata | undefined
   lastCreatedAppName: string | undefined
+  lastCreatedAgentName: string | undefined
+  lastCreatedAgentRole: string | undefined
   createdAppIds: string[] = []
+  createdAgentIds: string[] = []
   capturedDownloads: Download[] = []
   shareURL: string | undefined
 
@@ -26,7 +29,10 @@ export class DifyWorld extends World {
     this.consoleErrors = []
     this.pageErrors = []
     this.lastCreatedAppName = undefined
+    this.lastCreatedAgentName = undefined
+    this.lastCreatedAgentRole = undefined
     this.createdAppIds = []
+    this.createdAgentIds = []
     this.capturedDownloads = []
     this.shareURL = undefined
   }

e2e/scripts/common.ts

@@ -1,3 +1,4 @@
+import type { Buffer } from 'node:buffer'
 import type { ChildProcess } from 'node:child_process'
 import { spawn } from 'node:child_process'
 import { createHash } from 'node:crypto'
@@ -42,6 +43,7 @@ export const webEnvExampleFile = path.join(webDir, '.env.example')
 export const apiEnvExampleFile = path.join(apiDir, 'tests', 'integration_tests', '.env.example')
 export const e2eWebEnvOverrides = {
   NEXT_PUBLIC_API_PREFIX: 'http://127.0.0.1:5001/console/api',
+  NEXT_PUBLIC_ENABLE_AGENT_V2: 'true',
   NEXT_PUBLIC_PUBLIC_API_PREFIX: 'http://127.0.0.1:5001/api',
 } satisfies Record<string, string>
 

e2e/support/agent.ts

@@ -0,0 +1,229 @@
+import { createApiContext, expectApiResponseOK, setAppSiteEnabled } from './api'
+
+export type AgentSeed = {
+  app_id?: string
+  backing_app_id?: string
+  description?: string
+  enable_site?: boolean
+  id: string
+  name: string
+  role?: string
+  site?: {
+    access_token?: string | null
+    app_base_url?: string | null
+    code?: string | null
+  } | null
+}
+
+export type AgentSoulConfig = Record<string, unknown>
+
+export type AgentComposerResponse = {
+  agent_soul?: AgentSoulConfig
+}
+
+export type AgentBuildDraftResponse = {
+  agent_soul: AgentSoulConfig
+  draft: Record<string, unknown>
+  variant: 'agent_app'
+}
+
+export type AgentApiAccess = {
+  api_key_count: number
+  api_reference_url: string
+  endpoint: string
+  enabled: boolean
+  files_upload_endpoint: string
+}
+
+export type AgentApiKey = {
+  id: string
+  token?: string
+}
+
+export const defaultAgentSoulConfig: AgentSoulConfig = {
+  prompt: {
+    system_prompt: 'You are a Dify Agent E2E test assistant.',
+  },
+}
+
+export const getAgentConfigurePath = (agentId: string) => `/roster/agent/${agentId}/configure`
+export const getAgentAccessPath = (agentId: string) => `/roster/agent/${agentId}/access`
+
+export async function createTestAgent({
+  description = 'Created by Dify E2E.',
+  name = `E2E Agent ${Date.now()}`,
+  role = 'E2E test assistant',
+}: {
+  description?: string
+  name?: string
+  role?: string
+} = {}): Promise<AgentSeed> {
+  const ctx = await createApiContext()
+  try {
+    const response = await ctx.post('/console/api/agent', {
+      data: {
+        description,
+        icon: '🤖',
+        icon_background: '#FFEAD5',
+        icon_type: 'emoji',
+        name,
+        role,
+      },
+    })
+    await expectApiResponseOK(response, 'Create Agent v2 test agent')
+    return (await response.json()) as AgentSeed
+  }
+  finally {
+    await ctx.dispose()
+  }
+}
+
+export async function getTestAgent(agentId: string): Promise<AgentSeed> {
+  const ctx = await createApiContext()
+  try {
+    const response = await ctx.get(`/console/api/agent/${agentId}`)
+    await expectApiResponseOK(response, `Get Agent v2 test agent ${agentId}`)
+    return (await response.json()) as AgentSeed
+  }
+  finally {
+    await ctx.dispose()
+  }
+}
+
+export async function deleteTestAgent(agentId: string): Promise<void> {
+  const ctx = await createApiContext()
+  try {
+    const response = await ctx.delete(`/console/api/agent/${agentId}`)
+    await expectApiResponseOK(response, `Delete Agent v2 test agent ${agentId}`)
+  }
+  finally {
+    await ctx.dispose()
+  }
+}
+
+export async function saveAgentComposerDraft(
+  agentId: string,
+  agentSoul: AgentSoulConfig = defaultAgentSoulConfig,
+): Promise<AgentComposerResponse> {
+  const ctx = await createApiContext()
+  try {
+    const response = await ctx.put(`/console/api/agent/${agentId}/composer`, {
+      data: {
+        agent_soul: agentSoul,
+        save_strategy: 'save_to_current_version',
+        variant: 'agent_app',
+      },
+    })
+    await expectApiResponseOK(response, `Save Agent v2 composer draft for ${agentId}`)
+    return (await response.json()) as AgentComposerResponse
+  }
+  finally {
+    await ctx.dispose()
+  }
+}
+
+export async function checkoutAgentBuildDraft(agentId: string): Promise<AgentBuildDraftResponse> {
+  const ctx = await createApiContext()
+  try {
+    const response = await ctx.post(`/console/api/agent/${agentId}/build-draft/checkout`, {
+      data: { force: true },
+    })
+    await expectApiResponseOK(response, `Checkout Agent v2 build draft for ${agentId}`)
+    return (await response.json()) as AgentBuildDraftResponse
+  }
+  finally {
+    await ctx.dispose()
+  }
+}
+
+export async function discardAgentBuildDraft(agentId: string): Promise<void> {
+  const ctx = await createApiContext()
+  try {
+    const response = await ctx.delete(`/console/api/agent/${agentId}/build-draft`)
+    await expectApiResponseOK(response, `Discard Agent v2 build draft for ${agentId}`)
+  }
+  finally {
+    await ctx.dispose()
+  }
+}
+
+export async function publishAgent(agentId: string, versionNote = 'E2E publish'): Promise<void> {
+  const ctx = await createApiContext()
+  try {
+    const response = await ctx.post(`/console/api/agent/${agentId}/publish`, {
+      data: { version_note: versionNote },
+    })
+    await expectApiResponseOK(response, `Publish Agent v2 test agent ${agentId}`)
+  }
+  finally {
+    await ctx.dispose()
+  }
+}
+
+export async function enableAgentSiteAndGetURL(agentId: string): Promise<string> {
+  const agent = await getTestAgent(agentId)
+  const appId = agent.app_id ?? agent.backing_app_id
+  if (!appId)
+    throw new Error(`Agent v2 ${agentId} does not expose a backing app ID.`)
+
+  const appDetail = await setAppSiteEnabled(appId, true)
+  const token = agent.site?.access_token ?? agent.site?.code ?? appDetail.site.access_token
+  const baseURL = agent.site?.app_base_url ?? appDetail.site.app_base_url
+
+  return `${baseURL.replace(/\/$/, '')}/agent/${token}`
+}
+
+export async function getAgentApiAccess(agentId: string): Promise<AgentApiAccess> {
+  const ctx = await createApiContext()
+  try {
+    const response = await ctx.get(`/console/api/agent/${agentId}/api-access`)
+    await expectApiResponseOK(response, `Get Agent v2 API access for ${agentId}`)
+    return (await response.json()) as AgentApiAccess
+  }
+  finally {
+    await ctx.dispose()
+  }
+}
+
+export async function setAgentApiAccess(
+  agentId: string,
+  enabled: boolean,
+): Promise<AgentApiAccess> {
+  const ctx = await createApiContext()
+  try {
+    const response = await ctx.post(`/console/api/agent/${agentId}/api-enable`, {
+      data: { enable_api: enabled },
+    })
+    await expectApiResponseOK(
+      response,
+      `${enabled ? 'Enable' : 'Disable'} Agent v2 API access for ${agentId}`,
+    )
+    return (await response.json()) as AgentApiAccess
+  }
+  finally {
+    await ctx.dispose()
+  }
+}
+
+export async function createAgentApiKey(agentId: string): Promise<AgentApiKey> {
+  const ctx = await createApiContext()
+  try {
+    const response = await ctx.post(`/console/api/agent/${agentId}/api-keys`)
+    await expectApiResponseOK(response, `Create Agent v2 API key for ${agentId}`)
+    return (await response.json()) as AgentApiKey
+  }
+  finally {
+    await ctx.dispose()
+  }
+}
+
+export async function deleteAgentApiKey(agentId: string, apiKeyId: string): Promise<void> {
+  const ctx = await createApiContext()
+  try {
+    const response = await ctx.delete(`/console/api/agent/${agentId}/api-keys/${apiKeyId}`)
+    await expectApiResponseOK(response, `Delete Agent v2 API key ${apiKeyId} for ${agentId}`)
+  }
+  finally {
+    await ctx.dispose()
+  }
+}

e2e/support/api.ts

@@ -1,3 +1,4 @@
+import type { APIResponse } from '@playwright/test'
 import { readFile } from 'node:fs/promises'
 import { request } from '@playwright/test'
 import { authStatePath } from '../fixtures/auth'
@@ -7,7 +8,7 @@ type StorageState = {
   cookies: Array<{ name: string, value: string }>
 }
 
-async function createApiContext() {
+export async function createApiContext() {
   const state = JSON.parse(await readFile(authStatePath, 'utf8')) as StorageState
   const csrfToken = state.cookies.find(c => c.name.endsWith('csrf_token'))?.value ?? ''
 
@@ -18,6 +19,14 @@ async function createApiContext() {
   })
 }
 
+export async function expectApiResponseOK(response: APIResponse, action: string): Promise<void> {
+  if (response.ok())
+    return
+
+  const body = await response.text().catch(() => '')
+  throw new Error(`${action} failed with ${response.status()} ${response.statusText()}: ${body}`)
+}
+
 export type AppSeed = {
   id: string
   name: string
@@ -141,20 +150,34 @@ export async function publishWorkflowApp(appId: string): Promise<void> {
   }
 }
 
-type AppDetailWithSite = {
+export type AppDetailWithSite = {
+  mode?: string
   site: { access_token: string, app_base_url: string, enable_site: boolean }
 }
 
+export function getAppSiteURL({ mode, site }: AppDetailWithSite): string {
+  const webAppMode = mode === 'completion' || mode === 'workflow' ? mode : 'chat'
+  return `${site.app_base_url}/${webAppMode}/${site.access_token}`
+}
+
 export async function enableAppSiteAndGetURL(appId: string): Promise<string> {
+  return getAppSiteURL(await setAppSiteEnabled(appId, true))
+}
+
+export async function setAppSiteEnabled(
+  appId: string,
+  enabled: boolean,
+): Promise<AppDetailWithSite> {
   const ctx = await createApiContext()
   try {
-    await ctx.post(`/console/api/apps/${appId}/site-enable`, {
-      data: { enable_site: true },
+    const enableResponse = await ctx.post(`/console/api/apps/${appId}/site-enable`, {
+      data: { enable_site: enabled },
     })
-    const res = await ctx.get(`/console/api/apps/${appId}`)
-    const body = (await res.json()) as AppDetailWithSite
-    const { app_base_url, access_token } = body.site
-    return `${app_base_url}/workflow/${access_token}`
+    await expectApiResponseOK(enableResponse, `${enabled ? 'Enable' : 'Disable'} app site ${appId}`)
+
+    const detailResponse = await ctx.get(`/console/api/apps/${appId}`)
+    await expectApiResponseOK(detailResponse, `Get app site detail for ${appId}`)
+    return (await detailResponse.json()) as AppDetailWithSite
   }
   finally {
     await ctx.dispose()

e2e/support/process.ts

@@ -122,21 +122,23 @@ const waitForProcessExit = (childProcess: ChildProcess, timeoutMs: number) =>
       return
     }
 
-    const timeout = setTimeout(() => {
-      cleanup()
-      resolve()
-    }, timeoutMs)
+    let timeout: ReturnType<typeof setTimeout>
 
-    const onExit = () => {
-      cleanup()
-      resolve()
-    }
-
-    const cleanup = () => {
+    function cleanup() {
       clearTimeout(timeout)
       childProcess.off('exit', onExit)
     }
 
+    function onExit() {
+      cleanup()
+      resolve()
+    }
+
+    timeout = setTimeout(() => {
+      cleanup()
+      resolve()
+    }, timeoutMs)
+
     childProcess.once('exit', onExit)
   })
 

eslint-suppressions.json

@@ -1,22 +1,4 @@
 {
-  "e2e/features/support/hooks.ts": {
-    "no-console": {
-      "count": 3
-    },
-    "node/prefer-global/buffer": {
-      "count": 1
-    }
-  },
-  "e2e/scripts/common.ts": {
-    "node/prefer-global/buffer": {
-      "count": 2
-    }
-  },
-  "e2e/support/process.ts": {
-    "ts/no-use-before-define": {
-      "count": 2
-    }
-  },
   "packages/migrate-no-unchecked-indexed-access/src/no-unchecked-indexed-access/migrate.ts": {
     "no-console": {
       "count": 11