opensource/dify
2
0
Fork 0
mirror of https://github.com/langgenius/dify.git synced 2026-07-19 16:38:31 +08:00
Code Issues Packages Projects Releases Wiki Activity

test(e2e): enforce behavior-driven quality gates (#39148)

Browse Source
This commit is contained in:
yyh 2026-07-16 21:42:10 +08:00 committed by GitHub
parent 33fe0dfd60
commit 48e536ba39
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
73 changed files with 915 additions and 2435 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
.github/workflows/web-e2e.yml vendored
View File

@ -59,7 +59,6 @@ jobs:
E2E_ADMIN_EMAIL: e2e-admin@example.com
E2E_ADMIN_NAME: E2E Admin
E2E_ADMIN_PASSWORD: E2eAdmin12345
E2E_CUCUMBER_REPORT_PROFILE: core
E2E_FORCE_WEB_BUILD: "1"
E2E_INIT_PASSWORD: E2eInit12345
run: vp run e2e:full
@ -81,7 +80,6 @@ jobs:
E2E_ADMIN_NAME: E2E Admin
E2E_ADMIN_PASSWORD: E2eAdmin12345
E2E_BROWSER: webkit
E2E_CUCUMBER_REPORT_PROFILE: webkit-browser-smoke
E2E_INIT_PASSWORD: E2eInit12345
run: |
teardown_webkit_smoke() {
@ -110,7 +108,7 @@ jobs:
mv e2e/.logs e2e/.logs-webkit
fi
- name: Run external runtime E2E tests
- name: Run prepared and external runtime E2E tests
if: ${{ inputs.run-external-runtime }}
working-directory: ./e2e
env:
@ -120,13 +118,10 @@ jobs:
E2E_AGENT_DECISION_MODEL_NAME: ${{ vars.E2E_AGENT_DECISION_MODEL_NAME || 'gpt-5.5' }}
E2E_AGENT_DECISION_MODEL_PROVIDER: ${{ vars.E2E_AGENT_DECISION_MODEL_PROVIDER || 'openai' }}
E2E_AGENT_DECISION_MODEL_TYPE: ${{ vars.E2E_AGENT_DECISION_MODEL_TYPE || 'llm' }}
E2E_CUCUMBER_REPORT_PROFILE: external
E2E_EXTERNAL_RUNTIME_SEED_SPECS: ${{ vars.E2E_EXTERNAL_RUNTIME_SEED_SPECS }}
E2E_FORCE_WEB_BUILD: "1"
E2E_INIT_PASSWORD: E2eInit12345
E2E_MARKETPLACE_API_URL: ${{ vars.E2E_MARKETPLACE_API_URL }}
E2E_MARKETPLACE_PLUGIN_IDS: ${{ vars.E2E_MARKETPLACE_PLUGIN_IDS }}
E2E_MARKETPLACE_PLUGIN_UNIQUE_IDENTIFIERS: ${{ vars.E2E_MARKETPLACE_PLUGIN_UNIQUE_IDENTIFIERS }}
E2E_MODEL_PROVIDER_CREDENTIALS_JSON: ${{ secrets.E2E_MODEL_PROVIDER_CREDENTIALS_JSON }}
E2E_SPEECH_TO_TEXT_MODEL_NAME: ${{ vars.E2E_SPEECH_TO_TEXT_MODEL_NAME || 'gpt-4o-mini-transcribe' }}
E2E_SPEECH_TO_TEXT_MODEL_PROVIDER: ${{ vars.E2E_SPEECH_TO_TEXT_MODEL_PROVIDER || 'openai' }}
@ -164,8 +159,8 @@ jobs:
trap teardown_external_runtime EXIT
vp run e2e:middleware:up
vp run e2e:external:prepare
vp run e2e:external
vp run e2e:post-merge:prepare
vp run e2e:post-merge
- name: Upload Cucumber report
if: ${{ !cancelled() }}

19
e2e/.env.example
View File

@ -17,11 +17,6 @@ E2E_SPEECH_TO_TEXT_MODEL_NAME=gpt-4o-mini-transcribe
E2E_MODEL_PROVIDER_CREDENTIALS_JSON='{"openai_api_key":"replace-with-real-key"}'
# Optional: external runtime CI/local profile. Defaults prepare Agent V2 runtime
# seed resources and then runs every @external-model/@external-tool scenario.
# E2E_EXTERNAL_RUNTIME_SEED_SPECS=agent-v2:external-runtime
# E2E_EXTERNAL_RUNTIME_TAGS=(@external-model or @external-tool) and not @feature-gated and not @skip and not @preview
# Optional: start the standalone dify-agent backend for Agent V2 runtime scenarios.
# This is required for scenarios tagged @agent-backend-runtime. It is separate
# from @external-model because other external-model scenarios may not use dify-agent.
@ -36,18 +31,6 @@ E2E_MODEL_PROVIDER_CREDENTIALS_JSON='{"openai_api_key":"replace-with-real-key"}'
# E2E_SHELLCTL_AUTH_TOKEN=
# Optional: Agent V2 seed installs the latest OpenAI, JSON Process, and Tavily
# marketplace plugins by default. Override ids when testing a different provider
# set, or pin exact marketplace package identifiers for reproducible CI runs.
# marketplace plugins by default. Override ids when testing a different provider set.
# E2E_MARKETPLACE_API_URL=https://marketplace.dify.ai
# E2E_MARKETPLACE_PLUGIN_IDS=langgenius/openai,langgenius/json_process,langgenius/tavily
# E2E_MARKETPLACE_PLUGIN_UNIQUE_IDENTIFIERS=
# Optional: required only for OAuth tool fixture scenarios.
# E2E_OAUTH_TOOL_CREDENTIAL_ID=
# E2E_OAUTH_TOOL_PROVIDER=
# E2E_OAUTH_TOOL_NAME=
# Optional: required only for broken model recovery scenarios.
# E2E_BROKEN_MODEL_PROVIDER=
# E2E_BROKEN_MODEL_NAME=broken-model
# E2E_BROKEN_MODEL_TYPE=llm

45
e2e/AGENTS.md
View File

@ -35,18 +35,18 @@ Run only one `pnpm -C e2e e2e*` process against a local workspace at a time. Sep
Use root lint plus the package type check as the default local verification step after editing E2E TypeScript, Cucumber support code, or feature glue:
```bash
vpr lint --fix --quiet
vp lint --fix --quiet
pnpm -C e2e type-check
```
Common commands:
```bash
# authenticated-only regression (default excludes @fresh)
# deterministic regression against an initialized instance
# expects backend API, frontend artifact, and middleware stack to already be running
pnpm -C e2e e2e
# full reset + fresh install + authenticated scenarios
# reset, initialize, and run deterministic scenarios
# starts required middleware/dependencies for you
pnpm -C e2e e2e:full
@ -83,8 +83,8 @@ flowchart TD
C --> D["Cucumber loads config, steps, and support modules"]
D --> E["BeforeAll bootstraps shared auth state via /install"]
E --> F{"Which command is running?"}
F -->|`pnpm -C e2e e2e`| G["Run config default tags: not @fresh and not @skip and not @preview and not @external-model and not @external-tool"]
F -->|`pnpm -C e2e e2e:full*`| H["Override tags to not @skip and not @preview and not @external-model and not @external-tool"]
F -->|`pnpm -C e2e e2e`| G["Run deterministic scenarios; exclude @prepared and external runtime"]
F -->|`pnpm -C e2e e2e:full*`| H["Reset and run deterministic scenarios; exclude @prepared and external runtime"]
G --> I["Per-scenario BrowserContext from shared browser"]
H --> I
I --> J["Failure artifacts written to cucumber-report/artifacts"]
@ -114,7 +114,7 @@ Behavior depends on instance state:
- uninitialized instance: completes install and stores authenticated state
- initialized instance: signs in and reuses authenticated state
Because of that, the `@fresh` install scenario only runs in the `pnpm -C e2e e2e:full*` flows. The default `pnpm -C e2e e2e*` flows exclude `@fresh`, `@preview`, `@external-model`, and `@external-tool` via Cucumber config tags so they can be re-run against an already initialized instance while keeping Builder Preview and real external runtime scenarios opt-in.
The `pnpm -C e2e e2e:full*` flows prove reset and authentication bootstrap by failing setup when initialization cannot complete; they do not model bootstrap state as a Gherkin scenario. Deterministic runs exclude `@prepared`, `@external-model`, and `@external-tool`. Post-merge first seeds required fixtures, then runs prepared and external scenarios.
Reset all persisted E2E state:
@ -175,7 +175,7 @@ pnpm -C e2e e2e:middleware:down
Artifacts and diagnostics:
- `cucumber-report/report.html`: HTML report
- `cucumber-report/report.json`: JSON report
- `cucumber-report/report.ndjson`: Cucumber Messages report
- `cucumber-report/artifacts/`: failure screenshots and HTML captures
- `cucumber-report-non-external/`: Chromium core report preserved before later CI lanes
- `cucumber-report-webkit/`: focused WebKit keyboard/browser smoke report
@ -185,12 +185,9 @@ Artifacts and diagnostics:
- `.logs-webkit/`: focused WebKit lane logs
- `seed-report/`: JSON readiness reports emitted by external runtime seed packs
CI enables a JSON report gate after Cucumber exits. The gate asserts minimum selected and passed
scenario counts, maximum skipped counts, and zero unexplained skips. A skipped scenario counts as
an explained blocked precondition only when its skipped step attaches a `Blocked precondition:`
reason. This keeps feature-gated and preflight readiness reporting visible without allowing a
zero-coverage or silently skipped CI run to pass. Set `E2E_CUCUMBER_REPORT_PROFILE` to select the
checked-in `core`, `webkit-browser-smoke`, or `external` thresholds and allowed blocked tags.
Cucumber's exit status is the behavior gate. The runner also requires at least one
`testCaseStarted` message so an empty or broken tag selector cannot pass silently. Do not add
scenario-count baselines or skipped-scenario allowlists.
Open the HTML report locally with:
@ -206,7 +203,7 @@ open cucumber-report/report.html
1. Add step definitions under `features/step-definitions/<capability>/`
1. Reuse existing steps from `common/` and other definition files before writing new ones
1. Run with `pnpm -C e2e e2e -- --tags @your-tag` to verify
1. Run `vpr lint --fix --quiet` from the repository root and `pnpm -C e2e type-check` before committing
1. Run `vp lint --fix --quiet` from the repository root and `pnpm -C e2e type-check` before committing
### Feature file conventions
@ -226,14 +223,12 @@ Feature: Create dataset
- default behavior — scenarios run with the shared authenticated storageState unless marked otherwise
- `@unauthenticated` — uses a clean BrowserContext with no cookies or storage
- `@authenticated` — optional intent tag for readability or selective runs; it does not currently change hook behavior on its own
- `@fresh` — only runs in `e2e:full` mode (requires uninitialized instance)
- `@prepared` — deterministic user behavior that requires the strict post-merge seed profile
- `@external-model` — scenario execution can call a real model provider. Use this only for runtime requests, not for scenarios that only require an active model fixture.
- `@external-tool` — scenario execution can call a real third-party tool provider. Use this only for runtime tool execution, not for plugin installation, discovery, or local deterministic tools.
- `@microphone` — runs the scenario in an isolated Chromium instance backed by the checked-in fake audio fixture and grants microphone permission only to that scenario context.
- `@browser-smoke` — focused keyboard and navigation coverage that runs in Chromium with the core suite and again in WebKit on CI.
- `@skip` — excluded from all runs
External runtime commands are opt-in. `pnpm -C e2e e2e:external:prepare` reads `E2E_EXTERNAL_RUNTIME_SEED_SPECS`, defaulting to `agent-v2:external-runtime`, and runs the matching seed packs before the external suite. `pnpm -C e2e e2e:external` reads `E2E_EXTERNAL_RUNTIME_TAGS`, defaulting to `(@external-model or @external-tool) and not @feature-gated and not @skip and not @preview`.
External runtime commands are opt-in. `pnpm -C e2e e2e:external:prepare` prepares the fixed Agent v2 external-runtime seed and `pnpm -C e2e e2e:external` runs every `@external-model` or `@external-tool` scenario. CI uses `e2e:post-merge:prepare` followed by `e2e:post-merge` to run `@prepared` and external scenarios against one strict seed.
The Agent v2 external runtime seed also prepares the workspace default Speech-to-Text model. `E2E_SPEECH_TO_TEXT_MODEL_PROVIDER` and `E2E_SPEECH_TO_TEXT_MODEL_NAME` select an existing model or the model configured through `E2E_MODEL_PROVIDER_CREDENTIALS_JSON`; they default to `openai` and `gpt-4o-mini-transcribe`.
@ -328,25 +323,21 @@ The `After` hook automatically captures diagnostics for failed, ambiguous, pendi
Artifacts are saved to `cucumber-report/artifacts/` and attached to the HTML report. No extra code needed in step definitions.
Skipped preflight scenarios should attach the blocked-precondition reason to the skipped step and should not create screenshot or HTML artifacts.
### Seed resources and preflight checks
### Seed and fixture contracts
Use `support/naming.ts` for generated test resource names. New app, Agent, dataset, file, or credential seeds should start with `E2E` so local and shared environments can identify disposable resources.
Use `fixtures/test-materials/` for checked-in files that scenarios upload, preview, index, or retrieve. Keep these fixtures small and deterministic, and use `support/test-materials.ts` to resolve their absolute paths.
Use scoped feature support for scenarios that require optional external resources such as a model provider, plugin/tool credential, knowledge base seed, or fixed app. Prefer an explicit `Given` step that returns a skipped result with a clear blocked-precondition reason over hidden setup in hooks.
Seed scripts own long-lived models, plugins, datasets, and fixed apps. Selected scenarios may resolve and verify those fixtures through explicit `Given` steps, but a missing or drifted fixture must fail the scenario. Do not represent environment readiness as Gherkin scenarios and do not conditionally skip behavior.
Treat preflight checks as read-only readiness checks. A preflight step may query the environment, record typed state on `DifyWorld`, attach a blocked-precondition reason, or return `skipped`; it must not create, repair, publish, reconfigure, or mutate shared seed resources. Treat the preflight suite as a readiness and drift report, not as a seed manager. Long-lived resources belong to the environment seed/setup process and need an explicit owner outside individual scenarios.
Keep package-level support limited to broadly reusable primitives such as API clients, naming, fixture path resolution, and cleanup helpers. Feature-specific seed and fixture contracts belong under the owning feature's support folder.
Keep package-level support limited to broadly reusable primitives such as API clients, naming, fixture path resolution, and cleanup helpers. Feature-specific seed contracts and preflight checks belong under the owning feature's support folder.
Use generated API contracts for Console/Web/Service API request, response, and payload shapes. Import the concrete type directly from `@dify/contracts/.../types.gen` when it exists, and do not hand-write duplicate response shapes or wrap generated types in local aliases just to preserve an older helper name. Keep local E2E types only for scenario state, fixture registries, helper input options, preflight resource state, and intentionally narrowed test view models that are not complete API responses.
Use generated API contracts for Console/Web/Service API request, response, and payload shapes. Import the concrete type directly from `@dify/contracts/.../types.gen` when it exists, and do not hand-write duplicate response shapes or wrap generated types in local aliases just to preserve an older helper name. Keep local E2E types only for scenario state, fixture registries, helper input options, and intentionally narrowed test view models that are not complete API responses.
Use typed cleanup fields on `DifyWorld` for resource types created by scenarios, and use `DifyWorld.registerCleanup(...)` when a scenario creates any resource type that is not covered by typed cleanup fields. Typed cleanup should remove child or referencing resources before their owners, such as Agent files before Agents and workflow apps before Agents they reference. Cleanup failures should be attached to the report instead of being swallowed silently. Cleanup callbacks run after typed cleanup queues, even when the scenario fails.
Scenario-owned setup may create disposable apps, Agents, files, credentials, drafts, or access toggles when the scenario owns their lifecycle and cleanup. Do not use scenario setup to silently fix or complete a shared preseeded resource; if a fixed resource is missing or drifted, report it as blocked and route it to the seed owner.
Scenario-owned setup may create disposable apps, Agents, files, credentials, drafts, or access toggles when the scenario owns their lifecycle and cleanup. Do not use scenario setup to silently fix a shared fixture; a missing or drifted fixed resource is a seed failure.
Feature-specific seed contracts, resource readiness rules, tags, and scenario ownership can be documented in one scoped `AGENTS.md` at the feature root when a module becomes large enough to need it. Do not add deeper `AGENTS.md` files unless the nested module becomes independently owned.

6
e2e/cucumber.config.ts
View File

@ -2,8 +2,7 @@ import type { IConfiguration } from '@cucumber/cucumber'
import './scripts/env-register'
const hasCliTags = process.argv.some((arg) => arg === '--tags' || arg.startsWith('--tags='))
const defaultNonExternalTags =
'not @fresh and not @skip and not @preview and not @external-model and not @external-tool'
const defaultNonExternalTags = 'not @prepared and not @external-model and not @external-tool'
const defaultTags =
process.env.E2E_CUCUMBER_TAGS || (hasCliTags ? undefined : defaultNonExternalTags)
@ -12,10 +11,9 @@ const config = {
'progress-bar',
'summary',
'html:./cucumber-report/report.html',
'json:./cucumber-report/report.json',
'message:./cucumber-report/report.ndjson',
],
import: ['./tsx-register.js', 'features/**/*.ts'],
parallel: 1,
paths: ['features/**/*.feature'],
...(defaultTags ? { tags: defaultTags } : {}),
timeout: 60_000,

252
e2e/features/agent-v2/AGENTS.md
View File

@ -1,236 +1,96 @@
# Agent V2 E2E
This file scopes Agent v2 E2E conventions for `e2e/features/agent-v2/` and its step definitions under `e2e/features/step-definitions/agent-v2/`. Keep package-wide runner, lifecycle, hook, fixture, locator, assertion, and cleanup rules in `e2e/AGENTS.md`.
Do not add deeper `AGENTS.md` files unless an Agent v2 submodule becomes independently owned.
This file scopes Agent v2 conventions under `features/agent-v2/` and its step definitions. Package-wide runner, locator, assertion, lifecycle, and cleanup rules live in `e2e/AGENTS.md`.
## Scope
Agent v2 scenarios live under `features/agent-v2/` and use the `@agent-v2` capability tag.
Agent v2 scenarios use the `@agent-v2` capability tag. The E2E web environment enables Agent v2 through `NEXT_PUBLIC_ENABLE_AGENT_V2=true`.
The E2E web environment enables Agent v2 through `NEXT_PUBLIC_ENABLE_AGENT_V2=true` in `scripts/common.ts`, because `/agents` routes are guarded by that feature flag.
Cover user-observable Configure, Build draft, saved configuration, publish, Access Point, files, advanced settings, and runtime behavior. Do not keep readiness-only, unavailable, or permanently skipped scenarios. Use API setup for prerequisites, then assert visible behavior or a persisted public contract.
Preview/Test Run scenarios are not part of the current build-mode slice unless explicitly requested. Current Agent v2 coverage should prioritize Configure, Build draft, saved configuration display, publish state, Access Point, preflight, files, advanced settings, and other build-mode behavior. Published Web app runtime is not Builder Preview; keep it as a separate `@web-app-runtime` slice because it exercises the public app surface and real model-backed responses after publish.
## Execution tags
Use API setup for prerequisite state, then use Playwright only for user-observable navigation, editing, and assertions. Do not make assertions pass by mirroring the current implementation blindly; if a failure exposes a product ambiguity, resource gap, or test-quality problem, identify the owner before changing the test.
- `@core` — stable, non-runtime user behavior.
- `@prepared` — deterministic user behavior that requires the checked-in post-merge seed.
- `@external-model` — execution can call a real model provider.
- `@external-tool` — execution can call a real third-party tool provider.
- `@agent-backend-runtime` — execution requires the standalone `dify-agent` server and shellctl sandbox.
- `@web-app-runtime` — published Web app chat behavior.
- `@service-api-runtime` — Backend service API chat behavior.
- `@microphone` — isolated Chromium context with the fake audio fixture.
## Tags
Fixture tags such as `@stable-model`, `@tool-fixture`, `@knowledge-fixture`, `@full-config-agent`, `@tool-states-agent`, `@dual-retrieval-fixture`, and `@workflow-reference` describe the concrete seeded dependency. They do not change execution by themselves.
Use tags in three layers:
Use `@external-model` and `@external-tool` only for runtime calls. A scenario that merely selects or persists an active model or tool is `@prepared`, not external.
- Capability tags describe the product area: `@build`, `@files`, `@advanced-settings`, `@agent-edit`, `@publish`, `@access-point`, `@output-variables`, and similar tags.
- Execution-scope tags describe how the scenario should be selected: `@core`, `@infra`, `@web-app-runtime`, `@service-api-runtime`, `@agent-backend-runtime`, `@preview`, `@feature-gated`, `@external-model`, and `@external-tool`.
- Narrow fixture or sub-surface tags describe a specific dependency or slice: `@stable-model`, `@agent-decision-model`, `@skill-fixture`, `@web-app-access`, `@workflow-reference`, `@files-limits`, and similar tags.
## Step organization
- `@agent-v2` — required capability tag for all Agent v2 scenarios.
- `@core` — stable non-runtime scenario expected to run in the regular Agent v2 suite when its explicit preconditions are met. Do not apply `@core` to Preview/Test Run, Web app chat runtime, or Backend service API chat runtime scenarios.
- `@infra` — infrastructure or readiness checks.
- `@build` — Build mode and Build draft behavior.
- `@files` — Files section upload, display, and fixture behavior.
- `@files-limits` — stable file limit behavior, currently covering multiple-file drop rejection.
- `@knowledge` — Knowledge Retrieval configuration display, persistence, and reference cleanup.
- `@advanced-settings` — Env Editor, Content Moderation, and related Advanced Settings behavior.
- `@agent-create` — Agent Roster creation and initial Configure navigation.
- `@agent-edit` — saved Agent detail/configuration display surfaces.
- `@publish` — publish and publish-bar state.
- `@access-point` — Web app, Backend service API, and Workflow access surfaces.
- `@stable-model` — active model fixture dependency. Apply this to every scenario that includes `the Agent Builder stable chat model is available` or otherwise requires an active model configured in the workspace.
- `@speech-to-text-model` — active workspace default Speech-to-Text model dependency. Apply this to scenarios that include `the workspace default speech-to-text model is active`.
- `@microphone` — deterministic Chromium fake microphone dependency. The scenario hook launches a fake-audio browser and grants microphone permission only to tagged scenarios.
- `@agent-decision-model` — stronger active model fixture dependency for scenarios that specifically validate Agent autonomous planning or resource-selection behavior, such as generated-query Knowledge Retrieval. Do not use it for scenarios that only need a model to exist or answer a deterministic prompt.
- `@tool-fixture` — preseeded Tool dependency such as `JSON Process / JSON Replace` or `Tavily / Tavily Search`.
- `@skill-fixture` — checked-in or preseeded Skill dependency such as `e2e-summary-skill`.
- `@knowledge-fixture` — preseeded dataset dependency such as `E2E Agent Knowledge Base`.
- `@full-config-agent` — fixed `E2E New Agent Builder Full Config` Agent dependency.
- `@tool-states-agent` — fixed `E2E New Agent Builder Tool States` Agent dependency.
- `@oauth-tool-agent` — fixed `E2E Agent With OAuth Tool` Agent dependency for OAuth2 tool credential preservation.
- `@dual-retrieval-fixture` — fixed dual Knowledge Retrieval Agent dependency.
- `@backend-api-access` — fixed or scenario-owned Backend service API access dependency.
- `@published-web-app` — fixed or scenario-owned published Web app access dependency.
- `@web-app-runtime` — published public Web app runtime behavior. Use it for scenarios that open the public Web app and assert real chat responses. Access Point URL, launch, customization, and settings surfaces remain `@access-point` behavior unless they send messages through the public Web app.
- `@service-api-runtime` — Backend service API runtime behavior. Use it for scenarios that call the published service API and assert real chat responses. Endpoint display, copy, API key, and API reference surfaces remain `@access-point` behavior.
- `@agent-backend-runtime` — standalone `dify-agent` run-server dependency. Apply it only when the scenario goes through Agent v2 runtime execution that requires `AGENT_BACKEND_BASE_URL`, such as Build chat/finalize, published Web app chat, or Backend service API chat. This is separate from `@external-model`; a scenario can call a real model without using `dify-agent`, and a future Agent v2 runtime scenario might exercise `dify-agent` without a paid model provider.
- `@external-model` — package-wide external runtime tag. In Agent v2, apply it to Build chat generation/finalization, published Web app chat, Backend service API chat, and other Agent runtime requests that can invoke a real model provider. Do not use this for scenarios that only require a configured active model.
- `@external-tool` — package-wide external runtime tag. In Agent v2, apply it only when the scenario executes a real external tool provider at runtime. Do not use this for plugin installation, tool discovery, tool configuration persistence, or local deterministic tools.
- `@feature-gated` — product capability is optional. This tag alone does not skip execution; the scenario must include an explicit step that returns `skipped` with a blocked-precondition reason when the feature is unavailable.
Keep steps grouped by user capability:
Use feature-level `@core` only when every scenario in the file is stable, non-runtime, and not feature-gated. If a feature file mixes stable scenarios with runtime, Preview, or feature-gated scenarios, put `@core` only on the stable scenarios. Keep runtime tags scenario-level so the regular core suite cannot inherit them accidentally.
- `configure.steps.ts` — navigation, editing, autosave, and saved draft behavior.
- `build-draft.steps.ts` — checkout, apply, discard, and isolation.
- `files.steps.ts`, `knowledge.steps.ts`, `tools.steps.ts` — resource configuration behavior.
- `advanced-settings.steps.ts`, `env-editor.steps.ts` — supported Advanced Settings behavior.
- `agent-roster.steps.ts`, `agent-edit.steps.ts`, `publish.steps.ts` — Agent lifecycle surfaces.
- `access-point*.steps.ts` — Web app, service API, and Workflow access.
- `fixtures.steps.ts` — strict fixture resolution for behavior scenarios.
- `speech-to-text.steps.ts` — voice input and transcription behavior.
## Step Organization
Cucumber step definitions are globally registered. Do not duplicate step text across files.
Keep Agent v2 step definitions grouped by user capability, not by DOM component or Cucumber keyword:
## World state
- `configure.steps.ts` — common configure navigation, refresh, autosave, and normal draft assertions.
- `build-draft.steps.ts` — Build mode checkout, apply, discard, supported writeback, and Build draft isolation.
- `files.steps.ts` — Files upload, display, and fixture-list assertions.
- `knowledge.steps.ts` — Knowledge Retrieval configuration persistence and reference cleanup.
- `tools.steps.ts` — Tools selector, search, and configuration-boundary behavior.
- `advanced-settings.steps.ts` — common Advanced Settings shell and supported-entry assertions.
- `env-editor.steps.ts` — Env Editor add, import, delete, persistence, and restored-display behavior.
- `content-moderation.steps.ts` — Content Moderation availability, keyword settings, and feature-gated assertions.
- `agent-roster.steps.ts` — Agent Roster creation and Roster-level user actions.
- `agent-edit.steps.ts` — saved Agent detail display assertions.
- `publish.steps.ts` — publish and publish-bar assertions.
- `access-point.steps.ts` — common Access Point navigation and overview.
- `access-point-web-app.steps.ts` — Web app access entrypoints and public Web app assertions.
- `access-point-service-api.steps.ts` — Backend service API entrypoints, keys, API reference, and service requests.
- `access-point-workflow.steps.ts` — Workflow access references.
- `preflight.steps.ts` — explicit `Given` entrypoints for Agent Builder preflight resources.
- `speech-to-text.steps.ts` — Agent Build voice input, multipart request, and transcribed input behavior.
Agent v2 state belongs under `world.agentBuilder`:
Cucumber step definitions are globally registered. Do not duplicate the same step text across files, even if one is written as `Given` and another as `Then`.
- `fixtures` stores resolved models and seeded resources.
- `accessPoint`, `configure`, `speechToText`, and `workflow` store per-scenario state.
## World State
Do not add Agent v2 fields to the top level of `DifyWorld`. Store created Agent IDs, drive files, and tool credentials in the existing typed cleanup fields.
`DifyWorld` owns generic scenario state such as `page`, `context`, errors, downloads, cleanup queues, and created resource IDs.
## Setup boundary
Agent v2 business state belongs under `world.agentBuilder`; do not keep adding Agent v2-specific fields to the top level of `DifyWorld`.
Use `a basic configured Agent v2 test agent has been created via API` for model-free, scenario-owned state. Use `a runnable Agent v2 test agent has been created via API` only after resolving the stable model fixture. Use the agent-decision variant only for autonomous planning or resource-selection behavior.
Use the existing namespace shape:
API setup may create scenario-owned Agents, workflows, drafts, access toggles, and files. Register every created resource for cleanup. Do not mutate a fixed seeded fixture to make a scenario pass.
- `world.agentBuilder.preflight.stableModel`
- `world.agentBuilder.preflight.speechToTextModel`
- `world.agentBuilder.preflight.brokenModel`
- `world.agentBuilder.preflight.preseededResources`
- `world.agentBuilder.accessPoint.serviceApiBaseURL`
- `world.agentBuilder.accessPoint.generatedApiKey`
- `world.agentBuilder.accessPoint.serviceApiResponse`
- `world.agentBuilder.accessPoint.apiReferencePage`
- `world.agentBuilder.accessPoint.webAppPage`
- `world.agentBuilder.accessPoint.webAppURL`
- `world.agentBuilder.accessPoint.workflowReferencePage`
- `world.agentBuilder.accessPoint.composerDraftSnapshot`
- `world.agentBuilder.configure.concurrentPage`
- `world.agentBuilder.speechToText.request`
- `world.agentBuilder.workflow.agentConsolePage`
- `world.agentBuilder.workflow.outputVariables`
Use `the Agent v2 configuration should be saved automatically` for Configure autosave. It waits for the visible publish-bar saved state; do not replace it with network-idle waits or internal store assertions.
Use `features/agent-v2/support/agent.ts` for Agent v2 core API fixtures. It owns roster-shaped Agent IDs, configure/access route helpers, composer draft sync, version details, workflow references, publish, and Agent cleanup. Use `features/agent-v2/support/agent-soul.ts` for reusable Agent Soul fixture configuration, prompts, and model/dataset config builders. Use `features/agent-v2/support/agent-build-draft.ts` for Build draft checkout/save/discard API helpers. Use `features/agent-v2/support/agent-drive.ts` for Agent drive/config file and Skill upload plus cleanup helpers. Use `features/agent-v2/support/access-point.ts` for Web app access, Backend service API access, API keys, and service API request helpers. Store created roster Agent IDs in `DifyWorld.createdAgentIds`; the shared `After` hook deletes them after each scenario.
## Seed and fixture contract
Use `DifyWorld.createdAgentDriveFiles` for Agent drive files committed during a scenario and `DifyWorld.createdBuiltinToolCredentials` for built-in tool credentials created during a scenario. The shared `After` hook deletes Agent drive files first so cleanup also works for scenarios that upload into a preseeded Agent.
Seed scripts create or update environment-owned models, plugins, datasets, Agents, and workflows. `fixtures.steps.ts` resolves and validates those resources before a dependent behavior runs. Missing, inactive, unindexed, or drifted fixtures must throw and fail the scenario; never return `skipped`.
## Setup Boundary
Use `a basic configured Agent v2 test agent has been created via API` when a scenario only needs a created Agent with a composer draft. Do not use that basic shell for runtime, model, tool, skill, knowledge, environment variable, moderation, or output-variable coverage until those resources have explicit seed helpers and readiness checks.
Use `a runnable Agent v2 test agent has been created via API` after `the Agent Builder stable chat model is available` when a scenario needs a real model-backed Agent. The step writes the preflight model into the Agent Soul model config through `features/agent-v2/support/agent-soul.ts` with deterministic E2E model settings; do not duplicate provider/model payload construction in individual steps.
Use `a runnable Agent v2 test agent using the agent-decision model has been created via API` after `the Agent Builder agent-decision chat model is available` only when the scenario is asserting Agent autonomous decision quality rather than generic runtime availability. Keep this scoped to scenarios tagged `@agent-decision-model`.
Use `the Agent v2 configuration should be saved automatically` after UI edits that rely on Configure autosave. It waits for the user-visible publish bar saved state; do not replace it with network-idle waits or internal store checks.
API setup is acceptable for creating scenario-owned Agents, enabling Backend service API, writing composer drafts, seeding Build drafts, and preparing fixed state. The scenario must still assert user-visible behavior or a real persisted product contract through the public Console API. Do not assert only that a setup API call succeeded.
Do not use scenario API setup to repair an environment-owned Agent Builder seed. If a scenario depends on a fixed Agent, dataset, workflow, Skill, Tool, credential, published Web app, or active model, use the matching preflight step to verify it and block when it is missing or drifted. Create or mutate resources only when they are scenario-owned and registered for cleanup.
## API Contract Types
Agent v2 support helpers consume Console API contracts from `@dify/contracts/api/console/.../types.gen`. When a generated request, response, or payload type exists, import and use that exact type name at the helper boundary. Do not keep an old local response type name as an alias for the generated type.
Keep local types for Agent v2 E2E-owned state only, such as `DifyWorld.agentBuilder` state, scenario preflight resource records, fixture registry entries, helper input options, and deliberately narrowed test view models. If an endpoint response needs a field that the generated contract does not expose yet, fix the backend schema and regenerate contracts before broadening E2E types.
## Build Mode And Preview
Build mode scope means:
- Configure page behavior.
- Build draft checkout, pending state, apply, discard, and route isolation.
- Supported Build draft writeback for files, skills, and env.
- Saved configuration display after apply/discard/refresh.
Preview/Test Run scope means:
- Model-backed runtime responses.
- Duplicate run prevention.
- Runtime failure recovery.
- Tool/knowledge hit behavior proven through Agent replies.
Keep Preview/Test Run scenarios out of the current build-mode slice unless the task explicitly reintroduces them.
## Preflight Resources
Agent Builder resource checks live under `features/agent-v2/support/preflight/`. Import from the specific module that owns the resource contract; do not add a preflight barrel file:
- `models.ts`
- `agents.ts`
- `datasets.ts`
- `tools.ts`
- `access.ts`
`preflight.steps.ts` should remain the explicit `Given` entrypoint. Do not move preflight into hidden hooks.
Agent Builder preflight is read-only. It checks long-lived seed resources and records their IDs or normalized metadata for later steps, but it must not create missing resources, toggle fixed access settings, upload missing Skills/files, publish fixed Agents, or patch model/provider credentials. Seed creation and repair belong to the environment setup process, not to Cucumber scenarios.
Treat preseeded Agent Builder resources as environment contracts. Preflight can report that a stable model, dataset, Skill, Tool credential, fixed Agent, published Web app, or workflow reference is missing, inactive, not indexed, or drifted, but it must not repair that drift during a scenario. Seed scripts, CI bootstrap, or the documented environment maintenance flow own creating and keeping those resources valid.
Use `the Agent v2 runtime backend is available` before scenarios tagged `@agent-backend-runtime`. The step checks the standalone `dify-agent` run server through `E2E_AGENT_BACKEND_URL`, `AGENT_BACKEND_BASE_URL`, or the default `http://127.0.0.1:5050` when `E2E_START_AGENT_BACKEND=1`. The E2E runner starts this service with `uv run --project dify-agent --extra server uvicorn dify_agent.server.app:app --host 127.0.0.1 --port 5050` only when `E2E_START_AGENT_BACKEND=1` is explicit. Because Agent App runtime always carries the `dify.config` layer and that layer depends on `dify.shell`, the same runner path also starts the `dify-agent-runtime/docker` shellctl container on `E2E_SHELLCTL_PORT` (default `5004`) and injects `DIFY_AGENT_SHELLCTL_ENTRYPOINT` into the Agent backend. Missing or unreachable runtime backend or shellctl sandbox should be reported as a blocked precondition, not discovered later as a `/build-chat/finalize` 400 or Web app response timeout.
Use `the Agent Builder stable chat model is available` before scenarios that need a real Agent Soul model configuration. This includes true runtime scenarios, model-backed build-mode assertions, and Workflow Agent v2 node setup because the backend rejects Agent nodes without model config. Do not add the model preflight to pure navigation or identity checks unless the setup API itself requires model config. `E2E_STABLE_MODEL_PROVIDER`, `E2E_STABLE_MODEL_NAME`, and optional `E2E_STABLE_MODEL_TYPE` are selectors for a model already configured in the workspace; they are not provider credentials. The step defaults to `openai` / `gpt-5-nano` / `llm`, verifies the selected model is present and `active` through `/console/api/workspaces/current/models/model-types/{type}`, then stores it on `DifyWorld.agentBuilder.preflight.stableModel`.
Use `the workspace default speech-to-text model is active` before real speech-to-text scenarios. The preflight reads `/console/api/workspaces/current/default-model?model_type=speech2text`, verifies that same provider/model is active in the Speech-to-Text model list, and stores it on `DifyWorld.agentBuilder.preflight.speechToTextModel`. It must not configure a model or provider credential. External runtime preparation selects `E2E_SPEECH_TO_TEXT_MODEL_PROVIDER` / `E2E_SPEECH_TO_TEXT_MODEL_NAME`, defaulting to `openai` / `gpt-4o-mini-transcribe`, reuses `E2E_MODEL_PROVIDER_CREDENTIALS_JSON` when provider setup is required, and selects the active model as the workspace default.
Use `the Agent Builder agent-decision chat model is available` before scenarios that need a stronger model to exercise Agent autonomous planning, generated query selection, or tool/resource choice. `E2E_AGENT_DECISION_MODEL_PROVIDER`, `E2E_AGENT_DECISION_MODEL_NAME`, and optional `E2E_AGENT_DECISION_MODEL_TYPE` are selectors for a second active model fixture, defaulting to `openai` / `gpt-5.5` / `llm`. The step stores the model on `DifyWorld.agentBuilder.preflight.agentDecisionModel`. Do not use this fixture as a broad replacement for `@stable-model`; it is intentionally narrower and costlier.
Keep `@stable-model` on Build draft apply scenarios that click `Apply`. The current product path calls `/build-chat/finalize` before applying the draft, and the backend returns `model is required` when the Agent Soul has no model config. Discard-only and pending-draft isolation scenarios can stay model-free when they do not finalize the Build draft.
Do not pass model provider API keys through Cucumber or Playwright env vars. Provider credentials belong to the Dify environment seed/admin setup. If the selected provider/model is missing or inactive, the scenario must be blocked by preflight instead of trying to create or patch provider credentials during the test.
Override the default selector only when a scenario or environment explicitly needs a different stable model:
`@prepared` scenarios are excluded from deterministic PR core. Post-merge runs:
```bash
E2E_STABLE_MODEL_PROVIDER=openai
E2E_STABLE_MODEL_NAME=gpt-5-nano
E2E_STABLE_MODEL_TYPE=llm
E2E_AGENT_DECISION_MODEL_PROVIDER=openai
E2E_AGENT_DECISION_MODEL_NAME=gpt-5.5
E2E_AGENT_DECISION_MODEL_TYPE=llm
pnpm -C e2e e2e:post-merge:prepare
pnpm -C e2e e2e:post-merge
```
Dify may expose OpenAI as either `openai` or a plugin provider ID such as `langgenius/openai/openai`. The preflight accepts both forms for selection and stores the actual Console API provider ID for Agent Soul setup.
The strict seed must finish without blocked tasks. It prepares the stable and decision models, Speech-to-Text default, marketplace plugins, JSON Replace and Tavily tools, ready knowledge base, Full Config Agent, Tool States Agent, Dual Retrieval Agent, and Workflow reference.
Use `the Agent Builder broken chat model is available` before model-recovery scenarios that intentionally start from an invalid model. The step requires `E2E_BROKEN_MODEL_PROVIDER`, defaults `E2E_BROKEN_MODEL_NAME` to `e2e-broken-model`, defaults `E2E_BROKEN_MODEL_TYPE` to `llm`, and only verifies that the model entry exists. The scenario must still assert the user-visible failure and recovery behavior.
Fixture helpers live under `features/agent-v2/support/fixtures/`:
Use `the Agent Builder preseeded Agent "{name}" is available`, `the Agent Builder preseeded workflow "{name}" is available`, `the Agent Builder preseeded dataset "{name}" is available`, and `the Agent Builder preseeded tool "{provider} / {tool}" is available` when a scenario depends on a fixed environment resource. These steps verify the resource through Console APIs, store the result in `DifyWorld.agentBuilder.preflight.preseededResources`, and return `skipped` when the resource is missing.
- `models.ts` — stable, decision, and Speech-to-Text models.
- `agents.ts` — fixed Agent and configuration contracts.
- `datasets.ts` — indexed knowledge contract.
- `tools.ts` — installed built-in tool contract.
- `access.ts` — Workflow reference contract.
- `agent-backend.ts` — runtime server and shellctl readiness.
Use `the Agent Builder preseeded dataset "{name}" is indexed and ready` for knowledge retrieval scenarios that require a completed knowledge base. It verifies that the dataset exists, has documents, all listed documents are available, and every document indexing status is `completed`. For `E2E Agent Knowledge Base`, it also verifies through the Console segment list API that at least one enabled segment contains `AGENT_KNOWLEDGE_PASS`.
The stable model selectors default to `openai` / `gpt-5-nano` / `llm`. The decision model defaults to `openai` / `gpt-5.5` / `llm`. The Speech-to-Text model defaults to `openai` / `gpt-4o-mini-transcribe`. Provider credentials belong to seed/admin setup through `E2E_MODEL_PROVIDER_CREDENTIALS_JSON`, never to Cucumber steps.
Use `the Agent Builder preseeded dataset "{name}" is indexing` for failure-recovery scenarios that require an indexing or queued knowledge base. It verifies at least one document is in `waiting`, `parsing`, `cleaning`, `splitting`, or `indexing`.
The Full Config Agent contract includes the stable model, prompt marker, checked-in files, Summary Skill, JSON Replace tool, and indexed knowledge reference. Tool States includes Summary Skill, JSON Replace, Tavily, and its credential reference. Dual Retrieval includes generated-query and custom-query knowledge sets. Workflow reference verifies the same Console API used by the Access Point table.
`e2e-summary-skill` has two separate E2E contracts. The checked-in package under `e2e/fixtures/test-materials/e2e-summary-skill/SKILL.md` is used by scenario-owned Agents to verify that the Skill package can be uploaded to an Agent drive. Fixed display/configuration scenarios use `e2e-summary-skill` as a preseeded resource: the environment-owned `E2E New Agent Builder Full Config` Agent must already include that drive-backed Skill before the scenario starts. Do not mutate fixed preseeded Agents during a scenario to add missing Skills.
## Runtime contract
Use `the Agent Builder preseeded Agent "{agent}" includes drive skill "{skill}"` to verify that a fixed Agent has a drive-backed Skill attached. If it is missing, return a blocked precondition owned by seed/product instead of uploading the Skill into the fixed Agent.
Scenarios tagged `@agent-backend-runtime` must include `the Agent v2 runtime backend is available`. Run with `E2E_START_AGENT_BACKEND=1` to start `dify-agent` and shellctl, or point `E2E_AGENT_BACKEND_URL` / `AGENT_BACKEND_BASE_URL` to an existing server.
Use `the Agent Builder preseeded Agent "{agent}" has Backend service API access with an API key` to verify that a fixed Agent has Backend service API enabled and at least one key. The API key step does not validate a human-readable key name because the Console API key response does not expose one.
Use the stable model for generic runtime behavior and the decision model only where model reasoning quality is part of the contract. Do not broaden external runtime tags to cover configuration-only scenarios.
Use `the Agent Builder preseeded Agent "{agent}" includes the core fixture configuration` for the fixed Full Config Agent prerequisite. It composes the stable model, Summary Skill, JSON Replace tool, and indexed knowledge-base preflights, then reads `/console/api/agent/{agent_id}/composer` to verify the Agent Soul contains the selected model, prompt success token, required file fixtures, JSON Replace tool entry, and knowledge dataset reference. Do not use this step for Agent node output variables; those live in workflow node-job `declared_outputs`, not the roster Agent App composer response.
## Build and Preview
Use `the Agent Builder preseeded Agent "{agent}" includes the tool state fixture configuration` for the fixed Tool States Agent prerequisite. It composes the Summary Skill, JSON Replace tool, and Tavily Search tool preflights, then reads `/console/api/agent/{agent_id}/composer` to verify the Agent Soul includes JSON Replace, Tavily Search, and a Tavily credential reference. This proves the seed is configured to exercise tool status UI; keep actual invalid-credential errors in dependent user-visible configuration or runtime scenarios.
Build mode covers Configure and Build draft persistence. Preview/Test Run covers real generation, runtime failure recovery, and tool or knowledge hits proven through replies. Do not add Preview scenarios until they are executable in the selected CI lane.
Use `the Agent Builder preseeded Agent "{agent}" includes an OAuth2 tool credential` for the fixed OAuth Tool Agent prerequisite. It reads the Agent composer and verifies at least one Dify tool has `credential_type: oauth2` with a `credential_ref.id`. Scenarios that need to save or mutate this configuration should copy that tool config into a scenario-owned Agent instead of mutating the fixed preseeded Agent.
## API contracts
Use `the Agent Builder preseeded Agent "{agent}" includes the dual retrieval fixture configuration` for the fixed Dual Retrieval Agent prerequisite. It composes the indexed knowledge-base preflight, then reads `/console/api/agent/{agent_id}/composer` to verify `agent_soul.knowledge.sets` includes both an Agent-decide generated query set and a custom user-query set using the fixed custom query.
Use `the Agent Builder preseeded Agent "{agent}" has published Web app access` to verify that a fixed Agent is published, Web app access is enabled, and the Agent detail response includes the site token and base URL needed to open the Web app.
Use `the Agent Builder preseeded Agent "{agent}" is referenced by workflow "{workflow}"` to verify Workflow access prerequisites. It checks both fixed resources exist, then uses `/console/api/agent/{agent_id}/referencing-workflows`, the same Console API used by the Access Point Workflow references table, to verify the workflow references the Agent through at least one published Agent node.
Run `pnpm -C e2e e2e -- --tags @agent-v2-preflight` against a seeded environment to verify Agent Builder preseeded resource readiness before running dependent scenarios. Keep each resource as a separate preflight scenario so a missing resource marks only its dependent precondition as blocked instead of hiding the rest of the readiness report.
## Blocked And Partial Policy
Use explicit skipped steps for missing resources, disabled feature flags, and product capabilities that are not currently implemented. `@feature-gated` is only a label; it is not execution semantics.
Blocked messages should be specific enough to route ownership:
```text
Blocked precondition: <resource> missing <capability>. Owner: seed/product. Remediation: <what to seed or decide>.
```
Order blocked steps by the real owner of the first unresolved condition. If a scenario is not automatable yet because the product behavior or test fixture contract is undefined, put that explicit availability step before model/tool/dataset preflights so the report is not masked by an unrelated missing seed. If the scenario is otherwise automatable and only depends on an external seed resource, run the matching resource preflight before creating scenario-owned state. When an availability check must inspect a real Configure UI surface, create only scenario-owned disposable state first and rely on the shared cleanup path after the skipped result.
Use partial coverage only when current product behavior is intentionally narrower than the written requirement and the test still asserts a real user-visible behavior. Example: Files are currently flat in Agent config files, so the flat Files list can be asserted while tree display remains blocked until product support exists.
Multiple-file drop is covered as stable `@core @files-limits` behavior. Add file format or size rejection coverage only after the product exposes stable Agent config file restrictions and user-visible error states; do not encode undefined behavior as permanently skipped scenarios.
Do not mark a scenario as complete if it only proves setup state and does not assert the user-visible behavior or persisted product contract required by the case.
Import generated Console/Web/Service API types directly from `@dify/contracts/.../types.gen`. Keep local types only for E2E-owned state, fixture registry entries, helper inputs, and intentionally narrowed views. If the generated contract is incomplete, fix the backend schema and regenerate it instead of duplicating the response shape.

2
e2e/features/agent-v2/access-point.feature
View File

@ -81,7 +81,7 @@ Feature: Agent v2 Access Point
When I refresh the current page
Then Agent v2 Web app access should be in service
@core @workflow-reference
@core @prepared @workflow-reference
Scenario: Workflow access shows the referencing workflow
Given I am signed in as the default E2E admin
And the Agent Builder preseeded Agent "E2E Agent With Workflow Reference" is available

14
e2e/features/agent-v2/advanced-settings.feature
View File

@ -60,17 +60,3 @@ Feature: Agent v2 advanced settings
And the Agent v2 configuration should be saved automatically
When I refresh the current page
Then I should see the Agent v2 environment variables from the invalid import in Advanced Settings
@content-moderation @feature-gated
Scenario: Content Moderation keyword preset replies are saved and restored
Given I am signed in as the default E2E admin
And a basic configured Agent v2 test agent has been created via API
When I open the Agent v2 configure page
And I expand Agent v2 Advanced Settings
Then Agent v2 Content Moderation Settings should be available
When I configure Agent v2 Content Moderation keyword preset replies
Then Agent v2 Content Moderation keyword preset replies should be saved in the Agent v2 draft
And the Agent v2 configuration should be saved automatically
When I refresh the current page
And I expand Agent v2 Advanced Settings
Then I should see the Agent v2 Content Moderation keyword preset replies in Advanced Settings

11
e2e/features/agent-v2/agent-edit.feature
View File

@ -1,6 +1,6 @@
@agent-v2 @authenticated @agent-edit
Feature: Agent v2 Agent Edit page
@core @stable-model @full-config-agent
@core @prepared @stable-model @full-config-agent
Scenario: Saved orchestration sections are visible on the Agent Edit page
Given I am signed in as the default E2E admin
And the Agent Builder stable chat model is available
@ -9,12 +9,13 @@ Feature: Agent v2 Agent Edit page
When I open the preseeded Agent v2 configure page for "E2E New Agent Builder Full Config" from the Agent Roster
Then I should see the Agent v2 full-config fixture sections
@core @stable-model @full-config-agent
@core @prepared @stable-model @full-config-agent
Scenario: Duplicated Agent inherits configuration without changing the original Agent
Given I am signed in as the default E2E admin
And the Agent Builder stable chat model is available
And the Agent Builder preseeded Agent "E2E New Agent Builder Full Config" is available
And the Agent Builder preseeded Agent "E2E New Agent Builder Full Config" includes the core fixture configuration
And the preseeded Agent v2 "E2E New Agent Builder Full Config" has been published via API
When I duplicate the preseeded Agent v2 "E2E New Agent Builder Full Config" from the Agent Roster
Then the duplicated Agent v2 should inherit the full-config fixture from "E2E New Agent Builder Full Config"
When I open the Agent v2 configure page
@ -23,7 +24,7 @@ Feature: Agent v2 Agent Edit page
And the normal Agent v2 draft should use the updated E2E prompt
And the preseeded Agent v2 "E2E New Agent Builder Full Config" should still use the normal E2E prompt
@core @tool-states-agent
@core @prepared @tool-states-agent
Scenario: Tool states are visible on the Agent Edit page
Given I am signed in as the default E2E admin
And the Agent Builder preseeded Agent "E2E New Agent Builder Tool States" is available
@ -31,7 +32,7 @@ Feature: Agent v2 Agent Edit page
When I open the preseeded Agent v2 configure page for "E2E New Agent Builder Tool States" from the Agent Roster
Then I should see the Agent v2 tool state fixture tools
@core @dual-retrieval-fixture
@core @prepared @dual-retrieval-fixture
Scenario: Dual Knowledge Retrieval settings are visible on the Agent Edit page
Given I am signed in as the default E2E admin
And the Agent Builder preseeded Agent "E2E Agent With Dual Retrieval" is available
@ -39,7 +40,7 @@ Feature: Agent v2 Agent Edit page
When I open the preseeded Agent v2 configure page for "E2E Agent With Dual Retrieval" from the Agent Roster
Then I should see the Agent v2 dual retrieval fixture settings
@core @stable-model
@core @prepared @stable-model
Scenario: Agent Edit opens the same Agent in Agent Console
Given I am signed in as the default E2E admin
And the Agent Builder stable chat model is available

29
e2e/features/agent-v2/build-draft.feature
View File

@ -79,38 +79,27 @@ Feature: Agent v2 build draft
Then I should see the updated E2E prompt in the Agent v2 prompt editor
And the Agent v2 Build draft should no longer be active
@core @skill-fixture
Scenario: Applying a Build draft updates supported configuration sections
@core @external-model @stable-model @skill-fixture
Scenario: Applying a Build draft updates configuration without duplicating an existing Skill
Given I am signed in as the default E2E admin
And the Agent Builder stable chat model is available
And a basic configured Agent v2 test agent has been created via API
And an Agent v2 Build draft adds the supported E2E files, skills, and env
And an Agent v2 Build draft adds supported E2E files and env while retaining the existing Skill
When I open the Agent v2 configure page
Then I should see the Agent v2 Build draft pending changes
And I should see the updated E2E prompt in the Agent v2 prompt editor
And I should see one e2e-summary-skill Skill in the Skills section
And the normal Agent v2 draft should still use the normal E2E prompt
When I apply the Agent v2 Build draft via API
And the Agent v2 draft should include one e2e-summary-skill Skill
When I apply the Agent v2 Build draft
Then the Agent v2 draft should include the supported Build draft config
And the Agent v2 draft should include one e2e-summary-skill Skill
When I refresh the current page
Then I should see the updated E2E prompt in the Agent v2 prompt editor
And I should see the small Agent v2 file in the Files section
And I should see the e2e-summary-skill Skill in the Skills section
And I should see the supported E2E environment variable in Advanced Settings
And the Agent v2 Build draft should no longer be active
@core @skill-fixture
Scenario: Applying a Build draft with an existing Skill keeps a single Skill entry
Given I am signed in as the default E2E admin
And a basic configured Agent v2 test agent has been created via API
And an Agent v2 Build draft includes the existing e2e-summary-skill Skill
When I open the Agent v2 configure page
Then I should see the Agent v2 Build draft pending changes
And I should see one e2e-summary-skill Skill in the Skills section
And the Agent v2 draft should include one e2e-summary-skill Skill
When I apply the Agent v2 Build draft via API
Then the Agent v2 draft should include one e2e-summary-skill Skill
When I refresh the current page
Then I should see one e2e-summary-skill Skill in the Skills section
And the Agent v2 draft should include one e2e-summary-skill Skill
And I should see the supported E2E environment variable in Advanced Settings
And the Agent v2 Build draft should no longer be active
@core

16
e2e/features/agent-v2/configure-persistence.feature
View File

@ -1,22 +1,22 @@
@agent-v2 @authenticated @core
Feature: Agent v2 configure persistence
@configure-persistence @stable-model
Scenario: Selecting a stable model in Configure persists after refresh
@prepared @configure-persistence @agent-decision-model
Scenario: Selecting a compatible model in Configure persists after refresh
Given I am signed in as the default E2E admin
And the Agent Builder stable chat model is available
And the Agent Builder agent-decision chat model is available
And an Agent v2 test agent has been created via API
When I open the Agent v2 configure page
And I select the stable E2E model in the Agent v2 model selector
And I select the agent-decision E2E model in the Agent v2 model selector
And I fill the Agent v2 prompt editor with the normal E2E prompt
Then the Agent v2 configuration should be saved automatically
And the Agent v2 draft should use the stable E2E model
And the Agent v2 draft should use the agent-decision E2E model
And the normal Agent v2 draft should use the normal E2E prompt
When I refresh the current page
Then I should see the stable E2E model in the Agent v2 model selector
Then I should see the agent-decision E2E model in the Agent v2 model selector
And I should see the normal E2E prompt in the Agent v2 prompt editor
And the Agent v2 draft should use the stable E2E model
And the Agent v2 draft should use the agent-decision E2E model
@configure-persistence @stable-model
@prepared @configure-persistence @stable-model
Scenario: Persisted Agent v2 instructions remain visible after refresh
Given I am signed in as the default E2E admin
And the Agent Builder stable chat model is available

9
e2e/features/agent-v2/configure-validation.feature
View File

@ -1,9 +0,0 @@
@agent-v2 @authenticated @preview
Feature: Agent v2 configure validation
Scenario: Preview is unavailable until a required model is configured
Given I am signed in as the default E2E admin
And an Agent v2 test agent has been created via API
And the Agent v2 composer draft uses the normal E2E prompt
When I open the Agent v2 configure page
Then Agent v2 Preview should be unavailable until a model is configured
And I should see the normal E2E prompt in the Agent v2 prompt editor

6
e2e/features/agent-v2/knowledge.feature
View File

@ -1,6 +1,6 @@
@agent-v2 @authenticated @knowledge @knowledge-fixture
Feature: Agent v2 Knowledge Retrieval
@core
@core @prepared
Scenario: Agent decide Knowledge Retrieval settings are saved and restored
Given I am signed in as the default E2E admin
And the Agent Builder preseeded dataset "E2E Agent Knowledge Base" is indexed and ready
@ -12,7 +12,7 @@ Feature: Agent v2 Knowledge Retrieval
When I refresh the current page
Then I should see the Agent v2 Agent decide Knowledge Retrieval settings
@core
@core @prepared
Scenario: Custom query Knowledge Retrieval settings are saved and restored
Given I am signed in as the default E2E admin
And the Agent Builder preseeded dataset "E2E Agent Knowledge Base" is indexed and ready
@ -58,7 +58,7 @@ Feature: Agent v2 Knowledge Retrieval
When I send the Agent v2 Backend service API knowledge request
Then the Agent v2 Backend service API response should include the knowledge E2E marker
@core
@core @prepared
Scenario: Removing Knowledge Retrieval clears the saved dataset reference
Given I am signed in as the default E2E admin
And the Agent Builder preseeded dataset "E2E Agent Knowledge Base" is indexed and ready

14
e2e/features/agent-v2/output-variables.feature
View File

@ -1,6 +1,6 @@
@agent-v2 @authenticated @output-variables
Feature: Agent v2 output variables
@core @stable-model
@core @prepared @stable-model
Scenario: Workflow Agent v2 output variables persist after refresh
Given I am signed in as the default E2E admin
And the Agent Builder stable chat model is available
@ -17,7 +17,7 @@ Feature: Agent v2 output variables
And I open the Agent v2 workflow node panel
Then I should see the Agent v2 workflow node output variables
@core @stable-model
@core @prepared @stable-model
Scenario: Workflow Agent v2 nested object output variables persist after refresh
Given I am signed in as the default E2E admin
And the Agent Builder stable chat model is available
@ -30,17 +30,17 @@ Feature: Agent v2 output variables
And I open the Agent v2 workflow node panel
Then I should see the Agent v2 workflow node nested object output variable
@core @stable-model
@core @prepared @stable-model
Scenario: Workflow Agent v2 prompt output reference stays synced when renamed
Given I am signed in as the default E2E admin
And the Agent Builder stable chat model is available
And a workflow app with an Agent v2 node has been created via API
When I open the app from the app list
And I open the Agent v2 workflow node panel
And I insert a file output reference from the Agent v2 workflow node task editor
Then the Agent v2 workflow node task should reference the file output
And I insert an output reference from the Agent v2 workflow node task editor
Then the Agent v2 workflow node task should reference the output
When I rename the Agent v2 workflow node task output reference
Then the Agent v2 workflow node task should reference the renamed file output
Then the Agent v2 workflow node task should reference the renamed output
When I refresh the current page
And I open the Agent v2 workflow node panel
Then the Agent v2 workflow node task should reference the renamed file output
Then the Agent v2 workflow node task should reference the renamed output

135
e2e/features/agent-v2/preflight.feature
View File

@ -1,135 +0,0 @@
@agent-v2 @authenticated @infra @agent-v2-preflight
Feature: Agent Builder preseeded environment
@agent-lifecycle
Scenario: Agent lifecycle permissions are available
Given I am signed in as the default E2E admin
And an Agent v2 test agent has been created via API
And the Agent v2 composer draft is publishable
When I open the Agent v2 configure page
And I publish the Agent v2 draft
Then the Agent v2 draft should be published and up to date
@stable-model
Scenario: Stable chat model is available
Given I am signed in as the default E2E admin
And the Agent Builder stable chat model is available
@speech-to-text-model
Scenario: Default speech-to-text model is available
Given I am signed in as the default E2E admin
And the workspace default speech-to-text model is active
@agent-decision-model
Scenario: Agent-decision chat model is available
Given I am signed in as the default E2E admin
And the Agent Builder agent-decision chat model is available
@broken-model
Scenario: Broken chat model is available for recovery scenarios
Given I am signed in as the default E2E admin
And the Agent Builder broken chat model is available
@tool-fixture
Scenario: JSON Replace tool is available
Given I am signed in as the default E2E admin
And the Agent Builder preseeded tool "JSON Process / JSON Replace" is available
@tool-fixture
Scenario: Tavily Search tool is available
Given I am signed in as the default E2E admin
And the Agent Builder preseeded tool "Tavily / Tavily Search" is available
@skill-fixture
Scenario: Summary Skill package fixture uploads to Agent drive
Given I am signed in as the default E2E admin
And an Agent v2 test agent has been created via API
And the e2e-summary-skill Skill is available to the Agent v2 test agent
Then the Agent v2 test agent should include drive skill "e2e-summary-skill"
@knowledge-fixture
Scenario: Agent knowledge base is available
Given I am signed in as the default E2E admin
And the Agent Builder preseeded dataset "E2E Agent Knowledge Base" is indexed and ready
@knowledge-fixture
Scenario: Indexing knowledge base is available
Given I am signed in as the default E2E admin
And the Agent Builder preseeded dataset "E2E Agent Knowledge Base Indexing" is indexing
@full-config-agent
Scenario: Full config Agent is available
Given I am signed in as the default E2E admin
And the Agent Builder preseeded Agent "E2E New Agent Builder Full Config" is available
@full-config-agent @skill-fixture
Scenario: Full config Agent includes the summary Skill
Given I am signed in as the default E2E admin
And the Agent Builder preseeded Agent "E2E New Agent Builder Full Config" includes drive skill "e2e-summary-skill"
@full-config-agent @stable-model @tool-fixture @skill-fixture @knowledge-fixture
Scenario: Full config Agent includes core fixture configuration
Given I am signed in as the default E2E admin
And the Agent Builder preseeded Agent "E2E New Agent Builder Full Config" includes the core fixture configuration
@content-moderation @feature-gated
Scenario: Content Moderation Settings is enabled
Given I am signed in as the default E2E admin
And a basic configured Agent v2 test agent has been created via API
When I open the Agent v2 configure page
And I expand Agent v2 Advanced Settings
Then Agent v2 Content Moderation Settings should be available
@tool-states-agent
Scenario: Tool states Agent is available
Given I am signed in as the default E2E admin
And the Agent Builder preseeded Agent "E2E New Agent Builder Tool States" is available
@tool-states-agent @tool-fixture @skill-fixture
Scenario: Tool states Agent includes tool state fixture configuration
Given I am signed in as the default E2E admin
And the Agent Builder preseeded Agent "E2E New Agent Builder Tool States" includes the tool state fixture configuration
@oauth-tool-agent
Scenario: OAuth2 tool Agent includes credential fixture configuration
Given I am signed in as the default E2E admin
And the Agent Builder preseeded Agent "E2E Agent With OAuth Tool" includes an OAuth2 tool credential
@dual-retrieval-fixture
Scenario: Dual retrieval Agent is available
Given I am signed in as the default E2E admin
And the Agent Builder preseeded Agent "E2E Agent With Dual Retrieval" is available
@dual-retrieval-fixture @knowledge-fixture
Scenario: Dual retrieval Agent includes dual retrieval fixture configuration
Given I am signed in as the default E2E admin
And the Agent Builder preseeded Agent "E2E Agent With Dual Retrieval" includes the dual retrieval fixture configuration
@published-web-app
Scenario: Published Web app Agent exposes Web app access
Given I am signed in as the default E2E admin
And the Agent Builder preseeded Agent "E2E Agent Published Web App" has published Web app access
@backend-api-access
Scenario: Backend API-enabled Agent is available
Given I am signed in as the default E2E admin
And the Agent Builder preseeded Agent "E2E Agent Backend API Enabled" is available
@backend-api-access
Scenario: Backend API-enabled Agent exposes API access with a key
Given I am signed in as the default E2E admin
And the Agent Builder preseeded Agent "E2E Agent Backend API Enabled" has Backend service API access with an API key
@workflow-reference
Scenario: Workflow reference Agent is available
Given I am signed in as the default E2E admin
And the Agent Builder preseeded Agent "E2E Agent With Workflow Reference" is available
@workflow-reference
Scenario: Reference workflow is available
Given I am signed in as the default E2E admin
And the Agent Builder preseeded workflow "E2E Agent Reference Workflow" is available
@workflow-reference
Scenario: Workflow reference Agent is used by the reference workflow
Given I am signed in as the default E2E admin
And the Agent Builder preseeded Agent "E2E Agent With Workflow Reference" is referenced by workflow "E2E Agent Reference Workflow"

19
e2e/features/agent-v2/publish.feature
View File

@ -10,7 +10,7 @@ Feature: Agent v2 publish
Then Agent v2 publish should be blocked until a model is configured
And the Agent v2 draft should remain unpublished
@core @stable-model
@core @prepared @stable-model
Scenario: Publish a configured Agent v2 draft
Given I am signed in as the default E2E admin
And the Agent Builder stable chat model is available
@ -19,7 +19,7 @@ Feature: Agent v2 publish
And I publish the Agent v2 draft
Then the Agent v2 draft should be published and up to date
@core @stable-model
@core @prepared @stable-model
Scenario: Publish action follows unpublished changes
Given I am signed in as the default E2E admin
And the Agent Builder stable chat model is available
@ -33,20 +33,7 @@ Feature: Agent v2 publish
Then the Agent v2 configuration should be saved automatically
And the Agent v2 publish action should be available for unpublished changes
@core @stable-model
Scenario: Published Agent v2 version remains isolated from draft edits
Given I am signed in as the default E2E admin
And the Agent Builder stable chat model is available
And a runnable Agent v2 test agent has been created via API
When I open the Agent v2 configure page
And I publish the Agent v2 draft
Then the Agent v2 draft should be published and up to date
When I fill the Agent v2 prompt editor with the updated E2E prompt
Then the Agent v2 configuration should be saved automatically
And the normal Agent v2 draft should use the updated E2E prompt
And the active published Agent v2 version should still use the normal E2E prompt
@core @stable-model
@core @prepared @stable-model
Scenario: Restoring a published Agent v2 version shows the restored configuration in Builder
Given I am signed in as the default E2E admin
And the Agent Builder stable chat model is available

10
e2e/features/agent-v2/support/agent-build-draft.ts
View File

@ -61,16 +61,6 @@ export async function getAgentBuildDraft(agentId: string): Promise<AgentBuildDra
}
}
export async function applyAgentBuildDraft(agentId: string): Promise<void> {
const ctx = await createApiContext()
try {
const response = await ctx.post(`/console/api/agent/${agentId}/build-draft/apply`)
await expectApiResponseOK(response, `Apply Agent v2 build draft for ${agentId}`)
} finally {
await ctx.dispose()
}
}
export async function discardAgentBuildDraft(agentId: string): Promise<void> {
const ctx = await createApiContext()
try {

10
e2e/features/agent-v2/support/agent-builder-resources.ts
View File

@ -5,19 +5,12 @@ export const agentBuilderPreseededResources = {
jsonReplaceTool: 'JSON Process / JSON Replace',
tavilySearchTool: 'Tavily / Tavily Search',
agentKnowledgeBase: 'E2E Agent Knowledge Base',
indexingKnowledgeBase: 'E2E Agent Knowledge Base Indexing',
agentDecisionChatModel: 'E2E Agent Decision Chat Model',
brokenModelProvider: 'E2E Broken Model Provider',
brokenModel: 'e2e-broken-model',
fullConfigAgent: 'E2E New Agent Builder Full Config',
toolStatesAgent: 'E2E New Agent Builder Tool States',
oauthToolAgent: 'E2E Agent With OAuth Tool',
dualRetrievalAgent: 'E2E Agent With Dual Retrieval',
publishedWebAppAgent: 'E2E Agent Published Web App',
backendApiEnabledAgent: 'E2E Agent Backend API Enabled',
workflowReferenceAgent: 'E2E Agent With Workflow Reference',
referenceWorkflow: 'E2E Agent Reference Workflow',
backendApiKey: 'E2E Backend API Key',
} as const
export const agentBuilderFixedInputs = {
@ -34,9 +27,6 @@ export const agentBuilderFixedInputs = {
envModeValue: 'plain',
envAfterInvalidImportKey: 'E2E_AGENT_AFTER_INVALID',
envAfterInvalidImportValue: 'still-valid',
moderationKeyword: 'E2E_BLOCKED_KEYWORD',
inputModerationReply: 'E2E_INPUT_BLOCKED_REPLY',
outputModerationReply: 'E2E_OUTPUT_BLOCKED_REPLY',
backendApiUser: 'e2e-agent-access-point',
} as const

1
e2e/features/agent-v2/support/agent-soul.ts
View File

@ -48,7 +48,6 @@ export const updatedAgentSoulConfig: AgentSoulConfig = {
}
const stableAgentModelSettings = {
max_tokens: 4096,
temperature: 0,
}

15
e2e/features/agent-v2/support/agent.ts
View File

@ -1,7 +1,6 @@
import type {
AgentAppComposerResponse,
AgentAppDetailWithSite,
AgentConfigSnapshotDetailResponse,
AgentReferencingWorkflowResponse,
AgentReferencingWorkflowsResponse,
AgentSoulConfig,
@ -86,20 +85,6 @@ export async function getTestAgent(agentId: string): Promise<AgentSeed> {
}
}
export async function getAgentVersionDetail(
agentId: string,
versionId: string,
): Promise<AgentConfigSnapshotDetailResponse> {
const ctx = await createApiContext()
try {
const response = await ctx.get(`/console/api/agent/${agentId}/versions/${versionId}`)
await expectApiResponseOK(response, `Get Agent v2 version ${versionId} for ${agentId}`)
return (await response.json()) as AgentConfigSnapshotDetailResponse
} finally {
await ctx.dispose()
}
}
export async function deleteTestAgent(agentId: string): Promise<void> {
const ctx = await createApiContext()
try {

48
e2e/features/agent-v2/support/fixtures/access.ts Normal file
View File

@ -0,0 +1,48 @@
import type { AgentReferencingWorkflowsResponse } from '@dify/contracts/api/console/agent/types.gen'
import type { DifyWorld } from '../../../support/world'
import type { PreseededResource } from './common'
import { createApiContext, expectApiResponseOK } from '../../../../support/api'
import { requirePreseededAgent, requirePreseededWorkflow } from './agents'
import { failFixturePrerequisite } from './common'
export async function requirePreseededAgentWorkflowReference(
world: DifyWorld,
agentName: string,
workflowName: string,
): Promise<PreseededResource> {
const agent = await requirePreseededAgent(world, agentName)
const workflow = await requirePreseededWorkflow(world, workflowName)
const ctx = await createApiContext()
try {
const response = await ctx.get(`/console/api/agent/${agent.id}/referencing-workflows`)
await expectApiResponseOK(response, `Check preseeded Agent workflow reference ${agentName}`)
const references = (await response.json()) as AgentReferencingWorkflowsResponse
const reference = references.data?.find(
(item) => item.app_id === workflow.id || item.app_name === workflow.name,
)
if (!reference) {
return failFixturePrerequisite(
world,
`Preseeded Agent "${agentName}" is not referenced by workflow "${workflowName}".`,
)
}
if (!reference.node_ids || reference.node_ids.length < 1) {
return failFixturePrerequisite(
world,
`Preseeded workflow "${workflowName}" does not expose Agent reference nodes for "${agentName}".`,
)
}
return {
id: workflow.id,
kind: 'workflow',
name: workflow.name,
}
} finally {
await ctx.dispose()
}
}

10
e2e/features/agent-v2/support/preflight/agent-backend.ts → e2e/features/agent-v2/support/fixtures/agent-backend.ts
View File

@ -1,5 +1,5 @@
import type { DifyWorld } from '../../../support/world'
import { skipBlockedPrecondition } from './common'
import { failFixturePrerequisite } from './common'
const isTruthyEnv = (value: string | undefined) => value === '1' || value === 'true'
@ -54,7 +54,7 @@ const checkRuntimeEndpoint = async ({
const response = await fetch(healthURL)
if (response.ok) return undefined
return skipBlockedPrecondition(
return failFixturePrerequisite(
world,
`${title} did not respond successfully at ${healthURL}: ${response.status} ${response.statusText}.`,
{
@ -65,18 +65,18 @@ const checkRuntimeEndpoint = async ({
} catch (error) {
const message = error instanceof Error ? error.message : String(error)
return skipBlockedPrecondition(world, `${title} is unreachable at ${healthURL}: ${message}.`, {
return failFixturePrerequisite(world, `${title} is unreachable at ${healthURL}: ${message}.`, {
owner: 'e2e/runtime',
remediation,
})
}
}
export async function skipMissingAgentBackendRuntime(world: DifyWorld) {
export async function requireAgentBackendRuntime(world: DifyWorld) {
const agentBackendURL = getAgentBackendURL()
if (!agentBackendURL) {
return skipBlockedPrecondition(
return failFixturePrerequisite(
world,
'Agent v2 runtime backend is not configured. This scenario needs the standalone dify-agent run server, not just an active model provider.',
{

158
e2e/features/agent-v2/support/preflight/agents.ts → e2e/features/agent-v2/support/fixtures/agents.ts
View File

@ -1,7 +1,6 @@
import type {
AgentAppComposerResponse,
AgentDriveSkillListResponse,
AgentSoulConfig,
} from '@dify/contracts/api/console/agent/types.gen'
import type { DifyWorld } from '../../../support/world'
import type { PreseededResource } from './common'
@ -17,25 +16,21 @@ import {
asRecord,
asString,
buildQuery,
failFixturePrerequisite,
findConsoleResourceByName,
hasNamedOrKeyedEntry,
skipBlockedPrecondition,
} from './common'
import { skipMissingReadyPreseededDataset } from './datasets'
import { skipMissingAgentBuilderStableChatModel } from './models'
import { requireReadyPreseededDataset } from './datasets'
import { requireAgentBuilderStableChatModel } from './models'
import {
findToolEntry,
hasToolEntry,
hasUnauthorizedToolCredentialState,
skipMissingPreseededTool,
requirePreseededTool,
splitToolDisplayName,
splitToolResourceId,
} from './tools'
type AgentSoulDifyToolConfig = NonNullable<
NonNullable<AgentSoulConfig['tools']>['dify_tools']
>[number]
const hasKnowledgeDataset = (soul: Record<string, unknown>, dataset: PreseededResource) => {
const knowledge = asRecord(soul.knowledge)
const sets = asArray(knowledge.sets)
@ -80,19 +75,10 @@ const hasKnowledgeSet = (
})
}
const getOAuth2ToolCredentialFixture = (toolItems: unknown[]) =>
toolItems.find((item) => {
const record = asRecord(item)
return (
record.credential_type === 'oauth2' && Boolean(asString(asRecord(record.credential_ref).id))
)
}) as AgentSoulDifyToolConfig | undefined
export async function skipMissingPreseededAgent(
export async function requirePreseededAgent(
world: DifyWorld,
resourceName: string,
): Promise<'skipped' | PreseededResource> {
): Promise<PreseededResource> {
const query = buildQuery({ limit: '20', name: resourceName, page: '1' })
const resource = await findConsoleResourceByName({
action: `Check preseeded Agent ${resourceName}`,
@ -101,7 +87,7 @@ export async function skipMissingPreseededAgent(
})
if (!resource)
return skipBlockedPrecondition(world, `Preseeded Agent "${resourceName}" was not found.`)
return failFixturePrerequisite(world, `Preseeded Agent "${resourceName}" was not found.`)
return {
id: resource.id,
@ -110,10 +96,10 @@ export async function skipMissingPreseededAgent(
}
}
export async function skipMissingPreseededWorkflow(
export async function requirePreseededWorkflow(
world: DifyWorld,
resourceName: string,
): Promise<'skipped' | PreseededResource> {
): Promise<PreseededResource> {
const query = buildQuery({ limit: '20', mode: 'workflow', name: resourceName, page: '1' })
const resource = await findConsoleResourceByName({
action: `Check preseeded workflow ${resourceName}`,
@ -122,7 +108,7 @@ export async function skipMissingPreseededWorkflow(
})
if (!resource)
return skipBlockedPrecondition(world, `Preseeded workflow "${resourceName}" was not found.`)
return failFixturePrerequisite(world, `Preseeded workflow "${resourceName}" was not found.`)
return {
id: resource.id,
@ -131,13 +117,12 @@ export async function skipMissingPreseededWorkflow(
}
}
export async function skipMissingPreseededAgentDriveSkill(
export async function requirePreseededAgentDriveSkill(
world: DifyWorld,
agentName: string,
skillName: string,
): Promise<'skipped' | PreseededResource> {
const agent = await skipMissingPreseededAgent(world, agentName)
if (agent === 'skipped') return agent
): Promise<PreseededResource> {
const agent = await requirePreseededAgent(world, agentName)
const ctx = await createApiContext()
try {
@ -147,7 +132,7 @@ export async function skipMissingPreseededAgentDriveSkill(
const skill = body.items?.find((item) => item.name === skillName)
if (!skill) {
return skipBlockedPrecondition(
return failFixturePrerequisite(
world,
`Preseeded Agent "${agentName}" does not include drive skill "${skillName}".`,
)
@ -163,34 +148,26 @@ export async function skipMissingPreseededAgentDriveSkill(
}
}
export async function skipMissingPreseededFullConfigAgentCoreConfiguration(
export async function requirePreseededFullConfigAgentCoreConfiguration(
world: DifyWorld,
agentName: string,
): Promise<'skipped' | PreseededResource> {
const stableModel = await skipMissingAgentBuilderStableChatModel(world)
if (stableModel === 'skipped') return stableModel
): Promise<PreseededResource> {
const stableModel = await requireAgentBuilderStableChatModel(world)
const agent = await skipMissingPreseededAgent(world, agentName)
if (agent === 'skipped') return agent
const agent = await requirePreseededAgent(world, agentName)
const summarySkill = await skipMissingPreseededAgentDriveSkill(
await requirePreseededAgentDriveSkill(
world,
agentName,
agentBuilderPreseededResources.summarySkill,
)
if (summarySkill === 'skipped') return summarySkill
const jsonTool = await skipMissingPreseededTool(
world,
agentBuilderPreseededResources.jsonReplaceTool,
)
if (jsonTool === 'skipped') return jsonTool
const jsonTool = await requirePreseededTool(world, agentBuilderPreseededResources.jsonReplaceTool)
const knowledgeBase = await skipMissingReadyPreseededDataset(
const knowledgeBase = await requireReadyPreseededDataset(
world,
agentBuilderPreseededResources.agentKnowledgeBase,
)
if (knowledgeBase === 'skipped') return knowledgeBase
const ctx = await createApiContext()
try {
@ -238,7 +215,7 @@ export async function skipMissingPreseededFullConfigAgentCoreConfiguration(
missing.push(agentBuilderPreseededResources.agentKnowledgeBase)
if (missing.length > 0) {
return skipBlockedPrecondition(
return failFixturePrerequisite(
world,
`Preseeded Agent "${agentName}" is missing core fixture configuration: ${missing.join(', ')}.`,
)
@ -250,31 +227,24 @@ export async function skipMissingPreseededFullConfigAgentCoreConfiguration(
}
}
export async function skipMissingPreseededToolStatesAgentConfiguration(
export async function requirePreseededToolStatesAgentConfiguration(
world: DifyWorld,
agentName: string,
): Promise<'skipped' | PreseededResource> {
const agent = await skipMissingPreseededAgent(world, agentName)
if (agent === 'skipped') return agent
): Promise<PreseededResource> {
const agent = await requirePreseededAgent(world, agentName)
const summarySkill = await skipMissingPreseededAgentDriveSkill(
await requirePreseededAgentDriveSkill(
world,
agentName,
agentBuilderPreseededResources.summarySkill,
)
if (summarySkill === 'skipped') return summarySkill
const jsonTool = await skipMissingPreseededTool(
world,
agentBuilderPreseededResources.jsonReplaceTool,
)
if (jsonTool === 'skipped') return jsonTool
const jsonTool = await requirePreseededTool(world, agentBuilderPreseededResources.jsonReplaceTool)
const tavilyTool = await skipMissingPreseededTool(
const tavilyTool = await requirePreseededTool(
world,
agentBuilderPreseededResources.tavilySearchTool,
)
if (tavilyTool === 'skipped') return tavilyTool
const ctx = await createApiContext()
try {
@ -327,7 +297,7 @@ export async function skipMissingPreseededToolStatesAgentConfiguration(
}
if (missing.length > 0) {
return skipBlockedPrecondition(
return failFixturePrerequisite(
world,
`Preseeded Agent "${agentName}" is missing tool state fixture configuration: ${missing.join(', ')}.`,
)
@ -339,76 +309,16 @@ export async function skipMissingPreseededToolStatesAgentConfiguration(
}
}
export async function skipMissingPreseededOAuthToolAgentConfiguration(
export async function requirePreseededDualRetrievalAgentConfiguration(
world: DifyWorld,
agentName: string,
): Promise<'skipped' | PreseededResource> {
const agent = await skipMissingPreseededAgent(world, agentName)
if (agent === 'skipped') return agent
): Promise<PreseededResource> {
const agent = await requirePreseededAgent(world, agentName)
const ctx = await createApiContext()
try {
const response = await ctx.get(`/console/api/agent/${agent.id}/composer`)
await expectApiResponseOK(
response,
`Check preseeded Agent OAuth2 tool configuration ${agentName}`,
)
const body = (await response.json()) as AgentAppComposerResponse
const toolItems = asArray(asRecord(body.agent_soul?.tools).dify_tools)
const oauthTool = getOAuth2ToolCredentialFixture(toolItems)
if (!oauthTool) {
return skipBlockedPrecondition(
world,
`Preseeded Agent "${agentName}" is missing an OAuth2 tool with a credential reference.`,
{
owner: 'seed',
remediation:
'Seed an Agent v2 fixture that includes a built-in OAuth2 tool with credential_type oauth2 and credential_ref.id.',
},
)
}
return agent
} finally {
await ctx.dispose()
}
}
export async function getPreseededOAuthToolConfig(
agentId: string,
): Promise<AgentSoulDifyToolConfig> {
const ctx = await createApiContext()
try {
const response = await ctx.get(`/console/api/agent/${agentId}/composer`)
await expectApiResponseOK(response, `Get preseeded Agent OAuth2 tool config ${agentId}`)
const body = (await response.json()) as AgentAppComposerResponse
const toolItems = asArray(asRecord(body.agent_soul?.tools).dify_tools)
const oauthTool = getOAuth2ToolCredentialFixture(toolItems)
if (!oauthTool)
throw new Error(
`Preseeded Agent ${agentId} does not include an OAuth2 tool credential fixture.`,
)
return oauthTool
} finally {
await ctx.dispose()
}
}
export async function skipMissingPreseededDualRetrievalAgentConfiguration(
world: DifyWorld,
agentName: string,
): Promise<'skipped' | PreseededResource> {
const agent = await skipMissingPreseededAgent(world, agentName)
if (agent === 'skipped') return agent
const knowledgeBase = await skipMissingReadyPreseededDataset(
const knowledgeBase = await requireReadyPreseededDataset(
world,
agentBuilderPreseededResources.agentKnowledgeBase,
)
if (knowledgeBase === 'skipped') return knowledgeBase
const ctx = await createApiContext()
try {
@ -431,7 +341,7 @@ export async function skipMissingPreseededDualRetrievalAgentConfiguration(
}
if (missing.length > 0) {
return skipBlockedPrecondition(
return failFixturePrerequisite(
world,
`Preseeded Agent "${agentName}" is missing dual retrieval fixture configuration: ${missing.join(', ')}.`,
)

60
e2e/features/agent-v2/support/preflight/common.ts → e2e/features/agent-v2/support/fixtures/common.ts
View File

@ -1,21 +1,10 @@
import type { DifyWorld } from '../../../support/world'
import { createApiContext, expectApiResponseOK } from '../../../../support/api'
import { agentBuilderPreseededResources } from '../agent-builder-resources'
export type PreseededResource = NonNullable<
DifyWorld['agentBuilder']['preflight']['preseededResources'][string]
DifyWorld['agentBuilder']['fixtures']['preseededResources'][string]
>
export type E2EResourcePrecondition =
| {
ok: true
value: string
}
| {
ok: false
reason: string
}
export type NamedResource = {
id: string
name: string
@ -30,60 +19,21 @@ export type LocalizedLabel = {
zh_Hans?: string
}
export const readRequiredEnvResource = (
envName: string,
description: string,
): E2EResourcePrecondition => {
const value = process.env[envName]?.trim()
if (value) return { ok: true, value }
return {
ok: false,
reason: `${description} requires ${envName}.`,
}
}
export function skipBlockedPrecondition(
export function failFixturePrerequisite(
world: DifyWorld,
reason: string,
options: {
owner?: string
remediation?: string
} = {},
): 'skipped' {
): never {
const owner = options.owner ?? 'seed/product'
const remediation =
options.remediation ??
'Seed the required resource or align the product capability before running this scenario.'
const message = `Blocked precondition: ${reason} Owner: ${owner}. Remediation: ${remediation}`
console.warn(`[e2e] ${message}`)
const message = `Fixture prerequisite failed: ${reason} Owner: ${owner}. Remediation: ${remediation}`
world.attach(message, 'text/plain')
return 'skipped'
}
export function skipMissingEnvResource(
world: DifyWorld,
envName: string,
description: string,
): 'skipped' | string {
const resource = readRequiredEnvResource(envName, description)
if (resource.ok) return resource.value
return skipBlockedPrecondition(world, resource.reason)
}
export const requiredAgentBuilderPreseededResources = Object.values(agentBuilderPreseededResources)
export function skipMissingAgentBuilderPreseed(
world: DifyWorld,
resourceName: string,
envName: string,
): 'skipped' | string {
return skipMissingEnvResource(
world,
envName,
`Preseeded Agent Builder resource "${resourceName}"`,
)
throw new Error(message)
}
export const findConsoleResourceByName = async <T extends NamedResource = NamedResource>({

67
e2e/features/agent-v2/support/preflight/datasets.ts → e2e/features/agent-v2/support/fixtures/datasets.ts
View File

@ -12,7 +12,7 @@ import {
agentBuilderFixedInputs,
agentBuilderPreseededResources,
} from '../agent-builder-resources'
import { buildQuery, findConsoleResourceByName, skipBlockedPrecondition } from './common'
import { buildQuery, failFixturePrerequisite, findConsoleResourceByName } from './common'
type DocumentIndexingStatus =
| 'cleaning'
@ -23,14 +23,6 @@ type DocumentIndexingStatus =
| 'waiting'
const completedDocumentIndexingStatus: DocumentIndexingStatus = 'completed'
const activeDocumentIndexingStatuses = new Set<string>([
'cleaning',
'indexing',
'parsing',
'splitting',
'waiting',
])
export const getPreseededDataset = async (resourceName: string) => {
const query = buildQuery({ keyword: resourceName, limit: '20', page: '1' })
@ -117,39 +109,27 @@ const datasetHasEnabledSegmentContainingTokens = async (
}
}
export const toDatasetResource = (resource: DatasetListItemResponse): PreseededResource => ({
const toDatasetResource = (resource: DatasetListItemResponse): PreseededResource => ({
id: resource.id,
kind: 'dataset',
name: resource.name,
})
export async function skipMissingPreseededDataset(
export async function requireReadyPreseededDataset(
world: DifyWorld,
resourceName: string,
): Promise<'skipped' | PreseededResource> {
): Promise<PreseededResource> {
const resource = await getPreseededDataset(resourceName)
if (!resource)
return skipBlockedPrecondition(world, `Preseeded dataset "${resourceName}" was not found.`)
return toDatasetResource(resource)
}
export async function skipMissingReadyPreseededDataset(
world: DifyWorld,
resourceName: string,
): Promise<'skipped' | PreseededResource> {
const resource = await getPreseededDataset(resourceName)
if (!resource)
return skipBlockedPrecondition(world, `Preseeded dataset "${resourceName}" was not found.`)
return failFixturePrerequisite(world, `Preseeded dataset "${resourceName}" was not found.`)
if (resource.document_count < 1) {
return skipBlockedPrecondition(world, `Preseeded dataset "${resourceName}" has no documents.`)
return failFixturePrerequisite(world, `Preseeded dataset "${resourceName}" has no documents.`)
}
if (resource.total_available_documents !== resource.document_count) {
return skipBlockedPrecondition(
return failFixturePrerequisite(
world,
`Preseeded dataset "${resourceName}" has ${resource.total_available_documents}/${resource.document_count} available documents.`,
)
@ -157,7 +137,7 @@ export async function skipMissingReadyPreseededDataset(
const statuses = await getDatasetIndexingStatuses(resource.id, resourceName)
if (statuses.length < 1) {
return skipBlockedPrecondition(
return failFixturePrerequisite(
world,
`Preseeded dataset "${resourceName}" has no document indexing status.`,
)
@ -167,7 +147,7 @@ export async function skipMissingReadyPreseededDataset(
(item) => item.indexing_status !== completedDocumentIndexingStatus,
)
if (incompleteStatus) {
return skipBlockedPrecondition(
return failFixturePrerequisite(
world,
`Preseeded dataset "${resourceName}" includes document ${incompleteStatus.id} with indexing status "${incompleteStatus.indexing_status ?? 'missing'}".`,
)
@ -186,7 +166,7 @@ export async function skipMissingReadyPreseededDataset(
)
if (!hasExpectedToken) {
return skipBlockedPrecondition(
return failFixturePrerequisite(
world,
`Preseeded dataset "${resourceName}" has no enabled segment containing "${requiredTokens.join('" and "')}".`,
{
@ -198,30 +178,3 @@ export async function skipMissingReadyPreseededDataset(
return toDatasetResource(resource)
}
export async function skipMissingIndexingPreseededDataset(
world: DifyWorld,
resourceName: string,
): Promise<'skipped' | PreseededResource> {
const resource = await getPreseededDataset(resourceName)
if (!resource)
return skipBlockedPrecondition(world, `Preseeded dataset "${resourceName}" was not found.`)
const statuses = await getDatasetIndexingStatuses(resource.id, resourceName)
const indexingStatus = statuses.find((item) =>
activeDocumentIndexingStatuses.has(item.indexing_status ?? ''),
)
if (!indexingStatus) {
const actualStatuses =
statuses.map((item) => item.indexing_status ?? 'missing').join(', ') || 'none'
return skipBlockedPrecondition(
world,
`Preseeded dataset "${resourceName}" is not indexing or queued; document statuses: ${actualStatuses}.`,
)
}
return toDatasetResource(resource)
}

73
e2e/features/agent-v2/support/preflight/models.ts → e2e/features/agent-v2/support/fixtures/models.ts
View File

@ -5,7 +5,7 @@ import type {
import type { DifyWorld } from '../../../support/world'
import { createApiContext, expectApiResponseOK } from '../../../../support/api'
import { agentBuilderPreseededResources } from '../agent-builder-resources'
import { skipBlockedPrecondition } from './common'
import { failFixturePrerequisite } from './common'
const stableChatModelProviderEnv = 'E2E_STABLE_MODEL_PROVIDER'
const stableChatModelNameEnv = 'E2E_STABLE_MODEL_NAME'
@ -13,9 +13,6 @@ const stableChatModelTypeEnv = 'E2E_STABLE_MODEL_TYPE'
const agentDecisionChatModelProviderEnv = 'E2E_AGENT_DECISION_MODEL_PROVIDER'
const agentDecisionChatModelNameEnv = 'E2E_AGENT_DECISION_MODEL_NAME'
const agentDecisionChatModelTypeEnv = 'E2E_AGENT_DECISION_MODEL_TYPE'
const brokenChatModelProviderEnv = 'E2E_BROKEN_MODEL_PROVIDER'
const brokenChatModelNameEnv = 'E2E_BROKEN_MODEL_NAME'
const brokenChatModelTypeEnv = 'E2E_BROKEN_MODEL_TYPE'
const activeModelStatus = 'active'
const defaultStableChatModelProvider = 'openai'
const defaultStableChatModelName = 'gpt-5-nano'
@ -23,7 +20,6 @@ const defaultStableChatModelType = 'llm'
const defaultAgentDecisionChatModelProvider = 'openai'
const defaultAgentDecisionChatModelName = 'gpt-5.5'
const defaultAgentDecisionChatModelType = 'llm'
const defaultBrokenChatModelName = agentBuilderPreseededResources.brokenModel
const getProviderAlias = (provider: string) =>
provider.split('/').filter(Boolean).at(-1) ?? provider
@ -31,7 +27,7 @@ const getProviderAlias = (provider: string) =>
const matchesProvider = (actual: string, expected: string) =>
actual === expected || getProviderAlias(actual) === getProviderAlias(expected)
type ModelPreflightConfig =
type ModelFixtureConfig =
| {
ok: true
provider: string
@ -44,7 +40,7 @@ type ModelPreflightConfig =
reason: string
}
export function readAgentBuilderStableChatModelConfig(): ModelPreflightConfig {
export function readAgentBuilderStableChatModelConfig(): ModelFixtureConfig {
const provider = process.env[stableChatModelProviderEnv]?.trim() || defaultStableChatModelProvider
const name = process.env[stableChatModelNameEnv]?.trim() || defaultStableChatModelName
const type = process.env[stableChatModelTypeEnv]?.trim() || defaultStableChatModelType
@ -58,7 +54,7 @@ export function readAgentBuilderStableChatModelConfig(): ModelPreflightConfig {
}
}
export function readAgentBuilderAgentDecisionChatModelConfig(): ModelPreflightConfig {
export function readAgentBuilderAgentDecisionChatModelConfig(): ModelFixtureConfig {
const provider =
process.env[agentDecisionChatModelProviderEnv]?.trim() || defaultAgentDecisionChatModelProvider
const name =
@ -75,37 +71,16 @@ export function readAgentBuilderAgentDecisionChatModelConfig(): ModelPreflightCo
}
}
export function readAgentBuilderBrokenChatModelConfig(): ModelPreflightConfig {
const provider = process.env[brokenChatModelProviderEnv]?.trim()
const name = process.env[brokenChatModelNameEnv]?.trim() || defaultBrokenChatModelName
const type = process.env[brokenChatModelTypeEnv]?.trim() || defaultStableChatModelType
if (!provider) {
return {
ok: false,
reason: `${agentBuilderPreseededResources.brokenModelProvider} requires ${brokenChatModelProviderEnv}.`,
}
}
return {
ok: true,
provider,
resourceName: agentBuilderPreseededResources.brokenModelProvider,
type,
value: name,
}
}
async function skipMissingAgentBuilderModel(
async function requireAgentBuilderModel(
world: DifyWorld,
config: ModelPreflightConfig,
config: ModelFixtureConfig,
{
requireActive,
}: {
requireActive: boolean
},
): Promise<'skipped' | NonNullable<DifyWorld['agentBuilder']['preflight']['stableModel']>> {
if (!config.ok) return skipBlockedPrecondition(world, config.reason)
): Promise<NonNullable<DifyWorld['agentBuilder']['fixtures']['stableModel']>> {
if (!config.ok) return failFixturePrerequisite(world, config.reason)
const ctx = await createApiContext()
try {
@ -123,14 +98,14 @@ async function skipMissingAgentBuilderModel(
)
if (!provider || !model) {
return skipBlockedPrecondition(
return failFixturePrerequisite(
world,
`${config.resourceName} was not found as ${config.provider}/${config.value} (${config.type}).`,
)
}
if (requireActive && model.status !== activeModelStatus) {
return skipBlockedPrecondition(
return failFixturePrerequisite(
world,
`${config.resourceName} is ${model.status ?? 'missing status'} instead of ${activeModelStatus}.`,
)
@ -146,17 +121,17 @@ async function skipMissingAgentBuilderModel(
}
}
export async function skipMissingAgentBuilderStableChatModel(
export async function requireAgentBuilderStableChatModel(
world: DifyWorld,
): Promise<'skipped' | NonNullable<DifyWorld['agentBuilder']['preflight']['stableModel']>> {
return skipMissingAgentBuilderModel(world, readAgentBuilderStableChatModelConfig(), {
): Promise<NonNullable<DifyWorld['agentBuilder']['fixtures']['stableModel']>> {
return requireAgentBuilderModel(world, readAgentBuilderStableChatModelConfig(), {
requireActive: true,
})
}
export async function skipMissingAgentBuilderSpeechToTextModel(
export async function requireAgentBuilderSpeechToTextModel(
world: DifyWorld,
): Promise<'skipped' | NonNullable<DifyWorld['agentBuilder']['preflight']['speechToTextModel']>> {
): Promise<NonNullable<DifyWorld['agentBuilder']['fixtures']['speechToTextModel']>> {
const ctx = await createApiContext()
let defaultModel: NonNullable<DefaultModelDataResponse['data']>
@ -167,7 +142,7 @@ export async function skipMissingAgentBuilderSpeechToTextModel(
await expectApiResponseOK(response, `Check ${agentBuilderPreseededResources.speechToTextModel}`)
const body = (await response.json()) as DefaultModelDataResponse
if (!body.data) {
return skipBlockedPrecondition(
return failFixturePrerequisite(
world,
`${agentBuilderPreseededResources.speechToTextModel} is not configured.`,
{
@ -182,7 +157,7 @@ export async function skipMissingAgentBuilderSpeechToTextModel(
await ctx.dispose()
}
return skipMissingAgentBuilderModel(
return requireAgentBuilderModel(
world,
{
ok: true,
@ -197,18 +172,10 @@ export async function skipMissingAgentBuilderSpeechToTextModel(
)
}
export async function skipMissingAgentBuilderAgentDecisionChatModel(
export async function requireAgentBuilderAgentDecisionChatModel(
world: DifyWorld,
): Promise<'skipped' | NonNullable<DifyWorld['agentBuilder']['preflight']['stableModel']>> {
return skipMissingAgentBuilderModel(world, readAgentBuilderAgentDecisionChatModelConfig(), {
): Promise<NonNullable<DifyWorld['agentBuilder']['fixtures']['stableModel']>> {
return requireAgentBuilderModel(world, readAgentBuilderAgentDecisionChatModelConfig(), {
requireActive: true,
})
}
export async function skipMissingAgentBuilderBrokenChatModel(
world: DifyWorld,
): Promise<'skipped' | NonNullable<DifyWorld['agentBuilder']['preflight']['stableModel']>> {
return skipMissingAgentBuilderModel(world, readAgentBuilderBrokenChatModelConfig(), {
requireActive: false,
})
}

10
e2e/features/agent-v2/support/preflight/tools.ts → e2e/features/agent-v2/support/fixtures/tools.ts
View File

@ -1,7 +1,7 @@
import type { DifyWorld } from '../../../support/world'
import type { LocalizedLabel, PreseededResource } from './common'
import { createApiContext, expectApiResponseOK } from '../../../../support/api'
import { asRecord, asString, matchesNameOrLabel, skipBlockedPrecondition } from './common'
import { asRecord, asString, failFixturePrerequisite, matchesNameOrLabel } from './common'
type BuiltinToolProvider = {
label?: LocalizedLabel
@ -86,12 +86,12 @@ export const hasUnauthorizedToolCredentialState = (item: unknown) => {
return asString(record.credential_type) === 'unauthorized'
}
export async function skipMissingPreseededTool(
export async function requirePreseededTool(
world: DifyWorld,
resourceName: string,
): Promise<'skipped' | PreseededResource> {
): Promise<PreseededResource> {
const parsed = splitToolDisplayName(resourceName)
if (!parsed.ok) return skipBlockedPrecondition(world, parsed.reason)
if (!parsed.ok) return failFixturePrerequisite(world, parsed.reason)
const ctx = await createApiContext()
try {
@ -106,7 +106,7 @@ export async function skipMissingPreseededTool(
)
if (!provider || !tool)
return skipBlockedPrecondition(world, `Preseeded tool "${resourceName}" was not found.`)
return failFixturePrerequisite(world, `Preseeded tool "${resourceName}" was not found.`)
return {
id: `${provider.name}/${tool.name}`,

136
e2e/features/agent-v2/support/preflight/access.ts
View File

@ -1,136 +0,0 @@
import type {
AgentApiAccessResponse,
AgentAppDetailWithSite,
AgentReferencingWorkflowsResponse,
ApiKeyList,
} from '@dify/contracts/api/console/agent/types.gen'
import type { DifyWorld } from '../../../support/world'
import type { PreseededResource } from './common'
import { createApiContext, expectApiResponseOK } from '../../../../support/api'
import { skipMissingPreseededAgent, skipMissingPreseededWorkflow } from './agents'
import { skipBlockedPrecondition } from './common'
export async function skipMissingPreseededAgentBackendApiKey(
world: DifyWorld,
agentName: string,
): Promise<'skipped' | PreseededResource> {
const agent = await skipMissingPreseededAgent(world, agentName)
if (agent === 'skipped') return agent
const ctx = await createApiContext()
try {
const accessResponse = await ctx.get(`/console/api/agent/${agent.id}/api-access`)
await expectApiResponseOK(accessResponse, `Check preseeded Agent API access ${agentName}`)
const access = (await accessResponse.json()) as AgentApiAccessResponse
if (!access.enabled || access.api_key_count < 1) {
return skipBlockedPrecondition(
world,
`Preseeded Agent "${agentName}" does not have Backend service API enabled with an API key.`,
)
}
const keyResponse = await ctx.get(`/console/api/agent/${agent.id}/api-keys`)
await expectApiResponseOK(keyResponse, `Check preseeded Agent API key ${agentName}`)
const keys = (await keyResponse.json()) as ApiKeyList
const key = keys.data.at(0)
if (!key) {
return skipBlockedPrecondition(
world,
`Preseeded Agent "${agentName}" Backend service API key list is empty.`,
)
}
return {
id: key.id,
kind: 'api-key',
name: `${agentName} Backend service API key`,
}
} finally {
await ctx.dispose()
}
}
export async function skipMissingPreseededAgentPublishedWebApp(
world: DifyWorld,
agentName: string,
): Promise<'skipped' | PreseededResource> {
const agent = await skipMissingPreseededAgent(world, agentName)
if (agent === 'skipped') return agent
const ctx = await createApiContext()
try {
const response = await ctx.get(`/console/api/agent/${agent.id}`)
await expectApiResponseOK(response, `Check preseeded Agent published Web app ${agentName}`)
const detail = (await response.json()) as AgentAppDetailWithSite
if (detail.active_config_is_published !== true) {
return skipBlockedPrecondition(world, `Preseeded Agent "${agentName}" is not published.`)
}
if (detail.enable_site !== true) {
return skipBlockedPrecondition(
world,
`Preseeded Agent "${agentName}" Web app is not enabled.`,
)
}
const siteToken = detail.site?.access_token ?? detail.site?.code
if (!siteToken || !detail.site?.app_base_url) {
return skipBlockedPrecondition(
world,
`Preseeded Agent "${agentName}" Web app URL is not available.`,
)
}
return {
id: agent.id,
kind: 'agent',
name: agent.name,
}
} finally {
await ctx.dispose()
}
}
export async function skipMissingPreseededAgentWorkflowReference(
world: DifyWorld,
agentName: string,
workflowName: string,
): Promise<'skipped' | PreseededResource> {
const agent = await skipMissingPreseededAgent(world, agentName)
if (agent === 'skipped') return agent
const workflow = await skipMissingPreseededWorkflow(world, workflowName)
if (workflow === 'skipped') return workflow
const ctx = await createApiContext()
try {
const response = await ctx.get(`/console/api/agent/${agent.id}/referencing-workflows`)
await expectApiResponseOK(response, `Check preseeded Agent workflow reference ${agentName}`)
const references = (await response.json()) as AgentReferencingWorkflowsResponse
const reference = references.data?.find(
(item) => item.app_id === workflow.id || item.app_name === workflow.name,
)
if (!reference) {
return skipBlockedPrecondition(
world,
`Preseeded Agent "${agentName}" is not referenced by workflow "${workflowName}".`,
)
}
if (!reference.node_ids || reference.node_ids.length < 1) {
return skipBlockedPrecondition(
world,
`Preseeded workflow "${workflowName}" does not expose Agent reference nodes for "${agentName}".`,
)
}
return {
id: workflow.id,
kind: 'workflow',
name: workflow.name,
}
} finally {
await ctx.dispose()
}
}

147
e2e/features/agent-v2/support/seed.ts
View File

@ -27,7 +27,6 @@ import {
import { bootstrapMarketplacePlugins } from '../../../support/marketplace-plugins'
import { sleep } from '../../../support/process'
import { blocked, created, skipped, updated, verified } from '../../../support/seed'
import { createAgentApiKey, setAgentApiAccess, setAgentSiteAccessAndGetURL } from './access-point'
import { createTestAgent, publishAgent, saveAgentComposerDraft } from './agent'
import {
agentBuilderExpectedTokens,
@ -50,8 +49,8 @@ import {
findConsoleResourceByName,
isRecord,
matchesNameOrLabel,
} from './preflight/common'
import { splitToolDisplayName } from './preflight/tools'
} from './fixtures/common'
import { splitToolDisplayName } from './fixtures/tools'
import { agentBuilderTestMaterials, getAgentBuilderTestMaterialPath } from './test-materials'
type StableModel = {
@ -69,10 +68,6 @@ const modelCredentialEnv = 'E2E_MODEL_PROVIDER_CREDENTIALS_JSON'
const speechToTextModelProviderEnv = 'E2E_SPEECH_TO_TEXT_MODEL_PROVIDER'
const speechToTextModelNameEnv = 'E2E_SPEECH_TO_TEXT_MODEL_NAME'
const marketplacePluginIdsEnv = 'E2E_MARKETPLACE_PLUGIN_IDS'
const marketplacePluginUniqueIdentifiersEnv = 'E2E_MARKETPLACE_PLUGIN_UNIQUE_IDENTIFIERS'
const oauthToolCredentialIdEnv = 'E2E_OAUTH_TOOL_CREDENTIAL_ID'
const oauthToolProviderEnv = 'E2E_OAUTH_TOOL_PROVIDER'
const oauthToolNameEnv = 'E2E_OAUTH_TOOL_NAME'
const activeModelStatus = 'active'
const stableModelCredentialName = 'E2E Stable Model'
const agentV2MarketplacePluginIds = [
@ -882,45 +877,6 @@ const seedDualRetrievalAgent = async (context: SeedContext) => {
return wasCreated ? created(title, resource) : updated(title, resource)
}
const seedPublishedWebAppAgent = async (context: SeedContext) => {
const title = agentBuilderPreseededResources.publishedWebAppAgent
const model = getStableModelResource(context)
if (!model)
return blocked(title, `${agentBuilderPreseededResources.stableChatModel} is not ready.`)
if (context.dryRun) return skipped(title, `Would create or update Agent "${title}".`)
const { agent, created: wasCreated } = await ensureAgent(title)
await saveSeededAgentComposer({
agentId: agent.id,
config: createAgentSoulConfigWithModel(normalAgentSoulConfig, model),
shouldPublish: true,
})
await setAgentSiteAccessAndGetURL(agent.id, true)
const resource = { id: agent.id, kind: 'agent', name: title }
return wasCreated ? created(title, resource) : updated(title, resource)
}
const seedBackendApiAgent = async (context: SeedContext) => {
const title = agentBuilderPreseededResources.backendApiEnabledAgent
const model = getStableModelResource(context)
if (!model)
return blocked(title, `${agentBuilderPreseededResources.stableChatModel} is not ready.`)
if (context.dryRun) return skipped(title, `Would create or update Agent "${title}".`)
const { agent, created: wasCreated } = await ensureAgent(title)
await saveSeededAgentComposer({
agentId: agent.id,
config: createAgentSoulConfigWithModel(normalAgentSoulConfig, model),
shouldPublish: true,
})
const access = await setAgentApiAccess(agent.id, true)
if (access.api_key_count < 1) await createAgentApiKey(agent.id)
const resource = { id: agent.id, kind: 'agent', name: title }
return wasCreated ? created(title, resource) : updated(title, resource)
}
const findWorkflow = (name: string) => {
const query = buildQuery({ limit: '20', mode: 'workflow', name, page: '1' })
return findConsoleResourceByName({
@ -961,52 +917,6 @@ const seedWorkflowReference = async (context: SeedContext) => {
: updated(`${title} / ${workflowName}`, resource)
}
const seedOAuthToolAgent = async (context: SeedContext) => {
const title = agentBuilderPreseededResources.oauthToolAgent
const credentialId = process.env[oauthToolCredentialIdEnv]?.trim()
const providerName = process.env[oauthToolProviderEnv]?.trim()
const toolName = process.env[oauthToolNameEnv]?.trim()
if (!credentialId || !providerName || !toolName) {
return blocked(
title,
`${oauthToolCredentialIdEnv}, ${oauthToolProviderEnv}, and ${oauthToolNameEnv} are required for OAuth2 tool seed.`,
)
}
if (context.dryRun)
return skipped(
title,
`Would create or update Agent "${title}" with OAuth2 tool ${providerName}/${toolName}.`,
)
const { agent, created: wasCreated } = await ensureAgent(title)
await saveSeededAgentComposer({
agentId: agent.id,
config: {
...normalAgentSoulConfig,
tools: {
dify_tools: [
{
credential_ref: {
id: credentialId,
provider: providerName,
type: 'provider',
},
credential_type: 'oauth2',
enabled: true,
provider_id: providerName,
provider_type: 'builtin',
runtime_parameters: {},
tool_name: toolName,
},
],
},
},
})
const resource = { id: agent.id, kind: 'agent', name: title }
return wasCreated ? created(title, resource) : updated(title, resource)
}
const agentV2BaseSeedTasks = (): SeedTask[] => [
{
id: 'marketplace-plugins',
@ -1015,7 +925,6 @@ const agentV2BaseSeedTasks = (): SeedTask[] => [
bootstrapMarketplacePlugins(context, {
defaultPluginIds: agentV2MarketplacePluginIds,
pluginIdsEnv: marketplacePluginIdsEnv,
pluginUniqueIdentifiersEnv: marketplacePluginUniqueIdentifiersEnv,
title: 'Agent V2 marketplace plugins',
}),
},
@ -1038,26 +947,7 @@ const agentV2BaseSeedTasks = (): SeedTask[] => [
},
]
const agentV2FullSeedTasks = (): SeedTask[] => [
...agentV2BaseSeedTasks(),
{
id: 'indexing-knowledge',
title: agentBuilderPreseededResources.indexingKnowledgeBase,
run: async () =>
blocked(
agentBuilderPreseededResources.indexingKnowledgeBase,
'A deterministic long-lived "currently indexing" dataset seed is not implemented yet.',
),
},
{
id: 'broken-model',
title: agentBuilderPreseededResources.brokenModelProvider,
run: async () =>
blocked(
agentBuilderPreseededResources.brokenModelProvider,
'Broken model fixture is validation-only for now; provide E2E_BROKEN_MODEL_PROVIDER and keep the model entry externally.',
),
},
const agentV2PreparedFixtureSeedTasks = (): SeedTask[] => [
{
id: 'full-config-agent',
title: agentBuilderPreseededResources.fullConfigAgent,
@ -1068,26 +958,11 @@ const agentV2FullSeedTasks = (): SeedTask[] => [
title: agentBuilderPreseededResources.toolStatesAgent,
run: seedToolStatesAgent,
},
{
id: 'oauth-tool-agent',
title: agentBuilderPreseededResources.oauthToolAgent,
run: seedOAuthToolAgent,
},
{
id: 'dual-retrieval-agent',
title: agentBuilderPreseededResources.dualRetrievalAgent,
run: seedDualRetrievalAgent,
},
{
id: 'published-web-app-agent',
title: agentBuilderPreseededResources.publishedWebAppAgent,
run: seedPublishedWebAppAgent,
},
{
id: 'backend-api-agent',
title: agentBuilderPreseededResources.backendApiEnabledAgent,
run: seedBackendApiAgent,
},
{
id: 'workflow-reference',
title: `${agentBuilderPreseededResources.workflowReferenceAgent} / ${agentBuilderPreseededResources.referenceWorkflow}`,
@ -1095,6 +970,11 @@ const agentV2FullSeedTasks = (): SeedTask[] => [
},
]
const agentV2PreparedSeedTasks = (): SeedTask[] => [
...agentV2BaseSeedTasks(),
...agentV2PreparedFixtureSeedTasks(),
]
const agentV2ExternalRuntimeSeedTasks = (): SeedTask[] => [
...agentV2BaseSeedTasks(),
{
@ -1104,8 +984,15 @@ const agentV2ExternalRuntimeSeedTasks = (): SeedTask[] => [
},
]
export const createAgentV2SeedTasks = (profile: string = 'full'): SeedTask[] => {
if (profile === 'full') return agentV2FullSeedTasks()
const agentV2PostMergeSeedTasks = (): SeedTask[] => [
...agentV2ExternalRuntimeSeedTasks(),
...agentV2PreparedFixtureSeedTasks(),
]
export const createAgentV2SeedTasks = (profile: string = 'post-merge'): SeedTask[] => {
if (profile === 'post-merge') return agentV2PostMergeSeedTasks()
if (profile === 'prepared') return agentV2PreparedSeedTasks()
if (profile === 'external-runtime') return agentV2ExternalRuntimeSeedTasks()

6
e2e/features/agent-v2/support/tools.ts
View File

@ -1,11 +1,11 @@
import type { DifyWorld } from '../../support/world'
import { splitToolDisplayName, splitToolResourceId } from './preflight/tools'
import { splitToolDisplayName, splitToolResourceId } from './fixtures/tools'
export const getPreseededToolContract = (world: DifyWorld, resourceName: string) => {
const resource = world.agentBuilder.preflight.preseededResources[resourceName]
const resource = world.agentBuilder.fixtures.preseededResources[resourceName]
if (!resource || resource.kind !== 'tool') {
throw new Error(
`Preseeded tool "${resourceName}" is not available. Run the matching preflight step first.`,
`Preseeded tool "${resourceName}" is not available. Run the matching fixture setup step first.`,
)
}

16
e2e/features/agent-v2/tools.feature
View File

@ -1,6 +1,6 @@
@agent-v2 @authenticated @tools
Feature: Agent v2 tools
@core @tool-fixture
@core @prepared @tool-fixture
Scenario: JSON Replace tool is saved after adding it from the Tools selector
Given I am signed in as the default E2E admin
And the Agent Builder preseeded tool "JSON Process / JSON Replace" is available
@ -12,20 +12,6 @@ Feature: Agent v2 tools
When I refresh the current page
Then I should see the Agent v2 JSON Replace tool in the Tools section
@core @oauth-tool-agent
Scenario: OAuth2 tool credentials stay authorized after Configure autosaves
Given I am signed in as the default E2E admin
And the Agent Builder preseeded Agent "E2E Agent With OAuth Tool" includes an OAuth2 tool credential
And an Agent v2 test agent with the OAuth2 tool credential fixture has been created via API
When I open the Agent v2 configure page
Then I should see the Agent v2 OAuth2 tool authorized in the Tools section
When I fill the Agent v2 prompt editor with the updated E2E prompt
Then the Agent v2 configuration should be saved automatically
And the Agent v2 OAuth2 tool credential should remain saved in the Agent v2 draft
When I refresh the current page
Then I should see the Agent v2 OAuth2 tool authorized in the Tools section
And the Agent v2 OAuth2 tool credential should remain saved in the Agent v2 draft
@service-api-runtime @external-model @agent-backend-runtime @stable-model @tool-fixture
Scenario: JSON Replace tool runtime returns the replacement marker
Given I am signed in as the default E2E admin

12
e2e/features/apps/app-detail-navigation.feature
View File

@ -12,15 +12,3 @@ Feature: App detail navigation
And a "chat" app has been created via API
When I open the app from the app list
Then I should land on the app configuration page
Scenario: The develop tab is accessible from a workflow app
Given I am signed in as the default E2E admin
And a "workflow" app has been created via API
When I navigate to the app develop page
Then I should be on the app develop page
Scenario: The overview tab is accessible from a workflow app
Given I am signed in as the default E2E admin
And a "workflow" app has been created via API
When I navigate to the app overview page
Then I should be on the app overview page

6
e2e/features/smoke/install.feature
View File

@ -1,6 +0,0 @@
@smoke @fresh
Feature: Fresh installation bootstrap
Scenario: Complete the initial installation bootstrap on a fresh instance
Given the last authentication bootstrap came from a fresh install
When I open the apps console
Then I should stay on the apps console

4
e2e/features/step-definitions/agent-v2/access-point-helpers.ts
View File

@ -12,10 +12,10 @@ export const getPreseededResource = (
name: string,
kind: 'agent' | 'workflow',
) => {
const resource = world.agentBuilder.preflight.preseededResources[name]
const resource = world.agentBuilder.fixtures.preseededResources[name]
if (!resource || resource.kind !== kind) {
throw new Error(
`Preseeded ${kind} "${name}" is not available. Run the matching preflight step first.`,
`Preseeded ${kind} "${name}" is not available. Run the matching fixture setup step first.`,
)
}

31
e2e/features/step-definitions/agent-v2/agent-edit.steps.ts
View File

@ -1,16 +1,16 @@
import type { PostAgentByAgentIdCopyResponse } from '@dify/contracts/api/console/agent/types.gen'
import type { DifyWorld } from '../../support/world'
import { Then, When } from '@cucumber/cucumber'
import { Given, Then, When } from '@cucumber/cucumber'
import { expect } from '@playwright/test'
import { createE2EResourceName } from '../../../support/naming'
import { getAgentComposerDraft, getTestAgent } from '../../agent-v2/support/agent'
import { getAgentComposerDraft, getTestAgent, publishAgent } from '../../agent-v2/support/agent'
import {
agentBuilderExpectedTokens,
agentBuilderFixedInputs,
agentBuilderPreseededResources,
} from '../../agent-v2/support/agent-builder-resources'
import { normalAgentPrompt } from '../../agent-v2/support/agent-soul'
import { asArray, asRecord, asString } from '../../agent-v2/support/preflight/common'
import { asArray, asRecord, asString } from '../../agent-v2/support/fixtures/common'
import { agentBuilderTestMaterials } from '../../agent-v2/support/test-materials'
import {
expectProviderToolActionVisible,
@ -65,6 +65,13 @@ const getComposerInheritanceSnapshot = async (agentId: string) => {
}
}
Given(
'the preseeded Agent v2 {string} has been published via API',
async function (this: DifyWorld, agentName: string) {
await publishAgent(getPreseededAgent(this, agentName).id)
},
)
When(
'I duplicate the preseeded Agent v2 {string} from the Agent Roster',
async function (this: DifyWorld, agentName: string) {
@ -89,7 +96,7 @@ When(
response.request().method() === 'POST' &&
new URL(response.url()).pathname.endsWith(`/console/api/agent/${agent.id}/copy`),
)
await dialog.getByRole('button', { name: 'Duplicate' }).click()
await dialog.getByRole('button', { exact: true, name: 'Duplicate' }).click()
const copyResponse = await copyResponsePromise
expect(copyResponse.status()).toBe(201)
@ -107,9 +114,11 @@ When(
Then('I should see the Agent v2 full-config fixture sections', async function (this: DifyWorld) {
const page = this.getPage()
const stableModel = this.agentBuilder.preflight.stableModel
const stableModel = this.agentBuilder.fixtures.stableModel
if (!stableModel)
throw new Error('Stable chat model preflight must run before asserting the full-config Agent.')
throw new Error(
'Stable chat model fixture setup must run before asserting the full-config Agent.',
)
await expect(page.getByRole('heading', { name: 'Configure' })).toBeVisible({ timeout: 30_000 })
await expect(
@ -130,7 +139,7 @@ Then('I should see the Agent v2 full-config fixture sections', async function (t
}),
).toBeVisible()
const filesSection = page.getByRole('region', { name: 'Files' })
const filesSection = page.getByRole('region', { exact: true, name: 'Files' })
await expect(filesSection).toBeVisible()
await expect(
filesSection.getByRole('button', {
@ -168,9 +177,11 @@ Then(
async function (this: DifyWorld, agentName: string) {
const sourceAgent = getPreseededAgent(this, agentName)
const duplicatedAgentId = getCurrentAgentId(this)
const stableModel = this.agentBuilder.preflight.stableModel
const stableModel = this.agentBuilder.fixtures.stableModel
if (!stableModel)
throw new Error('Stable chat model preflight must run before asserting the duplicated Agent.')
throw new Error(
'Stable chat model fixture setup must run before asserting the duplicated Agent.',
)
const [sourceDetail, duplicatedDetail, sourceSnapshot, duplicatedSnapshot] = await Promise.all([
getTestAgent(sourceAgent.id),
@ -232,7 +243,7 @@ Then('I should see the Agent v2 tool state fixture tools', async function (this:
await expect(toolsSection).toBeVisible({ timeout: 30_000 })
await expect(
toolsSection.getByRole('button', { exact: true, name: 'Not authorized' }),
).toBeVisible()
).toHaveCount(2)
const { action: jsonReplaceAction, tool: jsonTool } = await expectProviderToolActionVisible(
toolsSection,

129
e2e/features/step-definitions/agent-v2/build-draft.steps.ts
View File

@ -6,7 +6,6 @@ import { expect } from '@playwright/test'
import { getAgentComposerDraft, saveAgentComposerDraft } from '../../agent-v2/support/agent'
import {
agentBuildDraftExists,
applyAgentBuildDraft,
getAgentBuildDraft,
saveAgentBuildDraft,
} from '../../agent-v2/support/agent-build-draft'
@ -61,81 +60,65 @@ const getLastBuildChatAnswerText = async (page: Page) => {
const formatBuildChatAnswerText = (text: string) =>
text.length > 500 ? `${text.slice(0, 500)}...` : text
const saveSupportedBuildDraft = async (
world: DifyWorld,
{ retainSkillInNormalDraft }: { retainSkillInNormalDraft: boolean },
) => {
const agentId = getCurrentAgentId(world)
const configFile = await uploadAgentConfigFileToDraft({
agentId,
fileName: agentBuilderTestMaterials.smallFile,
filePath: getAgentBuilderTestMaterialPath('smallFile'),
})
const skill = await uploadSummaryConfigSkillForBuildDraft(world)
if (!configFile.file_id)
throw new Error('Agent v2 build draft config file fixture did not return a file_id.')
world.createdAgentConfigFiles.push({ agentId, name: configFile.name })
const stableModel = world.agentBuilder.fixtures.stableModel
const normalConfig = stableModel
? createAgentSoulConfigWithModel(normalAgentSoulConfig, stableModel)
: normalAgentSoulConfig
const updatedConfig = stableModel
? createAgentSoulConfigWithModel(updatedAgentSoulConfig, stableModel)
: updatedAgentSoulConfig
const configSkills = [skill]
await saveAgentComposerDraft(agentId, {
...normalConfig,
...(retainSkillInNormalDraft ? { config_skills: configSkills } : {}),
})
await saveAgentBuildDraft(agentId, {
...updatedConfig,
config_files: [configFile],
config_skills: configSkills,
env: {
secret_refs: [],
variables: [
{
id: agentBuilderFixedInputs.envPlainKey,
key: agentBuilderFixedInputs.envPlainKey,
name: agentBuilderFixedInputs.envPlainKey,
value: agentBuilderFixedInputs.envPlainValue,
variable: agentBuilderFixedInputs.envPlainKey,
},
],
},
})
}
Given(
'an Agent v2 Build draft adds the supported E2E files, skills, and env',
async function (this: DifyWorld) {
const agentId = getCurrentAgentId(this)
const configFile = await uploadAgentConfigFileToDraft({
agentId,
fileName: agentBuilderTestMaterials.smallFile,
filePath: getAgentBuilderTestMaterialPath('smallFile'),
})
const skill = await uploadSummaryConfigSkillForBuildDraft(this)
if (!configFile.file_id)
throw new Error('Agent v2 build draft config file fixture did not return a file_id.')
this.createdAgentConfigFiles.push({ agentId, name: configFile.name })
const normalConfig = this.agentBuilder.preflight.stableModel
? createAgentSoulConfigWithModel(
normalAgentSoulConfig,
this.agentBuilder.preflight.stableModel,
)
: normalAgentSoulConfig
const updatedConfig = this.agentBuilder.preflight.stableModel
? createAgentSoulConfigWithModel(
updatedAgentSoulConfig,
this.agentBuilder.preflight.stableModel,
)
: updatedAgentSoulConfig
await saveAgentComposerDraft(agentId, normalConfig)
await saveAgentBuildDraft(agentId, {
...updatedConfig,
config_files: [configFile],
config_skills: [skill],
env: {
secret_refs: [],
variables: [
{
id: agentBuilderFixedInputs.envPlainKey,
key: agentBuilderFixedInputs.envPlainKey,
name: agentBuilderFixedInputs.envPlainKey,
value: agentBuilderFixedInputs.envPlainValue,
variable: agentBuilderFixedInputs.envPlainKey,
},
],
},
})
await saveSupportedBuildDraft(this, { retainSkillInNormalDraft: false })
},
)
Given(
'an Agent v2 Build draft includes the existing e2e-summary-skill Skill',
'an Agent v2 Build draft adds supported E2E files and env while retaining the existing Skill',
async function (this: DifyWorld) {
const skill = await uploadSummaryConfigSkillForBuildDraft(this)
const normalConfig = this.agentBuilder.preflight.stableModel
? createAgentSoulConfigWithModel(
normalAgentSoulConfig,
this.agentBuilder.preflight.stableModel,
)
: normalAgentSoulConfig
const updatedConfig = this.agentBuilder.preflight.stableModel
? createAgentSoulConfigWithModel(
updatedAgentSoulConfig,
this.agentBuilder.preflight.stableModel,
)
: updatedAgentSoulConfig
const configSkills = [skill]
await saveAgentComposerDraft(getCurrentAgentId(this), {
...normalConfig,
config_skills: configSkills,
})
await saveAgentBuildDraft(getCurrentAgentId(this), {
...updatedConfig,
config_skills: configSkills,
})
await saveSupportedBuildDraft(this, { retainSkillInNormalDraft: true })
},
)
@ -146,16 +129,16 @@ Given('an Agent v2 Build draft uses the updated E2E prompt', async function (thi
Given(
'an Agent v2 Build draft uses the updated E2E prompt with the stable E2E model',
async function (this: DifyWorld) {
if (!this.agentBuilder.preflight.stableModel)
if (!this.agentBuilder.fixtures.stableModel)
throw new Error(
'Create an Agent v2 Build draft with a stable model after stable model preflight.',
'Create an Agent v2 Build draft with a stable model after stable model fixture setup.',
)
await saveAgentBuildDraft(
getCurrentAgentId(this),
createAgentSoulConfigWithModel(
updatedAgentSoulConfig,
this.agentBuilder.preflight.stableModel,
this.agentBuilder.fixtures.stableModel,
),
)
},
@ -279,10 +262,6 @@ When(
},
)
When('I apply the Agent v2 Build draft via API', async function (this: DifyWorld) {
await applyAgentBuildDraft(getCurrentAgentId(this))
})
Then('I should see the Agent v2 Build draft pending changes', async function (this: DifyWorld) {
const page = this.getPage()

4
e2e/features/step-definitions/agent-v2/configure-helpers.ts
View File

@ -18,10 +18,10 @@ export const getCurrentAgentId = (world: DifyWorld) => {
}
export const getPreseededAgent = (world: DifyWorld, name: string) => {
const resource = world.agentBuilder.preflight.preseededResources[name]
const resource = world.agentBuilder.fixtures.preseededResources[name]
if (!resource || resource.kind !== 'agent') {
throw new Error(
`Preseeded Agent "${name}" is not available. Run the matching preflight step first.`,
`Preseeded Agent "${name}" is not available. Run the matching fixture setup step first.`,
)
}

102
e2e/features/step-definitions/agent-v2/configure.steps.ts
View File

@ -43,9 +43,10 @@ async function fillAgentPromptEditor(page: Page, prompt: string) {
}
async function selectAgentModel(page: Page, modelName: string) {
await page.getByRole('combobox', { name: 'Configure model' }).click()
await page.getByLabel('Search model').fill(modelName)
await page.getByRole('option', { name: modelName }).click()
await page.getByRole('combobox').first().click()
await page.getByPlaceholder('Search model').fill(modelName)
const escapedModelName = modelName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
await page.getByRole('option', { name: new RegExp(`${escapedModelName}(?:\\s|$)`) }).click()
}
async function expectAgentComposerPrompt(agentId: string, prompt: string) {
@ -74,13 +75,13 @@ Given(
)
Given('a runnable Agent v2 test agent has been created via API', async function (this: DifyWorld) {
if (!this.agentBuilder.preflight.stableModel)
throw new Error('Create a runnable Agent v2 test agent after stable model preflight.')
if (!this.agentBuilder.fixtures.stableModel)
throw new Error('Create a runnable Agent v2 test agent after stable model fixture setup.')
const agent = await createConfiguredTestAgent({
agentSoul: createAgentSoulConfigWithModel(
normalAgentSoulConfig,
this.agentBuilder.preflight.stableModel,
this.agentBuilder.fixtures.stableModel,
),
})
this.createdAgentIds.push(agent.id)
@ -91,16 +92,16 @@ Given('a runnable Agent v2 test agent has been created via API', async function
Given(
'a runnable Agent v2 test agent using the agent-decision model has been created via API',
async function (this: DifyWorld) {
if (!this.agentBuilder.preflight.agentDecisionModel) {
if (!this.agentBuilder.fixtures.agentDecisionModel) {
throw new Error(
'Create an agent-decision Agent v2 test agent after agent-decision model preflight.',
'Create an agent-decision Agent v2 test agent after agent-decision model fixture setup.',
)
}
const agent = await createConfiguredTestAgent({
agentSoul: createAgentSoulConfigWithModel(
normalAgentSoulConfig,
this.agentBuilder.preflight.agentDecisionModel,
this.agentBuilder.fixtures.agentDecisionModel,
),
})
this.createdAgentIds.push(agent.id)
@ -157,14 +158,27 @@ When('I open the Agent v2 configure page', async function (this: DifyWorld) {
When(
'I select the stable E2E model in the Agent v2 model selector',
async function (this: DifyWorld) {
const stableModel = this.agentBuilder.preflight.stableModel
const stableModel = this.agentBuilder.fixtures.stableModel
if (!stableModel)
throw new Error('Stable chat model preflight must run before selecting the Agent model.')
throw new Error('Stable chat model fixture setup must run before selecting the Agent model.')
await selectAgentModel(this.getPage(), stableModel.name)
},
)
When(
'I select the agent-decision E2E model in the Agent v2 model selector',
async function (this: DifyWorld) {
const model = this.agentBuilder.fixtures.agentDecisionModel
if (!model)
throw new Error(
'Agent-decision chat model fixture setup must run before selecting the Agent model.',
)
await selectAgentModel(this.getPage(), model.name)
},
)
When('I switch to the Agent v2 Configure section', async function (this: DifyWorld) {
const page = this.getPage()
const agentId = getCurrentAgentId(this)
@ -298,9 +312,9 @@ Then(
Then(
'I should see the stable E2E model in the Agent v2 model selector',
async function (this: DifyWorld) {
const stableModel = this.agentBuilder.preflight.stableModel
const stableModel = this.agentBuilder.fixtures.stableModel
if (!stableModel)
throw new Error('Stable chat model preflight must run before asserting the Agent model.')
throw new Error('Stable chat model fixture setup must run before asserting the Agent model.')
await expect(this.getPage().getByText(stableModel.name, { exact: true })).toBeVisible({
timeout: 30_000,
@ -308,6 +322,21 @@ Then(
},
)
Then(
'I should see the agent-decision E2E model in the Agent v2 model selector',
async function (this: DifyWorld) {
const model = this.agentBuilder.fixtures.agentDecisionModel
if (!model)
throw new Error(
'Agent-decision chat model fixture setup must run before asserting the Agent model.',
)
await expect(this.getPage().getByText(model.name, { exact: true })).toBeVisible({
timeout: 30_000,
})
},
)
Then(
'I should see the updated E2E prompt in the Agent v2 prompt editor',
async function (this: DifyWorld) {
@ -345,16 +374,6 @@ Then(
},
)
Then(
'Agent v2 Preview should be unavailable until a model is configured',
async function (this: DifyWorld) {
const page = this.getPage()
await expect(page.getByRole('heading', { name: 'Configure' })).toBeVisible({ timeout: 30_000 })
await expect(page.getByRole('button', { name: /^Preview$/i })).toBeDisabled()
},
)
Then(
'the normal Agent v2 draft should still use the normal E2E prompt',
async function (this: DifyWorld) {
@ -381,9 +400,9 @@ Then(
)
Then('the Agent v2 draft should use the stable E2E model', async function (this: DifyWorld) {
const stableModel = this.agentBuilder.preflight.stableModel
const stableModel = this.agentBuilder.fixtures.stableModel
if (!stableModel)
throw new Error('Stable chat model preflight must run before asserting the Agent model.')
throw new Error('Stable chat model fixture setup must run before asserting the Agent model.')
await expect
.poll(
@ -406,3 +425,36 @@ Then('the Agent v2 draft should use the stable E2E model', async function (this:
provider: stableModel.provider,
})
})
Then(
'the Agent v2 draft should use the agent-decision E2E model',
async function (this: DifyWorld) {
const model = this.agentBuilder.fixtures.agentDecisionModel
if (!model)
throw new Error(
'Agent-decision chat model fixture setup must run before asserting the Agent model.',
)
await expect
.poll(
async () => {
const draftModel = (await getAgentComposerDraft(getCurrentAgentId(this))).agent_soul
?.model
const modelConfig =
typeof draftModel === 'object' && draftModel !== null && !Array.isArray(draftModel)
? (draftModel as Record<string, unknown>)
: undefined
return {
model: modelConfig?.model,
provider: modelConfig?.model_provider,
}
},
{ timeout: 30_000 },
)
.toEqual({
model: model.name,
provider: model.provider,
})
},
)

151
e2e/features/step-definitions/agent-v2/content-moderation.steps.ts
View File

@ -1,151 +0,0 @@
import type { Locator } from '@playwright/test'
import type { DifyWorld } from '../../support/world'
import { Then, When } from '@cucumber/cucumber'
import { expect } from '@playwright/test'
import { getAgentComposerDraft } from '../../agent-v2/support/agent'
import { agentBuilderFixedInputs } from '../../agent-v2/support/agent-builder-resources'
import { skipBlockedPrecondition } from '../../agent-v2/support/preflight/common'
import { getCurrentAgentId } from './configure-helpers'
const getModerationSettingsDialog = (world: DifyWorld) =>
world.getPage().getByRole('dialog').filter({ hasText: 'Content moderation settings' })
const getContentModerationRegion = (world: DifyWorld) =>
world.getPage().getByRole('region', { name: 'Content moderation' })
const ensureSwitchChecked = async (switchLocator: Locator) => {
if ((await switchLocator.getAttribute('aria-checked')) !== 'true') await switchLocator.click()
}
When(
'I configure Agent v2 Content Moderation keyword preset replies',
async function (this: DifyWorld) {
const page = this.getPage()
const contentModeration = getContentModerationRegion(this)
const enabledSwitch = contentModeration.getByRole('switch', { name: 'Content moderation' })
if ((await enabledSwitch.getAttribute('aria-checked')) === 'true')
await contentModeration.getByRole('button', { name: 'Settings' }).click()
else await enabledSwitch.click()
const dialog = getModerationSettingsDialog(this)
await expect(dialog).toBeVisible()
await dialog.getByRole('button', { name: 'Keywords' }).click()
await dialog
.getByRole('textbox', { name: 'Keywords' })
.fill(agentBuilderFixedInputs.moderationKeyword)
const inputModeration = dialog.getByRole('region', { name: 'Moderate INPUT Content' })
const outputModeration = dialog.getByRole('region', { name: 'Moderate OUTPUT Content' })
await ensureSwitchChecked(
inputModeration.getByRole('switch', { name: 'Moderate INPUT Content' }),
)
await dialog.getByRole('button', { name: 'Save' }).click()
await expect(page.getByText('Preset replies cannot be empty')).toBeVisible()
await expect(dialog).toBeVisible()
await inputModeration
.getByRole('textbox', { name: 'Preset replies' })
.fill(agentBuilderFixedInputs.inputModerationReply)
await ensureSwitchChecked(
outputModeration.getByRole('switch', { name: 'Moderate OUTPUT Content' }),
)
await outputModeration
.getByRole('textbox', { name: 'Preset replies' })
.fill(agentBuilderFixedInputs.outputModerationReply)
await dialog.getByRole('button', { name: 'Save' }).click()
await expect(dialog).not.toBeVisible()
},
)
Then('Agent v2 Content Moderation Settings should be available', async function (this: DifyWorld) {
const advancedSettings = this.getPage().getByRole('region', { name: 'Advanced Settings' })
const contentModeration = advancedSettings.getByRole('region', { name: 'Content moderation' })
try {
await expect(contentModeration).toBeVisible({ timeout: 3_000 })
} catch {
return skipBlockedPrecondition(
this,
'Agent v2 Content Moderation Settings is not available in this build.',
{
owner: 'product',
remediation:
'Enable the Agent v2 Content Moderation feature flag in the product or keep this scenario feature-gated.',
},
)
}
})
Then(
'Agent v2 Content Moderation keyword preset replies should be saved in the Agent v2 draft',
async function (this: DifyWorld) {
const agentId = getCurrentAgentId(this)
await expect
.poll(
async () => {
const draft = await getAgentComposerDraft(agentId)
const appFeatures = draft.agent_soul?.app_features as Record<string, unknown> | undefined
const moderation = appFeatures?.sensitive_word_avoidance as
| Record<string, unknown>
| undefined
const config = moderation?.config as Record<string, unknown> | undefined
const inputsConfig = config?.inputs_config as Record<string, unknown> | undefined
const outputsConfig = config?.outputs_config as Record<string, unknown> | undefined
return {
enabled: moderation?.enabled,
inputEnabled: inputsConfig?.enabled,
inputPreset: inputsConfig?.preset_response,
keywords: config?.keywords,
outputEnabled: outputsConfig?.enabled,
outputPreset: outputsConfig?.preset_response,
type: moderation?.type,
}
},
{
timeout: 30_000,
},
)
.toEqual({
enabled: true,
inputEnabled: true,
inputPreset: agentBuilderFixedInputs.inputModerationReply,
keywords: agentBuilderFixedInputs.moderationKeyword,
outputEnabled: true,
outputPreset: agentBuilderFixedInputs.outputModerationReply,
type: 'keywords',
})
},
)
Then(
'I should see the Agent v2 Content Moderation keyword preset replies in Advanced Settings',
async function (this: DifyWorld) {
const contentModeration = getContentModerationRegion(this)
await expect(contentModeration).toContainText('Keywords')
await expect(contentModeration).toContainText('INPUT & OUTPUT')
await contentModeration.getByRole('button', { name: 'Settings' }).click()
const dialog = getModerationSettingsDialog(this)
await expect(dialog).toBeVisible()
await expect(dialog.getByRole('textbox', { name: 'Keywords' })).toHaveValue(
agentBuilderFixedInputs.moderationKeyword,
)
await expect(
dialog
.getByRole('region', { name: 'Moderate INPUT Content' })
.getByRole('textbox', { name: 'Preset replies' }),
).toHaveValue(agentBuilderFixedInputs.inputModerationReply)
await expect(
dialog
.getByRole('region', { name: 'Moderate OUTPUT Content' })
.getByRole('textbox', { name: 'Preset replies' }),
).toHaveValue(agentBuilderFixedInputs.outputModerationReply)
await dialog.getByRole('button', { name: 'Cancel' }).click()
await expect(dialog).not.toBeVisible()
},
)

117
e2e/features/step-definitions/agent-v2/fixtures.steps.ts Normal file
View File

@ -0,0 +1,117 @@
import type { DifyWorld } from '../../support/world'
import { Given } from '@cucumber/cucumber'
import { requirePreseededAgentWorkflowReference } from '../../agent-v2/support/fixtures/access'
import { requireAgentBackendRuntime } from '../../agent-v2/support/fixtures/agent-backend'
import {
requirePreseededAgent,
requirePreseededDualRetrievalAgentConfiguration,
requirePreseededFullConfigAgentCoreConfiguration,
requirePreseededToolStatesAgentConfiguration,
requirePreseededWorkflow,
} from '../../agent-v2/support/fixtures/agents'
import { requireReadyPreseededDataset } from '../../agent-v2/support/fixtures/datasets'
import {
requireAgentBuilderAgentDecisionChatModel,
requireAgentBuilderSpeechToTextModel,
requireAgentBuilderStableChatModel,
} from '../../agent-v2/support/fixtures/models'
import { requirePreseededTool } from '../../agent-v2/support/fixtures/tools'
Given('the Agent Builder stable chat model is available', async function (this: DifyWorld) {
const stableModel = await requireAgentBuilderStableChatModel(this)
this.agentBuilder.fixtures.stableModel = stableModel
})
Given('the workspace default speech-to-text model is active', async function (this: DifyWorld) {
const speechToTextModel = await requireAgentBuilderSpeechToTextModel(this)
this.agentBuilder.fixtures.speechToTextModel = speechToTextModel
})
Given('the Agent Builder agent-decision chat model is available', async function (this: DifyWorld) {
const agentDecisionModel = await requireAgentBuilderAgentDecisionChatModel(this)
this.agentBuilder.fixtures.agentDecisionModel = agentDecisionModel
})
Given('the Agent v2 runtime backend is available', async function (this: DifyWorld) {
await requireAgentBackendRuntime(this)
})
Given(
'the Agent Builder preseeded Agent {string} is available',
async function (this: DifyWorld, resourceName: string) {
const resource = await requirePreseededAgent(this, resourceName)
this.agentBuilder.fixtures.preseededResources[resourceName] = resource
},
)
Given(
'the Agent Builder preseeded workflow {string} is available',
async function (this: DifyWorld, resourceName: string) {
const resource = await requirePreseededWorkflow(this, resourceName)
this.agentBuilder.fixtures.preseededResources[resourceName] = resource
},
)
Given(
'the Agent Builder preseeded dataset {string} is indexed and ready',
async function (this: DifyWorld, resourceName: string) {
const resource = await requireReadyPreseededDataset(this, resourceName)
this.agentBuilder.fixtures.preseededResources[resourceName] = resource
},
)
Given(
'the Agent Builder preseeded tool {string} is available',
async function (this: DifyWorld, resourceName: string) {
const resource = await requirePreseededTool(this, resourceName)
this.agentBuilder.fixtures.preseededResources[resourceName] = resource
},
)
Given(
'the Agent Builder preseeded Agent {string} includes the core fixture configuration',
async function (this: DifyWorld, agentName: string) {
const resource = await requirePreseededFullConfigAgentCoreConfiguration(this, agentName)
this.agentBuilder.fixtures.preseededResources[`${agentName} / core fixture configuration`] =
resource
},
)
Given(
'the Agent Builder preseeded Agent {string} includes the tool state fixture configuration',
async function (this: DifyWorld, agentName: string) {
const resource = await requirePreseededToolStatesAgentConfiguration(this, agentName)
this.agentBuilder.fixtures.preseededResources[
`${agentName} / tool state fixture configuration`
] = resource
},
)
Given(
'the Agent Builder preseeded Agent {string} includes the dual retrieval fixture configuration',
async function (this: DifyWorld, agentName: string) {
const resource = await requirePreseededDualRetrievalAgentConfiguration(this, agentName)
this.agentBuilder.fixtures.preseededResources[
`${agentName} / dual retrieval fixture configuration`
] = resource
},
)
Given(
'the Agent Builder preseeded Agent {string} is referenced by workflow {string}',
async function (this: DifyWorld, agentName: string, workflowName: string) {
const resource = await requirePreseededAgentWorkflowReference(this, agentName, workflowName)
this.agentBuilder.fixtures.preseededResources[`${agentName} / ${workflowName}`] = resource
},
)

6
e2e/features/step-definitions/agent-v2/knowledge.steps.ts
View File

@ -11,17 +11,17 @@ import {
createAgentSoulConfigWithKnowledgeDataset,
normalAgentSoulConfig,
} from '../../agent-v2/support/agent-soul'
import { asArray, asRecord } from '../../agent-v2/support/preflight/common'
import { asArray, asRecord } from '../../agent-v2/support/fixtures/common'
import { getCurrentAgentId } from './configure-helpers'
const getPreseededKnowledgeBase = (world: DifyWorld) => {
const resource =
world.agentBuilder.preflight.preseededResources[
world.agentBuilder.fixtures.preseededResources[
agentBuilderPreseededResources.agentKnowledgeBase
]
if (!resource || resource.kind !== 'dataset') {
throw new Error(
`Preseeded dataset "${agentBuilderPreseededResources.agentKnowledgeBase}" is not available. Run the matching preflight step first.`,
`Preseeded dataset "${agentBuilderPreseededResources.agentKnowledgeBase}" is not available. Run the matching fixture setup step first.`,
)
}

50
e2e/features/step-definitions/agent-v2/output-variables.steps.ts
View File

@ -6,8 +6,8 @@ import { expect } from '@playwright/test'
import { getWorkflowDraft } from '../../../support/api'
const agentV2WorkflowNodeId = 'agent-v2'
const taskFileOutputName = 'e2e_report.pdf'
const renamedTaskFileOutputName = 'e2e_final_report.pdf'
const taskOutputName = 'e2e_report'
const renamedTaskOutputName = 'e2e_final_report'
const getAgentOutputToken = (name: string) => `[§output:${name}:${name}§]`
@ -77,14 +77,14 @@ const fillOutputVariableEditor = async (
await expect(editor).toBeVisible()
await editor.getByRole('textbox', { name: 'Field name' }).fill(name)
if (type !== 'string') {
await editor.getByRole('button', { name: 'Output type' }).click()
await editor.getByLabel('Output type').click()
await page.getByRole('option', { name: type, exact: true }).click()
}
if (required) await editor.getByRole('switch', { name: 'Required' }).click()
}
When(
'I insert a file output reference from the Agent v2 workflow node task editor',
'I insert an output reference from the Agent v2 workflow node task editor',
async function (this: DifyWorld) {
const page = this.getPage()
const appId = getCurrentAppId(this)
@ -92,27 +92,28 @@ When(
await expect(taskEditor).toBeVisible()
await taskEditor.click()
await page.getByRole('button', { name: 'Insert' }).click()
await page.keyboard.type('/')
const insertResponse = waitForWorkflowDraftSave(this, appId)
await page.getByRole('button', { name: 'New output' }).click()
expect((await insertResponse).ok()).toBe(true)
const nameInput = page.getByRole('textbox', { name: 'Field name' })
await expect(nameInput).toBeVisible()
await nameInput.fill(taskFileOutputName)
const saveResponse = waitForWorkflowDraftSave(this, appId)
await nameInput.fill(taskOutputName)
const renameResponse = waitForWorkflowDraftSave(this, appId)
await nameInput.press('Enter')
expect((await saveResponse).ok()).toBe(true)
expect((await renameResponse).ok()).toBe(true)
},
)
When('I rename the Agent v2 workflow node task output reference', async function (this: DifyWorld) {
const page = this.getPage()
const appId = getCurrentAppId(this)
const taskEditor = page.getByRole('textbox', { name: 'Agent task' })
await page.getByText(taskFileOutputName, { exact: true }).hover()
await taskEditor.getByRole('button', { name: `Edit ${taskOutputName}`, exact: true }).click()
const editor = page.getByRole('form', { name: 'Output variable editor' })
await expect(editor).toBeVisible()
await editor.getByRole('textbox', { name: 'Field name' }).fill(renamedTaskFileOutputName)
await editor.getByRole('textbox', { name: 'Field name' }).fill(renamedTaskOutputName)
const saveResponse = waitForWorkflowDraftSave(this, appId)
await editor.getByRole('button', { name: 'Confirm' }).click()
@ -221,7 +222,6 @@ Then('I should see the Agent v2 workflow node output variables', async function
for (const output of expectedOutputVariables) {
await expect(page.getByText(output.name, { exact: true })).toBeVisible()
await expect(page.getByText(output.type, { exact: true })).toBeVisible()
}
})
@ -272,16 +272,16 @@ Then(
)
Then(
'the Agent v2 workflow node task should reference the file output',
'the Agent v2 workflow node task should reference the output',
async function (this: DifyWorld) {
await expectAgentTaskOutputReference(this, taskFileOutputName)
await expectAgentTaskOutputReference(this, taskOutputName)
},
)
Then(
'the Agent v2 workflow node task should reference the renamed file output',
'the Agent v2 workflow node task should reference the renamed output',
async function (this: DifyWorld) {
await expectAgentTaskOutputReference(this, renamedTaskFileOutputName, taskFileOutputName)
await expectAgentTaskOutputReference(this, renamedTaskOutputName, taskOutputName)
},
)
@ -292,11 +292,8 @@ Then(
await openWorkflowOutputVariablesPanel(this)
await expect(page.getByText('response', { exact: true })).toBeVisible()
await expect(page.getByText('object', { exact: true })).toBeVisible()
await expect(page.getByText('Required', { exact: true })).toBeVisible()
await expect(page.getByText('text', { exact: true })).toBeVisible()
await expect(page.getByText('analysis', { exact: true })).toBeVisible()
await expect(page.getByText('string', { exact: true })).toBeVisible()
},
)
@ -307,6 +304,7 @@ async function expectAgentTaskOutputReference(
) {
const page = world.getPage()
const appId = getCurrentAppId(world)
const taskEditor = page.getByRole('textbox', { name: 'Agent task' })
await expect
.poll(
@ -336,12 +334,16 @@ async function expectAgentTaskOutputReference(
agentTask: expect.stringContaining(getAgentOutputToken(expectedName)),
expectedOutput: {
name: expectedName,
type: 'file',
type: 'string',
},
unexpectedOutput: false,
})
await expect(page.getByText(expectedName, { exact: true })).toBeVisible()
await expect(page.getByText('file', { exact: true })).toBeVisible()
if (unexpectedName) await expect(page.getByText(unexpectedName, { exact: true })).toHaveCount(0)
await expect(
taskEditor.getByRole('button', { name: `Edit ${expectedName}`, exact: true }),
).toBeVisible()
if (unexpectedName)
await expect(
taskEditor.getByRole('button', { name: `Edit ${unexpectedName}`, exact: true }),
).toHaveCount(0)
}

198
e2e/features/step-definitions/agent-v2/preflight.steps.ts
View File

@ -1,198 +0,0 @@
import type { DifyWorld } from '../../support/world'
import { Given } from '@cucumber/cucumber'
import {
skipMissingPreseededAgentBackendApiKey,
skipMissingPreseededAgentPublishedWebApp,
skipMissingPreseededAgentWorkflowReference,
} from '../../agent-v2/support/preflight/access'
import { skipMissingAgentBackendRuntime } from '../../agent-v2/support/preflight/agent-backend'
import {
skipMissingPreseededAgent,
skipMissingPreseededAgentDriveSkill,
skipMissingPreseededDualRetrievalAgentConfiguration,
skipMissingPreseededFullConfigAgentCoreConfiguration,
skipMissingPreseededOAuthToolAgentConfiguration,
skipMissingPreseededToolStatesAgentConfiguration,
skipMissingPreseededWorkflow,
} from '../../agent-v2/support/preflight/agents'
import {
skipMissingIndexingPreseededDataset,
skipMissingReadyPreseededDataset,
} from '../../agent-v2/support/preflight/datasets'
import {
skipMissingAgentBuilderAgentDecisionChatModel,
skipMissingAgentBuilderBrokenChatModel,
skipMissingAgentBuilderSpeechToTextModel,
skipMissingAgentBuilderStableChatModel,
} from '../../agent-v2/support/preflight/models'
import { skipMissingPreseededTool } from '../../agent-v2/support/preflight/tools'
Given('the Agent Builder stable chat model is available', async function (this: DifyWorld) {
const stableModel = await skipMissingAgentBuilderStableChatModel(this)
if (stableModel === 'skipped') return stableModel
this.agentBuilder.preflight.stableModel = stableModel
})
Given('the workspace default speech-to-text model is active', async function (this: DifyWorld) {
const speechToTextModel = await skipMissingAgentBuilderSpeechToTextModel(this)
if (speechToTextModel === 'skipped') return speechToTextModel
this.agentBuilder.preflight.speechToTextModel = speechToTextModel
})
Given('the Agent Builder agent-decision chat model is available', async function (this: DifyWorld) {
const agentDecisionModel = await skipMissingAgentBuilderAgentDecisionChatModel(this)
if (agentDecisionModel === 'skipped') return agentDecisionModel
this.agentBuilder.preflight.agentDecisionModel = agentDecisionModel
})
Given('the Agent Builder broken chat model is available', async function (this: DifyWorld) {
const brokenModel = await skipMissingAgentBuilderBrokenChatModel(this)
if (brokenModel === 'skipped') return brokenModel
this.agentBuilder.preflight.brokenModel = brokenModel
})
Given('the Agent v2 runtime backend is available', async function (this: DifyWorld) {
const runtimeBackend = await skipMissingAgentBackendRuntime(this)
if (runtimeBackend === 'skipped') return runtimeBackend
})
Given(
'the Agent Builder preseeded Agent {string} is available',
async function (this: DifyWorld, resourceName: string) {
const resource = await skipMissingPreseededAgent(this, resourceName)
if (resource === 'skipped') return resource
this.agentBuilder.preflight.preseededResources[resourceName] = resource
},
)
Given(
'the Agent Builder preseeded workflow {string} is available',
async function (this: DifyWorld, resourceName: string) {
const resource = await skipMissingPreseededWorkflow(this, resourceName)
if (resource === 'skipped') return resource
this.agentBuilder.preflight.preseededResources[resourceName] = resource
},
)
Given(
'the Agent Builder preseeded dataset {string} is indexed and ready',
async function (this: DifyWorld, resourceName: string) {
const resource = await skipMissingReadyPreseededDataset(this, resourceName)
if (resource === 'skipped') return resource
this.agentBuilder.preflight.preseededResources[resourceName] = resource
},
)
Given(
'the Agent Builder preseeded dataset {string} is indexing',
async function (this: DifyWorld, resourceName: string) {
const resource = await skipMissingIndexingPreseededDataset(this, resourceName)
if (resource === 'skipped') return resource
this.agentBuilder.preflight.preseededResources[resourceName] = resource
},
)
Given(
'the Agent Builder preseeded tool {string} is available',
async function (this: DifyWorld, resourceName: string) {
const resource = await skipMissingPreseededTool(this, resourceName)
if (resource === 'skipped') return resource
this.agentBuilder.preflight.preseededResources[resourceName] = resource
},
)
Given(
'the Agent Builder preseeded Agent {string} includes drive skill {string}',
async function (this: DifyWorld, agentName: string, skillName: string) {
const resource = await skipMissingPreseededAgentDriveSkill(this, agentName, skillName)
if (resource === 'skipped') return resource
this.agentBuilder.preflight.preseededResources[`${agentName} / ${skillName}`] = resource
},
)
Given(
'the Agent Builder preseeded Agent {string} includes the core fixture configuration',
async function (this: DifyWorld, agentName: string) {
const resource = await skipMissingPreseededFullConfigAgentCoreConfiguration(this, agentName)
if (resource === 'skipped') return resource
this.agentBuilder.preflight.preseededResources[`${agentName} / core fixture configuration`] =
resource
},
)
Given(
'the Agent Builder preseeded Agent {string} includes the tool state fixture configuration',
async function (this: DifyWorld, agentName: string) {
const resource = await skipMissingPreseededToolStatesAgentConfiguration(this, agentName)
if (resource === 'skipped') return resource
this.agentBuilder.preflight.preseededResources[
`${agentName} / tool state fixture configuration`
] = resource
},
)
Given(
'the Agent Builder preseeded Agent {string} includes an OAuth2 tool credential',
async function (this: DifyWorld, agentName: string) {
const resource = await skipMissingPreseededOAuthToolAgentConfiguration(this, agentName)
if (resource === 'skipped') return resource
this.agentBuilder.preflight.preseededResources[`${agentName} / OAuth2 tool credential`] =
resource
},
)
Given(
'the Agent Builder preseeded Agent {string} includes the dual retrieval fixture configuration',
async function (this: DifyWorld, agentName: string) {
const resource = await skipMissingPreseededDualRetrievalAgentConfiguration(this, agentName)
if (resource === 'skipped') return resource
this.agentBuilder.preflight.preseededResources[
`${agentName} / dual retrieval fixture configuration`
] = resource
},
)
Given(
'the Agent Builder preseeded Agent {string} has Backend service API access with an API key',
async function (this: DifyWorld, agentName: string) {
const resource = await skipMissingPreseededAgentBackendApiKey(this, agentName)
if (resource === 'skipped') return resource
this.agentBuilder.preflight.preseededResources[`${agentName} / Backend service API key`] =
resource
},
)
Given(
'the Agent Builder preseeded Agent {string} has published Web app access',
async function (this: DifyWorld, agentName: string) {
const resource = await skipMissingPreseededAgentPublishedWebApp(this, agentName)
if (resource === 'skipped') return resource
this.agentBuilder.preflight.preseededResources[`${agentName} / Web app`] = resource
},
)
Given(
'the Agent Builder preseeded Agent {string} is referenced by workflow {string}',
async function (this: DifyWorld, agentName: string, workflowName: string) {
const resource = await skipMissingPreseededAgentWorkflowReference(this, agentName, workflowName)
if (resource === 'skipped') return resource
this.agentBuilder.preflight.preseededResources[`${agentName} / ${workflowName}`] = resource
},
)

29
e2e/features/step-definitions/agent-v2/publish.steps.ts
View File

@ -2,8 +2,7 @@ import type { DifyWorld } from '../../support/world'
import { Then, When } from '@cucumber/cucumber'
import { expect } from '@playwright/test'
import { waitForAgentConfigureAutosaved } from '../../../support/agent-configure'
import { getAgentVersionDetail, getTestAgent } from '../../agent-v2/support/agent'
import { normalAgentPrompt } from '../../agent-v2/support/agent-soul'
import { getTestAgent } from '../../agent-v2/support/agent'
import { expectAgentModelRequiredFeedback, getCurrentAgentId } from './configure-helpers'
When('I publish the Agent v2 draft', async function (this: DifyWorld) {
@ -55,15 +54,11 @@ Then(
'the Agent v2 publish action should be available for unpublished changes',
async function (this: DifyWorld) {
const page = this.getPage()
const agentId = getCurrentAgentId(this)
await expect(
page.getByRole('status', { name: /^(?:Draft|Unpublished changes)\./ }),
).toBeVisible({ timeout: 30_000 })
await expect(page.getByRole('button', { name: /^Publish(?: update)?$/ })).toBeEnabled()
await expect
.poll(async () => (await getTestAgent(agentId)).active_config_is_published)
.toBe(false)
},
)
@ -123,25 +118,3 @@ When('I restore the selected Agent v2 version', async function (this: DifyWorld)
const response = await restoreResponse
expect(response.ok()).toBe(true)
})
Then(
'the active published Agent v2 version should still use the normal E2E prompt',
async function (this: DifyWorld) {
const agentId = getCurrentAgentId(this)
await expect
.poll(async () => (await getTestAgent(agentId)).active_config_is_published, {
timeout: 30_000,
})
.toBe(false)
const agent = await getTestAgent(agentId)
const activeSnapshotId = agent?.active_config_snapshot_id
if (!activeSnapshotId)
throw new Error(`Agent v2 ${agentId} does not have an active published snapshot.`)
const version = await getAgentVersionDetail(agentId, activeSnapshotId)
expect(version.config_snapshot.prompt).toEqual({ system_prompt: normalAgentPrompt })
},
)

4
e2e/features/step-definitions/agent-v2/speech-to-text.steps.ts
View File

@ -21,9 +21,9 @@ const normalizeFixturePhrase = (value: string) =>
Given(
'an Agent v2 test agent with speech-to-text enabled has been created via API',
async function (this: DifyWorld) {
if (!this.agentBuilder.preflight.speechToTextModel) {
if (!this.agentBuilder.fixtures.speechToTextModel) {
throw new Error(
'Create a speech-to-text Agent v2 test agent after the default Speech-to-Text model preflight.',
'Create a speech-to-text Agent v2 test agent after the default Speech-to-Text model fixture setup.',
)
}

123
e2e/features/step-definitions/agent-v2/tools.steps.ts
View File

@ -14,9 +14,8 @@ import {
createAgentSoulConfigWithModel,
normalAgentSoulConfig,
} from '../../agent-v2/support/agent-soul'
import { getPreseededOAuthToolConfig } from '../../agent-v2/support/preflight/agents'
import { asArray, asRecord, asString } from '../../agent-v2/support/preflight/common'
import { hasToolEntry } from '../../agent-v2/support/preflight/tools'
import { asArray, asRecord, asString } from '../../agent-v2/support/fixtures/common'
import { hasToolEntry } from '../../agent-v2/support/fixtures/tools'
import { SERVICE_API_RUNTIME_STEP_TIMEOUT_MS } from '../../agent-v2/support/service-api-sse'
import { getPreseededToolContract } from '../../agent-v2/support/tools'
import { expectProviderToolActionVisible, getCurrentAgentId } from './configure-helpers'
@ -51,45 +50,6 @@ const expectJsonReplaceToolDraft = async (world: DifyWorld) => {
.toBe(true)
}
const getOAuth2ToolEntries = async (agentId: string) => {
const draft = await getAgentComposerDraft(agentId)
return asArray(asRecord(draft.agent_soul?.tools).dify_tools).filter((item) => {
const record = asRecord(item)
return (
record.credential_type === 'oauth2' && Boolean(asString(asRecord(record.credential_ref).id))
)
})
}
const getOAuth2ToolDisplayName = async (world: DifyWorld) => {
const [tool] = await getOAuth2ToolEntries(getCurrentAgentId(world))
const record = asRecord(tool)
const providerName =
asString(record.provider) || asString(record.provider_id) || asString(record.plugin_id)
const toolName = asString(record.name) || asString(record.tool_name)
if (!providerName || !toolName)
throw new Error('Agent v2 OAuth2 tool fixture must include provider and tool names.')
return `${providerName} / ${toolName}`
}
const getPreseededOAuthToolAgent = (world: DifyWorld) => {
const resource =
world.agentBuilder.preflight.preseededResources[
`${agentBuilderPreseededResources.oauthToolAgent} / OAuth2 tool credential`
]
if (!resource || resource.kind !== 'agent') {
throw new Error(
`Preseeded Agent "${agentBuilderPreseededResources.oauthToolAgent}" OAuth2 tool credential fixture is not available. Run the matching preflight step first.`,
)
}
return resource
}
const jsonReplaceToolConfig = (world: DifyWorld): AgentSoulDifyToolConfig => {
const tool = getPreseededToolContract(world, agentBuilderPreseededResources.jsonReplaceTool)
@ -150,63 +110,11 @@ const findJsonReplaceRuntimeThought = (body: unknown) =>
)
})
const expectOAuth2CredentialPreserved = async (world: DifyWorld) => {
const preseededAgent = getPreseededOAuthToolAgent(world)
const expectedTool = await getPreseededOAuthToolConfig(preseededAgent.id)
const expected = asRecord(expectedTool)
const expectedCredentialRef = asRecord(expected.credential_ref)
const expectedCredentialId = asString(expectedCredentialRef.id)
const expectedProvider =
asString(expected.provider_id) || asString(expected.provider) || asString(expected.plugin_id)
const expectedToolName = asString(expected.tool_name) || asString(expected.name)
await expect
.poll(
async () => {
const tools = await getOAuth2ToolEntries(getCurrentAgentId(world))
const matchingTool = tools.find((item) => {
const record = asRecord(item)
const provider =
asString(record.provider_id) || asString(record.provider) || asString(record.plugin_id)
const toolName = asString(record.tool_name) || asString(record.name)
return provider === expectedProvider && toolName === expectedToolName
})
const record = asRecord(matchingTool)
return {
credentialId: asString(asRecord(record.credential_ref).id),
credentialType: asString(record.credential_type),
}
},
{ timeout: 30_000 },
)
.toEqual({
credentialId: expectedCredentialId,
credentialType: 'oauth2',
})
}
Given(
'an Agent v2 test agent with the OAuth2 tool credential fixture has been created via API',
async function (this: DifyWorld) {
const preseededAgent = getPreseededOAuthToolAgent(this)
const oauthTool = await getPreseededOAuthToolConfig(preseededAgent.id)
const agent = await createConfiguredTestAgent({
agentSoul: createAgentSoulConfigWithDifyTool(normalAgentSoulConfig, oauthTool),
})
this.createdAgentIds.push(agent.id)
this.lastCreatedAgentName = agent.name
this.lastCreatedAgentRole = agent.role ?? undefined
},
)
Given(
'a runnable Agent v2 test agent with the JSON Replace tool has been created via API',
async function (this: DifyWorld) {
if (!this.agentBuilder.preflight.stableModel)
throw new Error('Create a JSON Replace runtime Agent after stable model preflight.')
if (!this.agentBuilder.fixtures.stableModel)
throw new Error('Create a JSON Replace runtime Agent after stable model fixture setup.')
const agent = await createConfiguredTestAgent({
agentSoul: createAgentSoulConfigWithDifyTool(
@ -217,7 +125,7 @@ Given(
system_prompt: jsonReplaceRuntimePrompt,
},
},
this.agentBuilder.preflight.stableModel,
this.agentBuilder.fixtures.stableModel,
),
jsonReplaceToolConfig(this),
),
@ -237,7 +145,6 @@ When(
await expect(toolsSection).toBeVisible({ timeout: 30_000 })
await toolsSection.getByRole('button', { name: 'Add tool' }).click()
await page.getByRole('button', { name: /^Tool\b/ }).click()
const search = getToolSelectorSearch(this)
await expect(search).toBeVisible()
@ -342,26 +249,6 @@ Then(
},
)
Then(
'I should see the Agent v2 OAuth2 tool authorized in the Tools section',
async function (this: DifyWorld) {
const toolsSection = getToolsSection(this)
const displayName = await getOAuth2ToolDisplayName(this)
await expectProviderToolActionVisible(toolsSection, displayName)
await expect(
toolsSection.getByRole('button', { exact: true, name: 'Not authorized' }),
).not.toBeVisible()
},
)
Then(
'the Agent v2 OAuth2 tool credential should remain saved in the Agent v2 draft',
async function (this: DifyWorld) {
await expectOAuth2CredentialPreserved(this)
},
)
Then('I should see the Agent v2 tool selector empty state', async function (this: DifyWorld) {
const page = this.getPage()

23
e2e/features/step-definitions/agent-v2/workflow-node.steps.ts
View File

@ -3,7 +3,7 @@ import { Given, Then, When } from '@cucumber/cucumber'
import { expect } from '@playwright/test'
import { createTestApp, syncAgentV2WorkflowDraft } from '../../../support/api'
import { createE2EResourceName } from '../../../support/naming'
import { createConfiguredTestAgent } from '../../agent-v2/support/agent'
import { createConfiguredTestAgent, publishAgent } from '../../agent-v2/support/agent'
import {
createAgentSoulConfigWithModel,
normalAgentPrompt,
@ -13,18 +13,19 @@ import {
Given(
'a workflow app with an Agent v2 node has been created via API',
async function (this: DifyWorld) {
if (!this.agentBuilder.preflight.stableModel)
throw new Error('Create an Agent v2 workflow node after stable model preflight.')
if (!this.agentBuilder.fixtures.stableModel)
throw new Error('Create an Agent v2 workflow node after stable model fixture setup.')
const agent = await createConfiguredTestAgent({
agentSoul: createAgentSoulConfigWithModel(
normalAgentSoulConfig,
this.agentBuilder.preflight.stableModel,
this.agentBuilder.fixtures.stableModel,
),
})
this.createdAgentIds.push(agent.id)
this.lastCreatedAgentName = agent.name
this.lastCreatedAgentRole = agent.role ?? undefined
await publishAgent(agent.id)
const app = await createTestApp(createE2EResourceName('App', 'workflow-agent-v2'), 'workflow')
this.createdAppIds.push(app.id)
@ -49,7 +50,7 @@ When('I open the Agent v2 workflow Agent details', async function (this: DifyWor
if (!agentName) throw new Error('No Agent v2 name found. Create a workflow Agent v2 node first.')
await page.getByRole('button', { name: `Open ${agentName} details` }).click()
await expect(page.getByRole('dialog', { name: `${agentName} details` })).toBeVisible()
await expect(page.getByRole('dialog', { exact: true, name: agentName })).toBeVisible()
})
When('I open the Agent v2 workflow Agent in Agent Console', async function (this: DifyWorld) {
@ -57,7 +58,7 @@ When('I open the Agent v2 workflow Agent in Agent Console', async function (this
const agentName = this.lastCreatedAgentName
if (!agentName) throw new Error('No Agent v2 name found. Create a workflow Agent v2 node first.')
const detailsDialog = page.getByRole('dialog', { name: `${agentName} details` })
const detailsDialog = page.getByRole('dialog', { exact: true, name: agentName })
const [agentConsolePage] = await Promise.all([
page.waitForEvent('popup'),
detailsDialog.getByRole('link', { name: 'Edit in Agent Console' }).click(),
@ -72,15 +73,15 @@ Then(
const page = this.getPage()
const agentName = this.lastCreatedAgentName
const agentRole = this.lastCreatedAgentRole
const stableModel = this.agentBuilder.preflight.stableModel
const stableModel = this.agentBuilder.fixtures.stableModel
if (!agentName)
throw new Error('No Agent v2 name found. Create a workflow Agent v2 node first.')
if (!stableModel)
throw new Error(
'Stable chat model preflight must run before asserting workflow Agent details.',
'Stable chat model fixture setup must run before asserting workflow Agent details.',
)
const detailsDialog = page.getByRole('dialog', { name: `${agentName} details` })
const detailsDialog = page.getByRole('dialog', { exact: true, name: agentName })
await expect(detailsDialog).toBeVisible()
await expect(detailsDialog.getByText(agentName, { exact: true })).toBeVisible()
@ -99,12 +100,12 @@ Then(
const agentConsolePage = this.agentBuilder.workflow.agentConsolePage
const agentId = this.createdAgentIds.at(-1)
const agentName = this.lastCreatedAgentName
const stableModel = this.agentBuilder.preflight.stableModel
const stableModel = this.agentBuilder.fixtures.stableModel
if (!agentConsolePage) throw new Error('Agent Console page was not opened.')
if (!agentId || !agentName)
throw new Error('No Agent v2 ID or name found. Create a workflow Agent v2 node first.')
if (!stableModel)
throw new Error('Stable chat model preflight must run before asserting Agent Console.')
throw new Error('Stable chat model fixture setup must run before asserting Agent Console.')
await expect(agentConsolePage).toHaveURL(new RegExp(`/agents/${agentId}/configure(?:\\?.*)?$`))
await expect(agentConsolePage.getByRole('heading', { name: 'Configure' })).toBeVisible({

16
e2e/features/step-definitions/apps/app-detail-navigation.steps.ts
View File

@ -1,21 +1,7 @@
import type { DifyWorld } from '../../support/world'
import { Then, When } from '@cucumber/cucumber'
import { expect } from '@playwright/test'
When('I navigate to the app develop page', async function (this: DifyWorld) {
const appId = this.createdAppIds.at(-1)
await this.getPage().goto(`/app/${appId}/develop`)
})
import { When } from '@cucumber/cucumber'
When('I navigate to the app overview page', async function (this: DifyWorld) {
const appId = this.createdAppIds.at(-1)
await this.getPage().goto(`/app/${appId}/overview`)
})
Then('I should be on the app develop page', async function (this: DifyWorld) {
await expect(this.getPage()).toHaveURL(/\/app\/[^/]+\/develop(?:\?.*)?$/, { timeout: 30_000 })
})
Then('I should be on the app overview page', async function (this: DifyWorld) {
await expect(this.getPage()).toHaveURL(/\/app\/[^/]+\/overview(?:\?.*)?$/, { timeout: 30_000 })
})

48
e2e/features/step-definitions/apps/create-app.steps.ts
View File

@ -4,6 +4,21 @@ import { expect } from '@playwright/test'
import { openBlankAppCreation } from '../../../support/apps'
import { createE2EResourceName } from '../../../support/naming'
const appModeByType: Record<string, string> = {
Agent: 'agent-chat',
Chatbot: 'chat',
Chatflow: 'advanced-chat',
'Text Generator': 'completion',
Workflow: 'workflow',
}
const getLatestCreatedAppId = (world: DifyWorld) => {
const appId = world.createdAppIds.at(-1)
if (!appId) throw new Error('No created app ID was recorded from the UI response.')
return appId
}
When('I start creating a blank app', async function (this: DifyWorld) {
await openBlankAppCreation(this.getPage())
})
@ -15,12 +30,27 @@ When('I enter a unique E2E app name', async function (this: DifyWorld) {
})
When('I confirm app creation', async function (this: DifyWorld) {
const createButton = this.getPage()
.getByRole('dialog')
.getByRole('button', { name: /^Create(?:\s|$)/ })
const page = this.getPage()
const createButton = page.getByRole('dialog').getByRole('button', { name: /^Create(?:\s|$)/ })
const responsePromise = page.waitForResponse(
(response) =>
response.request().method() === 'POST' &&
new URL(response.url()).pathname.endsWith('/console/api/apps'),
)
await expect(createButton).toBeEnabled()
await createButton.click()
const response = await responsePromise
expect(response.ok()).toBe(true)
const createdApp = (await response.json()) as { id?: string; mode?: string }
if (!createdApp.id) throw new Error('Create app response did not include an app ID.')
const expectedMode = this.lastSelectedAppType
? appModeByType[this.lastSelectedAppType]
: undefined
if (expectedMode) expect(createdApp.mode).toBe(expectedMode)
this.createdAppIds.push(createdApp.id)
})
When('I select the {string} app type', async function (this: DifyWorld, appType: string) {
@ -31,6 +61,7 @@ When('I select the {string} app type', async function (this: DifyWorld, appType:
await expect(appTypeCard).toBeVisible()
await appTypeCard.click()
this.lastSelectedAppType = appType
})
When('I expand the beginner app types', async function (this: DifyWorld) {
@ -42,13 +73,18 @@ When('I expand the beginner app types', async function (this: DifyWorld) {
})
Then('I should land on the app editor', async function (this: DifyWorld) {
await expect(this.getPage()).toHaveURL(/\/app\/[^/]+\/(workflow|configuration)(?:\?.*)?$/)
const appId = getLatestCreatedAppId(this)
await expect(this.getPage()).toHaveURL(
new RegExp(`/app/${appId}/(workflow|configuration)(?:\\?.*)?$`),
)
})
Then('I should land on the workflow editor', async function (this: DifyWorld) {
await expect(this.getPage()).toHaveURL(/\/app\/[^/]+\/workflow(?:\?.*)?$/)
const appId = getLatestCreatedAppId(this)
await expect(this.getPage()).toHaveURL(new RegExp(`/app/${appId}/workflow(?:\\?.*)?$`))
})
Then('I should land on the app configuration page', async function (this: DifyWorld) {
await expect(this.getPage()).toHaveURL(/\/app\/[^/]+\/configuration(?:\?.*)?$/)
const appId = getLatestCreatedAppId(this)
await expect(this.getPage()).toHaveURL(new RegExp(`/app/${appId}/configuration(?:\\?.*)?$`))
})

18
e2e/features/step-definitions/apps/duplicate-app.steps.ts
View File

@ -27,5 +27,21 @@ When('I click {string} in the app options menu', async function (this: DifyWorld
})
When('I confirm the app duplication', async function (this: DifyWorld) {
await this.getPage().getByRole('button', { name: 'Duplicate' }).click()
const sourceAppId = this.createdAppIds.at(-1)
if (!sourceAppId) throw new Error('No source app ID was recorded before duplication.')
const page = this.getPage()
const responsePromise = page.waitForResponse(
(response) =>
response.request().method() === 'POST' &&
new URL(response.url()).pathname.endsWith(`/console/api/apps/${sourceAppId}/copy`),
)
await page.getByRole('button', { exact: true, name: 'Duplicate' }).click()
const response = await responsePromise
expect(response.ok()).toBe(true)
const copiedApp = (await response.json()) as { id?: string }
if (!copiedApp.id) throw new Error('Duplicate app response did not include an app ID.')
expect(copiedApp.id).not.toBe(sourceAppId)
this.createdAppIds.push(copiedApp.id)
})

12
e2e/features/step-definitions/smoke/install.steps.ts
View File

@ -1,12 +0,0 @@
import type { DifyWorld } from '../../support/world'
import { Given } from '@cucumber/cucumber'
import { expect } from '@playwright/test'
Given(
'the last authentication bootstrap came from a fresh install',
async function (this: DifyWorld) {
const session = await this.getAuthSession()
expect(session.mode).toBe('install')
},
)

30
e2e/features/support/hooks.ts
View File

@ -5,7 +5,7 @@ import type { DifyWorld } from './world'
import { mkdir, writeFile } from 'node:fs/promises'
import path from 'node:path'
import { fileURLToPath } from 'node:url'
import { After, AfterAll, Before, BeforeAll, setDefaultTimeout, Status } from '@cucumber/cucumber'
import { After, AfterAll, Before, setDefaultTimeout, Status } from '@cucumber/cucumber'
import { chromium, webkit } from '@playwright/test'
import { AUTH_BOOTSTRAP_TIMEOUT_MS, ensureAuthenticatedState } from '../../fixtures/auth'
import { deleteTestApp } from '../../support/api'
@ -88,19 +88,6 @@ const captureDiagnosticPage = async (
return [screenshotPath, htmlPath]
}
BeforeAll({ timeout: AUTH_BOOTSTRAP_TIMEOUT_MS }, async () => {
await mkdir(artifactsDir, { recursive: true })
const browserType = e2eBrowser === 'webkit' ? webkit : chromium
browser = await browserType.launch({
headless: cucumberHeadless,
slowMo: cucumberSlowMo,
})
console.warn(`[e2e] ${e2eBrowser} session cache bootstrap against ${baseURL}`)
await ensureAuthenticatedState(browser, baseURL)
})
const getMicrophoneBrowser = () => {
microphoneBrowserPromise ??= chromium.launch({
args: [
@ -115,7 +102,20 @@ const getMicrophoneBrowser = () => {
return microphoneBrowserPromise
}
Before(async function (this: DifyWorld, { pickle }) {
Before({ timeout: AUTH_BOOTSTRAP_TIMEOUT_MS }, async function (this: DifyWorld, { pickle }) {
await mkdir(artifactsDir, { recursive: true })
if (!browser) {
const browserType = e2eBrowser === 'webkit' ? webkit : chromium
browser = await browserType.launch({
headless: cucumberHeadless,
slowMo: cucumberSlowMo,
})
console.warn(`[e2e] ${e2eBrowser} session cache bootstrap against ${baseURL}`)
await ensureAuthenticatedState(browser, baseURL)
}
if (!browser) throw new Error('Shared Playwright browser is not available.')
const scenarioTags = pickle.tags.map((tag) => tag.name)

7
e2e/features/support/world.ts
View File

@ -30,7 +30,7 @@ export type AgentBuilderChatModel = {
}
export type AgentBuilderPreseededResource = {
id: string
kind: 'agent' | 'api-key' | 'dataset' | 'skill' | 'tool' | 'workflow'
kind: 'agent' | 'dataset' | 'skill' | 'tool' | 'workflow'
name: string
}
export type AgentV2WorkflowOutputVariable = {
@ -44,9 +44,8 @@ export type AgentBuilderSpeechToTextRequest = {
}
export const createAgentBuilderWorldState = () => ({
preflight: {
fixtures: {
agentDecisionModel: undefined as AgentBuilderChatModel | undefined,
brokenModel: undefined as AgentBuilderChatModel | undefined,
preseededResources: {} as Record<string, AgentBuilderPreseededResource>,
speechToTextModel: undefined as AgentBuilderChatModel | undefined,
stableModel: undefined as AgentBuilderChatModel | undefined,
@ -83,6 +82,7 @@ export class DifyWorld extends World {
scenarioStartedAt: number | undefined
session: AuthSessionMetadata | undefined
lastCreatedAppName: string | undefined
lastSelectedAppType: string | undefined
lastCreatedAgentName: string | undefined
lastCreatedAgentRole: string | undefined
createdAppIds: string[] = []
@ -106,6 +106,7 @@ export class DifyWorld extends World {
this.consoleErrors = []
this.pageErrors = []
this.lastCreatedAppName = undefined
this.lastSelectedAppType = undefined
this.lastCreatedAgentName = undefined
this.lastCreatedAgentRole = undefined
this.createdAppIds = []

10
e2e/fixtures/auth.ts
View File

@ -3,7 +3,6 @@ import { Buffer } from 'node:buffer'
import { mkdir, readFile, writeFile } from 'node:fs/promises'
import path from 'node:path'
import { fileURLToPath } from 'node:url'
import { waitForConsoleHome } from '../support/home'
import { apiURL, defaultBaseURL, defaultLocale } from '../test-env'
export type AuthSessionMetadata = {
@ -37,7 +36,6 @@ export const readAuthSessionMetadata = async () => {
return JSON.parse(content) as AuthSessionMetadata
}
const appURL = (baseURL: string, pathname: string) => new URL(pathname, baseURL).toString()
const apiEndpoint = (pathname: string) => new URL(pathname, apiURL).toString()
type SetupStatusResponse = {
@ -142,19 +140,11 @@ export const ensureAuthenticatedState = async (browser: Browser, configuredBaseU
baseURL,
locale: defaultLocale,
})
const page = await context.newPage()
try {
const { mode, usedInitPassword } = await ensureAdminAccount(context, deadline)
await loginAdmin(context, deadline)
console.warn('[e2e] auth bootstrap: verifying console home')
await page.goto(appURL(baseURL, '/'), {
timeout: getRemainingTimeout(deadline),
waitUntil: 'domcontentloaded',
})
await waitForConsoleHome(page, getRemainingTimeout(deadline))
await context.storageState({ path: authStatePath })
const metadata: AuthSessionMetadata = {

2
e2e/package.json
View File

@ -12,6 +12,8 @@
"e2e:install": "playwright install --with-deps chromium webkit",
"e2e:middleware:down": "tsx ./scripts/setup.ts middleware-down",
"e2e:middleware:up": "tsx ./scripts/setup.ts middleware-up",
"e2e:post-merge": "tsx ./scripts/run-post-merge.ts",
"e2e:post-merge:prepare": "tsx ./scripts/seed.ts --pack agent-v2 --profile post-merge",
"e2e:reset": "tsx ./scripts/setup.ts reset",
"seed": "tsx ./scripts/seed.ts",
"test:unit": "vitest run",

19
e2e/scripts/env.ts
View File

@ -82,22 +82,12 @@ export const validateE2eEnv = () =>
E2E_API_URL: process.env.E2E_API_URL,
E2E_BASE_URL: process.env.E2E_BASE_URL,
E2E_BROWSER: process.env.E2E_BROWSER,
E2E_BROKEN_MODEL_NAME: process.env.E2E_BROKEN_MODEL_NAME,
E2E_BROKEN_MODEL_PROVIDER: process.env.E2E_BROKEN_MODEL_PROVIDER,
E2E_BROKEN_MODEL_TYPE: process.env.E2E_BROKEN_MODEL_TYPE,
E2E_CUCUMBER_TAGS: process.env.E2E_CUCUMBER_TAGS,
E2E_EXTERNAL_RUNTIME_SEED_SPECS: process.env.E2E_EXTERNAL_RUNTIME_SEED_SPECS,
E2E_EXTERNAL_RUNTIME_TAGS: process.env.E2E_EXTERNAL_RUNTIME_TAGS,
E2E_FORCE_WEB_BUILD: process.env.E2E_FORCE_WEB_BUILD,
E2E_INIT_PASSWORD: process.env.E2E_INIT_PASSWORD,
E2E_MARKETPLACE_API_URL: process.env.E2E_MARKETPLACE_API_URL,
E2E_MARKETPLACE_PLUGIN_IDS: process.env.E2E_MARKETPLACE_PLUGIN_IDS,
E2E_MARKETPLACE_PLUGIN_UNIQUE_IDENTIFIERS:
process.env.E2E_MARKETPLACE_PLUGIN_UNIQUE_IDENTIFIERS,
E2E_MODEL_PROVIDER_CREDENTIALS_JSON: process.env.E2E_MODEL_PROVIDER_CREDENTIALS_JSON,
E2E_OAUTH_TOOL_CREDENTIAL_ID: process.env.E2E_OAUTH_TOOL_CREDENTIAL_ID,
E2E_OAUTH_TOOL_NAME: process.env.E2E_OAUTH_TOOL_NAME,
E2E_OAUTH_TOOL_PROVIDER: process.env.E2E_OAUTH_TOOL_PROVIDER,
E2E_REUSE_WEB_SERVER: process.env.E2E_REUSE_WEB_SERVER,
E2E_SLOW_MO: process.env.E2E_SLOW_MO,
E2E_SHELLCTL_AUTH_TOKEN: process.env.E2E_SHELLCTL_AUTH_TOKEN,
@ -125,21 +115,12 @@ export const validateE2eEnv = () =>
E2E_API_URL: z.url().optional(),
E2E_BASE_URL: z.url().optional(),
E2E_BROWSER: z.enum(['chromium', 'webkit']).optional(),
E2E_BROKEN_MODEL_NAME: z.string().min(1).optional(),
E2E_BROKEN_MODEL_PROVIDER: z.string().min(1).optional(),
E2E_BROKEN_MODEL_TYPE: z.string().min(1).optional(),
E2E_CUCUMBER_TAGS: z.string().min(1).optional(),
E2E_EXTERNAL_RUNTIME_SEED_SPECS: z.string().min(1).optional(),
E2E_EXTERNAL_RUNTIME_TAGS: z.string().min(1).optional(),
E2E_FORCE_WEB_BUILD: z.literal('1').optional(),
E2E_INIT_PASSWORD: z.string().min(1).optional(),
E2E_MARKETPLACE_API_URL: z.url().optional(),
E2E_MARKETPLACE_PLUGIN_IDS: z.string().min(1).optional(),
E2E_MARKETPLACE_PLUGIN_UNIQUE_IDENTIFIERS: z.string().min(1).optional(),
E2E_MODEL_PROVIDER_CREDENTIALS_JSON: jsonObjectString.optional(),
E2E_OAUTH_TOOL_CREDENTIAL_ID: z.string().min(1).optional(),
E2E_OAUTH_TOOL_NAME: z.string().min(1).optional(),
E2E_OAUTH_TOOL_PROVIDER: z.string().min(1).optional(),
E2E_REUSE_WEB_SERVER: booleanString.optional(),
E2E_SLOW_MO: z.coerce.number().nonnegative().optional(),
E2E_SHELLCTL_AUTH_TOKEN: z.string().min(1).optional(),

33
e2e/scripts/prepare-external-runtime.ts
View File

@ -1,35 +1,12 @@
import { e2eDir, isMainModule, runCommandOrThrow } from './common'
import './env-register'
const defaultExternalRuntimeSeedSpecs = 'agent-v2:external-runtime'
const parseSeedSpecs = (value: string) =>
value
.split(',')
.map((entry) => entry.trim())
.filter(Boolean)
.map((entry) => {
const [pack, profile = 'full'] = entry.split(':').map((part) => part.trim())
if (!pack)
throw new Error(
`Invalid external runtime seed spec "${entry}". Expected "pack" or "pack:profile".`,
)
return { pack, profile }
})
const main = async () => {
const seedSpecs = parseSeedSpecs(
process.env.E2E_EXTERNAL_RUNTIME_SEED_SPECS?.trim() || defaultExternalRuntimeSeedSpecs,
)
for (const { pack, profile } of seedSpecs) {
await runCommandOrThrow({
command: 'npx',
args: ['tsx', './scripts/seed.ts', '--pack', pack, '--profile', profile],
cwd: e2eDir,
})
}
await runCommandOrThrow({
command: 'npx',
args: ['tsx', './scripts/seed.ts', '--pack', 'agent-v2', '--profile', 'external-runtime'],
cwd: e2eDir,
})
}
if (isMainModule(import.meta.url)) {

21
e2e/scripts/run-cucumber.ts
View File

@ -2,12 +2,7 @@ import type { ManagedProcess } from '../support/process'
import { mkdir, readFile, rm } from 'node:fs/promises'
import path from 'node:path'
import { runCleanupTasks } from '../support/cleanup'
import {
assertCucumberReport,
formatCucumberReportSummary,
getCucumberReportGate,
readCucumberReportSummary,
} from '../support/cucumber-report'
import { assertCucumberScenariosStarted } from '../support/cucumber-messages'
import { startLoggedProcess, stopManagedProcess, waitForUrl } from '../support/process'
import { startWebServer, stopWebServer } from '../support/web-server'
import { apiURL, baseURL, reuseExistingWebServer } from '../test-env'
@ -51,8 +46,7 @@ const parseArgs = (argv: string[]): RunOptions => {
const hasCustomTags = (forwardArgs: string[]) =>
forwardArgs.some((arg) => arg === '--tags' || arg.startsWith('--tags='))
const fullNonExternalTags =
'not @skip and not @preview and not @external-model and not @external-tool'
const fullNonExternalTags = 'not @prepared and not @external-model and not @external-tool'
const isTruthyEnv = (value: string | undefined) => value === '1' || value === 'true'
@ -274,14 +268,9 @@ const main = async () => {
env: cucumberEnv,
})
const reportGate = getCucumberReportGate(cucumberEnv)
if (reportGate) {
const reportPath = path.join(cucumberReportDir, 'report.json')
const reportSummary = await readCucumberReportSummary(reportPath)
console.warn(
`[e2e] cucumber report ${reportGate.profile}: ${formatCucumberReportSummary(reportSummary)}`,
)
assertCucumberReport(reportSummary, reportGate)
if (result.exitCode === 0) {
const messages = await readFile(path.join(cucumberReportDir, 'report.ndjson'), 'utf8')
assertCucumberScenariosStarted(messages)
}
process.exitCode = result.exitCode

7
e2e/scripts/run-external-runtime.ts
View File

@ -1,15 +1,12 @@
import { e2eDir, isMainModule, runForegroundProcess } from './common'
import './env-register'
const defaultExternalRuntimeTags =
'(@external-model or @external-tool) and not @feature-gated and not @skip and not @preview'
const defaultExternalRuntimeTags = '@external-model or @external-tool'
const main = async () => {
const tags = process.env.E2E_EXTERNAL_RUNTIME_TAGS?.trim() || defaultExternalRuntimeTags
await runForegroundProcess({
command: 'npx',
args: ['tsx', './scripts/run-cucumber.ts', '--', '--tags', tags],
args: ['tsx', './scripts/run-cucumber.ts', '--', '--tags', defaultExternalRuntimeTags],
cwd: e2eDir,
})
}

14
e2e/scripts/run-post-merge.ts Normal file
View File

@ -0,0 +1,14 @@
import { e2eDir, isMainModule, runForegroundProcess } from './common'
import './env-register'
const postMergeTags = '@prepared or @external-model or @external-tool'
const main = async () => {
await runForegroundProcess({
command: 'npx',
args: ['tsx', './scripts/run-cucumber.ts', '--', '--tags', postMergeTags],
cwd: e2eDir,
})
}
if (isMainModule(import.meta.url)) void main()

5
e2e/scripts/seed.ts
View File

@ -23,7 +23,7 @@ const parseArgs = (argv: string[]): SeedOptions => {
allowBlocked: false,
dryRun: false,
pack: 'agent-v2',
profile: 'full',
profile: 'post-merge',
}
for (const [index, arg] of argv.entries()) {
@ -133,8 +133,7 @@ const main = async () => {
dryRun: options.dryRun,
resources: new Map(),
})
const reportName =
options.profile === 'full' ? options.pack : `${options.pack}-${options.profile}`
const reportName = `${options.pack}-${options.profile}`
const reportPath = await writeSeedReport(reportName, results)
const blockedCount = results.filter((result) => result.status === 'blocked').length

4
e2e/scripts/setup.ts
View File

@ -78,11 +78,15 @@ const composeArgs = [
const getApiEnvironment = async (): Promise<Record<string, string>> => {
const envFromExample = await readSimpleDotenv(apiEnvExampleFile)
const agentBackendBaseUrl = getAgentBackendBaseUrl()
const marketplaceApiUrl = process.env.E2E_MARKETPLACE_API_URL?.trim()
return {
...envFromExample,
...(agentBackendBaseUrl ? { AGENT_BACKEND_BASE_URL: agentBackendBaseUrl } : {}),
...(marketplaceApiUrl ? { MARKETPLACE_API_URL: marketplaceApiUrl } : {}),
FLASK_APP: 'app.py',
HTTP_PROXY: process.env.HTTP_PROXY || '',
HTTPS_PROXY: process.env.HTTPS_PROXY || '',
}
}

17
e2e/support/cucumber-messages.ts Normal file
View File

@ -0,0 +1,17 @@
type CucumberMessage = {
testCaseStarted?: unknown
}
export const countStartedCucumberScenarios = (contents: string) =>
contents
.split(/\r?\n/)
.filter(Boolean)
.map((line) => JSON.parse(line) as CucumberMessage)
.filter((message) => message.testCaseStarted !== undefined).length
export const assertCucumberScenariosStarted = (contents: string) => {
const started = countStartedCucumberScenarios(contents)
if (started === 0)
throw new Error('Cucumber selected zero scenarios. Check the active tag expression and paths.')
}

279
e2e/support/cucumber-report.ts
View File

@ -1,279 +0,0 @@
import { Buffer } from 'node:buffer'
import { readFile } from 'node:fs/promises'
type CucumberEmbedding = {
data?: string
mime_type?: string
}
type CucumberStep = {
embeddings?: CucumberEmbedding[]
hidden?: boolean
result?: {
status?: string
}
}
type CucumberScenario = {
name?: string
steps?: CucumberStep[]
tags?: { name?: string }[]
type?: string
}
export type CucumberReport = {
elements?: CucumberScenario[]
uri?: string
}[]
export type CucumberReportSummary = {
blockedScenarios: {
name: string
tags: string[]
uri: string
}[]
blockedSkipped: number
failed: number
other: number
passed: number
selected: number
skipped: number
unexpectedSkipped: number
}
export type CucumberReportGate = {
allowedBlockedScenarios: Record<string, string[]>
maxSkipped: number
maxUnexpectedSkipped: number
minPassed: number
minSelected: number
profile: string
}
const reportGateProfiles = {
core: {
allowedBlockedScenarios: {
'features/agent-v2/access-point.feature': ['Workflow access shows the referencing workflow'],
'features/agent-v2/advanced-settings.feature': [
'Content Moderation keyword preset replies are saved and restored',
],
'features/agent-v2/agent-edit.feature': [
'Saved orchestration sections are visible on the Agent Edit page',
'Duplicated Agent inherits configuration without changing the original Agent',
'Tool states are visible on the Agent Edit page',
'Dual Knowledge Retrieval settings are visible on the Agent Edit page',
'Agent Edit opens the same Agent in Agent Console',
],
'features/agent-v2/configure-persistence.feature': [
'Selecting a stable model in Configure persists after refresh',
'Persisted Agent v2 instructions remain visible after refresh',
],
'features/agent-v2/knowledge.feature': [
'Agent decide Knowledge Retrieval settings are saved and restored',
'Custom query Knowledge Retrieval settings are saved and restored',
'Removing Knowledge Retrieval clears the saved dataset reference',
],
'features/agent-v2/output-variables.feature': [
'Workflow Agent v2 output variables persist after refresh',
'Workflow Agent v2 nested object output variables persist after refresh',
'Workflow Agent v2 prompt output reference stays synced when renamed',
],
'features/agent-v2/preflight.feature': [
'Stable chat model is available',
'Default speech-to-text model is available',
'Agent-decision chat model is available',
'Broken chat model is available for recovery scenarios',
'JSON Replace tool is available',
'Tavily Search tool is available',
'Agent knowledge base is available',
'Indexing knowledge base is available',
'Full config Agent is available',
'Full config Agent includes the summary Skill',
'Full config Agent includes core fixture configuration',
'Content Moderation Settings is enabled',
'Tool states Agent is available',
'Tool states Agent includes tool state fixture configuration',
'OAuth2 tool Agent includes credential fixture configuration',
'Dual retrieval Agent is available',
'Dual retrieval Agent includes dual retrieval fixture configuration',
'Published Web app Agent exposes Web app access',
'Backend API-enabled Agent is available',
'Backend API-enabled Agent exposes API access with a key',
'Workflow reference Agent is available',
'Reference workflow is available',
'Workflow reference Agent is used by the reference workflow',
],
'features/agent-v2/publish.feature': [
'Publish a configured Agent v2 draft',
'Publish action follows unpublished changes',
'Published Agent v2 version remains isolated from draft edits',
'Restoring a published Agent v2 version shows the restored configuration in Builder',
],
'features/agent-v2/tools.feature': [
'JSON Replace tool is saved after adding it from the Tools selector',
'OAuth2 tool credentials stay authorized after Configure autosaves',
],
},
maxSkipped: 44,
maxUnexpectedSkipped: 0,
minPassed: 65,
minSelected: 109,
},
external: {
allowedBlockedScenarios: {},
maxSkipped: 0,
maxUnexpectedSkipped: 0,
minPassed: 11,
minSelected: 11,
},
'webkit-browser-smoke': {
allowedBlockedScenarios: {},
maxSkipped: 0,
maxUnexpectedSkipped: 0,
minPassed: 4,
minSelected: 4,
},
} satisfies Record<string, Omit<CucumberReportGate, 'profile'>>
export const getCucumberReportGate = (env: NodeJS.ProcessEnv): CucumberReportGate | undefined => {
const profile = env.E2E_CUCUMBER_REPORT_PROFILE?.trim()
if (!profile) return undefined
const gate = reportGateProfiles[profile as keyof typeof reportGateProfiles]
if (!gate) throw new Error(`Unknown Cucumber report gate profile "${profile}".`)
return {
...gate,
allowedBlockedScenarios: Object.fromEntries(
Object.entries(gate.allowedBlockedScenarios).map(([uri, names]) => [uri, [...names]]),
),
profile,
}
}
const failureStatuses = new Set(['ambiguous', 'failed', 'pending', 'undefined', 'unknown'])
const hasBlockedPrecondition = (steps: CucumberStep[]) =>
steps
.filter((step) => !step.hidden && step.result?.status?.toLowerCase() === 'skipped')
.flatMap((step) => step.embeddings || [])
.filter((embedding) => embedding.mime_type === 'text/plain' && embedding.data)
.some((embedding) => {
const contents = Buffer.from(embedding.data || '', 'base64').toString('utf8')
return contents.startsWith('Blocked precondition:')
})
export const summarizeCucumberReport = (report: CucumberReport): CucumberReportSummary => {
const summary: CucumberReportSummary = {
blockedScenarios: [],
blockedSkipped: 0,
failed: 0,
other: 0,
passed: 0,
selected: 0,
skipped: 0,
unexpectedSkipped: 0,
}
for (const feature of report) {
for (const scenario of feature.elements || []) {
if (scenario.type !== 'scenario') continue
summary.selected += 1
const steps = scenario.steps || []
const statuses = steps
.map((step) => step.result?.status?.toLowerCase())
.filter((status): status is string => Boolean(status))
if (statuses.some((status) => failureStatuses.has(status))) {
summary.failed += 1
continue
}
if (steps.length === 0 || statuses.length !== steps.length) {
summary.other += 1
continue
}
if (statuses.includes('skipped')) {
summary.skipped += 1
if (hasBlockedPrecondition(steps)) {
summary.blockedSkipped += 1
summary.blockedScenarios.push({
name: scenario.name || '<unnamed scenario>',
tags: (scenario.tags || []).flatMap((tag) => (tag.name ? [tag.name] : [])),
uri: feature.uri || '<unknown feature>',
})
} else summary.unexpectedSkipped += 1
continue
}
if (statuses.length > 0 && statuses.every((status) => status === 'passed')) {
summary.passed += 1
continue
}
summary.other += 1
}
}
return summary
}
export const readCucumberReportSummary = async (reportPath: string) => {
const contents = await readFile(reportPath, 'utf8')
const report = JSON.parse(contents) as CucumberReport
return summarizeCucumberReport(report)
}
export const assertCucumberReport = (summary: CucumberReportSummary, gate: CucumberReportGate) => {
const errors: string[] = []
const allowedBlockedScenarios = new Set(
Object.entries(gate.allowedBlockedScenarios).flatMap(([uri, names]) =>
names.map((name) => `${uri}\0${name}`),
),
)
const disallowedBlockedScenarios = summary.blockedScenarios.filter(
(scenario) => !allowedBlockedScenarios.has(`${scenario.uri}\0${scenario.name}`),
)
if (summary.selected < gate.minSelected)
errors.push(`selected scenarios ${summary.selected} is below minimum ${gate.minSelected}`)
if (summary.passed < gate.minPassed)
errors.push(`passed scenarios ${summary.passed} is below minimum ${gate.minPassed}`)
if (summary.skipped > gate.maxSkipped)
errors.push(`skipped scenarios ${summary.skipped} exceeds maximum ${gate.maxSkipped}`)
if (summary.unexpectedSkipped > gate.maxUnexpectedSkipped) {
errors.push(
`unexpected skipped scenarios ${summary.unexpectedSkipped} exceeds maximum ${gate.maxUnexpectedSkipped}`,
)
}
if (disallowedBlockedScenarios.length > 0) {
errors.push(
`blocked scenarios not present in the checked-in allowlist: ${disallowedBlockedScenarios
.map((scenario) => `${scenario.uri}: ${scenario.name}`)
.join(', ')}`,
)
}
if (summary.failed > 0) errors.push(`failed scenarios ${summary.failed} exceeds maximum 0`)
if (summary.other > 0) errors.push(`unclassified scenarios ${summary.other} exceeds maximum 0`)
if (errors.length > 0)
throw new Error(
[
`Cucumber report gate "${gate.profile}" failed:`,
...errors.map((error) => `- ${error}`),
].join('\n'),
)
}
export const formatCucumberReportSummary = (summary: CucumberReportSummary) =>
[
`selected=${summary.selected}`,
`passed=${summary.passed}`,
`skipped=${summary.skipped}`,
`blockedSkipped=${summary.blockedSkipped}`,
`unexpectedSkipped=${summary.unexpectedSkipped}`,
`failed=${summary.failed}`,
`other=${summary.other}`,
].join(' ')

33
e2e/support/marketplace-plugins.ts
View File

@ -34,7 +34,6 @@ type PluginInstallStartResponse = {
type MarketplacePluginBootstrapConfig = {
defaultPluginIds: string[]
pluginIdsEnv: string
pluginUniqueIdentifiersEnv: string
title: string
}
@ -54,12 +53,6 @@ const unique = (values: string[]) => Array.from(new Set(values))
const getPluginId = (pluginUniqueIdentifier: string) =>
pluginUniqueIdentifier.split(':')[0]?.trim() || pluginUniqueIdentifier.trim()
const findPlaceholderPluginIdentifier = (pluginUniqueIdentifiers: string[]) =>
pluginUniqueIdentifiers.find((identifier) => identifier.includes('replace-with-'))
const withoutPlaceholderPluginIdentifiers = (pluginUniqueIdentifiers: string[]) =>
pluginUniqueIdentifiers.filter((identifier) => !identifier.includes('replace-with-'))
const resolveLatestPluginIdentifiers = async (pluginIds: string[]) => {
if (pluginIds.length === 0) return { identifiers: [] as string[], missing: [] as string[] }
@ -249,25 +242,8 @@ export const bootstrapMarketplacePlugins = async (
config: MarketplacePluginBootstrapConfig,
): Promise<SeedResult> => {
const requestedPluginIds = parseListEnv(config.pluginIdsEnv)
const rawExactPluginUniqueIdentifiers = unique(parseListEnv(config.pluginUniqueIdentifiersEnv))
const exactPluginUniqueIdentifiers = withoutPlaceholderPluginIdentifiers(
rawExactPluginUniqueIdentifiers,
)
const placeholderPluginIdentifier = findPlaceholderPluginIdentifier(
rawExactPluginUniqueIdentifiers,
)
if (placeholderPluginIdentifier) {
console.warn(
`[seed] ignoring example marketplace package placeholder for ${getPluginId(placeholderPluginIdentifier)}.`,
)
}
const pluginIds = unique(
exactPluginUniqueIdentifiers.length > 0
? []
: requestedPluginIds.length > 0
? requestedPluginIds
: config.defaultPluginIds,
requestedPluginIds.length > 0 ? requestedPluginIds : config.defaultPluginIds,
)
if (pluginIds.length > 0) {
@ -287,14 +263,11 @@ export const bootstrapMarketplacePlugins = async (
if (resolved.missing.length > 0) {
return blocked(
config.title,
`Marketplace metadata was not found for plugin ids: ${resolved.missing.join(', ')}. Set ${config.pluginUniqueIdentifiersEnv} to exact package identifiers to bypass latest-version lookup.`,
`Marketplace metadata was not found for plugin ids: ${resolved.missing.join(', ')}.`,
)
}
const requiredPluginUniqueIdentifiers = unique([
...resolved.identifiers,
...exactPluginUniqueIdentifiers,
])
const requiredPluginUniqueIdentifiers = unique(resolved.identifiers)
const requiredPluginIds = unique(requiredPluginUniqueIdentifiers.map(getPluginId))
if (requiredPluginUniqueIdentifiers.length === 0)

6
e2e/tests/agent-backend-preflight.test.ts → e2e/tests/agent-backend-readiness.test.ts
View File

@ -1,8 +1,8 @@
import type { DifyWorld } from '../features/support/world'
import { afterEach, describe, expect, it, vi } from 'vitest'
import { skipMissingAgentBackendRuntime } from '../features/agent-v2/support/preflight/agent-backend'
import { requireAgentBackendRuntime } from '../features/agent-v2/support/fixtures/agent-backend'
describe('skipMissingAgentBackendRuntime', () => {
describe('requireAgentBackendRuntime', () => {
afterEach(() => {
vi.unstubAllEnvs()
vi.unstubAllGlobals()
@ -26,7 +26,7 @@ describe('skipMissingAgentBackendRuntime', () => {
const attach = vi.fn()
const world = { attach } as unknown as DifyWorld
await expect(skipMissingAgentBackendRuntime(world)).resolves.toBe('http://agent-backend.test')
await expect(requireAgentBackendRuntime(world)).resolves.toBe('http://agent-backend.test')
expect(fetchMock).toHaveBeenCalledTimes(2)
expect(attach).not.toHaveBeenCalled()
})

40
e2e/tests/agent-v2-seed.test.ts Normal file
View File

@ -0,0 +1,40 @@
import { describe, expect, it } from 'vitest'
import { createAgentV2SeedTasks } from '../features/agent-v2/support/seed'
const baseTaskIds = [
'marketplace-plugins',
'stable-model',
'agent-decision-model',
'tool:JSON Process / JSON Replace',
'tool:Tavily / Tavily Search',
'ready-knowledge',
]
const preparedFixtureTaskIds = [
'full-config-agent',
'tool-states-agent',
'dual-retrieval-agent',
'workflow-reference',
]
const taskIds = (profile: string) => createAgentV2SeedTasks(profile).map((task) => task.id)
describe('Agent v2 seed profiles', () => {
it('prepares every fixture required by prepared behavior', () => {
expect(taskIds('prepared')).toEqual([...baseTaskIds, ...preparedFixtureTaskIds])
})
it('prepares external runtime and prepared behavior in post-merge', () => {
expect(taskIds('post-merge')).toEqual([
...baseTaskIds,
'speech-to-text-model',
...preparedFixtureTaskIds,
])
})
it('rejects unknown profiles', () => {
expect(() => createAgentV2SeedTasks('unknown')).toThrow(
'Unknown Agent V2 seed profile "unknown".',
)
})
})

31
e2e/tests/cucumber-messages.test.ts Normal file
View File

@ -0,0 +1,31 @@
import { describe, expect, it } from 'vitest'
import {
assertCucumberScenariosStarted,
countStartedCucumberScenarios,
} from '../support/cucumber-messages'
describe('countStartedCucumberScenarios', () => {
it('counts test case start messages', () => {
const report = [
JSON.stringify({ meta: { protocolVersion: '32.3.1' } }),
JSON.stringify({ testCaseStarted: { id: 'first' } }),
JSON.stringify({ testCaseStarted: { id: 'second' } }),
'',
].join('\n')
expect(countStartedCucumberScenarios(report)).toBe(2)
})
it('returns zero when the selector starts no scenarios', () => {
const report = [
JSON.stringify({ meta: { protocolVersion: '32.3.1' } }),
JSON.stringify({ testRunStarted: { timestamp: {} } }),
JSON.stringify({ testRunFinished: { timestamp: {} } }),
].join('\n')
expect(countStartedCucumberScenarios(report)).toBe(0)
expect(() => assertCucumberScenariosStarted(report)).toThrow(
'Cucumber selected zero scenarios.',
)
})
})

214
e2e/tests/cucumber-report.test.ts
View File

@ -1,214 +0,0 @@
import { Buffer } from 'node:buffer'
import { describe, expect, it } from 'vitest'
import {
assertCucumberReport,
getCucumberReportGate,
summarizeCucumberReport,
} from '../support/cucumber-report'
const step = (status?: string, options: { blockedReason?: string; hidden?: boolean } = {}) => ({
...(options.blockedReason
? {
embeddings: [
{
data: Buffer.from(`Blocked precondition: ${options.blockedReason}`).toString('base64'),
mime_type: 'text/plain',
},
],
}
: {}),
...(options.hidden ? { hidden: true } : {}),
...(status ? { result: { status } } : {}),
})
const scenario = (name: string, steps: ReturnType<typeof step>[], tags: string[] = []) => ({
name,
steps,
tags: tags.map((tag) => ({ name: tag })),
type: 'scenario',
})
describe('summarizeCucumberReport', () => {
it('separates passed, explicitly blocked, and unexpected skipped scenarios', () => {
const summary = summarizeCucumberReport([
{
elements: [
scenario('passes', [step('passed')]),
scenario(
'blocked',
[step('passed'), step('skipped', { blockedReason: 'fixture' })],
['@fixture'],
),
scenario('unexpected skip', [step('skipped')]),
],
uri: 'features/example.feature',
},
])
expect(summary).toEqual({
blockedScenarios: [
{
name: 'blocked',
tags: ['@fixture'],
uri: 'features/example.feature',
},
],
blockedSkipped: 1,
failed: 0,
other: 0,
passed: 1,
selected: 3,
skipped: 2,
unexpectedSkipped: 1,
})
})
it('treats a failed hook as a failed scenario', () => {
const summary = summarizeCucumberReport([
{
elements: [scenario('cleanup fails', [step('passed'), step('failed', { hidden: true })])],
uri: 'features/example.feature',
},
])
expect(summary.failed).toBe(1)
expect(summary.passed).toBe(0)
})
it('classifies a scenario with a missing step status as unclassified', () => {
const summary = summarizeCucumberReport([
{
elements: [scenario('missing status', [step('passed', { hidden: true }), step()])],
uri: 'features/example.feature',
},
])
expect(summary.other).toBe(1)
expect(summary.passed).toBe(0)
})
})
describe('assertCucumberReport', () => {
it('accepts an explicitly blocked readiness report within configured limits', () => {
const summary = {
blockedScenarios: [
{ name: 'fixture blocked', tags: ['@fixture'], uri: 'features/example.feature' },
{ name: 'preflight blocked', tags: ['@preflight'], uri: 'features/example.feature' },
],
blockedSkipped: 2,
failed: 0,
other: 0,
passed: 3,
selected: 5,
skipped: 2,
unexpectedSkipped: 0,
}
expect(() =>
assertCucumberReport(summary, {
allowedBlockedScenarios: {
'features/example.feature': ['fixture blocked', 'preflight blocked'],
},
maxSkipped: 2,
maxUnexpectedSkipped: 0,
minPassed: 3,
minSelected: 5,
profile: 'core',
}),
).not.toThrow()
})
it('rejects zero coverage, all-skipped coverage, and unexplained skips', () => {
const summary = {
blockedScenarios: [
{ name: 'fixture blocked', tags: ['@fixture'], uri: 'features/example.feature' },
],
blockedSkipped: 1,
failed: 0,
other: 0,
passed: 0,
selected: 2,
skipped: 2,
unexpectedSkipped: 1,
}
expect(() =>
assertCucumberReport(summary, {
allowedBlockedScenarios: {
'features/example.feature': ['fixture blocked'],
},
maxSkipped: 2,
maxUnexpectedSkipped: 0,
minPassed: 1,
minSelected: 3,
profile: 'external',
}),
).toThrow(
[
'Cucumber report gate "external" failed:',
'- selected scenarios 2 is below minimum 3',
'- passed scenarios 0 is below minimum 1',
'- unexpected skipped scenarios 1 exceeds maximum 0',
].join('\n'),
)
})
it('rejects a blocked scenario that replaces an allowed scenario with the same dependency tag', () => {
const summary = {
blockedScenarios: [
{ name: 'core regression', tags: ['@fixture'], uri: 'features/example.feature' },
],
blockedSkipped: 1,
failed: 0,
other: 0,
passed: 1,
selected: 2,
skipped: 1,
unexpectedSkipped: 0,
}
expect(() =>
assertCucumberReport(summary, {
allowedBlockedScenarios: {
'features/example.feature': ['fixture blocked'],
},
maxSkipped: 1,
maxUnexpectedSkipped: 0,
minPassed: 1,
minSelected: 2,
profile: 'core',
}),
).toThrow(
'blocked scenarios not present in the checked-in allowlist: features/example.feature: core regression',
)
})
})
describe('getCucumberReportGate', () => {
it('returns no gate unless a profile is configured', () => {
expect(getCucumberReportGate({})).toBeUndefined()
})
it('returns the checked-in gate for a known profile', () => {
expect(
getCucumberReportGate({
E2E_CUCUMBER_REPORT_PROFILE: 'webkit-browser-smoke',
}),
).toEqual({
allowedBlockedScenarios: {},
maxSkipped: 0,
maxUnexpectedSkipped: 0,
minPassed: 4,
minSelected: 4,
profile: 'webkit-browser-smoke',
})
})
it('rejects an unknown profile instead of silently weakening the gate', () => {
expect(() =>
getCucumberReportGate({
E2E_CUCUMBER_REPORT_PROFILE: 'custom',
}),
).toThrow('Unknown Cucumber report gate profile "custom".')
})
})

81
web/features/agent-v2/agent-detail/configure/__tests__/page.spec.tsx
View File

@ -225,7 +225,9 @@ vi.mock('../components/orchestrate', async () => {
AgentOrchestratePanel: (props: {
bottomAction?: ReactNode
isBuildDraftActive?: boolean
onExitVersions?: () => void
onOpenVersions?: () => void
onVersionRestored?: () => void | Promise<void>
readOnly?: boolean
showPublishBar?: boolean
}) => {
@ -240,6 +242,15 @@ vi.mock('../components/orchestrate', async () => {
<button type="button" onClick={props.onOpenVersions}>
open versions
</button>
<button
type="button"
onClick={async () => {
await props.onVersionRestored?.()
props.onExitVersions?.()
}}
>
restore version
</button>
{props.bottomAction}
</div>
)
@ -1661,6 +1672,76 @@ describe('AgentConfigurePage', () => {
expect(screen.queryByRole('region', { name: 'build-draft-bar' })).not.toBeInTheDocument()
})
it('should rebase the composer from the restored version', async () => {
const user = userEvent.setup()
const queryClient = new QueryClient()
const refetchComposer = vi.fn(async () => {
mocks.queryState.composer = {
...mocks.queryState.composer,
data: {
agent_soul: {
prompt: {
system_prompt: 'restored prompt',
},
},
},
}
return {}
})
mocks.queryState.composer = {
data: {
agent_soul: {
prompt: {
system_prompt: 'draft prompt',
},
},
},
isFetching: false,
isError: false,
isPending: false,
isSuccess: true,
refetch: refetchComposer,
}
mocks.queryState.version = {
data: {
config_snapshot: {
prompt: {
system_prompt: 'published prompt',
},
},
},
isFetching: false,
isError: false,
isPending: false,
isSuccess: true,
}
render(
<QueryClientProvider client={queryClient}>
<AgentConfigurePage agentId="agent-1" />
</QueryClientProvider>,
)
expect(screen.getByRole('region', { name: 'orchestrate-panel' })).toHaveTextContent(
'prompt:draft prompt',
)
await user.click(screen.getByRole('button', { name: 'open versions' }))
await user.click(screen.getByRole('button', { name: 'select version' }))
expect(screen.getByRole('region', { name: 'orchestrate-panel' })).toHaveTextContent(
'prompt:published prompt',
)
await user.click(screen.getByRole('button', { name: 'restore version' }))
expect(refetchComposer).toHaveBeenCalled()
await waitFor(() => {
expect(screen.getByRole('region', { name: 'orchestrate-panel' })).toHaveTextContent(
'prompt:restored prompt',
)
})
})
it('should apply the build draft and rebase the composer store from the refetched normal draft', async () => {
const user = userEvent.setup()
const queryClient = new QueryClient()

4
web/features/agent-v2/agent-detail/configure/components/composer-session.tsx
View File

@ -366,6 +366,10 @@ function AgentConfigurePageComposerContent({
setShowPreviewVersions(true)
}}
onExitVersions={() => selectVersion(null)}
onVersionRestored={async () => {
await composerQuery.refetch()
onComposerRebase()
}}
/>
}
rightPanel={

11
web/features/agent-v2/agent-detail/configure/components/orchestrate/__tests__/publish-bar.spec.tsx
View File

@ -187,6 +187,7 @@ function renderPublishBar({
isPublishing,
onPublish = vi.fn<PublishHandler>(),
onExitVersions = vi.fn(),
onVersionRestored = vi.fn(),
prompt = '',
selectedVersionSnapshot,
setupStore,
@ -199,6 +200,7 @@ function renderPublishBar({
isPublishing?: boolean
onPublish?: PublishMock
onExitVersions?: Mock<() => void>
onVersionRestored?: Mock<() => void | Promise<void>>
prompt?: string
selectedVersionSnapshot?: AgentConfigSnapshotSummaryResponse | null
setupStore?: (store: ReturnType<typeof createStore>) => void
@ -243,6 +245,7 @@ function renderPublishBar({
onPublish={onPublish}
onExitVersions={onExitVersions}
onOpenVersions={vi.fn()}
onVersionRestored={onVersionRestored}
/>
</JotaiProvider>
</QueryClientProvider>
@ -254,6 +257,7 @@ function renderPublishBar({
queryClient,
onExitVersions,
onPublish,
onVersionRestored,
rerenderPublishBar: renderPublishBarTree,
}
}
@ -342,7 +346,9 @@ describe('AgentConfigurePublishBar', () => {
created_by: 'Alice',
}
const onVersionRestored = vi.fn().mockResolvedValue(undefined)
const { onExitVersions } = renderPublishBar({
onVersionRestored,
selectedVersionSnapshot,
})
@ -362,7 +368,10 @@ describe('AgentConfigurePublishBar', () => {
version_id: 'snapshot-2',
},
})
expect(onExitVersions).toHaveBeenCalled()
await waitFor(() => expect(onVersionRestored).toHaveBeenCalled())
expect(onVersionRestored.mock.invocationCallOrder[0]).toBeLessThan(
onExitVersions.mock.invocationCallOrder[0]!,
)
expect(toastMock.success).toHaveBeenCalledWith('common.api.actionSuccess')
})

3
web/features/agent-v2/agent-detail/configure/components/orchestrate/index.tsx
View File

@ -55,6 +55,7 @@ type AgentOrchestratePanelProps = {
onPublish?: () => void | Promise<void>
onExitVersions?: () => void
onOpenVersions?: () => void
onVersionRestored?: () => void | Promise<void>
}
export function AgentOrchestratePanel({
@ -83,6 +84,7 @@ export function AgentOrchestratePanel({
onPublish,
onExitVersions,
onOpenVersions,
onVersionRestored,
}: AgentOrchestratePanelProps) {
const { t } = useTranslation('agentV2')
const orchestrateHeadingId = 'agent-configure-orchestrate-heading'
@ -102,6 +104,7 @@ export function AgentOrchestratePanel({
onPublish={onPublish}
onExitVersions={onExitVersions}
onOpenVersions={onOpenVersions}
onVersionRestored={onVersionRestored}
/>
) : null)
const hasBottomAction = !!orchestrateBottomAction

41
web/features/agent-v2/agent-detail/configure/components/orchestrate/publish-bar/index.tsx
View File

@ -45,6 +45,7 @@ type AgentConfigurePublishBarProps = {
onPublish?: () => void | Promise<void>
onExitVersions?: () => void
onOpenVersions?: () => void
onVersionRestored?: () => void | Promise<void>
}
function getPublishState({
@ -99,6 +100,7 @@ export function AgentConfigurePublishBar({
onPublish,
onExitVersions,
onOpenVersions,
onVersionRestored,
}: AgentConfigurePublishBarProps) {
const { t } = useTranslation('agentV2')
const { t: tCommon } = useTranslation('common')
@ -154,28 +156,31 @@ export function AgentConfigurePublishBar({
},
},
{
onSuccess: () => {
void queryClient.invalidateQueries({
queryKey: consoleQuery.agent.byAgentId.get.queryKey({
input: {
params: {
agent_id: agentId,
onSuccess: async () => {
await Promise.all([
queryClient.invalidateQueries({
queryKey: consoleQuery.agent.byAgentId.get.queryKey({
input: {
params: {
agent_id: agentId,
},
},
},
}),
}),
})
void queryClient.invalidateQueries({
queryKey: consoleQuery.agent.byAgentId.composer.get.queryKey({
input: {
params: {
agent_id: agentId,
queryClient.invalidateQueries({
queryKey: consoleQuery.agent.byAgentId.composer.get.queryKey({
input: {
params: {
agent_id: agentId,
},
},
},
}),
}),
})
void queryClient.invalidateQueries({
queryKey: consoleQuery.agent.byAgentId.versions.get.key(),
})
queryClient.invalidateQueries({
queryKey: consoleQuery.agent.byAgentId.versions.get.key(),
}),
])
await onVersionRestored?.()
onExitVersions?.()
toast.success(tCommon(($) => $['api.actionSuccess']))
},