From 800c9f4fca005cd399383e87cc8b3865ee9a4102 Mon Sep 17 00:00:00 2001 From: Stephen Zhou Date: Tue, 7 Jul 2026 12:48:12 +0800 Subject: [PATCH] test(e2e): stabilize Agent v2 external runtime checks (#38493) Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> --- .../common/agent_app_parameters.py | 4 +- api/core/app/apps/agent_app/app_generator.py | 4 +- .../common/test_agent_app_parameters.py | 33 +++++++-- .../app/apps/agent_app/test_resolve_agent.py | 25 ++++++- .../agent-v2/access-point-web-app.steps.ts | 8 ++- .../agent-v2/build-draft.steps.ts | 5 +- e2e/features/support/hooks.ts | 70 +++++++++++++++---- 7 files changed, 122 insertions(+), 27 deletions(-) diff --git a/api/controllers/common/agent_app_parameters.py b/api/controllers/common/agent_app_parameters.py index 32e338957ea..8c2fbccd513 100644 --- a/api/controllers/common/agent_app_parameters.py +++ b/api/controllers/common/agent_app_parameters.py @@ -32,7 +32,9 @@ def get_published_agent_app_feature_dict_and_user_input_form( ) if agent is None: raise AgentAppGeneratorError("Agent App has no bound Agent") - if not agent.active_config_snapshot_id or not agent.active_config_is_published: + # active_config_is_published means the draft has no unpublished edits; the public app + # can still read parameters from the active snapshot while a newer draft is pending. + if not agent.active_config_snapshot_id: raise AgentAppNotPublishedError("Agent has not been published") snapshot = db.session.scalar( diff --git a/api/core/app/apps/agent_app/app_generator.py b/api/core/app/apps/agent_app/app_generator.py index a940ccf6ceb..9531f9092a4 100644 --- a/api/core/app/apps/agent_app/app_generator.py +++ b/api/core/app/apps/agent_app/app_generator.py @@ -611,7 +611,9 @@ class AgentAppGenerator(MessageBasedAppGenerator): "build_draft" if draft.draft_type == AgentConfigDraftType.DEBUG_BUILD else "draft" ) return agent, draft.id, config_version_kind, agent_soul - if not agent.active_config_snapshot_id or not agent.active_config_is_published: + # active_config_is_published tracks whether the editable draft matches the active snapshot. + # Public runtime must keep serving the active snapshot even when unpublished draft edits exist. + if not agent.active_config_snapshot_id: raise AgentAppNotPublishedError("Agent has not been published") _, snapshot, agent_soul = self._resolve_agent_by_id( tenant_id=app_model.tenant_id, diff --git a/api/tests/unit_tests/controllers/common/test_agent_app_parameters.py b/api/tests/unit_tests/controllers/common/test_agent_app_parameters.py index f963729c8a2..b4639d54d2f 100644 --- a/api/tests/unit_tests/controllers/common/test_agent_app_parameters.py +++ b/api/tests/unit_tests/controllers/common/test_agent_app_parameters.py @@ -85,15 +85,13 @@ def test_published_agent_app_parameters_requires_existing_active_agent(monkeypat @pytest.mark.parametrize( - ("active_config_snapshot_id", "active_config_is_published"), + "active_config_is_published", [ - (None, True), - ("snapshot-1", False), + True, + False, ], ) -def test_published_agent_app_parameters_requires_published_agent( - monkeypatch, active_config_snapshot_id, active_config_is_published -): +def test_published_agent_app_parameters_requires_published_agent(monkeypatch, active_config_is_published): app_model = SimpleNamespace( tenant_id="tenant-1", bound_agent_id="agent-1", @@ -101,7 +99,7 @@ def test_published_agent_app_parameters_requires_published_agent( ) agent = SimpleNamespace( id="agent-1", - active_config_snapshot_id=active_config_snapshot_id, + active_config_snapshot_id=None, active_config_is_published=active_config_is_published, ) monkeypatch.setattr(agent_app_parameters.db.session, "scalar", lambda _: agent) @@ -110,6 +108,27 @@ def test_published_agent_app_parameters_requires_published_agent( get_published_agent_app_feature_dict_and_user_input_form(app_model) +def test_published_agent_app_parameters_allows_unpublished_draft_with_active_snapshot(monkeypatch): + app_model = SimpleNamespace( + tenant_id="tenant-1", + bound_agent_id="agent-1", + app_model_config=None, + ) + agent = SimpleNamespace( + id="agent-1", + active_config_snapshot_id="snapshot-1", + active_config_is_published=False, + ) + snapshot = SimpleNamespace(config_snapshot_dict={}) + query_results = iter([agent, snapshot]) + monkeypatch.setattr(agent_app_parameters.db.session, "scalar", lambda _: next(query_results)) + + features_dict, user_input_form = get_published_agent_app_feature_dict_and_user_input_form(app_model) + + assert features_dict["file_upload"]["enabled"] is True + assert user_input_form == [] + + def test_published_agent_app_parameters_requires_published_snapshot(monkeypatch): app_model = SimpleNamespace( tenant_id="tenant-1", diff --git a/api/tests/unit_tests/core/app/apps/agent_app/test_resolve_agent.py b/api/tests/unit_tests/core/app/apps/agent_app/test_resolve_agent.py index 293c6676e3c..ec0ce7d23b1 100644 --- a/api/tests/unit_tests/core/app/apps/agent_app/test_resolve_agent.py +++ b/api/tests/unit_tests/core/app/apps/agent_app/test_resolve_agent.py @@ -97,12 +97,35 @@ class TestResolveAgent: assert config_version_kind == "snapshot" assert soul.model is not None - def test_unpublished_agent_raises_before_model_resolution(self, monkeypatch: pytest.MonkeyPatch): + def test_unpublished_draft_still_resolves_active_snapshot(self, monkeypatch: pytest.MonkeyPatch): bound_agent = SimpleNamespace( id="agent-1", active_config_snapshot_id="snap-1", active_config_is_published=False, ) + inner_agent = SimpleNamespace(id="agent-1") + snapshot = _snapshot() + _patch_session(monkeypatch, [bound_agent, inner_agent, snapshot]) + app_model = SimpleNamespace(id="app-1", tenant_id="t1") + + agent, config_id, config_version_kind, soul = AgentAppGenerator()._resolve_agent( + app_model, + invoke_from=InvokeFrom.WEB_APP, + draft_type=None, + user=SimpleNamespace(id="user-1"), + ) # type: ignore[arg-type] + + assert agent is bound_agent + assert config_id == snapshot.id + assert config_version_kind == "snapshot" + assert soul.prompt.system_prompt == "You are Iris." + + def test_agent_without_active_snapshot_raises_before_model_resolution(self, monkeypatch: pytest.MonkeyPatch): + bound_agent = SimpleNamespace( + id="agent-1", + active_config_snapshot_id=None, + active_config_is_published=False, + ) _patch_session(monkeypatch, [bound_agent]) app_model = SimpleNamespace(id="app-1", tenant_id="t1") diff --git a/e2e/features/step-definitions/agent-v2/access-point-web-app.steps.ts b/e2e/features/step-definitions/agent-v2/access-point-web-app.steps.ts index d9d85e83772..e5de7e1ac26 100644 --- a/e2e/features/step-definitions/agent-v2/access-point-web-app.steps.ts +++ b/e2e/features/step-definitions/agent-v2/access-point-web-app.steps.ts @@ -1,3 +1,4 @@ +import type { Page } from '@playwright/test' import type { DifyWorld } from '../../support/world' import { Given, Then, When } from '@cucumber/cucumber' import { expect } from '@playwright/test' @@ -12,6 +13,9 @@ import { const WEB_APP_RUNTIME_RESPONSE_STEP_TIMEOUT_MS = 180_000 +const getWebAppMessageInput = (webAppPage: Page) => + webAppPage.getByPlaceholder(/^Talk to /).last() + Then('I should see the Agent v2 Web app access URL', async function (this: DifyWorld) { const webAppCard = getWebAppCard(this) @@ -71,7 +75,7 @@ When('I send an E2E message in the Agent v2 Web app', async function (this: Dify if (!webAppPage) throw new Error('No Agent v2 Web app page was opened.') - const messageInput = webAppPage.getByRole('textbox').last() + const messageInput = getWebAppMessageInput(webAppPage) await expect(messageInput).toBeEditable({ timeout: 30_000 }) await messageInput.fill('Please reply with the test success marker.') await messageInput.press('Enter') @@ -84,7 +88,7 @@ Then('the Agent v2 Web app should open in a new tab', async function (this: Dify throw new Error('No Agent v2 Web app page was opened.') await expect(webAppPage).toHaveURL(webAppURL) - await expect(webAppPage.getByRole('textbox').last()).toBeEditable({ timeout: 30_000 }) + await expect(getWebAppMessageInput(webAppPage)).toBeEditable({ timeout: 30_000 }) await webAppPage.close() this.agentBuilder.accessPoint.webAppPage = undefined this.agentBuilder.accessPoint.webAppURL = undefined diff --git a/e2e/features/step-definitions/agent-v2/build-draft.steps.ts b/e2e/features/step-definitions/agent-v2/build-draft.steps.ts index 22e11f7bd63..87901434035 100644 --- a/e2e/features/step-definitions/agent-v2/build-draft.steps.ts +++ b/e2e/features/step-definitions/agent-v2/build-draft.steps.ts @@ -254,9 +254,8 @@ Then('I should see the Agent v2 Build mode confirmation state', async function ( const page = this.getPage() await expect(page.getByText('Build mode', { exact: true })).toBeVisible() - await expect( - page.getByText('You\'re in build mode. Shape this setup through the chat on the right, then Apply.'), - ).toBeVisible() + await expect(page.getByText('Configure can only be updated by the agent in this mode.')).toBeVisible() + await expect(page.getByText('Shape this setup through the chat on the right, then Apply.')).toBeVisible() }) Then('Agent v2 Build chat should be blocked until a model is configured', async function (this: DifyWorld) { diff --git a/e2e/features/support/hooks.ts b/e2e/features/support/hooks.ts index 8fd59934dae..658f3efc69b 100644 --- a/e2e/features/support/hooks.ts +++ b/e2e/features/support/hooks.ts @@ -1,4 +1,4 @@ -import type { Browser } from '@playwright/test' +import type { Browser, Page } from '@playwright/test' import type { Buffer } from 'node:buffer' import type { DifyWorld } from './world' import { mkdir, writeFile } from 'node:fs/promises' @@ -34,18 +34,50 @@ const sanitizeForPath = (value: string) => const writeArtifact = async ( scenarioName: string, + label: string, extension: 'html' | 'png', contents: Buffer | string, ) => { const artifactPath = path.join( artifactsDir, - `${Date.now()}-${sanitizeForPath(scenarioName || 'scenario')}.${extension}`, + `${Date.now()}-${sanitizeForPath(scenarioName || 'scenario')}-${sanitizeForPath(label)}.${extension}`, ) await writeFile(artifactPath, contents) return artifactPath } +const uniqueDiagnosticPages = (pages: { label: string, page: Page | undefined }[]) => { + const seen = new Set() + + return pages.filter(({ page }) => { + if (!page || page.isClosed() || seen.has(page)) + return false + + seen.add(page) + return true + }) as { label: string, page: Page }[] +} + +const captureDiagnosticPage = async ( + world: DifyWorld, + scenarioName: string, + label: string, + page: Page, +) => { + const screenshot = await page.screenshot({ + fullPage: true, + }) + const screenshotPath = await writeArtifact(scenarioName, label, 'png', screenshot) + world.attach(screenshot, 'image/png') + + const html = await page.content() + const htmlPath = await writeArtifact(scenarioName, label, 'html', html) + world.attach(html, 'text/html') + + return [screenshotPath, htmlPath] +} + const recordCleanup = async ( errors: string[], label: string, @@ -91,16 +123,26 @@ After(async function (this: DifyWorld, { pickle, result }) { const elapsedMs = this.scenarioStartedAt ? Date.now() - this.scenarioStartedAt : undefined const status = result?.status || Status.UNKNOWN - if (diagnosticArtifactStatuses.has(status) && this.page) { - const screenshot = await this.page.screenshot({ - fullPage: true, - }) - const screenshotPath = await writeArtifact(pickle.name, 'png', screenshot) - this.attach(screenshot, 'image/png') + if (diagnosticArtifactStatuses.has(status)) { + const artifactPaths: string[] = [] + const artifactErrors: string[] = [] + const diagnosticPages = uniqueDiagnosticPages([ + { label: 'main-page', page: this.page }, + { label: 'agent-v2-web-app', page: this.agentBuilder.accessPoint.webAppPage }, + { label: 'agent-v2-api-reference', page: this.agentBuilder.accessPoint.apiReferencePage }, + { label: 'agent-v2-workflow-reference', page: this.agentBuilder.accessPoint.workflowReferencePage }, + { label: 'agent-v2-concurrent-configure', page: this.agentBuilder.configure.concurrentPage }, + { label: 'agent-v2-workflow-console', page: this.agentBuilder.workflow.agentConsolePage }, + ]) - const html = await this.page.content() - const htmlPath = await writeArtifact(pickle.name, 'html', html) - this.attach(html, 'text/html') + for (const { label, page } of diagnosticPages) { + try { + artifactPaths.push(...await captureDiagnosticPage(this, pickle.name, label, page)) + } + catch (error) { + artifactErrors.push(`${label}: ${error instanceof Error ? error.message : String(error)}`) + } + } if (this.consoleErrors.length > 0) this.attach(`Console Errors:\n${this.consoleErrors.join('\n')}`, 'text/plain') @@ -108,7 +150,11 @@ After(async function (this: DifyWorld, { pickle, result }) { if (this.pageErrors.length > 0) this.attach(`Page Errors:\n${this.pageErrors.join('\n')}`, 'text/plain') - this.attach(`Artifacts:\n${[screenshotPath, htmlPath].join('\n')}`, 'text/plain') + if (artifactErrors.length > 0) + this.attach(`Artifact Errors:\n${artifactErrors.join('\n')}`, 'text/plain') + + if (artifactPaths.length > 0) + this.attach(`Artifacts:\n${artifactPaths.join('\n')}`, 'text/plain') } console.warn(