test(e2e): stabilize Agent v2 external runtime checks (#38493)

Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>

api/controllers/common/agent_app_parameters.py

@@ -32,7 +32,9 @@ def get_published_agent_app_feature_dict_and_user_input_form(
     )
     if agent is None:
         raise AgentAppGeneratorError("Agent App has no bound Agent")
-    if not agent.active_config_snapshot_id or not agent.active_config_is_published:
+    # active_config_is_published means the draft has no unpublished edits; the public app
+    # can still read parameters from the active snapshot while a newer draft is pending.
+    if not agent.active_config_snapshot_id:
         raise AgentAppNotPublishedError("Agent has not been published")
 
     snapshot = db.session.scalar(

api/core/app/apps/agent_app/app_generator.py

@@ -611,7 +611,9 @@ class AgentAppGenerator(MessageBasedAppGenerator):
                 "build_draft" if draft.draft_type == AgentConfigDraftType.DEBUG_BUILD else "draft"
             )
             return agent, draft.id, config_version_kind, agent_soul
-        if not agent.active_config_snapshot_id or not agent.active_config_is_published:
+        # active_config_is_published tracks whether the editable draft matches the active snapshot.
+        # Public runtime must keep serving the active snapshot even when unpublished draft edits exist.
+        if not agent.active_config_snapshot_id:
             raise AgentAppNotPublishedError("Agent has not been published")
         _, snapshot, agent_soul = self._resolve_agent_by_id(
             tenant_id=app_model.tenant_id,

api/tests/unit_tests/controllers/common/test_agent_app_parameters.py

@@ -85,15 +85,13 @@ def test_published_agent_app_parameters_requires_existing_active_agent(monkeypat
 
 
 @pytest.mark.parametrize(
-    ("active_config_snapshot_id", "active_config_is_published"),
+    "active_config_is_published",
     [
-        (None, True),
-        ("snapshot-1", False),
+        True,
+        False,
     ],
 )
-def test_published_agent_app_parameters_requires_published_agent(
-    monkeypatch, active_config_snapshot_id, active_config_is_published
-):
+def test_published_agent_app_parameters_requires_published_agent(monkeypatch, active_config_is_published):
     app_model = SimpleNamespace(
         tenant_id="tenant-1",
         bound_agent_id="agent-1",
@@ -101,7 +99,7 @@ def test_published_agent_app_parameters_requires_published_agent(
     )
     agent = SimpleNamespace(
         id="agent-1",
-        active_config_snapshot_id=active_config_snapshot_id,
+        active_config_snapshot_id=None,
         active_config_is_published=active_config_is_published,
     )
     monkeypatch.setattr(agent_app_parameters.db.session, "scalar", lambda _: agent)
@@ -110,6 +108,27 @@ def test_published_agent_app_parameters_requires_published_agent(
         get_published_agent_app_feature_dict_and_user_input_form(app_model)
 
 
+def test_published_agent_app_parameters_allows_unpublished_draft_with_active_snapshot(monkeypatch):
+    app_model = SimpleNamespace(
+        tenant_id="tenant-1",
+        bound_agent_id="agent-1",
+        app_model_config=None,
+    )
+    agent = SimpleNamespace(
+        id="agent-1",
+        active_config_snapshot_id="snapshot-1",
+        active_config_is_published=False,
+    )
+    snapshot = SimpleNamespace(config_snapshot_dict={})
+    query_results = iter([agent, snapshot])
+    monkeypatch.setattr(agent_app_parameters.db.session, "scalar", lambda _: next(query_results))
+
+    features_dict, user_input_form = get_published_agent_app_feature_dict_and_user_input_form(app_model)
+
+    assert features_dict["file_upload"]["enabled"] is True
+    assert user_input_form == []
+
+
 def test_published_agent_app_parameters_requires_published_snapshot(monkeypatch):
     app_model = SimpleNamespace(
         tenant_id="tenant-1",

api/tests/unit_tests/core/app/apps/agent_app/test_resolve_agent.py

@@ -97,12 +97,35 @@ class TestResolveAgent:
         assert config_version_kind == "snapshot"
         assert soul.model is not None
 
-    def test_unpublished_agent_raises_before_model_resolution(self, monkeypatch: pytest.MonkeyPatch):
+    def test_unpublished_draft_still_resolves_active_snapshot(self, monkeypatch: pytest.MonkeyPatch):
         bound_agent = SimpleNamespace(
             id="agent-1",
             active_config_snapshot_id="snap-1",
             active_config_is_published=False,
         )
+        inner_agent = SimpleNamespace(id="agent-1")
+        snapshot = _snapshot()
+        _patch_session(monkeypatch, [bound_agent, inner_agent, snapshot])
+        app_model = SimpleNamespace(id="app-1", tenant_id="t1")
+
+        agent, config_id, config_version_kind, soul = AgentAppGenerator()._resolve_agent(
+            app_model,
+            invoke_from=InvokeFrom.WEB_APP,
+            draft_type=None,
+            user=SimpleNamespace(id="user-1"),
+        )  # type: ignore[arg-type]
+
+        assert agent is bound_agent
+        assert config_id == snapshot.id
+        assert config_version_kind == "snapshot"
+        assert soul.prompt.system_prompt == "You are Iris."
+
+    def test_agent_without_active_snapshot_raises_before_model_resolution(self, monkeypatch: pytest.MonkeyPatch):
+        bound_agent = SimpleNamespace(
+            id="agent-1",
+            active_config_snapshot_id=None,
+            active_config_is_published=False,
+        )
         _patch_session(monkeypatch, [bound_agent])
         app_model = SimpleNamespace(id="app-1", tenant_id="t1")
 

e2e/features/step-definitions/agent-v2/access-point-web-app.steps.ts

@@ -1,3 +1,4 @@
+import type { Page } from '@playwright/test'
 import type { DifyWorld } from '../../support/world'
 import { Given, Then, When } from '@cucumber/cucumber'
 import { expect } from '@playwright/test'
@@ -12,6 +13,9 @@ import {
 
 const WEB_APP_RUNTIME_RESPONSE_STEP_TIMEOUT_MS = 180_000
 
+const getWebAppMessageInput = (webAppPage: Page) =>
+  webAppPage.getByPlaceholder(/^Talk to /).last()
+
 Then('I should see the Agent v2 Web app access URL', async function (this: DifyWorld) {
   const webAppCard = getWebAppCard(this)
 
@@ -71,7 +75,7 @@ When('I send an E2E message in the Agent v2 Web app', async function (this: Dify
   if (!webAppPage)
     throw new Error('No Agent v2 Web app page was opened.')
 
-  const messageInput = webAppPage.getByRole('textbox').last()
+  const messageInput = getWebAppMessageInput(webAppPage)
   await expect(messageInput).toBeEditable({ timeout: 30_000 })
   await messageInput.fill('Please reply with the test success marker.')
   await messageInput.press('Enter')
@@ -84,7 +88,7 @@ Then('the Agent v2 Web app should open in a new tab', async function (this: Dify
     throw new Error('No Agent v2 Web app page was opened.')
 
   await expect(webAppPage).toHaveURL(webAppURL)
-  await expect(webAppPage.getByRole('textbox').last()).toBeEditable({ timeout: 30_000 })
+  await expect(getWebAppMessageInput(webAppPage)).toBeEditable({ timeout: 30_000 })
   await webAppPage.close()
   this.agentBuilder.accessPoint.webAppPage = undefined
   this.agentBuilder.accessPoint.webAppURL = undefined

e2e/features/step-definitions/agent-v2/build-draft.steps.ts

@@ -254,9 +254,8 @@ Then('I should see the Agent v2 Build mode confirmation state', async function (
   const page = this.getPage()
 
   await expect(page.getByText('Build mode', { exact: true })).toBeVisible()
-  await expect(
-    page.getByText('You\'re in build mode. Shape this setup through the chat on the right, then Apply.'),
-  ).toBeVisible()
+  await expect(page.getByText('Configure can only be updated by the agent in this mode.')).toBeVisible()
+  await expect(page.getByText('Shape this setup through the chat on the right, then Apply.')).toBeVisible()
 })
 
 Then('Agent v2 Build chat should be blocked until a model is configured', async function (this: DifyWorld) {

e2e/features/support/hooks.ts

@@ -1,4 +1,4 @@
-import type { Browser } from '@playwright/test'
+import type { Browser, Page } from '@playwright/test'
 import type { Buffer } from 'node:buffer'
 import type { DifyWorld } from './world'
 import { mkdir, writeFile } from 'node:fs/promises'
@@ -34,18 +34,50 @@ const sanitizeForPath = (value: string) =>
 
 const writeArtifact = async (
   scenarioName: string,
+  label: string,
   extension: 'html' | 'png',
   contents: Buffer | string,
 ) => {
   const artifactPath = path.join(
     artifactsDir,
-    `${Date.now()}-${sanitizeForPath(scenarioName || 'scenario')}.${extension}`,
+    `${Date.now()}-${sanitizeForPath(scenarioName || 'scenario')}-${sanitizeForPath(label)}.${extension}`,
   )
   await writeFile(artifactPath, contents)
 
   return artifactPath
 }
 
+const uniqueDiagnosticPages = (pages: { label: string, page: Page | undefined }[]) => {
+  const seen = new Set<Page>()
+
+  return pages.filter(({ page }) => {
+    if (!page || page.isClosed() || seen.has(page))
+      return false
+
+    seen.add(page)
+    return true
+  }) as { label: string, page: Page }[]
+}
+
+const captureDiagnosticPage = async (
+  world: DifyWorld,
+  scenarioName: string,
+  label: string,
+  page: Page,
+) => {
+  const screenshot = await page.screenshot({
+    fullPage: true,
+  })
+  const screenshotPath = await writeArtifact(scenarioName, label, 'png', screenshot)
+  world.attach(screenshot, 'image/png')
+
+  const html = await page.content()
+  const htmlPath = await writeArtifact(scenarioName, label, 'html', html)
+  world.attach(html, 'text/html')
+
+  return [screenshotPath, htmlPath]
+}
+
 const recordCleanup = async (
   errors: string[],
   label: string,
@@ -91,16 +123,26 @@ After(async function (this: DifyWorld, { pickle, result }) {
   const elapsedMs = this.scenarioStartedAt ? Date.now() - this.scenarioStartedAt : undefined
   const status = result?.status || Status.UNKNOWN
 
-  if (diagnosticArtifactStatuses.has(status) && this.page) {
-    const screenshot = await this.page.screenshot({
-      fullPage: true,
-    })
-    const screenshotPath = await writeArtifact(pickle.name, 'png', screenshot)
-    this.attach(screenshot, 'image/png')
+  if (diagnosticArtifactStatuses.has(status)) {
+    const artifactPaths: string[] = []
+    const artifactErrors: string[] = []
+    const diagnosticPages = uniqueDiagnosticPages([
+      { label: 'main-page', page: this.page },
+      { label: 'agent-v2-web-app', page: this.agentBuilder.accessPoint.webAppPage },
+      { label: 'agent-v2-api-reference', page: this.agentBuilder.accessPoint.apiReferencePage },
+      { label: 'agent-v2-workflow-reference', page: this.agentBuilder.accessPoint.workflowReferencePage },
+      { label: 'agent-v2-concurrent-configure', page: this.agentBuilder.configure.concurrentPage },
+      { label: 'agent-v2-workflow-console', page: this.agentBuilder.workflow.agentConsolePage },
+    ])
 
-    const html = await this.page.content()
-    const htmlPath = await writeArtifact(pickle.name, 'html', html)
-    this.attach(html, 'text/html')
+    for (const { label, page } of diagnosticPages) {
+      try {
+        artifactPaths.push(...await captureDiagnosticPage(this, pickle.name, label, page))
+      }
+      catch (error) {
+        artifactErrors.push(`${label}: ${error instanceof Error ? error.message : String(error)}`)
+      }
+    }
 
     if (this.consoleErrors.length > 0)
       this.attach(`Console Errors:\n${this.consoleErrors.join('\n')}`, 'text/plain')
@@ -108,7 +150,11 @@ After(async function (this: DifyWorld, { pickle, result }) {
     if (this.pageErrors.length > 0)
       this.attach(`Page Errors:\n${this.pageErrors.join('\n')}`, 'text/plain')
 
-    this.attach(`Artifacts:\n${[screenshotPath, htmlPath].join('\n')}`, 'text/plain')
+    if (artifactErrors.length > 0)
+      this.attach(`Artifact Errors:\n${artifactErrors.join('\n')}`, 'text/plain')
+
+    if (artifactPaths.length > 0)
+      this.attach(`Artifacts:\n${artifactPaths.join('\n')}`, 'text/plain')
   }
 
   console.warn(