fix(cli/e2e): remove LLM nodes from fixture DSLs and fix test assertions (#37463)

Co-authored-by: yunlu.wen <yunlu.wen@dify.ai>
This commit is contained in:
gigglewang 2026-06-16 16:58:53 +08:00 committed by GitHub
parent dcc0b95e11
commit 56a026505e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 138 additions and 327 deletions

View File

@ -14,7 +14,7 @@ export type TokenStore = {
const DOC_VERSION = 1
type TokenDoc = {
export type TokenDoc = {
version?: number
tokens?: Record<string, Record<string, string>>
}

View File

@ -6,12 +6,7 @@ app:
mode: advanced-chat
name: echo-bot
use_icon_as_answer_icon: false
dependencies:
- current_identifier: null
type: marketplace
value:
marketplace_plugin_unique_identifier: langgenius/tongyi:0.1.48@966d88dc40611f067311c1c9839139ebc4b55bff471bc5e736dc3e828bc67b46
version: null
dependencies: []
kind: app
version: 0.6.0
workflow:
@ -68,18 +63,9 @@ workflow:
edges:
- data:
sourceType: start
targetType: llm
id: 1779690795511-llm
source: '1779690795511'
sourceHandle: source
target: llm
targetHandle: target
type: custom
- data:
sourceType: llm
targetType: answer
id: llm-answer
source: llm
id: 1779690795511-answer
source: '1779690795511'
sourceHandle: source
target: answer
targetHandle: target
@ -87,7 +73,7 @@ workflow:
nodes:
- data:
selected: false
title: 用户输入
title: User Input
type: start
variables:
- default: ''
@ -107,66 +93,24 @@ workflow:
positionAbsolute:
x: 79
y: 282
selected: true
sourcePosition: right
targetPosition: left
type: custom
width: 242
- data:
context:
enabled: false
variable_selector: []
memory:
query_prompt_template: '{{#sys.query#}}
{{#sys.files#}}'
role_prefix:
assistant: ''
user: ''
window:
enabled: false
size: 10
model:
completion_params:
temperature: 0.7
mode: chat
name: qwen3.6-plus
provider: langgenius/tongyi/tongyi
prompt_template:
- id: 9b866a63-3619-4f5c-a46f-0aed04078587
role: system
text: 'User says: {{{#sys.query#}} Reply exactly: echo:{{#sys.query#}}'
selected: false
title: LLM
type: llm
vision:
enabled: false
height: 88
id: llm
position:
x: 380
y: 282
positionAbsolute:
x: 380
y: 282
selected: false
sourcePosition: right
targetPosition: left
type: custom
width: 242
- data:
answer: '{{#llm.text#}}'
answer: 'echo:{{#sys.query#}}'
selected: false
title: 直接回复
title: Reply
type: answer
variables: []
height: 103
id: answer
position:
x: 680
x: 380
y: 282
positionAbsolute:
x: 680
x: 380
y: 282
selected: false
sourcePosition: right

View File

@ -6,12 +6,7 @@ app:
mode: workflow
name: basic_auto_test
use_icon_as_answer_icon: false
dependencies:
- current_identifier: null
type: marketplace
value:
marketplace_plugin_unique_identifier: langgenius/tongyi:0.1.48@966d88dc40611f067311c1c9839139ebc4b55bff471bc5e736dc3e828bc67b46
version: null
dependencies: []
kind: app
version: 0.6.0
workflow:
@ -70,20 +65,9 @@ workflow:
isInIteration: false
isInLoop: false
sourceType: start
targetType: llm
id: 1779097154262-source-1779097204645-target
source: '1779097154262'
sourceHandle: source
target: '1779097204645'
targetHandle: target
type: custom
zIndex: 0
- data:
isInLoop: false
sourceType: llm
targetType: end
id: 1779097204645-source-1779171097399-target
source: '1779097204645'
id: 1779097154262-source-1779171097399-target
source: '1779097154262'
sourceHandle: source
target: '1779171097399'
targetHandle: target
@ -92,7 +76,7 @@ workflow:
nodes:
- data:
selected: true
title: 用户输入
title: User Input
type: start
variables:
- default: ''
@ -143,49 +127,15 @@ workflow:
targetPosition: left
type: custom
width: 242
- data:
context:
enabled: false
variable_selector: []
model:
completion_params:
temperature: 0.7
mode: chat
name: qwen3.6-plus
provider: langgenius/tongyi/tongyi
prompt_template:
- id: 1ddb3202-d84c-4faf-afe3-424eedc9049a
role: system
text: 'User says:{{#1779097154262.x#}}. Reply exactly: echo:{{#1779097154262.x#}}
'
selected: false
title: LLM
type: llm
vision:
enabled: false
height: 88
id: '1779097204645'
position:
x: 382
y: 282
positionAbsolute:
x: 382
y: 282
selected: false
sourcePosition: right
targetPosition: left
type: custom
width: 242
- data:
outputs:
- value_selector:
- '1779097204645'
- text
- '1779097154262'
- x
value_type: string
variable: x
selected: false
title: 输出
title: Output
type: end
height: 88
id: '1779171097399'

View File

@ -6,12 +6,7 @@ app:
mode: advanced-chat
name: DIFY_E2E_FILE_CHAT
use_icon_as_answer_icon: false
dependencies:
- current_identifier: null
type: marketplace
value:
marketplace_plugin_unique_identifier: langgenius/tongyi:0.1.48@966d88dc40611f067311c1c9839139ebc4b55bff471bc5e736dc3e828bc67b46
version: null
dependencies: []
kind: app
version: 0.6.0
workflow:
@ -68,18 +63,9 @@ workflow:
edges:
- data:
sourceType: start
targetType: llm
id: 1780453002656-llm
source: '1780453002656'
sourceHandle: source
target: llm
targetHandle: target
type: custom
- data:
sourceType: llm
targetType: answer
id: llm-answer
source: llm
id: 1780453002656-answer
source: '1780453002656'
sourceHandle: source
target: answer
targetHandle: target
@ -87,7 +73,7 @@ workflow:
nodes:
- data:
selected: false
title: 用户输入
title: User Input
type: start
variables:
- allowed_file_extensions: []
@ -119,60 +105,18 @@ workflow:
type: custom
width: 242
- data:
context:
enabled: false
variable_selector: []
memory:
query_prompt_template: '{{#sys.query#}}
{{#sys.files#}}'
role_prefix:
assistant: ''
user: ''
window:
enabled: false
size: 10
model:
completion_params:
temperature: 0.7
mode: chat
name: qwen3.6-plus
provider: langgenius/tongyi/tongyi
prompt_template:
- id: ebc516ad-be6b-4a78-af32-77f447305b34
role: system
text: 输出固定内容:""hello
answer: ok
selected: false
title: LLM
type: llm
vision:
enabled: false
height: 88
id: llm
position:
x: 380
y: 282
positionAbsolute:
x: 380
y: 282
selected: true
sourcePosition: right
targetPosition: left
type: custom
width: 242
- data:
answer: '{{#llm.text#}}'
selected: false
title: 直接回复
title: Reply
type: answer
variables: []
height: 103
id: answer
position:
x: 680
x: 380
y: 282
positionAbsolute:
x: 680
x: 380
y: 282
selected: false
sourcePosition: right

View File

@ -6,12 +6,7 @@ app:
mode: workflow
name: file_auto_test
use_icon_as_answer_icon: false
dependencies:
- current_identifier: null
type: marketplace
value:
marketplace_plugin_unique_identifier: langgenius/tongyi:0.1.48@966d88dc40611f067311c1c9839139ebc4b55bff471bc5e736dc3e828bc67b46
version: null
dependencies: []
kind: app
version: 0.6.0
workflow:
@ -70,21 +65,9 @@ workflow:
isInIteration: false
isInLoop: false
sourceType: start
targetType: llm
id: 1779693724732-source-1779693759949-target
source: '1779693724732'
sourceHandle: source
target: '1779693759949'
targetHandle: target
type: custom
zIndex: 0
- data:
isInIteration: false
isInLoop: false
sourceType: llm
targetType: end
id: 1779693759949-source-1779693765299-target
source: '1779693759949'
id: 1779693724732-source-1779693765299-target
source: '1779693724732'
sourceHandle: source
target: '1779693765299'
targetHandle: target
@ -93,7 +76,7 @@ workflow:
nodes:
- data:
selected: true
title: 用户输入
title: User Input
type: start
variables:
- allowed_file_extensions: []
@ -140,46 +123,9 @@ workflow:
type: custom
width: 242
- data:
context:
enabled: false
variable_selector: []
model:
completion_params:
temperature: 0.7
mode: chat
name: qwen3.6-plus
provider: langgenius/tongyi/tongyi
prompt_template:
- id: bb929f8f-5fa9-415b-91c3-c30228488dcf
role: system
text: 直接输出内容:hello
outputs: []
selected: false
title: LLM
type: llm
vision:
enabled: false
height: 88
id: '1779693759949'
position:
x: 382
y: 282
positionAbsolute:
x: 382
y: 282
selected: false
sourcePosition: right
targetPosition: left
type: custom
width: 242
- data:
outputs:
- value_selector:
- '1779693759949'
- text
value_type: string
variable: x
selected: false
title: 输出
title: Output
type: end
height: 88
id: '1779693765299'

View File

@ -6,12 +6,7 @@ app:
mode: workflow
name: auto_test_workspace2
use_icon_as_answer_icon: false
dependencies:
- current_identifier: null
type: marketplace
value:
marketplace_plugin_unique_identifier: langgenius/tongyi:0.1.48@966d88dc40611f067311c1c9839139ebc4b55bff471bc5e736dc3e828bc67b46
version: null
dependencies: []
kind: app
version: 0.6.0
workflow:
@ -70,21 +65,9 @@ workflow:
isInIteration: false
isInLoop: false
sourceType: start
targetType: llm
id: 1780305524693-source-1780305526186-target
source: '1780305524693'
sourceHandle: source
target: '1780305526186'
targetHandle: target
type: custom
zIndex: 0
- data:
isInIteration: false
isInLoop: false
sourceType: llm
targetType: end
id: 1780305526186-source-1780305600095-target
source: '1780305526186'
id: 1780305524693-source-1780305600095-target
source: '1780305524693'
sourceHandle: source
target: '1780305600095'
targetHandle: target
@ -93,7 +76,7 @@ workflow:
nodes:
- data:
selected: false
title: 用户输入
title: User Input
type: start
variables: []
height: 73
@ -109,45 +92,9 @@ workflow:
type: custom
width: 242
- data:
context:
enabled: false
variable_selector: []
model:
completion_params:
temperature: 0.7
mode: chat
name: qwen3.6-plus
provider: langgenius/tongyi/tongyi
prompt_template:
- id: cd753cdd-d950-44bf-99ad-7cb19f42d5b6
role: system
text: 输出内容:hello
outputs: []
selected: false
title: LLM
type: llm
vision:
enabled: false
height: 88
id: '1780305526186'
position:
x: 382
y: 282
positionAbsolute:
x: 382
y: 282
sourcePosition: right
targetPosition: left
type: custom
width: 242
- data:
outputs:
- value_selector:
- '1780305526186'
- text
value_type: string
variable: x
selected: false
title: 输出
title: Output
type: end
height: 88
id: '1780305600095'
@ -157,6 +104,7 @@ workflow:
positionAbsolute:
x: 684
y: 282
selected: false
sourcePosition: right
targetPosition: left
type: custom

View File

@ -8,11 +8,13 @@
* withTempConfig) to prevent session state leaking between tests.
*/
import type { TokenDoc } from '@/store/token-store'
import { Buffer } from 'node:buffer'
import { execSync, spawn } from 'node:child_process'
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join, resolve } from 'node:path'
import yaml from 'js-yaml'
/** Path to the dev entry point — no build required. */
export const BIN = resolve(__dirname, '../../../bin/dev.js')
@ -208,13 +210,11 @@ function splitHost(host: string): { bare: string, scheme: string } {
}
async function writeFileToken(configDir: string, host: string, email: string, bearer: string): Promise<void> {
const dotParts = `tokens.${host}.${email}`.split('.')
let yaml = ''
for (let i = 0; i < dotParts.length - 1; i++) {
yaml += `${' '.repeat(i) + dotParts[i]}:\n`
const doc: TokenDoc = {
version: 1,
tokens: { [host]: { [email]: bearer } },
}
yaml += `${' '.repeat(dotParts.length - 1) + (dotParts[dotParts.length - 1] ?? '')}: "${bearer}"\n`
await writeFile(join(configDir, 'tokens.yml'), yaml, { mode: 0o600 })
await writeFile(join(configDir, 'tokens.yml'), yaml.dump(doc, { lineWidth: -1, noRefs: true }), { mode: 0o600 })
}
/**
@ -289,11 +289,8 @@ export async function injectAuth(configDir: string, opts: AuthInjectionOptions):
new Entry('difyctl', account).setPassword(JSON.stringify(opts.bearer))
}
else {
// Fall back to tokens.yml.
// YamlStore.doGet splits the key on '.' and traverses the nested object,
// so "tokens.localhost.user@dify.ai" splits into 4 parts:
// tokens -> localhost -> user@dify -> ai
// The YAML must mirror that exact nesting.
// Fall back to tokens.yml — FileTokenStore uses getTyped<TokenDoc>()
// which expects flat tokens[host][email] with version: 1.
await writeFileToken(configDir, bare, email, opts.bearer)
}
}

View File

@ -349,9 +349,11 @@ describe('E2E / agent skill — describe app -o json (auth required)', () => {
assertPipeFriendlyJson(r)
})
itWithAuth('[P0] nonexistent app → exit 1 + JSON error envelope', async () => {
itWithAuth('[P0] invalid (non-UUID) app id → exit 2 + usage error envelope', async () => {
// 'app-id-nonexistent-e2e-xyz' is not a valid UUID; describe app rejects it
// client-side via isValidUuid() with usage_invalid_flag (exit 2).
const r = await fx.r(['describe', 'app', 'app-id-nonexistent-e2e-xyz', '-o', 'json'])
expect(r.exitCode).toBe(1)
expect(r.exitCode).toBe(2)
assertErrorEnvelope(r)
})
})

View File

@ -131,11 +131,12 @@ describe('E2E / difyctl describe app', () => {
// ── Not found ─────────────────────────────────────────────────────────────
it('[P0] non-existent app returns exit code 1 with not-found error (3.83)', async () => {
// Spec 3.83: describe non-existent app → stderr contains not-found, exit code 1.
it('[P0] invalid (non-UUID) app id returns usage error (exit code 2)', async () => {
// NONEXISTENT_ID is not a valid UUID, so the CLI rejects it client-side via
// isValidUuid() before making any network request → usage_invalid_flag (exit 2).
const result = await fx.r(['describe', 'app', NONEXISTENT_ID])
expect(result.exitCode, 'non-existent app should exit with code 1').toBe(1)
expect(result.stderr).toMatch(/not.?found|404|does not exist|server_5xx/i)
expect(result.exitCode, 'invalid UUID should exit with code 2').toBe(2)
expect(result.stderr).toMatch(/uuid|valid|usage_invalid_flag/i)
})
it('[P1] non-existent app in JSON mode outputs JSON error envelope', async () => {

View File

@ -141,7 +141,7 @@ describe('E2E / error message standards (spec 5.3)', () => {
// Spec 5.70: submitting a value of the wrong type must fail.
// The workflow app (workflowAppId) expects x as a string; passing a JSON
// number causes the server to reject the request.
// In v1.0 the server returns HTTP 500 for type validation failures.
// After the @accepts/@returns contract unification, the server returns HTTP 422 for request schema failures.
const result = await fx.r([
'run',
'app',
@ -156,6 +156,65 @@ describe('E2E / error message standards (spec 5.3)', () => {
expect(result.stderr.trim().length).toBeGreaterThan(0)
})
// ── 5.70a/b/c P4 sanitization — 422 error body is clean (no leaks) ────────
it('[P0] 5.70a validation failure message is a plain string, not double-encoded JSON', async () => {
// After the @accepts contract fix, the server aborts with
// abort(422, message="Request validation failed", errors=[...])
// The CLI wraps this into its envelope. The message field must be a plain
// human-readable string — NOT a JSON-serialised string that itself contains
// pydantic error details (which was the double-encoding bug in P4).
const result = await fx.r([
'run',
'app',
E.workflowAppId,
'--inputs',
JSON.stringify({ x: 'hello', num: 'not-a-number', enum_var: 'A', paragraph: 'ok' }),
'-o',
'json',
])
assertNonZeroExit(result)
const envelope = assertErrorEnvelope(result)
// message must be a plain string, not a JSON string (no double encoding)
expect(typeof envelope.error.message).toBe('string')
expect(() => JSON.parse(envelope.error.message)).toThrow()
})
it('[P1] 5.70b validation error response does not leak pydantic version URL', async () => {
// Before the P4 fix, exc.json() included a "url" field pointing to
// https://errors.pydantic.dev/<version>/... — exposing the server's pydantic
// version. The sanitised response must not contain this URL.
const result = await fx.r([
'run',
'app',
E.workflowAppId,
'--inputs',
JSON.stringify({ x: 'hello', num: 'not-a-number', enum_var: 'A', paragraph: 'ok' }),
'-o',
'json',
])
assertNonZeroExit(result)
expect(result.stderr).not.toMatch(/errors\.pydantic\.dev|pydantic\.dev\//)
})
it('[P1] 5.70c validation error response does not echo back user input', async () => {
// Before the P4 fix, exc.json() included the user's original "input" value
// inside the error details. The sanitised response must not repeat the
// submitted value so that sensitive payloads are not reflected to callers.
const sentValue = 'not-a-number-sentinel-12345'
const result = await fx.r([
'run',
'app',
E.workflowAppId,
'--inputs',
JSON.stringify({ x: 'hello', num: sentValue, enum_var: 'A', paragraph: 'ok' }),
'-o',
'json',
])
assertNonZeroExit(result)
expect(result.stderr).not.toContain(sentValue)
})
// ── 5.76 Failed command + -o yaml → stderr is still JSON envelope ────────
it('[P1] 5.76 failed command with -o yaml still outputs a JSON error envelope on stderr', async () => {

View File

@ -8,7 +8,7 @@
*
* Staging app prerequisites (specified via DIFY_E2E_* env vars):
* echo-chat mode=chat, query variable, outputs "echo: {query}"
* echo-workflow mode=workflow, x variable (required), outputs "echo: {x}"
* echo-workflow mode=workflow, x variable (required), outputs x directly (no echo prefix)
*/
import type { AuthFixture } from '../../helpers/cli.js'
@ -263,7 +263,7 @@ describe('E2E / difyctl run app', () => {
JSON.stringify({ x: 'hello', num: 42, enum_var: 'A', paragraph: 'short text' }),
])
assertExitCode(happyResult, 0)
assertStdoutContains(happyResult, 'echo:hello')
assertStdoutContains(happyResult, 'hello') // workflow outputs x directly; echo: prefix removed (no sandbox on server)
// ── 4.1.17: number field receives a string value ─────────────────────
const typedResult = await fx.r([
@ -289,6 +289,26 @@ describe('E2E / difyctl run app', () => {
})
})
it('[P1] validation failure returns http_status 422 in JSON error envelope', async () => {
// After the @accepts/@returns server contract unification, input schema
// validation failures consistently return HTTP 422 (not 400 or 500).
// This verifies the CLI propagates the unified status code.
const result = await fx.r([
'run',
'app',
E.workflowAppId,
'--inputs',
JSON.stringify({ x: 'hello', num: 'not-a-number', enum_var: 'A', paragraph: 'ok' }),
'-o',
'json',
])
expect(result.exitCode).not.toBe(0)
const envelope = JSON.parse(result.stderr.trim()) as {
error: { code: string, message: string, http_status?: number }
}
expect(envelope.error.http_status, 'validation failure must return http_status 422').toBe(422)
})
// =========================================================================
// Error scenarios
// =========================================================================

View File

@ -118,7 +118,7 @@ describe('E2E / difyctl run app --conversation', () => {
'invalid-conv-id-xyz-not-exist',
])
assertExitCode(result, 1)
expect(result.stderr).toMatch(/not.?found|conversation|404/i)
expect(result.stderr).toMatch(/not.?found|conversation|404|422|validation/i)
})
// ── Combined flags ──────────────────────────────────────────────────────

View File

@ -277,7 +277,7 @@ describe('E2E / difyctl run app --stream (specialisation)', () => {
'--stream',
])
expect(result.exitCode, 'wrong-type input should cause non-zero exit').not.toBe(0)
expect(result.stderr).toMatch(/validation|invalid|type|400|server_5xx|must be/i)
expect(result.stderr).toMatch(/validation|invalid|type|422|must be/i)
})
// ── Non-existent app with positional query (4.2.16) ────────────────────