mirror of
https://github.com/langgenius/dify.git
synced 2026-06-10 18:24:09 +08:00
198 lines
8.6 KiB
TypeScript
198 lines
8.6 KiB
TypeScript
/**
|
|
* E2E: Exit Code standards — spec 5.4
|
|
*
|
|
* Exit code contract:
|
|
* 0 — success (also: --help, version, empty command)
|
|
* 1 — server / resource error (not_found, server_5xx, network)
|
|
* 2 — usage / argument error (unknown flag, invalid value, missing arg)
|
|
* 4 — authentication error (not_logged_in, token expired)
|
|
* 6 — config schema error (config parse failure, unsupported version)
|
|
*
|
|
* Already covered in other suites (not duplicated here):
|
|
* 5.90 success exit 0 → all passing tests in every suite
|
|
* 5.91 usage error exit 2 → get-app-list (--limit 0/201), run-app-basic
|
|
* 5.92 app not found exit 1 → get-app-single
|
|
* 5.93 auth error exit 4 → get-app-list, auth suites, run-app-basic
|
|
* 5.94 insufficient_scope → get-app-list (SSO guard)
|
|
* 5.96 network error → get-app-list, get-app-single, devices
|
|
* 5.98 Ctrl+C streaming → run-app-streaming.e2e.ts
|
|
* 5.99 Ctrl+C streaming → run-app-streaming.e2e.ts
|
|
* 5.100 server 500 exit 1 → get-app-single, error-messages
|
|
* 5.101 invalid input exit 2/1 → run-app-basic (many cases)
|
|
* 5.109 unknown command exit 1 → error-handling/error-messages.e2e.ts (5.59)
|
|
* 5.111 failed stdout empty → run-app-basic, get-app-list (many)
|
|
*
|
|
* Non-automatable cases (excluded):
|
|
* 5.97 timeout exit — cannot reliably control request timeout
|
|
* 5.102 file upload failure — hard to trigger non-ENOENT upload failure
|
|
* 5.103 workflow node failure — no stable staging fixture
|
|
* 5.115 shell stays healthy after failure — needs real shell context
|
|
* 5.116 crash exit — cannot reliably trigger CLI crash
|
|
* 5.117 panic output — cannot reliably trigger panic
|
|
*/
|
|
|
|
import type { AuthFixture } from '../../helpers/cli.js'
|
|
import { writeFile } from 'node:fs/promises'
|
|
import { join } from 'node:path'
|
|
import { afterEach, beforeEach, describe, expect, inject, it } from 'vitest'
|
|
import { assertExitCode, assertNonZeroExit } from '../../helpers/assert.js'
|
|
import { run, withAuthFixture, withTempConfig } from '../../helpers/cli.js'
|
|
import { resolveEnv } from '../../setup/env.js'
|
|
|
|
// @ts-expect-error — see test/e2e/helpers/vitest-context.ts for explanation
|
|
const caps = inject('e2eCapabilities') as import('../../setup/env.js').E2ECapabilities
|
|
const E = resolveEnv(caps)
|
|
|
|
describe('E2E / exit code standards (spec 5.4)', () => {
|
|
let fx: AuthFixture
|
|
|
|
beforeEach(async () => {
|
|
fx = await withAuthFixture(E)
|
|
})
|
|
afterEach(async () => {
|
|
await fx.cleanup()
|
|
})
|
|
|
|
// ── 5.95 Corrupt config → exit 6 ─────────────────────────────────────────
|
|
|
|
it('[P0] 5.95 corrupt config.yml causes a non-zero exit (exit 6 — config_schema_unsupported)', async () => {
|
|
// Spec 5.95: when config.yml contains invalid YAML the CLI must exit with
|
|
// a non-zero code. In practice the CLI exits 6 (config_schema_unsupported).
|
|
const corruptTmp = await withTempConfig()
|
|
try {
|
|
await writeFile(
|
|
join(corruptTmp.configDir, 'config.yml'),
|
|
': broken yaml [[[',
|
|
{ mode: 0o600 },
|
|
)
|
|
const result = await run(['config', 'get', 'defaults.format'], {
|
|
configDir: corruptTmp.configDir,
|
|
})
|
|
expect(result.exitCode).toBe(6)
|
|
}
|
|
finally {
|
|
await corruptTmp.cleanup()
|
|
}
|
|
})
|
|
|
|
// ── 5.104 Failed + -o json exit code (WTA-249) ───────────────────────────
|
|
|
|
it('[P0] 5.104 failed command with -o json returns non-zero exit — documents WTA-249 known defect', async () => {
|
|
// Spec 5.104: a failed command with -o json must return a non-zero exit code.
|
|
// WTA-249 has been fixed: app not found now correctly returns exit 1.
|
|
//
|
|
// Scenario: get app with a non-existent UUID + -o json → server_4xx_other
|
|
const result = await fx.r([
|
|
'get',
|
|
'app',
|
|
'00000000-0000-0000-0000-000000000000',
|
|
'-o',
|
|
'json',
|
|
])
|
|
// WTA-249 has been fixed in the current build: 4xx with -o json now
|
|
// correctly returns exit 1.
|
|
expect(result.exitCode, 'app not found with -o json must exit 1 (WTA-249 fixed)').toBe(1)
|
|
// stderr must still contain the JSON error envelope
|
|
expect(result.stderr).toMatch(/app not found|server_4xx|error/i)
|
|
})
|
|
|
|
// ── 5.105 Failed + -o yaml exit code ────────────────────────────────────
|
|
|
|
it('[P1] 5.105 failed command with -o yaml returns a non-zero exit code', async () => {
|
|
// Spec 5.105: -o yaml on a failing command must not swallow the exit code.
|
|
const unauthTmp = await withTempConfig()
|
|
try {
|
|
const result = await run(['get', 'app', '-o', 'yaml'], {
|
|
configDir: unauthTmp.configDir,
|
|
})
|
|
assertNonZeroExit(result)
|
|
}
|
|
finally {
|
|
await unauthTmp.cleanup()
|
|
}
|
|
})
|
|
|
|
// ── 5.106 --help exit 0 ──────────────────────────────────────────────────
|
|
|
|
it('[P1] 5.106 difyctl --help exits with code 0', async () => {
|
|
// Spec 5.106: help output must not be treated as an error.
|
|
const result = await fx.r(['--help'])
|
|
assertExitCode(result, 0)
|
|
})
|
|
|
|
// ── 5.107 version exit 0 ─────────────────────────────────────────────────
|
|
|
|
it('[P1] 5.107 difyctl version exits with code 0', async () => {
|
|
// Spec 5.107: --version does not exist; the correct command is "version".
|
|
const result = await fx.r(['version'])
|
|
assertExitCode(result, 0)
|
|
})
|
|
|
|
// ── 5.108 Empty command exit 0 ───────────────────────────────────────────
|
|
|
|
it('[P1] 5.108 difyctl with no arguments exits with code 0 (displays help)', async () => {
|
|
// Spec 5.108: running difyctl without arguments prints help and exits 0.
|
|
const result = await fx.r([])
|
|
assertExitCode(result, 0)
|
|
// Must print some usage/command output
|
|
expect(result.stdout.length).toBeGreaterThan(0)
|
|
})
|
|
|
|
// ── 5.112 Successful command stderr is empty ─────────────────────────────
|
|
|
|
it('[P1] 5.112 a successful query command produces no stderr output', async () => {
|
|
// Spec 5.112: on success stderr must be empty (no spurious warnings).
|
|
// Using get app -o json --limit 1 which has no hint or side-channel output.
|
|
const result = await fx.r(['get', 'app', '-o', 'json', '--limit', '1'])
|
|
assertExitCode(result, 0)
|
|
expect(result.stderr.trim(), 'stderr must be empty on successful query').toBe('')
|
|
})
|
|
|
|
// ── 5.113 Repeated identical failure → consistent exit code ──────────────
|
|
|
|
it('[P1] 5.113 repeated identical failure commands return the same exit code each time', async () => {
|
|
// Spec 5.113: exit codes must be deterministic — the same error condition
|
|
// must always produce the same exit code.
|
|
const unauthTmp = await withTempConfig()
|
|
try {
|
|
const r1 = await run(['get', 'app'], { configDir: unauthTmp.configDir })
|
|
const r2 = await run(['get', 'app'], { configDir: unauthTmp.configDir })
|
|
expect(r1.exitCode).toBe(r2.exitCode)
|
|
expect(r1.exitCode).not.toBe(0)
|
|
}
|
|
finally {
|
|
await unauthTmp.cleanup()
|
|
}
|
|
})
|
|
|
|
// ── 5.114 Exit code classification ──────────────────────────────────────
|
|
|
|
it('[P1] 5.114 exit codes follow the classification: usage=2, auth=4, server=1', async () => {
|
|
// Spec 5.114: the three main exit code classes must be distinct and correct.
|
|
|
|
// Class 2 — usage/argument error
|
|
const usageResult = await fx.r(['get', 'app', '-o', 'table'])
|
|
expect(usageResult.exitCode, 'illegal -o value must exit 2').toBe(2)
|
|
|
|
// Class 4 — authentication error
|
|
const unauthTmp = await withTempConfig()
|
|
let authExitCode: number
|
|
try {
|
|
const authResult = await run(['get', 'app'], { configDir: unauthTmp.configDir })
|
|
authExitCode = authResult.exitCode
|
|
}
|
|
finally {
|
|
await unauthTmp.cleanup()
|
|
}
|
|
expect(authExitCode!, 'not_logged_in must exit 4').toBe(4)
|
|
|
|
// Class 1 — server/resource error (not_found, server_5xx, network)
|
|
const serverResult = await fx.r([
|
|
'use',
|
|
'workspace',
|
|
'nonexistent-workspace-id-xyz',
|
|
])
|
|
expect(serverResult.exitCode, 'workspace not found must exit 1').toBe(1)
|
|
})
|
|
})
|