/** * E2E: difyctl run app — separated-mode reasoning (PR #37460) * * Exercises the out-of-band `reasoning_chunk` SSE channel against a real server. * Requires a chatflow whose LLM node uses reasoning_format=separated AND a * workspace with a configured chat model. The whole suite is skipped unless * DIFY_E2E_REASONING_APP_ID resolves (set it directly, or provision the * reasoning-chat.yml fixture with DIFY_E2E_REASONING_PROVISION=1). * * Verifies the client adaptation: * - --think surfaces the separated reasoning to stderr, framed as * - the answer (stdout) stays free of * - -o json persists the reasoning under metadata.reasoning * - without --think, reasoning stays hidden */ import type { AuthFixture } from '../../helpers/cli.js' import { afterEach, beforeEach, describe, expect, inject } from 'vitest' import { assertExitCode, assertJson, assertStderrContains } from '../../helpers/assert.js' import { registerConversation } from '../../helpers/cleanup-registry.js' import { withAuthFixture } from '../../helpers/cli.js' import { withRetry } from '../../helpers/retry.js' import { optionalIt } from '../../helpers/skip.js' import { resolveEnv } from '../../setup/env.js' // @ts-expect-error — see test/e2e/helpers/vitest-context.ts for explanation const caps = inject('e2eCapabilities') as import('../../setup/env.js').E2ECapabilities const E = resolveEnv(caps) // Skipped unless a separated-reasoning chatflow is wired up (needs a real model). const reasoningIt = optionalIt(Boolean(E.reasoningAppId)) const QUERY = 'In one short sentence, why is the sky blue?' describe('E2E / difyctl run app — separated reasoning', () => { let fx: AuthFixture beforeEach(async () => { fx = await withAuthFixture(E) }) afterEach(async () => { await fx.cleanup() }) reasoningIt( '[P1] --think --stream surfaces reasoning on stderr, clean answer on stdout', async () => { const result = await withRetry( () => fx.r(['run', 'app', E.reasoningAppId, QUERY, '--think', '--stream']), { attempts: 3, delayMs: 1000 }, ) assertExitCode(result, 0) expect(result.stdout.trim().length).toBeGreaterThan(0) // Separated mode keeps the answer free of ; reasoning is framed on stderr. expect(result.stdout).not.toContain('') assertStderrContains(result, '') }, ) reasoningIt('[P1] --think -o json persists reasoning under metadata.reasoning', async () => { const result = await withRetry( () => fx.r(['run', 'app', E.reasoningAppId, QUERY, '--think', '-o', 'json']), { attempts: 3, delayMs: 1000 }, ) assertExitCode(result, 0) const parsed = assertJson<{ conversation_id?: string answer: string metadata?: { reasoning?: Record } }>(result) if (parsed.conversation_id) registerConversation(E.host, E.token, E.reasoningAppId, parsed.conversation_id) const reasoning = parsed.metadata?.reasoning ?? {} expect(Object.keys(reasoning).length).toBeGreaterThan(0) expect(Object.values(reasoning).join('').length).toBeGreaterThan(0) // --think also echoes the separated reasoning to stderr. assertStderrContains(result, '') }) reasoningIt('[P1] without --think, reasoning stays hidden', async () => { const result = await withRetry( () => fx.r(['run', 'app', E.reasoningAppId, QUERY, '--stream']), { attempts: 3, delayMs: 1000 }, ) assertExitCode(result, 0) expect(result.stdout.trim().length).toBeGreaterThan(0) expect(result.stderr).not.toContain('') }) })