From 47724ec764fa965c0fd098342f415cad46b846f7 Mon Sep 17 00:00:00 2001 From: CodingOnStar Date: Tue, 16 Dec 2025 14:15:09 +0800 Subject: [PATCH] feat: enhance E2E testing setup with Cloudflare Access support and improved authentication handling - Added support for Cloudflare Access headers in Playwright configuration and teardown. - Updated global setup for E2E tests to validate authentication credentials and handle login more robustly. - Enhanced README with authentication configuration details and supported methods. - Updated Playwright reporter configuration to include JSON output for test results. --- web/e2e/README.md | 22 +++++++ web/e2e/global.setup.ts | 123 ++++++++++++++++++++++++++----------- web/e2e/global.teardown.ts | 17 ++++- web/playwright.config.ts | 20 +++++- web/pnpm-lock.yaml | 5 +- 5 files changed, 146 insertions(+), 41 deletions(-) diff --git a/web/e2e/README.md b/web/e2e/README.md index 2c9219b4c3..7df74e79f5 100644 --- a/web/e2e/README.md +++ b/web/e2e/README.md @@ -28,6 +28,28 @@ E2E_SKIP_WEB_SERVER=true # API URL (optional, defaults to http://localhost:5001/console/api) NEXT_PUBLIC_API_PREFIX=http://localhost:5001/console/api + +# Authentication Configuration +# Test user credentials +NEXT_PUBLIC_E2E_USER_EMAIL=test@example.com +NEXT_PUBLIC_E2E_USER_PASSWORD=your-password +``` + +### Authentication Methods + +Dify supports multiple login methods, but not all are suitable for E2E testing: + +| Method | E2E Support | Configuration | +|--------|-------------|---------------| +| **Email + Password** | ✅ Recommended | Set `NEXT_PUBLIC_E2E_USER_EMAIL` and `NEXT_PUBLIC_E2E_USER_PASSWORD` | + +#### Email + Password (Default) + +The most reliable method for E2E testing. Simply set the credentials: + +```env +NEXT_PUBLIC_E2E_USER_EMAIL=test@example.com +NEXT_PUBLIC_E2E_USER_PASSWORD=your-password ``` ### 3. Run Tests diff --git a/web/e2e/global.setup.ts b/web/e2e/global.setup.ts index ec7001ddcd..63f7134499 100644 --- a/web/e2e/global.setup.ts +++ b/web/e2e/global.setup.ts @@ -4,75 +4,124 @@ import path from 'node:path' const authFile = path.join(__dirname, '.auth/user.json') +/** + * Supported authentication methods for E2E tests + * - password: Email + Password login (default, recommended) + * + * OAuth (GitHub/Google) and SSO are not supported in E2E tests + * as they require third-party authentication which cannot be reliably automated. + */ + /** * Global setup for E2E tests * * This runs before all tests and handles authentication. * The authenticated state is saved and reused across all tests. * - * Based on signin implementation: - * - web/app/signin/components/mail-and-password-auth.tsx + * Environment variables: + * - NEXT_PUBLIC_E2E_USER_EMAIL: Test user email (required) + * - NEXT_PUBLIC_E2E_USER_PASSWORD: Test user password (required for 'password' method) */ setup('authenticate', async ({ page }) => { - // Get test user credentials from environment const email = process.env.NEXT_PUBLIC_E2E_USER_EMAIL const password = process.env.NEXT_PUBLIC_E2E_USER_PASSWORD - if (!email || !password) { + // Validate required credentials based on auth method + if (!email) { console.warn( - '⚠️ NEXT_PUBLIC_E2E_USER_EMAIL or NEXT_PUBLIC_E2E_USER_PASSWORD not set.', + '⚠️ NEXT_PUBLIC_E2E_USER_EMAIL not set.', 'Creating empty auth state. Tests requiring auth will fail.', ) - // Create empty auth state directory if it doesn't exist - const authDir = path.dirname(authFile) - if (!fs.existsSync(authDir)) - fs.mkdirSync(authDir, { recursive: true }) + await saveEmptyAuthState(page) + return + } - // Save empty state - await page.context().storageState({ path: authFile }) + if (!password) { + console.warn( + '⚠️ NEXT_PUBLIC_E2E_USER_PASSWORD not set for password auth method.', + 'Creating empty auth state. Tests requiring auth will fail.', + ) + await saveEmptyAuthState(page) return } // Navigate to login page await page.goto('/signin') - - // Wait for the page to load await page.waitForLoadState('networkidle') - // Fill in login form using actual Dify selectors + // Execute login + await loginWithPassword(page, email, password!) + + // Wait for successful redirect to /apps + await expect(page).toHaveURL(/\/apps/, { timeout: 30000 }) + + // Save authenticated state + await page.context().storageState({ path: authFile }) + console.log('✅ Authentication successful, state saved.') +}) + +/** + * Save empty auth state when credentials are not available + */ +async function saveEmptyAuthState(page: import('@playwright/test').Page): Promise { + const authDir = path.dirname(authFile) + if (!fs.existsSync(authDir)) + fs.mkdirSync(authDir, { recursive: true }) + await page.context().storageState({ path: authFile }) +} + +/** + * Login using email and password + * Based on: web/app/signin/components/mail-and-password-auth.tsx + */ +async function loginWithPassword( + page: import('@playwright/test').Page, + email: string, + password: string, +): Promise { + console.log('📧 Logging in with email and password...') + + // Fill in login form // Email input has id="email" await page.locator('#email').fill(email) // Password input has id="password" await page.locator('#password').fill(password) // Wait for button to be enabled (form validation passes) - const signInButton = page.getByRole('button', { name: 'Sign in' }) + const signInButton = page.getByRole('button', { name: /sign in/i }) await expect(signInButton).toBeEnabled({ timeout: 5000 }) - // Click login button and wait for the login API response - const [response] = await Promise.all([ - page.waitForResponse(resp => - resp.url().includes('/login') && resp.request().method() === 'POST', - ), - signInButton.click(), - ]) + // Click login button and wait for navigation or API response + // The app uses ky library which follows redirects automatically + // Some environments may have WAF/CDN that adds extra redirects + // So we use a more flexible approach: wait for either URL change or API response + const responsePromise = page.waitForResponse( + resp => resp.url().includes('login') && resp.request().method() === 'POST', + { timeout: 15000 }, + ).catch(() => null) // Don't fail if we can't catch the response - // Check if login request was successful - const status = response.status() - if (status === 200) { - // Redirect response means login successful (server-side redirect) - console.log('✅ Login successful (redirect response)') - // Wait for navigation to complete (redirect to /apps) - // See: mail-and-password-auth.tsx line 71 - router.replace(redirectUrl || '/apps') - await expect(page).toHaveURL(/\/apps/, { timeout: 30000 }) + await signInButton.click() + + // Try to get the response, but don't fail if we can't + const response = await responsePromise + if (response) { + const status = response.status() + console.log(`📡 Login API response status: ${status}`) + // 200 = success, 302 = redirect (some WAF/CDN setups) + if (status !== 200 && status !== 302) { + // Try to get error details + try { + const body = await response.json() + console.error('❌ Login failed:', body) + } + catch { + console.error(`❌ Login failed with status ${status}`) + } + } } else { - // Other status codes indicate failure - throw new Error(`Login request failed with status ${status}`) + console.log('⚠️ Could not capture login API response, will verify via URL redirect') } - // Save authenticated state - await page.context().storageState({ path: authFile }) - - console.log('✅ Authentication successful, state saved.') -}) + console.log('✅ Password login request sent') +} diff --git a/web/e2e/global.teardown.ts b/web/e2e/global.teardown.ts index c4187b86ac..5d1823091b 100644 --- a/web/e2e/global.teardown.ts +++ b/web/e2e/global.teardown.ts @@ -20,6 +20,17 @@ import { request, test as teardown } from '@playwright/test' // Ensure baseURL ends with '/' for proper path concatenation const API_BASE_URL = (process.env.NEXT_PUBLIC_API_PREFIX || 'http://localhost:5001/console/api').replace(/\/?$/, '/') +// Cloudflare Access headers (for protected environments). +// Prefer environment variables to avoid hardcoding secrets in repo. +const CF_ACCESS_CLIENT_ID = process.env.CF_ACCESS_CLIENT_ID +const CF_ACCESS_CLIENT_SECRET = process.env.CF_ACCESS_CLIENT_SECRET + +const cfAccessHeaders: Record = {} +if (CF_ACCESS_CLIENT_ID && CF_ACCESS_CLIENT_SECRET) { + cfAccessHeaders['CF-Access-Client-Id'] = CF_ACCESS_CLIENT_ID + cfAccessHeaders['CF-Access-Client-Secret'] = CF_ACCESS_CLIENT_SECRET +} + // Test data prefixes - used to identify test-created data // Should match the prefix used in generateTestId() const TEST_DATA_PREFIXES = ['e2e-', 'test-'] @@ -87,7 +98,10 @@ teardown('cleanup test data', async () => { return } // Extract CSRF token from cookies for API requests - const csrfCookie = authState.cookies.find((c: { name: string }) => c.name === 'csrf_token') + // Cookie name may be 'csrf_token' or '__Host-csrf_token' depending on environment + const csrfCookie = authState.cookies.find((c: { name: string }) => + c.name === 'csrf_token' || c.name === '__Host-csrf_token', + ) csrfToken = csrfCookie?.value || '' } catch { @@ -103,6 +117,7 @@ teardown('cleanup test data', async () => { storageState: authPath, extraHTTPHeaders: { 'X-CSRF-Token': csrfToken, + ...cfAccessHeaders, }, }) diff --git a/web/playwright.config.ts b/web/playwright.config.ts index 565e620b26..3c1c8a68fe 100644 --- a/web/playwright.config.ts +++ b/web/playwright.config.ts @@ -25,6 +25,17 @@ const BASE_URL = process.env.E2E_BASE_URL || 'http://localhost:3000' // - CI/CD with deployed env: true (use existing server) const SKIP_WEB_SERVER = process.env.E2E_SKIP_WEB_SERVER === 'true' +// Cloudflare Access headers (for protected environments). +// Prefer environment variables to avoid hardcoding secrets in repo. +const CF_ACCESS_CLIENT_ID = process.env.CF_ACCESS_CLIENT_ID +const CF_ACCESS_CLIENT_SECRET = process.env.CF_ACCESS_CLIENT_SECRET + +const cfAccessHeaders: Record = {} +if (CF_ACCESS_CLIENT_ID && CF_ACCESS_CLIENT_SECRET) { + cfAccessHeaders['CF-Access-Client-Id'] = CF_ACCESS_CLIENT_ID + cfAccessHeaders['CF-Access-Client-Secret'] = CF_ACCESS_CLIENT_SECRET +} + export default defineConfig({ // Directory containing test files testDir: './e2e/tests', @@ -43,7 +54,7 @@ export default defineConfig({ // Reporter to use reporter: process.env.CI - ? [['html', { open: 'never' }], ['github']] + ? [['html', { open: 'never', outputFolder: 'playwright-report' }], ['github'], ['json', { outputFile: 'e2e/test-results/results.json' }]] : [['html', { open: 'on-failure' }], ['list']], // Shared settings for all the projects below @@ -51,6 +62,13 @@ export default defineConfig({ // Base URL for all page.goto() calls baseURL: BASE_URL, + // Extra headers for all requests made by the browser context. + extraHTTPHeaders: cfAccessHeaders, + + // Bypass Content Security Policy to allow test automation + // This is needed when testing against environments with strict CSP headers + bypassCSP: true, + // Collect trace when retrying the failed test trace: 'on-first-retry', diff --git a/web/pnpm-lock.yaml b/web/pnpm-lock.yaml index 8b23d20d92..8e2d34148a 100644 --- a/web/pnpm-lock.yaml +++ b/web/pnpm-lock.yaml @@ -386,6 +386,9 @@ importers: '@next/mdx': specifier: 15.5.9 version: 15.5.9(@mdx-js/loader@3.1.1(webpack@5.103.0(esbuild@0.25.0)(uglify-js@3.19.3)))(@mdx-js/react@3.1.1(@types/react@19.2.7)(react@19.2.3)) + '@playwright/test': + specifier: ^1.56.1 + version: 1.57.0 '@rgrove/parse-xml': specifier: ^4.2.0 version: 4.2.0 @@ -6899,7 +6902,6 @@ packages: next@15.5.9: resolution: {integrity: sha512-agNLK89seZEtC5zUHwtut0+tNrc0Xw4FT/Dg+B/VLEo9pAcS9rtTKpek3V6kVcVwsB2YlqMaHdfZL4eLEVYuCg==} engines: {node: ^18.18.0 || ^19.8.0 || >= 20.0.0} - deprecated: This version has a security vulnerability. Please upgrade to a patched version. See https://nextjs.org/blog/security-update-2025-12-11 for more details. hasBin: true peerDependencies: '@opentelemetry/api': ^1.1.0 @@ -11463,7 +11465,6 @@ snapshots: '@playwright/test@1.57.0': dependencies: playwright: 1.57.0 - optional: true '@pmmmwh/react-refresh-webpack-plugin@0.5.17(react-refresh@0.14.2)(type-fest@4.2.0)(webpack-hot-middleware@2.26.1)(webpack@5.103.0(esbuild@0.25.0)(uglify-js@3.19.3))': dependencies: