From d5cfb26db60a6d310f6520adee464e80e613f518 Mon Sep 17 00:00:00 2001 From: Joel Date: Thu, 2 Jan 2025 16:28:57 +0800 Subject: [PATCH] feat: support make retry data --- .../format-log/graph-to-log-struct.spec.ts | 28 ++++ .../utils/format-log/graph-to-log-struct.ts | 122 ++++++++++++++++ .../run/utils/format-log/index.backup.ts | 105 -------------- .../run/utils/format-log/retry/data.ts | 133 ------------------ .../run/utils/format-log/retry/index.spec.ts | 16 ++- .../format-log/simple-graph-to-log-struct.ts | 14 -- 6 files changed, 163 insertions(+), 255 deletions(-) create mode 100644 web/app/components/workflow/run/utils/format-log/graph-to-log-struct.spec.ts create mode 100644 web/app/components/workflow/run/utils/format-log/graph-to-log-struct.ts delete mode 100644 web/app/components/workflow/run/utils/format-log/index.backup.ts delete mode 100644 web/app/components/workflow/run/utils/format-log/retry/data.ts delete mode 100644 web/app/components/workflow/run/utils/format-log/simple-graph-to-log-struct.ts diff --git a/web/app/components/workflow/run/utils/format-log/graph-to-log-struct.spec.ts b/web/app/components/workflow/run/utils/format-log/graph-to-log-struct.spec.ts new file mode 100644 index 0000000000..8a783c9644 --- /dev/null +++ b/web/app/components/workflow/run/utils/format-log/graph-to-log-struct.spec.ts @@ -0,0 +1,28 @@ +import graphToLogStruct, { parseNodeString } from './graph-to-log-struct' + +describe('graphToLogStruct', () => { + test('parseNodeString', () => { + expect(parseNodeString('(node1, param1, (node2, param2, (node3, param1)), param4)')).toEqual({ + node: 'node1', + params: [ + 'param1', + { + node: 'node2', + params: [ + 'param2', + { + node: 'node3', + params: [ + 'param1', + ], + }, + ], + }, + 'param4', + ], + }) + }) + test('retry nodes', () => { + console.log(graphToLogStruct('start -> (retry, 1, 3)')) + }) +}) diff --git a/web/app/components/workflow/run/utils/format-log/graph-to-log-struct.ts b/web/app/components/workflow/run/utils/format-log/graph-to-log-struct.ts new file mode 100644 index 0000000000..faa16d53c6 --- /dev/null +++ b/web/app/components/workflow/run/utils/format-log/graph-to-log-struct.ts @@ -0,0 +1,122 @@ +const STEP_SPLIT = '->' + +const toNodeData = (step: string, info: Record = {}): any => { + const [nodeId, title] = step.split('@') + const data = { + id: nodeId, + node_id: nodeId, + title: title || nodeId, + execution_metadata: {}, + status: 'succeeded', + } + // const executionMetadata = data.execution_metadata + const { isRetry } = info + if (isRetry) + data.status = 'retry' + + return data +} + +const toRetryNodeData = ({ + nodeId, + repeatTimes, +}: { + nodeId: string, + repeatTimes: number, +}): any => { + const res = [toNodeData(nodeId)] + for (let i = 0; i < repeatTimes; i++) + res.push(toNodeData(nodeId, { isRetry: true })) + return res +} + +type NodeStructure = { + node: string; + params: Array; +} + +export function parseNodeString(input: string): NodeStructure { + input = input.trim() + if (input.startsWith('(') && input.endsWith(')')) + input = input.slice(1, -1) + + const parts: Array = [] + let current = '' + let depth = 0 + + for (let i = 0; i < input.length; i++) { + const char = input[i] + + if (char === '(') + depth++ + else if (char === ')') + depth-- + + if (char === ',' && depth === 0) { + parts.push(current.trim()) + current = '' + } + else { + current += char + } + } + + if (current) + parts.push(current.trim()) + + const result: NodeStructure = { + node: '', + params: [], + } + + for (let i = 0; i < parts.length; i++) { + const part = parts[i] + + if (typeof part === 'string' && part.startsWith('(')) + result.params.push(parseNodeString(part)) + else if (i === 0) + result.node = part as string + else + result.params.push(part as string) + } + + return result +} + +const toNodes = (input: string): any[] => { + const list = input.split(STEP_SPLIT) + .map(step => step.trim()) + + const res: any[] = [] + list.forEach((step) => { + const isPlainStep = !step.includes('(') + if (isPlainStep) { + res.push(toNodeData(step)) + return + } + + const { node, params } = parseNodeString(step) + switch (node) { + case 'retry': + res.push(...toRetryNodeData({ + nodeId: params[0] as string, + repeatTimes: Number.parseInt(params[1] as string), + })) + break + } + }) + return res +} + +/* +* : 1 -> 2 -> 3 +* iteration: (iteration, 1, [2, 3]) -> 4. (1, [2, 3]) means 1 is parent, [2, 3] is children +* parallel: 1 -> (parallel, [1,2,3], [4, (parallel: (6,7))]). +* retry: (retry, 1, 3). 1 is parent, 3 is retry times +*/ +const graphToLogStruct = (input: string): any[] => { + const list = toNodes(input) + return list +} + +export default graphToLogStruct diff --git a/web/app/components/workflow/run/utils/format-log/index.backup.ts b/web/app/components/workflow/run/utils/format-log/index.backup.ts deleted file mode 100644 index f35e029490..0000000000 --- a/web/app/components/workflow/run/utils/format-log/index.backup.ts +++ /dev/null @@ -1,105 +0,0 @@ -import type { NodeTracing } from '@/types/workflow' -import { BlockEnum } from '../../../types' - -type IterationNodeId = string -type RunIndex = string -type IterationGroupMap = Map> - -const processIterationNode = (item: NodeTracing) => { - return { - ...item, - details: [], // to add the sub nodes in the iteration - } -} - -const updateParallelModeGroup = (nodeGroupMap: IterationGroupMap, runIndex: string, item: NodeTracing, iterationNode: NodeTracing) => { - if (!nodeGroupMap.has(iterationNode.node_id)) - nodeGroupMap.set(iterationNode.node_id, new Map()) - - const groupMap = nodeGroupMap.get(iterationNode.node_id)! - - if (!groupMap.has(runIndex)) - groupMap.set(runIndex, [item]) - - else - groupMap.get(runIndex)!.push(item) - - if (item.status === 'failed') { - iterationNode.status = 'failed' - iterationNode.error = item.error - } - - iterationNode.details = Array.from(groupMap.values()) -} - -const updateSequentialModeGroup = (runIndex: number, item: NodeTracing, iterationNode: NodeTracing) => { - const { details } = iterationNode - if (details) { - if (!details[runIndex]) - details[runIndex] = [item] - else - details[runIndex].push(item) - } - - if (item.status === 'failed') { - iterationNode.status = 'failed' - iterationNode.error = item.error - } -} - -const addRetryDetail = (result: NodeTracing[], item: NodeTracing) => { - const retryNode = result.find(node => node.node_id === item.node_id) - - if (retryNode) { - if (retryNode?.retryDetail) - retryNode.retryDetail.push(item) - else - retryNode.retryDetail = [item] - } -} - -const processNonIterationNode = (result: NodeTracing[], nodeGroupMap: IterationGroupMap, item: NodeTracing) => { - const { execution_metadata } = item - if (!execution_metadata?.iteration_id) { - if (item.status === 'retry') { - addRetryDetail(result, item) - return - } - result.push(item) - return - } - - const parentIterationNode = result.find(node => node.node_id === execution_metadata.iteration_id) - const isInIteration = !!parentIterationNode && Array.isArray(parentIterationNode.details) - if (!isInIteration) - return - - // the parallel in the iteration in mode. - const { parallel_mode_run_id, iteration_index = 0 } = execution_metadata - const isInParallel = !!parallel_mode_run_id - - if (isInParallel) - updateParallelModeGroup(nodeGroupMap, parallel_mode_run_id, item, parentIterationNode) - else - updateSequentialModeGroup(iteration_index, item, parentIterationNode) -} - -// list => tree. Put the iteration node's children into the details field. -const formatToTracingNodeList = (list: NodeTracing[]) => { - const allItems = [...list].reverse() - const result: NodeTracing[] = [] - const iterationGroupMap = new Map>() - - allItems.forEach((item) => { - item.node_type === BlockEnum.Iteration - ? result.push(processIterationNode(item)) - : processNonIterationNode(result, iterationGroupMap, item) - }) - - // console.log(allItems) - // console.log(result) - - return result -} - -export default formatToTracingNodeList diff --git a/web/app/components/workflow/run/utils/format-log/retry/data.ts b/web/app/components/workflow/run/utils/format-log/retry/data.ts deleted file mode 100644 index e22c8b8982..0000000000 --- a/web/app/components/workflow/run/utils/format-log/retry/data.ts +++ /dev/null @@ -1,133 +0,0 @@ -export const simpleRetryData = (() => { - const startNode = { - id: 'f7938b2b-77cd-43f0-814c-2f0ade7cbc60', - index: 1, - predecessor_node_id: null, - node_id: '1735112903395', - node_type: 'start', - title: 'Start', - inputs: { - 'sys.files': [], - 'sys.user_id': '6d8ad01f-edf9-43a6-b863-a034b1828ac7', - 'sys.app_id': '6180ead7-2190-4a61-975c-ec3bf29653da', - 'sys.workflow_id': 'eef6da45-244b-4c79-958e-f3573f7c12bb', - 'sys.workflow_run_id': 'fc8970ef-1406-484e-afde-8567dc22f34c', - }, - process_data: null, - outputs: { - 'sys.files': [], - 'sys.user_id': '6d8ad01f-edf9-43a6-b863-a034b1828ac7', - 'sys.app_id': '6180ead7-2190-4a61-975c-ec3bf29653da', - 'sys.workflow_id': 'eef6da45-244b-4c79-958e-f3573f7c12bb', - 'sys.workflow_run_id': 'fc8970ef-1406-484e-afde-8567dc22f34c', - }, - status: 'succeeded', - error: null, - elapsed_time: 0.008715, - execution_metadata: null, - extras: {}, - created_at: 1735112940, - created_by_role: 'account', - created_by_account: { - id: '6d8ad01f-edf9-43a6-b863-a034b1828ac7', - name: '九彩拼盘', - email: 'iamjoel007@gmail.com', - }, - created_by_end_user: null, - finished_at: 1735112940, - } - - const httpNode = { - id: '50220407-3420-4ad4-89da-c6959710d1aa', - index: 2, - predecessor_node_id: '1735112903395', - node_id: '1735112908006', - node_type: 'http-request', - title: 'HTTP Request', - inputs: null, - process_data: { - request: 'GET / HTTP/1.1\r\nHost: 404\r\n\r\n', - }, - outputs: null, - status: 'failed', - error: 'timed out', - elapsed_time: 30.247757, - execution_metadata: null, - extras: {}, - created_at: 1735112940, - created_by_role: 'account', - created_by_account: { - id: '6d8ad01f-edf9-43a6-b863-a034b1828ac7', - name: '九彩拼盘', - email: 'iamjoel007@gmail.com', - }, - created_by_end_user: null, - finished_at: 1735112970, - } - - const retry1 = { - id: 'ed352b36-27fb-49c6-9e8f-cc755bfc25fc', - index: 3, - predecessor_node_id: '1735112903395', - node_id: '1735112908006', - node_type: 'http-request', - title: 'HTTP Request', - inputs: null, - process_data: null, - outputs: null, - status: 'retry', - error: 'timed out', - elapsed_time: 10.011833, - execution_metadata: { - iteration_id: null, - parallel_mode_run_id: null, - }, - extras: {}, - created_at: 1735112940, - created_by_role: 'account', - created_by_account: { - id: '6d8ad01f-edf9-43a6-b863-a034b1828ac7', - name: '九彩拼盘', - email: 'iamjoel007@gmail.com', - }, - created_by_end_user: null, - finished_at: 1735112950, - } - - const retry2 = { - id: '74dfb3d3-dacf-44f2-8784-e36bfa2d6c4e', - index: 4, - predecessor_node_id: '1735112903395', - node_id: '1735112908006', - node_type: 'http-request', - title: 'HTTP Request', - inputs: null, - process_data: null, - outputs: null, - status: 'retry', - error: 'timed out', - elapsed_time: 10.010368, - execution_metadata: { - iteration_id: null, - parallel_mode_run_id: null, - }, - extras: {}, - created_at: 1735112950, - created_by_role: 'account', - created_by_account: { - id: '6d8ad01f-edf9-43a6-b863-a034b1828ac7', - name: '九彩拼盘', - email: 'iamjoel007@gmail.com', - }, - created_by_end_user: null, - finished_at: 1735112960, - } - - return { - in: [startNode, httpNode, retry1, retry2], - expect: [startNode, { - ...httpNode, - retryDetail: [retry1, retry2], - }], - } -})() diff --git a/web/app/components/workflow/run/utils/format-log/retry/index.spec.ts b/web/app/components/workflow/run/utils/format-log/retry/index.spec.ts index 5ae6c385fd..099b987843 100644 --- a/web/app/components/workflow/run/utils/format-log/retry/index.spec.ts +++ b/web/app/components/workflow/run/utils/format-log/retry/index.spec.ts @@ -1,11 +1,21 @@ import format from '.' -import { simpleRetryData } from './data' +import graphToLogStruct from '../graph-to-log-struct' describe('retry', () => { + // retry nodeId:1 3 times. + const steps = graphToLogStruct('start -> (retry, 1, 3)') + const [startNode, retryNode, ...retryDetail] = steps + const result = format(steps) test('should have no retry status nodes', () => { - expect(format(simpleRetryData.in as any).find(item => (item as any).status === 'retry')).toBeUndefined() + expect(result.find(item => (item as any).status === 'retry')).toBeUndefined() }) test('should put retry nodes in retryDetail', () => { - expect(format(simpleRetryData.in as any)).toEqual(simpleRetryData.expect) + expect(result).toEqual([ + startNode, + { + ...retryNode, + retryDetail, + }, + ]) }) }) diff --git a/web/app/components/workflow/run/utils/format-log/simple-graph-to-log-struct.ts b/web/app/components/workflow/run/utils/format-log/simple-graph-to-log-struct.ts deleted file mode 100644 index 4aea146a7f..0000000000 --- a/web/app/components/workflow/run/utils/format-log/simple-graph-to-log-struct.ts +++ /dev/null @@ -1,14 +0,0 @@ -const STEP_SPLIT = '->' - -/* -* : 1 -> 2 -> 3 -* iteration: (iteration, 1, [2, 3]) -> 4. (1, [2, 3]) means 1 is parent, [2, 3] is children -* parallel: 1 -> (parallel, [1,2,3], [4, (parallel: (6,7))]). -* retry: (retry, 1, [2,3]). 1 is parent, [2, 3] is retry nodes -*/ -const simpleGraphToLogStruct = (input: string): any[] => { - const list = input.split(STEP_SPLIT) - return list -} - -export default simpleGraphToLogStruct