Merge branch 'feat/rag-pipeline' of https://github.com/langgenius/dify into feat/rag-pipeline

This commit is contained in:
twwu 2025-06-11 17:17:15 +08:00
commit b7f9d7e94a
25 changed files with 172 additions and 29 deletions

View File

@ -9,7 +9,7 @@ class PackagingInfo(BaseSettings):
CURRENT_VERSION: str = Field(
description="Dify version",
default="1.4.1",
default="1.4.2",
)
COMMIT_SHA: str = Field(

View File

@ -29,7 +29,7 @@ from core.plugin.entities.request import (
RequestRequestUploadFile,
)
from core.tools.entities.tool_entities import ToolProviderType
from libs.helper import compact_generate_response
from libs.helper import length_prefixed_response
from models.account import Account, Tenant
from models.model import EndUser
@ -44,7 +44,7 @@ class PluginInvokeLLMApi(Resource):
response = PluginModelBackwardsInvocation.invoke_llm(user_model.id, tenant_model, payload)
return PluginModelBackwardsInvocation.convert_to_event_stream(response)
return compact_generate_response(generator())
return length_prefixed_response(0xF, generator())
class PluginInvokeTextEmbeddingApi(Resource):
@ -101,7 +101,7 @@ class PluginInvokeTTSApi(Resource):
)
return PluginModelBackwardsInvocation.convert_to_event_stream(response)
return compact_generate_response(generator())
return length_prefixed_response(0xF, generator())
class PluginInvokeSpeech2TextApi(Resource):
@ -162,7 +162,7 @@ class PluginInvokeToolApi(Resource):
),
)
return compact_generate_response(generator())
return length_prefixed_response(0xF, generator())
class PluginInvokeParameterExtractorNodeApi(Resource):
@ -228,7 +228,7 @@ class PluginInvokeAppApi(Resource):
files=payload.files,
)
return compact_generate_response(PluginAppBackwardsInvocation.convert_to_event_stream(response))
return length_prefixed_response(0xF, PluginAppBackwardsInvocation.convert_to_event_stream(response))
class PluginInvokeEncryptApi(Resource):

View File

@ -11,14 +11,12 @@ class BaseBackwardsInvocation:
try:
for chunk in response:
if isinstance(chunk, BaseModel | dict):
yield BaseBackwardsInvocationResponse(data=chunk).model_dump_json().encode() + b"\n\n"
elif isinstance(chunk, str):
yield f"event: {chunk}\n\n".encode()
yield BaseBackwardsInvocationResponse(data=chunk).model_dump_json().encode()
except Exception as e:
error_message = BaseBackwardsInvocationResponse(error=str(e)).model_dump_json()
yield f"{error_message}\n\n".encode()
yield error_message.encode()
else:
yield BaseBackwardsInvocationResponse(data=response).model_dump_json().encode() + b"\n\n"
yield BaseBackwardsInvocationResponse(data=response).model_dump_json().encode()
T = TypeVar("T", bound=dict | Mapping | str | bool | int | BaseModel)

View File

@ -3,6 +3,7 @@ import logging
import re
import secrets
import string
import struct
import subprocess
import time
import uuid
@ -14,6 +15,7 @@ from zoneinfo import available_timezones
from flask import Response, stream_with_context
from flask_restful import fields
from pydantic import BaseModel
from configs import dify_config
from core.app.features.rate_limiting.rate_limit import RateLimitGenerator
@ -183,7 +185,7 @@ def generate_string(n):
def extract_remote_ip(request) -> str:
if request.headers.get("CF-Connecting-IP"):
return cast(str, request.headers.get("Cf-Connecting-Ip"))
return cast(str, request.headers.get("CF-Connecting-IP"))
elif request.headers.getlist("X-Forwarded-For"):
return cast(str, request.headers.getlist("X-Forwarded-For")[0])
else:
@ -206,6 +208,60 @@ def compact_generate_response(response: Union[Mapping, Generator, RateLimitGener
return Response(stream_with_context(generate()), status=200, mimetype="text/event-stream")
def length_prefixed_response(magic_number: int, response: Union[Mapping, Generator, RateLimitGenerator]) -> Response:
"""
This function is used to return a response with a length prefix.
Magic number is a one byte number that indicates the type of the response.
For a compatibility with latest plugin daemon https://github.com/langgenius/dify-plugin-daemon/pull/341
Avoid using line-based response, it leads a memory issue.
We uses following format:
| Field | Size | Description |
|---------------|----------|---------------------------------|
| Magic Number | 1 byte | Magic number identifier |
| Reserved | 1 byte | Reserved field |
| Header Length | 2 bytes | Header length (usually 0xa) |
| Data Length | 4 bytes | Length of the data |
| Reserved | 6 bytes | Reserved fields |
| Data | Variable | Actual data content |
| Reserved Fields | Header | Data |
|-----------------|----------|----------|
| 4 bytes total | Variable | Variable |
all data is in little endian
"""
def pack_response_with_length_prefix(response: bytes) -> bytes:
header_length = 0xA
data_length = len(response)
# | Magic Number 1byte | Reserved 1byte | Header Length 2bytes | Data Length 4bytes | Reserved 6bytes | Data
return struct.pack("<BBHI", magic_number, 0, header_length, data_length) + b"\x00" * 6 + response
if isinstance(response, dict):
return Response(
response=pack_response_with_length_prefix(json.dumps(jsonable_encoder(response)).encode("utf-8")),
status=200,
mimetype="application/json",
)
elif isinstance(response, BaseModel):
return Response(
response=pack_response_with_length_prefix(response.model_dump_json().encode("utf-8")),
status=200,
mimetype="application/json",
)
def generate() -> Generator:
for chunk in response:
if isinstance(chunk, str):
yield pack_response_with_length_prefix(chunk.encode("utf-8"))
else:
yield pack_response_with_length_prefix(chunk)
return Response(stream_with_context(generate()), status=200, mimetype="text/event-stream")
class TokenManager:
@classmethod
def generate_token(

View File

@ -2,7 +2,7 @@ x-shared-env: &shared-api-worker-env
services:
# API service
api:
image: langgenius/dify-api:1.4.1
image: langgenius/dify-api:1.4.2
restart: always
environment:
# Use the shared environment variables.
@ -31,7 +31,7 @@ services:
# worker service
# The Celery worker for processing the queue.
worker:
image: langgenius/dify-api:1.4.1
image: langgenius/dify-api:1.4.2
restart: always
environment:
# Use the shared environment variables.
@ -57,7 +57,7 @@ services:
# Frontend web application.
web:
image: langgenius/dify-web:1.4.1
image: langgenius/dify-web:1.4.2
restart: always
environment:
CONSOLE_API_URL: ${CONSOLE_API_URL:-}
@ -142,7 +142,7 @@ services:
# plugin daemon
plugin_daemon:
image: langgenius/dify-plugin-daemon:0.1.1-local
image: langgenius/dify-plugin-daemon:0.1.2-local
restart: always
environment:
# Use the shared environment variables.

View File

@ -71,7 +71,7 @@ services:
# plugin daemon
plugin_daemon:
image: langgenius/dify-plugin-daemon:0.1.1-local
image: langgenius/dify-plugin-daemon:0.1.2-local
restart: always
env_file:
- ./middleware.env

View File

@ -508,7 +508,7 @@ x-shared-env: &shared-api-worker-env
services:
# API service
api:
image: langgenius/dify-api:1.4.1
image: langgenius/dify-api:1.4.2
restart: always
environment:
# Use the shared environment variables.
@ -537,7 +537,7 @@ services:
# worker service
# The Celery worker for processing the queue.
worker:
image: langgenius/dify-api:1.4.1
image: langgenius/dify-api:1.4.2
restart: always
environment:
# Use the shared environment variables.
@ -563,7 +563,7 @@ services:
# Frontend web application.
web:
image: langgenius/dify-web:1.4.1
image: langgenius/dify-web:1.4.2
restart: always
environment:
CONSOLE_API_URL: ${CONSOLE_API_URL:-}
@ -648,7 +648,7 @@ services:
# plugin daemon
plugin_daemon:
image: langgenius/dify-plugin-daemon:0.1.1-local
image: langgenius/dify-plugin-daemon:0.1.2-local
restart: always
environment:
# Use the shared environment variables.

View File

@ -12,12 +12,18 @@ const Layout: FC<{
}> = ({ children }) => {
const isGlobalPending = useGlobalPublicStore(s => s.isGlobalPending)
const setWebAppAccessMode = useGlobalPublicStore(s => s.setWebAppAccessMode)
const systemFeatures = useGlobalPublicStore(s => s.systemFeatures)
const pathname = usePathname()
const searchParams = useSearchParams()
const redirectUrl = searchParams.get('redirect_url')
const [isLoading, setIsLoading] = useState(true)
useEffect(() => {
(async () => {
if (!systemFeatures.webapp_auth.enabled) {
setIsLoading(false)
return
}
let appCode: string | null = null
if (redirectUrl)
appCode = redirectUrl?.split('/').pop() || null

View File

@ -165,6 +165,8 @@ const ComponentPicker = ({
isSupportFileVar={isSupportFileVar}
onClose={handleClose}
onBlur={handleClose}
showManageInputField={workflowVariableBlock.showManageInputField}
onManageInputField={workflowVariableBlock.onManageInputField}
/>
</div>
)
@ -205,7 +207,7 @@ const ComponentPicker = ({
}
</>
)
}, [allFlattenOptions.length, workflowVariableBlock?.show, refs, isPositioned, floatingStyles, queryString, workflowVariableOptions, handleSelectWorkflowVariable, handleClose, isSupportFileVar])
}, [allFlattenOptions.length, workflowVariableBlock?.show, refs, isPositioned, floatingStyles, queryString, workflowVariableOptions, handleSelectWorkflowVariable, handleClose, isSupportFileVar, workflowVariableBlock?.showManageInputField, workflowVariableBlock?.onManageInputField])
return (
<LexicalTypeaheadMenuPlugin

View File

@ -68,6 +68,8 @@ export type WorkflowVariableBlockType = {
onInsert?: () => void
onDelete?: () => void
getVarType?: GetVarType
showManageInputField?: boolean
onManageInputField?: () => void
}
export type MenuTextMatch = {

View File

@ -21,7 +21,7 @@ const DialogWrapper = ({
const close = useCallback(() => onClose?.(), [onClose])
return (
<Transition appear show={show} as={Fragment}>
<Dialog as='div' className='relative z-40' onClose={close}>
<Dialog as='div' className='relative z-[2000]' onClose={close}>
<TransitionChild>
<div className={cn(
'fixed inset-0 bg-black/25',

View File

@ -21,7 +21,7 @@ const DialogWrapper = ({
const close = useCallback(() => onClose?.(), [onClose])
return (
<Transition appear show={show} as={Fragment}>
<Dialog as='div' className='relative z-40' onClose={close}>
<Dialog as='div' className='relative z-[2001]' onClose={close}>
<TransitionChild>
<div className={cn(
'fixed inset-0 bg-black/25',

View File

@ -1251,7 +1251,7 @@ export const useNodesInteractions = () => {
}
else {
// If no nodeId is provided, fall back to the current behavior
const bundledNodes = nodes.filter(node => node.data._isBundled && node.data.type !== BlockEnum.Start && node.data.type !== BlockEnum.DataSource
const bundledNodes = nodes.filter(node => node.data._isBundled && node.data.type !== BlockEnum.Start && node.data.type !== BlockEnum.DataSource && node.data.type !== BlockEnum.KnowledgeBase
&& !node.data.isInIteration && !node.data.isInLoop)
if (bundledNodes.length) {

View File

@ -13,6 +13,7 @@ import PromptEditor from '@/app/components/base/prompt-editor'
import { Variable02 } from '@/app/components/base/icons/src/vender/solid/development'
import Tooltip from '@/app/components/base/tooltip'
import { noop } from 'lodash-es'
import { useStore } from '@/app/components/workflow/store'
type Props = {
instanceId?: string
@ -56,6 +57,9 @@ const Editor: FC<Props> = ({
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [isFocus])
const pipelineId = useStore(s => s.pipelineId)
const setShowInputFieldDialog = useStore(s => s.setShowInputFieldDialog)
return (
<div className={cn(className, 'relative')}>
<>
@ -103,6 +107,8 @@ const Editor: FC<Props> = ({
}
return acc
}, {} as any),
showManageInputField: !!pipelineId,
onManageInputField: () => setShowInputFieldDialog?.(true),
}}
onChange={onChange}
editable={!readOnly}

View File

@ -254,7 +254,7 @@ const Editor: FC<Props> = ({
workflowVariableBlock={{
show: true,
variables: nodesOutputVars || [],
getVarType,
getVarType: getVarType as any,
workflowNodesMap: availableNodes.reduce((acc, node) => {
acc[node.id] = {
title: node.data.title,

View File

@ -0,0 +1,38 @@
import { useTranslation } from 'react-i18next'
import { RiAddLine } from '@remixicon/react'
type ManageInputFieldProps = {
onManage: () => void
}
const ManageInputField = ({
onManage,
}: ManageInputFieldProps) => {
const { t } = useTranslation()
return (
<div className='flex items-center border-t border-divider-subtle pt-1'>
<div
className='flex h-8 grow cursor-pointer items-center px-3'
onClick={onManage}
>
<RiAddLine className='mr-1 h-4 w-4 text-text-tertiary' />
<div
className='system-xs-medium truncate text-text-tertiary'
title='Create user input field'
>
{t('pipeline.inputField.create')}
</div>
</div>
<div className='mx-1 h-3 w-[1px] shrink-0 bg-divider-regular'></div>
<div
className='system-xs-medium flex h-8 shrink-0 cursor-pointer items-center justify-center px-3 text-text-tertiary'
onClick={onManage}
>
{t('pipeline.inputField.manage')}
</div>
</div>
)
}
export default ManageInputField

View File

@ -319,6 +319,7 @@ const VarReferencePicker: FC<Props> = ({
return null
}, [isValidVar, isShowAPart, hasValue, t, outputVarNode?.title, outputVarNode?.type, value, type])
return (
<div className={cn(className, !readonly && 'cursor-pointer')}>
<PortalToFollowElem

View File

@ -1,6 +1,6 @@
'use client'
import type { FC } from 'react'
import React from 'react'
import React, { useMemo } from 'react'
import { useTranslation } from 'react-i18next'
import { useContext } from 'use-context-selector'
import VarReferenceVars from './var-reference-vars'
@ -8,6 +8,7 @@ import type { NodeOutPutVar, ValueSelector, Var } from '@/app/components/workflo
import ListEmpty from '@/app/components/base/list-empty'
import { LanguagesSupported } from '@/i18n/language'
import I18n from '@/context/i18n'
import { useStore } from '@/app/components/workflow/store'
type Props = {
vars: NodeOutPutVar[]
@ -25,6 +26,9 @@ const VarReferencePopup: FC<Props> = ({
}) => {
const { t } = useTranslation()
const { locale } = useContext(I18n)
const pipelineId = useStore(s => s.pipelineId)
const showManageRagInputFields = useMemo(() => !!pipelineId, [pipelineId])
const setShowInputFieldDialog = useStore(s => s.setShowInputFieldDialog)
// max-h-[300px] overflow-y-auto todo: use portal to handle long list
return (
<div className='space-y-1 rounded-lg border border-components-panel-border bg-components-panel-bg p-1 shadow-lg' style={{
@ -57,6 +61,8 @@ const VarReferencePopup: FC<Props> = ({
onChange={onChange}
itemWidth={itemWidth}
isSupportFileVar={isSupportFileVar}
showManageInputField={showManageRagInputFields}
onManageInputField={() => setShowInputFieldDialog?.(true)}
/>
}
</div >

View File

@ -24,6 +24,7 @@ import { FILE_STRUCT } from '@/app/components/workflow/constants'
import { Loop } from '@/app/components/base/icons/src/vender/workflow'
import { noop } from 'lodash-es'
import { InputField } from '@/app/components/base/icons/src/vender/pipeline'
import ManageInputField from './manage-input-field'
type ObjectChildrenProps = {
nodeId: string
@ -266,6 +267,8 @@ type Props = {
maxHeightClass?: string
onClose?: () => void
onBlur?: () => void
showManageInputField?: boolean
onManageInputField?: () => void
}
const VarReferenceVars: FC<Props> = ({
hideSearch,
@ -277,6 +280,8 @@ const VarReferenceVars: FC<Props> = ({
maxHeightClass,
onClose,
onBlur,
showManageInputField,
onManageInputField,
}) => {
const { t } = useTranslation()
const [searchText, setSearchText] = useState('')
@ -367,6 +372,13 @@ const VarReferenceVars: FC<Props> = ({
}
</div>
: <div className='pl-3 text-xs font-medium uppercase leading-[18px] text-gray-500'>{t('workflow.common.noVar')}</div>}
{
showManageInputField && (
<ManageInputField
onManage={onManageInputField || noop}
/>
)
}
</>
)
}

View File

@ -23,6 +23,8 @@ const ConditionInput = ({
}: ConditionInputProps) => {
const { t } = useTranslation()
const controlPromptEditorRerenderKey = useStore(s => s.controlPromptEditorRerenderKey)
const pipelineId = useStore(s => s.pipelineId)
const setShowInputFieldDialog = useStore(s => s.setShowInputFieldDialog)
return (
<PromptEditor
@ -49,6 +51,8 @@ const ConditionInput = ({
}
return acc
}, {} as any),
showManageInputField: !!pipelineId,
onManageInputField: () => setShowInputFieldDialog?.(true),
}}
onChange={onChange}
editable={!disabled}

View File

@ -13,7 +13,7 @@ const nodeDefault: NodeDefault<KnowledgeBaseNodeType> = {
index_chunk_variable_selector: [],
keyword_number: 10,
retrieval_model: {
top_k: 2,
top_k: 3,
score_threshold_enabled: false,
score_threshold: 0.5,
},

View File

@ -20,6 +20,8 @@ const ConditionInput = ({
}: ConditionInputProps) => {
const { t } = useTranslation()
const controlPromptEditorRerenderKey = useStore(s => s.controlPromptEditorRerenderKey)
const pipelineId = useStore(s => s.pipelineId)
const setShowInputFieldDialog = useStore(s => s.setShowInputFieldDialog)
return (
<PromptEditor
@ -43,6 +45,8 @@ const ConditionInput = ({
}
return acc
}, {} as any),
showManageInputField: !!pipelineId,
onManageInputField: () => setShowInputFieldDialog?.(true),
}}
onChange={onChange}
editable={!disabled}

View File

@ -11,6 +11,10 @@ const translation = {
descriptionPlaceholder: 'Please enter the description of this Knowledge Pipeline. (Optional) ',
},
},
inputField: {
create: 'Create user input field',
manage: 'Manage',
},
}
export default translation

View File

@ -11,6 +11,10 @@ const translation = {
descriptionPlaceholder: '请输入此 Pipeline 的描述。 (可选)',
},
},
inputField: {
create: '创建用户输入字段',
manage: '管理',
},
}
export default translation

View File

@ -1,6 +1,6 @@
{
"name": "dify-web",
"version": "1.4.1",
"version": "1.4.2",
"private": true,
"engines": {
"node": ">=v22.11.0"