opensource/dify
2
0
Fork 0
mirror of https://github.com/langgenius/dify.git synced 2026-06-07 16:13:59 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor(api): migrate console.datasets.rag_pipeline partially to BaseModel (#36649)

Browse Source
This commit is contained in:
chariri 2026-06-04 02:44:10 +09:00 committed by GitHub
parent e14cb209a4
commit 4fc62d3b38
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
15 changed files with 1435 additions and 549 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
api/controllers/common/schema.py
View File

@ -36,6 +36,8 @@ QueryParamDoc = TypedDict(
},
)
JsonResponseWithStatus = tuple[dict[str, Any], int]
class QueryArgs(Protocol):
def to_dict(self, flat: bool = True) -> dict[str, str]: ...

129
api/controllers/console/datasets/rag_pipeline/rag_pipeline.py
View File

@ -1,13 +1,20 @@
import logging
from typing import Any
from flask import request
from flask_restx import Resource
from pydantic import BaseModel, Field
from sqlalchemy import select
from sqlalchemy.orm import sessionmaker
from werkzeug.exceptions import NotFound
from controllers.common.fields import SimpleDataResponse
from controllers.common.schema import register_response_schema_models, register_schema_models
from controllers.common.schema import (
JsonResponseWithStatus,
query_params_from_model,
register_response_schema_models,
register_schema_models,
)
from controllers.console import console_ns
from controllers.console.wraps import (
account_initialization_required,
@ -16,79 +23,132 @@ from controllers.console.wraps import (
setup_required,
)
from extensions.ext_database import db
from fields.base import ResponseModel
from libs.helper import dump_response
from libs.login import login_required
from models.dataset import PipelineCustomizedTemplate
from services.entities.knowledge_entities.rag_pipeline_entities import PipelineTemplateInfoEntity
from services.entities.knowledge_entities.rag_pipeline_entities import IconInfo, PipelineTemplateInfoEntity
from services.rag_pipeline.rag_pipeline import RagPipelineService
logger = logging.getLogger(__name__)
logger: logging.Logger = logging.getLogger(__name__)
class PipelineTemplateListQuery(BaseModel):
type: str = Field(default="built-in", description="Template source: built-in or customized")
language: str = Field(default="en-US", description="Template language")
class PipelineTemplateDetailQuery(BaseModel):
type: str = Field(default="built-in", description="Template source: built-in or customized")
class PipelineTemplateItemResponse(ResponseModel):
id: str
name: str
icon: dict[str, Any]
description: str
position: int
chunk_structure: str
copyright: str | None = None
privacy_policy: str | None = None
class PipelineTemplateListResponse(ResponseModel):
pipeline_templates: list[PipelineTemplateItemResponse]
class PipelineTemplateDetailResponse(ResponseModel):
id: str
name: str
icon_info: dict[str, Any]
description: str
chunk_structure: str
export_data: str
graph: dict[str, Any]
created_by: str | None = None
class CustomizedPipelineTemplatePayload(BaseModel):
name: str = Field(..., min_length=1, max_length=40)
description: str = Field(default="", max_length=400)
icon_info: dict[str, object] = Field(default_factory=lambda: IconInfo(icon="").model_dump())
register_schema_models(
console_ns,
CustomizedPipelineTemplatePayload,
PipelineTemplateDetailQuery,
PipelineTemplateListQuery,
)
register_response_schema_models(
console_ns,
PipelineTemplateDetailResponse,
PipelineTemplateListResponse,
SimpleDataResponse,
)
@console_ns.route("/rag/pipeline/templates")
class PipelineTemplateListApi(Resource):
@console_ns.doc(params=query_params_from_model(PipelineTemplateListQuery))
@console_ns.response(200, "Pipeline templates", console_ns.models[PipelineTemplateListResponse.__name__])
@setup_required
@login_required
@account_initialization_required
@enterprise_license_required
def get(self):
type = request.args.get("type", default="built-in", type=str)
language = request.args.get("language", default="en-US", type=str)
def get(self) -> JsonResponseWithStatus:
query = PipelineTemplateListQuery.model_validate(request.args.to_dict(flat=True))
# get pipeline templates
pipeline_templates = RagPipelineService.get_pipeline_templates(type, language)
return pipeline_templates, 200
pipeline_templates = RagPipelineService.get_pipeline_templates(query.type, query.language)
return dump_response(PipelineTemplateListResponse, pipeline_templates), 200
@console_ns.route("/rag/pipeline/templates/<string:template_id>")
class PipelineTemplateDetailApi(Resource):
@console_ns.doc(params=query_params_from_model(PipelineTemplateDetailQuery))
@console_ns.response(200, "Pipeline template", console_ns.models[PipelineTemplateDetailResponse.__name__])
@setup_required
@login_required
@account_initialization_required
@enterprise_license_required
def get(self, template_id: str):
type = request.args.get("type", default="built-in", type=str)
def get(self, template_id: str) -> JsonResponseWithStatus:
query = PipelineTemplateDetailQuery.model_validate(request.args.to_dict(flat=True))
rag_pipeline_service = RagPipelineService()
pipeline_template = rag_pipeline_service.get_pipeline_template_detail(template_id, type)
pipeline_template = rag_pipeline_service.get_pipeline_template_detail(template_id, query.type)
if pipeline_template is None:
return {"error": "Pipeline template not found from upstream service."}, 404
return pipeline_template, 200
class Payload(BaseModel):
name: str = Field(..., min_length=1, max_length=40)
description: str = Field(default="", max_length=400)
icon_info: dict[str, object] | None = None
register_schema_models(console_ns, Payload)
register_response_schema_models(console_ns, SimpleDataResponse)
raise NotFound("Pipeline template not found from upstream service.")
return dump_response(PipelineTemplateDetailResponse, pipeline_template), 200
@console_ns.route("/rag/pipeline/customized/templates/<string:template_id>")
class CustomizedPipelineTemplateApi(Resource):
@console_ns.expect(console_ns.models[CustomizedPipelineTemplatePayload.__name__])
@console_ns.response(204, "Pipeline template updated")
@setup_required
@login_required
@account_initialization_required
@enterprise_license_required
def patch(self, template_id: str):
payload = Payload.model_validate(console_ns.payload or {})
def patch(self, template_id: str) -> tuple[str, int]:
payload = CustomizedPipelineTemplatePayload.model_validate(console_ns.payload or {})
pipeline_template_info = PipelineTemplateInfoEntity.model_validate(payload.model_dump())
RagPipelineService.update_customized_pipeline_template(template_id, pipeline_template_info)
return 200
return "", 204
@console_ns.response(204, "Pipeline template deleted")
@setup_required
@login_required
@account_initialization_required
@enterprise_license_required
def delete(self, template_id: str):
def delete(self, template_id: str) -> tuple[str, int]:
RagPipelineService.delete_customized_pipeline_template(template_id)
return 200
return "", 204
@setup_required
@login_required
@account_initialization_required
@enterprise_license_required
@console_ns.response(200, "Success", console_ns.models[SimpleDataResponse.__name__])
def post(self, template_id: str):
def post(self, template_id: str) -> JsonResponseWithStatus:
with sessionmaker(db.engine, expire_on_commit=False).begin() as session:
template = session.scalar(
select(PipelineCustomizedTemplate).where(PipelineCustomizedTemplate.id == template_id).limit(1)
@ -96,19 +156,20 @@ class CustomizedPipelineTemplateApi(Resource):
if not template:
raise ValueError("Customized pipeline template not found.")
return {"data": template.yaml_content}, 200
return dump_response(SimpleDataResponse, {"data": template.yaml_content}), 200
@console_ns.route("/rag/pipelines/<string:pipeline_id>/customized/publish")
class PublishCustomizedPipelineTemplateApi(Resource):
@console_ns.expect(console_ns.models[Payload.__name__])
@console_ns.expect(console_ns.models[CustomizedPipelineTemplatePayload.__name__])
@console_ns.response(204, "Pipeline template published")
@setup_required
@login_required
@account_initialization_required
@enterprise_license_required
@knowledge_pipeline_publish_enabled
def post(self, pipeline_id: str):
payload = Payload.model_validate(console_ns.payload or {})
def post(self, pipeline_id: str) -> tuple[str, int]:
payload = CustomizedPipelineTemplatePayload.model_validate(console_ns.payload or {})
rag_pipeline_service = RagPipelineService()
rag_pipeline_service.publish_customized_pipeline_template(pipeline_id, payload.model_dump())
return {"result": "success"}
return "", 204

25
api/controllers/console/datasets/rag_pipeline/rag_pipeline_datasets.py
View File

@ -1,12 +1,13 @@
from flask_restx import Resource, marshal
from flask_restx import Resource
from pydantic import BaseModel
from sqlalchemy.orm import Session
from werkzeug.exceptions import Forbidden
import services
from controllers.common.schema import register_schema_model
from controllers.common.schema import JsonResponseWithStatus, register_response_schema_models, register_schema_models
from controllers.console import console_ns
from controllers.console.datasets.error import DatasetNameDuplicateError
from controllers.console.datasets.rag_pipeline.rag_pipeline_import import RagPipelineImportResponse
from controllers.console.wraps import (
account_initialization_required,
cloud_edition_billing_rate_limit_check,
@ -15,7 +16,8 @@ from controllers.console.wraps import (
with_current_user,
)
from extensions.ext_database import db
from fields.dataset_fields import dataset_detail_fields
from fields.dataset_fields import DatasetDetailResponse
from libs.helper import dump_response
from libs.login import login_required
from models import Account
from models.dataset import DatasetPermissionEnum
@ -28,19 +30,25 @@ class RagPipelineDatasetImportPayload(BaseModel):
yaml_content: str
register_schema_model(console_ns, RagPipelineDatasetImportPayload)
register_schema_models(console_ns, RagPipelineDatasetImportPayload)
register_response_schema_models(console_ns, DatasetDetailResponse, RagPipelineImportResponse)
@console_ns.route("/rag/pipeline/dataset")
class CreateRagPipelineDatasetApi(Resource):
@console_ns.expect(console_ns.models[RagPipelineDatasetImportPayload.__name__])
@console_ns.response(
201,
"RAG pipeline dataset import started",
console_ns.models[RagPipelineImportResponse.__name__],
)
@setup_required
@login_required
@account_initialization_required
@cloud_edition_billing_rate_limit_check("knowledge")
@with_current_user
@with_current_tenant_id
def post(self, current_tenant_id: str, current_user: Account):
def post(self, current_tenant_id: str, current_user: Account) -> JsonResponseWithStatus:
payload = RagPipelineDatasetImportPayload.model_validate(console_ns.payload or {})
# The role of the current user in the ta table must be admin, owner, or editor, or dataset_operator
if not current_user.is_dataset_editor:
@ -74,18 +82,19 @@ class CreateRagPipelineDatasetApi(Resource):
except services.errors.dataset.DatasetNameDuplicateError:
raise DatasetNameDuplicateError()
return import_info, 201
return dump_response(RagPipelineImportResponse, import_info), 201
@console_ns.route("/rag/pipeline/empty-dataset")
class CreateEmptyRagPipelineDatasetApi(Resource):
@console_ns.response(201, "RAG pipeline dataset created", console_ns.models[DatasetDetailResponse.__name__])
@setup_required
@login_required
@account_initialization_required
@cloud_edition_billing_rate_limit_check("knowledge")
@with_current_user
@with_current_tenant_id
def post(self, current_tenant_id: str, current_user: Account):
def post(self, current_tenant_id: str, current_user: Account) -> JsonResponseWithStatus:
# The role of the current user in the ta table must be admin, owner, or editor, or dataset_operator
if not current_user.is_dataset_editor:
raise Forbidden()
@ -103,4 +112,4 @@ class CreateEmptyRagPipelineDatasetApi(Resource):
partial_member_list=None,
),
)
return marshal(dataset, dataset_detail_fields), 201
return dump_response(DatasetDetailResponse, dataset), 201

89
api/controllers/console/datasets/rag_pipeline/rag_pipeline_import.py
View File

@ -1,9 +1,15 @@
from flask import request
from flask_restx import Resource, fields, marshal_with # type: ignore
from flask_restx import Resource
from pydantic import BaseModel, Field
from sqlalchemy.orm import Session
from controllers.common.schema import get_or_create_model, register_schema_models
from controllers.common.fields import SimpleDataResponse
from controllers.common.schema import (
JsonResponseWithStatus,
query_params_from_model,
register_response_schema_models,
register_schema_models,
)
from controllers.console import console_ns
from controllers.console.datasets.wraps import get_rag_pipeline
from controllers.console.wraps import (
@ -12,12 +18,10 @@ from controllers.console.wraps import (
setup_required,
with_current_user,
)
from core.plugin.entities.plugin import PluginDependency
from extensions.ext_database import db
from fields.rag_pipeline_fields import (
leaked_dependency_fields,
pipeline_import_check_dependencies_fields,
pipeline_import_fields,
)
from fields.base import ResponseModel
from libs.helper import dump_response
from libs.login import login_required
from models.account import Account
from models.dataset import Pipeline
@ -38,34 +42,44 @@ class RagPipelineImportPayload(BaseModel):
class IncludeSecretQuery(BaseModel):
include_secret: str = Field(default="false")
include_secret: str = Field(default="false", description="Whether to include secret values in the exported DSL")
class RagPipelineImportResponse(ResponseModel):
id: str
status: ImportStatus
pipeline_id: str | None = None
dataset_id: str | None = None
current_dsl_version: str
imported_dsl_version: str
error: str = ""
class RagPipelineImportCheckDependenciesResponse(ResponseModel):
leaked_dependencies: list[PluginDependency] = Field(default_factory=list)
register_schema_models(console_ns, RagPipelineImportPayload, IncludeSecretQuery)
pipeline_import_model = get_or_create_model("RagPipelineImport", pipeline_import_fields)
leaked_dependency_model = get_or_create_model("RagPipelineLeakedDependency", leaked_dependency_fields)
pipeline_import_check_dependencies_fields_copy = pipeline_import_check_dependencies_fields.copy()
pipeline_import_check_dependencies_fields_copy["leaked_dependencies"] = fields.List(
fields.Nested(leaked_dependency_model)
)
pipeline_import_check_dependencies_model = get_or_create_model(
"RagPipelineImportCheckDependencies", pipeline_import_check_dependencies_fields_copy
register_response_schema_models(
console_ns,
RagPipelineImportCheckDependenciesResponse,
RagPipelineImportResponse,
SimpleDataResponse,
)
@console_ns.route("/rag/pipelines/imports")
class RagPipelineImportApi(Resource):
@console_ns.expect(console_ns.models[RagPipelineImportPayload.__name__])
@console_ns.response(200, "Import completed", console_ns.models[RagPipelineImportResponse.__name__])
@console_ns.response(202, "Import pending confirmation", console_ns.models[RagPipelineImportResponse.__name__])
@console_ns.response(400, "Import failed", console_ns.models[RagPipelineImportResponse.__name__])
@setup_required
@login_required
@account_initialization_required
@edit_permission_required
@marshal_with(pipeline_import_model)
@console_ns.expect(console_ns.models[RagPipelineImportPayload.__name__])
@with_current_user
def post(self, current_user: Account):
def post(self, current_user: Account) -> JsonResponseWithStatus:
# Check user role first
payload = RagPipelineImportPayload.model_validate(console_ns.payload or {})
@ -93,22 +107,23 @@ class RagPipelineImportApi(Resource):
status = result.status
match status:
case ImportStatus.FAILED:
return result.model_dump(mode="json"), 400
return dump_response(RagPipelineImportResponse, result), 400
case ImportStatus.PENDING:
return result.model_dump(mode="json"), 202
return dump_response(RagPipelineImportResponse, result), 202
case ImportStatus.COMPLETED | ImportStatus.COMPLETED_WITH_WARNINGS:
return result.model_dump(mode="json"), 200
return dump_response(RagPipelineImportResponse, result), 200
@console_ns.route("/rag/pipelines/imports/<string:import_id>/confirm")
class RagPipelineImportConfirmApi(Resource):
@console_ns.response(200, "Import confirmed", console_ns.models[RagPipelineImportResponse.__name__])
@console_ns.response(400, "Import failed", console_ns.models[RagPipelineImportResponse.__name__])
@setup_required
@login_required
@account_initialization_required
@edit_permission_required
@marshal_with(pipeline_import_model)
@with_current_user
def post(self, current_user: Account, import_id: str):
def post(self, current_user: Account, import_id: str) -> JsonResponseWithStatus:
with Session(db.engine, expire_on_commit=False) as session:
import_service = RagPipelineDslService(session)
account = current_user
@ -120,34 +135,40 @@ class RagPipelineImportConfirmApi(Resource):
# Return appropriate status code based on result
if result.status == ImportStatus.FAILED:
return result.model_dump(mode="json"), 400
return result.model_dump(mode="json"), 200
return dump_response(RagPipelineImportResponse, result), 400
return dump_response(RagPipelineImportResponse, result), 200
@console_ns.route("/rag/pipelines/imports/<string:pipeline_id>/check-dependencies")
class RagPipelineImportCheckDependenciesApi(Resource):
@console_ns.response(
200,
"Dependencies checked",
console_ns.models[RagPipelineImportCheckDependenciesResponse.__name__],
)
@setup_required
@login_required
@get_rag_pipeline
@account_initialization_required
@edit_permission_required
@marshal_with(pipeline_import_check_dependencies_model)
def get(self, pipeline: Pipeline):
def get(self, pipeline: Pipeline) -> JsonResponseWithStatus:
with Session(db.engine, expire_on_commit=False) as session:
import_service = RagPipelineDslService(session)
result = import_service.check_dependencies(pipeline=pipeline)
return result.model_dump(mode="json"), 200
return dump_response(RagPipelineImportCheckDependenciesResponse, result), 200
@console_ns.route("/rag/pipelines/<string:pipeline_id>/exports")
class RagPipelineExportApi(Resource):
@console_ns.doc(params=query_params_from_model(IncludeSecretQuery))
@console_ns.response(200, "Pipeline exported", console_ns.models[SimpleDataResponse.__name__])
@setup_required
@login_required
@get_rag_pipeline
@account_initialization_required
@edit_permission_required
def get(self, pipeline: Pipeline):
def get(self, pipeline: Pipeline) -> JsonResponseWithStatus:
# Add include_secret params
query = IncludeSecretQuery.model_validate(request.args.to_dict())
@ -157,4 +178,4 @@ class RagPipelineExportApi(Resource):
pipeline=pipeline, include_secret=query.include_secret == "true"
)
return {"data": result}, 200
return dump_response(SimpleDataResponse, {"data": result}), 200

164
api/fields/rag_pipeline_fields.py
View File

@ -1,164 +0,0 @@
from flask_restx import fields
from fields.workflow_fields import workflow_partial_fields
from libs.helper import AppIconUrlField, TimestampField
pipeline_detail_kernel_fields = {
"id": fields.String,
"name": fields.String,
"description": fields.String,
"icon_type": fields.String,
"icon": fields.String,
"icon_background": fields.String,
"icon_url": AppIconUrlField,
}
related_app_list = {
"data": fields.List(fields.Nested(pipeline_detail_kernel_fields)),
"total": fields.Integer,
}
app_detail_fields = {
"id": fields.String,
"name": fields.String,
"description": fields.String,
"mode": fields.String(attribute="mode_compatible_with_agent"),
"icon": fields.String,
"icon_background": fields.String,
"workflow": fields.Nested(workflow_partial_fields, allow_null=True),
"tracing": fields.Raw,
"created_by": fields.String,
"created_at": TimestampField,
"updated_by": fields.String,
"updated_at": TimestampField,
}
tag_fields = {"id": fields.String, "name": fields.String, "type": fields.String}
app_partial_fields = {
"id": fields.String,
"name": fields.String,
"description": fields.String(attribute="desc_or_prompt"),
"icon_type": fields.String,
"icon": fields.String,
"icon_background": fields.String,
"icon_url": AppIconUrlField,
"workflow": fields.Nested(workflow_partial_fields, allow_null=True),
"created_by": fields.String,
"created_at": TimestampField,
"updated_by": fields.String,
"updated_at": TimestampField,
"tags": fields.List(fields.Nested(tag_fields)),
}
app_pagination_fields = {
"page": fields.Integer,
"limit": fields.Integer(attribute="per_page"),
"total": fields.Integer,
"has_more": fields.Boolean(attribute="has_next"),
"data": fields.List(fields.Nested(app_partial_fields), attribute="items"),
}
template_fields = {
"name": fields.String,
"icon": fields.String,
"icon_background": fields.String,
"description": fields.String,
"mode": fields.String,
}
template_list_fields = {
"data": fields.List(fields.Nested(template_fields)),
}
site_fields = {
"access_token": fields.String(attribute="code"),
"code": fields.String,
"title": fields.String,
"icon_type": fields.String,
"icon": fields.String,
"icon_background": fields.String,
"icon_url": AppIconUrlField,
"description": fields.String,
"default_language": fields.String,
"chat_color_theme": fields.String,
"chat_color_theme_inverted": fields.Boolean,
"customize_domain": fields.String,
"copyright": fields.String,
"privacy_policy": fields.String,
"custom_disclaimer": fields.String,
"customize_token_strategy": fields.String,
"prompt_public": fields.Boolean,
"app_base_url": fields.String,
"show_workflow_steps": fields.Boolean,
"use_icon_as_answer_icon": fields.Boolean,
"created_by": fields.String,
"created_at": TimestampField,
"updated_by": fields.String,
"updated_at": TimestampField,
}
deleted_tool_fields = {
"type": fields.String,
"tool_name": fields.String,
"provider_id": fields.String,
}
app_detail_fields_with_site = {
"id": fields.String,
"name": fields.String,
"description": fields.String,
"mode": fields.String(attribute="mode_compatible_with_agent"),
"icon_type": fields.String,
"icon": fields.String,
"icon_background": fields.String,
"icon_url": AppIconUrlField,
"enable_site": fields.Boolean,
"enable_api": fields.Boolean,
"workflow": fields.Nested(workflow_partial_fields, allow_null=True),
"site": fields.Nested(site_fields),
"api_base_url": fields.String,
"use_icon_as_answer_icon": fields.Boolean,
"created_by": fields.String,
"created_at": TimestampField,
"updated_by": fields.String,
"updated_at": TimestampField,
}
app_site_fields = {
"app_id": fields.String,
"access_token": fields.String(attribute="code"),
"code": fields.String,
"title": fields.String,
"icon": fields.String,
"icon_background": fields.String,
"description": fields.String,
"default_language": fields.String,
"customize_domain": fields.String,
"copyright": fields.String,
"privacy_policy": fields.String,
"custom_disclaimer": fields.String,
"customize_token_strategy": fields.String,
"prompt_public": fields.Boolean,
"show_workflow_steps": fields.Boolean,
"use_icon_as_answer_icon": fields.Boolean,
}
leaked_dependency_fields = {"type": fields.String, "value": fields.Raw, "current_identifier": fields.String}
pipeline_import_fields = {
"id": fields.String,
"status": fields.String,
"pipeline_id": fields.String,
"dataset_id": fields.String,
"current_dsl_version": fields.String,
"imported_dsl_version": fields.String,
"error": fields.String,
}
pipeline_import_check_dependencies_fields = {
"leaked_dependencies": fields.List(fields.Nested(leaked_dependency_fields)),
}

160
api/openapi/markdown/console-swagger.md
View File

@ -6939,7 +6939,7 @@ Handle OAuth callback for trigger provider
| Code | Description |
| ---- | ----------- |
| 200 | Success |
| 204 | Pipeline template deleted |
#### PATCH
##### Parameters
@ -6947,12 +6947,13 @@ Handle OAuth callback for trigger provider
| Name | Located in | Description | Required | Schema |
| ---- | ---------- | ----------- | -------- | ------ |
| template_id | path | | Yes | string |
| payload | body | | Yes | [CustomizedPipelineTemplatePayload](#customizedpipelinetemplatepayload) |
##### Responses
| Code | Description |
| ---- | ----------- |
| 200 | Success |
| 204 | Pipeline template updated |
#### POST
##### Parameters
@ -6978,27 +6979,34 @@ Handle OAuth callback for trigger provider
##### Responses
| Code | Description |
| ---- | ----------- |
| 200 | Success |
| Code | Description | Schema |
| ---- | ----------- | ------ |
| 201 | RAG pipeline dataset import started | [RagPipelineImportResponse](#ragpipelineimportresponse) |
### /rag/pipeline/empty-dataset
#### POST
##### Responses
| Code | Description |
| ---- | ----------- |
| 200 | Success |
| Code | Description | Schema |
| ---- | ----------- | ------ |
| 201 | RAG pipeline dataset created | [DatasetDetailResponse](#datasetdetailresponse) |
### /rag/pipeline/templates
#### GET
##### Parameters
| Name | Located in | Description | Required | Schema |
| ---- | ---------- | ----------- | -------- | ------ |
| language | query | Template language | No | string |
| type | query | Template source: built-in or customized | No | string |
##### Responses
| Code | Description |
| ---- | ----------- |
| 200 | Success |
| Code | Description | Schema |
| ---- | ----------- | ------ |
| 200 | Pipeline templates | [PipelineTemplateListResponse](#pipelinetemplatelistresponse) |
### /rag/pipeline/templates/{template_id}
@ -7008,12 +7016,13 @@ Handle OAuth callback for trigger provider
| Name | Located in | Description | Required | Schema |
| ---- | ---------- | ----------- | -------- | ------ |
| template_id | path | | Yes | string |
| type | query | Template source: built-in or customized | No | string |
##### Responses
| Code | Description |
| ---- | ----------- |
| 200 | Success |
| Code | Description | Schema |
| ---- | ----------- | ------ |
| 200 | Pipeline template | [PipelineTemplateDetailResponse](#pipelinetemplatedetailresponse) |
### /rag/pipelines/datasource-plugins
@ -7035,9 +7044,11 @@ Handle OAuth callback for trigger provider
##### Responses
| Code | Description |
| ---- | ----------- |
| 200 | Success |
| Code | Description | Schema |
| ---- | ----------- | ------ |
| 200 | Import completed | [RagPipelineImportResponse](#ragpipelineimportresponse) |
| 202 | Import pending confirmation | [RagPipelineImportResponse](#ragpipelineimportresponse) |
| 400 | Import failed | [RagPipelineImportResponse](#ragpipelineimportresponse) |
### /rag/pipelines/imports/{import_id}/confirm
@ -7050,9 +7061,10 @@ Handle OAuth callback for trigger provider
##### Responses
| Code | Description |
| ---- | ----------- |
| 200 | Success |
| Code | Description | Schema |
| ---- | ----------- | ------ |
| 200 | Import confirmed | [RagPipelineImportResponse](#ragpipelineimportresponse) |
| 400 | Import failed | [RagPipelineImportResponse](#ragpipelineimportresponse) |
### /rag/pipelines/imports/{pipeline_id}/check-dependencies
@ -7065,9 +7077,9 @@ Handle OAuth callback for trigger provider
##### Responses
| Code | Description |
| ---- | ----------- |
| 200 | Success |
| Code | Description | Schema |
| ---- | ----------- | ------ |
| 200 | Dependencies checked | [RagPipelineImportCheckDependenciesResponse](#ragpipelineimportcheckdependenciesresponse) |
### /rag/pipelines/recommended-plugins
@ -7101,13 +7113,13 @@ Handle OAuth callback for trigger provider
| Name | Located in | Description | Required | Schema |
| ---- | ---------- | ----------- | -------- | ------ |
| pipeline_id | path | | Yes | string |
| payload | body | | Yes | [Payload](#payload) |
| payload | body | | Yes | [CustomizedPipelineTemplatePayload](#customizedpipelinetemplatepayload) |
##### Responses
| Code | Description |
| ---- | ----------- |
| 200 | Success |
| 204 | Pipeline template published |
### /rag/pipelines/{pipeline_id}/exports
@ -7117,12 +7129,13 @@ Handle OAuth callback for trigger provider
| Name | Located in | Description | Required | Schema |
| ---- | ---------- | ----------- | -------- | ------ |
| pipeline_id | path | | Yes | string |
| include_secret | query | Whether to include secret values in the exported DSL | No | string |
##### Responses
| Code | Description |
| ---- | ----------- |
| 200 | Success |
| Code | Description | Schema |
| ---- | ----------- | ------ |
| 200 | Pipeline exported | [SimpleDataResponse](#simpledataresponse) |
### /rag/pipelines/{pipeline_id}/workflow-runs
@ -12429,6 +12442,14 @@ Condition detail
| ---- | ---- | ----------- | -------- |
| CredentialType | string | | |
#### CustomizedPipelineTemplatePayload
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| description | string | | No |
| icon_info | object | | No |
| name | string | | Yes |
#### DataSource
| Name | Type | Description | Required |
@ -13883,7 +13904,7 @@ Request payload for bulk downloading documents as a zip archive.
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| include_secret | string | | No |
| include_secret | string | Whether to include secret values in the exported DSL | No |
#### IndexingEstimatePayload
@ -14860,14 +14881,6 @@ Form input definition.
| node_title | string | | Yes |
| pause_type | [HumanInputPauseTypeResponse](#humaninputpausetyperesponse) | | Yes |
#### Payload
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| description | string | | No |
| icon_info | object | | No |
| name | string | | Yes |
#### PermissionEnum
Shared permission levels for resources (datasets, credentials, etc.)
@ -14876,6 +14889,51 @@ Shared permission levels for resources (datasets, credentials, etc.)
| ---- | ---- | ----------- | -------- |
| PermissionEnum | string | Shared permission levels for resources (datasets, credentials, etc.) | |
#### PipelineTemplateDetailQuery
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| type | string | Template source: built-in or customized | No |
#### PipelineTemplateDetailResponse
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| chunk_structure | string | | Yes |
| created_by | string | | No |
| description | string | | Yes |
| export_data | string | | Yes |
| graph | object | | Yes |
| icon_info | object | | Yes |
| id | string | | Yes |
| name | string | | Yes |
#### PipelineTemplateItemResponse
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| chunk_structure | string | | Yes |
| copyright | string | | No |
| description | string | | Yes |
| icon | object | | Yes |
| id | string | | Yes |
| name | string | | Yes |
| position | integer | | Yes |
| privacy_policy | string | | No |
#### PipelineTemplateListQuery
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| language | string | Template language | No |
| type | string | Template source: built-in or customized | No |
#### PipelineTemplateListResponse
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| pipeline_templates | [ [PipelineTemplateItemResponse](#pipelinetemplateitemresponse) ] | | Yes |
#### PipelineVariableResponse
| Name | Type | Description | Required |
@ -15000,23 +15058,11 @@ Shared permission levels for resources (datasets, credentials, etc.)
| ---- | ---- | ----------- | -------- |
| yaml_content | string | | Yes |
#### RagPipelineImport
#### RagPipelineImportCheckDependenciesResponse
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| current_dsl_version | string | | No |
| dataset_id | string | | No |
| error | string | | No |
| id | string | | No |
| imported_dsl_version | string | | No |
| pipeline_id | string | | No |
| status | string | | No |
#### RagPipelineImportCheckDependencies
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| leaked_dependencies | [ [RagPipelineLeakedDependency](#ragpipelineleakeddependency) ] | | No |
| leaked_dependencies | [ [PluginDependency](#plugindependency) ] | | No |
#### RagPipelineImportPayload
@ -15032,13 +15078,17 @@ Shared permission levels for resources (datasets, credentials, etc.)
| yaml_content | string | | No |
| yaml_url | string | | No |
#### RagPipelineLeakedDependency
#### RagPipelineImportResponse
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| current_identifier | string | | No |
| type | string | | No |
| value | object | | No |
| current_dsl_version | string | | Yes |
| dataset_id | string | | No |
| error | string | | No |
| id | string | | Yes |
| imported_dsl_version | string | | Yes |
| pipeline_id | string | | No |
| status | [ImportStatus](#importstatus) | | Yes |
#### RagPipelineRecommendedPluginQuery

15
api/services/entities/knowledge_entities/rag_pipeline_entities.py
View File

@ -1,4 +1,4 @@
from typing import Any, Literal
from typing import Literal
from pydantic import BaseModel, field_validator
@ -73,18 +73,11 @@ class KnowledgeConfiguration(BaseModel):
keyword_number: int | None = 10
retrieval_model: RetrievalSetting
# add summary index setting
summary_index_setting: dict[str, Any] | None = None
summary_index_setting: dict[str, object] | None = None
@field_validator("embedding_model_provider", mode="before")
@field_validator("embedding_model_provider", "embedding_model", mode="before")
@classmethod
def validate_embedding_model_provider(cls, v):
if v is None:
return ""
return v
@field_validator("embedding_model", mode="before")
@classmethod
def validate_embedding_model(cls, v):
def validate_embedding_model_fields(cls, v: str | None) -> str:
if v is None:
return ""
return v

106
api/tests/test_containers_integration_tests/controllers/console/datasets/rag_pipeline/test_rag_pipeline.py
View File

@ -2,13 +2,17 @@
from __future__ import annotations
from collections.abc import Callable
from typing import cast
from unittest.mock import MagicMock, patch
from uuid import uuid4
import pytest
from flask import Flask
from sqlalchemy.orm import Session
from werkzeug.exceptions import NotFound
from controllers.common.schema import JsonResponseWithStatus
from controllers.console import console_ns
from controllers.console.datasets.rag_pipeline.rag_pipeline import (
CustomizedPipelineTemplateApi,
@ -17,24 +21,30 @@ from controllers.console.datasets.rag_pipeline.rag_pipeline import (
PublishCustomizedPipelineTemplateApi,
)
from models.dataset import PipelineCustomizedTemplate
def unwrap(func):
while hasattr(func, "__wrapped__"):
func = func.__wrapped__
return func
from tests.test_containers_integration_tests.controllers.console.helpers import unwrap
class TestPipelineTemplateListApi:
@pytest.fixture
def app(self, flask_app_with_containers: Flask):
def app(self, flask_app_with_containers: Flask) -> Flask:
return flask_app_with_containers
def test_get_success(self, app: Flask):
def test_get_success(self, app: Flask) -> None:
api = PipelineTemplateListApi()
method = unwrap(api.get)
templates = [{"id": "t1"}]
templates = {
"pipeline_templates": [
{
"id": "t1",
"name": "Template",
"description": "Description",
"icon": {"icon": "📘", "icon_type": "emoji", "icon_background": "#fff"},
"position": 1,
"chunk_structure": "general",
}
]
}
with (
app.test_request_context("/?type=built-in&language=en-US"),
@ -46,19 +56,38 @@ class TestPipelineTemplateListApi:
response, status = method(api)
assert status == 200
assert response == templates
assert response == {
"pipeline_templates": [
{
**templates["pipeline_templates"][0],
"copyright": None,
"privacy_policy": None,
}
]
}
class TestPipelineTemplateDetailApi:
@pytest.fixture
def app(self, flask_app_with_containers: Flask):
def app(self, flask_app_with_containers: Flask) -> Flask:
return flask_app_with_containers
def test_get_success(self, app: Flask):
def test_get_success(self, app: Flask) -> None:
api = PipelineTemplateDetailApi()
method = unwrap(api.get)
template = {"id": "tpl-1"}
nodes: list[dict[str, object]] = []
edges: list[dict[str, object]] = []
viewport: dict[str, int] = {"x": 0, "y": 0, "zoom": 1}
template = {
"id": "tpl-1",
"name": "Template",
"icon_info": {"icon": "📘", "icon_type": "emoji", "icon_background": "#fff"},
"description": "Description",
"chunk_structure": "general",
"export_data": "yaml-data",
"graph": {"nodes": nodes, "edges": edges, "viewport": viewport},
}
service = MagicMock()
service.get_pipeline_template_detail.return_value = template
@ -73,9 +102,9 @@ class TestPipelineTemplateDetailApi:
response, status = method(api, "tpl-1")
assert status == 200
assert response == template
assert response == {**template, "created_by": None}
def test_get_returns_404_when_template_not_found(self, app: Flask):
def test_get_returns_404_when_template_not_found(self, app: Flask) -> None:
api = PipelineTemplateDetailApi()
method = unwrap(api.get)
@ -89,12 +118,10 @@ class TestPipelineTemplateDetailApi:
return_value=service,
),
):
response, status = method(api, "non-existent-id")
with pytest.raises(NotFound):
method(api, "non-existent-id")
assert status == 404
assert "error" in response
def test_get_returns_404_for_customized_type_not_found(self, app: Flask):
def test_get_returns_404_for_customized_type_not_found(self, app: Flask) -> None:
api = PipelineTemplateDetailApi()
method = unwrap(api.get)
@ -108,18 +135,16 @@ class TestPipelineTemplateDetailApi:
return_value=service,
),
):
response, status = method(api, "non-existent-id")
assert status == 404
assert "error" in response
with pytest.raises(NotFound):
method(api, "non-existent-id")
class TestCustomizedPipelineTemplateApi:
@pytest.fixture
def app(self, flask_app_with_containers: Flask):
def app(self, flask_app_with_containers: Flask) -> Flask:
return flask_app_with_containers
def test_patch_success(self, app: Flask):
def test_patch_success(self, app: Flask) -> None:
api = CustomizedPipelineTemplateApi()
method = unwrap(api.patch)
@ -136,12 +161,13 @@ class TestCustomizedPipelineTemplateApi:
"controllers.console.datasets.rag_pipeline.rag_pipeline.RagPipelineService.update_customized_pipeline_template"
) as update_mock,
):
response = method(api, "tpl-1")
response, status = method(api, "tpl-1")
update_mock.assert_called_once()
assert response == 200
assert status == 204
assert response == ""
def test_delete_success(self, app: Flask):
def test_delete_success(self, app: Flask) -> None:
api = CustomizedPipelineTemplateApi()
method = unwrap(api.delete)
@ -151,14 +177,15 @@ class TestCustomizedPipelineTemplateApi:
"controllers.console.datasets.rag_pipeline.rag_pipeline.RagPipelineService.delete_customized_pipeline_template"
) as delete_mock,
):
response = method(api, "tpl-1")
response, status = method(api, "tpl-1")
delete_mock.assert_called_once_with("tpl-1")
assert response == 200
assert status == 204
assert response == ""
def test_post_success(self, app: Flask, db_session_with_containers: Session):
def test_post_success(self, app: Flask, db_session_with_containers: Session) -> None:
api = CustomizedPipelineTemplateApi()
method = unwrap(api.post)
method = unwrap(cast(Callable[..., JsonResponseWithStatus], api.post))
tenant_id = str(uuid4())
template = PipelineCustomizedTemplate(
@ -183,9 +210,9 @@ class TestCustomizedPipelineTemplateApi:
assert status == 200
assert response == {"data": "yaml-data"}
def test_post_template_not_found(self, app: Flask):
def test_post_template_not_found(self, app: Flask) -> None:
api = CustomizedPipelineTemplateApi()
method = unwrap(api.post)
method = unwrap(cast(Callable[..., JsonResponseWithStatus], api.post))
with app.test_request_context("/"):
with pytest.raises(ValueError):
@ -194,10 +221,10 @@ class TestCustomizedPipelineTemplateApi:
class TestPublishCustomizedPipelineTemplateApi:
@pytest.fixture
def app(self, flask_app_with_containers: Flask):
def app(self, flask_app_with_containers: Flask) -> Flask:
return flask_app_with_containers
def test_post_success(self, app: Flask):
def test_post_success(self, app: Flask) -> None:
api = PublishCustomizedPipelineTemplateApi()
method = unwrap(api.post)
@ -217,7 +244,8 @@ class TestPublishCustomizedPipelineTemplateApi:
return_value=service,
),
):
response = method(api, "pipeline-1")
response, status = method(api, "pipeline-1")
service.publish_customized_pipeline_template.assert_called_once()
assert response == {"result": "success"}
assert status == 204
assert response == ""

84
api/tests/test_containers_integration_tests/controllers/console/datasets/rag_pipeline/test_rag_pipeline_datasets.py
View File

@ -2,12 +2,12 @@
from __future__ import annotations
from collections.abc import Callable
from typing import Protocol, cast
from unittest.mock import MagicMock, patch
from uuid import uuid4
import pytest
from flask import Flask
from sqlalchemy.orm import Session
from werkzeug.exceptions import Forbidden
import services
@ -17,17 +17,9 @@ from controllers.console.datasets.rag_pipeline.rag_pipeline_datasets import (
CreateEmptyRagPipelineDatasetApi,
CreateRagPipelineDatasetApi,
)
class _WrappedCallable(Protocol):
__wrapped__: Callable[..., object]
def unwrap(func: Callable[..., object]) -> Callable[..., object]:
current: Callable[..., object] | _WrappedCallable = func
while hasattr(current, "__wrapped__"):
current = cast(_WrappedCallable, current).__wrapped__
return cast(Callable[..., object], current)
from models.dataset import Dataset, DatasetPermissionEnum, DatasetRuntimeMode, Pipeline
from services.entities.dsl_entities import ImportStatus
from tests.test_containers_integration_tests.controllers.console.helpers import unwrap
class TestCreateRagPipelineDatasetApi:
@ -44,7 +36,15 @@ class TestCreateRagPipelineDatasetApi:
payload = self._valid_payload()
user = MagicMock(is_dataset_editor=True)
import_info = {"dataset_id": "ds-1"}
import_info = {
"id": "import-1",
"status": ImportStatus.COMPLETED,
"dataset_id": "ds-1",
"pipeline_id": "pipeline-1",
"current_dsl_version": "0.1.0",
"imported_dsl_version": "0.1.0",
"error": "",
}
mock_service = MagicMock()
mock_service.create_rag_pipeline_dataset.return_value = import_info
@ -57,10 +57,18 @@ class TestCreateRagPipelineDatasetApi:
return_value=mock_service,
),
):
response, status = cast(tuple[dict[str, str], int], method(api, "tenant-1", user))
response, status = method(api, "tenant-1", user)
assert status == 201
assert response == import_info
assert response == {
"id": "import-1",
"status": "completed",
"dataset_id": "ds-1",
"pipeline_id": "pipeline-1",
"current_dsl_version": "0.1.0",
"imported_dsl_version": "0.1.0",
"error": "",
}
def test_post_forbidden_non_editor(self, app: Flask) -> None:
api = CreateRagPipelineDatasetApi()
@ -117,12 +125,35 @@ class TestCreateEmptyRagPipelineDatasetApi:
def app(self, flask_app_with_containers: Flask) -> Flask:
return flask_app_with_containers
def test_post_success(self, app: Flask) -> None:
def test_post_success(self, app: Flask, db_session_with_containers: Session) -> None:
api = CreateEmptyRagPipelineDatasetApi()
method = unwrap(api.post)
user = MagicMock(is_dataset_editor=True)
dataset = MagicMock()
tenant_id = str(uuid4())
account_id = str(uuid4())
pipeline = Pipeline(
tenant_id=tenant_id,
name="Pipeline dataset",
description="",
created_by=account_id,
)
db_session_with_containers.add(pipeline)
db_session_with_containers.flush()
dataset = Dataset(
tenant_id=tenant_id,
name="Pipeline dataset",
description="",
permission=DatasetPermissionEnum.ONLY_ME,
provider="vendor",
runtime_mode=DatasetRuntimeMode.RAG_PIPELINE,
icon_info={"icon": "📙", "icon_background": "#FFF4ED", "icon_type": "emoji"},
created_by=account_id,
pipeline_id=pipeline.id,
)
db_session_with_containers.add(dataset)
db_session_with_containers.commit()
db_session_with_containers.expire_all()
with (
app.test_request_context("/"),
@ -130,15 +161,20 @@ class TestCreateEmptyRagPipelineDatasetApi:
"controllers.console.datasets.rag_pipeline.rag_pipeline_datasets.DatasetService.create_empty_rag_pipeline_dataset",
return_value=dataset,
),
patch(
"controllers.console.datasets.rag_pipeline.rag_pipeline_datasets.marshal",
return_value={"id": "ds-1"},
),
):
response, status = cast(tuple[dict[str, str], int], method(api, "tenant-1", user))
response, status = method(api, tenant_id, user)
assert status == 201
assert response == {"id": "ds-1"}
assert response["id"] == dataset.id
assert response["name"] == "Pipeline dataset"
assert response["pipeline_id"] == pipeline.id
assert response["runtime_mode"] == "rag_pipeline"
assert response["icon_info"] == {
"icon": "📙",
"icon_background": "#FFF4ED",
"icon_type": "emoji",
"icon_url": None,
}
def test_post_forbidden_non_editor(self, app: Flask) -> None:
api = CreateEmptyRagPipelineDatasetApi()

161
api/tests/test_containers_integration_tests/controllers/console/datasets/rag_pipeline/test_rag_pipeline_import.py
View File

@ -14,37 +14,39 @@ from controllers.console.datasets.rag_pipeline.rag_pipeline_import import (
RagPipelineImportCheckDependenciesApi,
RagPipelineImportConfirmApi,
)
from core.plugin.entities.plugin import PluginDependency
from models.dataset import Pipeline
from services.app_dsl_service import ImportStatus
def unwrap(func):
while hasattr(func, "__wrapped__"):
func = func.__wrapped__
return func
from services.entities.dsl_entities import CheckDependenciesResult, ImportStatus
from services.rag_pipeline.rag_pipeline_dsl_service import RagPipelineImportInfo
from tests.test_containers_integration_tests.controllers.console.helpers import unwrap
class TestRagPipelineImportApi:
@pytest.fixture
def app(self, flask_app_with_containers: Flask):
def app(self, flask_app_with_containers: Flask) -> Flask:
return flask_app_with_containers
def _payload(self, mode="create"):
def _payload(self, mode: str = "create") -> dict[str, str]:
return {
"mode": mode,
"yaml_content": "content",
"name": "Test",
}
def test_post_success_200(self, app: Flask):
def test_post_success_200(self, app: Flask) -> None:
api = RagPipelineImportApi()
method = unwrap(api.post)
payload = self._payload()
user = MagicMock()
result = MagicMock()
result.status = "completed"
result.model_dump.return_value = {"status": "success"}
result = RagPipelineImportInfo(
id="import-1",
status=ImportStatus.COMPLETED,
pipeline_id="pipeline-1",
dataset_id="dataset-1",
current_dsl_version="0.1.0",
imported_dsl_version="0.1.0",
)
service = MagicMock()
service.import_rag_pipeline.return_value = result
@ -60,17 +62,29 @@ class TestRagPipelineImportApi:
response, status = method(api, user)
assert status == 200
assert response == {"status": "success"}
assert response == {
"id": "import-1",
"status": "completed",
"pipeline_id": "pipeline-1",
"dataset_id": "dataset-1",
"current_dsl_version": "0.1.0",
"imported_dsl_version": "0.1.0",
"error": "",
}
def test_post_failed_400(self, app: Flask):
def test_post_failed_400(self, app: Flask) -> None:
api = RagPipelineImportApi()
method = unwrap(api.post)
payload = self._payload()
user = MagicMock()
result = MagicMock()
result.status = ImportStatus.FAILED
result.model_dump.return_value = {"status": "failed"}
result = RagPipelineImportInfo(
id="import-1",
status=ImportStatus.FAILED,
current_dsl_version="0.1.0",
imported_dsl_version="",
error="bad dsl",
)
service = MagicMock()
service.import_rag_pipeline.return_value = result
@ -86,17 +100,25 @@ class TestRagPipelineImportApi:
response, status = method(api, user)
assert status == 400
assert response == {"status": "failed"}
assert response["status"] == "failed"
assert response["error"] == "bad dsl"
assert response["pipeline_id"] is None
assert response["dataset_id"] is None
def test_post_pending_202(self, app: Flask):
def test_post_pending_202(self, app: Flask) -> None:
api = RagPipelineImportApi()
method = unwrap(api.post)
payload = self._payload()
user = MagicMock()
result = MagicMock()
result.status = ImportStatus.PENDING
result.model_dump.return_value = {"status": "pending"}
result = RagPipelineImportInfo(
id="import-1",
status=ImportStatus.PENDING,
pipeline_id="pipeline-1",
dataset_id="dataset-1",
current_dsl_version="0.1.0",
imported_dsl_version="0.2.0",
)
service = MagicMock()
service.import_rag_pipeline.return_value = result
@ -112,22 +134,29 @@ class TestRagPipelineImportApi:
response, status = method(api, user)
assert status == 202
assert response == {"status": "pending"}
assert response["status"] == "pending"
assert response["pipeline_id"] == "pipeline-1"
assert response["dataset_id"] == "dataset-1"
class TestRagPipelineImportConfirmApi:
@pytest.fixture
def app(self, flask_app_with_containers: Flask):
def app(self, flask_app_with_containers: Flask) -> Flask:
return flask_app_with_containers
def test_confirm_success(self, app: Flask):
def test_confirm_success(self, app: Flask) -> None:
api = RagPipelineImportConfirmApi()
method = unwrap(api.post)
user = MagicMock()
result = MagicMock()
result.status = "completed"
result.model_dump.return_value = {"ok": True}
result = RagPipelineImportInfo(
id="import-1",
status=ImportStatus.COMPLETED,
pipeline_id="pipeline-1",
dataset_id="dataset-1",
current_dsl_version="0.1.0",
imported_dsl_version="0.1.0",
)
service = MagicMock()
service.confirm_import.return_value = result
@ -142,16 +171,21 @@ class TestRagPipelineImportConfirmApi:
response, status = method(api, user, "import-1")
assert status == 200
assert response == {"ok": True}
assert response["status"] == "completed"
assert response["pipeline_id"] == "pipeline-1"
def test_confirm_failed(self, app: Flask):
def test_confirm_failed(self, app: Flask) -> None:
api = RagPipelineImportConfirmApi()
method = unwrap(api.post)
user = MagicMock()
result = MagicMock()
result.status = ImportStatus.FAILED
result.model_dump.return_value = {"ok": False}
result = RagPipelineImportInfo(
id="import-1",
status=ImportStatus.FAILED,
current_dsl_version="0.1.0",
imported_dsl_version="",
error="missing dependency",
)
service = MagicMock()
service.confirm_import.return_value = result
@ -166,21 +200,21 @@ class TestRagPipelineImportConfirmApi:
response, status = method(api, user, "import-1")
assert status == 400
assert response == {"ok": False}
assert response["status"] == "failed"
assert response["error"] == "missing dependency"
class TestRagPipelineImportCheckDependenciesApi:
@pytest.fixture
def app(self, flask_app_with_containers: Flask):
def app(self, flask_app_with_containers: Flask) -> Flask:
return flask_app_with_containers
def test_get_success(self, app: Flask):
def test_get_success(self, app: Flask) -> None:
api = RagPipelineImportCheckDependenciesApi()
method = unwrap(api.get)
pipeline = MagicMock(spec=Pipeline)
result = MagicMock()
result.model_dump.return_value = {"deps": []}
result = CheckDependenciesResult()
service = MagicMock()
service.check_dependencies.return_value = result
@ -195,21 +229,60 @@ class TestRagPipelineImportCheckDependenciesApi:
response, status = method(api, pipeline)
assert status == 200
assert response == {"deps": []}
assert response == {"leaked_dependencies": []}
def test_get_serializes_leaked_dependencies(self, app: Flask) -> None:
api = RagPipelineImportCheckDependenciesApi()
method = unwrap(api.get)
pipeline = MagicMock(spec=Pipeline)
dependency = PluginDependency(
type=PluginDependency.Type.Marketplace,
value=PluginDependency.Marketplace(
marketplace_plugin_unique_identifier="langgenius/example:0.1.0",
version="0.1.0",
),
current_identifier="langgenius/example:0.0.1",
)
service = MagicMock()
service.check_dependencies.return_value = CheckDependenciesResult(leaked_dependencies=[dependency])
with (
app.test_request_context("/"),
patch(
"controllers.console.datasets.rag_pipeline.rag_pipeline_import.RagPipelineDslService",
return_value=service,
),
):
response, status = method(api, pipeline)
assert status == 200
assert response == {
"leaked_dependencies": [
{
"type": "marketplace",
"value": {
"marketplace_plugin_unique_identifier": "langgenius/example:0.1.0",
"version": "0.1.0",
},
"current_identifier": "langgenius/example:0.0.1",
}
]
}
class TestRagPipelineExportApi:
@pytest.fixture
def app(self, flask_app_with_containers: Flask):
def app(self, flask_app_with_containers: Flask) -> Flask:
return flask_app_with_containers
def test_get_with_include_secret(self, app: Flask):
def test_get_with_include_secret(self, app: Flask) -> None:
api = RagPipelineExportApi()
method = unwrap(api.get)
pipeline = MagicMock(spec=Pipeline)
service = MagicMock()
service.export_rag_pipeline_dsl.return_value = {"yaml": "data"}
service.export_rag_pipeline_dsl.return_value = "yaml: data"
with (
app.test_request_context("/?include_secret=true"),
@ -221,4 +294,4 @@ class TestRagPipelineExportApi:
response, status = method(api, pipeline)
assert status == 200
assert response == {"data": {"yaml": "data"}}
assert response == {"data": "yaml: data"}

11
api/tests/test_containers_integration_tests/controllers/console/helpers.py
View File

@ -1,6 +1,8 @@
"""Shared helpers for authenticated console controller integration tests."""
"""Shared helpers for console controller integration tests."""
import uuid
from collections.abc import Callable
from typing import cast
from flask.testing import FlaskClient
from sqlalchemy import select
@ -83,3 +85,10 @@ def authenticate_console_client(test_client: FlaskClient, account: Account) -> d
"Authorization": f"Bearer {access_token}",
HEADER_NAME_CSRF_TOKEN: csrf_token,
}
def unwrap[R](func: Callable[..., R]) -> Callable[..., R]:
"""Return the undecorated callable for controller methods wrapped by Flask/RESTX."""
while hasattr(func, "__wrapped__"):
func = cast(Callable[..., R], func.__wrapped__)
return func

352
api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_rag_pipeline.py Normal file
View File

@ -0,0 +1,352 @@
from __future__ import annotations
from collections.abc import Callable
from typing import Any, cast
from unittest.mock import PropertyMock, patch
import pytest
from flask import Flask
from werkzeug.exceptions import NotFound
from controllers.console import console_ns
from controllers.console.datasets.rag_pipeline import rag_pipeline as module
from controllers.console.datasets.rag_pipeline.rag_pipeline import (
CustomizedPipelineTemplateApi,
PipelineTemplateDetailApi,
PipelineTemplateListApi,
PublishCustomizedPipelineTemplateApi,
)
from models.dataset import PipelineCustomizedTemplate
from services.entities.knowledge_entities.rag_pipeline_entities import PipelineTemplateInfoEntity
def _unwrap(func: object) -> Callable[..., Any]:
while hasattr(func, "__wrapped__"):
func = func.__wrapped__
return cast(Callable[..., Any], func)
def _template_item() -> dict[str, object]:
return {
"id": "template-1",
"name": "Template",
"icon": {"icon": "book", "icon_type": "emoji", "icon_background": "#fff"},
"description": "Description",
"position": 1,
"chunk_structure": "general",
}
def _template_detail() -> dict[str, object]:
return {
"id": "template-1",
"name": "Template",
"icon_info": {"icon": "book", "icon_type": "emoji", "icon_background": "#fff"},
"description": "Description",
"chunk_structure": "general",
"export_data": "dsl: value",
"graph": {"nodes": [], "edges": [], "viewport": {"x": 0, "y": 0, "zoom": 1}},
}
def _payload() -> dict[str, object]:
return {
"name": "Updated template",
"description": "Updated description",
"icon_info": {"icon": "book", "icon_type": "emoji", "icon_background": "#fff"},
}
class TestPipelineTemplateListApi:
def test_get_uses_query_defaults_and_serializes_nullable_fields(self, app: Flask) -> None:
api = PipelineTemplateListApi()
method = _unwrap(api.get)
service_calls: list[tuple[str, str]] = []
def get_pipeline_templates(template_type: str, language: str) -> dict[str, object]:
service_calls.append((template_type, language))
return {"pipeline_templates": [_template_item()]}
with (
app.test_request_context("/rag/pipeline/templates"),
patch.object(module.RagPipelineService, "get_pipeline_templates", side_effect=get_pipeline_templates),
):
response, status = method(api)
assert status == 200
assert service_calls == [("built-in", "en-US")]
assert response == {
"pipeline_templates": [
{
**_template_item(),
"copyright": None,
"privacy_policy": None,
}
]
}
def test_get_passes_explicit_query_to_service(self, app: Flask) -> None:
api = PipelineTemplateListApi()
method = _unwrap(api.get)
service_calls: list[tuple[str, str]] = []
def get_pipeline_templates(template_type: str, language: str) -> dict[str, object]:
service_calls.append((template_type, language))
return {"pipeline_templates": []}
with (
app.test_request_context("/rag/pipeline/templates?type=customized&language=ja-JP"),
patch.object(module.RagPipelineService, "get_pipeline_templates", side_effect=get_pipeline_templates),
):
response, status = method(api)
assert status == 200
assert response == {"pipeline_templates": []}
assert service_calls == [("customized", "ja-JP")]
class TestPipelineTemplateDetailApi:
def test_get_serializes_template_detail(self, app: Flask) -> None:
api = PipelineTemplateDetailApi()
method = _unwrap(api.get)
service_calls: list[tuple[str, str]] = []
class Service:
def get_pipeline_template_detail(self, template_id: str, template_type: str) -> dict[str, object]:
service_calls.append((template_id, template_type))
return _template_detail()
with (
app.test_request_context("/rag/pipeline/templates/template-1?type=customized"),
patch.object(module, "RagPipelineService", Service),
):
response, status = method(api, "template-1")
assert status == 200
assert response == {**_template_detail(), "created_by": None}
assert service_calls == [("template-1", "customized")]
def test_get_raises_not_found_without_custom_response_body(self, app: Flask) -> None:
api = PipelineTemplateDetailApi()
method = _unwrap(api.get)
class Service:
def get_pipeline_template_detail(self, template_id: str, template_type: str) -> None:
return None
with (
app.test_request_context("/rag/pipeline/templates/missing"),
patch.object(module, "RagPipelineService", Service),
):
with pytest.raises(NotFound):
method(api, "missing")
class TestCustomizedPipelineTemplateApi:
def test_patch_validates_payload_and_returns_empty_204(self, app: Flask) -> None:
api = CustomizedPipelineTemplateApi()
method = _unwrap(api.patch)
payload = _payload()
service_calls: list[tuple[str, PipelineTemplateInfoEntity]] = []
def update_template(template_id: str, template_info: PipelineTemplateInfoEntity) -> None:
service_calls.append((template_id, template_info))
with (
app.test_request_context("/rag/pipeline/customized/templates/template-1", method="PATCH", json=payload),
patch.object(type(console_ns), "payload", new_callable=PropertyMock, return_value=payload),
patch.object(module.RagPipelineService, "update_customized_pipeline_template", side_effect=update_template),
):
response, status = method(api, "template-1")
assert (response, status) == ("", 204)
assert len(service_calls) == 1
template_id, template_info = service_calls[0]
assert template_id == "template-1"
assert template_info.name == "Updated template"
assert template_info.description == "Updated description"
assert template_info.icon_info.model_dump() == {
"icon": "book",
"icon_background": "#fff",
"icon_type": "emoji",
"icon_url": None,
}
def test_patch_defaults_missing_icon_info_before_service_call(self, app: Flask) -> None:
api = CustomizedPipelineTemplateApi()
method = _unwrap(api.patch)
payload: dict[str, object] = {
"name": "Updated template",
"description": "Updated description",
}
service_calls: list[tuple[str, PipelineTemplateInfoEntity]] = []
def update_template(template_id: str, template_info: PipelineTemplateInfoEntity) -> None:
service_calls.append((template_id, template_info))
with (
app.test_request_context("/rag/pipeline/customized/templates/template-1", method="PATCH", json=payload),
patch.object(type(console_ns), "payload", new_callable=PropertyMock, return_value=payload),
patch.object(module.RagPipelineService, "update_customized_pipeline_template", side_effect=update_template),
):
response, status = method(api, "template-1")
assert (response, status) == ("", 204)
assert len(service_calls) == 1
template_id, template_info = service_calls[0]
assert template_id == "template-1"
assert template_info.icon_info.model_dump() == {
"icon": "",
"icon_background": None,
"icon_type": None,
"icon_url": None,
}
def test_delete_returns_empty_204(self, app: Flask) -> None:
api = CustomizedPipelineTemplateApi()
method = _unwrap(api.delete)
deleted_template_ids: list[str] = []
def delete_template(template_id: str) -> None:
deleted_template_ids.append(template_id)
with (
app.test_request_context("/rag/pipeline/customized/templates/template-1", method="DELETE"),
patch.object(module.RagPipelineService, "delete_customized_pipeline_template", side_effect=delete_template),
):
response, status = method(api, "template-1")
assert (response, status) == ("", 204)
assert deleted_template_ids == ["template-1"]
def test_post_exports_yaml_from_orm_template(self, app: Flask) -> None:
api = CustomizedPipelineTemplateApi()
method = _unwrap(api.post)
template = PipelineCustomizedTemplate(
tenant_id="00000000-0000-0000-0000-000000000001",
name="Template",
description="Description",
chunk_structure="general",
icon={"icon": "book", "icon_type": "emoji", "icon_background": "#fff"},
position=1,
yaml_content="dsl: value",
install_count=0,
language="en-US",
created_by="00000000-0000-0000-0000-000000000002",
)
class Session:
def scalar(self, stmt: object) -> PipelineCustomizedTemplate:
return template
class SessionContext:
def __enter__(self) -> Session:
return Session()
def __exit__(self, exc_type: object, exc: object, tb: object) -> bool:
return False
class SessionMaker:
def begin(self) -> SessionContext:
return SessionContext()
class Database:
engine = object()
with (
app.test_request_context("/rag/pipeline/customized/templates/template-1", method="POST"),
patch.object(module, "db", Database()),
patch.object(module, "sessionmaker", return_value=SessionMaker()),
):
response, status = method(api, "template-1")
assert status == 200
assert response == {"data": "dsl: value"}
def test_post_raises_when_template_is_missing(self, app: Flask) -> None:
api = CustomizedPipelineTemplateApi()
method = _unwrap(api.post)
class Session:
def scalar(self, stmt: object) -> None:
return None
class SessionContext:
def __enter__(self) -> Session:
return Session()
def __exit__(self, exc_type: object, exc: object, tb: object) -> bool:
return False
class SessionMaker:
def begin(self) -> SessionContext:
return SessionContext()
class Database:
engine = object()
with (
app.test_request_context("/rag/pipeline/customized/templates/missing", method="POST"),
patch.object(module, "db", Database()),
patch.object(module, "sessionmaker", return_value=SessionMaker()),
):
with pytest.raises(ValueError, match="Customized pipeline template not found"):
method(api, "missing")
class TestPublishCustomizedPipelineTemplateApi:
def test_post_validates_payload_and_returns_empty_204(self, app: Flask) -> None:
api = PublishCustomizedPipelineTemplateApi()
method = _unwrap(api.post)
payload = _payload()
service_calls: list[tuple[str, dict[str, object]]] = []
class Service:
def publish_customized_pipeline_template(self, pipeline_id: str, data: dict[str, object]) -> None:
service_calls.append((pipeline_id, data))
with (
app.test_request_context("/rag/pipelines/pipeline-1/customized/publish", method="POST", json=payload),
patch.object(type(console_ns), "payload", new_callable=PropertyMock, return_value=payload),
patch.object(module, "RagPipelineService", Service),
):
response, status = method(api, "pipeline-1")
assert (response, status) == ("", 204)
assert service_calls == [("pipeline-1", payload)]
def test_post_allows_missing_icon_info_for_publish_service_fallback(self, app: Flask) -> None:
api = PublishCustomizedPipelineTemplateApi()
method = _unwrap(api.post)
payload: dict[str, object] = {
"name": "Published template",
"description": "Description",
}
service_calls: list[tuple[str, dict[str, object]]] = []
class Service:
def publish_customized_pipeline_template(self, pipeline_id: str, data: dict[str, object]) -> None:
service_calls.append((pipeline_id, data))
with (
app.test_request_context("/rag/pipelines/pipeline-1/customized/publish", method="POST", json=payload),
patch.object(type(console_ns), "payload", new_callable=PropertyMock, return_value=payload),
patch.object(module, "RagPipelineService", Service),
):
response, status = method(api, "pipeline-1")
assert (response, status) == ("", 204)
assert service_calls == [
(
"pipeline-1",
{
**payload,
"icon_info": {
"icon": "",
"icon_background": None,
"icon_type": None,
"icon_url": None,
},
},
)
]

87
packages/contracts/generated/api/console/rag/orpc.gen.ts
View File

@ -15,6 +15,7 @@ import {
zDeleteRagPipelinesByPipelineIdWorkflowsDraftVariablesPath,
zDeleteRagPipelinesByPipelineIdWorkflowsDraftVariablesResponse,
zGetRagPipelinesByPipelineIdExportsPath,
zGetRagPipelinesByPipelineIdExportsQuery,
zGetRagPipelinesByPipelineIdExportsResponse,
zGetRagPipelinesByPipelineIdWorkflowRunsByRunIdNodeExecutionsPath,
zGetRagPipelinesByPipelineIdWorkflowRunsByRunIdNodeExecutionsResponse,
@ -57,8 +58,11 @@ import {
zGetRagPipelinesImportsByPipelineIdCheckDependenciesResponse,
zGetRagPipelinesRecommendedPluginsResponse,
zGetRagPipelineTemplatesByTemplateIdPath,
zGetRagPipelineTemplatesByTemplateIdQuery,
zGetRagPipelineTemplatesByTemplateIdResponse,
zGetRagPipelineTemplatesQuery,
zGetRagPipelineTemplatesResponse,
zPatchRagPipelineCustomizedTemplatesByTemplateIdBody,
zPatchRagPipelineCustomizedTemplatesByTemplateIdPath,
zPatchRagPipelineCustomizedTemplatesByTemplateIdResponse,
zPatchRagPipelinesByPipelineIdWorkflowsByWorkflowIdPath,
@ -118,20 +122,13 @@ import {
zPutRagPipelinesByPipelineIdWorkflowsDraftVariablesByVariableIdResetResponse,
} from './zod.gen'
/**
* Generated contract types may be inaccurate because backend OpenAPI annotations are incomplete. Do not migrate callers until the generated contract is accurate.
*
* @deprecated
*/
export const delete_ = oc
.route({
deprecated: true,
description:
'Generated contract types may be inaccurate because backend OpenAPI annotations are incomplete. Do not migrate callers until the generated contract is accurate.',
inputStructure: 'detailed',
method: 'DELETE',
operationId: 'deleteRagPipelineCustomizedTemplatesByTemplateId',
path: '/rag/pipeline/customized/templates/{template_id}',
successStatus: 204,
tags: ['console'],
})
.input(z.object({ params: zDeleteRagPipelineCustomizedTemplatesByTemplateIdPath }))
@ -151,9 +148,15 @@ export const patch = oc
method: 'PATCH',
operationId: 'patchRagPipelineCustomizedTemplatesByTemplateId',
path: '/rag/pipeline/customized/templates/{template_id}',
successStatus: 204,
tags: ['console'],
})
.input(z.object({ params: zPatchRagPipelineCustomizedTemplatesByTemplateIdPath }))
.input(
z.object({
body: zPatchRagPipelineCustomizedTemplatesByTemplateIdBody,
params: zPatchRagPipelineCustomizedTemplatesByTemplateIdPath,
}),
)
.output(zPatchRagPipelineCustomizedTemplatesByTemplateIdResponse)
export const post = oc
@ -181,20 +184,13 @@ export const customized = {
templates,
}
/**
* Generated contract types may be inaccurate because backend OpenAPI annotations are incomplete. Do not migrate callers until the generated contract is accurate.
*
* @deprecated
*/
export const post2 = oc
.route({
deprecated: true,
description:
'Generated contract types may be inaccurate because backend OpenAPI annotations are incomplete. Do not migrate callers until the generated contract is accurate.',
inputStructure: 'detailed',
method: 'POST',
operationId: 'postRagPipelineDataset',
path: '/rag/pipeline/dataset',
successStatus: 201,
tags: ['console'],
})
.input(z.object({ body: zPostRagPipelineDatasetBody }))
@ -204,20 +200,13 @@ export const dataset = {
post: post2,
}
/**
* Generated contract types may be inaccurate because backend OpenAPI annotations are incomplete. Do not migrate callers until the generated contract is accurate.
*
* @deprecated
*/
export const post3 = oc
.route({
deprecated: true,
description:
'Generated contract types may be inaccurate because backend OpenAPI annotations are incomplete. Do not migrate callers until the generated contract is accurate.',
inputStructure: 'detailed',
method: 'POST',
operationId: 'postRagPipelineEmptyDataset',
path: '/rag/pipeline/empty-dataset',
successStatus: 201,
tags: ['console'],
})
.output(zPostRagPipelineEmptyDatasetResponse)
@ -242,7 +231,12 @@ export const get = oc
path: '/rag/pipeline/templates/{template_id}',
tags: ['console'],
})
.input(z.object({ params: zGetRagPipelineTemplatesByTemplateIdPath }))
.input(
z.object({
params: zGetRagPipelineTemplatesByTemplateIdPath,
query: zGetRagPipelineTemplatesByTemplateIdQuery.optional(),
}),
)
.output(zGetRagPipelineTemplatesByTemplateIdResponse)
export const byTemplateId2 = {
@ -265,6 +259,7 @@ export const get2 = oc
path: '/rag/pipeline/templates',
tags: ['console'],
})
.input(z.object({ query: zGetRagPipelineTemplatesQuery.optional() }))
.output(zGetRagPipelineTemplatesResponse)
export const templates2 = {
@ -301,16 +296,8 @@ export const datasourcePlugins = {
get: get3,
}
/**
* Generated contract types may be inaccurate because backend OpenAPI annotations are incomplete. Do not migrate callers until the generated contract is accurate.
*
* @deprecated
*/
export const post4 = oc
.route({
deprecated: true,
description:
'Generated contract types may be inaccurate because backend OpenAPI annotations are incomplete. Do not migrate callers until the generated contract is accurate.',
inputStructure: 'detailed',
method: 'POST',
operationId: 'postRagPipelinesImportsByImportIdConfirm',
@ -328,16 +315,8 @@ export const byImportId = {
confirm,
}
/**
* Generated contract types may be inaccurate because backend OpenAPI annotations are incomplete. Do not migrate callers until the generated contract is accurate.
*
* @deprecated
*/
export const get4 = oc
.route({
deprecated: true,
description:
'Generated contract types may be inaccurate because backend OpenAPI annotations are incomplete. Do not migrate callers until the generated contract is accurate.',
inputStructure: 'detailed',
method: 'GET',
operationId: 'getRagPipelinesImportsByPipelineIdCheckDependencies',
@ -355,16 +334,8 @@ export const byPipelineId = {
checkDependencies,
}
/**
* Generated contract types may be inaccurate because backend OpenAPI annotations are incomplete. Do not migrate callers until the generated contract is accurate.
*
* @deprecated
*/
export const post5 = oc
.route({
deprecated: true,
description:
'Generated contract types may be inaccurate because backend OpenAPI annotations are incomplete. Do not migrate callers until the generated contract is accurate.',
inputStructure: 'detailed',
method: 'POST',
operationId: 'postRagPipelinesImports',
@ -447,6 +418,7 @@ export const post7 = oc
method: 'POST',
operationId: 'postRagPipelinesByPipelineIdCustomizedPublish',
path: '/rag/pipelines/{pipeline_id}/customized/publish',
successStatus: 204,
tags: ['console'],
})
.input(
@ -465,23 +437,20 @@ export const customized2 = {
publish,
}
/**
* Generated contract types may be inaccurate because backend OpenAPI annotations are incomplete. Do not migrate callers until the generated contract is accurate.
*
* @deprecated
*/
export const get6 = oc
.route({
deprecated: true,
description:
'Generated contract types may be inaccurate because backend OpenAPI annotations are incomplete. Do not migrate callers until the generated contract is accurate.',
inputStructure: 'detailed',
method: 'GET',
operationId: 'getRagPipelinesByPipelineIdExports',
path: '/rag/pipelines/{pipeline_id}/exports',
tags: ['console'],
})
.input(z.object({ params: zGetRagPipelinesByPipelineIdExportsPath }))
.input(
z.object({
params: zGetRagPipelinesByPipelineIdExportsPath,
query: zGetRagPipelinesByPipelineIdExportsQuery.optional(),
}),
)
.output(zGetRagPipelinesByPipelineIdExportsResponse)
export const exports_ = {

270
packages/contracts/generated/api/console/rag/types.gen.ts
View File

@ -4,6 +4,14 @@ export type ClientOptions = {
baseUrl: `${string}://${string}/console/api` | (string & {})
}
export type CustomizedPipelineTemplatePayload = {
description?: string
icon_info?: {
[key: string]: unknown
}
name: string
}
export type SimpleDataResponse = {
data: string
}
@ -12,6 +20,73 @@ export type RagPipelineDatasetImportPayload = {
yaml_content: string
}
export type RagPipelineImportResponse = {
current_dsl_version: string
dataset_id?: string | null
error?: string
id: string
imported_dsl_version: string
pipeline_id?: string | null
status: ImportStatus
}
export type DatasetDetailResponse = {
app_count: number
author_name: string | null
built_in_field_enabled: boolean
chunk_structure: string | null
created_at: number
created_by: string
data_source_type: string | null
description: string | null
doc_form: string | null
doc_metadata: Array<DatasetDocMetadataResponse>
document_count: number
embedding_available?: boolean | null
embedding_model: string | null
embedding_model_provider: string | null
enable_api: boolean
external_knowledge_info?: DatasetExternalKnowledgeInfoResponse
external_retrieval_model: DatasetExternalRetrievalModelResponse
icon_info?: DatasetIconInfoResponse
id: string
indexing_technique: string | null
is_multimodal: boolean
is_published: boolean
name: string
permission: string
pipeline_id: string | null
provider: string
retrieval_model_dict: DatasetRetrievalModelResponse
runtime_mode: string | null
summary_index_setting?: DatasetSummaryIndexSettingResponse
tags: Array<DatasetTagResponse>
total_available_documents: number
total_documents: number
updated_at: number
updated_by: string | null
word_count: number
}
export type PipelineTemplateListResponse = {
pipeline_templates: Array<PipelineTemplateItemResponse>
}
export type PipelineTemplateDetailResponse = {
chunk_structure: string
created_by?: string | null
description: string
export_data: string
graph: {
[key: string]: unknown
}
icon_info: {
[key: string]: unknown
}
id: string
name: string
}
export type RagPipelineImportPayload = {
description?: string | null
icon?: string | null
@ -24,12 +99,8 @@ export type RagPipelineImportPayload = {
yaml_url?: string | null
}
export type Payload = {
description?: string
icon_info?: {
[key: string]: unknown
} | null
name: string
export type RagPipelineImportCheckDependenciesResponse = {
leaked_dependencies?: Array<PluginDependency>
}
export type WorkflowRunPaginationResponse = {
@ -192,6 +263,77 @@ export type PublishedWorkflowRunPayload = {
start_node_id: string
}
export type ImportStatus = 'completed' | 'completed-with-warnings' | 'failed' | 'pending'
export type DatasetDocMetadataResponse = {
id: string
name: string
type: string
}
export type DatasetExternalKnowledgeInfoResponse = {
external_knowledge_api_endpoint?: string | null
external_knowledge_api_id?: string | null
external_knowledge_api_name?: string | null
external_knowledge_id?: string | null
}
export type DatasetExternalRetrievalModelResponse = {
score_threshold?: number | null
score_threshold_enabled?: boolean | null
top_k: number
}
export type DatasetIconInfoResponse = {
icon?: string | null
icon_background?: string | null
icon_type?: string | null
icon_url?: string | null
}
export type DatasetRetrievalModelResponse = {
reranking_enable: boolean
reranking_mode?: string | null
reranking_model?: DatasetRerankingModelResponse
score_threshold?: number | null
score_threshold_enabled: boolean
search_method: string
top_k: number
weights?: DatasetWeightedScoreResponse
}
export type DatasetSummaryIndexSettingResponse = {
enable?: boolean | null
model_name?: string | null
model_provider_name?: string | null
summary_prompt?: string | null
}
export type DatasetTagResponse = {
id: string
name: string
type: string
}
export type PipelineTemplateItemResponse = {
chunk_structure: string
copyright?: string | null
description: string
icon: {
[key: string]: unknown
}
id: string
name: string
position: number
privacy_policy?: string | null
}
export type PluginDependency = {
current_identifier?: string | null
type: Type
value: unknown
}
export type WorkflowRunForListResponse = {
created_at?: number | null
created_by_account?: SimpleAccount
@ -258,6 +400,46 @@ export type PipelineVariableResponse = {
variable: string
}
export type DatasetRerankingModelResponse = {
reranking_model_name?: string | null
reranking_provider_name?: string | null
}
export type DatasetWeightedScoreResponse = {
keyword_setting?: DatasetKeywordSettingResponse
vector_setting?: DatasetVectorSettingResponse
weight_type?: string | null
}
export type Type = 'github' | 'marketplace' | 'package'
export type Github = {
github_plugin_unique_identifier: string
package: string
repo: string
version: string
}
export type Marketplace = {
marketplace_plugin_unique_identifier: string
version?: string | null
}
export type Package = {
plugin_unique_identifier: string
version?: string | null
}
export type DatasetKeywordSettingResponse = {
keyword_weight?: number | null
}
export type DatasetVectorSettingResponse = {
embedding_model_name?: string | null
embedding_provider_name?: string | null
vector_weight?: number | null
}
export type DeleteRagPipelineCustomizedTemplatesByTemplateIdData = {
body?: never
path: {
@ -268,8 +450,8 @@ export type DeleteRagPipelineCustomizedTemplatesByTemplateIdData = {
}
export type DeleteRagPipelineCustomizedTemplatesByTemplateIdResponses = {
200: {
[key: string]: unknown
204: {
[key: string]: never
}
}
@ -277,7 +459,7 @@ export type DeleteRagPipelineCustomizedTemplatesByTemplateIdResponse
= DeleteRagPipelineCustomizedTemplatesByTemplateIdResponses[keyof DeleteRagPipelineCustomizedTemplatesByTemplateIdResponses]
export type PatchRagPipelineCustomizedTemplatesByTemplateIdData = {
body?: never
body: CustomizedPipelineTemplatePayload
path: {
template_id: string
}
@ -286,8 +468,8 @@ export type PatchRagPipelineCustomizedTemplatesByTemplateIdData = {
}
export type PatchRagPipelineCustomizedTemplatesByTemplateIdResponses = {
200: {
[key: string]: unknown
204: {
[key: string]: never
}
}
@ -318,9 +500,7 @@ export type PostRagPipelineDatasetData = {
}
export type PostRagPipelineDatasetResponses = {
200: {
[key: string]: unknown
}
201: RagPipelineImportResponse
}
export type PostRagPipelineDatasetResponse
@ -334,9 +514,7 @@ export type PostRagPipelineEmptyDatasetData = {
}
export type PostRagPipelineEmptyDatasetResponses = {
200: {
[key: string]: unknown
}
201: DatasetDetailResponse
}
export type PostRagPipelineEmptyDatasetResponse
@ -345,14 +523,15 @@ export type PostRagPipelineEmptyDatasetResponse
export type GetRagPipelineTemplatesData = {
body?: never
path?: never
query?: never
query?: {
language?: string
type?: string
}
url: '/rag/pipeline/templates'
}
export type GetRagPipelineTemplatesResponses = {
200: {
[key: string]: unknown
}
200: PipelineTemplateListResponse
}
export type GetRagPipelineTemplatesResponse
@ -363,14 +542,14 @@ export type GetRagPipelineTemplatesByTemplateIdData = {
path: {
template_id: string
}
query?: never
query?: {
type?: string
}
url: '/rag/pipeline/templates/{template_id}'
}
export type GetRagPipelineTemplatesByTemplateIdResponses = {
200: {
[key: string]: unknown
}
200: PipelineTemplateDetailResponse
}
export type GetRagPipelineTemplatesByTemplateIdResponse
@ -399,10 +578,16 @@ export type PostRagPipelinesImportsData = {
url: '/rag/pipelines/imports'
}
export type PostRagPipelinesImportsErrors = {
400: RagPipelineImportResponse
}
export type PostRagPipelinesImportsError
= PostRagPipelinesImportsErrors[keyof PostRagPipelinesImportsErrors]
export type PostRagPipelinesImportsResponses = {
200: {
[key: string]: unknown
}
200: RagPipelineImportResponse
202: RagPipelineImportResponse
}
export type PostRagPipelinesImportsResponse
@ -417,10 +602,15 @@ export type PostRagPipelinesImportsByImportIdConfirmData = {
url: '/rag/pipelines/imports/{import_id}/confirm'
}
export type PostRagPipelinesImportsByImportIdConfirmErrors = {
400: RagPipelineImportResponse
}
export type PostRagPipelinesImportsByImportIdConfirmError
= PostRagPipelinesImportsByImportIdConfirmErrors[keyof PostRagPipelinesImportsByImportIdConfirmErrors]
export type PostRagPipelinesImportsByImportIdConfirmResponses = {
200: {
[key: string]: unknown
}
200: RagPipelineImportResponse
}
export type PostRagPipelinesImportsByImportIdConfirmResponse
@ -436,9 +626,7 @@ export type GetRagPipelinesImportsByPipelineIdCheckDependenciesData = {
}
export type GetRagPipelinesImportsByPipelineIdCheckDependenciesResponses = {
200: {
[key: string]: unknown
}
200: RagPipelineImportCheckDependenciesResponse
}
export type GetRagPipelinesImportsByPipelineIdCheckDependenciesResponse
@ -479,7 +667,7 @@ export type PostRagPipelinesTransformDatasetsByDatasetIdResponse
= PostRagPipelinesTransformDatasetsByDatasetIdResponses[keyof PostRagPipelinesTransformDatasetsByDatasetIdResponses]
export type PostRagPipelinesByPipelineIdCustomizedPublishData = {
body: Payload
body: CustomizedPipelineTemplatePayload
path: {
pipeline_id: string
}
@ -488,8 +676,8 @@ export type PostRagPipelinesByPipelineIdCustomizedPublishData = {
}
export type PostRagPipelinesByPipelineIdCustomizedPublishResponses = {
200: {
[key: string]: unknown
204: {
[key: string]: never
}
}
@ -501,14 +689,14 @@ export type GetRagPipelinesByPipelineIdExportsData = {
path: {
pipeline_id: string
}
query?: never
query?: {
include_secret?: string
}
url: '/rag/pipelines/{pipeline_id}/exports'
}
export type GetRagPipelinesByPipelineIdExportsResponses = {
200: {
[key: string]: unknown
}
200: SimpleDataResponse
}
export type GetRagPipelinesByPipelineIdExportsResponse

329
packages/contracts/generated/api/console/rag/zod.gen.ts
View File

@ -2,6 +2,15 @@
import * as z from 'zod'
/**
* CustomizedPipelineTemplatePayload
*/
export const zCustomizedPipelineTemplatePayload = z.object({
description: z.string().max(400).optional().default(''),
icon_info: z.record(z.string(), z.unknown()).optional(),
name: z.string().min(1).max(40),
})
/**
* SimpleDataResponse
*/
@ -16,6 +25,20 @@ export const zRagPipelineDatasetImportPayload = z.object({
yaml_content: z.string(),
})
/**
* PipelineTemplateDetailResponse
*/
export const zPipelineTemplateDetailResponse = z.object({
chunk_structure: z.string(),
created_by: z.string().nullish(),
description: z.string(),
export_data: z.string(),
graph: z.record(z.string(), z.unknown()),
icon_info: z.record(z.string(), z.unknown()),
id: z.string(),
name: z.string(),
})
/**
* RagPipelineImportPayload
*/
@ -31,15 +54,6 @@ export const zRagPipelineImportPayload = z.object({
yaml_url: z.string().nullish(),
})
/**
* Payload
*/
export const zPayload = z.object({
description: z.string().max(400).optional().default(''),
icon_info: z.record(z.string(), z.unknown()).nullish(),
name: z.string().min(1).max(40),
})
/**
* SimpleResultResponse
*/
@ -129,6 +143,102 @@ export const zPublishedWorkflowRunPayload = z.object({
start_node_id: z.string(),
})
/**
* ImportStatus
*/
export const zImportStatus = z.enum(['completed', 'completed-with-warnings', 'failed', 'pending'])
/**
* RagPipelineImportResponse
*/
export const zRagPipelineImportResponse = z.object({
current_dsl_version: z.string(),
dataset_id: z.string().nullish(),
error: z.string().optional().default(''),
id: z.string(),
imported_dsl_version: z.string(),
pipeline_id: z.string().nullish(),
status: zImportStatus,
})
/**
* DatasetDocMetadataResponse
*/
export const zDatasetDocMetadataResponse = z.object({
id: z.string(),
name: z.string(),
type: z.string(),
})
/**
* DatasetExternalKnowledgeInfoResponse
*/
export const zDatasetExternalKnowledgeInfoResponse = z.object({
external_knowledge_api_endpoint: z.string().nullish(),
external_knowledge_api_id: z.string().nullish(),
external_knowledge_api_name: z.string().nullish(),
external_knowledge_id: z.string().nullish(),
})
/**
* DatasetExternalRetrievalModelResponse
*/
export const zDatasetExternalRetrievalModelResponse = z.object({
score_threshold: z.number().nullish(),
score_threshold_enabled: z.boolean().nullish(),
top_k: z.int(),
})
/**
* DatasetIconInfoResponse
*/
export const zDatasetIconInfoResponse = z.object({
icon: z.string().nullish(),
icon_background: z.string().nullish(),
icon_type: z.string().nullish(),
icon_url: z.string().nullish(),
})
/**
* DatasetSummaryIndexSettingResponse
*/
export const zDatasetSummaryIndexSettingResponse = z.object({
enable: z.boolean().nullish(),
model_name: z.string().nullish(),
model_provider_name: z.string().nullish(),
summary_prompt: z.string().nullish(),
})
/**
* DatasetTagResponse
*/
export const zDatasetTagResponse = z.object({
id: z.string(),
name: z.string(),
type: z.string(),
})
/**
* PipelineTemplateItemResponse
*/
export const zPipelineTemplateItemResponse = z.object({
chunk_structure: z.string(),
copyright: z.string().nullish(),
description: z.string(),
icon: z.record(z.string(), z.unknown()),
id: z.string(),
name: z.string(),
position: z.int(),
privacy_policy: z.string().nullish(),
})
/**
* PipelineTemplateListResponse
*/
export const zPipelineTemplateListResponse = z.object({
pipeline_templates: z.array(zPipelineTemplateItemResponse),
})
/**
* SimpleAccount
*/
@ -304,28 +414,166 @@ export const zWorkflowPaginationResponse = z.object({
page: z.int(),
})
/**
* DatasetRerankingModelResponse
*/
export const zDatasetRerankingModelResponse = z.object({
reranking_model_name: z.string().nullish(),
reranking_provider_name: z.string().nullish(),
})
/**
* Type
*/
export const zType = z.enum(['github', 'marketplace', 'package'])
/**
* PluginDependency
*/
export const zPluginDependency = z.object({
current_identifier: z.string().nullish(),
type: zType,
value: z.unknown(),
})
/**
* RagPipelineImportCheckDependenciesResponse
*/
export const zRagPipelineImportCheckDependenciesResponse = z.object({
leaked_dependencies: z.array(zPluginDependency).optional(),
})
/**
* Github
*/
export const zGithub = z.object({
github_plugin_unique_identifier: z.string(),
package: z.string(),
repo: z.string(),
version: z.string(),
})
/**
* Marketplace
*/
export const zMarketplace = z.object({
marketplace_plugin_unique_identifier: z.string(),
version: z.string().nullish(),
})
/**
* Package
*/
export const zPackage = z.object({
plugin_unique_identifier: z.string(),
version: z.string().nullish(),
})
/**
* DatasetKeywordSettingResponse
*/
export const zDatasetKeywordSettingResponse = z.object({
keyword_weight: z.number().nullish(),
})
/**
* DatasetVectorSettingResponse
*/
export const zDatasetVectorSettingResponse = z.object({
embedding_model_name: z.string().nullish(),
embedding_provider_name: z.string().nullish(),
vector_weight: z.number().nullish(),
})
/**
* DatasetWeightedScoreResponse
*/
export const zDatasetWeightedScoreResponse = z.object({
keyword_setting: zDatasetKeywordSettingResponse.optional(),
vector_setting: zDatasetVectorSettingResponse.optional(),
weight_type: z.string().nullish(),
})
/**
* DatasetRetrievalModelResponse
*/
export const zDatasetRetrievalModelResponse = z.object({
reranking_enable: z.boolean(),
reranking_mode: z.string().nullish(),
reranking_model: zDatasetRerankingModelResponse.optional(),
score_threshold: z.number().nullish(),
score_threshold_enabled: z.boolean(),
search_method: z.string(),
top_k: z.int(),
weights: zDatasetWeightedScoreResponse.optional(),
})
/**
* DatasetDetailResponse
*/
export const zDatasetDetailResponse = z.object({
app_count: z.int(),
author_name: z.string().nullable(),
built_in_field_enabled: z.boolean(),
chunk_structure: z.string().nullable(),
created_at: z.int(),
created_by: z.string(),
data_source_type: z.string().nullable(),
description: z.string().nullable(),
doc_form: z.string().nullable(),
doc_metadata: z.array(zDatasetDocMetadataResponse),
document_count: z.int(),
embedding_available: z.boolean().nullish(),
embedding_model: z.string().nullable(),
embedding_model_provider: z.string().nullable(),
enable_api: z.boolean(),
external_knowledge_info: zDatasetExternalKnowledgeInfoResponse.optional(),
external_retrieval_model: zDatasetExternalRetrievalModelResponse,
icon_info: zDatasetIconInfoResponse.optional(),
id: z.string(),
indexing_technique: z.string().nullable(),
is_multimodal: z.boolean(),
is_published: z.boolean(),
name: z.string(),
permission: z.string(),
pipeline_id: z.string().nullable(),
provider: z.string(),
retrieval_model_dict: zDatasetRetrievalModelResponse,
runtime_mode: z.string().nullable(),
summary_index_setting: zDatasetSummaryIndexSettingResponse.optional(),
tags: z.array(zDatasetTagResponse),
total_available_documents: z.int(),
total_documents: z.int(),
updated_at: z.int(),
updated_by: z.string().nullable(),
word_count: z.int(),
})
export const zDeleteRagPipelineCustomizedTemplatesByTemplateIdPath = z.object({
template_id: z.string(),
})
/**
* Success
* Pipeline template deleted
*/
export const zDeleteRagPipelineCustomizedTemplatesByTemplateIdResponse = z.record(
z.string(),
z.unknown(),
z.never(),
)
export const zPatchRagPipelineCustomizedTemplatesByTemplateIdBody
= zCustomizedPipelineTemplatePayload
export const zPatchRagPipelineCustomizedTemplatesByTemplateIdPath = z.object({
template_id: z.string(),
})
/**
* Success
* Pipeline template updated
*/
export const zPatchRagPipelineCustomizedTemplatesByTemplateIdResponse = z.record(
z.string(),
z.unknown(),
z.never(),
)
export const zPostRagPipelineCustomizedTemplatesByTemplateIdPath = z.object({
@ -340,28 +588,37 @@ export const zPostRagPipelineCustomizedTemplatesByTemplateIdResponse = zSimpleDa
export const zPostRagPipelineDatasetBody = zRagPipelineDatasetImportPayload
/**
* Success
* RAG pipeline dataset import started
*/
export const zPostRagPipelineDatasetResponse = z.record(z.string(), z.unknown())
export const zPostRagPipelineDatasetResponse = zRagPipelineImportResponse
/**
* Success
* RAG pipeline dataset created
*/
export const zPostRagPipelineEmptyDatasetResponse = z.record(z.string(), z.unknown())
export const zPostRagPipelineEmptyDatasetResponse = zDatasetDetailResponse
export const zGetRagPipelineTemplatesQuery = z.object({
language: z.string().optional().default('en-US'),
type: z.string().optional().default('built-in'),
})
/**
* Success
* Pipeline templates
*/
export const zGetRagPipelineTemplatesResponse = z.record(z.string(), z.unknown())
export const zGetRagPipelineTemplatesResponse = zPipelineTemplateListResponse
export const zGetRagPipelineTemplatesByTemplateIdPath = z.object({
template_id: z.string(),
})
export const zGetRagPipelineTemplatesByTemplateIdQuery = z.object({
type: z.string().optional().default('built-in'),
})
/**
* Success
* Pipeline template
*/
export const zGetRagPipelineTemplatesByTemplateIdResponse = z.record(z.string(), z.unknown())
export const zGetRagPipelineTemplatesByTemplateIdResponse = zPipelineTemplateDetailResponse
/**
* Success
@ -371,30 +628,28 @@ export const zGetRagPipelinesDatasourcePluginsResponse = z.record(z.string(), z.
export const zPostRagPipelinesImportsBody = zRagPipelineImportPayload
/**
* Success
* Import completed
*/
export const zPostRagPipelinesImportsResponse = z.record(z.string(), z.unknown())
export const zPostRagPipelinesImportsResponse = zRagPipelineImportResponse
export const zPostRagPipelinesImportsByImportIdConfirmPath = z.object({
import_id: z.string(),
})
/**
* Success
* Import confirmed
*/
export const zPostRagPipelinesImportsByImportIdConfirmResponse = z.record(z.string(), z.unknown())
export const zPostRagPipelinesImportsByImportIdConfirmResponse = zRagPipelineImportResponse
export const zGetRagPipelinesImportsByPipelineIdCheckDependenciesPath = z.object({
pipeline_id: z.string(),
})
/**
* Success
* Dependencies checked
*/
export const zGetRagPipelinesImportsByPipelineIdCheckDependenciesResponse = z.record(
z.string(),
z.unknown(),
)
export const zGetRagPipelinesImportsByPipelineIdCheckDependenciesResponse
= zRagPipelineImportCheckDependenciesResponse
/**
* Success
@ -413,28 +668,32 @@ export const zPostRagPipelinesTransformDatasetsByDatasetIdResponse = z.record(
z.unknown(),
)
export const zPostRagPipelinesByPipelineIdCustomizedPublishBody = zPayload
export const zPostRagPipelinesByPipelineIdCustomizedPublishBody = zCustomizedPipelineTemplatePayload
export const zPostRagPipelinesByPipelineIdCustomizedPublishPath = z.object({
pipeline_id: z.string(),
})
/**
* Success
* Pipeline template published
*/
export const zPostRagPipelinesByPipelineIdCustomizedPublishResponse = z.record(
z.string(),
z.unknown(),
z.never(),
)
export const zGetRagPipelinesByPipelineIdExportsPath = z.object({
pipeline_id: z.string(),
})
export const zGetRagPipelinesByPipelineIdExportsQuery = z.object({
include_secret: z.string().optional().default('false'),
})
/**
* Success
* Pipeline exported
*/
export const zGetRagPipelinesByPipelineIdExportsResponse = z.record(z.string(), z.unknown())
export const zGetRagPipelinesByPipelineIdExportsResponse = zSimpleDataResponse
export const zGetRagPipelinesByPipelineIdWorkflowRunsPath = z.object({
pipeline_id: z.string(),