Merge remote-tracking branch 'upstream/feat/rag-2' into feat/rag-2

2025-09-01 17:06:53 +08:00 · 2025-09-01 17:06:53 +08:00 · eb7b21c7f1
parent ad5a2c77c1 1676207776
commit eb7b21c7f1
13 changed files with 118 additions and 12 deletions
--- a/api/controllers/console/datasets/rag_pipeline/rag_pipeline_workflow.py
+++ b/api/controllers/console/datasets/rag_pipeline/rag_pipeline_workflow.py
@ -981,6 +981,14 @@ class RagPipelineDatasourceVariableApi(Resource):
        )
        return workflow_node_execution

+class RagPipelineRecommendedPluginApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self):
+        rag_pipeline_service = RagPipelineService()
+        recommended_plugins = rag_pipeline_service.get_recommended_plugins()
+        return recommended_plugins

 api.add_resource(
    DraftRagPipelineApi,
@ -1090,3 +1098,8 @@ api.add_resource(
    RagPipelineDatasourceVariableApi,
    "/rag/pipelines/<uuid:pipeline_id>/workflows/draft/datasource/variables-inspect",
 )
+
+api.add_resource(
+    RagPipelineRecommendedPluginApi,
+    "/rag/pipelines/recommended-plugins",
+)
--- a/api/migrations/versions/2025_05_15_1558-b35c3db83d09_add_pipeline_info.py
+++ b/api/migrations/versions/2025_05_15_1558-b35c3db83d09_add_pipeline_info.py
@ -12,7 +12,7 @@ from sqlalchemy.dialects import postgresql

 # revision identifiers, used by Alembic.
 revision = 'b35c3db83d09'
-down_revision = '0ab65e1cc7fa'
+down_revision = '0e154742a5fa'
 branch_labels = None
 depends_on = None

--- a/api/migrations/versions/2025_09_01_1443-8c5088481127_add_pipeline_info_17.py
+++ b/api/migrations/versions/2025_09_01_1443-8c5088481127_add_pipeline_info_17.py
@ -0,0 +1,38 @@
+"""add_pipeline_info_17
+
+Revision ID: 8c5088481127
+Revises: 17d4db47800c
+Create Date: 2025-09-01 14:43:48.417869
+
+"""
+from alembic import op
+import models as models
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '8c5088481127'
+down_revision = '17d4db47800c'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('pipeline_recommended_plugins',
+    sa.Column('id', models.types.StringUUID(), server_default=sa.text('uuid_generate_v4()'), nullable=False),
+    sa.Column('plugin_id', sa.Text(), nullable=False),
+    sa.Column('provider_name', sa.Text(), nullable=False),
+    sa.Column('position', sa.Integer(), nullable=False),
+    sa.Column('active', sa.Boolean(), nullable=False),
+    sa.Column('created_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False),
+    sa.Column('updated_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False),
+    sa.PrimaryKeyConstraint('id', name='pipeline_recommended_plugin_pkey')
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('pipeline_recommended_plugins')
+    # ### end Alembic commands ###
--- a/api/models/dataset.py
+++ b/api/models/dataset.py
@ -1307,3 +1307,15 @@ class DocumentPipelineExecutionLog(Base):
    input_data = db.Column(db.JSON, nullable=False)
    created_by = db.Column(StringUUID, nullable=True)
    created_at = db.Column(db.DateTime, nullable=False, server_default=func.current_timestamp())
+
+class PipelineRecommendedPlugin(Base):
+    __tablename__ = "pipeline_recommended_plugins"
+    __table_args__ = (db.PrimaryKeyConstraint("id", name="pipeline_recommended_plugin_pkey"),)
+
+    id = db.Column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    plugin_id = db.Column(db.Text, nullable=False)
+    provider_name = db.Column(db.Text, nullable=False)
+    position = db.Column(db.Integer, nullable=False, default=0)
+    active = db.Column(db.Boolean, nullable=False, default=True)
+    created_at = db.Column(db.DateTime, nullable=False, server_default=func.current_timestamp())
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=func.current_timestamp())
--- a/api/services/rag_pipeline/rag_pipeline.py
+++ b/api/services/rag_pipeline/rag_pipeline.py
@ -27,6 +27,7 @@ from core.datasource.entities.datasource_entities import (
 from core.datasource.online_document.online_document_plugin import OnlineDocumentDatasourcePlugin
 from core.datasource.online_drive.online_drive_plugin import OnlineDriveDatasourcePlugin
 from core.datasource.website_crawl.website_crawl_plugin import WebsiteCrawlDatasourcePlugin
+from core.helper import marketplace
 from core.rag.entities.event import (
    DatasourceCompletedEvent,
    DatasourceErrorEvent,
@ -52,7 +53,7 @@ from core.workflow.workflow_entry import WorkflowEntry
 from extensions.ext_database import db
 from libs.infinite_scroll_pagination import InfiniteScrollPagination
 from models.account import Account
-from models.dataset import Document, Pipeline, PipelineCustomizedTemplate  # type: ignore
+from models.dataset import Document, Pipeline, PipelineCustomizedTemplate, PipelineRecommendedPlugin  # type: ignore
 from models.enums import WorkflowRunTriggeredFrom
 from models.model import EndUser
 from models.workflow import (
@ -70,6 +71,7 @@ from services.entities.knowledge_entities.rag_pipeline_entities import (
 )
 from services.errors.app import WorkflowHashNotEqualError
 from services.rag_pipeline.pipeline_template.pipeline_template_factory import PipelineTemplateRetrievalFactory
+from services.tools.builtin_tools_manage_service import BuiltinToolManageService
 from services.workflow_draft_variable_service import DraftVariableSaver, DraftVarLoader

 logger = logging.getLogger(__name__)
@ -1226,3 +1228,37 @@ class RagPipelineService:
            )
            session.commit()
        return workflow_node_execution_db_model
+
+    def get_recommended_plugins(self) -> list[dict]:
+        # Query active recommended plugins
+        pipeline_recommended_plugins = (
+            db.session.query(PipelineRecommendedPlugin)
+            .filter(PipelineRecommendedPlugin.active == True)
+            .order_by(PipelineRecommendedPlugin.position.asc())
+            .all()
+        )
+        
+        if not pipeline_recommended_plugins:
+            return []
+        
+        # Batch fetch plugin manifests
+        plugin_ids = [plugin.plugin_id for plugin in pipeline_recommended_plugins]
+        plugin_manifests = marketplace.batch_fetch_plugin_manifests(plugin_ids)
+        
+        builtin_tools = BuiltinToolManageService.list_builtin_tools(
+            user_id=current_user.id,
+            tenant_id=current_user.current_tenant_id,
+        )
+        installed_plugin_ids = {tool.plugin_id for tool in builtin_tools}
+        
+        # Build recommended plugins list
+        return [
+            {
+                "plugin_id": manifest.plugin_id,
+                "name": manifest.name,
+                "icon": manifest.icon,
+                "plugin_unique_identifier": manifest.latest_package_identifier,
+                "installed": manifest.plugin_id in installed_plugin_ids,
+            }
+            for manifest in plugin_manifests
+        ]
--- a/web/app/components/app-sidebar/dataset-info/dropdown.tsx
+++ b/web/app/components/app-sidebar/dataset-info/dropdown.tsx
@ -66,7 +66,7 @@ const DropDown = ({
      const a = document.createElement('a')
      const file = new Blob([data], { type: 'application/yaml' })
      a.href = URL.createObjectURL(file)
-      a.download = `${name}.yml`
+      a.download = `${name}.pipeline`
      a.click()
    }
    catch {
--- a/web/app/components/app/create-from-dsl-modal/uploader.tsx
+++ b/web/app/components/app/create-from-dsl-modal/uploader.tsx
@ -17,12 +17,16 @@ export type Props = {
  file: File | undefined
  updateFile: (file?: File) => void
  className?: string
+  accept?: string
+  displayName?: string
 }

 const Uploader: FC<Props> = ({
  file,
  updateFile,
  className,
+  accept = '.yaml,.yml',
+  displayName = 'YAML',
 }) => {
  const { t } = useTranslation()
  const { notify } = useContext(ToastContext)
@ -95,9 +99,9 @@ const Uploader: FC<Props> = ({
      <input
        ref={fileUploader}
        style={{ display: 'none' }}
-        type="file"
-        id="fileUploader"
-        accept='.yaml,.yml'
+        type='file'
+        id='fileUploader'
+        accept={accept}
        onChange={fileChangeHandle}
      />
      <div ref={dropRef}>
@ -116,12 +120,12 @@ const Uploader: FC<Props> = ({
        {file && (
          <div className={cn('group flex items-center rounded-lg border-[0.5px] border-components-panel-border bg-components-panel-on-panel-item-bg shadow-xs', ' hover:bg-components-panel-on-panel-item-bg-hover')}>
            <div className='flex items-center justify-center p-3'>
-              <YamlIcon className="h-6 w-6 shrink-0" />
+              <YamlIcon className='h-6 w-6 shrink-0' />
            </div>
            <div className='flex grow flex-col items-start gap-0.5 py-1 pr-2'>
              <span className='font-inter max-w-[calc(100%_-_30px)] overflow-hidden text-ellipsis whitespace-nowrap text-[12px] font-medium leading-4 text-text-secondary'>{file.name}</span>
              <div className='font-inter flex h-3 items-center gap-1 self-stretch text-[10px] font-medium uppercase leading-3 text-text-tertiary'>
-                <span>YAML</span>
+                <span>{displayName}</span>
                <span className='text-text-quaternary'>·</span>
                <span>{formatFileSize(file.size)}</span>
              </div>
--- a/web/app/components/datasets/create-from-pipeline/create-options/create-from-dsl-modal/uploader.tsx
+++ b/web/app/components/datasets/create-from-pipeline/create-options/create-from-dsl-modal/uploader.tsx
@ -98,7 +98,7 @@ const Uploader: FC<Props> = ({
        style={{ display: 'none' }}
        type='file'
        id='fileUploader'
-        accept='.yaml,.yml'
+        accept='.pipeline'
        onChange={fileChangeHandle}
      />
      <div ref={dropRef}>
--- a/web/app/components/datasets/create-from-pipeline/list/template-card/index.tsx
+++ b/web/app/components/datasets/create-from-pipeline/list/template-card/index.tsx
@ -103,7 +103,7 @@ const TemplateCard = ({
        const blob = new Blob([res.data], { type: 'application/yaml' })
        downloadFile({
          data: blob,
-          fileName: `${pipeline.name}.yml`,
+          fileName: `${pipeline.name}.pipeline`,
        })
        Toast.notify({
          type: 'success',
--- a/web/app/components/datasets/list/dataset-card/index.tsx
+++ b/web/app/components/datasets/list/dataset-card/index.tsx
@ -115,7 +115,7 @@ const DatasetCard = ({
      const a = document.createElement('a')
      const file = new Blob([data], { type: 'application/yaml' })
      a.href = URL.createObjectURL(file)
-      a.download = `${name}.yml`
+      a.download = `${name}.pipeline`
      a.click()
    }
    catch {
--- a/web/app/components/rag-pipeline/components/update-dsl-modal.tsx
+++ b/web/app/components/rag-pipeline/components/update-dsl-modal.tsx
@ -233,6 +233,8 @@ const UpdateDSLModal = ({
              file={currentFile}
              updateFile={handleFile}
              className='!mt-0 w-full'
+              accept='.pipeline'
+              displayName='PIPELINE'
            />
          </div>
        </div>
--- a/web/app/components/rag-pipeline/hooks/use-DSL.ts
+++ b/web/app/components/rag-pipeline/hooks/use-DSL.ts
@ -40,7 +40,7 @@ export const useDSL = () => {
      const a = document.createElement('a')
      const file = new Blob([data], { type: 'application/yaml' })
      a.href = URL.createObjectURL(file)
-      a.download = `${knowledgeName}.yml`
+      a.download = `${knowledgeName}.pipeline`
      a.click()
    }
    catch {
--- a/web/app/components/rag-pipeline/hooks/use-pipeline-template.ts
+++ b/web/app/components/rag-pipeline/hooks/use-pipeline-template.ts
@ -15,6 +15,7 @@ export const usePipelineTemplate = () => {
      ...knowledgeBaseDefault.defaultValue as KnowledgeBaseNodeType,
      type: knowledgeBaseDefault.metaData.type,
      title: t(`workflow.blocks.${knowledgeBaseDefault.metaData.type}`),
+      selected: true,
    },
    position: {
      x: START_INITIAL_POSITION.x + 500,