opensource/dify
2
0
Fork 0
mirror of https://github.com/langgenius/dify.git synced 2026-05-10 05:56:31 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into feat/support-agent-sandbox

Browse Source 
# Conflicts:
#	api/models/model.py
#	web/contract/router.ts
This commit is contained in:
yyh 2026-01-15 10:59:45 +08:00
commit 657739d48b
No known key found for this signature in database
48 changed files with 6825 additions and 1040 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/style.yml vendored
View File

@ -90,7 +90,7 @@ jobs:
uses: actions/setup-node@v6
if: steps.changed-files.outputs.any_changed == 'true'
with:
node-version: 22
node-version: 24
cache: pnpm
cache-dependency-path: ./web/pnpm-lock.yaml

8
.github/workflows/tool-test-sdks.yaml vendored
View File

@ -16,10 +16,6 @@ jobs:
name: unit test for Node.js SDK
runs-on: ubuntu-latest
strategy:
matrix:
node-version: [16, 18, 20, 22]
defaults:
run:
working-directory: sdks/nodejs-client
@ -29,10 +25,10 @@ jobs:
with:
persist-credentials: false
- name: Use Node.js ${{ matrix.node-version }}
- name: Use Node.js
uses: actions/setup-node@v6
with:
node-version: ${{ matrix.node-version }}
node-version: 24
cache: ''
cache-dependency-path: 'pnpm-lock.yaml'

2
.github/workflows/translate-i18n-claude.yml vendored
View File

@ -57,7 +57,7 @@ jobs:
- name: Set up Node.js
uses: actions/setup-node@v6
with:
node-version: 'lts/*'
node-version: 24
cache: pnpm
cache-dependency-path: ./web/pnpm-lock.yaml

2
.github/workflows/web-tests.yml vendored
View File

@ -31,7 +31,7 @@ jobs:
- name: Setup Node.js
uses: actions/setup-node@v6
with:
node-version: 22
node-version: 24
cache: pnpm
cache-dependency-path: ./web/pnpm-lock.yaml

11
api/controllers/console/app/conversation.py
View File

@ -592,9 +592,12 @@ def _get_conversation(app_model, conversation_id):
if not conversation:
raise NotFound("Conversation Not Exists.")
if not conversation.read_at:
conversation.read_at = naive_utc_now()
conversation.read_account_id = current_user.id
db.session.commit()
db.session.execute(
sa.update(Conversation)
.where(Conversation.id == conversation_id, Conversation.read_at.is_(None))
.values(read_at=naive_utc_now(), read_account_id=current_user.id)
)
db.session.commit()
db.session.refresh(conversation)
return conversation

32
api/core/plugin/impl/endpoint.py
View File

@ -1,5 +1,6 @@
from core.plugin.entities.endpoint import EndpointEntityWithInstance
from core.plugin.impl.base import BasePluginClient
from core.plugin.impl.exc import PluginDaemonInternalServerError
class PluginEndpointClient(BasePluginClient):
@ -70,18 +71,27 @@ class PluginEndpointClient(BasePluginClient):
def delete_endpoint(self, tenant_id: str, user_id: str, endpoint_id: str):
"""
Delete the given endpoint.
This operation is idempotent: if the endpoint is already deleted (record not found),
it will return True instead of raising an error.
"""
return self._request_with_plugin_daemon_response(
"POST",
f"plugin/{tenant_id}/endpoint/remove",
bool,
data={
"endpoint_id": endpoint_id,
},
headers={
"Content-Type": "application/json",
},
)
try:
return self._request_with_plugin_daemon_response(
"POST",
f"plugin/{tenant_id}/endpoint/remove",
bool,
data={
"endpoint_id": endpoint_id,
},
headers={
"Content-Type": "application/json",
},
)
except PluginDaemonInternalServerError as e:
# Make delete idempotent: if record is not found, consider it a success
if "record not found" in str(e.description).lower():
return True
raise
def enable_endpoint(self, tenant_id: str, user_id: str, endpoint_id: str):
"""

13
api/factories/file_factory.py
View File

@ -115,7 +115,18 @@ def build_from_mappings(
# TODO(QuantumGhost): Performance concern - each mapping triggers a separate database query.
# Implement batch processing to reduce database load when handling multiple files.
# Filter out None/empty mappings to avoid errors
valid_mappings = [m for m in mappings if m and m.get("transfer_method")]
def is_valid_mapping(m: Mapping[str, Any]) -> bool:
if not m or not m.get("transfer_method"):
return False
# For REMOTE_URL transfer method, ensure url or remote_url is provided and not None
transfer_method = m.get("transfer_method")
if transfer_method == FileTransferMethod.REMOTE_URL:
url = m.get("url") or m.get("remote_url")
if not url:
return False
return True
valid_mappings = [m for m in mappings if is_valid_mapping(m)]
files = [
build_from_mapping(
mapping=mapping,

5
api/fields/message_fields.py
View File

@ -2,6 +2,7 @@ from __future__ import annotations
from datetime import datetime
from typing import TypeAlias
from uuid import uuid4
from pydantic import BaseModel, ConfigDict, Field, field_validator
@ -20,8 +21,8 @@ class SimpleFeedback(ResponseModel):
class RetrieverResource(ResponseModel):
id: str
message_id: str
id: str = Field(default_factory=lambda: str(uuid4()))
message_id: str = Field(default_factory=lambda: str(uuid4()))
position: int
dataset_id: str | None = None
dataset_name: str | None = None

14
api/models/dataset.py
View File

@ -1149,7 +1149,7 @@ class DatasetCollectionBinding(TypeBase):
)
class TidbAuthBinding(Base):
class TidbAuthBinding(TypeBase):
__tablename__ = "tidb_auth_bindings"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="tidb_auth_bindings_pkey"),
@ -1158,7 +1158,13 @@ class TidbAuthBinding(Base):
sa.Index("tidb_auth_bindings_created_at_idx", "created_at"),
sa.Index("tidb_auth_bindings_status_idx", "status"),
)
id: Mapped[str] = mapped_column(StringUUID, primary_key=True, default=lambda: str(uuid4()))
id: Mapped[str] = mapped_column(
StringUUID,
primary_key=True,
insert_default=lambda: str(uuid4()),
default_factory=lambda: str(uuid4()),
init=False,
)
tenant_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True)
cluster_id: Mapped[str] = mapped_column(String(255), nullable=False)
cluster_name: Mapped[str] = mapped_column(String(255), nullable=False)
@ -1166,7 +1172,9 @@ class TidbAuthBinding(Base):
status: Mapped[str] = mapped_column(sa.String(255), nullable=False, server_default=sa.text("'CREATING'"))
account: Mapped[str] = mapped_column(String(255), nullable=False)
password: Mapped[str] = mapped_column(String(255), nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
created_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=func.current_timestamp(), init=False
)
class Whitelist(TypeBase):

48
api/models/model.py
View File

@ -1461,7 +1461,7 @@ class MessageAnnotation(Base):
return account
class AppAnnotationHitHistory(Base):
class AppAnnotationHitHistory(TypeBase):
__tablename__ = "app_annotation_hit_histories"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="app_annotation_hit_histories_pkey"),
@ -1471,17 +1471,19 @@ class AppAnnotationHitHistory(Base):
sa.Index("app_annotation_hit_histories_message_idx", "message_id"),
)
id = mapped_column(StringUUID, default=lambda: str(uuid4()))
app_id = mapped_column(StringUUID, nullable=False)
id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False)
app_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
annotation_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
source = mapped_column(LongText, nullable=False)
question = mapped_column(LongText, nullable=False)
account_id = mapped_column(StringUUID, nullable=False)
created_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
score = mapped_column(Float, nullable=False, server_default=sa.text("0"))
message_id = mapped_column(StringUUID, nullable=False)
annotation_question = mapped_column(LongText, nullable=False)
annotation_content = mapped_column(LongText, nullable=False)
source: Mapped[str] = mapped_column(LongText, nullable=False)
question: Mapped[str] = mapped_column(LongText, nullable=False)
account_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
created_at: Mapped[datetime] = mapped_column(
sa.DateTime, nullable=False, server_default=func.current_timestamp(), init=False
)
score: Mapped[float] = mapped_column(Float, nullable=False, server_default=sa.text("0"))
message_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
annotation_question: Mapped[str] = mapped_column(LongText, nullable=False)
annotation_content: Mapped[str] = mapped_column(LongText, nullable=False)
@property
def account(self):
@ -2181,7 +2183,7 @@ class LLMGenerationDetail(Base):
)
class TenantCreditPool(Base):
class TenantCreditPool(TypeBase):
__tablename__ = "tenant_credit_pools"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="tenant_credit_pool_pkey"),
@ -2189,14 +2191,20 @@ class TenantCreditPool(Base):
sa.Index("tenant_credit_pool_pool_type_idx", "pool_type"),
)
id = mapped_column(StringUUID, primary_key=True, server_default=text("uuid_generate_v4()"))
tenant_id = mapped_column(StringUUID, nullable=False)
pool_type = mapped_column(String(40), nullable=False, default="trial", server_default="trial")
quota_limit = mapped_column(BigInteger, nullable=False, default=0)
quota_used = mapped_column(BigInteger, nullable=False, default=0)
created_at = mapped_column(sa.DateTime, nullable=False, server_default=text("CURRENT_TIMESTAMP"))
updated_at = mapped_column(
sa.DateTime, nullable=False, server_default=func.current_timestamp(), onupdate=func.current_timestamp()
id: Mapped[str] = mapped_column(StringUUID, primary_key=True, server_default=text("uuid_generate_v4()"), init=False)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
pool_type: Mapped[str] = mapped_column(String(40), nullable=False, default="trial", server_default="trial")
quota_limit: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
quota_used: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
created_at: Mapped[datetime] = mapped_column(
sa.DateTime, nullable=False, server_default=text("CURRENT_TIMESTAMP"), init=False
)
updated_at: Mapped[datetime] = mapped_column(
sa.DateTime,
nullable=False,
server_default=func.current_timestamp(),
onupdate=func.current_timestamp(),
init=False,
)
@property

3
api/schedule/create_tidb_serverless_task.py
View File

@ -50,10 +50,13 @@ def create_clusters(batch_size):
)
for new_cluster in new_clusters:
tidb_auth_binding = TidbAuthBinding(
tenant_id=None,
cluster_id=new_cluster["cluster_id"],
cluster_name=new_cluster["cluster_name"],
account=new_cluster["account"],
password=new_cluster["password"],
active=False,
status="CREATING",
)
db.session.add(tidb_auth_binding)
db.session.commit()

279
api/tests/unit_tests/core/plugin/test_endpoint_client.py Normal file
View File

@ -0,0 +1,279 @@
"""Unit tests for PluginEndpointClient functionality.
This test module covers the endpoint client operations including:
- Successful endpoint deletion
- Idempotent delete behavior (record not found)
- Non-idempotent delete behavior (other errors)
Tests follow the Arrange-Act-Assert pattern for clarity.
"""
from unittest.mock import MagicMock, patch
import pytest
from core.plugin.impl.endpoint import PluginEndpointClient
from core.plugin.impl.exc import PluginDaemonInternalServerError
class TestPluginEndpointClientDelete:
"""Unit tests for PluginEndpointClient delete_endpoint operation.
Tests cover:
- Successful endpoint deletion
- Idempotent behavior when endpoint is already deleted (record not found)
- Non-idempotent behavior for other errors
"""
@pytest.fixture
def endpoint_client(self):
"""Create a PluginEndpointClient instance for testing."""
return PluginEndpointClient()
@pytest.fixture
def mock_config(self):
"""Mock plugin daemon configuration."""
with (
patch("core.plugin.impl.base.dify_config.PLUGIN_DAEMON_URL", "http://127.0.0.1:5002"),
patch("core.plugin.impl.base.dify_config.PLUGIN_DAEMON_KEY", "test-api-key"),
):
yield
def test_delete_endpoint_success(self, endpoint_client, mock_config):
"""Test successful endpoint deletion.
Given:
- A valid tenant_id, user_id, and endpoint_id
- The plugin daemon returns success response
When:
- delete_endpoint is called
Then:
- The method should return True
- The request should be made with correct parameters
"""
# Arrange
tenant_id = "tenant-123"
user_id = "user-456"
endpoint_id = "endpoint-789"
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.json.return_value = {
"code": 0,
"message": "success",
"data": True,
}
with patch("httpx.request", return_value=mock_response):
# Act
result = endpoint_client.delete_endpoint(
tenant_id=tenant_id,
user_id=user_id,
endpoint_id=endpoint_id,
)
# Assert
assert result is True
def test_delete_endpoint_idempotent_record_not_found(self, endpoint_client, mock_config):
"""Test idempotent delete behavior when endpoint is already deleted.
Given:
- A valid tenant_id, user_id, and endpoint_id
- The plugin daemon returns "record not found" error
When:
- delete_endpoint is called
Then:
- The method should return True (idempotent behavior)
- No exception should be raised
"""
# Arrange
tenant_id = "tenant-123"
user_id = "user-456"
endpoint_id = "endpoint-789"
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.json.return_value = {
"code": -1,
"message": (
'{"error_type": "PluginDaemonInternalServerError", '
'"message": "failed to remove endpoint: record not found"}'
),
}
with patch("httpx.request", return_value=mock_response):
# Act
result = endpoint_client.delete_endpoint(
tenant_id=tenant_id,
user_id=user_id,
endpoint_id=endpoint_id,
)
# Assert - should return True instead of raising an error
assert result is True
def test_delete_endpoint_non_idempotent_other_errors(self, endpoint_client, mock_config):
"""Test non-idempotent delete behavior for other errors.
Given:
- A valid tenant_id, user_id, and endpoint_id
- The plugin daemon returns a different error (not "record not found")
When:
- delete_endpoint is called
Then:
- The method should raise PluginDaemonInternalServerError
"""
# Arrange
tenant_id = "tenant-123"
user_id = "user-456"
endpoint_id = "endpoint-789"
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.json.return_value = {
"code": -1,
"message": (
'{"error_type": "PluginDaemonInternalServerError", '
'"message": "failed to remove endpoint: internal server error"}'
),
}
with patch("httpx.request", return_value=mock_response):
# Act & Assert
with pytest.raises(PluginDaemonInternalServerError) as exc_info:
endpoint_client.delete_endpoint(
tenant_id=tenant_id,
user_id=user_id,
endpoint_id=endpoint_id,
)
# Assert - the error message should not be "record not found"
assert "record not found" not in str(exc_info.value.description)
def test_delete_endpoint_idempotent_case_insensitive(self, endpoint_client, mock_config):
"""Test idempotent delete behavior with case-insensitive error message.
Given:
- A valid tenant_id, user_id, and endpoint_id
- The plugin daemon returns "Record Not Found" error (different case)
When:
- delete_endpoint is called
Then:
- The method should return True (idempotent behavior)
"""
# Arrange
tenant_id = "tenant-123"
user_id = "user-456"
endpoint_id = "endpoint-789"
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.json.return_value = {
"code": -1,
"message": '{"error_type": "PluginDaemonInternalServerError", "message": "Record Not Found"}',
}
with patch("httpx.request", return_value=mock_response):
# Act
result = endpoint_client.delete_endpoint(
tenant_id=tenant_id,
user_id=user_id,
endpoint_id=endpoint_id,
)
# Assert - should still return True
assert result is True
def test_delete_endpoint_multiple_calls_idempotent(self, endpoint_client, mock_config):
"""Test that multiple delete calls are idempotent.
Given:
- A valid tenant_id, user_id, and endpoint_id
- The first call succeeds
- Subsequent calls return "record not found"
When:
- delete_endpoint is called multiple times
Then:
- All calls should return True
"""
# Arrange
tenant_id = "tenant-123"
user_id = "user-456"
endpoint_id = "endpoint-789"
# First call - success
mock_response_success = MagicMock()
mock_response_success.status_code = 200
mock_response_success.json.return_value = {
"code": 0,
"message": "success",
"data": True,
}
# Second call - record not found
mock_response_not_found = MagicMock()
mock_response_not_found.status_code = 200
mock_response_not_found.json.return_value = {
"code": -1,
"message": (
'{"error_type": "PluginDaemonInternalServerError", '
'"message": "failed to remove endpoint: record not found"}'
),
}
with patch("httpx.request") as mock_request:
# Act - first call
mock_request.return_value = mock_response_success
result1 = endpoint_client.delete_endpoint(
tenant_id=tenant_id,
user_id=user_id,
endpoint_id=endpoint_id,
)
# Act - second call (already deleted)
mock_request.return_value = mock_response_not_found
result2 = endpoint_client.delete_endpoint(
tenant_id=tenant_id,
user_id=user_id,
endpoint_id=endpoint_id,
)
# Assert - both should return True
assert result1 is True
assert result2 is True
def test_delete_endpoint_non_idempotent_unauthorized_error(self, endpoint_client, mock_config):
"""Test that authorization errors are not treated as idempotent.
Given:
- A valid tenant_id, user_id, and endpoint_id
- The plugin daemon returns an unauthorized error
When:
- delete_endpoint is called
Then:
- The method should raise the appropriate error (not return True)
"""
# Arrange
tenant_id = "tenant-123"
user_id = "user-456"
endpoint_id = "endpoint-789"
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.json.return_value = {
"code": -1,
"message": '{"error_type": "PluginDaemonUnauthorizedError", "message": "unauthorized access"}',
}
with patch("httpx.request", return_value=mock_response):
# Act & Assert
with pytest.raises(Exception) as exc_info:
endpoint_client.delete_endpoint(
tenant_id=tenant_id,
user_id=user_id,
endpoint_id=endpoint_id,
)
# Assert - should not return True for unauthorized errors
assert exc_info.value.__class__.__name__ == "PluginDaemonUnauthorizedError"

2
web/.nvmrc
View File

@ -1 +1 @@
22.21.1
24

2
web/Dockerfile
View File

@ -1,5 +1,5 @@
# base image
FROM node:22.21.1-alpine3.23 AS base
FROM node:24-alpine AS base
LABEL maintainer="takatost@gmail.com"
# if you located in China, you can use aliyun mirror to speed up

4
web/README.md
View File

@ -8,8 +8,8 @@ This is a [Next.js](https://nextjs.org/) project bootstrapped with [`create-next
Before starting the web frontend service, please make sure the following environment is ready.
- [Node.js](https://nodejs.org) >= v22.11.x
- [pnpm](https://pnpm.io) v10.x
- [Node.js](https://nodejs.org)
- [pnpm](https://pnpm.io)
> [!TIP]
> It is recommended to install and enable Corepack to manage package manager versions automatically:

12
web/app/components/billing/pricing/plans/cloud-plan-item/index.spec.tsx
View File

@ -27,7 +27,9 @@ vi.mock('@/service/billing', () => ({
vi.mock('@/service/client', () => ({
consoleClient: {
billingUrl: vi.fn(),
billing: {
invoices: vi.fn(),
},
},
}))
@ -43,7 +45,7 @@ vi.mock('../../assets', () => ({
const mockUseAppContext = useAppContext as Mock
const mockUseAsyncWindowOpen = useAsyncWindowOpen as Mock
const mockBillingUrl = consoleClient.billingUrl as Mock
const mockBillingInvoices = consoleClient.billing.invoices as Mock
const mockFetchSubscriptionUrls = fetchSubscriptionUrls as Mock
const mockToastNotify = Toast.notify as Mock
@ -75,7 +77,7 @@ beforeEach(() => {
vi.clearAllMocks()
mockUseAppContext.mockReturnValue({ isCurrentWorkspaceManager: true })
mockUseAsyncWindowOpen.mockReturnValue(vi.fn(async open => await open()))
mockBillingUrl.mockResolvedValue({ url: 'https://billing.example' })
mockBillingInvoices.mockResolvedValue({ url: 'https://billing.example' })
mockFetchSubscriptionUrls.mockResolvedValue({ url: 'https://subscription.example' })
assignedHref = ''
})
@ -149,7 +151,7 @@ describe('CloudPlanItem', () => {
type: 'error',
message: 'billing.buyPermissionDeniedTip',
}))
expect(mockBillingUrl).not.toHaveBeenCalled()
expect(mockBillingInvoices).not.toHaveBeenCalled()
})
it('should open billing portal when upgrading current paid plan', async () => {
@ -168,7 +170,7 @@ describe('CloudPlanItem', () => {
fireEvent.click(screen.getByRole('button', { name: 'billing.plansCommon.currentPlan' }))
await waitFor(() => {
expect(mockBillingUrl).toHaveBeenCalledTimes(1)
expect(mockBillingInvoices).toHaveBeenCalledTimes(1)
})
expect(openWindow).toHaveBeenCalledTimes(1)
})

2
web/app/components/billing/pricing/plans/cloud-plan-item/index.tsx
View File

@ -77,7 +77,7 @@ const CloudPlanItem: FC<CloudPlanItemProps> = ({
try {
if (isCurrentPaidPlan) {
await openAsyncWindow(async () => {
const res = await consoleClient.billingUrl()
const res = await consoleClient.billing.invoices()
if (res.url)
return res.url
throw new Error('Failed to open billing page')

2028
web/app/components/datasets/create-from-pipeline/create-options/create-from-dsl-modal/index.spec.tsx Normal file
View File

File diff suppressed because it is too large Load Diff

201
web/app/components/datasets/documents/components/documents-header.tsx Normal file
View File

@ -0,0 +1,201 @@
'use client'
import type { FC } from 'react'
import type { Item } from '@/app/components/base/select'
import type { BuiltInMetadataItem, MetadataItemWithValueLength } from '@/app/components/datasets/metadata/types'
import type { SortType } from '@/service/datasets'
import { PlusIcon } from '@heroicons/react/24/solid'
import { RiDraftLine, RiExternalLinkLine } from '@remixicon/react'
import { useMemo } from 'react'
import { useTranslation } from 'react-i18next'
import Button from '@/app/components/base/button'
import Chip from '@/app/components/base/chip'
import Input from '@/app/components/base/input'
import Sort from '@/app/components/base/sort'
import AutoDisabledDocument from '@/app/components/datasets/common/document-status-with-action/auto-disabled-document'
import IndexFailed from '@/app/components/datasets/common/document-status-with-action/index-failed'
import StatusWithAction from '@/app/components/datasets/common/document-status-with-action/status-with-action'
import DatasetMetadataDrawer from '@/app/components/datasets/metadata/metadata-dataset/dataset-metadata-drawer'
import { useDocLink } from '@/context/i18n'
import { DataSourceType } from '@/models/datasets'
import { useIndexStatus } from '../status-item/hooks'
type DocumentsHeaderProps = {
// Dataset info
datasetId: string
dataSourceType?: DataSourceType
embeddingAvailable: boolean
isFreePlan: boolean
// Filter & sort
statusFilterValue: string
sortValue: SortType
inputValue: string
onStatusFilterChange: (value: string) => void
onStatusFilterClear: () => void
onSortChange: (value: string) => void
onInputChange: (value: string) => void
// Metadata modal
isShowEditMetadataModal: boolean
showEditMetadataModal: () => void
hideEditMetadataModal: () => void
datasetMetaData?: MetadataItemWithValueLength[]
builtInMetaData?: BuiltInMetadataItem[]
builtInEnabled: boolean
onAddMetaData: (payload: BuiltInMetadataItem) => Promise<void>
onRenameMetaData: (payload: MetadataItemWithValueLength) => Promise<void>
onDeleteMetaData: (metaDataId: string) => Promise<void>
onBuiltInEnabledChange: (enabled: boolean) => void
// Actions
onAddDocument: () => void
}
const DocumentsHeader: FC<DocumentsHeaderProps> = ({
datasetId,
dataSourceType,
embeddingAvailable,
isFreePlan,
statusFilterValue,
sortValue,
inputValue,
onStatusFilterChange,
onStatusFilterClear,
onSortChange,
onInputChange,
isShowEditMetadataModal,
showEditMetadataModal,
hideEditMetadataModal,
datasetMetaData,
builtInMetaData,
builtInEnabled,
onAddMetaData,
onRenameMetaData,
onDeleteMetaData,
onBuiltInEnabledChange,
onAddDocument,
}) => {
const { t } = useTranslation()
const docLink = useDocLink()
const DOC_INDEX_STATUS_MAP = useIndexStatus()
const isDataSourceNotion = dataSourceType === DataSourceType.NOTION
const isDataSourceWeb = dataSourceType === DataSourceType.WEB
const statusFilterItems: Item[] = useMemo(() => [
{ value: 'all', name: t('list.index.all', { ns: 'datasetDocuments' }) as string },
{ value: 'queuing', name: DOC_INDEX_STATUS_MAP.queuing.text },
{ value: 'indexing', name: DOC_INDEX_STATUS_MAP.indexing.text },
{ value: 'paused', name: DOC_INDEX_STATUS_MAP.paused.text },
{ value: 'error', name: DOC_INDEX_STATUS_MAP.error.text },
{ value: 'available', name: DOC_INDEX_STATUS_MAP.available.text },
{ value: 'enabled', name: DOC_INDEX_STATUS_MAP.enabled.text },
{ value: 'disabled', name: DOC_INDEX_STATUS_MAP.disabled.text },
{ value: 'archived', name: DOC_INDEX_STATUS_MAP.archived.text },
], [DOC_INDEX_STATUS_MAP, t])
const sortItems: Item[] = useMemo(() => [
{ value: 'created_at', name: t('list.sort.uploadTime', { ns: 'datasetDocuments' }) as string },
{ value: 'hit_count', name: t('list.sort.hitCount', { ns: 'datasetDocuments' }) as string },
], [t])
// Determine add button text based on data source type
const addButtonText = useMemo(() => {
if (isDataSourceNotion)
return t('list.addPages', { ns: 'datasetDocuments' })
if (isDataSourceWeb)
return t('list.addUrl', { ns: 'datasetDocuments' })
return t('list.addFile', { ns: 'datasetDocuments' })
}, [isDataSourceNotion, isDataSourceWeb, t])
return (
<>
{/* Title section */}
<div className="flex flex-col justify-center gap-1 px-6 pt-4">
<h1 className="text-base font-semibold text-text-primary">
{t('list.title', { ns: 'datasetDocuments' })}
</h1>
<div className="flex items-center space-x-0.5 text-sm font-normal text-text-tertiary">
<span>{t('list.desc', { ns: 'datasetDocuments' })}</span>
<a
className="flex items-center text-text-accent"
target="_blank"
rel="noopener noreferrer"
href={docLink('/guides/knowledge-base/integrate-knowledge-within-application')}
>
<span>{t('list.learnMore', { ns: 'datasetDocuments' })}</span>
<RiExternalLinkLine className="h-3 w-3" />
</a>
</div>
</div>
{/* Toolbar section */}
<div className="flex flex-wrap items-center justify-between px-6 pt-4">
{/* Left: Filters */}
<div className="flex items-center gap-2">
<Chip
className="w-[160px]"
showLeftIcon={false}
value={statusFilterValue}
items={statusFilterItems}
onSelect={item => onStatusFilterChange(item?.value ? String(item.value) : '')}
onClear={onStatusFilterClear}
/>
<Input
showLeftIcon
showClearIcon
wrapperClassName="!w-[200px]"
value={inputValue}
onChange={e => onInputChange(e.target.value)}
onClear={() => onInputChange('')}
/>
<div className="h-3.5 w-px bg-divider-regular"></div>
<Sort
order={sortValue.startsWith('-') ? '-' : ''}
value={sortValue.replace('-', '')}
items={sortItems}
onSelect={value => onSortChange(String(value))}
/>
</div>
{/* Right: Actions */}
<div className="flex !h-8 items-center justify-center gap-2">
{!isFreePlan && <AutoDisabledDocument datasetId={datasetId} />}
<IndexFailed datasetId={datasetId} />
{!embeddingAvailable && (
<StatusWithAction
type="warning"
description={t('embeddingModelNotAvailable', { ns: 'dataset' })}
/>
)}
{embeddingAvailable && (
<Button variant="secondary" className="shrink-0" onClick={showEditMetadataModal}>
<RiDraftLine className="mr-1 size-4" />
{t('metadata.metadata', { ns: 'dataset' })}
</Button>
)}
{isShowEditMetadataModal && (
<DatasetMetadataDrawer
userMetadata={datasetMetaData ?? []}
onClose={hideEditMetadataModal}
onAdd={onAddMetaData}
onRename={onRenameMetaData}
onRemove={onDeleteMetaData}
builtInMetadata={builtInMetaData ?? []}
isBuiltInEnabled={builtInEnabled}
onIsBuiltInEnabledChange={onBuiltInEnabledChange}
/>
)}
{embeddingAvailable && (
<Button variant="primary" onClick={onAddDocument} className="shrink-0">
<PlusIcon className="mr-2 h-4 w-4 stroke-current" />
{addButtonText}
</Button>
)}
</div>
</div>
</>
)
}
export default DocumentsHeader

41
web/app/components/datasets/documents/components/empty-element.tsx Normal file
View File

@ -0,0 +1,41 @@
'use client'
import type { FC } from 'react'
import { PlusIcon } from '@heroicons/react/24/solid'
import { useTranslation } from 'react-i18next'
import Button from '@/app/components/base/button'
import s from '../style.module.css'
import { FolderPlusIcon, NotionIcon, ThreeDotsIcon } from './icons'
type EmptyElementProps = {
canAdd: boolean
onClick: () => void
type?: 'upload' | 'sync'
}
const EmptyElement: FC<EmptyElementProps> = ({ canAdd = true, onClick, type = 'upload' }) => {
const { t } = useTranslation()
return (
<div className={s.emptyWrapper}>
<div className={s.emptyElement}>
<div className={s.emptySymbolIconWrapper}>
{type === 'upload' ? <FolderPlusIcon /> : <NotionIcon />}
</div>
<span className={s.emptyTitle}>
{t('list.empty.title', { ns: 'datasetDocuments' })}
<ThreeDotsIcon className="relative -left-1.5 -top-3 inline" />
</span>
<div className={s.emptyTip}>
{t(`list.empty.${type}.tip`, { ns: 'datasetDocuments' })}
</div>
{type === 'upload' && canAdd && (
<Button onClick={onClick} className={s.addFileBtn} variant="secondary-accent">
<PlusIcon className={s.plusIcon} />
{t('list.addFile', { ns: 'datasetDocuments' })}
</Button>
)}
</div>
</div>
)
}
export default EmptyElement

34
web/app/components/datasets/documents/components/icons.tsx Normal file
View File

@ -0,0 +1,34 @@
import type * as React from 'react'
export const FolderPlusIcon = ({ className }: React.SVGProps<SVGElement>) => {
return (
<svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg" className={className ?? ''}>
<path d="M10.8332 5.83333L9.90355 3.9741C9.63601 3.439 9.50222 3.17144 9.30265 2.97597C9.12615 2.80311 8.91344 2.67164 8.6799 2.59109C8.41581 2.5 8.11668 2.5 7.51841 2.5H4.33317C3.39975 2.5 2.93304 2.5 2.57652 2.68166C2.26292 2.84144 2.00795 3.09641 1.84816 3.41002C1.6665 3.76654 1.6665 4.23325 1.6665 5.16667V5.83333M1.6665 5.83333H14.3332C15.7333 5.83333 16.4334 5.83333 16.9681 6.10582C17.4386 6.3455 17.821 6.72795 18.0607 7.19836C18.3332 7.73314 18.3332 8.4332 18.3332 9.83333V13.5C18.3332 14.9001 18.3332 15.6002 18.0607 16.135C17.821 16.6054 17.4386 16.9878 16.9681 17.2275C16.4334 17.5 15.7333 17.5 14.3332 17.5H5.6665C4.26637 17.5 3.56631 17.5 3.03153 17.2275C2.56112 16.9878 2.17867 16.6054 1.93899 16.135C1.6665 15.6002 1.6665 14.9001 1.6665 13.5V5.83333ZM9.99984 14.1667V9.16667M7.49984 11.6667H12.4998" stroke="#667085" strokeWidth="1.5" strokeLinecap="round" strokeLinejoin="round" />
</svg>
)
}
export const ThreeDotsIcon = ({ className }: React.SVGProps<SVGElement>) => {
return (
<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg" className={className ?? ''}>
<path d="M5 6.5V5M8.93934 7.56066L10 6.5M10.0103 11.5H11.5103" stroke="#374151" strokeWidth="2" strokeLinecap="round" strokeLinejoin="round" />
</svg>
)
}
export const NotionIcon = ({ className }: React.SVGProps<SVGElement>) => {
return (
<svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg" className={className ?? ''}>
<g clipPath="url(#clip0_2164_11263)">
<path fillRule="evenodd" clipRule="evenodd" d="M3.5725 18.2611L1.4229 15.5832C0.905706 14.9389 0.625 14.1466 0.625 13.3312V3.63437C0.625 2.4129 1.60224 1.39936 2.86295 1.31328L12.8326 0.632614C13.5569 0.583164 14.2768 0.775682 14.8717 1.17794L18.3745 3.5462C19.0015 3.97012 19.375 4.66312 19.375 5.40266V16.427C19.375 17.6223 18.4141 18.6121 17.1798 18.688L6.11458 19.3692C5.12958 19.4298 4.17749 19.0148 3.5725 18.2611Z" fill="white" />
<path d="M7.03006 8.48669V8.35974C7.03006 8.03794 7.28779 7.77104 7.61997 7.74886L10.0396 7.58733L13.3857 12.5147V8.19009L12.5244 8.07528V8.01498C12.5244 7.68939 12.788 7.42074 13.1244 7.4035L15.326 7.29073V7.60755C15.326 7.75628 15.2154 7.88349 15.0638 7.90913L14.534 7.99874V15.0023L13.8691 15.231C13.3136 15.422 12.6952 15.2175 12.3772 14.7377L9.12879 9.83574V14.5144L10.1287 14.7057L10.1147 14.7985C10.0711 15.089 9.82028 15.3087 9.51687 15.3222L7.03006 15.4329C6.99718 15.1205 7.23132 14.841 7.55431 14.807L7.88143 14.7727V8.53453L7.03006 8.48669Z" fill="black" />
<path fillRule="evenodd" clipRule="evenodd" d="M12.9218 1.85424L2.95217 2.53491C2.35499 2.57568 1.89209 3.05578 1.89209 3.63437V13.3312C1.89209 13.8748 2.07923 14.403 2.42402 14.8325L4.57362 17.5104C4.92117 17.9434 5.46812 18.1818 6.03397 18.147L17.0991 17.4658C17.6663 17.4309 18.1078 16.9762 18.1078 16.427V5.40266C18.1078 5.06287 17.9362 4.74447 17.6481 4.54969L14.1453 2.18143C13.7883 1.94008 13.3564 1.82457 12.9218 1.85424ZM3.44654 3.78562C3.30788 3.68296 3.37387 3.46909 3.54806 3.4566L12.9889 2.77944C13.2897 2.75787 13.5886 2.8407 13.8318 3.01305L15.7261 4.35508C15.798 4.40603 15.7642 4.51602 15.6752 4.52086L5.67742 5.0646C5.37485 5.08106 5.0762 4.99217 4.83563 4.81406L3.44654 3.78562ZM5.20848 6.76919C5.20848 6.4444 5.47088 6.1761 5.80642 6.15783L16.3769 5.58216C16.7039 5.56435 16.9792 5.81583 16.9792 6.13239V15.6783C16.9792 16.0025 16.7177 16.2705 16.3829 16.2896L5.8793 16.8872C5.51537 16.9079 5.20848 16.6283 5.20848 16.2759V6.76919Z" fill="black" />
</g>
<defs>
<clipPath id="clip0_2164_11263">
<rect width="20" height="20" fill="white" />
</clipPath>
</defs>
</svg>
)
}

12
web/app/components/datasets/documents/list.tsx → web/app/components/datasets/documents/components/list.tsx
View File

@ -16,13 +16,16 @@ import * as React from 'react'
import { useCallback, useEffect, useMemo, useState } from 'react'
import { useTranslation } from 'react-i18next'
import Checkbox from '@/app/components/base/checkbox'
import FileTypeIcon from '@/app/components/base/file-uploader/file-type-icon'
import NotionIcon from '@/app/components/base/notion-icon'
import Pagination from '@/app/components/base/pagination'
import Toast from '@/app/components/base/toast'
import Tooltip from '@/app/components/base/tooltip'
import ChunkingModeLabel from '@/app/components/datasets/common/chunking-mode-label'
import { normalizeStatusForQuery } from '@/app/components/datasets/documents/status-filter'
import { extensionToFileType } from '@/app/components/datasets/hit-testing/utils/extension-to-file-type'
import EditMetadataBatchModal from '@/app/components/datasets/metadata/edit-metadata-batch/modal'
import useBatchEditDocumentMetadata from '@/app/components/datasets/metadata/hooks/use-batch-edit-document-metadata'
import { useDatasetDetailContextWithSelector as useDatasetDetailContext } from '@/context/dataset-detail'
import useTimestamp from '@/hooks/use-timestamp'
import { ChunkingMode, DataSourceType, DocumentActionType } from '@/models/datasets'
@ -31,14 +34,11 @@ import { useDocumentArchive, useDocumentBatchRetryIndex, useDocumentDelete, useD
import { asyncRunSafe } from '@/utils'
import { cn } from '@/utils/classnames'
import { formatNumber } from '@/utils/format'
import FileTypeIcon from '../../base/file-uploader/file-type-icon'
import ChunkingModeLabel from '../common/chunking-mode-label'
import useBatchEditDocumentMetadata from '../metadata/hooks/use-batch-edit-document-metadata'
import BatchAction from './detail/completed/common/batch-action'
import BatchAction from '../detail/completed/common/batch-action'
import StatusItem from '../status-item'
import s from '../style.module.css'
import Operations from './operations'
import RenameModal from './rename-modal'
import StatusItem from './status-item'
import s from './style.module.css'
export const renderTdValue = (value: string | number | null, isEmptyStyle = false) => {
return (

16
web/app/components/datasets/documents/operations.tsx → web/app/components/datasets/documents/components/operations.tsx
View File

@ -1,4 +1,4 @@
import type { OperationName } from './types'
import type { OperationName } from '../types'
import type { CommonResponse } from '@/models/common'
import {
RiArchive2Line,
@ -17,6 +17,12 @@ import * as React from 'react'
import { useCallback, useState } from 'react'
import { useTranslation } from 'react-i18next'
import { useContext } from 'use-context-selector'
import Confirm from '@/app/components/base/confirm'
import Divider from '@/app/components/base/divider'
import CustomPopover from '@/app/components/base/popover'
import Switch from '@/app/components/base/switch'
import { ToastContext } from '@/app/components/base/toast'
import Tooltip from '@/app/components/base/tooltip'
import { DataSourceType, DocumentActionType } from '@/models/datasets'
import {
useDocumentArchive,
@ -31,14 +37,8 @@ import {
} from '@/service/knowledge/use-document'
import { asyncRunSafe } from '@/utils'
import { cn } from '@/utils/classnames'
import Confirm from '../../base/confirm'
import Divider from '../../base/divider'
import CustomPopover from '../../base/popover'
import Switch from '../../base/switch'
import { ToastContext } from '../../base/toast'
import Tooltip from '../../base/tooltip'
import s from '../style.module.css'
import RenameModal from './rename-modal'
import s from './style.module.css'
type OperationsProps = {
embeddingAvailable: boolean

2
web/app/components/datasets/documents/rename-modal.tsx → web/app/components/datasets/documents/components/rename-modal.tsx
View File

@ -7,8 +7,8 @@ import { useTranslation } from 'react-i18next'
import Button from '@/app/components/base/button'
import Input from '@/app/components/base/input'
import Modal from '@/app/components/base/modal'
import Toast from '@/app/components/base/toast'
import { renameDocumentName } from '@/service/datasets'
import Toast from '../../base/toast'
type Props = {
datasetId: string

5
web/app/components/datasets/documents/create-from-pipeline/hooks/index.ts Normal file
View File

@ -0,0 +1,5 @@
export { useAddDocumentsSteps } from './use-add-documents-steps'
export { useDatasourceActions } from './use-datasource-actions'
export { useDatasourceOptions } from './use-datasource-options'
export { useLocalFile, useOnlineDocument, useOnlineDrive, useWebsiteCrawl } from './use-datasource-store'
export { useDatasourceUIState } from './use-datasource-ui-state'

41
web/app/components/datasets/documents/create-from-pipeline/hooks/use-add-documents-steps.ts Normal file
View File

@ -0,0 +1,41 @@
import { useCallback, useState } from 'react'
import { useTranslation } from 'react-i18next'
import { AddDocumentsStep } from '../types'
/**
* Hook for managing add documents wizard steps
*/
export const useAddDocumentsSteps = () => {
const { t } = useTranslation()
const [currentStep, setCurrentStep] = useState(1)
const handleNextStep = useCallback(() => {
setCurrentStep(preStep => preStep + 1)
}, [])
const handleBackStep = useCallback(() => {
setCurrentStep(preStep => preStep - 1)
}, [])
const steps = [
{
label: t('addDocuments.steps.chooseDatasource', { ns: 'datasetPipeline' }),
value: AddDocumentsStep.dataSource,
},
{
label: t('addDocuments.steps.processDocuments', { ns: 'datasetPipeline' }),
value: AddDocumentsStep.processDocuments,
},
{
label: t('addDocuments.steps.processingDocuments', { ns: 'datasetPipeline' }),
value: AddDocumentsStep.processingDocuments,
},
]
return {
steps,
currentStep,
handleNextStep,
handleBackStep,
}
}

321
web/app/components/datasets/documents/create-from-pipeline/hooks/use-datasource-actions.ts Normal file
View File

@ -0,0 +1,321 @@
import type { StoreApi } from 'zustand'
import type { DataSourceShape } from '@/app/components/datasets/documents/create-from-pipeline/data-source/store'
import type { Datasource } from '@/app/components/rag-pipeline/components/panel/test-run/types'
import type { DataSourceNotionPageMap, NotionPage } from '@/models/common'
import type { CrawlResultItem, DocumentItem, CustomFile as File, FileIndexingEstimateResponse } from '@/models/datasets'
import type {
OnlineDriveFile,
PublishedPipelineRunPreviewResponse,
PublishedPipelineRunResponse,
} from '@/models/pipeline'
import { useCallback, useRef } from 'react'
import { trackEvent } from '@/app/components/base/amplitude'
import { DatasourceType } from '@/models/pipeline'
import { useRunPublishedPipeline } from '@/service/use-pipeline'
import {
buildLocalFileDatasourceInfo,
buildOnlineDocumentDatasourceInfo,
buildOnlineDriveDatasourceInfo,
buildWebsiteCrawlDatasourceInfo,
} from '../utils/datasource-info-builder'
type DatasourceActionsParams = {
datasource: Datasource | undefined
datasourceType: string | undefined
pipelineId: string | undefined
dataSourceStore: StoreApi<DataSourceShape>
setEstimateData: (data: FileIndexingEstimateResponse | undefined) => void
setBatchId: (id: string) => void
setDocuments: (docs: PublishedPipelineRunResponse['documents']) => void
handleNextStep: () => void
PagesMapAndSelectedPagesId: DataSourceNotionPageMap
currentWorkspacePages: { page_id: string }[] | undefined
clearOnlineDocumentData: () => void
clearWebsiteCrawlData: () => void
clearOnlineDriveData: () => void
setDatasource: (ds: Datasource) => void
}
/**
* Hook for datasource-related actions (preview, process, etc.)
*/
export const useDatasourceActions = ({
datasource,
datasourceType,
pipelineId,
dataSourceStore,
setEstimateData,
setBatchId,
setDocuments,
handleNextStep,
PagesMapAndSelectedPagesId,
currentWorkspacePages,
clearOnlineDocumentData,
clearWebsiteCrawlData,
clearOnlineDriveData,
setDatasource,
}: DatasourceActionsParams) => {
const isPreview = useRef(false)
const formRef = useRef<{ submit: () => void } | null>(null)
const { mutateAsync: runPublishedPipeline, isIdle, isPending } = useRunPublishedPipeline()
// Build datasource info for preview (single item)
const buildPreviewDatasourceInfo = useCallback(() => {
const {
previewLocalFileRef,
previewOnlineDocumentRef,
previewWebsitePageRef,
previewOnlineDriveFileRef,
currentCredentialId,
bucket,
} = dataSourceStore.getState()
const datasourceInfoList: Record<string, unknown>[] = []
if (datasourceType === DatasourceType.localFile && previewLocalFileRef.current) {
datasourceInfoList.push(buildLocalFileDatasourceInfo(
previewLocalFileRef.current as File,
currentCredentialId,
))
}
if (datasourceType === DatasourceType.onlineDocument && previewOnlineDocumentRef.current) {
datasourceInfoList.push(buildOnlineDocumentDatasourceInfo(
previewOnlineDocumentRef.current,
currentCredentialId,
))
}
if (datasourceType === DatasourceType.websiteCrawl && previewWebsitePageRef.current) {
datasourceInfoList.push(buildWebsiteCrawlDatasourceInfo(
previewWebsitePageRef.current,
currentCredentialId,
))
}
if (datasourceType === DatasourceType.onlineDrive && previewOnlineDriveFileRef.current) {
datasourceInfoList.push(buildOnlineDriveDatasourceInfo(
previewOnlineDriveFileRef.current,
bucket,
currentCredentialId,
))
}
return datasourceInfoList
}, [dataSourceStore, datasourceType])
// Build datasource info for processing (all items)
const buildProcessDatasourceInfo = useCallback(() => {
const {
currentCredentialId,
localFileList,
onlineDocuments,
websitePages,
bucket,
selectedFileIds,
onlineDriveFileList,
} = dataSourceStore.getState()
const datasourceInfoList: Record<string, unknown>[] = []
if (datasourceType === DatasourceType.localFile) {
localFileList.forEach((file) => {
datasourceInfoList.push(buildLocalFileDatasourceInfo(file.file, currentCredentialId))
})
}
if (datasourceType === DatasourceType.onlineDocument) {
onlineDocuments.forEach((page) => {
datasourceInfoList.push(buildOnlineDocumentDatasourceInfo(page, currentCredentialId))
})
}
if (datasourceType === DatasourceType.websiteCrawl) {
websitePages.forEach((page) => {
datasourceInfoList.push(buildWebsiteCrawlDatasourceInfo(page, currentCredentialId))
})
}
if (datasourceType === DatasourceType.onlineDrive) {
selectedFileIds.forEach((id) => {
const file = onlineDriveFileList.find(f => f.id === id)
if (file)
datasourceInfoList.push(buildOnlineDriveDatasourceInfo(file, bucket, currentCredentialId))
})
}
return datasourceInfoList
}, [dataSourceStore, datasourceType])
// Handle chunk preview
const handlePreviewChunks = useCallback(async (data: Record<string, unknown>) => {
if (!datasource || !pipelineId)
return
const datasourceInfoList = buildPreviewDatasourceInfo()
await runPublishedPipeline({
pipeline_id: pipelineId,
inputs: data,
start_node_id: datasource.nodeId,
datasource_type: datasourceType as DatasourceType,
datasource_info_list: datasourceInfoList,
is_preview: true,
}, {
onSuccess: (res) => {
setEstimateData((res as PublishedPipelineRunPreviewResponse).data.outputs)
},
})
}, [datasource, pipelineId, datasourceType, buildPreviewDatasourceInfo, runPublishedPipeline, setEstimateData])
// Handle document processing
const handleProcess = useCallback(async (data: Record<string, unknown>) => {
if (!datasource || !pipelineId)
return
const datasourceInfoList = buildProcessDatasourceInfo()
await runPublishedPipeline({
pipeline_id: pipelineId,
inputs: data,
start_node_id: datasource.nodeId,
datasource_type: datasourceType as DatasourceType,
datasource_info_list: datasourceInfoList,
is_preview: false,
}, {
onSuccess: (res) => {
setBatchId((res as PublishedPipelineRunResponse).batch || '')
setDocuments((res as PublishedPipelineRunResponse).documents || [])
handleNextStep()
trackEvent('dataset_document_added', {
data_source_type: datasourceType,
indexing_technique: 'pipeline',
})
},
})
}, [datasource, pipelineId, datasourceType, buildProcessDatasourceInfo, runPublishedPipeline, setBatchId, setDocuments, handleNextStep])
// Form submission handlers
const onClickProcess = useCallback(() => {
isPreview.current = false
formRef.current?.submit()
}, [])
const onClickPreview = useCallback(() => {
isPreview.current = true
formRef.current?.submit()
}, [])
const handleSubmit = useCallback((data: Record<string, unknown>) => {
if (isPreview.current)
handlePreviewChunks(data)
else
handleProcess(data)
}, [handlePreviewChunks, handleProcess])
// Preview change handlers
const handlePreviewFileChange = useCallback((file: DocumentItem) => {
const { previewLocalFileRef } = dataSourceStore.getState()
previewLocalFileRef.current = file
onClickPreview()
}, [dataSourceStore, onClickPreview])
const handlePreviewOnlineDocumentChange = useCallback((page: NotionPage) => {
const { previewOnlineDocumentRef } = dataSourceStore.getState()
previewOnlineDocumentRef.current = page
onClickPreview()
}, [dataSourceStore, onClickPreview])
const handlePreviewWebsiteChange = useCallback((website: CrawlResultItem) => {
const { previewWebsitePageRef } = dataSourceStore.getState()
previewWebsitePageRef.current = website
onClickPreview()
}, [dataSourceStore, onClickPreview])
const handlePreviewOnlineDriveFileChange = useCallback((file: OnlineDriveFile) => {
const { previewOnlineDriveFileRef } = dataSourceStore.getState()
previewOnlineDriveFileRef.current = file
onClickPreview()
}, [dataSourceStore, onClickPreview])
// Select all handler
const handleSelectAll = useCallback(() => {
const {
onlineDocuments,
onlineDriveFileList,
selectedFileIds,
setOnlineDocuments,
setSelectedFileIds,
setSelectedPagesId,
} = dataSourceStore.getState()
if (datasourceType === DatasourceType.onlineDocument) {
const allIds = currentWorkspacePages?.map(page => page.page_id) || []
if (onlineDocuments.length < allIds.length) {
const selectedPages = Array.from(allIds).map(pageId => PagesMapAndSelectedPagesId[pageId])
setOnlineDocuments(selectedPages)
setSelectedPagesId(new Set(allIds))
}
else {
setOnlineDocuments([])
setSelectedPagesId(new Set())
}
}
if (datasourceType === DatasourceType.onlineDrive) {
const allKeys = onlineDriveFileList.filter(item => item.type !== 'bucket').map(file => file.id)
if (selectedFileIds.length < allKeys.length)
setSelectedFileIds(allKeys)
else
setSelectedFileIds([])
}
}, [PagesMapAndSelectedPagesId, currentWorkspacePages, dataSourceStore, datasourceType])
// Clear datasource data based on type
const clearDataSourceData = useCallback((dataSource: Datasource) => {
const providerType = dataSource.nodeData.provider_type
const clearFunctions: Record<string, () => void> = {
[DatasourceType.onlineDocument]: clearOnlineDocumentData,
[DatasourceType.websiteCrawl]: clearWebsiteCrawlData,
[DatasourceType.onlineDrive]: clearOnlineDriveData,
[DatasourceType.localFile]: () => {},
}
clearFunctions[providerType]?.()
}, [clearOnlineDocumentData, clearOnlineDriveData, clearWebsiteCrawlData])
// Switch datasource handler
const handleSwitchDataSource = useCallback((dataSource: Datasource) => {
const {
setCurrentCredentialId,
currentNodeIdRef,
} = dataSourceStore.getState()
clearDataSourceData(dataSource)
setCurrentCredentialId('')
currentNodeIdRef.current = dataSource.nodeId
setDatasource(dataSource)
}, [clearDataSourceData, dataSourceStore, setDatasource])
// Credential change handler
const handleCredentialChange = useCallback((credentialId: string) => {
const { setCurrentCredentialId } = dataSourceStore.getState()
if (datasource)
clearDataSourceData(datasource)
setCurrentCredentialId(credentialId)
}, [clearDataSourceData, dataSourceStore, datasource])
return {
isPreview,
formRef,
isIdle,
isPending,
onClickProcess,
onClickPreview,
handleSubmit,
handlePreviewFileChange,
handlePreviewOnlineDocumentChange,
handlePreviewWebsiteChange,
handlePreviewOnlineDriveFileChange,
handleSelectAll,
handleSwitchDataSource,
handleCredentialChange,
}
}

27
web/app/components/datasets/documents/create-from-pipeline/hooks/use-datasource-options.ts Normal file
View File

@ -0,0 +1,27 @@
import type { DataSourceOption } from '@/app/components/rag-pipeline/components/panel/test-run/types'
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
import type { Node } from '@/app/components/workflow/types'
import { useMemo } from 'react'
import { BlockEnum } from '@/app/components/workflow/types'
/**
* Hook for getting datasource options from pipeline nodes
*/
export const useDatasourceOptions = (pipelineNodes: Node<DataSourceNodeType>[]) => {
const datasourceNodes = pipelineNodes.filter(node => node.data.type === BlockEnum.DataSource)
const options = useMemo(() => {
const options: DataSourceOption[] = []
datasourceNodes.forEach((node) => {
const label = node.data.title
options.push({
label,
value: node.id,
data: node.data,
})
})
return options
}, [datasourceNodes])
return options
}

76
web/app/components/datasets/documents/create-from-pipeline/hooks.ts → web/app/components/datasets/documents/create-from-pipeline/hooks/use-datasource-store.ts
View File

@ -1,69 +1,12 @@
import type { DataSourceOption } from '@/app/components/rag-pipeline/components/panel/test-run/types'
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
import type { Node } from '@/app/components/workflow/types'
import type { DataSourceNotionPageMap, DataSourceNotionWorkspace } from '@/models/common'
import { useCallback, useMemo, useState } from 'react'
import { useTranslation } from 'react-i18next'
import { useCallback, useMemo } from 'react'
import { useShallow } from 'zustand/react/shallow'
import { BlockEnum } from '@/app/components/workflow/types'
import { CrawlStep } from '@/models/datasets'
import { useDataSourceStore, useDataSourceStoreWithSelector } from './data-source/store'
import { AddDocumentsStep } from './types'
export const useAddDocumentsSteps = () => {
const { t } = useTranslation()
const [currentStep, setCurrentStep] = useState(1)
const handleNextStep = useCallback(() => {
setCurrentStep(preStep => preStep + 1)
}, [])
const handleBackStep = useCallback(() => {
setCurrentStep(preStep => preStep - 1)
}, [])
const steps = [
{
label: t('addDocuments.steps.chooseDatasource', { ns: 'datasetPipeline' }),
value: AddDocumentsStep.dataSource,
},
{
label: t('addDocuments.steps.processDocuments', { ns: 'datasetPipeline' }),
value: AddDocumentsStep.processDocuments,
},
{
label: t('addDocuments.steps.processingDocuments', { ns: 'datasetPipeline' }),
value: AddDocumentsStep.processingDocuments,
},
]
return {
steps,
currentStep,
handleNextStep,
handleBackStep,
}
}
export const useDatasourceOptions = (pipelineNodes: Node<DataSourceNodeType>[]) => {
const datasourceNodes = pipelineNodes.filter(node => node.data.type === BlockEnum.DataSource)
const options = useMemo(() => {
const options: DataSourceOption[] = []
datasourceNodes.forEach((node) => {
const label = node.data.title
options.push({
label,
value: node.id,
data: node.data,
})
})
return options
}, [datasourceNodes])
return options
}
import { useDataSourceStore, useDataSourceStoreWithSelector } from '../data-source/store'
/**
* Hook for local file datasource store operations
*/
export const useLocalFile = () => {
const {
localFileList,
@ -89,6 +32,9 @@ export const useLocalFile = () => {
}
}
/**
* Hook for online document datasource store operations
*/
export const useOnlineDocument = () => {
const {
documentsData,
@ -147,6 +93,9 @@ export const useOnlineDocument = () => {
}
}
/**
* Hook for website crawl datasource store operations
*/
export const useWebsiteCrawl = () => {
const {
websitePages,
@ -186,6 +135,9 @@ export const useWebsiteCrawl = () => {
}
}
/**
* Hook for online drive datasource store operations
*/
export const useOnlineDrive = () => {
const {
onlineDriveFileList,

132
web/app/components/datasets/documents/create-from-pipeline/hooks/use-datasource-ui-state.ts Normal file
View File

@ -0,0 +1,132 @@
import type { Datasource } from '@/app/components/rag-pipeline/components/panel/test-run/types'
import type { OnlineDriveFile } from '@/models/pipeline'
import { useMemo } from 'react'
import { useTranslation } from 'react-i18next'
import { DatasourceType } from '@/models/pipeline'
type DatasourceUIStateParams = {
datasource: Datasource | undefined
allFileLoaded: boolean
localFileListLength: number
onlineDocumentsLength: number
websitePagesLength: number
selectedFileIdsLength: number
onlineDriveFileList: OnlineDriveFile[]
isVectorSpaceFull: boolean
enableBilling: boolean
currentWorkspacePagesLength: number
fileUploadConfig: { file_size_limit: number, batch_count_limit: number }
}
/**
* Hook for computing datasource UI state based on datasource type
*/
export const useDatasourceUIState = ({
datasource,
allFileLoaded,
localFileListLength,
onlineDocumentsLength,
websitePagesLength,
selectedFileIdsLength,
onlineDriveFileList,
isVectorSpaceFull,
enableBilling,
currentWorkspacePagesLength,
fileUploadConfig,
}: DatasourceUIStateParams) => {
const { t } = useTranslation()
const datasourceType = datasource?.nodeData.provider_type
const isShowVectorSpaceFull = useMemo(() => {
if (!datasource || !datasourceType)
return false
// Lookup table for vector space full condition check
const vectorSpaceFullConditions: Record<string, boolean> = {
[DatasourceType.localFile]: allFileLoaded,
[DatasourceType.onlineDocument]: onlineDocumentsLength > 0,
[DatasourceType.websiteCrawl]: websitePagesLength > 0,
[DatasourceType.onlineDrive]: onlineDriveFileList.length > 0,
}
const condition = vectorSpaceFullConditions[datasourceType]
return condition && isVectorSpaceFull && enableBilling
}, [datasource, datasourceType, allFileLoaded, onlineDocumentsLength, websitePagesLength, onlineDriveFileList.length, isVectorSpaceFull, enableBilling])
// Lookup table for next button disabled conditions
const nextBtnDisabled = useMemo(() => {
if (!datasource || !datasourceType)
return true
const disabledConditions: Record<string, boolean> = {
[DatasourceType.localFile]: isShowVectorSpaceFull || localFileListLength === 0 || !allFileLoaded,
[DatasourceType.onlineDocument]: isShowVectorSpaceFull || onlineDocumentsLength === 0,
[DatasourceType.websiteCrawl]: isShowVectorSpaceFull || websitePagesLength === 0,
[DatasourceType.onlineDrive]: isShowVectorSpaceFull || selectedFileIdsLength === 0,
}
return disabledConditions[datasourceType] ?? true
}, [datasource, datasourceType, isShowVectorSpaceFull, localFileListLength, allFileLoaded, onlineDocumentsLength, websitePagesLength, selectedFileIdsLength])
// Check if select all should be shown
const showSelect = useMemo(() => {
if (datasourceType === DatasourceType.onlineDocument)
return currentWorkspacePagesLength > 0
if (datasourceType === DatasourceType.onlineDrive) {
const nonBucketItems = onlineDriveFileList.filter(item => item.type !== 'bucket')
const isBucketList = onlineDriveFileList.some(file => file.type === 'bucket')
return !isBucketList && nonBucketItems.length > 0
}
return false
}, [currentWorkspacePagesLength, datasourceType, onlineDriveFileList])
// Total selectable options count
const totalOptions = useMemo(() => {
if (datasourceType === DatasourceType.onlineDocument)
return currentWorkspacePagesLength
if (datasourceType === DatasourceType.onlineDrive)
return onlineDriveFileList.filter(item => item.type !== 'bucket').length
return undefined
}, [currentWorkspacePagesLength, datasourceType, onlineDriveFileList])
// Selected options count
const selectedOptions = useMemo(() => {
if (datasourceType === DatasourceType.onlineDocument)
return onlineDocumentsLength
if (datasourceType === DatasourceType.onlineDrive)
return selectedFileIdsLength
return undefined
}, [datasourceType, onlineDocumentsLength, selectedFileIdsLength])
// Tip message for selection
const tip = useMemo(() => {
if (datasourceType === DatasourceType.onlineDocument)
return t('addDocuments.selectOnlineDocumentTip', { ns: 'datasetPipeline', count: 50 })
if (datasourceType === DatasourceType.onlineDrive) {
return t('addDocuments.selectOnlineDriveTip', {
ns: 'datasetPipeline',
count: fileUploadConfig.batch_count_limit,
fileSize: fileUploadConfig.file_size_limit,
})
}
return ''
}, [datasourceType, fileUploadConfig.batch_count_limit, fileUploadConfig.file_size_limit, t])
return {
datasourceType,
isShowVectorSpaceFull,
nextBtnDisabled,
showSelect,
totalOptions,
selectedOptions,
tip,
}
}

2698
web/app/components/datasets/documents/create-from-pipeline/index.spec.tsx Normal file
View File

File diff suppressed because it is too large Load Diff

683
web/app/components/datasets/documents/create-from-pipeline/index.tsx
View File

@ -2,75 +2,71 @@
import type { Datasource } from '@/app/components/rag-pipeline/components/panel/test-run/types'
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
import type { Node } from '@/app/components/workflow/types'
import type { NotionPage } from '@/models/common'
import type { CrawlResultItem, DocumentItem, CustomFile as File, FileIndexingEstimateResponse } from '@/models/datasets'
import type {
InitialDocumentDetail,
OnlineDriveFile,
PublishedPipelineRunPreviewResponse,
PublishedPipelineRunResponse,
} from '@/models/pipeline'
import type { FileIndexingEstimateResponse } from '@/models/datasets'
import type { InitialDocumentDetail } from '@/models/pipeline'
import { useBoolean } from 'ahooks'
import { useCallback, useMemo, useRef, useState } from 'react'
import { useCallback, useMemo, useState } from 'react'
import { useTranslation } from 'react-i18next'
import { trackEvent } from '@/app/components/base/amplitude'
import Divider from '@/app/components/base/divider'
import Loading from '@/app/components/base/loading'
import PlanUpgradeModal from '@/app/components/billing/plan-upgrade-modal'
import VectorSpaceFull from '@/app/components/billing/vector-space-full'
import LocalFile from '@/app/components/datasets/documents/create-from-pipeline/data-source/local-file'
import OnlineDocuments from '@/app/components/datasets/documents/create-from-pipeline/data-source/online-documents'
import OnlineDrive from '@/app/components/datasets/documents/create-from-pipeline/data-source/online-drive'
import WebsiteCrawl from '@/app/components/datasets/documents/create-from-pipeline/data-source/website-crawl'
import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
import { useProviderContextSelector } from '@/context/provider-context'
import { DatasourceType } from '@/models/pipeline'
import { useFileUploadConfig } from '@/service/use-common'
import { usePublishedPipelineInfo, useRunPublishedPipeline } from '@/service/use-pipeline'
import { TransferMethod } from '@/types/app'
import UpgradeCard from '../../create/step-one/upgrade-card'
import Actions from './actions'
import DataSourceOptions from './data-source-options'
import { usePublishedPipelineInfo } from '@/service/use-pipeline'
import { useDataSourceStore } from './data-source/store'
import DataSourceProvider from './data-source/store/provider'
import { useAddDocumentsSteps, useLocalFile, useOnlineDocument, useOnlineDrive, useWebsiteCrawl } from './hooks'
import {
useAddDocumentsSteps,
useDatasourceActions,
useDatasourceUIState,
useLocalFile,
useOnlineDocument,
useOnlineDrive,
useWebsiteCrawl,
} from './hooks'
import LeftHeader from './left-header'
import ChunkPreview from './preview/chunk-preview'
import FilePreview from './preview/file-preview'
import OnlineDocumentPreview from './preview/online-document-preview'
import WebsitePreview from './preview/web-preview'
import ProcessDocuments from './process-documents'
import Processing from './processing'
import { StepOneContent, StepThreeContent, StepTwoContent } from './steps'
import { StepOnePreview, StepTwoPreview } from './steps/preview-panel'
const CreateFormPipeline = () => {
const { t } = useTranslation()
const plan = useProviderContextSelector(state => state.plan)
const enableBilling = useProviderContextSelector(state => state.enableBilling)
const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id)
const dataSourceStore = useDataSourceStore()
// Core state
const [datasource, setDatasource] = useState<Datasource>()
const [estimateData, setEstimateData] = useState<FileIndexingEstimateResponse | undefined>(undefined)
const [batchId, setBatchId] = useState('')
const [documents, setDocuments] = useState<InitialDocumentDetail[]>([])
const dataSourceStore = useDataSourceStore()
const isPreview = useRef(false)
const formRef = useRef<any>(null)
// Data fetching
const { data: pipelineInfo, isFetching: isFetchingPipelineInfo } = usePublishedPipelineInfo(pipelineId || '')
const { data: fileUploadConfigResponse } = useFileUploadConfig()
const fileUploadConfig = useMemo(() => fileUploadConfigResponse ?? {
file_size_limit: 15,
batch_count_limit: 5,
}, [fileUploadConfigResponse])
// Steps management
const {
steps,
currentStep,
handleNextStep: doHandleNextStep,
handleBackStep,
} = useAddDocumentsSteps()
// Datasource-specific hooks
const {
localFileList,
allFileLoaded,
currentLocalFile,
hidePreviewLocalFile,
} = useLocalFile()
const {
currentWorkspace,
onlineDocuments,
@ -79,12 +75,14 @@ const CreateFormPipeline = () => {
hidePreviewOnlineDocument,
clearOnlineDocumentData,
} = useOnlineDocument()
const {
websitePages,
currentWebsite,
hideWebsitePreview,
clearWebsiteCrawlData,
} = useWebsiteCrawl()
const {
onlineDriveFileList,
selectedFileIds,
@ -92,43 +90,50 @@ const CreateFormPipeline = () => {
clearOnlineDriveData,
} = useOnlineDrive()
const datasourceType = useMemo(() => datasource?.nodeData.provider_type, [datasource])
// Computed values
const isVectorSpaceFull = plan.usage.vectorSpace >= plan.total.vectorSpace
const isShowVectorSpaceFull = useMemo(() => {
if (!datasource)
return false
if (datasourceType === DatasourceType.localFile)
return allFileLoaded && isVectorSpaceFull && enableBilling
if (datasourceType === DatasourceType.onlineDocument)
return onlineDocuments.length > 0 && isVectorSpaceFull && enableBilling
if (datasourceType === DatasourceType.websiteCrawl)
return websitePages.length > 0 && isVectorSpaceFull && enableBilling
if (datasourceType === DatasourceType.onlineDrive)
return onlineDriveFileList.length > 0 && isVectorSpaceFull && enableBilling
return false
}, [allFileLoaded, datasource, datasourceType, enableBilling, isVectorSpaceFull, onlineDocuments.length, onlineDriveFileList.length, websitePages.length])
const supportBatchUpload = !enableBilling || plan.type !== 'sandbox'
// UI state
const {
datasourceType,
isShowVectorSpaceFull,
nextBtnDisabled,
showSelect,
totalOptions,
selectedOptions,
tip,
} = useDatasourceUIState({
datasource,
allFileLoaded,
localFileListLength: localFileList.length,
onlineDocumentsLength: onlineDocuments.length,
websitePagesLength: websitePages.length,
selectedFileIdsLength: selectedFileIds.length,
onlineDriveFileList,
isVectorSpaceFull,
enableBilling,
currentWorkspacePagesLength: currentWorkspace?.pages.length ?? 0,
fileUploadConfig,
})
// Plan upgrade modal
const [isShowPlanUpgradeModal, {
setTrue: showPlanUpgradeModal,
setFalse: hidePlanUpgradeModal,
}] = useBoolean(false)
// Next step with batch upload check
const handleNextStep = useCallback(() => {
if (!supportBatchUpload) {
let isMultiple = false
if (datasourceType === DatasourceType.localFile && localFileList.length > 1)
isMultiple = true
if (datasourceType === DatasourceType.onlineDocument && onlineDocuments.length > 1)
isMultiple = true
if (datasourceType === DatasourceType.websiteCrawl && websitePages.length > 1)
isMultiple = true
if (datasourceType === DatasourceType.onlineDrive && selectedFileIds.length > 1)
isMultiple = true
if (isMultiple) {
const multipleCheckMap: Record<string, number> = {
[DatasourceType.localFile]: localFileList.length,
[DatasourceType.onlineDocument]: onlineDocuments.length,
[DatasourceType.websiteCrawl]: websitePages.length,
[DatasourceType.onlineDrive]: selectedFileIds.length,
}
const count = datasourceType ? multipleCheckMap[datasourceType] : 0
if (count > 1) {
showPlanUpgradeModal()
return
}
@ -136,334 +141,44 @@ const CreateFormPipeline = () => {
doHandleNextStep()
}, [datasourceType, doHandleNextStep, localFileList.length, onlineDocuments.length, selectedFileIds.length, showPlanUpgradeModal, supportBatchUpload, websitePages.length])
const nextBtnDisabled = useMemo(() => {
if (!datasource)
return true
if (datasourceType === DatasourceType.localFile)
return isShowVectorSpaceFull || !localFileList.length || !allFileLoaded
if (datasourceType === DatasourceType.onlineDocument)
return isShowVectorSpaceFull || !onlineDocuments.length
if (datasourceType === DatasourceType.websiteCrawl)
return isShowVectorSpaceFull || !websitePages.length
if (datasourceType === DatasourceType.onlineDrive)
return isShowVectorSpaceFull || !selectedFileIds.length
return false
}, [datasource, datasourceType, isShowVectorSpaceFull, localFileList.length, allFileLoaded, onlineDocuments.length, websitePages.length, selectedFileIds.length])
// Datasource actions
const {
isPreview,
formRef,
isIdle,
isPending,
onClickProcess,
onClickPreview,
handleSubmit,
handlePreviewFileChange,
handlePreviewOnlineDocumentChange,
handlePreviewWebsiteChange,
handlePreviewOnlineDriveFileChange,
handleSelectAll,
handleSwitchDataSource,
handleCredentialChange,
} = useDatasourceActions({
datasource,
datasourceType,
pipelineId,
dataSourceStore,
setEstimateData,
setBatchId,
setDocuments,
handleNextStep,
PagesMapAndSelectedPagesId,
currentWorkspacePages: currentWorkspace?.pages,
clearOnlineDocumentData,
clearWebsiteCrawlData,
clearOnlineDriveData,
setDatasource,
})
const fileUploadConfig = useMemo(() => fileUploadConfigResponse ?? {
file_size_limit: 15,
batch_count_limit: 5,
}, [fileUploadConfigResponse])
const showSelect = useMemo(() => {
if (datasourceType === DatasourceType.onlineDocument) {
const pagesCount = currentWorkspace?.pages.length ?? 0
return pagesCount > 0
}
if (datasourceType === DatasourceType.onlineDrive) {
const isBucketList = onlineDriveFileList.some(file => file.type === 'bucket')
return !isBucketList && onlineDriveFileList.filter((item) => {
return item.type !== 'bucket'
}).length > 0
}
return false
}, [currentWorkspace?.pages.length, datasourceType, onlineDriveFileList])
const totalOptions = useMemo(() => {
if (datasourceType === DatasourceType.onlineDocument)
return currentWorkspace?.pages.length
if (datasourceType === DatasourceType.onlineDrive) {
return onlineDriveFileList.filter((item) => {
return item.type !== 'bucket'
}).length
}
}, [currentWorkspace?.pages.length, datasourceType, onlineDriveFileList])
const selectedOptions = useMemo(() => {
if (datasourceType === DatasourceType.onlineDocument)
return onlineDocuments.length
if (datasourceType === DatasourceType.onlineDrive)
return selectedFileIds.length
}, [datasourceType, onlineDocuments.length, selectedFileIds.length])
const tip = useMemo(() => {
if (datasourceType === DatasourceType.onlineDocument)
return t('addDocuments.selectOnlineDocumentTip', { ns: 'datasetPipeline', count: 50 })
if (datasourceType === DatasourceType.onlineDrive) {
return t('addDocuments.selectOnlineDriveTip', {
ns: 'datasetPipeline',
count: fileUploadConfig.batch_count_limit,
fileSize: fileUploadConfig.file_size_limit,
})
}
return ''
}, [datasourceType, fileUploadConfig.batch_count_limit, fileUploadConfig.file_size_limit, t])
const { mutateAsync: runPublishedPipeline, isIdle, isPending } = useRunPublishedPipeline()
const handlePreviewChunks = useCallback(async (data: Record<string, any>) => {
if (!datasource)
return
const {
previewLocalFileRef,
previewOnlineDocumentRef,
previewWebsitePageRef,
previewOnlineDriveFileRef,
currentCredentialId,
} = dataSourceStore.getState()
const datasourceInfoList: Record<string, any>[] = []
if (datasourceType === DatasourceType.localFile) {
const { id, name, type, size, extension, mime_type } = previewLocalFileRef.current as File
const documentInfo = {
related_id: id,
name,
type,
size,
extension,
mime_type,
url: '',
transfer_method: TransferMethod.local_file,
credential_id: currentCredentialId,
}
datasourceInfoList.push(documentInfo)
}
if (datasourceType === DatasourceType.onlineDocument) {
const { workspace_id, ...rest } = previewOnlineDocumentRef.current!
const documentInfo = {
workspace_id,
page: rest,
credential_id: currentCredentialId,
}
datasourceInfoList.push(documentInfo)
}
if (datasourceType === DatasourceType.websiteCrawl) {
datasourceInfoList.push({
...previewWebsitePageRef.current!,
credential_id: currentCredentialId,
})
}
if (datasourceType === DatasourceType.onlineDrive) {
const { bucket } = dataSourceStore.getState()
const { id, type, name } = previewOnlineDriveFileRef.current!
datasourceInfoList.push({
bucket,
id,
name,
type,
credential_id: currentCredentialId,
})
}
await runPublishedPipeline({
pipeline_id: pipelineId!,
inputs: data,
start_node_id: datasource.nodeId,
datasource_type: datasourceType as DatasourceType,
datasource_info_list: datasourceInfoList,
is_preview: true,
}, {
onSuccess: (res) => {
setEstimateData((res as PublishedPipelineRunPreviewResponse).data.outputs)
},
})
}, [datasource, datasourceType, runPublishedPipeline, pipelineId, dataSourceStore])
const handleProcess = useCallback(async (data: Record<string, any>) => {
if (!datasource)
return
const { currentCredentialId } = dataSourceStore.getState()
const datasourceInfoList: Record<string, any>[] = []
if (datasourceType === DatasourceType.localFile) {
const {
localFileList,
} = dataSourceStore.getState()
localFileList.forEach((file) => {
const { id, name, type, size, extension, mime_type } = file.file
const documentInfo = {
related_id: id,
name,
type,
size,
extension,
mime_type,
url: '',
transfer_method: TransferMethod.local_file,
credential_id: currentCredentialId,
}
datasourceInfoList.push(documentInfo)
})
}
if (datasourceType === DatasourceType.onlineDocument) {
const {
onlineDocuments,
} = dataSourceStore.getState()
onlineDocuments.forEach((page) => {
const { workspace_id, ...rest } = page
const documentInfo = {
workspace_id,
page: rest,
credential_id: currentCredentialId,
}
datasourceInfoList.push(documentInfo)
})
}
if (datasourceType === DatasourceType.websiteCrawl) {
const {
websitePages,
} = dataSourceStore.getState()
websitePages.forEach((websitePage) => {
datasourceInfoList.push({
...websitePage,
credential_id: currentCredentialId,
})
})
}
if (datasourceType === DatasourceType.onlineDrive) {
const {
bucket,
selectedFileIds,
onlineDriveFileList,
} = dataSourceStore.getState()
selectedFileIds.forEach((id) => {
const file = onlineDriveFileList.find(file => file.id === id)
datasourceInfoList.push({
bucket,
id: file?.id,
name: file?.name,
type: file?.type,
credential_id: currentCredentialId,
})
})
}
await runPublishedPipeline({
pipeline_id: pipelineId!,
inputs: data,
start_node_id: datasource.nodeId,
datasource_type: datasourceType as DatasourceType,
datasource_info_list: datasourceInfoList,
is_preview: false,
}, {
onSuccess: (res) => {
setBatchId((res as PublishedPipelineRunResponse).batch || '')
setDocuments((res as PublishedPipelineRunResponse).documents || [])
handleNextStep()
trackEvent('dataset_document_added', {
data_source_type: datasourceType,
indexing_technique: 'pipeline',
})
},
})
}, [dataSourceStore, datasource, datasourceType, handleNextStep, pipelineId, runPublishedPipeline])
const onClickProcess = useCallback(() => {
isPreview.current = false
formRef.current?.submit()
}, [])
const onClickPreview = useCallback(() => {
isPreview.current = true
formRef.current?.submit()
}, [])
const handleSubmit = useCallback((data: Record<string, any>) => {
if (isPreview.current)
handlePreviewChunks(data)
else
handleProcess(data)
}, [handlePreviewChunks, handleProcess])
const handlePreviewFileChange = useCallback((file: DocumentItem) => {
const { previewLocalFileRef } = dataSourceStore.getState()
previewLocalFileRef.current = file
onClickPreview()
}, [dataSourceStore, onClickPreview])
const handlePreviewOnlineDocumentChange = useCallback((page: NotionPage) => {
const { previewOnlineDocumentRef } = dataSourceStore.getState()
previewOnlineDocumentRef.current = page
onClickPreview()
}, [dataSourceStore, onClickPreview])
const handlePreviewWebsiteChange = useCallback((website: CrawlResultItem) => {
const { previewWebsitePageRef } = dataSourceStore.getState()
previewWebsitePageRef.current = website
onClickPreview()
}, [dataSourceStore, onClickPreview])
const handlePreviewOnlineDriveFileChange = useCallback((file: OnlineDriveFile) => {
const { previewOnlineDriveFileRef } = dataSourceStore.getState()
previewOnlineDriveFileRef.current = file
onClickPreview()
}, [dataSourceStore, onClickPreview])
const handleSelectAll = useCallback(() => {
const {
onlineDocuments,
onlineDriveFileList,
selectedFileIds,
setOnlineDocuments,
setSelectedFileIds,
setSelectedPagesId,
} = dataSourceStore.getState()
if (datasourceType === DatasourceType.onlineDocument) {
const allIds = currentWorkspace?.pages.map(page => page.page_id) || []
if (onlineDocuments.length < allIds.length) {
const selectedPages = Array.from(allIds).map(pageId => PagesMapAndSelectedPagesId[pageId])
setOnlineDocuments(selectedPages)
setSelectedPagesId(new Set(allIds))
}
else {
setOnlineDocuments([])
setSelectedPagesId(new Set())
}
}
if (datasourceType === DatasourceType.onlineDrive) {
const allKeys = onlineDriveFileList.filter((item) => {
return item.type !== 'bucket'
}).map(file => file.id)
if (selectedFileIds.length < allKeys.length)
setSelectedFileIds(allKeys)
else
setSelectedFileIds([])
}
}, [PagesMapAndSelectedPagesId, currentWorkspace?.pages, dataSourceStore, datasourceType])
const clearDataSourceData = useCallback((dataSource: Datasource) => {
const providerType = dataSource.nodeData.provider_type
if (providerType === DatasourceType.onlineDocument)
clearOnlineDocumentData()
else if (providerType === DatasourceType.websiteCrawl)
clearWebsiteCrawlData()
else if (providerType === DatasourceType.onlineDrive)
clearOnlineDriveData()
}, [clearOnlineDocumentData, clearOnlineDriveData, clearWebsiteCrawlData])
const handleSwitchDataSource = useCallback((dataSource: Datasource) => {
const {
setCurrentCredentialId,
currentNodeIdRef,
} = dataSourceStore.getState()
clearDataSourceData(dataSource)
setCurrentCredentialId('')
currentNodeIdRef.current = dataSource.nodeId
setDatasource(dataSource)
}, [clearDataSourceData, dataSourceStore])
const handleCredentialChange = useCallback((credentialId: string) => {
const { setCurrentCredentialId } = dataSourceStore.getState()
clearDataSourceData(datasource!)
setCurrentCredentialId(credentialId)
}, [clearDataSourceData, dataSourceStore, datasource])
if (isFetchingPipelineInfo) {
return (
<Loading type="app" />
)
}
if (isFetchingPipelineInfo)
return <Loading type="app" />
return (
<div
className="relative flex h-[calc(100vh-56px)] w-full min-w-[1024px] overflow-x-auto rounded-t-2xl border-t border-effects-highlight bg-background-default-subtle"
>
<div className="relative flex h-[calc(100vh-56px)] w-full min-w-[1024px] overflow-x-auto rounded-t-2xl border-t border-effects-highlight bg-background-default-subtle">
<div className="h-full min-w-0 flex-1">
<div className="flex h-full flex-col px-14">
<LeftHeader
@ -472,139 +187,77 @@ const CreateFormPipeline = () => {
currentStep={currentStep}
/>
<div className="grow overflow-y-auto">
{
currentStep === 1 && (
<div className="flex flex-col gap-y-5 pt-4">
<DataSourceOptions
datasourceNodeId={datasource?.nodeId || ''}
onSelect={handleSwitchDataSource}
pipelineNodes={(pipelineInfo?.graph.nodes || []) as Node<DataSourceNodeType>[]}
/>
{datasourceType === DatasourceType.localFile && (
<LocalFile
allowedExtensions={datasource!.nodeData.fileExtensions || []}
supportBatchUpload={supportBatchUpload}
/>
)}
{datasourceType === DatasourceType.onlineDocument && (
<OnlineDocuments
nodeId={datasource!.nodeId}
nodeData={datasource!.nodeData}
onCredentialChange={handleCredentialChange}
/>
)}
{datasourceType === DatasourceType.websiteCrawl && (
<WebsiteCrawl
nodeId={datasource!.nodeId}
nodeData={datasource!.nodeData}
onCredentialChange={handleCredentialChange}
/>
)}
{datasourceType === DatasourceType.onlineDrive && (
<OnlineDrive
nodeId={datasource!.nodeId}
nodeData={datasource!.nodeData}
onCredentialChange={handleCredentialChange}
/>
)}
{isShowVectorSpaceFull && (
<VectorSpaceFull />
)}
<Actions
showSelect={showSelect}
totalOptions={totalOptions}
selectedOptions={selectedOptions}
onSelectAll={handleSelectAll}
disabled={nextBtnDisabled}
handleNextStep={handleNextStep}
tip={tip}
/>
{
!supportBatchUpload && datasourceType === DatasourceType.localFile && localFileList.length > 0 && (
<>
<Divider type="horizontal" className="my-4 h-px bg-divider-subtle" />
<UpgradeCard />
</>
)
}
</div>
)
}
{
currentStep === 2 && (
<ProcessDocuments
ref={formRef}
dataSourceNodeId={datasource!.nodeId}
isRunning={isPending}
onProcess={onClickProcess}
onPreview={onClickPreview}
onSubmit={handleSubmit}
onBack={handleBackStep}
/>
)
}
{
currentStep === 3 && (
<Processing
batchId={batchId}
documents={documents}
/>
)
}
{currentStep === 1 && (
<StepOneContent
datasource={datasource}
datasourceType={datasourceType}
pipelineNodes={(pipelineInfo?.graph.nodes || []) as Node<DataSourceNodeType>[]}
supportBatchUpload={supportBatchUpload}
localFileListLength={localFileList.length}
isShowVectorSpaceFull={isShowVectorSpaceFull}
showSelect={showSelect}
totalOptions={totalOptions}
selectedOptions={selectedOptions}
tip={tip}
nextBtnDisabled={nextBtnDisabled}
onSelectDataSource={handleSwitchDataSource}
onCredentialChange={handleCredentialChange}
onSelectAll={handleSelectAll}
onNextStep={handleNextStep}
/>
)}
{currentStep === 2 && (
<StepTwoContent
formRef={formRef}
dataSourceNodeId={datasource!.nodeId}
isRunning={isPending}
onProcess={onClickProcess}
onPreview={onClickPreview}
onSubmit={handleSubmit}
onBack={handleBackStep}
/>
)}
{currentStep === 3 && (
<StepThreeContent
batchId={batchId}
documents={documents}
/>
)}
</div>
</div>
</div>
{/* Preview */}
{
currentStep === 1 && (
<div className="h-full min-w-0 flex-1">
<div className="flex h-full flex-col pl-2 pt-2">
{currentLocalFile && (
<FilePreview
file={currentLocalFile}
hidePreview={hidePreviewLocalFile}
/>
)}
{currentDocument && (
<OnlineDocumentPreview
datasourceNodeId={datasource!.nodeId}
currentPage={currentDocument}
hidePreview={hidePreviewOnlineDocument}
/>
)}
{currentWebsite && (
<WebsitePreview
currentWebsite={currentWebsite}
hidePreview={hideWebsitePreview}
/>
)}
</div>
</div>
)
}
{
currentStep === 2 && (
<div className="h-full min-w-0 flex-1">
<div className="flex h-full flex-col pl-2 pt-2">
<ChunkPreview
dataSourceType={datasourceType as DatasourceType}
localFiles={localFileList.map(file => file.file)}
onlineDocuments={onlineDocuments}
websitePages={websitePages}
onlineDriveFiles={selectedOnlineDriveFileList}
isIdle={isIdle}
isPending={isPending && isPreview.current}
estimateData={estimateData}
onPreview={onClickPreview}
handlePreviewFileChange={handlePreviewFileChange}
handlePreviewOnlineDocumentChange={handlePreviewOnlineDocumentChange}
handlePreviewWebsitePageChange={handlePreviewWebsiteChange}
handlePreviewOnlineDriveFileChange={handlePreviewOnlineDriveFileChange}
/>
</div>
</div>
)
}
{/* Preview Panel */}
{currentStep === 1 && (
<StepOnePreview
datasource={datasource}
currentLocalFile={currentLocalFile}
currentDocument={currentDocument}
currentWebsite={currentWebsite}
hidePreviewLocalFile={hidePreviewLocalFile}
hidePreviewOnlineDocument={hidePreviewOnlineDocument}
hideWebsitePreview={hideWebsitePreview}
/>
)}
{currentStep === 2 && (
<StepTwoPreview
datasourceType={datasourceType}
localFileList={localFileList}
onlineDocuments={onlineDocuments}
websitePages={websitePages}
selectedOnlineDriveFileList={selectedOnlineDriveFileList}
isIdle={isIdle}
isPendingPreview={isPending && isPreview.current}
estimateData={estimateData}
onPreview={onClickPreview}
handlePreviewFileChange={handlePreviewFileChange}
handlePreviewOnlineDocumentChange={handlePreviewOnlineDocumentChange}
handlePreviewWebsitePageChange={handlePreviewWebsiteChange}
handlePreviewOnlineDriveFileChange={handlePreviewOnlineDriveFileChange}
/>
)}
{/* Plan Upgrade Modal */}
{isShowPlanUpgradeModal && (
<PlanUpgradeModal
show

3
web/app/components/datasets/documents/create-from-pipeline/steps/index.ts Normal file
View File

@ -0,0 +1,3 @@
export { default as StepOneContent } from './step-one-content'
export { default as StepThreeContent } from './step-three-content'
export { default as StepTwoContent } from './step-two-content'

112
web/app/components/datasets/documents/create-from-pipeline/steps/preview-panel.tsx Normal file
View File

@ -0,0 +1,112 @@
'use client'
import type { Datasource } from '@/app/components/rag-pipeline/components/panel/test-run/types'
import type { NotionPage } from '@/models/common'
import type { CrawlResultItem, CustomFile, DocumentItem, FileIndexingEstimateResponse, FileItem } from '@/models/datasets'
import type { DatasourceType, OnlineDriveFile } from '@/models/pipeline'
import { memo } from 'react'
import ChunkPreview from '../preview/chunk-preview'
import FilePreview from '../preview/file-preview'
import OnlineDocumentPreview from '../preview/online-document-preview'
import WebsitePreview from '../preview/web-preview'
type StepOnePreviewProps = {
datasource: Datasource | undefined
currentLocalFile: CustomFile | undefined
currentDocument: (NotionPage & { workspace_id: string }) | undefined
currentWebsite: CrawlResultItem | undefined
hidePreviewLocalFile: () => void
hidePreviewOnlineDocument: () => void
hideWebsitePreview: () => void
}
export const StepOnePreview = memo(({
datasource,
currentLocalFile,
currentDocument,
currentWebsite,
hidePreviewLocalFile,
hidePreviewOnlineDocument,
hideWebsitePreview,
}: StepOnePreviewProps) => {
return (
<div className="h-full min-w-0 flex-1">
<div className="flex h-full flex-col pl-2 pt-2">
{currentLocalFile && (
<FilePreview
file={currentLocalFile}
hidePreview={hidePreviewLocalFile}
/>
)}
{currentDocument && (
<OnlineDocumentPreview
datasourceNodeId={datasource!.nodeId}
currentPage={currentDocument}
hidePreview={hidePreviewOnlineDocument}
/>
)}
{currentWebsite && (
<WebsitePreview
currentWebsite={currentWebsite}
hidePreview={hideWebsitePreview}
/>
)}
</div>
</div>
)
})
StepOnePreview.displayName = 'StepOnePreview'
type StepTwoPreviewProps = {
datasourceType: string | undefined
localFileList: FileItem[]
onlineDocuments: (NotionPage & { workspace_id: string })[]
websitePages: CrawlResultItem[]
selectedOnlineDriveFileList: OnlineDriveFile[]
isIdle: boolean
isPendingPreview: boolean
estimateData: FileIndexingEstimateResponse | undefined
onPreview: () => void
handlePreviewFileChange: (file: DocumentItem) => void
handlePreviewOnlineDocumentChange: (page: NotionPage) => void
handlePreviewWebsitePageChange: (website: CrawlResultItem) => void
handlePreviewOnlineDriveFileChange: (file: OnlineDriveFile) => void
}
export const StepTwoPreview = memo(({
datasourceType,
localFileList,
onlineDocuments,
websitePages,
selectedOnlineDriveFileList,
isIdle,
isPendingPreview,
estimateData,
onPreview,
handlePreviewFileChange,
handlePreviewOnlineDocumentChange,
handlePreviewWebsitePageChange,
handlePreviewOnlineDriveFileChange,
}: StepTwoPreviewProps) => {
return (
<div className="h-full min-w-0 flex-1">
<div className="flex h-full flex-col pl-2 pt-2">
<ChunkPreview
dataSourceType={datasourceType as DatasourceType}
localFiles={localFileList.map(file => file.file)}
onlineDocuments={onlineDocuments}
websitePages={websitePages}
onlineDriveFiles={selectedOnlineDriveFileList}
isIdle={isIdle}
isPending={isPendingPreview}
estimateData={estimateData}
onPreview={onPreview}
handlePreviewFileChange={handlePreviewFileChange}
handlePreviewOnlineDocumentChange={handlePreviewOnlineDocumentChange}
handlePreviewWebsitePageChange={handlePreviewWebsitePageChange}
handlePreviewOnlineDriveFileChange={handlePreviewOnlineDriveFileChange}
/>
</div>
</div>
)
})
StepTwoPreview.displayName = 'StepTwoPreview'

110
web/app/components/datasets/documents/create-from-pipeline/steps/step-one-content.tsx Normal file
View File

@ -0,0 +1,110 @@
'use client'
import type { Datasource } from '@/app/components/rag-pipeline/components/panel/test-run/types'
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
import type { Node } from '@/app/components/workflow/types'
import { memo } from 'react'
import Divider from '@/app/components/base/divider'
import VectorSpaceFull from '@/app/components/billing/vector-space-full'
import LocalFile from '@/app/components/datasets/documents/create-from-pipeline/data-source/local-file'
import OnlineDocuments from '@/app/components/datasets/documents/create-from-pipeline/data-source/online-documents'
import OnlineDrive from '@/app/components/datasets/documents/create-from-pipeline/data-source/online-drive'
import WebsiteCrawl from '@/app/components/datasets/documents/create-from-pipeline/data-source/website-crawl'
import { DatasourceType } from '@/models/pipeline'
import UpgradeCard from '../../../create/step-one/upgrade-card'
import Actions from '../actions'
import DataSourceOptions from '../data-source-options'
type StepOneContentProps = {
datasource: Datasource | undefined
datasourceType: string | undefined
pipelineNodes: Node<DataSourceNodeType>[]
supportBatchUpload: boolean
localFileListLength: number
isShowVectorSpaceFull: boolean
showSelect: boolean
totalOptions: number | undefined
selectedOptions: number | undefined
tip: string
nextBtnDisabled: boolean
onSelectDataSource: (dataSource: Datasource) => void
onCredentialChange: (credentialId: string) => void
onSelectAll: () => void
onNextStep: () => void
}
const StepOneContent = ({
datasource,
datasourceType,
pipelineNodes,
supportBatchUpload,
localFileListLength,
isShowVectorSpaceFull,
showSelect,
totalOptions,
selectedOptions,
tip,
nextBtnDisabled,
onSelectDataSource,
onCredentialChange,
onSelectAll,
onNextStep,
}: StepOneContentProps) => {
const showUpgradeCard = !supportBatchUpload
&& datasourceType === DatasourceType.localFile
&& localFileListLength > 0
return (
<div className="flex flex-col gap-y-5 pt-4">
<DataSourceOptions
datasourceNodeId={datasource?.nodeId || ''}
onSelect={onSelectDataSource}
pipelineNodes={pipelineNodes}
/>
{datasourceType === DatasourceType.localFile && (
<LocalFile
allowedExtensions={datasource!.nodeData.fileExtensions || []}
supportBatchUpload={supportBatchUpload}
/>
)}
{datasourceType === DatasourceType.onlineDocument && (
<OnlineDocuments
nodeId={datasource!.nodeId}
nodeData={datasource!.nodeData}
onCredentialChange={onCredentialChange}
/>
)}
{datasourceType === DatasourceType.websiteCrawl && (
<WebsiteCrawl
nodeId={datasource!.nodeId}
nodeData={datasource!.nodeData}
onCredentialChange={onCredentialChange}
/>
)}
{datasourceType === DatasourceType.onlineDrive && (
<OnlineDrive
nodeId={datasource!.nodeId}
nodeData={datasource!.nodeData}
onCredentialChange={onCredentialChange}
/>
)}
{isShowVectorSpaceFull && <VectorSpaceFull />}
<Actions
showSelect={showSelect}
totalOptions={totalOptions}
selectedOptions={selectedOptions}
onSelectAll={onSelectAll}
disabled={nextBtnDisabled}
handleNextStep={onNextStep}
tip={tip}
/>
{showUpgradeCard && (
<>
<Divider type="horizontal" className="my-4 h-px bg-divider-subtle" />
<UpgradeCard />
</>
)}
</div>
)
}
export default memo(StepOneContent)

23
web/app/components/datasets/documents/create-from-pipeline/steps/step-three-content.tsx Normal file
View File

@ -0,0 +1,23 @@
'use client'
import type { InitialDocumentDetail } from '@/models/pipeline'
import { memo } from 'react'
import Processing from '../processing'
type StepThreeContentProps = {
batchId: string
documents: InitialDocumentDetail[]
}
const StepThreeContent = ({
batchId,
documents,
}: StepThreeContentProps) => {
return (
<Processing
batchId={batchId}
documents={documents}
/>
)
}
export default memo(StepThreeContent)

38
web/app/components/datasets/documents/create-from-pipeline/steps/step-two-content.tsx Normal file
View File

@ -0,0 +1,38 @@
'use client'
import type { RefObject } from 'react'
import { memo } from 'react'
import ProcessDocuments from '../process-documents'
type StepTwoContentProps = {
formRef: RefObject<{ submit: () => void } | null>
dataSourceNodeId: string
isRunning: boolean
onProcess: () => void
onPreview: () => void
onSubmit: (data: Record<string, unknown>) => void
onBack: () => void
}
const StepTwoContent = ({
formRef,
dataSourceNodeId,
isRunning,
onProcess,
onPreview,
onSubmit,
onBack,
}: StepTwoContentProps) => {
return (
<ProcessDocuments
ref={formRef}
dataSourceNodeId={dataSourceNodeId}
isRunning={isRunning}
onProcess={onProcess}
onPreview={onPreview}
onSubmit={onSubmit}
onBack={onBack}
/>
)
}
export default memo(StepTwoContent)

63
web/app/components/datasets/documents/create-from-pipeline/utils/datasource-info-builder.ts Normal file
View File

@ -0,0 +1,63 @@
import type { NotionPage } from '@/models/common'
import type { CrawlResultItem, CustomFile as File } from '@/models/datasets'
import type { OnlineDriveFile } from '@/models/pipeline'
import { TransferMethod } from '@/types/app'
/**
* Build datasource info for local files
*/
export const buildLocalFileDatasourceInfo = (
file: File,
credentialId: string,
): Record<string, unknown> => ({
related_id: file.id,
name: file.name,
type: file.type,
size: file.size,
extension: file.extension,
mime_type: file.mime_type,
url: '',
transfer_method: TransferMethod.local_file,
credential_id: credentialId,
})
/**
* Build datasource info for online documents
*/
export const buildOnlineDocumentDatasourceInfo = (
page: NotionPage & { workspace_id: string },
credentialId: string,
): Record<string, unknown> => {
const { workspace_id, ...rest } = page
return {
workspace_id,
page: rest,
credential_id: credentialId,
}
}
/**
* Build datasource info for website crawl
*/
export const buildWebsiteCrawlDatasourceInfo = (
page: CrawlResultItem,
credentialId: string,
): Record<string, unknown> => ({
...page,
credential_id: credentialId,
})
/**
* Build datasource info for online drive
*/
export const buildOnlineDriveDatasourceInfo = (
file: OnlineDriveFile,
bucket: string,
credentialId: string,
): Record<string, unknown> => ({
bucket,
id: file.id,
name: file.name,
type: file.type,
credential_id: credentialId,
})

2
web/app/components/datasets/documents/detail/index.tsx
View File

@ -18,7 +18,7 @@ import { useDocumentDetail, useDocumentMetadata, useInvalidDocumentList } from '
import { useCheckSegmentBatchImportProgress, useChildSegmentListKey, useSegmentBatchImport, useSegmentListKey } from '@/service/knowledge/use-segment'
import { useInvalid } from '@/service/use-base'
import { cn } from '@/utils/classnames'
import Operations from '../operations'
import Operations from '../components/operations'
import StatusItem from '../status-item'
import BatchModal from './batch-modal'
import Completed from './completed'

197
web/app/components/datasets/documents/hooks/use-documents-page-state.ts Normal file
View File

@ -0,0 +1,197 @@
import type { DocumentListResponse } from '@/models/datasets'
import type { SortType } from '@/service/datasets'
import { useDebounce, useDebounceFn } from 'ahooks'
import { useCallback, useEffect, useMemo, useState } from 'react'
import { normalizeStatusForQuery, sanitizeStatusValue } from '../status-filter'
import useDocumentListQueryState from './use-document-list-query-state'
/**
* Custom hook to manage documents page state including:
* - Search state (input value, debounced search value)
* - Filter state (status filter, sort value)
* - Pagination state (current page, limit)
* - Selection state (selected document ids)
* - Polling state (timer control for auto-refresh)
*/
export function useDocumentsPageState() {
const { query, updateQuery } = useDocumentListQueryState()
// Search state
const [inputValue, setInputValue] = useState<string>('')
const [searchValue, setSearchValue] = useState<string>('')
const debouncedSearchValue = useDebounce(searchValue, { wait: 500 })
// Filter & sort state
const [statusFilterValue, setStatusFilterValue] = useState<string>(() => sanitizeStatusValue(query.status))
const [sortValue, setSortValue] = useState<SortType>(query.sort)
const normalizedStatusFilterValue = useMemo(
() => normalizeStatusForQuery(statusFilterValue),
[statusFilterValue],
)
// Pagination state
const [currPage, setCurrPage] = useState<number>(query.page - 1)
const [limit, setLimit] = useState<number>(query.limit)
// Selection state
const [selectedIds, setSelectedIds] = useState<string[]>([])
// Polling state
const [timerCanRun, setTimerCanRun] = useState(true)
// Initialize search value from URL on mount
useEffect(() => {
if (query.keyword) {
setInputValue(query.keyword)
setSearchValue(query.keyword)
}
}, []) // Only run on mount
// Sync local state with URL query changes
useEffect(() => {
setCurrPage(query.page - 1)
setLimit(query.limit)
if (query.keyword !== searchValue) {
setInputValue(query.keyword)
setSearchValue(query.keyword)
}
setStatusFilterValue((prev) => {
const nextValue = sanitizeStatusValue(query.status)
return prev === nextValue ? prev : nextValue
})
setSortValue(query.sort)
}, [query])
// Update URL when search changes
useEffect(() => {
if (debouncedSearchValue !== query.keyword) {
setCurrPage(0)
updateQuery({ keyword: debouncedSearchValue, page: 1 })
}
}, [debouncedSearchValue, query.keyword, updateQuery])
// Clear selection when search changes
useEffect(() => {
if (searchValue !== query.keyword)
setSelectedIds([])
}, [searchValue, query.keyword])
// Clear selection when status filter changes
useEffect(() => {
setSelectedIds([])
}, [normalizedStatusFilterValue])
// Page change handler
const handlePageChange = useCallback((newPage: number) => {
setCurrPage(newPage)
updateQuery({ page: newPage + 1 })
}, [updateQuery])
// Limit change handler
const handleLimitChange = useCallback((newLimit: number) => {
setLimit(newLimit)
setCurrPage(0)
updateQuery({ limit: newLimit, page: 1 })
}, [updateQuery])
// Debounced search handler
const { run: handleSearch } = useDebounceFn(() => {
setSearchValue(inputValue)
}, { wait: 500 })
// Input change handler
const handleInputChange = useCallback((value: string) => {
setInputValue(value)
handleSearch()
}, [handleSearch])
// Status filter change handler
const handleStatusFilterChange = useCallback((value: string) => {
const selectedValue = sanitizeStatusValue(value)
setStatusFilterValue(selectedValue)
setCurrPage(0)
updateQuery({ status: selectedValue, page: 1 })
}, [updateQuery])
// Status filter clear handler
const handleStatusFilterClear = useCallback(() => {
if (statusFilterValue === 'all')
return
setStatusFilterValue('all')
setCurrPage(0)
updateQuery({ status: 'all', page: 1 })
}, [statusFilterValue, updateQuery])
// Sort change handler
const handleSortChange = useCallback((value: string) => {
const next = value as SortType
if (next === sortValue)
return
setSortValue(next)
setCurrPage(0)
updateQuery({ sort: next, page: 1 })
}, [sortValue, updateQuery])
// Update polling state based on documents response
const updatePollingState = useCallback((documentsRes: DocumentListResponse | undefined) => {
if (!documentsRes?.data)
return
let completedNum = 0
documentsRes.data.forEach((documentItem) => {
const { indexing_status } = documentItem
const isEmbedded = indexing_status === 'completed' || indexing_status === 'paused' || indexing_status === 'error'
if (isEmbedded)
completedNum++
})
const hasIncompleteDocuments = completedNum !== documentsRes.data.length
const transientStatuses = ['queuing', 'indexing', 'paused']
const shouldForcePolling = normalizedStatusFilterValue === 'all'
? false
: transientStatuses.includes(normalizedStatusFilterValue)
setTimerCanRun(shouldForcePolling || hasIncompleteDocuments)
}, [normalizedStatusFilterValue])
// Adjust page when total pages change
const adjustPageForTotal = useCallback((documentsRes: DocumentListResponse | undefined) => {
if (!documentsRes)
return
const totalPages = Math.ceil(documentsRes.total / limit)
if (currPage > 0 && currPage + 1 > totalPages)
handlePageChange(totalPages > 0 ? totalPages - 1 : 0)
}, [limit, currPage, handlePageChange])
return {
// Search state
inputValue,
searchValue,
debouncedSearchValue,
handleInputChange,
// Filter & sort state
statusFilterValue,
sortValue,
normalizedStatusFilterValue,
handleStatusFilterChange,
handleStatusFilterClear,
handleSortChange,
// Pagination state
currPage,
limit,
handlePageChange,
handleLimitChange,
// Selection state
selectedIds,
setSelectedIds,
// Polling state
timerCanRun,
updatePollingState,
adjustPageForTotal,
}
}
export default useDocumentsPageState

468
web/app/components/datasets/documents/index.tsx
View File

@ -1,185 +1,55 @@
'use client'
import type { FC } from 'react'
import type { Item } from '@/app/components/base/select'
import type { SortType } from '@/service/datasets'
import { PlusIcon } from '@heroicons/react/24/solid'
import { RiDraftLine, RiExternalLinkLine } from '@remixicon/react'
import { useDebounce, useDebounceFn } from 'ahooks'
import { useRouter } from 'next/navigation'
import * as React from 'react'
import { useCallback, useEffect, useMemo, useState } from 'react'
import { useTranslation } from 'react-i18next'
import Button from '@/app/components/base/button'
import Input from '@/app/components/base/input'
import { useCallback, useEffect } from 'react'
import Loading from '@/app/components/base/loading'
import IndexFailed from '@/app/components/datasets/common/document-status-with-action/index-failed'
import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
import { useDocLink } from '@/context/i18n'
import { useProviderContext } from '@/context/provider-context'
import { DataSourceType } from '@/models/datasets'
import { useDocumentList, useInvalidDocumentDetail, useInvalidDocumentList } from '@/service/knowledge/use-document'
import { useChildSegmentListKey, useSegmentListKey } from '@/service/knowledge/use-segment'
import { useInvalid } from '@/service/use-base'
import { cn } from '@/utils/classnames'
import Chip from '../../base/chip'
import Sort from '../../base/sort'
import AutoDisabledDocument from '../common/document-status-with-action/auto-disabled-document'
import StatusWithAction from '../common/document-status-with-action/status-with-action'
import useEditDocumentMetadata from '../metadata/hooks/use-edit-dataset-metadata'
import DatasetMetadataDrawer from '../metadata/metadata-dataset/dataset-metadata-drawer'
import useDocumentListQueryState from './hooks/use-document-list-query-state'
import List from './list'
import { normalizeStatusForQuery, sanitizeStatusValue } from './status-filter'
import { useIndexStatus } from './status-item/hooks'
import s from './style.module.css'
const FolderPlusIcon = ({ className }: React.SVGProps<SVGElement>) => {
return (
<svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg" className={className ?? ''}>
<path d="M10.8332 5.83333L9.90355 3.9741C9.63601 3.439 9.50222 3.17144 9.30265 2.97597C9.12615 2.80311 8.91344 2.67164 8.6799 2.59109C8.41581 2.5 8.11668 2.5 7.51841 2.5H4.33317C3.39975 2.5 2.93304 2.5 2.57652 2.68166C2.26292 2.84144 2.00795 3.09641 1.84816 3.41002C1.6665 3.76654 1.6665 4.23325 1.6665 5.16667V5.83333M1.6665 5.83333H14.3332C15.7333 5.83333 16.4334 5.83333 16.9681 6.10582C17.4386 6.3455 17.821 6.72795 18.0607 7.19836C18.3332 7.73314 18.3332 8.4332 18.3332 9.83333V13.5C18.3332 14.9001 18.3332 15.6002 18.0607 16.135C17.821 16.6054 17.4386 16.9878 16.9681 17.2275C16.4334 17.5 15.7333 17.5 14.3332 17.5H5.6665C4.26637 17.5 3.56631 17.5 3.03153 17.2275C2.56112 16.9878 2.17867 16.6054 1.93899 16.135C1.6665 15.6002 1.6665 14.9001 1.6665 13.5V5.83333ZM9.99984 14.1667V9.16667M7.49984 11.6667H12.4998" stroke="#667085" strokeWidth="1.5" strokeLinecap="round" strokeLinejoin="round" />
</svg>
)
}
const ThreeDotsIcon = ({ className }: React.SVGProps<SVGElement>) => {
return (
<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg" className={className ?? ''}>
<path d="M5 6.5V5M8.93934 7.56066L10 6.5M10.0103 11.5H11.5103" stroke="#374151" strokeWidth="2" strokeLinecap="round" strokeLinejoin="round" />
</svg>
)
}
const NotionIcon = ({ className }: React.SVGProps<SVGElement>) => {
return (
<svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg" className={className ?? ''}>
<g clipPath="url(#clip0_2164_11263)">
<path fillRule="evenodd" clipRule="evenodd" d="M3.5725 18.2611L1.4229 15.5832C0.905706 14.9389 0.625 14.1466 0.625 13.3312V3.63437C0.625 2.4129 1.60224 1.39936 2.86295 1.31328L12.8326 0.632614C13.5569 0.583164 14.2768 0.775682 14.8717 1.17794L18.3745 3.5462C19.0015 3.97012 19.375 4.66312 19.375 5.40266V16.427C19.375 17.6223 18.4141 18.6121 17.1798 18.688L6.11458 19.3692C5.12958 19.4298 4.17749 19.0148 3.5725 18.2611Z" fill="white" />
<path d="M7.03006 8.48669V8.35974C7.03006 8.03794 7.28779 7.77104 7.61997 7.74886L10.0396 7.58733L13.3857 12.5147V8.19009L12.5244 8.07528V8.01498C12.5244 7.68939 12.788 7.42074 13.1244 7.4035L15.326 7.29073V7.60755C15.326 7.75628 15.2154 7.88349 15.0638 7.90913L14.534 7.99874V15.0023L13.8691 15.231C13.3136 15.422 12.6952 15.2175 12.3772 14.7377L9.12879 9.83574V14.5144L10.1287 14.7057L10.1147 14.7985C10.0711 15.089 9.82028 15.3087 9.51687 15.3222L7.03006 15.4329C6.99718 15.1205 7.23132 14.841 7.55431 14.807L7.88143 14.7727V8.53453L7.03006 8.48669Z" fill="black" />
<path fillRule="evenodd" clipRule="evenodd" d="M12.9218 1.85424L2.95217 2.53491C2.35499 2.57568 1.89209 3.05578 1.89209 3.63437V13.3312C1.89209 13.8748 2.07923 14.403 2.42402 14.8325L4.57362 17.5104C4.92117 17.9434 5.46812 18.1818 6.03397 18.147L17.0991 17.4658C17.6663 17.4309 18.1078 16.9762 18.1078 16.427V5.40266C18.1078 5.06287 17.9362 4.74447 17.6481 4.54969L14.1453 2.18143C13.7883 1.94008 13.3564 1.82457 12.9218 1.85424ZM3.44654 3.78562C3.30788 3.68296 3.37387 3.46909 3.54806 3.4566L12.9889 2.77944C13.2897 2.75787 13.5886 2.8407 13.8318 3.01305L15.7261 4.35508C15.798 4.40603 15.7642 4.51602 15.6752 4.52086L5.67742 5.0646C5.37485 5.08106 5.0762 4.99217 4.83563 4.81406L3.44654 3.78562ZM5.20848 6.76919C5.20848 6.4444 5.47088 6.1761 5.80642 6.15783L16.3769 5.58216C16.7039 5.56435 16.9792 5.81583 16.9792 6.13239V15.6783C16.9792 16.0025 16.7177 16.2705 16.3829 16.2896L5.8793 16.8872C5.51537 16.9079 5.20848 16.6283 5.20848 16.2759V6.76919Z" fill="black" />
</g>
<defs>
<clipPath id="clip0_2164_11263">
<rect width="20" height="20" fill="white" />
</clipPath>
</defs>
</svg>
)
}
const EmptyElement: FC<{ canAdd: boolean, onClick: () => void, type?: 'upload' | 'sync' }> = ({ canAdd = true, onClick, type = 'upload' }) => {
const { t } = useTranslation()
return (
<div className={s.emptyWrapper}>
<div className={s.emptyElement}>
<div className={s.emptySymbolIconWrapper}>
{type === 'upload' ? <FolderPlusIcon /> : <NotionIcon />}
</div>
<span className={s.emptyTitle}>
{t('list.empty.title', { ns: 'datasetDocuments' })}
<ThreeDotsIcon className="relative -left-1.5 -top-3 inline" />
</span>
<div className={s.emptyTip}>
{t(`list.empty.${type}.tip`, { ns: 'datasetDocuments' })}
</div>
{type === 'upload' && canAdd && (
<Button onClick={onClick} className={s.addFileBtn} variant="secondary-accent">
<PlusIcon className={s.plusIcon} />
{t('list.addFile', { ns: 'datasetDocuments' })}
</Button>
)}
</div>
</div>
)
}
import DocumentsHeader from './components/documents-header'
import EmptyElement from './components/empty-element'
import List from './components/list'
import useDocumentsPageState from './hooks/use-documents-page-state'
type IDocumentsProps = {
datasetId: string
}
const Documents: FC<IDocumentsProps> = ({ datasetId }) => {
const { t } = useTranslation()
const docLink = useDocLink()
const router = useRouter()
const { plan } = useProviderContext()
const isFreePlan = plan.type === 'sandbox'
const { query, updateQuery } = useDocumentListQueryState()
const [inputValue, setInputValue] = useState<string>('') // the input value
const [searchValue, setSearchValue] = useState<string>('')
const [statusFilterValue, setStatusFilterValue] = useState<string>(() => sanitizeStatusValue(query.status))
const [sortValue, setSortValue] = useState<SortType>(query.sort)
const DOC_INDEX_STATUS_MAP = useIndexStatus()
const [currPage, setCurrPage] = React.useState<number>(query.page - 1) // Convert to 0-based index
const [limit, setLimit] = useState<number>(query.limit)
const router = useRouter()
const dataset = useDatasetDetailContextWithSelector(s => s.dataset)
const [timerCanRun, setTimerCanRun] = useState(true)
const isDataSourceNotion = dataset?.data_source_type === DataSourceType.NOTION
const isDataSourceWeb = dataset?.data_source_type === DataSourceType.WEB
const isDataSourceFile = dataset?.data_source_type === DataSourceType.FILE
const embeddingAvailable = !!dataset?.embedding_available
const debouncedSearchValue = useDebounce(searchValue, { wait: 500 })
const statusFilterItems: Item[] = useMemo(() => [
{ value: 'all', name: t('list.index.all', { ns: 'datasetDocuments' }) as string },
{ value: 'queuing', name: DOC_INDEX_STATUS_MAP.queuing.text },
{ value: 'indexing', name: DOC_INDEX_STATUS_MAP.indexing.text },
{ value: 'paused', name: DOC_INDEX_STATUS_MAP.paused.text },
{ value: 'error', name: DOC_INDEX_STATUS_MAP.error.text },
{ value: 'available', name: DOC_INDEX_STATUS_MAP.available.text },
{ value: 'enabled', name: DOC_INDEX_STATUS_MAP.enabled.text },
{ value: 'disabled', name: DOC_INDEX_STATUS_MAP.disabled.text },
{ value: 'archived', name: DOC_INDEX_STATUS_MAP.archived.text },
], [DOC_INDEX_STATUS_MAP, t])
const normalizedStatusFilterValue = useMemo(() => normalizeStatusForQuery(statusFilterValue), [statusFilterValue])
const sortItems: Item[] = useMemo(() => [
{ value: 'created_at', name: t('list.sort.uploadTime', { ns: 'datasetDocuments' }) as string },
{ value: 'hit_count', name: t('list.sort.hitCount', { ns: 'datasetDocuments' }) as string },
], [t])
// Initialize search value from URL on mount
useEffect(() => {
if (query.keyword) {
setInputValue(query.keyword)
setSearchValue(query.keyword)
}
}, []) // Only run on mount
// Sync local state with URL query changes
useEffect(() => {
setCurrPage(query.page - 1)
setLimit(query.limit)
if (query.keyword !== searchValue) {
setInputValue(query.keyword)
setSearchValue(query.keyword)
}
setStatusFilterValue((prev) => {
const nextValue = sanitizeStatusValue(query.status)
return prev === nextValue ? prev : nextValue
})
setSortValue(query.sort)
}, [query])
// Update URL when pagination changes
const handlePageChange = (newPage: number) => {
setCurrPage(newPage)
updateQuery({ page: newPage + 1 }) // Pagination emits 0-based page, convert to 1-based for URL
}
// Update URL when limit changes
const handleLimitChange = (newLimit: number) => {
setLimit(newLimit)
setCurrPage(0) // Reset to first page when limit changes
updateQuery({ limit: newLimit, page: 1 })
}
// Update URL when search changes
useEffect(() => {
if (debouncedSearchValue !== query.keyword) {
setCurrPage(0) // Reset to first page when search changes
updateQuery({ keyword: debouncedSearchValue, page: 1 })
}
}, [debouncedSearchValue, query.keyword, updateQuery])
// Use custom hook for page state management
const {
inputValue,
debouncedSearchValue,
handleInputChange,
statusFilterValue,
sortValue,
normalizedStatusFilterValue,
handleStatusFilterChange,
handleStatusFilterClear,
handleSortChange,
currPage,
limit,
handlePageChange,
handleLimitChange,
selectedIds,
setSelectedIds,
timerCanRun,
updatePollingState,
adjustPageForTotal,
} = useDocumentsPageState()
// Fetch document list
const { data: documentsRes, isLoading: isListLoading } = useDocumentList({
datasetId,
query: {
@ -192,16 +62,18 @@ const Documents: FC<IDocumentsProps> = ({ datasetId }) => {
refetchInterval: timerCanRun ? 2500 : 0,
})
const invalidDocumentList = useInvalidDocumentList(datasetId)
// Update polling state when documents change
useEffect(() => {
if (documentsRes) {
const totalPages = Math.ceil(documentsRes.total / limit)
if (totalPages < currPage + 1)
setCurrPage(totalPages === 0 ? 0 : totalPages - 1)
}
}, [documentsRes])
updatePollingState(documentsRes)
}, [documentsRes, updatePollingState])
// Adjust page when total changes
useEffect(() => {
adjustPageForTotal(documentsRes)
}, [documentsRes, adjustPageForTotal])
// Invalidation hooks
const invalidDocumentList = useInvalidDocumentList(datasetId)
const invalidDocumentDetail = useInvalidDocumentDetail()
const invalidChunkList = useInvalid(useSegmentListKey)
const invalidChildChunkList = useInvalid(useChildSegmentListKey)
@ -213,73 +85,9 @@ const Documents: FC<IDocumentsProps> = ({ datasetId }) => {
invalidChunkList()
invalidChildChunkList()
}, 5000)
}, [])
useEffect(() => {
let completedNum = 0
let percent = 0
documentsRes?.data?.forEach((documentItem) => {
const { indexing_status, completed_segments, total_segments } = documentItem
const isEmbedded = indexing_status === 'completed' || indexing_status === 'paused' || indexing_status === 'error'
if (isEmbedded)
completedNum++
const completedCount = completed_segments || 0
const totalCount = total_segments || 0
if (totalCount === 0 && completedCount === 0) {
percent = isEmbedded ? 100 : 0
}
else {
const per = Math.round(completedCount * 100 / totalCount)
percent = per > 100 ? 100 : per
}
return {
...documentItem,
percent,
}
})
const hasIncompleteDocuments = completedNum !== documentsRes?.data?.length
const transientStatuses = ['queuing', 'indexing', 'paused']
const shouldForcePolling = normalizedStatusFilterValue === 'all'
? false
: transientStatuses.includes(normalizedStatusFilterValue)
setTimerCanRun(shouldForcePolling || hasIncompleteDocuments)
}, [documentsRes, normalizedStatusFilterValue])
const total = documentsRes?.total || 0
const routeToDocCreate = () => {
// if dataset is created from pipeline, go to create from pipeline page
if (dataset?.runtime_mode === 'rag_pipeline') {
router.push(`/datasets/${datasetId}/documents/create-from-pipeline`)
return
}
router.push(`/datasets/${datasetId}/documents/create`)
}
const documentsList = documentsRes?.data
const [selectedIds, setSelectedIds] = useState<string[]>([])
// Clear selection when search changes to avoid confusion
useEffect(() => {
if (searchValue !== query.keyword)
setSelectedIds([])
}, [searchValue, query.keyword])
useEffect(() => {
setSelectedIds([])
}, [normalizedStatusFilterValue])
const { run: handleSearch } = useDebounceFn(() => {
setSearchValue(inputValue)
}, { wait: 500 })
const handleInputChange = (value: string) => {
setInputValue(value)
handleSearch()
}
}, [invalidDocumentList, invalidDocumentDetail, invalidChunkList, invalidChildChunkList])
// Metadata editing hook
const {
isShowEditModal: isShowEditMetadataModal,
showEditModal: showEditMetadataModal,
@ -297,130 +105,84 @@ const Documents: FC<IDocumentsProps> = ({ datasetId }) => {
onUpdateDocList: invalidDocumentList,
})
// Route to document creation page
const routeToDocCreate = useCallback(() => {
if (dataset?.runtime_mode === 'rag_pipeline') {
router.push(`/datasets/${datasetId}/documents/create-from-pipeline`)
return
}
router.push(`/datasets/${datasetId}/documents/create`)
}, [dataset?.runtime_mode, datasetId, router])
const total = documentsRes?.total || 0
const documentsList = documentsRes?.data
// Render content based on loading and data state
const renderContent = () => {
if (isListLoading)
return <Loading type="app" />
if (total > 0) {
return (
<List
embeddingAvailable={embeddingAvailable}
documents={documentsList || []}
datasetId={datasetId}
onUpdate={handleUpdate}
selectedIds={selectedIds}
onSelectedIdChange={setSelectedIds}
statusFilterValue={normalizedStatusFilterValue}
remoteSortValue={sortValue}
pagination={{
total,
limit,
onLimitChange: handleLimitChange,
current: currPage,
onChange: handlePageChange,
}}
onManageMetadata={showEditMetadataModal}
/>
)
}
const isDataSourceNotion = dataset?.data_source_type === DataSourceType.NOTION
return (
<EmptyElement
canAdd={embeddingAvailable}
onClick={routeToDocCreate}
type={isDataSourceNotion ? 'sync' : 'upload'}
/>
)
}
return (
<div className="flex h-full flex-col">
<div className="flex flex-col justify-center gap-1 px-6 pt-4">
<h1 className="text-base font-semibold text-text-primary">{t('list.title', { ns: 'datasetDocuments' })}</h1>
<div className="flex items-center space-x-0.5 text-sm font-normal text-text-tertiary">
<span>{t('list.desc', { ns: 'datasetDocuments' })}</span>
<a
className="flex items-center text-text-accent"
target="_blank"
href={docLink('/guides/knowledge-base/integrate-knowledge-within-application')}
>
<span>{t('list.learnMore', { ns: 'datasetDocuments' })}</span>
<RiExternalLinkLine className="h-3 w-3" />
</a>
</div>
</div>
<DocumentsHeader
datasetId={datasetId}
dataSourceType={dataset?.data_source_type}
embeddingAvailable={embeddingAvailable}
isFreePlan={isFreePlan}
statusFilterValue={statusFilterValue}
sortValue={sortValue}
inputValue={inputValue}
onStatusFilterChange={handleStatusFilterChange}
onStatusFilterClear={handleStatusFilterClear}
onSortChange={handleSortChange}
onInputChange={handleInputChange}
isShowEditMetadataModal={isShowEditMetadataModal}
showEditMetadataModal={showEditMetadataModal}
hideEditMetadataModal={hideEditMetadataModal}
datasetMetaData={datasetMetaData}
builtInMetaData={builtInMetaData}
builtInEnabled={!!builtInEnabled}
onAddMetaData={handleAddMetaData}
onRenameMetaData={handleRename}
onDeleteMetaData={handleDeleteMetaData}
onBuiltInEnabledChange={setBuiltInEnabled}
onAddDocument={routeToDocCreate}
/>
<div className="flex h-0 grow flex-col px-6 pt-4">
<div className="flex flex-wrap items-center justify-between">
<div className="flex items-center gap-2">
<Chip
className="w-[160px]"
showLeftIcon={false}
value={statusFilterValue}
items={statusFilterItems}
onSelect={(item) => {
const selectedValue = sanitizeStatusValue(item?.value ? String(item.value) : '')
setStatusFilterValue(selectedValue)
setCurrPage(0)
updateQuery({ status: selectedValue, page: 1 })
}}
onClear={() => {
if (statusFilterValue === 'all')
return
setStatusFilterValue('all')
setCurrPage(0)
updateQuery({ status: 'all', page: 1 })
}}
/>
<Input
showLeftIcon
showClearIcon
wrapperClassName="!w-[200px]"
value={inputValue}
onChange={e => handleInputChange(e.target.value)}
onClear={() => handleInputChange('')}
/>
<div className="h-3.5 w-px bg-divider-regular"></div>
<Sort
order={sortValue.startsWith('-') ? '-' : ''}
value={sortValue.replace('-', '')}
items={sortItems}
onSelect={(value) => {
const next = String(value) as SortType
if (next === sortValue)
return
setSortValue(next)
setCurrPage(0)
updateQuery({ sort: next, page: 1 })
}}
/>
</div>
<div className="flex !h-8 items-center justify-center gap-2">
{!isFreePlan && <AutoDisabledDocument datasetId={datasetId} />}
<IndexFailed datasetId={datasetId} />
{!embeddingAvailable && <StatusWithAction type="warning" description={t('embeddingModelNotAvailable', { ns: 'dataset' })} />}
{embeddingAvailable && (
<Button variant="secondary" className="shrink-0" onClick={showEditMetadataModal}>
<RiDraftLine className="mr-1 size-4" />
{t('metadata.metadata', { ns: 'dataset' })}
</Button>
)}
{isShowEditMetadataModal && (
<DatasetMetadataDrawer
userMetadata={datasetMetaData || []}
onClose={hideEditMetadataModal}
onAdd={handleAddMetaData}
onRename={handleRename}
onRemove={handleDeleteMetaData}
builtInMetadata={builtInMetaData || []}
isBuiltInEnabled={!!builtInEnabled}
onIsBuiltInEnabledChange={setBuiltInEnabled}
/>
)}
{embeddingAvailable && (
<Button variant="primary" onClick={routeToDocCreate} className="shrink-0">
<PlusIcon className={cn('mr-2 h-4 w-4 stroke-current')} />
{isDataSourceNotion && t('list.addPages', { ns: 'datasetDocuments' })}
{isDataSourceWeb && t('list.addUrl', { ns: 'datasetDocuments' })}
{(!dataset?.data_source_type || isDataSourceFile) && t('list.addFile', { ns: 'datasetDocuments' })}
</Button>
)}
</div>
</div>
{isListLoading
? <Loading type="app" />
// eslint-disable-next-line sonarjs/no-nested-conditional
: total > 0
? (
<List
embeddingAvailable={embeddingAvailable}
documents={documentsList || []}
datasetId={datasetId}
onUpdate={handleUpdate}
selectedIds={selectedIds}
onSelectedIdChange={setSelectedIds}
statusFilterValue={normalizedStatusFilterValue}
remoteSortValue={sortValue}
pagination={{
total,
limit,
onLimitChange: handleLimitChange,
current: currPage,
onChange: handlePageChange,
}}
onManageMetadata={showEditMetadataModal}
/>
)
: (
<EmptyElement
canAdd={embeddingAvailable}
onClick={routeToDocCreate}
type={isDataSourceNotion ? 'sync' : 'upload'}
/>
)}
{renderContent()}
</div>
</div>
)

9
web/app/components/plugins/plugin-detail-panel/app-selector/app-picker.tsx
View File

@ -180,7 +180,14 @@ const AppPicker: FC<Props> = ({
background={app.icon_background}
imageUrl={app.icon_url}
/>
<div title={app.name} className="system-sm-medium grow text-components-input-text-filled">{app.name}</div>
<div title={`${app.name} (${app.id})`} className="system-sm-medium grow text-components-input-text-filled">
<span className="mr-1">{app.name}</span>
<span className="text-text-tertiary">
(
{app.id.slice(0, 8)}
)
</span>
</div>
<div className="system-2xs-medium-uppercase shrink-0 text-text-tertiary">{getAppType(app)}</div>
</div>
))}

34
web/app/components/plugins/plugin-detail-panel/app-selector/index.tsx
View File

@ -16,7 +16,7 @@ import {
import AppInputsPanel from '@/app/components/plugins/plugin-detail-panel/app-selector/app-inputs-panel'
import AppPicker from '@/app/components/plugins/plugin-detail-panel/app-selector/app-picker'
import AppTrigger from '@/app/components/plugins/plugin-detail-panel/app-selector/app-trigger'
import { useInfiniteAppList } from '@/service/use-apps'
import { useAppDetail, useInfiniteAppList } from '@/service/use-apps'
const PAGE_SIZE = 20
@ -70,6 +70,30 @@ const AppSelector: FC<Props> = ({
return pages.flatMap(({ data: apps }) => apps)
}, [pages])
// fetch selected app by id to avoid pagination gaps
const { data: selectedAppDetail } = useAppDetail(value?.app_id || '')
// Ensure the currently selected app is available for display and in the picker options
const currentAppInfo = useMemo(() => {
if (!value?.app_id)
return undefined
return selectedAppDetail || displayedApps.find(app => app.id === value.app_id)
}, [value?.app_id, selectedAppDetail, displayedApps])
const appsForPicker = useMemo(() => {
if (!currentAppInfo)
return displayedApps
const appIndex = displayedApps.findIndex(a => a.id === currentAppInfo.id)
if (appIndex === -1)
return [currentAppInfo, ...displayedApps]
const updatedApps = [...displayedApps]
updatedApps[appIndex] = currentAppInfo
return updatedApps
}, [currentAppInfo, displayedApps])
const hasMore = hasNextPage ?? true
const handleLoadMore = useCallback(async () => {
@ -127,12 +151,6 @@ const AppSelector: FC<Props> = ({
}
}, [value])
const currentAppInfo = useMemo(() => {
if (!displayedApps || !value)
return undefined
return displayedApps.find(app => app.id === value.app_id)
}, [displayedApps, value])
return (
<>
<PortalToFollowElem
@ -168,7 +186,7 @@ const AppSelector: FC<Props> = ({
disabled={false}
onSelect={handleSelectApp}
scope={scope || 'all'}
apps={displayedApps}
apps={appsForPicker}
isLoading={isLoading || isLoadingMore || isFetchingNextPage}
hasMore={hasMore}
onLoadMore={handleLoadMore}

13
web/contract/console.ts → web/contract/console/billing.ts
View File

@ -1,16 +1,7 @@
import type { SystemFeatures } from '@/types/feature'
import { type } from '@orpc/contract'
import { base } from './base'
import { base } from '../base'
export const systemFeaturesContract = base
.route({
path: '/system-features',
method: 'GET',
})
.input(type<unknown>())
.output(type<SystemFeatures>())
export const billingUrlContract = base
export const invoicesContract = base
.route({
path: '/billing/invoices',
method: 'GET',

11
web/contract/console/system.ts Normal file
View File

@ -0,0 +1,11 @@
import type { SystemFeatures } from '@/types/feature'
import { type } from '@orpc/contract'
import { base } from '../base'
export const systemFeaturesContract = base
.route({
path: '/system-features',
method: 'GET',
})
.input(type<unknown>())
.output(type<SystemFeatures>())

13
web/contract/router.ts
View File

@ -1,9 +1,4 @@
import type { InferContractRouterInputs } from '@orpc/contract'
import {
billingUrlContract,
bindPartnerStackContract,
systemFeaturesContract,
} from './console'
import {
createFileContract,
createFolderContract,
@ -16,6 +11,7 @@ import {
treeContract,
updateFileContentContract,
} from './console/app-asset'
import { bindPartnerStackContract, invoicesContract } from './console/billing'
import {
activateSandboxProviderContract,
deleteSandboxProviderConfigContract,
@ -24,6 +20,7 @@ import {
getSandboxProviderListContract,
saveSandboxProviderConfigContract,
} from './console/sandbox-provider'
import { systemFeaturesContract } from './console/system'
import { collectionPluginsContract, collectionsContract, searchAdvancedContract } from './marketplace'
export const marketplaceRouterContract = {
@ -36,8 +33,10 @@ export type MarketPlaceInputs = InferContractRouterInputs<typeof marketplaceRout
export const consoleRouterContract = {
systemFeatures: systemFeaturesContract,
billingUrl: billingUrlContract,
bindPartnerStack: bindPartnerStackContract,
billing: {
invoices: invoicesContract,
bindPartnerStack: bindPartnerStackContract,
},
getSandboxProviderList: getSandboxProviderListContract,
getSandboxProvider: getSandboxProviderContract,
saveSandboxProviderConfig: saveSandboxProviderConfigContract,

3
web/package.json
View File

@ -10,9 +10,6 @@
"default": "./i18n-config/lib.client.ts"
}
},
"engines": {
"node": ">=22.12.0"
},
"browserslist": [
"last 1 Chrome version",
"last 1 Firefox version",

8
web/service/use-billing.ts
View File

@ -3,8 +3,8 @@ import { consoleClient, consoleQuery } from '@/service/client'
export const useBindPartnerStackInfo = () => {
return useMutation({
mutationKey: consoleQuery.bindPartnerStack.mutationKey(),
mutationFn: (data: { partnerKey: string, clickId: string }) => consoleClient.bindPartnerStack({
mutationKey: consoleQuery.billing.bindPartnerStack.mutationKey(),
mutationFn: (data: { partnerKey: string, clickId: string }) => consoleClient.billing.bindPartnerStack({
params: { partnerKey: data.partnerKey },
body: { click_id: data.clickId },
}),
@ -13,10 +13,10 @@ export const useBindPartnerStackInfo = () => {
export const useBillingUrl = (enabled: boolean) => {
return useQuery({
queryKey: consoleQuery.billingUrl.queryKey(),
queryKey: consoleQuery.billing.invoices.queryKey(),
enabled,
queryFn: async () => {
const res = await consoleClient.billingUrl()
const res = await consoleClient.billing.invoices()
return res.url
},
})