import type { CreateExternalAPIReq } from '@/app/components/datasets/external-api/declarations' import type { CreateKnowledgeBaseReq } from '@/app/components/datasets/external-knowledge-base/create/declarations' import type { BuiltInMetadataItem, MetadataBatchEditToServer, MetadataItemWithValueLength } from '@/app/components/datasets/metadata/types' import type { ApiKeysListResponse, CreateApiKeyResponse, } from '@/models/app' import type { CommonResponse, DataSourceNotionWorkspace } from '@/models/common' import type { BatchImportResponse, ChildChunkDetail, ChildSegmentsResponse, ChunkingMode, CreateDocumentReq, createDocumentResponse, DataSet, DataSetListResponse, DocumentActionType, DocumentDetailResponse, DocumentListResponse, ErrorDocsResponse, ExternalAPIDeleteResponse, ExternalAPIItem, ExternalAPIListResponse, ExternalAPIUsage, ExternalKnowledgeBaseHitTestingResponse, ExternalKnowledgeItem, FetchDatasetsParams, FileIndexingEstimateResponse, HitTestingRecordsResponse, HitTestingRequest, HitTestingResponse, IndexingEstimateParams, IndexingEstimateResponse, IndexingStatusBatchResponse, IndexingStatusResponse, ProcessRuleResponse, RelatedAppResponse, SegmentDetailModel, SegmentsResponse, SegmentUpdater, } from '@/models/datasets' import qs from 'qs' import { DataSourceProvider } from '@/models/common' import { del, get, patch, post, put } from './base' // apis for documents in a dataset type CommonDocReq = { datasetId: string documentId: string } type BatchReq = { datasetId: string batchId: string } export type SortType = 'created_at' | 'hit_count' | '-created_at' | '-hit_count' export type MetadataType = 'all' | 'only' | 'without' export const fetchDatasetDetail = (datasetId: string): Promise => { return get(`/datasets/${datasetId}`) } export const updateDatasetSetting = ({ datasetId, body, }: { datasetId: string body: Partial> }): Promise => { return patch(`/datasets/${datasetId}`, { body }) } export const fetchDatasetRelatedApps = (datasetId: string): Promise => { return get(`/datasets/${datasetId}/related-apps`) } export const fetchDatasets = ({ url, params }: FetchDatasetsParams): Promise => { const urlParams = qs.stringify(params, { indices: false }) return get(`${url}?${urlParams}`) } export const createEmptyDataset = ({ name }: { name: string }): Promise => { return post('/datasets', { body: { name } }) } export const checkIsUsedInApp = (id: string): Promise<{ is_using: boolean }> => { return get<{ is_using: boolean }>(`/datasets/${id}/use-check`, {}, { silent: true, }) } export const deleteDataset = (datasetID: string): Promise => { return del(`/datasets/${datasetID}`) } export const fetchExternalAPIList = ({ url }: { url: string }): Promise => { return get(url) } export const fetchExternalAPI = ({ apiTemplateId }: { apiTemplateId: string }): Promise => { return get(`/datasets/external-knowledge-api/${apiTemplateId}`) } export const updateExternalAPI = ({ apiTemplateId, body }: { apiTemplateId: string, body: ExternalAPIItem }): Promise => { return patch(`/datasets/external-knowledge-api/${apiTemplateId}`, { body }) } export const deleteExternalAPI = ({ apiTemplateId }: { apiTemplateId: string }): Promise => { return del(`/datasets/external-knowledge-api/${apiTemplateId}`) } export const checkUsageExternalAPI = ({ apiTemplateId }: { apiTemplateId: string }): Promise => { return get(`/datasets/external-knowledge-api/${apiTemplateId}/use-check`) } export const createExternalAPI = ({ body }: { body: CreateExternalAPIReq }): Promise => { return post('/datasets/external-knowledge-api', { body }) } export const createExternalKnowledgeBase = ({ body }: { body: CreateKnowledgeBaseReq }): Promise => { return post('/datasets/external', { body }) } export const fetchDefaultProcessRule = ({ url }: { url: string }): Promise => { return get(url) } export const fetchProcessRule = ({ params: { documentId } }: { params: { documentId: string } }): Promise => { return get('/datasets/process-rule', { params: { document_id: documentId } }) } export const createFirstDocument = ({ body }: { body: CreateDocumentReq }): Promise => { return post('/datasets/init', { body }) } export const createDocument = ({ datasetId, body }: { datasetId: string, body: CreateDocumentReq }): Promise => { return post(`/datasets/${datasetId}/documents`, { body }) } export const fetchIndexingEstimate = ({ datasetId, documentId }: CommonDocReq): Promise => { return get(`/datasets/${datasetId}/documents/${documentId}/indexing-estimate`, {}) } export const fetchIndexingEstimateBatch = ({ datasetId, batchId }: BatchReq): Promise => { return get(`/datasets/${datasetId}/batch/${batchId}/indexing-estimate`, {}) } export const fetchIndexingStatus = ({ datasetId, documentId }: CommonDocReq): Promise => { return get(`/datasets/${datasetId}/documents/${documentId}/indexing-status`, {}) } export const fetchIndexingStatusBatch = ({ datasetId, batchId }: BatchReq): Promise => { return get(`/datasets/${datasetId}/batch/${batchId}/indexing-status`, {}) } export const renameDocumentName = ({ datasetId, documentId, name }: CommonDocReq & { name: string }): Promise => { return post(`/datasets/${datasetId}/documents/${documentId}/rename`, { body: { name }, }) } export const pauseDocIndexing = ({ datasetId, documentId }: CommonDocReq): Promise => { return patch(`/datasets/${datasetId}/documents/${documentId}/processing/pause`) } export const resumeDocIndexing = ({ datasetId, documentId }: CommonDocReq): Promise => { return patch(`/datasets/${datasetId}/documents/${documentId}/processing/resume`) } export const preImportNotionPages = ({ url, datasetId }: { url: string, datasetId?: string }): Promise<{ notion_info: DataSourceNotionWorkspace[] }> => { return get<{ notion_info: DataSourceNotionWorkspace[] }>(url, { params: { dataset_id: datasetId } }) } export const fetchPreImportNotionPages = ({ datasetId, credentialId }: { datasetId: string, credentialId?: string }): Promise<{ notion_info: DataSourceNotionWorkspace[] }> => { return get<{ notion_info: DataSourceNotionWorkspace[] }>('/notion/pre-import/pages', { params: { dataset_id: datasetId, credential_id: credentialId, }, }) } export const modifyDocMetadata = ({ datasetId, documentId, body }: CommonDocReq & { body: { doc_type: string, doc_metadata: Record } }): Promise => { return put(`/datasets/${datasetId}/documents/${documentId}/metadata`, { body }) } // hit testing export const hitTesting = ({ datasetId, query, retrieval_model, attachment_ids }: { datasetId: string } & HitTestingRequest): Promise => { return post(`/datasets/${datasetId}/hit-testing`, { body: { query, retrieval_model, attachment_ids } }) } export const externalKnowledgeBaseHitTesting = ({ datasetId, query, external_retrieval_model }: { datasetId: string, query: string, external_retrieval_model: { top_k: number, score_threshold: number, score_threshold_enabled: boolean } }): Promise => { return post(`/datasets/${datasetId}/external-hit-testing`, { body: { query, external_retrieval_model } }) } export const fetchTestingRecords = ({ datasetId, params }: { datasetId: string, params: { page: number, limit: number } }): Promise => { return get(`/datasets/${datasetId}/queries`, { params }) } export const fetchFileIndexingEstimate = (body: IndexingEstimateParams): Promise => { return post('/datasets/indexing-estimate', { body }) } export const fetchNotionPagePreview = ({ pageID, pageType, credentialID }: { pageID: string, pageType: string, credentialID: string }): Promise<{ content: string }> => { return get<{ content: string }>(`notion/pages/${pageID}/${pageType}/preview`, { params: { credential_id: credentialID, }, }) } export const fetchApiKeysList = ({ url, params }: { url: string, params: Record }): Promise => { return get(url, params) } export const delApikey = ({ url, params }: { url: string, params: Record }): Promise => { return del(url, params) } export const createApikey = ({ url, body }: { url: string, body: Record }): Promise => { return post(url, body) } export const fetchDataSources = (): Promise => { return get('api-key-auth/data-source') } export const createDataSourceApiKeyBinding = (body: Record): Promise => { return post('api-key-auth/data-source/binding', { body }) } export const removeDataSourceApiKeyBinding = (id: string): Promise => { return del(`api-key-auth/data-source/${id}`) } export const createFirecrawlTask = (body: Record): Promise => { return post('website/crawl', { body: { ...body, provider: DataSourceProvider.fireCrawl, }, }) } export const checkFirecrawlTaskStatus = (jobId: string): Promise => { return get(`website/crawl/status/${jobId}`, { params: { provider: DataSourceProvider.fireCrawl, }, }, { silent: true, }) } export const createJinaReaderTask = (body: Record): Promise => { return post('website/crawl', { body: { ...body, provider: DataSourceProvider.jinaReader, }, }) } export const checkJinaReaderTaskStatus = (jobId: string): Promise => { return get(`website/crawl/status/${jobId}`, { params: { provider: 'jinareader', }, }, { silent: true, }) } export const createWatercrawlTask = (body: Record): Promise => { return post('website/crawl', { body: { ...body, provider: DataSourceProvider.waterCrawl, }, }) } export const checkWatercrawlTaskStatus = (jobId: string): Promise => { return get(`website/crawl/status/${jobId}`, { params: { provider: DataSourceProvider.waterCrawl, }, }, { silent: true, }) } export type FileTypesRes = { allowed_extensions: string[] } export const fetchSupportFileTypes = ({ url }: { url: string }): Promise => { return get(url) } export const getErrorDocs = ({ datasetId }: { datasetId: string }): Promise => { return get(`/datasets/${datasetId}/error-docs`) } export const retryErrorDocs = ({ datasetId, document_ids }: { datasetId: string, document_ids: string[] }): Promise => { return post(`/datasets/${datasetId}/retry`, { body: { document_ids } }) } export const fetchDatasetApiBaseInfo = (): Promise<{ api_base_url: string }> => { return get<{ api_base_url: string }>('/datasets/api-base-info') } export const enableDatasetServiceApi = (datasetId: string): Promise => { return post(`/datasets/${datasetId}/api-keys/enable`) } export const disableDatasetServiceApi = (datasetId: string): Promise => { return post(`/datasets/${datasetId}/api-keys/disable`) } export const fetchDatasetApiKeys = (): Promise => { return get('/datasets/api-keys') } export const fetchExternalKnowledgeApiList = (): Promise => { return get('/datasets/external-knowledge-api') } export const fetchDatasetTestingRecords = (datasetId: string, params?: { page: number, limit: number }): Promise => { return get(`/datasets/${datasetId}/queries`, { params }) } export const fetchDatasetErrorDocs = (datasetId: string): Promise => { return get(`/datasets/${datasetId}/error-docs`) } export const fetchDocumentList = (datasetId: string, params: { keyword: string, page: number, limit: number, sort?: string, status?: string }): Promise => { return get(`/datasets/${datasetId}/documents`, { params }) } export const fetchAutoDisabledDocuments = (datasetId: string): Promise<{ document_ids: string[] }> => { return get<{ document_ids: string[] }>(`/datasets/${datasetId}/auto-disable-logs`) } const buildDocumentIdsQuery = (documentIds: string[] | string) => { const ids = Array.isArray(documentIds) ? documentIds : [documentIds] return ids.map(id => `document_id=${id}`).join('&') } export const updateDocumentStatusBatch = (datasetId: string, action: DocumentActionType, documentIds: string[] | string): Promise => { return patch(`/datasets/${datasetId}/documents/status/${action}/batch?${buildDocumentIdsQuery(documentIds)}`) } export const deleteDocumentBatch = (datasetId: string, documentIds: string[] | string): Promise => { return del(`/datasets/${datasetId}/documents?${buildDocumentIdsQuery(documentIds)}`) } export const syncNotionDocument = (datasetId: string, documentId: string): Promise => { return get(`/datasets/${datasetId}/documents/${documentId}/notion/sync`) } export const syncWebsiteDocument = (datasetId: string, documentId: string): Promise => { return get(`/datasets/${datasetId}/documents/${documentId}/website-sync`) } export const fetchDocumentDetail = (datasetId: string, documentId: string, params?: { metadata?: string }): Promise => { return get(`/datasets/${datasetId}/documents/${documentId}`, { params }) } export const retryDocumentBatch = (datasetId: string, documentIds: string[]): Promise => { return post(`/datasets/${datasetId}/retry`, { body: { document_ids: documentIds, }, }) } export const fetchDatasetMetadata = (datasetId: string): Promise<{ doc_metadata: MetadataItemWithValueLength[], built_in_field_enabled: boolean }> => { return get<{ doc_metadata: MetadataItemWithValueLength[], built_in_field_enabled: boolean }>(`/datasets/${datasetId}/metadata`) } export const createDatasetMetadata = (datasetId: string, payload: BuiltInMetadataItem): Promise => { return post(`/datasets/${datasetId}/metadata`, { body: payload, }) as Promise } export const updateDatasetMetadataName = (datasetId: string, metaDataId: string, name: string): Promise => { return patch(`/datasets/${datasetId}/metadata/${metaDataId}`, { body: { name, }, }) as Promise } export const deleteDatasetMetadata = (datasetId: string, metaDataId: string): Promise => { return del(`/datasets/${datasetId}/metadata/${metaDataId}`) as Promise } export const fetchBuiltInMetadataFields = (): Promise<{ fields: BuiltInMetadataItem[] }> => { return get<{ fields: BuiltInMetadataItem[] }>('/datasets/metadata/built-in') } export const fetchDocumentMetadata = (datasetId: string, documentId: string): Promise => { return get(`/datasets/${datasetId}/documents/${documentId}`, { params: { metadata: 'only' } }) } export const batchUpdateDocumentMetadata = (datasetId: string, metadataList: MetadataBatchEditToServer): Promise => { return post(`/datasets/${datasetId}/documents/metadata`, { body: { operation_data: metadataList, }, }) as Promise } export const updateBuiltInMetadataStatus = (datasetId: string, enabled: boolean): Promise => { return post(`/datasets/${datasetId}/metadata/built-in/${enabled ? 'enable' : 'disable'}`) as Promise } export const fetchSegmentList = (datasetId: string, documentId: string, params: { page: number, limit: number, keyword: string, enabled: boolean | 'all' | '' }): Promise => { return get(`/datasets/${datasetId}/documents/${documentId}/segments`, { params }) } export const updateSegment = (datasetId: string, documentId: string, segmentId: string, body: SegmentUpdater): Promise<{ data: SegmentDetailModel, doc_form: ChunkingMode }> => { return patch<{ data: SegmentDetailModel, doc_form: ChunkingMode }>(`/datasets/${datasetId}/documents/${documentId}/segments/${segmentId}`, { body }) } export const addSegment = (datasetId: string, documentId: string, body: SegmentUpdater): Promise<{ data: SegmentDetailModel, doc_form: ChunkingMode }> => { return post<{ data: SegmentDetailModel, doc_form: ChunkingMode }>(`/datasets/${datasetId}/documents/${documentId}/segment`, { body }) } export const enableSegments = (datasetId: string, documentId: string, segmentIds: string[]): Promise => { const query = segmentIds.map(id => `segment_id=${id}`).join('&') return patch(`/datasets/${datasetId}/documents/${documentId}/segment/enable?${query}`) } export const disableSegments = (datasetId: string, documentId: string, segmentIds: string[]): Promise => { const query = segmentIds.map(id => `segment_id=${id}`).join('&') return patch(`/datasets/${datasetId}/documents/${documentId}/segment/disable?${query}`) } export const deleteSegments = (datasetId: string, documentId: string, segmentIds: string[]): Promise => { const query = segmentIds.map(id => `segment_id=${id}`).join('&') return del(`/datasets/${datasetId}/documents/${documentId}/segments?${query}`) } export const fetchChildSegments = (datasetId: string, documentId: string, segmentId: string, params: { page: number, limit: number, keyword: string }): Promise => { return get(`/datasets/${datasetId}/documents/${documentId}/segments/${segmentId}/child_chunks`, { params }) } export const deleteChildSegment = (datasetId: string, documentId: string, segmentId: string, childChunkId: string): Promise => { return del(`/datasets/${datasetId}/documents/${documentId}/segments/${segmentId}/child_chunks/${childChunkId}`) } export const addChildSegment = (datasetId: string, documentId: string, segmentId: string, body: { content: string }): Promise<{ data: ChildChunkDetail }> => { return post<{ data: ChildChunkDetail }>(`/datasets/${datasetId}/documents/${documentId}/segments/${segmentId}/child_chunks`, { body }) } export const updateChildSegment = (datasetId: string, documentId: string, segmentId: string, childChunkId: string, body: { content: string }): Promise<{ data: ChildChunkDetail }> => { return patch<{ data: ChildChunkDetail }>(`/datasets/${datasetId}/documents/${documentId}/segments/${segmentId}/child_chunks/${childChunkId}`, { body }) } export const batchImportSegments = (url: string, body: { upload_file_id: string }): Promise => { return post(url, { body }) } export const checkSegmentBatchImportStatus = (jobID: string): Promise => { return get(`/datasets/batch_import_status/${jobID}`) }