From d3058d63bddba1fdab9220be0e9743aedd26d3b5 Mon Sep 17 00:00:00 2001 From: chariri Date: Thu, 4 Jun 2026 04:38:39 +0900 Subject: [PATCH] refactor(api): migrate console.datasets.data_source to BaseModel (#36624) --- .../console/datasets/data_source.py | 163 ++++++++++------- api/fields/data_source_fields.py | 55 ------ api/openapi/markdown/console-swagger.md | 155 ++++++++-------- .../console/datasets/test_data_source.py | 67 ++++++- .../console/datasets/test_data_source.py | 173 ++++++++++++++++++ .../api/console/data-source/orpc.gen.ts | 16 -- .../api/console/data-source/types.gen.ts | 44 ++++- .../api/console/data-source/zod.gen.ts | 55 +++++- .../api/console/datasets/orpc.gen.ts | 128 ++++++------- .../api/console/datasets/types.gen.ts | 39 ++-- .../generated/api/console/datasets/zod.gen.ts | 40 ++-- .../generated/api/console/notion/orpc.gen.ts | 40 +--- .../generated/api/console/notion/types.gen.ts | 65 ++++--- .../generated/api/console/notion/zod.gen.ts | 61 ++++-- 14 files changed, 682 insertions(+), 419 deletions(-) delete mode 100644 api/fields/data_source_fields.py create mode 100644 api/tests/unit_tests/controllers/console/datasets/test_data_source.py diff --git a/api/controllers/console/datasets/data_source.py b/api/controllers/console/datasets/data_source.py index 71aac4b160..b736c3129d 100644 --- a/api/controllers/console/datasets/data_source.py +++ b/api/controllers/console/datasets/data_source.py @@ -1,33 +1,29 @@ import json from collections.abc import Generator +from datetime import datetime from typing import Any, Literal, cast from uuid import UUID from flask import request -from flask_restx import Resource, fields, marshal_with -from pydantic import BaseModel, Field +from flask_restx import Resource +from pydantic import BaseModel, Field, field_serializer from sqlalchemy import select from sqlalchemy.orm import sessionmaker from werkzeug.exceptions import NotFound from controllers.common.fields import SimpleResultResponse, TextContentResponse -from controllers.common.schema import get_or_create_model, register_response_schema_models, register_schema_model +from controllers.common.schema import query_params_from_model, register_response_schema_models, register_schema_models from core.datasource.entities.datasource_entities import DatasourceProviderType, OnlineDocumentPagesMessage from core.datasource.online_document.online_document_plugin import OnlineDocumentDatasourcePlugin +from core.entities.knowledge_entities import IndexingEstimate from core.indexing_runner import IndexingRunner from core.rag.extractor.entity.datasource_type import DatasourceType from core.rag.extractor.entity.extract_setting import ExtractSetting, NotionInfo from core.rag.extractor.notion_extractor import NotionExtractor from extensions.ext_database import db -from fields.data_source_fields import ( - integrate_fields, - integrate_icon_fields, - integrate_list_fields, - integrate_notion_info_list_fields, - integrate_page_fields, - integrate_workspace_fields, -) +from fields.base import ResponseModel from libs.datetime_utils import naive_utc_now +from libs.helper import dump_response, to_timestamp from libs.login import current_account_with_tenant, login_required from models import DataSourceOauthBinding, Document from services.dataset_service import DatasetService, DocumentService @@ -54,50 +50,74 @@ class DataSourceNotionPreviewQuery(BaseModel): credential_id: str = Field(..., description="Credential ID", min_length=1) -register_schema_model(console_ns, NotionEstimatePayload) -register_response_schema_models(console_ns, SimpleResultResponse, TextContentResponse) +class DataSourceIntegrateIconResponse(ResponseModel): + type: str | None = None + url: str | None = None + emoji: str | None = None -integrate_icon_model = get_or_create_model("DataSourceIntegrateIcon", integrate_icon_fields) +class DataSourceIntegratePageResponse(ResponseModel): + page_name: str + page_id: str + page_icon: DataSourceIntegrateIconResponse | None + parent_id: str + type: str -integrate_page_fields_copy = integrate_page_fields.copy() -integrate_page_fields_copy["page_icon"] = fields.Nested(integrate_icon_model, allow_null=True) -integrate_page_model = get_or_create_model("DataSourceIntegratePage", integrate_page_fields_copy) -integrate_workspace_fields_copy = integrate_workspace_fields.copy() -integrate_workspace_fields_copy["pages"] = fields.List(fields.Nested(integrate_page_model)) -integrate_workspace_model = get_or_create_model("DataSourceIntegrateWorkspace", integrate_workspace_fields_copy) +class DataSourceIntegrateWorkspaceResponse(ResponseModel): + workspace_name: str | None + workspace_id: str | None + workspace_icon: str | None + pages: list[DataSourceIntegratePageResponse] + total: int -integrate_fields_copy = integrate_fields.copy() -integrate_fields_copy["source_info"] = fields.Nested(integrate_workspace_model) -integrate_model = get_or_create_model("DataSourceIntegrate", integrate_fields_copy) -integrate_list_fields_copy = integrate_list_fields.copy() -integrate_list_fields_copy["data"] = fields.List(fields.Nested(integrate_model)) -integrate_list_model = get_or_create_model("DataSourceIntegrateList", integrate_list_fields_copy) +class DataSourceIntegrateResponse(ResponseModel): + id: str | None + provider: str + created_at: datetime | int | None + is_bound: bool + disabled: bool | None + link: str + source_info: DataSourceIntegrateWorkspaceResponse | None -notion_page_fields = { - "page_name": fields.String, - "page_id": fields.String, - "page_icon": fields.Nested(integrate_icon_model, allow_null=True), - "is_bound": fields.Boolean, - "parent_id": fields.String, - "type": fields.String, -} -notion_page_model = get_or_create_model("NotionIntegratePage", notion_page_fields) + @field_serializer("created_at") + def serialize_created_at(self, value: datetime | int | None) -> int | None: + return to_timestamp(value) -notion_workspace_fields = { - "workspace_name": fields.String, - "workspace_id": fields.String, - "workspace_icon": fields.String, - "pages": fields.List(fields.Nested(notion_page_model)), -} -notion_workspace_model = get_or_create_model("NotionIntegrateWorkspace", notion_workspace_fields) -integrate_notion_info_list_fields_copy = integrate_notion_info_list_fields.copy() -integrate_notion_info_list_fields_copy["notion_info"] = fields.List(fields.Nested(notion_workspace_model)) -integrate_notion_info_list_model = get_or_create_model( - "NotionIntegrateInfoList", integrate_notion_info_list_fields_copy +class DataSourceIntegrateListResponse(ResponseModel): + data: list[DataSourceIntegrateResponse] + + +class NotionIntegratePageResponse(ResponseModel): + page_name: str + page_id: str + page_icon: DataSourceIntegrateIconResponse | None + parent_id: str | None + type: str + is_bound: bool + + +class NotionIntegrateWorkspaceResponse(ResponseModel): + workspace_name: str | None + workspace_id: str | None + workspace_icon: str | None + pages: list[NotionIntegratePageResponse] + + +class NotionIntegrateInfoListResponse(ResponseModel): + notion_info: list[NotionIntegrateWorkspaceResponse] + + +register_schema_models(console_ns, NotionEstimatePayload) +register_response_schema_models( + console_ns, + DataSourceIntegrateListResponse, + IndexingEstimate, + NotionIntegrateInfoListResponse, + SimpleResultResponse, + TextContentResponse, ) @@ -109,8 +129,8 @@ class DataSourceApi(Resource): @setup_required @login_required @account_initialization_required - @marshal_with(integrate_list_model) - def get(self): + @console_ns.response(200, "Success", console_ns.models[DataSourceIntegrateListResponse.__name__]) + def get(self) -> tuple[dict[str, Any], int]: _, current_tenant_id = current_account_with_tenant() # get workspace data source integrates @@ -154,19 +174,19 @@ class DataSourceApi(Resource): "link": f"{base_url}{data_source_oauth_base_path}/{provider}", } ) - return {"data": integrate_data}, 200 + return dump_response(DataSourceIntegrateListResponse, {"data": integrate_data}), 200 @setup_required @login_required @account_initialization_required @console_ns.response(200, "Success", console_ns.models[SimpleResultResponse.__name__]) - def patch(self, binding_id, action: Literal["enable", "disable"]): + def patch(self, binding_id: UUID, action: Literal["enable", "disable"]) -> tuple[dict[str, str], int]: _, current_tenant_id = current_account_with_tenant() - binding_id = str(binding_id) + binding_id_str = str(binding_id) with sessionmaker(db.engine, expire_on_commit=False).begin() as session: data_source_binding = session.execute( select(DataSourceOauthBinding).where( - DataSourceOauthBinding.id == binding_id, DataSourceOauthBinding.tenant_id == current_tenant_id + DataSourceOauthBinding.id == binding_id_str, DataSourceOauthBinding.tenant_id == current_tenant_id ) ).scalar_one_or_none() if data_source_binding is None: @@ -198,12 +218,12 @@ class DataSourceNotionListApi(Resource): @setup_required @login_required @account_initialization_required - @marshal_with(integrate_notion_info_list_model) - def get(self): + @console_ns.doc(params=query_params_from_model(DataSourceNotionListQuery)) + @console_ns.response(200, "Success", console_ns.models[NotionIntegrateInfoListResponse.__name__]) + def get(self) -> tuple[dict[str, Any], int]: current_user, current_tenant_id = current_account_with_tenant() - query = DataSourceNotionListQuery.model_validate(request.args.to_dict()) - + query = DataSourceNotionListQuery.model_validate(request.args.to_dict(flat=True)) datasource_provider_service = DatasourceProviderService() credential = datasource_provider_service.get_datasource_credentials( tenant_id=current_tenant_id, @@ -278,22 +298,23 @@ class DataSourceNotionListApi(Resource): pages.append(page_info) except Exception as e: raise e - return {"notion_info": {**workspace_info, "pages": pages}}, 200 + notion_info = [{**workspace_info, "pages": pages}] if workspace_info else [] + return dump_response(NotionIntegrateInfoListResponse, {"notion_info": notion_info}), 200 -@console_ns.route( - "/notion/pages///preview", - "/datasets/notion-indexing-estimate", -) -class DataSourceNotionApi(Resource): +@console_ns.route("/notion/pages///preview") +class DataSourceNotionPreviewApi(Resource): + """Preview one authorized Notion page through the datasource credential.""" + @setup_required @login_required @account_initialization_required + @console_ns.doc(params=query_params_from_model(DataSourceNotionPreviewQuery)) @console_ns.response(200, "Success", console_ns.models[TextContentResponse.__name__]) - def get(self, page_id: UUID, page_type: str): + def get(self, page_id: UUID, page_type: str) -> tuple[dict[str, str], int]: _, current_tenant_id = current_account_with_tenant() - query = DataSourceNotionPreviewQuery.model_validate(request.args.to_dict()) + query = DataSourceNotionPreviewQuery.model_validate(request.args.to_dict(flat=True)) datasource_provider_service = DatasourceProviderService() credential = datasource_provider_service.get_datasource_credentials( @@ -316,11 +337,17 @@ class DataSourceNotionApi(Resource): text_docs = extractor.extract() return {"content": "\n".join([doc.page_content for doc in text_docs])}, 200 + +@console_ns.route("/datasets/notion-indexing-estimate") +class DataSourceNotionIndexingEstimateApi(Resource): + """Estimate indexing work for selected Notion pages.""" + @setup_required @login_required @account_initialization_required @console_ns.expect(console_ns.models[NotionEstimatePayload.__name__]) - def post(self): + @console_ns.response(200, "Success", console_ns.models[IndexingEstimate.__name__]) + def post(self) -> tuple[dict[str, Any], int]: _, current_tenant_id = current_account_with_tenant() payload = NotionEstimatePayload.model_validate(console_ns.payload or {}) @@ -355,7 +382,7 @@ class DataSourceNotionApi(Resource): args["doc_form"], args["doc_language"], ) - return response.model_dump(), 200 + return dump_response(IndexingEstimate, response), 200 @console_ns.route("/datasets//notion/sync") @@ -364,7 +391,7 @@ class DataSourceNotionDatasetSyncApi(Resource): @login_required @account_initialization_required @console_ns.response(200, "Success", console_ns.models[SimpleResultResponse.__name__]) - def get(self, dataset_id: UUID): + def get(self, dataset_id: UUID) -> tuple[dict[str, str], int]: dataset_id_str = str(dataset_id) dataset = DatasetService.get_dataset(dataset_id_str) if dataset is None: @@ -382,7 +409,7 @@ class DataSourceNotionDocumentSyncApi(Resource): @login_required @account_initialization_required @console_ns.response(200, "Success", console_ns.models[SimpleResultResponse.__name__]) - def get(self, dataset_id: UUID, document_id: UUID): + def get(self, dataset_id: UUID, document_id: UUID) -> tuple[dict[str, str], int]: dataset_id_str = str(dataset_id) document_id_str = str(document_id) dataset = DatasetService.get_dataset(dataset_id_str) diff --git a/api/fields/data_source_fields.py b/api/fields/data_source_fields.py deleted file mode 100644 index 27ab505376..0000000000 --- a/api/fields/data_source_fields.py +++ /dev/null @@ -1,55 +0,0 @@ -from flask_restx import fields - -from libs.helper import TimestampField - -integrate_icon_fields = {"type": fields.String, "url": fields.String, "emoji": fields.String} - -integrate_page_fields = { - "page_name": fields.String, - "page_id": fields.String, - "page_icon": fields.Nested(integrate_icon_fields, allow_null=True), - "is_bound": fields.Boolean, - "parent_id": fields.String, - "type": fields.String, -} - -integrate_workspace_fields = { - "workspace_name": fields.String, - "workspace_id": fields.String, - "workspace_icon": fields.String, - "pages": fields.List(fields.Nested(integrate_page_fields)), -} - -integrate_notion_info_list_fields = { - "notion_info": fields.List(fields.Nested(integrate_workspace_fields)), -} - -integrate_page_fields = { - "page_name": fields.String, - "page_id": fields.String, - "page_icon": fields.Nested(integrate_icon_fields, allow_null=True), - "parent_id": fields.String, - "type": fields.String, -} - -integrate_workspace_fields = { - "workspace_name": fields.String, - "workspace_id": fields.String, - "workspace_icon": fields.String, - "pages": fields.List(fields.Nested(integrate_page_fields)), - "total": fields.Integer, -} - -integrate_fields = { - "id": fields.String, - "provider": fields.String, - "created_at": TimestampField, - "is_bound": fields.Boolean, - "disabled": fields.Boolean, - "link": fields.String, - "source_info": fields.Nested(integrate_workspace_fields), -} - -integrate_list_fields = { - "data": fields.List(fields.Nested(integrate_fields)), -} diff --git a/api/openapi/markdown/console-swagger.md b/api/openapi/markdown/console-swagger.md index f4e121a26e..04e00459af 100644 --- a/api/openapi/markdown/console-swagger.md +++ b/api/openapi/markdown/console-swagger.md @@ -4326,9 +4326,9 @@ Get compliance document download link #### GET ##### Responses -| Code | Description | -| ---- | ----------- | -| 200 | Success | +| Code | Description | Schema | +| ---- | ----------- | ------ | +| 200 | Success | [DataSourceIntegrateListResponse](#datasourceintegratelistresponse) | #### PATCH ##### Responses @@ -4349,9 +4349,9 @@ Get compliance document download link ##### Responses -| Code | Description | -| ---- | ----------- | -| 200 | Success | +| Code | Description | Schema | +| ---- | ----------- | ------ | +| 200 | Success | [DataSourceIntegrateListResponse](#datasourceintegratelistresponse) | #### PATCH ##### Parameters @@ -4662,13 +4662,6 @@ Initialize dataset with documents ### /datasets/notion-indexing-estimate -#### GET -##### Responses - -| Code | Description | Schema | -| ---- | ----------- | ------ | -| 200 | Success | [TextContentResponse](#textcontentresponse) | - #### POST ##### Parameters @@ -4678,9 +4671,9 @@ Initialize dataset with documents ##### Responses -| Code | Description | -| ---- | ----------- | -| 200 | Success | +| Code | Description | Schema | +| ---- | ----------- | ------ | +| 200 | Success | [IndexingEstimate](#indexingestimate) | ### /datasets/process-rule @@ -6652,6 +6645,7 @@ Mark a notification as dismissed for the current user. | ---- | ---------- | ----------- | -------- | ------ | | page_id | path | | Yes | string | | page_type | path | | Yes | string | +| credential_id | query | Credential ID | Yes | string | ##### Responses @@ -6659,29 +6653,21 @@ Mark a notification as dismissed for the current user. | ---- | ----------- | ------ | | 200 | Success | [TextContentResponse](#textcontentresponse) | -#### POST +### /notion/pre-import/pages + +#### GET ##### Parameters | Name | Located in | Description | Required | Schema | | ---- | ---------- | ----------- | -------- | ------ | -| page_id | path | | Yes | string | -| page_type | path | | Yes | string | -| payload | body | | Yes | [NotionEstimatePayload](#notionestimatepayload) | +| credential_id | query | Credential ID | Yes | string | +| dataset_id | query | Dataset ID | No | string | ##### Responses -| Code | Description | -| ---- | ----------- | -| 200 | Success | - -### /notion/pre-import/pages - -#### GET -##### Responses - -| Code | Description | -| ---- | ----------- | -| 200 | Success | +| Code | Description | Schema | +| ---- | ----------- | ------ | +| 200 | Success | [NotionIntegrateInfoListResponse](#notionintegrateinfolistresponse) | ### /oauth/authorize/{provider} @@ -12456,19 +12442,7 @@ Condition detail | ---- | ---- | ----------- | -------- | | info_list | [InfoList](#infolist) | | Yes | -#### DataSourceIntegrate - -| Name | Type | Description | Required | -| ---- | ---- | ----------- | -------- | -| created_at | object | | No | -| disabled | boolean | | No | -| id | string | | No | -| is_bound | boolean | | No | -| link | string | | No | -| provider | string | | No | -| source_info | [DataSourceIntegrateWorkspace](#datasourceintegrateworkspace) | | No | - -#### DataSourceIntegrateIcon +#### DataSourceIntegrateIconResponse | Name | Type | Description | Required | | ---- | ---- | ----------- | -------- | @@ -12476,31 +12450,43 @@ Condition detail | type | string | | No | | url | string | | No | -#### DataSourceIntegrateList +#### DataSourceIntegrateListResponse | Name | Type | Description | Required | | ---- | ---- | ----------- | -------- | -| data | [ [DataSourceIntegrate](#datasourceintegrate) ] | | No | +| data | [ [DataSourceIntegrateResponse](#datasourceintegrateresponse) ] | | Yes | -#### DataSourceIntegratePage +#### DataSourceIntegratePageResponse | Name | Type | Description | Required | | ---- | ---- | ----------- | -------- | -| page_icon | [DataSourceIntegrateIcon](#datasourceintegrateicon) | | No | -| page_id | string | | No | -| page_name | string | | No | -| parent_id | string | | No | -| type | string | | No | +| page_icon | [DataSourceIntegrateIconResponse](#datasourceintegrateiconresponse) | | Yes | +| page_id | string | | Yes | +| page_name | string | | Yes | +| parent_id | string | | Yes | +| type | string | | Yes | -#### DataSourceIntegrateWorkspace +#### DataSourceIntegrateResponse | Name | Type | Description | Required | | ---- | ---- | ----------- | -------- | -| pages | [ [DataSourceIntegratePage](#datasourceintegratepage) ] | | No | -| total | integer | | No | -| workspace_icon | string | | No | -| workspace_id | string | | No | -| workspace_name | string | | No | +| created_at | integer | | Yes | +| disabled | boolean | | Yes | +| id | string | | Yes | +| is_bound | boolean | | Yes | +| link | string | | Yes | +| provider | string | | Yes | +| source_info | [DataSourceIntegrateWorkspaceResponse](#datasourceintegrateworkspaceresponse) | | Yes | + +#### DataSourceIntegrateWorkspaceResponse + +| Name | Type | Description | Required | +| ---- | ---- | ----------- | -------- | +| pages | [ [DataSourceIntegratePageResponse](#datasourceintegratepageresponse) ] | | Yes | +| total | integer | | Yes | +| workspace_icon | string | | Yes | +| workspace_id | string | | Yes | +| workspace_name | string | | Yes | #### DatasetAndDocumentResponse @@ -13906,6 +13892,14 @@ Request payload for bulk downloading documents as a zip archive. | ---- | ---- | ----------- | -------- | | include_secret | string | Whether to include secret values in the exported DSL | No | +#### IndexingEstimate + +| Name | Type | Description | Required | +| ---- | ---- | ----------- | -------- | +| preview | [ [PreviewDetail](#previewdetail) ] | | Yes | +| qa_preview | [ [QAPreviewDetail](#qapreviewdetail) ] | | No | +| total_segments | integer | | Yes | + #### IndexingEstimatePayload | Name | Type | Description | Required | @@ -14462,31 +14456,31 @@ Enum class for model type. | pages | [ [NotionPage](#notionpage) ] | | Yes | | workspace_id | string | | Yes | -#### NotionIntegrateInfoList +#### NotionIntegrateInfoListResponse | Name | Type | Description | Required | | ---- | ---- | ----------- | -------- | -| notion_info | [ [NotionIntegrateWorkspace](#notionintegrateworkspace) ] | | No | +| notion_info | [ [NotionIntegrateWorkspaceResponse](#notionintegrateworkspaceresponse) ] | | Yes | -#### NotionIntegratePage +#### NotionIntegratePageResponse | Name | Type | Description | Required | | ---- | ---- | ----------- | -------- | -| is_bound | boolean | | No | -| page_icon | [DataSourceIntegrateIcon](#datasourceintegrateicon) | | No | -| page_id | string | | No | -| page_name | string | | No | -| parent_id | string | | No | -| type | string | | No | +| is_bound | boolean | | Yes | +| page_icon | [DataSourceIntegrateIconResponse](#datasourceintegrateiconresponse) | | Yes | +| page_id | string | | Yes | +| page_name | string | | Yes | +| parent_id | string | | Yes | +| type | string | | Yes | -#### NotionIntegrateWorkspace +#### NotionIntegrateWorkspaceResponse | Name | Type | Description | Required | | ---- | ---- | ----------- | -------- | -| pages | [ [NotionIntegratePage](#notionintegratepage) ] | | No | -| workspace_icon | string | | No | -| workspace_id | string | | No | -| workspace_name | string | | No | +| pages | [ [NotionIntegratePageResponse](#notionintegratepageresponse) ] | | Yes | +| workspace_icon | string | | Yes | +| workspace_id | string | | Yes | +| workspace_name | string | | Yes | #### NotionPage @@ -15018,6 +15012,14 @@ Shared permission levels for resources (datasets, credentials, etc.) | enabled | boolean | | Yes | | id | string | | Yes | +#### PreviewDetail + +| Name | Type | Description | Required | +| ---- | ---- | ----------- | -------- | +| child_chunks | [ string ] | | No | +| content | string | | Yes | +| summary | string | | No | + #### ProcessRule | Name | Type | Description | Required | @@ -15044,6 +15046,13 @@ Shared permission levels for resources (datasets, credentials, etc.) | response_mode | string | *Enum:* `"blocking"`, `"streaming"` | No | | start_node_id | string | | Yes | +#### QAPreviewDetail + +| Name | Type | Description | Required | +| ---- | ---- | ----------- | -------- | +| answer | string | | Yes | +| question | string | | Yes | + #### Quota | Name | Type | Description | Required | diff --git a/api/tests/test_containers_integration_tests/controllers/console/datasets/test_data_source.py b/api/tests/test_containers_integration_tests/controllers/console/datasets/test_data_source.py index b59009f7c4..54b82df4e5 100644 --- a/api/tests/test_containers_integration_tests/controllers/console/datasets/test_data_source.py +++ b/api/tests/test_containers_integration_tests/controllers/console/datasets/test_data_source.py @@ -2,6 +2,7 @@ from __future__ import annotations +from datetime import UTC, datetime from unittest.mock import MagicMock, PropertyMock, patch import pytest @@ -11,12 +12,14 @@ from werkzeug.exceptions import NotFound from controllers.console.datasets import data_source from controllers.console.datasets.data_source import ( DataSourceApi, - DataSourceNotionApi, DataSourceNotionDatasetSyncApi, DataSourceNotionDocumentSyncApi, + DataSourceNotionIndexingEstimateApi, DataSourceNotionListApi, + DataSourceNotionPreviewApi, ) from core.rag.index_processor.constant.index_type import IndexStructureType +from models import DataSourceOauthBinding def unwrap(func): @@ -59,13 +62,29 @@ class TestDataSourceApi: api = DataSourceApi() method = unwrap(api.get) - binding = MagicMock( - id="b1", + binding = DataSourceOauthBinding( + tenant_id="tenant-1", + access_token="token", provider="notion", - created_at="now", - disabled=False, - source_info={}, + source_info={ + "workspace_name": "Workspace", + "workspace_id": "workspace-1", + "workspace_icon": None, + "total": 1, + "pages": [ + { + "page_id": "page-1", + "page_name": "Page", + "page_icon": {"type": "emoji", "emoji": "P", "url": None}, + "parent_id": "parent-1", + "type": "page", + } + ], + }, ) + binding.id = "b1" + binding.created_at = datetime(2026, 5, 25, 1, 2, 3, tzinfo=UTC) + binding.disabled = False with ( app.test_request_context("/"), @@ -77,7 +96,29 @@ class TestDataSourceApi: response, status = method(api) assert status == 200 - assert response["data"][0]["is_bound"] is True + assert response["data"][0] == { + "id": "b1", + "provider": "notion", + "created_at": 1779670923, + "is_bound": True, + "disabled": False, + "source_info": { + "workspace_name": "Workspace", + "workspace_id": "workspace-1", + "workspace_icon": None, + "pages": [ + { + "page_name": "Page", + "page_id": "page-1", + "page_icon": {"type": "emoji", "url": None, "emoji": "P"}, + "parent_id": "parent-1", + "type": "page", + } + ], + "total": 1, + }, + "link": "http://localhost/console/api/oauth/data-source/notion", + } def test_get_no_bindings(self, app: Flask, patch_tenant): api = DataSourceApi() @@ -322,13 +363,13 @@ class TestDataSourceNotionListApi: method(api) -class TestDataSourceNotionApi: +class TestDataSourceNotionPreviewApi: @pytest.fixture def app(self, flask_app_with_containers: Flask): return flask_app_with_containers def test_get_preview_success(self, app: Flask, patch_tenant): - api = DataSourceNotionApi() + api = DataSourceNotionPreviewApi() method = unwrap(api.get) extractor = MagicMock(extract=lambda: [MagicMock(page_content="hello")]) @@ -348,8 +389,14 @@ class TestDataSourceNotionApi: assert status == 200 + +class TestDataSourceNotionIndexingEstimateApi: + @pytest.fixture + def app(self, flask_app_with_containers: Flask): + return flask_app_with_containers + def test_post_indexing_estimate_success(self, app: Flask, patch_tenant): - api = DataSourceNotionApi() + api = DataSourceNotionIndexingEstimateApi() method = unwrap(api.post) payload = { diff --git a/api/tests/unit_tests/controllers/console/datasets/test_data_source.py b/api/tests/unit_tests/controllers/console/datasets/test_data_source.py new file mode 100644 index 0000000000..5890175c97 --- /dev/null +++ b/api/tests/unit_tests/controllers/console/datasets/test_data_source.py @@ -0,0 +1,173 @@ +from __future__ import annotations + +import inspect +from collections.abc import Callable +from datetime import UTC, datetime +from typing import cast +from unittest.mock import MagicMock, PropertyMock, patch + +import pytest +from flask import Flask + +from controllers.console.datasets import data_source as module +from controllers.console.datasets.data_source import DataSourceApi, DataSourceNotionListApi +from models import DataSourceOauthBinding + +ControllerMethod = Callable[..., tuple[dict[str, object], int]] + + +def unwrap(func: object) -> ControllerMethod: + return cast(ControllerMethod, inspect.unwrap(cast(Callable[..., object], func))) + + +@pytest.fixture +def flask_app() -> Flask: + app = Flask(__name__) + app.config["TESTING"] = True + return app + + +@pytest.fixture +def tenant_context() -> tuple[MagicMock, str]: + return MagicMock(id="user-1"), "tenant-1" + + +def test_get_data_source_integrates_serializes_orm_binding( + flask_app: Flask, tenant_context: tuple[MagicMock, str] +) -> None: + binding = DataSourceOauthBinding( + tenant_id="tenant-1", + access_token="token", + provider="notion", + source_info={ + "workspace_name": "Workspace", + "workspace_id": "workspace-1", + "workspace_icon": None, + "total": 1, + "pages": [ + { + "page_id": "page-1", + "page_name": "Page", + "page_icon": {"type": "emoji", "emoji": "P", "url": None}, + "parent_id": "parent-1", + "type": "page", + } + ], + }, + ) + binding.id = "binding-1" + binding.created_at = datetime(2026, 5, 25, 1, 2, 3, tzinfo=UTC) + binding.disabled = False + + with ( + flask_app.test_request_context("/"), + patch.object(module, "current_account_with_tenant", return_value=tenant_context), + patch.object(module.db.session, "scalars", return_value=MagicMock(all=lambda: [binding])), + ): + response, status = unwrap(DataSourceApi().get)(DataSourceApi()) + + assert status == 200 + assert response == { + "data": [ + { + "id": "binding-1", + "provider": "notion", + "created_at": 1779670923, + "is_bound": True, + "disabled": False, + "source_info": { + "workspace_name": "Workspace", + "workspace_id": "workspace-1", + "workspace_icon": None, + "pages": [ + { + "page_name": "Page", + "page_id": "page-1", + "page_icon": {"type": "emoji", "url": None, "emoji": "P"}, + "parent_id": "parent-1", + "type": "page", + } + ], + "total": 1, + }, + "link": "http://localhost/console/api/oauth/data-source/notion", + } + ] + } + + +def test_get_data_source_integrates_preserves_empty_list_when_no_binding( + flask_app: Flask, tenant_context: tuple[MagicMock, str] +) -> None: + with ( + flask_app.test_request_context("/"), + patch.object(module, "current_account_with_tenant", return_value=tenant_context), + patch.object(module.db.session, "scalars", return_value=MagicMock(all=lambda: [])), + ): + response, status = unwrap(DataSourceApi().get)(DataSourceApi()) + + assert status == 200 + assert response == {"data": []} + + +def test_notion_pre_import_pages_serializes_frontend_list_shape( + flask_app: Flask, tenant_context: tuple[MagicMock, str] +) -> None: + page = MagicMock( + page_id="page-1", + page_name="Page", + type="page", + parent_id="parent-1", + page_icon={"type": "emoji", "emoji": "P", "url": None}, + ) + online_document_message = MagicMock( + result=[ + MagicMock( + workspace_id="workspace-1", + workspace_name="Workspace", + workspace_icon=None, + pages=[page], + ) + ] + ) + runtime = MagicMock( + get_online_document_pages=MagicMock(return_value=iter([online_document_message])), + datasource_provider_type=MagicMock(return_value="online_document"), + ) + + with ( + flask_app.test_request_context("/?credential_id=credential-1"), + patch.object(module, "current_account_with_tenant", return_value=tenant_context), + patch.object( + module.DatasourceProviderService, + "get_datasource_credentials", + return_value={"token": "token"}, + ), + patch.object(type(module.db), "engine", new_callable=PropertyMock, return_value=MagicMock()), + patch.object(module, "sessionmaker"), + patch("core.datasource.datasource_manager.DatasourceManager.get_datasource_runtime", return_value=runtime), + ): + response, status = unwrap(DataSourceNotionListApi().get)(DataSourceNotionListApi()) + + assert status == 200 + assert response == { + "notion_info": [ + { + "workspace_name": "Workspace", + "workspace_id": "workspace-1", + "workspace_icon": None, + "pages": [ + { + "page_name": "Page", + "page_id": "page-1", + "page_icon": {"type": "emoji", "url": None, "emoji": "P"}, + "parent_id": "parent-1", + "type": "page", + "is_bound": False, + } + ], + } + ] + } + runtime.get_online_document_pages.assert_called_once() + assert runtime.get_online_document_pages.call_args.kwargs["datasource_parameters"] == {} diff --git a/packages/contracts/generated/api/console/data-source/orpc.gen.ts b/packages/contracts/generated/api/console/data-source/orpc.gen.ts index 51c86e86bb..209447236a 100644 --- a/packages/contracts/generated/api/console/data-source/orpc.gen.ts +++ b/packages/contracts/generated/api/console/data-source/orpc.gen.ts @@ -12,16 +12,8 @@ import { zPatchDataSourceIntegratesResponse, } from './zod.gen' -/** - * Generated contract types may be inaccurate because backend OpenAPI annotations are incomplete. Do not migrate callers until the generated contract is accurate. - * - * @deprecated - */ export const get = oc .route({ - deprecated: true, - description: - 'Generated contract types may be inaccurate because backend OpenAPI annotations are incomplete. Do not migrate callers until the generated contract is accurate.', inputStructure: 'detailed', method: 'GET', operationId: 'getDataSourceIntegratesByBindingIdByAction', @@ -51,16 +43,8 @@ export const byBindingId = { byAction, } -/** - * Generated contract types may be inaccurate because backend OpenAPI annotations are incomplete. Do not migrate callers until the generated contract is accurate. - * - * @deprecated - */ export const get2 = oc .route({ - deprecated: true, - description: - 'Generated contract types may be inaccurate because backend OpenAPI annotations are incomplete. Do not migrate callers until the generated contract is accurate.', inputStructure: 'detailed', method: 'GET', operationId: 'getDataSourceIntegrates', diff --git a/packages/contracts/generated/api/console/data-source/types.gen.ts b/packages/contracts/generated/api/console/data-source/types.gen.ts index eb68f30bd5..6c2ef8db58 100644 --- a/packages/contracts/generated/api/console/data-source/types.gen.ts +++ b/packages/contracts/generated/api/console/data-source/types.gen.ts @@ -4,10 +4,46 @@ export type ClientOptions = { baseUrl: `${string}://${string}/console/api` | (string & {}) } +export type DataSourceIntegrateListResponse = { + data: Array +} + export type SimpleResultResponse = { result: string } +export type DataSourceIntegrateResponse = { + created_at: number | null + disabled: boolean | null + id: string | null + is_bound: boolean + link: string + provider: string + source_info: DataSourceIntegrateWorkspaceResponse +} + +export type DataSourceIntegrateWorkspaceResponse = { + pages: Array + total: number + workspace_icon: string | null + workspace_id: string | null + workspace_name: string | null +} + +export type DataSourceIntegratePageResponse = { + page_icon: DataSourceIntegrateIconResponse + page_id: string + page_name: string + parent_id: string + type: string +} + +export type DataSourceIntegrateIconResponse = { + emoji?: string | null + type?: string | null + url?: string | null +} + export type GetDataSourceIntegratesData = { body?: never path?: never @@ -16,9 +52,7 @@ export type GetDataSourceIntegratesData = { } export type GetDataSourceIntegratesResponses = { - 200: { - [key: string]: unknown - } + 200: DataSourceIntegrateListResponse } export type GetDataSourceIntegratesResponse @@ -49,9 +83,7 @@ export type GetDataSourceIntegratesByBindingIdByActionData = { } export type GetDataSourceIntegratesByBindingIdByActionResponses = { - 200: { - [key: string]: unknown - } + 200: DataSourceIntegrateListResponse } export type GetDataSourceIntegratesByBindingIdByActionResponse diff --git a/packages/contracts/generated/api/console/data-source/zod.gen.ts b/packages/contracts/generated/api/console/data-source/zod.gen.ts index f1bd5c652f..1511f3de86 100644 --- a/packages/contracts/generated/api/console/data-source/zod.gen.ts +++ b/packages/contracts/generated/api/console/data-source/zod.gen.ts @@ -9,10 +9,61 @@ export const zSimpleResultResponse = z.object({ result: z.string(), }) +/** + * DataSourceIntegrateIconResponse + */ +export const zDataSourceIntegrateIconResponse = z.object({ + emoji: z.string().nullish(), + type: z.string().nullish(), + url: z.string().nullish(), +}) + +/** + * DataSourceIntegratePageResponse + */ +export const zDataSourceIntegratePageResponse = z.object({ + page_icon: zDataSourceIntegrateIconResponse, + page_id: z.string(), + page_name: z.string(), + parent_id: z.string(), + type: z.string(), +}) + +/** + * DataSourceIntegrateWorkspaceResponse + */ +export const zDataSourceIntegrateWorkspaceResponse = z.object({ + pages: z.array(zDataSourceIntegratePageResponse), + total: z.int(), + workspace_icon: z.string().nullable(), + workspace_id: z.string().nullable(), + workspace_name: z.string().nullable(), +}) + +/** + * DataSourceIntegrateResponse + */ +export const zDataSourceIntegrateResponse = z.object({ + created_at: z.int().nullable(), + disabled: z.boolean().nullable(), + id: z.string().nullable(), + is_bound: z.boolean(), + link: z.string(), + provider: z.string(), + source_info: zDataSourceIntegrateWorkspaceResponse, +}) + +/** + * DataSourceIntegrateListResponse + */ +export const zDataSourceIntegrateListResponse = z.object({ + data: z.array(zDataSourceIntegrateResponse), +}) + /** * Success */ -export const zGetDataSourceIntegratesResponse = z.record(z.string(), z.unknown()) +export const zGetDataSourceIntegratesResponse = zDataSourceIntegrateListResponse /** * Success @@ -27,7 +78,7 @@ export const zGetDataSourceIntegratesByBindingIdByActionPath = z.object({ /** * Success */ -export const zGetDataSourceIntegratesByBindingIdByActionResponse = z.record(z.string(), z.unknown()) +export const zGetDataSourceIntegratesByBindingIdByActionResponse = zDataSourceIntegrateListResponse export const zPatchDataSourceIntegratesByBindingIdByActionPath = z.object({ action: z.string(), diff --git a/packages/contracts/generated/api/console/datasets/orpc.gen.ts b/packages/contracts/generated/api/console/datasets/orpc.gen.ts index 0b2c04d64b..13ce55e275 100644 --- a/packages/contracts/generated/api/console/datasets/orpc.gen.ts +++ b/packages/contracts/generated/api/console/datasets/orpc.gen.ts @@ -90,7 +90,6 @@ import { zGetDatasetsExternalKnowledgeApiQuery, zGetDatasetsExternalKnowledgeApiResponse, zGetDatasetsMetadataBuiltInResponse, - zGetDatasetsNotionIndexingEstimateResponse, zGetDatasetsProcessRuleQuery, zGetDatasetsProcessRuleResponse, zGetDatasetsQuery, @@ -518,16 +517,6 @@ export const metadata = { builtIn, } -export const get8 = oc - .route({ - inputStructure: 'detailed', - method: 'GET', - operationId: 'getDatasetsNotionIndexingEstimate', - path: '/datasets/notion-indexing-estimate', - tags: ['console'], - }) - .output(zGetDatasetsNotionIndexingEstimateResponse) - /** * Generated contract types may be inaccurate because backend OpenAPI annotations are incomplete. Do not migrate callers until the generated contract is accurate. * @@ -548,7 +537,6 @@ export const post7 = oc .output(zPostDatasetsNotionIndexingEstimateResponse) export const notionIndexingEstimate = { - get: get8, post: post7, } @@ -559,7 +547,7 @@ export const notionIndexingEstimate = { * * @deprecated */ -export const get9 = oc +export const get8 = oc .route({ deprecated: true, description: @@ -574,13 +562,13 @@ export const get9 = oc .output(zGetDatasetsProcessRuleResponse) export const processRule = { - get: get9, + get: get8, } /** * Get mock dataset retrieval settings by vector type */ -export const get10 = oc +export const get9 = oc .route({ description: 'Get mock dataset retrieval settings by vector type', inputStructure: 'detailed', @@ -593,13 +581,13 @@ export const get10 = oc .output(zGetDatasetsRetrievalSettingByVectorTypeResponse) export const byVectorType = { - get: get10, + get: get9, } /** * Get dataset retrieval settings */ -export const get11 = oc +export const get10 = oc .route({ description: 'Get dataset retrieval settings', inputStructure: 'detailed', @@ -611,7 +599,7 @@ export const get11 = oc .output(zGetDatasetsRetrievalSettingResponse) export const retrievalSetting = { - get: get11, + get: get10, byVectorType, } @@ -637,7 +625,7 @@ export const apiKeys2 = { /** * Get dataset auto disable logs */ -export const get12 = oc +export const get11 = oc .route({ description: 'Get dataset auto disable logs', inputStructure: 'detailed', @@ -650,7 +638,7 @@ export const get12 = oc .output(zGetDatasetsByDatasetIdAutoDisableLogsResponse) export const autoDisableLogs = { - get: get12, + get: get11, } /** @@ -658,7 +646,7 @@ export const autoDisableLogs = { * * @deprecated */ -export const get13 = oc +export const get12 = oc .route({ deprecated: true, description: @@ -673,10 +661,10 @@ export const get13 = oc .output(zGetDatasetsByDatasetIdBatchByBatchIndexingEstimateResponse) export const indexingEstimate2 = { - get: get13, + get: get12, } -export const get14 = oc +export const get13 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -688,7 +676,7 @@ export const get14 = oc .output(zGetDatasetsByDatasetIdBatchByBatchIndexingStatusResponse) export const indexingStatus = { - get: get14, + get: get13, } export const byBatch = { @@ -811,7 +799,7 @@ export const status = { /** * Get a signed download URL for a dataset document's original uploaded file */ -export const get15 = oc +export const get14 = oc .route({ description: 'Get a signed download URL for a dataset document\'s original uploaded file', inputStructure: 'detailed', @@ -824,7 +812,7 @@ export const get15 = oc .output(zGetDatasetsByDatasetIdDocumentsByDocumentIdDownloadResponse) export const download = { - get: get15, + get: get14, } /** @@ -834,7 +822,7 @@ export const download = { * * @deprecated */ -export const get16 = oc +export const get15 = oc .route({ deprecated: true, description: @@ -849,13 +837,13 @@ export const get16 = oc .output(zGetDatasetsByDatasetIdDocumentsByDocumentIdIndexingEstimateResponse) export const indexingEstimate3 = { - get: get16, + get: get15, } /** * Get document indexing status */ -export const get17 = oc +export const get16 = oc .route({ description: 'Get document indexing status', inputStructure: 'detailed', @@ -868,7 +856,7 @@ export const get17 = oc .output(zGetDatasetsByDatasetIdDocumentsByDocumentIdIndexingStatusResponse) export const indexingStatus2 = { - get: get17, + get: get16, } /** @@ -895,7 +883,7 @@ export const metadata3 = { put, } -export const get18 = oc +export const get17 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -907,7 +895,7 @@ export const get18 = oc .output(zGetDatasetsByDatasetIdDocumentsByDocumentIdNotionSyncResponse) export const sync = { - get: get18, + get: get17, } export const notion = { @@ -919,7 +907,7 @@ export const notion = { * * @deprecated */ -export const get19 = oc +export const get18 = oc .route({ deprecated: true, description: @@ -934,7 +922,7 @@ export const get19 = oc .output(zGetDatasetsByDatasetIdDocumentsByDocumentIdPipelineExecutionLogResponse) export const pipelineExecutionLog = { - get: get19, + get: get18, } /** @@ -1063,7 +1051,7 @@ export const segment = { byAction: byAction3, } -export const get20 = oc +export const get19 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -1091,7 +1079,7 @@ export const post14 = oc .output(zPostDatasetsByDatasetIdDocumentsByDocumentIdSegmentsBatchImportResponse) export const batchImport = { - get: get20, + get: get19, post: post14, } @@ -1140,7 +1128,7 @@ export const byChildChunkId = { patch: patch7, } -export const get21 = oc +export const get20 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -1190,7 +1178,7 @@ export const post15 = oc .output(zPostDatasetsByDatasetIdDocumentsByDocumentIdSegmentsBySegmentIdChildChunksResponse) export const childChunks = { - get: get21, + get: get20, patch: patch8, post: post15, byChildChunkId, @@ -1249,7 +1237,7 @@ export const delete5 = oc ) .output(zDeleteDatasetsByDatasetIdDocumentsByDocumentIdSegmentsResponse) -export const get22 = oc +export const get21 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -1267,7 +1255,7 @@ export const get22 = oc export const segments = { delete: delete5, - get: get22, + get: get21, batchImport, bySegmentId, } @@ -1289,7 +1277,7 @@ export const segments = { * * @deprecated */ -export const get23 = oc +export const get22 = oc .route({ deprecated: true, description: @@ -1305,13 +1293,13 @@ export const get23 = oc .output(zGetDatasetsByDatasetIdDocumentsByDocumentIdSummaryStatusResponse) export const summaryStatus = { - get: get23, + get: get22, } /** * sync website document */ -export const get24 = oc +export const get23 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -1324,7 +1312,7 @@ export const get24 = oc .output(zGetDatasetsByDatasetIdDocumentsByDocumentIdWebsiteSyncResponse) export const websiteSync = { - get: get24, + get: get23, } export const delete6 = oc @@ -1346,7 +1334,7 @@ export const delete6 = oc * * @deprecated */ -export const get25 = oc +export const get24 = oc .route({ deprecated: true, description: @@ -1367,7 +1355,7 @@ export const get25 = oc export const byDocumentId = { delete: delete6, - get: get25, + get: get24, download, indexingEstimate: indexingEstimate3, indexingStatus: indexingStatus2, @@ -1397,7 +1385,7 @@ export const delete7 = oc /** * Get documents in a dataset */ -export const get26 = oc +export const get25 = oc .route({ description: 'Get documents in a dataset', inputStructure: 'detailed', @@ -1440,7 +1428,7 @@ export const post16 = oc export const documents = { delete: delete7, - get: get26, + get: get25, post: post16, downloadZip, generateSummary, @@ -1452,7 +1440,7 @@ export const documents = { /** * Get dataset error documents */ -export const get27 = oc +export const get26 = oc .route({ description: 'Get dataset error documents', inputStructure: 'detailed', @@ -1465,7 +1453,7 @@ export const get27 = oc .output(zGetDatasetsByDatasetIdErrorDocsResponse) export const errorDocs = { - get: get27, + get: get26, } /** @@ -1531,7 +1519,7 @@ export const hitTesting = { /** * Get dataset indexing status */ -export const get28 = oc +export const get27 = oc .route({ description: 'Get dataset indexing status', inputStructure: 'detailed', @@ -1544,7 +1532,7 @@ export const get28 = oc .output(zGetDatasetsByDatasetIdIndexingStatusResponse) export const indexingStatus3 = { - get: get28, + get: get27, } export const post19 = oc @@ -1600,7 +1588,7 @@ export const byMetadataId = { patch: patch10, } -export const get29 = oc +export const get28 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -1629,13 +1617,13 @@ export const post20 = oc .output(zPostDatasetsByDatasetIdMetadataResponse) export const metadata4 = { - get: get29, + get: get28, post: post20, builtIn: builtIn2, byMetadataId, } -export const get30 = oc +export const get29 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -1647,7 +1635,7 @@ export const get30 = oc .output(zGetDatasetsByDatasetIdNotionSyncResponse) export const sync2 = { - get: get30, + get: get29, } export const notion2 = { @@ -1657,7 +1645,7 @@ export const notion2 = { /** * Get dataset permission user list */ -export const get31 = oc +export const get30 = oc .route({ description: 'Get dataset permission user list', inputStructure: 'detailed', @@ -1670,13 +1658,13 @@ export const get31 = oc .output(zGetDatasetsByDatasetIdPermissionPartUsersResponse) export const permissionPartUsers = { - get: get31, + get: get30, } /** * Get dataset query history */ -export const get32 = oc +export const get31 = oc .route({ description: 'Get dataset query history', inputStructure: 'detailed', @@ -1689,13 +1677,13 @@ export const get32 = oc .output(zGetDatasetsByDatasetIdQueriesResponse) export const queries = { - get: get32, + get: get31, } /** * Get applications related to dataset */ -export const get33 = oc +export const get32 = oc .route({ description: 'Get applications related to dataset', inputStructure: 'detailed', @@ -1708,7 +1696,7 @@ export const get33 = oc .output(zGetDatasetsByDatasetIdRelatedAppsResponse) export const relatedApps = { - get: get33, + get: get32, } /** @@ -1739,7 +1727,7 @@ export const retry = { /** * Check if dataset is in use */ -export const get34 = oc +export const get33 = oc .route({ description: 'Check if dataset is in use', inputStructure: 'detailed', @@ -1752,7 +1740,7 @@ export const get34 = oc .output(zGetDatasetsByDatasetIdUseCheckResponse) export const useCheck2 = { - get: get34, + get: get33, } export const delete9 = oc @@ -1770,7 +1758,7 @@ export const delete9 = oc /** * Get dataset details */ -export const get35 = oc +export const get34 = oc .route({ description: 'Get dataset details', inputStructure: 'detailed', @@ -1805,7 +1793,7 @@ export const patch11 = oc export const byDatasetId = { delete: delete9, - get: get35, + get: get34, patch: patch11, apiKeys: apiKeys2, autoDisableLogs, @@ -1852,7 +1840,7 @@ export const byApiKeyId2 = { * * Get all API keys for a dataset */ -export const get36 = oc +export const get35 = oc .route({ description: 'Get all API keys for a dataset', inputStructure: 'detailed', @@ -1885,7 +1873,7 @@ export const post22 = oc .output(zPostDatasetsByResourceIdApiKeysResponse) export const apiKeys3 = { - get: get36, + get: get35, post: post22, byApiKeyId: byApiKeyId2, } @@ -1897,7 +1885,7 @@ export const byResourceId = { /** * Get list of datasets */ -export const get37 = oc +export const get36 = oc .route({ description: 'Get list of datasets', inputStructure: 'detailed', @@ -1926,7 +1914,7 @@ export const post23 = oc .output(zPostDatasetsResponse) export const datasets = { - get: get37, + get: get36, post: post23, apiBaseInfo, apiKeys, diff --git a/packages/contracts/generated/api/console/datasets/types.gen.ts b/packages/contracts/generated/api/console/datasets/types.gen.ts index cb7a869214..4d15231980 100644 --- a/packages/contracts/generated/api/console/datasets/types.gen.ts +++ b/packages/contracts/generated/api/console/datasets/types.gen.ts @@ -196,10 +196,6 @@ export type DatasetMetadataBuiltInFieldsResponse = { fields: Array } -export type TextContentResponse = { - content: string -} - export type NotionEstimatePayload = { doc_form?: string doc_language?: string @@ -211,6 +207,12 @@ export type NotionEstimatePayload = { } } +export type IndexingEstimate = { + preview: Array + qa_preview?: Array | null + total_segments: number +} + export type RetrievalSettingResponse = { retrieval_method: Array } @@ -694,6 +696,17 @@ export type DatasetMetadataBuiltInFieldResponse = { type: string } +export type PreviewDetail = { + child_chunks?: Array | null + content: string + summary?: string | null +} + +export type QaPreviewDetail = { + answer: string + question: string +} + export type DocumentWithSegmentsResponse = { archived?: boolean | null completed_segments?: number | null @@ -1387,20 +1400,6 @@ export type GetDatasetsMetadataBuiltInResponses = { export type GetDatasetsMetadataBuiltInResponse = GetDatasetsMetadataBuiltInResponses[keyof GetDatasetsMetadataBuiltInResponses] -export type GetDatasetsNotionIndexingEstimateData = { - body?: never - path?: never - query?: never - url: '/datasets/notion-indexing-estimate' -} - -export type GetDatasetsNotionIndexingEstimateResponses = { - 200: TextContentResponse -} - -export type GetDatasetsNotionIndexingEstimateResponse - = GetDatasetsNotionIndexingEstimateResponses[keyof GetDatasetsNotionIndexingEstimateResponses] - export type PostDatasetsNotionIndexingEstimateData = { body: NotionEstimatePayload path?: never @@ -1409,9 +1408,7 @@ export type PostDatasetsNotionIndexingEstimateData = { } export type PostDatasetsNotionIndexingEstimateResponses = { - 200: { - [key: string]: unknown - } + 200: IndexingEstimate } export type PostDatasetsNotionIndexingEstimateResponse diff --git a/packages/contracts/generated/api/console/datasets/zod.gen.ts b/packages/contracts/generated/api/console/datasets/zod.gen.ts index 87361f07c8..e4e2f37234 100644 --- a/packages/contracts/generated/api/console/datasets/zod.gen.ts +++ b/packages/contracts/generated/api/console/datasets/zod.gen.ts @@ -81,13 +81,6 @@ export const zIndexingEstimatePayload = z.object({ process_rule: z.record(z.string(), z.unknown()), }) -/** - * TextContentResponse - */ -export const zTextContentResponse = z.object({ - content: z.string(), -}) - /** * NotionEstimatePayload */ @@ -483,6 +476,32 @@ export const zDatasetMetadataBuiltInFieldsResponse = z.object({ fields: z.array(zDatasetMetadataBuiltInFieldResponse), }) +/** + * PreviewDetail + */ +export const zPreviewDetail = z.object({ + child_chunks: z.array(z.string()).nullish(), + content: z.string(), + summary: z.string().nullish(), +}) + +/** + * QAPreviewDetail + */ +export const zQaPreviewDetail = z.object({ + answer: z.string(), + question: z.string(), +}) + +/** + * IndexingEstimate + */ +export const zIndexingEstimate = z.object({ + preview: z.array(zPreviewDetail), + qa_preview: z.array(zQaPreviewDetail).nullish(), + total_segments: z.int(), +}) + /** * DocumentMetadataResponse */ @@ -1530,17 +1549,12 @@ export const zPostDatasetsInitResponse = zDatasetAndDocumentResponse */ export const zGetDatasetsMetadataBuiltInResponse = zDatasetMetadataBuiltInFieldsResponse -/** - * Success - */ -export const zGetDatasetsNotionIndexingEstimateResponse = zTextContentResponse - export const zPostDatasetsNotionIndexingEstimateBody = zNotionEstimatePayload /** * Success */ -export const zPostDatasetsNotionIndexingEstimateResponse = z.record(z.string(), z.unknown()) +export const zPostDatasetsNotionIndexingEstimateResponse = zIndexingEstimate export const zGetDatasetsProcessRuleQuery = z.object({ document_id: z.string().optional(), diff --git a/packages/contracts/generated/api/console/notion/orpc.gen.ts b/packages/contracts/generated/api/console/notion/orpc.gen.ts index 28b2704a4b..1ab71ffc81 100644 --- a/packages/contracts/generated/api/console/notion/orpc.gen.ts +++ b/packages/contracts/generated/api/console/notion/orpc.gen.ts @@ -5,11 +5,10 @@ import * as z from 'zod' import { zGetNotionPagesByPageIdByPageTypePreviewPath, + zGetNotionPagesByPageIdByPageTypePreviewQuery, zGetNotionPagesByPageIdByPageTypePreviewResponse, + zGetNotionPreImportPagesQuery, zGetNotionPreImportPagesResponse, - zPostNotionPagesByPageIdByPageTypePreviewBody, - zPostNotionPagesByPageIdByPageTypePreviewPath, - zPostNotionPagesByPageIdByPageTypePreviewResponse, } from './zod.gen' export const get = oc @@ -20,36 +19,16 @@ export const get = oc path: '/notion/pages/{page_id}/{page_type}/preview', tags: ['console'], }) - .input(z.object({ params: zGetNotionPagesByPageIdByPageTypePreviewPath })) - .output(zGetNotionPagesByPageIdByPageTypePreviewResponse) - -/** - * Generated contract types may be inaccurate because backend OpenAPI annotations are incomplete. Do not migrate callers until the generated contract is accurate. - * - * @deprecated - */ -export const post = oc - .route({ - deprecated: true, - description: - 'Generated contract types may be inaccurate because backend OpenAPI annotations are incomplete. Do not migrate callers until the generated contract is accurate.', - inputStructure: 'detailed', - method: 'POST', - operationId: 'postNotionPagesByPageIdByPageTypePreview', - path: '/notion/pages/{page_id}/{page_type}/preview', - tags: ['console'], - }) .input( z.object({ - body: zPostNotionPagesByPageIdByPageTypePreviewBody, - params: zPostNotionPagesByPageIdByPageTypePreviewPath, + params: zGetNotionPagesByPageIdByPageTypePreviewPath, + query: zGetNotionPagesByPageIdByPageTypePreviewQuery, }), ) - .output(zPostNotionPagesByPageIdByPageTypePreviewResponse) + .output(zGetNotionPagesByPageIdByPageTypePreviewResponse) export const preview = { get, - post, } export const byPageType = { @@ -64,22 +43,15 @@ export const pages = { byPageId, } -/** - * Generated contract types may be inaccurate because backend OpenAPI annotations are incomplete. Do not migrate callers until the generated contract is accurate. - * - * @deprecated - */ export const get2 = oc .route({ - deprecated: true, - description: - 'Generated contract types may be inaccurate because backend OpenAPI annotations are incomplete. Do not migrate callers until the generated contract is accurate.', inputStructure: 'detailed', method: 'GET', operationId: 'getNotionPreImportPages', path: '/notion/pre-import/pages', tags: ['console'], }) + .input(z.object({ query: zGetNotionPreImportPagesQuery })) .output(zGetNotionPreImportPagesResponse) export const pages2 = { diff --git a/packages/contracts/generated/api/console/notion/types.gen.ts b/packages/contracts/generated/api/console/notion/types.gen.ts index 90c9f50894..ac0431b2d3 100644 --- a/packages/contracts/generated/api/console/notion/types.gen.ts +++ b/packages/contracts/generated/api/console/notion/types.gen.ts @@ -8,15 +8,30 @@ export type TextContentResponse = { content: string } -export type NotionEstimatePayload = { - doc_form?: string - doc_language?: string - notion_info_list: Array<{ - [key: string]: unknown - }> - process_rule: { - [key: string]: unknown - } +export type NotionIntegrateInfoListResponse = { + notion_info: Array +} + +export type NotionIntegrateWorkspaceResponse = { + pages: Array + workspace_icon: string | null + workspace_id: string | null + workspace_name: string | null +} + +export type NotionIntegratePageResponse = { + is_bound: boolean + page_icon: DataSourceIntegrateIconResponse + page_id: string + page_name: string + parent_id: string | null + type: string +} + +export type DataSourceIntegrateIconResponse = { + emoji?: string | null + type?: string | null + url?: string | null } export type GetNotionPagesByPageIdByPageTypePreviewData = { @@ -25,7 +40,9 @@ export type GetNotionPagesByPageIdByPageTypePreviewData = { page_id: string page_type: string } - query?: never + query: { + credential_id: string + } url: '/notion/pages/{page_id}/{page_type}/preview' } @@ -36,36 +53,18 @@ export type GetNotionPagesByPageIdByPageTypePreviewResponses = { export type GetNotionPagesByPageIdByPageTypePreviewResponse = GetNotionPagesByPageIdByPageTypePreviewResponses[keyof GetNotionPagesByPageIdByPageTypePreviewResponses] -export type PostNotionPagesByPageIdByPageTypePreviewData = { - body: NotionEstimatePayload - path: { - page_id: string - page_type: string - } - query?: never - url: '/notion/pages/{page_id}/{page_type}/preview' -} - -export type PostNotionPagesByPageIdByPageTypePreviewResponses = { - 200: { - [key: string]: unknown - } -} - -export type PostNotionPagesByPageIdByPageTypePreviewResponse - = PostNotionPagesByPageIdByPageTypePreviewResponses[keyof PostNotionPagesByPageIdByPageTypePreviewResponses] - export type GetNotionPreImportPagesData = { body?: never path?: never - query?: never + query: { + credential_id: string + dataset_id?: string + } url: '/notion/pre-import/pages' } export type GetNotionPreImportPagesResponses = { - 200: { - [key: string]: unknown - } + 200: NotionIntegrateInfoListResponse } export type GetNotionPreImportPagesResponse diff --git a/packages/contracts/generated/api/console/notion/zod.gen.ts b/packages/contracts/generated/api/console/notion/zod.gen.ts index 3362dd6b34..6e371766b9 100644 --- a/packages/contracts/generated/api/console/notion/zod.gen.ts +++ b/packages/contracts/generated/api/console/notion/zod.gen.ts @@ -10,13 +10,41 @@ export const zTextContentResponse = z.object({ }) /** - * NotionEstimatePayload + * DataSourceIntegrateIconResponse */ -export const zNotionEstimatePayload = z.object({ - doc_form: z.string().optional().default('text_model'), - doc_language: z.string().optional().default('English'), - notion_info_list: z.array(z.record(z.string(), z.unknown())), - process_rule: z.record(z.string(), z.unknown()), +export const zDataSourceIntegrateIconResponse = z.object({ + emoji: z.string().nullish(), + type: z.string().nullish(), + url: z.string().nullish(), +}) + +/** + * NotionIntegratePageResponse + */ +export const zNotionIntegratePageResponse = z.object({ + is_bound: z.boolean(), + page_icon: zDataSourceIntegrateIconResponse, + page_id: z.string(), + page_name: z.string(), + parent_id: z.string().nullable(), + type: z.string(), +}) + +/** + * NotionIntegrateWorkspaceResponse + */ +export const zNotionIntegrateWorkspaceResponse = z.object({ + pages: z.array(zNotionIntegratePageResponse), + workspace_icon: z.string().nullable(), + workspace_id: z.string().nullable(), + workspace_name: z.string().nullable(), +}) + +/** + * NotionIntegrateInfoListResponse + */ +export const zNotionIntegrateInfoListResponse = z.object({ + notion_info: z.array(zNotionIntegrateWorkspaceResponse), }) export const zGetNotionPagesByPageIdByPageTypePreviewPath = z.object({ @@ -24,24 +52,21 @@ export const zGetNotionPagesByPageIdByPageTypePreviewPath = z.object({ page_type: z.string(), }) -/** - * Success - */ -export const zGetNotionPagesByPageIdByPageTypePreviewResponse = zTextContentResponse - -export const zPostNotionPagesByPageIdByPageTypePreviewBody = zNotionEstimatePayload - -export const zPostNotionPagesByPageIdByPageTypePreviewPath = z.object({ - page_id: z.string(), - page_type: z.string(), +export const zGetNotionPagesByPageIdByPageTypePreviewQuery = z.object({ + credential_id: z.string().min(1), }) /** * Success */ -export const zPostNotionPagesByPageIdByPageTypePreviewResponse = z.record(z.string(), z.unknown()) +export const zGetNotionPagesByPageIdByPageTypePreviewResponse = zTextContentResponse + +export const zGetNotionPreImportPagesQuery = z.object({ + credential_id: z.string().min(1), + dataset_id: z.string().optional(), +}) /** * Success */ -export const zGetNotionPreImportPagesResponse = z.record(z.string(), z.unknown()) +export const zGetNotionPreImportPagesResponse = zNotionIntegrateInfoListResponse