diff --git a/api/core/rag/extractor/notion_extractor.py b/api/core/rag/extractor/notion_extractor.py index 4e14800d0a..eca955ddd1 100644 --- a/api/core/rag/extractor/notion_extractor.py +++ b/api/core/rag/extractor/notion_extractor.py @@ -79,55 +79,71 @@ class NotionExtractor(BaseExtractor): def _get_notion_database_data(self, database_id: str, query_dict: dict[str, Any] = {}) -> list[Document]: """Get all the pages from a Notion database.""" assert self._notion_access_token is not None, "Notion access token is required" - res = requests.post( - DATABASE_URL_TMPL.format(database_id=database_id), - headers={ - "Authorization": "Bearer " + self._notion_access_token, - "Content-Type": "application/json", - "Notion-Version": "2022-06-28", - }, - json=query_dict, - ) - - data = res.json() database_content = [] - if "results" not in data or data["results"] is None: + next_cursor = None + has_more = True + + while has_more: + current_query = query_dict.copy() + if next_cursor: + current_query["start_cursor"] = next_cursor + + res = requests.post( + DATABASE_URL_TMPL.format(database_id=database_id), + headers={ + "Authorization": "Bearer " + self._notion_access_token, + "Content-Type": "application/json", + "Notion-Version": "2022-06-28", + }, + json=current_query, + ) + + response_data = res.json() + + if "results" not in response_data or response_data["results"] is None: + break + + for result in response_data["results"]: + properties = result["properties"] + data = {} + value: Any + for property_name, property_value in properties.items(): + type = property_value["type"] + if type == "multi_select": + value = [] + multi_select_list = property_value[type] + for multi_select in multi_select_list: + value.append(multi_select["name"]) + elif type in {"rich_text", "title"}: + if len(property_value[type]) > 0: + value = property_value[type][0]["plain_text"] + else: + value = "" + elif type in {"select", "status"}: + if property_value[type]: + value = property_value[type]["name"] + else: + value = "" + else: + value = property_value[type] + data[property_name] = value + row_dict = {k: v for k, v in data.items() if v} + row_content = "" + for key, value in row_dict.items(): + if isinstance(value, dict): + value_dict = {k: v for k, v in value.items() if v} + value_content = "".join(f"{k}:{v} " for k, v in value_dict.items()) + row_content = row_content + f"{key}:{value_content}\n" + else: + row_content = row_content + f"{key}:{value}\n" + database_content.append(row_content) + + has_more = response_data.get("has_more", False) + next_cursor = response_data.get("next_cursor") + + if not database_content: return [] - for result in data["results"]: - properties = result["properties"] - data = {} - value: Any - for property_name, property_value in properties.items(): - type = property_value["type"] - if type == "multi_select": - value = [] - multi_select_list = property_value[type] - for multi_select in multi_select_list: - value.append(multi_select["name"]) - elif type in {"rich_text", "title"}: - if len(property_value[type]) > 0: - value = property_value[type][0]["plain_text"] - else: - value = "" - elif type in {"select", "status"}: - if property_value[type]: - value = property_value[type]["name"] - else: - value = "" - else: - value = property_value[type] - data[property_name] = value - row_dict = {k: v for k, v in data.items() if v} - row_content = "" - for key, value in row_dict.items(): - if isinstance(value, dict): - value_dict = {k: v for k, v in value.items() if v} - value_content = "".join(f"{k}:{v} " for k, v in value_dict.items()) - row_content = row_content + f"{key}:{value_content}\n" - else: - row_content = row_content + f"{key}:{value}\n" - database_content.append(row_content) return [Document(page_content="\n".join(database_content))] diff --git a/docker/.env.example b/docker/.env.example index d4d59936eb..195446b7ba 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -1067,7 +1067,7 @@ PLUGIN_MEDIA_CACHE_PATH=assets # Plugin oss bucket PLUGIN_STORAGE_OSS_BUCKET= # Plugin oss s3 credentials -PLUGIN_S3_USE_AWS= +PLUGIN_S3_USE_AWS=false PLUGIN_S3_USE_AWS_MANAGED_IAM=false PLUGIN_S3_ENDPOINT= PLUGIN_S3_USE_PATH_STYLE=false diff --git a/docker/docker-compose.middleware.yaml b/docker/docker-compose.middleware.yaml index 6d38c02bbb..4761e73178 100644 --- a/docker/docker-compose.middleware.yaml +++ b/docker/docker-compose.middleware.yaml @@ -103,7 +103,7 @@ services: PLUGIN_PACKAGE_CACHE_PATH: ${PLUGIN_PACKAGE_CACHE_PATH:-plugin_packages} PLUGIN_MEDIA_CACHE_PATH: ${PLUGIN_MEDIA_CACHE_PATH:-assets} PLUGIN_STORAGE_OSS_BUCKET: ${PLUGIN_STORAGE_OSS_BUCKET:-} - S3_USE_AWS: ${PLUGIN_S3_USE_AWS:-} + S3_USE_AWS: ${PLUGIN_S3_USE_AWS:-false} S3_USE_AWS_MANAGED_IAM: ${PLUGIN_S3_USE_AWS_MANAGED_IAM:-false} S3_ENDPOINT: ${PLUGIN_S3_ENDPOINT:-} S3_USE_PATH_STYLE: ${PLUGIN_S3_USE_PATH_STYLE:-false} diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index dddce106b9..2b98d098b3 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -467,7 +467,7 @@ x-shared-env: &shared-api-worker-env PLUGIN_PACKAGE_CACHE_PATH: ${PLUGIN_PACKAGE_CACHE_PATH:-plugin_packages} PLUGIN_MEDIA_CACHE_PATH: ${PLUGIN_MEDIA_CACHE_PATH:-assets} PLUGIN_STORAGE_OSS_BUCKET: ${PLUGIN_STORAGE_OSS_BUCKET:-} - PLUGIN_S3_USE_AWS: ${PLUGIN_S3_USE_AWS:-} + PLUGIN_S3_USE_AWS: ${PLUGIN_S3_USE_AWS:-false} PLUGIN_S3_USE_AWS_MANAGED_IAM: ${PLUGIN_S3_USE_AWS_MANAGED_IAM:-false} PLUGIN_S3_ENDPOINT: ${PLUGIN_S3_ENDPOINT:-} PLUGIN_S3_USE_PATH_STYLE: ${PLUGIN_S3_USE_PATH_STYLE:-false} diff --git a/docker/middleware.env.example b/docker/middleware.env.example index 338b057ae8..f261d88d48 100644 --- a/docker/middleware.env.example +++ b/docker/middleware.env.example @@ -133,7 +133,7 @@ PLUGIN_MEDIA_CACHE_PATH=assets PLUGIN_STORAGE_OSS_BUCKET= # Plugin oss s3 credentials PLUGIN_S3_USE_AWS_MANAGED_IAM=false -PLUGIN_S3_USE_AWS= +PLUGIN_S3_USE_AWS=false PLUGIN_S3_ENDPOINT= PLUGIN_S3_USE_PATH_STYLE=false PLUGIN_AWS_ACCESS_KEY= diff --git a/web/app/components/app/configuration/debug/index.tsx b/web/app/components/app/configuration/debug/index.tsx index 477328dad3..38b0c890e2 100644 --- a/web/app/components/app/configuration/debug/index.tsx +++ b/web/app/components/app/configuration/debug/index.tsx @@ -156,12 +156,11 @@ const Debug: FC = ({ } let hasEmptyInput = '' const requiredVars = modelConfig.configs.prompt_variables.filter(({ key, name, required, type }) => { - if (type !== 'string' && type !== 'paragraph' && type !== 'select') + if (type !== 'string' && type !== 'paragraph' && type !== 'select' && type !== 'number') return false const res = (!key || !key.trim()) || (!name || !name.trim()) || (required || required === undefined || required === null) return res }) // compatible with old version - // debugger requiredVars.forEach(({ key, name }) => { if (hasEmptyInput) return diff --git a/web/app/components/base/markdown/markdown-utils.ts b/web/app/components/base/markdown/markdown-utils.ts index 0aa385a1d1..dc3c7a9784 100644 --- a/web/app/components/base/markdown/markdown-utils.ts +++ b/web/app/components/base/markdown/markdown-utils.ts @@ -28,7 +28,7 @@ export const preprocessLaTeX = (content: string) => { } export const preprocessThinkTag = (content: string) => { - const thinkOpenTagRegex = /\n/g + const thinkOpenTagRegex = /(\n)+/g const thinkCloseTagRegex = /\n<\/think>/g return flow([ (str: string) => str.replace(thinkOpenTagRegex, '
\n'), diff --git a/web/app/components/rag-pipeline/components/input-field/dialog-wrapper.tsx b/web/app/components/rag-pipeline/components/input-field/dialog-wrapper.tsx index 552be5cd6c..42fdf46567 100644 --- a/web/app/components/rag-pipeline/components/input-field/dialog-wrapper.tsx +++ b/web/app/components/rag-pipeline/components/input-field/dialog-wrapper.tsx @@ -1,47 +1,58 @@ -import { Fragment, useCallback } from 'react' +import { useCallback } from 'react' import type { ReactNode } from 'react' import { Dialog, DialogPanel, Transition, TransitionChild } from '@headlessui/react' import cn from '@/utils/classnames' type DialogWrapperProps = { + dialogClassName?: string className?: string panelWrapperClassName?: string + outerWrapperClassName?: string children: ReactNode show: boolean onClose?: () => void } const DialogWrapper = ({ + dialogClassName, className, panelWrapperClassName, + outerWrapperClassName, children, show, onClose, }: DialogWrapperProps) => { const close = useCallback(() => onClose?.(), [onClose]) return ( - - + + -
+
-
+
- + {children} diff --git a/web/app/components/rag-pipeline/components/input-field/editor/dialog-wrapper.tsx b/web/app/components/rag-pipeline/components/input-field/editor/dialog-wrapper.tsx deleted file mode 100644 index 1cfaebb69d..0000000000 --- a/web/app/components/rag-pipeline/components/input-field/editor/dialog-wrapper.tsx +++ /dev/null @@ -1,54 +0,0 @@ -import { Fragment, useCallback } from 'react' -import type { ReactNode } from 'react' -import { Dialog, DialogPanel, Transition, TransitionChild } from '@headlessui/react' -import cn from '@/utils/classnames' - -type DialogWrapperProps = { - className?: string - panelWrapperClassName?: string - children: ReactNode - show: boolean - onClose?: () => void -} - -const DialogWrapper = ({ - className, - panelWrapperClassName, - children, - show, - onClose, -}: DialogWrapperProps) => { - const close = useCallback(() => onClose?.(), [onClose]) - return ( - - - -
- - -
-
- - - {children} - - -
-
-
- - ) -} - -export default DialogWrapper diff --git a/web/app/components/rag-pipeline/components/input-field/editor/index.tsx b/web/app/components/rag-pipeline/components/input-field/editor/index.tsx index 6aef4e4186..494846a627 100644 --- a/web/app/components/rag-pipeline/components/input-field/editor/index.tsx +++ b/web/app/components/rag-pipeline/components/input-field/editor/index.tsx @@ -1,5 +1,5 @@ import { RiCloseLine } from '@remixicon/react' -import DialogWrapper from './dialog-wrapper' +import DialogWrapper from '../dialog-wrapper' import InputFieldForm from './form' import { convertFormDataToINputField, convertToInputFieldFormData } from './utils' import { useCallback } from 'react' @@ -33,7 +33,9 @@ const InputFieldEditor = ({
{initialData ? t('datasetPipeline.inputFieldPanel.editInputField') : t('datasetPipeline.inputFieldPanel.addInputField')} diff --git a/web/app/components/rag-pipeline/components/input-field/index.tsx b/web/app/components/rag-pipeline/components/input-field/index.tsx index 81eb4c7809..401001d418 100644 --- a/web/app/components/rag-pipeline/components/input-field/index.tsx +++ b/web/app/components/rag-pipeline/components/input-field/index.tsx @@ -94,6 +94,7 @@ const InputFieldDialog = ({
diff --git a/web/app/components/rag-pipeline/components/input-field/preview/dialog-wrapper.tsx b/web/app/components/rag-pipeline/components/input-field/preview/dialog-wrapper.tsx deleted file mode 100644 index 087ad42089..0000000000 --- a/web/app/components/rag-pipeline/components/input-field/preview/dialog-wrapper.tsx +++ /dev/null @@ -1,54 +0,0 @@ -import { Fragment, useCallback } from 'react' -import type { ReactNode } from 'react' -import { Dialog, DialogPanel, Transition, TransitionChild } from '@headlessui/react' -import cn from '@/utils/classnames' - -type DialogWrapperProps = { - className?: string - panelWrapperClassName?: string - children: ReactNode - show: boolean - onClose?: () => void -} - -const DialogWrapper = ({ - className, - panelWrapperClassName, - children, - show, - onClose, -}: DialogWrapperProps) => { - const close = useCallback(() => onClose?.(), [onClose]) - return ( - - - -
- - -
-
- - - {children} - - -
-
-
- - ) -} - -export default DialogWrapper diff --git a/web/app/components/rag-pipeline/components/input-field/preview/index.tsx b/web/app/components/rag-pipeline/components/input-field/preview/index.tsx index d3ca95cb56..6693444356 100644 --- a/web/app/components/rag-pipeline/components/input-field/preview/index.tsx +++ b/web/app/components/rag-pipeline/components/input-field/preview/index.tsx @@ -1,6 +1,6 @@ import { useState } from 'react' import { RiCloseLine } from '@remixicon/react' -import DialogWrapper from './dialog-wrapper' +import DialogWrapper from '../dialog-wrapper' import { useTranslation } from 'react-i18next' import Badge from '@/app/components/base/badge' import DataSource from './data-source' @@ -24,7 +24,8 @@ const PreviewPanel = ({
diff --git a/web/types/app.ts b/web/types/app.ts index e4227adbe9..3de5c446ec 100644 --- a/web/types/app.ts +++ b/web/types/app.ts @@ -90,6 +90,7 @@ export type TextTypeFormItem = { variable: string required: boolean max_length: number + hide: boolean } export type SelectTypeFormItem = { @@ -98,6 +99,7 @@ export type SelectTypeFormItem = { variable: string required: boolean options: string[] + hide: boolean } export type ParagraphTypeFormItem = { @@ -105,6 +107,7 @@ export type ParagraphTypeFormItem = { label: string variable: string required: boolean + hide: boolean } /** * User Input Form Item diff --git a/web/utils/model-config.ts b/web/utils/model-config.ts index 74d8848a98..330d8f9b52 100644 --- a/web/utils/model-config.ts +++ b/web/utils/model-config.ts @@ -40,6 +40,7 @@ export const userInputsFormToPromptVariables = (useInputs: UserInputFormItem[] | max_length: content.max_length, options: [], is_context_var, + hide: content.hide, }) } else if (type === 'number') { @@ -49,6 +50,7 @@ export const userInputsFormToPromptVariables = (useInputs: UserInputFormItem[] | required: content.required, type, options: [], + hide: content.hide, }) } else if (type === 'select') { @@ -59,6 +61,7 @@ export const userInputsFormToPromptVariables = (useInputs: UserInputFormItem[] | type: 'select', options: content.options, is_context_var, + hide: content.hide, }) } else if (type === 'file') { @@ -73,6 +76,7 @@ export const userInputsFormToPromptVariables = (useInputs: UserInputFormItem[] | allowed_file_upload_methods: content.allowed_file_upload_methods, number_limits: 1, }, + hide: content.hide, }) } else if (type === 'file-list') { @@ -87,6 +91,7 @@ export const userInputsFormToPromptVariables = (useInputs: UserInputFormItem[] | allowed_file_upload_methods: content.allowed_file_upload_methods, number_limits: content.max_length, }, + hide: content.hide, }) } else { @@ -100,6 +105,7 @@ export const userInputsFormToPromptVariables = (useInputs: UserInputFormItem[] | icon: content.icon, icon_background: content.icon_background, is_context_var, + hide: content.hide, }) } }) @@ -119,6 +125,7 @@ export const promptVariablesToUserInputsForm = (promptVariables: PromptVariable[ required: item.required !== false, // default true max_length: item.max_length, default: '', + hide: item.hide, }, } as any) return @@ -130,6 +137,7 @@ export const promptVariablesToUserInputsForm = (promptVariables: PromptVariable[ variable: item.key, required: item.required !== false, // default true default: '', + hide: item.hide, }, } as any) } @@ -141,6 +149,7 @@ export const promptVariablesToUserInputsForm = (promptVariables: PromptVariable[ required: item.required !== false, // default true options: item.options, default: '', + hide: item.hide, }, } as any) } @@ -155,6 +164,7 @@ export const promptVariablesToUserInputsForm = (promptVariables: PromptVariable[ required: item.required, icon: item.icon, icon_background: item.icon_background, + hide: item.hide, }, } as any) }