diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml index 06584c1b78..e652657705 100644 --- a/.github/workflows/style.yml +++ b/.github/workflows/style.yml @@ -103,6 +103,11 @@ jobs: run: | pnpm run lint + - name: Web type check + if: steps.changed-files.outputs.any_changed == 'true' + working-directory: ./web + run: pnpm run type-check + docker-compose-template: name: Docker Compose Template runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index 32ec6914cd..c6067e96cd 100644 --- a/.gitignore +++ b/.gitignore @@ -100,6 +100,7 @@ __pypackages__/ # Celery stuff celerybeat-schedule +celerybeat-schedule.db celerybeat.pid # SageMath parsed files diff --git a/.vscode/launch.json.template b/.vscode/launch.json.template index f5a7f0893b..bf04d31998 100644 --- a/.vscode/launch.json.template +++ b/.vscode/launch.json.template @@ -40,7 +40,7 @@ "-c", "1", "-Q", - "dataset,generation,mail,ops_trace", + "dataset,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,priority_pipeline,pipeline", "--loglevel", "INFO" ], diff --git a/api/.vscode/launch.json.example b/api/.vscode/launch.json.example index a52eca63d9..092c66e798 100644 --- a/api/.vscode/launch.json.example +++ b/api/.vscode/launch.json.example @@ -54,7 +54,7 @@ "--loglevel", "DEBUG", "-Q", - "dataset,generation,mail,ops_trace,app_deletion,workflow" + "dataset,priority_pipeline,pipeline,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,workflow,schedule_poller,schedule_executor,triggered_workflow_dispatcher,trigger_refresh_executor" ] } ] diff --git a/api/README.md b/api/README.md index ea6f547a0a..fc21158c7a 100644 --- a/api/README.md +++ b/api/README.md @@ -80,7 +80,7 @@ 1. If you need to handle and debug the async tasks (e.g. dataset importing and documents indexing), please start the worker service. ```bash -uv run celery -A app.celery worker -P gevent -c 2 --loglevel INFO -Q dataset,priority_pipeline,pipeline,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation +uv run celery -A app.celery worker -P gevent -c 2 --loglevel INFO -Q dataset,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,priority_pipeline,pipeline ``` Additionally, if you want to debug the celery scheduled tasks, you can run the following command in another terminal to start the beat service: diff --git a/api/controllers/console/extension.py b/api/controllers/console/extension.py index 4e1a8aeb3e..a1d36def0d 100644 --- a/api/controllers/console/extension.py +++ b/api/controllers/console/extension.py @@ -66,13 +66,7 @@ class APIBasedExtensionAPI(Resource): @account_initialization_required @marshal_with(api_based_extension_fields) def post(self): - parser = ( - reqparse.RequestParser() - .add_argument("name", type=str, required=True, location="json") - .add_argument("api_endpoint", type=str, required=True, location="json") - .add_argument("api_key", type=str, required=True, location="json") - ) - args = parser.parse_args() + args = api.payload _, current_tenant_id = current_account_with_tenant() extension_data = APIBasedExtension( @@ -125,13 +119,7 @@ class APIBasedExtensionDetailAPI(Resource): extension_data_from_db = APIBasedExtensionService.get_with_tenant_id(current_tenant_id, api_based_extension_id) - parser = ( - reqparse.RequestParser() - .add_argument("name", type=str, required=True, location="json") - .add_argument("api_endpoint", type=str, required=True, location="json") - .add_argument("api_key", type=str, required=True, location="json") - ) - args = parser.parse_args() + args = api.payload extension_data_from_db.name = args["name"] extension_data_from_db.api_endpoint = args["api_endpoint"] diff --git a/api/core/workflow/nodes/document_extractor/node.py b/api/core/workflow/nodes/document_extractor/node.py index cd5f50aaab..12cd7e2bd9 100644 --- a/api/core/workflow/nodes/document_extractor/node.py +++ b/api/core/workflow/nodes/document_extractor/node.py @@ -171,6 +171,7 @@ def _extract_text_by_file_extension(*, file_content: bytes, file_extension: str) ".txt" | ".markdown" | ".md" + | ".mdx" | ".html" | ".htm" | ".xml" diff --git a/api/docker/entrypoint.sh b/api/docker/entrypoint.sh index dfaaf712a5..d7b30648fa 100755 --- a/api/docker/entrypoint.sh +++ b/api/docker/entrypoint.sh @@ -34,10 +34,10 @@ if [[ "${MODE}" == "worker" ]]; then if [[ -z "${CELERY_QUEUES}" ]]; then if [[ "${EDITION}" == "CLOUD" ]]; then # Cloud edition: separate queues for dataset and trigger tasks - DEFAULT_QUEUES="dataset,pipeline,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,workflow_professional,workflow_team,workflow_sandbox,schedule_poller,schedule_executor,triggered_workflow_dispatcher,trigger_refresh_executor" + DEFAULT_QUEUES="dataset,priority_pipeline,pipeline,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,workflow_professional,workflow_team,workflow_sandbox,schedule_poller,schedule_executor,triggered_workflow_dispatcher,trigger_refresh_executor" else # Community edition (SELF_HOSTED): dataset, pipeline and workflow have separate queues - DEFAULT_QUEUES="dataset,pipeline,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,workflow,schedule_poller,schedule_executor,triggered_workflow_dispatcher,trigger_refresh_executor" + DEFAULT_QUEUES="dataset,priority_pipeline,pipeline,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,workflow,schedule_poller,schedule_executor,triggered_workflow_dispatcher,trigger_refresh_executor" fi else DEFAULT_QUEUES="${CELERY_QUEUES}" diff --git a/api/tests/unit_tests/core/app/apps/common/test_workflow_response_converter_truncation.py b/api/tests/unit_tests/core/app/apps/common/test_workflow_response_converter_truncation.py index 964d62be1f..1c9f577a50 100644 --- a/api/tests/unit_tests/core/app/apps/common/test_workflow_response_converter_truncation.py +++ b/api/tests/unit_tests/core/app/apps/common/test_workflow_response_converter_truncation.py @@ -360,7 +360,7 @@ class TestWorkflowResponseConverterServiceApiTruncation: app_id="test_app_id", app_config=app_config, tenant_id="test_tenant", - app_mode="workflow", + app_mode=AppMode.WORKFLOW, invoke_from=invoke_from, inputs={"test_input": "test_value"}, user_id="test_user_id", diff --git a/dev/start-worker b/dev/start-worker index 0c9b55a6fb..e5e103692f 100755 --- a/dev/start-worker +++ b/dev/start-worker @@ -26,12 +26,16 @@ show_help() { echo " workflow_sandbox - Sandbox tier workflows (cloud edition)" echo " schedule_poller - Schedule polling tasks" echo " schedule_executor - Schedule execution tasks" - echo " generation - Content generation tasks" echo " mail - Email notifications" echo " ops_trace - Operations tracing" echo " app_deletion - Application cleanup" echo " plugin - Plugin operations" echo " workflow_storage - Workflow storage tasks" + echo " conversation - Conversation tasks" + echo " priority_pipeline - High priority pipeline tasks" + echo " pipeline - Standard pipeline tasks" + echo " triggered_workflow_dispatcher - Trigger dispatcher tasks" + echo " trigger_refresh_executor - Trigger refresh tasks" } # Parse command line arguments @@ -81,10 +85,10 @@ if [[ -z "${QUEUES}" ]]; then # Configure queues based on edition if [[ "${EDITION}" == "CLOUD" ]]; then # Cloud edition: separate queues for dataset and trigger tasks - QUEUES="dataset,generation,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,workflow_professional,workflow_team,workflow_sandbox,schedule_poller,schedule_executor,priority_pipeline,pipeline" + QUEUES="dataset,priority_pipeline,pipeline,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,workflow_professional,workflow_team,workflow_sandbox,schedule_poller,schedule_executor,triggered_workflow_dispatcher,trigger_refresh_executor" else # Community edition (SELF_HOSTED): dataset and workflow have separate queues - QUEUES="dataset,generation,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,workflow,schedule_poller,schedule_executor,priority_pipeline,pipeline" + QUEUES="dataset,priority_pipeline,pipeline,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,workflow,schedule_poller,schedule_executor,triggered_workflow_dispatcher,trigger_refresh_executor" fi echo "No queues specified, using edition-based defaults: ${QUEUES}" diff --git a/docker/.env.example b/docker/.env.example index afb49e5c3b..fb815ea160 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -1258,6 +1258,7 @@ MARKETPLACE_ENABLED=true MARKETPLACE_API_URL=https://marketplace.dify.ai FORCE_VERIFYING_SIGNATURE=true +ENFORCE_LANGGENIUS_PLUGIN_SIGNATURES=true PLUGIN_STDIO_BUFFER_SIZE=1024 PLUGIN_STDIO_MAX_BUFFER_SIZE=5242880 diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index b0f98adaff..81f2a5e625 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -548,6 +548,7 @@ x-shared-env: &shared-api-worker-env MARKETPLACE_ENABLED: ${MARKETPLACE_ENABLED:-true} MARKETPLACE_API_URL: ${MARKETPLACE_API_URL:-https://marketplace.dify.ai} FORCE_VERIFYING_SIGNATURE: ${FORCE_VERIFYING_SIGNATURE:-true} + ENFORCE_LANGGENIUS_PLUGIN_SIGNATURES: ${ENFORCE_LANGGENIUS_PLUGIN_SIGNATURES:-true} PLUGIN_STDIO_BUFFER_SIZE: ${PLUGIN_STDIO_BUFFER_SIZE:-1024} PLUGIN_STDIO_MAX_BUFFER_SIZE: ${PLUGIN_STDIO_MAX_BUFFER_SIZE:-5242880} PLUGIN_PYTHON_ENV_INIT_TIMEOUT: ${PLUGIN_PYTHON_ENV_INIT_TIMEOUT:-120}