Merge branch 'langgenius:main' into feature/tool-invocation-add-inputs-param

.github/workflows/autofix.yml

@@ -28,6 +28,11 @@ jobs:
           # Format code
           uv run ruff format ..
 
+      - name: count migration progress
+        run: |
+          cd api
+          ./cnt_base.sh
+
       - name: ast-grep
         run: |
           uvx --from ast-grep-cli sg --pattern 'db.session.query($WHATEVER).filter($HERE)' --rewrite 'db.session.query($WHATEVER).where($HERE)' -l py --update-all

api/cnt_base.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euxo pipefail
+
+for pattern in "Base" "TypeBase"; do
+    printf "%s " "$pattern"
+    grep "($pattern):" -r --include='*.py' --exclude-dir=".venv" --exclude-dir="tests" . | wc -l
+done

api/models/dataset.py

@@ -225,7 +225,7 @@ class Dataset(Base):
                 ExternalKnowledgeApis.id == external_knowledge_binding.external_knowledge_api_id
             )
         )
-        if not external_knowledge_api:
+        if external_knowledge_api is None or external_knowledge_api.settings is None:
             return None
         return {
             "external_knowledge_id": external_knowledge_binding.external_knowledge_id,
@@ -945,18 +945,20 @@ class DatasetQuery(Base):
     created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=sa.func.current_timestamp())
 
 
-class DatasetKeywordTable(Base):
+class DatasetKeywordTable(TypeBase):
     __tablename__ = "dataset_keyword_tables"
     __table_args__ = (
         sa.PrimaryKeyConstraint("id", name="dataset_keyword_table_pkey"),
         sa.Index("dataset_keyword_table_dataset_id_idx", "dataset_id"),
     )
 
-    id = mapped_column(StringUUID, primary_key=True, server_default=sa.text("uuid_generate_v4()"))
+    id: Mapped[str] = mapped_column(
-    dataset_id = mapped_column(StringUUID, nullable=False, unique=True)
+        StringUUID, primary_key=True, server_default=sa.text("uuid_generate_v4()"), init=False
-    keyword_table = mapped_column(sa.Text, nullable=False)
+    )
-    data_source_type = mapped_column(
+    dataset_id: Mapped[str] = mapped_column(StringUUID, nullable=False, unique=True)
-        String(255), nullable=False, server_default=sa.text("'database'::character varying")
+    keyword_table: Mapped[str] = mapped_column(sa.Text, nullable=False)
+    data_source_type: Mapped[str] = mapped_column(
+        String(255), nullable=False, server_default=sa.text("'database'::character varying"), default="database"
     )
 
     @property
@@ -1054,19 +1056,23 @@ class TidbAuthBinding(Base):
     created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
 
 
-class Whitelist(Base):
+class Whitelist(TypeBase):
     __tablename__ = "whitelists"
     __table_args__ = (
         sa.PrimaryKeyConstraint("id", name="whitelists_pkey"),
         sa.Index("whitelists_tenant_idx", "tenant_id"),
     )
-    id = mapped_column(StringUUID, primary_key=True, server_default=sa.text("uuid_generate_v4()"))
+    id: Mapped[str] = mapped_column(
-    tenant_id = mapped_column(StringUUID, nullable=True)
+        StringUUID, primary_key=True, server_default=sa.text("uuid_generate_v4()"), init=False
+    )
+    tenant_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True)
     category: Mapped[str] = mapped_column(String(255), nullable=False)
-    created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime, nullable=False, server_default=func.current_timestamp(), init=False
+    )
 
 
-class DatasetPermission(Base):
+class DatasetPermission(TypeBase):
     __tablename__ = "dataset_permissions"
     __table_args__ = (
         sa.PrimaryKeyConstraint("id", name="dataset_permission_pkey"),
@@ -1075,15 +1081,21 @@ class DatasetPermission(Base):
         sa.Index("idx_dataset_permissions_tenant_id", "tenant_id"),
     )
 
-    id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"), primary_key=True)
+    id: Mapped[str] = mapped_column(
-    dataset_id = mapped_column(StringUUID, nullable=False)
+        StringUUID, server_default=sa.text("uuid_generate_v4()"), primary_key=True, init=False
-    account_id = mapped_column(StringUUID, nullable=False)
+    )
-    tenant_id = mapped_column(StringUUID, nullable=False)
+    dataset_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
-    has_permission: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.text("true"))
+    account_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
-    created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
+    tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
+    has_permission: Mapped[bool] = mapped_column(
+        sa.Boolean, nullable=False, server_default=sa.text("true"), default=True
+    )
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime, nullable=False, server_default=func.current_timestamp(), init=False
+    )
 
 
-class ExternalKnowledgeApis(Base):
+class ExternalKnowledgeApis(TypeBase):
     __tablename__ = "external_knowledge_apis"
     __table_args__ = (
         sa.PrimaryKeyConstraint("id", name="external_knowledge_apis_pkey"),
@@ -1091,16 +1103,20 @@ class ExternalKnowledgeApis(Base):
         sa.Index("external_knowledge_apis_name_idx", "name"),
     )
 
-    id = mapped_column(StringUUID, nullable=False, server_default=sa.text("uuid_generate_v4()"))
+    id: Mapped[str] = mapped_column(
+        StringUUID, nullable=False, server_default=sa.text("uuid_generate_v4()"), init=False
+    )
     name: Mapped[str] = mapped_column(String(255), nullable=False)
     description: Mapped[str] = mapped_column(String(255), nullable=False)
-    tenant_id = mapped_column(StringUUID, nullable=False)
+    tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
-    settings = mapped_column(sa.Text, nullable=True)
+    settings: Mapped[str | None] = mapped_column(sa.Text, nullable=True)
-    created_by = mapped_column(StringUUID, nullable=False)
+    created_by: Mapped[str] = mapped_column(StringUUID, nullable=False)
-    created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
+    created_at: Mapped[datetime] = mapped_column(
-    updated_by = mapped_column(StringUUID, nullable=True)
+        DateTime, nullable=False, server_default=func.current_timestamp(), init=False
+    )
+    updated_by: Mapped[str | None] = mapped_column(StringUUID, nullable=True)
     updated_at: Mapped[datetime] = mapped_column(
-        DateTime, nullable=False, server_default=func.current_timestamp(), onupdate=func.current_timestamp()
+        DateTime, nullable=False, server_default=func.current_timestamp(), onupdate=func.current_timestamp(), init=False
     )
 
     def to_dict(self) -> dict[str, Any]:
@@ -1178,7 +1194,7 @@ class DatasetAutoDisableLog(Base):
     )
 
 
-class RateLimitLog(Base):
+class RateLimitLog(TypeBase):
     __tablename__ = "rate_limit_logs"
     __table_args__ = (
         sa.PrimaryKeyConstraint("id", name="rate_limit_log_pkey"),
@@ -1186,12 +1202,12 @@ class RateLimitLog(Base):
         sa.Index("rate_limit_log_operation_idx", "operation"),
     )
 
-    id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
+    id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"), init=False)
-    tenant_id = mapped_column(StringUUID, nullable=False)
+    tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
     subscription_plan: Mapped[str] = mapped_column(String(255), nullable=False)
     operation: Mapped[str] = mapped_column(String(255), nullable=False)
     created_at: Mapped[datetime] = mapped_column(
-        DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)")
+        DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)"), init=False
     )
 
 

api/models/trigger.py

@@ -14,7 +14,7 @@ from core.trigger.entities.api_entities import TriggerProviderSubscriptionApiEnt
 from core.trigger.entities.entities import Subscription
 from core.trigger.utils.endpoint import generate_plugin_trigger_endpoint_url, generate_webhook_trigger_endpoint
 from libs.datetime_utils import naive_utc_now
-from models.base import Base
+from models.base import Base, TypeBase
 from models.engine import db
 from models.enums import AppTriggerStatus, AppTriggerType, CreatorUserRole, WorkflowTriggerStatus
 from models.model import Account
@@ -399,7 +399,7 @@ class AppTrigger(Base):
     )
 
 
-class WorkflowSchedulePlan(Base):
+class WorkflowSchedulePlan(TypeBase):
     """
     Workflow Schedule Configuration
 
@@ -425,7 +425,7 @@ class WorkflowSchedulePlan(Base):
         sa.Index("workflow_schedule_plan_next_idx", "next_run_at"),
     )
 
-    id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuidv7()"))
+    id: Mapped[str] = mapped_column(StringUUID, primary_key=True, server_default=sa.text("uuidv7()"), init=False)
     app_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
     node_id: Mapped[str] = mapped_column(String(64), nullable=False)
     tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
@@ -436,9 +436,11 @@ class WorkflowSchedulePlan(Base):
 
     # Schedule control
     next_run_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
-    created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime, nullable=False, server_default=func.current_timestamp(), init=False
+    )
     updated_at: Mapped[datetime] = mapped_column(
-        DateTime, nullable=False, server_default=func.current_timestamp(), onupdate=func.current_timestamp()
+        DateTime, nullable=False, server_default=func.current_timestamp(), onupdate=func.current_timestamp(), init=False
     )
 
     def to_dict(self) -> dict[str, Any]:

api/services/external_knowledge_service.py

@@ -62,7 +62,7 @@ class ExternalDatasetService:
             tenant_id=tenant_id,
             created_by=user_id,
             updated_by=user_id,
-            name=args.get("name"),
+            name=str(args.get("name")),
             description=args.get("description", ""),
             settings=json.dumps(args.get("settings"), ensure_ascii=False),
         )
@@ -163,7 +163,7 @@ class ExternalDatasetService:
         external_knowledge_api = (
             db.session.query(ExternalKnowledgeApis).filter_by(id=external_knowledge_api_id, tenant_id=tenant_id).first()
         )
-        if external_knowledge_api is None:
+        if external_knowledge_api is None or external_knowledge_api.settings is None:
             raise ValueError("api template not found")
         settings = json.loads(external_knowledge_api.settings)
         for setting in settings:
@@ -290,7 +290,7 @@ class ExternalDatasetService:
             .filter_by(id=external_knowledge_binding.external_knowledge_api_id)
             .first()
         )
-        if not external_knowledge_api:
+        if external_knowledge_api is None or external_knowledge_api.settings is None:
             raise ValueError("external api template not found")
 
         settings = json.loads(external_knowledge_api.settings)

api/tasks/async_workflow_tasks.py

@@ -13,13 +13,13 @@ from sqlalchemy import select
 from sqlalchemy.orm import Session, sessionmaker
 
 from configs import dify_config
-from core.app.apps.workflow.app_generator import WorkflowAppGenerator
+from core.app.apps.workflow.app_generator import SKIP_PREPARE_USER_INPUTS_KEY, WorkflowAppGenerator
 from core.app.entities.app_invoke_entities import InvokeFrom
 from core.app.layers.timeslice_layer import TimeSliceLayer
 from core.app.layers.trigger_post_layer import TriggerPostLayer
 from extensions.ext_database import db
 from models.account import Account
-from models.enums import CreatorUserRole, WorkflowTriggerStatus
+from models.enums import AppTriggerType, CreatorUserRole, WorkflowTriggerStatus
 from models.model import App, EndUser, Tenant
 from models.trigger import WorkflowTriggerLog
 from models.workflow import Workflow
@@ -81,6 +81,19 @@ def execute_workflow_sandbox(task_data_dict: dict[str, Any]):
     )
 
 
+def _build_generator_args(trigger_data: TriggerData) -> dict[str, Any]:
+    """Build args passed into WorkflowAppGenerator.generate for Celery executions."""
+    args: dict[str, Any] = {
+        "inputs": dict(trigger_data.inputs),
+        "files": list(trigger_data.files),
+    }
+
+    if trigger_data.trigger_type == AppTriggerType.TRIGGER_WEBHOOK:
+        args[SKIP_PREPARE_USER_INPUTS_KEY] = True  # Webhooks already provide structured inputs
+
+    return args
+
+
 def _execute_workflow_common(
     task_data: WorkflowTaskData,
     cfs_plan_scheduler: AsyncWorkflowCFSPlanScheduler,
@@ -128,7 +141,7 @@ def _execute_workflow_common(
             generator = WorkflowAppGenerator()
 
             # Prepare args matching AppGenerateService.generate format
-            args: dict[str, Any] = {"inputs": dict(trigger_data.inputs), "files": list(trigger_data.files)}
+            args = _build_generator_args(trigger_data)
 
             # If workflow_id was specified, add it to args
             if trigger_data.workflow_id:

api/tests/unit_tests/tasks/test_async_workflow_tasks.py

@@ -0,0 +1,37 @@
+from core.app.apps.workflow.app_generator import SKIP_PREPARE_USER_INPUTS_KEY
+from models.enums import AppTriggerType, WorkflowRunTriggeredFrom
+from services.workflow.entities import TriggerData, WebhookTriggerData
+from tasks import async_workflow_tasks
+
+
+def test_build_generator_args_sets_skip_flag_for_webhook():
+    trigger_data = WebhookTriggerData(
+        app_id="app",
+        tenant_id="tenant",
+        workflow_id="workflow",
+        root_node_id="node",
+        inputs={"webhook_data": {"body": {"foo": "bar"}}},
+    )
+
+    args = async_workflow_tasks._build_generator_args(trigger_data)
+
+    assert args[SKIP_PREPARE_USER_INPUTS_KEY] is True
+    assert args["inputs"]["webhook_data"]["body"]["foo"] == "bar"
+
+
+def test_build_generator_args_keeps_validation_for_other_triggers():
+    trigger_data = TriggerData(
+        app_id="app",
+        tenant_id="tenant",
+        workflow_id="workflow",
+        root_node_id="node",
+        inputs={"foo": "bar"},
+        files=[],
+        trigger_type=AppTriggerType.TRIGGER_SCHEDULE,
+        trigger_from=WorkflowRunTriggeredFrom.SCHEDULE,
+    )
+
+    args = async_workflow_tasks._build_generator_args(trigger_data)
+
+    assert SKIP_PREPARE_USER_INPUTS_KEY not in args
+    assert args["inputs"] == {"foo": "bar"}