-Dify is an open-source LLM app development platform. Its intuitive interface combines agentic AI workflow, RAG pipeline, agent capabilities, model management, observability features and more, letting you quickly go from prototype to production.
+Dify is an open-source LLM app development platform. Its intuitive interface combines agentic AI workflow, RAG pipeline, agent capabilities, model management, observability features, and more, allowing you to quickly move from prototype to production.
## Quick start
@@ -188,7 +188,7 @@ All of Dify's offerings come with corresponding APIs, so you could effortlessly
- **Dify for enterprise / organizations**
We provide additional enterprise-centric features. [Log your questions for us through this chatbot](https://udify.app/chat/22L1zSxg6yW1cWQg) or [send us an email](mailto:business@dify.ai?subject=[GitHub]Business%20License%20Inquiry) to discuss enterprise needs.
- > For startups and small businesses using AWS, check out [Dify Premium on AWS Marketplace](https://aws.amazon.com/marketplace/pp/prodview-t22mebxzwjhu6) and deploy it to your own AWS VPC with one-click. It's an affordable AMI offering with the option to create apps with custom logo and branding.
+ > For startups and small businesses using AWS, check out [Dify Premium on AWS Marketplace](https://aws.amazon.com/marketplace/pp/prodview-t22mebxzwjhu6) and deploy it to your own AWS VPC with one click. It's an affordable AMI offering with the option to create apps with custom logo and branding.
## Staying ahead
@@ -233,7 +233,7 @@ Deploy Dify to AWS with [CDK](https://aws.amazon.com/cdk/)
For those who'd like to contribute code, see our [Contribution Guide](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md).
At the same time, please consider supporting Dify by sharing it on social media and at events and conferences.
-> We are looking for contributors to help with translating Dify to languages other than Mandarin or English. If you are interested in helping, please see the [i18n README](https://github.com/langgenius/dify/blob/main/web/i18n/README.md) for more information, and leave us a comment in the `global-users` channel of our [Discord Community Server](https://discord.gg/8Tpq4AcN9c).
+> We are looking for contributors to help translate Dify into languages other than Mandarin or English. If you are interested in helping, please see the [i18n README](https://github.com/langgenius/dify/blob/main/web/i18n/README.md) for more information, and leave us a comment in the `global-users` channel of our [Discord Community Server](https://discord.gg/8Tpq4AcN9c).
## Community & contact
diff --git a/api/.env.example b/api/.env.example
index b5820fcdc2..2cc6410cdd 100644
--- a/api/.env.example
+++ b/api/.env.example
@@ -297,6 +297,7 @@ LINDORM_URL=http://ld-*******************-proxy-search-pub.lindorm.aliyuncs.com:
LINDORM_USERNAME=admin
LINDORM_PASSWORD=admin
USING_UGC_INDEX=False
+LINDORM_QUERY_TIMEOUT=1
# OceanBase Vector configuration
OCEANBASE_VECTOR_HOST=127.0.0.1
diff --git a/api/app_factory.py b/api/app_factory.py
index 9648d770ab..586f2ded9e 100644
--- a/api/app_factory.py
+++ b/api/app_factory.py
@@ -52,7 +52,6 @@ def initialize_extensions(app: DifyApp):
ext_mail,
ext_migrate,
ext_otel,
- ext_otel_patch,
ext_proxy_fix,
ext_redis,
ext_repositories,
@@ -85,7 +84,6 @@ def initialize_extensions(app: DifyApp):
ext_proxy_fix,
ext_blueprints,
ext_commands,
- ext_otel_patch, # Apply patch before initializing OpenTelemetry
ext_otel,
]
for ext in extensions:
diff --git a/api/commands.py b/api/commands.py
index e70d6e0b49..99a3211baf 100644
--- a/api/commands.py
+++ b/api/commands.py
@@ -17,6 +17,7 @@ from core.rag.models.document import Document
from events.app_event import app_was_created
from extensions.ext_database import db
from extensions.ext_redis import redis_client
+from extensions.ext_storage import storage
from libs.helper import email as email_validate
from libs.password import hash_password, password_pattern, valid_password
from libs.rsa import generate_key_pair
@@ -271,6 +272,7 @@ def migrate_knowledge_vector_database():
upper_collection_vector_types = {
VectorType.MILVUS,
VectorType.PGVECTOR,
+ VectorType.VASTBASE,
VectorType.RELYT,
VectorType.WEAVIATE,
VectorType.ORACLE,
@@ -814,3 +816,331 @@ def clear_free_plan_tenant_expired_logs(days: int, batch: int, tenant_ids: list[
ClearFreePlanTenantExpiredLogs.process(days, batch, tenant_ids)
click.echo(click.style("Clear free plan tenant expired logs completed.", fg="green"))
+
+
+@click.option("-f", "--force", is_flag=True, help="Skip user confirmation and force the command to execute.")
+@click.command("clear-orphaned-file-records", help="Clear orphaned file records.")
+def clear_orphaned_file_records(force: bool):
+ """
+ Clear orphaned file records in the database.
+ """
+
+ # define tables and columns to process
+ files_tables = [
+ {"table": "upload_files", "id_column": "id", "key_column": "key"},
+ {"table": "tool_files", "id_column": "id", "key_column": "file_key"},
+ ]
+ ids_tables = [
+ {"type": "uuid", "table": "message_files", "column": "upload_file_id"},
+ {"type": "text", "table": "documents", "column": "data_source_info"},
+ {"type": "text", "table": "document_segments", "column": "content"},
+ {"type": "text", "table": "messages", "column": "answer"},
+ {"type": "text", "table": "workflow_node_executions", "column": "inputs"},
+ {"type": "text", "table": "workflow_node_executions", "column": "process_data"},
+ {"type": "text", "table": "workflow_node_executions", "column": "outputs"},
+ {"type": "text", "table": "conversations", "column": "introduction"},
+ {"type": "text", "table": "conversations", "column": "system_instruction"},
+ {"type": "json", "table": "messages", "column": "inputs"},
+ {"type": "json", "table": "messages", "column": "message"},
+ ]
+
+ # notify user and ask for confirmation
+ click.echo(
+ click.style(
+ "This command will first find and delete orphaned file records from the message_files table,", fg="yellow"
+ )
+ )
+ click.echo(
+ click.style(
+ "and then it will find and delete orphaned file records in the following tables:",
+ fg="yellow",
+ )
+ )
+ for files_table in files_tables:
+ click.echo(click.style(f"- {files_table['table']}", fg="yellow"))
+ click.echo(
+ click.style("The following tables and columns will be scanned to find orphaned file records:", fg="yellow")
+ )
+ for ids_table in ids_tables:
+ click.echo(click.style(f"- {ids_table['table']} ({ids_table['column']})", fg="yellow"))
+ click.echo("")
+
+ click.echo(click.style("!!! USE WITH CAUTION !!!", fg="red"))
+ click.echo(
+ click.style(
+ (
+ "Since not all patterns have been fully tested, "
+ "please note that this command may delete unintended file records."
+ ),
+ fg="yellow",
+ )
+ )
+ click.echo(
+ click.style("This cannot be undone. Please make sure to back up your database before proceeding.", fg="yellow")
+ )
+ click.echo(
+ click.style(
+ (
+ "It is also recommended to run this during the maintenance window, "
+ "as this may cause high load on your instance."
+ ),
+ fg="yellow",
+ )
+ )
+ if not force:
+ click.confirm("Do you want to proceed?", abort=True)
+
+ # start the cleanup process
+ click.echo(click.style("Starting orphaned file records cleanup.", fg="white"))
+
+ # clean up the orphaned records in the message_files table where message_id doesn't exist in messages table
+ try:
+ click.echo(
+ click.style("- Listing message_files records where message_id doesn't exist in messages table", fg="white")
+ )
+ query = (
+ "SELECT mf.id, mf.message_id "
+ "FROM message_files mf LEFT JOIN messages m ON mf.message_id = m.id "
+ "WHERE m.id IS NULL"
+ )
+ orphaned_message_files = []
+ with db.engine.begin() as conn:
+ rs = conn.execute(db.text(query))
+ for i in rs:
+ orphaned_message_files.append({"id": str(i[0]), "message_id": str(i[1])})
+
+ if orphaned_message_files:
+ click.echo(click.style(f"Found {len(orphaned_message_files)} orphaned message_files records:", fg="white"))
+ for record in orphaned_message_files:
+ click.echo(click.style(f" - id: {record['id']}, message_id: {record['message_id']}", fg="black"))
+
+ if not force:
+ click.confirm(
+ (
+ f"Do you want to proceed "
+ f"to delete all {len(orphaned_message_files)} orphaned message_files records?"
+ ),
+ abort=True,
+ )
+
+ click.echo(click.style("- Deleting orphaned message_files records", fg="white"))
+ query = "DELETE FROM message_files WHERE id IN :ids"
+ with db.engine.begin() as conn:
+ conn.execute(db.text(query), {"ids": tuple([record["id"] for record in orphaned_message_files])})
+ click.echo(
+ click.style(f"Removed {len(orphaned_message_files)} orphaned message_files records.", fg="green")
+ )
+ else:
+ click.echo(click.style("No orphaned message_files records found. There is nothing to delete.", fg="green"))
+ except Exception as e:
+ click.echo(click.style(f"Error deleting orphaned message_files records: {str(e)}", fg="red"))
+
+ # clean up the orphaned records in the rest of the *_files tables
+ try:
+ # fetch file id and keys from each table
+ all_files_in_tables = []
+ for files_table in files_tables:
+ click.echo(click.style(f"- Listing file records in table {files_table['table']}", fg="white"))
+ query = f"SELECT {files_table['id_column']}, {files_table['key_column']} FROM {files_table['table']}"
+ with db.engine.begin() as conn:
+ rs = conn.execute(db.text(query))
+ for i in rs:
+ all_files_in_tables.append({"table": files_table["table"], "id": str(i[0]), "key": i[1]})
+ click.echo(click.style(f"Found {len(all_files_in_tables)} files in tables.", fg="white"))
+
+ # fetch referred table and columns
+ guid_regexp = "[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}"
+ all_ids_in_tables = []
+ for ids_table in ids_tables:
+ query = ""
+ if ids_table["type"] == "uuid":
+ click.echo(
+ click.style(
+ f"- Listing file ids in column {ids_table['column']} in table {ids_table['table']}", fg="white"
+ )
+ )
+ query = (
+ f"SELECT {ids_table['column']} FROM {ids_table['table']} WHERE {ids_table['column']} IS NOT NULL"
+ )
+ with db.engine.begin() as conn:
+ rs = conn.execute(db.text(query))
+ for i in rs:
+ all_ids_in_tables.append({"table": ids_table["table"], "id": str(i[0])})
+ elif ids_table["type"] == "text":
+ click.echo(
+ click.style(
+ f"- Listing file-id-like strings in column {ids_table['column']} in table {ids_table['table']}",
+ fg="white",
+ )
+ )
+ query = (
+ f"SELECT regexp_matches({ids_table['column']}, '{guid_regexp}', 'g') AS extracted_id "
+ f"FROM {ids_table['table']}"
+ )
+ with db.engine.begin() as conn:
+ rs = conn.execute(db.text(query))
+ for i in rs:
+ for j in i[0]:
+ all_ids_in_tables.append({"table": ids_table["table"], "id": j})
+ elif ids_table["type"] == "json":
+ click.echo(
+ click.style(
+ (
+ f"- Listing file-id-like JSON string in column {ids_table['column']} "
+ f"in table {ids_table['table']}"
+ ),
+ fg="white",
+ )
+ )
+ query = (
+ f"SELECT regexp_matches({ids_table['column']}::text, '{guid_regexp}', 'g') AS extracted_id "
+ f"FROM {ids_table['table']}"
+ )
+ with db.engine.begin() as conn:
+ rs = conn.execute(db.text(query))
+ for i in rs:
+ for j in i[0]:
+ all_ids_in_tables.append({"table": ids_table["table"], "id": j})
+ click.echo(click.style(f"Found {len(all_ids_in_tables)} file ids in tables.", fg="white"))
+
+ except Exception as e:
+ click.echo(click.style(f"Error fetching keys: {str(e)}", fg="red"))
+ return
+
+ # find orphaned files
+ all_files = [file["id"] for file in all_files_in_tables]
+ all_ids = [file["id"] for file in all_ids_in_tables]
+ orphaned_files = list(set(all_files) - set(all_ids))
+ if not orphaned_files:
+ click.echo(click.style("No orphaned file records found. There is nothing to delete.", fg="green"))
+ return
+ click.echo(click.style(f"Found {len(orphaned_files)} orphaned file records.", fg="white"))
+ for file in orphaned_files:
+ click.echo(click.style(f"- orphaned file id: {file}", fg="black"))
+ if not force:
+ click.confirm(f"Do you want to proceed to delete all {len(orphaned_files)} orphaned file records?", abort=True)
+
+ # delete orphaned records for each file
+ try:
+ for files_table in files_tables:
+ click.echo(click.style(f"- Deleting orphaned file records in table {files_table['table']}", fg="white"))
+ query = f"DELETE FROM {files_table['table']} WHERE {files_table['id_column']} IN :ids"
+ with db.engine.begin() as conn:
+ conn.execute(db.text(query), {"ids": tuple(orphaned_files)})
+ except Exception as e:
+ click.echo(click.style(f"Error deleting orphaned file records: {str(e)}", fg="red"))
+ return
+ click.echo(click.style(f"Removed {len(orphaned_files)} orphaned file records.", fg="green"))
+
+
+@click.option("-f", "--force", is_flag=True, help="Skip user confirmation and force the command to execute.")
+@click.command("remove-orphaned-files-on-storage", help="Remove orphaned files on the storage.")
+def remove_orphaned_files_on_storage(force: bool):
+ """
+ Remove orphaned files on the storage.
+ """
+
+ # define tables and columns to process
+ files_tables = [
+ {"table": "upload_files", "key_column": "key"},
+ {"table": "tool_files", "key_column": "file_key"},
+ ]
+ storage_paths = ["image_files", "tools", "upload_files"]
+
+ # notify user and ask for confirmation
+ click.echo(click.style("This command will find and remove orphaned files on the storage,", fg="yellow"))
+ click.echo(
+ click.style("by comparing the files on the storage with the records in the following tables:", fg="yellow")
+ )
+ for files_table in files_tables:
+ click.echo(click.style(f"- {files_table['table']}", fg="yellow"))
+ click.echo(click.style("The following paths on the storage will be scanned to find orphaned files:", fg="yellow"))
+ for storage_path in storage_paths:
+ click.echo(click.style(f"- {storage_path}", fg="yellow"))
+ click.echo("")
+
+ click.echo(click.style("!!! USE WITH CAUTION !!!", fg="red"))
+ click.echo(
+ click.style(
+ "Currently, this command will work only for opendal based storage (STORAGE_TYPE=opendal).", fg="yellow"
+ )
+ )
+ click.echo(
+ click.style(
+ "Since not all patterns have been fully tested, please note that this command may delete unintended files.",
+ fg="yellow",
+ )
+ )
+ click.echo(
+ click.style("This cannot be undone. Please make sure to back up your storage before proceeding.", fg="yellow")
+ )
+ click.echo(
+ click.style(
+ (
+ "It is also recommended to run this during the maintenance window, "
+ "as this may cause high load on your instance."
+ ),
+ fg="yellow",
+ )
+ )
+ if not force:
+ click.confirm("Do you want to proceed?", abort=True)
+
+ # start the cleanup process
+ click.echo(click.style("Starting orphaned files cleanup.", fg="white"))
+
+ # fetch file id and keys from each table
+ all_files_in_tables = []
+ try:
+ for files_table in files_tables:
+ click.echo(click.style(f"- Listing files from table {files_table['table']}", fg="white"))
+ query = f"SELECT {files_table['key_column']} FROM {files_table['table']}"
+ with db.engine.begin() as conn:
+ rs = conn.execute(db.text(query))
+ for i in rs:
+ all_files_in_tables.append(str(i[0]))
+ click.echo(click.style(f"Found {len(all_files_in_tables)} files in tables.", fg="white"))
+ except Exception as e:
+ click.echo(click.style(f"Error fetching keys: {str(e)}", fg="red"))
+
+ all_files_on_storage = []
+ for storage_path in storage_paths:
+ try:
+ click.echo(click.style(f"- Scanning files on storage path {storage_path}", fg="white"))
+ files = storage.scan(path=storage_path, files=True, directories=False)
+ all_files_on_storage.extend(files)
+ except FileNotFoundError as e:
+ click.echo(click.style(f" -> Skipping path {storage_path} as it does not exist.", fg="yellow"))
+ continue
+ except Exception as e:
+ click.echo(click.style(f" -> Error scanning files on storage path {storage_path}: {str(e)}", fg="red"))
+ continue
+ click.echo(click.style(f"Found {len(all_files_on_storage)} files on storage.", fg="white"))
+
+ # find orphaned files
+ orphaned_files = list(set(all_files_on_storage) - set(all_files_in_tables))
+ if not orphaned_files:
+ click.echo(click.style("No orphaned files found. There is nothing to remove.", fg="green"))
+ return
+ click.echo(click.style(f"Found {len(orphaned_files)} orphaned files.", fg="white"))
+ for file in orphaned_files:
+ click.echo(click.style(f"- orphaned file: {file}", fg="black"))
+ if not force:
+ click.confirm(f"Do you want to proceed to remove all {len(orphaned_files)} orphaned files?", abort=True)
+
+ # delete orphaned files
+ removed_files = 0
+ error_files = 0
+ for file in orphaned_files:
+ try:
+ storage.delete(file)
+ removed_files += 1
+ click.echo(click.style(f"- Removing orphaned file: {file}", fg="white"))
+ except Exception as e:
+ error_files += 1
+ click.echo(click.style(f"- Error deleting orphaned file {file}: {str(e)}", fg="red"))
+ continue
+ if error_files == 0:
+ click.echo(click.style(f"Removed {removed_files} orphaned files without errors.", fg="green"))
+ else:
+ click.echo(click.style(f"Removed {removed_files} orphaned files, with {error_files} errors.", fg="yellow"))
diff --git a/api/configs/middleware/__init__.py b/api/configs/middleware/__init__.py
index c2ad24094a..d285515998 100644
--- a/api/configs/middleware/__init__.py
+++ b/api/configs/middleware/__init__.py
@@ -39,6 +39,7 @@ from .vdb.tencent_vector_config import TencentVectorDBConfig
from .vdb.tidb_on_qdrant_config import TidbOnQdrantConfig
from .vdb.tidb_vector_config import TiDBVectorConfig
from .vdb.upstash_config import UpstashConfig
+from .vdb.vastbase_vector_config import VastbaseVectorConfig
from .vdb.vikingdb_config import VikingDBConfig
from .vdb.weaviate_config import WeaviateConfig
@@ -270,6 +271,7 @@ class MiddlewareConfig(
OpenSearchConfig,
OracleConfig,
PGVectorConfig,
+ VastbaseVectorConfig,
PGVectoRSConfig,
QdrantConfig,
RelytConfig,
diff --git a/api/configs/middleware/vdb/lindorm_config.py b/api/configs/middleware/vdb/lindorm_config.py
index 95e1d1cfca..e80e3f4a35 100644
--- a/api/configs/middleware/vdb/lindorm_config.py
+++ b/api/configs/middleware/vdb/lindorm_config.py
@@ -32,3 +32,4 @@ class LindormConfig(BaseSettings):
description="Using UGC index will store the same type of Index in a single index but can retrieve separately.",
default=False,
)
+ LINDORM_QUERY_TIMEOUT: Optional[float] = Field(description="The lindorm search request timeout (s)", default=2.0)
diff --git a/api/configs/middleware/vdb/opensearch_config.py b/api/configs/middleware/vdb/opensearch_config.py
index 81dde4c04d..96f478e9a6 100644
--- a/api/configs/middleware/vdb/opensearch_config.py
+++ b/api/configs/middleware/vdb/opensearch_config.py
@@ -1,4 +1,5 @@
-from typing import Optional
+import enum
+from typing import Literal, Optional
from pydantic import Field, PositiveInt
from pydantic_settings import BaseSettings
@@ -9,6 +10,14 @@ class OpenSearchConfig(BaseSettings):
Configuration settings for OpenSearch
"""
+ class AuthMethod(enum.StrEnum):
+ """
+ Authentication method for OpenSearch
+ """
+
+ BASIC = "basic"
+ AWS_MANAGED_IAM = "aws_managed_iam"
+
OPENSEARCH_HOST: Optional[str] = Field(
description="Hostname or IP address of the OpenSearch server (e.g., 'localhost' or 'opensearch.example.com')",
default=None,
@@ -19,6 +28,16 @@ class OpenSearchConfig(BaseSettings):
default=9200,
)
+ OPENSEARCH_SECURE: bool = Field(
+ description="Whether to use SSL/TLS encrypted connection for OpenSearch (True for HTTPS, False for HTTP)",
+ default=False,
+ )
+
+ OPENSEARCH_AUTH_METHOD: AuthMethod = Field(
+ description="Authentication method for OpenSearch connection (default is 'basic')",
+ default=AuthMethod.BASIC,
+ )
+
OPENSEARCH_USER: Optional[str] = Field(
description="Username for authenticating with OpenSearch",
default=None,
@@ -29,7 +48,11 @@ class OpenSearchConfig(BaseSettings):
default=None,
)
- OPENSEARCH_SECURE: bool = Field(
- description="Whether to use SSL/TLS encrypted connection for OpenSearch (True for HTTPS, False for HTTP)",
- default=False,
+ OPENSEARCH_AWS_REGION: Optional[str] = Field(
+ description="AWS region for OpenSearch (e.g. 'us-west-2')",
+ default=None,
+ )
+
+ OPENSEARCH_AWS_SERVICE: Optional[Literal["es", "aoss"]] = Field(
+ description="AWS service for OpenSearch (e.g. 'aoss' for OpenSearch Serverless)", default=None
)
diff --git a/api/configs/middleware/vdb/vastbase_vector_config.py b/api/configs/middleware/vdb/vastbase_vector_config.py
new file mode 100644
index 0000000000..816d6df90a
--- /dev/null
+++ b/api/configs/middleware/vdb/vastbase_vector_config.py
@@ -0,0 +1,45 @@
+from typing import Optional
+
+from pydantic import Field, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class VastbaseVectorConfig(BaseSettings):
+ """
+ Configuration settings for Vector (Vastbase with vector extension)
+ """
+
+ VASTBASE_HOST: Optional[str] = Field(
+ description="Hostname or IP address of the Vastbase server with Vector extension (e.g., 'localhost')",
+ default=None,
+ )
+
+ VASTBASE_PORT: PositiveInt = Field(
+ description="Port number on which the Vastbase server is listening (default is 5432)",
+ default=5432,
+ )
+
+ VASTBASE_USER: Optional[str] = Field(
+ description="Username for authenticating with the Vastbase database",
+ default=None,
+ )
+
+ VASTBASE_PASSWORD: Optional[str] = Field(
+ description="Password for authenticating with the Vastbase database",
+ default=None,
+ )
+
+ VASTBASE_DATABASE: Optional[str] = Field(
+ description="Name of the Vastbase database to connect to",
+ default=None,
+ )
+
+ VASTBASE_MIN_CONNECTION: PositiveInt = Field(
+ description="Min connection of the Vastbase database",
+ default=1,
+ )
+
+ VASTBASE_MAX_CONNECTION: PositiveInt = Field(
+ description="Max connection of the Vastbase database",
+ default=5,
+ )
diff --git a/api/configs/packaging/__init__.py b/api/configs/packaging/__init__.py
index c7aedc5b8a..c7960e1356 100644
--- a/api/configs/packaging/__init__.py
+++ b/api/configs/packaging/__init__.py
@@ -9,7 +9,7 @@ class PackagingInfo(BaseSettings):
CURRENT_VERSION: str = Field(
description="Dify version",
- default="1.2.0",
+ default="1.3.1",
)
COMMIT_SHA: str = Field(
diff --git a/api/constants/__init__.py b/api/constants/__init__.py
index 9162357466..a84de0a451 100644
--- a/api/constants/__init__.py
+++ b/api/constants/__init__.py
@@ -16,11 +16,25 @@ AUDIO_EXTENSIONS.extend([ext.upper() for ext in AUDIO_EXTENSIONS])
if dify_config.ETL_TYPE == "Unstructured":
- DOCUMENT_EXTENSIONS = ["txt", "markdown", "md", "mdx", "pdf", "html", "htm", "xlsx", "xls"]
+ DOCUMENT_EXTENSIONS = ["txt", "markdown", "md", "mdx", "pdf", "html", "htm", "xlsx", "xls", "vtt", "properties"]
DOCUMENT_EXTENSIONS.extend(("doc", "docx", "csv", "eml", "msg", "pptx", "xml", "epub"))
if dify_config.UNSTRUCTURED_API_URL:
DOCUMENT_EXTENSIONS.append("ppt")
DOCUMENT_EXTENSIONS.extend([ext.upper() for ext in DOCUMENT_EXTENSIONS])
else:
- DOCUMENT_EXTENSIONS = ["txt", "markdown", "md", "mdx", "pdf", "html", "htm", "xlsx", "xls", "docx", "csv"]
+ DOCUMENT_EXTENSIONS = [
+ "txt",
+ "markdown",
+ "md",
+ "mdx",
+ "pdf",
+ "html",
+ "htm",
+ "xlsx",
+ "xls",
+ "docx",
+ "csv",
+ "vtt",
+ "properties",
+ ]
DOCUMENT_EXTENSIONS.extend([ext.upper() for ext in DOCUMENT_EXTENSIONS])
diff --git a/api/controllers/console/app/annotation.py b/api/controllers/console/app/annotation.py
index fcd8ed1882..48353a63af 100644
--- a/api/controllers/console/app/annotation.py
+++ b/api/controllers/console/app/annotation.py
@@ -186,7 +186,7 @@ class AnnotationUpdateDeleteApi(Resource):
app_id = str(app_id)
annotation_id = str(annotation_id)
AppAnnotationService.delete_app_annotation(app_id, annotation_id)
- return {"result": "success"}, 200
+ return {"result": "success"}, 204
class AnnotationBatchImportApi(Resource):
diff --git a/api/controllers/console/app/audio.py b/api/controllers/console/app/audio.py
index 12d9157dda..7519ae96c0 100644
--- a/api/controllers/console/app/audio.py
+++ b/api/controllers/console/app/audio.py
@@ -80,8 +80,6 @@ class ChatMessageTextApi(Resource):
@account_initialization_required
@get_app_model
def post(self, app_model: App):
- from werkzeug.exceptions import InternalServerError
-
try:
parser = reqparse.RequestParser()
parser.add_argument("message_id", type=str, location="json")
diff --git a/api/controllers/console/app/ops_trace.py b/api/controllers/console/app/ops_trace.py
index dd25af8ebf..7176440e16 100644
--- a/api/controllers/console/app/ops_trace.py
+++ b/api/controllers/console/app/ops_trace.py
@@ -84,7 +84,7 @@ class TraceAppConfigApi(Resource):
result = OpsService.delete_tracing_app_config(app_id=app_id, tracing_provider=args["tracing_provider"])
if not result:
raise TracingConfigNotExist()
- return {"result": "success"}
+ return {"result": "success"}, 204
except Exception as e:
raise BadRequest(str(e))
diff --git a/api/controllers/console/auth/data_source_bearer_auth.py b/api/controllers/console/auth/data_source_bearer_auth.py
index ea00c2b8c2..5f0762e4a5 100644
--- a/api/controllers/console/auth/data_source_bearer_auth.py
+++ b/api/controllers/console/auth/data_source_bearer_auth.py
@@ -65,7 +65,7 @@ class ApiKeyAuthDataSourceBindingDelete(Resource):
ApiKeyAuthService.delete_provider_auth(current_user.current_tenant_id, binding_id)
- return {"result": "success"}, 200
+ return {"result": "success"}, 204
api.add_resource(ApiKeyAuthDataSource, "/api-key-auth/data-source")
diff --git a/api/controllers/console/datasets/datasets.py b/api/controllers/console/datasets/datasets.py
index 752d124735..43615af709 100644
--- a/api/controllers/console/datasets/datasets.py
+++ b/api/controllers/console/datasets/datasets.py
@@ -657,6 +657,7 @@ class DatasetRetrievalSettingApi(Resource):
| VectorType.ELASTICSEARCH
| VectorType.ELASTICSEARCH_JA
| VectorType.PGVECTOR
+ | VectorType.VASTBASE
| VectorType.TIDB_ON_QDRANT
| VectorType.LINDORM
| VectorType.COUCHBASE
@@ -706,6 +707,7 @@ class DatasetRetrievalSettingMockApi(Resource):
| VectorType.ELASTICSEARCH_JA
| VectorType.COUCHBASE
| VectorType.PGVECTOR
+ | VectorType.VASTBASE
| VectorType.LINDORM
| VectorType.OPENGAUSS
| VectorType.OCEANBASE
diff --git a/api/controllers/console/datasets/datasets_document.py b/api/controllers/console/datasets/datasets_document.py
index 0b40312368..3588abeff5 100644
--- a/api/controllers/console/datasets/datasets_document.py
+++ b/api/controllers/console/datasets/datasets_document.py
@@ -40,7 +40,7 @@ from core.indexing_runner import IndexingRunner
from core.model_manager import ModelManager
from core.model_runtime.entities.model_entities import ModelType
from core.model_runtime.errors.invoke import InvokeAuthorizationError
-from core.plugin.manager.exc import PluginDaemonClientSideError
+from core.plugin.impl.exc import PluginDaemonClientSideError
from core.rag.extractor.entity.extract_setting import ExtractSetting
from extensions.ext_database import db
from extensions.ext_redis import redis_client
diff --git a/api/controllers/console/datasets/datasets_segments.py b/api/controllers/console/datasets/datasets_segments.py
index 696aaa94db..5c54ecbe81 100644
--- a/api/controllers/console/datasets/datasets_segments.py
+++ b/api/controllers/console/datasets/datasets_segments.py
@@ -131,7 +131,7 @@ class DatasetDocumentSegmentListApi(Resource):
except services.errors.account.NoPermissionError as e:
raise Forbidden(str(e))
SegmentService.delete_segments(segment_ids, document, dataset)
- return {"result": "success"}, 200
+ return {"result": "success"}, 204
class DatasetDocumentSegmentApi(Resource):
@@ -333,7 +333,7 @@ class DatasetDocumentSegmentUpdateApi(Resource):
except services.errors.account.NoPermissionError as e:
raise Forbidden(str(e))
SegmentService.delete_segment(segment, document, dataset)
- return {"result": "success"}, 200
+ return {"result": "success"}, 204
class DatasetDocumentSegmentBatchImportApi(Resource):
@@ -590,7 +590,7 @@ class ChildChunkUpdateApi(Resource):
SegmentService.delete_child_chunk(child_chunk, dataset)
except ChildChunkDeleteIndexServiceError as e:
raise ChildChunkDeleteIndexError(str(e))
- return {"result": "success"}, 200
+ return {"result": "success"}, 204
@setup_required
@login_required
diff --git a/api/controllers/console/datasets/external.py b/api/controllers/console/datasets/external.py
index 2c031172bf..aee8323f23 100644
--- a/api/controllers/console/datasets/external.py
+++ b/api/controllers/console/datasets/external.py
@@ -135,7 +135,7 @@ class ExternalApiTemplateApi(Resource):
raise Forbidden()
ExternalDatasetService.delete_external_knowledge_api(current_user.current_tenant_id, external_knowledge_api_id)
- return {"result": "success"}, 200
+ return {"result": "success"}, 204
class ExternalApiUseCheckApi(Resource):
diff --git a/api/controllers/console/datasets/metadata.py b/api/controllers/console/datasets/metadata.py
index fc9711169f..e4cac40ca1 100644
--- a/api/controllers/console/datasets/metadata.py
+++ b/api/controllers/console/datasets/metadata.py
@@ -82,7 +82,7 @@ class DatasetMetadataApi(Resource):
DatasetService.check_dataset_permission(dataset, current_user)
MetadataService.delete_metadata(dataset_id_str, metadata_id_str)
- return 200
+ return {"result": "success"}, 204
class DatasetMetadataBuiltInFieldApi(Resource):
diff --git a/api/controllers/console/explore/installed_app.py b/api/controllers/console/explore/installed_app.py
index 86550b2bdf..132da11878 100644
--- a/api/controllers/console/explore/installed_app.py
+++ b/api/controllers/console/explore/installed_app.py
@@ -113,7 +113,7 @@ class InstalledAppApi(InstalledAppResource):
db.session.delete(installed_app)
db.session.commit()
- return {"result": "success", "message": "App uninstalled successfully"}
+ return {"result": "success", "message": "App uninstalled successfully"}, 204
def patch(self, installed_app):
parser = reqparse.RequestParser()
diff --git a/api/controllers/console/explore/saved_message.py b/api/controllers/console/explore/saved_message.py
index 9f0c496645..3a1655d0ee 100644
--- a/api/controllers/console/explore/saved_message.py
+++ b/api/controllers/console/explore/saved_message.py
@@ -72,7 +72,7 @@ class SavedMessageApi(InstalledAppResource):
SavedMessageService.delete(app_model, current_user, message_id)
- return {"result": "success"}
+ return {"result": "success"}, 204
api.add_resource(
diff --git a/api/controllers/console/extension.py b/api/controllers/console/extension.py
index ed6cedb220..833da0d03c 100644
--- a/api/controllers/console/extension.py
+++ b/api/controllers/console/extension.py
@@ -99,7 +99,7 @@ class APIBasedExtensionDetailAPI(Resource):
APIBasedExtensionService.delete(extension_data_from_db)
- return {"result": "success"}
+ return {"result": "success"}, 204
api.add_resource(CodeBasedExtensionAPI, "/code-based-extension")
diff --git a/api/controllers/console/tag/tags.py b/api/controllers/console/tag/tags.py
index da83f64019..0d0d7ae95f 100644
--- a/api/controllers/console/tag/tags.py
+++ b/api/controllers/console/tag/tags.py
@@ -86,7 +86,7 @@ class TagUpdateDeleteApi(Resource):
TagService.delete_tag(tag_id)
- return 200
+ return 204
class TagBindingCreateApi(Resource):
diff --git a/api/controllers/console/workspace/endpoint.py b/api/controllers/console/workspace/endpoint.py
index 46dee20f8b..aa1a78935d 100644
--- a/api/controllers/console/workspace/endpoint.py
+++ b/api/controllers/console/workspace/endpoint.py
@@ -5,7 +5,7 @@ from werkzeug.exceptions import Forbidden
from controllers.console import api
from controllers.console.wraps import account_initialization_required, setup_required
from core.model_runtime.utils.encoders import jsonable_encoder
-from core.plugin.manager.exc import PluginPermissionDeniedError
+from core.plugin.impl.exc import PluginPermissionDeniedError
from libs.login import login_required
from services.plugin.endpoint_service import EndpointService
diff --git a/api/controllers/console/workspace/plugin.py b/api/controllers/console/workspace/plugin.py
index e9c1884c60..6f9ae18750 100644
--- a/api/controllers/console/workspace/plugin.py
+++ b/api/controllers/console/workspace/plugin.py
@@ -10,7 +10,7 @@ from controllers.console import api
from controllers.console.workspace import plugin_permission_required
from controllers.console.wraps import account_initialization_required, setup_required
from core.model_runtime.utils.encoders import jsonable_encoder
-from core.plugin.manager.exc import PluginDaemonClientSideError
+from core.plugin.impl.exc import PluginDaemonClientSideError
from libs.login import login_required
from models.account import TenantPluginPermission
from services.plugin.plugin_permission_service import PluginPermissionService
diff --git a/api/controllers/console/wraps.py b/api/controllers/console/wraps.py
index e5e8038ad7..6911181d82 100644
--- a/api/controllers/console/wraps.py
+++ b/api/controllers/console/wraps.py
@@ -10,6 +10,7 @@ from configs import dify_config
from controllers.console.workspace.error import AccountNotInitializedError
from extensions.ext_database import db
from extensions.ext_redis import redis_client
+from models.account import AccountStatus
from models.dataset import RateLimitLog
from models.model import DifySetup
from services.feature_service import FeatureService, LicenseStatus
@@ -24,7 +25,7 @@ def account_initialization_required(view):
# check account initialization
account = current_user
- if account.status == "uninitialized":
+ if account.status == AccountStatus.UNINITIALIZED:
raise AccountNotInitializedError()
return view(*args, **kwargs)
diff --git a/api/controllers/files/image_preview.py b/api/controllers/files/image_preview.py
index 5adfe16a79..5bb28b3897 100644
--- a/api/controllers/files/image_preview.py
+++ b/api/controllers/files/image_preview.py
@@ -75,7 +75,7 @@ class FilePreviewApi(Resource):
if args["as_attachment"]:
encoded_filename = quote(upload_file.name)
response.headers["Content-Disposition"] = f"attachment; filename*=UTF-8''{encoded_filename}"
- response.headers["Content-Type"] = "application/octet-stream"
+ response.headers["Content-Type"] = "application/octet-stream"
return response
diff --git a/api/controllers/service_api/app/annotation.py b/api/controllers/service_api/app/annotation.py
index cffa3665b1..c50f551faf 100644
--- a/api/controllers/service_api/app/annotation.py
+++ b/api/controllers/service_api/app/annotation.py
@@ -79,7 +79,7 @@ class AnnotationListApi(Resource):
class AnnotationUpdateDeleteApi(Resource):
@validate_app_token(fetch_user_arg=FetchUserArg(fetch_from=WhereisUserArg.JSON))
@marshal_with(annotation_fields)
- def post(self, app_model: App, end_user: EndUser, annotation_id):
+ def put(self, app_model: App, end_user: EndUser, annotation_id):
if not current_user.is_editor:
raise Forbidden()
@@ -98,7 +98,7 @@ class AnnotationUpdateDeleteApi(Resource):
annotation_id = str(annotation_id)
AppAnnotationService.delete_app_annotation(app_model.id, annotation_id)
- return {"result": "success"}, 200
+ return {"result": "success"}, 204
api.add_resource(AnnotationReplyActionApi, "/apps/annotation-reply/