diff --git a/api/.env.example b/api/.env.example index 4026d024fb9..daee3058a0e 100644 --- a/api/.env.example +++ b/api/.env.example @@ -530,7 +530,7 @@ LOG_FORMAT=%(asctime)s,%(msecs)d %(levelname)-2s [%(filename)s:%(lineno)d] %(req # Indexing configuration INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH=4000 -# Maximum number of worker threads used for high-quality dataset indexing. +# Maximum number of worker threads used for high-quality dataset indexing (1-10). # Lower this value to reduce memory usage and avoid OOM freezes during re-indexing. INDEXING_MAX_WORKERS=2 diff --git a/api/configs/feature/__init__.py b/api/configs/feature/__init__.py index b1d9950730d..6c604705ecf 100644 --- a/api/configs/feature/__init__.py +++ b/api/configs/feature/__init__.py @@ -1109,7 +1109,7 @@ class IndexingConfig(BaseSettings): ) INDEXING_MAX_WORKERS: PositiveInt = Field( - description="Maximum number of worker threads used for high-quality dataset indexing", + description="Maximum number of worker threads used for high-quality dataset indexing (1-10)", default=2, ) diff --git a/api/core/indexing_runner.py b/api/core/indexing_runner.py index e9ad2aaa7e2..9147630ddb4 100644 --- a/api/core/indexing_runner.py +++ b/api/core/indexing_runner.py @@ -604,7 +604,7 @@ class IndexingRunner: # High-quality indexing is memory intensive (embedding generation + vector writes). # Running too many chunks in parallel can trigger OOM and freeze the service until reboot. - max_workers = max(1, min(10, int(dify_config.INDEXING_MAX_WORKERS))) + max_workers = max(1, int(dify_config.INDEXING_MAX_WORKERS)) max_workers = min(max_workers, len(documents)) if documents else 1 if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY: with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: diff --git a/docker/.env.example b/docker/.env.example index 8f843d1699b..f9701506fb1 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -1009,7 +1009,7 @@ SENDGRID_API_KEY= # Maximum length of segmentation tokens for indexing INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH=4000 -# Maximum number of worker threads used for high-quality dataset indexing. +# Maximum number of worker threads used for high-quality dataset indexing (1-10). # Lower this value to reduce memory usage and avoid OOM/freezes during re-indexing. INDEXING_MAX_WORKERS=2