opensource
/
dify
mirror of https://github.com/langgenius/dify.git
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat: optimize weaviate batch size (#348)

This commit is contained in:
John Wang 2023-06-13 11:28:15 +08:00 committed by GitHub
parent 1af968e73a
commit 6a3ab36101
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 13 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
api/core/vector_store/weaviate_vector_store_client.py
View File

@ -26,13 +26,25 @@ class WeaviateVectorStoreClient(BaseVectorStoreClient):
weaviate.connect.connection.has_grpc = grpc_enabled
return weaviate.Client(
client = weaviate.Client(
url=endpoint,
auth_client_secret=auth_config,
timeout_config=(5, 60),
startup_period=None
)
client.batch.configure(
# `batch_size` takes an `int` value to enable auto-batching
# (`None` is used for manual batching)
batch_size=100,
# dynamically update the `batch_size` based on import speed
dynamic=True,
# `timeout_retries` takes an `int` value to retry on time outs
timeout_retries=3,
)
return client
def get_index(self, service_context: ServiceContext, config: dict) -> GPTVectorStoreIndex:
index_struct = WeaviateIndexDict()