opensource
/
dify
mirror of https://github.com/langgenius/dify.git
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

deal dataset vector

This commit is contained in:
Jyong 2023-05-23 22:42:03 +08:00
parent 63a4f836e9
commit 8d6546cf7f
1 changed files with 12 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
api/tasks/deal_dataset_vector_index_task.py
View File

@ -8,6 +8,7 @@ from core.index.vector_index import VectorIndex
from extensions.ext_database import db
from models.dataset import DocumentSegment, Document, Dataset
@shared_task
def deal_dataset_vector_index_task(dataset_id: str, action: str):
"""
@ -36,8 +37,7 @@ def deal_dataset_vector_index_task(dataset_id: str, action: str):
segments = db.session.query(DocumentSegment).filter(
DocumentSegment.document_id == document.id,
DocumentSegment.enabled == True
) \
.order_by(DocumentSegment.position.asc()).all()
) .order_by(DocumentSegment.position.asc()).all()
nodes = []
previous_node = None
@ -51,18 +51,17 @@ def deal_dataset_vector_index_task(dataset_id: str, action: str):
previous_node.relationships[DocumentRelationship.NEXT] = segment.index_node_id
node = Node(
doc_id=segment.index_node_id,
doc_hash=segment.index_node_hash,
text=segment.content,
extra_info=None,
node_info=None,
relationships=relationships
)
node = Node(
doc_id=segment.index_node_id,
doc_hash=segment.index_node_hash,
text=segment.content,
extra_info=None,
node_info=None,
relationships=relationships
)
previous_node = node
nodes.append(node)
previous_node = node
nodes.append(node)
# save vector index
vector_index.add_nodes(
nodes=nodes,