zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-07-18 21:57:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: improve pipeline compilation, template updates, and document resets (#16815)

Browse Source 
### What problem does this PR solve?

- Clear stale pipeline IDs and generated data when updating documents
without `pipeline_id`.
- Support tree compilation results in pipeline workflows.
- Update compilation templates in place while preserving existing
template IDs.
- Improve duplicate-template validation messages.

### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)
- [x] New Feature (non-breaking change which adds functionality)

Co-authored-by: Jin Hai <haijin.chn@gmail.com>
This commit is contained in:
buua436
2026-07-10 21:17:19 +08:00
committed by GitHub
parent 30739b7f8e
commit d291e4641d
7 changed files with 266 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
api/apps/restful_apis/document_api.py
View File

@@ -272,19 +272,25 @@ async def update_document(tenant_id, dataset_id, document_id):
parser_config_template_group_changed = parser_config_template_group_touched and _compilation_template_group_id_changed(old_parser_config, req["parser_config"])
DocumentService.update_parser_config(doc.id, req["parser_config"])
# pipeline_id provided - reset document for reparse
if update_doc_req.pipeline_id:
if error := reset_document_for_reparse(doc, tenant_id, pipeline_id=update_doc_req.pipeline_id):
# A non-empty pipeline_id selects pipeline parsing; an explicitly empty
# value clears it and switches back to the direct parser path.
if "pipeline_id" in req:
if error := reset_document_for_reparse(doc, tenant_id, pipeline_id=update_doc_req.pipeline_id or ""):
return error
# chunk method provided - the update method will check if it's different with existing one
elif update_doc_req.chunk_method:
if error := update_chunk_method(req, doc, tenant_id):
return error
if parser_config_template_group_changed and doc.parser_id.lower() == req["chunk_method"].lower():
if error := reset_document_for_reparse(doc, tenant_id):
if error := reset_document_for_reparse(doc, tenant_id, pipeline_id=""):
return error
elif parser_config_template_group_changed:
if error := reset_document_for_reparse(doc, tenant_id):
if error := reset_document_for_reparse(doc, tenant_id, pipeline_id=""):
return error
else:
# Direct-parser updates do not carry a pipeline_id. Clear any stale
# pipeline selection and its generated document-store data.
if error := reset_document_for_reparse(doc, tenant_id, pipeline_id=""):
return error
if "enabled" in req: # already checked in UpdateDocumentReq - it's int if present

14
api/apps/services/document_api_service.py
View File

@@ -80,7 +80,13 @@ def update_chunk_method(req, doc, tenant_id):
"""
if doc.parser_id.lower() != req["chunk_method"].lower():
# if chunk method changed, reset document for reparse
result = reset_document_for_reparse(doc, tenant_id, parser_id=req["chunk_method"])
result = reset_document_for_reparse(doc, tenant_id, parser_id=req["chunk_method"], pipeline_id="")
if result:
return result
elif doc.pipeline_id:
# An explicit chunk method selects the direct parser path. Clear the
# previous pipeline even when the parser method itself is unchanged.
result = reset_document_for_reparse(doc, tenant_id, pipeline_id="")
if result:
return result
if not req.get("parser_config"):
@@ -122,7 +128,9 @@ def reset_document_for_reparse(doc, tenant_id, parser_id=None, pipeline_id=None)
if not e:
return get_error_data_result(message="Document not found!")
# Delete chunks from document store
# Update document statistics before deleting all document rows. Pipeline
# compilation rows may exist even when token_num is zero, so the doc-store
# cleanup must not be gated by the document counters.
if doc.token_num > 0:
try:
e = DocumentService.increment_chunk_num(
@@ -136,7 +144,7 @@ def reset_document_for_reparse(doc, tenant_id, parser_id=None, pipeline_id=None)
return get_error_data_result(message="Document not found!")
if not e:
return get_error_data_result(message="Document not found!")
settings.docStoreConn.delete({"doc_id": doc.id}, search.index_name(tenant_id), doc.kb_id)
settings.docStoreConn.delete({"doc_id": doc.id}, search.index_name(tenant_id), doc.kb_id)
# Delete chunk images
try:

112
api/db/services/compilation_template_group_service.py
View File

@@ -296,16 +296,84 @@ class CompilationTemplateGroupService(CommonService):
cls.model.update(**updates).where(cls.model.id == group_id).execute()
if templates is not None:
# Soft-delete previous children (en-bloc replace). Simpler than
# diffing and acceptable given small N — child IDs are not
# referenced externally (parser_config keys the group, not its
# children).
CompilationTemplate.update(status=StatusEnum.INVALID.value).where(
CompilationTemplate.group_id == group_id,
CompilationTemplate.status == StatusEnum.VALID.value,
).execute()
for i, child in enumerate(templates):
cls._insert_child(group_id, tenant_id, child, index=i)
current_children = list(
CompilationTemplate.select()
.where(
CompilationTemplate.group_id == group_id,
CompilationTemplate.status == StatusEnum.VALID.value,
)
.order_by(CompilationTemplate.create_time.asc())
)
current_by_id = {child.id: child for child in current_children}
retained_ids: set[str] = set()
seen_names: set[str] = set()
for index, child in enumerate(templates):
child_id = str((child or {}).get("id") or "").strip()
target = current_by_id.get(child_id) if child_id else None
if child_id and target is None:
raise GroupValidationError(f"Template {child_id} does not belong to this group.")
# Older clients did not send child ids. Preserve their
# existing ids by matching the submitted order.
if target is None and not child_id and index < len(current_children):
target = current_children[index]
kind = str((child or {}).get("kind") or "").strip()
name = str((child or {}).get("name") or "").strip()
config = (child or {}).get("config") or {}
if not kind or not name or not isinstance(config, dict):
raise GroupValidationError("Each template must include a name, kind, and config object.")
if name in seen_names:
raise GroupValidationError(f"Template name '{name}' is duplicated in this group.")
seen_names.add(name)
from api.db.services.compilation_template_service import CompilationTemplateService
config = CompilationTemplateService.fill_config_default_llm(config, tenant_id)
duplicate_query = CompilationTemplate.select().where(
CompilationTemplate.tenant_id == tenant_id,
CompilationTemplate.name == name,
~CompilationTemplate.is_builtin,
CompilationTemplate.status == StatusEnum.VALID.value,
)
if target is not None:
duplicate_query = duplicate_query.where(CompilationTemplate.id != target.id)
if duplicate_query.exists():
raise GroupValidationError(f"Template name '{name}' already exists. Please choose another name.")
if target is None:
new_id = cls._insert_child(
group_id,
tenant_id,
{
"name": name,
"description": (child or {}).get("description") or "",
"kind": kind,
"config": config,
},
index=index,
)
retained_ids.add(new_id)
continue
CompilationTemplate.update(
name=name,
description=str((child or {}).get("description") or ""),
kind=kind,
config=config,
).where(CompilationTemplate.id == target.id).execute()
retained_ids.add(target.id)
removed_children = [child for child in current_children if child.id not in retained_ids]
if removed_children:
removed_names = [child.name for child in removed_children]
removed_ids = [child.id for child in removed_children]
cls._purge_stale_invalid_children(tenant_id, removed_names)
CompilationTemplate.update(status=StatusEnum.INVALID.value).where(
CompilationTemplate.group_id == group_id,
CompilationTemplate.id.in_(removed_ids),
CompilationTemplate.status == StatusEnum.VALID.value,
).execute()
return cls.get_saved(group_id, tenant_id)
@@ -336,7 +404,7 @@ class CompilationTemplateGroupService(CommonService):
child: dict,
*,
index: int,
) -> None:
) -> str:
kind = str((child or {}).get("kind") or "").strip()
name = str((child or {}).get("name") or "").strip()
config = (child or {}).get("config") or {}
@@ -345,6 +413,15 @@ class CompilationTemplateGroupService(CommonService):
from api.db.services.compilation_template_service import CompilationTemplateService
config = CompilationTemplateService.fill_config_default_llm(config, tenant_id)
duplicate = CompilationTemplate.select().where(
CompilationTemplate.tenant_id == tenant_id,
CompilationTemplate.name == name,
~CompilationTemplate.is_builtin,
CompilationTemplate.status == StatusEnum.VALID.value,
)
if duplicate.exists():
raise GroupValidationError(f"A compilation template named '{name}' already exists. Please choose a different name.")
template_id = get_uuid()
CompilationTemplate.create(
id=template_id,
@@ -357,6 +434,19 @@ class CompilationTemplateGroupService(CommonService):
is_builtin=False,
status=StatusEnum.VALID.value,
)
return template_id
@classmethod
def _purge_stale_invalid_children(cls, tenant_id: str, names: list[str]) -> None:
cleaned_names = [name for name in {str(name).strip() for name in names} if name]
if not cleaned_names:
return
CompilationTemplate.delete().where(
CompilationTemplate.tenant_id == tenant_id,
CompilationTemplate.name.in_(cleaned_names),
~CompilationTemplate.is_builtin,
CompilationTemplate.status == StatusEnum.INVALID.value,
).execute()
# ------------------------------------------------------------------
# Lookup helpers used by the orchestrator