mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-07-25 01:43:27 +08:00
feat: parser pages range and parse type validation for dataset/document (#17293)
## Summary Adds page-range parsing support to the Go-native pipeline path and introduces strict `parse_type` validation for both dataset and document update endpoints. ## What changed ### Pages range parsing - **`internal/utility/pdf_pages.go`** — `NormalizePDFPages`: normalizes raw page ranges (list of `[from,to]` 1-indexed inclusive ranges) into sorted, merged, deduplicated `[][]int`. Invalid ranges are dropped. - **`internal/ingestion/pipeline/pdf_pages.go`** — `NormalizeParserConfigPages`: walks any parser_config map and normalizes `"pages"` values under every component → filetype setup, so the persisted config always carries clean, merged ranges. - **`internal/deepdoc/parser/pdf/parser.go`** — integrates `resolvePagesToProcess` to filter parsed PDF pages by the configured ranges. - Pipeline integration (parser pages): `internal/parser/parser/pdf_parser_common.go`, `chunk_process.go`, plus associated e2e and unit tests. ### Parse type validation (shared logic) - **`internal/service/parser_mode.go`** (new) — `ValidateParseTypeMode`: shared function that validates `parse_type` (1=BuiltIn/parser_id, 2=Pipeline/pipeline_id) and ensures the corresponding field is present. Used by both dataset and document update endpoints. - **`internal/service/dataset/crud.go`** / `update.go` — replaces inline `isPipelineMode`/`isBuiltinMode` computation with the shared `service.ValidateParseTypeMode`. - **`internal/service/document/document_dataset_update.go`** — adds strict `parse_type` validation in `validateDatasetDocumentUpdate`, simplifies the reparse logic to a two-way switch (isBuiltin/isPipeline) now that parse_type is always valid. - **`internal/service/document/document.go`** — adds `ParseType` field to `UpdateDatasetDocumentRequest`. - **`internal/service/document/document_dataset_update.go`** — `updateDocumentParserConfig` fallback path when DSL loading fails. - **`internal/service/parser_mode_test.go`** (new) — test coverage for nil, invalid, and missing-field scenarios. ### Frontend - **`web/src/interfaces/request/document.ts`** — adds `parseType` to `IChangeParserRequestBody`. - **`web/src/hooks/use-document-request.ts`** — `useSetDocumentPipelineParser` sends `parse_type` in the PATCH payload. - **`web/src/pages/dataset/dataset/use-change-document-parser.ts`** — Go/Python branching for the document parser config dialog. - **`web/src/components/document-pipeline-dialog/use-document-pipeline-form.ts`** — `buildSubmitData` returns `parseType` (bugfix: was dropped from the return value). ### Test changes - **Removed**: 2 tests that verified the old "mutually exclusive" error (replaced by `ValidateParseTypeMode` coverage). - **Modified**: 6 tests across document and dataset packages to include `ParseType` in request structs. - **Added**: new e2e tests for pages parsing (`pages_e2e_test.go`, `pdf_parser_pages_e2e_test.go`) and unit tests for `NormalizePDFPages`, `NormalizeParserConfigPages`, `resolvePagesToProcess`. ## Backward compatibility - The `parse_type` field is **required** when `parser_id` or `pipeline_id` is sent. This changes the contract for both dataset and document PATCH endpoints, but aligns the Go backend with the existing frontend behavior (the frontend already sends `parse_type`). Callers that omit `parse_type` when updating parser/pipeline selections will receive a clear error message. - Existing callers that only update fields like `name`, `enabled`, or `meta_fields` are unaffected. - Test updates ensure all known call sites are compliant.
This commit is contained in:
@@ -141,6 +141,20 @@ export function ChunkMethodDialog({
|
||||
entity_types: z.array(z.string()).optional(),
|
||||
pages: z
|
||||
.array(z.object({ from: z.coerce.number(), to: z.coerce.number() }))
|
||||
.refine(
|
||||
(ranges) =>
|
||||
ranges.every(
|
||||
(r) =>
|
||||
Number.isInteger(r.from) &&
|
||||
Number.isInteger(r.to) &&
|
||||
r.from >= 1 &&
|
||||
r.from <= r.to,
|
||||
),
|
||||
{
|
||||
message:
|
||||
'page range invalid: from/to must be integers, from >= 1, from <= to',
|
||||
},
|
||||
)
|
||||
.optional(),
|
||||
metadata: z.any().optional(),
|
||||
built_in_metadata: z
|
||||
|
||||
@@ -185,6 +185,7 @@ export function useDocumentPipelineForm({
|
||||
parser_id: isPipeline ? '' : data.parser_id,
|
||||
pipeline_id: isPipeline ? data.pipeline_id : '',
|
||||
parser_config: transformedConfig,
|
||||
parseType: data.parseType,
|
||||
};
|
||||
},
|
||||
[],
|
||||
|
||||
@@ -524,12 +524,14 @@ export const useSetDocumentPipelineParser = () => {
|
||||
mutationFn: async ({
|
||||
parserId,
|
||||
pipelineId,
|
||||
parseType,
|
||||
documentId,
|
||||
datasetId,
|
||||
parserConfig,
|
||||
}: {
|
||||
parserId: string;
|
||||
pipelineId: string;
|
||||
parseType?: number;
|
||||
documentId: string;
|
||||
datasetId: string;
|
||||
parserConfig?: IChangeParserConfigRequestBody;
|
||||
@@ -539,6 +541,10 @@ export const useSetDocumentPipelineParser = () => {
|
||||
pipeline_id: pipelineId,
|
||||
};
|
||||
|
||||
if (parseType !== undefined) {
|
||||
updateData.parse_type = parseType;
|
||||
}
|
||||
|
||||
if (parserConfig) {
|
||||
updateData.parser_config = isPipelineParserConfig(parserConfig)
|
||||
? parserConfig
|
||||
|
||||
@@ -40,6 +40,8 @@ export interface IChangeParserConfigRequestBody {
|
||||
export interface IChangeParserRequestBody {
|
||||
parser_id: string;
|
||||
pipeline_id?: string;
|
||||
// 1 = BuiltIn (parser_id), 2 = Pipeline (pipeline_id); omitted = unspecified.
|
||||
parseType?: number;
|
||||
doc_id?: string;
|
||||
parser_config: IChangeParserConfigRequestBody;
|
||||
}
|
||||
|
||||
@@ -25,16 +25,19 @@ export const useChangeDocumentParser = () => {
|
||||
if (record?.id && record?.dataset_id) {
|
||||
// The Go document endpoint takes `parser_id` and a pipeline-shaped
|
||||
// parser_config; the Python one keeps the legacy payload shape.
|
||||
const setParser = isGoBackend()
|
||||
? setDocumentPipelineParser
|
||||
: setDocumentParser;
|
||||
const ret = await setParser({
|
||||
const common = {
|
||||
parserId: parserConfigInfo.parser_id,
|
||||
pipelineId: parserConfigInfo.pipeline_id || '',
|
||||
documentId: record?.id,
|
||||
datasetId: record?.dataset_id,
|
||||
parserConfig: parserConfigInfo.parser_config,
|
||||
});
|
||||
};
|
||||
const ret = isGoBackend()
|
||||
? await setDocumentPipelineParser({
|
||||
...common,
|
||||
parseType: parserConfigInfo.parseType,
|
||||
})
|
||||
: await setDocumentParser(common);
|
||||
if (ret === 0) {
|
||||
hideChangeParserModal();
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user