diff --git a/internal/agent/templates/advanced_ingestion_pipeline.json b/internal/agent/templates/advanced_ingestion_pipeline.json index 26b2f85d6d..f2c84bce55 100644 --- a/internal/agent/templates/advanced_ingestion_pipeline.json +++ b/internal/agent/templates/advanced_ingestion_pipeline.json @@ -133,7 +133,6 @@ ] }, "docx": { - "flatten_media_to_text": false, "preprocess": "main_content", "suffix": [ "docx" @@ -176,7 +175,6 @@ "system_prompt": "" }, "markdown": { - "flatten_media_to_text": false, "preprocess": "main_content", "suffix": [ "md", @@ -186,7 +184,6 @@ "vlm": {} }, "pdf": { - "flatten_media_to_text": false, "parse_method": "DeepDOC", "preprocess": "main_content", "suffix": [ @@ -203,7 +200,6 @@ ] }, "spreadsheet": { - "flatten_media_to_text": false, "parse_method": "DeepDOC", "preprocess": "main_content", "suffix": [ @@ -363,13 +359,11 @@ "setups": [ { "fileFormat": "pdf", - "flatten_media_to_text": false, "parse_method": "DeepDOC", "preprocess": "main_content" }, { "fileFormat": "spreadsheet", - "flatten_media_to_text": false, "parse_method": "DeepDOC", "preprocess": "main_content" }, @@ -395,7 +389,6 @@ }, { "fileFormat": "markdown", - "flatten_media_to_text": false, "preprocess": "main_content" }, { @@ -412,7 +405,6 @@ }, { "fileFormat": "docx", - "flatten_media_to_text": false, "preprocess": "main_content" }, { diff --git a/internal/agent/templates/chunk_summary.json b/internal/agent/templates/chunk_summary.json index 24a936e6c0..f0de49be76 100644 --- a/internal/agent/templates/chunk_summary.json +++ b/internal/agent/templates/chunk_summary.json @@ -90,7 +90,6 @@ ] }, "docx": { - "flatten_media_to_text": false, "preprocess": "main_content", "suffix": [ "docx" @@ -133,7 +132,6 @@ "system_prompt": "" }, "markdown": { - "flatten_media_to_text": false, "preprocess": "main_content", "suffix": [ "md", @@ -143,7 +141,6 @@ "vlm": {} }, "pdf": { - "flatten_media_to_text": false, "parse_method": "DeepDOC", "preprocess": "main_content", "suffix": [ @@ -160,7 +157,6 @@ ] }, "spreadsheet": { - "flatten_media_to_text": false, "parse_method": "DeepDOC", "preprocess": "main_content", "suffix": [ @@ -323,13 +319,11 @@ "setups": [ { "fileFormat": "pdf", - "flatten_media_to_text": false, "parse_method": "DeepDOC", "preprocess": "main_content" }, { "fileFormat": "spreadsheet", - "flatten_media_to_text": false, "parse_method": "DeepDOC", "preprocess": "main_content" }, @@ -355,7 +349,6 @@ }, { "fileFormat": "markdown", - "flatten_media_to_text": false, "preprocess": "main_content" }, { @@ -372,7 +365,6 @@ }, { "fileFormat": "docx", - "flatten_media_to_text": false, "preprocess": "main_content" }, { diff --git a/internal/agent/templates/compiler.json b/internal/agent/templates/compiler.json index 7e2c54b4cc..ecb04c3089 100644 --- a/internal/agent/templates/compiler.json +++ b/internal/agent/templates/compiler.json @@ -57,7 +57,6 @@ }, "setups": { "doc": { - "flatten_media_to_text": false, "order_index": 7, "remove_header_footer": false, "suffix": [ @@ -66,7 +65,6 @@ "vlm": {} }, "docx": { - "flatten_media_to_text": false, "order_index": 8, "remove_header_footer": false, "suffix": [ @@ -110,7 +108,6 @@ ] }, "markdown": { - "flatten_media_to_text": false, "order_index": 4, "suffix": [ "md", @@ -120,7 +117,6 @@ "vlm": {} }, "pdf": { - "flatten_media_to_text": false, "order_index": 0, "parse_method": "DeepDOC", "remove_header_footer": false, @@ -138,7 +134,6 @@ ] }, "spreadsheet": { - "flatten_media_to_text": false, "order_index": 1, "parse_method": "DeepDOC", "suffix": [ @@ -274,14 +269,12 @@ "setups": [ { "fileFormat": "pdf", - "flatten_media_to_text": false, "parse_method": "DeepDOC", "remove_header_footer": false, "vlm": {} }, { "fileFormat": "spreadsheet", - "flatten_media_to_text": false, "parse_method": "DeepDOC", "vlm": {} }, @@ -304,7 +297,6 @@ }, { "fileFormat": "markdown", - "flatten_media_to_text": false, "vlm": {} }, { @@ -316,13 +308,11 @@ }, { "fileFormat": "doc", - "flatten_media_to_text": false, "remove_header_footer": false, "vlm": {} }, { "fileFormat": "docx", - "flatten_media_to_text": false, "remove_header_footer": false, "vlm": {} }, diff --git a/internal/agent/templates/title_chunker.json b/internal/agent/templates/title_chunker.json index ad845fbb88..e7a7d92240 100644 --- a/internal/agent/templates/title_chunker.json +++ b/internal/agent/templates/title_chunker.json @@ -90,7 +90,6 @@ ] }, "docx": { - "flatten_media_to_text": false, "preprocess": "main_content", "suffix": [ "docx" @@ -133,7 +132,6 @@ "system_prompt": "" }, "markdown": { - "flatten_media_to_text": false, "preprocess": "main_content", "suffix": [ "md", @@ -143,7 +141,6 @@ "vlm": {} }, "pdf": { - "flatten_media_to_text": false, "parse_method": "DeepDOC", "preprocess": "main_content", "suffix": [ @@ -160,7 +157,6 @@ ] }, "spreadsheet": { - "flatten_media_to_text": false, "parse_method": "DeepDOC", "preprocess": "main_content", "suffix": [ @@ -342,13 +338,11 @@ "setups": [ { "fileFormat": "pdf", - "flatten_media_to_text": false, "parse_method": "DeepDOC", "preprocess": "main_content" }, { "fileFormat": "spreadsheet", - "flatten_media_to_text": false, "parse_method": "DeepDOC", "preprocess": "main_content" }, @@ -374,7 +368,6 @@ }, { "fileFormat": "markdown", - "flatten_media_to_text": false, "preprocess": "main_content" }, { @@ -391,7 +384,6 @@ }, { "fileFormat": "docx", - "flatten_media_to_text": false, "preprocess": "main_content" }, { diff --git a/internal/dao/ingestion_parser_config_runtime_test.go b/internal/dao/ingestion_parser_config_runtime_test.go index 91fea3ca05..cf6ede236d 100644 --- a/internal/dao/ingestion_parser_config_runtime_test.go +++ b/internal/dao/ingestion_parser_config_runtime_test.go @@ -634,15 +634,14 @@ var overriddenFamilyKeys = map[string]bool{ // questions (this one, "is the template's value safe to lose"; that one, "may // the value the migration writes differ from the template's"). var inertFamilyKeys = map[string]string{ - "preprocess": "the value has no Go reader; the preprocessing stage is selected by the chunker's own options", - "suffix": "no Go reader; file routing uses the document's own suffix column, not the family list", - "vlm": "empty in every template; absent falls back to the family llm_id and then the tenant model, and only Markdown and audio read a family vlm at all", - "flatten_media_to_text": "false in every template, which is also defaultSetups()'s default", - "pages": "the only template that bakes it lists every page ([[1,100000]]), which is what an absent pages means", - "parse_method": "a written pdf family always restates it; for the spreadsheet family defaultSetups() re-derives the same deepdoc engine", - "column_mode": "the table template bakes auto, which is what an absent column_mode means to indexdoc's resolveTableColumnConfig", - "column_roles": "the table template bakes the empty role map, which is what an absent column_roles means", - "column_names": "the table template bakes the empty name list, which is what an absent column_names means", + "preprocess": "the value has no Go reader; the preprocessing stage is selected by the chunker's own options", + "suffix": "no Go reader; file routing uses the document's own suffix column, not the family list", + "vlm": "empty in every template; absent falls back to the family llm_id and then the tenant model, and only Markdown and audio read a family vlm at all", + "pages": "the only template that bakes it lists every page ([[1,100000]]), which is what an absent pages means", + "parse_method": "a written pdf family always restates it; for the spreadsheet family defaultSetups() re-derives the same deepdoc engine", + "column_mode": "the table template bakes auto, which is what an absent column_mode means to indexdoc's resolveTableColumnConfig", + "column_roles": "the table template bakes the empty role map, which is what an absent column_roles means", + "column_names": "the table template bakes the empty name list, which is what an absent column_names means", } // assertFamiliesPreserved checks that no template-baked key of a written family diff --git a/internal/ingestion/component/parser.go b/internal/ingestion/component/parser.go index 0e27602f67..8ce1bb0ed6 100644 --- a/internal/ingestion/component/parser.go +++ b/internal/ingestion/component/parser.go @@ -301,17 +301,15 @@ func imageSetupRunsOCR(setup schema.ParserSetup) bool { func defaultSetups() map[string]schema.ParserSetup { s := map[string]schema.ParserSetup{ "pdf": { - "parse_method": "deepdoc", - "flatten_media_to_text": false, - "remove_toc": false, - "remove_header_footer": false, - "suffix": []string{"pdf"}, + "parse_method": "deepdoc", + "remove_toc": false, + "remove_header_footer": false, + "suffix": []string{"pdf"}, }, "spreadsheet": { - "parse_method": "deepdoc", - "flatten_media_to_text": false, - "html4excel": false, - "suffix": []string{"xls", "xlsx", "csv"}, + "parse_method": "deepdoc", + "html4excel": false, + "suffix": []string{"xls", "xlsx", "csv"}, }, "doc": { "remove_toc": false, @@ -319,15 +317,13 @@ func defaultSetups() map[string]schema.ParserSetup { "suffix": []string{"doc"}, }, "docx": { - "flatten_media_to_text": false, - "remove_toc": false, - "remove_header_footer": false, - "suffix": []string{"docx"}, + "remove_toc": false, + "remove_header_footer": false, + "suffix": []string{"docx"}, }, "markdown": { - "flatten_media_to_text": false, - "suffix": []string{"md", "markdown", "mdx"}, - "remove_toc": false, + "suffix": []string{"md", "markdown", "mdx"}, + "remove_toc": false, }, "text&code": { "suffix": []string{ diff --git a/internal/ingestion/component/pdf_vision_dispatch.go b/internal/ingestion/component/pdf_vision_dispatch.go index 2ea50dc414..7e880331d5 100644 --- a/internal/ingestion/component/pdf_vision_dispatch.go +++ b/internal/ingestion/component/pdf_vision_dispatch.go @@ -89,9 +89,8 @@ func maybeDispatchPDFVision( // fallback) are converted to JSON by NormalizeMarkdown before dispatch // returns, so the NormalizeResult closure below only ever applies item // policies to already-normalized JSON and stays idempotent. - flatten, _ := setup["flatten_media_to_text"].(bool) normalize := func(result parser.ParseResult) parser.ParseResult { - pdf := parser.PDFParser{FlattenMediaToText: flatten} + pdf := parser.PDFParser{} return parser.AttachPDFOutlines(pdf.NormalizeResult(ctx, filename, result), binary) } if result, handled, err := maybeDispatchTenantPDFParser(ctx, db, filename, binary, tenantID, setup); handled { @@ -347,7 +346,7 @@ func dispatchMonkeyOCRv2PDF(ctx context.Context, db *gorm.DB, filename string, b return parser.ParseResult{}, err } md := strings.Join(sections, "\n\n") - return normalizePDFMarkdown(ctx, filename, md, setup), nil + return normalizePDFMarkdown(ctx, filename, md), nil } // normalizePDFMarkdown converts a PDF backend Markdown response (MonkeyOCRv2, @@ -355,9 +354,8 @@ func dispatchMonkeyOCRv2PDF(ctx context.Context, db *gorm.DB, filename string, b // package's NormalizeMarkdown. This keeps the markdown→JSON boundary in the // parser package, matching the NormalizeResult path used for JSON backends, and // lets the downstream NormalizeResult closure stay idempotent for JSON. -func normalizePDFMarkdown(ctx context.Context, filename, md string, setup schema.ParserSetup) parser.ParseResult { - flatten, _ := setup["flatten_media_to_text"].(bool) - return (&parser.PDFParser{FlattenMediaToText: flatten}).NormalizeMarkdown(ctx, filename, md) +func normalizePDFMarkdown(ctx context.Context, filename, md string) parser.ParseResult { + return (&parser.PDFParser{}).NormalizeMarkdown(ctx, filename, md) } var isMonkeyOCRv2LayoutModelID = defaultIsMonkeyOCRv2LayoutModelID @@ -652,7 +650,7 @@ func dispatchMinerUPDF( if err != nil { return parser.ParseResult{}, fmt.Errorf("parser: MinerU cloud: %w", err) } - return finishMinerURemoteResult(ctx, filename, binary, result, setup) + return finishMinerURemoteResult(ctx, filename, binary, result) } v1, err := modelModule.MinerUSupportsV1(ctx, baseURL, apiKeyRaw) @@ -673,7 +671,7 @@ func dispatchMinerUPDF( if err != nil { return parser.ParseResult{}, fmt.Errorf("parser: MinerU V1: %w", err) } - return finishMinerURemoteResult(ctx, filename, binary, result, setup) + return finishMinerURemoteResult(ctx, filename, binary, result) } apiURL := strings.TrimRight(baseURL, "/") + "/file_parse" @@ -694,7 +692,7 @@ func dispatchMinerUPDF( return res, nil } -func finishMinerURemoteResult(ctx context.Context, filename string, binary []byte, result *modelModule.MinerUV1Result, setup schema.ParserSetup) (parser.ParseResult, error) { +func finishMinerURemoteResult(ctx context.Context, filename string, binary []byte, result *modelModule.MinerUV1Result) (parser.ParseResult, error) { markdown := "" var zipBytes []byte if result != nil { @@ -714,7 +712,7 @@ func finishMinerURemoteResult(ctx context.Context, filename string, binary []byt return res, nil } if markdown != "" { - return normalizePDFMarkdown(ctx, filename, result.Markdown, setup), nil + return normalizePDFMarkdown(ctx, filename, result.Markdown), nil } if extractErr != nil { return parser.ParseResult{}, fmt.Errorf("parser: MinerU extract: %w", extractErr) @@ -870,7 +868,7 @@ func dispatchMonkeyOCRPDF( } } md := strings.Join(parts, "\n") - return normalizePDFMarkdown(ctx, filename, md, setup), nil + return normalizePDFMarkdown(ctx, filename, md), nil } // resolveMinerUModelForDispatch resolves the OCR model used by the MinerU PDF diff --git a/internal/ingestion/component/pdf_vision_dispatch_test.go b/internal/ingestion/component/pdf_vision_dispatch_test.go index f3b0f6af2f..7d07408f6d 100644 --- a/internal/ingestion/component/pdf_vision_dispatch_test.go +++ b/internal/ingestion/component/pdf_vision_dispatch_test.go @@ -749,7 +749,7 @@ func TestFinishMinerURemoteResultPrefersStructuredZip(t *testing.T) { res, err := finishMinerURemoteResult(t.Context(), "sample.pdf", []byte("%PDF-1.4"), &modelModule.MinerUV1Result{ Markdown: "# markdown only", Zip: buf.Bytes(), - }, nil) + }) if err != nil { t.Fatal(err) } diff --git a/internal/ingestion/component/vision_enhancement.go b/internal/ingestion/component/vision_enhancement.go index b246bd5e62..cf72e7fb97 100644 --- a/internal/ingestion/component/vision_enhancement.go +++ b/internal/ingestion/component/vision_enhancement.go @@ -204,8 +204,7 @@ func maybeDispatchVisionEnhancement( // (pdf_vision_dispatch.go), not an input here — see issue #20727. language := resolveVisionLanguage(inputs) - // Collect visual resources, including Markdown images whose type is text - // because flatten_media_to_text is enabled. Items whose text already + // Collect visual resources. Items whose text already // carries a backend-provided description (MinerU vlm_description) are // skipped to avoid a duplicate second description and an extra VLM call. var items []int diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_book.json b/internal/ingestion/pipeline/template/ingestion_pipeline_book.json index c688afb197..35b5b58a7d 100644 --- a/internal/ingestion/pipeline/template/ingestion_pipeline_book.json +++ b/internal/ingestion/pipeline/template/ingestion_pipeline_book.json @@ -62,7 +62,6 @@ ] }, "docx": { - "flatten_media_to_text": false, "preprocess": [ "main_content" ], @@ -81,7 +80,6 @@ ] }, "pdf": { - "flatten_media_to_text": false, "parse_method": "DeepDOC", "preprocess": [ "main_content" @@ -473,7 +471,6 @@ "setups": [ { "fileFormat": "pdf", - "flatten_media_to_text": false, "parse_method": "DeepDOC", "preprocess": [ "main_content" @@ -502,7 +499,6 @@ }, { "fileFormat": "docx", - "flatten_media_to_text": false, "preprocess": [ "main_content" ], diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_general.json b/internal/ingestion/pipeline/template/ingestion_pipeline_general.json index 3f8efb9e01..7fe0e36b3c 100644 --- a/internal/ingestion/pipeline/template/ingestion_pipeline_general.json +++ b/internal/ingestion/pipeline/template/ingestion_pipeline_general.json @@ -44,7 +44,6 @@ ] }, "docx": { - "flatten_media_to_text": false, "preprocess": [ "main_content" ], @@ -63,7 +62,6 @@ ] }, "markdown": { - "flatten_media_to_text": false, "preprocess": [ "main_content" ], @@ -75,7 +73,6 @@ "vlm": {} }, "pdf": { - "flatten_media_to_text": false, "parse_method": "DeepDOC", "preprocess": [ "main_content" @@ -92,7 +89,6 @@ ] }, "spreadsheet": { - "flatten_media_to_text": false, "parse_method": "DeepDOC", "preprocess": [ "main_content" @@ -315,7 +311,6 @@ "setups": [ { "fileFormat": "pdf", - "flatten_media_to_text": false, "parse_method": "DeepDOC", "preprocess": [ "main_content" @@ -324,7 +319,6 @@ }, { "fileFormat": "spreadsheet", - "flatten_media_to_text": false, "parse_method": "DeepDOC", "preprocess": [ "main_content" @@ -333,7 +327,6 @@ }, { "fileFormat": "markdown", - "flatten_media_to_text": false, "preprocess": [ "main_content" ], @@ -359,7 +352,6 @@ }, { "fileFormat": "docx", - "flatten_media_to_text": false, "preprocess": [ "main_content" ], diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_laws.json b/internal/ingestion/pipeline/template/ingestion_pipeline_laws.json index ae561c1c71..5368855475 100644 --- a/internal/ingestion/pipeline/template/ingestion_pipeline_laws.json +++ b/internal/ingestion/pipeline/template/ingestion_pipeline_laws.json @@ -62,7 +62,6 @@ ] }, "docx": { - "flatten_media_to_text": false, "preprocess": [ "main_content" ], @@ -81,7 +80,6 @@ ] }, "markdown": { - "flatten_media_to_text": false, "preprocess": [ "main_content" ], @@ -93,7 +91,6 @@ "vlm": {} }, "pdf": { - "flatten_media_to_text": false, "parse_method": "DeepDOC", "preprocess": [ "main_content" @@ -488,7 +485,6 @@ "setups": [ { "fileFormat": "pdf", - "flatten_media_to_text": false, "parse_method": "DeepDOC", "preprocess": [ "main_content" @@ -496,7 +492,6 @@ }, { "fileFormat": "markdown", - "flatten_media_to_text": false, "preprocess": [ "main_content" ] @@ -521,7 +516,6 @@ }, { "fileFormat": "docx", - "flatten_media_to_text": false, "preprocess": [ "main_content" ] diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_manual.json b/internal/ingestion/pipeline/template/ingestion_pipeline_manual.json index f239ccba37..cce3776c52 100644 --- a/internal/ingestion/pipeline/template/ingestion_pipeline_manual.json +++ b/internal/ingestion/pipeline/template/ingestion_pipeline_manual.json @@ -62,7 +62,6 @@ ] }, "docx": { - "flatten_media_to_text": false, "preprocess": [ "main_content" ], @@ -72,7 +71,6 @@ "vlm": {} }, "pdf": { - "flatten_media_to_text": false, "parse_method": "DeepDOC", "preprocess": [ "main_content" @@ -455,7 +453,6 @@ "setups": [ { "fileFormat": "pdf", - "flatten_media_to_text": false, "parse_method": "DeepDOC", "preprocess": [ "main_content" @@ -469,7 +466,6 @@ }, { "fileFormat": "docx", - "flatten_media_to_text": false, "preprocess": [ "main_content" ] diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_one.json b/internal/ingestion/pipeline/template/ingestion_pipeline_one.json index c32e8a71b5..0429def36d 100644 --- a/internal/ingestion/pipeline/template/ingestion_pipeline_one.json +++ b/internal/ingestion/pipeline/template/ingestion_pipeline_one.json @@ -62,7 +62,6 @@ ] }, "docx": { - "flatten_media_to_text": false, "preprocess": [ "main_content" ], @@ -81,7 +80,6 @@ ] }, "markdown": { - "flatten_media_to_text": false, "preprocess": [ "main_content" ], @@ -93,7 +91,6 @@ "vlm": {} }, "pdf": { - "flatten_media_to_text": false, "parse_method": "DeepDOC", "preprocess": [ "main_content" @@ -104,7 +101,6 @@ "vlm": {} }, "spreadsheet": { - "flatten_media_to_text": false, "parse_method": "DeepDOC", "preprocess": [ "main_content" @@ -280,7 +276,6 @@ "setups": [ { "fileFormat": "pdf", - "flatten_media_to_text": false, "parse_method": "DeepDOC", "preprocess": [ "main_content" @@ -288,7 +283,6 @@ }, { "fileFormat": "spreadsheet", - "flatten_media_to_text": false, "parse_method": "DeepDOC", "preprocess": [ "main_content" @@ -296,7 +290,6 @@ }, { "fileFormat": "markdown", - "flatten_media_to_text": false, "preprocess": [ "main_content" ] @@ -321,7 +314,6 @@ }, { "fileFormat": "docx", - "flatten_media_to_text": false, "preprocess": [ "main_content" ] diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_paper.json b/internal/ingestion/pipeline/template/ingestion_pipeline_paper.json index a38c8522d7..2d288b8b07 100644 --- a/internal/ingestion/pipeline/template/ingestion_pipeline_paper.json +++ b/internal/ingestion/pipeline/template/ingestion_pipeline_paper.json @@ -55,7 +55,6 @@ }, "pdf": { "enable_multi_column": true, - "flatten_media_to_text": false, "parse_method": "DeepDOC", "preprocess": [ "main_content" @@ -442,7 +441,6 @@ { "enable_multi_column": true, "fileFormat": "pdf", - "flatten_media_to_text": false, "parse_method": "DeepDOC", "preprocess": [ "main_content" diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_presentation.json b/internal/ingestion/pipeline/template/ingestion_pipeline_presentation.json index 6cea29ef4f..14fff21e90 100644 --- a/internal/ingestion/pipeline/template/ingestion_pipeline_presentation.json +++ b/internal/ingestion/pipeline/template/ingestion_pipeline_presentation.json @@ -54,7 +54,6 @@ } }, "pdf": { - "flatten_media_to_text": false, "parse_method": "DeepDOC", "preprocess": [ "main_content" @@ -233,7 +232,6 @@ "setups": [ { "fileFormat": "pdf", - "flatten_media_to_text": false, "parse_method": "DeepDOC", "preprocess": [ "main_content" diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_qa.json b/internal/ingestion/pipeline/template/ingestion_pipeline_qa.json index 910156bffe..27eee1d37e 100644 --- a/internal/ingestion/pipeline/template/ingestion_pipeline_qa.json +++ b/internal/ingestion/pipeline/template/ingestion_pipeline_qa.json @@ -54,7 +54,6 @@ } }, "docx": { - "flatten_media_to_text": false, "preprocess": [ "main_content" ], @@ -64,7 +63,6 @@ "vlm": {} }, "markdown": { - "flatten_media_to_text": false, "preprocess": [ "main_content" ], @@ -76,7 +74,6 @@ "vlm": {} }, "pdf": { - "flatten_media_to_text": false, "parse_method": "DeepDOC", "preprocess": [ "main_content" @@ -87,7 +84,6 @@ "vlm": {} }, "spreadsheet": { - "flatten_media_to_text": false, "parse_method": "DeepDOC", "preprocess": [ "main_content" @@ -225,7 +221,6 @@ "setups": [ { "fileFormat": "pdf", - "flatten_media_to_text": false, "parse_method": "DeepDOC", "preprocess": [ "main_content" @@ -234,7 +229,6 @@ }, { "fileFormat": "spreadsheet", - "flatten_media_to_text": false, "parse_method": "DeepDOC", "preprocess": [ "main_content" @@ -243,7 +237,6 @@ }, { "fileFormat": "markdown", - "flatten_media_to_text": false, "preprocess": [ "main_content" ], @@ -257,7 +250,6 @@ }, { "fileFormat": "docx", - "flatten_media_to_text": false, "preprocess": [ "main_content" ], diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_table.json b/internal/ingestion/pipeline/template/ingestion_pipeline_table.json index 3da45d9c6f..844389e879 100644 --- a/internal/ingestion/pipeline/template/ingestion_pipeline_table.json +++ b/internal/ingestion/pipeline/template/ingestion_pipeline_table.json @@ -54,7 +54,6 @@ } }, "spreadsheet": { - "flatten_media_to_text": false, "parse_method": "DeepDOC", "preprocess": [ "main_content" @@ -195,7 +194,6 @@ "setups": [ { "fileFormat": "spreadsheet", - "flatten_media_to_text": false, "parse_method": "DeepDOC", "preprocess": [ "main_content" diff --git a/internal/ingestion/task/pipeline_executor_defaults_test.go b/internal/ingestion/task/pipeline_executor_defaults_test.go index 5a0699b5b3..421c8a23e7 100644 --- a/internal/ingestion/task/pipeline_executor_defaults_test.go +++ b/internal/ingestion/task/pipeline_executor_defaults_test.go @@ -39,17 +39,17 @@ import ( // serialization order inside each entry is irrelevant. var builtinComponentParamsGolden = map[string]string{ "audio": "{\"File\": {}, \"Parser:SongsFillAir\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"audio\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"aac\", \"aiff\", \"ape\", \"au\", \"da\", \"flac\", \"midi\", \"mp3\", \"ogg\", \"oggvorbis\", \"realaudio\", \"vqf\", \"wav\", \"wave\", \"wma\"]}}, \"OneChunker:BlueSkiesLaugh\": {}, \"Tokenizer:KindEyesWatch\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}", - "book": "{\"File\":{},\"Parser:HipSignsRhyme\":{\"enable_vision_enhancement\":false,\"vlm\":{\"llm_id\":\"\"},\"doc\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"doc\"]},\"docx\":{\"flatten_media_to_text\":false,\"preprocess\":[\"main_content\"],\"suffix\":[\"docx\"],\"vlm\":{}},\"html\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"htm\",\"html\"]},\"pdf\":{\"flatten_media_to_text\":false,\"parse_method\":\"DeepDOC\",\"preprocess\":[\"main_content\"],\"remove_toc\":true,\"suffix\":[\"pdf\"],\"vlm\":{}},\"text&code\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"txt\"]}},\"TitleChunker:GrumpyGarlicsBake\":{\"hierarchy\":5,\"include_heading_content\":true,\"levels\":[[\"^#[^#]\",\"^##[^#]\",\"^###[^#]\",\"^####[^#]\"],[\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\",\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"第[零一二三四五六七八九十百0-9]+条\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"],[\"第[0-9]+章\",\"第[0-9]+节\",\"[0-9]{1,2}[\\\\. 、]\",\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\",\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"],[\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"[零一二三四五六七八九十百]+[ 、]\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\",\"[\\\\(（][0-9]{,2}[\\\\)）]\"],[\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\",\"Chapter (I+V?|VI*|XI|IX|X)\",\"Section [0-9]+\",\"Article [0-9]+\"]],\"hierarchyRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"groupRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"hierarchyHierarchy\":\"5\",\"hierarchyGroup\":\"0\",\"method\":\"hierarchy\"},\"Tokenizer:HotDonutsRing\":{\"fields\":\"text\",\"filename_embd_weight\":0.1,\"search_method\":[\"embedding\",\"full_text\"]},\"Extractor:AutoExtractDefault\":{\"llm_id\":\"\",\"keywords\":{\"top_n\":0},\"questions\":{\"top_n\":0},\"tags\":{\"top_n\":0,\"tag_file_id\":\"\"},\"summary\":{\"enabled\":false},\"metadata\":{\"enabled\":false,\"metadata\":[],\"built_in_metadata\":[]}}}", + "book": "{\"File\":{},\"Parser:HipSignsRhyme\":{\"enable_vision_enhancement\":false,\"vlm\":{\"llm_id\":\"\"},\"doc\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"doc\"]},\"docx\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"docx\"],\"vlm\":{}},\"html\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"htm\",\"html\"]},\"pdf\":{\"parse_method\":\"DeepDOC\",\"preprocess\":[\"main_content\"],\"remove_toc\":true,\"suffix\":[\"pdf\"],\"vlm\":{}},\"text&code\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"txt\"]}},\"TitleChunker:GrumpyGarlicsBake\":{\"hierarchy\":5,\"include_heading_content\":true,\"levels\":[[\"^#[^#]\",\"^##[^#]\",\"^###[^#]\",\"^####[^#]\"],[\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\",\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"第[零一二三四五六七八九十百0-9]+条\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"],[\"第[0-9]+章\",\"第[0-9]+节\",\"[0-9]{1,2}[\\\\. 、]\",\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\",\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"],[\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"[零一二三四五六七八九十百]+[ 、]\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\",\"[\\\\(（][0-9]{,2}[\\\\)）]\"],[\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\",\"Chapter (I+V?|VI*|XI|IX|X)\",\"Section [0-9]+\",\"Article [0-9]+\"]],\"hierarchyRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"groupRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"hierarchyHierarchy\":\"5\",\"hierarchyGroup\":\"0\",\"method\":\"hierarchy\"},\"Tokenizer:HotDonutsRing\":{\"fields\":\"text\",\"filename_embd_weight\":0.1,\"search_method\":[\"embedding\",\"full_text\"]},\"Extractor:AutoExtractDefault\":{\"llm_id\":\"\",\"keywords\":{\"top_n\":0},\"questions\":{\"top_n\":0},\"tags\":{\"top_n\":0,\"tag_file_id\":\"\"},\"summary\":{\"enabled\":false},\"metadata\":{\"enabled\":false,\"metadata\":[],\"built_in_metadata\":[]}}}", "email": "{\"File\": {}, \"Parser:BirdsFlutterHigh\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"email\": {\"fields\": [\"from\", \"to\", \"cc\", \"bcc\", \"date\", \"subject\", \"body\", \"attachments\"], \"preprocess\": [\"main_content\"], \"suffix\": [\"eml\"]}}, \"TokenChunker:WarmBreadSmells\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"delimiter\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \"；\", \"！\", \"？\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:NiceWordsSpoken\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}", - "general": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"doc\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"pages\": [[1, 100000]], \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"txt\", \"py\", \"js\", \"java\", \"c\", \"cpp\", \"h\", \"php\", \"go\", \"ts\", \"sh\", \"cs\", \"kt\", \"sql\"]}, \"image\": {\"ocr_enabled\": true, \"preprocess\": [\"main_content\"], \"suffix\": [\"bmp\", \"gif\", \"jpeg\", \"jpg\", \"png\", \"svg\", \"tif\", \"tiff\", \"webp\"]}, \"email\": {\"fields\": [\"from\", \"to\", \"cc\", \"bcc\", \"date\", \"subject\", \"body\", \"attachments\"], \"preprocess\": [\"main_content\"], \"suffix\": [\"eml\", \"msg\"]}}, \"GeneralChunker:SixApplesFall\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiters\": [\"\\n\", \"!\", \"?\", \";\", \"。\", \"；\", \"！\", \"？\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:LegalReadersDecide\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}", - "laws": "{\"File\":{},\"Parser:HipSignsRhyme\":{\"enable_vision_enhancement\":false,\"vlm\":{\"llm_id\":\"\"},\"doc\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"doc\"]},\"docx\":{\"flatten_media_to_text\":false,\"preprocess\":[\"main_content\"],\"suffix\":[\"docx\"],\"vlm\":{}},\"html\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"htm\",\"html\"]},\"markdown\":{\"flatten_media_to_text\":false,\"preprocess\":[\"main_content\"],\"suffix\":[\"md\",\"markdown\",\"mdx\"],\"vlm\":{}},\"pdf\":{\"flatten_media_to_text\":false,\"parse_method\":\"DeepDOC\",\"preprocess\":[\"main_content\"],\"remove_toc\":true,\"suffix\":[\"pdf\"],\"vlm\":{}},\"text&code\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"txt\"]}},\"TitleChunker:SpicyKeysKick\":{\"hierarchy\":2,\"include_heading_content\":false,\"levels\":[[\"^#[^#]\",\"^##[^#]\",\"^###[^#]\",\"^####[^#]\"],[\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\",\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"第[零一二三四五六七八九十百0-9]+条\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"],[\"第[0-9]+章\",\"第[0-9]+节\",\"[0-9]{1,2}[\\\\. 、]\",\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\",\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"],[\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"[零一二三四五六七八九十百]+[ 、]\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\",\"[\\\\(（][0-9]{,2}[\\\\)）]\"],[\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\",\"Chapter (I+V?|VI*|XI|IX|X)\",\"Section [0-9]+\",\"Article [0-9]+\"]],\"hierarchyRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"groupRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"hierarchyHierarchy\":\"2\",\"hierarchyGroup\":\"0\",\"method\":\"hierarchy\"},\"Tokenizer:PublicJobsTake\":{\"fields\":\"text\",\"filename_embd_weight\":0.1,\"search_method\":[\"embedding\",\"full_text\"]},\"Extractor:AutoExtractDefault\":{\"llm_id\":\"\",\"keywords\":{\"top_n\":0},\"questions\":{\"top_n\":0},\"tags\":{\"top_n\":0,\"tag_file_id\":\"\"},\"summary\":{\"enabled\":false},\"metadata\":{\"enabled\":false,\"metadata\":[],\"built_in_metadata\":[]}}}", - "manual": "{\"File\":{},\"Parser:HipSignsRhyme\":{\"enable_vision_enhancement\":false,\"vlm\":{\"llm_id\":\"\"},\"doc\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"doc\"]},\"docx\":{\"flatten_media_to_text\":false,\"preprocess\":[\"main_content\"],\"suffix\":[\"docx\"],\"vlm\":{}},\"pdf\":{\"flatten_media_to_text\":false,\"parse_method\":\"DeepDOC\",\"preprocess\":[\"main_content\"],\"suffix\":[\"pdf\"],\"vlm\":{}}},\"ManualChunker:NineInsectsFind\":{\"hierarchy\":0,\"include_heading_content\":false,\"levels\":[[\"^#[^#]\",\"^##[^#]\",\"^###[^#]\",\"^####[^#]\"],[\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\",\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"第[零一二三四五六七八九十百0-9]+条\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"],[\"第[0-9]+章\",\"第[0-9]+节\",\"[0-9]{1,2}[\\\\. 、]\",\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\",\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"],[\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"[零一二三四五六七八九十百]+[ 、]\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\",\"[\\\\(（][0-9]{,2}[\\\\)）]\"],[\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\",\"Chapter (I+V?|VI*|XI|IX|X)\",\"Section [0-9]+\",\"Article [0-9]+\"]],\"hierarchyRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"groupRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"hierarchyHierarchy\":\"0\",\"hierarchyGroup\":\"0\",\"method\":\"group\"},\"Tokenizer:FunnyBalloonsGrin\":{\"fields\":\"text\",\"filename_embd_weight\":0.1,\"search_method\":[\"embedding\",\"full_text\"]},\"Extractor:AutoExtractDefault\":{\"llm_id\":\"\",\"keywords\":{\"top_n\":0},\"questions\":{\"top_n\":0},\"tags\":{\"top_n\":0,\"tag_file_id\":\"\"},\"summary\":{\"enabled\":false},\"metadata\":{\"enabled\":false,\"metadata\":[],\"built_in_metadata\":[]}}}", - "one": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"doc\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\"], \"vlm\": {}}, \"text&code\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"OneChunker:DryDrinksVisit\": {}, \"Tokenizer:FrankWeeksListen\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}", - "paper": "{\"File\":{},\"Parser:HipSignsRhyme\":{\"enable_vision_enhancement\":false,\"vlm\":{\"llm_id\":\"\"},\"pdf\":{\"enable_multi_column\":true,\"flatten_media_to_text\":false,\"parse_method\":\"DeepDOC\",\"preprocess\":[\"main_content\"],\"suffix\":[\"pdf\"],\"vlm\":{}}},\"TitleChunker:SparklySchoolsTravel\":{\"hierarchy\":0,\"include_heading_content\":false,\"levels\":[[\"^#[^#]\",\"^##[^#]\",\"^###[^#]\",\"^####[^#]\"],[\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\",\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"第[零一二三四五六七八九十百0-9]+条\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"],[\"第[0-9]+章\",\"第[0-9]+节\",\"[0-9]{1,2}[\\\\. 、]\",\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\",\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"],[\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"[零一二三四五六七八九十百]+[ 、]\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\",\"[\\\\(（][0-9]{,2}[\\\\)）]\"],[\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\",\"Chapter (I+V?|VI*|XI|IX|X)\",\"Section [0-9]+\",\"Article [0-9]+\"]],\"hierarchyRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"groupRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"hierarchyHierarchy\":\"0\",\"hierarchyGroup\":\"0\",\"method\":\"group\"},\"Tokenizer:GreatCarsWash\":{\"fields\":\"text\",\"filename_embd_weight\":0.1,\"search_method\":[\"embedding\",\"full_text\"]},\"Extractor:AutoExtractDefault\":{\"llm_id\":\"\",\"keywords\":{\"top_n\":0},\"questions\":{\"top_n\":0},\"tags\":{\"top_n\":0,\"tag_file_id\":\"\"},\"summary\":{\"enabled\":false},\"metadata\":{\"enabled\":false,\"metadata\":[],\"built_in_metadata\":[]}}}", + "general": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"doc\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"pages\": [[1, 100000]], \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"txt\", \"py\", \"js\", \"java\", \"c\", \"cpp\", \"h\", \"php\", \"go\", \"ts\", \"sh\", \"cs\", \"kt\", \"sql\"]}, \"image\": {\"ocr_enabled\": true, \"preprocess\": [\"main_content\"], \"suffix\": [\"bmp\", \"gif\", \"jpeg\", \"jpg\", \"png\", \"svg\", \"tif\", \"tiff\", \"webp\"]}, \"email\": {\"fields\": [\"from\", \"to\", \"cc\", \"bcc\", \"date\", \"subject\", \"body\", \"attachments\"], \"preprocess\": [\"main_content\"], \"suffix\": [\"eml\", \"msg\"]}}, \"GeneralChunker:SixApplesFall\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiters\": [\"\\n\", \"!\", \"?\", \";\", \"。\", \"；\", \"！\", \"？\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:LegalReadersDecide\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}", + "laws": "{\"File\":{},\"Parser:HipSignsRhyme\":{\"enable_vision_enhancement\":false,\"vlm\":{\"llm_id\":\"\"},\"doc\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"doc\"]},\"docx\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"docx\"],\"vlm\":{}},\"html\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"htm\",\"html\"]},\"markdown\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"md\",\"markdown\",\"mdx\"],\"vlm\":{}},\"pdf\":{\"parse_method\":\"DeepDOC\",\"preprocess\":[\"main_content\"],\"remove_toc\":true,\"suffix\":[\"pdf\"],\"vlm\":{}},\"text&code\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"txt\"]}},\"TitleChunker:SpicyKeysKick\":{\"hierarchy\":2,\"include_heading_content\":false,\"levels\":[[\"^#[^#]\",\"^##[^#]\",\"^###[^#]\",\"^####[^#]\"],[\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\",\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"第[零一二三四五六七八九十百0-9]+条\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"],[\"第[0-9]+章\",\"第[0-9]+节\",\"[0-9]{1,2}[\\\\. 、]\",\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\",\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"],[\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"[零一二三四五六七八九十百]+[ 、]\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\",\"[\\\\(（][0-9]{,2}[\\\\)）]\"],[\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\",\"Chapter (I+V?|VI*|XI|IX|X)\",\"Section [0-9]+\",\"Article [0-9]+\"]],\"hierarchyRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"groupRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"hierarchyHierarchy\":\"2\",\"hierarchyGroup\":\"0\",\"method\":\"hierarchy\"},\"Tokenizer:PublicJobsTake\":{\"fields\":\"text\",\"filename_embd_weight\":0.1,\"search_method\":[\"embedding\",\"full_text\"]},\"Extractor:AutoExtractDefault\":{\"llm_id\":\"\",\"keywords\":{\"top_n\":0},\"questions\":{\"top_n\":0},\"tags\":{\"top_n\":0,\"tag_file_id\":\"\"},\"summary\":{\"enabled\":false},\"metadata\":{\"enabled\":false,\"metadata\":[],\"built_in_metadata\":[]}}}", + "manual": "{\"File\":{},\"Parser:HipSignsRhyme\":{\"enable_vision_enhancement\":false,\"vlm\":{\"llm_id\":\"\"},\"doc\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"doc\"]},\"docx\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"docx\"],\"vlm\":{}},\"pdf\":{\"parse_method\":\"DeepDOC\",\"preprocess\":[\"main_content\"],\"suffix\":[\"pdf\"],\"vlm\":{}}},\"ManualChunker:NineInsectsFind\":{\"hierarchy\":0,\"include_heading_content\":false,\"levels\":[[\"^#[^#]\",\"^##[^#]\",\"^###[^#]\",\"^####[^#]\"],[\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\",\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"第[零一二三四五六七八九十百0-9]+条\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"],[\"第[0-9]+章\",\"第[0-9]+节\",\"[0-9]{1,2}[\\\\. 、]\",\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\",\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"],[\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"[零一二三四五六七八九十百]+[ 、]\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\",\"[\\\\(（][0-9]{,2}[\\\\)）]\"],[\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\",\"Chapter (I+V?|VI*|XI|IX|X)\",\"Section [0-9]+\",\"Article [0-9]+\"]],\"hierarchyRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"groupRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"hierarchyHierarchy\":\"0\",\"hierarchyGroup\":\"0\",\"method\":\"group\"},\"Tokenizer:FunnyBalloonsGrin\":{\"fields\":\"text\",\"filename_embd_weight\":0.1,\"search_method\":[\"embedding\",\"full_text\"]},\"Extractor:AutoExtractDefault\":{\"llm_id\":\"\",\"keywords\":{\"top_n\":0},\"questions\":{\"top_n\":0},\"tags\":{\"top_n\":0,\"tag_file_id\":\"\"},\"summary\":{\"enabled\":false},\"metadata\":{\"enabled\":false,\"metadata\":[],\"built_in_metadata\":[]}}}", + "one": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"doc\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\"], \"vlm\": {}}, \"text&code\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"OneChunker:DryDrinksVisit\": {}, \"Tokenizer:FrankWeeksListen\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}", + "paper": "{\"File\":{},\"Parser:HipSignsRhyme\":{\"enable_vision_enhancement\":false,\"vlm\":{\"llm_id\":\"\"},\"pdf\":{\"enable_multi_column\":true,\"parse_method\":\"DeepDOC\",\"preprocess\":[\"main_content\"],\"suffix\":[\"pdf\"],\"vlm\":{}}},\"TitleChunker:SparklySchoolsTravel\":{\"hierarchy\":0,\"include_heading_content\":false,\"levels\":[[\"^#[^#]\",\"^##[^#]\",\"^###[^#]\",\"^####[^#]\"],[\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\",\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"第[零一二三四五六七八九十百0-9]+条\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"],[\"第[0-9]+章\",\"第[0-9]+节\",\"[0-9]{1,2}[\\\\. 、]\",\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\",\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"],[\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"[零一二三四五六七八九十百]+[ 、]\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\",\"[\\\\(（][0-9]{,2}[\\\\)）]\"],[\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\",\"Chapter (I+V?|VI*|XI|IX|X)\",\"Section [0-9]+\",\"Article [0-9]+\"]],\"hierarchyRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"groupRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"hierarchyHierarchy\":\"0\",\"hierarchyGroup\":\"0\",\"method\":\"group\"},\"Tokenizer:GreatCarsWash\":{\"fields\":\"text\",\"filename_embd_weight\":0.1,\"search_method\":[\"embedding\",\"full_text\"]},\"Extractor:AutoExtractDefault\":{\"llm_id\":\"\",\"keywords\":{\"top_n\":0},\"questions\":{\"top_n\":0},\"tags\":{\"top_n\":0,\"tag_file_id\":\"\"},\"summary\":{\"enabled\":false},\"metadata\":{\"enabled\":false,\"metadata\":[],\"built_in_metadata\":[]}}}", "picture": "{\"File\": {}, \"Parser:ViewsCaptureLight\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"image\": {\"ocr_enabled\": true, \"preprocess\": [\"main_content\"], \"suffix\": [\"bmp\", \"gif\", \"jpeg\", \"jpg\", \"png\", \"svg\", \"tif\", \"tiff\", \"webp\"]}, \"video\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"3gp\", \"3gpp\", \"avi\", \"flv\", \"mkv\", \"mov\", \"mp4\", \"mpeg\", \"mpg\", \"webm\", \"wmv\"]}}, \"OneChunker:BrightColorsGlow\": {}, \"Tokenizer:SharpLensFocus\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}", - "presentation": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"pdf\": {\"flatten_media_to_text\": false, \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"slides\": {\"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pptx\", \"ppt\", \"dps\", \"dpt\"]}}, \"Tokenizer:TallTreesDance\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"PageChunker:HappyHillsGlow\": {}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}", - "qa": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"docx\": {\"flatten_media_to_text\": false, \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"markdown\": {\"flatten_media_to_text\": false, \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"Tokenizer:ColdCloudsDream\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"QAChunker:TidyCloudsThink\": {}}", - "table": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}, \"column_mode\": \"auto\", \"column_roles\": {}, \"column_names\": []}, \"text&code\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"TableChunker:FastFoxesJump\": {}, \"Tokenizer:DeepLakesShine\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}}", + "presentation": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"pdf\": {\"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"slides\": {\"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pptx\", \"ppt\", \"dps\", \"dpt\"]}}, \"Tokenizer:TallTreesDance\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"PageChunker:HappyHillsGlow\": {}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}", + "qa": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"docx\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"markdown\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"Tokenizer:ColdCloudsDream\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"QAChunker:TidyCloudsThink\": {}}", + "table": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"spreadsheet\": {\"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}, \"column_mode\": \"auto\", \"column_roles\": {}, \"column_names\": []}, \"text&code\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"TableChunker:FastFoxesJump\": {}, \"Tokenizer:DeepLakesShine\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}}", } // Per-template test methods. Each resolves default component params from a diff --git a/internal/parser/parser/markdown_media_test.go b/internal/parser/parser/markdown_media_test.go index d3028d9590..2fcbf500ec 100644 --- a/internal/parser/parser/markdown_media_test.go +++ b/internal/parser/parser/markdown_media_test.go @@ -374,29 +374,19 @@ func TestPDFMarkdownKeepsInlineDataURIImage(t *testing.T) { } } -func TestMarkdownParserIngestionFlattenKeepsImageAsText(t *testing.T) { +func TestMarkdownParserIngestionKeepsImageMedia(t *testing.T) { ctx, cache := recordingMediaContext() - p := markdownIngestionParser(t) - markdown, ok := p.(*MarkdownParser) - if !ok { - t.Fatalf("GetParser(FileTypeMarkdown) = %T", p) - } - markdown.FlattenMediaToText = true - - res := p.ParseWithResult(ctx, "flatten.md", []byte("![figure]("+markdownMediaDataURI+")\n")) + res := markdownIngestionParser(t).ParseWithResult(ctx, "image.md", []byte("![figure]("+markdownMediaDataURI+")\n")) if res.Err != nil { - t.Fatalf("ParseWithResult: %v", res.Err) + t.Fatal(res.Err) + } + items := ItemsToMaps(res.JSON) + if len(items) != 1 || items[0]["doc_type_kwd"] != DocTypeImage || entity.MediaRefFromAny(items[0]["media_ref"]) == nil { + t.Fatalf("image item lost its type or media reference: %+v", items) } if len(cache.puts) != 1 || string(cache.puts[0].data) != "hello" { t.Fatalf("cache puts = %+v", cache.puts) } - items := ItemsToMaps(res.JSON) - if len(items) != 1 || items[0]["doc_type_kwd"] != DocTypeText { - t.Fatalf("flattened items = %+v, want one text item", items) - } - if entity.MediaRefFromAny(items[0]["media_ref"]) == nil { - t.Fatalf("flattened image lost its media_ref: %+v", items[0]) - } } // TestMarkdownParserIngestionCountsInvalidDataURIOnce pins the shared source diff --git a/internal/parser/parser/markdown_parser.go b/internal/parser/parser/markdown_parser.go index 260d045e63..ebc1a12978 100644 --- a/internal/parser/parser/markdown_parser.go +++ b/internal/parser/parser/markdown_parser.go @@ -42,10 +42,9 @@ const dataURIPrefix = "data:image/" const GoMarkdown = "go_markdown" type MarkdownParser struct { - libType string - ParseMethod string - VLM map[string]any - FlattenMediaToText bool + libType string + ParseMethod string + VLM map[string]any // FetchRemoteImages controls whether HTTP(S) Markdown images are // downloaded. NewMarkdownParser enables it for direct Markdown parsing; // callers normalizing untrusted backend responses may disable it. @@ -81,9 +80,6 @@ func (p *MarkdownParser) ConfigureFromSetup(setup map[string]any) { if v, ok := setup["vlm"].(map[string]any); ok { p.VLM = v } - if v, ok := setup["flatten_media_to_text"].(bool); ok { - p.FlattenMediaToText = v - } } // ParseWithResult implements ParseResultProducer (plan §6.5) and @@ -179,7 +175,7 @@ func (p *MarkdownParser) parseWithState(ctx context.Context, filename string, da state.resolveMedia = markdownMediaResolver(state, walkCtx, p.FetchRemoteImages) defer func() { state.resolveMedia = previousMediaResolver }() } - unresolvedImages := walkMarkdownBlocksWithImages(walkCtx, doc, &items, p.FlattenMediaToText, p.FetchRemoteImages, state) + unresolvedImages := walkMarkdownBlocksWithImages(walkCtx, doc, &items, p.FetchRemoteImages, state) if state.err != nil { return ParseResult{Err: state.err} } @@ -463,22 +459,12 @@ func markdownTableCells(line string) []string { // findBlockImage (per-block AST walk) and the item carries // `doc_type_kwd: "image"` together with either a run-cache reference // (refOnly), the base64-encoded payload (inline mode), or nothing when the -// source was skipped. When flatten is true, image items use doc_type_kwd="text". +// source was skipped. // // Tables are emitted through the HTML walker as structured items in source // order, with embedded images linked to their parent table. -func walkMarkdownBlocksWithImages(ctx context.Context, doc ast.Node, out *[]map[string]any, flatten, fetchRemoteImages bool, htmlState *htmlWalkState) int { +func walkMarkdownBlocksWithImages(ctx context.Context, doc ast.Node, out *[]map[string]any, fetchRemoteImages bool, htmlState *htmlWalkState) int { budget := htmlState.budget - start := len(*out) - defer func() { - if flatten { - for _, item := range (*out)[start:] { - if item["doc_type_kwd"] == "image" { - item["doc_type_kwd"] = "text" - } - } - } - }() unresolvedImages := 0 previousResolver := htmlState.resolveImage if !htmlState.refOnly && !htmlState.skipImages { @@ -534,7 +520,7 @@ func walkMarkdownBlocksWithImages(ctx context.Context, doc ast.Node, out *[]map[ for i, entry := range n.GetChildren() { if containsMarkdownTable(entry) { start := len(*out) - unresolvedImages += walkMarkdownBlocksWithImages(ctx, entry, out, flatten, fetchRemoteImages, htmlState) + unresolvedImages += walkMarkdownBlocksWithImages(ctx, entry, out, fetchRemoteImages, htmlState) for _, item := range (*out)[start:] { if item["doc_type_kwd"] == "text" && item["layout"] == nil { item["layout"] = "list" @@ -554,7 +540,7 @@ func walkMarkdownBlocksWithImages(ctx context.Context, doc ast.Node, out *[]map[ continue case *ast.BlockQuote: if containsMarkdownTable(n) { - unresolvedImages += walkMarkdownBlocksWithImages(ctx, n, out, flatten, fetchRemoteImages, htmlState) + unresolvedImages += walkMarkdownBlocksWithImages(ctx, n, out, fetchRemoteImages, htmlState) continue } txt = leafText(n) @@ -591,9 +577,7 @@ func walkMarkdownBlocksWithImages(ctx context.Context, doc ast.Node, out *[]map[ // the image payload (and doc_type_kwd:"image") is attached to the // single block that actually contains the ![alt](src) reference. // Scanning the whole document (the old approach) wrongly tagged - // every block as an image whenever any image was present. When - // flatten is true, keep doc_type_kwd="text" (Python - // parser.py:1034: flatten_media_to_text overrides image). + // every block as an image whenever any image was present. if !htmlState.skipImages { if imgURL, ok := findBlockImage(child); ok { if htmlState.refOnly { diff --git a/internal/parser/parser/markdown_parser_test.go b/internal/parser/parser/markdown_parser_test.go index f7b116fce0..7cb7eff46e 100644 --- a/internal/parser/parser/markdown_parser_test.go +++ b/internal/parser/parser/markdown_parser_test.go @@ -513,9 +513,8 @@ func TestMarkdownParser_ParseWithResult_InlineRawHTMLSurvives(t *testing.T) { func TestMarkdownParser_ConfigureFromSetup(t *testing.T) { p, _ := NewMarkdownParser(GoMarkdown) p.ConfigureFromSetup(map[string]any{ - "parse_method": "deepdoc", - "vlm": map[string]any{"llm_id": "gpt-4-vision"}, - "flatten_media_to_text": false, + "parse_method": "deepdoc", + "vlm": map[string]any{"llm_id": "gpt-4-vision"}, }) if p.ParseMethod != "deepdoc" { t.Fatalf("ParseMethod = %q, want %q", p.ParseMethod, "deepdoc") @@ -536,44 +535,6 @@ func TestMarkdownParser_ConfigureFromSetup_NilSafe(t *testing.T) { } } -// TestMarkdownParser_FlattenMediaToText verifies that when -// flatten_media_to_text is true, image items are emitted with -// doc_type_kwd="text" (mirroring Python parser.py:1034). When false, -// image items keep doc_type_kwd="image". -func TestMarkdownParser_FlattenMediaToText(t *testing.T) { - ctx := t.Context() - pixelB64 := base64.StdEncoding.EncodeToString([]byte("fake-png-data")) - md := "Some text with an image\n![test](data:image/png;base64," + pixelB64 + ")\n" - - cases := []struct { - name string - flatten bool - wantDocType string - }{ - {"flatten=false keeps image", false, "image"}, - {"flatten=true forces text", true, "text"}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - p, _ := NewMarkdownParser(GoMarkdown) - p.ConfigureFromSetup(map[string]any{"flatten_media_to_text": tc.flatten}) - if p.FlattenMediaToText != tc.flatten { - t.Fatalf("FlattenMediaToText field = %v, want %v", p.FlattenMediaToText, tc.flatten) - } - res := p.ParseWithResult(ctx, "test.md", []byte(md)) - if res.Err != nil { - t.Fatalf("ParseWithResult: %v", res.Err) - } - for _, item := range ItemsToMaps(res.JSON) { - if kd, _ := item["doc_type_kwd"].(string); kd != tc.wantDocType { - t.Errorf("doc_type_kwd = %q, want %q (item text=%q)", - kd, tc.wantDocType, item["text"]) - } - } - }) - } -} - func TestResolveImageURL_DataURI(t *testing.T) { b64 := base64.StdEncoding.EncodeToString([]byte("fakeimage")) budget := newLimitedMediaBudgetForTest(t, len("fakeimage"), int64(len("fakeimage")), 2) @@ -644,7 +605,7 @@ func TestMarkdownImageBudgetSkipsFetchAfterCountLimit(t *testing.T) { doc := markdownNew().Parse([]byte(markdown)) budget := newLimitedMediaBudgetForTest(t, 4, 4, 2) var items []map[string]any - if unresolved := walkMarkdownBlocksWithImages(t.Context(), doc, &items, false, true, &htmlWalkState{budget: budget}); unresolved != 0 { + if unresolved := walkMarkdownBlocksWithImages(t.Context(), doc, &items, true, &htmlWalkState{budget: budget}); unresolved != 0 { t.Fatalf("unresolved images = %d, want 0", unresolved) } if got := fetches.Load(); got != 2 { diff --git a/internal/parser/parser/pdf_parser_common.go b/internal/parser/parser/pdf_parser_common.go index 5e79224fb3..52fc835017 100644 --- a/internal/parser/parser/pdf_parser_common.go +++ b/internal/parser/parser/pdf_parser_common.go @@ -59,7 +59,6 @@ type PDFParser struct { Model string // DeepDoc@buildin@ragflow LibType string // pdf_oxide, used by DeepDoc - FlattenMediaToText bool RemoveTOC bool RemoveHeaderFooter bool EnableMultiColumn bool @@ -142,9 +141,6 @@ func (p *PDFParser) ConfigureFromSetup(setup map[string]any) { if p == nil || setup == nil { return } - if v, ok := setup["flatten_media_to_text"].(bool); ok { - p.FlattenMediaToText = v - } if v, ok := setup["remove_toc"].(bool); ok { p.RemoveTOC = v } @@ -404,10 +400,9 @@ func (p *PDFParser) parseWithResult(ctx context.Context, filename string, data [ cfg.RemoveHeaderFooter = p.RemoveHeaderFooter cfg.OnPageDone = p.OnPageDone res := parsePDFWithDeepDocOptions(ctx, filename, data, pdfPostProcessOptions{ - zoom: cfg.Zoom, - enableMultiColumn: p.EnableMultiColumn, - flattenMediaToText: p.FlattenMediaToText, - parseMethod: p.ParseMethod, + zoom: cfg.Zoom, + enableMultiColumn: p.EnableMultiColumn, + parseMethod: p.ParseMethod, }, deepdocpdf.NewParser(cfg).Parse) if res.Err != nil { if errors.Is(res.Err, deepdocpdf.ErrNoPDFData) || @@ -555,8 +550,6 @@ func pdfParseResultToJSONWithOptions(filename string, parsed *deepdoctype.ParseR if opts.enableMultiColumn && opts.pageWidth <= 0 { opts.pageWidth = firstPDFPageWidth(processed.PageWidth) } - // JSON media policy is applied once at the PDFParser result boundary. - opts.flattenMediaToText = false applyPDFPostProcess(&processed, opts) defer processed.Close() // NOTE: PDF media (figure/table) is intentionally NOT cropped here under diff --git a/internal/parser/parser/pdf_parser_common_test.go b/internal/parser/parser/pdf_parser_common_test.go index 45991339a9..54e85cc866 100644 --- a/internal/parser/parser/pdf_parser_common_test.go +++ b/internal/parser/parser/pdf_parser_common_test.go @@ -280,11 +280,10 @@ func TestPDFParseResultToJSON_DefaultKeepsHeaderFooterLikePython(t *testing.T) { func TestPDFParser_ConfigureFromSetup(t *testing.T) { p := NewPDFParser() p.ConfigureFromSetup(map[string]any{ - "parse_method": "deepdoc", - "enable_multi_column": true, - "flatten_media_to_text": true, - "remove_toc": true, - "remove_header_footer": true, + "parse_method": "deepdoc", + "enable_multi_column": true, + "remove_toc": true, + "remove_header_footer": true, }) if !p.EnableMultiColumn { t.Fatal("EnableMultiColumn = false, want true") @@ -292,9 +291,6 @@ func TestPDFParser_ConfigureFromSetup(t *testing.T) { if got, want := p.ParseMethod, "deepdoc"; got != want { t.Fatalf("ParseMethod = %q, want %q", got, want) } - if !p.FlattenMediaToText { - t.Fatal("FlattenMediaToText = false, want true") - } if !p.RemoveTOC { t.Fatal("RemoveTOC = false, want true") } diff --git a/internal/parser/parser/pdf_parser_pages_test.go b/internal/parser/parser/pdf_parser_pages_test.go index 3a1ca88a42..41427f77ad 100644 --- a/internal/parser/parser/pdf_parser_pages_test.go +++ b/internal/parser/parser/pdf_parser_pages_test.go @@ -54,7 +54,7 @@ func TestConfigureFromSetup_Pages(t *testing.T) { t.Run("missing pages key -> nil", func(t *testing.T) { p := &PDFParser{} - setup := map[string]any{"flatten_media_to_text": true} + setup := map[string]any{} p.ConfigureFromSetup(setup) if p.Pages != nil { t.Errorf("Pages = %v, want nil", p.Pages) @@ -65,16 +65,12 @@ func TestConfigureFromSetup_Pages(t *testing.T) { t.Run("other fields still read (no regression)", func(t *testing.T) { p := &PDFParser{} setup := map[string]any{ - "flatten_media_to_text": true, - "parse_method": "DeepDOC", + "parse_method": "DeepDOC", "pages": []any{ []any{float64(1), float64(100)}, }, } p.ConfigureFromSetup(setup) - if !p.FlattenMediaToText { - t.Error("FlattenMediaToText not read") - } if p.ParseMethod != "DeepDOC" { t.Errorf("ParseMethod = %q, want DeepDOC", p.ParseMethod) } diff --git a/internal/parser/parser/pdf_postprocess.go b/internal/parser/parser/pdf_postprocess.go index bce6649e49..8bbcadbb4f 100644 --- a/internal/parser/parser/pdf_postprocess.go +++ b/internal/parser/parser/pdf_postprocess.go @@ -14,10 +14,9 @@ import ( // per-box geometry signals those detectors rely on are still intact; they are // intentionally not repeated here. type pdfPostProcessOptions struct { - pageWidth float64 - zoom float64 - enableMultiColumn bool - flattenMediaToText bool + pageWidth float64 + zoom float64 + enableMultiColumn bool // parseMethod carries the PDFParser.ParseMethod so the DeepDoc analyzer can // be resolved per request (in-process vs a registered external backend). // Empty means the historical in-process default. @@ -33,7 +32,7 @@ func applyPDFPostProcess(result *deepdoctype.ParseResult, opts pdfPostProcessOpt reorderPDFMultiColumn(result, opts.pageWidth, opts.zoom) } normalizePDFLayoutTypes(result) - assignPDFDocTypeKeywords(result, opts.flattenMediaToText) + assignPDFDocTypeKeywords(result) } func normalizePDFLayoutTypes(result *deepdoctype.ParseResult) { @@ -46,13 +45,9 @@ func normalizePDFLayoutTypes(result *deepdoctype.ParseResult) { } } -func assignPDFDocTypeKeywords(result *deepdoctype.ParseResult, flatten bool) { +func assignPDFDocTypeKeywords(result *deepdoctype.ParseResult) { for i := range result.Sections { section := &result.Sections[i] - if flatten { - section.DocTypeKwd = "text" - continue - } switch strings.TrimSpace(section.LayoutType) { case deepdoctype.LayoutTypeTable: section.DocTypeKwd = "table" diff --git a/internal/parser/parser/pdf_postprocess_test.go b/internal/parser/parser/pdf_postprocess_test.go index bef11936a0..7404bb3dc7 100644 --- a/internal/parser/parser/pdf_postprocess_test.go +++ b/internal/parser/parser/pdf_postprocess_test.go @@ -59,24 +59,6 @@ func TestApplyPDFPostProcess_AssignsDocTypeKeywords(t *testing.T) { } } -func TestApplyPDFPostProcess_FlattenMediaKeepsImagesButMarksText(t *testing.T) { - result := &deepdoctype.ParseResult{ - Sections: []deepdoctype.Section{ - {Text: "a", LayoutType: "figure", Image: "abc"}, - {Text: "b", LayoutType: "table"}, - }, - } - applyPDFPostProcess(result, pdfPostProcessOptions{flattenMediaToText: true}) - for i, s := range result.Sections { - if s.DocTypeKwd != "text" { - t.Fatalf("Sections[%d].DocTypeKwd = %q, want text", i, s.DocTypeKwd) - } - } - if got, want := result.Sections[0].Image, "abc"; got != want { - t.Fatalf("Sections[0].Image = %q, want %q", got, want) - } -} - func TestApplyPDFPostProcess_ReordersMultiColumnText(t *testing.T) { cases := []struct { name string diff --git a/internal/parser/parser/pdf_result.go b/internal/parser/parser/pdf_result.go index 62f3d1f01f..16797557eb 100644 --- a/internal/parser/parser/pdf_result.go +++ b/internal/parser/parser/pdf_result.go @@ -81,28 +81,6 @@ func (p *PDFParser) NormalizeResult(ctx context.Context, filename string, result fields["doc_type_kwd"] = "text" } } - if p.FlattenMediaToText { - // Flattened output keeps one text carrier per item. Structured - // tables (MinerU content_list, projected HTML) carry cells in - // `table` and footnotes in `text`; rendering the table preserves - // cells, caption, and footnote in that order. The render - // normalizes markup (header inference, escaping, whitespace), so - // it is not byte-identical to the source HTML, but it is - // content-preserving. Items without a table keep their text verbatim. - if table, ok := fields["table"].(*entity.TableData); ok && table != nil { - text, _ := fields["text"].(string) - fields["text"] = strings.TrimSpace(tableutil.RenderTableHTML(table) + "\n" + text) - delete(fields, "table") - } - // Flattening moves an item's content into text. A crop-only media - // item has no content to move: rewriting it to text would hide the - // region from vision enhancement and leave an empty item the - // chunker drops, so it keeps its media type and stays available to - // the on-demand cropper. - if hasPDFTextCarrier(fields) || len(positions) == 0 { - fields["doc_type_kwd"] = "text" - } - } items = appendPDFItem(items, fields) } if droppedPositions > 0 { @@ -311,18 +289,6 @@ func pdfPositionHasRegion(values [5]float64) bool { return values[2] > values[1] && values[4] > values[3] } -// hasPDFTextCarrier reports whether flattening an item has any content to move -// into text: a body, an inline image payload, or a deferred media reference. -func hasPDFTextCarrier(fields map[string]any) bool { - if text, _ := fields["text"].(string); strings.TrimSpace(text) != "" { - return true - } - if img, _ := fields["image"].(string); img != "" { - return true - } - return entity.MediaRefFromAny(fields["media_ref"]) != nil -} - // HasPDFCropRegion reports whether the item carries at least one PDF crop // region, i.e. a position with a positive-area box. A page-only position // carries page identity but no region, so it must not trigger a crop. diff --git a/internal/parser/parser/pdf_result_normalize_markdown_test.go b/internal/parser/parser/pdf_result_normalize_markdown_test.go index 655d4eb406..0a3d7d9041 100644 --- a/internal/parser/parser/pdf_result_normalize_markdown_test.go +++ b/internal/parser/parser/pdf_result_normalize_markdown_test.go @@ -70,22 +70,6 @@ func TestPDFParser_NormalizeMarkdown_DoesNotFetchRemoteImages(t *testing.T) { } } -// TestPDFParser_NormalizeMarkdown_FlattenMediaToText pins that, with flatten -// enabled, image markdown is flattened to a text carrier (doc_type_kwd == text). -func TestPDFParser_NormalizeMarkdown_FlattenMediaToText(t *testing.T) { - p := NewPDFParser() - p.FlattenMediaToText = true - res := p.NormalizeMarkdown(context.Background(), "doc.md", "![img](data:image/png;base64,abc)\n") - if res.Err != nil { - t.Fatalf("err: %v", res.Err) - } - for _, it := range res.JSON { - if dt, _ := it.ToMap()["doc_type_kwd"].(string); dt != "text" { - t.Errorf("doc_type_kwd = %q, want text", dt) - } - } -} - // TestPDFParser_NormalizeMarkdown_PreservesFileOnCancel pins that a cancelled // context returns the error (not silently swallowed) and still carries the // filename in File so callers can report which document failed. diff --git a/internal/parser/parser/pdf_result_test.go b/internal/parser/parser/pdf_result_test.go index 3beddc0ffb..66b59852bc 100644 --- a/internal/parser/parser/pdf_result_test.go +++ b/internal/parser/parser/pdf_result_test.go @@ -121,66 +121,22 @@ func TestPDFNormalizeResultInfersStructuredTable(t *testing.T) { } } -func TestPDFNormalizeResultFlattenPreservesContentAndClearsMediaSubtype(t *testing.T) { - p := NewPDFParser() - p.FlattenMediaToText = true - table := &entity.TableData{Grid: [][]entity.TableCell{{{Text: "Cell"}}}, Caption: "Caption"} - result := p.NormalizeResult(t.Context(), "sample.pdf", ParseResult{JSON: []DocItem{ - TableItem{Table: table, Extra: map[string]any{"text": "Footnote"}}, - ImageItem{Text: "Figure", Image: "image-data", Positions: Positions{AbsoluteBox{PageNumber: 2, Left: 1, Right: 5, Top: 2, Bottom: 6}}}, - TextItem{Text: "Title", Extra: map[string]any{"layout": "heading"}}, - }}) - if result.Err != nil { - t.Fatal(result.Err) - } - for i, item := range result.JSON[:2] { - fields := item.ToMap() - if fields["doc_type_kwd"] != "text" { - t.Fatalf("flattened item %d still has media type: %#v", i, fields) - } - } - text, _ := result.JSON[0].ToMap()["text"].(string) - for _, want := range []string{"Cell", "Caption", "Footnote"} { - if !strings.Contains(text, want) { - t.Fatalf("missing %q in %q", want, text) - } - } - if result.JSON[1].ToMap()["image"] != "image-data" { - t.Fatal("flatten lost image") - } - if _, ok := ExtractPDFPositions(result.JSON[1].ToMap()); !ok { - t.Fatal("flatten lost positions") - } - if result.JSON[2].ToMap()["layout"] != "heading" { - t.Fatal("heading layout changed") - } -} - -func TestPDFNormalizeResultFlattenKeepsCropOnlyMedia(t *testing.T) { - p := NewPDFParser() - p.FlattenMediaToText = true - result := p.NormalizeResult(t.Context(), "sample.pdf", ParseResult{JSON: MapsToItems([]map[string]any{ +func TestPDFNormalizeResultPreservesMediaAndCropRegion(t *testing.T) { + result := NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", ParseResult{JSON: MapsToItems([]map[string]any{ {"doc_type_kwd": "image", "layout": "figure", "positions": [][]any{{1, 10, 30, 20, 40}}}, {"doc_type_kwd": "image", "layout": "figure", "text": "Figure 1", "positions": [][]any{{1, 10, 30, 50, 70}}}, - {"doc_type_kwd": "image", "layout": "figure"}, })}) - if result.Err != nil || len(result.JSON) != 3 { + if result.Err != nil || len(result.JSON) != 2 { t.Fatalf("unexpected result: %+v", result) } - // A crop-only media item has no content to flatten; vision enhancement and - // the on-demand cropper still need its type and region. - cropOnly := result.JSON[0].ToMap() - if cropOnly["doc_type_kwd"] != "image" || !HasPDFCropRegion(cropOnly) { - t.Fatalf("crop-only media lost its type or region: %#v", cropOnly) + for _, item := range result.JSON { + fields := item.ToMap() + if fields["doc_type_kwd"] != "image" || !HasPDFCropRegion(fields) { + t.Fatalf("image type or crop region lost: %#v", fields) + } } - captioned := result.JSON[1].ToMap() - if captioned["doc_type_kwd"] != "text" || captioned["text"] != "Figure 1" { - t.Fatalf("captioned media not flattened: %#v", captioned) - } - // Without a region there is nothing to crop, so the empty item still - // flattens to text and is dropped downstream as before. - if bare := result.JSON[2].ToMap(); bare["doc_type_kwd"] != "text" { - t.Fatalf("regionless media kept a media type: %#v", bare) + if result.JSON[1].ToMap()["text"] != "Figure 1" { + t.Fatalf("image caption lost: %#v", result.JSON[1].ToMap()) } } @@ -198,27 +154,18 @@ func TestPDFNormalizeResultDowngradesUnstructuredTableSubtype(t *testing.T) { } func TestPDFNormalizeResultKeepsPositionedUnstructuredTables(t *testing.T) { - for _, flatten := range []bool{false, true} { - p := NewPDFParser() - p.FlattenMediaToText = flatten - result := p.NormalizeResult(t.Context(), "sample.pdf", ParseResult{JSON: MapsToItems([]map[string]any{{ - "doc_type_kwd": "table", "text": "Recognized table without cells", - "positions": [][]any{{2, 10, 30, 20, 40}}, - }})}) - if result.Err != nil || len(result.JSON) != 1 { - t.Fatalf("unexpected result: %+v", result) - } - item := result.JSON[0].ToMap() - want := "table" - if flatten { - want = "text" - } - if item["doc_type_kwd"] != want || item["text"] != "Recognized table without cells" { - t.Fatalf("flatten=%v: table media identity or text lost: %#v", flatten, item) - } - if _, ok := ExtractPDFPositions(item); !ok { - t.Fatalf("flatten=%v: table crop region lost: %#v", flatten, item) - } + result := NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", ParseResult{JSON: MapsToItems([]map[string]any{{ + "doc_type_kwd": "table", "text": "Recognized table without cells", "positions": [][]any{{2, 10, 30, 20, 40}}, + }})}) + if result.Err != nil || len(result.JSON) != 1 { + t.Fatalf("unexpected result: %+v", result) + } + item := result.JSON[0].ToMap() + if item["doc_type_kwd"] != "table" || item["text"] != "Recognized table without cells" { + t.Fatalf("table identity or text lost: %#v", item) + } + if _, ok := ExtractPDFPositions(item); !ok { + t.Fatalf("table crop region lost: %#v", item) } } @@ -244,34 +191,6 @@ func TestPDFNormalizeResultRecoversUnclosedTableCells(t *testing.T) { } } -func TestPDFParserSoMarkFlattensStructuredMedia(t *testing.T) { - withSSRFBypass(t) - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == "/parse/async" { - _, _ = w.Write([]byte(`{"code":0,"data":{"task_id":"task"}}`)) - return - } - _, _ = w.Write([]byte(`{"code":0,"data":{"status":"SUCCESS","result":{"outputs":{"json":{"pages":[{"blocks":[{"type":"figure","content":"Figure caption"},{"type":"table","content":"
Cell value
"}]}]}}}}}`)) - })) - defer server.Close() - p := NewPDFParser() - p.ConfigureFromSetup(map[string]any{"parse_method": "somark", "somark_base_url": server.URL, "flatten_media_to_text": true}) - result := p.ParseWithResult(t.Context(), "sample.pdf", []byte("pdf")) - if result.Err != nil { - t.Fatal(result.Err) - } - if len(result.JSON) != 2 { - t.Fatalf("items = %#v", result.JSON) - } - for i, want := range []string{"Figure caption", "Cell value"} { - item := result.JSON[i].ToMap() - text, _ := item["text"].(string) - if item["doc_type_kwd"] != "text" || !strings.Contains(text, want) { - t.Fatalf("item %d = %#v; want text containing %q", i, item, want) - } - } -} - func TestProjectPDFTablePreservesExistingStructureAndNotes(t *testing.T) { table := &entity.TableData{Grid: [][]entity.TableCell{{{Text: "Original cell"}}}} item := map[string]any{"doc_type_kwd": "table", "table": table, "text": "
Other
\nFootnote"} @@ -323,8 +242,8 @@ func TestPDFWhitespaceTableDescriptionCannotHideCells(t *testing.T) { func TestPDFNormalizeResultPreservesAdjacentTables(t *testing.T) { const markup = "
First cell
\n
Second cell
\nFootnote" for _, projected := range []bool{false, true} { - for _, flatten := range []bool{false, true} { - name := fmt.Sprintf("projected=%v/flatten=%v", projected, flatten) + { + name := fmt.Sprintf("projected=%v", projected) t.Run(name, func(t *testing.T) { item := map[string]any{ "doc_type_kwd": "table", "text": markup, "page_number": 3, @@ -335,45 +254,29 @@ func TestPDFNormalizeResultPreservesAdjacentTables(t *testing.T) { } originalText := item["text"] p := NewPDFParser() - p.FlattenMediaToText = flatten + result := p.NormalizeResult(t.Context(), "sample.pdf", pdfItemsToResult("sample.pdf", []map[string]any{item}, 3)) if result.Err != nil { t.Fatal(result.Err) } - if flatten { - var text string - for _, doc := range result.JSON { - fields := doc.ToMap() - if fields["doc_type_kwd"] != "text" || fields["table"] != nil { - t.Fatalf("media not flattened: %#v", fields) - } - value, _ := fields["text"].(string) - text += value + + if len(result.JSON) != 3 { + t.Fatalf("want two tables and a footnote, got %#v", result.JSON) + } + for i, want := range []string{"First cell", "Second cell"} { + fields := result.JSON[i].ToMap() + table, _ := fields["table"].(*entity.TableData) + if table == nil || table.Page != 3 || len(table.Grid) != 1 || len(table.Grid[0]) != 1 || table.Grid[0][0].Text != want || fields["text"] != nil { + t.Fatalf("table %d lost cells or metadata: %#v", i, fields) } - for _, want := range []string{"First cell", "Second cell", "Footnote"} { - if strings.Count(text, want) != 1 { - t.Fatalf("want %q exactly once in %q", want, text) - } - } - } else { - if len(result.JSON) != 3 { - t.Fatalf("want two tables and a footnote, got %#v", result.JSON) - } - for i, want := range []string{"First cell", "Second cell"} { - fields := result.JSON[i].ToMap() - table, _ := fields["table"].(*entity.TableData) - if table == nil || table.Page != 3 || len(table.Grid) != 1 || len(table.Grid[0]) != 1 || table.Grid[0][0].Text != want || fields["text"] != nil { - t.Fatalf("table %d lost cells or metadata: %#v", i, fields) - } - if _, ok := ExtractPDFPositions(fields); !ok { - t.Fatalf("table %d lost crop positions", i) - } - } - note := result.JSON[2].ToMap() - if note["text"] != "Footnote" || note["doc_type_kwd"] != "text" || note["page_number"] != 3 || note["_preserve_order"] != true { - t.Fatalf("footnote lost text or metadata: %#v", note) + if _, ok := ExtractPDFPositions(fields); !ok { + t.Fatalf("table %d lost crop positions", i) } } + note := result.JSON[2].ToMap() + if note["text"] != "Footnote" || note["doc_type_kwd"] != "text" || note["page_number"] != 3 || note["_preserve_order"] != true { + t.Fatalf("footnote lost text or metadata: %#v", note) + } if item["text"] != originalText { t.Fatal("source item mutated") } @@ -384,34 +287,29 @@ func TestPDFNormalizeResultPreservesAdjacentTables(t *testing.T) { func TestPDFNormalizeResultIsIdempotent(t *testing.T) { const markup = "
First cell
\n
Second cell
\nFootnote" - for _, flatten := range []bool{false, true} { - p := NewPDFParser() - p.FlattenMediaToText = flatten - item := map[string]any{ - "doc_type_kwd": "table", "text": markup, "page_number": 3, - "positions": [][]any{{3, 10, 30, 20, 40}}, "_preserve_order": true, + p := NewPDFParser() + item := map[string]any{ + "doc_type_kwd": "table", "text": markup, "page_number": 3, + "positions": [][]any{{3, 10, 30, 20, 40}}, "_preserve_order": true, + } + once := p.NormalizeResult(t.Context(), "sample.pdf", pdfItemsToResult("sample.pdf", []map[string]any{item}, 3)) + if once.Err != nil { + t.Fatal(once.Err) + } + twice := p.NormalizeResult(t.Context(), "sample.pdf", once) + if twice.Err != nil { + t.Fatal(twice.Err) + } + if len(once.JSON) != len(twice.JSON) { + t.Fatalf("second normalize changed item count %d -> %d", len(once.JSON), len(twice.JSON)) + } + for i := range once.JSON { + a, b := once.JSON[i].ToMap(), twice.JSON[i].ToMap() + if a["doc_type_kwd"] != b["doc_type_kwd"] { + t.Fatalf("item %d type changed %v -> %v", i, a["doc_type_kwd"], b["doc_type_kwd"]) } - once := p.NormalizeResult(t.Context(), "sample.pdf", pdfItemsToResult("sample.pdf", []map[string]any{item}, 3)) - if once.Err != nil { - t.Fatal(once.Err) - } - twice := p.NormalizeResult(t.Context(), "sample.pdf", once) - if twice.Err != nil { - t.Fatal(twice.Err) - } - if len(once.JSON) != len(twice.JSON) { - t.Fatalf("flatten=%v: second normalize changed item count %d -> %d", flatten, len(once.JSON), len(twice.JSON)) - } - for i := range once.JSON { - a, b := once.JSON[i].ToMap(), twice.JSON[i].ToMap() - if a["doc_type_kwd"] != b["doc_type_kwd"] { - t.Fatalf("flatten=%v: item %d type changed %v -> %v", flatten, i, a["doc_type_kwd"], b["doc_type_kwd"]) - } - ta, _ := a["text"].(string) - tb, _ := b["text"].(string) - if ta != tb { - t.Fatalf("flatten=%v: item %d text changed %q -> %q", flatten, i, ta, tb) - } + if a["text"] != b["text"] { + t.Fatalf("item %d text changed %v -> %v", i, a["text"], b["text"]) } } } diff --git a/internal/parser/parser/table_regression_test.go b/internal/parser/parser/table_regression_test.go index ebbe76e2e4..d04676acbe 100644 --- a/internal/parser/parser/table_regression_test.go +++ b/internal/parser/parser/table_regression_test.go @@ -44,10 +44,9 @@ func TestTableRegressionDOCXCaptionAfterNestedTable(t *testing.T) { } } -func TestTableRegressionMarkdownFlattenBesideTable(t *testing.T) { +func TestTableRegressionMarkdownImageBesideTable(t *testing.T) { source := "![logo](data:image/png;base64,aGVsbG8=) Before
AB
After" p, _ := NewMarkdownParser(GoMarkdown) - p.FlattenMediaToText = true after := p.ParseWithResult(t.Context(), "x.md", []byte(source)) if after.Err != nil || len(after.JSON) == 0 { t.Fatalf("result=%+v", after) @@ -57,9 +56,6 @@ func TestTableRegressionMarkdownFlattenBesideTable(t *testing.T) { if item.ToMap()["image"] != nil { payloads++ } - if item.ToMap()["doc_type_kwd"] == "image" { - t.Error("flatten_media_to_text ignored") - } } if payloads != 1 { t.Fatalf("image payloads=%d, want 1", payloads) @@ -148,7 +144,7 @@ func TestTableRegressionMarkdownRawHTMLImageBudget(t *testing.T) { doc := markdownNew().Parse([]byte(source)) budget := newMediaBudgetForTest() var items []map[string]any - walkMarkdownBlocksWithImages(t.Context(), doc, &items, false, true, &htmlWalkState{budget: budget}) + walkMarkdownBlocksWithImages(t.Context(), doc, &items, true, &htmlWalkState{budget: budget}) images := 0 for _, item := range items { if item["doc_type_kwd"] == "image" && item["image"] != nil { diff --git a/internal/parser/parser/table_structure_test.go b/internal/parser/parser/table_structure_test.go index 77dd4e4150..f904dd3fa6 100644 --- a/internal/parser/parser/table_structure_test.go +++ b/internal/parser/parser/table_structure_test.go @@ -98,7 +98,7 @@ func TestMarkdownTableImageTimeoutDoesNotConsumeBudget(t *testing.T) { doc := markdownNew().Parse([]byte("| A | B |\n|---|---|\n| Logo | ![logo](http://127.0.0.1/logo.png) |\n")) budget := newMediaBudgetForTest() var items []map[string]any - unresolved := walkMarkdownBlocksWithImages(ctx, doc, &items, false, true, &htmlWalkState{budget: budget}) + unresolved := walkMarkdownBlocksWithImages(ctx, doc, &items, true, &htmlWalkState{budget: budget}) if budget.Count() != 0 || unresolved != 0 { t.Fatalf("timed out image consumed budget: attempts=%d unresolved=%d", budget.Count(), unresolved) } diff --git a/web/src/locales/az.ts b/web/src/locales/az.ts index 69641615a9..eae30cba11 100644 --- a/web/src/locales/az.ts +++ b/web/src/locales/az.ts @@ -2785,9 +2785,6 @@ Nümunə: Virtual host üslubu`, oneChunkTitle: 'Qeyd', oneChunkDescription: 'Təhlil edilmiş bütün bölmələr ardıcıllıqla bir hissədə birləşdiriləcək.', - flattenMediaToText: 'Görmə modelini söndür', - flattenMediaToTextTip: - 'Şəkil və cədvəl bölmələrini adi mətn kimi qəbul edir və görmə modeli ilə təkmilləşdirməni ötürür.', enableChildrenDelimiters: 'Alt hissələr axtarış üçün istifadə olunur', merge: 'Birləşdir', split: 'Böl', diff --git a/web/src/locales/fr.ts b/web/src/locales/fr.ts index 73e8082494..d2c97ac03f 100644 --- a/web/src/locales/fr.ts +++ b/web/src/locales/fr.ts @@ -2551,9 +2551,6 @@ Idéal pour : les documents avec un contenu fluide et contextuellement lié — oneChunkTitle: 'Note', oneChunkDescription: 'Toutes les sections analysées seront fusionnées dans un seul bloc.', - flattenMediaToText: 'Désactiver le modèle de vision', - flattenMediaToTextTip: - "Traite les sections d'images et de tableaux comme du texte brut et ignore l'amélioration par vision.", enableChildrenDelimiters: 'Les blocs enfants sont utilisés pour la récupération', merge: 'Fusionner', diff --git a/web/src/locales/it.ts b/web/src/locales/it.ts index e7a8856baa..50332e1d66 100644 --- a/web/src/locales/it.ts +++ b/web/src/locales/it.ts @@ -1925,9 +1925,6 @@ Ideale per: documenti con contenuto fluente e contestualmente connesso — come oneChunkTitle: 'Nota', oneChunkDescription: 'Tutte le sezioni analizzate saranno unite in ordine in un singolo chunk.', - flattenMediaToText: 'Disabilita modello visivo', - flattenMediaToTextTip: - 'Tratta le sezioni di immagini e tabelle come testo semplice e salta il miglioramento visivo.', enableChildrenDelimiters: 'I chunk figli sono usati per il recupero', merge: 'Unisci', split: 'Dividi', diff --git a/web/src/locales/ja.ts b/web/src/locales/ja.ts index 61f8a5f2e7..18f941ce26 100644 --- a/web/src/locales/ja.ts +++ b/web/src/locales/ja.ts @@ -2483,9 +2483,6 @@ export default { oneChunkTitle: '注記', oneChunkDescription: '解析されたすべてのセクションが順番に1つのチャンクにマージされます。', - flattenMediaToText: 'ビジョンモデルを無効化', - flattenMediaToTextTip: - '画像や表のセクションをプレーンテキストとして扱い、ビジョンによる拡張処理をスキップします。', enableChildrenDelimiters: '子チャンクを検索に使用する', merge: 'マージ', split: '分割', diff --git a/web/src/locales/ko.ts b/web/src/locales/ko.ts index 1bcde753ab..05ec780c5a 100644 --- a/web/src/locales/ko.ts +++ b/web/src/locales/ko.ts @@ -1928,9 +1928,6 @@ export default { oneChunkTitle: '참고', oneChunkDescription: '파싱된 모든 섹션이 순서대로 단일 chunk로 병합됩니다.', - flattenMediaToText: '비전 모델 비활성화', - flattenMediaToTextTip: - '이미지 및 표 섹션을 일반 텍스트로 처리하고 비전 향상을 건너뜁니다.', enableChildrenDelimiters: '하위 chunk를 검색에 사용', merge: '병합', split: '분할', diff --git a/web/src/locales/tr.ts b/web/src/locales/tr.ts index c4df1ddd67..3c915797de 100644 --- a/web/src/locales/tr.ts +++ b/web/src/locales/tr.ts @@ -2876,9 +2876,6 @@ En uygun olduğu durumlar: Anlatı bütünlüğünün bitişik paragrafları bir "Görüntüler için bir metin kaynağı gerekir: OCR'yi veya görsel geliştirmeyi açık tutun.", visionEnhancementPromptTip: 'Görsel model için özel talimat. Yerleşik istemi kullanmak için boş bırakın.', - flattenMediaToText: 'Görsel modeli devre dışı bırak', - flattenMediaToTextTip: - 'Görüntü ve tablo bölümlerini düz metin olarak ele alır ve görsel geliştirmeyi atlar.', enableChildrenDelimiters: 'Alt parçalar alım için kullanılır', merge: 'Birleştir', split: 'Böl', diff --git a/web/src/locales/uz.ts b/web/src/locales/uz.ts index a14f501fc1..6e7d2fdca2 100644 --- a/web/src/locales/uz.ts +++ b/web/src/locales/uz.ts @@ -2685,9 +2685,6 @@ export default { oneChunkTitle: 'Eslatma', oneChunkDescription: 'Tahlil qilingan barcha bo‘limlar tartib bilan bitta bo‘lakka birlashtiriladi.', - flattenMediaToText: 'Vizual modelni o‘chirish', - flattenMediaToTextTip: - 'Rasm va jadval bo‘limlarini oddiy matn sifatida qarab, vizual yaxshilashni o‘tkazib yuboradi.', enableChildrenDelimiters: 'Qidiruv uchun bola bo‘laklar ishlatiladi', merge: 'Birlashtirish', split: 'Bo‘lish', diff --git a/web/src/pages/agent/form/parser-form/utils.test.ts b/web/src/pages/agent/form/parser-form/utils.test.ts index 8688fd55c9..c229842b36 100644 --- a/web/src/pages/agent/form/parser-form/utils.test.ts +++ b/web/src/pages/agent/form/parser-form/utils.test.ts @@ -93,7 +93,6 @@ describe('parser-form utils', () => { setups: [ { fileFormat: FileType.PDF, - flatten_media_to_text: false, vlm: { llm_id: 'gpt-4o@OpenAI' }, }, { fileFormat: FileType.Audio, vlm: { llm_id: 'whisper@OpenAI' } }, @@ -115,10 +114,7 @@ describe('parser-form utils', () => { it('turns the enhancement off when no setup used the vision model', () => { const values = normalizeParserFormValues({ - setups: [ - { fileFormat: FileType.PDF, flatten_media_to_text: true }, - { fileFormat: FileType.Docx, flatten_media_to_text: true }, - ], + setups: [{ fileFormat: FileType.PDF }, { fileFormat: FileType.Docx }], }); expect(values.enable_vision_enhancement).toBe(false); @@ -147,7 +143,6 @@ describe('parser-form utils', () => { setups: [ { fileFormat: FileType.PDF, - flatten_media_to_text: true, vlm: { llm_id: 'stale@Model' }, }, { fileFormat: FileType.Audio, vlm: { llm_id: 'whisper@OpenAI' } }, diff --git a/web/src/pages/agent/form/parser-form/utils.ts b/web/src/pages/agent/form/parser-form/utils.ts index bdb76cb387..06a86d189f 100644 --- a/web/src/pages/agent/form/parser-form/utils.ts +++ b/web/src/pages/agent/form/parser-form/utils.ts @@ -24,8 +24,7 @@ export const VisionEnhancementFileTypes: FileType[] = [ FileType.Video, ]; -// Lifts the legacy per-setup vision options (vlm.llm_id / flatten_media_to_text) -// onto the top level and migrates the image family onto its own contract: +// Lifts the per-setup vision model onto the top level and migrates the image family onto its own contract: // `ocr_enabled` replaces parse_method, and the prompt the image setup used to // own moves into the global vlm block. Idempotent, so every boundary (form // defaults, canvas save, dataset load) can apply it: values already carrying a @@ -65,9 +64,7 @@ export function normalizeParserFormValues>( const enableVisionEnhancement = typeof values?.enable_vision_enhancement === 'boolean' ? values.enable_vision_enhancement - : visionSetups.some( - (x) => x?.flatten_media_to_text === false || !isEmpty(x?.vlm?.llm_id), - ); + : visionSetups.some((x) => !isEmpty(x?.vlm?.llm_id)); const llmId = legacyImageModel || @@ -80,7 +77,7 @@ export function normalizeParserFormValues>( // setups to JSON, so a stored value is dead config that must not survive // a load/save round-trip. if (x?.fileFormat === FileType.Audio) return omit(x, ['output_format']); - const stripped = omit(x, ['vlm', 'flatten_media_to_text', 'output_format']); + const stripped = omit(x, ['vlm', 'output_format']); if (x?.fileFormat !== FileType.Image) return stripped; const { parse_method, ...rest } = omit(stripped, ['lang', 'system_prompt']); // Materialize the switch either way. Absent or "ocr" both mean "run local