refactor(ingestion): remove media flattening option (#22144)

internal/agent/templates/advanced_ingestion_pipeline.json

@@ -133,7 +133,6 @@
                             ]
                         },
                         "docx": {
-                            "flatten_media_to_text": false,
                             "preprocess": "main_content",
                             "suffix": [
                                 "docx"
@@ -176,7 +175,6 @@
                             "system_prompt": ""
                         },
                         "markdown": {
-                            "flatten_media_to_text": false,
                             "preprocess": "main_content",
                             "suffix": [
                                 "md",
@@ -186,7 +184,6 @@
                             "vlm": {}
                         },
                         "pdf": {
-                            "flatten_media_to_text": false,
                             "parse_method": "DeepDOC",
                             "preprocess": "main_content",
                             "suffix": [
@@ -203,7 +200,6 @@
                             ]
                         },
                         "spreadsheet": {
-                            "flatten_media_to_text": false,
                             "parse_method": "DeepDOC",
                             "preprocess": "main_content",
                             "suffix": [
@@ -363,13 +359,11 @@
                             "setups": [
                                 {
                                     "fileFormat": "pdf",
-                                    "flatten_media_to_text": false,
                                     "parse_method": "DeepDOC",
                                     "preprocess": "main_content"
                                 },
                                 {
                                     "fileFormat": "spreadsheet",
-                                    "flatten_media_to_text": false,
                                     "parse_method": "DeepDOC",
                                     "preprocess": "main_content"
                                 },
@@ -395,7 +389,6 @@
                                 },
                                 {
                                     "fileFormat": "markdown",
-                                    "flatten_media_to_text": false,
                                     "preprocess": "main_content"
                                 },
                                 {
@@ -412,7 +405,6 @@
                                 },
                                 {
                                     "fileFormat": "docx",
-                                    "flatten_media_to_text": false,
                                     "preprocess": "main_content"
                                 },
                                 {

internal/agent/templates/chunk_summary.json

@@ -90,7 +90,6 @@
                             ]
                         },
                         "docx": {
-                            "flatten_media_to_text": false,
                             "preprocess": "main_content",
                             "suffix": [
                                 "docx"
@@ -133,7 +132,6 @@
                             "system_prompt": ""
                         },
                         "markdown": {
-                            "flatten_media_to_text": false,
                             "preprocess": "main_content",
                             "suffix": [
                                 "md",
@@ -143,7 +141,6 @@
                             "vlm": {}
                         },
                         "pdf": {
-                            "flatten_media_to_text": false,
                             "parse_method": "DeepDOC",
                             "preprocess": "main_content",
                             "suffix": [
@@ -160,7 +157,6 @@
                             ]
                         },
                         "spreadsheet": {
-                            "flatten_media_to_text": false,
                             "parse_method": "DeepDOC",
                             "preprocess": "main_content",
                             "suffix": [
@@ -323,13 +319,11 @@
                             "setups": [
                                 {
                                     "fileFormat": "pdf",
-                                    "flatten_media_to_text": false,
                                     "parse_method": "DeepDOC",
                                     "preprocess": "main_content"
                                 },
                                 {
                                     "fileFormat": "spreadsheet",
-                                    "flatten_media_to_text": false,
                                     "parse_method": "DeepDOC",
                                     "preprocess": "main_content"
                                 },
@@ -355,7 +349,6 @@
                                 },
                                 {
                                     "fileFormat": "markdown",
-                                    "flatten_media_to_text": false,
                                     "preprocess": "main_content"
                                 },
                                 {
@@ -372,7 +365,6 @@
                                 },
                                 {
                                     "fileFormat": "docx",
-                                    "flatten_media_to_text": false,
                                     "preprocess": "main_content"
                                 },
                                 {

internal/agent/templates/compiler.json

@@ -57,7 +57,6 @@
           },
           "setups": {
             "doc": {
-              "flatten_media_to_text": false,
               "order_index": 7,
               "remove_header_footer": false,
               "suffix": [
@@ -66,7 +65,6 @@
               "vlm": {}
             },
             "docx": {
-              "flatten_media_to_text": false,
               "order_index": 8,
               "remove_header_footer": false,
               "suffix": [
@@ -110,7 +108,6 @@
               ]
             },
             "markdown": {
-              "flatten_media_to_text": false,
               "order_index": 4,
               "suffix": [
                 "md",
@@ -120,7 +117,6 @@
               "vlm": {}
             },
             "pdf": {
-              "flatten_media_to_text": false,
               "order_index": 0,
               "parse_method": "DeepDOC",
               "remove_header_footer": false,
@@ -138,7 +134,6 @@
               ]
             },
             "spreadsheet": {
-              "flatten_media_to_text": false,
               "order_index": 1,
               "parse_method": "DeepDOC",
               "suffix": [
@@ -274,14 +269,12 @@
             "setups": [
               {
                 "fileFormat": "pdf",
-                "flatten_media_to_text": false,
                 "parse_method": "DeepDOC",
                 "remove_header_footer": false,
                 "vlm": {}
               },
               {
                 "fileFormat": "spreadsheet",
-                "flatten_media_to_text": false,
                 "parse_method": "DeepDOC",
                 "vlm": {}
               },
@@ -304,7 +297,6 @@
               },
               {
                 "fileFormat": "markdown",
-                "flatten_media_to_text": false,
                 "vlm": {}
               },
               {
@@ -316,13 +308,11 @@
               },
               {
                 "fileFormat": "doc",
-                "flatten_media_to_text": false,
                 "remove_header_footer": false,
                 "vlm": {}
               },
               {
                 "fileFormat": "docx",
-                "flatten_media_to_text": false,
                 "remove_header_footer": false,
                 "vlm": {}
               },

internal/agent/templates/title_chunker.json

@@ -90,7 +90,6 @@
                                 ]
                             },
                             "docx": {
-                                "flatten_media_to_text": false,
                                 "preprocess": "main_content",
                                 "suffix": [
                                     "docx"
@@ -133,7 +132,6 @@
                                 "system_prompt": ""
                             },
                             "markdown": {
-                                "flatten_media_to_text": false,
                                 "preprocess": "main_content",
                                 "suffix": [
                                     "md",
@@ -143,7 +141,6 @@
                                 "vlm": {}
                             },
                             "pdf": {
-                                "flatten_media_to_text": false,
                                 "parse_method": "DeepDOC",
                                 "preprocess": "main_content",
                                 "suffix": [
@@ -160,7 +157,6 @@
                                 ]
                             },
                             "spreadsheet": {
-                                "flatten_media_to_text": false,
                                 "parse_method": "DeepDOC",
                                 "preprocess": "main_content",
                                 "suffix": [
@@ -342,13 +338,11 @@
                                 "setups": [
                                     {
                                         "fileFormat": "pdf",
-                                        "flatten_media_to_text": false,
                                         "parse_method": "DeepDOC",
                                         "preprocess": "main_content"
                                     },
                                     {
                                         "fileFormat": "spreadsheet",
-                                        "flatten_media_to_text": false,
                                         "parse_method": "DeepDOC",
                                         "preprocess": "main_content"
                                     },
@@ -374,7 +368,6 @@
                                     },
                                     {
                                         "fileFormat": "markdown",
-                                        "flatten_media_to_text": false,
                                         "preprocess": "main_content"
                                     },
                                     {
@@ -391,7 +384,6 @@
                                     },
                                     {
                                         "fileFormat": "docx",
-                                        "flatten_media_to_text": false,
                                         "preprocess": "main_content"
                                     },
                                     {

internal/dao/ingestion_parser_config_runtime_test.go

@@ -634,15 +634,14 @@ var overriddenFamilyKeys = map[string]bool{
 // questions (this one, "is the template's value safe to lose"; that one, "may
 // the value the migration writes differ from the template's").
 var inertFamilyKeys = map[string]string{
-	"preprocess":            "the value has no Go reader; the preprocessing stage is selected by the chunker's own options",
-	"suffix":                "no Go reader; file routing uses the document's own suffix column, not the family list",
-	"vlm":                   "empty in every template; absent falls back to the family llm_id and then the tenant model, and only Markdown and audio read a family vlm at all",
-	"flatten_media_to_text": "false in every template, which is also defaultSetups()'s default",
-	"pages":                 "the only template that bakes it lists every page ([[1,100000]]), which is what an absent pages means",
-	"parse_method":          "a written pdf family always restates it; for the spreadsheet family defaultSetups() re-derives the same deepdoc engine",
-	"column_mode":           "the table template bakes auto, which is what an absent column_mode means to indexdoc's resolveTableColumnConfig",
-	"column_roles":          "the table template bakes the empty role map, which is what an absent column_roles means",
-	"column_names":          "the table template bakes the empty name list, which is what an absent column_names means",
+	"preprocess":   "the value has no Go reader; the preprocessing stage is selected by the chunker's own options",
+	"suffix":       "no Go reader; file routing uses the document's own suffix column, not the family list",
+	"vlm":          "empty in every template; absent falls back to the family llm_id and then the tenant model, and only Markdown and audio read a family vlm at all",
+	"pages":        "the only template that bakes it lists every page ([[1,100000]]), which is what an absent pages means",
+	"parse_method": "a written pdf family always restates it; for the spreadsheet family defaultSetups() re-derives the same deepdoc engine",
+	"column_mode":  "the table template bakes auto, which is what an absent column_mode means to indexdoc's resolveTableColumnConfig",
+	"column_roles": "the table template bakes the empty role map, which is what an absent column_roles means",
+	"column_names": "the table template bakes the empty name list, which is what an absent column_names means",
 }
 
 // assertFamiliesPreserved checks that no template-baked key of a written family

internal/ingestion/component/parser.go

@@ -301,17 +301,15 @@ func imageSetupRunsOCR(setup schema.ParserSetup) bool {
 func defaultSetups() map[string]schema.ParserSetup {
 	s := map[string]schema.ParserSetup{
 		"pdf": {
-			"parse_method":          "deepdoc",
-			"flatten_media_to_text": false,
-			"remove_toc":            false,
-			"remove_header_footer":  false,
-			"suffix":                []string{"pdf"},
+			"parse_method":         "deepdoc",
+			"remove_toc":           false,
+			"remove_header_footer": false,
+			"suffix":               []string{"pdf"},
 		},
 		"spreadsheet": {
-			"parse_method":          "deepdoc",
-			"flatten_media_to_text": false,
-			"html4excel":            false,
-			"suffix":                []string{"xls", "xlsx", "csv"},
+			"parse_method": "deepdoc",
+			"html4excel":   false,
+			"suffix":       []string{"xls", "xlsx", "csv"},
 		},
 		"doc": {
 			"remove_toc":           false,
@@ -319,15 +317,13 @@ func defaultSetups() map[string]schema.ParserSetup {
 			"suffix":               []string{"doc"},
 		},
 		"docx": {
-			"flatten_media_to_text": false,
-			"remove_toc":            false,
-			"remove_header_footer":  false,
-			"suffix":                []string{"docx"},
+			"remove_toc":           false,
+			"remove_header_footer": false,
+			"suffix":               []string{"docx"},
 		},
 		"markdown": {
-			"flatten_media_to_text": false,
-			"suffix":                []string{"md", "markdown", "mdx"},
-			"remove_toc":            false,
+			"suffix":     []string{"md", "markdown", "mdx"},
+			"remove_toc": false,
 		},
 		"text&code": {
 			"suffix": []string{

internal/ingestion/component/pdf_vision_dispatch.go

@@ -89,9 +89,8 @@ func maybeDispatchPDFVision(
 	// fallback) are converted to JSON by NormalizeMarkdown before dispatch
 	// returns, so the NormalizeResult closure below only ever applies item
 	// policies to already-normalized JSON and stays idempotent.
-	flatten, _ := setup["flatten_media_to_text"].(bool)
 	normalize := func(result parser.ParseResult) parser.ParseResult {
-		pdf := parser.PDFParser{FlattenMediaToText: flatten}
+		pdf := parser.PDFParser{}
 		return parser.AttachPDFOutlines(pdf.NormalizeResult(ctx, filename, result), binary)
 	}
 	if result, handled, err := maybeDispatchTenantPDFParser(ctx, db, filename, binary, tenantID, setup); handled {
@@ -347,7 +346,7 @@ func dispatchMonkeyOCRv2PDF(ctx context.Context, db *gorm.DB, filename string, b
 		return parser.ParseResult{}, err
 	}
 	md := strings.Join(sections, "\n\n")
-	return normalizePDFMarkdown(ctx, filename, md, setup), nil
+	return normalizePDFMarkdown(ctx, filename, md), nil
 }
 
 // normalizePDFMarkdown converts a PDF backend Markdown response (MonkeyOCRv2,
@@ -355,9 +354,8 @@ func dispatchMonkeyOCRv2PDF(ctx context.Context, db *gorm.DB, filename string, b
 // package's NormalizeMarkdown. This keeps the markdown→JSON boundary in the
 // parser package, matching the NormalizeResult path used for JSON backends, and
 // lets the downstream NormalizeResult closure stay idempotent for JSON.
-func normalizePDFMarkdown(ctx context.Context, filename, md string, setup schema.ParserSetup) parser.ParseResult {
-	flatten, _ := setup["flatten_media_to_text"].(bool)
-	return (&parser.PDFParser{FlattenMediaToText: flatten}).NormalizeMarkdown(ctx, filename, md)
+func normalizePDFMarkdown(ctx context.Context, filename, md string) parser.ParseResult {
+	return (&parser.PDFParser{}).NormalizeMarkdown(ctx, filename, md)
 }
 
 var isMonkeyOCRv2LayoutModelID = defaultIsMonkeyOCRv2LayoutModelID
@@ -652,7 +650,7 @@ func dispatchMinerUPDF(
 		if err != nil {
 			return parser.ParseResult{}, fmt.Errorf("parser: MinerU cloud: %w", err)
 		}
-		return finishMinerURemoteResult(ctx, filename, binary, result, setup)
+		return finishMinerURemoteResult(ctx, filename, binary, result)
 	}
 
 	v1, err := modelModule.MinerUSupportsV1(ctx, baseURL, apiKeyRaw)
@@ -673,7 +671,7 @@ func dispatchMinerUPDF(
 		if err != nil {
 			return parser.ParseResult{}, fmt.Errorf("parser: MinerU V1: %w", err)
 		}
-		return finishMinerURemoteResult(ctx, filename, binary, result, setup)
+		return finishMinerURemoteResult(ctx, filename, binary, result)
 	}
 
 	apiURL := strings.TrimRight(baseURL, "/") + "/file_parse"
@@ -694,7 +692,7 @@ func dispatchMinerUPDF(
 	return res, nil
 }
 
-func finishMinerURemoteResult(ctx context.Context, filename string, binary []byte, result *modelModule.MinerUV1Result, setup schema.ParserSetup) (parser.ParseResult, error) {
+func finishMinerURemoteResult(ctx context.Context, filename string, binary []byte, result *modelModule.MinerUV1Result) (parser.ParseResult, error) {
 	markdown := ""
 	var zipBytes []byte
 	if result != nil {
@@ -714,7 +712,7 @@ func finishMinerURemoteResult(ctx context.Context, filename string, binary []byt
 		return res, nil
 	}
 	if markdown != "" {
-		return normalizePDFMarkdown(ctx, filename, result.Markdown, setup), nil
+		return normalizePDFMarkdown(ctx, filename, result.Markdown), nil
 	}
 	if extractErr != nil {
 		return parser.ParseResult{}, fmt.Errorf("parser: MinerU extract: %w", extractErr)
@@ -870,7 +868,7 @@ func dispatchMonkeyOCRPDF(
 		}
 	}
 	md := strings.Join(parts, "\n")
-	return normalizePDFMarkdown(ctx, filename, md, setup), nil
+	return normalizePDFMarkdown(ctx, filename, md), nil
 }
 
 // resolveMinerUModelForDispatch resolves the OCR model used by the MinerU PDF

internal/ingestion/component/pdf_vision_dispatch_test.go

@@ -749,7 +749,7 @@ func TestFinishMinerURemoteResultPrefersStructuredZip(t *testing.T) {
 	res, err := finishMinerURemoteResult(t.Context(), "sample.pdf", []byte("%PDF-1.4"), &modelModule.MinerUV1Result{
 		Markdown: "# markdown only",
 		Zip:      buf.Bytes(),
-	}, nil)
+	})
 	if err != nil {
 		t.Fatal(err)
 	}

internal/ingestion/component/vision_enhancement.go

@@ -204,8 +204,7 @@ func maybeDispatchVisionEnhancement(
 	// (pdf_vision_dispatch.go), not an input here — see issue #20727.
 	language := resolveVisionLanguage(inputs)
 
-	// Collect visual resources, including Markdown images whose type is text
-	// because flatten_media_to_text is enabled. Items whose text already
+	// Collect visual resources. Items whose text already
 	// carries a backend-provided description (MinerU vlm_description) are
 	// skipped to avoid a duplicate second description and an extra VLM call.
 	var items []int

internal/ingestion/pipeline/template/ingestion_pipeline_book.json

@@ -62,7 +62,6 @@
                             ]
                         },
                         "docx": {
-                            "flatten_media_to_text": false,
                             "preprocess": [
                                 "main_content"
                             ],
@@ -81,7 +80,6 @@
                             ]
                         },
                         "pdf": {
-                            "flatten_media_to_text": false,
                             "parse_method": "DeepDOC",
                             "preprocess": [
                                 "main_content"
@@ -473,7 +471,6 @@
                             "setups": [
                                 {
                                     "fileFormat": "pdf",
-                                    "flatten_media_to_text": false,
                                     "parse_method": "DeepDOC",
                                     "preprocess": [
                                         "main_content"
@@ -502,7 +499,6 @@
                                 },
                                 {
                                     "fileFormat": "docx",
-                                    "flatten_media_to_text": false,
                                     "preprocess": [
                                         "main_content"
                                     ],

internal/ingestion/pipeline/template/ingestion_pipeline_general.json

@@ -44,7 +44,6 @@
                             ]
                         },
                         "docx": {
-                            "flatten_media_to_text": false,
                             "preprocess": [
                                 "main_content"
                             ],
@@ -63,7 +62,6 @@
                             ]
                         },
                         "markdown": {
-                            "flatten_media_to_text": false,
                             "preprocess": [
                                 "main_content"
                             ],
@@ -75,7 +73,6 @@
                             "vlm": {}
                         },
                         "pdf": {
-                            "flatten_media_to_text": false,
                             "parse_method": "DeepDOC",
                             "preprocess": [
                                 "main_content"
@@ -92,7 +89,6 @@
                             ]
                         },
                         "spreadsheet": {
-                            "flatten_media_to_text": false,
                             "parse_method": "DeepDOC",
                             "preprocess": [
                                 "main_content"
@@ -315,7 +311,6 @@
                             "setups": [
                                 {
                                     "fileFormat": "pdf",
-                                    "flatten_media_to_text": false,
                                     "parse_method": "DeepDOC",
                                     "preprocess": [
                                         "main_content"
@@ -324,7 +319,6 @@
                                 },
                                 {
                                     "fileFormat": "spreadsheet",
-                                    "flatten_media_to_text": false,
                                     "parse_method": "DeepDOC",
                                     "preprocess": [
                                         "main_content"
@@ -333,7 +327,6 @@
                                 },
                                 {
                                     "fileFormat": "markdown",
-                                    "flatten_media_to_text": false,
                                     "preprocess": [
                                         "main_content"
                                     ],
@@ -359,7 +352,6 @@
                                 },
                                 {
                                     "fileFormat": "docx",
-                                    "flatten_media_to_text": false,
                                     "preprocess": [
                                         "main_content"
                                     ],

internal/ingestion/pipeline/template/ingestion_pipeline_laws.json

@@ -62,7 +62,6 @@
                             ]
                         },
                         "docx": {
-                            "flatten_media_to_text": false,
                             "preprocess": [
                                 "main_content"
                             ],
@@ -81,7 +80,6 @@
                             ]
                         },
                         "markdown": {
-                            "flatten_media_to_text": false,
                             "preprocess": [
                                 "main_content"
                             ],
@@ -93,7 +91,6 @@
                             "vlm": {}
                         },
                         "pdf": {
-                            "flatten_media_to_text": false,
                             "parse_method": "DeepDOC",
                             "preprocess": [
                                 "main_content"
@@ -488,7 +485,6 @@
                             "setups": [
                                 {
                                     "fileFormat": "pdf",
-                                    "flatten_media_to_text": false,
                                     "parse_method": "DeepDOC",
                                     "preprocess": [
                                         "main_content"
@@ -496,7 +492,6 @@
                                 },
                                 {
                                     "fileFormat": "markdown",
-                                    "flatten_media_to_text": false,
                                     "preprocess": [
                                         "main_content"
                                     ]
@@ -521,7 +516,6 @@
                                 },
                                 {
                                     "fileFormat": "docx",
-                                    "flatten_media_to_text": false,
                                     "preprocess": [
                                         "main_content"
                                     ]

internal/ingestion/pipeline/template/ingestion_pipeline_manual.json

@@ -62,7 +62,6 @@
                             ]
                         },
                         "docx": {
-                            "flatten_media_to_text": false,
                             "preprocess": [
                                 "main_content"
                             ],
@@ -72,7 +71,6 @@
                             "vlm": {}
                         },
                         "pdf": {
-                            "flatten_media_to_text": false,
                             "parse_method": "DeepDOC",
                             "preprocess": [
                                 "main_content"
@@ -455,7 +453,6 @@
                             "setups": [
                                 {
                                     "fileFormat": "pdf",
-                                    "flatten_media_to_text": false,
                                     "parse_method": "DeepDOC",
                                     "preprocess": [
                                         "main_content"
@@ -469,7 +466,6 @@
                                 },
                                 {
                                     "fileFormat": "docx",
-                                    "flatten_media_to_text": false,
                                     "preprocess": [
                                         "main_content"
                                     ]

internal/ingestion/pipeline/template/ingestion_pipeline_one.json

@@ -62,7 +62,6 @@
                             ]
                         },
                         "docx": {
-                            "flatten_media_to_text": false,
                             "preprocess": [
                                 "main_content"
                             ],
@@ -81,7 +80,6 @@
                             ]
                         },
                         "markdown": {
-                            "flatten_media_to_text": false,
                             "preprocess": [
                                 "main_content"
                             ],
@@ -93,7 +91,6 @@
                             "vlm": {}
                         },
                         "pdf": {
-                            "flatten_media_to_text": false,
                             "parse_method": "DeepDOC",
                             "preprocess": [
                                 "main_content"
@@ -104,7 +101,6 @@
                             "vlm": {}
                         },
                         "spreadsheet": {
-                            "flatten_media_to_text": false,
                             "parse_method": "DeepDOC",
                             "preprocess": [
                                 "main_content"
@@ -280,7 +276,6 @@
                             "setups": [
                                 {
                                     "fileFormat": "pdf",
-                                    "flatten_media_to_text": false,
                                     "parse_method": "DeepDOC",
                                     "preprocess": [
                                         "main_content"
@@ -288,7 +283,6 @@
                                 },
                                 {
                                     "fileFormat": "spreadsheet",
-                                    "flatten_media_to_text": false,
                                     "parse_method": "DeepDOC",
                                     "preprocess": [
                                         "main_content"
@@ -296,7 +290,6 @@
                                 },
                                 {
                                     "fileFormat": "markdown",
-                                    "flatten_media_to_text": false,
                                     "preprocess": [
                                         "main_content"
                                     ]
@@ -321,7 +314,6 @@
                                 },
                                 {
                                     "fileFormat": "docx",
-                                    "flatten_media_to_text": false,
                                     "preprocess": [
                                         "main_content"
                                     ]

internal/ingestion/pipeline/template/ingestion_pipeline_paper.json

@@ -55,7 +55,6 @@
                         },
                         "pdf": {
                             "enable_multi_column": true,
-                            "flatten_media_to_text": false,
                             "parse_method": "DeepDOC",
                             "preprocess": [
                                 "main_content"
@@ -442,7 +441,6 @@
                                 {
                                     "enable_multi_column": true,
                                     "fileFormat": "pdf",
-                                    "flatten_media_to_text": false,
                                     "parse_method": "DeepDOC",
                                     "preprocess": [
                                         "main_content"

internal/ingestion/pipeline/template/ingestion_pipeline_presentation.json

@@ -54,7 +54,6 @@
                             }
                         },
                         "pdf": {
-                            "flatten_media_to_text": false,
                             "parse_method": "DeepDOC",
                             "preprocess": [
                                 "main_content"
@@ -233,7 +232,6 @@
                             "setups": [
                                 {
                                     "fileFormat": "pdf",
-                                    "flatten_media_to_text": false,
                                     "parse_method": "DeepDOC",
                                     "preprocess": [
                                         "main_content"

internal/ingestion/pipeline/template/ingestion_pipeline_qa.json

@@ -54,7 +54,6 @@
                             }
                         },
                         "docx": {
-                            "flatten_media_to_text": false,
                             "preprocess": [
                                 "main_content"
                             ],
@@ -64,7 +63,6 @@
                             "vlm": {}
                         },
                         "markdown": {
-                            "flatten_media_to_text": false,
                             "preprocess": [
                                 "main_content"
                             ],
@@ -76,7 +74,6 @@
                             "vlm": {}
                         },
                         "pdf": {
-                            "flatten_media_to_text": false,
                             "parse_method": "DeepDOC",
                             "preprocess": [
                                 "main_content"
@@ -87,7 +84,6 @@
                             "vlm": {}
                         },
                         "spreadsheet": {
-                            "flatten_media_to_text": false,
                             "parse_method": "DeepDOC",
                             "preprocess": [
                                 "main_content"
@@ -225,7 +221,6 @@
                             "setups": [
                                 {
                                     "fileFormat": "pdf",
-                                    "flatten_media_to_text": false,
                                     "parse_method": "DeepDOC",
                                     "preprocess": [
                                         "main_content"
@@ -234,7 +229,6 @@
                                 },
                                 {
                                     "fileFormat": "spreadsheet",
-                                    "flatten_media_to_text": false,
                                     "parse_method": "DeepDOC",
                                     "preprocess": [
                                         "main_content"
@@ -243,7 +237,6 @@
                                 },
                                 {
                                     "fileFormat": "markdown",
-                                    "flatten_media_to_text": false,
                                     "preprocess": [
                                         "main_content"
                                     ],
@@ -257,7 +250,6 @@
                                 },
                                 {
                                     "fileFormat": "docx",
-                                    "flatten_media_to_text": false,
                                     "preprocess": [
                                         "main_content"
                                     ],

internal/ingestion/pipeline/template/ingestion_pipeline_table.json

@@ -54,7 +54,6 @@
                             }
                         },
                         "spreadsheet": {
-                            "flatten_media_to_text": false,
                             "parse_method": "DeepDOC",
                             "preprocess": [
                                 "main_content"
@@ -195,7 +194,6 @@
                             "setups": [
                                 {
                                     "fileFormat": "spreadsheet",
-                                    "flatten_media_to_text": false,
                                     "parse_method": "DeepDOC",
                                     "preprocess": [
                                         "main_content"

internal/ingestion/task/pipeline_executor_defaults_test.go

@@ -39,17 +39,17 @@ import (
 // serialization order inside each entry is irrelevant.
 var builtinComponentParamsGolden = map[string]string{
 	"audio":        "{\"File\": {}, \"Parser:SongsFillAir\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"audio\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"aac\", \"aiff\", \"ape\", \"au\", \"da\", \"flac\", \"midi\", \"mp3\", \"ogg\", \"oggvorbis\", \"realaudio\", \"vqf\", \"wav\", \"wave\", \"wma\"]}}, \"OneChunker:BlueSkiesLaugh\": {}, \"Tokenizer:KindEyesWatch\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
-	"book":         "{\"File\":{},\"Parser:HipSignsRhyme\":{\"enable_vision_enhancement\":false,\"vlm\":{\"llm_id\":\"\"},\"doc\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"doc\"]},\"docx\":{\"flatten_media_to_text\":false,\"preprocess\":[\"main_content\"],\"suffix\":[\"docx\"],\"vlm\":{}},\"html\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"htm\",\"html\"]},\"pdf\":{\"flatten_media_to_text\":false,\"parse_method\":\"DeepDOC\",\"preprocess\":[\"main_content\"],\"remove_toc\":true,\"suffix\":[\"pdf\"],\"vlm\":{}},\"text&code\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"txt\"]}},\"TitleChunker:GrumpyGarlicsBake\":{\"hierarchy\":5,\"include_heading_content\":true,\"levels\":[[\"^#[^#]\",\"^##[^#]\",\"^###[^#]\",\"^####[^#]\"],[\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\",\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"第[零一二三四五六七八九十百0-9]+条\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"],[\"第[0-9]+章\",\"第[0-9]+节\",\"[0-9]{1,2}[\\\\. 、]\",\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\",\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"],[\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"[零一二三四五六七八九十百]+[ 、]\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\",\"[\\\\(（][0-9]{,2}[\\\\)）]\"],[\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\",\"Chapter (I+V?|VI*|XI|IX|X)\",\"Section [0-9]+\",\"Article [0-9]+\"]],\"hierarchyRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"groupRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"hierarchyHierarchy\":\"5\",\"hierarchyGroup\":\"0\",\"method\":\"hierarchy\"},\"Tokenizer:HotDonutsRing\":{\"fields\":\"text\",\"filename_embd_weight\":0.1,\"search_method\":[\"embedding\",\"full_text\"]},\"Extractor:AutoExtractDefault\":{\"llm_id\":\"\",\"keywords\":{\"top_n\":0},\"questions\":{\"top_n\":0},\"tags\":{\"top_n\":0,\"tag_file_id\":\"\"},\"summary\":{\"enabled\":false},\"metadata\":{\"enabled\":false,\"metadata\":[],\"built_in_metadata\":[]}}}",
+	"book":         "{\"File\":{},\"Parser:HipSignsRhyme\":{\"enable_vision_enhancement\":false,\"vlm\":{\"llm_id\":\"\"},\"doc\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"doc\"]},\"docx\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"docx\"],\"vlm\":{}},\"html\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"htm\",\"html\"]},\"pdf\":{\"parse_method\":\"DeepDOC\",\"preprocess\":[\"main_content\"],\"remove_toc\":true,\"suffix\":[\"pdf\"],\"vlm\":{}},\"text&code\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"txt\"]}},\"TitleChunker:GrumpyGarlicsBake\":{\"hierarchy\":5,\"include_heading_content\":true,\"levels\":[[\"^#[^#]\",\"^##[^#]\",\"^###[^#]\",\"^####[^#]\"],[\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\",\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"第[零一二三四五六七八九十百0-9]+条\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"],[\"第[0-9]+章\",\"第[0-9]+节\",\"[0-9]{1,2}[\\\\. 、]\",\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\",\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"],[\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"[零一二三四五六七八九十百]+[ 、]\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\",\"[\\\\(（][0-9]{,2}[\\\\)）]\"],[\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\",\"Chapter (I+V?|VI*|XI|IX|X)\",\"Section [0-9]+\",\"Article [0-9]+\"]],\"hierarchyRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"groupRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"hierarchyHierarchy\":\"5\",\"hierarchyGroup\":\"0\",\"method\":\"hierarchy\"},\"Tokenizer:HotDonutsRing\":{\"fields\":\"text\",\"filename_embd_weight\":0.1,\"search_method\":[\"embedding\",\"full_text\"]},\"Extractor:AutoExtractDefault\":{\"llm_id\":\"\",\"keywords\":{\"top_n\":0},\"questions\":{\"top_n\":0},\"tags\":{\"top_n\":0,\"tag_file_id\":\"\"},\"summary\":{\"enabled\":false},\"metadata\":{\"enabled\":false,\"metadata\":[],\"built_in_metadata\":[]}}}",
 	"email":        "{\"File\": {}, \"Parser:BirdsFlutterHigh\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"email\": {\"fields\": [\"from\", \"to\", \"cc\", \"bcc\", \"date\", \"subject\", \"body\", \"attachments\"], \"preprocess\": [\"main_content\"], \"suffix\": [\"eml\"]}}, \"TokenChunker:WarmBreadSmells\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"delimiter\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \"；\", \"！\", \"？\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:NiceWordsSpoken\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
-	"general":      "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"doc\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"pages\": [[1, 100000]], \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"txt\", \"py\", \"js\", \"java\", \"c\", \"cpp\", \"h\", \"php\", \"go\", \"ts\", \"sh\", \"cs\", \"kt\", \"sql\"]}, \"image\": {\"ocr_enabled\": true, \"preprocess\": [\"main_content\"], \"suffix\": [\"bmp\", \"gif\", \"jpeg\", \"jpg\", \"png\", \"svg\", \"tif\", \"tiff\", \"webp\"]}, \"email\": {\"fields\": [\"from\", \"to\", \"cc\", \"bcc\", \"date\", \"subject\", \"body\", \"attachments\"], \"preprocess\": [\"main_content\"], \"suffix\": [\"eml\", \"msg\"]}}, \"GeneralChunker:SixApplesFall\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiters\": [\"\\n\", \"!\", \"?\", \";\", \"。\", \"；\", \"！\", \"？\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:LegalReadersDecide\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
-	"laws":         "{\"File\":{},\"Parser:HipSignsRhyme\":{\"enable_vision_enhancement\":false,\"vlm\":{\"llm_id\":\"\"},\"doc\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"doc\"]},\"docx\":{\"flatten_media_to_text\":false,\"preprocess\":[\"main_content\"],\"suffix\":[\"docx\"],\"vlm\":{}},\"html\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"htm\",\"html\"]},\"markdown\":{\"flatten_media_to_text\":false,\"preprocess\":[\"main_content\"],\"suffix\":[\"md\",\"markdown\",\"mdx\"],\"vlm\":{}},\"pdf\":{\"flatten_media_to_text\":false,\"parse_method\":\"DeepDOC\",\"preprocess\":[\"main_content\"],\"remove_toc\":true,\"suffix\":[\"pdf\"],\"vlm\":{}},\"text&code\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"txt\"]}},\"TitleChunker:SpicyKeysKick\":{\"hierarchy\":2,\"include_heading_content\":false,\"levels\":[[\"^#[^#]\",\"^##[^#]\",\"^###[^#]\",\"^####[^#]\"],[\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\",\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"第[零一二三四五六七八九十百0-9]+条\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"],[\"第[0-9]+章\",\"第[0-9]+节\",\"[0-9]{1,2}[\\\\. 、]\",\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\",\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"],[\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"[零一二三四五六七八九十百]+[ 、]\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\",\"[\\\\(（][0-9]{,2}[\\\\)）]\"],[\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\",\"Chapter (I+V?|VI*|XI|IX|X)\",\"Section [0-9]+\",\"Article [0-9]+\"]],\"hierarchyRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"groupRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"hierarchyHierarchy\":\"2\",\"hierarchyGroup\":\"0\",\"method\":\"hierarchy\"},\"Tokenizer:PublicJobsTake\":{\"fields\":\"text\",\"filename_embd_weight\":0.1,\"search_method\":[\"embedding\",\"full_text\"]},\"Extractor:AutoExtractDefault\":{\"llm_id\":\"\",\"keywords\":{\"top_n\":0},\"questions\":{\"top_n\":0},\"tags\":{\"top_n\":0,\"tag_file_id\":\"\"},\"summary\":{\"enabled\":false},\"metadata\":{\"enabled\":false,\"metadata\":[],\"built_in_metadata\":[]}}}",
-	"manual":       "{\"File\":{},\"Parser:HipSignsRhyme\":{\"enable_vision_enhancement\":false,\"vlm\":{\"llm_id\":\"\"},\"doc\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"doc\"]},\"docx\":{\"flatten_media_to_text\":false,\"preprocess\":[\"main_content\"],\"suffix\":[\"docx\"],\"vlm\":{}},\"pdf\":{\"flatten_media_to_text\":false,\"parse_method\":\"DeepDOC\",\"preprocess\":[\"main_content\"],\"suffix\":[\"pdf\"],\"vlm\":{}}},\"ManualChunker:NineInsectsFind\":{\"hierarchy\":0,\"include_heading_content\":false,\"levels\":[[\"^#[^#]\",\"^##[^#]\",\"^###[^#]\",\"^####[^#]\"],[\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\",\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"第[零一二三四五六七八九十百0-9]+条\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"],[\"第[0-9]+章\",\"第[0-9]+节\",\"[0-9]{1,2}[\\\\. 、]\",\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\",\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"],[\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"[零一二三四五六七八九十百]+[ 、]\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\",\"[\\\\(（][0-9]{,2}[\\\\)）]\"],[\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\",\"Chapter (I+V?|VI*|XI|IX|X)\",\"Section [0-9]+\",\"Article [0-9]+\"]],\"hierarchyRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"groupRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"hierarchyHierarchy\":\"0\",\"hierarchyGroup\":\"0\",\"method\":\"group\"},\"Tokenizer:FunnyBalloonsGrin\":{\"fields\":\"text\",\"filename_embd_weight\":0.1,\"search_method\":[\"embedding\",\"full_text\"]},\"Extractor:AutoExtractDefault\":{\"llm_id\":\"\",\"keywords\":{\"top_n\":0},\"questions\":{\"top_n\":0},\"tags\":{\"top_n\":0,\"tag_file_id\":\"\"},\"summary\":{\"enabled\":false},\"metadata\":{\"enabled\":false,\"metadata\":[],\"built_in_metadata\":[]}}}",
-	"one":          "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"doc\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\"], \"vlm\": {}}, \"text&code\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"OneChunker:DryDrinksVisit\": {}, \"Tokenizer:FrankWeeksListen\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
-	"paper":        "{\"File\":{},\"Parser:HipSignsRhyme\":{\"enable_vision_enhancement\":false,\"vlm\":{\"llm_id\":\"\"},\"pdf\":{\"enable_multi_column\":true,\"flatten_media_to_text\":false,\"parse_method\":\"DeepDOC\",\"preprocess\":[\"main_content\"],\"suffix\":[\"pdf\"],\"vlm\":{}}},\"TitleChunker:SparklySchoolsTravel\":{\"hierarchy\":0,\"include_heading_content\":false,\"levels\":[[\"^#[^#]\",\"^##[^#]\",\"^###[^#]\",\"^####[^#]\"],[\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\",\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"第[零一二三四五六七八九十百0-9]+条\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"],[\"第[0-9]+章\",\"第[0-9]+节\",\"[0-9]{1,2}[\\\\. 、]\",\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\",\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"],[\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"[零一二三四五六七八九十百]+[ 、]\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\",\"[\\\\(（][0-9]{,2}[\\\\)）]\"],[\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\",\"Chapter (I+V?|VI*|XI|IX|X)\",\"Section [0-9]+\",\"Article [0-9]+\"]],\"hierarchyRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"groupRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"hierarchyHierarchy\":\"0\",\"hierarchyGroup\":\"0\",\"method\":\"group\"},\"Tokenizer:GreatCarsWash\":{\"fields\":\"text\",\"filename_embd_weight\":0.1,\"search_method\":[\"embedding\",\"full_text\"]},\"Extractor:AutoExtractDefault\":{\"llm_id\":\"\",\"keywords\":{\"top_n\":0},\"questions\":{\"top_n\":0},\"tags\":{\"top_n\":0,\"tag_file_id\":\"\"},\"summary\":{\"enabled\":false},\"metadata\":{\"enabled\":false,\"metadata\":[],\"built_in_metadata\":[]}}}",
+	"general":      "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"doc\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"pages\": [[1, 100000]], \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"txt\", \"py\", \"js\", \"java\", \"c\", \"cpp\", \"h\", \"php\", \"go\", \"ts\", \"sh\", \"cs\", \"kt\", \"sql\"]}, \"image\": {\"ocr_enabled\": true, \"preprocess\": [\"main_content\"], \"suffix\": [\"bmp\", \"gif\", \"jpeg\", \"jpg\", \"png\", \"svg\", \"tif\", \"tiff\", \"webp\"]}, \"email\": {\"fields\": [\"from\", \"to\", \"cc\", \"bcc\", \"date\", \"subject\", \"body\", \"attachments\"], \"preprocess\": [\"main_content\"], \"suffix\": [\"eml\", \"msg\"]}}, \"GeneralChunker:SixApplesFall\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiters\": [\"\\n\", \"!\", \"?\", \";\", \"。\", \"；\", \"！\", \"？\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:LegalReadersDecide\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
+	"laws":         "{\"File\":{},\"Parser:HipSignsRhyme\":{\"enable_vision_enhancement\":false,\"vlm\":{\"llm_id\":\"\"},\"doc\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"doc\"]},\"docx\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"docx\"],\"vlm\":{}},\"html\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"htm\",\"html\"]},\"markdown\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"md\",\"markdown\",\"mdx\"],\"vlm\":{}},\"pdf\":{\"parse_method\":\"DeepDOC\",\"preprocess\":[\"main_content\"],\"remove_toc\":true,\"suffix\":[\"pdf\"],\"vlm\":{}},\"text&code\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"txt\"]}},\"TitleChunker:SpicyKeysKick\":{\"hierarchy\":2,\"include_heading_content\":false,\"levels\":[[\"^#[^#]\",\"^##[^#]\",\"^###[^#]\",\"^####[^#]\"],[\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\",\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"第[零一二三四五六七八九十百0-9]+条\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"],[\"第[0-9]+章\",\"第[0-9]+节\",\"[0-9]{1,2}[\\\\. 、]\",\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\",\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"],[\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"[零一二三四五六七八九十百]+[ 、]\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\",\"[\\\\(（][0-9]{,2}[\\\\)）]\"],[\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\",\"Chapter (I+V?|VI*|XI|IX|X)\",\"Section [0-9]+\",\"Article [0-9]+\"]],\"hierarchyRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"groupRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"hierarchyHierarchy\":\"2\",\"hierarchyGroup\":\"0\",\"method\":\"hierarchy\"},\"Tokenizer:PublicJobsTake\":{\"fields\":\"text\",\"filename_embd_weight\":0.1,\"search_method\":[\"embedding\",\"full_text\"]},\"Extractor:AutoExtractDefault\":{\"llm_id\":\"\",\"keywords\":{\"top_n\":0},\"questions\":{\"top_n\":0},\"tags\":{\"top_n\":0,\"tag_file_id\":\"\"},\"summary\":{\"enabled\":false},\"metadata\":{\"enabled\":false,\"metadata\":[],\"built_in_metadata\":[]}}}",
+	"manual":       "{\"File\":{},\"Parser:HipSignsRhyme\":{\"enable_vision_enhancement\":false,\"vlm\":{\"llm_id\":\"\"},\"doc\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"doc\"]},\"docx\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"docx\"],\"vlm\":{}},\"pdf\":{\"parse_method\":\"DeepDOC\",\"preprocess\":[\"main_content\"],\"suffix\":[\"pdf\"],\"vlm\":{}}},\"ManualChunker:NineInsectsFind\":{\"hierarchy\":0,\"include_heading_content\":false,\"levels\":[[\"^#[^#]\",\"^##[^#]\",\"^###[^#]\",\"^####[^#]\"],[\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\",\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"第[零一二三四五六七八九十百0-9]+条\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"],[\"第[0-9]+章\",\"第[0-9]+节\",\"[0-9]{1,2}[\\\\. 、]\",\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\",\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"],[\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"[零一二三四五六七八九十百]+[ 、]\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\",\"[\\\\(（][0-9]{,2}[\\\\)）]\"],[\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\",\"Chapter (I+V?|VI*|XI|IX|X)\",\"Section [0-9]+\",\"Article [0-9]+\"]],\"hierarchyRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"groupRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"hierarchyHierarchy\":\"0\",\"hierarchyGroup\":\"0\",\"method\":\"group\"},\"Tokenizer:FunnyBalloonsGrin\":{\"fields\":\"text\",\"filename_embd_weight\":0.1,\"search_method\":[\"embedding\",\"full_text\"]},\"Extractor:AutoExtractDefault\":{\"llm_id\":\"\",\"keywords\":{\"top_n\":0},\"questions\":{\"top_n\":0},\"tags\":{\"top_n\":0,\"tag_file_id\":\"\"},\"summary\":{\"enabled\":false},\"metadata\":{\"enabled\":false,\"metadata\":[],\"built_in_metadata\":[]}}}",
+	"one":          "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"doc\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\"], \"vlm\": {}}, \"text&code\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"OneChunker:DryDrinksVisit\": {}, \"Tokenizer:FrankWeeksListen\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
+	"paper":        "{\"File\":{},\"Parser:HipSignsRhyme\":{\"enable_vision_enhancement\":false,\"vlm\":{\"llm_id\":\"\"},\"pdf\":{\"enable_multi_column\":true,\"parse_method\":\"DeepDOC\",\"preprocess\":[\"main_content\"],\"suffix\":[\"pdf\"],\"vlm\":{}}},\"TitleChunker:SparklySchoolsTravel\":{\"hierarchy\":0,\"include_heading_content\":false,\"levels\":[[\"^#[^#]\",\"^##[^#]\",\"^###[^#]\",\"^####[^#]\"],[\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\",\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"第[零一二三四五六七八九十百0-9]+条\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"],[\"第[0-9]+章\",\"第[0-9]+节\",\"[0-9]{1,2}[\\\\. 、]\",\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\",\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"],[\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"[零一二三四五六七八九十百]+[ 、]\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\",\"[\\\\(（][0-9]{,2}[\\\\)）]\"],[\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\",\"Chapter (I+V?|VI*|XI|IX|X)\",\"Section [0-9]+\",\"Article [0-9]+\"]],\"hierarchyRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"groupRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"hierarchyHierarchy\":\"0\",\"hierarchyGroup\":\"0\",\"method\":\"group\"},\"Tokenizer:GreatCarsWash\":{\"fields\":\"text\",\"filename_embd_weight\":0.1,\"search_method\":[\"embedding\",\"full_text\"]},\"Extractor:AutoExtractDefault\":{\"llm_id\":\"\",\"keywords\":{\"top_n\":0},\"questions\":{\"top_n\":0},\"tags\":{\"top_n\":0,\"tag_file_id\":\"\"},\"summary\":{\"enabled\":false},\"metadata\":{\"enabled\":false,\"metadata\":[],\"built_in_metadata\":[]}}}",
 	"picture":      "{\"File\": {}, \"Parser:ViewsCaptureLight\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"image\": {\"ocr_enabled\": true, \"preprocess\": [\"main_content\"], \"suffix\": [\"bmp\", \"gif\", \"jpeg\", \"jpg\", \"png\", \"svg\", \"tif\", \"tiff\", \"webp\"]}, \"video\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"3gp\", \"3gpp\", \"avi\", \"flv\", \"mkv\", \"mov\", \"mp4\", \"mpeg\", \"mpg\", \"webm\", \"wmv\"]}}, \"OneChunker:BrightColorsGlow\": {}, \"Tokenizer:SharpLensFocus\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
-	"presentation": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"pdf\": {\"flatten_media_to_text\": false, \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"slides\": {\"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pptx\", \"ppt\", \"dps\", \"dpt\"]}}, \"Tokenizer:TallTreesDance\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"PageChunker:HappyHillsGlow\": {}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
-	"qa":           "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"docx\": {\"flatten_media_to_text\": false, \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"markdown\": {\"flatten_media_to_text\": false, \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"Tokenizer:ColdCloudsDream\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"QAChunker:TidyCloudsThink\": {}}",
-	"table":        "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}, \"column_mode\": \"auto\", \"column_roles\": {}, \"column_names\": []}, \"text&code\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"TableChunker:FastFoxesJump\": {}, \"Tokenizer:DeepLakesShine\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}}",
+	"presentation": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"pdf\": {\"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"slides\": {\"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pptx\", \"ppt\", \"dps\", \"dpt\"]}}, \"Tokenizer:TallTreesDance\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"PageChunker:HappyHillsGlow\": {}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
+	"qa":           "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"docx\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"markdown\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"Tokenizer:ColdCloudsDream\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"QAChunker:TidyCloudsThink\": {}}",
+	"table":        "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"spreadsheet\": {\"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}, \"column_mode\": \"auto\", \"column_roles\": {}, \"column_names\": []}, \"text&code\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"TableChunker:FastFoxesJump\": {}, \"Tokenizer:DeepLakesShine\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}}",
 }
 
 // Per-template test methods. Each resolves default component params from a

internal/parser/parser/markdown_media_test.go

@@ -374,29 +374,19 @@ func TestPDFMarkdownKeepsInlineDataURIImage(t *testing.T) {
 	}
 }
 
-func TestMarkdownParserIngestionFlattenKeepsImageAsText(t *testing.T) {
+func TestMarkdownParserIngestionKeepsImageMedia(t *testing.T) {
 	ctx, cache := recordingMediaContext()
-	p := markdownIngestionParser(t)
-	markdown, ok := p.(*MarkdownParser)
-	if !ok {
-		t.Fatalf("GetParser(FileTypeMarkdown) = %T", p)
-	}
-	markdown.FlattenMediaToText = true
-
-	res := p.ParseWithResult(ctx, "flatten.md", []byte("![figure]("+markdownMediaDataURI+")\n"))
+	res := markdownIngestionParser(t).ParseWithResult(ctx, "image.md", []byte("![figure]("+markdownMediaDataURI+")\n"))
 	if res.Err != nil {
-		t.Fatalf("ParseWithResult: %v", res.Err)
+		t.Fatal(res.Err)
+	}
+	items := ItemsToMaps(res.JSON)
+	if len(items) != 1 || items[0]["doc_type_kwd"] != DocTypeImage || entity.MediaRefFromAny(items[0]["media_ref"]) == nil {
+		t.Fatalf("image item lost its type or media reference: %+v", items)
 	}
 	if len(cache.puts) != 1 || string(cache.puts[0].data) != "hello" {
 		t.Fatalf("cache puts = %+v", cache.puts)
 	}
-	items := ItemsToMaps(res.JSON)
-	if len(items) != 1 || items[0]["doc_type_kwd"] != DocTypeText {
-		t.Fatalf("flattened items = %+v, want one text item", items)
-	}
-	if entity.MediaRefFromAny(items[0]["media_ref"]) == nil {
-		t.Fatalf("flattened image lost its media_ref: %+v", items[0])
-	}
 }
 
 // TestMarkdownParserIngestionCountsInvalidDataURIOnce pins the shared source

internal/parser/parser/markdown_parser.go

@@ -42,10 +42,9 @@ const dataURIPrefix = "data:image/"
 const GoMarkdown = "go_markdown"
 
 type MarkdownParser struct {
-	libType            string
-	ParseMethod        string
-	VLM                map[string]any
-	FlattenMediaToText bool
+	libType     string
+	ParseMethod string
+	VLM         map[string]any
 	// FetchRemoteImages controls whether HTTP(S) Markdown images are
 	// downloaded. NewMarkdownParser enables it for direct Markdown parsing;
 	// callers normalizing untrusted backend responses may disable it.
@@ -81,9 +80,6 @@ func (p *MarkdownParser) ConfigureFromSetup(setup map[string]any) {
 	if v, ok := setup["vlm"].(map[string]any); ok {
 		p.VLM = v
 	}
-	if v, ok := setup["flatten_media_to_text"].(bool); ok {
-		p.FlattenMediaToText = v
-	}
 }
 
 // ParseWithResult implements ParseResultProducer (plan §6.5) and
@@ -179,7 +175,7 @@ func (p *MarkdownParser) parseWithState(ctx context.Context, filename string, da
 		state.resolveMedia = markdownMediaResolver(state, walkCtx, p.FetchRemoteImages)
 		defer func() { state.resolveMedia = previousMediaResolver }()
 	}
-	unresolvedImages := walkMarkdownBlocksWithImages(walkCtx, doc, &items, p.FlattenMediaToText, p.FetchRemoteImages, state)
+	unresolvedImages := walkMarkdownBlocksWithImages(walkCtx, doc, &items, p.FetchRemoteImages, state)
 	if state.err != nil {
 		return ParseResult{Err: state.err}
 	}
@@ -463,22 +459,12 @@ func markdownTableCells(line string) []string {
 // findBlockImage (per-block AST walk) and the item carries
 // `doc_type_kwd: "image"` together with either a run-cache reference
 // (refOnly), the base64-encoded payload (inline mode), or nothing when the
-// source was skipped. When flatten is true, image items use doc_type_kwd="text".
+// source was skipped.
 //
 // Tables are emitted through the HTML walker as structured items in source
 // order, with embedded images linked to their parent table.
-func walkMarkdownBlocksWithImages(ctx context.Context, doc ast.Node, out *[]map[string]any, flatten, fetchRemoteImages bool, htmlState *htmlWalkState) int {
+func walkMarkdownBlocksWithImages(ctx context.Context, doc ast.Node, out *[]map[string]any, fetchRemoteImages bool, htmlState *htmlWalkState) int {
 	budget := htmlState.budget
-	start := len(*out)
-	defer func() {
-		if flatten {
-			for _, item := range (*out)[start:] {
-				if item["doc_type_kwd"] == "image" {
-					item["doc_type_kwd"] = "text"
-				}
-			}
-		}
-	}()
 	unresolvedImages := 0
 	previousResolver := htmlState.resolveImage
 	if !htmlState.refOnly && !htmlState.skipImages {
@@ -534,7 +520,7 @@ func walkMarkdownBlocksWithImages(ctx context.Context, doc ast.Node, out *[]map[
 			for i, entry := range n.GetChildren() {
 				if containsMarkdownTable(entry) {
 					start := len(*out)
-					unresolvedImages += walkMarkdownBlocksWithImages(ctx, entry, out, flatten, fetchRemoteImages, htmlState)
+					unresolvedImages += walkMarkdownBlocksWithImages(ctx, entry, out, fetchRemoteImages, htmlState)
 					for _, item := range (*out)[start:] {
 						if item["doc_type_kwd"] == "text" && item["layout"] == nil {
 							item["layout"] = "list"
@@ -554,7 +540,7 @@ func walkMarkdownBlocksWithImages(ctx context.Context, doc ast.Node, out *[]map[
 			continue
 		case *ast.BlockQuote:
 			if containsMarkdownTable(n) {
-				unresolvedImages += walkMarkdownBlocksWithImages(ctx, n, out, flatten, fetchRemoteImages, htmlState)
+				unresolvedImages += walkMarkdownBlocksWithImages(ctx, n, out, fetchRemoteImages, htmlState)
 				continue
 			}
 			txt = leafText(n)
@@ -591,9 +577,7 @@ func walkMarkdownBlocksWithImages(ctx context.Context, doc ast.Node, out *[]map[
 		// the image payload (and doc_type_kwd:"image") is attached to the
 		// single block that actually contains the ![alt](src) reference.
 		// Scanning the whole document (the old approach) wrongly tagged
-		// every block as an image whenever any image was present. When
-		// flatten is true, keep doc_type_kwd="text" (Python
-		// parser.py:1034: flatten_media_to_text overrides image).
+		// every block as an image whenever any image was present.
 		if !htmlState.skipImages {
 			if imgURL, ok := findBlockImage(child); ok {
 				if htmlState.refOnly {

internal/parser/parser/markdown_parser_test.go

@@ -513,9 +513,8 @@ func TestMarkdownParser_ParseWithResult_InlineRawHTMLSurvives(t *testing.T) {
 func TestMarkdownParser_ConfigureFromSetup(t *testing.T) {
 	p, _ := NewMarkdownParser(GoMarkdown)
 	p.ConfigureFromSetup(map[string]any{
-		"parse_method":          "deepdoc",
-		"vlm":                   map[string]any{"llm_id": "gpt-4-vision"},
-		"flatten_media_to_text": false,
+		"parse_method": "deepdoc",
+		"vlm":          map[string]any{"llm_id": "gpt-4-vision"},
 	})
 	if p.ParseMethod != "deepdoc" {
 		t.Fatalf("ParseMethod = %q, want %q", p.ParseMethod, "deepdoc")
@@ -536,44 +535,6 @@ func TestMarkdownParser_ConfigureFromSetup_NilSafe(t *testing.T) {
 	}
 }
 
-// TestMarkdownParser_FlattenMediaToText verifies that when
-// flatten_media_to_text is true, image items are emitted with
-// doc_type_kwd="text" (mirroring Python parser.py:1034). When false,
-// image items keep doc_type_kwd="image".
-func TestMarkdownParser_FlattenMediaToText(t *testing.T) {
-	ctx := t.Context()
-	pixelB64 := base64.StdEncoding.EncodeToString([]byte("fake-png-data"))
-	md := "Some text with an image\n![test](data:image/png;base64," + pixelB64 + ")\n"
-
-	cases := []struct {
-		name        string
-		flatten     bool
-		wantDocType string
-	}{
-		{"flatten=false keeps image", false, "image"},
-		{"flatten=true forces text", true, "text"},
-	}
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			p, _ := NewMarkdownParser(GoMarkdown)
-			p.ConfigureFromSetup(map[string]any{"flatten_media_to_text": tc.flatten})
-			if p.FlattenMediaToText != tc.flatten {
-				t.Fatalf("FlattenMediaToText field = %v, want %v", p.FlattenMediaToText, tc.flatten)
-			}
-			res := p.ParseWithResult(ctx, "test.md", []byte(md))
-			if res.Err != nil {
-				t.Fatalf("ParseWithResult: %v", res.Err)
-			}
-			for _, item := range ItemsToMaps(res.JSON) {
-				if kd, _ := item["doc_type_kwd"].(string); kd != tc.wantDocType {
-					t.Errorf("doc_type_kwd = %q, want %q (item text=%q)",
-						kd, tc.wantDocType, item["text"])
-				}
-			}
-		})
-	}
-}
-
 func TestResolveImageURL_DataURI(t *testing.T) {
 	b64 := base64.StdEncoding.EncodeToString([]byte("fakeimage"))
 	budget := newLimitedMediaBudgetForTest(t, len("fakeimage"), int64(len("fakeimage")), 2)
@@ -644,7 +605,7 @@ func TestMarkdownImageBudgetSkipsFetchAfterCountLimit(t *testing.T) {
 	doc := markdownNew().Parse([]byte(markdown))
 	budget := newLimitedMediaBudgetForTest(t, 4, 4, 2)
 	var items []map[string]any
-	if unresolved := walkMarkdownBlocksWithImages(t.Context(), doc, &items, false, true, &htmlWalkState{budget: budget}); unresolved != 0 {
+	if unresolved := walkMarkdownBlocksWithImages(t.Context(), doc, &items, true, &htmlWalkState{budget: budget}); unresolved != 0 {
 		t.Fatalf("unresolved images = %d, want 0", unresolved)
 	}
 	if got := fetches.Load(); got != 2 {

internal/parser/parser/pdf_parser_common.go

@@ -59,7 +59,6 @@ type PDFParser struct {
 	Model      string // DeepDoc@buildin@ragflow
 	LibType    string // pdf_oxide, used by DeepDoc
 
-	FlattenMediaToText bool
 	RemoveTOC          bool
 	RemoveHeaderFooter bool
 	EnableMultiColumn  bool
@@ -142,9 +141,6 @@ func (p *PDFParser) ConfigureFromSetup(setup map[string]any) {
 	if p == nil || setup == nil {
 		return
 	}
-	if v, ok := setup["flatten_media_to_text"].(bool); ok {
-		p.FlattenMediaToText = v
-	}
 	if v, ok := setup["remove_toc"].(bool); ok {
 		p.RemoveTOC = v
 	}
@@ -404,10 +400,9 @@ func (p *PDFParser) parseWithResult(ctx context.Context, filename string, data [
 	cfg.RemoveHeaderFooter = p.RemoveHeaderFooter
 	cfg.OnPageDone = p.OnPageDone
 	res := parsePDFWithDeepDocOptions(ctx, filename, data, pdfPostProcessOptions{
-		zoom:               cfg.Zoom,
-		enableMultiColumn:  p.EnableMultiColumn,
-		flattenMediaToText: p.FlattenMediaToText,
-		parseMethod:        p.ParseMethod,
+		zoom:              cfg.Zoom,
+		enableMultiColumn: p.EnableMultiColumn,
+		parseMethod:       p.ParseMethod,
 	}, deepdocpdf.NewParser(cfg).Parse)
 	if res.Err != nil {
 		if errors.Is(res.Err, deepdocpdf.ErrNoPDFData) ||
@@ -555,8 +550,6 @@ func pdfParseResultToJSONWithOptions(filename string, parsed *deepdoctype.ParseR
 	if opts.enableMultiColumn && opts.pageWidth <= 0 {
 		opts.pageWidth = firstPDFPageWidth(processed.PageWidth)
 	}
-	// JSON media policy is applied once at the PDFParser result boundary.
-	opts.flattenMediaToText = false
 	applyPDFPostProcess(&processed, opts)
 	defer processed.Close()
 	// NOTE: PDF media (figure/table) is intentionally NOT cropped here under

internal/parser/parser/pdf_parser_common_test.go

@@ -280,11 +280,10 @@ func TestPDFParseResultToJSON_DefaultKeepsHeaderFooterLikePython(t *testing.T) {
 func TestPDFParser_ConfigureFromSetup(t *testing.T) {
 	p := NewPDFParser()
 	p.ConfigureFromSetup(map[string]any{
-		"parse_method":          "deepdoc",
-		"enable_multi_column":   true,
-		"flatten_media_to_text": true,
-		"remove_toc":            true,
-		"remove_header_footer":  true,
+		"parse_method":         "deepdoc",
+		"enable_multi_column":  true,
+		"remove_toc":           true,
+		"remove_header_footer": true,
 	})
 	if !p.EnableMultiColumn {
 		t.Fatal("EnableMultiColumn = false, want true")
@@ -292,9 +291,6 @@ func TestPDFParser_ConfigureFromSetup(t *testing.T) {
 	if got, want := p.ParseMethod, "deepdoc"; got != want {
 		t.Fatalf("ParseMethod = %q, want %q", got, want)
 	}
-	if !p.FlattenMediaToText {
-		t.Fatal("FlattenMediaToText = false, want true")
-	}
 	if !p.RemoveTOC {
 		t.Fatal("RemoveTOC = false, want true")
 	}

internal/parser/parser/pdf_parser_pages_test.go

@@ -54,7 +54,7 @@ func TestConfigureFromSetup_Pages(t *testing.T) {
 
 	t.Run("missing pages key -> nil", func(t *testing.T) {
 		p := &PDFParser{}
-		setup := map[string]any{"flatten_media_to_text": true}
+		setup := map[string]any{}
 		p.ConfigureFromSetup(setup)
 		if p.Pages != nil {
 			t.Errorf("Pages = %v, want nil", p.Pages)
@@ -65,16 +65,12 @@ func TestConfigureFromSetup_Pages(t *testing.T) {
 	t.Run("other fields still read (no regression)", func(t *testing.T) {
 		p := &PDFParser{}
 		setup := map[string]any{
-			"flatten_media_to_text": true,
-			"parse_method":          "DeepDOC",
+			"parse_method": "DeepDOC",
 			"pages": []any{
 				[]any{float64(1), float64(100)},
 			},
 		}
 		p.ConfigureFromSetup(setup)
-		if !p.FlattenMediaToText {
-			t.Error("FlattenMediaToText not read")
-		}
 		if p.ParseMethod != "DeepDOC" {
 			t.Errorf("ParseMethod = %q, want DeepDOC", p.ParseMethod)
 		}

internal/parser/parser/pdf_postprocess.go

@@ -14,10 +14,9 @@ import (
 // per-box geometry signals those detectors rely on are still intact; they are
 // intentionally not repeated here.
 type pdfPostProcessOptions struct {
-	pageWidth          float64
-	zoom               float64
-	enableMultiColumn  bool
-	flattenMediaToText bool
+	pageWidth         float64
+	zoom              float64
+	enableMultiColumn bool
 	// parseMethod carries the PDFParser.ParseMethod so the DeepDoc analyzer can
 	// be resolved per request (in-process vs a registered external backend).
 	// Empty means the historical in-process default.
@@ -33,7 +32,7 @@ func applyPDFPostProcess(result *deepdoctype.ParseResult, opts pdfPostProcessOpt
 		reorderPDFMultiColumn(result, opts.pageWidth, opts.zoom)
 	}
 	normalizePDFLayoutTypes(result)
-	assignPDFDocTypeKeywords(result, opts.flattenMediaToText)
+	assignPDFDocTypeKeywords(result)
 }
 
 func normalizePDFLayoutTypes(result *deepdoctype.ParseResult) {
@@ -46,13 +45,9 @@ func normalizePDFLayoutTypes(result *deepdoctype.ParseResult) {
 	}
 }
 
-func assignPDFDocTypeKeywords(result *deepdoctype.ParseResult, flatten bool) {
+func assignPDFDocTypeKeywords(result *deepdoctype.ParseResult) {
 	for i := range result.Sections {
 		section := &result.Sections[i]
-		if flatten {
-			section.DocTypeKwd = "text"
-			continue
-		}
 		switch strings.TrimSpace(section.LayoutType) {
 		case deepdoctype.LayoutTypeTable:
 			section.DocTypeKwd = "table"

internal/parser/parser/pdf_postprocess_test.go

@@ -59,24 +59,6 @@ func TestApplyPDFPostProcess_AssignsDocTypeKeywords(t *testing.T) {
 	}
 }
 
-func TestApplyPDFPostProcess_FlattenMediaKeepsImagesButMarksText(t *testing.T) {
-	result := &deepdoctype.ParseResult{
-		Sections: []deepdoctype.Section{
-			{Text: "a", LayoutType: "figure", Image: "abc"},
-			{Text: "b", LayoutType: "table"},
-		},
-	}
-	applyPDFPostProcess(result, pdfPostProcessOptions{flattenMediaToText: true})
-	for i, s := range result.Sections {
-		if s.DocTypeKwd != "text" {
-			t.Fatalf("Sections[%d].DocTypeKwd = %q, want text", i, s.DocTypeKwd)
-		}
-	}
-	if got, want := result.Sections[0].Image, "abc"; got != want {
-		t.Fatalf("Sections[0].Image = %q, want %q", got, want)
-	}
-}
-
 func TestApplyPDFPostProcess_ReordersMultiColumnText(t *testing.T) {
 	cases := []struct {
 		name      string

internal/parser/parser/pdf_result.go

@@ -81,28 +81,6 @@ func (p *PDFParser) NormalizeResult(ctx context.Context, filename string, result
 				fields["doc_type_kwd"] = "text"
 			}
 		}
-		if p.FlattenMediaToText {
-			// Flattened output keeps one text carrier per item. Structured
-			// tables (MinerU content_list, projected HTML) carry cells in
-			// `table` and footnotes in `text`; rendering the table preserves
-			// cells, caption, and footnote in that order. The render
-			// normalizes markup (header inference, escaping, whitespace), so
-			// it is not byte-identical to the source HTML, but it is
-			// content-preserving. Items without a table keep their text verbatim.
-			if table, ok := fields["table"].(*entity.TableData); ok && table != nil {
-				text, _ := fields["text"].(string)
-				fields["text"] = strings.TrimSpace(tableutil.RenderTableHTML(table) + "\n" + text)
-				delete(fields, "table")
-			}
-			// Flattening moves an item's content into text. A crop-only media
-			// item has no content to move: rewriting it to text would hide the
-			// region from vision enhancement and leave an empty item the
-			// chunker drops, so it keeps its media type and stays available to
-			// the on-demand cropper.
-			if hasPDFTextCarrier(fields) || len(positions) == 0 {
-				fields["doc_type_kwd"] = "text"
-			}
-		}
 		items = appendPDFItem(items, fields)
 	}
 	if droppedPositions > 0 {
@@ -311,18 +289,6 @@ func pdfPositionHasRegion(values [5]float64) bool {
 	return values[2] > values[1] && values[4] > values[3]
 }
 
-// hasPDFTextCarrier reports whether flattening an item has any content to move
-// into text: a body, an inline image payload, or a deferred media reference.
-func hasPDFTextCarrier(fields map[string]any) bool {
-	if text, _ := fields["text"].(string); strings.TrimSpace(text) != "" {
-		return true
-	}
-	if img, _ := fields["image"].(string); img != "" {
-		return true
-	}
-	return entity.MediaRefFromAny(fields["media_ref"]) != nil
-}
-
 // HasPDFCropRegion reports whether the item carries at least one PDF crop
 // region, i.e. a position with a positive-area box. A page-only position
 // carries page identity but no region, so it must not trigger a crop.

internal/parser/parser/pdf_result_normalize_markdown_test.go

@@ -70,22 +70,6 @@ func TestPDFParser_NormalizeMarkdown_DoesNotFetchRemoteImages(t *testing.T) {
 	}
 }
 
-// TestPDFParser_NormalizeMarkdown_FlattenMediaToText pins that, with flatten
-// enabled, image markdown is flattened to a text carrier (doc_type_kwd == text).
-func TestPDFParser_NormalizeMarkdown_FlattenMediaToText(t *testing.T) {
-	p := NewPDFParser()
-	p.FlattenMediaToText = true
-	res := p.NormalizeMarkdown(context.Background(), "doc.md", "![img](data:image/png;base64,abc)\n")
-	if res.Err != nil {
-		t.Fatalf("err: %v", res.Err)
-	}
-	for _, it := range res.JSON {
-		if dt, _ := it.ToMap()["doc_type_kwd"].(string); dt != "text" {
-			t.Errorf("doc_type_kwd = %q, want text", dt)
-		}
-	}
-}
-
 // TestPDFParser_NormalizeMarkdown_PreservesFileOnCancel pins that a cancelled
 // context returns the error (not silently swallowed) and still carries the
 // filename in File so callers can report which document failed.

internal/parser/parser/pdf_result_test.go

@@ -121,66 +121,22 @@ func TestPDFNormalizeResultInfersStructuredTable(t *testing.T) {
 	}
 }
 
-func TestPDFNormalizeResultFlattenPreservesContentAndClearsMediaSubtype(t *testing.T) {
-	p := NewPDFParser()
-	p.FlattenMediaToText = true
-	table := &entity.TableData{Grid: [][]entity.TableCell{{{Text: "Cell"}}}, Caption: "Caption"}
-	result := p.NormalizeResult(t.Context(), "sample.pdf", ParseResult{JSON: []DocItem{
-		TableItem{Table: table, Extra: map[string]any{"text": "Footnote"}},
-		ImageItem{Text: "Figure", Image: "image-data", Positions: Positions{AbsoluteBox{PageNumber: 2, Left: 1, Right: 5, Top: 2, Bottom: 6}}},
-		TextItem{Text: "Title", Extra: map[string]any{"layout": "heading"}},
-	}})
-	if result.Err != nil {
-		t.Fatal(result.Err)
-	}
-	for i, item := range result.JSON[:2] {
-		fields := item.ToMap()
-		if fields["doc_type_kwd"] != "text" {
-			t.Fatalf("flattened item %d still has media type: %#v", i, fields)
-		}
-	}
-	text, _ := result.JSON[0].ToMap()["text"].(string)
-	for _, want := range []string{"Cell", "Caption", "Footnote"} {
-		if !strings.Contains(text, want) {
-			t.Fatalf("missing %q in %q", want, text)
-		}
-	}
-	if result.JSON[1].ToMap()["image"] != "image-data" {
-		t.Fatal("flatten lost image")
-	}
-	if _, ok := ExtractPDFPositions(result.JSON[1].ToMap()); !ok {
-		t.Fatal("flatten lost positions")
-	}
-	if result.JSON[2].ToMap()["layout"] != "heading" {
-		t.Fatal("heading layout changed")
-	}
-}
-
-func TestPDFNormalizeResultFlattenKeepsCropOnlyMedia(t *testing.T) {
-	p := NewPDFParser()
-	p.FlattenMediaToText = true
-	result := p.NormalizeResult(t.Context(), "sample.pdf", ParseResult{JSON: MapsToItems([]map[string]any{
+func TestPDFNormalizeResultPreservesMediaAndCropRegion(t *testing.T) {
+	result := NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", ParseResult{JSON: MapsToItems([]map[string]any{
 		{"doc_type_kwd": "image", "layout": "figure", "positions": [][]any{{1, 10, 30, 20, 40}}},
 		{"doc_type_kwd": "image", "layout": "figure", "text": "Figure 1", "positions": [][]any{{1, 10, 30, 50, 70}}},
-		{"doc_type_kwd": "image", "layout": "figure"},
 	})})
-	if result.Err != nil || len(result.JSON) != 3 {
+	if result.Err != nil || len(result.JSON) != 2 {
 		t.Fatalf("unexpected result: %+v", result)
 	}
-	// A crop-only media item has no content to flatten; vision enhancement and
-	// the on-demand cropper still need its type and region.
-	cropOnly := result.JSON[0].ToMap()
-	if cropOnly["doc_type_kwd"] != "image" || !HasPDFCropRegion(cropOnly) {
-		t.Fatalf("crop-only media lost its type or region: %#v", cropOnly)
+	for _, item := range result.JSON {
+		fields := item.ToMap()
+		if fields["doc_type_kwd"] != "image" || !HasPDFCropRegion(fields) {
+			t.Fatalf("image type or crop region lost: %#v", fields)
+		}
 	}
-	captioned := result.JSON[1].ToMap()
-	if captioned["doc_type_kwd"] != "text" || captioned["text"] != "Figure 1" {
-		t.Fatalf("captioned media not flattened: %#v", captioned)
-	}
-	// Without a region there is nothing to crop, so the empty item still
-	// flattens to text and is dropped downstream as before.
-	if bare := result.JSON[2].ToMap(); bare["doc_type_kwd"] != "text" {
-		t.Fatalf("regionless media kept a media type: %#v", bare)
+	if result.JSON[1].ToMap()["text"] != "Figure 1" {
+		t.Fatalf("image caption lost: %#v", result.JSON[1].ToMap())
 	}
 }
 
@@ -198,27 +154,18 @@ func TestPDFNormalizeResultDowngradesUnstructuredTableSubtype(t *testing.T) {
 }
 
 func TestPDFNormalizeResultKeepsPositionedUnstructuredTables(t *testing.T) {
-	for _, flatten := range []bool{false, true} {
-		p := NewPDFParser()
-		p.FlattenMediaToText = flatten
-		result := p.NormalizeResult(t.Context(), "sample.pdf", ParseResult{JSON: MapsToItems([]map[string]any{{
-			"doc_type_kwd": "table", "text": "Recognized table without cells",
-			"positions": [][]any{{2, 10, 30, 20, 40}},
-		}})})
-		if result.Err != nil || len(result.JSON) != 1 {
-			t.Fatalf("unexpected result: %+v", result)
-		}
-		item := result.JSON[0].ToMap()
-		want := "table"
-		if flatten {
-			want = "text"
-		}
-		if item["doc_type_kwd"] != want || item["text"] != "Recognized table without cells" {
-			t.Fatalf("flatten=%v: table media identity or text lost: %#v", flatten, item)
-		}
-		if _, ok := ExtractPDFPositions(item); !ok {
-			t.Fatalf("flatten=%v: table crop region lost: %#v", flatten, item)
-		}
+	result := NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", ParseResult{JSON: MapsToItems([]map[string]any{{
+		"doc_type_kwd": "table", "text": "Recognized table without cells", "positions": [][]any{{2, 10, 30, 20, 40}},
+	}})})
+	if result.Err != nil || len(result.JSON) != 1 {
+		t.Fatalf("unexpected result: %+v", result)
+	}
+	item := result.JSON[0].ToMap()
+	if item["doc_type_kwd"] != "table" || item["text"] != "Recognized table without cells" {
+		t.Fatalf("table identity or text lost: %#v", item)
+	}
+	if _, ok := ExtractPDFPositions(item); !ok {
+		t.Fatalf("table crop region lost: %#v", item)
 	}
 }
 
@@ -244,34 +191,6 @@ func TestPDFNormalizeResultRecoversUnclosedTableCells(t *testing.T) {
 	}
 }
 
-func TestPDFParserSoMarkFlattensStructuredMedia(t *testing.T) {
-	withSSRFBypass(t)
-	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path == "/parse/async" {
-			_, _ = w.Write([]byte(`{"code":0,"data":{"task_id":"task"}}`))
-			return
-		}
-		_, _ = w.Write([]byte(`{"code":0,"data":{"status":"SUCCESS","result":{"outputs":{"json":{"pages":[{"blocks":[{"type":"figure","content":"Figure caption"},{"type":"table","content":"<table><tr><td>Cell value</td></tr></table>"}]}]}}}}}`))
-	}))
-	defer server.Close()
-	p := NewPDFParser()
-	p.ConfigureFromSetup(map[string]any{"parse_method": "somark", "somark_base_url": server.URL, "flatten_media_to_text": true})
-	result := p.ParseWithResult(t.Context(), "sample.pdf", []byte("pdf"))
-	if result.Err != nil {
-		t.Fatal(result.Err)
-	}
-	if len(result.JSON) != 2 {
-		t.Fatalf("items = %#v", result.JSON)
-	}
-	for i, want := range []string{"Figure caption", "Cell value"} {
-		item := result.JSON[i].ToMap()
-		text, _ := item["text"].(string)
-		if item["doc_type_kwd"] != "text" || !strings.Contains(text, want) {
-			t.Fatalf("item %d = %#v; want text containing %q", i, item, want)
-		}
-	}
-}
-
 func TestProjectPDFTablePreservesExistingStructureAndNotes(t *testing.T) {
 	table := &entity.TableData{Grid: [][]entity.TableCell{{{Text: "Original cell"}}}}
 	item := map[string]any{"doc_type_kwd": "table", "table": table, "text": "<table><tr><td>Other</td></tr></table>\nFootnote"}
@@ -323,8 +242,8 @@ func TestPDFWhitespaceTableDescriptionCannotHideCells(t *testing.T) {
 func TestPDFNormalizeResultPreservesAdjacentTables(t *testing.T) {
 	const markup = "<table><tr><td>First cell</td></tr></table>\n<table><tr><td>Second cell</td></tr></table>\nFootnote"
 	for _, projected := range []bool{false, true} {
-		for _, flatten := range []bool{false, true} {
-			name := fmt.Sprintf("projected=%v/flatten=%v", projected, flatten)
+		{
+			name := fmt.Sprintf("projected=%v", projected)
 			t.Run(name, func(t *testing.T) {
 				item := map[string]any{
 					"doc_type_kwd": "table", "text": markup, "page_number": 3,
@@ -335,45 +254,29 @@ func TestPDFNormalizeResultPreservesAdjacentTables(t *testing.T) {
 				}
 				originalText := item["text"]
 				p := NewPDFParser()
-				p.FlattenMediaToText = flatten
+
 				result := p.NormalizeResult(t.Context(), "sample.pdf", pdfItemsToResult("sample.pdf", []map[string]any{item}, 3))
 				if result.Err != nil {
 					t.Fatal(result.Err)
 				}
-				if flatten {
-					var text string
-					for _, doc := range result.JSON {
-						fields := doc.ToMap()
-						if fields["doc_type_kwd"] != "text" || fields["table"] != nil {
-							t.Fatalf("media not flattened: %#v", fields)
-						}
-						value, _ := fields["text"].(string)
-						text += value
+
+				if len(result.JSON) != 3 {
+					t.Fatalf("want two tables and a footnote, got %#v", result.JSON)
+				}
+				for i, want := range []string{"First cell", "Second cell"} {
+					fields := result.JSON[i].ToMap()
+					table, _ := fields["table"].(*entity.TableData)
+					if table == nil || table.Page != 3 || len(table.Grid) != 1 || len(table.Grid[0]) != 1 || table.Grid[0][0].Text != want || fields["text"] != nil {
+						t.Fatalf("table %d lost cells or metadata: %#v", i, fields)
 					}
-					for _, want := range []string{"First cell", "Second cell", "Footnote"} {
-						if strings.Count(text, want) != 1 {
-							t.Fatalf("want %q exactly once in %q", want, text)
-						}
-					}
-				} else {
-					if len(result.JSON) != 3 {
-						t.Fatalf("want two tables and a footnote, got %#v", result.JSON)
-					}
-					for i, want := range []string{"First cell", "Second cell"} {
-						fields := result.JSON[i].ToMap()
-						table, _ := fields["table"].(*entity.TableData)
-						if table == nil || table.Page != 3 || len(table.Grid) != 1 || len(table.Grid[0]) != 1 || table.Grid[0][0].Text != want || fields["text"] != nil {
-							t.Fatalf("table %d lost cells or metadata: %#v", i, fields)
-						}
-						if _, ok := ExtractPDFPositions(fields); !ok {
-							t.Fatalf("table %d lost crop positions", i)
-						}
-					}
-					note := result.JSON[2].ToMap()
-					if note["text"] != "Footnote" || note["doc_type_kwd"] != "text" || note["page_number"] != 3 || note["_preserve_order"] != true {
-						t.Fatalf("footnote lost text or metadata: %#v", note)
+					if _, ok := ExtractPDFPositions(fields); !ok {
+						t.Fatalf("table %d lost crop positions", i)
 					}
 				}
+				note := result.JSON[2].ToMap()
+				if note["text"] != "Footnote" || note["doc_type_kwd"] != "text" || note["page_number"] != 3 || note["_preserve_order"] != true {
+					t.Fatalf("footnote lost text or metadata: %#v", note)
+				}
 				if item["text"] != originalText {
 					t.Fatal("source item mutated")
 				}
@@ -384,34 +287,29 @@ func TestPDFNormalizeResultPreservesAdjacentTables(t *testing.T) {
 
 func TestPDFNormalizeResultIsIdempotent(t *testing.T) {
 	const markup = "<table><tr><td>First cell</td></tr></table>\n<table><tr><td>Second cell</td></tr></table>\nFootnote"
-	for _, flatten := range []bool{false, true} {
-		p := NewPDFParser()
-		p.FlattenMediaToText = flatten
-		item := map[string]any{
-			"doc_type_kwd": "table", "text": markup, "page_number": 3,
-			"positions": [][]any{{3, 10, 30, 20, 40}}, "_preserve_order": true,
+	p := NewPDFParser()
+	item := map[string]any{
+		"doc_type_kwd": "table", "text": markup, "page_number": 3,
+		"positions": [][]any{{3, 10, 30, 20, 40}}, "_preserve_order": true,
+	}
+	once := p.NormalizeResult(t.Context(), "sample.pdf", pdfItemsToResult("sample.pdf", []map[string]any{item}, 3))
+	if once.Err != nil {
+		t.Fatal(once.Err)
+	}
+	twice := p.NormalizeResult(t.Context(), "sample.pdf", once)
+	if twice.Err != nil {
+		t.Fatal(twice.Err)
+	}
+	if len(once.JSON) != len(twice.JSON) {
+		t.Fatalf("second normalize changed item count %d -> %d", len(once.JSON), len(twice.JSON))
+	}
+	for i := range once.JSON {
+		a, b := once.JSON[i].ToMap(), twice.JSON[i].ToMap()
+		if a["doc_type_kwd"] != b["doc_type_kwd"] {
+			t.Fatalf("item %d type changed %v -> %v", i, a["doc_type_kwd"], b["doc_type_kwd"])
 		}
-		once := p.NormalizeResult(t.Context(), "sample.pdf", pdfItemsToResult("sample.pdf", []map[string]any{item}, 3))
-		if once.Err != nil {
-			t.Fatal(once.Err)
-		}
-		twice := p.NormalizeResult(t.Context(), "sample.pdf", once)
-		if twice.Err != nil {
-			t.Fatal(twice.Err)
-		}
-		if len(once.JSON) != len(twice.JSON) {
-			t.Fatalf("flatten=%v: second normalize changed item count %d -> %d", flatten, len(once.JSON), len(twice.JSON))
-		}
-		for i := range once.JSON {
-			a, b := once.JSON[i].ToMap(), twice.JSON[i].ToMap()
-			if a["doc_type_kwd"] != b["doc_type_kwd"] {
-				t.Fatalf("flatten=%v: item %d type changed %v -> %v", flatten, i, a["doc_type_kwd"], b["doc_type_kwd"])
-			}
-			ta, _ := a["text"].(string)
-			tb, _ := b["text"].(string)
-			if ta != tb {
-				t.Fatalf("flatten=%v: item %d text changed %q -> %q", flatten, i, ta, tb)
-			}
+		if a["text"] != b["text"] {
+			t.Fatalf("item %d text changed %v -> %v", i, a["text"], b["text"])
 		}
 	}
 }

internal/parser/parser/table_regression_test.go

@@ -44,10 +44,9 @@ func TestTableRegressionDOCXCaptionAfterNestedTable(t *testing.T) {
 	}
 }
 
-func TestTableRegressionMarkdownFlattenBesideTable(t *testing.T) {
+func TestTableRegressionMarkdownImageBesideTable(t *testing.T) {
 	source := "![logo](data:image/png;base64,aGVsbG8=) Before <table><tr><td>A</td><td>B</td></tr></table> After"
 	p, _ := NewMarkdownParser(GoMarkdown)
-	p.FlattenMediaToText = true
 	after := p.ParseWithResult(t.Context(), "x.md", []byte(source))
 	if after.Err != nil || len(after.JSON) == 0 {
 		t.Fatalf("result=%+v", after)
@@ -57,9 +56,6 @@ func TestTableRegressionMarkdownFlattenBesideTable(t *testing.T) {
 		if item.ToMap()["image"] != nil {
 			payloads++
 		}
-		if item.ToMap()["doc_type_kwd"] == "image" {
-			t.Error("flatten_media_to_text ignored")
-		}
 	}
 	if payloads != 1 {
 		t.Fatalf("image payloads=%d, want 1", payloads)
@@ -148,7 +144,7 @@ func TestTableRegressionMarkdownRawHTMLImageBudget(t *testing.T) {
 	doc := markdownNew().Parse([]byte(source))
 	budget := newMediaBudgetForTest()
 	var items []map[string]any
-	walkMarkdownBlocksWithImages(t.Context(), doc, &items, false, true, &htmlWalkState{budget: budget})
+	walkMarkdownBlocksWithImages(t.Context(), doc, &items, true, &htmlWalkState{budget: budget})
 	images := 0
 	for _, item := range items {
 		if item["doc_type_kwd"] == "image" && item["image"] != nil {

internal/parser/parser/table_structure_test.go

@@ -98,7 +98,7 @@ func TestMarkdownTableImageTimeoutDoesNotConsumeBudget(t *testing.T) {
 	doc := markdownNew().Parse([]byte("| A | B |\n|---|---|\n| Logo | ![logo](http://127.0.0.1/logo.png) |\n"))
 	budget := newMediaBudgetForTest()
 	var items []map[string]any
-	unresolved := walkMarkdownBlocksWithImages(ctx, doc, &items, false, true, &htmlWalkState{budget: budget})
+	unresolved := walkMarkdownBlocksWithImages(ctx, doc, &items, true, &htmlWalkState{budget: budget})
 	if budget.Count() != 0 || unresolved != 0 {
 		t.Fatalf("timed out image consumed budget: attempts=%d unresolved=%d", budget.Count(), unresolved)
 	}

web/src/locales/az.ts

@@ -2785,9 +2785,6 @@ Nümunə: Virtual host üslubu`,
       oneChunkTitle: 'Qeyd',
       oneChunkDescription:
         'Təhlil edilmiş bütün bölmələr ardıcıllıqla bir hissədə birləşdiriləcək.',
-      flattenMediaToText: 'Görmə modelini söndür',
-      flattenMediaToTextTip:
-        'Şəkil və cədvəl bölmələrini adi mətn kimi qəbul edir və görmə modeli ilə təkmilləşdirməni ötürür.',
       enableChildrenDelimiters: 'Alt hissələr axtarış üçün istifadə olunur',
       merge: 'Birləşdir',
       split: 'Böl',

web/src/locales/fr.ts

@@ -2551,9 +2551,6 @@ Idéal pour : les documents avec un contenu fluide et contextuellement lié —
       oneChunkTitle: 'Note',
       oneChunkDescription:
         'Toutes les sections analysées seront fusionnées dans un seul bloc.',
-      flattenMediaToText: 'Désactiver le modèle de vision',
-      flattenMediaToTextTip:
-        "Traite les sections d'images et de tableaux comme du texte brut et ignore l'amélioration par vision.",
       enableChildrenDelimiters:
         'Les blocs enfants sont utilisés pour la récupération',
       merge: 'Fusionner',

web/src/locales/it.ts

@@ -1925,9 +1925,6 @@ Ideale per: documenti con contenuto fluente e contestualmente connesso — come
       oneChunkTitle: 'Nota',
       oneChunkDescription:
         'Tutte le sezioni analizzate saranno unite in ordine in un singolo chunk.',
-      flattenMediaToText: 'Disabilita modello visivo',
-      flattenMediaToTextTip:
-        'Tratta le sezioni di immagini e tabelle come testo semplice e salta il miglioramento visivo.',
       enableChildrenDelimiters: 'I chunk figli sono usati per il recupero',
       merge: 'Unisci',
       split: 'Dividi',

web/src/locales/ja.ts

@@ -2483,9 +2483,6 @@ export default {
       oneChunkTitle: '注記',
       oneChunkDescription:
         '解析されたすべてのセクションが順番に1つのチャンクにマージされます。',
-      flattenMediaToText: 'ビジョンモデルを無効化',
-      flattenMediaToTextTip:
-        '画像や表のセクションをプレーンテキストとして扱い、ビジョンによる拡張処理をスキップします。',
       enableChildrenDelimiters: '子チャンクを検索に使用する',
       merge: 'マージ',
       split: '分割',

web/src/locales/ko.ts

@@ -1928,9 +1928,6 @@ export default {
       oneChunkTitle: '참고',
       oneChunkDescription:
         '파싱된 모든 섹션이 순서대로 단일 chunk로 병합됩니다.',
-      flattenMediaToText: '비전 모델 비활성화',
-      flattenMediaToTextTip:
-        '이미지 및 표 섹션을 일반 텍스트로 처리하고 비전 향상을 건너뜁니다.',
       enableChildrenDelimiters: '하위 chunk를 검색에 사용',
       merge: '병합',
       split: '분할',

web/src/locales/tr.ts

@@ -2876,9 +2876,6 @@ En uygun olduğu durumlar: Anlatı bütünlüğünün bitişik paragrafları bir
         "Görüntüler için bir metin kaynağı gerekir: OCR'yi veya görsel geliştirmeyi açık tutun.",
       visionEnhancementPromptTip:
         'Görsel model için özel talimat. Yerleşik istemi kullanmak için boş bırakın.',
-      flattenMediaToText: 'Görsel modeli devre dışı bırak',
-      flattenMediaToTextTip:
-        'Görüntü ve tablo bölümlerini düz metin olarak ele alır ve görsel geliştirmeyi atlar.',
       enableChildrenDelimiters: 'Alt parçalar alım için kullanılır',
       merge: 'Birleştir',
       split: 'Böl',

web/src/locales/uz.ts

@@ -2685,9 +2685,6 @@ export default {
       oneChunkTitle: 'Eslatma',
       oneChunkDescription:
         'Tahlil qilingan barcha bo‘limlar tartib bilan bitta bo‘lakka birlashtiriladi.',
-      flattenMediaToText: 'Vizual modelni o‘chirish',
-      flattenMediaToTextTip:
-        'Rasm va jadval bo‘limlarini oddiy matn sifatida qarab, vizual yaxshilashni o‘tkazib yuboradi.',
       enableChildrenDelimiters: 'Qidiruv uchun bola bo‘laklar ishlatiladi',
       merge: 'Birlashtirish',
       split: 'Bo‘lish',

web/src/pages/agent/form/parser-form/utils.test.ts

@@ -93,7 +93,6 @@ describe('parser-form utils', () => {
         setups: [
           {
             fileFormat: FileType.PDF,
-            flatten_media_to_text: false,
             vlm: { llm_id: 'gpt-4o@OpenAI' },
           },
           { fileFormat: FileType.Audio, vlm: { llm_id: 'whisper@OpenAI' } },
@@ -115,10 +114,7 @@ describe('parser-form utils', () => {
 
     it('turns the enhancement off when no setup used the vision model', () => {
       const values = normalizeParserFormValues({
-        setups: [
-          { fileFormat: FileType.PDF, flatten_media_to_text: true },
-          { fileFormat: FileType.Docx, flatten_media_to_text: true },
-        ],
+        setups: [{ fileFormat: FileType.PDF }, { fileFormat: FileType.Docx }],
       });
 
       expect(values.enable_vision_enhancement).toBe(false);
@@ -147,7 +143,6 @@ describe('parser-form utils', () => {
         setups: [
           {
             fileFormat: FileType.PDF,
-            flatten_media_to_text: true,
             vlm: { llm_id: 'stale@Model' },
           },
           { fileFormat: FileType.Audio, vlm: { llm_id: 'whisper@OpenAI' } },

web/src/pages/agent/form/parser-form/utils.ts

@@ -24,8 +24,7 @@ export const VisionEnhancementFileTypes: FileType[] = [
   FileType.Video,
 ];
 
-// Lifts the legacy per-setup vision options (vlm.llm_id / flatten_media_to_text)
-// onto the top level and migrates the image family onto its own contract:
+// Lifts the per-setup vision model onto the top level and migrates the image family onto its own contract:
 // `ocr_enabled` replaces parse_method, and the prompt the image setup used to
 // own moves into the global vlm block. Idempotent, so every boundary (form
 // defaults, canvas save, dataset load) can apply it: values already carrying a
@@ -65,9 +64,7 @@ export function normalizeParserFormValues<T extends Record<string, any>>(
   const enableVisionEnhancement =
     typeof values?.enable_vision_enhancement === 'boolean'
       ? values.enable_vision_enhancement
-      : visionSetups.some(
-          (x) => x?.flatten_media_to_text === false || !isEmpty(x?.vlm?.llm_id),
-        );
+      : visionSetups.some((x) => !isEmpty(x?.vlm?.llm_id));
 
   const llmId =
     legacyImageModel ||
@@ -80,7 +77,7 @@ export function normalizeParserFormValues<T extends Record<string, any>>(
     // setups to JSON, so a stored value is dead config that must not survive
     // a load/save round-trip.
     if (x?.fileFormat === FileType.Audio) return omit(x, ['output_format']);
-    const stripped = omit(x, ['vlm', 'flatten_media_to_text', 'output_format']);
+    const stripped = omit(x, ['vlm', 'output_format']);
     if (x?.fileFormat !== FileType.Image) return stripped;
     const { parse_method, ...rest } = omit(stripped, ['lang', 'system_prompt']);
     // Materialize the switch either way. Absent or "ocr" both mean "run local