diff --git a/internal/agent/templates/advanced_ingestion_pipeline.json b/internal/agent/templates/advanced_ingestion_pipeline.json index f2c84bce55..1039b56b21 100644 --- a/internal/agent/templates/advanced_ingestion_pipeline.json +++ b/internal/agent/templates/advanced_ingestion_pipeline.json @@ -127,17 +127,14 @@ } }, "doc": { - "preprocess": "main_content", "suffix": [ "doc" ] }, "docx": { - "preprocess": "main_content", "suffix": [ "docx" - ], - "vlm": {} + ] }, "email": { "fields": [ @@ -150,14 +147,12 @@ "body", "attachments" ], - "preprocess": "main_content", "suffix": [ "eml", "msg" ] }, "html": { - "preprocess": "main_content", "suffix": [ "htm", "html" @@ -165,7 +160,6 @@ }, "image": { "parse_method": "ocr", - "preprocess": "main_content", "suffix": [ "jpg", "jpeg", @@ -175,25 +169,20 @@ "system_prompt": "" }, "markdown": { - "preprocess": "main_content", "suffix": [ "md", "markdown", "mdx" - ], - "vlm": {} + ] }, "pdf": { "parse_method": "DeepDOC", - "preprocess": "main_content", "suffix": [ "pdf" - ], - "vlm": {} + ] }, "slides": { "parse_method": "DeepDOC", - "preprocess": "main_content", "suffix": [ "pptx", "ppt" @@ -201,16 +190,13 @@ }, "spreadsheet": { "parse_method": "DeepDOC", - "preprocess": "main_content", "suffix": [ "xls", "xlsx", "csv" - ], - "vlm": {} + ] }, "text&code": { - "preprocess": "main_content", "suffix": [ "txt", "py", @@ -359,18 +345,15 @@ "setups": [ { "fileFormat": "pdf", - "parse_method": "DeepDOC", - "preprocess": "main_content" + "parse_method": "DeepDOC" }, { "fileFormat": "spreadsheet", - "parse_method": "DeepDOC", - "preprocess": "main_content" + "parse_method": "DeepDOC" }, { "fileFormat": "image", "parse_method": "ocr", - "preprocess": "main_content", "system_prompt": "" }, { @@ -384,33 +367,26 @@ "body", "attachments" ], - "fileFormat": "email", - "preprocess": "main_content" + "fileFormat": "email" }, { - "fileFormat": "markdown", - "preprocess": "main_content" + "fileFormat": "markdown" }, { - "fileFormat": "text&code", - "preprocess": "main_content" + "fileFormat": "text&code" }, { - "fileFormat": "html", - "preprocess": "main_content" + "fileFormat": "html" }, { - "fileFormat": "doc", - "preprocess": "main_content" + "fileFormat": "doc" }, { - "fileFormat": "docx", - "preprocess": "main_content" + "fileFormat": "docx" }, { "fileFormat": "slides", - "parse_method": "DeepDOC", - "preprocess": "main_content" + "parse_method": "DeepDOC" } ] }, diff --git a/internal/agent/templates/chunk_summary.json b/internal/agent/templates/chunk_summary.json index f0de49be76..f42e703154 100644 --- a/internal/agent/templates/chunk_summary.json +++ b/internal/agent/templates/chunk_summary.json @@ -84,17 +84,14 @@ } }, "doc": { - "preprocess": "main_content", "suffix": [ "doc" ] }, "docx": { - "preprocess": "main_content", "suffix": [ "docx" - ], - "vlm": {} + ] }, "email": { "fields": [ @@ -107,14 +104,12 @@ "body", "attachments" ], - "preprocess": "main_content", "suffix": [ "eml", "msg" ] }, "html": { - "preprocess": "main_content", "suffix": [ "htm", "html" @@ -122,7 +117,6 @@ }, "image": { "parse_method": "ocr", - "preprocess": "main_content", "suffix": [ "jpg", "jpeg", @@ -132,25 +126,20 @@ "system_prompt": "" }, "markdown": { - "preprocess": "main_content", "suffix": [ "md", "markdown", "mdx" - ], - "vlm": {} + ] }, "pdf": { "parse_method": "DeepDOC", - "preprocess": "main_content", "suffix": [ "pdf" - ], - "vlm": {} + ] }, "slides": { "parse_method": "DeepDOC", - "preprocess": "main_content", "suffix": [ "pptx", "ppt" @@ -158,16 +147,13 @@ }, "spreadsheet": { "parse_method": "DeepDOC", - "preprocess": "main_content", "suffix": [ "xls", "xlsx", "csv" - ], - "vlm": {} + ] }, "text&code": { - "preprocess": "main_content", "suffix": [ "txt", "py", @@ -319,18 +305,15 @@ "setups": [ { "fileFormat": "pdf", - "parse_method": "DeepDOC", - "preprocess": "main_content" + "parse_method": "DeepDOC" }, { "fileFormat": "spreadsheet", - "parse_method": "DeepDOC", - "preprocess": "main_content" + "parse_method": "DeepDOC" }, { "fileFormat": "image", "parse_method": "ocr", - "preprocess": "main_content", "system_prompt": "" }, { @@ -344,33 +327,26 @@ "body", "attachments" ], - "fileFormat": "email", - "preprocess": "main_content" + "fileFormat": "email" }, { - "fileFormat": "markdown", - "preprocess": "main_content" + "fileFormat": "markdown" }, { - "fileFormat": "text&code", - "preprocess": "main_content" + "fileFormat": "text&code" }, { - "fileFormat": "html", - "preprocess": "main_content" + "fileFormat": "html" }, { - "fileFormat": "doc", - "preprocess": "main_content" + "fileFormat": "doc" }, { - "fileFormat": "docx", - "preprocess": "main_content" + "fileFormat": "docx" }, { "fileFormat": "slides", - "parse_method": "DeepDOC", - "preprocess": "main_content" + "parse_method": "DeepDOC" } ] }, diff --git a/internal/agent/templates/compiler.json b/internal/agent/templates/compiler.json index ecb04c3089..96ec16e753 100644 --- a/internal/agent/templates/compiler.json +++ b/internal/agent/templates/compiler.json @@ -61,16 +61,14 @@ "remove_header_footer": false, "suffix": [ "doc" - ], - "vlm": {} + ] }, "docx": { "order_index": 8, "remove_header_footer": false, "suffix": [ "docx" - ], - "vlm": {} + ] }, "email": { "fields": [ @@ -113,8 +111,7 @@ "md", "markdown", "mdx" - ], - "vlm": {} + ] }, "pdf": { "order_index": 0, @@ -122,8 +119,7 @@ "remove_header_footer": false, "suffix": [ "pdf" - ], - "vlm": {} + ] }, "slides": { "order_index": 9, @@ -140,8 +136,7 @@ "xls", "xlsx", "csv" - ], - "vlm": {} + ] }, "text&code": { "order_index": 5, @@ -270,13 +265,11 @@ { "fileFormat": "pdf", "parse_method": "DeepDOC", - "remove_header_footer": false, - "vlm": {} + "remove_header_footer": false }, { "fileFormat": "spreadsheet", - "parse_method": "DeepDOC", - "vlm": {} + "parse_method": "DeepDOC" }, { "fileFormat": "image", @@ -296,8 +289,7 @@ "fileFormat": "email" }, { - "fileFormat": "markdown", - "vlm": {} + "fileFormat": "markdown" }, { "fileFormat": "text&code" @@ -308,13 +300,11 @@ }, { "fileFormat": "doc", - "remove_header_footer": false, - "vlm": {} + "remove_header_footer": false }, { "fileFormat": "docx", - "remove_header_footer": false, - "vlm": {} + "remove_header_footer": false }, { "fileFormat": "slides", diff --git a/internal/agent/templates/title_chunker.json b/internal/agent/templates/title_chunker.json index e7a7d92240..1867463c8c 100644 --- a/internal/agent/templates/title_chunker.json +++ b/internal/agent/templates/title_chunker.json @@ -84,17 +84,14 @@ } }, "doc": { - "preprocess": "main_content", "suffix": [ "doc" ] }, "docx": { - "preprocess": "main_content", "suffix": [ "docx" - ], - "vlm": {} + ] }, "email": { "fields": [ @@ -107,14 +104,12 @@ "body", "attachments" ], - "preprocess": "main_content", "suffix": [ "eml", "msg" ] }, "html": { - "preprocess": "main_content", "suffix": [ "htm", "html" @@ -122,7 +117,6 @@ }, "image": { "parse_method": "ocr", - "preprocess": "main_content", "suffix": [ "jpg", "jpeg", @@ -132,25 +126,20 @@ "system_prompt": "" }, "markdown": { - "preprocess": "main_content", "suffix": [ "md", "markdown", "mdx" - ], - "vlm": {} + ] }, "pdf": { "parse_method": "DeepDOC", - "preprocess": "main_content", "suffix": [ "pdf" - ], - "vlm": {} + ] }, "slides": { "parse_method": "DeepDOC", - "preprocess": "main_content", "suffix": [ "pptx", "ppt" @@ -158,16 +147,13 @@ }, "spreadsheet": { "parse_method": "DeepDOC", - "preprocess": "main_content", "suffix": [ "xls", "xlsx", "csv" - ], - "vlm": {} + ] }, "text&code": { - "preprocess": "main_content", "suffix": [ "txt", "py", @@ -338,18 +324,15 @@ "setups": [ { "fileFormat": "pdf", - "parse_method": "DeepDOC", - "preprocess": "main_content" + "parse_method": "DeepDOC" }, { "fileFormat": "spreadsheet", - "parse_method": "DeepDOC", - "preprocess": "main_content" + "parse_method": "DeepDOC" }, { "fileFormat": "image", "parse_method": "ocr", - "preprocess": "main_content", "system_prompt": "" }, { @@ -363,33 +346,26 @@ "body", "attachments" ], - "fileFormat": "email", - "preprocess": "main_content" + "fileFormat": "email" }, { - "fileFormat": "markdown", - "preprocess": "main_content" + "fileFormat": "markdown" }, { - "fileFormat": "text&code", - "preprocess": "main_content" + "fileFormat": "text&code" }, { - "fileFormat": "html", - "preprocess": "main_content" + "fileFormat": "html" }, { - "fileFormat": "doc", - "preprocess": "main_content" + "fileFormat": "doc" }, { - "fileFormat": "docx", - "preprocess": "main_content" + "fileFormat": "docx" }, { "fileFormat": "slides", - "parse_method": "DeepDOC", - "preprocess": "main_content" + "parse_method": "DeepDOC" } ] }, diff --git a/internal/dao/ingestion_parser_config_migration_test.go b/internal/dao/ingestion_parser_config_migration_test.go index 183edb9bd6..fd05cf9f0c 100644 --- a/internal/dao/ingestion_parser_config_migration_test.go +++ b/internal/dao/ingestion_parser_config_migration_test.go @@ -116,8 +116,8 @@ func TestMigrateParserChunkerTokenizerConfig(t *testing.T) { { name: "an existing family map is merged, not replaced", parser: "book", - input: `{"layout_recognize":true,"Parser:HipSignsRhyme":{"pdf":{"preprocess":["main_content"],"remove_toc":false}}}`, - want: `{"Parser:HipSignsRhyme":{"pdf":{"preprocess":["main_content"],"remove_toc":false,"parse_method":"deepdoc"}}}`, + input: `{"layout_recognize":true,"Parser:HipSignsRhyme":{"pdf":{"remove_header_footer":true,"remove_toc":false}}}`, + want: `{"Parser:HipSignsRhyme":{"pdf":{"remove_header_footer":true,"remove_toc":false,"parse_method":"deepdoc"}}}`, }, {name: "retired settings are deleted", input: `{"html4excel":true,"task_page_size":256,"video_prompt":"x"}`, want: `{}`}, {name: "no owned keys", input: `{"chunk_method":"naive","topn_tags":5}`, want: `{"chunk_method":"naive","topn_tags":5}`}, diff --git a/internal/dao/ingestion_parser_config_runtime_test.go b/internal/dao/ingestion_parser_config_runtime_test.go index cf6ede236d..4bc2cb6d51 100644 --- a/internal/dao/ingestion_parser_config_runtime_test.go +++ b/internal/dao/ingestion_parser_config_runtime_test.go @@ -634,9 +634,7 @@ var overriddenFamilyKeys = map[string]bool{ // questions (this one, "is the template's value safe to lose"; that one, "may // the value the migration writes differ from the template's"). var inertFamilyKeys = map[string]string{ - "preprocess": "the value has no Go reader; the preprocessing stage is selected by the chunker's own options", "suffix": "no Go reader; file routing uses the document's own suffix column, not the family list", - "vlm": "empty in every template; absent falls back to the family llm_id and then the tenant model, and only Markdown and audio read a family vlm at all", "pages": "the only template that bakes it lists every page ([[1,100000]]), which is what an absent pages means", "parse_method": "a written pdf family always restates it; for the spreadsheet family defaultSetups() re-derives the same deepdoc engine", "column_mode": "the table template bakes auto, which is what an absent column_mode means to indexdoc's resolveTableColumnConfig", diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_audio.json b/internal/ingestion/pipeline/template/ingestion_pipeline_audio.json index 1dc7997d99..d81a564222 100644 --- a/internal/ingestion/pipeline/template/ingestion_pipeline_audio.json +++ b/internal/ingestion/pipeline/template/ingestion_pipeline_audio.json @@ -54,9 +54,6 @@ } }, "audio": { - "preprocess": [ - "main_content" - ], "suffix": [ "aac", "aiff", @@ -231,10 +228,7 @@ }, "setups": [ { - "fileFormat": "audio", - "preprocess": [ - "main_content" - ] + "fileFormat": "audio" } ] }, diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_book.json b/internal/ingestion/pipeline/template/ingestion_pipeline_book.json index 35b5b58a7d..a83d322fda 100644 --- a/internal/ingestion/pipeline/template/ingestion_pipeline_book.json +++ b/internal/ingestion/pipeline/template/ingestion_pipeline_book.json @@ -54,26 +54,16 @@ } }, "doc": { - "preprocess": [ - "main_content" - ], "suffix": [ "doc" ] }, "docx": { - "preprocess": [ - "main_content" - ], "suffix": [ "docx" - ], - "vlm": {} + ] }, "html": { - "preprocess": [ - "main_content" - ], "suffix": [ "htm", "html" @@ -81,19 +71,12 @@ }, "pdf": { "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ], "remove_toc": true, "suffix": [ "pdf" - ], - "vlm": {} + ] }, "text&code": { - "preprocess": [ - "main_content" - ], "suffix": [ "txt" ] @@ -472,38 +455,21 @@ { "fileFormat": "pdf", "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ], - "remove_toc": true, - "vlm": {} - }, - { - "fileFormat": "text&code", - "preprocess": [ - "main_content" - ] - }, - { - "fileFormat": "html", - "preprocess": [ - "main_content" - ], "remove_toc": true }, { - "fileFormat": "doc", - "preprocess": [ - "main_content" - ] + "fileFormat": "text&code" + }, + { + "fileFormat": "html", + "remove_toc": true + }, + { + "fileFormat": "doc" }, { "fileFormat": "docx", - "preprocess": [ - "main_content" - ], - "remove_toc": true, - "vlm": {} + "remove_toc": true } ] }, diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_email.json b/internal/ingestion/pipeline/template/ingestion_pipeline_email.json index 1ce058d009..645e871451 100644 --- a/internal/ingestion/pipeline/template/ingestion_pipeline_email.json +++ b/internal/ingestion/pipeline/template/ingestion_pipeline_email.json @@ -64,9 +64,6 @@ "body", "attachments" ], - "preprocess": [ - "main_content" - ], "suffix": [ "eml" ] @@ -246,10 +243,7 @@ "body", "attachments" ], - "fileFormat": "email", - "preprocess": [ - "main_content" - ] + "fileFormat": "email" } ] }, diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_general.json b/internal/ingestion/pipeline/template/ingestion_pipeline_general.json index 7fe0e36b3c..5858d7df0b 100644 --- a/internal/ingestion/pipeline/template/ingestion_pipeline_general.json +++ b/internal/ingestion/pipeline/template/ingestion_pipeline_general.json @@ -36,51 +36,33 @@ "llm_id": "" }, "doc": { - "preprocess": [ - "main_content" - ], "suffix": [ "doc" ] }, "docx": { - "preprocess": [ - "main_content" - ], "suffix": [ "docx" - ], - "vlm": {} + ] }, "html": { - "preprocess": [ - "main_content" - ], "suffix": [ "htm", "html" ] }, "markdown": { - "preprocess": [ - "main_content" - ], "suffix": [ "md", "markdown", "mdx" - ], - "vlm": {} + ] }, "pdf": { "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ], "suffix": [ "pdf" ], - "vlm": {}, "pages": [ [ 1, @@ -90,20 +72,13 @@ }, "spreadsheet": { "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ], "suffix": [ "xls", "xlsx", "csv" - ], - "vlm": {} + ] }, "text&code": { - "preprocess": [ - "main_content" - ], "suffix": [ "txt", "py", @@ -123,9 +98,6 @@ }, "image": { "ocr_enabled": true, - "preprocess": [ - "main_content" - ], "suffix": [ "bmp", "gif", @@ -149,9 +121,6 @@ "body", "attachments" ], - "preprocess": [ - "main_content" - ], "suffix": [ "eml", "msg" @@ -311,58 +280,30 @@ "setups": [ { "fileFormat": "pdf", - "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ], - "vlm": {} + "parse_method": "DeepDOC" }, { "fileFormat": "spreadsheet", - "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ], - "vlm": {} + "parse_method": "DeepDOC" }, { - "fileFormat": "markdown", - "preprocess": [ - "main_content" - ], - "vlm": {} + "fileFormat": "markdown" }, { - "fileFormat": "text&code", - "preprocess": [ - "main_content" - ] + "fileFormat": "text&code" }, { - "fileFormat": "html", - "preprocess": [ - "main_content" - ] + "fileFormat": "html" }, { - "fileFormat": "doc", - "preprocess": [ - "main_content" - ] + "fileFormat": "doc" }, { - "fileFormat": "docx", - "preprocess": [ - "main_content" - ], - "vlm": {} + "fileFormat": "docx" }, { "fileFormat": "image", - "ocr_enabled": true, - "preprocess": [ - "main_content" - ] + "ocr_enabled": true }, { "fileFormat": "email", @@ -375,9 +316,6 @@ "subject", "body", "attachments" - ], - "preprocess": [ - "main_content" ] } ] diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_laws.json b/internal/ingestion/pipeline/template/ingestion_pipeline_laws.json index 5368855475..13c7dd469b 100644 --- a/internal/ingestion/pipeline/template/ingestion_pipeline_laws.json +++ b/internal/ingestion/pipeline/template/ingestion_pipeline_laws.json @@ -54,57 +54,36 @@ } }, "doc": { - "preprocess": [ - "main_content" - ], "suffix": [ "doc" ] }, "docx": { - "preprocess": [ - "main_content" - ], "suffix": [ "docx" - ], - "vlm": {} + ] }, "html": { - "preprocess": [ - "main_content" - ], "suffix": [ "htm", "html" ] }, "markdown": { - "preprocess": [ - "main_content" - ], "suffix": [ "md", "markdown", "mdx" - ], - "vlm": {} + ] }, "pdf": { "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ], "remove_toc": true, "suffix": [ "pdf" - ], - "vlm": {} + ] }, "text&code": { - "preprocess": [ - "main_content" - ], "suffix": [ "txt" ] @@ -485,40 +464,22 @@ "setups": [ { "fileFormat": "pdf", - "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ] + "parse_method": "DeepDOC" }, { - "fileFormat": "markdown", - "preprocess": [ - "main_content" - ] + "fileFormat": "markdown" }, { - "fileFormat": "text&code", - "preprocess": [ - "main_content" - ] + "fileFormat": "text&code" }, { - "fileFormat": "html", - "preprocess": [ - "main_content" - ] + "fileFormat": "html" }, { - "fileFormat": "doc", - "preprocess": [ - "main_content" - ] + "fileFormat": "doc" }, { - "fileFormat": "docx", - "preprocess": [ - "main_content" - ] + "fileFormat": "docx" } ] }, diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_manual.json b/internal/ingestion/pipeline/template/ingestion_pipeline_manual.json index cce3776c52..5765145493 100644 --- a/internal/ingestion/pipeline/template/ingestion_pipeline_manual.json +++ b/internal/ingestion/pipeline/template/ingestion_pipeline_manual.json @@ -54,31 +54,20 @@ } }, "doc": { - "preprocess": [ - "main_content" - ], "suffix": [ "doc" ] }, "docx": { - "preprocess": [ - "main_content" - ], "suffix": [ "docx" - ], - "vlm": {} + ] }, "pdf": { "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ], "suffix": [ "pdf" - ], - "vlm": {} + ] } } }, @@ -453,22 +442,13 @@ "setups": [ { "fileFormat": "pdf", - "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ] + "parse_method": "DeepDOC" }, { - "fileFormat": "doc", - "preprocess": [ - "main_content" - ] + "fileFormat": "doc" }, { - "fileFormat": "docx", - "preprocess": [ - "main_content" - ] + "fileFormat": "docx" } ] }, diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_one.json b/internal/ingestion/pipeline/template/ingestion_pipeline_one.json index 0429def36d..f54bf9215a 100644 --- a/internal/ingestion/pipeline/template/ingestion_pipeline_one.json +++ b/internal/ingestion/pipeline/template/ingestion_pipeline_one.json @@ -54,67 +54,42 @@ } }, "doc": { - "preprocess": [ - "main_content" - ], "suffix": [ "doc" ] }, "docx": { - "preprocess": [ - "main_content" - ], "suffix": [ "docx" - ], - "vlm": {} + ] }, "html": { - "preprocess": [ - "main_content" - ], "suffix": [ "htm", "html" ] }, "markdown": { - "preprocess": [ - "main_content" - ], "suffix": [ "md", "markdown", "mdx" - ], - "vlm": {} + ] }, "pdf": { "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ], "suffix": [ "pdf" - ], - "vlm": {} + ] }, "spreadsheet": { "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ], "suffix": [ "xls", "xlsx" - ], - "vlm": {} + ] }, "text&code": { - "preprocess": [ - "main_content" - ], "suffix": [ "txt" ] @@ -276,47 +251,26 @@ "setups": [ { "fileFormat": "pdf", - "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ] + "parse_method": "DeepDOC" }, { "fileFormat": "spreadsheet", - "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ] + "parse_method": "DeepDOC" }, { - "fileFormat": "markdown", - "preprocess": [ - "main_content" - ] + "fileFormat": "markdown" }, { - "fileFormat": "text&code", - "preprocess": [ - "main_content" - ] + "fileFormat": "text&code" }, { - "fileFormat": "html", - "preprocess": [ - "main_content" - ] + "fileFormat": "html" }, { - "fileFormat": "doc", - "preprocess": [ - "main_content" - ] + "fileFormat": "doc" }, { - "fileFormat": "docx", - "preprocess": [ - "main_content" - ] + "fileFormat": "docx" } ] }, diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_paper.json b/internal/ingestion/pipeline/template/ingestion_pipeline_paper.json index 2d288b8b07..351555d336 100644 --- a/internal/ingestion/pipeline/template/ingestion_pipeline_paper.json +++ b/internal/ingestion/pipeline/template/ingestion_pipeline_paper.json @@ -56,13 +56,9 @@ "pdf": { "enable_multi_column": true, "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ], "suffix": [ "pdf" - ], - "vlm": {} + ] } } }, @@ -441,11 +437,7 @@ { "enable_multi_column": true, "fileFormat": "pdf", - "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ], - "vlm": {} + "parse_method": "DeepDOC" } ] }, diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_picture.json b/internal/ingestion/pipeline/template/ingestion_pipeline_picture.json index dd4011ad2f..47ec39bd9f 100644 --- a/internal/ingestion/pipeline/template/ingestion_pipeline_picture.json +++ b/internal/ingestion/pipeline/template/ingestion_pipeline_picture.json @@ -55,9 +55,6 @@ }, "image": { "ocr_enabled": true, - "preprocess": [ - "main_content" - ], "suffix": [ "bmp", "gif", @@ -71,9 +68,6 @@ ] }, "video": { - "preprocess": [ - "main_content" - ], "suffix": [ "3gp", "3gpp", @@ -245,16 +239,10 @@ "setups": [ { "fileFormat": "image", - "ocr_enabled": true, - "preprocess": [ - "main_content" - ] + "ocr_enabled": true }, { - "fileFormat": "video", - "preprocess": [ - "main_content" - ] + "fileFormat": "video" } ] }, diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_presentation.json b/internal/ingestion/pipeline/template/ingestion_pipeline_presentation.json index d1cea66cc8..5a1a30585c 100644 --- a/internal/ingestion/pipeline/template/ingestion_pipeline_presentation.json +++ b/internal/ingestion/pipeline/template/ingestion_pipeline_presentation.json @@ -55,19 +55,12 @@ }, "pdf": { "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ], "suffix": [ "pdf" - ], - "vlm": {} + ] }, "slides": { "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ], "suffix": [ "pptx", "ppt" @@ -230,18 +223,11 @@ "setups": [ { "fileFormat": "pdf", - "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ], - "vlm": {} + "parse_method": "DeepDOC" }, { "fileFormat": "slides", - "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ] + "parse_method": "DeepDOC" } ] }, diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_qa.json b/internal/ingestion/pipeline/template/ingestion_pipeline_qa.json index 27eee1d37e..4e4e919fc1 100644 --- a/internal/ingestion/pipeline/template/ingestion_pipeline_qa.json +++ b/internal/ingestion/pipeline/template/ingestion_pipeline_qa.json @@ -54,51 +54,32 @@ } }, "docx": { - "preprocess": [ - "main_content" - ], "suffix": [ "docx" - ], - "vlm": {} + ] }, "markdown": { - "preprocess": [ - "main_content" - ], "suffix": [ "md", "markdown", "mdx" - ], - "vlm": {} + ] }, "pdf": { "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ], "suffix": [ "pdf" - ], - "vlm": {} + ] }, "spreadsheet": { "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ], "suffix": [ "xls", "xlsx", "csv" - ], - "vlm": {} + ] }, "text&code": { - "preprocess": [ - "main_content" - ], "suffix": [ "txt" ] @@ -221,39 +202,20 @@ "setups": [ { "fileFormat": "pdf", - "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ], - "vlm": {} + "parse_method": "DeepDOC" }, { "fileFormat": "spreadsheet", - "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ], - "vlm": {} + "parse_method": "DeepDOC" }, { - "fileFormat": "markdown", - "preprocess": [ - "main_content" - ], - "vlm": {} + "fileFormat": "markdown" }, { - "fileFormat": "text&code", - "preprocess": [ - "main_content" - ] + "fileFormat": "text&code" }, { - "fileFormat": "docx", - "preprocess": [ - "main_content" - ], - "vlm": {} + "fileFormat": "docx" } ] }, diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_table.json b/internal/ingestion/pipeline/template/ingestion_pipeline_table.json index 844389e879..17a74a8d58 100644 --- a/internal/ingestion/pipeline/template/ingestion_pipeline_table.json +++ b/internal/ingestion/pipeline/template/ingestion_pipeline_table.json @@ -55,23 +55,16 @@ }, "spreadsheet": { "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ], "suffix": [ "xls", "xlsx", "csv" ], - "vlm": {}, "column_mode": "auto", "column_roles": {}, "column_names": [] }, "text&code": { - "preprocess": [ - "main_content" - ], "suffix": [ "txt" ] @@ -194,17 +187,10 @@ "setups": [ { "fileFormat": "spreadsheet", - "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ], - "vlm": {} + "parse_method": "DeepDOC" }, { - "fileFormat": "text&code", - "preprocess": [ - "main_content" - ] + "fileFormat": "text&code" } ] }, diff --git a/internal/ingestion/pipeline/template_output_format_test.go b/internal/ingestion/pipeline/template_output_format_test.go index 68617b00d9..c874233d7d 100644 --- a/internal/ingestion/pipeline/template_output_format_test.go +++ b/internal/ingestion/pipeline/template_output_format_test.go @@ -84,6 +84,12 @@ func assertNoParserOutputFormatInParams(t *testing.T, ref string, root map[strin continue } params, _ := obj["params"].(map[string]any) + // Canvas templates saved by the Python-era frontend nest the families + // under "setups" (compiler.json uses that shape); the runtime flattens + // it before merging, so check the families wherever they live. + if nested, ok := params["setups"].(map[string]any); ok { + params = nested + } for family, rawSetup := range params { if parserSetupParamsKeys[family] { continue diff --git a/internal/ingestion/pipeline/template_retired_parser_keys_test.go b/internal/ingestion/pipeline/template_retired_parser_keys_test.go new file mode 100644 index 0000000000..c6ac020bdf --- /dev/null +++ b/internal/ingestion/pipeline/template_retired_parser_keys_test.go @@ -0,0 +1,165 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package pipeline + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "ragflow/internal/common" +) + +// TestBuiltinTemplatesCarryNoRetiredParserFamilyKeys pins that no builtin +// ingestion template bakes the retired per-family keys: +// +// - preprocess: no Go reader; chunk preprocessing is a chunker option. +// - vlm on any family except audio: the vision model is the Parser's +// top-level vlm block, and audio's family vlm is the ASR model. +// +// Both copies are checked: dsl.components holds the runtime params and +// dsl.graph.nodes holds the Parser form setup list the frontend renders. +func TestBuiltinTemplatesCarryNoRetiredParserFamilyKeys(t *testing.T) { + registry, err := DefaultRegistry() + if err != nil { + t.Fatalf("DefaultRegistry: %v", err) + } + refs := registry.Refs() + if len(refs) == 0 { + t.Fatal("no builtin templates registered") + } + + for _, ref := range refs { + t.Run(ref, func(t *testing.T) { + dsl, err := LoadBuiltinDSL(ref) + if err != nil { + t.Fatalf("LoadBuiltinDSL(%q): %v", ref, err) + } + var root map[string]any + if err := json.Unmarshal([]byte(dsl), &root); err != nil { + t.Fatalf("decode %s DSL: %v", ref, err) + } + assertNoRetiredParserFamilyKeysInParams(t, ref, root) + assertNoRetiredParserFamilyKeysInGraph(t, ref, root) + }) + } +} + +func assertNoRetiredParserFamilyKeysInParams(t *testing.T, ref string, root map[string]any) { + t.Helper() + components, _ := root["components"].(map[string]any) + for cpnID, rawComponent := range components { + component, _ := rawComponent.(map[string]any) + obj, _ := component["obj"].(map[string]any) + if obj["component_name"] != "Parser" { + continue + } + params, _ := obj["params"].(map[string]any) + // Canvas templates saved by the Python-era frontend nest the families + // under "setups" (compiler.json uses that shape); the runtime flattens + // it before merging, so check the families wherever they live. + if nested, ok := params["setups"].(map[string]any); ok { + params = nested + } + for family, rawSetup := range params { + if parserSetupParamsKeys[family] { + continue + } + setup, ok := rawSetup.(map[string]any) + if !ok { + continue + } + assertNoRetiredFamilyKeys(t, ref, cpnID, family, setup) + } + } +} + +func assertNoRetiredParserFamilyKeysInGraph(t *testing.T, ref string, root map[string]any) { + t.Helper() + graph, _ := root["graph"].(map[string]any) + nodes, _ := graph["nodes"].([]any) + for _, rawNode := range nodes { + node, _ := rawNode.(map[string]any) + id, _ := node["id"].(string) + if !common.IsParserComponentID(id) { + continue + } + data, _ := node["data"].(map[string]any) + form, _ := data["form"].(map[string]any) + setups, _ := form["setups"].([]any) + for _, rawSetup := range setups { + setup, _ := rawSetup.(map[string]any) + fileFormat, _ := setup["fileFormat"].(string) + assertNoRetiredFamilyKeys(t, ref, id, fileFormat, setup) + } + } +} + +func assertNoRetiredFamilyKeys(t *testing.T, ref, cpnID, family string, setup map[string]any) { + t.Helper() + if value, found := setup["preprocess"]; found { + t.Errorf("%s: Parser %s family %q declares preprocess = %v; no parser reads the key", ref, cpnID, family, value) + } + if family == "audio" { + return + } + if value, found := setup["vlm"]; found { + t.Errorf("%s: Parser %s family %q declares vlm = %v; the vision model is the top-level vlm block", ref, cpnID, family, value) + } +} + +// TestAgentTemplatesCarryNoRetiredParserFamilyKeys extends the pin to the +// seeded canvas templates under internal/agent/templates. The dao seeder +// copies their DSL verbatim into the canvas_template table, so a baked key +// there reaches live canvases. compiler.json is a bare DSL fixture without +// the metadata wrapper, so the "dsl" key is optional. +func TestAgentTemplatesCarryNoRetiredParserFamilyKeys(t *testing.T) { + dir := filepath.Join("..", "..", "agent", "templates") + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatalf("read agent templates: %v", err) + } + checked := 0 + for _, entry := range entries { + name := entry.Name() + if entry.IsDir() || !strings.HasSuffix(name, ".json") { + continue + } + raw, err := os.ReadFile(filepath.Join(dir, name)) + if err != nil { + t.Fatalf("read %s: %v", name, err) + } + var wrapper map[string]any + if err := json.Unmarshal(raw, &wrapper); err != nil { + t.Fatalf("decode %s: %v", name, err) + } + root, ok := wrapper["dsl"].(map[string]any) + if !ok { + root = wrapper + } + checked++ + t.Run(name, func(t *testing.T) { + assertNoRetiredParserFamilyKeysInParams(t, name, root) + assertNoRetiredParserFamilyKeysInGraph(t, name, root) + }) + } + if checked == 0 { + t.Fatal("no agent templates found") + } +} diff --git a/internal/ingestion/task/pipeline_executor_defaults_test.go b/internal/ingestion/task/pipeline_executor_defaults_test.go index 94b7985a83..9007f70ce1 100644 --- a/internal/ingestion/task/pipeline_executor_defaults_test.go +++ b/internal/ingestion/task/pipeline_executor_defaults_test.go @@ -38,18 +38,18 @@ import ( // Comparison is done per-component (see assertComponentsMatch), so the JSON // serialization order inside each entry is irrelevant. var builtinComponentParamsGolden = map[string]string{ - "audio": "{\"File\": {}, \"Parser:SongsFillAir\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"audio\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"aac\", \"aiff\", \"ape\", \"au\", \"da\", \"flac\", \"midi\", \"mp3\", \"ogg\", \"oggvorbis\", \"realaudio\", \"vqf\", \"wav\", \"wave\", \"wma\"]}}, \"OneChunker:BlueSkiesLaugh\": {}, \"Tokenizer:KindEyesWatch\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}", - "book": "{\"File\":{},\"Parser:HipSignsRhyme\":{\"enable_vision_enhancement\":false,\"vlm\":{\"llm_id\":\"\"},\"doc\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"doc\"]},\"docx\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"docx\"],\"vlm\":{}},\"html\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"htm\",\"html\"]},\"pdf\":{\"parse_method\":\"DeepDOC\",\"preprocess\":[\"main_content\"],\"remove_toc\":true,\"suffix\":[\"pdf\"],\"vlm\":{}},\"text&code\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"txt\"]}},\"TitleChunker:GrumpyGarlicsBake\":{\"hierarchy\":5,\"include_heading_content\":true,\"levels\":[[\"^#[^#]\",\"^##[^#]\",\"^###[^#]\",\"^####[^#]\"],[\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\",\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"第[零一二三四五六七八九十百0-9]+条\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"],[\"第[0-9]+章\",\"第[0-9]+节\",\"[0-9]{1,2}[\\\\. 、]\",\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\",\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"],[\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"[零一二三四五六七八九十百]+[ 、]\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\",\"[\\\\(（][0-9]{,2}[\\\\)）]\"],[\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\",\"Chapter (I+V?|VI*|XI|IX|X)\",\"Section [0-9]+\",\"Article [0-9]+\"]],\"hierarchyRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"groupRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"hierarchyHierarchy\":\"5\",\"hierarchyGroup\":\"0\",\"method\":\"hierarchy\"},\"Tokenizer:HotDonutsRing\":{\"fields\":\"text\",\"filename_embd_weight\":0.1,\"search_method\":[\"embedding\",\"full_text\"]},\"Extractor:AutoExtractDefault\":{\"llm_id\":\"\",\"keywords\":{\"top_n\":0},\"questions\":{\"top_n\":0},\"tags\":{\"top_n\":0,\"tag_file_id\":\"\"},\"summary\":{\"enabled\":false},\"metadata\":{\"enabled\":false,\"metadata\":[],\"built_in_metadata\":[]}}}", - "email": "{\"File\": {}, \"Parser:BirdsFlutterHigh\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"email\": {\"fields\": [\"from\", \"to\", \"cc\", \"bcc\", \"date\", \"subject\", \"body\", \"attachments\"], \"preprocess\": [\"main_content\"], \"suffix\": [\"eml\"]}}, \"TokenChunker:WarmBreadSmells\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"delimiter\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \"；\", \"！\", \"？\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:NiceWordsSpoken\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}", - "general": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"doc\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"pages\": [[1, 100000]], \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"txt\", \"py\", \"js\", \"java\", \"c\", \"cpp\", \"h\", \"php\", \"go\", \"ts\", \"sh\", \"cs\", \"kt\", \"sql\"]}, \"image\": {\"ocr_enabled\": true, \"preprocess\": [\"main_content\"], \"suffix\": [\"bmp\", \"gif\", \"jpeg\", \"jpg\", \"png\", \"svg\", \"tif\", \"tiff\", \"webp\"]}, \"email\": {\"fields\": [\"from\", \"to\", \"cc\", \"bcc\", \"date\", \"subject\", \"body\", \"attachments\"], \"preprocess\": [\"main_content\"], \"suffix\": [\"eml\", \"msg\"]}}, \"GeneralChunker:SixApplesFall\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiters\": [\"\\n\", \"!\", \"?\", \";\", \"。\", \"；\", \"！\", \"？\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:LegalReadersDecide\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}", - "laws": "{\"File\":{},\"Parser:HipSignsRhyme\":{\"enable_vision_enhancement\":false,\"vlm\":{\"llm_id\":\"\"},\"doc\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"doc\"]},\"docx\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"docx\"],\"vlm\":{}},\"html\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"htm\",\"html\"]},\"markdown\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"md\",\"markdown\",\"mdx\"],\"vlm\":{}},\"pdf\":{\"parse_method\":\"DeepDOC\",\"preprocess\":[\"main_content\"],\"remove_toc\":true,\"suffix\":[\"pdf\"],\"vlm\":{}},\"text&code\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"txt\"]}},\"TitleChunker:SpicyKeysKick\":{\"hierarchy\":2,\"include_heading_content\":false,\"levels\":[[\"^#[^#]\",\"^##[^#]\",\"^###[^#]\",\"^####[^#]\"],[\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\",\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"第[零一二三四五六七八九十百0-9]+条\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"],[\"第[0-9]+章\",\"第[0-9]+节\",\"[0-9]{1,2}[\\\\. 、]\",\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\",\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"],[\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"[零一二三四五六七八九十百]+[ 、]\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\",\"[\\\\(（][0-9]{,2}[\\\\)）]\"],[\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\",\"Chapter (I+V?|VI*|XI|IX|X)\",\"Section [0-9]+\",\"Article [0-9]+\"]],\"hierarchyRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"groupRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"hierarchyHierarchy\":\"2\",\"hierarchyGroup\":\"0\",\"method\":\"hierarchy\"},\"Tokenizer:PublicJobsTake\":{\"fields\":\"text\",\"filename_embd_weight\":0.1,\"search_method\":[\"embedding\",\"full_text\"]},\"Extractor:AutoExtractDefault\":{\"llm_id\":\"\",\"keywords\":{\"top_n\":0},\"questions\":{\"top_n\":0},\"tags\":{\"top_n\":0,\"tag_file_id\":\"\"},\"summary\":{\"enabled\":false},\"metadata\":{\"enabled\":false,\"metadata\":[],\"built_in_metadata\":[]}}}", - "manual": "{\"File\":{},\"Parser:HipSignsRhyme\":{\"enable_vision_enhancement\":false,\"vlm\":{\"llm_id\":\"\"},\"doc\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"doc\"]},\"docx\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"docx\"],\"vlm\":{}},\"pdf\":{\"parse_method\":\"DeepDOC\",\"preprocess\":[\"main_content\"],\"suffix\":[\"pdf\"],\"vlm\":{}}},\"ManualChunker:NineInsectsFind\":{\"hierarchy\":0,\"include_heading_content\":false,\"levels\":[[\"^#[^#]\",\"^##[^#]\",\"^###[^#]\",\"^####[^#]\"],[\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\",\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"第[零一二三四五六七八九十百0-9]+条\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"],[\"第[0-9]+章\",\"第[0-9]+节\",\"[0-9]{1,2}[\\\\. 、]\",\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\",\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"],[\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"[零一二三四五六七八九十百]+[ 、]\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\",\"[\\\\(（][0-9]{,2}[\\\\)）]\"],[\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\",\"Chapter (I+V?|VI*|XI|IX|X)\",\"Section [0-9]+\",\"Article [0-9]+\"]],\"hierarchyRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"groupRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"hierarchyHierarchy\":\"0\",\"hierarchyGroup\":\"0\",\"method\":\"group\"},\"Tokenizer:FunnyBalloonsGrin\":{\"fields\":\"text\",\"filename_embd_weight\":0.1,\"search_method\":[\"embedding\",\"full_text\"]},\"Extractor:AutoExtractDefault\":{\"llm_id\":\"\",\"keywords\":{\"top_n\":0},\"questions\":{\"top_n\":0},\"tags\":{\"top_n\":0,\"tag_file_id\":\"\"},\"summary\":{\"enabled\":false},\"metadata\":{\"enabled\":false,\"metadata\":[],\"built_in_metadata\":[]}}}", - "one": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"doc\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\"], \"vlm\": {}}, \"text&code\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"OneChunker:DryDrinksVisit\": {}, \"Tokenizer:FrankWeeksListen\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}", - "paper": "{\"File\":{},\"Parser:HipSignsRhyme\":{\"enable_vision_enhancement\":false,\"vlm\":{\"llm_id\":\"\"},\"pdf\":{\"enable_multi_column\":true,\"parse_method\":\"DeepDOC\",\"preprocess\":[\"main_content\"],\"suffix\":[\"pdf\"],\"vlm\":{}}},\"TitleChunker:SparklySchoolsTravel\":{\"hierarchy\":0,\"include_heading_content\":false,\"levels\":[[\"^#[^#]\",\"^##[^#]\",\"^###[^#]\",\"^####[^#]\"],[\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\",\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"第[零一二三四五六七八九十百0-9]+条\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"],[\"第[0-9]+章\",\"第[0-9]+节\",\"[0-9]{1,2}[\\\\. 、]\",\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\",\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"],[\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"[零一二三四五六七八九十百]+[ 、]\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\",\"[\\\\(（][0-9]{,2}[\\\\)）]\"],[\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\",\"Chapter (I+V?|VI*|XI|IX|X)\",\"Section [0-9]+\",\"Article [0-9]+\"]],\"hierarchyRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"groupRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"hierarchyHierarchy\":\"0\",\"hierarchyGroup\":\"0\",\"method\":\"group\"},\"Tokenizer:GreatCarsWash\":{\"fields\":\"text\",\"filename_embd_weight\":0.1,\"search_method\":[\"embedding\",\"full_text\"]},\"Extractor:AutoExtractDefault\":{\"llm_id\":\"\",\"keywords\":{\"top_n\":0},\"questions\":{\"top_n\":0},\"tags\":{\"top_n\":0,\"tag_file_id\":\"\"},\"summary\":{\"enabled\":false},\"metadata\":{\"enabled\":false,\"metadata\":[],\"built_in_metadata\":[]}}}", - "picture": "{\"File\": {}, \"Parser:ViewsCaptureLight\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"image\": {\"ocr_enabled\": true, \"preprocess\": [\"main_content\"], \"suffix\": [\"bmp\", \"gif\", \"jpeg\", \"jpg\", \"png\", \"svg\", \"tif\", \"tiff\", \"webp\"]}, \"video\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"3gp\", \"3gpp\", \"avi\", \"flv\", \"mkv\", \"mov\", \"mp4\", \"mpeg\", \"mpg\", \"webm\", \"wmv\"]}}, \"OneChunker:BrightColorsGlow\": {}, \"Tokenizer:SharpLensFocus\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}", - "presentation": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"pdf\": {\"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"slides\": {\"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pptx\", \"ppt\"]}}, \"Tokenizer:TallTreesDance\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"PageChunker:HappyHillsGlow\": {}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}", - "qa": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"docx\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"markdown\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"Tokenizer:ColdCloudsDream\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"QAChunker:TidyCloudsThink\": {}}", - "table": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"spreadsheet\": {\"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}, \"column_mode\": \"auto\", \"column_roles\": {}, \"column_names\": []}, \"text&code\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"TableChunker:FastFoxesJump\": {}, \"Tokenizer:DeepLakesShine\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}}", + "audio": "{\"File\": {}, \"Parser:SongsFillAir\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"audio\": {\"suffix\": [\"aac\", \"aiff\", \"ape\", \"au\", \"da\", \"flac\", \"midi\", \"mp3\", \"ogg\", \"oggvorbis\", \"realaudio\", \"vqf\", \"wav\", \"wave\", \"wma\"]}}, \"OneChunker:BlueSkiesLaugh\": {}, \"Tokenizer:KindEyesWatch\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}", + "book": "{\"File\":{},\"Parser:HipSignsRhyme\":{\"enable_vision_enhancement\":false,\"vlm\":{\"llm_id\":\"\"},\"doc\":{\"suffix\":[\"doc\"]},\"docx\":{\"suffix\":[\"docx\"]},\"html\":{\"suffix\":[\"htm\",\"html\"]},\"pdf\":{\"parse_method\":\"DeepDOC\",\"remove_toc\":true,\"suffix\":[\"pdf\"]},\"text&code\":{\"suffix\":[\"txt\"]}},\"TitleChunker:GrumpyGarlicsBake\":{\"hierarchy\":5,\"include_heading_content\":true,\"levels\":[[\"^#[^#]\",\"^##[^#]\",\"^###[^#]\",\"^####[^#]\"],[\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\",\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"第[零一二三四五六七八九十百0-9]+条\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"],[\"第[0-9]+章\",\"第[0-9]+节\",\"[0-9]{1,2}[\\\\. 、]\",\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\",\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"],[\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"[零一二三四五六七八九十百]+[ 、]\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\",\"[\\\\(（][0-9]{,2}[\\\\)）]\"],[\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\",\"Chapter (I+V?|VI*|XI|IX|X)\",\"Section [0-9]+\",\"Article [0-9]+\"]],\"hierarchyRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"groupRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"hierarchyHierarchy\":\"5\",\"hierarchyGroup\":\"0\",\"method\":\"hierarchy\"},\"Tokenizer:HotDonutsRing\":{\"fields\":\"text\",\"filename_embd_weight\":0.1,\"search_method\":[\"embedding\",\"full_text\"]},\"Extractor:AutoExtractDefault\":{\"llm_id\":\"\",\"keywords\":{\"top_n\":0},\"questions\":{\"top_n\":0},\"tags\":{\"top_n\":0,\"tag_file_id\":\"\"},\"summary\":{\"enabled\":false},\"metadata\":{\"enabled\":false,\"metadata\":[],\"built_in_metadata\":[]}}}", + "email": "{\"File\": {}, \"Parser:BirdsFlutterHigh\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"email\": {\"fields\": [\"from\", \"to\", \"cc\", \"bcc\", \"date\", \"subject\", \"body\", \"attachments\"], \"suffix\": [\"eml\"]}}, \"TokenChunker:WarmBreadSmells\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"delimiter\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \"；\", \"！\", \"？\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:NiceWordsSpoken\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}", + "general": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"doc\": {\"suffix\": [\"doc\"]}, \"docx\": {\"suffix\": [\"docx\"]}, \"html\": {\"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"suffix\": [\"md\", \"markdown\", \"mdx\"]}, \"pdf\": {\"pages\": [[1, 100000]], \"parse_method\": \"DeepDOC\", \"suffix\": [\"pdf\"]}, \"spreadsheet\": {\"parse_method\": \"DeepDOC\", \"suffix\": [\"xls\", \"xlsx\", \"csv\"]}, \"text&code\": {\"suffix\": [\"txt\", \"py\", \"js\", \"java\", \"c\", \"cpp\", \"h\", \"php\", \"go\", \"ts\", \"sh\", \"cs\", \"kt\", \"sql\"]}, \"image\": {\"ocr_enabled\": true, \"suffix\": [\"bmp\", \"gif\", \"jpeg\", \"jpg\", \"png\", \"svg\", \"tif\", \"tiff\", \"webp\"]}, \"email\": {\"fields\": [\"from\", \"to\", \"cc\", \"bcc\", \"date\", \"subject\", \"body\", \"attachments\"], \"suffix\": [\"eml\", \"msg\"]}}, \"GeneralChunker:SixApplesFall\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiters\": [\"\\n\", \"!\", \"?\", \";\", \"。\", \"；\", \"！\", \"？\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:LegalReadersDecide\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}", + "laws": "{\"File\":{},\"Parser:HipSignsRhyme\":{\"enable_vision_enhancement\":false,\"vlm\":{\"llm_id\":\"\"},\"doc\":{\"suffix\":[\"doc\"]},\"docx\":{\"suffix\":[\"docx\"]},\"html\":{\"suffix\":[\"htm\",\"html\"]},\"markdown\":{\"suffix\":[\"md\",\"markdown\",\"mdx\"]},\"pdf\":{\"parse_method\":\"DeepDOC\",\"remove_toc\":true,\"suffix\":[\"pdf\"]},\"text&code\":{\"suffix\":[\"txt\"]}},\"TitleChunker:SpicyKeysKick\":{\"hierarchy\":2,\"include_heading_content\":false,\"levels\":[[\"^#[^#]\",\"^##[^#]\",\"^###[^#]\",\"^####[^#]\"],[\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\",\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"第[零一二三四五六七八九十百0-9]+条\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"],[\"第[0-9]+章\",\"第[0-9]+节\",\"[0-9]{1,2}[\\\\. 、]\",\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\",\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"],[\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"[零一二三四五六七八九十百]+[ 、]\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\",\"[\\\\(（][0-9]{,2}[\\\\)）]\"],[\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\",\"Chapter (I+V?|VI*|XI|IX|X)\",\"Section [0-9]+\",\"Article [0-9]+\"]],\"hierarchyRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"groupRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"hierarchyHierarchy\":\"2\",\"hierarchyGroup\":\"0\",\"method\":\"hierarchy\"},\"Tokenizer:PublicJobsTake\":{\"fields\":\"text\",\"filename_embd_weight\":0.1,\"search_method\":[\"embedding\",\"full_text\"]},\"Extractor:AutoExtractDefault\":{\"llm_id\":\"\",\"keywords\":{\"top_n\":0},\"questions\":{\"top_n\":0},\"tags\":{\"top_n\":0,\"tag_file_id\":\"\"},\"summary\":{\"enabled\":false},\"metadata\":{\"enabled\":false,\"metadata\":[],\"built_in_metadata\":[]}}}", + "manual": "{\"File\":{},\"Parser:HipSignsRhyme\":{\"enable_vision_enhancement\":false,\"vlm\":{\"llm_id\":\"\"},\"doc\":{\"suffix\":[\"doc\"]},\"docx\":{\"suffix\":[\"docx\"]},\"pdf\":{\"parse_method\":\"DeepDOC\",\"suffix\":[\"pdf\"]}},\"ManualChunker:NineInsectsFind\":{\"hierarchy\":0,\"include_heading_content\":false,\"levels\":[[\"^#[^#]\",\"^##[^#]\",\"^###[^#]\",\"^####[^#]\"],[\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\",\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"第[零一二三四五六七八九十百0-9]+条\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"],[\"第[0-9]+章\",\"第[0-9]+节\",\"[0-9]{1,2}[\\\\. 、]\",\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\",\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"],[\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"[零一二三四五六七八九十百]+[ 、]\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\",\"[\\\\(（][0-9]{,2}[\\\\)）]\"],[\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\",\"Chapter (I+V?|VI*|XI|IX|X)\",\"Section [0-9]+\",\"Article [0-9]+\"]],\"hierarchyRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"groupRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"hierarchyHierarchy\":\"0\",\"hierarchyGroup\":\"0\",\"method\":\"group\"},\"Tokenizer:FunnyBalloonsGrin\":{\"fields\":\"text\",\"filename_embd_weight\":0.1,\"search_method\":[\"embedding\",\"full_text\"]},\"Extractor:AutoExtractDefault\":{\"llm_id\":\"\",\"keywords\":{\"top_n\":0},\"questions\":{\"top_n\":0},\"tags\":{\"top_n\":0,\"tag_file_id\":\"\"},\"summary\":{\"enabled\":false},\"metadata\":{\"enabled\":false,\"metadata\":[],\"built_in_metadata\":[]}}}", + "one": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"doc\": {\"suffix\": [\"doc\"]}, \"docx\": {\"suffix\": [\"docx\"]}, \"html\": {\"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"suffix\": [\"md\", \"markdown\", \"mdx\"]}, \"pdf\": {\"parse_method\": \"DeepDOC\", \"suffix\": [\"pdf\"]}, \"spreadsheet\": {\"parse_method\": \"DeepDOC\", \"suffix\": [\"xls\", \"xlsx\"]}, \"text&code\": {\"suffix\": [\"txt\"]}}, \"OneChunker:DryDrinksVisit\": {}, \"Tokenizer:FrankWeeksListen\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}", + "paper": "{\"File\":{},\"Parser:HipSignsRhyme\":{\"enable_vision_enhancement\":false,\"vlm\":{\"llm_id\":\"\"},\"pdf\":{\"enable_multi_column\":true,\"parse_method\":\"DeepDOC\",\"suffix\":[\"pdf\"]}},\"TitleChunker:SparklySchoolsTravel\":{\"hierarchy\":0,\"include_heading_content\":false,\"levels\":[[\"^#[^#]\",\"^##[^#]\",\"^###[^#]\",\"^####[^#]\"],[\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\",\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"第[零一二三四五六七八九十百0-9]+条\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"],[\"第[0-9]+章\",\"第[0-9]+节\",\"[0-9]{1,2}[\\\\. 、]\",\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\",\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"],[\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"[零一二三四五六七八九十百]+[ 、]\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\",\"[\\\\(（][0-9]{,2}[\\\\)）]\"],[\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\",\"Chapter (I+V?|VI*|XI|IX|X)\",\"Section [0-9]+\",\"Article [0-9]+\"]],\"hierarchyRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"groupRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"hierarchyHierarchy\":\"0\",\"hierarchyGroup\":\"0\",\"method\":\"group\"},\"Tokenizer:GreatCarsWash\":{\"fields\":\"text\",\"filename_embd_weight\":0.1,\"search_method\":[\"embedding\",\"full_text\"]},\"Extractor:AutoExtractDefault\":{\"llm_id\":\"\",\"keywords\":{\"top_n\":0},\"questions\":{\"top_n\":0},\"tags\":{\"top_n\":0,\"tag_file_id\":\"\"},\"summary\":{\"enabled\":false},\"metadata\":{\"enabled\":false,\"metadata\":[],\"built_in_metadata\":[]}}}", + "picture": "{\"File\": {}, \"Parser:ViewsCaptureLight\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"image\": {\"ocr_enabled\": true, \"suffix\": [\"bmp\", \"gif\", \"jpeg\", \"jpg\", \"png\", \"svg\", \"tif\", \"tiff\", \"webp\"]}, \"video\": {\"suffix\": [\"3gp\", \"3gpp\", \"avi\", \"flv\", \"mkv\", \"mov\", \"mp4\", \"mpeg\", \"mpg\", \"webm\", \"wmv\"]}}, \"OneChunker:BrightColorsGlow\": {}, \"Tokenizer:SharpLensFocus\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}", + "presentation": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"pdf\": {\"parse_method\": \"DeepDOC\", \"suffix\": [\"pdf\"]}, \"slides\": {\"parse_method\": \"DeepDOC\", \"suffix\": [\"pptx\", \"ppt\"]}}, \"Tokenizer:TallTreesDance\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"PageChunker:HappyHillsGlow\": {}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}", + "qa": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"docx\": {\"suffix\": [\"docx\"]}, \"markdown\": {\"suffix\": [\"md\", \"markdown\", \"mdx\"]}, \"pdf\": {\"parse_method\": \"DeepDOC\", \"suffix\": [\"pdf\"]}, \"spreadsheet\": {\"parse_method\": \"DeepDOC\", \"suffix\": [\"xls\", \"xlsx\", \"csv\"]}, \"text&code\": {\"suffix\": [\"txt\"]}}, \"Tokenizer:ColdCloudsDream\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"QAChunker:TidyCloudsThink\": {}}", + "table": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"spreadsheet\": {\"parse_method\": \"DeepDOC\", \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"column_mode\": \"auto\", \"column_roles\": {}, \"column_names\": []}, \"text&code\": {\"suffix\": [\"txt\"]}}, \"TableChunker:FastFoxesJump\": {}, \"Tokenizer:DeepLakesShine\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}}", } // Per-template test methods. Each resolves default component params from a diff --git a/internal/parser/parser/markdown_parser.go b/internal/parser/parser/markdown_parser.go index ebc1a12978..fa881ba70b 100644 --- a/internal/parser/parser/markdown_parser.go +++ b/internal/parser/parser/markdown_parser.go @@ -44,7 +44,6 @@ const GoMarkdown = "go_markdown" type MarkdownParser struct { libType string ParseMethod string - VLM map[string]any // FetchRemoteImages controls whether HTTP(S) Markdown images are // downloaded. NewMarkdownParser enables it for direct Markdown parsing; // callers normalizing untrusted backend responses may disable it. @@ -77,9 +76,6 @@ func (p *MarkdownParser) ConfigureFromSetup(setup map[string]any) { if v, ok := setup["parse_method"].(string); ok && v != "" { p.ParseMethod = v } - if v, ok := setup["vlm"].(map[string]any); ok { - p.VLM = v - } } // ParseWithResult implements ParseResultProducer (plan §6.5) and diff --git a/internal/parser/parser/markdown_parser_test.go b/internal/parser/parser/markdown_parser_test.go index 7cb7eff46e..edd72d4190 100644 --- a/internal/parser/parser/markdown_parser_test.go +++ b/internal/parser/parser/markdown_parser_test.go @@ -512,19 +512,10 @@ func TestMarkdownParser_ParseWithResult_InlineRawHTMLSurvives(t *testing.T) { func TestMarkdownParser_ConfigureFromSetup(t *testing.T) { p, _ := NewMarkdownParser(GoMarkdown) - p.ConfigureFromSetup(map[string]any{ - "parse_method": "deepdoc", - "vlm": map[string]any{"llm_id": "gpt-4-vision"}, - }) + p.ConfigureFromSetup(map[string]any{"parse_method": "deepdoc"}) if p.ParseMethod != "deepdoc" { t.Fatalf("ParseMethod = %q, want %q", p.ParseMethod, "deepdoc") } - if p.VLM == nil { - t.Fatal("VLM is nil; want map") - } - if id, _ := p.VLM["llm_id"].(string); id != "gpt-4-vision" { - t.Fatalf("VLM[llm_id] = %q, want %q", id, "gpt-4-vision") - } } func TestMarkdownParser_ConfigureFromSetup_NilSafe(t *testing.T) { diff --git a/web/src/locales/ar.ts b/web/src/locales/ar.ts index 4dd9079fe8..114895c37c 100644 --- a/web/src/locales/ar.ts +++ b/web/src/locales/ar.ts @@ -1348,13 +1348,6 @@ export default { pleaseUploadAtLeastOneFile: 'يرجى تحميل ملف واحد على الأقل', }, flow: { - preprocess: { - preprocess: 'المعالجة المسبقة', - mainContent: 'المحتوى الرئيسي', - abstract: 'خلاصة', - author: 'مؤلف', - sectionTitle: 'عنوان القسم', - }, autoPlay: 'تشغيل الصوت تلقائيًا', downloadFileTypeTip: 'نوع الملف المراد تنزيله', downloadFileType: 'تنزيل نوع الملف', diff --git a/web/src/locales/az.ts b/web/src/locales/az.ts index 7af7ff1687..f838f592e9 100644 --- a/web/src/locales/az.ts +++ b/web/src/locales/az.ts @@ -2659,13 +2659,6 @@ Nümunə: Virtual host üslubu`, }, flow: { exportCurrentPage: 'Cari səhifəni ixrac et', - preprocess: { - preprocess: 'İlkin emal', - mainContent: 'Əsas məzmun', - abstract: 'Xülasə', - author: 'Müəllif', - sectionTitle: 'Bölmə başlığı', - }, tags: 'Etiketlər', agentCategory: 'Kətan kateqoriyası', editTags: 'Etiketləri redaktə et', diff --git a/web/src/locales/en.ts b/web/src/locales/en.ts index 6644f74e24..37edd185d6 100644 --- a/web/src/locales/en.ts +++ b/web/src/locales/en.ts @@ -2767,13 +2767,6 @@ Example: Virtual Hosted Style`, }, flow: { exportCurrentPage: 'Export current page', - preprocess: { - preprocess: 'Preprocess', - mainContent: 'Main content', - abstract: 'Abstract', - author: 'Author', - sectionTitle: 'Section title', - }, tags: 'Tags', agentCategory: 'Agent category', editTags: 'Edit tags', diff --git a/web/src/locales/fr.ts b/web/src/locales/fr.ts index 60b52c5dbb..74d8d0d4b5 100644 --- a/web/src/locales/fr.ts +++ b/web/src/locales/fr.ts @@ -2462,13 +2462,6 @@ Exemple : Virtual Hosted Style`, seconds: 'Secondes', subject: 'Sujet', // --- clés manquantes ajoutées --- - preprocess: { - preprocess: 'Prétraitement', - mainContent: 'Contenu principal', - abstract: 'Résumé', - author: 'Auteur', - sectionTitle: 'Titre de section', - }, editTags: 'Modifier les étiquettes', editTagsDescription: 'Ajoutez des étiquettes pour organiser et filtrer vos agents. Appuyez sur Entrée ou virgule pour ajouter.', diff --git a/web/src/locales/it.ts b/web/src/locales/it.ts index 2ecca75abc..ac30ee0543 100644 --- a/web/src/locales/it.ts +++ b/web/src/locales/it.ts @@ -1838,13 +1838,6 @@ Esempio: Virtual Hosted Style`, pleaseUploadAtLeastOneFile: 'Carica almeno un file', }, flow: { - preprocess: { - preprocess: 'Pre-elaborazione', - mainContent: 'Contenuto principale', - abstract: 'Abstract', - author: 'Autore', - sectionTitle: 'Titolo sezione', - }, editTags: 'Modifica tag', editTagsDescription: 'Aggiungi tag per organizzare e filtrare i tuoi agenti. Premi Invio o virgola per aggiungere.', diff --git a/web/src/locales/ja.ts b/web/src/locales/ja.ts index e5551a471d..f17d7148c8 100644 --- a/web/src/locales/ja.ts +++ b/web/src/locales/ja.ts @@ -2394,14 +2394,6 @@ export default { id: 'ID', logTitle: 'タイトル', - preprocess: { - preprocess: '前処理', - mainContent: '本文', - abstract: '概要', - author: '著者', - sectionTitle: 'セクションタイトル', - }, - editTags: 'タグを編集', editTagsDescription: 'タグを追加してエージェントを整理・絞り込みできます。Enterキーまたはカンマで追加します。', diff --git a/web/src/locales/ko.ts b/web/src/locales/ko.ts index 5a235898b4..282ec0bc21 100644 --- a/web/src/locales/ko.ts +++ b/web/src/locales/ko.ts @@ -1842,13 +1842,6 @@ export default { pleaseUploadAtLeastOneFile: '최소 하나의 파일을 업로드해 주세요', }, flow: { - preprocess: { - preprocess: '전처리', - mainContent: '본문 내용', - abstract: '요약', - author: '저자', - sectionTitle: '섹션 제목', - }, editTags: '태그 편집', editTagsDescription: '에이전트를 정리하고 필터링하기 위한 태그를 추가하세요. Enter 또는 쉼표를 눌러 추가합니다.', diff --git a/web/src/locales/ru.ts b/web/src/locales/ru.ts index d14a5a5771..870dce5db0 100644 --- a/web/src/locales/ru.ts +++ b/web/src/locales/ru.ts @@ -1537,13 +1537,6 @@ export default { pleaseUploadAtLeastOneFile: 'Пожалуйста, загрузите хотя бы один файл', }, flow: { - preprocess: { - preprocess: 'Предобработка', - mainContent: 'Основное содержимое', - abstract: 'Аннотация', - author: 'Автор', - sectionTitle: 'Заголовок раздела', - }, autoPlay: 'Автовоспроизведение аудио', downloadFileTypeTip: 'Тип файла для скачивания', downloadFileType: 'Тип скачиваемого файла', diff --git a/web/src/locales/tr.ts b/web/src/locales/tr.ts index 9202a40296..ba74cc68b4 100644 --- a/web/src/locales/tr.ts +++ b/web/src/locales/tr.ts @@ -2740,13 +2740,6 @@ Bu otomatik etiketleme özelliği, mevcut datasete alanına özgü bilgi katman }, flow: { exportCurrentPage: 'Geçerli sayfayı dışa aktar', - preprocess: { - preprocess: 'Ön işleme', - mainContent: 'Ana içerik', - abstract: 'Özet', - author: 'Yazar', - sectionTitle: 'Bölüm başlığı', - }, includeHeadingContent: 'Üst başlık içeriğini ayır', includeHeadingContentTip: 'Etkinleştirildiğinde parçalar yalnızca başlık yolunu ve içeriğini içerir; üst başlıktan hemen sonra gelen içerik ayrı bir parça olarak tutulur.', diff --git a/web/src/locales/uz.ts b/web/src/locales/uz.ts index fd726feefb..daaa22f6f5 100644 --- a/web/src/locales/uz.ts +++ b/web/src/locales/uz.ts @@ -2558,13 +2558,6 @@ export default { }, flow: { exportCurrentPage: 'Joriy sahifani eksport qilish', - preprocess: { - preprocess: 'Oldindan qayta ishlash', - mainContent: 'Asosiy kontent', - abstract: 'Annotatsiya', - author: 'Muallif', - sectionTitle: 'Bo‘lim sarlavhasi', - }, tags: 'Teglar', agentCategory: 'Canvas toifasi', editTags: 'Teglarni tahrirlash', diff --git a/web/src/locales/zh.ts b/web/src/locales/zh.ts index 3ec5a0d308..9b18d62316 100644 --- a/web/src/locales/zh.ts +++ b/web/src/locales/zh.ts @@ -2333,13 +2333,6 @@ NER：使用 spaCy NER 和基于规则的关键词提取来抽取 Entities 和 R }, flow: { exportCurrentPage: '导出当页', - preprocess: { - preprocess: '预处理', - mainContent: '主内容', - abstract: '摘要', - author: '作者', - sectionTitle: '章节标题', - }, agentCategory: '智能体分类', created: '创建于', editTags: '编辑标签', diff --git a/web/src/pages/agent/constant/pipeline.tsx b/web/src/pages/agent/constant/pipeline.tsx index 9d15fe6941..73a3ba1117 100644 --- a/web/src/pages/agent/constant/pipeline.tsx +++ b/web/src/pages/agent/constant/pipeline.tsx @@ -80,22 +80,6 @@ export enum StringTransformDelimiter { Space = ' ', } -export enum PreprocessValue { - main_content = 'main_content', - section_title = 'title', - abstract = 'abstract', - author = 'author', -} - -export const MAIN_CONTENT_PREPROCESS_VALUE: PreprocessValue = - PreprocessValue.main_content; - -export const PreprocessLabelKeyMap: Record = { - main_content: 'mainContent', - title: 'sectionTitle', - abstract: 'abstract', - author: 'author', -}; export const initialParserValues = { // The Go Parser component emits structured JSON items only; markdown/text/ // html are internal backend representations, never public Parser outputs. @@ -111,52 +95,42 @@ export const initialParserValues = { { fileFormat: FileType.PDF, parse_method: ParseDocumentType.DeepDOC, - preprocess: PreprocessValue.main_content, remove_header_footer: false, pages: [{ from: 1, to: 100000 }], }, { fileFormat: FileType.Spreadsheet, parse_method: ParseDocumentType.DeepDOC, - preprocess: PreprocessValue.main_content, }, { fileFormat: FileType.Image, ocr_enabled: true, - preprocess: PreprocessValue.main_content, }, { fileFormat: FileType.Email, fields: Object.values(ParserFields), - preprocess: PreprocessValue.main_content, }, { fileFormat: FileType.TextMarkdown, - preprocess: PreprocessValue.main_content, }, { fileFormat: FileType.Code, - preprocess: PreprocessValue.main_content, }, { fileFormat: FileType.Html, - preprocess: PreprocessValue.main_content, remove_header_footer: false, }, { fileFormat: FileType.Doc, - preprocess: PreprocessValue.main_content, remove_header_footer: false, }, { fileFormat: FileType.Docx, - preprocess: PreprocessValue.main_content, remove_header_footer: false, }, { fileFormat: FileType.PowerPoint, parse_method: ParseDocumentType.DeepDOC, - preprocess: PreprocessValue.main_content, }, // { // fileFormat: FileType.Video, diff --git a/web/src/pages/agent/form/parser-form/schema.ts b/web/src/pages/agent/form/parser-form/schema.ts index 594b7840a3..3c2ac1673c 100644 --- a/web/src/pages/agent/form/parser-form/schema.ts +++ b/web/src/pages/agent/form/parser-form/schema.ts @@ -5,7 +5,6 @@ import { z } from 'zod'; export const SetupSchema = z .object({ fileFormat: z.string().nullish(), - // preprocess: z.array(z.string()).optional(), parse_method: z.string().optional(), mineru_parse_method: z.string().optional(), mineru_lang: z.string().optional(), diff --git a/web/src/pages/agent/form/parser-form/utils.test.ts b/web/src/pages/agent/form/parser-form/utils.test.ts index c229842b36..b404a6a1f1 100644 --- a/web/src/pages/agent/form/parser-form/utils.test.ts +++ b/web/src/pages/agent/form/parser-form/utils.test.ts @@ -85,6 +85,19 @@ describe('parser-form utils', () => { buildInitialParserValues({}).setups, ); }); + + it('does not seed retired family keys', () => { + const values = buildInitialParserValues({}); + for (const setup of values.setups) { + expect(setup).not.toHaveProperty('preprocess'); + expect(setup).not.toHaveProperty('output_format'); + // Audio's family vlm is the ASR model and stays; every other family + // uses the Parser's top-level vision block. + if ((setup as any).fileFormat !== FileType.Audio) { + expect(setup).not.toHaveProperty('vlm'); + } + } + }); }); describe('normalizeParserFormValues', () => { @@ -307,21 +320,27 @@ describe('parser-form utils', () => { expect(normalizeParserFormValues(once)).toEqual(once); }); - it('drops the retired per-setup output_format from every family', () => { - // The Go Parser normalizes all setups to JSON, so a stored legacy value - // is dead config that must not survive a load/save round-trip. + it('drops the retired per-setup preprocess and output_format from every family', () => { + // The Parser derives preprocessing from the chunker and emits JSON only, + // so stored legacy values are dead config that must not survive a + // load/save round-trip. const normalized = normalizeParserFormValues({ setups: [ - { fileFormat: FileType.PDF, output_format: 'markdown' }, + { + fileFormat: FileType.PDF, + preprocess: ['main_content'], + output_format: 'markdown', + }, { fileFormat: FileType.Audio, + preprocess: ['main_content'], output_format: 'text', vlm: { llm_id: 'whisper@OpenAI' }, }, ], }); expect(normalized.setups[0]).toEqual({ fileFormat: FileType.PDF }); - // Audio keeps its per-setup ASR model; only output_format goes. + // Audio keeps its per-setup ASR model; only the retired keys go. expect(normalized.setups[1]).toEqual({ fileFormat: FileType.Audio, vlm: { llm_id: 'whisper@OpenAI' }, diff --git a/web/src/pages/agent/form/parser-form/utils.ts b/web/src/pages/agent/form/parser-form/utils.ts index 06a86d189f..efeb4e4ec0 100644 --- a/web/src/pages/agent/form/parser-form/utils.ts +++ b/web/src/pages/agent/form/parser-form/utils.ts @@ -73,11 +73,15 @@ export function normalizeParserFormValues>( ''); const nextSetups = setups.map((x) => { - // output_format is retired on every family: the Go Parser normalizes all - // setups to JSON, so a stored value is dead config that must not survive - // a load/save round-trip. - if (x?.fileFormat === FileType.Audio) return omit(x, ['output_format']); - const stripped = omit(x, ['vlm', 'output_format']); + // preprocess and output_format are retired on every family: the Parser + // derives preprocessing from the chunker and emits JSON only, so a stored + // value is dead config that must not survive a load/save round-trip. + // Per-setup vlm is dead for every family except Audio, whose vlm holds the + // ASR model. + if (x?.fileFormat === FileType.Audio) { + return omit(x, ['output_format', 'preprocess']); + } + const stripped = omit(x, ['vlm', 'output_format', 'preprocess']); if (x?.fileFormat !== FileType.Image) return stripped; const { parse_method, ...rest } = omit(stripped, ['lang', 'system_prompt']); // Materialize the switch either way. Absent or "ocr" both mean "run local