mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-11-10 13:24:05 +08:00
refactor(ingestion): drop inert parser family setup keys (#22163)
This commit is contained in:
jay77721 authored and 
GitHub committed
1 parent
d4ff41a57d
commit
2b4d628d51
38 files changed
+335
-642
No files matched your search
@@ -127,17 +127,14 @@
|
||||
}
|
||||
},
|
||||
"doc": {
|
||||
"preprocess": "main_content",
|
||||
"suffix": [
|
||||
"doc"
|
||||
]
|
||||
},
|
||||
"docx": {
|
||||
"preprocess": "main_content",
|
||||
"suffix": [
|
||||
"docx"
|
||||
],
|
||||
"vlm": {}
|
||||
]
|
||||
},
|
||||
"email": {
|
||||
"fields": [
|
||||
@@ -150,14 +147,12 @@
|
||||
"body",
|
||||
"attachments"
|
||||
],
|
||||
"preprocess": "main_content",
|
||||
"suffix": [
|
||||
"eml",
|
||||
"msg"
|
||||
]
|
||||
},
|
||||
"html": {
|
||||
"preprocess": "main_content",
|
||||
"suffix": [
|
||||
"htm",
|
||||
"html"
|
||||
@@ -165,7 +160,6 @@
|
||||
},
|
||||
"image": {
|
||||
"parse_method": "ocr",
|
||||
"preprocess": "main_content",
|
||||
"suffix": [
|
||||
"jpg",
|
||||
"jpeg",
|
||||
@@ -175,25 +169,20 @@
|
||||
"system_prompt": ""
|
||||
},
|
||||
"markdown": {
|
||||
"preprocess": "main_content",
|
||||
"suffix": [
|
||||
"md",
|
||||
"markdown",
|
||||
"mdx"
|
||||
],
|
||||
"vlm": {}
|
||||
]
|
||||
},
|
||||
"pdf": {
|
||||
"parse_method": "DeepDOC",
|
||||
"preprocess": "main_content",
|
||||
"suffix": [
|
||||
"pdf"
|
||||
],
|
||||
"vlm": {}
|
||||
]
|
||||
},
|
||||
"slides": {
|
||||
"parse_method": "DeepDOC",
|
||||
"preprocess": "main_content",
|
||||
"suffix": [
|
||||
"pptx",
|
||||
"ppt"
|
||||
@@ -201,16 +190,13 @@
|
||||
},
|
||||
"spreadsheet": {
|
||||
"parse_method": "DeepDOC",
|
||||
"preprocess": "main_content",
|
||||
"suffix": [
|
||||
"xls",
|
||||
"xlsx",
|
||||
"csv"
|
||||
],
|
||||
"vlm": {}
|
||||
]
|
||||
},
|
||||
"text&code": {
|
||||
"preprocess": "main_content",
|
||||
"suffix": [
|
||||
"txt",
|
||||
"py",
|
||||
@@ -359,18 +345,15 @@
|
||||
"setups": [
|
||||
{
|
||||
"fileFormat": "pdf",
|
||||
"parse_method": "DeepDOC",
|
||||
"preprocess": "main_content"
|
||||
"parse_method": "DeepDOC"
|
||||
},
|
||||
{
|
||||
"fileFormat": "spreadsheet",
|
||||
"parse_method": "DeepDOC",
|
||||
"preprocess": "main_content"
|
||||
"parse_method": "DeepDOC"
|
||||
},
|
||||
{
|
||||
"fileFormat": "image",
|
||||
"parse_method": "ocr",
|
||||
"preprocess": "main_content",
|
||||
"system_prompt": ""
|
||||
},
|
||||
{
|
||||
@@ -384,33 +367,26 @@
|
||||
"body",
|
||||
"attachments"
|
||||
],
|
||||
"fileFormat": "email",
|
||||
"preprocess": "main_content"
|
||||
"fileFormat": "email"
|
||||
},
|
||||
{
|
||||
"fileFormat": "markdown",
|
||||
"preprocess": "main_content"
|
||||
"fileFormat": "markdown"
|
||||
},
|
||||
{
|
||||
"fileFormat": "text&code",
|
||||
"preprocess": "main_content"
|
||||
"fileFormat": "text&code"
|
||||
},
|
||||
{
|
||||
"fileFormat": "html",
|
||||
"preprocess": "main_content"
|
||||
"fileFormat": "html"
|
||||
},
|
||||
{
|
||||
"fileFormat": "doc",
|
||||
"preprocess": "main_content"
|
||||
"fileFormat": "doc"
|
||||
},
|
||||
{
|
||||
"fileFormat": "docx",
|
||||
"preprocess": "main_content"
|
||||
"fileFormat": "docx"
|
||||
},
|
||||
{
|
||||
"fileFormat": "slides",
|
||||
"parse_method": "DeepDOC",
|
||||
"preprocess": "main_content"
|
||||
"parse_method": "DeepDOC"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -84,17 +84,14 @@
|
||||
}
|
||||
},
|
||||
"doc": {
|
||||
"preprocess": "main_content",
|
||||
"suffix": [
|
||||
"doc"
|
||||
]
|
||||
},
|
||||
"docx": {
|
||||
"preprocess": "main_content",
|
||||
"suffix": [
|
||||
"docx"
|
||||
],
|
||||
"vlm": {}
|
||||
]
|
||||
},
|
||||
"email": {
|
||||
"fields": [
|
||||
@@ -107,14 +104,12 @@
|
||||
"body",
|
||||
"attachments"
|
||||
],
|
||||
"preprocess": "main_content",
|
||||
"suffix": [
|
||||
"eml",
|
||||
"msg"
|
||||
]
|
||||
},
|
||||
"html": {
|
||||
"preprocess": "main_content",
|
||||
"suffix": [
|
||||
"htm",
|
||||
"html"
|
||||
@@ -122,7 +117,6 @@
|
||||
},
|
||||
"image": {
|
||||
"parse_method": "ocr",
|
||||
"preprocess": "main_content",
|
||||
"suffix": [
|
||||
"jpg",
|
||||
"jpeg",
|
||||
@@ -132,25 +126,20 @@
|
||||
"system_prompt": ""
|
||||
},
|
||||
"markdown": {
|
||||
"preprocess": "main_content",
|
||||
"suffix": [
|
||||
"md",
|
||||
"markdown",
|
||||
"mdx"
|
||||
],
|
||||
"vlm": {}
|
||||
]
|
||||
},
|
||||
"pdf": {
|
||||
"parse_method": "DeepDOC",
|
||||
"preprocess": "main_content",
|
||||
"suffix": [
|
||||
"pdf"
|
||||
],
|
||||
"vlm": {}
|
||||
]
|
||||
},
|
||||
"slides": {
|
||||
"parse_method": "DeepDOC",
|
||||
"preprocess": "main_content",
|
||||
"suffix": [
|
||||
"pptx",
|
||||
"ppt"
|
||||
@@ -158,16 +147,13 @@
|
||||
},
|
||||
"spreadsheet": {
|
||||
"parse_method": "DeepDOC",
|
||||
"preprocess": "main_content",
|
||||
"suffix": [
|
||||
"xls",
|
||||
"xlsx",
|
||||
"csv"
|
||||
],
|
||||
"vlm": {}
|
||||
]
|
||||
},
|
||||
"text&code": {
|
||||
"preprocess": "main_content",
|
||||
"suffix": [
|
||||
"txt",
|
||||
"py",
|
||||
@@ -319,18 +305,15 @@
|
||||
"setups": [
|
||||
{
|
||||
"fileFormat": "pdf",
|
||||
"parse_method": "DeepDOC",
|
||||
"preprocess": "main_content"
|
||||
"parse_method": "DeepDOC"
|
||||
},
|
||||
{
|
||||
"fileFormat": "spreadsheet",
|
||||
"parse_method": "DeepDOC",
|
||||
"preprocess": "main_content"
|
||||
"parse_method": "DeepDOC"
|
||||
},
|
||||
{
|
||||
"fileFormat": "image",
|
||||
"parse_method": "ocr",
|
||||
"preprocess": "main_content",
|
||||
"system_prompt": ""
|
||||
},
|
||||
{
|
||||
@@ -344,33 +327,26 @@
|
||||
"body",
|
||||
"attachments"
|
||||
],
|
||||
"fileFormat": "email",
|
||||
"preprocess": "main_content"
|
||||
"fileFormat": "email"
|
||||
},
|
||||
{
|
||||
"fileFormat": "markdown",
|
||||
"preprocess": "main_content"
|
||||
"fileFormat": "markdown"
|
||||
},
|
||||
{
|
||||
"fileFormat": "text&code",
|
||||
"preprocess": "main_content"
|
||||
"fileFormat": "text&code"
|
||||
},
|
||||
{
|
||||
"fileFormat": "html",
|
||||
"preprocess": "main_content"
|
||||
"fileFormat": "html"
|
||||
},
|
||||
{
|
||||
"fileFormat": "doc",
|
||||
"preprocess": "main_content"
|
||||
"fileFormat": "doc"
|
||||
},
|
||||
{
|
||||
"fileFormat": "docx",
|
||||
"preprocess": "main_content"
|
||||
"fileFormat": "docx"
|
||||
},
|
||||
{
|
||||
"fileFormat": "slides",
|
||||
"parse_method": "DeepDOC",
|
||||
"preprocess": "main_content"
|
||||
"parse_method": "DeepDOC"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -61,16 +61,14 @@
|
||||
"remove_header_footer": false,
|
||||
"suffix": [
|
||||
"doc"
|
||||
],
|
||||
"vlm": {}
|
||||
]
|
||||
},
|
||||
"docx": {
|
||||
"order_index": 8,
|
||||
"remove_header_footer": false,
|
||||
"suffix": [
|
||||
"docx"
|
||||
],
|
||||
"vlm": {}
|
||||
]
|
||||
},
|
||||
"email": {
|
||||
"fields": [
|
||||
@@ -113,8 +111,7 @@
|
||||
"md",
|
||||
"markdown",
|
||||
"mdx"
|
||||
],
|
||||
"vlm": {}
|
||||
]
|
||||
},
|
||||
"pdf": {
|
||||
"order_index": 0,
|
||||
@@ -122,8 +119,7 @@
|
||||
"remove_header_footer": false,
|
||||
"suffix": [
|
||||
"pdf"
|
||||
],
|
||||
"vlm": {}
|
||||
]
|
||||
},
|
||||
"slides": {
|
||||
"order_index": 9,
|
||||
@@ -140,8 +136,7 @@
|
||||
"xls",
|
||||
"xlsx",
|
||||
"csv"
|
||||
],
|
||||
"vlm": {}
|
||||
]
|
||||
},
|
||||
"text&code": {
|
||||
"order_index": 5,
|
||||
@@ -270,13 +265,11 @@
|
||||
{
|
||||
"fileFormat": "pdf",
|
||||
"parse_method": "DeepDOC",
|
||||
"remove_header_footer": false,
|
||||
"vlm": {}
|
||||
"remove_header_footer": false
|
||||
},
|
||||
{
|
||||
"fileFormat": "spreadsheet",
|
||||
"parse_method": "DeepDOC",
|
||||
"vlm": {}
|
||||
"parse_method": "DeepDOC"
|
||||
},
|
||||
{
|
||||
"fileFormat": "image",
|
||||
@@ -296,8 +289,7 @@
|
||||
"fileFormat": "email"
|
||||
},
|
||||
{
|
||||
"fileFormat": "markdown",
|
||||
"vlm": {}
|
||||
"fileFormat": "markdown"
|
||||
},
|
||||
{
|
||||
"fileFormat": "text&code"
|
||||
@@ -308,13 +300,11 @@
|
||||
},
|
||||
{
|
||||
"fileFormat": "doc",
|
||||
"remove_header_footer": false,
|
||||
"vlm": {}
|
||||
"remove_header_footer": false
|
||||
},
|
||||
{
|
||||
"fileFormat": "docx",
|
||||
"remove_header_footer": false,
|
||||
"vlm": {}
|
||||
"remove_header_footer": false
|
||||
},
|
||||
{
|
||||
"fileFormat": "slides",
|
||||
|
||||
@@ -84,17 +84,14 @@
|
||||
}
|
||||
},
|
||||
"doc": {
|
||||
"preprocess": "main_content",
|
||||
"suffix": [
|
||||
"doc"
|
||||
]
|
||||
},
|
||||
"docx": {
|
||||
"preprocess": "main_content",
|
||||
"suffix": [
|
||||
"docx"
|
||||
],
|
||||
"vlm": {}
|
||||
]
|
||||
},
|
||||
"email": {
|
||||
"fields": [
|
||||
@@ -107,14 +104,12 @@
|
||||
"body",
|
||||
"attachments"
|
||||
],
|
||||
"preprocess": "main_content",
|
||||
"suffix": [
|
||||
"eml",
|
||||
"msg"
|
||||
]
|
||||
},
|
||||
"html": {
|
||||
"preprocess": "main_content",
|
||||
"suffix": [
|
||||
"htm",
|
||||
"html"
|
||||
@@ -122,7 +117,6 @@
|
||||
},
|
||||
"image": {
|
||||
"parse_method": "ocr",
|
||||
"preprocess": "main_content",
|
||||
"suffix": [
|
||||
"jpg",
|
||||
"jpeg",
|
||||
@@ -132,25 +126,20 @@
|
||||
"system_prompt": ""
|
||||
},
|
||||
"markdown": {
|
||||
"preprocess": "main_content",
|
||||
"suffix": [
|
||||
"md",
|
||||
"markdown",
|
||||
"mdx"
|
||||
],
|
||||
"vlm": {}
|
||||
]
|
||||
},
|
||||
"pdf": {
|
||||
"parse_method": "DeepDOC",
|
||||
"preprocess": "main_content",
|
||||
"suffix": [
|
||||
"pdf"
|
||||
],
|
||||
"vlm": {}
|
||||
]
|
||||
},
|
||||
"slides": {
|
||||
"parse_method": "DeepDOC",
|
||||
"preprocess": "main_content",
|
||||
"suffix": [
|
||||
"pptx",
|
||||
"ppt"
|
||||
@@ -158,16 +147,13 @@
|
||||
},
|
||||
"spreadsheet": {
|
||||
"parse_method": "DeepDOC",
|
||||
"preprocess": "main_content",
|
||||
"suffix": [
|
||||
"xls",
|
||||
"xlsx",
|
||||
"csv"
|
||||
],
|
||||
"vlm": {}
|
||||
]
|
||||
},
|
||||
"text&code": {
|
||||
"preprocess": "main_content",
|
||||
"suffix": [
|
||||
"txt",
|
||||
"py",
|
||||
@@ -338,18 +324,15 @@
|
||||
"setups": [
|
||||
{
|
||||
"fileFormat": "pdf",
|
||||
"parse_method": "DeepDOC",
|
||||
"preprocess": "main_content"
|
||||
"parse_method": "DeepDOC"
|
||||
},
|
||||
{
|
||||
"fileFormat": "spreadsheet",
|
||||
"parse_method": "DeepDOC",
|
||||
"preprocess": "main_content"
|
||||
"parse_method": "DeepDOC"
|
||||
},
|
||||
{
|
||||
"fileFormat": "image",
|
||||
"parse_method": "ocr",
|
||||
"preprocess": "main_content",
|
||||
"system_prompt": ""
|
||||
},
|
||||
{
|
||||
@@ -363,33 +346,26 @@
|
||||
"body",
|
||||
"attachments"
|
||||
],
|
||||
"fileFormat": "email",
|
||||
"preprocess": "main_content"
|
||||
"fileFormat": "email"
|
||||
},
|
||||
{
|
||||
"fileFormat": "markdown",
|
||||
"preprocess": "main_content"
|
||||
"fileFormat": "markdown"
|
||||
},
|
||||
{
|
||||
"fileFormat": "text&code",
|
||||
"preprocess": "main_content"
|
||||
"fileFormat": "text&code"
|
||||
},
|
||||
{
|
||||
"fileFormat": "html",
|
||||
"preprocess": "main_content"
|
||||
"fileFormat": "html"
|
||||
},
|
||||
{
|
||||
"fileFormat": "doc",
|
||||
"preprocess": "main_content"
|
||||
"fileFormat": "doc"
|
||||
},
|
||||
{
|
||||
"fileFormat": "docx",
|
||||
"preprocess": "main_content"
|
||||
"fileFormat": "docx"
|
||||
},
|
||||
{
|
||||
"fileFormat": "slides",
|
||||
"parse_method": "DeepDOC",
|
||||
"preprocess": "main_content"
|
||||
"parse_method": "DeepDOC"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -116,8 +116,8 @@ func TestMigrateParserChunkerTokenizerConfig(t *testing.T) {
|
||||
{
|
||||
name: "an existing family map is merged, not replaced",
|
||||
parser: "book",
|
||||
input: `{"layout_recognize":true,"Parser:HipSignsRhyme":{"pdf":{"preprocess":["main_content"],"remove_toc":false}}}`,
|
||||
want: `{"Parser:HipSignsRhyme":{"pdf":{"preprocess":["main_content"],"remove_toc":false,"parse_method":"deepdoc"}}}`,
|
||||
input: `{"layout_recognize":true,"Parser:HipSignsRhyme":{"pdf":{"remove_header_footer":true,"remove_toc":false}}}`,
|
||||
want: `{"Parser:HipSignsRhyme":{"pdf":{"remove_header_footer":true,"remove_toc":false,"parse_method":"deepdoc"}}}`,
|
||||
},
|
||||
{name: "retired settings are deleted", input: `{"html4excel":true,"task_page_size":256,"video_prompt":"x"}`, want: `{}`},
|
||||
{name: "no owned keys", input: `{"chunk_method":"naive","topn_tags":5}`, want: `{"chunk_method":"naive","topn_tags":5}`},
|
||||
|
||||
@@ -634,9 +634,7 @@ var overriddenFamilyKeys = map[string]bool{
|
||||
// questions (this one, "is the template's value safe to lose"; that one, "may
|
||||
// the value the migration writes differ from the template's").
|
||||
var inertFamilyKeys = map[string]string{
|
||||
"preprocess": "the value has no Go reader; the preprocessing stage is selected by the chunker's own options",
|
||||
"suffix": "no Go reader; file routing uses the document's own suffix column, not the family list",
|
||||
"vlm": "empty in every template; absent falls back to the family llm_id and then the tenant model, and only Markdown and audio read a family vlm at all",
|
||||
"pages": "the only template that bakes it lists every page ([[1,100000]]), which is what an absent pages means",
|
||||
"parse_method": "a written pdf family always restates it; for the spreadsheet family defaultSetups() re-derives the same deepdoc engine",
|
||||
"column_mode": "the table template bakes auto, which is what an absent column_mode means to indexdoc's resolveTableColumnConfig",
|
||||
|
||||
@@ -54,9 +54,6 @@
|
||||
}
|
||||
},
|
||||
"audio": {
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"suffix": [
|
||||
"aac",
|
||||
"aiff",
|
||||
@@ -231,10 +228,7 @@
|
||||
},
|
||||
"setups": [
|
||||
{
|
||||
"fileFormat": "audio",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
]
|
||||
"fileFormat": "audio"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -54,26 +54,16 @@
|
||||
}
|
||||
},
|
||||
"doc": {
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"suffix": [
|
||||
"doc"
|
||||
]
|
||||
},
|
||||
"docx": {
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"suffix": [
|
||||
"docx"
|
||||
],
|
||||
"vlm": {}
|
||||
]
|
||||
},
|
||||
"html": {
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"suffix": [
|
||||
"htm",
|
||||
"html"
|
||||
@@ -81,19 +71,12 @@
|
||||
},
|
||||
"pdf": {
|
||||
"parse_method": "DeepDOC",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"remove_toc": true,
|
||||
"suffix": [
|
||||
"pdf"
|
||||
],
|
||||
"vlm": {}
|
||||
]
|
||||
},
|
||||
"text&code": {
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"suffix": [
|
||||
"txt"
|
||||
]
|
||||
@@ -472,38 +455,21 @@
|
||||
{
|
||||
"fileFormat": "pdf",
|
||||
"parse_method": "DeepDOC",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"remove_toc": true,
|
||||
"vlm": {}
|
||||
},
|
||||
{
|
||||
"fileFormat": "text&code",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
]
|
||||
},
|
||||
{
|
||||
"fileFormat": "html",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"remove_toc": true
|
||||
},
|
||||
{
|
||||
"fileFormat": "doc",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
]
|
||||
"fileFormat": "text&code"
|
||||
},
|
||||
{
|
||||
"fileFormat": "html",
|
||||
"remove_toc": true
|
||||
},
|
||||
{
|
||||
"fileFormat": "doc"
|
||||
},
|
||||
{
|
||||
"fileFormat": "docx",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"remove_toc": true,
|
||||
"vlm": {}
|
||||
"remove_toc": true
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -64,9 +64,6 @@
|
||||
"body",
|
||||
"attachments"
|
||||
],
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"suffix": [
|
||||
"eml"
|
||||
]
|
||||
@@ -246,10 +243,7 @@
|
||||
"body",
|
||||
"attachments"
|
||||
],
|
||||
"fileFormat": "email",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
]
|
||||
"fileFormat": "email"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -36,51 +36,33 @@
|
||||
"llm_id": ""
|
||||
},
|
||||
"doc": {
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"suffix": [
|
||||
"doc"
|
||||
]
|
||||
},
|
||||
"docx": {
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"suffix": [
|
||||
"docx"
|
||||
],
|
||||
"vlm": {}
|
||||
]
|
||||
},
|
||||
"html": {
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"suffix": [
|
||||
"htm",
|
||||
"html"
|
||||
]
|
||||
},
|
||||
"markdown": {
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"suffix": [
|
||||
"md",
|
||||
"markdown",
|
||||
"mdx"
|
||||
],
|
||||
"vlm": {}
|
||||
]
|
||||
},
|
||||
"pdf": {
|
||||
"parse_method": "DeepDOC",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"suffix": [
|
||||
"pdf"
|
||||
],
|
||||
"vlm": {},
|
||||
"pages": [
|
||||
[
|
||||
1,
|
||||
@@ -90,20 +72,13 @@
|
||||
},
|
||||
"spreadsheet": {
|
||||
"parse_method": "DeepDOC",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"suffix": [
|
||||
"xls",
|
||||
"xlsx",
|
||||
"csv"
|
||||
],
|
||||
"vlm": {}
|
||||
]
|
||||
},
|
||||
"text&code": {
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"suffix": [
|
||||
"txt",
|
||||
"py",
|
||||
@@ -123,9 +98,6 @@
|
||||
},
|
||||
"image": {
|
||||
"ocr_enabled": true,
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"suffix": [
|
||||
"bmp",
|
||||
"gif",
|
||||
@@ -149,9 +121,6 @@
|
||||
"body",
|
||||
"attachments"
|
||||
],
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"suffix": [
|
||||
"eml",
|
||||
"msg"
|
||||
@@ -311,58 +280,30 @@
|
||||
"setups": [
|
||||
{
|
||||
"fileFormat": "pdf",
|
||||
"parse_method": "DeepDOC",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"vlm": {}
|
||||
"parse_method": "DeepDOC"
|
||||
},
|
||||
{
|
||||
"fileFormat": "spreadsheet",
|
||||
"parse_method": "DeepDOC",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"vlm": {}
|
||||
"parse_method": "DeepDOC"
|
||||
},
|
||||
{
|
||||
"fileFormat": "markdown",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"vlm": {}
|
||||
"fileFormat": "markdown"
|
||||
},
|
||||
{
|
||||
"fileFormat": "text&code",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
]
|
||||
"fileFormat": "text&code"
|
||||
},
|
||||
{
|
||||
"fileFormat": "html",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
]
|
||||
"fileFormat": "html"
|
||||
},
|
||||
{
|
||||
"fileFormat": "doc",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
]
|
||||
"fileFormat": "doc"
|
||||
},
|
||||
{
|
||||
"fileFormat": "docx",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"vlm": {}
|
||||
"fileFormat": "docx"
|
||||
},
|
||||
{
|
||||
"fileFormat": "image",
|
||||
"ocr_enabled": true,
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
]
|
||||
"ocr_enabled": true
|
||||
},
|
||||
{
|
||||
"fileFormat": "email",
|
||||
@@ -375,9 +316,6 @@
|
||||
"subject",
|
||||
"body",
|
||||
"attachments"
|
||||
],
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
]
|
||||
}
|
||||
]
|
||||
|
||||
@@ -54,57 +54,36 @@
|
||||
}
|
||||
},
|
||||
"doc": {
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"suffix": [
|
||||
"doc"
|
||||
]
|
||||
},
|
||||
"docx": {
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"suffix": [
|
||||
"docx"
|
||||
],
|
||||
"vlm": {}
|
||||
]
|
||||
},
|
||||
"html": {
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"suffix": [
|
||||
"htm",
|
||||
"html"
|
||||
]
|
||||
},
|
||||
"markdown": {
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"suffix": [
|
||||
"md",
|
||||
"markdown",
|
||||
"mdx"
|
||||
],
|
||||
"vlm": {}
|
||||
]
|
||||
},
|
||||
"pdf": {
|
||||
"parse_method": "DeepDOC",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"remove_toc": true,
|
||||
"suffix": [
|
||||
"pdf"
|
||||
],
|
||||
"vlm": {}
|
||||
]
|
||||
},
|
||||
"text&code": {
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"suffix": [
|
||||
"txt"
|
||||
]
|
||||
@@ -485,40 +464,22 @@
|
||||
"setups": [
|
||||
{
|
||||
"fileFormat": "pdf",
|
||||
"parse_method": "DeepDOC",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
]
|
||||
"parse_method": "DeepDOC"
|
||||
},
|
||||
{
|
||||
"fileFormat": "markdown",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
]
|
||||
"fileFormat": "markdown"
|
||||
},
|
||||
{
|
||||
"fileFormat": "text&code",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
]
|
||||
"fileFormat": "text&code"
|
||||
},
|
||||
{
|
||||
"fileFormat": "html",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
]
|
||||
"fileFormat": "html"
|
||||
},
|
||||
{
|
||||
"fileFormat": "doc",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
]
|
||||
"fileFormat": "doc"
|
||||
},
|
||||
{
|
||||
"fileFormat": "docx",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
]
|
||||
"fileFormat": "docx"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -54,31 +54,20 @@
|
||||
}
|
||||
},
|
||||
"doc": {
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"suffix": [
|
||||
"doc"
|
||||
]
|
||||
},
|
||||
"docx": {
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"suffix": [
|
||||
"docx"
|
||||
],
|
||||
"vlm": {}
|
||||
]
|
||||
},
|
||||
"pdf": {
|
||||
"parse_method": "DeepDOC",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"suffix": [
|
||||
"pdf"
|
||||
],
|
||||
"vlm": {}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -453,22 +442,13 @@
|
||||
"setups": [
|
||||
{
|
||||
"fileFormat": "pdf",
|
||||
"parse_method": "DeepDOC",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
]
|
||||
"parse_method": "DeepDOC"
|
||||
},
|
||||
{
|
||||
"fileFormat": "doc",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
]
|
||||
"fileFormat": "doc"
|
||||
},
|
||||
{
|
||||
"fileFormat": "docx",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
]
|
||||
"fileFormat": "docx"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -54,67 +54,42 @@
|
||||
}
|
||||
},
|
||||
"doc": {
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"suffix": [
|
||||
"doc"
|
||||
]
|
||||
},
|
||||
"docx": {
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"suffix": [
|
||||
"docx"
|
||||
],
|
||||
"vlm": {}
|
||||
]
|
||||
},
|
||||
"html": {
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"suffix": [
|
||||
"htm",
|
||||
"html"
|
||||
]
|
||||
},
|
||||
"markdown": {
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"suffix": [
|
||||
"md",
|
||||
"markdown",
|
||||
"mdx"
|
||||
],
|
||||
"vlm": {}
|
||||
]
|
||||
},
|
||||
"pdf": {
|
||||
"parse_method": "DeepDOC",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"suffix": [
|
||||
"pdf"
|
||||
],
|
||||
"vlm": {}
|
||||
]
|
||||
},
|
||||
"spreadsheet": {
|
||||
"parse_method": "DeepDOC",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"suffix": [
|
||||
"xls",
|
||||
"xlsx"
|
||||
],
|
||||
"vlm": {}
|
||||
]
|
||||
},
|
||||
"text&code": {
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"suffix": [
|
||||
"txt"
|
||||
]
|
||||
@@ -276,47 +251,26 @@
|
||||
"setups": [
|
||||
{
|
||||
"fileFormat": "pdf",
|
||||
"parse_method": "DeepDOC",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
]
|
||||
"parse_method": "DeepDOC"
|
||||
},
|
||||
{
|
||||
"fileFormat": "spreadsheet",
|
||||
"parse_method": "DeepDOC",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
]
|
||||
"parse_method": "DeepDOC"
|
||||
},
|
||||
{
|
||||
"fileFormat": "markdown",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
]
|
||||
"fileFormat": "markdown"
|
||||
},
|
||||
{
|
||||
"fileFormat": "text&code",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
]
|
||||
"fileFormat": "text&code"
|
||||
},
|
||||
{
|
||||
"fileFormat": "html",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
]
|
||||
"fileFormat": "html"
|
||||
},
|
||||
{
|
||||
"fileFormat": "doc",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
]
|
||||
"fileFormat": "doc"
|
||||
},
|
||||
{
|
||||
"fileFormat": "docx",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
]
|
||||
"fileFormat": "docx"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -56,13 +56,9 @@
|
||||
"pdf": {
|
||||
"enable_multi_column": true,
|
||||
"parse_method": "DeepDOC",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"suffix": [
|
||||
"pdf"
|
||||
],
|
||||
"vlm": {}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -441,11 +437,7 @@
|
||||
{
|
||||
"enable_multi_column": true,
|
||||
"fileFormat": "pdf",
|
||||
"parse_method": "DeepDOC",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"vlm": {}
|
||||
"parse_method": "DeepDOC"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -55,9 +55,6 @@
|
||||
},
|
||||
"image": {
|
||||
"ocr_enabled": true,
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"suffix": [
|
||||
"bmp",
|
||||
"gif",
|
||||
@@ -71,9 +68,6 @@
|
||||
]
|
||||
},
|
||||
"video": {
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"suffix": [
|
||||
"3gp",
|
||||
"3gpp",
|
||||
@@ -245,16 +239,10 @@
|
||||
"setups": [
|
||||
{
|
||||
"fileFormat": "image",
|
||||
"ocr_enabled": true,
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
]
|
||||
"ocr_enabled": true
|
||||
},
|
||||
{
|
||||
"fileFormat": "video",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
]
|
||||
"fileFormat": "video"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -55,19 +55,12 @@
|
||||
},
|
||||
"pdf": {
|
||||
"parse_method": "DeepDOC",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"suffix": [
|
||||
"pdf"
|
||||
],
|
||||
"vlm": {}
|
||||
]
|
||||
},
|
||||
"slides": {
|
||||
"parse_method": "DeepDOC",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"suffix": [
|
||||
"pptx",
|
||||
"ppt"
|
||||
@@ -230,18 +223,11 @@
|
||||
"setups": [
|
||||
{
|
||||
"fileFormat": "pdf",
|
||||
"parse_method": "DeepDOC",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"vlm": {}
|
||||
"parse_method": "DeepDOC"
|
||||
},
|
||||
{
|
||||
"fileFormat": "slides",
|
||||
"parse_method": "DeepDOC",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
]
|
||||
"parse_method": "DeepDOC"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -54,51 +54,32 @@
|
||||
}
|
||||
},
|
||||
"docx": {
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"suffix": [
|
||||
"docx"
|
||||
],
|
||||
"vlm": {}
|
||||
]
|
||||
},
|
||||
"markdown": {
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"suffix": [
|
||||
"md",
|
||||
"markdown",
|
||||
"mdx"
|
||||
],
|
||||
"vlm": {}
|
||||
]
|
||||
},
|
||||
"pdf": {
|
||||
"parse_method": "DeepDOC",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"suffix": [
|
||||
"pdf"
|
||||
],
|
||||
"vlm": {}
|
||||
]
|
||||
},
|
||||
"spreadsheet": {
|
||||
"parse_method": "DeepDOC",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"suffix": [
|
||||
"xls",
|
||||
"xlsx",
|
||||
"csv"
|
||||
],
|
||||
"vlm": {}
|
||||
]
|
||||
},
|
||||
"text&code": {
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"suffix": [
|
||||
"txt"
|
||||
]
|
||||
@@ -221,39 +202,20 @@
|
||||
"setups": [
|
||||
{
|
||||
"fileFormat": "pdf",
|
||||
"parse_method": "DeepDOC",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"vlm": {}
|
||||
"parse_method": "DeepDOC"
|
||||
},
|
||||
{
|
||||
"fileFormat": "spreadsheet",
|
||||
"parse_method": "DeepDOC",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"vlm": {}
|
||||
"parse_method": "DeepDOC"
|
||||
},
|
||||
{
|
||||
"fileFormat": "markdown",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"vlm": {}
|
||||
"fileFormat": "markdown"
|
||||
},
|
||||
{
|
||||
"fileFormat": "text&code",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
]
|
||||
"fileFormat": "text&code"
|
||||
},
|
||||
{
|
||||
"fileFormat": "docx",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"vlm": {}
|
||||
"fileFormat": "docx"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -55,23 +55,16 @@
|
||||
},
|
||||
"spreadsheet": {
|
||||
"parse_method": "DeepDOC",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"suffix": [
|
||||
"xls",
|
||||
"xlsx",
|
||||
"csv"
|
||||
],
|
||||
"vlm": {},
|
||||
"column_mode": "auto",
|
||||
"column_roles": {},
|
||||
"column_names": []
|
||||
},
|
||||
"text&code": {
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"suffix": [
|
||||
"txt"
|
||||
]
|
||||
@@ -194,17 +187,10 @@
|
||||
"setups": [
|
||||
{
|
||||
"fileFormat": "spreadsheet",
|
||||
"parse_method": "DeepDOC",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
],
|
||||
"vlm": {}
|
||||
"parse_method": "DeepDOC"
|
||||
},
|
||||
{
|
||||
"fileFormat": "text&code",
|
||||
"preprocess": [
|
||||
"main_content"
|
||||
]
|
||||
"fileFormat": "text&code"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -84,6 +84,12 @@ func assertNoParserOutputFormatInParams(t *testing.T, ref string, root map[strin
|
||||
continue
|
||||
}
|
||||
params, _ := obj["params"].(map[string]any)
|
||||
// Canvas templates saved by the Python-era frontend nest the families
|
||||
// under "setups" (compiler.json uses that shape); the runtime flattens
|
||||
// it before merging, so check the families wherever they live.
|
||||
if nested, ok := params["setups"].(map[string]any); ok {
|
||||
params = nested
|
||||
}
|
||||
for family, rawSetup := range params {
|
||||
if parserSetupParamsKeys[family] {
|
||||
continue
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"ragflow/internal/common"
|
||||
)
|
||||
|
||||
// TestBuiltinTemplatesCarryNoRetiredParserFamilyKeys pins that no builtin
|
||||
// ingestion template bakes the retired per-family keys:
|
||||
//
|
||||
// - preprocess: no Go reader; chunk preprocessing is a chunker option.
|
||||
// - vlm on any family except audio: the vision model is the Parser's
|
||||
// top-level vlm block, and audio's family vlm is the ASR model.
|
||||
//
|
||||
// Both copies are checked: dsl.components holds the runtime params and
|
||||
// dsl.graph.nodes holds the Parser form setup list the frontend renders.
|
||||
func TestBuiltinTemplatesCarryNoRetiredParserFamilyKeys(t *testing.T) {
|
||||
registry, err := DefaultRegistry()
|
||||
if err != nil {
|
||||
t.Fatalf("DefaultRegistry: %v", err)
|
||||
}
|
||||
refs := registry.Refs()
|
||||
if len(refs) == 0 {
|
||||
t.Fatal("no builtin templates registered")
|
||||
}
|
||||
|
||||
for _, ref := range refs {
|
||||
t.Run(ref, func(t *testing.T) {
|
||||
dsl, err := LoadBuiltinDSL(ref)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadBuiltinDSL(%q): %v", ref, err)
|
||||
}
|
||||
var root map[string]any
|
||||
if err := json.Unmarshal([]byte(dsl), &root); err != nil {
|
||||
t.Fatalf("decode %s DSL: %v", ref, err)
|
||||
}
|
||||
assertNoRetiredParserFamilyKeysInParams(t, ref, root)
|
||||
assertNoRetiredParserFamilyKeysInGraph(t, ref, root)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func assertNoRetiredParserFamilyKeysInParams(t *testing.T, ref string, root map[string]any) {
|
||||
t.Helper()
|
||||
components, _ := root["components"].(map[string]any)
|
||||
for cpnID, rawComponent := range components {
|
||||
component, _ := rawComponent.(map[string]any)
|
||||
obj, _ := component["obj"].(map[string]any)
|
||||
if obj["component_name"] != "Parser" {
|
||||
continue
|
||||
}
|
||||
params, _ := obj["params"].(map[string]any)
|
||||
// Canvas templates saved by the Python-era frontend nest the families
|
||||
// under "setups" (compiler.json uses that shape); the runtime flattens
|
||||
// it before merging, so check the families wherever they live.
|
||||
if nested, ok := params["setups"].(map[string]any); ok {
|
||||
params = nested
|
||||
}
|
||||
for family, rawSetup := range params {
|
||||
if parserSetupParamsKeys[family] {
|
||||
continue
|
||||
}
|
||||
setup, ok := rawSetup.(map[string]any)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
assertNoRetiredFamilyKeys(t, ref, cpnID, family, setup)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func assertNoRetiredParserFamilyKeysInGraph(t *testing.T, ref string, root map[string]any) {
|
||||
t.Helper()
|
||||
graph, _ := root["graph"].(map[string]any)
|
||||
nodes, _ := graph["nodes"].([]any)
|
||||
for _, rawNode := range nodes {
|
||||
node, _ := rawNode.(map[string]any)
|
||||
id, _ := node["id"].(string)
|
||||
if !common.IsParserComponentID(id) {
|
||||
continue
|
||||
}
|
||||
data, _ := node["data"].(map[string]any)
|
||||
form, _ := data["form"].(map[string]any)
|
||||
setups, _ := form["setups"].([]any)
|
||||
for _, rawSetup := range setups {
|
||||
setup, _ := rawSetup.(map[string]any)
|
||||
fileFormat, _ := setup["fileFormat"].(string)
|
||||
assertNoRetiredFamilyKeys(t, ref, id, fileFormat, setup)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func assertNoRetiredFamilyKeys(t *testing.T, ref, cpnID, family string, setup map[string]any) {
|
||||
t.Helper()
|
||||
if value, found := setup["preprocess"]; found {
|
||||
t.Errorf("%s: Parser %s family %q declares preprocess = %v; no parser reads the key", ref, cpnID, family, value)
|
||||
}
|
||||
if family == "audio" {
|
||||
return
|
||||
}
|
||||
if value, found := setup["vlm"]; found {
|
||||
t.Errorf("%s: Parser %s family %q declares vlm = %v; the vision model is the top-level vlm block", ref, cpnID, family, value)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAgentTemplatesCarryNoRetiredParserFamilyKeys extends the pin to the
|
||||
// seeded canvas templates under internal/agent/templates. The dao seeder
|
||||
// copies their DSL verbatim into the canvas_template table, so a baked key
|
||||
// there reaches live canvases. compiler.json is a bare DSL fixture without
|
||||
// the metadata wrapper, so the "dsl" key is optional.
|
||||
func TestAgentTemplatesCarryNoRetiredParserFamilyKeys(t *testing.T) {
|
||||
dir := filepath.Join("..", "..", "agent", "templates")
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("read agent templates: %v", err)
|
||||
}
|
||||
checked := 0
|
||||
for _, entry := range entries {
|
||||
name := entry.Name()
|
||||
if entry.IsDir() || !strings.HasSuffix(name, ".json") {
|
||||
continue
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(dir, name))
|
||||
if err != nil {
|
||||
t.Fatalf("read %s: %v", name, err)
|
||||
}
|
||||
var wrapper map[string]any
|
||||
if err := json.Unmarshal(raw, &wrapper); err != nil {
|
||||
t.Fatalf("decode %s: %v", name, err)
|
||||
}
|
||||
root, ok := wrapper["dsl"].(map[string]any)
|
||||
if !ok {
|
||||
root = wrapper
|
||||
}
|
||||
checked++
|
||||
t.Run(name, func(t *testing.T) {
|
||||
assertNoRetiredParserFamilyKeysInParams(t, name, root)
|
||||
assertNoRetiredParserFamilyKeysInGraph(t, name, root)
|
||||
})
|
||||
}
|
||||
if checked == 0 {
|
||||
t.Fatal("no agent templates found")
|
||||
}
|
||||
}
|
||||
@@ -38,18 +38,18 @@ import (
|
||||
// Comparison is done per-component (see assertComponentsMatch), so the JSON
|
||||
// serialization order inside each entry is irrelevant.
|
||||
var builtinComponentParamsGolden = map[string]string{
|
||||
"audio": "{\"File\": {}, \"Parser:SongsFillAir\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"audio\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"aac\", \"aiff\", \"ape\", \"au\", \"da\", \"flac\", \"midi\", \"mp3\", \"ogg\", \"oggvorbis\", \"realaudio\", \"vqf\", \"wav\", \"wave\", \"wma\"]}}, \"OneChunker:BlueSkiesLaugh\": {}, \"Tokenizer:KindEyesWatch\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
|
||||
"book": "{\"File\":{},\"Parser:HipSignsRhyme\":{\"enable_vision_enhancement\":false,\"vlm\":{\"llm_id\":\"\"},\"doc\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"doc\"]},\"docx\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"docx\"],\"vlm\":{}},\"html\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"htm\",\"html\"]},\"pdf\":{\"parse_method\":\"DeepDOC\",\"preprocess\":[\"main_content\"],\"remove_toc\":true,\"suffix\":[\"pdf\"],\"vlm\":{}},\"text&code\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"txt\"]}},\"TitleChunker:GrumpyGarlicsBake\":{\"hierarchy\":5,\"include_heading_content\":true,\"levels\":[[\"^#[^#]\",\"^##[^#]\",\"^###[^#]\",\"^####[^#]\"],[\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\",\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"第[零一二三四五六七八九十百0-9]+条\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"],[\"第[0-9]+章\",\"第[0-9]+节\",\"[0-9]{1,2}[\\\\. 、]\",\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\",\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"],[\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"[零一二三四五六七八九十百]+[ 、]\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\",\"[\\\\(（][0-9]{,2}[\\\\)）]\"],[\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\",\"Chapter (I+V?|VI*|XI|IX|X)\",\"Section [0-9]+\",\"Article [0-9]+\"]],\"hierarchyRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"groupRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"hierarchyHierarchy\":\"5\",\"hierarchyGroup\":\"0\",\"method\":\"hierarchy\"},\"Tokenizer:HotDonutsRing\":{\"fields\":\"text\",\"filename_embd_weight\":0.1,\"search_method\":[\"embedding\",\"full_text\"]},\"Extractor:AutoExtractDefault\":{\"llm_id\":\"\",\"keywords\":{\"top_n\":0},\"questions\":{\"top_n\":0},\"tags\":{\"top_n\":0,\"tag_file_id\":\"\"},\"summary\":{\"enabled\":false},\"metadata\":{\"enabled\":false,\"metadata\":[],\"built_in_metadata\":[]}}}",
|
||||
"email": "{\"File\": {}, \"Parser:BirdsFlutterHigh\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"email\": {\"fields\": [\"from\", \"to\", \"cc\", \"bcc\", \"date\", \"subject\", \"body\", \"attachments\"], \"preprocess\": [\"main_content\"], \"suffix\": [\"eml\"]}}, \"TokenChunker:WarmBreadSmells\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"delimiter\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \"；\", \"！\", \"？\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:NiceWordsSpoken\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
|
||||
"general": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"doc\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"pages\": [[1, 100000]], \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"txt\", \"py\", \"js\", \"java\", \"c\", \"cpp\", \"h\", \"php\", \"go\", \"ts\", \"sh\", \"cs\", \"kt\", \"sql\"]}, \"image\": {\"ocr_enabled\": true, \"preprocess\": [\"main_content\"], \"suffix\": [\"bmp\", \"gif\", \"jpeg\", \"jpg\", \"png\", \"svg\", \"tif\", \"tiff\", \"webp\"]}, \"email\": {\"fields\": [\"from\", \"to\", \"cc\", \"bcc\", \"date\", \"subject\", \"body\", \"attachments\"], \"preprocess\": [\"main_content\"], \"suffix\": [\"eml\", \"msg\"]}}, \"GeneralChunker:SixApplesFall\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiters\": [\"\\n\", \"!\", \"?\", \";\", \"。\", \"；\", \"！\", \"？\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:LegalReadersDecide\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
|
||||
"laws": "{\"File\":{},\"Parser:HipSignsRhyme\":{\"enable_vision_enhancement\":false,\"vlm\":{\"llm_id\":\"\"},\"doc\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"doc\"]},\"docx\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"docx\"],\"vlm\":{}},\"html\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"htm\",\"html\"]},\"markdown\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"md\",\"markdown\",\"mdx\"],\"vlm\":{}},\"pdf\":{\"parse_method\":\"DeepDOC\",\"preprocess\":[\"main_content\"],\"remove_toc\":true,\"suffix\":[\"pdf\"],\"vlm\":{}},\"text&code\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"txt\"]}},\"TitleChunker:SpicyKeysKick\":{\"hierarchy\":2,\"include_heading_content\":false,\"levels\":[[\"^#[^#]\",\"^##[^#]\",\"^###[^#]\",\"^####[^#]\"],[\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\",\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"第[零一二三四五六七八九十百0-9]+条\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"],[\"第[0-9]+章\",\"第[0-9]+节\",\"[0-9]{1,2}[\\\\. 、]\",\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\",\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"],[\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"[零一二三四五六七八九十百]+[ 、]\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\",\"[\\\\(（][0-9]{,2}[\\\\)）]\"],[\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\",\"Chapter (I+V?|VI*|XI|IX|X)\",\"Section [0-9]+\",\"Article [0-9]+\"]],\"hierarchyRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"groupRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"hierarchyHierarchy\":\"2\",\"hierarchyGroup\":\"0\",\"method\":\"hierarchy\"},\"Tokenizer:PublicJobsTake\":{\"fields\":\"text\",\"filename_embd_weight\":0.1,\"search_method\":[\"embedding\",\"full_text\"]},\"Extractor:AutoExtractDefault\":{\"llm_id\":\"\",\"keywords\":{\"top_n\":0},\"questions\":{\"top_n\":0},\"tags\":{\"top_n\":0,\"tag_file_id\":\"\"},\"summary\":{\"enabled\":false},\"metadata\":{\"enabled\":false,\"metadata\":[],\"built_in_metadata\":[]}}}",
|
||||
"manual": "{\"File\":{},\"Parser:HipSignsRhyme\":{\"enable_vision_enhancement\":false,\"vlm\":{\"llm_id\":\"\"},\"doc\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"doc\"]},\"docx\":{\"preprocess\":[\"main_content\"],\"suffix\":[\"docx\"],\"vlm\":{}},\"pdf\":{\"parse_method\":\"DeepDOC\",\"preprocess\":[\"main_content\"],\"suffix\":[\"pdf\"],\"vlm\":{}}},\"ManualChunker:NineInsectsFind\":{\"hierarchy\":0,\"include_heading_content\":false,\"levels\":[[\"^#[^#]\",\"^##[^#]\",\"^###[^#]\",\"^####[^#]\"],[\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\",\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"第[零一二三四五六七八九十百0-9]+条\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"],[\"第[0-9]+章\",\"第[0-9]+节\",\"[0-9]{1,2}[\\\\. 、]\",\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\",\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"],[\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"[零一二三四五六七八九十百]+[ 、]\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\",\"[\\\\(（][0-9]{,2}[\\\\)）]\"],[\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\",\"Chapter (I+V?|VI*|XI|IX|X)\",\"Section [0-9]+\",\"Article [0-9]+\"]],\"hierarchyRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"groupRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"hierarchyHierarchy\":\"0\",\"hierarchyGroup\":\"0\",\"method\":\"group\"},\"Tokenizer:FunnyBalloonsGrin\":{\"fields\":\"text\",\"filename_embd_weight\":0.1,\"search_method\":[\"embedding\",\"full_text\"]},\"Extractor:AutoExtractDefault\":{\"llm_id\":\"\",\"keywords\":{\"top_n\":0},\"questions\":{\"top_n\":0},\"tags\":{\"top_n\":0,\"tag_file_id\":\"\"},\"summary\":{\"enabled\":false},\"metadata\":{\"enabled\":false,\"metadata\":[],\"built_in_metadata\":[]}}}",
|
||||
"one": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"doc\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\"], \"vlm\": {}}, \"text&code\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"OneChunker:DryDrinksVisit\": {}, \"Tokenizer:FrankWeeksListen\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
|
||||
"paper": "{\"File\":{},\"Parser:HipSignsRhyme\":{\"enable_vision_enhancement\":false,\"vlm\":{\"llm_id\":\"\"},\"pdf\":{\"enable_multi_column\":true,\"parse_method\":\"DeepDOC\",\"preprocess\":[\"main_content\"],\"suffix\":[\"pdf\"],\"vlm\":{}}},\"TitleChunker:SparklySchoolsTravel\":{\"hierarchy\":0,\"include_heading_content\":false,\"levels\":[[\"^#[^#]\",\"^##[^#]\",\"^###[^#]\",\"^####[^#]\"],[\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\",\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"第[零一二三四五六七八九十百0-9]+条\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"],[\"第[0-9]+章\",\"第[0-9]+节\",\"[0-9]{1,2}[\\\\. 、]\",\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\",\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"],[\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"[零一二三四五六七八九十百]+[ 、]\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\",\"[\\\\(（][0-9]{,2}[\\\\)）]\"],[\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\",\"Chapter (I+V?|VI*|XI|IX|X)\",\"Section [0-9]+\",\"Article [0-9]+\"]],\"hierarchyRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"groupRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"hierarchyHierarchy\":\"0\",\"hierarchyGroup\":\"0\",\"method\":\"group\"},\"Tokenizer:GreatCarsWash\":{\"fields\":\"text\",\"filename_embd_weight\":0.1,\"search_method\":[\"embedding\",\"full_text\"]},\"Extractor:AutoExtractDefault\":{\"llm_id\":\"\",\"keywords\":{\"top_n\":0},\"questions\":{\"top_n\":0},\"tags\":{\"top_n\":0,\"tag_file_id\":\"\"},\"summary\":{\"enabled\":false},\"metadata\":{\"enabled\":false,\"metadata\":[],\"built_in_metadata\":[]}}}",
|
||||
"picture": "{\"File\": {}, \"Parser:ViewsCaptureLight\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"image\": {\"ocr_enabled\": true, \"preprocess\": [\"main_content\"], \"suffix\": [\"bmp\", \"gif\", \"jpeg\", \"jpg\", \"png\", \"svg\", \"tif\", \"tiff\", \"webp\"]}, \"video\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"3gp\", \"3gpp\", \"avi\", \"flv\", \"mkv\", \"mov\", \"mp4\", \"mpeg\", \"mpg\", \"webm\", \"wmv\"]}}, \"OneChunker:BrightColorsGlow\": {}, \"Tokenizer:SharpLensFocus\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
|
||||
"presentation": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"pdf\": {\"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"slides\": {\"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pptx\", \"ppt\"]}}, \"Tokenizer:TallTreesDance\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"PageChunker:HappyHillsGlow\": {}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
|
||||
"qa": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"docx\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"markdown\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"Tokenizer:ColdCloudsDream\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"QAChunker:TidyCloudsThink\": {}}",
|
||||
"table": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"spreadsheet\": {\"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}, \"column_mode\": \"auto\", \"column_roles\": {}, \"column_names\": []}, \"text&code\": {\"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"TableChunker:FastFoxesJump\": {}, \"Tokenizer:DeepLakesShine\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}}",
|
||||
"audio": "{\"File\": {}, \"Parser:SongsFillAir\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"audio\": {\"suffix\": [\"aac\", \"aiff\", \"ape\", \"au\", \"da\", \"flac\", \"midi\", \"mp3\", \"ogg\", \"oggvorbis\", \"realaudio\", \"vqf\", \"wav\", \"wave\", \"wma\"]}}, \"OneChunker:BlueSkiesLaugh\": {}, \"Tokenizer:KindEyesWatch\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
|
||||
"book": "{\"File\":{},\"Parser:HipSignsRhyme\":{\"enable_vision_enhancement\":false,\"vlm\":{\"llm_id\":\"\"},\"doc\":{\"suffix\":[\"doc\"]},\"docx\":{\"suffix\":[\"docx\"]},\"html\":{\"suffix\":[\"htm\",\"html\"]},\"pdf\":{\"parse_method\":\"DeepDOC\",\"remove_toc\":true,\"suffix\":[\"pdf\"]},\"text&code\":{\"suffix\":[\"txt\"]}},\"TitleChunker:GrumpyGarlicsBake\":{\"hierarchy\":5,\"include_heading_content\":true,\"levels\":[[\"^#[^#]\",\"^##[^#]\",\"^###[^#]\",\"^####[^#]\"],[\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\",\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"第[零一二三四五六七八九十百0-9]+条\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"],[\"第[0-9]+章\",\"第[0-9]+节\",\"[0-9]{1,2}[\\\\. 、]\",\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\",\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"],[\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"[零一二三四五六七八九十百]+[ 、]\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\",\"[\\\\(（][0-9]{,2}[\\\\)）]\"],[\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\",\"Chapter (I+V?|VI*|XI|IX|X)\",\"Section [0-9]+\",\"Article [0-9]+\"]],\"hierarchyRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"groupRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"hierarchyHierarchy\":\"5\",\"hierarchyGroup\":\"0\",\"method\":\"hierarchy\"},\"Tokenizer:HotDonutsRing\":{\"fields\":\"text\",\"filename_embd_weight\":0.1,\"search_method\":[\"embedding\",\"full_text\"]},\"Extractor:AutoExtractDefault\":{\"llm_id\":\"\",\"keywords\":{\"top_n\":0},\"questions\":{\"top_n\":0},\"tags\":{\"top_n\":0,\"tag_file_id\":\"\"},\"summary\":{\"enabled\":false},\"metadata\":{\"enabled\":false,\"metadata\":[],\"built_in_metadata\":[]}}}",
|
||||
"email": "{\"File\": {}, \"Parser:BirdsFlutterHigh\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"email\": {\"fields\": [\"from\", \"to\", \"cc\", \"bcc\", \"date\", \"subject\", \"body\", \"attachments\"], \"suffix\": [\"eml\"]}}, \"TokenChunker:WarmBreadSmells\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"delimiter\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \"；\", \"！\", \"？\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:NiceWordsSpoken\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
|
||||
"general": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"doc\": {\"suffix\": [\"doc\"]}, \"docx\": {\"suffix\": [\"docx\"]}, \"html\": {\"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"suffix\": [\"md\", \"markdown\", \"mdx\"]}, \"pdf\": {\"pages\": [[1, 100000]], \"parse_method\": \"DeepDOC\", \"suffix\": [\"pdf\"]}, \"spreadsheet\": {\"parse_method\": \"DeepDOC\", \"suffix\": [\"xls\", \"xlsx\", \"csv\"]}, \"text&code\": {\"suffix\": [\"txt\", \"py\", \"js\", \"java\", \"c\", \"cpp\", \"h\", \"php\", \"go\", \"ts\", \"sh\", \"cs\", \"kt\", \"sql\"]}, \"image\": {\"ocr_enabled\": true, \"suffix\": [\"bmp\", \"gif\", \"jpeg\", \"jpg\", \"png\", \"svg\", \"tif\", \"tiff\", \"webp\"]}, \"email\": {\"fields\": [\"from\", \"to\", \"cc\", \"bcc\", \"date\", \"subject\", \"body\", \"attachments\"], \"suffix\": [\"eml\", \"msg\"]}}, \"GeneralChunker:SixApplesFall\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiters\": [\"\\n\", \"!\", \"?\", \";\", \"。\", \"；\", \"！\", \"？\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:LegalReadersDecide\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
|
||||
"laws": "{\"File\":{},\"Parser:HipSignsRhyme\":{\"enable_vision_enhancement\":false,\"vlm\":{\"llm_id\":\"\"},\"doc\":{\"suffix\":[\"doc\"]},\"docx\":{\"suffix\":[\"docx\"]},\"html\":{\"suffix\":[\"htm\",\"html\"]},\"markdown\":{\"suffix\":[\"md\",\"markdown\",\"mdx\"]},\"pdf\":{\"parse_method\":\"DeepDOC\",\"remove_toc\":true,\"suffix\":[\"pdf\"]},\"text&code\":{\"suffix\":[\"txt\"]}},\"TitleChunker:SpicyKeysKick\":{\"hierarchy\":2,\"include_heading_content\":false,\"levels\":[[\"^#[^#]\",\"^##[^#]\",\"^###[^#]\",\"^####[^#]\"],[\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\",\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"第[零一二三四五六七八九十百0-9]+条\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"],[\"第[0-9]+章\",\"第[0-9]+节\",\"[0-9]{1,2}[\\\\. 、]\",\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\",\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"],[\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"[零一二三四五六七八九十百]+[ 、]\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\",\"[\\\\(（][0-9]{,2}[\\\\)）]\"],[\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\",\"Chapter (I+V?|VI*|XI|IX|X)\",\"Section [0-9]+\",\"Article [0-9]+\"]],\"hierarchyRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"groupRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"hierarchyHierarchy\":\"2\",\"hierarchyGroup\":\"0\",\"method\":\"hierarchy\"},\"Tokenizer:PublicJobsTake\":{\"fields\":\"text\",\"filename_embd_weight\":0.1,\"search_method\":[\"embedding\",\"full_text\"]},\"Extractor:AutoExtractDefault\":{\"llm_id\":\"\",\"keywords\":{\"top_n\":0},\"questions\":{\"top_n\":0},\"tags\":{\"top_n\":0,\"tag_file_id\":\"\"},\"summary\":{\"enabled\":false},\"metadata\":{\"enabled\":false,\"metadata\":[],\"built_in_metadata\":[]}}}",
|
||||
"manual": "{\"File\":{},\"Parser:HipSignsRhyme\":{\"enable_vision_enhancement\":false,\"vlm\":{\"llm_id\":\"\"},\"doc\":{\"suffix\":[\"doc\"]},\"docx\":{\"suffix\":[\"docx\"]},\"pdf\":{\"parse_method\":\"DeepDOC\",\"suffix\":[\"pdf\"]}},\"ManualChunker:NineInsectsFind\":{\"hierarchy\":0,\"include_heading_content\":false,\"levels\":[[\"^#[^#]\",\"^##[^#]\",\"^###[^#]\",\"^####[^#]\"],[\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\",\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"第[零一二三四五六七八九十百0-9]+条\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"],[\"第[0-9]+章\",\"第[0-9]+节\",\"[0-9]{1,2}[\\\\. 、]\",\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\",\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"],[\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"[零一二三四五六七八九十百]+[ 、]\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\",\"[\\\\(（][0-9]{,2}[\\\\)）]\"],[\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\",\"Chapter (I+V?|VI*|XI|IX|X)\",\"Section [0-9]+\",\"Article [0-9]+\"]],\"hierarchyRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"groupRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"hierarchyHierarchy\":\"0\",\"hierarchyGroup\":\"0\",\"method\":\"group\"},\"Tokenizer:FunnyBalloonsGrin\":{\"fields\":\"text\",\"filename_embd_weight\":0.1,\"search_method\":[\"embedding\",\"full_text\"]},\"Extractor:AutoExtractDefault\":{\"llm_id\":\"\",\"keywords\":{\"top_n\":0},\"questions\":{\"top_n\":0},\"tags\":{\"top_n\":0,\"tag_file_id\":\"\"},\"summary\":{\"enabled\":false},\"metadata\":{\"enabled\":false,\"metadata\":[],\"built_in_metadata\":[]}}}",
|
||||
"one": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"doc\": {\"suffix\": [\"doc\"]}, \"docx\": {\"suffix\": [\"docx\"]}, \"html\": {\"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"suffix\": [\"md\", \"markdown\", \"mdx\"]}, \"pdf\": {\"parse_method\": \"DeepDOC\", \"suffix\": [\"pdf\"]}, \"spreadsheet\": {\"parse_method\": \"DeepDOC\", \"suffix\": [\"xls\", \"xlsx\"]}, \"text&code\": {\"suffix\": [\"txt\"]}}, \"OneChunker:DryDrinksVisit\": {}, \"Tokenizer:FrankWeeksListen\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
|
||||
"paper": "{\"File\":{},\"Parser:HipSignsRhyme\":{\"enable_vision_enhancement\":false,\"vlm\":{\"llm_id\":\"\"},\"pdf\":{\"enable_multi_column\":true,\"parse_method\":\"DeepDOC\",\"suffix\":[\"pdf\"]}},\"TitleChunker:SparklySchoolsTravel\":{\"hierarchy\":0,\"include_heading_content\":false,\"levels\":[[\"^#[^#]\",\"^##[^#]\",\"^###[^#]\",\"^####[^#]\"],[\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\",\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"第[零一二三四五六七八九十百0-9]+条\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"],[\"第[0-9]+章\",\"第[0-9]+节\",\"[0-9]{1,2}[\\\\. 、]\",\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\",\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"],[\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"[零一二三四五六七八九十百]+[ 、]\",\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\",\"[\\\\(（][0-9]{,2}[\\\\)）]\"],[\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\",\"Chapter (I+V?|VI*|XI|IX|X)\",\"Section [0-9]+\",\"Article [0-9]+\"]],\"hierarchyRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"groupRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"},{\"expression\":\"[\\\\(（][0-9]{,2}[\\\\)）]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"hierarchyHierarchy\":\"0\",\"hierarchyGroup\":\"0\",\"method\":\"group\"},\"Tokenizer:GreatCarsWash\":{\"fields\":\"text\",\"filename_embd_weight\":0.1,\"search_method\":[\"embedding\",\"full_text\"]},\"Extractor:AutoExtractDefault\":{\"llm_id\":\"\",\"keywords\":{\"top_n\":0},\"questions\":{\"top_n\":0},\"tags\":{\"top_n\":0,\"tag_file_id\":\"\"},\"summary\":{\"enabled\":false},\"metadata\":{\"enabled\":false,\"metadata\":[],\"built_in_metadata\":[]}}}",
|
||||
"picture": "{\"File\": {}, \"Parser:ViewsCaptureLight\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"image\": {\"ocr_enabled\": true, \"suffix\": [\"bmp\", \"gif\", \"jpeg\", \"jpg\", \"png\", \"svg\", \"tif\", \"tiff\", \"webp\"]}, \"video\": {\"suffix\": [\"3gp\", \"3gpp\", \"avi\", \"flv\", \"mkv\", \"mov\", \"mp4\", \"mpeg\", \"mpg\", \"webm\", \"wmv\"]}}, \"OneChunker:BrightColorsGlow\": {}, \"Tokenizer:SharpLensFocus\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
|
||||
"presentation": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"pdf\": {\"parse_method\": \"DeepDOC\", \"suffix\": [\"pdf\"]}, \"slides\": {\"parse_method\": \"DeepDOC\", \"suffix\": [\"pptx\", \"ppt\"]}}, \"Tokenizer:TallTreesDance\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"PageChunker:HappyHillsGlow\": {}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
|
||||
"qa": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"docx\": {\"suffix\": [\"docx\"]}, \"markdown\": {\"suffix\": [\"md\", \"markdown\", \"mdx\"]}, \"pdf\": {\"parse_method\": \"DeepDOC\", \"suffix\": [\"pdf\"]}, \"spreadsheet\": {\"parse_method\": \"DeepDOC\", \"suffix\": [\"xls\", \"xlsx\", \"csv\"]}, \"text&code\": {\"suffix\": [\"txt\"]}}, \"Tokenizer:ColdCloudsDream\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"QAChunker:TidyCloudsThink\": {}}",
|
||||
"table": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"spreadsheet\": {\"parse_method\": \"DeepDOC\", \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"column_mode\": \"auto\", \"column_roles\": {}, \"column_names\": []}, \"text&code\": {\"suffix\": [\"txt\"]}}, \"TableChunker:FastFoxesJump\": {}, \"Tokenizer:DeepLakesShine\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}}",
|
||||
}
|
||||
|
||||
// Per-template test methods. Each resolves default component params from a
|
||||
|
||||
@@ -44,7 +44,6 @@ const GoMarkdown = "go_markdown"
|
||||
type MarkdownParser struct {
|
||||
libType string
|
||||
ParseMethod string
|
||||
VLM map[string]any
|
||||
// FetchRemoteImages controls whether HTTP(S) Markdown images are
|
||||
// downloaded. NewMarkdownParser enables it for direct Markdown parsing;
|
||||
// callers normalizing untrusted backend responses may disable it.
|
||||
@@ -77,9 +76,6 @@ func (p *MarkdownParser) ConfigureFromSetup(setup map[string]any) {
|
||||
if v, ok := setup["parse_method"].(string); ok && v != "" {
|
||||
p.ParseMethod = v
|
||||
}
|
||||
if v, ok := setup["vlm"].(map[string]any); ok {
|
||||
p.VLM = v
|
||||
}
|
||||
}
|
||||
|
||||
// ParseWithResult implements ParseResultProducer (plan §6.5) and
|
||||
|
||||
@@ -512,19 +512,10 @@ func TestMarkdownParser_ParseWithResult_InlineRawHTMLSurvives(t *testing.T) {
|
||||
|
||||
func TestMarkdownParser_ConfigureFromSetup(t *testing.T) {
|
||||
p, _ := NewMarkdownParser(GoMarkdown)
|
||||
p.ConfigureFromSetup(map[string]any{
|
||||
"parse_method": "deepdoc",
|
||||
"vlm": map[string]any{"llm_id": "gpt-4-vision"},
|
||||
})
|
||||
p.ConfigureFromSetup(map[string]any{"parse_method": "deepdoc"})
|
||||
if p.ParseMethod != "deepdoc" {
|
||||
t.Fatalf("ParseMethod = %q, want %q", p.ParseMethod, "deepdoc")
|
||||
}
|
||||
if p.VLM == nil {
|
||||
t.Fatal("VLM is nil; want map")
|
||||
}
|
||||
if id, _ := p.VLM["llm_id"].(string); id != "gpt-4-vision" {
|
||||
t.Fatalf("VLM[llm_id] = %q, want %q", id, "gpt-4-vision")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarkdownParser_ConfigureFromSetup_NilSafe(t *testing.T) {
|
||||
|
||||
@@ -1348,13 +1348,6 @@ export default {
|
||||
pleaseUploadAtLeastOneFile: 'يرجى تحميل ملف واحد على الأقل',
|
||||
},
|
||||
flow: {
|
||||
preprocess: {
|
||||
preprocess: 'المعالجة المسبقة',
|
||||
mainContent: 'المحتوى الرئيسي',
|
||||
abstract: 'خلاصة',
|
||||
author: 'مؤلف',
|
||||
sectionTitle: 'عنوان القسم',
|
||||
},
|
||||
autoPlay: 'تشغيل الصوت تلقائيًا',
|
||||
downloadFileTypeTip: 'نوع الملف المراد تنزيله',
|
||||
downloadFileType: 'تنزيل نوع الملف',
|
||||
|
||||
@@ -2659,13 +2659,6 @@ Nümunə: Virtual host üslubu`,
|
||||
},
|
||||
flow: {
|
||||
exportCurrentPage: 'Cari səhifəni ixrac et',
|
||||
preprocess: {
|
||||
preprocess: 'İlkin emal',
|
||||
mainContent: 'Əsas məzmun',
|
||||
abstract: 'Xülasə',
|
||||
author: 'Müəllif',
|
||||
sectionTitle: 'Bölmə başlığı',
|
||||
},
|
||||
tags: 'Etiketlər',
|
||||
agentCategory: 'Kətan kateqoriyası',
|
||||
editTags: 'Etiketləri redaktə et',
|
||||
|
||||
@@ -2767,13 +2767,6 @@ Example: Virtual Hosted Style`,
|
||||
},
|
||||
flow: {
|
||||
exportCurrentPage: 'Export current page',
|
||||
preprocess: {
|
||||
preprocess: 'Preprocess',
|
||||
mainContent: 'Main content',
|
||||
abstract: 'Abstract',
|
||||
author: 'Author',
|
||||
sectionTitle: 'Section title',
|
||||
},
|
||||
tags: 'Tags',
|
||||
agentCategory: 'Agent category',
|
||||
editTags: 'Edit tags',
|
||||
|
||||
@@ -2462,13 +2462,6 @@ Exemple : Virtual Hosted Style`,
|
||||
seconds: 'Secondes',
|
||||
subject: 'Sujet',
|
||||
// --- clés manquantes ajoutées ---
|
||||
preprocess: {
|
||||
preprocess: 'Prétraitement',
|
||||
mainContent: 'Contenu principal',
|
||||
abstract: 'Résumé',
|
||||
author: 'Auteur',
|
||||
sectionTitle: 'Titre de section',
|
||||
},
|
||||
editTags: 'Modifier les étiquettes',
|
||||
editTagsDescription:
|
||||
'Ajoutez des étiquettes pour organiser et filtrer vos agents. Appuyez sur Entrée ou virgule pour ajouter.',
|
||||
|
||||
@@ -1838,13 +1838,6 @@ Esempio: Virtual Hosted Style`,
|
||||
pleaseUploadAtLeastOneFile: 'Carica almeno un file',
|
||||
},
|
||||
flow: {
|
||||
preprocess: {
|
||||
preprocess: 'Pre-elaborazione',
|
||||
mainContent: 'Contenuto principale',
|
||||
abstract: 'Abstract',
|
||||
author: 'Autore',
|
||||
sectionTitle: 'Titolo sezione',
|
||||
},
|
||||
editTags: 'Modifica tag',
|
||||
editTagsDescription:
|
||||
'Aggiungi tag per organizzare e filtrare i tuoi agenti. Premi Invio o virgola per aggiungere.',
|
||||
|
||||
@@ -2394,14 +2394,6 @@ export default {
|
||||
id: 'ID',
|
||||
logTitle: 'タイトル',
|
||||
|
||||
preprocess: {
|
||||
preprocess: '前処理',
|
||||
mainContent: '本文',
|
||||
abstract: '概要',
|
||||
author: '著者',
|
||||
sectionTitle: 'セクションタイトル',
|
||||
},
|
||||
|
||||
editTags: 'タグを編集',
|
||||
editTagsDescription:
|
||||
'タグを追加してエージェントを整理・絞り込みできます。Enterキーまたはカンマで追加します。',
|
||||
|
||||
@@ -1842,13 +1842,6 @@ export default {
|
||||
pleaseUploadAtLeastOneFile: '최소 하나의 파일을 업로드해 주세요',
|
||||
},
|
||||
flow: {
|
||||
preprocess: {
|
||||
preprocess: '전처리',
|
||||
mainContent: '본문 내용',
|
||||
abstract: '요약',
|
||||
author: '저자',
|
||||
sectionTitle: '섹션 제목',
|
||||
},
|
||||
editTags: '태그 편집',
|
||||
editTagsDescription:
|
||||
'에이전트를 정리하고 필터링하기 위한 태그를 추가하세요. Enter 또는 쉼표를 눌러 추가합니다.',
|
||||
|
||||
@@ -1537,13 +1537,6 @@ export default {
|
||||
pleaseUploadAtLeastOneFile: 'Пожалуйста, загрузите хотя бы один файл',
|
||||
},
|
||||
flow: {
|
||||
preprocess: {
|
||||
preprocess: 'Предобработка',
|
||||
mainContent: 'Основное содержимое',
|
||||
abstract: 'Аннотация',
|
||||
author: 'Автор',
|
||||
sectionTitle: 'Заголовок раздела',
|
||||
},
|
||||
autoPlay: 'Автовоспроизведение аудио',
|
||||
downloadFileTypeTip: 'Тип файла для скачивания',
|
||||
downloadFileType: 'Тип скачиваемого файла',
|
||||
|
||||
@@ -2740,13 +2740,6 @@ Bu otomatik etiketleme özelliği, mevcut datasete alanına özgü bilgi katman
|
||||
},
|
||||
flow: {
|
||||
exportCurrentPage: 'Geçerli sayfayı dışa aktar',
|
||||
preprocess: {
|
||||
preprocess: 'Ön işleme',
|
||||
mainContent: 'Ana içerik',
|
||||
abstract: 'Özet',
|
||||
author: 'Yazar',
|
||||
sectionTitle: 'Bölüm başlığı',
|
||||
},
|
||||
includeHeadingContent: 'Üst başlık içeriğini ayır',
|
||||
includeHeadingContentTip:
|
||||
'Etkinleştirildiğinde parçalar yalnızca başlık yolunu ve içeriğini içerir; üst başlıktan hemen sonra gelen içerik ayrı bir parça olarak tutulur.',
|
||||
|
||||
@@ -2558,13 +2558,6 @@ export default {
|
||||
},
|
||||
flow: {
|
||||
exportCurrentPage: 'Joriy sahifani eksport qilish',
|
||||
preprocess: {
|
||||
preprocess: 'Oldindan qayta ishlash',
|
||||
mainContent: 'Asosiy kontent',
|
||||
abstract: 'Annotatsiya',
|
||||
author: 'Muallif',
|
||||
sectionTitle: 'Bo‘lim sarlavhasi',
|
||||
},
|
||||
tags: 'Teglar',
|
||||
agentCategory: 'Canvas toifasi',
|
||||
editTags: 'Teglarni tahrirlash',
|
||||
|
||||
@@ -2333,13 +2333,6 @@ NER：使用 spaCy NER 和基于规则的关键词提取来抽取 Entities 和 R
|
||||
},
|
||||
flow: {
|
||||
exportCurrentPage: '导出当页',
|
||||
preprocess: {
|
||||
preprocess: '预处理',
|
||||
mainContent: '主内容',
|
||||
abstract: '摘要',
|
||||
author: '作者',
|
||||
sectionTitle: '章节标题',
|
||||
},
|
||||
agentCategory: '智能体分类',
|
||||
created: '创建于',
|
||||
editTags: '编辑标签',
|
||||
|
||||
@@ -80,22 +80,6 @@ export enum StringTransformDelimiter {
|
||||
Space = ' ',
|
||||
}
|
||||
|
||||
export enum PreprocessValue {
|
||||
main_content = 'main_content',
|
||||
section_title = 'title',
|
||||
abstract = 'abstract',
|
||||
author = 'author',
|
||||
}
|
||||
|
||||
export const MAIN_CONTENT_PREPROCESS_VALUE: PreprocessValue =
|
||||
PreprocessValue.main_content;
|
||||
|
||||
export const PreprocessLabelKeyMap: Record<PreprocessValue, string> = {
|
||||
main_content: 'mainContent',
|
||||
title: 'sectionTitle',
|
||||
abstract: 'abstract',
|
||||
author: 'author',
|
||||
};
|
||||
export const initialParserValues = {
|
||||
// The Go Parser component emits structured JSON items only; markdown/text/
|
||||
// html are internal backend representations, never public Parser outputs.
|
||||
@@ -111,52 +95,42 @@ export const initialParserValues = {
|
||||
{
|
||||
fileFormat: FileType.PDF,
|
||||
parse_method: ParseDocumentType.DeepDOC,
|
||||
preprocess: PreprocessValue.main_content,
|
||||
remove_header_footer: false,
|
||||
pages: [{ from: 1, to: 100000 }],
|
||||
},
|
||||
{
|
||||
fileFormat: FileType.Spreadsheet,
|
||||
parse_method: ParseDocumentType.DeepDOC,
|
||||
preprocess: PreprocessValue.main_content,
|
||||
},
|
||||
{
|
||||
fileFormat: FileType.Image,
|
||||
ocr_enabled: true,
|
||||
preprocess: PreprocessValue.main_content,
|
||||
},
|
||||
{
|
||||
fileFormat: FileType.Email,
|
||||
fields: Object.values(ParserFields),
|
||||
preprocess: PreprocessValue.main_content,
|
||||
},
|
||||
{
|
||||
fileFormat: FileType.TextMarkdown,
|
||||
preprocess: PreprocessValue.main_content,
|
||||
},
|
||||
{
|
||||
fileFormat: FileType.Code,
|
||||
preprocess: PreprocessValue.main_content,
|
||||
},
|
||||
{
|
||||
fileFormat: FileType.Html,
|
||||
preprocess: PreprocessValue.main_content,
|
||||
remove_header_footer: false,
|
||||
},
|
||||
{
|
||||
fileFormat: FileType.Doc,
|
||||
preprocess: PreprocessValue.main_content,
|
||||
remove_header_footer: false,
|
||||
},
|
||||
{
|
||||
fileFormat: FileType.Docx,
|
||||
preprocess: PreprocessValue.main_content,
|
||||
remove_header_footer: false,
|
||||
},
|
||||
{
|
||||
fileFormat: FileType.PowerPoint,
|
||||
parse_method: ParseDocumentType.DeepDOC,
|
||||
preprocess: PreprocessValue.main_content,
|
||||
},
|
||||
// {
|
||||
// fileFormat: FileType.Video,
|
||||
|
||||
@@ -5,7 +5,6 @@ import { z } from 'zod';
|
||||
export const SetupSchema = z
|
||||
.object({
|
||||
fileFormat: z.string().nullish(),
|
||||
// preprocess: z.array(z.string()).optional(),
|
||||
parse_method: z.string().optional(),
|
||||
mineru_parse_method: z.string().optional(),
|
||||
mineru_lang: z.string().optional(),
|
||||
|
||||
@@ -85,6 +85,19 @@ describe('parser-form utils', () => {
|
||||
buildInitialParserValues({}).setups,
|
||||
);
|
||||
});
|
||||
|
||||
it('does not seed retired family keys', () => {
|
||||
const values = buildInitialParserValues({});
|
||||
for (const setup of values.setups) {
|
||||
expect(setup).not.toHaveProperty('preprocess');
|
||||
expect(setup).not.toHaveProperty('output_format');
|
||||
// Audio's family vlm is the ASR model and stays; every other family
|
||||
// uses the Parser's top-level vision block.
|
||||
if ((setup as any).fileFormat !== FileType.Audio) {
|
||||
expect(setup).not.toHaveProperty('vlm');
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('normalizeParserFormValues', () => {
|
||||
@@ -307,21 +320,27 @@ describe('parser-form utils', () => {
|
||||
expect(normalizeParserFormValues(once)).toEqual(once);
|
||||
});
|
||||
|
||||
it('drops the retired per-setup output_format from every family', () => {
|
||||
// The Go Parser normalizes all setups to JSON, so a stored legacy value
|
||||
// is dead config that must not survive a load/save round-trip.
|
||||
it('drops the retired per-setup preprocess and output_format from every family', () => {
|
||||
// The Parser derives preprocessing from the chunker and emits JSON only,
|
||||
// so stored legacy values are dead config that must not survive a
|
||||
// load/save round-trip.
|
||||
const normalized = normalizeParserFormValues({
|
||||
setups: [
|
||||
{ fileFormat: FileType.PDF, output_format: 'markdown' },
|
||||
{
|
||||
fileFormat: FileType.PDF,
|
||||
preprocess: ['main_content'],
|
||||
output_format: 'markdown',
|
||||
},
|
||||
{
|
||||
fileFormat: FileType.Audio,
|
||||
preprocess: ['main_content'],
|
||||
output_format: 'text',
|
||||
vlm: { llm_id: 'whisper@OpenAI' },
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(normalized.setups[0]).toEqual({ fileFormat: FileType.PDF });
|
||||
// Audio keeps its per-setup ASR model; only output_format goes.
|
||||
// Audio keeps its per-setup ASR model; only the retired keys go.
|
||||
expect(normalized.setups[1]).toEqual({
|
||||
fileFormat: FileType.Audio,
|
||||
vlm: { llm_id: 'whisper@OpenAI' },
|
||||
|
||||
@@ -73,11 +73,15 @@ export function normalizeParserFormValues<T extends Record<string, any>>(
|
||||
'');
|
||||
|
||||
const nextSetups = setups.map((x) => {
|
||||
// output_format is retired on every family: the Go Parser normalizes all
|
||||
// setups to JSON, so a stored value is dead config that must not survive
|
||||
// a load/save round-trip.
|
||||
if (x?.fileFormat === FileType.Audio) return omit(x, ['output_format']);
|
||||
const stripped = omit(x, ['vlm', 'output_format']);
|
||||
// preprocess and output_format are retired on every family: the Parser
|
||||
// derives preprocessing from the chunker and emits JSON only, so a stored
|
||||
// value is dead config that must not survive a load/save round-trip.
|
||||
// Per-setup vlm is dead for every family except Audio, whose vlm holds the
|
||||
// ASR model.
|
||||
if (x?.fileFormat === FileType.Audio) {
|
||||
return omit(x, ['output_format', 'preprocess']);
|
||||
}
|
||||
const stripped = omit(x, ['vlm', 'output_format', 'preprocess']);
|
||||
if (x?.fileFormat !== FileType.Image) return stripped;
|
||||
const { parse_method, ...rest } = omit(stripped, ['lang', 'system_prompt']);
|
||||
// Materialize the switch either way. Absent or "ocr" both mean "run local
|
||||
|
||||
Reference in new issue
Block a user