diff --git a/internal/common/component_identity.go b/internal/common/component_identity.go new file mode 100644 index 0000000000..f97645da05 --- /dev/null +++ b/internal/common/component_identity.go @@ -0,0 +1,79 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package common + +import ( + "sort" + "strings" +) + +// hasComponentIDPrefix reports whether id is ":". +func hasComponentIDPrefix(id, componentName string) bool { + kind, instance, ok := strings.Cut(id, ":") + return ok && kind == componentName && instance != "" +} + +// IsExtractorComponentID reports whether id names an Extractor node in a +// component-scoped parser_config or pipeline DSL. Only the canonical +// "Extractor:" form is accepted. +func IsExtractorComponentID(id string) bool { + return hasComponentIDPrefix(id, ComponentNameExtractor) +} + +// IsExtractorComponent reports whether the component name or component ID +// identifies an Extractor. +func IsExtractorComponent(id, componentName string) bool { + return componentName == ComponentNameExtractor || IsExtractorComponentID(id) +} + +// IsParserComponentID reports whether id names a Parser node in a +// component-scoped parser_config or pipeline DSL. +func IsParserComponentID(id string) bool { + return hasComponentIDPrefix(id, ComponentNameParser) +} + +// IsParserComponent reports whether the component name or component ID +// identifies a Parser. +func IsParserComponent(id, componentName string) bool { + return componentName == ComponentNameParser || IsParserComponentID(id) +} + +// IsCompilerComponentID reports whether id names a Compiler node in a +// component-scoped parser_config or pipeline DSL. +func IsCompilerComponentID(id string) bool { + return hasComponentIDPrefix(id, ComponentNameCompiler) +} + +// IsComponentKey reports whether id is a canonical component-scoped +// parser_config key: ":". +func IsComponentKey(id string) bool { + kind, instance, ok := strings.Cut(id, ":") + return ok && instance != "" && IsComponentName(kind) +} + +// SortedExtractorKeys returns the parser_config keys that name Extractor nodes, +// sorted so callers resolve multiple Extractor nodes deterministically. +func SortedExtractorKeys(parserConfig map[string]any) []string { + keys := make([]string, 0) + for key := range parserConfig { + if IsExtractorComponentID(key) { + keys = append(keys, key) + } + } + sort.Strings(keys) + return keys +} diff --git a/internal/common/component_identity_test.go b/internal/common/component_identity_test.go new file mode 100644 index 0000000000..2f53313f8f --- /dev/null +++ b/internal/common/component_identity_test.go @@ -0,0 +1,210 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package common + +import ( + "reflect" + "testing" +) + +func TestIsExtractorComponentID(t *testing.T) { + tests := []struct { + id string + want bool + }{ + {id: "Extractor", want: false}, + {id: "extractor", want: false}, + {id: "Extractor:AutoExtractDefault", want: true}, + {id: "extractor:node", want: false}, + {id: "EXTRACTOR:Node", want: false}, + {id: "Extractor:", want: false}, + {id: "extractor_0", want: false}, + {id: "Extractor_0", want: false}, + {id: "extractors", want: false}, + {id: "my_extractor", want: false}, + {id: "Parser:Extractor", want: false}, + {id: "", want: false}, + } + for _, tt := range tests { + t.Run(tt.id, func(t *testing.T) { + if got := IsExtractorComponentID(tt.id); got != tt.want { + t.Fatalf("IsExtractorComponentID(%q) = %v, want %v", tt.id, got, tt.want) + } + }) + } +} + +func TestIsParserComponentID(t *testing.T) { + tests := []struct { + id string + want bool + }{ + {id: "Parser:HipSignsRhyme", want: true}, + {id: "Parser:", want: false}, + {id: "parser:node", want: false}, + {id: "PARSER:Node", want: false}, + {id: "Parser", want: false}, + {id: "parser_0", want: false}, + {id: "Extractor:Parser", want: false}, + {id: "", want: false}, + } + for _, tt := range tests { + t.Run(tt.id, func(t *testing.T) { + if got := IsParserComponentID(tt.id); got != tt.want { + t.Fatalf("IsParserComponentID(%q) = %v, want %v", tt.id, got, tt.want) + } + }) + } +} + +func TestIsCompilerComponentID(t *testing.T) { + tests := []struct { + id string + want bool + }{ + {id: "Compiler:CurvySunsLove", want: true}, + {id: "Compiler:", want: false}, + {id: "compiler:node", want: false}, + {id: "COMPILER:Node", want: false}, + {id: "compiler_0", want: false}, + {id: "Compiler", want: false}, + {id: "compilers", want: false}, + {id: "Parser:Compiler", want: false}, + {id: "", want: false}, + } + for _, tt := range tests { + t.Run(tt.id, func(t *testing.T) { + if got := IsCompilerComponentID(tt.id); got != tt.want { + t.Fatalf("IsCompilerComponentID(%q) = %v, want %v", tt.id, got, tt.want) + } + }) + } +} + +func TestIsComponentKey(t *testing.T) { + tests := []struct { + id string + want bool + }{ + {id: "Extractor:AutoExtractDefault", want: true}, + {id: "Parser:HipSignsRhyme", want: true}, + {id: "Parser:", want: false}, + {id: "GeneralChunker:SixApplesFall", want: true}, + {id: "File:node", want: true}, + {id: "Extractor:", want: false}, + {id: ":foo", want: false}, + {id: "foo:bar", want: false}, + {id: "extractor:0", want: false}, + {id: "extractor_0", want: false}, + {id: "compiler_0", want: false}, + {id: "Extractor", want: false}, + {id: "chunk_token_num", want: false}, + {id: "parser_0", want: false}, + {id: "", want: false}, + } + for _, tt := range tests { + t.Run(tt.id, func(t *testing.T) { + if got := IsComponentKey(tt.id); got != tt.want { + t.Fatalf("IsComponentKey(%q) = %v, want %v", tt.id, got, tt.want) + } + }) + } +} + +func TestIsComponentName(t *testing.T) { + names := []string{ + ComponentNameFile, + ComponentNameParser, + ComponentNameExtractor, + ComponentNameCompiler, + ComponentNameTokenizer, + ComponentNameGeneralChunker, + ComponentNameTokenChunker, + ComponentNameTitleChunker, + ComponentNameManualChunker, + ComponentNameOneChunker, + ComponentNamePageChunker, + ComponentNameQAChunker, + ComponentNameTableChunker, + ComponentNameGroupTitleChunker, + ComponentNameHierarchyTitleChunker, + } + for _, name := range names { + if !IsComponentName(name) { + t.Errorf("IsComponentName(%q) = false, want true", name) + } + } + for _, name := range []string{"", "Chunker", "extractor", "Extractor "} { + if IsComponentName(name) { + t.Errorf("IsComponentName(%q) = true, want false", name) + } + } +} + +func TestSortedExtractorKeys(t *testing.T) { + parserConfig := map[string]any{ + "Extractor:B": nil, + "Extractor:A": nil, + "Extractor:": nil, + "Extractor": nil, + "extractor_0": nil, + "Parser:X": nil, + } + want := []string{"Extractor:A", "Extractor:B"} + if got := SortedExtractorKeys(parserConfig); !reflect.DeepEqual(got, want) { + t.Fatalf("SortedExtractorKeys = %#v, want %#v", got, want) + } +} + +func TestIsExtractorComponent(t *testing.T) { + tests := []struct { + id string + name string + want bool + }{ + {id: "node-1", name: "Extractor", want: true}, + {id: "Extractor:node-1", name: "", want: true}, + {id: "extractor_0", name: "", want: false}, + {id: "node-1", name: "Parser", want: false}, + } + for _, tt := range tests { + t.Run(tt.name+"/"+tt.id, func(t *testing.T) { + if got := IsExtractorComponent(tt.id, tt.name); got != tt.want { + t.Fatalf("IsExtractorComponent(%q, %q) = %v, want %v", tt.id, tt.name, got, tt.want) + } + }) + } +} + +func TestIsParserComponent(t *testing.T) { + tests := []struct { + id string + name string + want bool + }{ + {id: "node-1", name: "Parser", want: true}, + {id: "Parser:node-1", name: "", want: true}, + {id: "node-1", name: "Extractor", want: false}, + } + for _, tt := range tests { + t.Run(tt.name+"/"+tt.id, func(t *testing.T) { + if got := IsParserComponent(tt.id, tt.name); got != tt.want { + t.Fatalf("IsParserComponent(%q, %q) = %v, want %v", tt.id, tt.name, got, tt.want) + } + }) + } +} diff --git a/internal/common/component_names.go b/internal/common/component_names.go new file mode 100644 index 0000000000..e40e2fbcbf --- /dev/null +++ b/internal/common/component_names.go @@ -0,0 +1,61 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package common + +// Canonical ingestion component names. +const ( + ComponentNameFile = "File" + ComponentNameParser = "Parser" + ComponentNameExtractor = "Extractor" + ComponentNameCompiler = "Compiler" + ComponentNameTokenizer = "Tokenizer" + + ComponentNameGeneralChunker = "GeneralChunker" + ComponentNameTokenChunker = "TokenChunker" + ComponentNameTitleChunker = "TitleChunker" + ComponentNameManualChunker = "ManualChunker" + ComponentNameOneChunker = "OneChunker" + ComponentNamePageChunker = "PageChunker" + ComponentNameQAChunker = "QAChunker" + ComponentNameTableChunker = "TableChunker" + ComponentNameGroupTitleChunker = "GroupTitleChunker" + ComponentNameHierarchyTitleChunker = "HierarchyTitleChunker" +) + +// IsComponentName reports whether name is a canonical ingestion component name. +func IsComponentName(name string) bool { + switch name { + case ComponentNameFile, + ComponentNameParser, + ComponentNameExtractor, + ComponentNameCompiler, + ComponentNameTokenizer, + ComponentNameGeneralChunker, + ComponentNameTokenChunker, + ComponentNameTitleChunker, + ComponentNameManualChunker, + ComponentNameOneChunker, + ComponentNamePageChunker, + ComponentNameQAChunker, + ComponentNameTableChunker, + ComponentNameGroupTitleChunker, + ComponentNameHierarchyTitleChunker: + return true + default: + return false + } +} diff --git a/internal/common/metadata_utils.go b/internal/common/metadata_utils.go index 0a4d8f8db8..3d461ec9c2 100644 --- a/internal/common/metadata_utils.go +++ b/internal/common/metadata_utils.go @@ -18,7 +18,6 @@ package common import ( "regexp" - "sort" "strconv" "strings" ) @@ -441,15 +440,7 @@ func ExtractorMetadataConfig(parserConfig map[string]any) (map[string]any, bool) if parserConfig == nil { return nil, false } - var extractorKeys []string - for k := range parserConfig { - lower := strings.ToLower(k) - if strings.HasPrefix(lower, "extractor:") || strings.HasPrefix(lower, "extractor_") { - extractorKeys = append(extractorKeys, k) - } - } - sort.Strings(extractorKeys) - for _, k := range extractorKeys { + for _, k := range SortedExtractorKeys(parserConfig) { node, ok := parserConfig[k].(map[string]any) if !ok { continue diff --git a/internal/common/parser_config.go b/internal/common/parser_config.go index 92f0163e68..4dc93c0b6d 100644 --- a/internal/common/parser_config.go +++ b/internal/common/parser_config.go @@ -105,7 +105,7 @@ func ExtractPipelineDefaults(dsl map[string]interface{}) map[string]interface{} continue } name, _ := obj["component_name"].(string) - if name == "" || name == "File" { + if name == "" || name == ComponentNameFile { continue } params, _ := obj["params"].(map[string]interface{}) diff --git a/internal/dao/ingestion_parser_config_migration.go b/internal/dao/ingestion_parser_config_migration.go index 0630165877..b1c781ea74 100644 --- a/internal/dao/ingestion_parser_config_migration.go +++ b/internal/dao/ingestion_parser_config_migration.go @@ -1164,12 +1164,9 @@ func migrateExtractorConfig(config map[string]any, parserID string, pipelineID * targets = append(targets, newNode) } } else { - for id, raw := range config { - id = strings.ToLower(id) - if strings.HasPrefix(id, "extractor:") || strings.HasPrefix(id, "extractor_") { - if node, ok := raw.(map[string]any); ok { - targets = append(targets, node) - } + for _, id := range common.SortedExtractorKeys(config) { + if node, ok := config[id].(map[string]any); ok { + targets = append(targets, node) } } if len(targets) == 0 { diff --git a/internal/ingestion/component/chunker/general.go b/internal/ingestion/component/chunker/general.go index d871355d56..cbb26b92db 100644 --- a/internal/ingestion/component/chunker/general.go +++ b/internal/ingestion/component/chunker/general.go @@ -44,8 +44,6 @@ import ( "ragflow/internal/parser/tableutil" ) -const ComponentNameGeneralChunker = "GeneralChunker" - var contextSentencePattern = regexp.MustCompile(`([。!??;!\n]|\. )`) type generalChunkerParam struct { diff --git a/internal/ingestion/component/chunker/group.go b/internal/ingestion/component/chunker/group.go index 38c8e7d238..6cd1cdd116 100644 --- a/internal/ingestion/component/chunker/group.go +++ b/internal/ingestion/component/chunker/group.go @@ -51,8 +51,6 @@ import ( "ragflow/internal/tokenizer" ) -const ComponentNameGroupTitleChunker = "GroupTitleChunker" - // MIN_GROUP_TOKENS / MAX_GROUP_TOKENS mirror the python constants in // group_chunker.py:22-23. They drive the merge heuristic for adjacent // text records. diff --git a/internal/ingestion/component/chunker/hierarchy.go b/internal/ingestion/component/chunker/hierarchy.go index 63e3b9d009..b59134c17d 100644 --- a/internal/ingestion/component/chunker/hierarchy.go +++ b/internal/ingestion/component/chunker/hierarchy.go @@ -51,8 +51,6 @@ import ( "ragflow/internal/ingestion/component/globals" ) -const ComponentNameHierarchyTitleChunker = "HierarchyTitleChunker" - // chunkNode mirrors python `_ChunkNode` (hierarchy_chunker.py:22-32). type chunkNode struct { level int diff --git a/internal/ingestion/component/chunker/manual.go b/internal/ingestion/component/chunker/manual.go index bfdf4d8a69..563390b8dd 100644 --- a/internal/ingestion/component/chunker/manual.go +++ b/internal/ingestion/component/chunker/manual.go @@ -33,8 +33,6 @@ import ( "gorm.io/gorm" ) -const ComponentNameManualChunker = "ManualChunker" - // ManualChunkerComponent is the standalone manual-layout chunker. It shares // the GroupTitleChunker grouping body (chunkFromRecords) and only differs by // the optional physical-position pre-sort. diff --git a/internal/ingestion/component/chunker/names.go b/internal/ingestion/component/chunker/names.go new file mode 100644 index 0000000000..fdb5544cd0 --- /dev/null +++ b/internal/ingestion/component/chunker/names.go @@ -0,0 +1,33 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package chunker + +import "ragflow/internal/common" + +// Chunker component names re-exported from the canonical component registry. +const ( + ComponentNameGeneralChunker = common.ComponentNameGeneralChunker + ComponentNameTokenChunker = common.ComponentNameTokenChunker + ComponentNameTitleChunker = common.ComponentNameTitleChunker + ComponentNameManualChunker = common.ComponentNameManualChunker + ComponentNameOneChunker = common.ComponentNameOneChunker + ComponentNamePageChunker = common.ComponentNamePageChunker + ComponentNameQAChunker = common.ComponentNameQAChunker + ComponentNameTableChunker = common.ComponentNameTableChunker + ComponentNameGroupTitleChunker = common.ComponentNameGroupTitleChunker + ComponentNameHierarchyTitleChunker = common.ComponentNameHierarchyTitleChunker +) diff --git a/internal/ingestion/component/chunker/one.go b/internal/ingestion/component/chunker/one.go index 1eeb11eed7..b569811480 100644 --- a/internal/ingestion/component/chunker/one.go +++ b/internal/ingestion/component/chunker/one.go @@ -38,8 +38,6 @@ import ( "gorm.io/gorm" ) -const ComponentNameOneChunker = "OneChunker" - type oneChunkerParam struct{} func (p *oneChunkerParam) Update(conf map[string]any) {} diff --git a/internal/ingestion/component/chunker/page.go b/internal/ingestion/component/chunker/page.go index f0428c38f4..deb6bc81f3 100644 --- a/internal/ingestion/component/chunker/page.go +++ b/internal/ingestion/component/chunker/page.go @@ -34,8 +34,6 @@ import ( "gorm.io/gorm" ) -const ComponentNamePageChunker = "PageChunker" - type pageChunkerParam struct{} func (p *pageChunkerParam) Update(conf map[string]any) {} diff --git a/internal/ingestion/component/chunker/qa.go b/internal/ingestion/component/chunker/qa.go index 520607e58d..6e9dbc7254 100644 --- a/internal/ingestion/component/chunker/qa.go +++ b/internal/ingestion/component/chunker/qa.go @@ -48,8 +48,6 @@ import ( "ragflow/internal/tokenizer" ) -const ComponentNameQAChunker = "QAChunker" - type qaChunkerParam struct { Lang string `json:"lang,omitempty"` } diff --git a/internal/ingestion/component/chunker/table.go b/internal/ingestion/component/chunker/table.go index c662b275af..6e451ce7b5 100644 --- a/internal/ingestion/component/chunker/table.go +++ b/internal/ingestion/component/chunker/table.go @@ -41,8 +41,6 @@ import ( "gorm.io/gorm" ) -const ComponentNameTableChunker = "TableChunker" - type tableChunkerParam struct{} func (p *tableChunkerParam) Update(conf map[string]any) {} diff --git a/internal/ingestion/component/chunker/title.go b/internal/ingestion/component/chunker/title.go index ad658cbe5e..629d44e598 100644 --- a/internal/ingestion/component/chunker/title.go +++ b/internal/ingestion/component/chunker/title.go @@ -82,8 +82,6 @@ import ( "gorm.io/gorm" ) -const ComponentNameTitleChunker = "TitleChunker" - type titleChunkerParam struct { schema.TitleChunkerParam } diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.go index 7989563828..f5793a56e0 100644 --- a/internal/ingestion/component/chunker/token.go +++ b/internal/ingestion/component/chunker/token.go @@ -90,8 +90,6 @@ import ( "ragflow/internal/parser/chunk" ) -const ComponentNameTokenChunker = "TokenChunker" - type tokenChunkerParam struct { schema.TokenChunkerParam } diff --git a/internal/ingestion/component/extractor.go b/internal/ingestion/component/extractor.go index 99db93bfe9..2e049e027d 100644 --- a/internal/ingestion/component/extractor.go +++ b/internal/ingestion/component/extractor.go @@ -66,8 +66,8 @@ import ( ) // componentNameExtractor is the canonical Extractor component name; the value -// lives in the schema package so the pipeline layer can reference the same -// identity. +// lives in common and is re-exported by the schema package so both layers +// reference the same identity. const componentNameExtractor = schema.ComponentNameExtractor // defaultExtractorChatTimeout bounds one Extractor LLM call, including diff --git a/internal/ingestion/component/file.go b/internal/ingestion/component/file.go index bbadda72a8..ebd98e0d69 100644 --- a/internal/ingestion/component/file.go +++ b/internal/ingestion/component/file.go @@ -50,7 +50,7 @@ import ( "gorm.io/gorm" ) -const ComponentNameFile = "File" +const ComponentNameFile = schema.ComponentNameFile // FileComponent resolves document/file metadata and forwards enough // identity for downstream Parser to fetch bytes. diff --git a/internal/ingestion/component/knowledge_compiler/component.go b/internal/ingestion/component/knowledge_compiler/component.go index 1fab7e5d2d..5740c14877 100644 --- a/internal/ingestion/component/knowledge_compiler/component.go +++ b/internal/ingestion/component/knowledge_compiler/component.go @@ -47,8 +47,8 @@ var chunkerOutputs = map[string]string{ // (rag/flow/compiler/compiler.py registers component_name = "Compiler"), so a // canvas saved by the Python frontend and Go's built-in ingestion templates // both reference the node as "Compiler" and resolve to the same component. -// The value itself lives in the schema package so the pipeline layer can -// reference the same identity. +// The value itself lives in common and is re-exported by the schema package so +// the pipeline layer references the same identity. const componentNameCompiler = schema.ComponentNameCompiler // KnowledgeCompilerComponent is the runtime.Component surface. Param is set at diff --git a/internal/ingestion/component/parser.go b/internal/ingestion/component/parser.go index 71cd9ca5b1..0e27602f67 100644 --- a/internal/ingestion/component/parser.go +++ b/internal/ingestion/component/parser.go @@ -96,7 +96,8 @@ import ( ) // ComponentNameParser is the canonical Parser component name; the value lives -// in the schema package so the pipeline layer can reference the same identity. +// in common and is re-exported by the schema package so both layers reference +// the same identity. const ComponentNameParser = schema.ComponentNameParser // pageFormFeed is the byte that text-page mode treats as a diff --git a/internal/ingestion/component/schema/component_names.go b/internal/ingestion/component/schema/component_names.go index 631d78dd02..809c41e82b 100644 --- a/internal/ingestion/component/schema/component_names.go +++ b/internal/ingestion/component/schema/component_names.go @@ -16,16 +16,19 @@ package schema -// Canonical ingestion component names. These are the identities components -// register under in runtime.DefaultRegistry and the values a DSL -// component_name carries. Layers that must recognise a component by name -// across the component/pipeline boundary (dispatch, param whitelists, -// operator-facing failure messages) reference these constants instead of -// repeating the literals; the schema package is the shared dependency both -// sides already import. More names can join this set as other layers need -// them. +import "ragflow/internal/common" + +// Canonical ingestion component names re-exported from common for callers that +// need them across the pipeline/component boundary. const ( - ComponentNameParser = "Parser" - ComponentNameExtractor = "Extractor" - ComponentNameCompiler = "Compiler" + ComponentNameFile = common.ComponentNameFile + ComponentNameParser = common.ComponentNameParser + ComponentNameExtractor = common.ComponentNameExtractor + ComponentNameCompiler = common.ComponentNameCompiler + ComponentNameTokenizer = common.ComponentNameTokenizer + + ComponentNameGeneralChunker = common.ComponentNameGeneralChunker + ComponentNameTokenChunker = common.ComponentNameTokenChunker + ComponentNameTitleChunker = common.ComponentNameTitleChunker + ComponentNameManualChunker = common.ComponentNameManualChunker ) diff --git a/internal/ingestion/component/tag_query.go b/internal/ingestion/component/tag_query.go index 14737f2386..db7760d568 100644 --- a/internal/ingestion/component/tag_query.go +++ b/internal/ingestion/component/tag_query.go @@ -22,6 +22,7 @@ import ( "sort" "strings" + "ragflow/internal/common" "ragflow/internal/entity" "ragflow/internal/ingestion/component/schema" "ragflow/internal/tokenizer" @@ -87,20 +88,9 @@ func TagFileIDFromParserConfig(parserConfig map[string]any) string { if v := tagFileIDFromTagsMap(parserConfig["tags"]); v != "" { return v } - // Several Extractor nodes can each declare their own tags block, so the - // winner has to be picked in a stable order: Go randomizes map iteration, - // and an unstable pick would let the dataset's tag source change between - // process restarts (different chunks tagged, different retrieval - // features). Sorting makes it deterministic; "Extractor" sorts before - // "Extractor:" because it is a prefix, so the unscoped node wins. - keys := make([]string, 0, len(parserConfig)) - for key := range parserConfig { - if key == "Extractor" || strings.HasPrefix(key, "Extractor:") { - keys = append(keys, key) - } - } - sort.Strings(keys) - for _, key := range keys { + // Several Extractor nodes can each declare their own tags block. Pick the + // winner deterministically by sorted component ID. + for _, key := range common.SortedExtractorKeys(parserConfig) { extractorMap, ok := asStringAnyMap(parserConfig[key]) if !ok { continue diff --git a/internal/ingestion/component/tag_query_test.go b/internal/ingestion/component/tag_query_test.go index 1fa2b4884e..902919ae4e 100644 --- a/internal/ingestion/component/tag_query_test.go +++ b/internal/ingestion/component/tag_query_test.go @@ -56,6 +56,33 @@ func TestTagFileIDFromParserConfig(t *testing.T) { }, want: "file-3", }, + { + name: "lowercase extractor form is not canonical", + parserConfig: map[string]any{ + "extractor:AutoExtractDefault": map[string]any{ + "tags": map[string]any{"tag_file_id": "file-5"}, + }, + }, + want: "", + }, + { + name: "bare extractor form is not scoped", + parserConfig: map[string]any{ + "Extractor": map[string]any{ + "tags": map[string]any{"tag_file_id": "file-6"}, + }, + }, + want: "", + }, + { + name: "underscore extractor form is not scoped", + parserConfig: map[string]any{ + "extractor_0": map[string]any{ + "tags": map[string]any{"tag_file_id": "file-7"}, + }, + }, + want: "", + }, { name: "entity.JSONMap nested value", parserConfig: map[string]any{ @@ -85,29 +112,17 @@ func TestTagFileIDFromParserConfig(t *testing.T) { func TestTagFileIDFromParserConfigPicksStableExtractorNode(t *testing.T) { // Several Extractor nodes can each declare their own tags block. Go - // randomizes map iteration, so an unsorted scan would return a different - // tag file - and thus tag the dataset from a different tag source - on - // every call and every process restart. - parserConfig := map[string]any{ - "Extractor": map[string]any{"tags": map[string]any{"tag_file_id": "file-unscoped"}}, - } + // randomizes map iteration, so the chosen tag source must not depend on + // iteration order. + parserConfig := map[string]any{} for i := 0; i < 8; i++ { parserConfig[fmt.Sprintf("Extractor:node-%d", i)] = map[string]any{ "tags": map[string]any{"tag_file_id": fmt.Sprintf("file-%d", i)}, } } - // The unscoped "Extractor" node wins: it is a prefix of the scoped keys, - // so it sorts first. - for i := 0; i < 200; i++ { - if got := TagFileIDFromParserConfig(parserConfig); got != "file-unscoped" { - t.Fatalf("TagFileIDFromParserConfig with unscoped node = %q, want %q", got, "file-unscoped") - } - } - // Without it the lowest-sorting scoped node wins, still stably. - delete(parserConfig, "Extractor") for i := 0; i < 200; i++ { if got := TagFileIDFromParserConfig(parserConfig); got != "file-0" { - t.Fatalf("TagFileIDFromParserConfig across scoped nodes = %q, want %q", got, "file-0") + t.Fatalf("TagFileIDFromParserConfig = %q, want %q", got, "file-0") } } } diff --git a/internal/ingestion/component/tokenizer.go b/internal/ingestion/component/tokenizer.go index bf3c5ba428..b14345754b 100644 --- a/internal/ingestion/component/tokenizer.go +++ b/internal/ingestion/component/tokenizer.go @@ -98,7 +98,7 @@ import ( "ragflow/internal/tokenizer" ) -const ComponentNameTokenizer = "Tokenizer" +const ComponentNameTokenizer = schema.ComponentNameTokenizer // titleExtRE strips a trailing file-extension (e.g. ".pdf") from the // upstream document name before tokenizing it. Mirrors the python diff --git a/internal/ingestion/pipeline/component_params.go b/internal/ingestion/pipeline/component_params.go index a06fab7d89..9f51a3e769 100644 --- a/internal/ingestion/pipeline/component_params.go +++ b/internal/ingestion/pipeline/component_params.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" + "ragflow/internal/common" "ragflow/internal/ingestion/component/schema" ) @@ -42,7 +43,10 @@ func ExtractAllComponentParams(dslJSON []byte) ([]ComponentParamsSchema, error) extractorCount := 0 for cpnID, comp := range dsl.Components { - if isExtractorComponent(cpnID, comp.Obj.ComponentName) { + if cpnID != common.ComponentNameFile && !common.IsComponentKey(cpnID) { + return nil, fmt.Errorf("ExtractAllComponentParams: component ID %q must be :; re-save the pipeline with canonical component IDs", cpnID) + } + if common.IsExtractorComponent(cpnID, comp.Obj.ComponentName) { extractorCount++ } } @@ -62,7 +66,7 @@ func ExtractAllComponentParams(dslJSON []byte) ([]ComponentParamsSchema, error) // Canvases saved by the Python-era frontend nest the Parser's // per-family setups under a "setups" key; lift them so the extracted // defaults (and the parser_config built from them) stay flat. - if isParserComponent(cpnID, comp.Obj.ComponentName) { + if common.IsParserComponent(cpnID, comp.Obj.ComponentName) { params = schema.FlattenLegacyParserSetups(params) } out = append(out, ComponentParamsSchema{ diff --git a/internal/ingestion/pipeline/component_params_test.go b/internal/ingestion/pipeline/component_params_test.go index 58c754ebdb..103239958e 100644 --- a/internal/ingestion/pipeline/component_params_test.go +++ b/internal/ingestion/pipeline/component_params_test.go @@ -196,6 +196,20 @@ func TestExtractAllComponentParams_MissingComponents(t *testing.T) { } } +func TestExtractAllComponentParams_RejectsUnscopedComponentIDs(t *testing.T) { + for _, id := range []string{"Extractor", "extractor_0", "parser_0"} { + dslJSON := []byte(`{"components":{"` + id + `":{"obj":{"component_name":"Extractor","params":{}}}}}`) + if _, err := ExtractAllComponentParams(dslJSON); err == nil { + t.Errorf("ExtractAllComponentParams(%q) expected error for unscoped component ID", id) + } + } + // File is the one bare component ID used by built-in ingestion templates. + dslJSON := []byte(`{"components":{"File":{"obj":{"component_name":"File","params":{}}}}}`) + if _, err := ExtractAllComponentParams(dslJSON); err != nil { + t.Fatalf("ExtractAllComponentParams(File): %v", err) + } +} + func componentIDs(schemas []ComponentParamsSchema) []string { ids := make([]string, len(schemas)) for i, s := range schemas { diff --git a/internal/ingestion/pipeline/pipeline.go b/internal/ingestion/pipeline/pipeline.go index 12af493f2e..9ba5d63f08 100644 --- a/internal/ingestion/pipeline/pipeline.go +++ b/internal/ingestion/pipeline/pipeline.go @@ -149,7 +149,7 @@ func ValidatePipeline(cnv *canvas.Canvas) error { } extractorCount := 0 for id, comp := range cnv.Components { - if isExtractorComponent(id, comp.Obj.ComponentName) { + if common.IsExtractorComponent(id, comp.Obj.ComponentName) { extractorCount++ } } diff --git a/internal/ingestion/pipeline/pipeline_params.go b/internal/ingestion/pipeline/pipeline_params.go index 000d44016e..9570e8fffa 100644 --- a/internal/ingestion/pipeline/pipeline_params.go +++ b/internal/ingestion/pipeline/pipeline_params.go @@ -83,14 +83,14 @@ var titleFamilyParamKeys = unionParamKeys( ) var componentParamSchemaKeys = map[string]map[string]struct{}{ - "titlechunker": titleFamilyParamKeys, + strings.ToLower(schema.ComponentNameTitleChunker): titleFamilyParamKeys, // ManualChunker reuses TitleChunkerParam. It pins method=group and is exempt // from the token cap, so method, chunk_token_cap and include_heading_content // (hierarchy-only) are accepted here but ignored by the component — the // operator form omits them for the same reason. - "manualchunker": titleFamilyParamKeys, - "tokenchunker": extractJSONTags(schema.TokenChunkerParam{}), - "tokenizer": extractJSONTags(schema.TokenizerParam{}), + strings.ToLower(schema.ComponentNameManualChunker): titleFamilyParamKeys, + strings.ToLower(schema.ComponentNameTokenChunker): extractJSONTags(schema.TokenChunkerParam{}), + strings.ToLower(schema.ComponentNameTokenizer): extractJSONTags(schema.TokenizerParam{}), } // extractJSONTags returns all top-level json tag names from a struct. @@ -119,38 +119,6 @@ func extractJSONTags(v any) map[string]struct{} { return tags } -// isExtractorComponent returns true if the component name or component ID -// indicates an Extractor component. -func isExtractorComponent(cpnID, componentName string) bool { - lowerName := strings.ToLower(componentName) - lowerID := strings.ToLower(cpnID) - return lowerName == strings.ToLower(schema.ComponentNameExtractor) || - strings.HasPrefix(lowerID, "extractor:") || - strings.HasPrefix(lowerID, "extractor_") -} - -// isCompilerComponent returns true if the component ID indicates a Compiler -// component, whose LLM-backed runtime parameters follow the same surface as -// the Extractor's. -func isCompilerComponent(cpnID string) bool { - lowerID := strings.ToLower(cpnID) - return strings.HasPrefix(lowerID, "compiler:") || - strings.HasPrefix(lowerID, "compiler_") -} - -// IsParserComponentID reports whether a component ID names a Parser component. -// A parser_config map is keyed by component ID, where the component name is not -// available, so this is the config-side half of isParserComponent. -func IsParserComponentID(cpnID string) bool { - return strings.HasPrefix(strings.ToLower(cpnID), "parser:") -} - -// isParserComponent returns true if the component name or component ID -// indicates a Parser component. -func isParserComponent(cpnID, componentName string) bool { - return strings.EqualFold(componentName, schema.ComponentNameParser) || IsParserComponentID(cpnID) -} - // NormalizeParserConfigSetups lifts legacy nested Parser "setups" entries in // a parser_config-shaped map ({cpnID: params}) into the flat per-family // shape. The runtime merges such a map over the DSL-baked component params @@ -163,7 +131,7 @@ func NormalizeParserConfigSetups(parserConfig map[string]any) map[string]any { return nil } for cpnID, raw := range parserConfig { - if !isParserComponent(cpnID, "") { + if !common.IsParserComponentID(cpnID) { continue } params, ok := raw.(map[string]any) @@ -177,22 +145,19 @@ func NormalizeParserConfigSetups(parserConfig map[string]any) map[string]any { // getComponentParamWhitelist returns the dynamic parameter whitelist for a component type. func getComponentParamWhitelist(cpnID string) (map[string]struct{}, bool) { - if isExtractorComponent(cpnID, "") { + if common.IsExtractorComponentID(cpnID) { return extractorValidParamKeys, true } - if isCompilerComponent(cpnID) { + if common.IsCompilerComponentID(cpnID) { return llmRuntimeParamKeys, true } return nil, false } -// cleanComponentParams filters rawConfig against the extracted DSL schemas. -// Keys containing ':' are treated as component IDs; they are kept only when both -// the cpnID AND the param name exist in one of the accepted-key sources: the DSL -// schema (the params the template bakes), the schema-derived per-component table -// (componentParamSchemaKeys), or the component's dynamic parameter schema (e.g. -// Extractor modular features). Keys without ':' (legacy flat fields) are dropped -// with a warning. +// cleanComponentParams keeps only component-scoped keys recognized by +// common.IsComponentKey, and only params accepted by the DSL schema, the +// per-component table, or the component's dynamic schema. Flat fields are +// dropped with a warning. func cleanComponentParams(schemas []ComponentParamsSchema, rawConfig map[string]interface{}) map[string]interface{} { validCPNs := make(map[string]map[string]struct{}, len(schemas)) componentNames := make(map[string]string, len(schemas)) @@ -201,10 +166,10 @@ func cleanComponentParams(schemas []ComponentParamsSchema, rawConfig map[string] for k := range s.ParamsDefaults { keys[k] = struct{}{} } - if s.ComponentName == "GeneralChunker" { + if s.ComponentName == schema.ComponentNameGeneralChunker { keys["delimiters"] = struct{}{} } - if isParserComponent(s.CpnID, s.ComponentName) { + if common.IsParserComponent(s.CpnID, s.ComponentName) { keys["enable_vision_enhancement"] = struct{}{} keys["vlm"] = struct{}{} } @@ -225,7 +190,7 @@ func cleanComponentParams(schemas []ComponentParamsSchema, rawConfig map[string] result := make(map[string]interface{}, len(rawConfig)) for key, val := range rawConfig { - if !strings.Contains(key, ":") { + if !common.IsComponentKey(key) { common.Warn("cleanComponentParams: dropping legacy flat field", zap.String("key", key)) continue @@ -240,18 +205,18 @@ func cleanComponentParams(schemas []ComponentParamsSchema, rawConfig map[string] if !ok { continue } - if componentNames[key] == "GeneralChunker" { + if componentNames[key] == schema.ComponentNameGeneralChunker { params = normalizeGeneralComponentParams(params) } // Legacy parser_config rows (or canvases) saved by the Python-era // frontend nest the Parser's setups under a "setups" key; lift them // before filtering, otherwise the families are dropped as unknown // params against the flat DSL schema keys. - if isParserComponent(key, componentNames[key]) { + if common.IsParserComponent(key, componentNames[key]) { params = schema.FlattenLegacyParserSetups(params) } dynamicWhitelist, hasDynamic := getComponentParamWhitelist(key) - if isExtractorComponent(key, "") { + if common.IsExtractorComponentID(key) { params = NormalizeExtractorParams(params) } cleaned := make(map[string]any, len(params)) @@ -430,5 +395,6 @@ func ApplyParentChildChunkerConfig(componentConfig entity.JSONMap, rawConfig map // chunks into children. func IsChunkerComponent(componentID string) bool { lowerID := strings.ToLower(componentID) - return strings.HasPrefix(lowerID, "generalchunker:") || strings.HasPrefix(lowerID, "tokenchunker:") + return strings.HasPrefix(lowerID, strings.ToLower(schema.ComponentNameGeneralChunker)+":") || + strings.HasPrefix(lowerID, strings.ToLower(schema.ComponentNameTokenChunker)+":") } diff --git a/internal/ingestion/pipeline/pipeline_params_test.go b/internal/ingestion/pipeline/pipeline_params_test.go index 533bc7468c..376595d99c 100644 --- a/internal/ingestion/pipeline/pipeline_params_test.go +++ b/internal/ingestion/pipeline/pipeline_params_test.go @@ -34,9 +34,9 @@ func generalDSL(t *testing.T) []byte { }, }, }, - "Chunker:LegalReadersDecide": map[string]any{ + "GeneralChunker:LegalReadersDecide": map[string]any{ "obj": map[string]any{ - "component_name": "Chunker", + "component_name": "GeneralChunker", "params": map[string]any{ "outputs": map[string]any{}, "chunk_size": float64(512), @@ -165,9 +165,9 @@ func TestBuildParserConfig_ShallowMerge_NestedParam(t *testing.T) { // A component with a nested-map param "chunk" that has sub-keys. dsl := map[string]any{ "components": map[string]any{ - "Chunker:xyz": map[string]any{ + "GeneralChunker:xyz": map[string]any{ "obj": map[string]any{ - "component_name": "Chunker", + "component_name": "GeneralChunker", "params": map[string]any{ "outputs": map[string]any{}, "chunk": map[string]any{ @@ -186,15 +186,15 @@ func TestBuildParserConfig_ShallowMerge_NestedParam(t *testing.T) { // User only overrides one sub-key of "chunk". overrides := map[string]any{ - "Chunker:xyz": map[string]any{ + "GeneralChunker:xyz": map[string]any{ "chunk": map[string]any{"size": float64(1024)}, }, } result := buildTestParserConfig(t, dslJSON, overrides) - chunker, ok := result["Chunker:xyz"].(map[string]any) + chunker, ok := result["GeneralChunker:xyz"].(map[string]any) if !ok { - t.Fatal("expected Chunker:xyz in result") + t.Fatal("expected GeneralChunker:xyz in result") } chunk, ok := chunker["chunk"].(map[string]any) if !ok { @@ -348,14 +348,14 @@ func TestNormalizeParserConfigSetups_EmptyNestedMap(t *testing.T) { func TestBuildParserConfig_ScalarOverridePreservesOtherDefaults(t *testing.T) { dslJSON := generalDSL(t) overrides := map[string]any{ - "Chunker:LegalReadersDecide": map[string]any{ + "GeneralChunker:LegalReadersDecide": map[string]any{ "chunk_size": float64(1024), }, } result := buildTestParserConfig(t, dslJSON, overrides) - chunker, ok := result["Chunker:LegalReadersDecide"].(map[string]any) + chunker, ok := result["GeneralChunker:LegalReadersDecide"].(map[string]any) if !ok { - t.Fatal("expected Chunker:LegalReadersDecide in result") + t.Fatal("expected GeneralChunker:LegalReadersDecide in result") } if chunker["chunk_size"] != float64(1024) { t.Errorf("expected chunk_size=1024, got %v", chunker["chunk_size"]) @@ -398,8 +398,8 @@ func TestBuildParserConfig_AllComponentsPresent(t *testing.T) { if _, ok := result["Parser:HipSignsRhyme"]; !ok { t.Error("expected Parser:HipSignsRhyme") } - if _, ok := result["Chunker:LegalReadersDecide"]; !ok { - t.Error("expected Chunker:LegalReadersDecide") + if _, ok := result["GeneralChunker:LegalReadersDecide"]; !ok { + t.Error("expected GeneralChunker:LegalReadersDecide") } } @@ -470,7 +470,7 @@ func TestComponentParamsDefaults_Basic(t *testing.T) { if _, ok := parser["pdf"]; !ok { t.Error("expected pdf to be present") } - chunker := result["Chunker:LegalReadersDecide"] + chunker := result["GeneralChunker:LegalReadersDecide"] if chunker["chunk_size"] != float64(512) { t.Errorf("expected chunk_size=512, got %v", chunker["chunk_size"]) } diff --git a/internal/ingestion/pipeline/template_output_format_test.go b/internal/ingestion/pipeline/template_output_format_test.go index fe14309cfb..68617b00d9 100644 --- a/internal/ingestion/pipeline/template_output_format_test.go +++ b/internal/ingestion/pipeline/template_output_format_test.go @@ -21,6 +21,8 @@ import ( "path/filepath" "strings" "testing" + + "ragflow/internal/common" ) // parserSetupParamsKeys are the Parser params keys that are NOT per-family @@ -108,7 +110,7 @@ func assertNoParserOutputFormatInGraph(t *testing.T, ref string, root map[string for _, rawNode := range nodes { node, _ := rawNode.(map[string]any) id, _ := node["id"].(string) - if !IsParserComponentID(id) { + if !common.IsParserComponentID(id) { continue } data, _ := node["data"].(map[string]any) diff --git a/internal/ingestion/task/indexdoc/process.go b/internal/ingestion/task/indexdoc/process.go index 2728bee4ed..636f3b671b 100644 --- a/internal/ingestion/task/indexdoc/process.go +++ b/internal/ingestion/task/indexdoc/process.go @@ -18,7 +18,6 @@ package indexdoc import ( "fmt" - "strings" "time" "ragflow/internal/common" @@ -376,7 +375,7 @@ func resolveTableColumnConfig(parserConfig map[string]interface{}) (mode string, return mode, roles, rawNames } for cid, raw := range parserConfig { - if !strings.HasPrefix(cid, "Parser:") { + if !common.IsParserComponentID(cid) { continue } comp, _ := raw.(map[string]interface{}) diff --git a/internal/service/component_scoped_parser_config.go b/internal/service/component_scoped_parser_config.go index c6d876ab01..004f78b1ce 100644 --- a/internal/service/component_scoped_parser_config.go +++ b/internal/service/component_scoped_parser_config.go @@ -3,6 +3,7 @@ package service import ( "strings" + "ragflow/internal/common" "ragflow/internal/entity" ) @@ -33,9 +34,8 @@ func ApplyComponentScopedParserConfig( if !ok { continue } - cpnLower := strings.ToLower(cpnID) switch { - case strings.HasPrefix(cpnLower, "extractor:") || strings.HasPrefix(cpnLower, "extractor_"): + case common.IsExtractorComponentID(cpnID): if value, _ := params["llm_id"].(string); strings.TrimSpace(value) == "" && strings.TrimSpace(llmID) != "" { params["llm_id"] = llmID } @@ -54,7 +54,7 @@ func ApplyComponentScopedParserConfig( "built_in_metadata": []any{}, } } - case strings.HasPrefix(cpnLower, "compiler:") || strings.HasPrefix(cpnLower, "compiler_"): + case common.IsCompilerComponentID(cpnID): if value, _ := params["llm_id"].(string); strings.TrimSpace(value) == "" && strings.TrimSpace(llmID) != "" { params["llm_id"] = llmID } diff --git a/internal/service/dataset/helpers.go b/internal/service/dataset/helpers.go index 58db80de31..7e739f71aa 100644 --- a/internal/service/dataset/helpers.go +++ b/internal/service/dataset/helpers.go @@ -8,6 +8,7 @@ import ( "strings" "unicode/utf8" + "ragflow/internal/common" "ragflow/internal/dao" "ragflow/internal/entity" pipelinepkg "ragflow/internal/ingestion/pipeline" @@ -202,23 +203,19 @@ func validateDatasetParserConfigSize(parserConfig map[string]interface{}) error return nil } -// DropUnscopedParserConfigKeys removes top-level parser_config keys that are -// not component-scoped (i.e. do not contain ":"), since the Go backend only -// consumes keys keyed by a component id (e.g. "Parser:abc123" or -// "Extractor:AutoExtractDefault"). Flat keys are legacy transport artifacts -// that downstream consumers (BuildParserConfig) never read, so they are -// dropped rather than rejected. Only top-level keys are affected; a key nested -// inside a component node (e.g. "Extractor:AutoExtractDefault.metadata") keeps -// its existing name. It mutates the provided map in place and returns the names -// of the dropped keys so callers can log the silent drop. A nil or empty map is -// returned untouched with a nil result. +// DropUnscopedParserConfigKeys removes top-level parser_config keys that +// common.IsComponentKey does not recognize as component-scoped. Flat keys are +// transport artifacts that BuildParserConfig never reads, so they are dropped +// rather than rejected. Only top-level keys are affected; nested keys keep +// their names. It mutates parserConfig and returns the dropped names so callers +// can log them. A nil or empty map is returned untouched. func DropUnscopedParserConfigKeys(parserConfig map[string]any) []string { if len(parserConfig) == 0 { return nil } var dropped []string for key := range parserConfig { - if !strings.Contains(key, ":") { + if !common.IsComponentKey(key) { dropped = append(dropped, key) } } @@ -369,10 +366,10 @@ func preserveDatasetParserConfigState(next, existing entity.JSONMap, incoming ma } var mm map[string]any if incoming != nil { - mm = extractorNodeMetadata(incoming) + mm = extractorNodeMetadata(incoming, next) } if mm == nil && existing != nil { - mm = extractorNodeMetadata(existing) + mm = extractorNodeMetadata(existing, next) } if mm != nil { next["metadata"] = mm @@ -447,18 +444,19 @@ func preserveDatasetParserConfigState(next, existing entity.JSONMap, incoming ma } // extractorNodeMetadata returns the modular metadata object ({enabled, metadata, -// built_in_metadata}) from the first Extractor node in a parser_config, or nil -// if none is present. Metadata is component-scoped under Extractor nodes; this -// helper reads it back for preservation across partial updates. -func extractorNodeMetadata(parserConfig map[string]interface{}) map[string]any { +// built_in_metadata}) from the first Extractor node present in validConfig, or +// nil if none is present. validConfig is the rebuilt parser_config for the +// current DSL; keys outside it are ignored so stale nodes cannot override the +// active configuration. +func extractorNodeMetadata(parserConfig, validConfig map[string]interface{}) map[string]any { if parserConfig == nil { return nil } - for cpnID, raw := range parserConfig { - lower := strings.ToLower(cpnID) - if !strings.HasPrefix(lower, "extractor:") && !strings.HasPrefix(lower, "extractor_") { + for _, cpnID := range common.SortedExtractorKeys(parserConfig) { + if _, ok := validConfig[cpnID]; !ok { continue } + raw := parserConfig[cpnID] params, ok := raw.(map[string]any) if !ok { continue diff --git a/internal/service/dataset/helpers_test.go b/internal/service/dataset/helpers_test.go index a52d810ac2..8ec5f364d1 100644 --- a/internal/service/dataset/helpers_test.go +++ b/internal/service/dataset/helpers_test.go @@ -227,6 +227,9 @@ func TestDropUnscopedParserConfigKeys_NilAndEmpty(t *testing.T) { func TestDropUnscopedParserConfigKeys_DropsFlatKeys(t *testing.T) { cfg := map[string]any{ "File": map[string]any{}, + "Extractor": map[string]any{}, + "extractor_0": map[string]any{}, + "compiler_0": map[string]any{}, "chunk_token_num": float64(128), "delimiter": "\n", "metadata": map[string]any{}, @@ -236,7 +239,7 @@ func TestDropUnscopedParserConfigKeys_DropsFlatKeys(t *testing.T) { } dropped := DropUnscopedParserConfigKeys(cfg) sort.Strings(dropped) - want := []string{"File", "chunk_token_num", "delimiter", "metadata", "parent_child"} + want := []string{"Extractor", "File", "chunk_token_num", "compiler_0", "delimiter", "extractor_0", "metadata", "parent_child"} if !reflect.DeepEqual(dropped, want) { t.Fatalf("dropped = %#v, want %#v", dropped, want) } @@ -510,7 +513,9 @@ func TestPreserveDatasetParserConfigState_FallsBackWhenIncomingNotMap(t *testing "metadata": incomingMetadata, }, } - got := preserveDatasetParserConfigState(entity.JSONMap{}, existing, incoming) + got := preserveDatasetParserConfigState(entity.JSONMap{ + "Extractor:AutoExtractDefault": map[string]any{}, + }, existing, incoming) meta, ok := got["metadata"].(map[string]any) if !ok { t.Fatalf("expected existing modular metadata preserved, got %#v", got["metadata"]) @@ -523,6 +528,38 @@ func TestPreserveDatasetParserConfigState_FallsBackWhenIncomingNotMap(t *testing } } +func TestPreserveDatasetParserConfigState_IgnoresStaleExtractor(t *testing.T) { + existing := entity.JSONMap{ + "Extractor:AutoExtractDefault": map[string]any{ + "metadata": map[string]any{ + "enabled": true, + "metadata": []any{map[string]any{"key": "current_field", "type": "string"}}, + "built_in_metadata": []any{}, + }, + }, + } + incoming := map[string]interface{}{ + "Extractor:Old": map[string]any{ + "metadata": map[string]any{ + "enabled": true, + "metadata": []any{map[string]any{"key": "stale_field", "type": "string"}}, + "built_in_metadata": []any{}, + }, + }, + } + got := preserveDatasetParserConfigState(entity.JSONMap{ + "Extractor:AutoExtractDefault": map[string]any{}, + }, existing, incoming) + meta, ok := got["metadata"].(map[string]any) + if !ok { + t.Fatalf("expected current modular metadata, got %#v", got["metadata"]) + } + fields, ok := meta["metadata"].([]any) + if !ok || len(fields) != 1 || fields[0].(map[string]any)["key"] != "current_field" { + t.Fatalf("expected current_field to win over stale Extractor, got %#v", meta["metadata"]) + } +} + func TestPreserveDatasetParserConfigState_UsesValidIncomingMap(t *testing.T) { existing := entity.JSONMap{ "Extractor:AutoExtractDefault": map[string]any{ @@ -542,7 +579,9 @@ func TestPreserveDatasetParserConfigState_UsesValidIncomingMap(t *testing.T) { }, }, } - got := preserveDatasetParserConfigState(entity.JSONMap{}, existing, incoming) + got := preserveDatasetParserConfigState(entity.JSONMap{ + "Extractor:AutoExtractDefault": map[string]any{}, + }, existing, incoming) meta, ok := got["metadata"].(map[string]any) if !ok { t.Fatalf("expected modular metadata map, got %#v", got["metadata"]) diff --git a/internal/service/dataset/metadata.go b/internal/service/dataset/metadata.go index 33e5e98ace..6d90140cb8 100644 --- a/internal/service/dataset/metadata.go +++ b/internal/service/dataset/metadata.go @@ -192,11 +192,8 @@ func modularMetadataConfig(parserConfig map[string]any) (bool, bool, []any, []an builtIn := anyOrEmptyList(metaObj["built_in_metadata"]) return true, enabled, metadata, builtIn } - for cpnID, raw := range parserConfig { - lower := strings.ToLower(cpnID) - if !strings.HasPrefix(lower, "extractor:") && !strings.HasPrefix(lower, "extractor_") { - continue - } + for _, cpnID := range common.SortedExtractorKeys(parserConfig) { + raw := parserConfig[cpnID] params, ok := raw.(map[string]any) if !ok { continue diff --git a/internal/service/dataset/parser_config_roundtrip_test.go b/internal/service/dataset/parser_config_roundtrip_test.go new file mode 100644 index 0000000000..48e36eb82d --- /dev/null +++ b/internal/service/dataset/parser_config_roundtrip_test.go @@ -0,0 +1,53 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package dataset + +import ( + "testing" + + "ragflow/internal/common" + "ragflow/internal/ingestion/component" + pipelinepkg "ragflow/internal/ingestion/pipeline" +) + +func TestParserConfigCanonicalExtractorRoundTrip(t *testing.T) { + dsl := []byte(`{"components":{"Extractor:AutoExtractDefault":{"obj":{"component_name":"Extractor","params":{"metadata":{"enabled":true,"metadata":[],"built_in_metadata":[]},"tags":{"top_n":5,"tag_file_id":""}}}}}}`) + raw := map[string]any{ + "chunk_token_num": float64(128), + "Extractor:AutoExtractDefault": map[string]any{ + "tags": map[string]any{"top_n": 5, "tag_file_id": "file-round-trip"}, + }, + } + if dropped, err := ValidateParserConfig(raw); err != nil { + t.Fatalf("ValidateParserConfig: %v", err) + } else if len(dropped) != 1 || dropped[0] != "chunk_token_num" { + t.Fatalf("dropped = %#v, want [chunk_token_num]", dropped) + } + + config, err := pipelinepkg.BuildParserConfig(dsl, raw) + if err != nil { + t.Fatalf("BuildParserConfig: %v", err) + } + if got := component.TagFileIDFromParserConfig(map[string]any(config)); got != "file-round-trip" { + t.Fatalf("TagFileIDFromParserConfig = %q, want file-round-trip", got) + } + if meta, ok := common.ExtractorMetadataConfig(map[string]any(config)); !ok { + t.Fatal("ExtractorMetadataConfig did not find the canonical Extractor node") + } else if enabled, _ := meta["enabled"].(bool); !enabled { + t.Fatalf("metadata enabled = %v, want true", meta["enabled"]) + } +} diff --git a/internal/service/dataset/update_test.go b/internal/service/dataset/update_test.go index 064d289b44..7bf5a11718 100644 --- a/internal/service/dataset/update_test.go +++ b/internal/service/dataset/update_test.go @@ -1356,8 +1356,7 @@ func TestUpdateDatasetUsesTenantModelIDForExtractorDefault(t *testing.T) { func extractorNodeMetadataInTest(t *testing.T, cfg map[string]interface{}) map[string]interface{} { t.Helper() for cpnID, raw := range cfg { - lower := strings.ToLower(cpnID) - if !strings.HasPrefix(lower, "extractor:") && !strings.HasPrefix(lower, "extractor_") { + if !common.IsExtractorComponentID(cpnID) { continue } params, ok := raw.(map[string]interface{}) diff --git a/internal/service/document/document_applykb_chain_test.go b/internal/service/document/document_applykb_chain_test.go index 3dc6027ac0..88a5579804 100644 --- a/internal/service/document/document_applykb_chain_test.go +++ b/internal/service/document/document_applykb_chain_test.go @@ -15,7 +15,6 @@ package document import ( "encoding/json" "sort" - "strings" "testing" "gorm.io/gorm" @@ -42,8 +41,7 @@ func firstExtractorMetadata(t *testing.T, parserConfig entity.JSONMap) map[strin t.Helper() var keys []string for k := range parserConfig { - lower := strings.ToLower(k) - if strings.HasPrefix(lower, "extractor:") || strings.HasPrefix(lower, "extractor_") { + if common.IsExtractorComponentID(k) { keys = append(keys, k) } } diff --git a/internal/service/document/document_parse.go b/internal/service/document/document_parse.go index 2e120ef3c6..8421e5aad3 100644 --- a/internal/service/document/document_parse.go +++ b/internal/service/document/document_parse.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "ragflow/internal/service" - "sort" "strconv" "strings" "sync" @@ -198,7 +197,7 @@ func (s *DocumentService) applyKnowledgebaseParserConfig(ctx context.Context, do if metaOK { merged["metadata"] = deepCopyValue(meta) } else { - for _, cpnID := range sortedExtractorKeys(merged) { + for _, cpnID := range common.SortedExtractorKeys(map[string]any(merged)) { params, ok := nodeParams(merged[cpnID]) if !ok { continue @@ -227,21 +226,6 @@ func (s *DocumentService) applyKnowledgebaseParserConfig(ctx context.Context, do return nil } -// sortedExtractorKeys returns the component IDs in parserConfig that name -// Extractor nodes, in sorted order so multi-Extractor configs resolve -// deterministically instead of depending on Go map iteration order. -func sortedExtractorKeys(parserConfig entity.JSONMap) []string { - var keys []string - for k := range parserConfig { - lower := strings.ToLower(k) - if strings.HasPrefix(lower, "extractor:") || strings.HasPrefix(lower, "extractor_") { - keys = append(keys, k) - } - } - sort.Strings(keys) - return keys -} - // nodeParams returns a writable view of a component node's params, accepting // both the decoded (map[string]any) and entity (entity.JSONMap) shapes. // The returned map shares storage with the input, so writes are visible. @@ -281,7 +265,7 @@ func resolveDatasetModularMetadata(parserConfig entity.JSONMap) (map[string]any, if mm, ok := parserConfig["metadata"].(entity.JSONMap); ok { return map[string]any(mm), true, true } - for _, cpnID := range sortedExtractorKeys(parserConfig) { + for _, cpnID := range common.SortedExtractorKeys(map[string]any(parserConfig)) { params, ok := nodeParams(parserConfig[cpnID]) if !ok { continue @@ -301,7 +285,7 @@ func resolveDatasetLLMID(parserConfig entity.JSONMap) string { if s, ok := parserConfig["llm_id"].(string); ok && strings.TrimSpace(s) != "" { return s } - for _, cpnID := range sortedExtractorKeys(parserConfig) { + for _, cpnID := range common.SortedExtractorKeys(map[string]any(parserConfig)) { params, ok := nodeParams(parserConfig[cpnID]) if !ok { continue diff --git a/internal/service/document/document_parser_selection.go b/internal/service/document/document_parser_selection.go index 8b8176dc58..18e69b6600 100644 --- a/internal/service/document/document_parser_selection.go +++ b/internal/service/document/document_parser_selection.go @@ -66,7 +66,7 @@ func parserParamsDeclaresFamily(params map[string]any, family string) bool { // config, or a malformed one): the config cannot answer then. func configDeclaredFamily(config entity.JSONMap, family string) (declared, found bool) { for cpnID, raw := range config { - if !pipelinepkg.IsParserComponentID(cpnID) { + if !common.IsParserComponentID(cpnID) { continue } params, ok := raw.(map[string]any) @@ -94,7 +94,7 @@ func templateDeclaredFamily(ctx context.Context, parserID, family string) (decla return false, false } for cpnID, params := range defaults { - if !pipelinepkg.IsParserComponentID(cpnID) { + if !common.IsParserComponentID(cpnID) { continue } return parserParamsDeclaresFamily(params, family), true diff --git a/internal/service/document/document_parser_selection_test.go b/internal/service/document/document_parser_selection_test.go index e89a0c6a06..e495a47a40 100644 --- a/internal/service/document/document_parser_selection_test.go +++ b/internal/service/document/document_parser_selection_test.go @@ -3,9 +3,9 @@ package document import ( "mime/multipart" "reflect" - "strings" "testing" + "ragflow/internal/common" "ragflow/internal/dao" "ragflow/internal/entity" "ragflow/internal/permission" @@ -763,7 +763,7 @@ func parserFamilySetup(t *testing.T, doc map[string]interface{}, family string) t.Fatalf("%v parser_config = %T, want map", doc["name"], doc["parser_config"]) } for cpnID, raw := range cfg { - if !strings.HasPrefix(cpnID, "Parser:") { + if !common.IsParserComponentID(cpnID) { continue } params, _ := raw.(map[string]interface{})