refactor(ingestion): centralize component names and identity checks (#22081)

This commit is contained in:
jay77721 authored and GitHub committed 2026-11-06 17:03:32 +08:00
1 parent fa813113e1
commit 3a6b6dca78
42 files changed
+629 -216

No files matched your search

+79
View File
@@ -0,0 +1,79 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package common
import (
"sort"
"strings"
)
// hasComponentIDPrefix reports whether id is "<componentName>:<non-empty instance>".
func hasComponentIDPrefix(id, componentName string) bool {
kind, instance, ok := strings.Cut(id, ":")
return ok && kind == componentName && instance != ""
}
// IsExtractorComponentID reports whether id names an Extractor node in a
// component-scoped parser_config or pipeline DSL. Only the canonical
// "Extractor:<instance>" form is accepted.
func IsExtractorComponentID(id string) bool {
return hasComponentIDPrefix(id, ComponentNameExtractor)
}
// IsExtractorComponent reports whether the component name or component ID
// identifies an Extractor.
func IsExtractorComponent(id, componentName string) bool {
return componentName == ComponentNameExtractor || IsExtractorComponentID(id)
}
// IsParserComponentID reports whether id names a Parser node in a
// component-scoped parser_config or pipeline DSL.
func IsParserComponentID(id string) bool {
return hasComponentIDPrefix(id, ComponentNameParser)
}
// IsParserComponent reports whether the component name or component ID
// identifies a Parser.
func IsParserComponent(id, componentName string) bool {
return componentName == ComponentNameParser || IsParserComponentID(id)
}
// IsCompilerComponentID reports whether id names a Compiler node in a
// component-scoped parser_config or pipeline DSL.
func IsCompilerComponentID(id string) bool {
return hasComponentIDPrefix(id, ComponentNameCompiler)
}
// IsComponentKey reports whether id is a canonical component-scoped
// parser_config key: "<ComponentName>:<non-empty instance>".
func IsComponentKey(id string) bool {
kind, instance, ok := strings.Cut(id, ":")
return ok && instance != "" && IsComponentName(kind)
}
// SortedExtractorKeys returns the parser_config keys that name Extractor nodes,
// sorted so callers resolve multiple Extractor nodes deterministically.
func SortedExtractorKeys(parserConfig map[string]any) []string {
keys := make([]string, 0)
for key := range parserConfig {
if IsExtractorComponentID(key) {
keys = append(keys, key)
}
}
sort.Strings(keys)
return keys
}
+210
View File
@@ -0,0 +1,210 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package common
import (
"reflect"
"testing"
)
func TestIsExtractorComponentID(t *testing.T) {
tests := []struct {
id string
want bool
}{
{id: "Extractor", want: false},
{id: "extractor", want: false},
{id: "Extractor:AutoExtractDefault", want: true},
{id: "extractor:node", want: false},
{id: "EXTRACTOR:Node", want: false},
{id: "Extractor:", want: false},
{id: "extractor_0", want: false},
{id: "Extractor_0", want: false},
{id: "extractors", want: false},
{id: "my_extractor", want: false},
{id: "Parser:Extractor", want: false},
{id: "", want: false},
}
for _, tt := range tests {
t.Run(tt.id, func(t *testing.T) {
if got := IsExtractorComponentID(tt.id); got != tt.want {
t.Fatalf("IsExtractorComponentID(%q) = %v, want %v", tt.id, got, tt.want)
}
})
}
}
func TestIsParserComponentID(t *testing.T) {
tests := []struct {
id string
want bool
}{
{id: "Parser:HipSignsRhyme", want: true},
{id: "Parser:", want: false},
{id: "parser:node", want: false},
{id: "PARSER:Node", want: false},
{id: "Parser", want: false},
{id: "parser_0", want: false},
{id: "Extractor:Parser", want: false},
{id: "", want: false},
}
for _, tt := range tests {
t.Run(tt.id, func(t *testing.T) {
if got := IsParserComponentID(tt.id); got != tt.want {
t.Fatalf("IsParserComponentID(%q) = %v, want %v", tt.id, got, tt.want)
}
})
}
}
func TestIsCompilerComponentID(t *testing.T) {
tests := []struct {
id string
want bool
}{
{id: "Compiler:CurvySunsLove", want: true},
{id: "Compiler:", want: false},
{id: "compiler:node", want: false},
{id: "COMPILER:Node", want: false},
{id: "compiler_0", want: false},
{id: "Compiler", want: false},
{id: "compilers", want: false},
{id: "Parser:Compiler", want: false},
{id: "", want: false},
}
for _, tt := range tests {
t.Run(tt.id, func(t *testing.T) {
if got := IsCompilerComponentID(tt.id); got != tt.want {
t.Fatalf("IsCompilerComponentID(%q) = %v, want %v", tt.id, got, tt.want)
}
})
}
}
func TestIsComponentKey(t *testing.T) {
tests := []struct {
id string
want bool
}{
{id: "Extractor:AutoExtractDefault", want: true},
{id: "Parser:HipSignsRhyme", want: true},
{id: "Parser:", want: false},
{id: "GeneralChunker:SixApplesFall", want: true},
{id: "File:node", want: true},
{id: "Extractor:", want: false},
{id: ":foo", want: false},
{id: "foo:bar", want: false},
{id: "extractor:0", want: false},
{id: "extractor_0", want: false},
{id: "compiler_0", want: false},
{id: "Extractor", want: false},
{id: "chunk_token_num", want: false},
{id: "parser_0", want: false},
{id: "", want: false},
}
for _, tt := range tests {
t.Run(tt.id, func(t *testing.T) {
if got := IsComponentKey(tt.id); got != tt.want {
t.Fatalf("IsComponentKey(%q) = %v, want %v", tt.id, got, tt.want)
}
})
}
}
func TestIsComponentName(t *testing.T) {
names := []string{
ComponentNameFile,
ComponentNameParser,
ComponentNameExtractor,
ComponentNameCompiler,
ComponentNameTokenizer,
ComponentNameGeneralChunker,
ComponentNameTokenChunker,
ComponentNameTitleChunker,
ComponentNameManualChunker,
ComponentNameOneChunker,
ComponentNamePageChunker,
ComponentNameQAChunker,
ComponentNameTableChunker,
ComponentNameGroupTitleChunker,
ComponentNameHierarchyTitleChunker,
}
for _, name := range names {
if !IsComponentName(name) {
t.Errorf("IsComponentName(%q) = false, want true", name)
}
}
for _, name := range []string{"", "Chunker", "extractor", "Extractor "} {
if IsComponentName(name) {
t.Errorf("IsComponentName(%q) = true, want false", name)
}
}
}
func TestSortedExtractorKeys(t *testing.T) {
parserConfig := map[string]any{
"Extractor:B": nil,
"Extractor:A": nil,
"Extractor:": nil,
"Extractor": nil,
"extractor_0": nil,
"Parser:X": nil,
}
want := []string{"Extractor:A", "Extractor:B"}
if got := SortedExtractorKeys(parserConfig); !reflect.DeepEqual(got, want) {
t.Fatalf("SortedExtractorKeys = %#v, want %#v", got, want)
}
}
func TestIsExtractorComponent(t *testing.T) {
tests := []struct {
id string
name string
want bool
}{
{id: "node-1", name: "Extractor", want: true},
{id: "Extractor:node-1", name: "", want: true},
{id: "extractor_0", name: "", want: false},
{id: "node-1", name: "Parser", want: false},
}
for _, tt := range tests {
t.Run(tt.name+"/"+tt.id, func(t *testing.T) {
if got := IsExtractorComponent(tt.id, tt.name); got != tt.want {
t.Fatalf("IsExtractorComponent(%q, %q) = %v, want %v", tt.id, tt.name, got, tt.want)
}
})
}
}
func TestIsParserComponent(t *testing.T) {
tests := []struct {
id string
name string
want bool
}{
{id: "node-1", name: "Parser", want: true},
{id: "Parser:node-1", name: "", want: true},
{id: "node-1", name: "Extractor", want: false},
}
for _, tt := range tests {
t.Run(tt.name+"/"+tt.id, func(t *testing.T) {
if got := IsParserComponent(tt.id, tt.name); got != tt.want {
t.Fatalf("IsParserComponent(%q, %q) = %v, want %v", tt.id, tt.name, got, tt.want)
}
})
}
}
+61
View File
@@ -0,0 +1,61 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package common
// Canonical ingestion component names.
const (
ComponentNameFile = "File"
ComponentNameParser = "Parser"
ComponentNameExtractor = "Extractor"
ComponentNameCompiler = "Compiler"
ComponentNameTokenizer = "Tokenizer"
ComponentNameGeneralChunker = "GeneralChunker"
ComponentNameTokenChunker = "TokenChunker"
ComponentNameTitleChunker = "TitleChunker"
ComponentNameManualChunker = "ManualChunker"
ComponentNameOneChunker = "OneChunker"
ComponentNamePageChunker = "PageChunker"
ComponentNameQAChunker = "QAChunker"
ComponentNameTableChunker = "TableChunker"
ComponentNameGroupTitleChunker = "GroupTitleChunker"
ComponentNameHierarchyTitleChunker = "HierarchyTitleChunker"
)
// IsComponentName reports whether name is a canonical ingestion component name.
func IsComponentName(name string) bool {
switch name {
case ComponentNameFile,
ComponentNameParser,
ComponentNameExtractor,
ComponentNameCompiler,
ComponentNameTokenizer,
ComponentNameGeneralChunker,
ComponentNameTokenChunker,
ComponentNameTitleChunker,
ComponentNameManualChunker,
ComponentNameOneChunker,
ComponentNamePageChunker,
ComponentNameQAChunker,
ComponentNameTableChunker,
ComponentNameGroupTitleChunker,
ComponentNameHierarchyTitleChunker:
return true
default:
return false
}
}
+1 -10
View File
@@ -18,7 +18,6 @@ package common
import (
"regexp"
"sort"
"strconv"
"strings"
)
@@ -441,15 +440,7 @@ func ExtractorMetadataConfig(parserConfig map[string]any) (map[string]any, bool)
if parserConfig == nil {
return nil, false
}
var extractorKeys []string
for k := range parserConfig {
lower := strings.ToLower(k)
if strings.HasPrefix(lower, "extractor:") || strings.HasPrefix(lower, "extractor_") {
extractorKeys = append(extractorKeys, k)
}
}
sort.Strings(extractorKeys)
for _, k := range extractorKeys {
for _, k := range SortedExtractorKeys(parserConfig) {
node, ok := parserConfig[k].(map[string]any)
if !ok {
continue
+1 -1
View File
@@ -105,7 +105,7 @@ func ExtractPipelineDefaults(dsl map[string]interface{}) map[string]interface{}
continue
}
name, _ := obj["component_name"].(string)
if name == "" || name == "File" {
if name == "" || name == ComponentNameFile {
continue
}
params, _ := obj["params"].(map[string]interface{})
@@ -1164,12 +1164,9 @@ func migrateExtractorConfig(config map[string]any, parserID string, pipelineID *
targets = append(targets, newNode)
}
} else {
for id, raw := range config {
id = strings.ToLower(id)
if strings.HasPrefix(id, "extractor:") || strings.HasPrefix(id, "extractor_") {
if node, ok := raw.(map[string]any); ok {
targets = append(targets, node)
}
for _, id := range common.SortedExtractorKeys(config) {
if node, ok := config[id].(map[string]any); ok {
targets = append(targets, node)
}
}
if len(targets) == 0 {
@@ -44,8 +44,6 @@ import (
"ragflow/internal/parser/tableutil"
)
const ComponentNameGeneralChunker = "GeneralChunker"
var contextSentencePattern = regexp.MustCompile(`([。!??;!\n]|\. )`)
type generalChunkerParam struct {
@@ -51,8 +51,6 @@ import (
"ragflow/internal/tokenizer"
)
const ComponentNameGroupTitleChunker = "GroupTitleChunker"
// MIN_GROUP_TOKENS / MAX_GROUP_TOKENS mirror the python constants in
// group_chunker.py:22-23. They drive the merge heuristic for adjacent
// text records.
@@ -51,8 +51,6 @@ import (
"ragflow/internal/ingestion/component/globals"
)
const ComponentNameHierarchyTitleChunker = "HierarchyTitleChunker"
// chunkNode mirrors python `_ChunkNode` (hierarchy_chunker.py:22-32).
type chunkNode struct {
level int
@@ -33,8 +33,6 @@ import (
"gorm.io/gorm"
)
const ComponentNameManualChunker = "ManualChunker"
// ManualChunkerComponent is the standalone manual-layout chunker. It shares
// the GroupTitleChunker grouping body (chunkFromRecords) and only differs by
// the optional physical-position pre-sort.
@@ -0,0 +1,33 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package chunker
import "ragflow/internal/common"
// Chunker component names re-exported from the canonical component registry.
const (
ComponentNameGeneralChunker = common.ComponentNameGeneralChunker
ComponentNameTokenChunker = common.ComponentNameTokenChunker
ComponentNameTitleChunker = common.ComponentNameTitleChunker
ComponentNameManualChunker = common.ComponentNameManualChunker
ComponentNameOneChunker = common.ComponentNameOneChunker
ComponentNamePageChunker = common.ComponentNamePageChunker
ComponentNameQAChunker = common.ComponentNameQAChunker
ComponentNameTableChunker = common.ComponentNameTableChunker
ComponentNameGroupTitleChunker = common.ComponentNameGroupTitleChunker
ComponentNameHierarchyTitleChunker = common.ComponentNameHierarchyTitleChunker
)
@@ -38,8 +38,6 @@ import (
"gorm.io/gorm"
)
const ComponentNameOneChunker = "OneChunker"
type oneChunkerParam struct{}
func (p *oneChunkerParam) Update(conf map[string]any) {}
@@ -34,8 +34,6 @@ import (
"gorm.io/gorm"
)
const ComponentNamePageChunker = "PageChunker"
type pageChunkerParam struct{}
func (p *pageChunkerParam) Update(conf map[string]any) {}
@@ -48,8 +48,6 @@ import (
"ragflow/internal/tokenizer"
)
const ComponentNameQAChunker = "QAChunker"
type qaChunkerParam struct {
Lang string `json:"lang,omitempty"`
}
@@ -41,8 +41,6 @@ import (
"gorm.io/gorm"
)
const ComponentNameTableChunker = "TableChunker"
type tableChunkerParam struct{}
func (p *tableChunkerParam) Update(conf map[string]any) {}
@@ -82,8 +82,6 @@ import (
"gorm.io/gorm"
)
const ComponentNameTitleChunker = "TitleChunker"
type titleChunkerParam struct {
schema.TitleChunkerParam
}
@@ -90,8 +90,6 @@ import (
"ragflow/internal/parser/chunk"
)
const ComponentNameTokenChunker = "TokenChunker"
type tokenChunkerParam struct {
schema.TokenChunkerParam
}
+2 -2
View File
@@ -66,8 +66,8 @@ import (
)
// componentNameExtractor is the canonical Extractor component name; the value
// lives in the schema package so the pipeline layer can reference the same
// identity.
// lives in common and is re-exported by the schema package so both layers
// reference the same identity.
const componentNameExtractor = schema.ComponentNameExtractor
// defaultExtractorChatTimeout bounds one Extractor LLM call, including
+1 -1
View File
@@ -50,7 +50,7 @@ import (
"gorm.io/gorm"
)
const ComponentNameFile = "File"
const ComponentNameFile = schema.ComponentNameFile
// FileComponent resolves document/file metadata and forwards enough
// identity for downstream Parser to fetch bytes.
@@ -47,8 +47,8 @@ var chunkerOutputs = map[string]string{
// (rag/flow/compiler/compiler.py registers component_name = "Compiler"), so a
// canvas saved by the Python frontend and Go's built-in ingestion templates
// both reference the node as "Compiler" and resolve to the same component.
// The value itself lives in the schema package so the pipeline layer can
// reference the same identity.
// The value itself lives in common and is re-exported by the schema package so
// the pipeline layer references the same identity.
const componentNameCompiler = schema.ComponentNameCompiler
// KnowledgeCompilerComponent is the runtime.Component surface. Param is set at
+2 -1
View File
@@ -96,7 +96,8 @@ import (
)
// ComponentNameParser is the canonical Parser component name; the value lives
// in the schema package so the pipeline layer can reference the same identity.
// in common and is re-exported by the schema package so both layers reference
// the same identity.
const ComponentNameParser = schema.ComponentNameParser
// pageFormFeed is the byte that text-page mode treats as a
@@ -16,16 +16,19 @@
package schema
// Canonical ingestion component names. These are the identities components
// register under in runtime.DefaultRegistry and the values a DSL
// component_name carries. Layers that must recognise a component by name
// across the component/pipeline boundary (dispatch, param whitelists,
// operator-facing failure messages) reference these constants instead of
// repeating the literals; the schema package is the shared dependency both
// sides already import. More names can join this set as other layers need
// them.
import "ragflow/internal/common"
// Canonical ingestion component names re-exported from common for callers that
// need them across the pipeline/component boundary.
const (
ComponentNameParser = "Parser"
ComponentNameExtractor = "Extractor"
ComponentNameCompiler = "Compiler"
ComponentNameFile = common.ComponentNameFile
ComponentNameParser = common.ComponentNameParser
ComponentNameExtractor = common.ComponentNameExtractor
ComponentNameCompiler = common.ComponentNameCompiler
ComponentNameTokenizer = common.ComponentNameTokenizer
ComponentNameGeneralChunker = common.ComponentNameGeneralChunker
ComponentNameTokenChunker = common.ComponentNameTokenChunker
ComponentNameTitleChunker = common.ComponentNameTitleChunker
ComponentNameManualChunker = common.ComponentNameManualChunker
)
+4 -14
View File
@@ -22,6 +22,7 @@ import (
"sort"
"strings"
"ragflow/internal/common"
"ragflow/internal/entity"
"ragflow/internal/ingestion/component/schema"
"ragflow/internal/tokenizer"
@@ -87,20 +88,9 @@ func TagFileIDFromParserConfig(parserConfig map[string]any) string {
if v := tagFileIDFromTagsMap(parserConfig["tags"]); v != "" {
return v
}
// Several Extractor nodes can each declare their own tags block, so the
// winner has to be picked in a stable order: Go randomizes map iteration,
// and an unstable pick would let the dataset's tag source change between
// process restarts (different chunks tagged, different retrieval
// features). Sorting makes it deterministic; "Extractor" sorts before
// "Extractor:<name>" because it is a prefix, so the unscoped node wins.
keys := make([]string, 0, len(parserConfig))
for key := range parserConfig {
if key == "Extractor" || strings.HasPrefix(key, "Extractor:") {
keys = append(keys, key)
}
}
sort.Strings(keys)
for _, key := range keys {
// Several Extractor nodes can each declare their own tags block. Pick the
// winner deterministically by sorted component ID.
for _, key := range common.SortedExtractorKeys(parserConfig) {
extractorMap, ok := asStringAnyMap(parserConfig[key])
if !ok {
continue
+31 -16
View File
@@ -56,6 +56,33 @@ func TestTagFileIDFromParserConfig(t *testing.T) {
},
want: "file-3",
},
{
name: "lowercase extractor form is not canonical",
parserConfig: map[string]any{
"extractor:AutoExtractDefault": map[string]any{
"tags": map[string]any{"tag_file_id": "file-5"},
},
},
want: "",
},
{
name: "bare extractor form is not scoped",
parserConfig: map[string]any{
"Extractor": map[string]any{
"tags": map[string]any{"tag_file_id": "file-6"},
},
},
want: "",
},
{
name: "underscore extractor form is not scoped",
parserConfig: map[string]any{
"extractor_0": map[string]any{
"tags": map[string]any{"tag_file_id": "file-7"},
},
},
want: "",
},
{
name: "entity.JSONMap nested value",
parserConfig: map[string]any{
@@ -85,29 +112,17 @@ func TestTagFileIDFromParserConfig(t *testing.T) {
func TestTagFileIDFromParserConfigPicksStableExtractorNode(t *testing.T) {
// Several Extractor nodes can each declare their own tags block. Go
// randomizes map iteration, so an unsorted scan would return a different
// tag file - and thus tag the dataset from a different tag source - on
// every call and every process restart.
parserConfig := map[string]any{
"Extractor": map[string]any{"tags": map[string]any{"tag_file_id": "file-unscoped"}},
}
// randomizes map iteration, so the chosen tag source must not depend on
// iteration order.
parserConfig := map[string]any{}
for i := 0; i < 8; i++ {
parserConfig[fmt.Sprintf("Extractor:node-%d", i)] = map[string]any{
"tags": map[string]any{"tag_file_id": fmt.Sprintf("file-%d", i)},
}
}
// The unscoped "Extractor" node wins: it is a prefix of the scoped keys,
// so it sorts first.
for i := 0; i < 200; i++ {
if got := TagFileIDFromParserConfig(parserConfig); got != "file-unscoped" {
t.Fatalf("TagFileIDFromParserConfig with unscoped node = %q, want %q", got, "file-unscoped")
}
}
// Without it the lowest-sorting scoped node wins, still stably.
delete(parserConfig, "Extractor")
for i := 0; i < 200; i++ {
if got := TagFileIDFromParserConfig(parserConfig); got != "file-0" {
t.Fatalf("TagFileIDFromParserConfig across scoped nodes = %q, want %q", got, "file-0")
t.Fatalf("TagFileIDFromParserConfig = %q, want %q", got, "file-0")
}
}
}
+1 -1
View File
@@ -98,7 +98,7 @@ import (
"ragflow/internal/tokenizer"
)
const ComponentNameTokenizer = "Tokenizer"
const ComponentNameTokenizer = schema.ComponentNameTokenizer
// titleExtRE strips a trailing file-extension (e.g. ".pdf") from the
// upstream document name before tokenizing it. Mirrors the python
@@ -4,6 +4,7 @@ import (
"encoding/json"
"fmt"
"ragflow/internal/common"
"ragflow/internal/ingestion/component/schema"
)
@@ -42,7 +43,10 @@ func ExtractAllComponentParams(dslJSON []byte) ([]ComponentParamsSchema, error)
extractorCount := 0
for cpnID, comp := range dsl.Components {
if isExtractorComponent(cpnID, comp.Obj.ComponentName) {
if cpnID != common.ComponentNameFile && !common.IsComponentKey(cpnID) {
return nil, fmt.Errorf("ExtractAllComponentParams: component ID %q must be <name>:<instance>; re-save the pipeline with canonical component IDs", cpnID)
}
if common.IsExtractorComponent(cpnID, comp.Obj.ComponentName) {
extractorCount++
}
}
@@ -62,7 +66,7 @@ func ExtractAllComponentParams(dslJSON []byte) ([]ComponentParamsSchema, error)
// Canvases saved by the Python-era frontend nest the Parser's
// per-family setups under a "setups" key; lift them so the extracted
// defaults (and the parser_config built from them) stay flat.
if isParserComponent(cpnID, comp.Obj.ComponentName) {
if common.IsParserComponent(cpnID, comp.Obj.ComponentName) {
params = schema.FlattenLegacyParserSetups(params)
}
out = append(out, ComponentParamsSchema{
@@ -196,6 +196,20 @@ func TestExtractAllComponentParams_MissingComponents(t *testing.T) {
}
}
func TestExtractAllComponentParams_RejectsUnscopedComponentIDs(t *testing.T) {
for _, id := range []string{"Extractor", "extractor_0", "parser_0"} {
dslJSON := []byte(`{"components":{"` + id + `":{"obj":{"component_name":"Extractor","params":{}}}}}`)
if _, err := ExtractAllComponentParams(dslJSON); err == nil {
t.Errorf("ExtractAllComponentParams(%q) expected error for unscoped component ID", id)
}
}
// File is the one bare component ID used by built-in ingestion templates.
dslJSON := []byte(`{"components":{"File":{"obj":{"component_name":"File","params":{}}}}}`)
if _, err := ExtractAllComponentParams(dslJSON); err != nil {
t.Fatalf("ExtractAllComponentParams(File): %v", err)
}
}
func componentIDs(schemas []ComponentParamsSchema) []string {
ids := make([]string, len(schemas))
for i, s := range schemas {
+1 -1
View File
@@ -149,7 +149,7 @@ func ValidatePipeline(cnv *canvas.Canvas) error {
}
extractorCount := 0
for id, comp := range cnv.Components {
if isExtractorComponent(id, comp.Obj.ComponentName) {
if common.IsExtractorComponent(id, comp.Obj.ComponentName) {
extractorCount++
}
}
+19 -53
View File
@@ -83,14 +83,14 @@ var titleFamilyParamKeys = unionParamKeys(
)
var componentParamSchemaKeys = map[string]map[string]struct{}{
"titlechunker": titleFamilyParamKeys,
strings.ToLower(schema.ComponentNameTitleChunker): titleFamilyParamKeys,
// ManualChunker reuses TitleChunkerParam. It pins method=group and is exempt
// from the token cap, so method, chunk_token_cap and include_heading_content
// (hierarchy-only) are accepted here but ignored by the component — the
// operator form omits them for the same reason.
"manualchunker": titleFamilyParamKeys,
"tokenchunker": extractJSONTags(schema.TokenChunkerParam{}),
"tokenizer": extractJSONTags(schema.TokenizerParam{}),
strings.ToLower(schema.ComponentNameManualChunker): titleFamilyParamKeys,
strings.ToLower(schema.ComponentNameTokenChunker): extractJSONTags(schema.TokenChunkerParam{}),
strings.ToLower(schema.ComponentNameTokenizer): extractJSONTags(schema.TokenizerParam{}),
}
// extractJSONTags returns all top-level json tag names from a struct.
@@ -119,38 +119,6 @@ func extractJSONTags(v any) map[string]struct{} {
return tags
}
// isExtractorComponent returns true if the component name or component ID
// indicates an Extractor component.
func isExtractorComponent(cpnID, componentName string) bool {
lowerName := strings.ToLower(componentName)
lowerID := strings.ToLower(cpnID)
return lowerName == strings.ToLower(schema.ComponentNameExtractor) ||
strings.HasPrefix(lowerID, "extractor:") ||
strings.HasPrefix(lowerID, "extractor_")
}
// isCompilerComponent returns true if the component ID indicates a Compiler
// component, whose LLM-backed runtime parameters follow the same surface as
// the Extractor's.
func isCompilerComponent(cpnID string) bool {
lowerID := strings.ToLower(cpnID)
return strings.HasPrefix(lowerID, "compiler:") ||
strings.HasPrefix(lowerID, "compiler_")
}
// IsParserComponentID reports whether a component ID names a Parser component.
// A parser_config map is keyed by component ID, where the component name is not
// available, so this is the config-side half of isParserComponent.
func IsParserComponentID(cpnID string) bool {
return strings.HasPrefix(strings.ToLower(cpnID), "parser:")
}
// isParserComponent returns true if the component name or component ID
// indicates a Parser component.
func isParserComponent(cpnID, componentName string) bool {
return strings.EqualFold(componentName, schema.ComponentNameParser) || IsParserComponentID(cpnID)
}
// NormalizeParserConfigSetups lifts legacy nested Parser "setups" entries in
// a parser_config-shaped map ({cpnID: params}) into the flat per-family
// shape. The runtime merges such a map over the DSL-baked component params
@@ -163,7 +131,7 @@ func NormalizeParserConfigSetups(parserConfig map[string]any) map[string]any {
return nil
}
for cpnID, raw := range parserConfig {
if !isParserComponent(cpnID, "") {
if !common.IsParserComponentID(cpnID) {
continue
}
params, ok := raw.(map[string]any)
@@ -177,22 +145,19 @@ func NormalizeParserConfigSetups(parserConfig map[string]any) map[string]any {
// getComponentParamWhitelist returns the dynamic parameter whitelist for a component type.
func getComponentParamWhitelist(cpnID string) (map[string]struct{}, bool) {
if isExtractorComponent(cpnID, "") {
if common.IsExtractorComponentID(cpnID) {
return extractorValidParamKeys, true
}
if isCompilerComponent(cpnID) {
if common.IsCompilerComponentID(cpnID) {
return llmRuntimeParamKeys, true
}
return nil, false
}
// cleanComponentParams filters rawConfig against the extracted DSL schemas.
// Keys containing ':' are treated as component IDs; they are kept only when both
// the cpnID AND the param name exist in one of the accepted-key sources: the DSL
// schema (the params the template bakes), the schema-derived per-component table
// (componentParamSchemaKeys), or the component's dynamic parameter schema (e.g.
// Extractor modular features). Keys without ':' (legacy flat fields) are dropped
// with a warning.
// cleanComponentParams keeps only component-scoped keys recognized by
// common.IsComponentKey, and only params accepted by the DSL schema, the
// per-component table, or the component's dynamic schema. Flat fields are
// dropped with a warning.
func cleanComponentParams(schemas []ComponentParamsSchema, rawConfig map[string]interface{}) map[string]interface{} {
validCPNs := make(map[string]map[string]struct{}, len(schemas))
componentNames := make(map[string]string, len(schemas))
@@ -201,10 +166,10 @@ func cleanComponentParams(schemas []ComponentParamsSchema, rawConfig map[string]
for k := range s.ParamsDefaults {
keys[k] = struct{}{}
}
if s.ComponentName == "GeneralChunker" {
if s.ComponentName == schema.ComponentNameGeneralChunker {
keys["delimiters"] = struct{}{}
}
if isParserComponent(s.CpnID, s.ComponentName) {
if common.IsParserComponent(s.CpnID, s.ComponentName) {
keys["enable_vision_enhancement"] = struct{}{}
keys["vlm"] = struct{}{}
}
@@ -225,7 +190,7 @@ func cleanComponentParams(schemas []ComponentParamsSchema, rawConfig map[string]
result := make(map[string]interface{}, len(rawConfig))
for key, val := range rawConfig {
if !strings.Contains(key, ":") {
if !common.IsComponentKey(key) {
common.Warn("cleanComponentParams: dropping legacy flat field",
zap.String("key", key))
continue
@@ -240,18 +205,18 @@ func cleanComponentParams(schemas []ComponentParamsSchema, rawConfig map[string]
if !ok {
continue
}
if componentNames[key] == "GeneralChunker" {
if componentNames[key] == schema.ComponentNameGeneralChunker {
params = normalizeGeneralComponentParams(params)
}
// Legacy parser_config rows (or canvases) saved by the Python-era
// frontend nest the Parser's setups under a "setups" key; lift them
// before filtering, otherwise the families are dropped as unknown
// params against the flat DSL schema keys.
if isParserComponent(key, componentNames[key]) {
if common.IsParserComponent(key, componentNames[key]) {
params = schema.FlattenLegacyParserSetups(params)
}
dynamicWhitelist, hasDynamic := getComponentParamWhitelist(key)
if isExtractorComponent(key, "") {
if common.IsExtractorComponentID(key) {
params = NormalizeExtractorParams(params)
}
cleaned := make(map[string]any, len(params))
@@ -430,5 +395,6 @@ func ApplyParentChildChunkerConfig(componentConfig entity.JSONMap, rawConfig map
// chunks into children.
func IsChunkerComponent(componentID string) bool {
lowerID := strings.ToLower(componentID)
return strings.HasPrefix(lowerID, "generalchunker:") || strings.HasPrefix(lowerID, "tokenchunker:")
return strings.HasPrefix(lowerID, strings.ToLower(schema.ComponentNameGeneralChunker)+":") ||
strings.HasPrefix(lowerID, strings.ToLower(schema.ComponentNameTokenChunker)+":")
}
@@ -34,9 +34,9 @@ func generalDSL(t *testing.T) []byte {
},
},
},
"Chunker:LegalReadersDecide": map[string]any{
"GeneralChunker:LegalReadersDecide": map[string]any{
"obj": map[string]any{
"component_name": "Chunker",
"component_name": "GeneralChunker",
"params": map[string]any{
"outputs": map[string]any{},
"chunk_size": float64(512),
@@ -165,9 +165,9 @@ func TestBuildParserConfig_ShallowMerge_NestedParam(t *testing.T) {
// A component with a nested-map param "chunk" that has sub-keys.
dsl := map[string]any{
"components": map[string]any{
"Chunker:xyz": map[string]any{
"GeneralChunker:xyz": map[string]any{
"obj": map[string]any{
"component_name": "Chunker",
"component_name": "GeneralChunker",
"params": map[string]any{
"outputs": map[string]any{},
"chunk": map[string]any{
@@ -186,15 +186,15 @@ func TestBuildParserConfig_ShallowMerge_NestedParam(t *testing.T) {
// User only overrides one sub-key of "chunk".
overrides := map[string]any{
"Chunker:xyz": map[string]any{
"GeneralChunker:xyz": map[string]any{
"chunk": map[string]any{"size": float64(1024)},
},
}
result := buildTestParserConfig(t, dslJSON, overrides)
chunker, ok := result["Chunker:xyz"].(map[string]any)
chunker, ok := result["GeneralChunker:xyz"].(map[string]any)
if !ok {
t.Fatal("expected Chunker:xyz in result")
t.Fatal("expected GeneralChunker:xyz in result")
}
chunk, ok := chunker["chunk"].(map[string]any)
if !ok {
@@ -348,14 +348,14 @@ func TestNormalizeParserConfigSetups_EmptyNestedMap(t *testing.T) {
func TestBuildParserConfig_ScalarOverridePreservesOtherDefaults(t *testing.T) {
dslJSON := generalDSL(t)
overrides := map[string]any{
"Chunker:LegalReadersDecide": map[string]any{
"GeneralChunker:LegalReadersDecide": map[string]any{
"chunk_size": float64(1024),
},
}
result := buildTestParserConfig(t, dslJSON, overrides)
chunker, ok := result["Chunker:LegalReadersDecide"].(map[string]any)
chunker, ok := result["GeneralChunker:LegalReadersDecide"].(map[string]any)
if !ok {
t.Fatal("expected Chunker:LegalReadersDecide in result")
t.Fatal("expected GeneralChunker:LegalReadersDecide in result")
}
if chunker["chunk_size"] != float64(1024) {
t.Errorf("expected chunk_size=1024, got %v", chunker["chunk_size"])
@@ -398,8 +398,8 @@ func TestBuildParserConfig_AllComponentsPresent(t *testing.T) {
if _, ok := result["Parser:HipSignsRhyme"]; !ok {
t.Error("expected Parser:HipSignsRhyme")
}
if _, ok := result["Chunker:LegalReadersDecide"]; !ok {
t.Error("expected Chunker:LegalReadersDecide")
if _, ok := result["GeneralChunker:LegalReadersDecide"]; !ok {
t.Error("expected GeneralChunker:LegalReadersDecide")
}
}
@@ -470,7 +470,7 @@ func TestComponentParamsDefaults_Basic(t *testing.T) {
if _, ok := parser["pdf"]; !ok {
t.Error("expected pdf to be present")
}
chunker := result["Chunker:LegalReadersDecide"]
chunker := result["GeneralChunker:LegalReadersDecide"]
if chunker["chunk_size"] != float64(512) {
t.Errorf("expected chunk_size=512, got %v", chunker["chunk_size"])
}
@@ -21,6 +21,8 @@ import (
"path/filepath"
"strings"
"testing"
"ragflow/internal/common"
)
// parserSetupParamsKeys are the Parser params keys that are NOT per-family
@@ -108,7 +110,7 @@ func assertNoParserOutputFormatInGraph(t *testing.T, ref string, root map[string
for _, rawNode := range nodes {
node, _ := rawNode.(map[string]any)
id, _ := node["id"].(string)
if !IsParserComponentID(id) {
if !common.IsParserComponentID(id) {
continue
}
data, _ := node["data"].(map[string]any)
+1 -2
View File
@@ -18,7 +18,6 @@ package indexdoc
import (
"fmt"
"strings"
"time"
"ragflow/internal/common"
@@ -376,7 +375,7 @@ func resolveTableColumnConfig(parserConfig map[string]interface{}) (mode string,
return mode, roles, rawNames
}
for cid, raw := range parserConfig {
if !strings.HasPrefix(cid, "Parser:") {
if !common.IsParserComponentID(cid) {
continue
}
comp, _ := raw.(map[string]interface{})
@@ -3,6 +3,7 @@ package service
import (
"strings"
"ragflow/internal/common"
"ragflow/internal/entity"
)
@@ -33,9 +34,8 @@ func ApplyComponentScopedParserConfig(
if !ok {
continue
}
cpnLower := strings.ToLower(cpnID)
switch {
case strings.HasPrefix(cpnLower, "extractor:") || strings.HasPrefix(cpnLower, "extractor_"):
case common.IsExtractorComponentID(cpnID):
if value, _ := params["llm_id"].(string); strings.TrimSpace(value) == "" && strings.TrimSpace(llmID) != "" {
params["llm_id"] = llmID
}
@@ -54,7 +54,7 @@ func ApplyComponentScopedParserConfig(
"built_in_metadata": []any{},
}
}
case strings.HasPrefix(cpnLower, "compiler:") || strings.HasPrefix(cpnLower, "compiler_"):
case common.IsCompilerComponentID(cpnID):
if value, _ := params["llm_id"].(string); strings.TrimSpace(value) == "" && strings.TrimSpace(llmID) != "" {
params["llm_id"] = llmID
}
+18 -20
View File
@@ -8,6 +8,7 @@ import (
"strings"
"unicode/utf8"
"ragflow/internal/common"
"ragflow/internal/dao"
"ragflow/internal/entity"
pipelinepkg "ragflow/internal/ingestion/pipeline"
@@ -202,23 +203,19 @@ func validateDatasetParserConfigSize(parserConfig map[string]interface{}) error
return nil
}
// DropUnscopedParserConfigKeys removes top-level parser_config keys that are
// not component-scoped (i.e. do not contain ":"), since the Go backend only
// consumes keys keyed by a component id (e.g. "Parser:abc123" or
// "Extractor:AutoExtractDefault"). Flat keys are legacy transport artifacts
// that downstream consumers (BuildParserConfig) never read, so they are
// dropped rather than rejected. Only top-level keys are affected; a key nested
// inside a component node (e.g. "Extractor:AutoExtractDefault.metadata") keeps
// its existing name. It mutates the provided map in place and returns the names
// of the dropped keys so callers can log the silent drop. A nil or empty map is
// returned untouched with a nil result.
// DropUnscopedParserConfigKeys removes top-level parser_config keys that
// common.IsComponentKey does not recognize as component-scoped. Flat keys are
// transport artifacts that BuildParserConfig never reads, so they are dropped
// rather than rejected. Only top-level keys are affected; nested keys keep
// their names. It mutates parserConfig and returns the dropped names so callers
// can log them. A nil or empty map is returned untouched.
func DropUnscopedParserConfigKeys(parserConfig map[string]any) []string {
if len(parserConfig) == 0 {
return nil
}
var dropped []string
for key := range parserConfig {
if !strings.Contains(key, ":") {
if !common.IsComponentKey(key) {
dropped = append(dropped, key)
}
}
@@ -369,10 +366,10 @@ func preserveDatasetParserConfigState(next, existing entity.JSONMap, incoming ma
}
var mm map[string]any
if incoming != nil {
mm = extractorNodeMetadata(incoming)
mm = extractorNodeMetadata(incoming, next)
}
if mm == nil && existing != nil {
mm = extractorNodeMetadata(existing)
mm = extractorNodeMetadata(existing, next)
}
if mm != nil {
next["metadata"] = mm
@@ -447,18 +444,19 @@ func preserveDatasetParserConfigState(next, existing entity.JSONMap, incoming ma
}
// extractorNodeMetadata returns the modular metadata object ({enabled, metadata,
// built_in_metadata}) from the first Extractor node in a parser_config, or nil
// if none is present. Metadata is component-scoped under Extractor nodes; this
// helper reads it back for preservation across partial updates.
func extractorNodeMetadata(parserConfig map[string]interface{}) map[string]any {
// built_in_metadata}) from the first Extractor node present in validConfig, or
// nil if none is present. validConfig is the rebuilt parser_config for the
// current DSL; keys outside it are ignored so stale nodes cannot override the
// active configuration.
func extractorNodeMetadata(parserConfig, validConfig map[string]interface{}) map[string]any {
if parserConfig == nil {
return nil
}
for cpnID, raw := range parserConfig {
lower := strings.ToLower(cpnID)
if !strings.HasPrefix(lower, "extractor:") && !strings.HasPrefix(lower, "extractor_") {
for _, cpnID := range common.SortedExtractorKeys(parserConfig) {
if _, ok := validConfig[cpnID]; !ok {
continue
}
raw := parserConfig[cpnID]
params, ok := raw.(map[string]any)
if !ok {
continue
+42 -3
View File
@@ -227,6 +227,9 @@ func TestDropUnscopedParserConfigKeys_NilAndEmpty(t *testing.T) {
func TestDropUnscopedParserConfigKeys_DropsFlatKeys(t *testing.T) {
cfg := map[string]any{
"File": map[string]any{},
"Extractor": map[string]any{},
"extractor_0": map[string]any{},
"compiler_0": map[string]any{},
"chunk_token_num": float64(128),
"delimiter": "\n",
"metadata": map[string]any{},
@@ -236,7 +239,7 @@ func TestDropUnscopedParserConfigKeys_DropsFlatKeys(t *testing.T) {
}
dropped := DropUnscopedParserConfigKeys(cfg)
sort.Strings(dropped)
want := []string{"File", "chunk_token_num", "delimiter", "metadata", "parent_child"}
want := []string{"Extractor", "File", "chunk_token_num", "compiler_0", "delimiter", "extractor_0", "metadata", "parent_child"}
if !reflect.DeepEqual(dropped, want) {
t.Fatalf("dropped = %#v, want %#v", dropped, want)
}
@@ -510,7 +513,9 @@ func TestPreserveDatasetParserConfigState_FallsBackWhenIncomingNotMap(t *testing
"metadata": incomingMetadata,
},
}
got := preserveDatasetParserConfigState(entity.JSONMap{}, existing, incoming)
got := preserveDatasetParserConfigState(entity.JSONMap{
"Extractor:AutoExtractDefault": map[string]any{},
}, existing, incoming)
meta, ok := got["metadata"].(map[string]any)
if !ok {
t.Fatalf("expected existing modular metadata preserved, got %#v", got["metadata"])
@@ -523,6 +528,38 @@ func TestPreserveDatasetParserConfigState_FallsBackWhenIncomingNotMap(t *testing
}
}
func TestPreserveDatasetParserConfigState_IgnoresStaleExtractor(t *testing.T) {
existing := entity.JSONMap{
"Extractor:AutoExtractDefault": map[string]any{
"metadata": map[string]any{
"enabled": true,
"metadata": []any{map[string]any{"key": "current_field", "type": "string"}},
"built_in_metadata": []any{},
},
},
}
incoming := map[string]interface{}{
"Extractor:Old": map[string]any{
"metadata": map[string]any{
"enabled": true,
"metadata": []any{map[string]any{"key": "stale_field", "type": "string"}},
"built_in_metadata": []any{},
},
},
}
got := preserveDatasetParserConfigState(entity.JSONMap{
"Extractor:AutoExtractDefault": map[string]any{},
}, existing, incoming)
meta, ok := got["metadata"].(map[string]any)
if !ok {
t.Fatalf("expected current modular metadata, got %#v", got["metadata"])
}
fields, ok := meta["metadata"].([]any)
if !ok || len(fields) != 1 || fields[0].(map[string]any)["key"] != "current_field" {
t.Fatalf("expected current_field to win over stale Extractor, got %#v", meta["metadata"])
}
}
func TestPreserveDatasetParserConfigState_UsesValidIncomingMap(t *testing.T) {
existing := entity.JSONMap{
"Extractor:AutoExtractDefault": map[string]any{
@@ -542,7 +579,9 @@ func TestPreserveDatasetParserConfigState_UsesValidIncomingMap(t *testing.T) {
},
},
}
got := preserveDatasetParserConfigState(entity.JSONMap{}, existing, incoming)
got := preserveDatasetParserConfigState(entity.JSONMap{
"Extractor:AutoExtractDefault": map[string]any{},
}, existing, incoming)
meta, ok := got["metadata"].(map[string]any)
if !ok {
t.Fatalf("expected modular metadata map, got %#v", got["metadata"])
+2 -5
View File
@@ -192,11 +192,8 @@ func modularMetadataConfig(parserConfig map[string]any) (bool, bool, []any, []an
builtIn := anyOrEmptyList(metaObj["built_in_metadata"])
return true, enabled, metadata, builtIn
}
for cpnID, raw := range parserConfig {
lower := strings.ToLower(cpnID)
if !strings.HasPrefix(lower, "extractor:") && !strings.HasPrefix(lower, "extractor_") {
continue
}
for _, cpnID := range common.SortedExtractorKeys(parserConfig) {
raw := parserConfig[cpnID]
params, ok := raw.(map[string]any)
if !ok {
continue
@@ -0,0 +1,53 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package dataset
import (
"testing"
"ragflow/internal/common"
"ragflow/internal/ingestion/component"
pipelinepkg "ragflow/internal/ingestion/pipeline"
)
func TestParserConfigCanonicalExtractorRoundTrip(t *testing.T) {
dsl := []byte(`{"components":{"Extractor:AutoExtractDefault":{"obj":{"component_name":"Extractor","params":{"metadata":{"enabled":true,"metadata":[],"built_in_metadata":[]},"tags":{"top_n":5,"tag_file_id":""}}}}}}`)
raw := map[string]any{
"chunk_token_num": float64(128),
"Extractor:AutoExtractDefault": map[string]any{
"tags": map[string]any{"top_n": 5, "tag_file_id": "file-round-trip"},
},
}
if dropped, err := ValidateParserConfig(raw); err != nil {
t.Fatalf("ValidateParserConfig: %v", err)
} else if len(dropped) != 1 || dropped[0] != "chunk_token_num" {
t.Fatalf("dropped = %#v, want [chunk_token_num]", dropped)
}
config, err := pipelinepkg.BuildParserConfig(dsl, raw)
if err != nil {
t.Fatalf("BuildParserConfig: %v", err)
}
if got := component.TagFileIDFromParserConfig(map[string]any(config)); got != "file-round-trip" {
t.Fatalf("TagFileIDFromParserConfig = %q, want file-round-trip", got)
}
if meta, ok := common.ExtractorMetadataConfig(map[string]any(config)); !ok {
t.Fatal("ExtractorMetadataConfig did not find the canonical Extractor node")
} else if enabled, _ := meta["enabled"].(bool); !enabled {
t.Fatalf("metadata enabled = %v, want true", meta["enabled"])
}
}
+1 -2
View File
@@ -1356,8 +1356,7 @@ func TestUpdateDatasetUsesTenantModelIDForExtractorDefault(t *testing.T) {
func extractorNodeMetadataInTest(t *testing.T, cfg map[string]interface{}) map[string]interface{} {
t.Helper()
for cpnID, raw := range cfg {
lower := strings.ToLower(cpnID)
if !strings.HasPrefix(lower, "extractor:") && !strings.HasPrefix(lower, "extractor_") {
if !common.IsExtractorComponentID(cpnID) {
continue
}
params, ok := raw.(map[string]interface{})
@@ -15,7 +15,6 @@ package document
import (
"encoding/json"
"sort"
"strings"
"testing"
"gorm.io/gorm"
@@ -42,8 +41,7 @@ func firstExtractorMetadata(t *testing.T, parserConfig entity.JSONMap) map[strin
t.Helper()
var keys []string
for k := range parserConfig {
lower := strings.ToLower(k)
if strings.HasPrefix(lower, "extractor:") || strings.HasPrefix(lower, "extractor_") {
if common.IsExtractorComponentID(k) {
keys = append(keys, k)
}
}
+3 -19
View File
@@ -5,7 +5,6 @@ import (
"errors"
"fmt"
"ragflow/internal/service"
"sort"
"strconv"
"strings"
"sync"
@@ -198,7 +197,7 @@ func (s *DocumentService) applyKnowledgebaseParserConfig(ctx context.Context, do
if metaOK {
merged["metadata"] = deepCopyValue(meta)
} else {
for _, cpnID := range sortedExtractorKeys(merged) {
for _, cpnID := range common.SortedExtractorKeys(map[string]any(merged)) {
params, ok := nodeParams(merged[cpnID])
if !ok {
continue
@@ -227,21 +226,6 @@ func (s *DocumentService) applyKnowledgebaseParserConfig(ctx context.Context, do
return nil
}
// sortedExtractorKeys returns the component IDs in parserConfig that name
// Extractor nodes, in sorted order so multi-Extractor configs resolve
// deterministically instead of depending on Go map iteration order.
func sortedExtractorKeys(parserConfig entity.JSONMap) []string {
var keys []string
for k := range parserConfig {
lower := strings.ToLower(k)
if strings.HasPrefix(lower, "extractor:") || strings.HasPrefix(lower, "extractor_") {
keys = append(keys, k)
}
}
sort.Strings(keys)
return keys
}
// nodeParams returns a writable view of a component node's params, accepting
// both the decoded (map[string]any) and entity (entity.JSONMap) shapes.
// The returned map shares storage with the input, so writes are visible.
@@ -281,7 +265,7 @@ func resolveDatasetModularMetadata(parserConfig entity.JSONMap) (map[string]any,
if mm, ok := parserConfig["metadata"].(entity.JSONMap); ok {
return map[string]any(mm), true, true
}
for _, cpnID := range sortedExtractorKeys(parserConfig) {
for _, cpnID := range common.SortedExtractorKeys(map[string]any(parserConfig)) {
params, ok := nodeParams(parserConfig[cpnID])
if !ok {
continue
@@ -301,7 +285,7 @@ func resolveDatasetLLMID(parserConfig entity.JSONMap) string {
if s, ok := parserConfig["llm_id"].(string); ok && strings.TrimSpace(s) != "" {
return s
}
for _, cpnID := range sortedExtractorKeys(parserConfig) {
for _, cpnID := range common.SortedExtractorKeys(map[string]any(parserConfig)) {
params, ok := nodeParams(parserConfig[cpnID])
if !ok {
continue
@@ -66,7 +66,7 @@ func parserParamsDeclaresFamily(params map[string]any, family string) bool {
// config, or a malformed one): the config cannot answer then.
func configDeclaredFamily(config entity.JSONMap, family string) (declared, found bool) {
for cpnID, raw := range config {
if !pipelinepkg.IsParserComponentID(cpnID) {
if !common.IsParserComponentID(cpnID) {
continue
}
params, ok := raw.(map[string]any)
@@ -94,7 +94,7 @@ func templateDeclaredFamily(ctx context.Context, parserID, family string) (decla
return false, false
}
for cpnID, params := range defaults {
if !pipelinepkg.IsParserComponentID(cpnID) {
if !common.IsParserComponentID(cpnID) {
continue
}
return parserParamsDeclaresFamily(params, family), true
@@ -3,9 +3,9 @@ package document
import (
"mime/multipart"
"reflect"
"strings"
"testing"
"ragflow/internal/common"
"ragflow/internal/dao"
"ragflow/internal/entity"
"ragflow/internal/permission"
@@ -763,7 +763,7 @@ func parserFamilySetup(t *testing.T, doc map[string]interface{}, family string)
t.Fatalf("%v parser_config = %T, want map", doc["name"], doc["parser_config"])
}
for cpnID, raw := range cfg {
if !strings.HasPrefix(cpnID, "Parser:") {
if !common.IsParserComponentID(cpnID) {
continue
}
params, _ := raw.(map[string]interface{})