mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-11-10 13:24:05 +08:00
refactor(ingestion): centralize component names and identity checks (#22081)
This commit is contained in:
1 parent
fa813113e1
commit
3a6b6dca78
42 files changed
+629
-216
No files matched your search
@@ -0,0 +1,79 @@
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package common
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// hasComponentIDPrefix reports whether id is "<componentName>:<non-empty instance>".
|
||||
func hasComponentIDPrefix(id, componentName string) bool {
|
||||
kind, instance, ok := strings.Cut(id, ":")
|
||||
return ok && kind == componentName && instance != ""
|
||||
}
|
||||
|
||||
// IsExtractorComponentID reports whether id names an Extractor node in a
|
||||
// component-scoped parser_config or pipeline DSL. Only the canonical
|
||||
// "Extractor:<instance>" form is accepted.
|
||||
func IsExtractorComponentID(id string) bool {
|
||||
return hasComponentIDPrefix(id, ComponentNameExtractor)
|
||||
}
|
||||
|
||||
// IsExtractorComponent reports whether the component name or component ID
|
||||
// identifies an Extractor.
|
||||
func IsExtractorComponent(id, componentName string) bool {
|
||||
return componentName == ComponentNameExtractor || IsExtractorComponentID(id)
|
||||
}
|
||||
|
||||
// IsParserComponentID reports whether id names a Parser node in a
|
||||
// component-scoped parser_config or pipeline DSL.
|
||||
func IsParserComponentID(id string) bool {
|
||||
return hasComponentIDPrefix(id, ComponentNameParser)
|
||||
}
|
||||
|
||||
// IsParserComponent reports whether the component name or component ID
|
||||
// identifies a Parser.
|
||||
func IsParserComponent(id, componentName string) bool {
|
||||
return componentName == ComponentNameParser || IsParserComponentID(id)
|
||||
}
|
||||
|
||||
// IsCompilerComponentID reports whether id names a Compiler node in a
|
||||
// component-scoped parser_config or pipeline DSL.
|
||||
func IsCompilerComponentID(id string) bool {
|
||||
return hasComponentIDPrefix(id, ComponentNameCompiler)
|
||||
}
|
||||
|
||||
// IsComponentKey reports whether id is a canonical component-scoped
|
||||
// parser_config key: "<ComponentName>:<non-empty instance>".
|
||||
func IsComponentKey(id string) bool {
|
||||
kind, instance, ok := strings.Cut(id, ":")
|
||||
return ok && instance != "" && IsComponentName(kind)
|
||||
}
|
||||
|
||||
// SortedExtractorKeys returns the parser_config keys that name Extractor nodes,
|
||||
// sorted so callers resolve multiple Extractor nodes deterministically.
|
||||
func SortedExtractorKeys(parserConfig map[string]any) []string {
|
||||
keys := make([]string, 0)
|
||||
for key := range parserConfig {
|
||||
if IsExtractorComponentID(key) {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return keys
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package common
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestIsExtractorComponentID(t *testing.T) {
|
||||
tests := []struct {
|
||||
id string
|
||||
want bool
|
||||
}{
|
||||
{id: "Extractor", want: false},
|
||||
{id: "extractor", want: false},
|
||||
{id: "Extractor:AutoExtractDefault", want: true},
|
||||
{id: "extractor:node", want: false},
|
||||
{id: "EXTRACTOR:Node", want: false},
|
||||
{id: "Extractor:", want: false},
|
||||
{id: "extractor_0", want: false},
|
||||
{id: "Extractor_0", want: false},
|
||||
{id: "extractors", want: false},
|
||||
{id: "my_extractor", want: false},
|
||||
{id: "Parser:Extractor", want: false},
|
||||
{id: "", want: false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.id, func(t *testing.T) {
|
||||
if got := IsExtractorComponentID(tt.id); got != tt.want {
|
||||
t.Fatalf("IsExtractorComponentID(%q) = %v, want %v", tt.id, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsParserComponentID(t *testing.T) {
|
||||
tests := []struct {
|
||||
id string
|
||||
want bool
|
||||
}{
|
||||
{id: "Parser:HipSignsRhyme", want: true},
|
||||
{id: "Parser:", want: false},
|
||||
{id: "parser:node", want: false},
|
||||
{id: "PARSER:Node", want: false},
|
||||
{id: "Parser", want: false},
|
||||
{id: "parser_0", want: false},
|
||||
{id: "Extractor:Parser", want: false},
|
||||
{id: "", want: false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.id, func(t *testing.T) {
|
||||
if got := IsParserComponentID(tt.id); got != tt.want {
|
||||
t.Fatalf("IsParserComponentID(%q) = %v, want %v", tt.id, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsCompilerComponentID(t *testing.T) {
|
||||
tests := []struct {
|
||||
id string
|
||||
want bool
|
||||
}{
|
||||
{id: "Compiler:CurvySunsLove", want: true},
|
||||
{id: "Compiler:", want: false},
|
||||
{id: "compiler:node", want: false},
|
||||
{id: "COMPILER:Node", want: false},
|
||||
{id: "compiler_0", want: false},
|
||||
{id: "Compiler", want: false},
|
||||
{id: "compilers", want: false},
|
||||
{id: "Parser:Compiler", want: false},
|
||||
{id: "", want: false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.id, func(t *testing.T) {
|
||||
if got := IsCompilerComponentID(tt.id); got != tt.want {
|
||||
t.Fatalf("IsCompilerComponentID(%q) = %v, want %v", tt.id, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsComponentKey(t *testing.T) {
|
||||
tests := []struct {
|
||||
id string
|
||||
want bool
|
||||
}{
|
||||
{id: "Extractor:AutoExtractDefault", want: true},
|
||||
{id: "Parser:HipSignsRhyme", want: true},
|
||||
{id: "Parser:", want: false},
|
||||
{id: "GeneralChunker:SixApplesFall", want: true},
|
||||
{id: "File:node", want: true},
|
||||
{id: "Extractor:", want: false},
|
||||
{id: ":foo", want: false},
|
||||
{id: "foo:bar", want: false},
|
||||
{id: "extractor:0", want: false},
|
||||
{id: "extractor_0", want: false},
|
||||
{id: "compiler_0", want: false},
|
||||
{id: "Extractor", want: false},
|
||||
{id: "chunk_token_num", want: false},
|
||||
{id: "parser_0", want: false},
|
||||
{id: "", want: false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.id, func(t *testing.T) {
|
||||
if got := IsComponentKey(tt.id); got != tt.want {
|
||||
t.Fatalf("IsComponentKey(%q) = %v, want %v", tt.id, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsComponentName(t *testing.T) {
|
||||
names := []string{
|
||||
ComponentNameFile,
|
||||
ComponentNameParser,
|
||||
ComponentNameExtractor,
|
||||
ComponentNameCompiler,
|
||||
ComponentNameTokenizer,
|
||||
ComponentNameGeneralChunker,
|
||||
ComponentNameTokenChunker,
|
||||
ComponentNameTitleChunker,
|
||||
ComponentNameManualChunker,
|
||||
ComponentNameOneChunker,
|
||||
ComponentNamePageChunker,
|
||||
ComponentNameQAChunker,
|
||||
ComponentNameTableChunker,
|
||||
ComponentNameGroupTitleChunker,
|
||||
ComponentNameHierarchyTitleChunker,
|
||||
}
|
||||
for _, name := range names {
|
||||
if !IsComponentName(name) {
|
||||
t.Errorf("IsComponentName(%q) = false, want true", name)
|
||||
}
|
||||
}
|
||||
for _, name := range []string{"", "Chunker", "extractor", "Extractor "} {
|
||||
if IsComponentName(name) {
|
||||
t.Errorf("IsComponentName(%q) = true, want false", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSortedExtractorKeys(t *testing.T) {
|
||||
parserConfig := map[string]any{
|
||||
"Extractor:B": nil,
|
||||
"Extractor:A": nil,
|
||||
"Extractor:": nil,
|
||||
"Extractor": nil,
|
||||
"extractor_0": nil,
|
||||
"Parser:X": nil,
|
||||
}
|
||||
want := []string{"Extractor:A", "Extractor:B"}
|
||||
if got := SortedExtractorKeys(parserConfig); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("SortedExtractorKeys = %#v, want %#v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsExtractorComponent(t *testing.T) {
|
||||
tests := []struct {
|
||||
id string
|
||||
name string
|
||||
want bool
|
||||
}{
|
||||
{id: "node-1", name: "Extractor", want: true},
|
||||
{id: "Extractor:node-1", name: "", want: true},
|
||||
{id: "extractor_0", name: "", want: false},
|
||||
{id: "node-1", name: "Parser", want: false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name+"/"+tt.id, func(t *testing.T) {
|
||||
if got := IsExtractorComponent(tt.id, tt.name); got != tt.want {
|
||||
t.Fatalf("IsExtractorComponent(%q, %q) = %v, want %v", tt.id, tt.name, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsParserComponent(t *testing.T) {
|
||||
tests := []struct {
|
||||
id string
|
||||
name string
|
||||
want bool
|
||||
}{
|
||||
{id: "node-1", name: "Parser", want: true},
|
||||
{id: "Parser:node-1", name: "", want: true},
|
||||
{id: "node-1", name: "Extractor", want: false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name+"/"+tt.id, func(t *testing.T) {
|
||||
if got := IsParserComponent(tt.id, tt.name); got != tt.want {
|
||||
t.Fatalf("IsParserComponent(%q, %q) = %v, want %v", tt.id, tt.name, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package common
|
||||
|
||||
// Canonical ingestion component names.
|
||||
const (
|
||||
ComponentNameFile = "File"
|
||||
ComponentNameParser = "Parser"
|
||||
ComponentNameExtractor = "Extractor"
|
||||
ComponentNameCompiler = "Compiler"
|
||||
ComponentNameTokenizer = "Tokenizer"
|
||||
|
||||
ComponentNameGeneralChunker = "GeneralChunker"
|
||||
ComponentNameTokenChunker = "TokenChunker"
|
||||
ComponentNameTitleChunker = "TitleChunker"
|
||||
ComponentNameManualChunker = "ManualChunker"
|
||||
ComponentNameOneChunker = "OneChunker"
|
||||
ComponentNamePageChunker = "PageChunker"
|
||||
ComponentNameQAChunker = "QAChunker"
|
||||
ComponentNameTableChunker = "TableChunker"
|
||||
ComponentNameGroupTitleChunker = "GroupTitleChunker"
|
||||
ComponentNameHierarchyTitleChunker = "HierarchyTitleChunker"
|
||||
)
|
||||
|
||||
// IsComponentName reports whether name is a canonical ingestion component name.
|
||||
func IsComponentName(name string) bool {
|
||||
switch name {
|
||||
case ComponentNameFile,
|
||||
ComponentNameParser,
|
||||
ComponentNameExtractor,
|
||||
ComponentNameCompiler,
|
||||
ComponentNameTokenizer,
|
||||
ComponentNameGeneralChunker,
|
||||
ComponentNameTokenChunker,
|
||||
ComponentNameTitleChunker,
|
||||
ComponentNameManualChunker,
|
||||
ComponentNameOneChunker,
|
||||
ComponentNamePageChunker,
|
||||
ComponentNameQAChunker,
|
||||
ComponentNameTableChunker,
|
||||
ComponentNameGroupTitleChunker,
|
||||
ComponentNameHierarchyTitleChunker:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
@@ -18,7 +18,6 @@ package common
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
@@ -441,15 +440,7 @@ func ExtractorMetadataConfig(parserConfig map[string]any) (map[string]any, bool)
|
||||
if parserConfig == nil {
|
||||
return nil, false
|
||||
}
|
||||
var extractorKeys []string
|
||||
for k := range parserConfig {
|
||||
lower := strings.ToLower(k)
|
||||
if strings.HasPrefix(lower, "extractor:") || strings.HasPrefix(lower, "extractor_") {
|
||||
extractorKeys = append(extractorKeys, k)
|
||||
}
|
||||
}
|
||||
sort.Strings(extractorKeys)
|
||||
for _, k := range extractorKeys {
|
||||
for _, k := range SortedExtractorKeys(parserConfig) {
|
||||
node, ok := parserConfig[k].(map[string]any)
|
||||
if !ok {
|
||||
continue
|
||||
|
||||
@@ -105,7 +105,7 @@ func ExtractPipelineDefaults(dsl map[string]interface{}) map[string]interface{}
|
||||
continue
|
||||
}
|
||||
name, _ := obj["component_name"].(string)
|
||||
if name == "" || name == "File" {
|
||||
if name == "" || name == ComponentNameFile {
|
||||
continue
|
||||
}
|
||||
params, _ := obj["params"].(map[string]interface{})
|
||||
|
||||
@@ -1164,12 +1164,9 @@ func migrateExtractorConfig(config map[string]any, parserID string, pipelineID *
|
||||
targets = append(targets, newNode)
|
||||
}
|
||||
} else {
|
||||
for id, raw := range config {
|
||||
id = strings.ToLower(id)
|
||||
if strings.HasPrefix(id, "extractor:") || strings.HasPrefix(id, "extractor_") {
|
||||
if node, ok := raw.(map[string]any); ok {
|
||||
targets = append(targets, node)
|
||||
}
|
||||
for _, id := range common.SortedExtractorKeys(config) {
|
||||
if node, ok := config[id].(map[string]any); ok {
|
||||
targets = append(targets, node)
|
||||
}
|
||||
}
|
||||
if len(targets) == 0 {
|
||||
|
||||
@@ -44,8 +44,6 @@ import (
|
||||
"ragflow/internal/parser/tableutil"
|
||||
)
|
||||
|
||||
const ComponentNameGeneralChunker = "GeneralChunker"
|
||||
|
||||
var contextSentencePattern = regexp.MustCompile(`([。!??;!\n]|\. )`)
|
||||
|
||||
type generalChunkerParam struct {
|
||||
|
||||
@@ -51,8 +51,6 @@ import (
|
||||
"ragflow/internal/tokenizer"
|
||||
)
|
||||
|
||||
const ComponentNameGroupTitleChunker = "GroupTitleChunker"
|
||||
|
||||
// MIN_GROUP_TOKENS / MAX_GROUP_TOKENS mirror the python constants in
|
||||
// group_chunker.py:22-23. They drive the merge heuristic for adjacent
|
||||
// text records.
|
||||
|
||||
@@ -51,8 +51,6 @@ import (
|
||||
"ragflow/internal/ingestion/component/globals"
|
||||
)
|
||||
|
||||
const ComponentNameHierarchyTitleChunker = "HierarchyTitleChunker"
|
||||
|
||||
// chunkNode mirrors python `_ChunkNode` (hierarchy_chunker.py:22-32).
|
||||
type chunkNode struct {
|
||||
level int
|
||||
|
||||
@@ -33,8 +33,6 @@ import (
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
const ComponentNameManualChunker = "ManualChunker"
|
||||
|
||||
// ManualChunkerComponent is the standalone manual-layout chunker. It shares
|
||||
// the GroupTitleChunker grouping body (chunkFromRecords) and only differs by
|
||||
// the optional physical-position pre-sort.
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package chunker
|
||||
|
||||
import "ragflow/internal/common"
|
||||
|
||||
// Chunker component names re-exported from the canonical component registry.
|
||||
const (
|
||||
ComponentNameGeneralChunker = common.ComponentNameGeneralChunker
|
||||
ComponentNameTokenChunker = common.ComponentNameTokenChunker
|
||||
ComponentNameTitleChunker = common.ComponentNameTitleChunker
|
||||
ComponentNameManualChunker = common.ComponentNameManualChunker
|
||||
ComponentNameOneChunker = common.ComponentNameOneChunker
|
||||
ComponentNamePageChunker = common.ComponentNamePageChunker
|
||||
ComponentNameQAChunker = common.ComponentNameQAChunker
|
||||
ComponentNameTableChunker = common.ComponentNameTableChunker
|
||||
ComponentNameGroupTitleChunker = common.ComponentNameGroupTitleChunker
|
||||
ComponentNameHierarchyTitleChunker = common.ComponentNameHierarchyTitleChunker
|
||||
)
|
||||
@@ -38,8 +38,6 @@ import (
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
const ComponentNameOneChunker = "OneChunker"
|
||||
|
||||
type oneChunkerParam struct{}
|
||||
|
||||
func (p *oneChunkerParam) Update(conf map[string]any) {}
|
||||
|
||||
@@ -34,8 +34,6 @@ import (
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
const ComponentNamePageChunker = "PageChunker"
|
||||
|
||||
type pageChunkerParam struct{}
|
||||
|
||||
func (p *pageChunkerParam) Update(conf map[string]any) {}
|
||||
|
||||
@@ -48,8 +48,6 @@ import (
|
||||
"ragflow/internal/tokenizer"
|
||||
)
|
||||
|
||||
const ComponentNameQAChunker = "QAChunker"
|
||||
|
||||
type qaChunkerParam struct {
|
||||
Lang string `json:"lang,omitempty"`
|
||||
}
|
||||
|
||||
@@ -41,8 +41,6 @@ import (
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
const ComponentNameTableChunker = "TableChunker"
|
||||
|
||||
type tableChunkerParam struct{}
|
||||
|
||||
func (p *tableChunkerParam) Update(conf map[string]any) {}
|
||||
|
||||
@@ -82,8 +82,6 @@ import (
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
const ComponentNameTitleChunker = "TitleChunker"
|
||||
|
||||
type titleChunkerParam struct {
|
||||
schema.TitleChunkerParam
|
||||
}
|
||||
|
||||
@@ -90,8 +90,6 @@ import (
|
||||
"ragflow/internal/parser/chunk"
|
||||
)
|
||||
|
||||
const ComponentNameTokenChunker = "TokenChunker"
|
||||
|
||||
type tokenChunkerParam struct {
|
||||
schema.TokenChunkerParam
|
||||
}
|
||||
|
||||
@@ -66,8 +66,8 @@ import (
|
||||
)
|
||||
|
||||
// componentNameExtractor is the canonical Extractor component name; the value
|
||||
// lives in the schema package so the pipeline layer can reference the same
|
||||
// identity.
|
||||
// lives in common and is re-exported by the schema package so both layers
|
||||
// reference the same identity.
|
||||
const componentNameExtractor = schema.ComponentNameExtractor
|
||||
|
||||
// defaultExtractorChatTimeout bounds one Extractor LLM call, including
|
||||
|
||||
@@ -50,7 +50,7 @@ import (
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
const ComponentNameFile = "File"
|
||||
const ComponentNameFile = schema.ComponentNameFile
|
||||
|
||||
// FileComponent resolves document/file metadata and forwards enough
|
||||
// identity for downstream Parser to fetch bytes.
|
||||
|
||||
@@ -47,8 +47,8 @@ var chunkerOutputs = map[string]string{
|
||||
// (rag/flow/compiler/compiler.py registers component_name = "Compiler"), so a
|
||||
// canvas saved by the Python frontend and Go's built-in ingestion templates
|
||||
// both reference the node as "Compiler" and resolve to the same component.
|
||||
// The value itself lives in the schema package so the pipeline layer can
|
||||
// reference the same identity.
|
||||
// The value itself lives in common and is re-exported by the schema package so
|
||||
// the pipeline layer references the same identity.
|
||||
const componentNameCompiler = schema.ComponentNameCompiler
|
||||
|
||||
// KnowledgeCompilerComponent is the runtime.Component surface. Param is set at
|
||||
|
||||
@@ -96,7 +96,8 @@ import (
|
||||
)
|
||||
|
||||
// ComponentNameParser is the canonical Parser component name; the value lives
|
||||
// in the schema package so the pipeline layer can reference the same identity.
|
||||
// in common and is re-exported by the schema package so both layers reference
|
||||
// the same identity.
|
||||
const ComponentNameParser = schema.ComponentNameParser
|
||||
|
||||
// pageFormFeed is the byte that text-page mode treats as a
|
||||
|
||||
@@ -16,16 +16,19 @@
|
||||
|
||||
package schema
|
||||
|
||||
// Canonical ingestion component names. These are the identities components
|
||||
// register under in runtime.DefaultRegistry and the values a DSL
|
||||
// component_name carries. Layers that must recognise a component by name
|
||||
// across the component/pipeline boundary (dispatch, param whitelists,
|
||||
// operator-facing failure messages) reference these constants instead of
|
||||
// repeating the literals; the schema package is the shared dependency both
|
||||
// sides already import. More names can join this set as other layers need
|
||||
// them.
|
||||
import "ragflow/internal/common"
|
||||
|
||||
// Canonical ingestion component names re-exported from common for callers that
|
||||
// need them across the pipeline/component boundary.
|
||||
const (
|
||||
ComponentNameParser = "Parser"
|
||||
ComponentNameExtractor = "Extractor"
|
||||
ComponentNameCompiler = "Compiler"
|
||||
ComponentNameFile = common.ComponentNameFile
|
||||
ComponentNameParser = common.ComponentNameParser
|
||||
ComponentNameExtractor = common.ComponentNameExtractor
|
||||
ComponentNameCompiler = common.ComponentNameCompiler
|
||||
ComponentNameTokenizer = common.ComponentNameTokenizer
|
||||
|
||||
ComponentNameGeneralChunker = common.ComponentNameGeneralChunker
|
||||
ComponentNameTokenChunker = common.ComponentNameTokenChunker
|
||||
ComponentNameTitleChunker = common.ComponentNameTitleChunker
|
||||
ComponentNameManualChunker = common.ComponentNameManualChunker
|
||||
)
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"ragflow/internal/common"
|
||||
"ragflow/internal/entity"
|
||||
"ragflow/internal/ingestion/component/schema"
|
||||
"ragflow/internal/tokenizer"
|
||||
@@ -87,20 +88,9 @@ func TagFileIDFromParserConfig(parserConfig map[string]any) string {
|
||||
if v := tagFileIDFromTagsMap(parserConfig["tags"]); v != "" {
|
||||
return v
|
||||
}
|
||||
// Several Extractor nodes can each declare their own tags block, so the
|
||||
// winner has to be picked in a stable order: Go randomizes map iteration,
|
||||
// and an unstable pick would let the dataset's tag source change between
|
||||
// process restarts (different chunks tagged, different retrieval
|
||||
// features). Sorting makes it deterministic; "Extractor" sorts before
|
||||
// "Extractor:<name>" because it is a prefix, so the unscoped node wins.
|
||||
keys := make([]string, 0, len(parserConfig))
|
||||
for key := range parserConfig {
|
||||
if key == "Extractor" || strings.HasPrefix(key, "Extractor:") {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, key := range keys {
|
||||
// Several Extractor nodes can each declare their own tags block. Pick the
|
||||
// winner deterministically by sorted component ID.
|
||||
for _, key := range common.SortedExtractorKeys(parserConfig) {
|
||||
extractorMap, ok := asStringAnyMap(parserConfig[key])
|
||||
if !ok {
|
||||
continue
|
||||
|
||||
@@ -56,6 +56,33 @@ func TestTagFileIDFromParserConfig(t *testing.T) {
|
||||
},
|
||||
want: "file-3",
|
||||
},
|
||||
{
|
||||
name: "lowercase extractor form is not canonical",
|
||||
parserConfig: map[string]any{
|
||||
"extractor:AutoExtractDefault": map[string]any{
|
||||
"tags": map[string]any{"tag_file_id": "file-5"},
|
||||
},
|
||||
},
|
||||
want: "",
|
||||
},
|
||||
{
|
||||
name: "bare extractor form is not scoped",
|
||||
parserConfig: map[string]any{
|
||||
"Extractor": map[string]any{
|
||||
"tags": map[string]any{"tag_file_id": "file-6"},
|
||||
},
|
||||
},
|
||||
want: "",
|
||||
},
|
||||
{
|
||||
name: "underscore extractor form is not scoped",
|
||||
parserConfig: map[string]any{
|
||||
"extractor_0": map[string]any{
|
||||
"tags": map[string]any{"tag_file_id": "file-7"},
|
||||
},
|
||||
},
|
||||
want: "",
|
||||
},
|
||||
{
|
||||
name: "entity.JSONMap nested value",
|
||||
parserConfig: map[string]any{
|
||||
@@ -85,29 +112,17 @@ func TestTagFileIDFromParserConfig(t *testing.T) {
|
||||
|
||||
func TestTagFileIDFromParserConfigPicksStableExtractorNode(t *testing.T) {
|
||||
// Several Extractor nodes can each declare their own tags block. Go
|
||||
// randomizes map iteration, so an unsorted scan would return a different
|
||||
// tag file - and thus tag the dataset from a different tag source - on
|
||||
// every call and every process restart.
|
||||
parserConfig := map[string]any{
|
||||
"Extractor": map[string]any{"tags": map[string]any{"tag_file_id": "file-unscoped"}},
|
||||
}
|
||||
// randomizes map iteration, so the chosen tag source must not depend on
|
||||
// iteration order.
|
||||
parserConfig := map[string]any{}
|
||||
for i := 0; i < 8; i++ {
|
||||
parserConfig[fmt.Sprintf("Extractor:node-%d", i)] = map[string]any{
|
||||
"tags": map[string]any{"tag_file_id": fmt.Sprintf("file-%d", i)},
|
||||
}
|
||||
}
|
||||
// The unscoped "Extractor" node wins: it is a prefix of the scoped keys,
|
||||
// so it sorts first.
|
||||
for i := 0; i < 200; i++ {
|
||||
if got := TagFileIDFromParserConfig(parserConfig); got != "file-unscoped" {
|
||||
t.Fatalf("TagFileIDFromParserConfig with unscoped node = %q, want %q", got, "file-unscoped")
|
||||
}
|
||||
}
|
||||
// Without it the lowest-sorting scoped node wins, still stably.
|
||||
delete(parserConfig, "Extractor")
|
||||
for i := 0; i < 200; i++ {
|
||||
if got := TagFileIDFromParserConfig(parserConfig); got != "file-0" {
|
||||
t.Fatalf("TagFileIDFromParserConfig across scoped nodes = %q, want %q", got, "file-0")
|
||||
t.Fatalf("TagFileIDFromParserConfig = %q, want %q", got, "file-0")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -98,7 +98,7 @@ import (
|
||||
"ragflow/internal/tokenizer"
|
||||
)
|
||||
|
||||
const ComponentNameTokenizer = "Tokenizer"
|
||||
const ComponentNameTokenizer = schema.ComponentNameTokenizer
|
||||
|
||||
// titleExtRE strips a trailing file-extension (e.g. ".pdf") from the
|
||||
// upstream document name before tokenizing it. Mirrors the python
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"ragflow/internal/common"
|
||||
"ragflow/internal/ingestion/component/schema"
|
||||
)
|
||||
|
||||
@@ -42,7 +43,10 @@ func ExtractAllComponentParams(dslJSON []byte) ([]ComponentParamsSchema, error)
|
||||
|
||||
extractorCount := 0
|
||||
for cpnID, comp := range dsl.Components {
|
||||
if isExtractorComponent(cpnID, comp.Obj.ComponentName) {
|
||||
if cpnID != common.ComponentNameFile && !common.IsComponentKey(cpnID) {
|
||||
return nil, fmt.Errorf("ExtractAllComponentParams: component ID %q must be <name>:<instance>; re-save the pipeline with canonical component IDs", cpnID)
|
||||
}
|
||||
if common.IsExtractorComponent(cpnID, comp.Obj.ComponentName) {
|
||||
extractorCount++
|
||||
}
|
||||
}
|
||||
@@ -62,7 +66,7 @@ func ExtractAllComponentParams(dslJSON []byte) ([]ComponentParamsSchema, error)
|
||||
// Canvases saved by the Python-era frontend nest the Parser's
|
||||
// per-family setups under a "setups" key; lift them so the extracted
|
||||
// defaults (and the parser_config built from them) stay flat.
|
||||
if isParserComponent(cpnID, comp.Obj.ComponentName) {
|
||||
if common.IsParserComponent(cpnID, comp.Obj.ComponentName) {
|
||||
params = schema.FlattenLegacyParserSetups(params)
|
||||
}
|
||||
out = append(out, ComponentParamsSchema{
|
||||
|
||||
@@ -196,6 +196,20 @@ func TestExtractAllComponentParams_MissingComponents(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractAllComponentParams_RejectsUnscopedComponentIDs(t *testing.T) {
|
||||
for _, id := range []string{"Extractor", "extractor_0", "parser_0"} {
|
||||
dslJSON := []byte(`{"components":{"` + id + `":{"obj":{"component_name":"Extractor","params":{}}}}}`)
|
||||
if _, err := ExtractAllComponentParams(dslJSON); err == nil {
|
||||
t.Errorf("ExtractAllComponentParams(%q) expected error for unscoped component ID", id)
|
||||
}
|
||||
}
|
||||
// File is the one bare component ID used by built-in ingestion templates.
|
||||
dslJSON := []byte(`{"components":{"File":{"obj":{"component_name":"File","params":{}}}}}`)
|
||||
if _, err := ExtractAllComponentParams(dslJSON); err != nil {
|
||||
t.Fatalf("ExtractAllComponentParams(File): %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func componentIDs(schemas []ComponentParamsSchema) []string {
|
||||
ids := make([]string, len(schemas))
|
||||
for i, s := range schemas {
|
||||
|
||||
@@ -149,7 +149,7 @@ func ValidatePipeline(cnv *canvas.Canvas) error {
|
||||
}
|
||||
extractorCount := 0
|
||||
for id, comp := range cnv.Components {
|
||||
if isExtractorComponent(id, comp.Obj.ComponentName) {
|
||||
if common.IsExtractorComponent(id, comp.Obj.ComponentName) {
|
||||
extractorCount++
|
||||
}
|
||||
}
|
||||
|
||||
@@ -83,14 +83,14 @@ var titleFamilyParamKeys = unionParamKeys(
|
||||
)
|
||||
|
||||
var componentParamSchemaKeys = map[string]map[string]struct{}{
|
||||
"titlechunker": titleFamilyParamKeys,
|
||||
strings.ToLower(schema.ComponentNameTitleChunker): titleFamilyParamKeys,
|
||||
// ManualChunker reuses TitleChunkerParam. It pins method=group and is exempt
|
||||
// from the token cap, so method, chunk_token_cap and include_heading_content
|
||||
// (hierarchy-only) are accepted here but ignored by the component — the
|
||||
// operator form omits them for the same reason.
|
||||
"manualchunker": titleFamilyParamKeys,
|
||||
"tokenchunker": extractJSONTags(schema.TokenChunkerParam{}),
|
||||
"tokenizer": extractJSONTags(schema.TokenizerParam{}),
|
||||
strings.ToLower(schema.ComponentNameManualChunker): titleFamilyParamKeys,
|
||||
strings.ToLower(schema.ComponentNameTokenChunker): extractJSONTags(schema.TokenChunkerParam{}),
|
||||
strings.ToLower(schema.ComponentNameTokenizer): extractJSONTags(schema.TokenizerParam{}),
|
||||
}
|
||||
|
||||
// extractJSONTags returns all top-level json tag names from a struct.
|
||||
@@ -119,38 +119,6 @@ func extractJSONTags(v any) map[string]struct{} {
|
||||
return tags
|
||||
}
|
||||
|
||||
// isExtractorComponent returns true if the component name or component ID
|
||||
// indicates an Extractor component.
|
||||
func isExtractorComponent(cpnID, componentName string) bool {
|
||||
lowerName := strings.ToLower(componentName)
|
||||
lowerID := strings.ToLower(cpnID)
|
||||
return lowerName == strings.ToLower(schema.ComponentNameExtractor) ||
|
||||
strings.HasPrefix(lowerID, "extractor:") ||
|
||||
strings.HasPrefix(lowerID, "extractor_")
|
||||
}
|
||||
|
||||
// isCompilerComponent returns true if the component ID indicates a Compiler
|
||||
// component, whose LLM-backed runtime parameters follow the same surface as
|
||||
// the Extractor's.
|
||||
func isCompilerComponent(cpnID string) bool {
|
||||
lowerID := strings.ToLower(cpnID)
|
||||
return strings.HasPrefix(lowerID, "compiler:") ||
|
||||
strings.HasPrefix(lowerID, "compiler_")
|
||||
}
|
||||
|
||||
// IsParserComponentID reports whether a component ID names a Parser component.
|
||||
// A parser_config map is keyed by component ID, where the component name is not
|
||||
// available, so this is the config-side half of isParserComponent.
|
||||
func IsParserComponentID(cpnID string) bool {
|
||||
return strings.HasPrefix(strings.ToLower(cpnID), "parser:")
|
||||
}
|
||||
|
||||
// isParserComponent returns true if the component name or component ID
|
||||
// indicates a Parser component.
|
||||
func isParserComponent(cpnID, componentName string) bool {
|
||||
return strings.EqualFold(componentName, schema.ComponentNameParser) || IsParserComponentID(cpnID)
|
||||
}
|
||||
|
||||
// NormalizeParserConfigSetups lifts legacy nested Parser "setups" entries in
|
||||
// a parser_config-shaped map ({cpnID: params}) into the flat per-family
|
||||
// shape. The runtime merges such a map over the DSL-baked component params
|
||||
@@ -163,7 +131,7 @@ func NormalizeParserConfigSetups(parserConfig map[string]any) map[string]any {
|
||||
return nil
|
||||
}
|
||||
for cpnID, raw := range parserConfig {
|
||||
if !isParserComponent(cpnID, "") {
|
||||
if !common.IsParserComponentID(cpnID) {
|
||||
continue
|
||||
}
|
||||
params, ok := raw.(map[string]any)
|
||||
@@ -177,22 +145,19 @@ func NormalizeParserConfigSetups(parserConfig map[string]any) map[string]any {
|
||||
|
||||
// getComponentParamWhitelist returns the dynamic parameter whitelist for a component type.
|
||||
func getComponentParamWhitelist(cpnID string) (map[string]struct{}, bool) {
|
||||
if isExtractorComponent(cpnID, "") {
|
||||
if common.IsExtractorComponentID(cpnID) {
|
||||
return extractorValidParamKeys, true
|
||||
}
|
||||
if isCompilerComponent(cpnID) {
|
||||
if common.IsCompilerComponentID(cpnID) {
|
||||
return llmRuntimeParamKeys, true
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// cleanComponentParams filters rawConfig against the extracted DSL schemas.
|
||||
// Keys containing ':' are treated as component IDs; they are kept only when both
|
||||
// the cpnID AND the param name exist in one of the accepted-key sources: the DSL
|
||||
// schema (the params the template bakes), the schema-derived per-component table
|
||||
// (componentParamSchemaKeys), or the component's dynamic parameter schema (e.g.
|
||||
// Extractor modular features). Keys without ':' (legacy flat fields) are dropped
|
||||
// with a warning.
|
||||
// cleanComponentParams keeps only component-scoped keys recognized by
|
||||
// common.IsComponentKey, and only params accepted by the DSL schema, the
|
||||
// per-component table, or the component's dynamic schema. Flat fields are
|
||||
// dropped with a warning.
|
||||
func cleanComponentParams(schemas []ComponentParamsSchema, rawConfig map[string]interface{}) map[string]interface{} {
|
||||
validCPNs := make(map[string]map[string]struct{}, len(schemas))
|
||||
componentNames := make(map[string]string, len(schemas))
|
||||
@@ -201,10 +166,10 @@ func cleanComponentParams(schemas []ComponentParamsSchema, rawConfig map[string]
|
||||
for k := range s.ParamsDefaults {
|
||||
keys[k] = struct{}{}
|
||||
}
|
||||
if s.ComponentName == "GeneralChunker" {
|
||||
if s.ComponentName == schema.ComponentNameGeneralChunker {
|
||||
keys["delimiters"] = struct{}{}
|
||||
}
|
||||
if isParserComponent(s.CpnID, s.ComponentName) {
|
||||
if common.IsParserComponent(s.CpnID, s.ComponentName) {
|
||||
keys["enable_vision_enhancement"] = struct{}{}
|
||||
keys["vlm"] = struct{}{}
|
||||
}
|
||||
@@ -225,7 +190,7 @@ func cleanComponentParams(schemas []ComponentParamsSchema, rawConfig map[string]
|
||||
|
||||
result := make(map[string]interface{}, len(rawConfig))
|
||||
for key, val := range rawConfig {
|
||||
if !strings.Contains(key, ":") {
|
||||
if !common.IsComponentKey(key) {
|
||||
common.Warn("cleanComponentParams: dropping legacy flat field",
|
||||
zap.String("key", key))
|
||||
continue
|
||||
@@ -240,18 +205,18 @@ func cleanComponentParams(schemas []ComponentParamsSchema, rawConfig map[string]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if componentNames[key] == "GeneralChunker" {
|
||||
if componentNames[key] == schema.ComponentNameGeneralChunker {
|
||||
params = normalizeGeneralComponentParams(params)
|
||||
}
|
||||
// Legacy parser_config rows (or canvases) saved by the Python-era
|
||||
// frontend nest the Parser's setups under a "setups" key; lift them
|
||||
// before filtering, otherwise the families are dropped as unknown
|
||||
// params against the flat DSL schema keys.
|
||||
if isParserComponent(key, componentNames[key]) {
|
||||
if common.IsParserComponent(key, componentNames[key]) {
|
||||
params = schema.FlattenLegacyParserSetups(params)
|
||||
}
|
||||
dynamicWhitelist, hasDynamic := getComponentParamWhitelist(key)
|
||||
if isExtractorComponent(key, "") {
|
||||
if common.IsExtractorComponentID(key) {
|
||||
params = NormalizeExtractorParams(params)
|
||||
}
|
||||
cleaned := make(map[string]any, len(params))
|
||||
@@ -430,5 +395,6 @@ func ApplyParentChildChunkerConfig(componentConfig entity.JSONMap, rawConfig map
|
||||
// chunks into children.
|
||||
func IsChunkerComponent(componentID string) bool {
|
||||
lowerID := strings.ToLower(componentID)
|
||||
return strings.HasPrefix(lowerID, "generalchunker:") || strings.HasPrefix(lowerID, "tokenchunker:")
|
||||
return strings.HasPrefix(lowerID, strings.ToLower(schema.ComponentNameGeneralChunker)+":") ||
|
||||
strings.HasPrefix(lowerID, strings.ToLower(schema.ComponentNameTokenChunker)+":")
|
||||
}
|
||||
@@ -34,9 +34,9 @@ func generalDSL(t *testing.T) []byte {
|
||||
},
|
||||
},
|
||||
},
|
||||
"Chunker:LegalReadersDecide": map[string]any{
|
||||
"GeneralChunker:LegalReadersDecide": map[string]any{
|
||||
"obj": map[string]any{
|
||||
"component_name": "Chunker",
|
||||
"component_name": "GeneralChunker",
|
||||
"params": map[string]any{
|
||||
"outputs": map[string]any{},
|
||||
"chunk_size": float64(512),
|
||||
@@ -165,9 +165,9 @@ func TestBuildParserConfig_ShallowMerge_NestedParam(t *testing.T) {
|
||||
// A component with a nested-map param "chunk" that has sub-keys.
|
||||
dsl := map[string]any{
|
||||
"components": map[string]any{
|
||||
"Chunker:xyz": map[string]any{
|
||||
"GeneralChunker:xyz": map[string]any{
|
||||
"obj": map[string]any{
|
||||
"component_name": "Chunker",
|
||||
"component_name": "GeneralChunker",
|
||||
"params": map[string]any{
|
||||
"outputs": map[string]any{},
|
||||
"chunk": map[string]any{
|
||||
@@ -186,15 +186,15 @@ func TestBuildParserConfig_ShallowMerge_NestedParam(t *testing.T) {
|
||||
|
||||
// User only overrides one sub-key of "chunk".
|
||||
overrides := map[string]any{
|
||||
"Chunker:xyz": map[string]any{
|
||||
"GeneralChunker:xyz": map[string]any{
|
||||
"chunk": map[string]any{"size": float64(1024)},
|
||||
},
|
||||
}
|
||||
|
||||
result := buildTestParserConfig(t, dslJSON, overrides)
|
||||
chunker, ok := result["Chunker:xyz"].(map[string]any)
|
||||
chunker, ok := result["GeneralChunker:xyz"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatal("expected Chunker:xyz in result")
|
||||
t.Fatal("expected GeneralChunker:xyz in result")
|
||||
}
|
||||
chunk, ok := chunker["chunk"].(map[string]any)
|
||||
if !ok {
|
||||
@@ -348,14 +348,14 @@ func TestNormalizeParserConfigSetups_EmptyNestedMap(t *testing.T) {
|
||||
func TestBuildParserConfig_ScalarOverridePreservesOtherDefaults(t *testing.T) {
|
||||
dslJSON := generalDSL(t)
|
||||
overrides := map[string]any{
|
||||
"Chunker:LegalReadersDecide": map[string]any{
|
||||
"GeneralChunker:LegalReadersDecide": map[string]any{
|
||||
"chunk_size": float64(1024),
|
||||
},
|
||||
}
|
||||
result := buildTestParserConfig(t, dslJSON, overrides)
|
||||
chunker, ok := result["Chunker:LegalReadersDecide"].(map[string]any)
|
||||
chunker, ok := result["GeneralChunker:LegalReadersDecide"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatal("expected Chunker:LegalReadersDecide in result")
|
||||
t.Fatal("expected GeneralChunker:LegalReadersDecide in result")
|
||||
}
|
||||
if chunker["chunk_size"] != float64(1024) {
|
||||
t.Errorf("expected chunk_size=1024, got %v", chunker["chunk_size"])
|
||||
@@ -398,8 +398,8 @@ func TestBuildParserConfig_AllComponentsPresent(t *testing.T) {
|
||||
if _, ok := result["Parser:HipSignsRhyme"]; !ok {
|
||||
t.Error("expected Parser:HipSignsRhyme")
|
||||
}
|
||||
if _, ok := result["Chunker:LegalReadersDecide"]; !ok {
|
||||
t.Error("expected Chunker:LegalReadersDecide")
|
||||
if _, ok := result["GeneralChunker:LegalReadersDecide"]; !ok {
|
||||
t.Error("expected GeneralChunker:LegalReadersDecide")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -470,7 +470,7 @@ func TestComponentParamsDefaults_Basic(t *testing.T) {
|
||||
if _, ok := parser["pdf"]; !ok {
|
||||
t.Error("expected pdf to be present")
|
||||
}
|
||||
chunker := result["Chunker:LegalReadersDecide"]
|
||||
chunker := result["GeneralChunker:LegalReadersDecide"]
|
||||
if chunker["chunk_size"] != float64(512) {
|
||||
t.Errorf("expected chunk_size=512, got %v", chunker["chunk_size"])
|
||||
}
|
||||
|
||||
@@ -21,6 +21,8 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"ragflow/internal/common"
|
||||
)
|
||||
|
||||
// parserSetupParamsKeys are the Parser params keys that are NOT per-family
|
||||
@@ -108,7 +110,7 @@ func assertNoParserOutputFormatInGraph(t *testing.T, ref string, root map[string
|
||||
for _, rawNode := range nodes {
|
||||
node, _ := rawNode.(map[string]any)
|
||||
id, _ := node["id"].(string)
|
||||
if !IsParserComponentID(id) {
|
||||
if !common.IsParserComponentID(id) {
|
||||
continue
|
||||
}
|
||||
data, _ := node["data"].(map[string]any)
|
||||
|
||||
@@ -18,7 +18,6 @@ package indexdoc
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"ragflow/internal/common"
|
||||
@@ -376,7 +375,7 @@ func resolveTableColumnConfig(parserConfig map[string]interface{}) (mode string,
|
||||
return mode, roles, rawNames
|
||||
}
|
||||
for cid, raw := range parserConfig {
|
||||
if !strings.HasPrefix(cid, "Parser:") {
|
||||
if !common.IsParserComponentID(cid) {
|
||||
continue
|
||||
}
|
||||
comp, _ := raw.(map[string]interface{})
|
||||
|
||||
@@ -3,6 +3,7 @@ package service
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"ragflow/internal/common"
|
||||
"ragflow/internal/entity"
|
||||
)
|
||||
|
||||
@@ -33,9 +34,8 @@ func ApplyComponentScopedParserConfig(
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
cpnLower := strings.ToLower(cpnID)
|
||||
switch {
|
||||
case strings.HasPrefix(cpnLower, "extractor:") || strings.HasPrefix(cpnLower, "extractor_"):
|
||||
case common.IsExtractorComponentID(cpnID):
|
||||
if value, _ := params["llm_id"].(string); strings.TrimSpace(value) == "" && strings.TrimSpace(llmID) != "" {
|
||||
params["llm_id"] = llmID
|
||||
}
|
||||
@@ -54,7 +54,7 @@ func ApplyComponentScopedParserConfig(
|
||||
"built_in_metadata": []any{},
|
||||
}
|
||||
}
|
||||
case strings.HasPrefix(cpnLower, "compiler:") || strings.HasPrefix(cpnLower, "compiler_"):
|
||||
case common.IsCompilerComponentID(cpnID):
|
||||
if value, _ := params["llm_id"].(string); strings.TrimSpace(value) == "" && strings.TrimSpace(llmID) != "" {
|
||||
params["llm_id"] = llmID
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"ragflow/internal/common"
|
||||
"ragflow/internal/dao"
|
||||
"ragflow/internal/entity"
|
||||
pipelinepkg "ragflow/internal/ingestion/pipeline"
|
||||
@@ -202,23 +203,19 @@ func validateDatasetParserConfigSize(parserConfig map[string]interface{}) error
|
||||
return nil
|
||||
}
|
||||
|
||||
// DropUnscopedParserConfigKeys removes top-level parser_config keys that are
|
||||
// not component-scoped (i.e. do not contain ":"), since the Go backend only
|
||||
// consumes keys keyed by a component id (e.g. "Parser:abc123" or
|
||||
// "Extractor:AutoExtractDefault"). Flat keys are legacy transport artifacts
|
||||
// that downstream consumers (BuildParserConfig) never read, so they are
|
||||
// dropped rather than rejected. Only top-level keys are affected; a key nested
|
||||
// inside a component node (e.g. "Extractor:AutoExtractDefault.metadata") keeps
|
||||
// its existing name. It mutates the provided map in place and returns the names
|
||||
// of the dropped keys so callers can log the silent drop. A nil or empty map is
|
||||
// returned untouched with a nil result.
|
||||
// DropUnscopedParserConfigKeys removes top-level parser_config keys that
|
||||
// common.IsComponentKey does not recognize as component-scoped. Flat keys are
|
||||
// transport artifacts that BuildParserConfig never reads, so they are dropped
|
||||
// rather than rejected. Only top-level keys are affected; nested keys keep
|
||||
// their names. It mutates parserConfig and returns the dropped names so callers
|
||||
// can log them. A nil or empty map is returned untouched.
|
||||
func DropUnscopedParserConfigKeys(parserConfig map[string]any) []string {
|
||||
if len(parserConfig) == 0 {
|
||||
return nil
|
||||
}
|
||||
var dropped []string
|
||||
for key := range parserConfig {
|
||||
if !strings.Contains(key, ":") {
|
||||
if !common.IsComponentKey(key) {
|
||||
dropped = append(dropped, key)
|
||||
}
|
||||
}
|
||||
@@ -369,10 +366,10 @@ func preserveDatasetParserConfigState(next, existing entity.JSONMap, incoming ma
|
||||
}
|
||||
var mm map[string]any
|
||||
if incoming != nil {
|
||||
mm = extractorNodeMetadata(incoming)
|
||||
mm = extractorNodeMetadata(incoming, next)
|
||||
}
|
||||
if mm == nil && existing != nil {
|
||||
mm = extractorNodeMetadata(existing)
|
||||
mm = extractorNodeMetadata(existing, next)
|
||||
}
|
||||
if mm != nil {
|
||||
next["metadata"] = mm
|
||||
@@ -447,18 +444,19 @@ func preserveDatasetParserConfigState(next, existing entity.JSONMap, incoming ma
|
||||
}
|
||||
|
||||
// extractorNodeMetadata returns the modular metadata object ({enabled, metadata,
|
||||
// built_in_metadata}) from the first Extractor node in a parser_config, or nil
|
||||
// if none is present. Metadata is component-scoped under Extractor nodes; this
|
||||
// helper reads it back for preservation across partial updates.
|
||||
func extractorNodeMetadata(parserConfig map[string]interface{}) map[string]any {
|
||||
// built_in_metadata}) from the first Extractor node present in validConfig, or
|
||||
// nil if none is present. validConfig is the rebuilt parser_config for the
|
||||
// current DSL; keys outside it are ignored so stale nodes cannot override the
|
||||
// active configuration.
|
||||
func extractorNodeMetadata(parserConfig, validConfig map[string]interface{}) map[string]any {
|
||||
if parserConfig == nil {
|
||||
return nil
|
||||
}
|
||||
for cpnID, raw := range parserConfig {
|
||||
lower := strings.ToLower(cpnID)
|
||||
if !strings.HasPrefix(lower, "extractor:") && !strings.HasPrefix(lower, "extractor_") {
|
||||
for _, cpnID := range common.SortedExtractorKeys(parserConfig) {
|
||||
if _, ok := validConfig[cpnID]; !ok {
|
||||
continue
|
||||
}
|
||||
raw := parserConfig[cpnID]
|
||||
params, ok := raw.(map[string]any)
|
||||
if !ok {
|
||||
continue
|
||||
|
||||
@@ -227,6 +227,9 @@ func TestDropUnscopedParserConfigKeys_NilAndEmpty(t *testing.T) {
|
||||
func TestDropUnscopedParserConfigKeys_DropsFlatKeys(t *testing.T) {
|
||||
cfg := map[string]any{
|
||||
"File": map[string]any{},
|
||||
"Extractor": map[string]any{},
|
||||
"extractor_0": map[string]any{},
|
||||
"compiler_0": map[string]any{},
|
||||
"chunk_token_num": float64(128),
|
||||
"delimiter": "\n",
|
||||
"metadata": map[string]any{},
|
||||
@@ -236,7 +239,7 @@ func TestDropUnscopedParserConfigKeys_DropsFlatKeys(t *testing.T) {
|
||||
}
|
||||
dropped := DropUnscopedParserConfigKeys(cfg)
|
||||
sort.Strings(dropped)
|
||||
want := []string{"File", "chunk_token_num", "delimiter", "metadata", "parent_child"}
|
||||
want := []string{"Extractor", "File", "chunk_token_num", "compiler_0", "delimiter", "extractor_0", "metadata", "parent_child"}
|
||||
if !reflect.DeepEqual(dropped, want) {
|
||||
t.Fatalf("dropped = %#v, want %#v", dropped, want)
|
||||
}
|
||||
@@ -510,7 +513,9 @@ func TestPreserveDatasetParserConfigState_FallsBackWhenIncomingNotMap(t *testing
|
||||
"metadata": incomingMetadata,
|
||||
},
|
||||
}
|
||||
got := preserveDatasetParserConfigState(entity.JSONMap{}, existing, incoming)
|
||||
got := preserveDatasetParserConfigState(entity.JSONMap{
|
||||
"Extractor:AutoExtractDefault": map[string]any{},
|
||||
}, existing, incoming)
|
||||
meta, ok := got["metadata"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("expected existing modular metadata preserved, got %#v", got["metadata"])
|
||||
@@ -523,6 +528,38 @@ func TestPreserveDatasetParserConfigState_FallsBackWhenIncomingNotMap(t *testing
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreserveDatasetParserConfigState_IgnoresStaleExtractor(t *testing.T) {
|
||||
existing := entity.JSONMap{
|
||||
"Extractor:AutoExtractDefault": map[string]any{
|
||||
"metadata": map[string]any{
|
||||
"enabled": true,
|
||||
"metadata": []any{map[string]any{"key": "current_field", "type": "string"}},
|
||||
"built_in_metadata": []any{},
|
||||
},
|
||||
},
|
||||
}
|
||||
incoming := map[string]interface{}{
|
||||
"Extractor:Old": map[string]any{
|
||||
"metadata": map[string]any{
|
||||
"enabled": true,
|
||||
"metadata": []any{map[string]any{"key": "stale_field", "type": "string"}},
|
||||
"built_in_metadata": []any{},
|
||||
},
|
||||
},
|
||||
}
|
||||
got := preserveDatasetParserConfigState(entity.JSONMap{
|
||||
"Extractor:AutoExtractDefault": map[string]any{},
|
||||
}, existing, incoming)
|
||||
meta, ok := got["metadata"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("expected current modular metadata, got %#v", got["metadata"])
|
||||
}
|
||||
fields, ok := meta["metadata"].([]any)
|
||||
if !ok || len(fields) != 1 || fields[0].(map[string]any)["key"] != "current_field" {
|
||||
t.Fatalf("expected current_field to win over stale Extractor, got %#v", meta["metadata"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreserveDatasetParserConfigState_UsesValidIncomingMap(t *testing.T) {
|
||||
existing := entity.JSONMap{
|
||||
"Extractor:AutoExtractDefault": map[string]any{
|
||||
@@ -542,7 +579,9 @@ func TestPreserveDatasetParserConfigState_UsesValidIncomingMap(t *testing.T) {
|
||||
},
|
||||
},
|
||||
}
|
||||
got := preserveDatasetParserConfigState(entity.JSONMap{}, existing, incoming)
|
||||
got := preserveDatasetParserConfigState(entity.JSONMap{
|
||||
"Extractor:AutoExtractDefault": map[string]any{},
|
||||
}, existing, incoming)
|
||||
meta, ok := got["metadata"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("expected modular metadata map, got %#v", got["metadata"])
|
||||
|
||||
@@ -192,11 +192,8 @@ func modularMetadataConfig(parserConfig map[string]any) (bool, bool, []any, []an
|
||||
builtIn := anyOrEmptyList(metaObj["built_in_metadata"])
|
||||
return true, enabled, metadata, builtIn
|
||||
}
|
||||
for cpnID, raw := range parserConfig {
|
||||
lower := strings.ToLower(cpnID)
|
||||
if !strings.HasPrefix(lower, "extractor:") && !strings.HasPrefix(lower, "extractor_") {
|
||||
continue
|
||||
}
|
||||
for _, cpnID := range common.SortedExtractorKeys(parserConfig) {
|
||||
raw := parserConfig[cpnID]
|
||||
params, ok := raw.(map[string]any)
|
||||
if !ok {
|
||||
continue
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package dataset
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"ragflow/internal/common"
|
||||
"ragflow/internal/ingestion/component"
|
||||
pipelinepkg "ragflow/internal/ingestion/pipeline"
|
||||
)
|
||||
|
||||
func TestParserConfigCanonicalExtractorRoundTrip(t *testing.T) {
|
||||
dsl := []byte(`{"components":{"Extractor:AutoExtractDefault":{"obj":{"component_name":"Extractor","params":{"metadata":{"enabled":true,"metadata":[],"built_in_metadata":[]},"tags":{"top_n":5,"tag_file_id":""}}}}}}`)
|
||||
raw := map[string]any{
|
||||
"chunk_token_num": float64(128),
|
||||
"Extractor:AutoExtractDefault": map[string]any{
|
||||
"tags": map[string]any{"top_n": 5, "tag_file_id": "file-round-trip"},
|
||||
},
|
||||
}
|
||||
if dropped, err := ValidateParserConfig(raw); err != nil {
|
||||
t.Fatalf("ValidateParserConfig: %v", err)
|
||||
} else if len(dropped) != 1 || dropped[0] != "chunk_token_num" {
|
||||
t.Fatalf("dropped = %#v, want [chunk_token_num]", dropped)
|
||||
}
|
||||
|
||||
config, err := pipelinepkg.BuildParserConfig(dsl, raw)
|
||||
if err != nil {
|
||||
t.Fatalf("BuildParserConfig: %v", err)
|
||||
}
|
||||
if got := component.TagFileIDFromParserConfig(map[string]any(config)); got != "file-round-trip" {
|
||||
t.Fatalf("TagFileIDFromParserConfig = %q, want file-round-trip", got)
|
||||
}
|
||||
if meta, ok := common.ExtractorMetadataConfig(map[string]any(config)); !ok {
|
||||
t.Fatal("ExtractorMetadataConfig did not find the canonical Extractor node")
|
||||
} else if enabled, _ := meta["enabled"].(bool); !enabled {
|
||||
t.Fatalf("metadata enabled = %v, want true", meta["enabled"])
|
||||
}
|
||||
}
|
||||
@@ -1356,8 +1356,7 @@ func TestUpdateDatasetUsesTenantModelIDForExtractorDefault(t *testing.T) {
|
||||
func extractorNodeMetadataInTest(t *testing.T, cfg map[string]interface{}) map[string]interface{} {
|
||||
t.Helper()
|
||||
for cpnID, raw := range cfg {
|
||||
lower := strings.ToLower(cpnID)
|
||||
if !strings.HasPrefix(lower, "extractor:") && !strings.HasPrefix(lower, "extractor_") {
|
||||
if !common.IsExtractorComponentID(cpnID) {
|
||||
continue
|
||||
}
|
||||
params, ok := raw.(map[string]interface{})
|
||||
|
||||
@@ -15,7 +15,6 @@ package document
|
||||
import (
|
||||
"encoding/json"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gorm.io/gorm"
|
||||
@@ -42,8 +41,7 @@ func firstExtractorMetadata(t *testing.T, parserConfig entity.JSONMap) map[strin
|
||||
t.Helper()
|
||||
var keys []string
|
||||
for k := range parserConfig {
|
||||
lower := strings.ToLower(k)
|
||||
if strings.HasPrefix(lower, "extractor:") || strings.HasPrefix(lower, "extractor_") {
|
||||
if common.IsExtractorComponentID(k) {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"ragflow/internal/service"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -198,7 +197,7 @@ func (s *DocumentService) applyKnowledgebaseParserConfig(ctx context.Context, do
|
||||
if metaOK {
|
||||
merged["metadata"] = deepCopyValue(meta)
|
||||
} else {
|
||||
for _, cpnID := range sortedExtractorKeys(merged) {
|
||||
for _, cpnID := range common.SortedExtractorKeys(map[string]any(merged)) {
|
||||
params, ok := nodeParams(merged[cpnID])
|
||||
if !ok {
|
||||
continue
|
||||
@@ -227,21 +226,6 @@ func (s *DocumentService) applyKnowledgebaseParserConfig(ctx context.Context, do
|
||||
return nil
|
||||
}
|
||||
|
||||
// sortedExtractorKeys returns the component IDs in parserConfig that name
|
||||
// Extractor nodes, in sorted order so multi-Extractor configs resolve
|
||||
// deterministically instead of depending on Go map iteration order.
|
||||
func sortedExtractorKeys(parserConfig entity.JSONMap) []string {
|
||||
var keys []string
|
||||
for k := range parserConfig {
|
||||
lower := strings.ToLower(k)
|
||||
if strings.HasPrefix(lower, "extractor:") || strings.HasPrefix(lower, "extractor_") {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return keys
|
||||
}
|
||||
|
||||
// nodeParams returns a writable view of a component node's params, accepting
|
||||
// both the decoded (map[string]any) and entity (entity.JSONMap) shapes.
|
||||
// The returned map shares storage with the input, so writes are visible.
|
||||
@@ -281,7 +265,7 @@ func resolveDatasetModularMetadata(parserConfig entity.JSONMap) (map[string]any,
|
||||
if mm, ok := parserConfig["metadata"].(entity.JSONMap); ok {
|
||||
return map[string]any(mm), true, true
|
||||
}
|
||||
for _, cpnID := range sortedExtractorKeys(parserConfig) {
|
||||
for _, cpnID := range common.SortedExtractorKeys(map[string]any(parserConfig)) {
|
||||
params, ok := nodeParams(parserConfig[cpnID])
|
||||
if !ok {
|
||||
continue
|
||||
@@ -301,7 +285,7 @@ func resolveDatasetLLMID(parserConfig entity.JSONMap) string {
|
||||
if s, ok := parserConfig["llm_id"].(string); ok && strings.TrimSpace(s) != "" {
|
||||
return s
|
||||
}
|
||||
for _, cpnID := range sortedExtractorKeys(parserConfig) {
|
||||
for _, cpnID := range common.SortedExtractorKeys(map[string]any(parserConfig)) {
|
||||
params, ok := nodeParams(parserConfig[cpnID])
|
||||
if !ok {
|
||||
continue
|
||||
|
||||
@@ -66,7 +66,7 @@ func parserParamsDeclaresFamily(params map[string]any, family string) bool {
|
||||
// config, or a malformed one): the config cannot answer then.
|
||||
func configDeclaredFamily(config entity.JSONMap, family string) (declared, found bool) {
|
||||
for cpnID, raw := range config {
|
||||
if !pipelinepkg.IsParserComponentID(cpnID) {
|
||||
if !common.IsParserComponentID(cpnID) {
|
||||
continue
|
||||
}
|
||||
params, ok := raw.(map[string]any)
|
||||
@@ -94,7 +94,7 @@ func templateDeclaredFamily(ctx context.Context, parserID, family string) (decla
|
||||
return false, false
|
||||
}
|
||||
for cpnID, params := range defaults {
|
||||
if !pipelinepkg.IsParserComponentID(cpnID) {
|
||||
if !common.IsParserComponentID(cpnID) {
|
||||
continue
|
||||
}
|
||||
return parserParamsDeclaresFamily(params, family), true
|
||||
|
||||
@@ -3,9 +3,9 @@ package document
|
||||
import (
|
||||
"mime/multipart"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"ragflow/internal/common"
|
||||
"ragflow/internal/dao"
|
||||
"ragflow/internal/entity"
|
||||
"ragflow/internal/permission"
|
||||
@@ -763,7 +763,7 @@ func parserFamilySetup(t *testing.T, doc map[string]interface{}, family string)
|
||||
t.Fatalf("%v parser_config = %T, want map", doc["name"], doc["parser_config"])
|
||||
}
|
||||
for cpnID, raw := range cfg {
|
||||
if !strings.HasPrefix(cpnID, "Parser:") {
|
||||
if !common.IsParserComponentID(cpnID) {
|
||||
continue
|
||||
}
|
||||
params, _ := raw.(map[string]interface{})
|
||||
|
||||
Reference in new issue
Block a user