mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-11-10 21:32:43 +08:00
fix(document): persist parser models with the selected pipeline (#21418)
This commit is contained in:
1 parent
67f9b211ab
commit
2fc2b8888d
19 files changed
+781
-351
No files matched your search
@@ -221,7 +221,10 @@ func TestMigratedMinerUControlsReachRequest(t *testing.T) {
|
||||
if err := dao.MigrateIngestionParserConfigsForTest(t.Context(), db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
config := pipeline.BuildParserConfig(dsl, migratedConfig(t, db, "kb-mineru"))
|
||||
config, err := pipeline.BuildParserConfig(dsl, migratedConfig(t, db, "kb-mineru"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
effective := effectiveComponentParams(t, "general", config)
|
||||
parser, err := component.NewParserComponent(effective["Parser:HipSignsRhyme"])
|
||||
if err != nil {
|
||||
@@ -400,9 +403,17 @@ func TestIngestionMigrationMatchesBuiltinTemplates(t *testing.T) {
|
||||
}
|
||||
// Snapshot before cleaning so mutations cannot hide a lost parameter.
|
||||
expected := jsonNormalize(t, config)
|
||||
cleaned := pipeline.CleanComponentParams(rawDSL, config)
|
||||
if !reflect.DeepEqual(jsonNormalize(t, cleaned), expected) {
|
||||
t.Fatalf("API cleaning changed migrated params: got %v; want %v", cleaned, expected)
|
||||
cleaned, err := pipeline.BuildParserConfig(rawDSL, config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for cpnID, raw := range expected.(map[string]any) {
|
||||
params, _ := cleaned[cpnID].(map[string]any)
|
||||
for key, value := range raw.(map[string]any) {
|
||||
if !reflect.DeepEqual(jsonNormalize(t, params[key]), value) {
|
||||
t.Fatalf("API config changed migrated %s.%s: got %v; want %v", cpnID, key, params[key], value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Parser: exactly the template's families the config targets,
|
||||
@@ -873,7 +884,11 @@ func extractorFromMigratedConfig(t *testing.T, config map[string]any) *component
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cleaned := service.ApplyComponentScopedParserConfig(pipeline.BuildParserConfig(dsl, config), "tenant-llm")
|
||||
cleaned, err := pipeline.BuildParserConfig(dsl, config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cleaned = service.ApplyComponentScopedParserConfig(cleaned, "tenant-llm")
|
||||
c, err := component.NewExtractorComponent(cleaned["Extractor:AutoExtractDefault"].(map[string]any))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -258,7 +258,7 @@ func pythonFamilyName(raw string) string {
|
||||
return "doc"
|
||||
case "docx":
|
||||
return "docx"
|
||||
case "ppt", "pptx", "slides":
|
||||
case "ppt", "pptx", "pages", "dps", "dpt", "slides":
|
||||
return "slides"
|
||||
case "xls", "xlsx", "spreadsheet":
|
||||
return "spreadsheet"
|
||||
|
||||
@@ -1961,6 +1961,14 @@ func tcadpZipFixtureForComponent(t *testing.T) []byte {
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func TestParserFileFamilyPresentationExtensions(t *testing.T) {
|
||||
for _, ext := range []string{"ppt", "pptx", "pages", "dps", "dpt"} {
|
||||
if got := ParserFileFamily(ext); got != "slides" {
|
||||
t.Errorf("ParserFileFamily(%q) = %q, want slides", ext, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestPythonFamilyName_FileTypeConstants pins the contract that every
|
||||
// utility.FileType semantic constant resolves to the setups key used by
|
||||
// defaultSetups. Before the fix, FileTypeVISUAL mapped
|
||||
|
||||
@@ -138,19 +138,19 @@ func TestAgentTemplateExtractorsUseModularParams(t *testing.T) {
|
||||
t.Errorf("%s: Extractor %s has every modular sub-task disabled; the template's purpose requires at least one active", f, cpnID)
|
||||
}
|
||||
|
||||
// The params must also survive CleanComponentParams: no
|
||||
// The params must also survive BuildParserConfig: no
|
||||
// dropped keys and any active sub-task left active with its
|
||||
// prompt intact.
|
||||
dslRaw, _ := json.Marshal(doc["dsl"])
|
||||
cleaned := CleanComponentParams(dslRaw, map[string]any{cpnID: params})
|
||||
cleaned := buildTestParserConfig(t, dslRaw, map[string]any{cpnID: params})
|
||||
got, _ := cleaned[cpnID].(map[string]any)
|
||||
if got == nil {
|
||||
t.Errorf("%s: Extractor %s dropped by CleanComponentParams", f, cpnID)
|
||||
t.Errorf("%s: Extractor %s dropped by BuildParserConfig", f, cpnID)
|
||||
continue
|
||||
}
|
||||
for k := range params {
|
||||
if _, kept := got[k]; !kept {
|
||||
t.Errorf("%s: Extractor %s param %q dropped by CleanComponentParams", f, cpnID, k)
|
||||
t.Errorf("%s: Extractor %s param %q dropped by BuildParserConfig", f, cpnID, k)
|
||||
}
|
||||
}
|
||||
for _, k := range modularKeys {
|
||||
@@ -159,14 +159,14 @@ func TestAgentTemplateExtractorsUseModularParams(t *testing.T) {
|
||||
continue
|
||||
}
|
||||
if !subTaskActive(k, got[k]) {
|
||||
t.Errorf("%s: Extractor %s sub-task %q no longer active after CleanComponentParams: %#v", f, cpnID, k, got[k])
|
||||
t.Errorf("%s: Extractor %s sub-task %q no longer active after BuildParserConfig: %#v", f, cpnID, k, got[k])
|
||||
continue
|
||||
}
|
||||
if k == "summary" {
|
||||
cleanedSummary, _ := got[k].(map[string]any)
|
||||
prompt, _ := cleanedSummary["system_prompt"].(string)
|
||||
if prompt == "" {
|
||||
t.Errorf("%s: Extractor %s summary.system_prompt empty after CleanComponentParams", f, cpnID)
|
||||
t.Errorf("%s: Extractor %s summary.system_prompt empty after BuildParserConfig", f, cpnID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -114,7 +114,7 @@ func TestImageOCRSavedConfigToSearchableChunk(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const parserID = "Parser:ViewsCaptureLight"
|
||||
config := BuildParserConfig(dsl, map[string]any{parserID: map[string]any{
|
||||
config := buildTestParserConfig(t, dsl, map[string]any{parserID: map[string]any{
|
||||
"image": map[string]any{"ocr_enabled": true, "ocr_model_id": "ocr-model"},
|
||||
"enable_vision_enhancement": false,
|
||||
}})
|
||||
|
||||
@@ -41,7 +41,7 @@ func TestBuildParserConfigPreservesMinerUExtractionSwitches(t *testing.T) {
|
||||
if nested {
|
||||
params = map[string]any{"setups": params}
|
||||
}
|
||||
config := BuildParserConfig(generalDSL(t), map[string]any{
|
||||
config := buildTestParserConfig(t, generalDSL(t), map[string]any{
|
||||
"Parser:HipSignsRhyme": params,
|
||||
})
|
||||
// Exercise the serialized shape stored in parser_config, not
|
||||
|
||||
@@ -27,7 +27,7 @@ func TestTenantPDFParserReferenceCleaning(t *testing.T) {
|
||||
for _, ref := range []string{"model-b", "SoMark-model@b@SoMark", "OpenDataLoader-model@b@OpenDataLoader"} {
|
||||
for _, field := range []string{"parse_method", "layout_recognizer"} {
|
||||
t.Run(ref+"/"+field, func(t *testing.T) {
|
||||
cleaned := CleanComponentParams(generalDSL(t), map[string]any{
|
||||
cleaned := buildTestParserConfig(t, generalDSL(t), map[string]any{
|
||||
"Parser:HipSignsRhyme": map[string]any{
|
||||
"pdf": map[string]any{"parse_method": "deepdoc", field: ref},
|
||||
},
|
||||
|
||||
@@ -11,7 +11,7 @@ func TestBuildParserConfigPreservesGlobalVisionSettings(t *testing.T) {
|
||||
for _, enabled := range []bool{true, false} {
|
||||
for _, model := range []string{"model-B", ""} {
|
||||
want := map[string]any{"enable_vision_enhancement": enabled, "vlm": map[string]any{"llm_id": model}}
|
||||
params := BuildParserConfig(generalDSL(t), map[string]any{id: want})[id].(map[string]any)
|
||||
params := buildTestParserConfig(t, generalDSL(t), map[string]any{id: want})[id].(map[string]any)
|
||||
for key, value := range want {
|
||||
if !reflect.DeepEqual(params[key], value) {
|
||||
t.Fatalf("existing canvas saved %s = %v, want %v", key, params[key], value)
|
||||
@@ -68,7 +68,7 @@ func TestBuiltinParserVisionSettings(t *testing.T) {
|
||||
}
|
||||
for _, enabled := range []bool{true, false} {
|
||||
want := map[string]any{"enable_vision_enhancement": enabled, "vlm": map[string]any{"llm_id": "model-B"}}
|
||||
params := BuildParserConfig(dsl, map[string]any{id: want})[id].(map[string]any)
|
||||
params := buildTestParserConfig(t, dsl, map[string]any{id: want})[id].(map[string]any)
|
||||
for key, value := range want {
|
||||
if !reflect.DeepEqual(params[key], value) {
|
||||
t.Fatalf("saved %s = %v, want %v", key, params[key], value)
|
||||
|
||||
@@ -40,7 +40,7 @@ var extractorValidParamKeys = func() map[string]struct{} {
|
||||
}()
|
||||
|
||||
// componentParamSchemaKeys maps a DSL component_name (lower-cased) to the
|
||||
// param keys its param schema declares. CleanComponentParams unions these with
|
||||
// param keys its param schema declares. cleanComponentParams unions these with
|
||||
// the keys a template already bakes, so a param the component can read is
|
||||
// never dropped just because a template — or a canvas snapshot saved before
|
||||
// the param existed — does not carry it.
|
||||
@@ -186,21 +186,14 @@ func getComponentParamWhitelist(cpnID string) (map[string]struct{}, bool) {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// CleanComponentParams filters rawConfig against the DSL schema given by dslJSON.
|
||||
// cleanComponentParams filters rawConfig against the extracted DSL schemas.
|
||||
// Keys containing ':' are treated as component IDs; they are kept only when both
|
||||
// the cpnID AND the param name exist in one of the accepted-key sources: the DSL
|
||||
// schema (the params the template bakes), the schema-derived per-component table
|
||||
// (componentParamSchemaKeys), or the component's dynamic parameter schema (e.g.
|
||||
// Extractor modular features). Keys without ':' (legacy flat fields) are dropped
|
||||
// with a warning.
|
||||
func CleanComponentParams(dslJSON []byte, rawConfig map[string]interface{}) map[string]interface{} {
|
||||
schemas, err := ExtractAllComponentParams(dslJSON)
|
||||
if err != nil {
|
||||
common.Warn("CleanComponentParams: failed to extract DSL schema, returning input as-is",
|
||||
zap.Error(err))
|
||||
return rawConfig
|
||||
}
|
||||
|
||||
func cleanComponentParams(schemas []ComponentParamsSchema, rawConfig map[string]interface{}) map[string]interface{} {
|
||||
validCPNs := make(map[string]map[string]struct{}, len(schemas))
|
||||
componentNames := make(map[string]string, len(schemas))
|
||||
for _, s := range schemas {
|
||||
@@ -233,13 +226,13 @@ func CleanComponentParams(dslJSON []byte, rawConfig map[string]interface{}) map[
|
||||
result := make(map[string]interface{}, len(rawConfig))
|
||||
for key, val := range rawConfig {
|
||||
if !strings.Contains(key, ":") {
|
||||
common.Warn("CleanComponentParams: dropping legacy flat field",
|
||||
common.Warn("cleanComponentParams: dropping legacy flat field",
|
||||
zap.String("key", key))
|
||||
continue
|
||||
}
|
||||
validKeys, ok := validCPNs[key]
|
||||
if !ok {
|
||||
common.Warn("CleanComponentParams: dropping unknown cpnID",
|
||||
common.Warn("cleanComponentParams: dropping unknown cpnID",
|
||||
zap.String("cpnID", key))
|
||||
continue
|
||||
}
|
||||
@@ -274,7 +267,7 @@ func CleanComponentParams(dslJSON []byte, rawConfig map[string]interface{}) map[
|
||||
continue
|
||||
}
|
||||
}
|
||||
common.Warn("CleanComponentParams: dropping unknown param",
|
||||
common.Warn("cleanComponentParams: dropping unknown param",
|
||||
zap.String("cpnID", key), zap.String("param", pk))
|
||||
}
|
||||
if len(cleaned) > 0 {
|
||||
@@ -374,17 +367,16 @@ func NormalizeExtractorParams(raw map[string]any) map[string]any {
|
||||
// BuildParserConfig builds the final parser_config by starting from the DSL
|
||||
// defaults for every component, then overlaying the cleaned incoming overrides.
|
||||
// This ensures all components from the current pipeline are present while
|
||||
// stripping stale params from other pipelines.
|
||||
func BuildParserConfig(dslJSON []byte, rawConfig map[string]interface{}) entity.JSONMap {
|
||||
cleaned := CleanComponentParams(dslJSON, rawConfig)
|
||||
defaults, err := ComponentParamsDefaults(dslJSON)
|
||||
// stripping stale params from other pipelines. Invalid DSL returns an error.
|
||||
func BuildParserConfig(dslJSON []byte, rawConfig map[string]interface{}) (entity.JSONMap, error) {
|
||||
schemas, err := ExtractAllComponentParams(dslJSON)
|
||||
if err != nil {
|
||||
common.Warn("BuildParserConfig: failed to extract DSL defaults, using cleaned only",
|
||||
zap.Error(err))
|
||||
return entity.JSONMap(cleaned)
|
||||
return nil, err
|
||||
}
|
||||
result := make(entity.JSONMap, len(defaults))
|
||||
for cpnID, params := range defaults {
|
||||
cleaned := cleanComponentParams(schemas, rawConfig)
|
||||
result := make(entity.JSONMap, len(schemas))
|
||||
for _, component := range schemas {
|
||||
cpnID, params := component.CpnID, component.ParamsDefaults
|
||||
base := make(map[string]interface{}, len(params))
|
||||
for k, v := range params {
|
||||
base[k] = v
|
||||
@@ -398,7 +390,7 @@ func BuildParserConfig(dslJSON []byte, rawConfig map[string]interface{}) entity.
|
||||
}
|
||||
result[cpnID] = base
|
||||
}
|
||||
return result
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// ApplyParentChildChunkerConfig derives runtime children_delimiters from the
|
||||
|
||||
@@ -9,6 +9,15 @@ import (
|
||||
"ragflow/internal/entity"
|
||||
)
|
||||
|
||||
func buildTestParserConfig(t *testing.T, dsl []byte, overrides map[string]any) entity.JSONMap {
|
||||
t.Helper()
|
||||
config, err := BuildParserConfig(dsl, overrides)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return config
|
||||
}
|
||||
|
||||
// generalDSL returns a minimal DSL resembling the "general" template's component
|
||||
// structure, with one Parser and one Chunker component.
|
||||
func generalDSL(t *testing.T) []byte {
|
||||
@@ -44,7 +53,7 @@ func generalDSL(t *testing.T) []byte {
|
||||
return raw
|
||||
}
|
||||
|
||||
func TestCleanComponentParamsNormalizesGeneralLegacyDelimiter(t *testing.T) {
|
||||
func TestBuildParserConfigNormalizesGeneralLegacyDelimiter(t *testing.T) {
|
||||
dsl := map[string]any{
|
||||
"components": map[string]any{
|
||||
"GeneralChunker:Legacy": map[string]any{
|
||||
@@ -61,7 +70,7 @@ func TestCleanComponentParamsNormalizesGeneralLegacyDelimiter(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("marshal dsl: %v", err)
|
||||
}
|
||||
result := CleanComponentParams(dslJSON, map[string]any{
|
||||
result := buildTestParserConfig(t, dslJSON, map[string]any{
|
||||
"GeneralChunker:Legacy": map[string]any{
|
||||
"delimiter": "\n!?;。;!?",
|
||||
},
|
||||
@@ -96,7 +105,11 @@ func TestCleanComponentParamsNormalizesDelimiterWhenDSLDeclaresOnlyLegacyKey(t *
|
||||
if err != nil {
|
||||
t.Fatalf("marshal DSL: %v", err)
|
||||
}
|
||||
result := CleanComponentParams(dslJSON, map[string]any{
|
||||
schemas, err := ExtractAllComponentParams(dslJSON)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result := cleanComponentParams(schemas, map[string]any{
|
||||
"GeneralChunker:Custom": map[string]any{
|
||||
"delimiter": "\n!?",
|
||||
},
|
||||
@@ -114,30 +127,21 @@ func TestCleanComponentParamsNormalizesDelimiterWhenDSLDeclaresOnlyLegacyKey(t *
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanComponentParams_DropsLegacyFlatFields(t *testing.T) {
|
||||
func TestBuildParserConfig_DropsLegacyFlatFields(t *testing.T) {
|
||||
dslJSON := generalDSL(t)
|
||||
raw := map[string]any{
|
||||
"chunk_token_num": 256,
|
||||
"image_context_size": 10,
|
||||
}
|
||||
result := CleanComponentParams(dslJSON, raw)
|
||||
if len(result) != 0 {
|
||||
t.Errorf("expected empty result, got %v", result)
|
||||
result := buildTestParserConfig(t, dslJSON, raw)
|
||||
for key := range raw {
|
||||
if _, exists := result[key]; exists {
|
||||
t.Errorf("flat field %q survived config building: %v", key, result)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanComponentParams_DropsUnknownCPNID(t *testing.T) {
|
||||
dslJSON := generalDSL(t)
|
||||
raw := map[string]any{
|
||||
"Parser:NoSuch": map[string]any{"chunk_size": float64(256)},
|
||||
}
|
||||
result := CleanComponentParams(dslJSON, raw)
|
||||
if _, ok := result["Parser:NoSuch"]; ok {
|
||||
t.Error("expected unknown cpnID to be dropped")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanComponentParams_DropsUnknownParamKey(t *testing.T) {
|
||||
func TestBuildParserConfig_DropsUnknownParamKey(t *testing.T) {
|
||||
dslJSON := generalDSL(t)
|
||||
raw := map[string]any{
|
||||
"Parser:HipSignsRhyme": map[string]any{
|
||||
@@ -145,7 +149,7 @@ func TestCleanComponentParams_DropsUnknownParamKey(t *testing.T) {
|
||||
"pdf": map[string]any{"parse_method": "deepdoc"},
|
||||
},
|
||||
}
|
||||
result := CleanComponentParams(dslJSON, raw)
|
||||
result := buildTestParserConfig(t, dslJSON, raw)
|
||||
params := result["Parser:HipSignsRhyme"].(map[string]any)
|
||||
if _, ok := params["no_such_param"]; ok {
|
||||
t.Error("expected unknown param key to be dropped")
|
||||
@@ -155,32 +159,6 @@ func TestCleanComponentParams_DropsUnknownParamKey(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanComponentParams_ReturnsInputOnDSLError(t *testing.T) {
|
||||
result := CleanComponentParams([]byte("not json"), map[string]any{"key": "val"})
|
||||
if result["key"] != "val" {
|
||||
t.Error("expected input returned as-is on DSL error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanComponentParams_ValidCPNIDPassesThrough(t *testing.T) {
|
||||
dslJSON := generalDSL(t)
|
||||
raw := map[string]any{
|
||||
"Parser:HipSignsRhyme": map[string]any{
|
||||
"pdf": map[string]any{"parse_method": "deepdoc"},
|
||||
},
|
||||
"Chunker:LegalReadersDecide": map[string]any{
|
||||
"chunk_size": float64(256),
|
||||
},
|
||||
}
|
||||
result := CleanComponentParams(dslJSON, raw)
|
||||
if _, ok := result["Parser:HipSignsRhyme"]; !ok {
|
||||
t.Error("expected Parser:HipSignsRhyme to pass through")
|
||||
}
|
||||
if _, ok := result["Chunker:LegalReadersDecide"]; !ok {
|
||||
t.Error("expected Chunker:LegalReadersDecide to pass through")
|
||||
}
|
||||
}
|
||||
|
||||
// --- BuildParserConfig ---
|
||||
|
||||
func TestBuildParserConfig_ShallowMerge_NestedParam(t *testing.T) {
|
||||
@@ -213,7 +191,7 @@ func TestBuildParserConfig_ShallowMerge_NestedParam(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
result := BuildParserConfig(dslJSON, overrides)
|
||||
result := buildTestParserConfig(t, dslJSON, overrides)
|
||||
chunker, ok := result["Chunker:xyz"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatal("expected Chunker:xyz in result")
|
||||
@@ -266,7 +244,7 @@ func TestBuildParserConfig_FlattensNestedParserSetups(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
result := BuildParserConfig(dslJSON, overrides)
|
||||
result := buildTestParserConfig(t, dslJSON, overrides)
|
||||
parser, ok := result["Parser:HipSignsRhyme"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatal("expected Parser:HipSignsRhyme in result")
|
||||
@@ -297,54 +275,6 @@ func TestBuildParserConfig_FlattensNestedParserSetups(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// CleanComponentParams must flatten a legacy nested "setups" group before
|
||||
// filtering, otherwise the lifted families are dropped as unknown params
|
||||
// against the (flat) DSL schema keys.
|
||||
func TestCleanComponentParams_FlattensNestedParserSetups(t *testing.T) {
|
||||
dsl := map[string]any{
|
||||
"components": map[string]any{
|
||||
"Parser:HipSignsRhyme": map[string]any{
|
||||
"obj": map[string]any{
|
||||
"component_name": "Parser",
|
||||
"params": map[string]any{
|
||||
"setups": map[string]any{
|
||||
"pdf": map[string]any{"parse_method": "deepdoc"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
dslJSON, err := json.Marshal(dsl)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal dsl: %v", err)
|
||||
}
|
||||
|
||||
rawConfig := map[string]any{
|
||||
"Parser:HipSignsRhyme": map[string]any{
|
||||
"setups": map[string]any{
|
||||
"pdf": map[string]any{"parse_method": "vision"},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
result := CleanComponentParams(dslJSON, rawConfig)
|
||||
parser, ok := result["Parser:HipSignsRhyme"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatal("expected Parser:HipSignsRhyme in result")
|
||||
}
|
||||
if _, ok := parser["setups"]; ok {
|
||||
t.Error("nested \"setups\" key must be flattened away")
|
||||
}
|
||||
pdf, ok := parser["pdf"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("expected flattened pdf family, got %T", parser["pdf"])
|
||||
}
|
||||
if pdf["parse_method"] != "vision" {
|
||||
t.Errorf("pdf parse_method = %v, want vision", pdf["parse_method"])
|
||||
}
|
||||
}
|
||||
|
||||
// The runtime passes Doc.ParserConfig as override_params, where top-level
|
||||
// keys from the override replace the DSL-baked ones. A legacy nested Parser
|
||||
// entry must be lifted BEFORE that merge — otherwise the DSL's flat families
|
||||
@@ -422,7 +352,7 @@ func TestBuildParserConfig_ScalarOverridePreservesOtherDefaults(t *testing.T) {
|
||||
"chunk_size": float64(1024),
|
||||
},
|
||||
}
|
||||
result := BuildParserConfig(dslJSON, overrides)
|
||||
result := buildTestParserConfig(t, dslJSON, overrides)
|
||||
chunker, ok := result["Chunker:LegalReadersDecide"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatal("expected Chunker:LegalReadersDecide in result")
|
||||
@@ -441,8 +371,8 @@ func TestBuildParserConfig_UnknownCPNIDNotPresentInResult(t *testing.T) {
|
||||
overrides := map[string]any{
|
||||
"Parser:Unknown": map[string]any{"chunk_size": float64(256)},
|
||||
}
|
||||
result := BuildParserConfig(dslJSON, overrides)
|
||||
// Unknown cpnID should be dropped by CleanComponentParams; the result should
|
||||
result := buildTestParserConfig(t, dslJSON, overrides)
|
||||
// Unknown cpnID should be dropped by BuildParserConfig; the result should
|
||||
// still contain the DSL-defined components with their defaults.
|
||||
if _, ok := result["Parser:Unknown"]; ok {
|
||||
t.Error("expected unknown cpnID to be absent from result")
|
||||
@@ -452,16 +382,18 @@ func TestBuildParserConfig_UnknownCPNIDNotPresentInResult(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildParserConfig_FallbackOnDSLError(t *testing.T) {
|
||||
result := BuildParserConfig([]byte("not json"), map[string]any{"key": "val"})
|
||||
if result["key"] != "val" {
|
||||
t.Error("expected fallback to return raw config on DSL error")
|
||||
func TestBuildParserConfigRejectsInvalidDSL(t *testing.T) {
|
||||
for _, dsl := range []string{"not json", `{}`, `{"components":{"Extractor:first":{},"Extractor:second":{}}}`} {
|
||||
result, err := BuildParserConfig([]byte(dsl), map[string]any{"key": "val"})
|
||||
if err == nil || result != nil {
|
||||
t.Fatalf("invalid DSL must return an error without config: config=%#v err=%v", result, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildParserConfig_AllComponentsPresent(t *testing.T) {
|
||||
dslJSON := generalDSL(t)
|
||||
result := BuildParserConfig(dslJSON, nil)
|
||||
result := buildTestParserConfig(t, dslJSON, nil)
|
||||
// Both components from the DSL fixture should be present.
|
||||
if _, ok := result["Parser:HipSignsRhyme"]; !ok {
|
||||
t.Error("expected Parser:HipSignsRhyme")
|
||||
@@ -500,7 +432,7 @@ func TestBuildParserConfig_BuiltinExtractorKeepsTagFileID(t *testing.T) {
|
||||
"tags": map[string]any{"top_n": 5, "tag_file_id": "file-123"},
|
||||
},
|
||||
}
|
||||
result := BuildParserConfig(dslJSON, overrides)
|
||||
result := buildTestParserConfig(t, dslJSON, overrides)
|
||||
params, ok := result[s.CpnID].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("template %q: expected component %q in result", ref, s.CpnID)
|
||||
@@ -606,13 +538,13 @@ func TestBuildParserConfig_BuiltinExtractorKeepsBuiltInMetadata(t *testing.T) {
|
||||
},
|
||||
},
|
||||
}
|
||||
result := BuildParserConfig(dslJSON, overrides)
|
||||
result := buildTestParserConfig(t, dslJSON, overrides)
|
||||
params, ok := result[s.CpnID].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("template %q: expected component %q in result", ref, s.CpnID)
|
||||
}
|
||||
if _, ok := params["metadata"]; !ok {
|
||||
t.Errorf("template %q: metadata dropped by CleanComponentParams", ref)
|
||||
t.Errorf("template %q: metadata dropped by BuildParserConfig", ref)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -642,7 +574,7 @@ func TestApplyParentChildChunkerConfig(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanComponentParams_KeepsModularExtractorParams(t *testing.T) {
|
||||
func TestBuildParserConfig_KeepsModularExtractorParams(t *testing.T) {
|
||||
// Builtin template with empty Extractor params: {}
|
||||
dsl := map[string]any{
|
||||
"components": map[string]any{
|
||||
@@ -680,7 +612,7 @@ func TestCleanComponentParams_KeepsModularExtractorParams(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
result := CleanComponentParams(dslJSON, overrides)
|
||||
result := buildTestParserConfig(t, dslJSON, overrides)
|
||||
ext, ok := result["Extractor:AutoExtractDefault"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("expected Extractor:AutoExtractDefault in result, got: %v", result)
|
||||
@@ -703,7 +635,7 @@ func TestCleanComponentParams_KeepsModularExtractorParams(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanComponentParams_ExtractorBuiltInMetadataRetained(t *testing.T) {
|
||||
func TestBuildParserConfig_ExtractorBuiltInMetadataRetained(t *testing.T) {
|
||||
dslJSON := []byte(`{
|
||||
"components": {
|
||||
"Extractor:AutoExtractDefault": {
|
||||
@@ -732,7 +664,7 @@ func TestCleanComponentParams_ExtractorBuiltInMetadataRetained(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
result := CleanComponentParams(dslJSON, rawConfig)
|
||||
result := buildTestParserConfig(t, dslJSON, rawConfig)
|
||||
ext, ok := result["Extractor:AutoExtractDefault"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("expected Extractor:AutoExtractDefault in result, got: %v", result)
|
||||
@@ -893,7 +825,7 @@ func compilerDSL(t *testing.T) []byte {
|
||||
return raw
|
||||
}
|
||||
|
||||
func TestCleanComponentParams_CompilerKeepsLLMRuntimeParams(t *testing.T) {
|
||||
func TestBuildParserConfig_CompilerKeepsLLMRuntimeParams(t *testing.T) {
|
||||
dslJSON := compilerDSL(t)
|
||||
raw := map[string]any{
|
||||
"Compiler:NewBoxesLove": map[string]any{
|
||||
@@ -914,7 +846,7 @@ func TestCleanComponentParams_CompilerKeepsLLMRuntimeParams(t *testing.T) {
|
||||
"mode": "structure",
|
||||
},
|
||||
}
|
||||
result := CleanComponentParams(dslJSON, raw)
|
||||
result := buildTestParserConfig(t, dslJSON, raw)
|
||||
params, ok := result["Compiler:NewBoxesLove"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("expected Compiler params to survive, got %#v", result)
|
||||
@@ -953,7 +885,7 @@ func TestBuildParserConfig_CompilerRuntimeParamsSurvive(t *testing.T) {
|
||||
"thinking": "disabled",
|
||||
},
|
||||
}
|
||||
result := BuildParserConfig(dslJSON, raw)
|
||||
result := buildTestParserConfig(t, dslJSON, raw)
|
||||
params, ok := result["Compiler:NewBoxesLove"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("expected Compiler entry in built parser_config, got %#v", result)
|
||||
@@ -1001,7 +933,7 @@ func titleFamilyDSL(t *testing.T) []byte {
|
||||
return raw
|
||||
}
|
||||
|
||||
func TestCleanComponentParams_TitleFamilyKeepsSchemaParamsWithoutBakedKeys(t *testing.T) {
|
||||
func TestBuildParserConfig_TitleFamilyKeepsSchemaParamsWithoutBakedKeys(t *testing.T) {
|
||||
dslJSON := titleFamilyDSL(t)
|
||||
raw := map[string]any{
|
||||
"TitleChunker:StaleSnapshot": map[string]any{
|
||||
@@ -1009,7 +941,7 @@ func TestCleanComponentParams_TitleFamilyKeepsSchemaParamsWithoutBakedKeys(t *te
|
||||
"root_chunk_as_heading": true,
|
||||
},
|
||||
}
|
||||
result := CleanComponentParams(dslJSON, raw)
|
||||
result := buildTestParserConfig(t, dslJSON, raw)
|
||||
params, ok := result["TitleChunker:StaleSnapshot"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("expected TitleChunker params to survive, got %#v", result)
|
||||
@@ -1025,7 +957,7 @@ func TestCleanComponentParams_TitleFamilyKeepsSchemaParamsWithoutBakedKeys(t *te
|
||||
// TestBuildParserConfig_KeepsTitleChunkerPerMethodKeys locks the persistence
|
||||
// contract for the title-chunker family: the frontend stores each method's
|
||||
// rules, hierarchy value and token cap under its own key so the dataset
|
||||
// config keeps Hierarchy and Group independent; CleanComponentParams must not
|
||||
// config keeps Hierarchy and Group independent; BuildParserConfig must not
|
||||
// drop them.
|
||||
func TestBuildParserConfig_KeepsTitleChunkerPerMethodKeys(t *testing.T) {
|
||||
dslJSON := titleFamilyDSL(t)
|
||||
@@ -1042,7 +974,7 @@ func TestBuildParserConfig_KeepsTitleChunkerPerMethodKeys(t *testing.T) {
|
||||
"groupChunkTokenCap": 2048,
|
||||
},
|
||||
}
|
||||
params, ok := BuildParserConfig(dslJSON, overrides)["TitleChunker:StaleSnapshot"].(map[string]any)
|
||||
params, ok := buildTestParserConfig(t, dslJSON, overrides)["TitleChunker:StaleSnapshot"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("expected TitleChunker params to survive, got %#v", params)
|
||||
}
|
||||
@@ -1091,7 +1023,7 @@ func TestBuildParserConfig_TitleFamilyParamsSurviveBuiltinTemplates(t *testing.T
|
||||
"root_chunk_as_heading": true,
|
||||
},
|
||||
}
|
||||
params, ok := BuildParserConfig(dslJSON, overrides)[s.CpnID].(map[string]any)
|
||||
params, ok := buildTestParserConfig(t, dslJSON, overrides)[s.CpnID].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("template %q: component %q missing from built parser_config", ref, s.CpnID)
|
||||
}
|
||||
@@ -1111,7 +1043,7 @@ func TestBuildParserConfig_TitleFamilyParamsSurviveBuiltinTemplates(t *testing.T
|
||||
// operatorFormParamKeys mirrors the params the dataset/document pipeline
|
||||
// operator forms submit (transformFormConfigToApi in
|
||||
// web/src/utils/pipeline-operator.ts). Every key listed for a component must be
|
||||
// accepted by CleanComponentParams for that component in every builtin
|
||||
// accepted by BuildParserConfig for that component in every builtin
|
||||
// template; otherwise a user edit is silently dropped.
|
||||
var operatorFormParamKeys = map[string][]string{
|
||||
"TitleChunker": {
|
||||
@@ -1140,7 +1072,7 @@ var operatorFormParamKeys = map[string][]string{
|
||||
"Extractor": {"llm_id", "keywords", "questions", "tags", "summary", "metadata"},
|
||||
}
|
||||
|
||||
func TestCleanComponentParams_AcceptsOperatorFormParams(t *testing.T) {
|
||||
func TestBuildParserConfig_AcceptsOperatorFormParams(t *testing.T) {
|
||||
registry, err := DefaultRegistry()
|
||||
if err != nil {
|
||||
t.Fatalf("DefaultRegistry: %v", err)
|
||||
@@ -1169,7 +1101,7 @@ func TestCleanComponentParams_AcceptsOperatorFormParams(t *testing.T) {
|
||||
for _, key := range expected {
|
||||
params[key] = "probe"
|
||||
}
|
||||
cleaned := CleanComponentParams(dslJSON, map[string]interface{}{s.CpnID: params})
|
||||
cleaned := buildTestParserConfig(t, dslJSON, map[string]interface{}{s.CpnID: params})
|
||||
got, ok := cleaned[s.CpnID].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("template %q: component %q missing from cleaned result", ref, s.CpnID)
|
||||
@@ -1245,9 +1177,9 @@ func TestBuiltinTemplateComponentsHaveWhitelistCoverage(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestCleanComponentParams_TitleFamilyStillDropsUnknownKeys keeps the union
|
||||
// TestBuildParserConfig_TitleFamilyStillDropsUnknownKeys keeps the union
|
||||
// honest: a key the component schema does not declare is still dropped.
|
||||
func TestCleanComponentParams_TitleFamilyStillDropsUnknownKeys(t *testing.T) {
|
||||
func TestBuildParserConfig_TitleFamilyStillDropsUnknownKeys(t *testing.T) {
|
||||
dslJSON := titleFamilyDSL(t)
|
||||
raw := map[string]any{
|
||||
"TitleChunker:StaleSnapshot": map[string]any{
|
||||
@@ -1255,7 +1187,7 @@ func TestCleanComponentParams_TitleFamilyStillDropsUnknownKeys(t *testing.T) {
|
||||
"method": "hierarchy",
|
||||
},
|
||||
}
|
||||
result := CleanComponentParams(dslJSON, raw)
|
||||
result := buildTestParserConfig(t, dslJSON, raw)
|
||||
params, ok := result["TitleChunker:StaleSnapshot"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("expected TitleChunker params to survive, got %#v", result)
|
||||
|
||||
@@ -144,7 +144,11 @@ func (d *DatasetService) CreateDataset(ctx context.Context, req *service.CreateD
|
||||
common.Warn("failed to load pipeline DSL for building parser_config",
|
||||
zap.String("parserID", parserID), zap.Error(dslErr))
|
||||
} else {
|
||||
parserConfig = pipelinepkg.BuildParserConfig(dslJSON, req.ParserConfig)
|
||||
var buildErr error
|
||||
parserConfig, buildErr = pipelinepkg.BuildParserConfig(dslJSON, req.ParserConfig)
|
||||
if buildErr != nil {
|
||||
return nil, common.CodeArgumentError, buildErr
|
||||
}
|
||||
}
|
||||
|
||||
// Preserve the public default shape without overriding pipeline delimiters
|
||||
|
||||
@@ -206,7 +206,7 @@ func validateDatasetParserConfigSize(parserConfig map[string]interface{}) error
|
||||
// not component-scoped (i.e. do not contain ":"), since the Go backend only
|
||||
// consumes keys keyed by a component id (e.g. "Parser:abc123" or
|
||||
// "Extractor:AutoExtractDefault"). Flat keys are legacy transport artifacts
|
||||
// that downstream consumers (CleanComponentParams) never read, so they are
|
||||
// that downstream consumers (BuildParserConfig) never read, so they are
|
||||
// dropped rather than rejected. Only top-level keys are affected; a key nested
|
||||
// inside a component node (e.g. "Extractor:AutoExtractDefault.metadata") keeps
|
||||
// its existing name. It mutates the provided map in place and returns the names
|
||||
|
||||
@@ -301,7 +301,11 @@ func (d *DatasetService) UpdateDataset(ctx context.Context, datasetID, userID st
|
||||
zap.String("parserID", effParserID), zap.Error(dslErr))
|
||||
}
|
||||
if dslJSON != nil {
|
||||
parserConfig := pipelinepkg.BuildParserConfig(dslJSON, map[string]interface{}(req.ParserConfig))
|
||||
parserConfig, buildErr := pipelinepkg.BuildParserConfig(dslJSON, map[string]interface{}(req.ParserConfig))
|
||||
if buildErr != nil {
|
||||
txCode = common.CodeDataError
|
||||
return buildErr
|
||||
}
|
||||
updates["parser_config"] = preserveDatasetParserConfigState(parserConfig, lockedKB.ParserConfig, req.ParserConfig)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"math"
|
||||
"path/filepath"
|
||||
"ragflow/internal/service"
|
||||
@@ -13,12 +14,12 @@ import (
|
||||
"ragflow/internal/common"
|
||||
"ragflow/internal/dao"
|
||||
"ragflow/internal/entity"
|
||||
componentpkg "ragflow/internal/ingestion/component"
|
||||
pipelinepkg "ragflow/internal/ingestion/pipeline"
|
||||
"ragflow/internal/permission"
|
||||
permissionresponse "ragflow/internal/permission/response"
|
||||
"ragflow/internal/tokenizer"
|
||||
|
||||
"go.uber.org/zap"
|
||||
"ragflow/internal/utility"
|
||||
)
|
||||
|
||||
func (s *DocumentService) BatchUpdateDocumentStatus(ctx context.Context, userID, datasetID, status string, documentIDs []string) (map[string]interface{}, common.ErrorCode, error) {
|
||||
@@ -155,35 +156,40 @@ func (s *DocumentService) UpdateDatasetDocument(ctx context.Context, userID, dat
|
||||
}
|
||||
return nil, common.CodeServerError, err
|
||||
}
|
||||
if code, err := s.validateDatasetDocumentUpdate(ctx, datasetID, documentID, userID, doc, req, present); err != nil {
|
||||
if code, err := s.validateDatasetDocumentUpdate(ctx, doc, req, present); err != nil {
|
||||
return nil, code, err
|
||||
}
|
||||
|
||||
if present["meta_fields"] {
|
||||
if err = s.replaceDocumentMetadata(ctx, documentID, req.MetaFields); err != nil {
|
||||
return nil, common.CodeDataError, err
|
||||
}
|
||||
// Normalize chunk_method into the same selection used to clean and save
|
||||
// parser_config. An omitted selection keeps the document's own template.
|
||||
parseType, parserID := req.ParseType, req.ParserID
|
||||
hasSelection := parseType != nil ||
|
||||
(parserID != nil && strings.TrimSpace(*parserID) != "") ||
|
||||
(req.PipelineID != nil && strings.TrimSpace(*req.PipelineID) != "")
|
||||
if !hasSelection && present["chunk_method"] {
|
||||
builtin := int(service.ParseModeBuiltIn)
|
||||
parseType, parserID = &builtin, req.ChunkMethod
|
||||
}
|
||||
|
||||
if present["name"] && req.Name != nil && (doc.Name == nil || *req.Name != *doc.Name) {
|
||||
if err = s.updateDocumentNameOnly(ctx, doc, kb, *req.Name); err != nil {
|
||||
return nil, common.CodeDataError, err
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve the effective parse mode once via the shared selection value
|
||||
// object. sel is the request selection (nil when the whole selection block
|
||||
// was omitted); current is the dataset's persisted selection. Resolve applies
|
||||
// PATCH semantics: an omitted block inherits the current mode. Both the
|
||||
// parser_config cleaning and the parser selection derive from this single
|
||||
// resolution so they can never disagree.
|
||||
sel, selErr := service.FromRequest(req.ParseType, req.ParserID, req.PipelineID)
|
||||
sel, selErr := service.FromRequest(parseType, parserID, req.PipelineID)
|
||||
if selErr != nil {
|
||||
return nil, common.CodeDataError, selErr
|
||||
}
|
||||
current := service.CurrentSelection(kb.ParserID, kb.PipelineID)
|
||||
current := service.CurrentSelection(doc.ParserID, doc.PipelineID)
|
||||
eff, _ := service.Resolve(current, sel)
|
||||
isPipeline, effParserID, effPipelineID := eff.Effective()
|
||||
selectionChanged := current.Mode != eff.Mode ||
|
||||
(isPipeline && current.PipelineID != eff.PipelineID) ||
|
||||
(!isPipeline && current.ParserID != effParserID)
|
||||
if selectionChanged && current.IsBuiltIn() && eff.IsBuiltIn() {
|
||||
registry, err := pipelinepkg.DefaultRegistry()
|
||||
if err != nil {
|
||||
return nil, common.CodeDataError, fmt.Errorf("builtin pipeline registry: %w", err)
|
||||
}
|
||||
previous, previousOK := registry.Get(current.ParserID)
|
||||
target, targetOK := registry.Get(effParserID)
|
||||
selectionChanged = !previousOK || !targetOK || previous.ParserID != target.ParserID
|
||||
}
|
||||
hasConfig := present["parser_config"] && req.ParserConfig != nil
|
||||
if (sel != nil || present["parser_config"]) && isPipeline && effPipelineID != nil && strings.TrimSpace(*effPipelineID) != "" {
|
||||
accessErr := permission.NewDatabaseChecker(dao.DB).CheckDependency(
|
||||
ctx,
|
||||
@@ -199,56 +205,66 @@ func (s *DocumentService) UpdateDatasetDocument(ctx context.Context, userID, dat
|
||||
}
|
||||
}
|
||||
|
||||
if present["parser_config"] && req.ParserConfig != nil {
|
||||
var dslJSON []byte
|
||||
dslJSON, err = service.LoadPipelineDSL(ctx, isPipeline, effParserID, effPipelineID)
|
||||
parserUpdates := map[string]interface{}{}
|
||||
if sel != nil || hasConfig {
|
||||
// Validate the target before any document, file, or metadata writes.
|
||||
dslJSON, err := service.LoadPipelineDSL(ctx, isPipeline, effParserID, effPipelineID)
|
||||
if err != nil {
|
||||
common.Warn("cleanAndUpdateDocumentParserConfig: failed to load DSL, falling back to merge",
|
||||
zap.Error(err))
|
||||
if err = s.updateDocumentParserConfig(ctx, doc.ID, req.ParserConfig); err != nil {
|
||||
return nil, common.CodeDataError, err
|
||||
return nil, common.CodeDataError, err
|
||||
}
|
||||
rawConfig := req.ParserConfig
|
||||
if !selectionChanged {
|
||||
rawConfig = common.DeepMergeMaps(doc.ParserConfig, req.ParserConfig)
|
||||
}
|
||||
cleaned, err := pipelinepkg.BuildParserConfig(dslJSON, rawConfig)
|
||||
if err != nil {
|
||||
return nil, common.CodeDataError, fmt.Errorf("invalid pipeline DSL: %w", err)
|
||||
}
|
||||
if doc.Name != nil {
|
||||
family := componentpkg.ParserFileFamily(utility.GetFileExtension(*doc.Name))
|
||||
if family != "" {
|
||||
if declared, found := configDeclaredFamily(cleaned, family); found && !declared {
|
||||
return nil, common.CodeDataError, fmt.Errorf("selected pipeline does not support file type %q", family)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
cleaned := pipelinepkg.BuildParserConfig(dslJSON, req.ParserConfig)
|
||||
pipelinepkg.ApplyParentChildChunkerConfig(cleaned, req.ParserConfig)
|
||||
}
|
||||
if selectionChanged || hasConfig {
|
||||
// Inherited runtime delimiters are not an explicit override when
|
||||
// this request changes parent_child.
|
||||
parentChildOverrides := maps.Clone(rawConfig)
|
||||
for componentID, raw := range req.ParserConfig {
|
||||
if requested, ok := raw.(map[string]any); ok {
|
||||
if _, provided := requested["parent_child"]; provided {
|
||||
parentChildOverrides[componentID] = requested
|
||||
}
|
||||
}
|
||||
}
|
||||
pipelinepkg.ApplyParentChildChunkerConfig(cleaned, parentChildOverrides)
|
||||
llmID, _, _ := service.NewTenantService().GetDefaultModelRefsForUser(ctx, userID, kb.TenantID, entity.ModelTypeChat)
|
||||
cleaned = service.ApplyComponentScopedParserConfig(cleaned, llmID)
|
||||
if err = s.documentDAO.UpdateByID(ctx, dao.DB, doc.ID, map[string]interface{}{
|
||||
"parser_config": cleaned,
|
||||
}); err != nil {
|
||||
return nil, common.CodeDataError, err
|
||||
}
|
||||
parserUpdates["parser_config"] = cleaned
|
||||
}
|
||||
}
|
||||
|
||||
// Save the selection without changing existing parse results. Cleanup is
|
||||
// owned by the explicit reparse request.
|
||||
parserUpdates := map[string]interface{}{}
|
||||
if req.ParseType != nil {
|
||||
switch {
|
||||
case !isPipeline: // BuiltIn
|
||||
if present["parser_id"] && req.ParserID != nil {
|
||||
if p := strings.TrimSpace(*req.ParserID); p != "" {
|
||||
parserUpdates["parser_id"] = p
|
||||
}
|
||||
}
|
||||
// Drop any prior canvas so the worker falls back to the builtin template.
|
||||
parserUpdates["pipeline_id"] = ""
|
||||
case isPipeline: // Pipeline
|
||||
if present["pipeline_id"] && req.PipelineID != nil {
|
||||
parserUpdates["pipeline_id"] = *req.PipelineID
|
||||
}
|
||||
}
|
||||
} else if present["chunk_method"] && req.ChunkMethod != nil {
|
||||
// chunk_method is the public alias for a direct parser change; it does
|
||||
// not require parse_type (mirrors Python's update_chunk_method).
|
||||
if p := strings.TrimSpace(*req.ChunkMethod); !strings.EqualFold(p, doc.ParserID) {
|
||||
parserUpdates["parser_id"] = p
|
||||
parserUpdates["pipeline_id"] = ""
|
||||
} else if doc.PipelineID != nil && *doc.PipelineID != "" {
|
||||
if sel != nil {
|
||||
if isPipeline {
|
||||
parserUpdates["pipeline_id"] = *effPipelineID
|
||||
} else {
|
||||
parserUpdates["parser_id"] = effParserID
|
||||
parserUpdates["pipeline_id"] = ""
|
||||
}
|
||||
}
|
||||
if present["meta_fields"] {
|
||||
if err = s.replaceDocumentMetadata(ctx, documentID, req.MetaFields); err != nil {
|
||||
return nil, common.CodeDataError, err
|
||||
}
|
||||
}
|
||||
if present["name"] && req.Name != nil && (doc.Name == nil || *req.Name != *doc.Name) {
|
||||
if err = s.updateDocumentNameOnly(ctx, doc, kb, *req.Name); err != nil {
|
||||
return nil, common.CodeDataError, err
|
||||
}
|
||||
}
|
||||
// Save selection and config together; parsed content changes only on reparse.
|
||||
if len(parserUpdates) > 0 {
|
||||
if err = s.documentDAO.UpdateByID(ctx, dao.DB, doc.ID, parserUpdates); err != nil {
|
||||
return nil, common.CodeDataError, err
|
||||
@@ -308,7 +324,7 @@ func (s *DocumentService) validateDocumentModifiable(ctx context.Context, doc *e
|
||||
}
|
||||
}
|
||||
|
||||
func (s *DocumentService) validateDatasetDocumentUpdate(ctx context.Context, datasetID, documentID, userID string, doc *entity.Document, req *UpdateDatasetDocumentRequest, present map[string]bool) (common.ErrorCode, error) {
|
||||
func (s *DocumentService) validateDatasetDocumentUpdate(ctx context.Context, doc *entity.Document, req *UpdateDatasetDocumentRequest, present map[string]bool) (common.ErrorCode, error) {
|
||||
if req == nil {
|
||||
return common.CodeDataError, errors.New("invalid request payload")
|
||||
}
|
||||
@@ -347,29 +363,15 @@ func (s *DocumentService) validateDatasetDocumentUpdate(ctx context.Context, dat
|
||||
if req.ChunkMethod == nil || strings.TrimSpace(*req.ChunkMethod) == "" {
|
||||
return common.CodeDataError, errors.New("`chunk_method` (empty string) is not valid")
|
||||
}
|
||||
cm := strings.TrimSpace(*req.ChunkMethod)
|
||||
if !validDocumentChunkMethods[cm] {
|
||||
return common.CodeDataError, fmt.Errorf("Field: <chunk_method> - Message: <`chunk_method` %s doesn't exist> - Value: <%s>", cm, cm)
|
||||
registry, err := pipelinepkg.DefaultRegistry()
|
||||
if err != nil {
|
||||
return common.CodeDataError, fmt.Errorf("builtin pipeline registry: %w", err)
|
||||
}
|
||||
if (doc.Type == "visual" && cm != "picture") || (isPresentationFile(doc.Name) && cm != "presentation") {
|
||||
return common.CodeDataError, errors.New("not supported yet")
|
||||
if !registry.IsValid(*req.ChunkMethod) {
|
||||
return common.CodeDataError, fmt.Errorf("unknown builtin parser_id: %s", strings.TrimSpace(*req.ChunkMethod))
|
||||
}
|
||||
}
|
||||
|
||||
if present["parse_type"] || present["parser_id"] || present["pipeline_id"] {
|
||||
sel, err := service.FromRequest(req.ParseType, req.ParserID, req.PipelineID)
|
||||
if err != nil {
|
||||
return common.CodeDataError, err
|
||||
}
|
||||
// The parser_id type constraint (visual/presentation) only applies in
|
||||
// builtin mode — in pipeline mode parser_id is not applicable.
|
||||
if sel != nil && sel.IsBuiltIn() && present["parser_id"] && req.ParserID != nil {
|
||||
parserID := strings.TrimSpace(*req.ParserID)
|
||||
if (doc.Type == "visual" && parserID != "picture") || (isPresentationFile(doc.Name) && parserID != "presentation") {
|
||||
return common.CodeDataError, errors.New("not supported yet")
|
||||
}
|
||||
}
|
||||
}
|
||||
if present["name"] {
|
||||
// An explicit null name is a type error, not an unset field.
|
||||
if req.Name == nil {
|
||||
@@ -431,14 +433,6 @@ func (s *DocumentService) validateDocumentName(ctx context.Context, doc *entity.
|
||||
return common.CodeSuccess, nil
|
||||
}
|
||||
|
||||
func isPresentationFile(name *string) bool {
|
||||
if name == nil {
|
||||
return false
|
||||
}
|
||||
ext := strings.ToLower(filepath.Ext(*name))
|
||||
return ext == ".ppt" || ext == ".pptx" || ext == ".pages" || ext == ".dps" || ext == ".dpt"
|
||||
}
|
||||
|
||||
func validateMetaFields(meta map[string]any) error {
|
||||
if meta == nil {
|
||||
return nil
|
||||
@@ -505,24 +499,6 @@ func (s *DocumentService) updateDocumentNameOnly(ctx context.Context, doc *entit
|
||||
)
|
||||
}
|
||||
|
||||
func (s *DocumentService) updateDocumentParserConfig(ctx context.Context, documentID string, config map[string]any) error {
|
||||
if len(config) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
doc, err := s.documentDAO.GetByID(ctx, dao.DB, documentID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("document(%s) not found", documentID)
|
||||
}
|
||||
|
||||
merged := common.DeepMergeMaps(doc.ParserConfig, config)
|
||||
pipelinepkg.ApplyParentChildChunkerConfig(merged, config)
|
||||
|
||||
return s.documentDAO.UpdateByID(ctx, dao.DB, documentID, map[string]interface{}{
|
||||
"parser_config": entity.JSONMap(merged),
|
||||
})
|
||||
}
|
||||
|
||||
func (s *DocumentService) toUpdateDatasetDocumentResponse(ctx context.Context, doc *entity.Document, metaFields map[string]interface{}) (*UpdateDatasetDocumentResponse, error) {
|
||||
if metaFields == nil {
|
||||
metaFields = map[string]interface{}{}
|
||||
@@ -576,13 +552,6 @@ func (s *DocumentService) toUpdateDatasetDocumentResponse(ctx context.Context, d
|
||||
}, nil
|
||||
}
|
||||
|
||||
// validDocumentChunkMethods mirrors Python's UpdateDocumentReq chunk_method set.
|
||||
var validDocumentChunkMethods = map[string]bool{
|
||||
"naive": true, "manual": true, "qa": true, "table": true, "paper": true,
|
||||
"book": true, "laws": true, "presentation": true, "picture": true,
|
||||
"one": true, "knowledge_graph": true, "email": true, "tag": true,
|
||||
}
|
||||
|
||||
// pythonFloatRepr formats a float the way Python's str()/repr() does:
|
||||
// integral floats keep a trailing ".0".
|
||||
func pythonFloatRepr(v float64) string {
|
||||
|
||||
@@ -1,40 +1,476 @@
|
||||
package document
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"ragflow/internal/common"
|
||||
"ragflow/internal/dao"
|
||||
"ragflow/internal/entity"
|
||||
"ragflow/internal/service"
|
||||
)
|
||||
|
||||
// TestIsPresentationFile pins the name-based presentation guard used by the
|
||||
// parser_id / chunk_method switch in document_dataset_update.go. A WPS
|
||||
// presentation (dps/dpt) must be treated as a presentation on a par with ppt/
|
||||
// pptx/pages, so the API allows switching its chunk_method to "presentation".
|
||||
func TestIsPresentationFile(t *testing.T) {
|
||||
p := func(s string) *string { return &s }
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
in *string
|
||||
want bool
|
||||
func TestUpdateDatasetDocumentParentChildPatchUsesRequestedFields(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
enabled bool
|
||||
children []any
|
||||
patch map[string]any
|
||||
want []any
|
||||
}{
|
||||
{"nil name is not presentation", nil, false},
|
||||
{"ppt is presentation", p("deck.ppt"), true},
|
||||
{"pptx is presentation", p("deck.pptx"), true},
|
||||
{"pages is presentation", p("deck.pages"), true},
|
||||
{"wps presentation dps is presentation", p("deck.dps"), true},
|
||||
{"wps presentation dpt is presentation", p("deck.dpt"), true},
|
||||
{"wps presentation dps is case-insensitive", p("DECK.DPS"), true},
|
||||
{"pdf is not presentation", p("report.pdf"), false},
|
||||
{"wps doc is not presentation", p("note.wps"), false},
|
||||
{"wps sheet et is not presentation", p("sheet.et"), false},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
{"enable", false, []any{}, map[string]any{"parent_child": map[string]any{"use_parent_child": true}}, []any{"|"}},
|
||||
{"disable", true, []any{"|"}, map[string]any{"parent_child": map[string]any{"use_parent_child": false}}, []any{}},
|
||||
{"delimiter", true, []any{"|"}, map[string]any{"parent_child": map[string]any{"children_delimiter": "!"}}, []any{"!"}},
|
||||
{"explicit children", true, []any{"|"}, map[string]any{"parent_child": map[string]any{"children_delimiter": "!"}, "children_delimiters": []any{"?"}}, []any{"?"}},
|
||||
{"unrelated patch", false, []any{"|"}, map[string]any{"chunk_token_size": 128}, []any{"|"}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := isPresentationFile(tc.in); got != tc.want {
|
||||
t.Fatalf("isPresentationFile(%v) = %v, want %v", tc.in, got, tc.want)
|
||||
db := setupServiceTestDB(t)
|
||||
pushServiceDB(t, db)
|
||||
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
|
||||
insertNamedTestDoc(t, "doc-1", "kb-1", "report.txt", 0, 0)
|
||||
if err := db.Model(&entity.Document{}).Where("id = ?", "doc-1").Update("parser_config", entity.JSONMap{
|
||||
"GeneralChunker:SixApplesFall": map[string]any{
|
||||
"children_delimiters": tc.children,
|
||||
"parent_child": map[string]any{"use_parent_child": tc.enabled, "children_delimiter": "|"},
|
||||
},
|
||||
}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resp, code, err := testDocumentService(t).UpdateDatasetDocument(documentTestContext(t.Context()), "user-1", "kb-1", "doc-1", &UpdateDatasetDocumentRequest{
|
||||
ParserConfig: map[string]any{"GeneralChunker:SixApplesFall": tc.patch},
|
||||
}, map[string]bool{"parser_config": true})
|
||||
if code != common.CodeSuccess || err != nil {
|
||||
t.Fatalf("update: code=%d err=%v", code, err)
|
||||
}
|
||||
doc, err := dao.NewDocumentDAO().GetByID(t.Context(), db, "doc-1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, config := range []map[string]any{resp.ParserConfig, doc.ParserConfig} {
|
||||
chunker, _ := config["GeneralChunker:SixApplesFall"].(map[string]any)
|
||||
if !reflect.DeepEqual(chunker["children_delimiters"], tc.want) {
|
||||
t.Fatalf("children_delimiters=%v, want=%v", chunker["children_delimiters"], tc.want)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateDatasetDocumentConfigPatchPreservesOmittedSettings(t *testing.T) {
|
||||
for _, selection := range []string{"omitted", "same builtin", "same canvas"} {
|
||||
t.Run(selection, func(t *testing.T) {
|
||||
db := setupServiceTestDB(t)
|
||||
pushServiceDB(t, db)
|
||||
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
|
||||
insertNamedTestDoc(t, "doc-1", "kb-1", "report.docx", 0, 0)
|
||||
config := entity.JSONMap{
|
||||
"Parser:HipSignsRhyme": map[string]any{
|
||||
"enable_vision_enhancement": true,
|
||||
"vlm": map[string]any{"llm_id": "old-model", "system_prompt": "keep this prompt"},
|
||||
},
|
||||
"GeneralChunker:SixApplesFall": map[string]any{"chunk_token_size": 123},
|
||||
"Parser:Removed": map[string]any{"pdf": map[string]any{}},
|
||||
}
|
||||
updates := map[string]any{"parser_id": "general", "parser_config": config}
|
||||
req := &UpdateDatasetDocumentRequest{ParserConfig: map[string]any{
|
||||
"Parser:HipSignsRhyme": map[string]any{"vlm": map[string]any{"llm_id": "new-model"}},
|
||||
}}
|
||||
present := map[string]bool{"parser_config": true}
|
||||
switch selection {
|
||||
case "same builtin":
|
||||
req.ParseType, req.ParserID = new(1), new("general")
|
||||
present["parse_type"], present["parser_id"] = true, true
|
||||
case "same canvas":
|
||||
const pipelineID = "1234567890abcdef1234567890abcdef"
|
||||
insertTestCanvas(t, pipelineID, "user-1")
|
||||
updates["pipeline_id"] = pipelineID
|
||||
req.ParseType, req.PipelineID = new(2), new(pipelineID)
|
||||
present["parse_type"], present["pipeline_id"] = true, true
|
||||
}
|
||||
if err := db.Model(&entity.Document{}).Where("id = ?", "doc-1").Updates(updates).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resp, code, err := testDocumentService(t).UpdateDatasetDocument(documentTestContext(t.Context()), "user-1", "kb-1", "doc-1", req, present)
|
||||
if err != nil || code != common.CodeSuccess {
|
||||
t.Fatalf("update: code=%d err=%v", code, err)
|
||||
}
|
||||
doc, err := dao.NewDocumentDAO().GetByID(t.Context(), db, "doc-1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, saved := range []map[string]any{resp.ParserConfig, doc.ParserConfig} {
|
||||
parser, _ := saved["Parser:HipSignsRhyme"].(map[string]any)
|
||||
vlm, _ := parser["vlm"].(map[string]any)
|
||||
chunker, _ := saved["GeneralChunker:SixApplesFall"].(map[string]any)
|
||||
if parser["enable_vision_enhancement"] != true || vlm["llm_id"] != "new-model" || vlm["system_prompt"] != "keep this prompt" || chunker["chunk_token_size"] != float64(123) {
|
||||
t.Fatalf("partial update lost settings: %#v", saved)
|
||||
}
|
||||
if _, exists := saved["Parser:Removed"]; exists {
|
||||
t.Fatalf("stale component survived cleaning: %#v", saved)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateDatasetDocumentRejectsUnsupportedFileFamilyBeforeWrites(t *testing.T) {
|
||||
for _, filename := range []string{"image.png", "deck.pptx", "deck.pages", "DECK.DPS", "deck.dpt"} {
|
||||
for _, selection := range []string{"chunk_method", "parser_id"} {
|
||||
for _, withConfig := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("%s/%s/config=%t", filename, selection, withConfig), func(t *testing.T) {
|
||||
db := setupServiceTestDB(t)
|
||||
pushServiceDB(t, db)
|
||||
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
|
||||
insertNamedTestDoc(t, "doc-1", "kb-1", filename, 0, 0)
|
||||
insertTestFile(t, "file-1", "folder-1", filename, &filename)
|
||||
insertTestFile2Document(t, "f2d-1", "file-1", "doc-1")
|
||||
if filename == "image.png" {
|
||||
if err := db.Model(&entity.Document{}).Where("id = ?", "doc-1").Updates(map[string]any{
|
||||
"type": "visual", "parser_id": "picture", "parser_config": pictureParserConfig(),
|
||||
}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
before, err := dao.NewDocumentDAO().GetByID(t.Context(), db, "doc-1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req := &UpdateDatasetDocumentRequest{Name: new("renamed." + strings.SplitN(filename, ".", 2)[1]), MetaFields: map[string]any{"author": "changed"}}
|
||||
present := map[string]bool{"name": true, "meta_fields": true, selection: true}
|
||||
if selection == "parser_id" {
|
||||
req.ParseType, req.ParserID = new(1), new("manual")
|
||||
present["parse_type"] = true
|
||||
} else {
|
||||
req.ChunkMethod = new("manual")
|
||||
}
|
||||
if withConfig {
|
||||
req.ParserConfig = map[string]any{"Parser:HipSignsRhyme": map[string]any{"enable_vision_enhancement": true}}
|
||||
present["parser_config"] = true
|
||||
}
|
||||
engine := newMetadataDocEngine(map[string]map[string]any{"doc-1": {"author": "original"}}, map[string]string{"doc-1": "kb-1"})
|
||||
svc := testDocumentService(t)
|
||||
svc.docEngine = engine
|
||||
svc.metadataSvc = service.NewMetadataServiceForTest(dao.NewKnowledgebaseDAO(), engine)
|
||||
_, code, err := svc.UpdateDatasetDocument(documentTestContext(t.Context()), "user-1", "kb-1", "doc-1", req, present)
|
||||
if code != common.CodeDataError || err == nil || !strings.Contains(err.Error(), "does not support") {
|
||||
t.Fatalf("expected unsupported file rejection: code=%d err=%v", code, err)
|
||||
}
|
||||
after, err := dao.NewDocumentDAO().GetByID(t.Context(), db, "doc-1")
|
||||
if err != nil || !reflect.DeepEqual(before, after) {
|
||||
t.Fatalf("rejected request changed document: before=%#v after=%#v err=%v", before, after, err)
|
||||
}
|
||||
file, err := dao.NewFileDAO().GetByID(t.Context(), db, "file-1")
|
||||
if err != nil || file.Name != filename {
|
||||
t.Fatalf("rejected request changed file: file=%#v err=%v", file, err)
|
||||
}
|
||||
if !reflect.DeepEqual(engine.records["doc-1"], map[string]any{"author": "original"}) {
|
||||
t.Fatalf("rejected request changed metadata: %#v", engine.records)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateDatasetDocumentParserAliasPreservesConfig(t *testing.T) {
|
||||
for _, current := range []string{"naive", "general"} {
|
||||
for _, selection := range []string{"chunk_method", "parser_id"} {
|
||||
for _, withConfig := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("%s/%s/config=%t", current, selection, withConfig), func(t *testing.T) {
|
||||
db := setupServiceTestDB(t)
|
||||
pushServiceDB(t, db)
|
||||
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
|
||||
insertNamedTestDoc(t, "doc-1", "kb-1", "report.docx", 0, 0)
|
||||
if err := db.Model(&entity.Document{}).Where("id = ?", "doc-1").Updates(map[string]any{
|
||||
"parser_id": current,
|
||||
"parser_config": entity.JSONMap{"GeneralChunker:SixApplesFall": map[string]any{"chunk_token_size": 123}},
|
||||
}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
target := "general"
|
||||
if current == "general" {
|
||||
target = "naive"
|
||||
}
|
||||
req := &UpdateDatasetDocumentRequest{}
|
||||
present := map[string]bool{selection: true}
|
||||
if selection == "parser_id" {
|
||||
req.ParseType, req.ParserID = new(1), &target
|
||||
present["parse_type"] = true
|
||||
} else {
|
||||
req.ChunkMethod = &target
|
||||
}
|
||||
if withConfig {
|
||||
req.ParserConfig = map[string]any{"Parser:HipSignsRhyme": map[string]any{"enable_vision_enhancement": true}}
|
||||
present["parser_config"] = true
|
||||
}
|
||||
resp, code, err := testDocumentService(t).UpdateDatasetDocument(documentTestContext(t.Context()), "user-1", "kb-1", "doc-1", req, present)
|
||||
if err != nil || code != common.CodeSuccess {
|
||||
t.Fatalf("update: code=%d err=%v", code, err)
|
||||
}
|
||||
doc, err := dao.NewDocumentDAO().GetByID(t.Context(), db, "doc-1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if resp.ParserID != target || doc.ParserID != target {
|
||||
t.Fatalf("parser ID not saved: response=%q stored=%q want=%q", resp.ParserID, doc.ParserID, target)
|
||||
}
|
||||
for _, config := range []map[string]any{resp.ParserConfig, doc.ParserConfig} {
|
||||
chunker, _ := config["GeneralChunker:SixApplesFall"].(map[string]any)
|
||||
if chunker["chunk_token_size"] != float64(123) {
|
||||
t.Fatalf("alias switch reset chunk size: %#v", config)
|
||||
}
|
||||
if withConfig {
|
||||
parser, _ := config["Parser:HipSignsRhyme"].(map[string]any)
|
||||
if parser["enable_vision_enhancement"] != true {
|
||||
t.Fatalf("alias switch lost config patch: %#v", config)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateDatasetDocumentSelectionWithoutConfig(t *testing.T) {
|
||||
for _, selection := range []string{"chunk_method", "parser_id"} {
|
||||
for _, current := range []string{"general", "manual"} {
|
||||
t.Run(selection+"/"+current, func(t *testing.T) {
|
||||
db := setupServiceTestDB(t)
|
||||
pushServiceDB(t, db)
|
||||
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
|
||||
insertNamedTestDoc(t, "doc-1", "kb-1", "report.docx", 0, 0)
|
||||
config := entity.JSONMap{"ManualChunker:NineInsectsFind": map[string]any{"hierarchy": float64(7)}}
|
||||
if current == "general" {
|
||||
config = entity.JSONMap{"GeneralChunker:SixApplesFall": map[string]any{"chunk_token_size": 123}}
|
||||
}
|
||||
if err := db.Model(&entity.Document{}).Where("id = ?", "doc-1").Updates(map[string]any{"parser_id": current, "parser_config": config}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req := &UpdateDatasetDocumentRequest{}
|
||||
present := map[string]bool{selection: true}
|
||||
if selection == "parser_id" {
|
||||
req.ParseType, req.ParserID = new(1), new("manual")
|
||||
present["parse_type"] = true
|
||||
} else {
|
||||
req.ChunkMethod = new("manual")
|
||||
}
|
||||
_, code, err := testDocumentService(t).UpdateDatasetDocument(documentTestContext(t.Context()), "user-1", "kb-1", "doc-1", req, present)
|
||||
if err != nil || code != common.CodeSuccess {
|
||||
t.Fatalf("update: code=%d err=%v", code, err)
|
||||
}
|
||||
doc, err := dao.NewDocumentDAO().GetByID(t.Context(), db, "doc-1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if doc.ParserID != "manual" {
|
||||
t.Fatalf("parser=%q", doc.ParserID)
|
||||
}
|
||||
if current == "manual" {
|
||||
if !reflect.DeepEqual(doc.ParserConfig, config) {
|
||||
t.Fatalf("unchanged selection reset config: %#v", doc.ParserConfig)
|
||||
}
|
||||
return
|
||||
}
|
||||
if _, ok := doc.ParserConfig["ManualChunker:NineInsectsFind"]; !ok {
|
||||
t.Fatalf("new template config missing: %#v", doc.ParserConfig)
|
||||
}
|
||||
if _, ok := doc.ParserConfig["GeneralChunker:SixApplesFall"]; ok {
|
||||
t.Fatalf("old template config retained: %#v", doc.ParserConfig)
|
||||
}
|
||||
parser, ok := doc.ParserConfig["Parser:HipSignsRhyme"].(map[string]any)
|
||||
if !ok || parser["docx"] == nil {
|
||||
t.Fatalf("target file family missing: %#v", doc.ParserConfig)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateDatasetDocumentChunkMethodAcceptsCurrentTemplates(t *testing.T) {
|
||||
for _, file := range []struct{ method, name string }{{"general", "report.docx"}, {"audio", "recording.mp3"}} {
|
||||
t.Run(file.method, func(t *testing.T) {
|
||||
db := setupServiceTestDB(t)
|
||||
pushServiceDB(t, db)
|
||||
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
|
||||
insertNamedTestDoc(t, "doc-1", "kb-1", file.name, 0, 0)
|
||||
resp, code, err := testDocumentService(t).UpdateDatasetDocument(documentTestContext(t.Context()), "user-1", "kb-1", "doc-1", &UpdateDatasetDocumentRequest{
|
||||
ChunkMethod: &file.method,
|
||||
}, map[string]bool{"chunk_method": true})
|
||||
if err != nil || code != common.CodeSuccess {
|
||||
t.Fatalf("select %s: code=%d err=%v", file.method, code, err)
|
||||
}
|
||||
doc, err := dao.NewDocumentDAO().GetByID(t.Context(), db, "doc-1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if resp.ParserID != file.method || doc.ParserID != file.method {
|
||||
t.Fatalf("selection not saved: response=%q stored=%q want=%q", resp.ParserID, doc.ParserID, file.method)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateDatasetDocumentUsesEffectiveParserForConfig(t *testing.T) {
|
||||
for _, changeMethod := range []bool{false, true} {
|
||||
name := "document selection"
|
||||
if changeMethod {
|
||||
name = "chunk method switch"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
db := setupServiceTestDB(t)
|
||||
pushServiceDB(t, db)
|
||||
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
|
||||
insertNamedTestDoc(t, "doc-1", "kb-1", "report.docx", 0, 0)
|
||||
req := &UpdateDatasetDocumentRequest{ParserConfig: map[string]any{
|
||||
"ManualChunker:NineInsectsFind": map[string]any{"hierarchy": 2},
|
||||
"GeneralChunker:SixApplesFall": map[string]any{"chunk_token_size": 123},
|
||||
}}
|
||||
present := map[string]bool{"parser_config": true}
|
||||
if changeMethod {
|
||||
req.ChunkMethod = new("manual")
|
||||
req.ParserID, req.PipelineID = new(""), new("")
|
||||
present["chunk_method"] = true
|
||||
present["parser_id"], present["pipeline_id"] = true, true
|
||||
} else if err := db.Model(&entity.Document{}).Where("id = ?", "doc-1").Update("parser_id", "manual").Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, code, err := testDocumentService(t).UpdateDatasetDocument(documentTestContext(t.Context()), "user-1", "kb-1", "doc-1", req, present)
|
||||
if err != nil || code != common.CodeSuccess {
|
||||
t.Fatalf("update: code=%d err=%v", code, err)
|
||||
}
|
||||
doc, err := dao.NewDocumentDAO().GetByID(t.Context(), db, "doc-1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
chunker, ok := doc.ParserConfig["ManualChunker:NineInsectsFind"].(map[string]any)
|
||||
if doc.ParserID != "manual" || !ok || chunker["hierarchy"] != float64(2) {
|
||||
t.Fatalf("target template config lost: parser=%q config=%#v", doc.ParserID, doc.ParserConfig)
|
||||
}
|
||||
if _, ok := doc.ParserConfig["GeneralChunker:SixApplesFall"]; ok {
|
||||
t.Fatalf("config retained a node from the dataset template: %#v", doc.ParserConfig)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateDatasetDocumentKeepsDocumentCanvasForConfigPatch(t *testing.T) {
|
||||
db := setupServiceTestDB(t)
|
||||
pushServiceDB(t, db)
|
||||
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
|
||||
insertNamedTestDoc(t, "doc-1", "kb-1", "report.docx", 0, 0)
|
||||
const pipelineID = "1234567890abcdef1234567890abcdef"
|
||||
if err := db.Create(&entity.UserCanvas{
|
||||
ID: pipelineID, UserID: "user-1", TenantID: new("tenant-1"), Permission: string(entity.TenantPermissionMe),
|
||||
DSL: entity.JSONMap{
|
||||
"components": map[string]any{
|
||||
"Parser:custom": map[string]any{"obj": map[string]any{
|
||||
"component_name": "Parser", "params": map[string]any{"docx": map[string]any{}},
|
||||
}},
|
||||
},
|
||||
},
|
||||
}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Model(&entity.Document{}).Where("id = ?", "doc-1").Update("pipeline_id", pipelineID).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, code, err := testDocumentService(t).UpdateDatasetDocument(documentTestContext(t.Context()), "user-1", "kb-1", "doc-1", &UpdateDatasetDocumentRequest{
|
||||
ParserConfig: map[string]any{"Parser:custom": map[string]any{"vlm": map[string]any{"llm_id": "vision-model"}}},
|
||||
}, map[string]bool{"parser_config": true})
|
||||
if err != nil || code != common.CodeSuccess {
|
||||
t.Fatalf("update: code=%d err=%v", code, err)
|
||||
}
|
||||
doc, err := dao.NewDocumentDAO().GetByID(t.Context(), db, "doc-1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
parser, ok := doc.ParserConfig["Parser:custom"].(map[string]any)
|
||||
if !ok || !reflect.DeepEqual(parser["vlm"], map[string]any{"llm_id": "vision-model"}) || doc.PipelineID == nil || *doc.PipelineID != pipelineID {
|
||||
t.Fatalf("document canvas config lost: %#v", doc)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateDatasetDocumentRejectsParserErrorsBeforeWrites(t *testing.T) {
|
||||
for _, failure := range []string{"unknown builtin", "unknown builtin without config", "empty canvas", "malformed canvas", "invalid selection", "knowledge_graph", "tag", "invalid chunk with builtin", "invalid chunk with canvas"} {
|
||||
t.Run(failure, func(t *testing.T) {
|
||||
db := setupServiceTestDB(t)
|
||||
pushServiceDB(t, db)
|
||||
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
|
||||
insertNamedTestDoc(t, "doc-1", "kb-1", "old.docx", 0, 0)
|
||||
insertTestFile(t, "file-1", "folder-1", "old.docx", new("old.docx"))
|
||||
insertTestFile2Document(t, "f2d-1", "file-1", "doc-1")
|
||||
req := &UpdateDatasetDocumentRequest{
|
||||
Name: new("new.docx"), ParseType: new(1), ParserID: new("missing-parser"),
|
||||
MetaFields: map[string]any{"author": "must-not-save"},
|
||||
ParserConfig: map[string]any{"Parser:HipSignsRhyme": map[string]any{"vlm": map[string]any{"llm_id": "must-not-save"}}},
|
||||
}
|
||||
present := map[string]bool{"name": true, "parse_type": true, "parser_id": true, "parser_config": true, "meta_fields": true}
|
||||
wantError := "unknown builtin parser_id"
|
||||
switch failure {
|
||||
case "invalid chunk with builtin", "invalid chunk with canvas":
|
||||
req.ParserID, req.ChunkMethod = new("general"), new("does-not-exist")
|
||||
present["chunk_method"] = true
|
||||
wantError = "unknown builtin parser_id: does-not-exist"
|
||||
if failure == "invalid chunk with canvas" {
|
||||
const pipelineID = "1234567890abcdef1234567890abcdef"
|
||||
insertTestCanvas(t, pipelineID, "user-1")
|
||||
req.ParseType, req.ParserID, req.PipelineID = new(2), nil, new(pipelineID)
|
||||
delete(present, "parser_id")
|
||||
present["pipeline_id"] = true
|
||||
}
|
||||
case "knowledge_graph", "tag":
|
||||
req.ParseType, req.ParserID, req.ChunkMethod = nil, nil, &failure
|
||||
delete(present, "parse_type")
|
||||
delete(present, "parser_id")
|
||||
present["chunk_method"] = true
|
||||
case "unknown builtin without config":
|
||||
req.ParserConfig = nil
|
||||
delete(present, "parser_config")
|
||||
case "empty canvas", "malformed canvas":
|
||||
const pipelineID = "1234567890abcdef1234567890abcdef"
|
||||
dsl := entity.JSONMap{}
|
||||
wantError = "has no DSL"
|
||||
if failure == "malformed canvas" {
|
||||
dsl["graph"] = map[string]any{}
|
||||
wantError = "DSL has no components"
|
||||
}
|
||||
if err := db.Create(&entity.UserCanvas{
|
||||
ID: pipelineID, UserID: "user-1", TenantID: new("tenant-1"), Permission: string(entity.TenantPermissionMe), DSL: dsl,
|
||||
}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req.ParseType, req.ParserID, req.PipelineID = new(2), nil, new(pipelineID)
|
||||
delete(present, "parser_id")
|
||||
present["pipeline_id"] = true
|
||||
case "invalid selection":
|
||||
req.ParseType = new(2)
|
||||
wantError = "parser_id must not be set"
|
||||
}
|
||||
engine := newMetadataDocEngine(map[string]map[string]any{"doc-1": {"author": "original"}}, map[string]string{"doc-1": "kb-1"})
|
||||
svc := testDocumentService(t)
|
||||
svc.docEngine = engine
|
||||
svc.metadataSvc = service.NewMetadataServiceForTest(dao.NewKnowledgebaseDAO(), engine)
|
||||
_, code, err := svc.UpdateDatasetDocument(documentTestContext(t.Context()), "user-1", "kb-1", "doc-1", req, present)
|
||||
if code != common.CodeDataError || err == nil || !strings.Contains(err.Error(), wantError) {
|
||||
t.Fatalf("expected %q, got code=%d err=%v", wantError, code, err)
|
||||
}
|
||||
doc, err := dao.NewDocumentDAO().GetByID(t.Context(), db, "doc-1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if doc.Name == nil || *doc.Name != "old.docx" || doc.ParserID != "naive" || (doc.PipelineID != nil && *doc.PipelineID != "") || len(doc.ParserConfig) != 0 {
|
||||
t.Fatalf("rejected request changed document: %#v", doc)
|
||||
}
|
||||
file, err := dao.NewFileDAO().GetByID(t.Context(), db, "file-1")
|
||||
if err != nil || file.Name != "old.docx" {
|
||||
t.Fatalf("rejected request changed file: file=%#v err=%v", file, err)
|
||||
}
|
||||
if !reflect.DeepEqual(engine.records["doc-1"], map[string]any{"author": "original"}) {
|
||||
t.Fatalf("rejected request changed metadata: %#v", engine.records)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -35,8 +35,7 @@ func dedicatedDocumentParserID(filetype utility.FileType, filename string) strin
|
||||
}
|
||||
|
||||
// The presentation and email extensions all classify as FileTypeDOC, so the
|
||||
// name decides (same name-based rule as Python's get_parser and the
|
||||
// document parser_id guard in document_dataset_update.go).
|
||||
// name decides which dedicated pipeline owns them.
|
||||
name := strings.ToLower(filename)
|
||||
switch {
|
||||
case strings.HasSuffix(name, ".ppt"),
|
||||
@@ -189,7 +188,12 @@ func rebuildParserConfig(ctx context.Context, kb *entity.Knowledgebase, target,
|
||||
return nil, false
|
||||
}
|
||||
raw := map[string]interface{}(base)
|
||||
cleaned := pipelinepkg.BuildParserConfig(dslJSON, raw)
|
||||
cleaned, err := pipelinepkg.BuildParserConfig(dslJSON, raw)
|
||||
if err != nil {
|
||||
common.Warn("document: invalid target pipeline DSL",
|
||||
zap.String("parser_id", target), zap.Error(err))
|
||||
return nil, false
|
||||
}
|
||||
// The dataset-level metadata group is parser-independent; keep it so the
|
||||
// target pipeline's Extractor node inherits the dataset setting instead of
|
||||
// falling back to disabled.
|
||||
|
||||
@@ -350,7 +350,7 @@ func TestResolveDocumentParser_Priority(t *testing.T) {
|
||||
wantFamilyIn: "text&code",
|
||||
},
|
||||
{
|
||||
name: "an unmappable extension still reaches its owner pipeline",
|
||||
name: "pages reaches its owner pipeline",
|
||||
kbParser: "general",
|
||||
kbConfig: generalParserConfig(),
|
||||
filename: "deck.pages",
|
||||
|
||||
@@ -688,11 +688,20 @@ func ensureTestTenantOwner(t *testing.T, userID, tenantID string) {
|
||||
func insertTestCanvas(t *testing.T, canvasID, ownerID string) {
|
||||
t.Helper()
|
||||
tenantID := "tenant-1"
|
||||
dslJSON, err := service.LoadPipelineDSL(t.Context(), false, "general", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var dsl entity.JSONMap
|
||||
if err := json.Unmarshal(dslJSON, &dsl); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := dao.DB.Create(&entity.UserCanvas{
|
||||
ID: canvasID,
|
||||
UserID: ownerID,
|
||||
TenantID: &tenantID,
|
||||
Permission: string(entity.TenantPermissionMe),
|
||||
DSL: dsl,
|
||||
}).Error; err != nil {
|
||||
t.Fatalf("insert test canvas: %v", err)
|
||||
}
|
||||
@@ -3025,31 +3034,71 @@ func float64Ptr(value float64) *float64 { return &value }
|
||||
|
||||
func int64Ptr(value int64) *int64 { return &value }
|
||||
|
||||
func TestUpdateDatasetDocumentRejectsUnsupportedParserIDForVisualDoc(t *testing.T) {
|
||||
db := setupServiceTestDB(t)
|
||||
pushServiceDB(t, db)
|
||||
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
|
||||
insertNamedTestDoc(t, "doc-1", "kb-1", "image.png", 0, 0)
|
||||
if err := dao.DB.Model(&entity.Document{}).Where("id = ?", "doc-1").Update("type", "visual").Error; err != nil {
|
||||
t.Fatalf("update doc type: %v", err)
|
||||
}
|
||||
func TestUpdateDatasetDocumentSavesVisionModelAcrossFileTypes(t *testing.T) {
|
||||
for _, file := range []struct {
|
||||
name string
|
||||
docType string
|
||||
parserID string
|
||||
}{
|
||||
{"image.png", "visual", "general"},
|
||||
{"deck.pptx", "ppt", "presentation"},
|
||||
{"deck.pages", "doc", "presentation"},
|
||||
{"DECK.DPS", "ppt", "presentation"},
|
||||
{"deck.dpt", "ppt", "presentation"},
|
||||
{"report.docx", "doc", "manual"},
|
||||
} {
|
||||
for _, selection := range []string{"parser_id", "chunk_method"} {
|
||||
t.Run(file.name+"/"+selection, func(t *testing.T) {
|
||||
db := setupServiceTestDB(t)
|
||||
pushServiceDB(t, db)
|
||||
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
|
||||
insertNamedTestDoc(t, "doc-1", "kb-1", file.name, 0, 0)
|
||||
if err := db.Model(&entity.Document{}).Where("id = ?", "doc-1").Update("type", file.docType).Error; err != nil {
|
||||
t.Fatalf("update doc type: %v", err)
|
||||
}
|
||||
|
||||
parserID := "naive"
|
||||
parseType := 1
|
||||
svc := testDocumentService(t)
|
||||
ctx := documentTestContext(t.Context())
|
||||
_, code, err := svc.UpdateDatasetDocument(ctx, "user-1", "kb-1", "doc-1", &UpdateDatasetDocumentRequest{
|
||||
ParserID: &parserID,
|
||||
ParseType: &parseType,
|
||||
}, map[string]bool{"parser_id": true, "parse_type": true})
|
||||
if err == nil {
|
||||
t.Fatal("expected parser_id visual error")
|
||||
}
|
||||
if code != common.CodeDataError {
|
||||
t.Fatalf("code = %v, want %v", code, common.CodeDataError)
|
||||
}
|
||||
if err.Error() != "not supported yet" {
|
||||
t.Fatalf("err = %q", err.Error())
|
||||
req := &UpdateDatasetDocumentRequest{
|
||||
ParserConfig: map[string]any{
|
||||
"Parser:HipSignsRhyme": map[string]any{
|
||||
"enable_vision_enhancement": true,
|
||||
"vlm": map[string]any{"llm_id": "b17886e8ff3f4000bcf46f5b58c76803"},
|
||||
},
|
||||
},
|
||||
}
|
||||
present := map[string]bool{"parser_config": true, selection: true}
|
||||
wantParser := file.parserID
|
||||
if selection == "parser_id" {
|
||||
req.ParseType = new(1)
|
||||
req.ParserID = &file.parserID
|
||||
req.PipelineID = new("")
|
||||
present["parse_type"] = true
|
||||
present["pipeline_id"] = true
|
||||
} else {
|
||||
req.ChunkMethod = &file.parserID
|
||||
}
|
||||
resp, code, err := testDocumentService(t).UpdateDatasetDocument(documentTestContext(t.Context()), "user-1", "kb-1", "doc-1", req, present)
|
||||
if err != nil || code != common.CodeSuccess {
|
||||
t.Fatalf("save vision model: code=%v err=%v", code, err)
|
||||
}
|
||||
doc, err := dao.NewDocumentDAO().GetByID(t.Context(), db, "doc-1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if resp.ParserID != wantParser || doc.ParserID != wantParser {
|
||||
t.Fatalf("parser selection: response=%q stored=%q, want %q", resp.ParserID, doc.ParserID, wantParser)
|
||||
}
|
||||
for _, config := range []map[string]any{resp.ParserConfig, doc.ParserConfig} {
|
||||
parser, ok := config["Parser:HipSignsRhyme"].(map[string]any)
|
||||
if !ok || parser["enable_vision_enhancement"] != true {
|
||||
t.Fatalf("vision enhancement not saved: %#v", config)
|
||||
}
|
||||
vlm, ok := parser["vlm"].(map[string]any)
|
||||
if !ok || vlm["llm_id"] != "b17886e8ff3f4000bcf46f5b58c76803" {
|
||||
t.Fatalf("vision model not saved: %#v", parser)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3143,7 +3192,7 @@ func TestUpdateDatasetDocumentParserConfigPreservesParseResults(t *testing.T) {
|
||||
db := setupServiceTestDB(t)
|
||||
pushServiceDB(t, db)
|
||||
insertTestKB(t, "kb-1", "tenant-1", 1, 10, 5)
|
||||
insertNamedTestDoc(t, "doc-1", "kb-1", "doc.txt", 10, 5)
|
||||
insertNamedTestDoc(t, "doc-1", "kb-1", "doc.docx", 10, 5)
|
||||
insertTestCanvas(t, pipelineID, "user-1")
|
||||
insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.COMPLETED)
|
||||
if err := db.Model(&entity.Document{}).Where("id = ?", "doc-1").Update("process_duration", 42).Error; err != nil {
|
||||
@@ -4091,7 +4140,7 @@ func TestUpdateDatasetDocumentParseTypeBuiltin(t *testing.T) {
|
||||
db := setupServiceTestDB(t)
|
||||
pushServiceDB(t, db)
|
||||
insertTestKB(t, "kb-1", "tenant-1", 1, 10, 5)
|
||||
insertNamedTestDoc(t, "doc-1", "kb-1", "doc.txt", 10, 5)
|
||||
insertNamedTestDoc(t, "doc-1", "kb-1", "doc.docx", 10, 5)
|
||||
|
||||
// Seed a prior canvas-backed document so the builtin transition must
|
||||
// actively clear the persisted pipeline_id. Without this, a regression in
|
||||
@@ -4189,19 +4238,11 @@ func TestUpdateDatasetDocumentParseTypePipelineRejectsParserID(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateDatasetDocumentParentChildConfigSurvivesDSLFailure(t *testing.T) {
|
||||
func TestUpdateDatasetDocumentParentChildConfigReachesCanvasChunker(t *testing.T) {
|
||||
db := setupServiceTestDB(t)
|
||||
pushServiceDB(t, db)
|
||||
insertTestKB(t, "kb-1", "tenant-1", 1, 10, 5)
|
||||
insertNamedTestDoc(t, "doc-1", "kb-1", "doc.txt", 10, 5)
|
||||
if err := db.Model(&entity.Document{}).Where("id = ?", "doc-1").Update("parser_config", entity.JSONMap{
|
||||
"GeneralChunker:SixApplesFall": map[string]any{
|
||||
"children_delimiters": []any{},
|
||||
},
|
||||
}).Error; err != nil {
|
||||
t.Fatalf("seed document parser config: %v", err)
|
||||
}
|
||||
|
||||
parseType := 2
|
||||
pipelineID := "1234567890abcdef1234567890abcdef"
|
||||
insertTestCanvas(t, pipelineID, "user-1")
|
||||
|
||||
@@ -290,6 +290,8 @@ func TestRESTDocumentUpdatesPersistAndRejectAtomically(t *testing.T) {
|
||||
{"nested_meta", apiObject{"meta_fields": apiObject{"nested": apiObject{"x": 1}}}, 102},
|
||||
{"nested_meta_list", apiObject{"meta_fields": apiObject{"nested": []any{apiObject{"x": 1}}}}, 102},
|
||||
{"invalid_chunk_method", apiObject{"chunk_method": "unknown"}, 102}, {"empty_chunk_method", apiObject{"chunk_method": ""}, 102},
|
||||
{"removed_knowledge_graph", apiObject{"name": "must_not_persist.txt", "chunk_method": "knowledge_graph"}, 102},
|
||||
{"removed_tag", apiObject{"name": "must_not_persist.txt", "chunk_method": "tag"}, 102},
|
||||
{"parser_not_object", apiObject{"parser_config": []any{}}, 102},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
@@ -303,7 +305,29 @@ func TestRESTDocumentUpdatesPersistAndRejectAtomically(t *testing.T) {
|
||||
}
|
||||
})
|
||||
}
|
||||
for _, method := range []string{"naive", "manual", "qa", "table", "paper", "book", "laws", "presentation", "picture", "one", "knowledge_graph", "email", "tag"} {
|
||||
catalog := object(t, restOK(t, restCall(t, TestConfig, http.MethodGet, "/pipelines", nil, apiObject{"type": "builtin"}))["data"])
|
||||
pipelines := array(t, catalog["canvas"])
|
||||
if len(pipelines) == 0 {
|
||||
t.Fatal("builtin pipeline catalog is empty")
|
||||
}
|
||||
methods := []string{"naive"}
|
||||
for _, pipeline := range pipelines {
|
||||
methods = append(methods, object(t, pipeline)["id"].(string))
|
||||
}
|
||||
for _, method := range methods {
|
||||
// This document is text: registry membership alone does not mean a
|
||||
// template supports its file family.
|
||||
if !slices.Contains([]string{"naive", "general", "book", "laws", "one", "qa", "table"}, method) {
|
||||
before := getRESTDocument(t, id, docID)
|
||||
restCode(t, restCall(t, TestConfig, http.MethodPatch, path, apiObject{"chunk_method": method, "name": "must_not_persist.txt"}, nil), 200, 102)
|
||||
after := getRESTDocument(t, id, docID)
|
||||
for _, key := range []string{"name", "parser_id", "pipeline_id", "parser_config"} {
|
||||
if !reflect.DeepEqual(before[key], after[key]) {
|
||||
t.Fatalf("unsupported template %s changed %s: before=%v after=%v", method, key, before[key], after[key])
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
restOK(t, restCall(t, TestConfig, http.MethodPatch, path, apiObject{"chunk_method": method}, nil))
|
||||
persisted := getRESTDocument(t, id, docID)
|
||||
if persisted["chunk_method"] != method || persisted["parser_id"] != method {
|
||||
@@ -375,6 +399,7 @@ func TestRESTDocumentParserConfigurationAndMetadataConfig(t *testing.T) {
|
||||
}
|
||||
}
|
||||
// Flat Python-era options are discarded by the component-scoped Go contract.
|
||||
beforeFlatConfig := persisted["parser_config"]
|
||||
restOK(t, restCall(t, TestConfig, http.MethodPatch, path, apiObject{"parser_config": apiObject{"chunk_token_num": -1, "raptor": apiObject{"use_raptor": true}}}, nil))
|
||||
persisted = getRESTDocument(t, id, docID)
|
||||
pc := object(t, persisted["parser_config"])
|
||||
@@ -383,8 +408,8 @@ func TestRESTDocumentParserConfigurationAndMetadataConfig(t *testing.T) {
|
||||
t.Fatalf("flat parser key retained: %s", key)
|
||||
}
|
||||
}
|
||||
if object(t, pc[component])["chunk_token_size"] != object(t, config[component])["chunk_token_size"] {
|
||||
t.Fatalf("flat options did not restore component defaults: got=%v defaults=%v", pc[component], config[component])
|
||||
if !reflect.DeepEqual(pc, beforeFlatConfig) {
|
||||
t.Fatalf("ignored flat options changed existing config: got=%v want=%v", pc, beforeFlatConfig)
|
||||
}
|
||||
metadata := apiObject{"author": "alice", "tags": []any{"one", "two"}}
|
||||
metadataPath := path + "/metadata/config"
|
||||
|
||||
Reference in new issue
Block a user