Fix: restrict metadata field key (#19999)

internal/ingestion/component/extractor.go

@@ -166,6 +166,7 @@ Propose questions about a given piece of text content.
 
 	autoMetadataPrompt = `## Role: Metadata extraction expert.
 ## Rules:
+ - Fixed Key Set: Output ONLY the keys that appear in the Schema. Never add, rename, translate, pluralize or re-case a key, and never invent an extra key even if the Content clearly carries that information. Any key that is not in the Schema MUST be omitted.
  - Strict Evidence Only: Extract a value ONLY if it is explicitly mentioned in the Content.
  - Enum Filter: For any field with an 'enum' list, the list acts as a strict filter. If no element from the list (or its direct synonym) is found in the Content, you MUST NOT extract that field.
  - No Meta-Inference: Do not infer values based on the document's nature, format, or category. If the text does not literally state the information, treat it as missing.
@@ -1059,6 +1060,11 @@ func (c *ExtractorComponent) runEnableMetadata(ctx context.Context, db *gorm.DB,
 			// Non-JSON or empty response — nothing to extract, not an error.
 			return nil
 		}
+		parsed = filterMetadataToDeclaredKeys(parsed, c.Param.Metadata.Metadata)
+		if len(parsed) == 0 {
+			// Every key fell outside the declared field set — nothing to keep.
+			return nil
+		}
 		setMetadataLLMCache(ctx, in.cache, modelID, schemaStr, chunkID, parsed)
 	}
 	// Merge into the chunk metadata map, preserving existing keys.
@@ -1081,6 +1087,33 @@ func (c *ExtractorComponent) runEnableMetadata(ctx context.Context, db *gorm.DB,
 	return nil
 }
 
+// filterMetadataToDeclaredKeys drops every key that is not part of the
+// declared metadata field set.
+//
+// The rendered schema carries additionalProperties:false, but that is only a
+// hint to the model: a model that renames, pluralizes or invents a field still
+// returns a parseable object, and such a key would otherwise widen the
+// dataset's metadata schema through mergeChunkMetadata. The declared set is
+// therefore enforced here, in code.
+func filterMetadataToDeclaredKeys(parsed map[string]any, fields []common.MetadataFieldDef) map[string]any {
+	allowed := make(map[string]struct{}, len(fields))
+	for _, f := range fields {
+		allowed[f.Key] = struct{}{}
+	}
+	kept := make(map[string]any, len(parsed))
+	for k, v := range parsed {
+		if _, ok := allowed[k]; !ok {
+			common.Debug("extractor stage",
+				zap.String("component", componentNameExtractor),
+				zap.String("dropped_metadata_key", k),
+			)
+			continue
+		}
+		kept[k] = v
+	}
+	return kept
+}
+
 // extractorChunkText resolves the body an extraction is run against.
 //
 // Canonical text wins. Media-only chunks may use their explicit surrounding

internal/ingestion/component/extractor_test.go

@@ -603,6 +603,49 @@ func TestExtractorComponent_runEnableMetadata_MergesIntoChunkMetadata(t *testing
 	}
 }
 
+// TestExtractorComponent_runEnableMetadata_DropsUndeclaredKeys verifies a model
+// that invents or re-cases a field cannot widen the dataset's metadata schema:
+// only keys from the declared field set survive.
+func TestExtractorComponent_runEnableMetadata_DropsUndeclaredKeys(t *testing.T) {
+	withStubChatInvoker(t, stubResponse{Content: `{"category":"finance","authors":["李白"],"Category":"finance"}`})
+	c := newMetadataExtractor(common.MetadataFieldDef{Key: "category", Type: "string"})
+	ck := map[string]any{}
+	if err := c.runEnableMetadata(t.Context(), nil, extractorInputs{llmID: "m"}, ck, "chunk text"); err != nil {
+		t.Fatalf("runEnableMetadata: %v", err)
+	}
+	meta, ok := ck["metadata"].(map[string]any)
+	if !ok {
+		t.Fatalf("ck[metadata] missing or wrong type: %T", ck["metadata"])
+	}
+	if meta["category"] != "finance" {
+		t.Errorf("metadata[category] = %v, want finance", meta["category"])
+	}
+	if _, leaked := meta["authors"]; leaked {
+		t.Errorf("undeclared key authors leaked into metadata: %v", meta)
+	}
+	if _, leaked := meta["Category"]; leaked {
+		t.Errorf("re-cased key Category leaked into metadata: %v", meta)
+	}
+}
+
+// TestExtractorComponent_runEnableMetadata_AllKeysUndeclared verifies a response
+// whose every key falls outside the declared set leaves existing metadata intact.
+func TestExtractorComponent_runEnableMetadata_AllKeysUndeclared(t *testing.T) {
+	withStubChatInvoker(t, stubResponse{Content: `{"authors":["李白"]}`})
+	c := newMetadataExtractor(common.MetadataFieldDef{Key: "category", Type: "string"})
+	ck := map[string]any{"metadata": map[string]any{"preexisting": "keep"}}
+	if err := c.runEnableMetadata(t.Context(), nil, extractorInputs{llmID: "m"}, ck, "chunk text"); err != nil {
+		t.Fatalf("runEnableMetadata: %v", err)
+	}
+	meta, ok := ck["metadata"].(map[string]any)
+	if !ok {
+		t.Fatalf("ck[metadata] missing or wrong type: %T", ck["metadata"])
+	}
+	if len(meta) != 1 || meta["preexisting"] != "keep" {
+		t.Errorf("metadata = %v, want only preexisting=keep", meta)
+	}
+}
+
 // TestExtractorComponent_runEnableMetadata_StripsJSONFence verifies the
 // extraction path tolerates a fenced ```json response.
 func TestExtractorComponent_runEnableMetadata_StripsJSONFence(t *testing.T) {

rag/prompts/meta_data.md

@@ -1,14 +1,15 @@
 ## Role: Metadata extraction expert.
 ## Rules:
- - Strict Evidence Only: Extract a value ONLY if it is explicitly mentioned in the Content. 
+ - Fixed Key Set: Output ONLY the keys that appear in the Schema. Never add, rename, translate, pluralize or re-case a key, and never invent an extra key even if the Content clearly carries that information. Any key that is not in the Schema MUST be omitted.
+ - Strict Evidence Only: Extract a value ONLY if it is explicitly mentioned in the Content.
  - Enum Filter: For any field with an 'enum' list, the list acts as a strict filter. If no element from the list (or its direct synonym) is found in the Content, you MUST NOT extract that field.
  - No Meta-Inference: Do not infer values based on the document's nature, format, or category. If the text does not literally state the information, treat it as missing.
  - Zero-Hallucination: Never invent information or pick a "likely" value from the enum to fill a field.
- - Empty Result: If no matches are found for any field, or if the content is irrelevant, output ONLY {}. 
+ - Empty Result: If no matches are found for any field, or if the content is irrelevant, output ONLY {}.
  - Output: ONLY a valid JSON string. No Markdown, no notes.
 
 ## Schema for extraction:
 {{ schema }}
 
 ## Content to analyze:
-{{ content }}
+{{ content }}