fix(deepdoc): keep DLA-labeled body prose out of table captions (#21979)

internal/deepdoc/parser/pdf/table/merge_captions.go

@@ -4,6 +4,8 @@ import (
 	"html"
 	"sort"
 	"strings"
+	"unicode"
+	"unicode/utf8"
 
 	pdf "ragflow/internal/deepdoc/parser/pdf/type"
 )
@@ -47,8 +49,81 @@ func captionSep(text string) string {
 	return ""
 }
 
+// captionContinuationGap is the maximum vertical gap (PDF points) between a
+// caption line and its predecessor for the line to count as the same caption
+// block. Real continuation lines sit within a point or two; 4 tolerates
+// box-boundary jitter without bridging a paragraph break.
+const captionContinuationGap = 4.0
+
+// labelOnlyCaption reports whether a caption kind came only from the layout
+// label: the text carries no caption marker of its own.
+func labelOnlyCaption(s pdf.Section) bool {
+	return strings.Contains(s.LayoutType, "caption") &&
+		!captionMarkerPrefix(strings.TrimSpace(s.Text))
+}
+
+// startsLowerLetter reports whether text starts with a lowercase letter — the
+// shape of a sentence that continues the previous line or column.
+func startsLowerLetter(text string) bool {
+	r, _ := utf8.DecodeRuneInString(strings.TrimSpace(text))
+	return unicode.IsLower(r)
+}
+
+// continuesCaption reports whether s follows a caption MergeCaptions already
+// accepted, on the same page and vertically contiguous. Real multi-line
+// captions continue with lowercase lines ("ering, lateral gathering, ..."), so
+// a lowercase start alone does not make a labeled caption prose.
+func continuesCaption(sections []pdf.Section, accepted []bool, i int) bool {
+	if i == 0 || !accepted[i-1] {
+		return false
+	}
+	prev, cur := sections[i-1], sections[i]
+	if len(prev.Positions) == 0 || len(cur.Positions) == 0 {
+		return false
+	}
+	cp := cur.Positions[0]
+	shared, constrained := sharedPageNumbers(cp, prev.Positions)
+	if constrained && len(shared) == 0 {
+		return false
+	}
+	pp := prev.Positions[0]
+	return cp.Top <= pp.Bottom+captionContinuationGap
+}
+
+// unfinishedTextBefore reports whether the section immediately before i is body
+// text whose sentence is still open. A lowercase caption candidate at the top
+// of a column is a paragraph tail when the preceding column ends mid-sentence.
+// Requiring a text section (or an unannotated box, which defaults to text) also
+// keeps the check away from tables, figures and equations, whose serialized
+// text never ends in sentence punctuation.
+func unfinishedTextBefore(sections []pdf.Section, i int) bool {
+	if i == 0 {
+		return false
+	}
+	prev := sections[i-1]
+	if prev.LayoutType != pdf.LayoutTypeText && prev.LayoutType != "" {
+		return false
+	}
+	text := strings.TrimSpace(prev.Text)
+	if text == "" {
+		return false
+	}
+	last, _ := utf8.DecodeLastRuneInString(text)
+	return !strings.ContainsRune(".!?。！？:：;；", last)
+}
+
+// MergeCaptions folds caption sections into their nearest table or figure.
+// Table captions are injected as one <caption> element and move their boxes
+// into the target's Positions; figure captions keep the historical raw-text
+// append. Target sections are mutated in place. Sections must be in reading
+// order: the prose guard compares each candidate with the section immediately
+// before it.
 func MergeCaptions(sections []pdf.Section) []pdf.Section {
 	captions := make([]int, 0, 4)
+	// accepted marks the sections already classified as captions, so a
+	// lowercase continuation can tell a caption's next line from a paragraph
+	// tail that the layout detector mislabelled as a caption.
+	accepted := make([]bool, len(sections))
 	// Group caption texts by the target section index they attach to, so
 	// multiple caption boxes for the SAME table/figure collapse into a single
 	// <caption> element. HTML allows only one <caption> per <table>; injecting
@@ -73,6 +148,28 @@ func MergeCaptions(sections []pdf.Section) []pdf.Section {
 		if captionType == "" {
 			continue
 		}
+		// A DLA table-caption label over a lowercase line that continues an
+		// unfinished body sentence is a paragraph tail, not a caption (#21884).
+		// The detector labels body text directly above a table as "table
+		// caption" (the reported 27-page RAG paper, Table 1), while real
+		// multi-line captions also continue with lowercase lines ("ering,
+		// lateral gathering, ..."), so the label alone cannot decide. Keep the
+		// prose as body text: merging it would move the sentence into the
+		// table's <caption> and extend the table's highlight over an unrelated
+		// region.
+		//
+		// Uppercase-starting prose (a new sentence mislabeled the same way) is
+		// still merged: without part-of-speech information it is
+		// indistinguishable from a title-case caption.
+		if captionType == pdf.LayoutTypeTable &&
+			labelOnlyCaption(s) &&
+			startsLowerLetter(s.Text) &&
+			unfinishedTextBefore(sections, i) &&
+			!continuesCaption(sections, accepted, i) {
+			sections[i].LayoutType = pdf.LayoutTypeText
+			continue
+		}
+		accepted[i] = true
 		target := findNearestParent(s, sections, captionType)
 		if target >= 0 {
 			// Emit the caption inside the target table's HTML as a <caption>

internal/deepdoc/parser/pdf/table/merge_captions_labeled_prose_test.go

@@ -0,0 +1,168 @@
+package table
+
+import (
+	"strings"
+	"testing"
+
+	pdf "ragflow/internal/deepdoc/parser/pdf/type"
+)
+
+// TestMergeCaptions_LabeledTableCaptionProseContinuationKept locks #21884 on
+// the shape measured in the 27-page RAG paper (page 6, 1-based): DLA marks the
+// two-line tail of a body paragraph as "table caption" at confidence 0.868
+// ([305,73]-[526,98] pt), higher than the real caption's 0.733
+// ([367,204]-[462,218] pt), because it sits directly above Table 1. The
+// sentence continues a line that ends mid-sentence ("... reducing redundant"),
+// so the label must not move it into <caption>.
+func TestMergeCaptions_LabeledTableCaptionProseContinuationKept(t *testing.T) {
+	const (
+		unfinished = "structure or evidence semantics. Instead, it improves inference efficiency by reducing redundant"
+		prose1     = "generation calls among sibling leaves that share"
+		prose2     = "overlapping retrieved evidence."
+		caption    = "Table 1: Data statistics."
+	)
+	sections := []pdf.Section{
+		{Text: unfinished, LayoutType: pdf.LayoutTypeText,
+			Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 305, Right: 527, Top: 750, Bottom: 763}}},
+		{Text: prose1, LayoutType: pdf.DLALabelTableCaption,
+			Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 304, Right: 526, Top: 73, Bottom: 86}}},
+		{Text: prose2, LayoutType: pdf.DLALabelTableCaption,
+			Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 305, Right: 443, Top: 87, Bottom: 99}}},
+		{Text: "<table><tr><td>data</td></tr></table>", LayoutType: pdf.LayoutTypeTable,
+			Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 305, Right: 528, Top: 111, Bottom: 197}}},
+		{Text: caption, LayoutType: pdf.DLALabelTableCaption,
+			Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 367, Right: 462, Top: 204, Bottom: 218}}},
+	}
+
+	result := MergeCaptions(sections)
+	if len(result) != 4 {
+		t.Fatalf("expected four sections (body tail + two prose lines + table), got %d: %v", len(result), textsOf(result))
+	}
+	var table *pdf.Section
+	kept := map[string]string{}
+	for i := range result {
+		if result[i].LayoutType == pdf.LayoutTypeTable {
+			table = &result[i]
+			continue
+		}
+		kept[result[i].Text] = result[i].LayoutType
+	}
+	if table == nil {
+		t.Fatal("table section missing")
+	}
+	if !strings.Contains(table.Text, "<caption>"+caption+"</caption>") {
+		t.Errorf("table caption must contain only the real caption, got %q", table.Text)
+	}
+	if strings.Contains(table.Text, "generation calls") {
+		t.Errorf("prose injected into the table caption: %q", table.Text)
+	}
+	if len(table.Positions) != 2 {
+		t.Errorf("table geometry must be table + real caption (2 boxes), got %d: %+v", len(table.Positions), table.Positions)
+	}
+	if hasBox(table.Positions, 73, 86) || hasBox(table.Positions, 87, 99) {
+		t.Errorf("prose boxes leaked into the table's highlight geometry: %+v", table.Positions)
+	}
+	for _, text := range []string{prose1, prose2} {
+		lt, ok := kept[text]
+		if !ok {
+			t.Errorf("prose %q dropped; sections = %v", text, textsOf(result))
+			continue
+		}
+		if lt != pdf.LayoutTypeText {
+			t.Errorf("prose %q kept as %q, want %q", text, lt, pdf.LayoutTypeText)
+		}
+	}
+}
+
+// TestMergeCaptions_LabeledTableCaptionContinuationKept pins the other side: a
+// real caption's lowercase continuation line must still merge. Table 3 in the
+// reported paper continues "... vertical gath-" / "ering, lateral gathering,
+// ..." / "respectively." — the continuation lines start lowercase.
+func TestMergeCaptions_LabeledTableCaptionContinuationKept(t *testing.T) {
+	sections := []pdf.Section{
+		{Text: "<table><tr><td>data</td></tr></table>", LayoutType: pdf.LayoutTypeTable,
+			Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 304, Right: 526, Top: 69, Bottom: 206}}},
+		// First caption line: the text marker makes it a caption even without
+		// the label, and it ends with a comma (an unfinished-looking line).
+		{Text: "Table 3: Component-wise ablation of APT-RAG. AP,", LayoutType: pdf.LayoutTypeText,
+			Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 304, Right: 527, Top: 214, Bottom: 226}}},
+		{Text: "ering, lateral gathering, and evidence-guided clustering,", LayoutType: pdf.DLALabelTableCaption,
+			Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 304, Right: 527, Top: 227, Bottom: 239}}},
+		{Text: "respectively.", LayoutType: pdf.DLALabelTableCaption,
+			Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 306, Right: 355, Top: 240, Bottom: 251}}},
+	}
+
+	result := MergeCaptions(sections)
+	if len(result) != 1 {
+		t.Fatalf("expected one table section, got %d: %v", len(result), textsOf(result))
+	}
+	for _, want := range []string{"Table 3: Component-wise ablation", "ering, lateral gathering", "respectively."} {
+		if !strings.Contains(result[0].Text, want) {
+			t.Errorf("caption lost %q: %q", want, result[0].Text)
+		}
+	}
+	if len(result[0].Positions) != 4 {
+		t.Errorf("expected table + three caption boxes, got %d: %+v", len(result[0].Positions), result[0].Positions)
+	}
+}
+
+// TestMergeCaptions_LabeledTableCaptionProseWithoutTargetKept locks the
+// content-loss half: a labeled-prose line with no nearby table would be deleted
+// by the no-target branch before #21884.
+func TestMergeCaptions_LabeledTableCaptionProseWithoutTargetKept(t *testing.T) {
+	const (
+		leading = "the previous sentence introduces the next clause and"
+		prose   = "continues on the following line."
+	)
+	sections := []pdf.Section{
+		// Unannotated body box: an empty LayoutType is text at this stage.
+		{Text: leading, LayoutType: "",
+			Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 100, Right: 500, Top: 100, Bottom: 112}}},
+		{Text: prose, LayoutType: pdf.DLALabelTableCaption,
+			Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 100, Right: 500, Top: 113, Bottom: 125}}},
+	}
+
+	result := MergeCaptions(sections)
+	if len(result) != 2 {
+		t.Fatalf("labeled prose with no target must be kept as body text, got %d: %v", len(result), textsOf(result))
+	}
+	if result[1].Text != prose || result[1].LayoutType != pdf.LayoutTypeText {
+		t.Errorf("prose = (%q, %q), want (%q, %q)", result[1].Text, result[1].LayoutType, prose, pdf.LayoutTypeText)
+	}
+}
+
+// TestMergeCaptions_LabelOnlyCaptionBelowTableAttaches pins the guard's
+// precondition: the preceding section must be body text that ends mid-sentence.
+// A label-only caption directly below a table is not prose just because it
+// starts lowercase (the table's serialized text never ends in punctuation).
+func TestMergeCaptions_LabelOnlyCaptionBelowTableAttaches(t *testing.T) {
+	sections := []pdf.Section{
+		{Text: "<table><tr><td>data</td></tr></table>", LayoutType: pdf.LayoutTypeTable,
+			Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 100, Right: 500, Top: 100, Bottom: 200}}},
+		{Text: "quarterly totals", LayoutType: pdf.DLALabelTableCaption,
+			Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 150, Right: 450, Top: 205, Bottom: 217}}},
+	}
+
+	result := MergeCaptions(sections)
+	if len(result) != 1 {
+		t.Fatalf("expected one table section, got %d: %v", len(result), textsOf(result))
+	}
+	if !strings.Contains(result[0].Text, "<caption>quarterly totals</caption>") {
+		t.Errorf("label-only caption below the table was rejected: %q", result[0].Text)
+	}
+}
+
+// TestCaptionKind_LabeledLowercaseProseStaysTable documents why #21884 is fixed
+// in MergeCaptions rather than CaptionKind: whether a labeled lowercase line is
+// prose depends on the preceding section (caption continuation versus paragraph
+// tail), context CaptionKind does not receive. A lowercase-first rule inside
+// CaptionKind would split real multi-line captions such as Table 3's.
+func TestCaptionKind_LabeledLowercaseProseStaysTable(t *testing.T) {
+	s := pdf.Section{
+		Text:       "generation calls among sibling leaves that share",
+		LayoutType: pdf.DLALabelTableCaption,
+	}
+	if got := CaptionKind(s); got != pdf.LayoutTypeTable {
+		t.Errorf("CaptionKind(%q) = %q, want %q (context-free label precedence)", s.Text, got, pdf.LayoutTypeTable)
+	}
+}

internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.json

@@ -16,6 +16,20 @@
       "permanent": true,
       "reason": "Python's is_caption uses the start-anchored pattern (?i)^Table\\s+\\d+ alone, so a body sentence that BEGINS with a table cross-reference ('Table 1, we present ...', 'Table 9 reports ...') is classified as a caption and _extract_table_figure moves it into the nearest table's <caption>, deleting it as a standalone section. Go requires a caption boundary after the number (table_cells.go captionBoundary): punctuation separator, an uppercase/digit/CJK start, a parenthesised continuation or panel label, or end of line. A comma, possessive, or lowercase sentence continuation keeps the line as body text. Real captions without a colon ('Table 2 Performance comparison'), with a letter suffix ('Table 1a. Results'), or with a Roman/multi-digit panel label ('Table 1 (iv) ...', 'Table 1 (12) ...') still classify. Deliberate go_intentional divergence, locked by TestCaptionKind_EnglishTableCrossReferenceNotCaption, TestCaptionKind_EnglishTableCaptionBoundaries and TestMergeCaptions_EnglishTable1OpeningParagraphKept. Figure captions are unchanged (sentence-style figure captions remain captions). A layout-type hint still names the kind for captions without a marker ('Quarterly revenue breakdown'), but it cannot override the boundary rule for a line that starts with 'Table N' followed by prose; such a box stays body text (TestCaptionKind_LabeledCrossReferenceNotCaption, TestMergeCaptions_LabeledCrossReferenceStaysBodyText). IsCaptionBox keeps the pattern-only verdict because cell filling, no-merge marking, and caption-row stripping depend on it. KNOWN LIMITATION: the boundary keys on the first character, so a cross-reference that continues with a proper noun ('Table 3 Smith et al. report ...') classifies as a caption, indistinguishable from a caption title that starts with one ('Table 3 Smith-Kappa-Rosenberg parameters') without part-of-speech tagging; the sentence is then merged into the nearest table's <caption>, so its text survives in the output rather than being dropped. Pinned by TestCaptionKind_ProperNounContinuationIsCaption and TestMergeCaptions_ProperNounCrossReferenceLandsInCaption."
     },
+    {
+      "id": "dla-table-caption-prose-continuation-kept",
+      "tag": "go_intentional",
+      "kind": "classification",
+      "applies_to": [
+        "*"
+      ],
+      "fields": [
+        "MergeCaptions",
+        "LayoutType"
+      ],
+      "permanent": true,
+      "reason": "A DLA 'table caption' label over a lowercase line that continues an unfinished body sentence is a paragraph tail, not a caption. Measured on the 27-page RAG paper (issue #21884): the two-line tail of a body paragraph sits directly above Table 1, DLA labels it 'table caption' (one region at confidence 0.868, covering two text boxes under layoutNo 'table caption-0' — higher than the real caption's 0.733), and Python merges it into the table's <caption>, deleting it as a standalone section and extending the table's highlight over the paragraph. Go keeps the line as body text (LayoutType reset to 'text') when all of: the caption marker is absent (label-only), the text starts with a lowercase letter, the preceding section is body text whose sentence is still open (no .!?。！？:：;；), and the line does not continue an already-accepted caption on the same page. The last condition is what preserves real multi-line captions: DLA also labels 'ering, lateral gathering, ...' and 'respectively.' as table captions, and those lowercase continuation lines must still merge (Table 3/Table 6 in the same paper). Scoped to table captions; figure captions keep the historical label-only behavior. KNOWN LIMITATION: prose that starts a new sentence with an uppercase word is still merged; distinguishing it from a title-case caption needs part-of-speech information the parser does not have. Locked by TestMergeCaptions_LabeledTableCaptionProseContinuationKept, TestMergeCaptions_LabeledTableCaptionContinuationKept, TestMergeCaptions_LabeledTableCaptionProseWithoutTargetKept, TestMergeCaptions_LabelOnlyCaptionBelowTableAttaches and TestCaptionKind_LabeledLowercaseProseStaysTable."
+    },
     {
       "id": "table-cell-fill-filled-threshold-0.85",
       "tag": "go_intentional",