fix(parser): preserve Office table cell spans (#22012)

internal/ingestion/component/chunker/general_table_policy_test.go

@@ -0,0 +1,67 @@
+//
+//  Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+//
+
+package chunker
+
+import (
+	"context"
+	"strings"
+	"testing"
+)
+
+// TestGeneralChunkerKeepsLargeTablesWhole pins the table policy shared by the
+// PDF and DOCX paths: a table is an atomic media unit, so chunk_token_size
+// never splits it. Text is still merged and split by the configured budget.
+func TestGeneralChunkerKeepsLargeTablesWhole(t *testing.T) {
+	component, err := NewGeneralChunker(map[string]any{"chunk_token_size": 512})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	table := "<table><tr><th>header</th></tr>" +
+		strings.Repeat("<tr><td>cell value</td></tr>", 400) +
+		"</table>"
+	if tokens := tokenizeStr(table); tokens <= 512 {
+		t.Fatalf("test table has %d tokens, want more than the 512 budget", tokens)
+	}
+
+	for _, fileType := range []string{"docx", "pdf"} {
+		t.Run(fileType, func(t *testing.T) {
+			out, err := component.Invoke(context.Background(), nil, map[string]any{
+				"name":          "large." + fileType,
+				"file_type":     fileType,
+				"output_format": "json",
+				"json": []map[string]any{{
+					"text":         table,
+					"doc_type_kwd": "table",
+				}},
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+			chunks := outputChunks(t, out)
+			if len(chunks) != 1 {
+				t.Fatalf("chunks = %d, want 1: a table is not split by the token budget", len(chunks))
+			}
+			if got := chunks[0]["doc_type_kwd"]; got != "table" {
+				t.Fatalf("doc_type_kwd = %v, want table", got)
+			}
+			if got, _ := chunks[0]["text"].(string); got != table {
+				t.Fatalf("table text changed: got %d bytes, want %d", len(got), len(table))
+			}
+		})
+	}
+}

internal/parser/parser/docx_ir.go

@@ -99,11 +99,14 @@ type docxIRRun struct {
 }
 
 type docxIRRow struct {
-	Cells []docxIRCell `json:"cells"`
+	IsHeader bool         `json:"is_header"` // repeated table header row
+	Cells    []docxIRCell `json:"cells"`
 }
 
 type docxIRCell struct {
 	Content []docxIRElement `json:"content"` // block-level content (typically paragraphs; may also hold list/table)
+	ColSpan int             `json:"col_span"`
+	RowSpan int             `json:"row_span"`
 }
 
 // joinDOCXIRRuns concatenates inline runs into plain text. Hard line
@@ -180,16 +183,26 @@ func joinCellText(cell docxIRCell) string {
 }
 
 // officeIRTableData projects native Office rows directly onto the table contract.
+// The row/column spans office_oxide recovers for merged cells are preserved so
+// BuildTableData can rebuild the same grid; header flags mark repeated header
+// rows, falling back to BuildTableData's default when the format does not carry
+// them.
 func officeIRTableData(el docxIRElement, caption string) *entity.TableData {
 	rows := make([][]tableutil.CellInput, 0, len(el.Rows))
+	headerFlags := make([]bool, 0, len(el.Rows))
 	for _, row := range el.Rows {
 		cells := make([]tableutil.CellInput, 0, len(row.Cells))
 		for _, cell := range row.Cells {
-			cells = append(cells, tableutil.CellInput{Text: joinCellText(cell)})
+			cells = append(cells, tableutil.CellInput{
+				Text:    joinCellText(cell),
+				ColSpan: cell.ColSpan,
+				RowSpan: cell.RowSpan,
+			})
 		}
 		rows = append(rows, cells)
+		headerFlags = append(headerFlags, row.IsHeader)
 	}
-	return tableutil.BuildTableData(rows, nil, strings.TrimSpace(caption))
+	return tableutil.BuildTableData(rows, headerFlags, strings.TrimSpace(caption))
 }
 
 type docxTableImage struct {
@@ -302,16 +315,6 @@ func docxElementText(el docxIRElement, cellSep string) string {
 	}
 }
 
-// docxIRTableCols returns the number of cells in a table's first row, used
-// to decide whether two adjacent table elements are fragments of one logical
-// table. It returns 0 for an empty table.
-func docxIRTableCols(el docxIRElement) int {
-	if len(el.Rows) == 0 {
-		return 0
-	}
-	return len(el.Rows[0].Cells)
-}
-
 // docxHeading is one entry of the heading outline stack.
 type docxHeading struct {
 	level int

internal/parser/parser/docx_ir_test.go

@@ -54,12 +54,126 @@ func TestBuildDOCXJSONSections_EmptyTableSkipped(t *testing.T) {
 	}
 }
 
-// TestBuildDOCXJSONSections_DifferentColumnTablesStaySeparate pins that
-// two adjacent tables of different column counts are NOT merged: they are
-// genuinely separate tables, not fragments of one.
+// TestBuildDOCXJSONSections_PreservesMergedHeaderSpans pins that office_oxide's
+// row/column spans and repeated-header flags survive the IR -> TableData
+// projection. The header is two rows: row 0 carries the colspan/rowspan
+// geometry, row 1 fills the seven columns the vertical merges leave open, and
+// the body has ten columns.
+func TestBuildDOCXJSONSections_PreservesMergedHeaderSpans(t *testing.T) {
+	cell := func(text string, colSpan, rowSpan int) docxIRCell {
+		return docxIRCell{
+			Content: []docxIRElement{para(text)},
+			ColSpan: colSpan,
+			RowSpan: rowSpan,
+		}
+	}
+	row := func(header bool, cells ...docxIRCell) docxIRRow {
+		return docxIRRow{IsHeader: header, Cells: cells}
+	}
+	ir := docxIRDocument{Sections: []docxIRSection{{
+		Elements: []docxIRElement{
+			{Type: "heading", Level: 1, Content: rawJSON([]docxIRRun{{Type: "text", Text: "H"}})},
+			{Type: "table", Rows: []docxIRRow{
+				row(true,
+					cell("单位名称", 1, 2),
+					cell("集团公司采用", 3, 1),
+					cell("中国中铁采用", 2, 1),
+					cell("国资委采用", 2, 1),
+					cell("得分", 1, 2),
+					cell("排名", 1, 2),
+				),
+				row(true,
+					cell("政务", 1, 1),
+					cell("政工", 1, 1),
+					cell("得分", 1, 1),
+					cell("数量", 1, 1),
+					cell("得分", 1, 1),
+					cell("数量", 1, 1),
+					cell("得分", 1, 1),
+				),
+				row(false,
+					cell("西安公司", 1, 1),
+					cell("15（1）", 1, 1),
+					cell("12", 1, 1),
+					cell("265", 1, 1),
+					cell("4（3）", 1, 1),
+					cell("50", 1, 1),
+					cell("2（2）", 1, 1),
+					cell("40", 1, 1),
+					cell("355", 1, 1),
+					cell("1", 1, 1),
+				),
+			}},
+		},
+	}}}
+	encoded, err := json.Marshal(ir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	got := buildDOCXJSONSections(string(encoded), nil)
+	var tables []map[string]any
+	for _, it := range got {
+		if dt, _ := it["doc_type_kwd"].(string); dt == "table" {
+			tables = append(tables, it)
+		}
+	}
+	if len(tables) != 1 {
+		t.Fatalf("expected 1 table item, got %d: %#v", len(tables), got)
+	}
+	td, ok := tables[0]["table"].(*entity.TableData)
+	if !ok || td == nil {
+		t.Fatalf("table has no TableData: %#v", tables[0])
+	}
+	if td.HeaderRows != 2 {
+		t.Fatalf("HeaderRows = %d, want 2", td.HeaderRows)
+	}
+	if td.Caption != "Table Location: H" {
+		t.Errorf("caption = %q, want %q", td.Caption, "Table Location: H")
+	}
+	if len(td.Grid) != 3 {
+		t.Fatalf("grid rows = %d, want 3", len(td.Grid))
+	}
+	for i, row := range td.Grid {
+		if len(row) != 10 {
+			t.Fatalf("grid row %d width = %d, want 10: %#v", i, len(row), row)
+		}
+	}
+	if got := td.Grid[0][0]; got.Text != "单位名称" || got.RowSpan != 2 {
+		t.Errorf("header cell 0 = %#v, want 单位名称 rowspan 2", got)
+	}
+	if got := td.Grid[0][1]; got.Text != "集团公司采用" || got.ColSpan != 3 {
+		t.Errorf("header cell 1 = %#v, want 集团公司采用 colspan 3", got)
+	}
+	if got := td.Grid[0][4]; got.Text != "中国中铁采用" || got.ColSpan != 2 {
+		t.Errorf("header cell at col 4 = %#v, want 中国中铁采用 colspan 2", got)
+	}
+	if got := td.Grid[0][6]; got.Text != "国资委采用" || got.ColSpan != 2 {
+		t.Errorf("header cell at col 6 = %#v, want 国资委采用 colspan 2", got)
+	}
+	if got := td.Grid[0][8]; got.Text != "得分" || got.RowSpan != 2 {
+		t.Errorf("header cell at col 8 = %#v, want 得分 rowspan 2", got)
+	}
+	if got := td.Grid[0][9]; got.Text != "排名" || got.RowSpan != 2 {
+		t.Errorf("header cell at col 9 = %#v, want 排名 rowspan 2", got)
+	}
+	for _, col := range []int{2, 3, 5, 7} {
+		if got := td.Grid[0][col]; !got.Covered {
+			t.Errorf("row 0 col %d = %#v, want covered by a colspan", col, got)
+		}
+	}
+	if got := td.Grid[1][0]; !got.Covered {
+		t.Errorf("row 1 col 0 = %#v, want covered by the rowspan", got)
+	}
+	if got := td.Grid[1][1]; got.Text != "政务" {
+		t.Errorf("row 1 col 1 = %#v, want 政务", got)
+	}
+}
+
+// TestBuildDOCXJSONSections_DifferentColumnTablesStaySeparate pins that two
+// adjacent tables of different column counts remain separate.
 func TestBuildDOCXJSONSections_DifferentColumnTablesStaySeparate(t *testing.T) {
 	const cell = `{"content":[{"type":"paragraph","content":[{"type":"text","text":"x"}]}]}`
-	// a 1-row 3-col table followed by a 2-row 2-col table: different widths.
 	irJSON := fmt.Sprintf(`{"sections":[{"title":"","elements":[
 		{"type":"table","rows":[{"cells":[%s,%s,%s]}]},
 		{"type":"table","rows":[{"cells":[%s,%s]},{"cells":[%s,%s]}]}
@@ -76,15 +190,11 @@ func TestBuildDOCXJSONSections_DifferentColumnTablesStaySeparate(t *testing.T) {
 	}
 }
 
-// TestBuildDOCXJSONSections_AdjacentMultiRowTablesStaySeparate is the key
-// regression guard for the 1-row merge gate: two genuinely separate multi-row
-// tables of the SAME width, stacked with only a blank paragraph between them
-// (so they are adjacent in the IR), must NOT be merged. Only a 1-row header
-// fragment is folded into the following table.
-func TestBuildDOCXJSONSections_AdjacentMultiRowTablesStaySeparate(t *testing.T) {
+// TestBuildDOCXJSONSections_AdjacentTablesStaySeparate pins that adjacent
+// tables remain separate items. The parser no longer merges table elements;
+// office_oxide now emits one element per logical table.
+func TestBuildDOCXJSONSections_AdjacentTablesStaySeparate(t *testing.T) {
 	const cell = `{"content":[{"type":"paragraph","content":[{"type":"text","text":"x"}]}]}`
-	// table A: 2 rows x 3 cols; table B: 2 rows x 3 cols — same width, both
-	// multi-row, no element between. These are two distinct tables.
 	irJSON := fmt.Sprintf(`{"sections":[{"title":"","elements":[
 		{"type":"table","rows":[{"cells":[%s,%s,%s]},{"cells":[%s,%s,%s]}]},
 		{"type":"table","rows":[{"cells":[%s,%s,%s]},{"cells":[%s,%s,%s]}]}
@@ -97,7 +207,7 @@ func TestBuildDOCXJSONSections_AdjacentMultiRowTablesStaySeparate(t *testing.T)
 		}
 	}
 	if len(tables) != 2 {
-		t.Fatalf("expected 2 separate multi-row tables, got %d: %#v", len(tables), got)
+		t.Fatalf("expected 2 separate tables, got %d: %#v", len(tables), got)
 	}
 	for i, tb := range tables {
 		td, ok := tb["table"].(*entity.TableData)
@@ -110,29 +220,6 @@ func TestBuildDOCXJSONSections_AdjacentMultiRowTablesStaySeparate(t *testing.T)
 	}
 }
 
-// TestBuildDOCXJSONSections_HeaderFragmentWithParagraphBetweenStaysSplit pins
-// that a 1-row header fragment is only merged when the body is the VERY NEXT
-// element. A paragraph (even blank) between them keeps them as distinct
-// tables, so the gate never reaches across intervening content.
-func TestBuildDOCXJSONSections_HeaderFragmentWithParagraphBetweenStaysSplit(t *testing.T) {
-	const cell = `{"content":[{"type":"paragraph","content":[{"type":"text","text":"x"}]}]}`
-	irJSON := fmt.Sprintf(`{"sections":[{"title":"","elements":[
-		{"type":"table","rows":[{"cells":[%s,%s,%s]}]},
-		{"type":"paragraph","content":[{"type":"text","text":"note"}]},
-		{"type":"table","rows":[{"cells":[%s,%s,%s]},{"cells":[%s,%s,%s]}]}
-	]}]}`, cell, cell, cell, cell, cell, cell, cell, cell, cell)
-	got := buildDOCXJSONSections(irJSON, nil)
-	var n int
-	for _, it := range got {
-		if dt, _ := it["doc_type_kwd"].(string); dt == "table" {
-			n++
-		}
-	}
-	if n != 2 {
-		t.Fatalf("expected 2 separate tables (paragraph between), got %d: %#v", n, got)
-	}
-}
-
 func TestBuildDOCXJSONSections_PreservesInlineImageOrder(t *testing.T) {
 	imageData := officeTestPNG(t, 1)
 	irJSON := `{"sections":[{"elements":[

internal/parser/parser/table_structure_test.go

@@ -252,13 +252,25 @@ func TestPPTXTableOnlySlideIsNotEmpty(t *testing.T) {
 	}
 }
 
-func TestOfficeTableRetainsDefaultHeaderCount(t *testing.T) {
+func TestOfficeTableUsesRepeatedHeaderFlags(t *testing.T) {
 	ir := `{"sections":[{"elements":[{"type":"table","rows":[{"is_header":true,"cells":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"H1"}]}]}]},{"is_header":true,"cells":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"H2"}]}]}]}]}]}]}`
 	items := buildDOCXJSONSections(ir, nil)
 	if len(items) != 1 {
 		t.Fatalf("items=%#v", items)
 	}
 	td := items[0]["table"].(*entity.TableData)
+	if td.HeaderRows != 2 || len(td.Grid) != 2 {
+		t.Fatalf("table=%+v, want two rows and two repeated header rows", td)
+	}
+}
+
+func TestOfficeTableDefaultsToOneHeaderWithoutFlags(t *testing.T) {
+	ir := `{"sections":[{"elements":[{"type":"table","rows":[{"cells":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"H1"}]}]}]},{"cells":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"H2"}]}]}]}]}]}]}`
+	items := buildDOCXJSONSections(ir, nil)
+	if len(items) != 1 {
+		t.Fatalf("items=%#v", items)
+	}
+	td := items[0]["table"].(*entity.TableData)
 	if td.HeaderRows != 1 || len(td.Grid) != 2 {
 		t.Fatalf("table=%+v, want two rows and one default header", td)
 	}