diff --git a/internal/ingestion/component/chunker/general_table_policy_test.go b/internal/ingestion/component/chunker/general_table_policy_test.go new file mode 100644 index 0000000000..42e0ded5f3 --- /dev/null +++ b/internal/ingestion/component/chunker/general_table_policy_test.go @@ -0,0 +1,67 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package chunker + +import ( + "context" + "strings" + "testing" +) + +// TestGeneralChunkerKeepsLargeTablesWhole pins the table policy shared by the +// PDF and DOCX paths: a table is an atomic media unit, so chunk_token_size +// never splits it. Text is still merged and split by the configured budget. +func TestGeneralChunkerKeepsLargeTablesWhole(t *testing.T) { + component, err := NewGeneralChunker(map[string]any{"chunk_token_size": 512}) + if err != nil { + t.Fatal(err) + } + + table := "" + + strings.Repeat("", 400) + + "
header
cell value
" + if tokens := tokenizeStr(table); tokens <= 512 { + t.Fatalf("test table has %d tokens, want more than the 512 budget", tokens) + } + + for _, fileType := range []string{"docx", "pdf"} { + t.Run(fileType, func(t *testing.T) { + out, err := component.Invoke(context.Background(), nil, map[string]any{ + "name": "large." + fileType, + "file_type": fileType, + "output_format": "json", + "json": []map[string]any{{ + "text": table, + "doc_type_kwd": "table", + }}, + }) + if err != nil { + t.Fatal(err) + } + chunks := outputChunks(t, out) + if len(chunks) != 1 { + t.Fatalf("chunks = %d, want 1: a table is not split by the token budget", len(chunks)) + } + if got := chunks[0]["doc_type_kwd"]; got != "table" { + t.Fatalf("doc_type_kwd = %v, want table", got) + } + if got, _ := chunks[0]["text"].(string); got != table { + t.Fatalf("table text changed: got %d bytes, want %d", len(got), len(table)) + } + }) + } +} diff --git a/internal/parser/parser/docx_ir.go b/internal/parser/parser/docx_ir.go index f7bf8113fd..f6781e9c28 100644 --- a/internal/parser/parser/docx_ir.go +++ b/internal/parser/parser/docx_ir.go @@ -99,11 +99,14 @@ type docxIRRun struct { } type docxIRRow struct { - Cells []docxIRCell `json:"cells"` + IsHeader bool `json:"is_header"` // repeated table header row + Cells []docxIRCell `json:"cells"` } type docxIRCell struct { Content []docxIRElement `json:"content"` // block-level content (typically paragraphs; may also hold list/table) + ColSpan int `json:"col_span"` + RowSpan int `json:"row_span"` } // joinDOCXIRRuns concatenates inline runs into plain text. Hard line @@ -180,16 +183,26 @@ func joinCellText(cell docxIRCell) string { } // officeIRTableData projects native Office rows directly onto the table contract. +// The row/column spans office_oxide recovers for merged cells are preserved so +// BuildTableData can rebuild the same grid; header flags mark repeated header +// rows, falling back to BuildTableData's default when the format does not carry +// them. func officeIRTableData(el docxIRElement, caption string) *entity.TableData { rows := make([][]tableutil.CellInput, 0, len(el.Rows)) + headerFlags := make([]bool, 0, len(el.Rows)) for _, row := range el.Rows { cells := make([]tableutil.CellInput, 0, len(row.Cells)) for _, cell := range row.Cells { - cells = append(cells, tableutil.CellInput{Text: joinCellText(cell)}) + cells = append(cells, tableutil.CellInput{ + Text: joinCellText(cell), + ColSpan: cell.ColSpan, + RowSpan: cell.RowSpan, + }) } rows = append(rows, cells) + headerFlags = append(headerFlags, row.IsHeader) } - return tableutil.BuildTableData(rows, nil, strings.TrimSpace(caption)) + return tableutil.BuildTableData(rows, headerFlags, strings.TrimSpace(caption)) } type docxTableImage struct { @@ -302,16 +315,6 @@ func docxElementText(el docxIRElement, cellSep string) string { } } -// docxIRTableCols returns the number of cells in a table's first row, used -// to decide whether two adjacent table elements are fragments of one logical -// table. It returns 0 for an empty table. -func docxIRTableCols(el docxIRElement) int { - if len(el.Rows) == 0 { - return 0 - } - return len(el.Rows[0].Cells) -} - // docxHeading is one entry of the heading outline stack. type docxHeading struct { level int diff --git a/internal/parser/parser/docx_ir_test.go b/internal/parser/parser/docx_ir_test.go index 1d1576b0a6..692edd47cd 100644 --- a/internal/parser/parser/docx_ir_test.go +++ b/internal/parser/parser/docx_ir_test.go @@ -54,12 +54,126 @@ func TestBuildDOCXJSONSections_EmptyTableSkipped(t *testing.T) { } } -// TestBuildDOCXJSONSections_DifferentColumnTablesStaySeparate pins that -// two adjacent tables of different column counts are NOT merged: they are -// genuinely separate tables, not fragments of one. +// TestBuildDOCXJSONSections_PreservesMergedHeaderSpans pins that office_oxide's +// row/column spans and repeated-header flags survive the IR -> TableData +// projection. The header is two rows: row 0 carries the colspan/rowspan +// geometry, row 1 fills the seven columns the vertical merges leave open, and +// the body has ten columns. +func TestBuildDOCXJSONSections_PreservesMergedHeaderSpans(t *testing.T) { + cell := func(text string, colSpan, rowSpan int) docxIRCell { + return docxIRCell{ + Content: []docxIRElement{para(text)}, + ColSpan: colSpan, + RowSpan: rowSpan, + } + } + row := func(header bool, cells ...docxIRCell) docxIRRow { + return docxIRRow{IsHeader: header, Cells: cells} + } + ir := docxIRDocument{Sections: []docxIRSection{{ + Elements: []docxIRElement{ + {Type: "heading", Level: 1, Content: rawJSON([]docxIRRun{{Type: "text", Text: "H"}})}, + {Type: "table", Rows: []docxIRRow{ + row(true, + cell("单位名称", 1, 2), + cell("集团公司采用", 3, 1), + cell("中国中铁采用", 2, 1), + cell("国资委采用", 2, 1), + cell("得分", 1, 2), + cell("排名", 1, 2), + ), + row(true, + cell("政务", 1, 1), + cell("政工", 1, 1), + cell("得分", 1, 1), + cell("数量", 1, 1), + cell("得分", 1, 1), + cell("数量", 1, 1), + cell("得分", 1, 1), + ), + row(false, + cell("西安公司", 1, 1), + cell("15（1）", 1, 1), + cell("12", 1, 1), + cell("265", 1, 1), + cell("4（3）", 1, 1), + cell("50", 1, 1), + cell("2（2）", 1, 1), + cell("40", 1, 1), + cell("355", 1, 1), + cell("1", 1, 1), + ), + }}, + }, + }}} + encoded, err := json.Marshal(ir) + if err != nil { + t.Fatal(err) + } + + got := buildDOCXJSONSections(string(encoded), nil) + var tables []map[string]any + for _, it := range got { + if dt, _ := it["doc_type_kwd"].(string); dt == "table" { + tables = append(tables, it) + } + } + if len(tables) != 1 { + t.Fatalf("expected 1 table item, got %d: %#v", len(tables), got) + } + td, ok := tables[0]["table"].(*entity.TableData) + if !ok || td == nil { + t.Fatalf("table has no TableData: %#v", tables[0]) + } + if td.HeaderRows != 2 { + t.Fatalf("HeaderRows = %d, want 2", td.HeaderRows) + } + if td.Caption != "Table Location: H" { + t.Errorf("caption = %q, want %q", td.Caption, "Table Location: H") + } + if len(td.Grid) != 3 { + t.Fatalf("grid rows = %d, want 3", len(td.Grid)) + } + for i, row := range td.Grid { + if len(row) != 10 { + t.Fatalf("grid row %d width = %d, want 10: %#v", i, len(row), row) + } + } + if got := td.Grid[0][0]; got.Text != "单位名称" || got.RowSpan != 2 { + t.Errorf("header cell 0 = %#v, want 单位名称 rowspan 2", got) + } + if got := td.Grid[0][1]; got.Text != "集团公司采用" || got.ColSpan != 3 { + t.Errorf("header cell 1 = %#v, want 集团公司采用 colspan 3", got) + } + if got := td.Grid[0][4]; got.Text != "中国中铁采用" || got.ColSpan != 2 { + t.Errorf("header cell at col 4 = %#v, want 中国中铁采用 colspan 2", got) + } + if got := td.Grid[0][6]; got.Text != "国资委采用" || got.ColSpan != 2 { + t.Errorf("header cell at col 6 = %#v, want 国资委采用 colspan 2", got) + } + if got := td.Grid[0][8]; got.Text != "得分" || got.RowSpan != 2 { + t.Errorf("header cell at col 8 = %#v, want 得分 rowspan 2", got) + } + if got := td.Grid[0][9]; got.Text != "排名" || got.RowSpan != 2 { + t.Errorf("header cell at col 9 = %#v, want 排名 rowspan 2", got) + } + for _, col := range []int{2, 3, 5, 7} { + if got := td.Grid[0][col]; !got.Covered { + t.Errorf("row 0 col %d = %#v, want covered by a colspan", col, got) + } + } + if got := td.Grid[1][0]; !got.Covered { + t.Errorf("row 1 col 0 = %#v, want covered by the rowspan", got) + } + if got := td.Grid[1][1]; got.Text != "政务" { + t.Errorf("row 1 col 1 = %#v, want 政务", got) + } +} + +// TestBuildDOCXJSONSections_DifferentColumnTablesStaySeparate pins that two +// adjacent tables of different column counts remain separate. func TestBuildDOCXJSONSections_DifferentColumnTablesStaySeparate(t *testing.T) { const cell = `{"content":[{"type":"paragraph","content":[{"type":"text","text":"x"}]}]}` - // a 1-row 3-col table followed by a 2-row 2-col table: different widths. irJSON := fmt.Sprintf(`{"sections":[{"title":"","elements":[ {"type":"table","rows":[{"cells":[%s,%s,%s]}]}, {"type":"table","rows":[{"cells":[%s,%s]},{"cells":[%s,%s]}]} @@ -76,15 +190,11 @@ func TestBuildDOCXJSONSections_DifferentColumnTablesStaySeparate(t *testing.T) { } } -// TestBuildDOCXJSONSections_AdjacentMultiRowTablesStaySeparate is the key -// regression guard for the 1-row merge gate: two genuinely separate multi-row -// tables of the SAME width, stacked with only a blank paragraph between them -// (so they are adjacent in the IR), must NOT be merged. Only a 1-row header -// fragment is folded into the following table. -func TestBuildDOCXJSONSections_AdjacentMultiRowTablesStaySeparate(t *testing.T) { +// TestBuildDOCXJSONSections_AdjacentTablesStaySeparate pins that adjacent +// tables remain separate items. The parser no longer merges table elements; +// office_oxide now emits one element per logical table. +func TestBuildDOCXJSONSections_AdjacentTablesStaySeparate(t *testing.T) { const cell = `{"content":[{"type":"paragraph","content":[{"type":"text","text":"x"}]}]}` - // table A: 2 rows x 3 cols; table B: 2 rows x 3 cols — same width, both - // multi-row, no element between. These are two distinct tables. irJSON := fmt.Sprintf(`{"sections":[{"title":"","elements":[ {"type":"table","rows":[{"cells":[%s,%s,%s]},{"cells":[%s,%s,%s]}]}, {"type":"table","rows":[{"cells":[%s,%s,%s]},{"cells":[%s,%s,%s]}]} @@ -97,7 +207,7 @@ func TestBuildDOCXJSONSections_AdjacentMultiRowTablesStaySeparate(t *testing.T) } } if len(tables) != 2 { - t.Fatalf("expected 2 separate multi-row tables, got %d: %#v", len(tables), got) + t.Fatalf("expected 2 separate tables, got %d: %#v", len(tables), got) } for i, tb := range tables { td, ok := tb["table"].(*entity.TableData) @@ -110,29 +220,6 @@ func TestBuildDOCXJSONSections_AdjacentMultiRowTablesStaySeparate(t *testing.T) } } -// TestBuildDOCXJSONSections_HeaderFragmentWithParagraphBetweenStaysSplit pins -// that a 1-row header fragment is only merged when the body is the VERY NEXT -// element. A paragraph (even blank) between them keeps them as distinct -// tables, so the gate never reaches across intervening content. -func TestBuildDOCXJSONSections_HeaderFragmentWithParagraphBetweenStaysSplit(t *testing.T) { - const cell = `{"content":[{"type":"paragraph","content":[{"type":"text","text":"x"}]}]}` - irJSON := fmt.Sprintf(`{"sections":[{"title":"","elements":[ - {"type":"table","rows":[{"cells":[%s,%s,%s]}]}, - {"type":"paragraph","content":[{"type":"text","text":"note"}]}, - {"type":"table","rows":[{"cells":[%s,%s,%s]},{"cells":[%s,%s,%s]}]} - ]}]}`, cell, cell, cell, cell, cell, cell, cell, cell, cell) - got := buildDOCXJSONSections(irJSON, nil) - var n int - for _, it := range got { - if dt, _ := it["doc_type_kwd"].(string); dt == "table" { - n++ - } - } - if n != 2 { - t.Fatalf("expected 2 separate tables (paragraph between), got %d: %#v", n, got) - } -} - func TestBuildDOCXJSONSections_PreservesInlineImageOrder(t *testing.T) { imageData := officeTestPNG(t, 1) irJSON := `{"sections":[{"elements":[ diff --git a/internal/parser/parser/table_structure_test.go b/internal/parser/parser/table_structure_test.go index bc95630196..77dd4e4150 100644 --- a/internal/parser/parser/table_structure_test.go +++ b/internal/parser/parser/table_structure_test.go @@ -252,13 +252,25 @@ func TestPPTXTableOnlySlideIsNotEmpty(t *testing.T) { } } -func TestOfficeTableRetainsDefaultHeaderCount(t *testing.T) { +func TestOfficeTableUsesRepeatedHeaderFlags(t *testing.T) { ir := `{"sections":[{"elements":[{"type":"table","rows":[{"is_header":true,"cells":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"H1"}]}]}]},{"is_header":true,"cells":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"H2"}]}]}]}]}]}]}` items := buildDOCXJSONSections(ir, nil) if len(items) != 1 { t.Fatalf("items=%#v", items) } td := items[0]["table"].(*entity.TableData) + if td.HeaderRows != 2 || len(td.Grid) != 2 { + t.Fatalf("table=%+v, want two rows and two repeated header rows", td) + } +} + +func TestOfficeTableDefaultsToOneHeaderWithoutFlags(t *testing.T) { + ir := `{"sections":[{"elements":[{"type":"table","rows":[{"cells":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"H1"}]}]}]},{"cells":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"H2"}]}]}]}]}]}]}` + items := buildDOCXJSONSections(ir, nil) + if len(items) != 1 { + t.Fatalf("items=%#v", items) + } + td := items[0]["table"].(*entity.TableData) if td.HeaderRows != 1 || len(td.Grid) != 2 { t.Fatalf("table=%+v, want two rows and one default header", td) }