From 6e2214aaa8cde4d5128b101201545ef0b7eb6143 Mon Sep 17 00:00:00 2001 From: jay77721 <164177721+jay77721@users.noreply.github.com> Date: Tue, 27 Oct 2026 16:54:40 +0800 Subject: [PATCH] fix(parser): preserve table structure across document formats (#21504) --- internal/ingestion/component/chunker/page.go | 7 +- .../component/chunker/page_pptx_cgo_test.go | 73 ++ .../ingestion/component/chunker/page_test.go | 59 +- .../component/chunker/parser_table_test.go | 115 ++++ internal/ingestion/component/chunker/table.go | 33 +- .../testdata/tables/retrieval_md_20_ret.md | 92 +++ internal/ingestion/component/chunker/token.go | 6 +- internal/parser/parser/charset_test.go | 2 +- internal/parser/parser/docx_caption_test.go | 3 +- internal/parser/parser/docx_ir.go | 234 ++++--- internal/parser/parser/docx_ir_test.go | 49 +- internal/parser/parser/docx_parser_test.go | 51 -- internal/parser/parser/epub_parser.go | 171 ++--- internal/parser/parser/epub_parser_test.go | 71 +- internal/parser/parser/html_media_test.go | 20 + internal/parser/parser/html_parser.go | 58 +- internal/parser/parser/markdown_parser.go | 640 ++++++++++-------- .../parser/parser/markdown_parser_test.go | 30 +- internal/parser/parser/pdf_parser_common.go | 2 +- internal/parser/parser/pdf_parser_somark.go | 62 +- .../parser/parser/pdf_parser_somark_test.go | 2 +- .../parser/pdf_parser_table_contract_test.go | 8 +- .../parser/pdf_table_no_html_text_test.go | 3 +- internal/parser/parser/pptx_ir.go | 119 ++-- internal/parser/parser/pptx_ir_test.go | 19 +- .../parser/parser/table_regression_test.go | 349 ++++++++++ .../parser/parser/table_structure_test.go | 311 +++++++++ .../service/document/document_artifact.go | 16 +- .../document/document_artifact_test.go | 46 ++ 29 files changed, 1931 insertions(+), 720 deletions(-) create mode 100644 internal/ingestion/component/chunker/page_pptx_cgo_test.go create mode 100644 internal/ingestion/component/chunker/parser_table_test.go create mode 100644 internal/ingestion/component/chunker/testdata/tables/retrieval_md_20_ret.md create mode 100644 internal/parser/parser/table_regression_test.go create mode 100644 internal/parser/parser/table_structure_test.go create mode 100644 internal/service/document/document_artifact_test.go diff --git a/internal/ingestion/component/chunker/page.go b/internal/ingestion/component/chunker/page.go index 10ae52dac1..f0428c38f4 100644 --- a/internal/ingestion/component/chunker/page.go +++ b/internal/ingestion/component/chunker/page.go @@ -21,11 +21,8 @@ // // Unlike TokenChunker (which merges slides into a single chunk) or // OneChunker (which collapses many slides into one), PageChunker -// keeps each slide as the unit of chunking. The upstream parser produces -// one record per slide with text and slide_number; this chunker passes -// each through unchanged. Note that the PPTX/PPT path does not emit image -// or position information (unlike the PDF path), so slide chunks carry no -// image and no bbox-based preview positioning. +// preserves the upstream slide records, including structured tables and +// images. Every record retains its slide_number for source-page identity. package chunker import ( diff --git a/internal/ingestion/component/chunker/page_pptx_cgo_test.go b/internal/ingestion/component/chunker/page_pptx_cgo_test.go new file mode 100644 index 0000000000..32925a965c --- /dev/null +++ b/internal/ingestion/component/chunker/page_pptx_cgo_test.go @@ -0,0 +1,73 @@ +//go:build cgo + +package chunker + +import ( + "archive/zip" + "bytes" + "io" + "strings" + "testing" + + officeOxide "github.com/yfedoseev/office_oxide/go" + "ragflow/internal/parser/parser" +) + +func TestPageChunkerPPTXTableEndToEnd(t *testing.T) { + writer := officeOxide.NewPptxWriter() + defer writer.Close() + slide := writer.AddSlide() + writer.SetSlideTitle(slide, "Revenue 2026") + writer.AddSlideText(slide, "Revenue rose") + original, err := writer.ToBytes() + if err != nil { + t.Fatal(err) + } + zr, err := zip.NewReader(bytes.NewReader(original), int64(len(original))) + if err != nil { + t.Fatal(err) + } + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + for _, file := range zr.File { + r, err := file.Open() + if err != nil { + t.Fatal(err) + } + data, err := io.ReadAll(r) + r.Close() + if err != nil { + t.Fatal(err) + } + if file.Name == "ppt/slides/slide1.xml" { + table := `Q1100` + source := string(data) + if !strings.Contains(source, "") { + t.Fatalf("slide does not contain shapes: %s", source) + } + data = []byte(strings.Replace(source, "", ""+table, 1)) + } + w, err := zw.Create(file.Name) + if err != nil { + t.Fatal(err) + } + if _, err := w.Write(data); err != nil { + t.Fatal(err) + } + } + if err := zw.Close(); err != nil { + t.Fatal(err) + } + parsed := parser.NewPPTXParser().ParseWithResult(t.Context(), "deck.pptx", buf.Bytes()) + if parsed.Err != nil { + t.Fatal(parsed.Err) + } + chunks := slideChunksOf(t, map[string]any{"name": "deck.pptx", "file_type": "pptx", "output_format": "json", "json": parsed.JSON}) + if len(chunks) != 3 { + t.Fatalf("items=%#v chunks=%#v", parser.ItemsToMaps(parsed.JSON), chunks) + } + text, _ := chunks[1]["text"].(string) + if chunks[0]["text"] != "Revenue 2026" || chunks[1]["doc_type_kwd"] != "table" || chunks[1]["table"] == nil || !strings.Contains(text, "Q1100") || chunks[2]["text"] != "Revenue rose" { + t.Fatalf("PPTX slide contents lost: items=%#v chunks=%#v", parser.ItemsToMaps(parsed.JSON), chunks) + } +} diff --git a/internal/ingestion/component/chunker/page_test.go b/internal/ingestion/component/chunker/page_test.go index 447bf9a208..e93f3df296 100644 --- a/internal/ingestion/component/chunker/page_test.go +++ b/internal/ingestion/component/chunker/page_test.go @@ -18,8 +18,12 @@ package chunker import ( "context" - "ragflow/internal/ingestion/component/schema" + "fmt" "testing" + + "ragflow/internal/entity" + "ragflow/internal/ingestion/component/schema" + "ragflow/internal/parser/parser" ) // slideChunksOf drives PageChunker.Invoke and returns the emitted maps. @@ -118,3 +122,56 @@ func TestPageChunker_NilInput(t *testing.T) { t.Fatalf("expected 0 chunks for nil input, got %d", len(chunks)) } } + +func TestPageChunkerPreservesSlideTableItems(t *testing.T) { + table := map[string]any{"grid": []any{[]any{map[string]any{"text": "cell"}}}, "header_rows": 1} + chunks := slideChunksOf(t, map[string]any{"name": "deck.pptx", "output_format": "json", "json": []map[string]any{ + {"text": "Before", "doc_type_kwd": "text", "slide_number": 1}, + {"doc_type_kwd": "table", "slide_number": 1, "source_table_id": "table-1", "table": table}, + {"doc_type_kwd": "image", "slide_number": 1, "parent_table_id": "table-1", "image": "one"}, + {"doc_type_kwd": "table", "slide_number": 1, "source_table_id": "table-2", "table": table}, + {"doc_type_kwd": "image", "slide_number": 1, "parent_table_id": "table-2", "image": "two"}, + {"text": "After", "doc_type_kwd": "text", "slide_number": 1}, + }}) + if len(chunks) != 6 { + t.Fatalf("expected ordered text, tables and images: %#v", chunks) + } + if chunks[0]["text"] != "Before" || chunks[5]["text"] != "After" { + t.Fatalf("text order lost: %#v", chunks) + } + for _, index := range []int{1, 3} { + if chunks[index]["doc_type_kwd"] != "table" || chunks[index]["table"] == nil || chunks[index+1]["parent_table_id"] != chunks[index]["source_table_id"] { + t.Fatalf("table structure or image association lost: %#v", chunks) + } + } + for _, chunk := range chunks { + if got := fmt.Sprint(chunk["slide_number"]); got != "1" { + t.Fatalf("slide_number lost: %#v", chunk) + } + } +} + +// TestPageChunkerPreservesTypedSlideTableItems covers the in-memory +// []parser.DocItem leg, which bypasses the JSON-map decode path. +func TestPageChunkerPreservesTypedSlideTableItems(t *testing.T) { + table := &entity.TableData{ + Grid: [][]entity.TableCell{{{Text: "cell"}}}, + HeaderRows: 1, + } + maps := []map[string]any{ + {"text": "Before", "doc_type_kwd": "text", "slide_number": 1}, + {"doc_type_kwd": "table", "slide_number": 1, "source_table_id": "table-1", "table": table}, + {"doc_type_kwd": "image", "slide_number": 1, "parent_table_id": "table-1", "image": "one"}, + } + chunks := slideChunksOf(t, map[string]any{ + "name": "deck.pptx", "output_format": "json", "json": parser.MapsToItems(maps), + }) + if len(chunks) != len(maps) { + t.Fatalf("chunks = %d, want %d: %#v", len(chunks), len(maps), chunks) + } + for _, chunk := range chunks { + if got := fmt.Sprint(chunk["slide_number"]); got != "1" { + t.Fatalf("slide_number lost through typed DocItem input: %#v", chunk) + } + } +} diff --git a/internal/ingestion/component/chunker/parser_table_test.go b/internal/ingestion/component/chunker/parser_table_test.go new file mode 100644 index 0000000000..b640ec13a3 --- /dev/null +++ b/internal/ingestion/component/chunker/parser_table_test.go @@ -0,0 +1,115 @@ +package chunker + +import ( + "archive/zip" + "bytes" + "encoding/json" + "os" + "strings" + "testing" + + "ragflow/internal/parser/parser" +) + +func TestGeneralChunkerPreservesRetrievalHandbookTables(t *testing.T) { + data, err := os.ReadFile("testdata/tables/retrieval_md_20_ret.md") + if err != nil { + t.Fatal(err) + } + p, err := parser.NewMarkdownParser(parser.GoMarkdown) + if err != nil { + t.Fatal(err) + } + parsed := p.ParseWithResult(t.Context(), "retrieval_md_20_ret.md", data) + if parsed.Err != nil { + t.Fatal(parsed.Err) + } + wire, err := json.Marshal(parsed.JSON) + if err != nil { + t.Fatal(err) + } + var items []map[string]any + if err := json.Unmarshal(wire, &items); err != nil { + t.Fatal(err) + } + component, err := NewGeneralChunker(map[string]any{"chunk_token_size": 512}) + if err != nil { + t.Fatal(err) + } + out, err := component.Invoke(t.Context(), nil, map[string]any{ + "name": "retrieval_md_20_ret.md", "file_type": "md", "output_format": "json", "json": items, + }) + if err != nil { + t.Fatal(err) + } + tables := 0 + metadataCopies := 0 + for _, chunk := range outputChunks(t, out) { + text, _ := chunk["text"].(string) + if chunk["doc_type_kwd"] == "table" { + tables++ + } + if strings.Contains(text, "Document ID") { + metadataCopies++ + if chunk["doc_type_kwd"] != "table" { + t.Fatalf("metadata flattened: %#v", chunk) + } + for _, row := range []string{ + "Document IDretrieval_md_20_ret.md", + "Component codeRET", + "OwnerPlatform Reliability (ret-team)", + "Last reviewed2026-09-10", + "Review cycleevery 90 days", + } { + if !strings.Contains(text, row) { + t.Errorf("metadata missing row %q: %s", row, text) + } + } + } + } + if tables != 4 || metadataCopies != 1 { + t.Fatalf("tables=%d metadata copies=%d, want 4 and 1", tables, metadataCopies) + } +} + +func TestGeneralChunkerPreservesEPUBParagraphBoundaries(t *testing.T) { + var data bytes.Buffer + writer := zip.NewWriter(&data) + for name, content := range map[string]string{ + "META-INF/container.xml": ``, + "OEBPS/content.opf": ``, + "OEBPS/chapter.xhtml": `

Alpha

Beta

ItemValue

Gamma

Delta

`, + } { + entry, err := writer.Create(name) + if err != nil { + t.Fatal(err) + } + if _, err := entry.Write([]byte(content)); err != nil { + t.Fatal(err) + } + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + parsed := parser.NewEPUBParser().ParseWithResult(t.Context(), "book.epub", data.Bytes()) + if parsed.Err != nil { + t.Fatal(parsed.Err) + } + component, err := NewGeneralChunker(map[string]any{"chunk_token_size": 512}) + if err != nil { + t.Fatal(err) + } + out, err := component.Invoke(t.Context(), nil, map[string]any{"name": "book.epub", "output_format": "json", "json": parsed.JSON}) + if err != nil { + t.Fatal(err) + } + chunks := outputChunks(t, out) + if len(chunks) != 3 { + t.Fatalf("chunks=%#v", chunks) + } + before, _ := chunks[0]["text"].(string) + after, _ := chunks[2]["text"].(string) + if before != "Alpha Beta" || chunks[1]["doc_type_kwd"] != "table" || after != "Gamma Delta" { + t.Fatalf("paragraph or table boundaries lost: %#v", chunks) + } +} diff --git a/internal/ingestion/component/chunker/table.go b/internal/ingestion/component/chunker/table.go index 0da8aa90ce..c662b275af 100644 --- a/internal/ingestion/component/chunker/table.go +++ b/internal/ingestion/component/chunker/table.go @@ -153,7 +153,6 @@ func expandHTMLTableRows(item schema.ChunkDoc) []schema.ChunkDoc { out.TableData = nil return []schema.ChunkDoc{out} } - names := rows[0] // R1: a row chunk must carry only its own tuple. That is only sound when // the item carries spreadsheet identity and its matrix was built // row-aligned (one five-field tuple per row, header included); a @@ -177,7 +176,14 @@ func expandHTMLTableRows(item schema.ChunkDoc) []schema.ChunkDoc { for i, row := range rows[headerCount:] { // Skip empty rows (no cell content); the HTML render of an empty row // would be a header-only table that duplicates the header-only chunk. - if tableRowRecordText(names, row) == "" { + hasText := false + for _, cell := range row { + if strings.TrimSpace(cell) != "" { + hasText = true + break + } + } + if !hasText { continue } doc := item @@ -231,29 +237,6 @@ func tableDataOf(item schema.ChunkDoc) *entity.TableData { return item.TableData } -// tableRowRecordText renders one row in the Python table chunker's line format: -// "- {field}: {value}" per non-empty cell, newline joined. A cell whose -// column has no header name keeps its value alone rather than being dropped. -func tableRowRecordText(names, cells []string) string { - lines := make([]string, 0, len(cells)) - for j, cell := range cells { - value := strings.TrimSpace(cell) - if value == "" { - continue - } - name := "" - if j < len(names) { - name = strings.TrimSpace(names[j]) - } - if name != "" { - lines = append(lines, "- "+name+": "+value) - continue - } - lines = append(lines, "- "+value) - } - return strings.Join(lines, "\n") -} - func init() { MustRegisterChunker(ComponentNameTableChunker) } diff --git a/internal/ingestion/component/chunker/testdata/tables/retrieval_md_20_ret.md b/internal/ingestion/component/chunker/testdata/tables/retrieval_md_20_ret.md new file mode 100644 index 0000000000..c107c3ac19 --- /dev/null +++ b/internal/ingestion/component/chunker/testdata/tables/retrieval_md_20_ret.md @@ -0,0 +1,92 @@ +# Retrieval Evaluation Handbook + +| | | +|---|---| +| Document ID | `retrieval_md_20_ret.md` | +| Component code | `RET` | +| Owner | Platform Reliability (ret-team) | +| Last reviewed | 2026-09-10 | +| Review cycle | every 90 days | + +## 1. Purpose + +This handbook covers **Retrieval Evaluation Handbook**. It exists to measure and improve retrieval quality, including recall, ranking and the effect of configuration changes. + +It is written for on-call engineers who already have access to the platform +console and to the read-only observability dashboards. Everything in this +document describes steady-state behaviour; emergency procedures are owned by +the incident commander and are not repeated here. + +## 2. Service endpoints + +| Endpoint | Description | +|---|---| +| `POST /v1/eval/runs` | 提交一次离线评测 | +| `GET /v1/eval/runs/{id}` | 读取评测指标与逐条明细 | + +All endpoints require a bearer token and are subject to the platform rate +limit. Requests without a valid token are rejected before any work is done. + +## 3. Configuration defaults + +Unless a site profile overrides them, the following values apply to every +deployment of Retrieval Evaluation Handbook. These are the values the operations team should quote +when sizing, tuning or comparing environments. + +- **default worker concurrency**: 16 workers +- **request timeout**: 80 seconds +- **maximum retry count**: 2 retries +- **health check interval**: 35 seconds +- **maintenance batch size**: 384 items + +Changing any of the values above is a configuration change and must go through +the normal review process. Values that differ from the defaults should be +recorded in the site profile so that later comparisons remain meaningful. + +## 4. Other tuning parameters + +The parameters below are environment-specific and are normally left at the +values shown. They are listed here for completeness. + +| Parameter | Typical value | +|---|---| +| Maximum concurrent uploads | 2 | +| Log retention | 34 days | +| Slow query threshold | 400 ms | +| Circuit breaker threshold | 5 consecutive failures | +| Trace sampling ratio | 0.1 | + +## 5. Normal operation + +1. Confirm that the health endpoint reports the component as serving before + starting any maintenance work. +2. Verify that no rebalance, rebuild or compaction is already in progress. +3. Apply changes to one node or one replica group at a time. +4. Watch the error rate and the latency dashboards for at least one full window + after each step. +5. Record the change in the operations log with the exact parameter values + used, so that the next reviewer can reconstruct the state. + +## 6. Failure handling + +| Symptom | First action | +|---|---| +| 评测结果不可复现 | 固定随机种子与候选集快照 | +| 标注与文档版本不匹配 | 用文档指纹校验标注时效 | + +If none of the actions above resolves the symptom, escalate to the component +owner rather than retrying indefinitely: repeated retries of a saturated +component make the original symptom harder to observe. + +## 7. Maintenance window procedure + +- Announce the window at least one business day in advance. +- Take a restorable snapshot before touching configuration. +- Prefer draining traffic over restarting under load. +- After the window, re-run the standard smoke checks and the retrieval + evaluation job before declaring the window closed. + +## 8. Related documents + +- `retrieval_md_05_chk.md` +- `retrieval_md_09_rank.md` diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.go index 517ceeeede..4cafa80311 100644 --- a/internal/ingestion/component/chunker/token.go +++ b/internal/ingestion/component/chunker/token.go @@ -247,9 +247,9 @@ func (c *TokenChunkerComponent) invoke(ctx context.Context, db *gorm.DB, inputs func decodeChunkerFromUpstream(inputs map[string]any) (schema.ChunkerFromUpstream, error) { var out schema.ChunkerFromUpstream // Fast path: the parser emits typed []parser.DocItem. Convert it to - // []ChunkDoc directly via schema.ChunkDocsFromDocItem (which reads the - // item's typed fields and never calls DocItem.ToMap), then let the JSON - // marshal below serialize the already-typed ChunkDoc. This keeps the + // []ChunkDoc directly via schema.ChunkDocsFromDocItem (which projects each + // item through its canonical ToMap wire shape), then let the JSON marshal + // below serialize the already-typed ChunkDoc. This keeps the // chunker free of the historical map intermediate for the in-memory // Parser -> Chunker leg. When inputs["json"] is already []map[string]any // (legacy fixtures, round-tripped JSON, or absent), the original path runs. diff --git a/internal/parser/parser/charset_test.go b/internal/parser/parser/charset_test.go index e1b3b06e7a..23c1c0988b 100644 --- a/internal/parser/parser/charset_test.go +++ b/internal/parser/parser/charset_test.go @@ -217,7 +217,7 @@ func TestEPUBParser_GB2312(t *testing.T) { if len(res.JSON) == 0 { t.Fatal("want non-empty JSON items from EPUB") } - text, _ := res.JSON[0].ToMap()["text"].(string) + text := epubTestText(res) if strings.ContainsRune(text, '\ufffd') { t.Errorf("EPUB text contains replacement runes: %s", text) } diff --git a/internal/parser/parser/docx_caption_test.go b/internal/parser/parser/docx_caption_test.go index 194c45296b..362dc1c2c5 100644 --- a/internal/parser/parser/docx_caption_test.go +++ b/internal/parser/parser/docx_caption_test.go @@ -16,8 +16,7 @@ const docxCaptionTableBody = `{"type":"table","rows":[{"cells":[{"content":[{"ty // docxCaptionFindTableCaption returns the Caption of the (single) table // section, which buildDOCXJSONSections stores on the structured -// entity.TableData (after projectPDFTable parses the injected -// out of the table HTML). +// entity.TableData through the Office IR table projection. func docxCaptionFindTableCaption(t *testing.T, sections []map[string]any) string { t.Helper() for _, s := range sections { diff --git a/internal/parser/parser/docx_ir.go b/internal/parser/parser/docx_ir.go index 3bb904be0a..dfe3707ad1 100644 --- a/internal/parser/parser/docx_ir.go +++ b/internal/parser/parser/docx_ir.go @@ -28,8 +28,10 @@ import ( "encoding/base64" "encoding/json" "fmt" - "html" "strings" + + "ragflow/internal/entity" + "ragflow/internal/parser/tableutil" ) // DOCXFigure represents one embedded image plus its surrounding text @@ -188,29 +190,17 @@ func joinCellText(cell docxIRCell) string { return strings.Join(parts, "\n") } -// docxIRTableToHTML converts a table IR element to an HTML table string. -// A non-empty caption is emitted as a leading element so the -// structured TableData contract (ParseTableHTML) carries it through to the -// chunk renderer, which escapes it again on output. -func docxIRTableToHTML(el docxIRElement, caption string) string { - var sb strings.Builder - sb.WriteString("") - if caption != "" { - sb.WriteString("") - } +// officeIRTableData projects native Office rows directly onto the table contract. +func officeIRTableData(el docxIRElement, caption string) *entity.TableData { + rows := make([][]tableutil.CellInput, 0, len(el.Rows)) for _, row := range el.Rows { - sb.WriteString("") + cells := make([]tableutil.CellInput, 0, len(row.Cells)) for _, cell := range row.Cells { - sb.WriteString("") + cells = append(cells, tableutil.CellInput{Text: joinCellText(cell)}) } - sb.WriteString("") + rows = append(rows, cells) } - sb.WriteString("
") - sb.WriteString(html.EscapeString(caption)) - sb.WriteString("
") - sb.WriteString(html.EscapeString(joinCellText(cell))) - sb.WriteString("
") - return sb.String() + return tableutil.BuildTableData(rows, nil, strings.TrimSpace(caption)) } type docxTableImage struct { @@ -423,15 +413,22 @@ func buildDOCXJSONSectionsCore(irJSON string, budget *embeddedMediaBudget, table } var sections []map[string]any tableSequence := 0 + captionSequence := 0 var headings []docxHeading - for si := range ir.Sections { - sec := &ir.Sections[si] - for ei := 0; ei < len(sec.Elements); ei++ { - el := sec.Elements[ei] - // TEMPORARY: this merge is a stopgap. Revisit it once office_oxide - // populates `TableRow.is_header` for .doc tables (upstream issue + var emitElements func([]docxIRElement, bool) + emitElements = func(elements []docxIRElement, topLevel bool) { + if !topLevel { + parentHeadings := headings + headings = append([]docxHeading(nil), headings...) + defer func() { headings = parentHeadings }() + } + for ei := 0; ei < len(elements); ei++ { + el := elements[ei] + // TEMPORARY: this merge is a stopgap. Revisit it once docxIRRow + // carries a three-state header field and office_oxide populates + // `TableRow.is_header` for .doc tables (upstream issue // yfedoseev/office_oxide#456) — at that point prefer rejoining via - // the real `is_header` flag and drop the structural proxy below. + // the real header flag and drop the structural proxy below. // // office_oxide splits a .doc table whose first row is a styled // header/title row into a 1-row "header" table immediately @@ -466,8 +463,8 @@ func buildDOCXJSONSectionsCore(irJSON string, budget *embeddedMediaBudget, table // separate. // - adjacency (no element between): any heading/paragraph/image // in between keeps the two tables distinct. - if el.Type == "table" && len(el.Rows) == 1 && ei+1 < len(sec.Elements) { - if nxt := sec.Elements[ei+1]; nxt.Type == "table" && + if topLevel && el.Type == "table" && len(el.Rows) == 1 && ei+1 < len(elements) { + if nxt := elements[ei+1]; nxt.Type == "table" && docxIRTableCols(el) > 0 && docxIRTableCols(el) == docxIRTableCols(nxt) { merged := nxt rows := make([]docxIRRow, 0, len(nxt.Rows)+len(el.Rows)) @@ -493,8 +490,8 @@ func buildDOCXJSONSectionsCore(irJSON string, budget *embeddedMediaBudget, table // to that table as the caption instead. The text match keeps // this safe if the document scan and the IR element order // ever desync: a non-matching paragraph is never dropped. - if tableSequence < len(tableCaptions) { - if explicit := strings.TrimSpace(tableCaptions[tableSequence]); explicit != "" && + if topLevel && captionSequence < len(tableCaptions) { + if explicit := strings.TrimSpace(tableCaptions[captionSequence]); explicit != "" && strings.TrimSpace(docxElementText(el, " ")) == explicit { continue } @@ -506,6 +503,9 @@ func buildDOCXJSONSectionsCore(irJSON string, budget *embeddedMediaBudget, table case "table": tableSequence++ + if topLevel { + captionSequence++ + } if len(el.Rows) == 0 { continue } @@ -513,71 +513,58 @@ func buildDOCXJSONSectionsCore(irJSON string, budget *embeddedMediaBudget, table // stopgap, see docx_caption_scan.go) over the synthesized // heading-location caption. caption := docxHeadingLocationCaption(headings) - if idx := tableSequence - 1; idx >= 0 && idx < len(tableCaptions) { + if idx := captionSequence - 1; topLevel && idx >= 0 && idx < len(tableCaptions) { if explicit := strings.TrimSpace(tableCaptions[idx]); explicit != "" { caption = explicit } } - html := docxIRTableToHTML(el, caption) - table := map[string]any{ - "text": html, - "image": nil, - "doc_type_kwd": "table", - } - projectPDFTable(table) - tableID := "" - foundImage := false - var tableImages []docxTableImage - forEachDOCXTableImage(el, func(tableImage docxTableImage) bool { - foundImage = true - included, keepWalking := budget.include(tableImage.data) - if included || keepWalking { - tableImage.included = included - tableImages = append(tableImages, tableImage) - } - return keepWalking - }) - if foundImage { - tableID = fmt.Sprintf("docx-table-%d", tableSequence) - table["source_table_id"] = tableID - } - sections = append(sections, table) - for mediaOrder, tableImage := range tableImages { - metadata := map[string]any{ - "parent_table_id": tableID, - "row_index": tableImage.row, - "column_index": tableImage.column, - "media_order": mediaOrder + 1, - } - sections = appendDOCXImagePayload(sections, tableImage.data, tableImage.included, metadata) - } + sections = appendOfficeTableSections(sections, el, caption, fmt.Sprintf("docx-table-%d", tableSequence), budget) case "list": for _, item := range el.Items { - text := extractTextFromListItem(item) - if text != "" { - sections = append(sections, map[string]any{ - "text": text, - "image": nil, - "doc_type_kwd": "text", - }) + emitElements(item.Content, false) + if item.Nested != nil { + emitElements([]docxIRElement{{Type: "list", Items: item.Nested.Items}}, false) } - sections = appendDOCXListImageSections(sections, docxIRList{Items: []docxIRListItem{item}}, budget) } - case "text_box": - text := extractTextFromBlockElements(el.contentBlocks()) - if text != "" { - sections = append(sections, map[string]any{ - "text": text, - "image": nil, - "doc_type_kwd": "text", - }) - } - sections = appendDOCXIRImageSections(sections, el.contentBlocks(), budget) + emitElements(el.contentBlocks(), false) } } } + for _, sec := range ir.Sections { + emitElements(sec.Elements, true) + } + return sections +} + +// appendOfficeTableSections emits a table and its embedded images together. +func appendOfficeTableSections(sections []map[string]any, el docxIRElement, caption, tableID string, budget *embeddedMediaBudget) []map[string]any { + table := NewTableDataJSONItem(officeIRTableData(el, caption), "", nil).ToMap() + + var tableImages []docxTableImage + forEachDOCXTableImage(el, func(tableImage docxTableImage) bool { + included, keepWalking := budget.include(tableImage.data) + if included || keepWalking { + tableImage.included = included + tableImages = append(tableImages, tableImage) + } + return keepWalking + }) + if len(tableImages) > 0 { + table["source_table_id"] = tableID + } + sections = append(sections, table) + for mediaOrder, tableImage := range tableImages { + metadata := map[string]any{ + "parent_table_id": tableID, + "row_index": tableImage.row, + "column_index": tableImage.column, + "media_order": mediaOrder + 1, + } + sections = appendDOCXImagePayload(sections, tableImage.data, tableImage.included, metadata) + } + return sections } @@ -603,8 +590,10 @@ func appendDOCXParagraphSections(sections []map[string]any, el docxIRElement, he } sections = append(sections, item) } - appendImage := func(data []byte) { - sections, _ = appendDOCXImageSection(sections, data, budget, nil) + appendImage := func(data []byte) bool { + var keepWalking bool + sections, keepWalking = appendDOCXImageSection(sections, data, budget, nil) + return keepWalking } for _, run := range el.contentRuns() { switch run.Type { @@ -615,12 +604,8 @@ func appendDOCXParagraphSections(sections []map[string]any, el docxIRElement, he case "image": flushText() appendImage(run.Data) - for _, nested := range run.Content { - if nested.Type == "image" { - appendImage(nested.Data) - } - } } + forEachDOCXIRImage(run.Content, func(data []byte) bool { flushText(); return appendImage(data) }) } flushText() return sections @@ -637,25 +622,6 @@ func appendDOCXImageSection(sections []map[string]any, data []byte, budget *embe return appendDOCXImagePayload(sections, data, included, metadata), keepWalking } -func appendDOCXListImageSections(sections []map[string]any, list docxIRList, budget *embeddedMediaBudget) []map[string]any { - for _, item := range list.Items { - sections = appendDOCXIRImageSections(sections, item.Content, budget) - if item.Nested != nil { - sections = appendDOCXListImageSections(sections, *item.Nested, budget) - } - } - return sections -} - -func appendDOCXIRImageSections(sections []map[string]any, elements []docxIRElement, budget *embeddedMediaBudget) []map[string]any { - forEachDOCXIRImage(elements, func(data []byte) bool { - var keepWalking bool - sections, keepWalking = appendDOCXImageSection(sections, data, budget, nil) - return keepWalking - }) - return sections -} - func appendDOCXImagePayload(sections []map[string]any, data []byte, included bool, metadata map[string]any) []map[string]any { item := map[string]any{ "text": "", @@ -694,18 +660,7 @@ func extractDOCXFiguresFromIR(irJSON string, budget *embeddedMediaBudget) []DOCX var flat []flatBlock for _, sec := range ir.Sections { - for _, el := range sec.Elements { - if el.Type == "image" { - flat = append(flat, flatBlock{imageData: el.Data}) - continue - } - if el.Type == "paragraph" || el.Type == "heading" { - flat = append(flat, docxInlineFlatBlocks(el)...) - continue - } - text := docxElementText(el, "\n") - flat = append(flat, flatBlock{text: text}) - } + flat = appendDOCXFigureBlocks(flat, sec.Elements) } var figures []DOCXFigure @@ -750,6 +705,43 @@ func extractDOCXFiguresFromIR(irJSON string, budget *embeddedMediaBudget) []DOCX return figures } +// appendDOCXFigureBlocks flattens IR elements in document order, including +// images nested in tables, lists and text boxes. Section emission recurses +// through the same containers, so figure metadata must use the same traversal +// instead of recording only top-level images. +func appendDOCXFigureBlocks(flat []flatBlock, elements []docxIRElement) []flatBlock { + for _, el := range elements { + switch el.Type { + case "image": + if len(el.Data) > 0 { + flat = append(flat, flatBlock{imageData: el.Data}) + } + case "paragraph", "heading": + flat = append(flat, docxInlineFlatBlocks(el)...) + case "table": + for _, row := range el.Rows { + for _, cell := range row.Cells { + flat = appendDOCXFigureBlocks(flat, cell.Content) + } + } + case "list": + for _, item := range el.Items { + flat = appendDOCXFigureBlocks(flat, item.Content) + if item.Nested != nil { + flat = appendDOCXFigureBlocks(flat, []docxIRElement{{Type: "list", Items: item.Nested.Items}}) + } + } + case "text_box": + flat = appendDOCXFigureBlocks(flat, el.contentBlocks()) + default: + if text := docxElementText(el, "\n"); text != "" { + flat = append(flat, flatBlock{text: text}) + } + } + } + return flat +} + // docxInlineFlatBlocks preserves the text/image sequence inside a paragraph // for figure context extraction. Top-level element order alone is not enough // for DOCX because office_oxide can encode an inline image as a paragraph run. diff --git a/internal/parser/parser/docx_ir_test.go b/internal/parser/parser/docx_ir_test.go index d26229c900..866ee36097 100644 --- a/internal/parser/parser/docx_ir_test.go +++ b/internal/parser/parser/docx_ir_test.go @@ -34,7 +34,7 @@ func TestBuildDOCXJSONSections_FromJSON(t *testing.T) { {"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"}, // The table sits under heading "Title", so it carries the // heading-location caption (mirrors Python __get_nearest_title). - {"image": nil, "doc_type_kwd": "table", "table": &entity.TableData{Grid: [][]entity.TableCell{{{Text: "cell", ColSpan: 1, RowSpan: 1}}}, HeaderRows: 1, Caption: "Table Location: Title"}}, + {"doc_type_kwd": "table", "table": &entity.TableData{Grid: [][]entity.TableCell{{{Text: "cell", ColSpan: 1, RowSpan: 1}}}, HeaderRows: 1, Caption: "Table Location: Title"}}, {"text": "item1", "image": nil, "doc_type_kwd": "text"}, } if !reflect.DeepEqual(got, want) { @@ -516,6 +516,28 @@ func TestExtractDOCXFiguresFromIR_NonParagraphContext(t *testing.T) { } } +// TestExtractDOCXFiguresFromIR_NestedImages verifies that images inside +// tables and text boxes reach the figure metadata. Section emission recurses +// into these containers, so a top-level-only figure walk would omit images +// the parser actually emits. +func TestExtractDOCXFiguresFromIR_NestedImages(t *testing.T) { + irJSON := `{"sections":[{"elements":[ + {"type":"paragraph","content":[{"type":"text","text":"before"}]}, + {"type":"table","rows":[{"cells":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"cell"},{"type":"image","data":"aGVsbG8="}]}]}]}]}, + {"type":"text_box","content":[{"type":"paragraph","content":[{"type":"image","data":"d29ybGQ="}]}]} + ]}]}` + figs := extractDOCXFiguresFromIR(irJSON, newEmbeddedMediaBudget()) + if len(figs) != 2 { + t.Fatalf("expected 2 nested figures, got %d: %+v", len(figs), figs) + } + if figs[0].Image != "aGVsbG8=" || figs[1].Image != "d29ybGQ=" { + t.Fatalf("nested figures = %+v", figs) + } + if !strings.Contains(figs[0].ContextAbove, "cell") { + t.Errorf("ContextAbove = %q, want the nested cell text", figs[0].ContextAbove) + } +} + // TestCollectDOCXText_BoundedByMaxLen pins that the joined context // never exceeds maxLen runes, even though newline separators are // inserted between blocks (they are not counted by the per-block @@ -611,8 +633,7 @@ func TestExtractTextFromListItem_NestedNull(t *testing.T) { } // TestExtractTextFromListItem_NestedTable verifies that a table nested -// inside a list item's content is flattened, not silently dropped. The old -// path only recognized paragraph/heading blocks in item.Content. +// inside a list item remains structured beside the preceding text. func TestExtractTextFromListItem_NestedTable(t *testing.T) { irJSON := `{"sections":[{"elements":[ {"type":"list","items":[ @@ -628,12 +649,10 @@ func TestExtractTextFromListItem_NestedTable(t *testing.T) { ]} ]}]}` sections := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget()) - if len(sections) != 1 { - t.Fatalf("expected 1 section, got %d: %+v", len(sections), sections) - } - if got := sections[0]["text"]; got != "before\nc1\nc2" { - t.Errorf("list item with nested table text = %q, want %q", got, "before\nc1\nc2") + if len(sections) != 2 || sections[0]["text"] != "before" { + t.Fatalf("sections=%#v", sections) } + requireTableRows(t, sections[1], [][]string{{"c1", "c2"}}) } // TestJoinCellText_NestedList verifies that a list nested inside a table @@ -682,9 +701,7 @@ func TestBuildDOCXJSONSections_TextBoxTable(t *testing.T) { if len(sections) != 1 { t.Fatalf("expected 1 section, got %d: %+v", len(sections), sections) } - if got := sections[0]["text"]; got != "Q1\nA1" { - t.Errorf("text_box table text = %q, want %q", got, "Q1\nA1") - } + requireTableRows(t, sections[0], [][]string{{"Q1", "A1"}}) } // TestBuildDOCXJSONSections_NestedList verifies end-to-end that a @@ -700,11 +717,13 @@ func TestBuildDOCXJSONSections_NestedList(t *testing.T) { ]} ]}]}` sections := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget()) - if len(sections) != 1 { - t.Fatalf("expected 1 section, got %d: %+v", len(sections), sections) + if len(sections) != 3 { + t.Fatalf("sections=%#v", sections) } - if got := sections[0]["text"]; got != "L1\nL2a\nL2b" { - t.Errorf("nested list text = %q, want %q", got, "L1\nL2a\nL2b") + for i, want := range []string{"L1", "L2a", "L2b"} { + if sections[i]["text"] != want { + t.Fatalf("section[%d]=%#v", i, sections[i]) + } } } diff --git a/internal/parser/parser/docx_parser_test.go b/internal/parser/parser/docx_parser_test.go index d724bf05aa..a4398a6b1b 100644 --- a/internal/parser/parser/docx_parser_test.go +++ b/internal/parser/parser/docx_parser_test.go @@ -20,7 +20,6 @@ package parser import ( "encoding/base64" - "encoding/json" "strings" "testing" @@ -225,53 +224,3 @@ func TestBuildDOCXJSONSections_EmptySkipped(t *testing.T) { t.Errorf("text = %q, want %q", got, "Only this matters") } } - -func TestDocxIRTableToHTML_Empty(t *testing.T) { - el := docxIRElement{Type: "table", Rows: []docxIRRow{}} - html := docxIRTableToHTML(el, "") - if html != "
" { - t.Errorf("empty table html = %q, want %q", html, "
") - } -} - -func TestDocxIRTableToHTML_Single(t *testing.T) { - el := docxIRElement{ - Type: "table", - Rows: []docxIRRow{{ - Cells: []docxIRCell{{ - Content: []docxIRElement{{ - Type: "paragraph", - Content: json.RawMessage(`[{"type":"text","text":"hello"}]`), - }}, - }}, - }}, - } - html := docxIRTableToHTML(el, "") - want := "
hello
" - if html != want { - t.Errorf("single cell html = %q, want %q", html, want) - } -} - -func TestDocxIRTableToHTML_MultiRowCol(t *testing.T) { - cell := func(text string) docxIRCell { - return docxIRCell{ - Content: []docxIRElement{{ - Type: "paragraph", - Content: rawJSON([]docxIRRun{{Type: "text", Text: text}}), - }}, - } - } - el := docxIRElement{ - Type: "table", - Rows: []docxIRRow{ - {Cells: []docxIRCell{cell("A1"), cell("B1")}}, - {Cells: []docxIRCell{cell("A2"), cell("B2")}}, - }, - } - html := docxIRTableToHTML(el, "") - want := "
A1B1
A2B2
" - if html != want { - t.Errorf("multi cell html = %q, want %q", html, want) - } -} diff --git a/internal/parser/parser/epub_parser.go b/internal/parser/parser/epub_parser.go index 7e0381f54d..df2ac36922 100644 --- a/internal/parser/parser/epub_parser.go +++ b/internal/parser/parser/epub_parser.go @@ -20,8 +20,7 @@ // 1. Opens the EPUB as a ZIP archive. // 2. Reads META-INF/container.xml to locate the OPF manifest. // 3. Walks the OPF spine to collect content documents in order. -// 4. Strips HTML tags from each content document to produce plain -// text items, emitting one JSON item per spine entry. +// 4. Emits text, tables and inline images from each chapter in document order. // // The output format is "json", matching the epub default in // ParserParam.Defaults(). @@ -34,12 +33,13 @@ import ( "context" "encoding/xml" "fmt" - "html" "io" "net/url" "path" "regexp" "strings" + + "golang.org/x/net/html" ) // EPUBParser extracts text from EPUB archives. @@ -54,8 +54,7 @@ func (p *EPUBParser) String() string { } // ParseWithResult implements ParseResultProducer. It extracts XHTML -// content from the EPUB spine and emits one JSON item per spine entry -// with {text, doc_type_kwd:"text"}. +// content from the EPUB spine using the shared HTML structure walker. func (p *EPUBParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult { if len(data) == 0 { return ParseResult{ @@ -83,7 +82,7 @@ func (p *EPUBParser) ParseWithResult(ctx context.Context, filename string, data return ParseResult{Err: fmt.Errorf("epub: opf: %w", err)} } - items, detectedEnc, err := extractEPUBTextItems(reader, opfPath, spineItems) + items, detectedEnc, warnings, err := extractEPUBTextItems(reader, opfPath, spineItems) if err != nil { return ParseResult{Err: fmt.Errorf("epub: %w", err)} } @@ -96,7 +95,8 @@ func (p *EPUBParser) ParseWithResult(ctx context.Context, filename string, data "size": len(data), "encoding": detectedEnc, }, - JSON: MapsToItems(items), + JSON: MapsToItems(items), + Warnings: warnings, } } @@ -189,25 +189,28 @@ func readOPFSpine(r *zip.Reader, opfPath string) ([]string, error) { // --- text extraction --- -var ( - epubScriptRe = regexp.MustCompile(`(?is)]*>.*?`) - epubStyleRe = regexp.MustCompile(`(?is)]*>.*?`) - epubTagRe = regexp.MustCompile(`<[^>]+>`) - epubEntityRe = regexp.MustCompile(`&(?:[a-zA-Z][a-zA-Z0-9]*|#[0-9]+|#[xX][0-9a-fA-F]+);`) - epubWSRe = regexp.MustCompile(`\s+`) - // Closing tags of block elements, or a void
, that end a run of text. - epubBlockEndRe = regexp.MustCompile(`^(?:]|])`) -) +var epubSpaceEntity = regexp.MustCompile(`&(?:nbsp|#0*160|#[xX]0*[aA]0);`) -// extractEPUBTextItems reads every spine item and returns one text item per -// item that has text. An item that cannot be read is skipped; when items were +// epubSelfClosingRawText matches a self-closing non-void XHTML element whose +// content HTML parsers treat as raw text. HTML5 parsing turns