fix(ingestion): preserve PDF sentences across page boundaries (#21982)

internal/ingestion/component/chunker/general.go

@@ -260,6 +260,7 @@ func (c *GeneralChunkerComponent) chunkPDF(ctx context.Context, db *gorm.DB, ups
 		}
 		joinSep := "\n"
 		if !customDelimiter {
+			body, bodySources = joinPDFContinuations(body, bodySources)
 			body = separateGeneralTextBlocks(body)
 			body, bodySources = splitPDFUnits(body, bodySources, primaryPattern, true)
 			joinSep = ""
@@ -268,7 +269,7 @@ func (c *GeneralChunkerComponent) chunkPDF(ctx context.Context, db *gorm.DB, ups
 		chunks = append(chunks, merged...)
 		chunkSources = append(chunkSources, mergedSources...)
 		body = body[:0]
-		bodySources = bodySources[:0]
+		bodySources = nil
 	}
 	for index, unit := range units {
 		if itemDocType(unit) == "text" {

internal/ingestion/component/chunker/general_pdf_continuation.go

@@ -0,0 +1,139 @@
+package chunker
+
+import (
+	"math"
+	"regexp"
+	"strings"
+	"unicode"
+	"unicode/utf8"
+
+	"ragflow/internal/ingestion/component/schema"
+	"ragflow/internal/parser/parser"
+)
+
+var pdfListStart = regexp.MustCompile(`^(?:[（(]?[0-9一二三四五六七八九十百]+[）).．、]|[a-zA-Z]\)|[a-zA-Z]\.(?:\s|$)|[-*•]\s)`)
+
+// joinPDFContinuations removes physical page breaks within continuing prose
+// before delimiter splitting turns source regions into independent merge units.
+// Each source keeps its own text range and page rectangle for later splitting.
+func joinPDFContinuations(units []schema.ChunkDoc, sources [][]pdfSourceSpan) ([]schema.ChunkDoc, [][]pdfSourceSpan) {
+	first := 1
+	for first < len(units) && !canJoinPDFContinuation(units[first-1], units[first]) {
+		first++
+	}
+	if first >= len(units) {
+		return units, sources
+	}
+	if len(sources) != len(units) {
+		sources = pdfUnitSources(units)
+	}
+	joined := append([]schema.ChunkDoc(nil), units[:first]...)
+	joinedSources := append([][]pdfSourceSpan(nil), sources[:first]...)
+	for i := first; i < len(units); i++ {
+		unit := units[i]
+		if !canJoinPDFContinuation(joined[len(joined)-1], unit) {
+			joined = append(joined, unit)
+			joinedSources = append(joinedSources, sources[i])
+			continue
+		}
+		index := len(joined) - 1
+		joined[index] = cloneChunkDoc(joined[index])
+		previous := &joined[index]
+		left := removeTag(normalizeGeneralNewlines(previous.Text))
+		right := removeTag(normalizeGeneralNewlines(unit.Text))
+		previous.Text = strings.TrimRightFunc(left, unicode.IsSpace)
+		unit.Text = strings.TrimLeftFunc(right, unicode.IsSpace)
+		leftSources := slicePDFSourceSpans(joinedSources[index], left, 0, len(previous.Text))
+		rightSources := slicePDFSourceSpans(sources[i], right, len(right)-len(unit.Text), len(right))
+		separator := pdfContinuationSeparator(previous.Text, unit.Text)
+		offset := len(previous.Text) + len(separator)
+		for j := range rightSources {
+			rightSources[j].start += offset
+			rightSources[j].end += offset
+		}
+		joinedSources[index] = append(leftSources, rightSources...)
+		previous.Text += separator + unit.Text
+		previous.PDFPositions = mergeGeneralPositions(previous.PDFPositions, unit.PDFPositions)
+		previous.Positions = mergeGeneralPositions(previous.Positions, unit.Positions)
+		mergeGeneralMetadata(previous, unit)
+		previous.TKNums = nil // Delimiter splitting counts the resulting units.
+	}
+	return joined, joinedSources
+}
+
+func canJoinPDFContinuation(previous, next schema.ChunkDoc) bool {
+	for _, unit := range []schema.ChunkDoc{previous, next} {
+		if itemDocType(unit) != "text" || unit.PreserveOrder {
+			return false
+		}
+		for _, layout := range []string{unit.Layout, unit.LayoutType} {
+			if layout != "" && layout != "text" {
+				return false
+			}
+		}
+	}
+	left := normalizeGeneralNewlines(removeTag(previous.Text))
+	right := normalizeGeneralNewlines(removeTag(next.Text))
+	// A blank line is an explicit paragraph break, including whitespace-only lines.
+	leftEnd := len(strings.TrimRightFunc(left, unicode.IsSpace))
+	rightStart := len(right) - len(strings.TrimLeftFunc(right, unicode.IsSpace))
+	if strings.Count(left[leftEnd:], "\n") > 1 || strings.Count(right[:rightStart], "\n") > 1 {
+		return false
+	}
+	left, right = strings.TrimSpace(left), strings.TrimSpace(right)
+	if left == "" || right == "" || pdfListStart.MatchString(right) {
+		return false
+	}
+	ending := strings.TrimRight(left, `"'”’）)]】》`)
+	if ending == "" || strings.ContainsRune("。！？.!?；;：:", lastPDFRune(ending)) {
+		return false
+	}
+	first, _ := utf8.DecodeRuneInString(right)
+	if unicode.IsUpper(first) || (!unicode.IsLetter(first) && !strings.ContainsRune("，、,", first)) {
+		return false
+	}
+	positions := func(unit schema.ChunkDoc) [][]float64 {
+		if len(unit.PDFPositions) > 0 {
+			return parser.PDFPositionRowsFromJSON(unit.PDFPositions)
+		}
+		return parser.PDFPositionRowsFromJSON(unit.Positions)
+	}
+	before, after := positions(previous), positions(next)
+	if len(before) == 0 || len(after) == 0 {
+		return false
+	}
+	a, b := before[len(before)-1], after[0]
+	// A continuation starts higher on the next page than the previous page's
+	// last region ended; a region at or below that point is unrelated.
+	if b[0] != a[0]+1 || b[3] >= a[4] {
+		return false
+	}
+	height := math.Min(a[4]-a[3], b[4]-b[3])
+	// Regions may cover whole paragraphs: their height must not make a
+	// first-line indent look like harmless alignment noise (PDF-point units).
+	tolerance := math.Min(height*0.6, 12)
+	return height > 0 && a[2] > a[1] && b[2] > b[1] && math.Abs(a[1]-b[1]) <= tolerance
+}
+
+func lastPDFRune(text string) rune {
+	r, _ := utf8.DecodeLastRuneInString(text)
+	return r
+}
+
+func pdfContinuationSeparator(left, right string) string {
+	a := lastPDFRune(left)
+	b, _ := utf8.DecodeRuneInString(right)
+	if pdfNoSpaceRune(a) && pdfNoSpaceRune(b) || strings.ContainsRune("-\u2010\u2011", a) || unicode.IsPunct(b) {
+		return ""
+	}
+	return " "
+}
+
+// pdfNoSpaceRune reports scripts that write words without separating spaces.
+// A boundary keeps the space when only one side is in this set because the
+// other side is space-delimited. The prolonged sound mark is Common script, so
+// it is listed explicitly; Hangul is excluded because Korean separates words
+// with spaces.
+func pdfNoSpaceRune(r rune) bool {
+	return r == '\u30fc' || unicode.In(r, unicode.Han, unicode.Hiragana, unicode.Katakana)
+}

internal/ingestion/component/chunker/general_pdf_continuation_test.go

@@ -0,0 +1,291 @@
+package chunker
+
+import (
+	"encoding/json"
+	"reflect"
+	"testing"
+
+	"ragflow/internal/ingestion/component/schema"
+)
+
+func TestGeneralPDFJoinsCrossPageSentenceBeforeTokenMerge(t *testing.T) {
+	for _, tc := range []struct {
+		name       string
+		delimiters []string
+	}{
+		{"newline", []string{"\n"}}, {"punctuation", []string{"。"}}, {"none", nil},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			first := "为确保工作有序推进，领导小组下设综合工作组、物资保障"
+			items := []map[string]any{
+				{"text": first, "doc_type_kwd": "text", "positions": [][]float64{{3, 80, 520, 619, 718}}},
+				{"text": "组、宣传工作组。", "doc_type_kwd": "text", "positions": [][]float64{{4, 80, 197, 115, 130}}},
+			}
+			// Force the budget boundary to fall between the two source regions.
+			out := invokePDFContinuation(t, items, tokenizeStr(first)-1, tc.delimiters, nil)
+			if got, want := outputTexts(t, out), []string{first + "组、宣传工作组。"}; !reflect.DeepEqual(got, want) {
+				t.Fatalf("texts = %q, want %q", got, want)
+			}
+			chunks := outputChunks(t, out)
+			if got, want := chunks[0]["positions"], [][]float64{{3, 80, 520, 619, 718}, {4, 80, 197, 115, 130}}; !reflect.DeepEqual(got, want) {
+				t.Fatalf("positions = %#v, want %#v", got, want)
+			}
+		})
+	}
+}
+
+func TestGeneralPDFCrossPageSentenceSplitKeepsSourcePositions(t *testing.T) {
+	items := []map[string]any{
+		{"text": "物资保障", "doc_type_kwd": "text", "positions": [][]float64{{3, 80, 520, 700, 715}}},
+		{"text": "组。下一段。", "doc_type_kwd": "text", "positions": [][]float64{{4, 80, 200, 100, 160}}},
+	}
+	out := invokePDFContinuation(t, items, 512, []string{"。"}, []string{"。"})
+	if got, want := outputTexts(t, out), []string{"物资保障组。", "下一段。"}; !reflect.DeepEqual(got, want) {
+		t.Fatalf("texts = %q, want %q", got, want)
+	}
+	chunks := outputChunks(t, out)
+	for i, want := range [][][]float64{
+		{{3, 80, 520, 700, 715}, {4, 80, 200, 100, 120}},
+		{{4, 80, 200, 120, 160}},
+	} {
+		if got := chunks[i]["positions"]; !reflect.DeepEqual(got, want) {
+			t.Errorf("chunk %d positions = %#v, want %#v", i, got, want)
+		}
+	}
+}
+
+func TestGeneralPDFCrossPageParentPositionsFollowSourceText(t *testing.T) {
+	items := []map[string]any{
+		{"text": "物资保障", "doc_type_kwd": "text", "positions": [][]float64{{3, 80, 520, 700, 715}}},
+		{"text": "组。下一段。", "doc_type_kwd": "text", "positions": [][]float64{{4, 80, 200, 100, 160}}},
+	}
+	out := invokePDFContinuation(t, items, 1, []string{"。"}, nil)
+	if got, want := outputTexts(t, out), []string{"物资保障组。", "下一段。"}; !reflect.DeepEqual(got, want) {
+		t.Fatalf("texts = %q, want %q", got, want)
+	}
+	chunks := outputChunks(t, out)
+	if got, want := chunks[1]["positions"], [][]float64{{4, 80, 200, 120, 160}}; !reflect.DeepEqual(got, want) {
+		t.Fatalf("positions = %#v, want %#v", got, want)
+	}
+}
+
+func TestGeneralPDFCrossPageEnglishKeepsWordSpace(t *testing.T) {
+	items := []map[string]any{
+		{"text": "work with the\r\n", "doc_type_kwd": "text", "positions": [][]float64{{1, 80, 520, 700, 715}}},
+		{"text": "team to finish.", "doc_type_kwd": "text", "positions": [][]float64{{2, 80, 200, 100, 115}}},
+	}
+	out := invokePDFContinuation(t, items, 1, []string{"\n"}, nil)
+	if got, want := outputTexts(t, out), []string{"work with the team to finish."}; !reflect.DeepEqual(got, want) {
+		t.Fatalf("texts=%q, want %q", got, want)
+	}
+}
+
+func TestGeneralPDFCrossPageKanaJoinsWithoutSpace(t *testing.T) {
+	items := []map[string]any{
+		{"text": "これは", "doc_type_kwd": "text", "positions": [][]float64{{1, 80, 520, 700, 715}}},
+		{"text": "テストです。", "doc_type_kwd": "text", "positions": [][]float64{{2, 80, 200, 100, 115}}},
+	}
+	out := invokePDFContinuation(t, items, 1, []string{"\n"}, nil)
+	if got, want := outputTexts(t, out), []string{"これはテストです。"}; !reflect.DeepEqual(got, want) {
+		t.Fatalf("texts=%q, want %q", got, want)
+	}
+}
+
+func TestGeneralPDFCrossPageMixedScriptKeepsSpace(t *testing.T) {
+	items := []map[string]any{
+		{"text": "The word is", "doc_type_kwd": "text", "positions": [][]float64{{1, 80, 520, 700, 715}}},
+		{"text": "ひらがな", "doc_type_kwd": "text", "positions": [][]float64{{2, 80, 200, 100, 115}}},
+	}
+	out := invokePDFContinuation(t, items, 1, []string{"\n"}, nil)
+	if got, want := outputTexts(t, out), []string{"The word is ひらがな"}; !reflect.DeepEqual(got, want) {
+		t.Fatalf("texts=%q, want %q", got, want)
+	}
+}
+
+func TestGeneralPDFCrossPageHanLatinKeepsSpace(t *testing.T) {
+	items := []map[string]any{
+		{"text": "使用", "doc_type_kwd": "text", "positions": [][]float64{{1, 80, 520, 700, 715}}},
+		{"text": "python 进行开发。", "doc_type_kwd": "text", "positions": [][]float64{{2, 80, 200, 100, 115}}},
+	}
+	out := invokePDFContinuation(t, items, 1, []string{"\n"}, nil)
+	if got, want := outputTexts(t, out), []string{"使用 python 进行开发。"}; !reflect.DeepEqual(got, want) {
+		t.Fatalf("texts=%q, want %q", got, want)
+	}
+}
+
+func TestGeneralPDFCrossPageHanKanaJoinsWithoutSpace(t *testing.T) {
+	items := []map[string]any{
+		{"text": "日本語", "doc_type_kwd": "text", "positions": [][]float64{{1, 80, 520, 700, 715}}},
+		{"text": "の文章。", "doc_type_kwd": "text", "positions": [][]float64{{2, 80, 200, 100, 115}}},
+	}
+	out := invokePDFContinuation(t, items, 1, []string{"\n"}, nil)
+	if got, want := outputTexts(t, out), []string{"日本語の文章。"}; !reflect.DeepEqual(got, want) {
+		t.Fatalf("texts=%q, want %q", got, want)
+	}
+}
+
+func TestGeneralPDFCrossPageProlongedSoundMarkJoinsWithoutSpace(t *testing.T) {
+	items := []map[string]any{
+		{"text": "コンピュー", "doc_type_kwd": "text", "positions": [][]float64{{1, 80, 520, 700, 715}}},
+		{"text": "ター。", "doc_type_kwd": "text", "positions": [][]float64{{2, 80, 200, 100, 115}}},
+	}
+	out := invokePDFContinuation(t, items, 1, []string{"\n"}, nil)
+	if got, want := outputTexts(t, out), []string{"コンピューター。"}; !reflect.DeepEqual(got, want) {
+		t.Fatalf("texts=%q, want %q", got, want)
+	}
+}
+
+func TestGeneralPDFCrossPageAbbreviationIsNotListMarker(t *testing.T) {
+	items := []map[string]any{
+		{"text": "references,", "doc_type_kwd": "text", "positions": [][]float64{{1, 80, 520, 700, 715}}},
+		{"text": "e.g. this article.", "doc_type_kwd": "text", "positions": [][]float64{{2, 80, 200, 100, 115}}},
+	}
+	out := invokePDFContinuation(t, items, 1, []string{"\n"}, nil)
+	if got, want := outputTexts(t, out), []string{"references, e.g. this article."}; !reflect.DeepEqual(got, want) {
+		t.Fatalf("texts=%q, want %q", got, want)
+	}
+}
+
+func TestGeneralPDFCrossPageRejectsRegionBelowPreviousBottom(t *testing.T) {
+	items := []map[string]any{
+		{"text": "负责物资保障", "doc_type_kwd": "text", "positions": [][]float64{{3, 80, 520, 100, 115}}},
+		{"text": "下一段内容", "doc_type_kwd": "text", "positions": [][]float64{{4, 80, 520, 500, 515}}},
+	}
+	out := invokePDFContinuation(t, items, 512, []string{"\n"}, nil)
+	if got, want := outputTexts(t, out), []string{"负责物资保障\n下一段内容"}; !reflect.DeepEqual(got, want) {
+		t.Fatalf("texts=%q, want %q", got, want)
+	}
+}
+
+func TestGeneralPDFContinuationRebuildsMismatchedSources(t *testing.T) {
+	units := []schema.ChunkDoc{
+		{Text: "物资保障", DocType: "text", Positions: json.RawMessage(`[[3,80,520,700,715]]`)},
+		{Text: "组、宣传工作组。", DocType: "text", Positions: json.RawMessage(`[[4,80,200,100,115]]`)},
+	}
+	joined, sources := joinPDFContinuations(units, [][]pdfSourceSpan{})
+	if len(joined) != 1 {
+		t.Fatalf("joined units = %d, want 1", len(joined))
+	}
+	if len(sources) != 1 || len(sources[0]) == 0 {
+		t.Fatalf("sources = %#v, want one populated source list", sources)
+	}
+}
+
+func TestGeneralPDFContinuationAcrossThreePagesKeepsEachSource(t *testing.T) {
+	items := []map[string]any{
+		{"text": "为保障工作", "doc_type_kwd": "text", "positions": [][]float64{{1, 80, 520, 700, 715}}},
+		{"text": "有序推进，设立物资保障", "doc_type_kwd": "text", "positions": [][]float64{{2, 80, 520, 100, 700}}},
+		{"text": "组。", "doc_type_kwd": "text", "positions": [][]float64{{3, 80, 200, 100, 115}}},
+	}
+	out := invokePDFContinuation(t, items, 1, []string{"\n"}, nil)
+	if got, want := outputTexts(t, out), []string{"为保障工作有序推进，设立物资保障组。"}; !reflect.DeepEqual(got, want) {
+		t.Fatalf("texts=%q, want %q", got, want)
+	}
+	chunks := outputChunks(t, out)
+	if got, want := chunks[0]["positions"], [][]float64{{1, 80, 520, 700, 715}, {2, 80, 520, 100, 700}, {3, 80, 200, 100, 115}}; !reflect.DeepEqual(got, want) {
+		t.Fatalf("positions=%#v, want %#v", got, want)
+	}
+}
+
+func TestGeneralPDFCrossPageContinuationHonorsCustomDelimiter(t *testing.T) {
+	items := []map[string]any{
+		{"text": "物资保障SEP", "doc_type_kwd": "text", "positions": [][]float64{{1, 80, 520, 700, 715}}},
+		{"text": "组、宣传工作组。", "doc_type_kwd": "text", "positions": [][]float64{{2, 80, 200, 100, 115}}},
+	}
+	out := invokePDFContinuation(t, items, 1, []string{"`SEP`"}, nil)
+	if got, want := outputTexts(t, out), []string{"物资保障", "组、宣传工作组。"}; !reflect.DeepEqual(got, want) {
+		t.Fatalf("texts=%q, want %q", got, want)
+	}
+}
+
+func TestGeneralPDFCrossPageContinuationDoesNotCrossMedia(t *testing.T) {
+	for _, kind := range []string{"image", "table"} {
+		t.Run(kind, func(t *testing.T) {
+			items := []map[string]any{
+				{"text": "物资保障", "doc_type_kwd": "text", "positions": [][]float64{{1, 80, 520, 700, 715}}},
+				{"text": "插图或表格", "doc_type_kwd": kind, "positions": [][]float64{{1, 80, 520, 720, 760}}},
+				{"text": "组、宣传工作组。", "doc_type_kwd": "text", "positions": [][]float64{{2, 80, 200, 100, 115}}},
+			}
+			out := invokePDFContinuation(t, items, 512, []string{"\n"}, nil)
+			if got, want := outputTexts(t, out), []string{"物资保障", "插图或表格", "组、宣传工作组。"}; !reflect.DeepEqual(got, want) {
+				t.Fatalf("texts=%q, want %q", got, want)
+			}
+		})
+	}
+}
+
+func TestGeneralPDFCrossPageSourcesResetAfterMedia(t *testing.T) {
+	items := []map[string]any{
+		{"text": "物资保障", "doc_type_kwd": "text", "positions": [][]float64{{1, 80, 520, 700, 715}}},
+		{"text": "组。", "doc_type_kwd": "text", "positions": [][]float64{{2, 80, 200, 100, 115}}},
+		{"text": "插图", "doc_type_kwd": "image", "positions": [][]float64{{2, 80, 520, 200, 300}}},
+		{"text": "后文。", "doc_type_kwd": "text", "positions": [][]float64{{2, 80, 520, 400, 415}}},
+	}
+	out := invokePDFContinuation(t, items, 512, []string{"\n"}, nil)
+	if got, want := outputTexts(t, out), []string{"物资保障组。", "插图", "后文。"}; !reflect.DeepEqual(got, want) {
+		t.Fatalf("texts=%q, want %q", got, want)
+	}
+}
+
+func TestGeneralPDFCrossPageTallParagraphKeepsIndentation(t *testing.T) {
+	items := []map[string]any{
+		{"text": "上一段没有结束标点", "doc_type_kwd": "text", "positions": [][]float64{{1, 80, 520, 600, 720}}},
+		{"text": "新的缩进段落。", "doc_type_kwd": "text", "positions": [][]float64{{2, 112, 520, 100, 250}}},
+	}
+	out := invokePDFContinuation(t, items, 512, []string{"\n"}, nil)
+	if got, want := outputTexts(t, out), []string{"上一段没有结束标点\n新的缩进段落。"}; !reflect.DeepEqual(got, want) {
+		t.Fatalf("texts=%q, want %q", got, want)
+	}
+}
+
+func TestGeneralPDFCrossPageContinuationKeepsParagraphBoundaries(t *testing.T) {
+	for _, tc := range []struct {
+		name, first, second, layout string
+		page, left                  float64
+	}{
+		{"sentence ended", "第一段。", "下一段。", "", 4, 80},
+		{"numbered item", "负责物资保障", "（二）宣传工作组", "", 4, 80},
+		{"title", "负责物资保障", "管理办法", "title", 4, 80},
+		{"new indentation", "负责物资保障", "下一段内容", "", 4, 112},
+		{"same page", "负责物资保障", "下一段内容", "", 3, 80},
+		{"missing page", "负责物资保障", "下一段内容", "", 5, 80},
+		{"blank line", "负责物资保障\n\n", "下一段内容", "", 4, 80},
+		{"spaced blank line", "负责物资保障\n \t\n", "下一段内容", "", 4, 80},
+		{"lettered item", "负责物资保障", "b) 宣传工作组", "", 4, 80},
+		{"lettered item no space", "负责物资保障", "b)宣传工作组", "", 4, 80},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			top := 100.0
+			if tc.page == 3 {
+				top = 740
+			}
+			items := []map[string]any{
+				{"text": tc.first, "doc_type_kwd": "text", "positions": [][]float64{{3, 80, 520, 700, 715}}},
+				{"text": tc.second, "doc_type_kwd": "text", "layout_type": tc.layout, "positions": [][]float64{{tc.page, tc.left, 520, top, top + 15}}},
+			}
+			out := invokePDFContinuation(t, items, 512, []string{"\n"}, nil)
+			blankLine := tc.name == "blank line" || tc.name == "spaced blank line"
+			if got, want := outputTexts(t, out), []string{tc.first + "\n" + tc.second}; !blankLine && !reflect.DeepEqual(got, want) {
+				t.Fatalf("texts=%q, want %q", got, want)
+			}
+			if blankLine {
+				if got := outputTexts(t, out); len(got) != 1 || got[0] != tc.first+tc.second {
+					t.Fatalf("lost paragraph boundary: %q", got)
+				}
+			}
+		})
+	}
+}
+
+func invokePDFContinuation(t *testing.T, items []map[string]any, budget int, delimiters, children []string) map[string]any {
+	t.Helper()
+	c, err := NewGeneralChunker(map[string]any{"chunk_token_size": budget, "delimiters": delimiters, "children_delimiters": children})
+	if err != nil {
+		t.Fatal(err)
+	}
+	out, err := c.Invoke(t.Context(), nil, map[string]any{"name": "continuation.pdf", "file_type": "pdf", "output_format": "json", "json": items})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return out
+}

internal/ingestion/component/chunker/general_pdf_source_positions.go

@@ -61,6 +61,9 @@ func splitPDFUnits(units []schema.ChunkDoc, sources [][]pdfSourceSpan, pattern *
 					cursor = end
 				}
 			}
+			if pdfPositions, positions := pdfSourcePositions(spans); len(pdfPositions) > 0 || len(positions) > 0 {
+				piece.PDFPositions, piece.Positions = pdfPositions, positions
+			}
 			result = append(result, piece)
 			resultSources = append(resultSources, spans)
 		}