fix(chunker): align PDF child previews with source positions (#21853)

internal/ingestion/component/chunker/common.go

@@ -125,21 +125,9 @@ func splitByDelim(text string, pattern *regexp.Regexp, keepDelim bool) []string
 		return []string{text}
 	}
 	if !keepDelim {
-		// Drop mode: keep only the text between delimiters, skipping any
-		// delimiter that sits at the very start (no preceding text).
 		var out []string
-		cursor := 0
-		for _, idx := range idxs {
-			start, end := idx[0], idx[1]
-			if start == cursor {
-				cursor = end
-				continue
-			}
-			out = append(out, text[cursor:start])
-			cursor = end
-		}
-		if cursor < len(text) {
-			out = append(out, text[cursor:])
+		for _, part := range splitDroppedDelimiterRanges(len(text), idxs) {
+			out = append(out, text[part[0]:part[1]])
 		}
 		return out
 	}
@@ -159,6 +147,21 @@ func splitByDelim(text string, pattern *regexp.Regexp, keepDelim bool) []string
 	return out
 }
 
+func splitDroppedDelimiterRanges(length int, matches [][]int) [][2]int {
+	var ranges [][2]int
+	cursor := 0
+	for _, match := range matches {
+		if match[0] > cursor {
+			ranges = append(ranges, [2]int{cursor, match[0]})
+		}
+		cursor = match[1]
+	}
+	if cursor < length {
+		ranges = append(ranges, [2]int{cursor, length})
+	}
+	return ranges
+}
+
 // foldWhitespaceSegments attaches blank runs to neighboring text without
 // changing their position in the reconstructed source.
 func foldWhitespaceSegments(parts []string) []string {

internal/ingestion/component/chunker/general.go

@@ -233,11 +233,19 @@ func (c *GeneralChunkerComponent) chunkPDF(ctx context.Context, db *gorm.DB, ups
 
 	primaryPattern := compileDelimPattern(c.param.Delimiters)
 	childrenPattern := compileDelimPattern(c.param.ChildrenDelimiters)
+	var unitSources [][]pdfSourceSpan
+	if childrenPattern != nil {
+		unitSources = pdfUnitSources(units)
+	}
 	customDelimiter := hasCustomDelim(c.param.Delimiters)
 	if customDelimiter {
-		units = splitGeneralUnits(units, primaryPattern, false)
+		units, unitSources = splitPDFUnits(units, unitSources, primaryPattern, false)
+	}
+	if unitSources != nil {
+		units, unitSources = sortPDFUnitsWithSources(units, unitSources)
+	} else {
+		units = sortPDFUnits(units)
 	}
-	units = sortPDFUnits(units)
 	if hasPDFPositions(units) && hasUnpositionedPDFMedia(units) {
 		common.Warn("GeneralChunker: PDF media is missing position metadata; using degraded context/order fallback")
 	}
@@ -245,6 +253,7 @@ func (c *GeneralChunkerComponent) chunkPDF(ctx context.Context, db *gorm.DB, ups
 
 	chunks := make([]schema.ChunkDoc, 0, len(units))
 	body := make([]schema.ChunkDoc, 0, len(units))
+	var chunkSources, bodySources [][]pdfSourceSpan
 	flushBody := func() {
 		if len(body) == 0 {
 			return
@@ -252,21 +261,30 @@ func (c *GeneralChunkerComponent) chunkPDF(ctx context.Context, db *gorm.DB, ups
 		joinSep := "\n"
 		if !customDelimiter {
 			body = separateGeneralTextBlocks(body)
-			body = splitGeneralUnits(body, primaryPattern, true)
+			body, bodySources = splitPDFUnits(body, bodySources, primaryPattern, true)
 			joinSep = ""
 		}
-		chunks = append(chunks, mergeGeneralUnits(body, c.param.ChunkTokenSize, c.param.OverlappedPercent, joinSep)...)
+		merged, mergedSources := mergeGeneralUnitsWithSources(body, bodySources, c.param.ChunkTokenSize, c.param.OverlappedPercent, joinSep)
+		chunks = append(chunks, merged...)
+		chunkSources = append(chunkSources, mergedSources...)
 		body = body[:0]
+		bodySources = bodySources[:0]
 	}
-	for _, unit := range units {
+	for index, unit := range units {
 		if itemDocType(unit) == "text" {
 			body = append(body, unit)
+			if unitSources != nil {
+				bodySources = append(bodySources, unitSources[index])
+			}
 		} else {
 			flushBody()
 			media := cloneChunkDoc(unit)
 			media.Text = normalizeGeneralNewlines(media.Text)
 			media.DocType = itemDocType(media)
 			chunks = append(chunks, media)
+			if unitSources != nil {
+				chunkSources = append(chunkSources, nil)
+			}
 		}
 	}
 	flushBody()
@@ -276,7 +294,7 @@ func (c *GeneralChunkerComponent) chunkPDF(ctx context.Context, db *gorm.DB, ups
 			chunks[i].Text = removeTag(chunks[i].Text)
 		}
 	}
-	chunks = splitGeneralChildren(chunks, childrenPattern)
+	chunks = splitGeneralChildrenWithSources(chunks, chunkSources, childrenPattern)
 	engine, err := newPDFEngineFromUpstream(ctx, db, upstream)
 	if err != nil {
 		common.Warn("GeneralChunker: could not open PDF for on-demand cropping", zap.Error(err))
@@ -300,15 +318,25 @@ func (c *GeneralChunkerComponent) chunkPDF(ctx context.Context, db *gorm.DB, ups
 // order. Units with explicit parser order or without coordinates are barriers:
 // they stay at their parser positions and preserve their relative order.
 func sortPDFUnits(units []schema.ChunkDoc) []schema.ChunkDoc {
+	ordered, _ := sortPDFUnitsWithSources(units, nil)
+	return ordered
+}
+
+func sortPDFUnitsWithSources(units []schema.ChunkDoc, sources [][]pdfSourceSpan) ([]schema.ChunkDoc, [][]pdfSourceSpan) {
 	type positionedUnit struct {
 		unit       schema.ChunkDoc
 		row        []float64
 		positioned bool
+		sources    []pdfSourceSpan
 	}
 	ordered := make([]positionedUnit, 0, len(units))
-	for _, unit := range units {
+	for index, unit := range units {
 		row, ok := firstPositionRow(lineRecord{pdfPositions: unit.PDFPositions, positions: unit.Positions})
-		ordered = append(ordered, positionedUnit{unit: cloneChunkDoc(unit), row: row, positioned: ok && !unit.PreserveOrder})
+		item := positionedUnit{unit: cloneChunkDoc(unit), row: row, positioned: ok && !unit.PreserveOrder}
+		if sources != nil {
+			item.sources = sources[index]
+		}
+		ordered = append(ordered, item)
 	}
 	runStart := -1
 	flush := func(end int) {
@@ -331,10 +359,14 @@ func sortPDFUnits(units []schema.ChunkDoc) []schema.ChunkDoc {
 		flush(i)
 	}
 	result := make([]schema.ChunkDoc, 0, len(ordered))
+	var orderedSources [][]pdfSourceSpan
 	for _, item := range ordered {
 		result = append(result, item.unit)
+		if sources != nil {
+			orderedSources = append(orderedSources, item.sources)
+		}
 	}
-	return result
+	return result, orderedSources
 }
 
 func hasPDFPositions(units []schema.ChunkDoc) bool {
@@ -1181,14 +1213,24 @@ func (c *GeneralChunkerComponent) chunkGeneral(ctx context.Context, upstream sch
 // appended while the current chunk is at or below the overlap-scaled target;
 // the append itself may cross the target. Oversized units remain indivisible.
 func mergeGeneralUnits(units []schema.ChunkDoc, target int, overlapPct float64, joinSep string) []schema.ChunkDoc {
+	merged, _ := mergeGeneralUnitsWithSources(units, nil, target, overlapPct, joinSep)
+	return merged
+}
+
+func mergeGeneralUnitsWithSources(units []schema.ChunkDoc, sources [][]pdfSourceSpan, target int, overlapPct float64, joinSep string) ([]schema.ChunkDoc, [][]pdfSourceSpan) {
 	overlapPct = max(0, min(100, overlapPct))
 	threshold := float64(target) * (100 - overlapPct) / 100
 	merged := make([]schema.ChunkDoc, 0, len(units))
+	var mergedSources [][]pdfSourceSpan
+	tracked := sources != nil
 	current := -1
 
-	for _, unit := range units {
+	for index, unit := range units {
 		if itemDocType(unit) != "text" {
 			merged = append(merged, cloneChunkDoc(unit))
+			if tracked {
+				mergedSources = append(mergedSources, sources[index])
+			}
 			current = -1
 			continue
 		}
@@ -1202,19 +1244,40 @@ func mergeGeneralUnits(units []schema.ChunkDoc, target int, overlapPct float64,
 		unit.TKNums = intPtr(count)
 		if count > target {
 			merged = append(merged, cloneChunkDoc(unit))
+			if tracked {
+				mergedSources = append(mergedSources, sources[index])
+			}
 			current = -1
 			continue
 		}
 		if current < 0 || float64(intValue(merged[current].TKNums)) > threshold {
 			merged = append(merged, cloneChunkDoc(unit))
+			if tracked {
+				mergedSources = append(mergedSources, sources[index])
+			}
 			current = len(merged) - 1
 			continue
 		}
 
+		if tracked {
+			if mergedSources[current] == nil || sources[index] == nil {
+				mergedSources[current] = nil
+			} else {
+				offset := len(removeTag(merged[current].Text))
+				if merged[current].Text != "" && unit.Text != "" {
+					offset += len(removeTag(joinSep))
+				}
+				for _, span := range sources[index] {
+					span.start += offset
+					span.end += offset
+					mergedSources[current] = append(mergedSources[current], span)
+				}
+			}
+		}
 		mergeGeneralChunk(&merged[current], unit, joinSep)
 	}
 
-	return applyGeneralOverlap(merged, overlapPct, joinSep)
+	return applyGeneralOverlapWithSources(merged, mergedSources, overlapPct, joinSep)
 }
 
 func sameSpreadsheetTable(first, second schema.ChunkDoc) bool {
@@ -1300,8 +1363,13 @@ func mergeGeneralPositions(first, second json.RawMessage) json.RawMessage {
 }
 
 func applyGeneralOverlap(chunks []schema.ChunkDoc, overlapPct float64, joinSep string) []schema.ChunkDoc {
+	result, _ := applyGeneralOverlapWithSources(chunks, nil, overlapPct, joinSep)
+	return result
+}
+
+func applyGeneralOverlapWithSources(chunks []schema.ChunkDoc, sources [][]pdfSourceSpan, overlapPct float64, joinSep string) ([]schema.ChunkDoc, [][]pdfSourceSpan) {
 	if overlapPct <= 0 {
-		return chunks
+		return chunks, sources
 	}
 	previousText := -1
 	for i := range chunks {
@@ -1313,6 +1381,24 @@ func applyGeneralOverlap(chunks []schema.ChunkDoc, overlapPct float64, joinSep s
 			prefix, _ := computeOverlapPrefix(chunks[previousText].Text, overlapPct)
 			if prefix != "" {
 				current := cloneChunkDoc(chunks[i])
+				if sources != nil {
+					previous := chunks[previousText]
+					previousVisible := removeTag(previous.Text)
+					if sources[previousText] == nil || sources[i] == nil || !strings.HasSuffix(previousVisible, prefix) {
+						sources[i] = nil
+					} else {
+						start := len(previousVisible) - len(prefix)
+						spans := slicePDFSourceSpans(sources[previousText], previousVisible, start, len(previousVisible))
+						joined := joinGeneralOverlapText(prefix, current.Text, joinSep)
+						offset := len(removeTag(joined)) - len(removeTag(current.Text))
+						for _, span := range sources[i] {
+							span.start += offset
+							span.end += offset
+							spans = append(spans, span)
+						}
+						sources[i] = spans
+					}
+				}
 				current.Text = joinGeneralOverlapText(prefix, current.Text, joinSep)
 				current.TKNums = intPtr(tokenizeStr(current.Text))
 				chunks[i] = current
@@ -1320,7 +1406,7 @@ func applyGeneralOverlap(chunks []schema.ChunkDoc, overlapPct float64, joinSep s
 		}
 		previousText = i
 	}
-	return chunks
+	return chunks, sources
 }
 
 func joinGeneralOverlapText(prefix, text, separator string) string {
@@ -1441,21 +1527,43 @@ func normalizeGeneralNewlines(text string) string {
 // (@@...##) carry no visual height and are excluded from the ratio, mirroring
 // the Token path (chunkFromItem/splitOversizedText).
 func sliceGeneralSplitPieces(unit schema.ChunkDoc, parts []string) []schema.ChunkDoc {
+	return sliceGeneralSplitPiecesWithSources(unit, parts, nil)
+}
+
+func sliceGeneralSplitPiecesWithSources(unit schema.ChunkDoc, parts []string, sources []pdfSourceSpan) []schema.ChunkDoc {
 	out := make([]schema.ChunkDoc, 0, len(parts))
 	if len(parts) == 0 {
 		return out
 	}
-	counts := make([]int, len(parts))
-	for i, part := range parts {
-		counts[i] = utf8.RuneCountInString(removeTag(part))
-	}
-	pdfSliced, posSliced := slicePositionsByCounts(unit.PDFPositions, unit.Positions, counts)
+	var pdfSliced, posSliced []json.RawMessage
+	cursor := 0
 	for i, part := range parts {
 		piece := cloneChunkDoc(unit)
 		piece.Text = part
 		piece.TKNums = intPtr(tokenizeStr(part))
-		piece.PDFPositions = pdfSliced[i]
-		piece.Positions = posSliced[i]
+		positioned := false
+		if sources != nil {
+			if offset := strings.Index(unit.Text[cursor:], part); offset >= 0 {
+				start := cursor + offset
+				cursor = start + len(part)
+				spans := slicePDFSourceSpans(sources, unit.Text, start, cursor)
+				pdfPositions, positions := pdfSourcePositions(spans)
+				if len(pdfPositions) > 0 || len(positions) > 0 {
+					piece.PDFPositions, piece.Positions = pdfPositions, positions
+					positioned = true
+				}
+			}
+		}
+		if !positioned {
+			if pdfSliced == nil {
+				counts := make([]int, len(parts))
+				for index, text := range parts {
+					counts[index] = utf8.RuneCountInString(removeTag(text))
+				}
+				pdfSliced, posSliced = slicePositionsByCounts(unit.PDFPositions, unit.Positions, counts)
+			}
+			piece.PDFPositions, piece.Positions = pdfSliced[i], posSliced[i]
+		}
 		out = append(out, piece)
 	}
 	return out
@@ -1491,11 +1599,15 @@ func finalizeGeneralChunks(chunks []schema.ChunkDoc, childrenPattern *regexp.Reg
 // TokenChunker splitByChildren helper intentionally drops delimiters and is
 // therefore not reused here.
 func splitGeneralChildren(chunks []schema.ChunkDoc, pattern *regexp.Regexp) []schema.ChunkDoc {
+	return splitGeneralChildrenWithSources(chunks, nil, pattern)
+}
+
+func splitGeneralChildrenWithSources(chunks []schema.ChunkDoc, sources [][]pdfSourceSpan, pattern *regexp.Regexp) []schema.ChunkDoc {
 	if pattern == nil {
 		return chunks
 	}
 	result := make([]schema.ChunkDoc, 0, len(chunks))
-	for _, chunk := range chunks {
+	for index, chunk := range chunks {
 		if itemDocType(chunk) != "text" {
 			result = append(result, chunk)
 			continue
@@ -1513,7 +1625,11 @@ func splitGeneralChildren(chunks []schema.ChunkDoc, pattern *regexp.Regexp) []sc
 			result = append(result, chunk)
 			continue
 		}
-		for _, piece := range sliceGeneralSplitPieces(chunk, parts) {
+		var spans []pdfSourceSpan
+		if sources != nil {
+			spans = sources[index]
+		}
+		for _, piece := range sliceGeneralSplitPiecesWithSources(chunk, parts, spans) {
 			piece.Mom = mom
 			result = append(result, piece)
 		}

internal/ingestion/component/chunker/general_pdf_source_positions.go

@@ -0,0 +1,122 @@
+package chunker
+
+import (
+	"encoding/json"
+	"regexp"
+	"strings"
+	"unicode"
+	"unicode/utf8"
+
+	"ragflow/internal/ingestion/component/schema"
+)
+
+type pdfSourceSpan struct {
+	start, end              int
+	pdfPositions, positions json.RawMessage
+}
+
+func pdfUnitSources(units []schema.ChunkDoc) [][]pdfSourceSpan {
+	sources := make([][]pdfSourceSpan, len(units))
+	for index, unit := range units {
+		text := removeTag(normalizeGeneralNewlines(unit.Text))
+		start := len(text) - len(strings.TrimLeftFunc(text, unicode.IsSpace))
+		sources[index] = []pdfSourceSpan{{start: start, end: start + len(strings.TrimSpace(text)), pdfPositions: unit.PDFPositions, positions: unit.Positions}}
+	}
+	return sources
+}
+
+func splitPDFUnits(units []schema.ChunkDoc, sources [][]pdfSourceSpan, pattern *regexp.Regexp, keepDelimiter bool) ([]schema.ChunkDoc, [][]pdfSourceSpan) {
+	if sources == nil {
+		return splitGeneralUnits(units, pattern, keepDelimiter), nil
+	}
+	var result []schema.ChunkDoc
+	var resultSources [][]pdfSourceSpan
+	for index, unit := range units {
+		pieces := splitGeneralUnits([]schema.ChunkDoc{unit}, pattern, keepDelimiter)
+		rawText := normalizeGeneralNewlines(unit.Text)
+		text := removeTag(rawText)
+		var ranges [][2]int
+		if !keepDelimiter && pattern != nil && itemDocType(unit) == "text" && pattern.MatchString(rawText) {
+			for _, part := range splitDroppedDelimiterRanges(len(rawText), pattern.FindAllStringIndex(rawText, -1)) {
+				if strings.TrimSpace(rawText[part[0]:part[1]]) != "" {
+					ranges = append(ranges, part)
+				}
+			}
+		}
+		cursor := 0
+		for pieceIndex, piece := range pieces {
+			pieceText := removeTag(piece.Text)
+			var spans []pdfSourceSpan
+			if ranges != nil && len(ranges) == len(pieces) && sources[index] != nil {
+				part := ranges[pieceIndex]
+				prefix, selected, suffix := removeTag(rawText[:part[0]]), removeTag(rawText[part[0]:part[1]]), removeTag(rawText[part[1]:])
+				if selected == pieceText && prefix+selected+suffix == text {
+					spans = slicePDFSourceSpans(sources[index], text, len(prefix), len(prefix)+len(selected))
+				}
+			} else if ranges == nil {
+				if offset := strings.Index(text[cursor:], pieceText); offset >= 0 && sources[index] != nil {
+					start := cursor + offset
+					end := start + len(pieceText)
+					spans = slicePDFSourceSpans(sources[index], text, start, end)
+					cursor = end
+				}
+			}
+			result = append(result, piece)
+			resultSources = append(resultSources, spans)
+		}
+	}
+	return result, resultSources
+}
+
+func slicePDFSourceSpans(spans []pdfSourceSpan, text string, start, end int) []pdfSourceSpan {
+	result := make([]pdfSourceSpan, 0, len(spans))
+	for _, span := range spans {
+		if span.start < 0 || span.end > len(text) || span.start >= span.end {
+			continue
+		}
+		left, right := max(start, span.start), min(end, span.end)
+		if left >= right {
+			continue
+		}
+		if left != span.start || right != span.end {
+			total := utf8.RuneCountInString(text[span.start:span.end])
+			if total > 0 {
+				begin := float64(utf8.RuneCountInString(text[span.start:left])) / float64(total)
+				finish := float64(utf8.RuneCountInString(text[span.start:right])) / float64(total)
+				if sliced := slicePositionsByTextRatio(span.pdfPositions, begin, finish); len(sliced) > 0 {
+					span.pdfPositions = sliced
+				}
+				if sliced := slicePositionsByTextRatio(span.positions, begin, finish); len(sliced) > 0 {
+					span.positions = sliced
+				}
+			}
+		}
+		span.start = left - start
+		span.end = right - start
+		result = append(result, span)
+	}
+	return result
+}
+
+func pdfSourcePositions(spans []pdfSourceSpan) (json.RawMessage, json.RawMessage) {
+	pdf := make([]json.RawMessage, 0, len(spans))
+	positions := make([]json.RawMessage, 0, len(spans))
+	for _, span := range spans {
+		pdf = append(pdf, span.pdfPositions)
+		positions = append(positions, span.positions)
+	}
+	return mergePDFSourcePositions(pdf), mergePDFSourcePositions(positions)
+}
+
+func mergePDFSourcePositions(sources []json.RawMessage) json.RawMessage {
+	if rows := mergePositionMatrix(sources...); rows != nil {
+		if encoded, err := json.Marshal(rows); err == nil {
+			return encoded
+		}
+	}
+	var positions json.RawMessage
+	for _, source := range sources {
+		positions = extendRawJSONArray(positions, source)
+	}
+	return positions
+}

internal/ingestion/component/chunker/general_pdf_source_positions_test.go

@@ -0,0 +1,280 @@
+package chunker
+
+import (
+	"encoding/json"
+	"reflect"
+	"testing"
+
+	"ragflow/internal/ingestion/component/schema"
+)
+
+func TestGeneralPDFChildPositionsFollowSourceText(t *testing.T) {
+	chunker, err := NewGeneralChunker(map[string]any{
+		"chunk_token_size":    512,
+		"delimiters":          []string{"#"},
+		"children_delimiters": []string{"#"},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	items := []map[string]any{
+		{"text": "上一段的长长长长长长长长长长长长长长长长长长长长长长长长长长长句。", "doc_type_kwd": "text", "positions": [][]float64{{1, 10, 200, 10, 40}}},
+		{"text": "#", "doc_type_kwd": "text", "positions": [][]float64{{1, 10, 20, 45, 55}}},
+		{"text": "本段第一句。", "doc_type_kwd": "text", "positions": [][]float64{{1, 10, 200, 60, 75}}},
+		{"text": "本段第二句。", "doc_type_kwd": "text", "positions": [][]float64{{1, 10, 200, 80, 95}}},
+	}
+	out, err := chunker.Invoke(t.Context(), nil, map[string]any{"name": "source.pdf", "file_type": "pdf", "output_format": "json", "json": items})
+	if err != nil {
+		t.Fatal(err)
+	}
+	chunks := outputChunks(t, out)
+	if len(chunks) != 2 {
+		t.Fatalf("got %d chunks, want 2: %#v", len(chunks), chunks)
+	}
+	for index, want := range [][][]float64{
+		{{1, 10, 200, 10, 40}, {1, 10, 20, 45, 55}},
+		{{1, 10, 200, 60, 75}, {1, 10, 200, 80, 95}},
+	} {
+		raw, err := json.Marshal(chunks[index]["positions"])
+		if err != nil {
+			t.Fatal(err)
+		}
+		var got [][]float64
+		if err := json.Unmarshal(raw, &got); err != nil {
+			t.Fatal(err)
+		}
+		if !reflect.DeepEqual(got, want) {
+			t.Errorf("chunk %d positions = %v, want %v", index, got, want)
+		}
+	}
+}
+
+func TestGeneralPDFOverlapChildPositionsIncludeSourceTail(t *testing.T) {
+	firstPosition := json.RawMessage(`[[1,10,20,0,100]]`)
+	secondPosition := json.RawMessage(`[[1,10,20,200,300]]`)
+	chunks := []schema.ChunkDoc{
+		{Text: "ABCDE", DocType: "text", Positions: firstPosition},
+		{Text: "FGHIJ", DocType: "text", Positions: secondPosition},
+	}
+	sources := [][]pdfSourceSpan{{{start: 0, end: 5, positions: firstPosition}}, {{start: 0, end: 5, positions: secondPosition}}}
+	chunks, sources = applyGeneralOverlapWithSources(chunks, sources, 40, "\n")
+	children := splitGeneralChildrenWithSources(chunks, sources, compileDelimPattern([]string{"#"}))
+	if len(children) != 2 || children[1].Text != "DE\nFGHIJ" {
+		t.Fatalf("unexpected overlapped children: %#v", children)
+	}
+	got := decodeGeneralMatrix(t, children[1].Positions)
+	want := [][]float64{{1, 10, 20, 60, 100}, {1, 10, 20, 200, 300}}
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("overlap positions = %v, want %v", got, want)
+	}
+}
+
+func TestGeneralPDFChildrenWithinOneSourceUseProportionalPositions(t *testing.T) {
+	chunker, err := NewGeneralChunker(map[string]any{
+		"chunk_token_size":    512,
+		"delimiters":          []string{"#"},
+		"children_delimiters": []string{"#"},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	out, err := chunker.Invoke(t.Context(), nil, map[string]any{
+		"name": "source.pdf", "file_type": "pdf", "output_format": "json",
+		"json": []map[string]any{{"text": "one#two", "doc_type_kwd": "text", "positions": [][]float64{{1, 10, 20, 0, 70}}}},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	chunks := outputChunks(t, out)
+	if len(chunks) != 2 {
+		t.Fatalf("got %d chunks, want 2", len(chunks))
+	}
+	for index, want := range [][][]float64{{{1, 10, 20, 0, 40}}, {{1, 10, 20, 40, 70}}} {
+		raw, err := json.Marshal(chunks[index]["positions"])
+		if err != nil {
+			t.Fatal(err)
+		}
+		var got [][]float64
+		if err := json.Unmarshal(raw, &got); err != nil {
+			t.Fatal(err)
+		}
+		if !reflect.DeepEqual(got, want) {
+			t.Errorf("chunk %d positions = %v, want %v", index, got, want)
+		}
+	}
+}
+
+func TestGeneralPDFChildPositionsFollowPageOrder(t *testing.T) {
+	chunker, err := NewGeneralChunker(map[string]any{"chunk_token_size": 512, "delimiters": []string{"#"}, "children_delimiters": []string{"#"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	items := []map[string]any{
+		{"text": "second page", "doc_type_kwd": "text", "positions": [][]float64{{2, 10, 100, 10, 20}}},
+		{"text": "first page", "doc_type_kwd": "text", "positions": [][]float64{{1, 10, 100, 10, 20}}},
+		{"text": "#", "doc_type_kwd": "text", "positions": [][]float64{{1, 10, 20, 25, 35}}},
+	}
+	out, err := chunker.Invoke(t.Context(), nil, map[string]any{"name": "source.pdf", "file_type": "pdf", "output_format": "json", "json": items})
+	if err != nil {
+		t.Fatal(err)
+	}
+	chunks := outputChunks(t, out)
+	if len(chunks) != 2 {
+		t.Fatalf("got %d chunks, want 2: %#v", len(chunks), chunks)
+	}
+	for index, want := range [][][]float64{{{1, 10, 100, 10, 20}, {1, 10, 20, 25, 35}}, {{2, 10, 100, 10, 20}}} {
+		raw, err := json.Marshal(chunks[index]["positions"])
+		if err != nil {
+			t.Fatal(err)
+		}
+		var got [][]float64
+		if err := json.Unmarshal(raw, &got); err != nil {
+			t.Fatal(err)
+		}
+		if !reflect.DeepEqual(got, want) {
+			t.Errorf("chunk %d positions = %v, want %v", index, got, want)
+		}
+	}
+}
+
+func TestPDFSourceOffsetsSkipCustomDelimiter(t *testing.T) {
+	for _, testCase := range []struct {
+		name       string
+		text       string
+		wantTexts  []string
+		wantBounds [][2]float64
+	}{
+		{"delimiter substring", "a###", []string{"a", "#"}, [][2]float64{{0, 25}, {75, 100}}},
+		{"leading and repeated", "##a####b", []string{"a", "b"}, [][2]float64{{25, 37.5}, {87.5, 100}}},
+	} {
+		t.Run(testCase.name, func(t *testing.T) {
+			unit := schema.ChunkDoc{Text: testCase.text, DocType: "text", Positions: json.RawMessage(`[[1,10,200,0,100]]`)}
+			pieces, sources := splitPDFUnits([]schema.ChunkDoc{unit}, pdfUnitSources([]schema.ChunkDoc{unit}), compileDelimPattern([]string{"`##`"}), false)
+			if len(pieces) != len(testCase.wantTexts) {
+				t.Fatalf("pieces = %v, want %v", generalChunkTexts(pieces), testCase.wantTexts)
+			}
+			for index, piece := range pieces {
+				if piece.Text != testCase.wantTexts[index] {
+					t.Errorf("piece %d text = %q, want %q", index, piece.Text, testCase.wantTexts[index])
+				}
+				_, raw := pdfSourcePositions(sources[index])
+				positions := decodeGeneralMatrix(t, raw)
+				if len(positions) != 1 || positions[0][3] != testCase.wantBounds[index][0] || positions[0][4] != testCase.wantBounds[index][1] {
+					t.Errorf("piece %d positions = %v, want bounds %v", index, positions, testCase.wantBounds[index])
+				}
+			}
+		})
+	}
+}
+
+func TestGeneralPDFCustomDelimiterPreservesSourceBounds(t *testing.T) {
+	chunker, err := NewGeneralChunker(map[string]any{
+		"chunk_token_size": 512, "delimiters": []string{"`##`"}, "children_delimiters": []string{"。"},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	out, err := chunker.Invoke(t.Context(), nil, map[string]any{
+		"name": "source.pdf", "file_type": "pdf", "output_format": "json",
+		"json": []map[string]any{{"text": "a###", "doc_type_kwd": "text", "positions": [][]float64{{1, 10, 200, 0, 100}}}},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	chunks := outputChunks(t, out)
+	if len(chunks) != 1 || chunks[0]["text"] != "a\n#" {
+		t.Fatalf("unexpected chunks: %#v", chunks)
+	}
+	raw, err := json.Marshal(chunks[0]["positions"])
+	if err != nil {
+		t.Fatal(err)
+	}
+	var got [][]float64
+	if err := json.Unmarshal(raw, &got); err != nil {
+		t.Fatal(err)
+	}
+	want := [][]float64{{1, 10, 200, 0, 25}, {1, 10, 200, 75, 100}}
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("positions = %v, want %v", got, want)
+	}
+}
+
+func TestPDFSourceOffsetsIgnorePositionTags(t *testing.T) {
+	unit := schema.ChunkDoc{
+		Text: "a@@1\t2\t3\t4\t5##xyyb", DocType: "text",
+		Positions: json.RawMessage(`[[1,10,200,0,100]]`),
+	}
+	pieces, sources := splitPDFUnits([]schema.ChunkDoc{unit}, pdfUnitSources([]schema.ChunkDoc{unit}), compileDelimPattern([]string{"`xy`"}), false)
+	if len(pieces) != 2 || removeTag(pieces[1].Text) != "yb" {
+		t.Fatalf("unexpected pieces: %v", generalChunkTexts(pieces))
+	}
+	_, raw := pdfSourcePositions(sources[1])
+	got := decodeGeneralMatrix(t, raw)
+	want := [][]float64{{1, 10, 200, 60, 100}}
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("positions = %v, want %v", got, want)
+	}
+}
+
+func TestPDFSourceOffsetsWithWhitespaceDelimiters(t *testing.T) {
+	for _, testCase := range []struct {
+		name      string
+		text      string
+		delimiter string
+		bounds    [][2]float64
+	}{
+		{"repeated spaces", "a  b", "` `", [][2]float64{{0, 25}, {75, 100}}},
+		{"repeated tabs", "a\t\tb", "`\t`", [][2]float64{{0, 25}, {75, 100}}},
+		{"spaces in delimiter", "a / b", "` / `", [][2]float64{{0, 20}, {80, 100}}},
+		{"position tag", "a@@1\t2\t3\t4\t5##  b", "` `", [][2]float64{{0, 25}, {75, 100}}},
+	} {
+		t.Run(testCase.name, func(t *testing.T) {
+			unit := schema.ChunkDoc{Text: testCase.text, DocType: "text", Positions: json.RawMessage(`[[1,10,200,0,100]]`)}
+			pieces, sources := splitPDFUnits([]schema.ChunkDoc{unit}, pdfUnitSources([]schema.ChunkDoc{unit}), compileDelimPattern([]string{testCase.delimiter}), false)
+			if len(pieces) != 2 || removeTag(pieces[0].Text) != "a" || removeTag(pieces[1].Text) != "b" {
+				t.Fatalf("unexpected pieces: %v", generalChunkTexts(pieces))
+			}
+			for index, want := range testCase.bounds {
+				_, raw := pdfSourcePositions(sources[index])
+				positions := decodeGeneralMatrix(t, raw)
+				if len(positions) != 1 || positions[0][3] != want[0] || positions[0][4] != want[1] {
+					t.Errorf("piece %d positions = %v, want bounds %v", index, positions, want)
+				}
+			}
+		})
+	}
+}
+
+func TestGeneralPDFWrappedChildExcludesPreviousParagraph(t *testing.T) {
+	chunker, err := NewGeneralChunker(map[string]any{
+		"chunk_token_size": 512, "delimiters": []string{"#"}, "children_delimiters": []string{"#"},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	items := []map[string]any{
+		{"text": "上一段很长，不能算进下一段。", "doc_type_kwd": "text", "_pdf_positions": [][]float64{{1, 10, 200, 10, 40}}},
+		{"text": "#", "doc_type_kwd": "text", "_pdf_positions": [][]float64{{1, 10, 20, 45, 55}}},
+		{"text": "将军向宠，性行淑均，\n晓畅军事。\n愚以为营中之事。", "doc_type_kwd": "text", "_pdf_positions": [][]float64{{1, 10, 200, 60, 100}}},
+	}
+	out, err := chunker.Invoke(t.Context(), nil, map[string]any{"name": "source.pdf", "file_type": "pdf", "output_format": "json", "json": items})
+	if err != nil {
+		t.Fatal(err)
+	}
+	chunks := outputChunks(t, out)
+	if len(chunks) != 2 {
+		t.Fatalf("got %d chunks, want 2", len(chunks))
+	}
+	raw, err := json.Marshal(chunks[1]["_pdf_positions"])
+	if err != nil {
+		t.Fatal(err)
+	}
+	var got [][]float64
+	if err := json.Unmarshal(raw, &got); err != nil {
+		t.Fatal(err)
+	}
+	want := [][]float64{{1, 10, 200, 60, 100}}
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("wrapped child positions = %v, want %v", got, want)
+	}
+}