fix(chunker): preserve PDF and Markdown sentence delimiters (#20812)

internal/ingestion/component/chunker/general.go

@@ -231,12 +231,15 @@ func (c *GeneralChunkerComponent) chunkPDF(ctx context.Context, db *gorm.DB, ups
 
 	primaryPattern := compileDelimPattern(c.param.Delimiters)
 	childrenPattern := compileChildrenPattern(c.param.ChildrenDelimiters)
-	units = splitGeneralUnits(units, primaryPattern, false)
+	customDelimiter := hasCustomDelim(c.param.Delimiters)
+	if customDelimiter {
+		units = splitGeneralUnits(units, primaryPattern, false)
+	}
 	units = sortPDFUnits(units)
 	if hasPDFPositions(units) && hasUnpositionedPDFMedia(units) {
 		common.Warn("GeneralChunker: PDF media is missing position metadata; using degraded context/order fallback")
 	}
-	attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize)
+	attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize, primaryPattern)
 
 	chunks := make([]schema.ChunkDoc, 0, len(units))
 	body := make([]schema.ChunkDoc, 0, len(units))
@@ -244,7 +247,13 @@ func (c *GeneralChunkerComponent) chunkPDF(ctx context.Context, db *gorm.DB, ups
 		if len(body) == 0 {
 			return
 		}
-		chunks = append(chunks, mergeGeneralUnits(body, c.param.ChunkTokenSize, c.param.OverlappedPercent, "\n")...)
+		joinSep := "\n"
+		if !customDelimiter {
+			body = separateGeneralTextBlocks(body)
+			body = splitGeneralUnits(body, primaryPattern, true)
+			joinSep = ""
+		}
+		chunks = append(chunks, mergeGeneralUnits(body, c.param.ChunkTokenSize, c.param.OverlappedPercent, joinSep)...)
 		body = body[:0]
 	}
 	for _, unit := range units {
@@ -252,7 +261,11 @@ func (c *GeneralChunkerComponent) chunkPDF(ctx context.Context, db *gorm.DB, ups
 			body = append(body, unit)
 		} else {
 			flushBody()
-			chunks = append(chunks, cloneChunkDoc(unit))
+			media := cloneChunkDoc(unit)
+			media.Text = normalizeGeneralNewlines(media.Text)
+			media.DocType = itemDocType(media)
+			media.CKType = media.DocType
+			chunks = append(chunks, media)
 		}
 	}
 	flushBody()
@@ -391,12 +404,12 @@ func (c *GeneralChunkerComponent) chunkDOCX(ctx context.Context, upstream schema
 	childrenPattern := compileChildrenPattern(c.param.ChildrenDelimiters)
 	customDelimiter := hasCustomDelim(c.param.Delimiters)
 	if !customDelimiter {
-		attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize)
-		units = separateDOCXParagraphs(units)
+		attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize, primaryPattern)
+		units = separateGeneralTextBlocks(units)
 	}
 	units = splitGeneralUnits(units, primaryPattern, !customDelimiter)
 	if customDelimiter {
-		attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize)
+		attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize, primaryPattern)
 	} else {
 		units = mergeDOCXUnits(units, c.param.ChunkTokenSize)
 	}
@@ -408,9 +421,9 @@ func (c *GeneralChunkerComponent) chunkDOCX(ctx context.Context, upstream schema
 	return chunkOutputs(units), nil
 }
 
-// separateDOCXParagraphs retains parser paragraph boundaries in the text before
+// separateGeneralTextBlocks retains parser block boundaries in the text before
 // delimiter splitting. Sentence fragments can then be concatenated directly.
-func separateDOCXParagraphs(units []schema.ChunkDoc) []schema.ChunkDoc {
+func separateGeneralTextBlocks(units []schema.ChunkDoc) []schema.ChunkDoc {
 	units = append([]schema.ChunkDoc(nil), units...)
 	previousText := -1
 	for i := range units {
@@ -456,11 +469,16 @@ func mergeDOCXUnits(units []schema.ChunkDoc, target int) []schema.ChunkDoc {
 	return merged
 }
 
-// attachGeneralMediaContext follows Python's sentence-aware media context
-// extraction. The shared TokenChunker helper intentionally returns the
-// smallest token-fitting rune suffix/prefix; DOCX/PDF general strategies use
-// complete sentence units instead.
-func attachGeneralMediaContext(units []schema.ChunkDoc, tableTokens, imageTokens int) {
+// attachGeneralMediaContext selects complete configured delimiter units or
+// sentences from the neighboring source blocks.
+func attachGeneralMediaContext(units []schema.ChunkDoc, tableTokens, imageTokens int, primaryPattern *regexp.Regexp) {
+	if tableTokens <= 0 && imageTokens <= 0 {
+		return
+	}
+	pattern := contextSentencePattern
+	if primaryPattern != nil {
+		pattern = regexp.MustCompile("(?:" + primaryPattern.String() + ")|(?:" + pattern.String() + ")")
+	}
 	for i := range units {
 		docType := itemDocType(units[i])
 		if docType != "table" && docType != "image" {
@@ -473,12 +491,12 @@ func attachGeneralMediaContext(units []schema.ChunkDoc, tableTokens, imageTokens
 		if budget <= 0 {
 			continue
 		}
-		units[i].ContextAbove = collectGeneralMediaContext(units, i, budget, true)
-		units[i].ContextBelow = collectGeneralMediaContext(units, i, budget, false)
+		units[i].ContextAbove = collectGeneralMediaContext(units, i, budget, true, pattern)
+		units[i].ContextBelow = collectGeneralMediaContext(units, i, budget, false, pattern)
 	}
 }
 
-func collectGeneralMediaContext(units []schema.ChunkDoc, index, budget int, above bool) string {
+func collectGeneralMediaContext(units []schema.ChunkDoc, index, budget int, above bool, pattern *regexp.Regexp) string {
 	var parts []string
 	remaining := budget
 	step := -1
@@ -496,7 +514,7 @@ func collectGeneralMediaContext(units []schema.ChunkDoc, index, budget int, abov
 		text = normalizeGeneralNewlines(text)
 		tokens := tokenizeStr(text)
 		if tokens >= remaining {
-			piece := takeContextSentences(text, remaining, above)
+			piece := takeContextSentences(text, remaining, above, pattern)
 			if above {
 				parts = append([]string{piece}, parts...)
 			} else {
@@ -568,13 +586,10 @@ func hasSpreadsheetIdentity(doc schema.ChunkDoc) bool {
 // combined token count reaches the budget, taking the last sentence even when
 // it overshoots: a sentence is the smallest unit a context window may cut at,
 // so a text without a sentence boundary comes back whole. Shared by the media
-// context collectors of both chunkers (general.go, token.go).
-func takeContextSentences(text string, budget int, fromEnd bool) string {
-	sentences := splitContextSentences(text)
-	if len(sentences) == 0 {
-		return text
-	}
-	// The sentence pieces are contiguous slices of text. Track byte offsets
+// context collectors of both chunkers use the supplied boundary pattern.
+func takeContextSentences(text string, budget int, fromEnd bool, pattern *regexp.Regexp) string {
+	sentences := splitByDelim(text, pattern, true)
+	// The sentence pieces cover contiguous source segments. Track byte offsets
 	// instead of repeatedly prepending slices and joining the selected text;
 	// each candidate is still tokenized independently because BPE token counts
 	// are not additive across sentence boundaries.
@@ -599,26 +614,6 @@ func takeContextSentences(text string, budget int, fromEnd bool) string {
 	return text[:end]
 }
 
-func splitContextSentences(text string) []string {
-	indices := contextSentencePattern.FindAllStringIndex(text, -1)
-	if len(indices) == 0 {
-		if text == "" {
-			return nil
-		}
-		return []string{text}
-	}
-	parts := make([]string, 0, len(indices)+1)
-	start := 0
-	for _, index := range indices {
-		parts = append(parts, text[start:index[1]])
-		start = index[1]
-	}
-	if start < len(text) {
-		parts = append(parts, text[start:])
-	}
-	return parts
-}
-
 func (c *GeneralChunkerComponent) chunkMarkdown(ctx context.Context, upstream schema.ChunkerFromUpstream) (map[string]any, error) {
 	if err := ctx.Err(); err != nil {
 		return nil, fmt.Errorf("GeneralChunker: %w", err)
@@ -632,9 +627,15 @@ func (c *GeneralChunkerComponent) chunkMarkdown(ctx context.Context, upstream sc
 	}
 	primaryPattern := compileDelimPattern(c.param.Delimiters)
 	childrenPattern := compileChildrenPattern(c.param.ChildrenDelimiters)
-	units = splitMarkdownUnits(units, primaryPattern)
-	attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize)
-	units = mergeMarkdownUnits(units, c.param.ChunkTokenSize, c.param.OverlappedPercent, "\n")
+	customDelimiter := hasCustomDelim(c.param.Delimiters)
+	if !customDelimiter {
+		attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize, primaryPattern)
+	}
+	parts := splitMarkdownUnits(units, primaryPattern, !customDelimiter)
+	if customDelimiter {
+		attachGeneralMediaContext(parts.chunks, c.param.TableContextSize, c.param.ImageContextSize, primaryPattern)
+	}
+	units = mergeMarkdownUnits(parts, c.param.ChunkTokenSize, c.param.OverlappedPercent)
 	units = finalizeGeneralChunks(units, childrenPattern)
 	if len(units) == 0 {
 		return emptyOutputs(), nil
@@ -642,44 +643,74 @@ func (c *GeneralChunkerComponent) chunkMarkdown(ctx context.Context, upstream sc
 	return chunkOutputs(units), nil
 }
 
+// markdownSplit keeps each fragment's parser block boundary beside its chunk.
+type markdownSplit struct {
+	chunks        []schema.ChunkDoc
+	blockStarts   []bool
+	shortHeadings []bool // Classified from the complete source heading.
+}
+
 // splitMarkdownUnits expands only text units while retaining the semantic
 // parser type on every piece. Markdown heading detection depends on ck_type,
 // so the generic splitter's type normalization cannot be used here.
-func splitMarkdownUnits(units []schema.ChunkDoc, pattern *regexp.Regexp) []schema.ChunkDoc {
-	result := make([]schema.ChunkDoc, 0, len(units))
+func splitMarkdownUnits(units []schema.ChunkDoc, pattern *regexp.Regexp, keepDelimiter bool) markdownSplit {
+	result := markdownSplit{
+		chunks:        make([]schema.ChunkDoc, 0, len(units)),
+		blockStarts:   make([]bool, 0, len(units)),
+		shortHeadings: make([]bool, 0, len(units)),
+	}
 	for _, unit := range units {
 		unit = cloneChunkDoc(unit)
-		unit.Text = strings.TrimSpace(normalizeGeneralNewlines(itemTextOrFallback(unit)))
+		unit.Text = normalizeGeneralNewlines(itemTextOrFallback(unit))
+		if !keepDelimiter || unit.CKType == "heading" {
+			unit.Text = strings.TrimSpace(unit.Text)
+		}
 		unit.DocType = itemDocType(unit)
 		if unit.DocType == "table" || unit.DocType == "image" {
 			unit.CKType = unit.DocType
 		}
+		shortHeading := isShortMarkdownHeading(unit)
 		if unit.DocType != "text" || pattern == nil || !pattern.MatchString(unit.Text) {
 			unit.TKNums = intPtr(tokenizeStr(unit.Text))
-			result = append(result, unit)
+			result.chunks = append(result.chunks, unit)
+			result.blockStarts = append(result.blockStarts, true)
+			result.shortHeadings = append(result.shortHeadings, shortHeading)
 			continue
 		}
-		for _, part := range splitByDelim(unit.Text, pattern, false) {
-			part = strings.TrimSpace(part)
-			if part == "" {
-				continue
+		parts := splitByDelim(unit.Text, pattern, keepDelimiter)
+		if keepDelimiter {
+			parts = foldWhitespaceSegments(parts)
+		}
+		startsBlock := true
+		for _, part := range parts {
+			if !keepDelimiter {
+				part = strings.TrimSpace(part)
+				if part == "" {
+					continue
+				}
 			}
 			piece := cloneChunkDoc(unit)
 			piece.Text = part
 			piece.TKNums = intPtr(tokenizeStr(part))
-			result = append(result, piece)
+			result.chunks = append(result.chunks, piece)
+			result.blockStarts = append(result.blockStarts, startsBlock)
+			result.shortHeadings = append(result.shortHeadings, shortHeading)
+			startsBlock = !keepDelimiter
 		}
 	}
 	return result
 }
 
 // stripTextLinesFromMediaContextAbove removes folded unit text from collected
-// media context so materializeMediaContext does not emit it twice.
+// media context, including a suffix selected by a small context budget.
 func stripTextLinesFromMediaContextAbove(contextAbove, folded string) string {
 	folded = strings.TrimSpace(folded)
 	if folded == "" || contextAbove == "" {
 		return contextAbove
 	}
+	if strings.HasSuffix(folded, strings.TrimSpace(contextAbove)) {
+		return ""
+	}
 	parts := strings.Split(contextAbove, "\n")
 	filtered := make([]string, 0, len(parts))
 	for _, part := range parts {
@@ -698,20 +729,25 @@ func markdownImageUnitHasMediaContext(unit schema.ChunkDoc) bool {
 	return unit.ContextAbove != "" || unit.ContextBelow != ""
 }
 
-// mergeMarkdownUnits mirrors the Markdown branch of Python naive.chunk:
-// ordinary units use a projected token cap, while a short heading is always
+// mergeMarkdownUnits uses a projected token cap, while a short heading is always
 // kept with the following unit. Markdown images are block attachments rather
 // than standalone media chunks, so image-bearing units participate in the
 // text merge and retain their image payload. Images with configured media
 // context are emitted as standalone chunks instead.
-func mergeMarkdownUnits(units []schema.ChunkDoc, target int, overlapPct float64, joinSep string) []schema.ChunkDoc {
+func mergeMarkdownUnits(parts markdownSplit, target int, overlapPct float64) []schema.ChunkDoc {
 	overlapPct = max(0, min(100, overlapPct))
-	merged := make([]schema.ChunkDoc, 0, len(units))
+	merged := make([]schema.ChunkDoc, 0, len(parts.chunks))
 	current := -1
 	currentTokens := 0
-	for _, unit := range units {
+	for index, unit := range parts.chunks {
+		startsBlock := parts.blockStarts[index]
+		previousShortHeading := index > 0 && parts.shortHeadings[index-1]
+		joinSep := ""
+		if startsBlock && current >= 0 && !strings.HasSuffix(merged[current].Text, "\n") && !strings.HasPrefix(unit.Text, "\n") {
+			joinSep = "\n"
+		}
 		if itemDocType(unit) == "table" {
-			if current >= 0 && isShortMarkdownHeading(merged[current]) {
+			if current >= 0 && merged[current].CKType == "heading" && previousShortHeading {
 				mergedImage, imagesOK := mergeMarkdownImagesChecked(merged[current].Image, unit.Image)
 				if !imagesOK {
 					merged = append(merged, cloneChunkDoc(unit))
@@ -775,13 +811,13 @@ func mergeMarkdownUnits(units []schema.ChunkDoc, target int, overlapPct float64,
 
 		previous := &merged[current]
 		unitTokens := generalUnitTokens(unit)
-		if unit.CKType == "heading" && previous.CKType != "heading" {
+		if startsBlock && unit.CKType == "heading" && previous.CKType != "heading" {
 			merged = append(merged, unit)
 			current = len(merged) - 1
 			currentTokens = unitTokens
 			continue
 		}
-		forceMerge := isShortMarkdownHeading(*previous)
+		forceMerge := previous.CKType == "heading" && previousShortHeading
 		mergedImage, imagesOK := mergeMarkdownImagesChecked(previous.Image, unit.Image)
 		if !imagesOK {
 			merged = append(merged, unit)
@@ -810,10 +846,12 @@ func mergeMarkdownUnits(units []schema.ChunkDoc, target int, overlapPct float64,
 			current = len(merged) - 1
 			continue
 		}
-		mergeMarkdownChunk(previous, unit, joinSep)
+		mergeGeneralChunk(previous, unit, joinSep)
 		previous.Image = mergedImage
 		previous.DocType = "text"
-		previous.CKType = "text"
+		if startsBlock || unit.CKType != "heading" {
+			previous.CKType = "text"
+		}
 		currentTokens = projected
 		previous.TKNums = intPtr(currentTokens)
 	}
@@ -842,10 +880,6 @@ func markdownHeadingContent(text string) string {
 	return strings.TrimSpace(text[level:])
 }
 
-func mergeMarkdownChunk(dst *schema.ChunkDoc, src schema.ChunkDoc, joinSep string) {
-	mergeGeneralChunk(dst, src, joinSep)
-}
-
 const (
 	maxMarkdownImageDimension = 8192
 	maxMarkdownImagePixels    = 32 * 1024 * 1024
@@ -925,7 +959,7 @@ func (c *GeneralChunkerComponent) chunkSpreadsheet(ctx context.Context, upstream
 	if len(units) == 0 {
 		return emptyOutputs(), nil
 	}
-	attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize)
+	attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize, nil)
 	chunks := make([]schema.ChunkDoc, 0, len(units))
 	for _, unit := range units {
 		if itemDocType(unit) == "table" && tableDataOf(unit) != nil {
@@ -1022,7 +1056,7 @@ func (c *GeneralChunkerComponent) chunkGeneral(ctx context.Context, upstream sch
 	childrenPattern := compileChildrenPattern(c.param.ChildrenDelimiters)
 	customDelimiter := hasCustomDelim(c.param.Delimiters)
 	units = splitGeneralUnits(units, primaryPattern, !customDelimiter)
-	attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize)
+	attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize, primaryPattern)
 	if !customDelimiter {
 		units = mergeGeneralUnits(units, c.param.ChunkTokenSize, c.param.OverlappedPercent, "")
 	}

internal/ingestion/component/chunker/general_block_delimiter_test.go

@@ -0,0 +1,184 @@
+package chunker
+
+import (
+	"reflect"
+	"testing"
+
+	"ragflow/internal/parser/parser"
+)
+
+func TestGeneralChunkerPDFAndMarkdownPreserveBlockBoundaries(t *testing.T) {
+	for _, fileType := range []string{"pdf", "md"} {
+		for _, tc := range []struct {
+			name, first, second string
+		}{
+			{"implicit", "第一段。还有一句。", "第二段。"},
+			{"trailing CRLF", "第一段。还有一句。\r\n", "第二段。"},
+			{"leading CR", "第一段。还有一句。", "\r第二段。"},
+		} {
+			t.Run(fileType+"/"+tc.name, func(t *testing.T) {
+				c, err := NewGeneralChunker(map[string]any{"chunk_token_size": 512, "delimiters": []string{"\n", "。"}})
+				if err != nil {
+					t.Fatal(err)
+				}
+				out, err := c.Invoke(t.Context(), nil, map[string]any{
+					"name": "document." + fileType, "file_type": fileType, "output_format": "json",
+					"json": []map[string]any{{"text": tc.first, "doc_type_kwd": "text"}, {"text": tc.second, "doc_type_kwd": "text"}},
+				})
+				if err != nil {
+					t.Fatal(err)
+				}
+				if got, want := outputTexts(t, out), []string{"第一段。还有一句。\n第二段。"}; !reflect.DeepEqual(got, want) {
+					t.Fatalf("got %q, want %q", got, want)
+				}
+			})
+		}
+	}
+}
+
+func TestGeneralChunkerPDFPunctuationPreservesMediaOrderAndContext(t *testing.T) {
+	c, err := NewGeneralChunker(map[string]any{
+		"chunk_token_size": 512, "delimiters": []string{"\n", "。"}, "table_context_size": 512,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	position := func(top int) [][]int { return [][]int{{1, 0, 10, top, top + 5}} }
+	out, err := c.Invoke(t.Context(), nil, map[string]any{
+		"name": "document.pdf", "file_type": "pdf", "output_format": "json",
+		"json": []map[string]any{
+			{"text": "后文。还有一句。", "doc_type_kwd": "text", "positions": position(30)},
+			{"text": "<table>A</table>", "doc_type_kwd": "table", "positions": position(20)},
+			{"text": "前文。还有一句。", "doc_type_kwd": "text", "positions": position(10)},
+		},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	chunks := outputChunks(t, out)
+	if len(chunks) != 3 || chunks[0]["text"] != "前文。还有一句。" || chunks[2]["text"] != "后文。还有一句。" {
+		t.Fatalf("chunks = %#v", chunks)
+	}
+	assertMaterializedMediaContext(t, chunks[1], "前文。还有一句。<table>A</table>后文。还有一句。")
+	if got, want := chunks[0]["positions"], [][]float64{{1, 0, 10, 10, 15}}; !reflect.DeepEqual(got, want) {
+		t.Fatalf("positions = %#v, want %#v", got, want)
+	}
+}
+
+func TestGeneralChunkerMarkdownParserPreservesPunctuationAndHeading(t *testing.T) {
+	p, err := parser.NewMarkdownParser(parser.GoMarkdown)
+	if err != nil {
+		t.Fatal(err)
+	}
+	parsed := p.ParseWithResult(t.Context(), "document.md", []byte("# 标题。还有标题！\n\n第一句。第二句。\n\n下一段。"))
+	if parsed.Err != nil {
+		t.Fatal(parsed.Err)
+	}
+	c, err := NewGeneralChunker(map[string]any{"chunk_token_size": 512, "delimiters": []string{"\n", "。", "！"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	out, err := c.Invoke(t.Context(), nil, map[string]any{"name": "document.md", "file_type": "md", "output_format": "json", "json": parsed.JSON})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got, want := outputTexts(t, out), []string{"# 标题。还有标题！\n第一句。第二句。\n下一段。"}; !reflect.DeepEqual(got, want) {
+		t.Fatalf("got %q, want %q", got, want)
+	}
+}
+
+func TestGeneralChunkerPDFAndMarkdownCustomDelimiterStillDropsMarker(t *testing.T) {
+	for _, fileType := range []string{"pdf", "md"} {
+		t.Run(fileType, func(t *testing.T) {
+			c, err := NewGeneralChunker(map[string]any{"chunk_token_size": 512, "delimiters": []string{"`SEP`"}})
+			if err != nil {
+				t.Fatal(err)
+			}
+			out, err := c.Invoke(t.Context(), nil, map[string]any{
+				"name": "document." + fileType, "file_type": fileType, "output_format": "json",
+				"json": []map[string]any{{"text": "第一句。SEP第二句。", "doc_type_kwd": "text"}},
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+			if got, want := outputTexts(t, out), []string{"第一句。\n第二句。"}; !reflect.DeepEqual(got, want) {
+				t.Fatalf("got %q, want %q", got, want)
+			}
+		})
+	}
+}
+
+func TestGeneralChunkerPDFAndMarkdownCustomMediaContextDropsMarker(t *testing.T) {
+	for _, fileType := range []string{"pdf", "md"} {
+		t.Run(fileType, func(t *testing.T) {
+			c, err := NewGeneralChunker(map[string]any{
+				"chunk_token_size": 512, "delimiters": []string{"`SEP`"}, "table_context_size": 512,
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+			out, err := c.Invoke(t.Context(), nil, map[string]any{
+				"name": "document." + fileType, "file_type": fileType, "output_format": "json",
+				"json": []map[string]any{
+					{"text": "前文。SEP还有一句。", "doc_type_kwd": "text"},
+					{"text": "<table>A</table>", "doc_type_kwd": "table"},
+					{"text": "后文。", "doc_type_kwd": "text"},
+				},
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+			chunks := outputChunks(t, out)
+			if len(chunks) != 3 {
+				t.Fatalf("chunks = %#v", chunks)
+			}
+			assertMaterializedMediaContext(t, chunks[1], "前文。\n还有一句。<table>A</table>后文。")
+		})
+	}
+}
+
+func TestGeneralChunkerMarkdownPunctuationPreservesMediaBoundaries(t *testing.T) {
+	for _, tc := range []struct {
+		name, docType, ckType, first, media string
+		contextSize                         int
+		want                                []string
+	}{
+		{"heading and table", "table", "heading", "标题。还有标题！", "<table>A</table>", 512,
+			[]string{"标题。还有标题！\n<table>A</table>后文。还有一句。", "后文。还有一句。"}},
+		{"image attachment", "image", "text", "前文。还有一句。", "图注。", 0,
+			[]string{"前文。还有一句。\n图注。\n后文。还有一句。"}},
+		{"standalone image", "image", "text", "前文。还有一句。", "图注。", 512,
+			[]string{"前文。还有一句。", "前文。还有一句。图注。后文。还有一句。", "后文。还有一句。"}},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			c, err := NewGeneralChunker(map[string]any{
+				"chunk_token_size": 512, "delimiters": []string{"\n", "。", "！"},
+				"table_context_size": tc.contextSize, "image_context_size": tc.contextSize,
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+			out, err := c.Invoke(t.Context(), nil, map[string]any{
+				"name": "document.md", "file_type": "md", "output_format": "json",
+				"json": []map[string]any{
+					{"text": tc.first, "doc_type_kwd": "text", "ck_type": tc.ckType},
+					{"text": tc.media, "doc_type_kwd": tc.docType, "img_id": "media-id"},
+					{"text": "后文。还有一句。", "doc_type_kwd": "text"},
+				},
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+			if got := outputTexts(t, out); !reflect.DeepEqual(got, tc.want) {
+				t.Fatalf("got %q, want %q", got, tc.want)
+			}
+			foundMedia := false
+			for _, chunk := range outputChunks(t, out) {
+				foundMedia = foundMedia || chunk["img_id"] == "media-id"
+			}
+			if !foundMedia {
+				t.Fatal("media identity was lost")
+			}
+		})
+	}
+}

internal/ingestion/component/chunker/general_context_regression_test.go

@@ -0,0 +1,119 @@
+package chunker
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestGeneralChunkerMediaContextUsesConfiguredDelimiters(t *testing.T) {
+	for _, fileType := range []string{"pdf", "md"} {
+		for _, tc := range []struct {
+			name, delimiter, text, above, below string
+		}{
+			{"semicolon", ";", "first;middle;last;", "last;<table>A</table>", "<table>A</table>first;"},
+			{"literal pipe", "|", "first|middle|last|", "last|<table>A</table>", "<table>A</table>first|"},
+			{"multi-character", "。END", "first。ENDmiddle。ENDlast。END", "last。END<table>A</table>", "<table>A</table>first。END"},
+		} {
+			for _, direction := range []struct {
+				name  string
+				above bool
+			}{{"above", true}, {"below", false}} {
+				t.Run(fileType+"/"+tc.name+"/"+direction.name, func(t *testing.T) {
+					c, err := NewGeneralChunker(map[string]any{
+						"chunk_token_size": 512, "delimiters": []string{"\n", tc.delimiter}, "table_context_size": 1,
+					})
+					if err != nil {
+						t.Fatal(err)
+					}
+					text := map[string]any{"text": tc.text, "doc_type_kwd": "text"}
+					table := map[string]any{"text": "<table>A</table>", "doc_type_kwd": "table"}
+					items := []map[string]any{table, text}
+					tableIndex, want := 0, tc.below
+					if direction.above {
+						items = []map[string]any{text, table}
+						tableIndex, want = 1, tc.above
+					}
+					out, err := c.Invoke(t.Context(), nil, map[string]any{
+						"name": "test." + fileType, "file_type": fileType, "output_format": "json", "json": items,
+					})
+					if err != nil {
+						t.Fatal(err)
+					}
+					chunks := outputChunks(t, out)
+					if len(chunks) != 2 {
+						t.Fatalf("chunks = %#v", chunks)
+					}
+					assertMaterializedMediaContext(t, chunks[tableIndex], want)
+				})
+			}
+		}
+	}
+}
+
+func TestGeneralChunkerMarkdownFoldedHeadingDropsPartialContext(t *testing.T) {
+	c, err := NewGeneralChunker(map[string]any{
+		"chunk_token_size": 512, "delimiters": []string{"\n", "。", "！"}, "table_context_size": 1,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	out, err := c.Invoke(t.Context(), nil, map[string]any{
+		"name": "test.md", "file_type": "md", "output_format": "json",
+		"json": []map[string]any{
+			{"text": "# 标题。还有标题！", "doc_type_kwd": "text", "ck_type": "heading"},
+			{"text": "<table>A</table>", "doc_type_kwd": "table"},
+		},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	chunks := outputChunks(t, out)
+	if len(chunks) != 1 {
+		t.Fatalf("chunks = %#v", chunks)
+	}
+	assertMaterializedMediaContext(t, chunks[0], "# 标题。还有标题！\n<table>A</table>")
+}
+
+func TestGeneralChunkerMarkdownLongHeadingUsesSourceClassification(t *testing.T) {
+	for _, tc := range []struct {
+		name, delimiter, marker string
+		minChunks, maxWords     int
+	}{
+		{"bare", "!", "! ", 20, 3},
+		{"custom", "`SEP`", "SEP", 12, 5},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			heading := "# " + strings.Repeat("heading"+tc.marker, 60)
+			c, err := NewGeneralChunker(map[string]any{"chunk_token_size": 5, "delimiters": []string{tc.delimiter}})
+			if err != nil {
+				t.Fatal(err)
+			}
+			out, err := c.Invoke(t.Context(), nil, map[string]any{
+				"name": "test.md", "file_type": "md", "output_format": "json",
+				"json": []map[string]any{
+					{"text": heading, "doc_type_kwd": "text", "ck_type": "heading"},
+					{"text": "body", "doc_type_kwd": "text"},
+				},
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+			texts := outputTexts(t, out)
+			if len(texts) < tc.minChunks {
+				t.Fatalf("long heading produced only %d chunks: %q", len(texts), texts)
+			}
+			wordCount := 0
+			for _, text := range texts {
+				wordCount += strings.Count(text, "heading")
+				// Retained punctuation costs tokens; removed custom markers
+				// leave one-token atoms, so more occurrences fit the budget.
+				if strings.Count(text, "heading") > tc.maxWords {
+					t.Fatalf("long heading received the short-heading merge exception: %q", text)
+				}
+			}
+			if wordCount != 60 || !strings.HasSuffix(texts[len(texts)-1], "body") {
+				t.Fatalf("heading or body content was lost: %q", texts)
+			}
+		})
+	}
+}

internal/ingestion/component/chunker/general_delimiter_test.go

@@ -9,7 +9,7 @@ import (
 )
 
 func TestGeneralChunkerPreservesBareDelimiters(t *testing.T) {
-	for _, fileType := range []string{"txt", "docx"} {
+	for _, fileType := range []string{"txt", "docx", "pdf", "md"} {
 		for _, source := range []string{
 			"第一句。第二句。",
 			"First!Second?Third;第四句；第五句！第六句？",
@@ -42,7 +42,7 @@ func TestGeneralChunkerPreservesBareDelimiters(t *testing.T) {
 
 func TestGeneralChunkerPreservesDelimitersAcrossChunks(t *testing.T) {
 	const source = "Alpha one.\nBeta two; Gamma three!\nDelta four? Epsilon five。Zeta six；Eta seven！Theta eight？Iota nine."
-	for _, fileType := range []string{"txt", "docx"} {
+	for _, fileType := range []string{"txt", "docx", "pdf", "md"} {
 		t.Run(fileType, func(t *testing.T) {
 			component, err := NewGeneralChunker(map[string]any{
 				"chunk_token_size": 5,
@@ -107,7 +107,7 @@ func TestGeneralChunkerDOCXPreservesParagraphBoundaries(t *testing.T) {
 }
 
 func TestGeneralChunkerOverlapPreservesSentenceDelimiters(t *testing.T) {
-	for _, fileType := range []string{"txt", "docx"} {
+	for _, fileType := range []string{"txt", "docx", "pdf", "md"} {
 		t.Run(fileType, func(t *testing.T) {
 			component, err := NewGeneralChunker(map[string]any{
 				"chunk_token_size": 1, "overlapped_percent": 50,
@@ -131,7 +131,7 @@ func TestGeneralChunkerOverlapPreservesSentenceDelimiters(t *testing.T) {
 }
 
 func TestGeneralChunkerDelimiterSplitPreservesMetadata(t *testing.T) {
-	for _, fileType := range []string{"txt", "docx"} {
+	for _, fileType := range []string{"txt", "docx", "pdf", "md"} {
 		t.Run(fileType, func(t *testing.T) {
 			component, err := NewGeneralChunker(map[string]any{
 				"chunk_token_size": 0, "delimiters": []string{";"},

internal/ingestion/component/chunker/general_merge_test.go

@@ -135,7 +135,7 @@ func TestMergeMarkdownUnitsCarriesCharacterOverlapIntoNextBudget(t *testing.T) {
 	}
 	target := intValue(units[0].TKNums) + intValue(units[1].TKNums)
 
-	got := mergeMarkdownUnits(units, target, 50, "\n")
+	got := mergeMarkdownUnits(markdownSplit{chunks: units, blockStarts: []bool{true, true, true, true}, shortHeadings: []bool{false, false, false, false}}, target, 50)
 	want := []string{
 		"alpha beta\ngamma delta",
 		"gamma delta\nepsilon zeta",
@@ -174,7 +174,7 @@ func TestMergeMarkdownUnitsOverlapKeepsCurrentChunkMetadata(t *testing.T) {
 	}
 
 	target := intValue(units[0].TKNums)
-	got := mergeMarkdownUnits(units, target, 50, "\n")
+	got := mergeMarkdownUnits(markdownSplit{chunks: units, blockStarts: []bool{true, true}, shortHeadings: []bool{false, false}}, target, 50)
 	if len(got) != 2 {
 		t.Fatalf("chunks = %#v, want previous and overlapped current chunks", got)
 	}
@@ -212,7 +212,7 @@ func TestMergeMarkdownUnitsStripsFoldedHeadingFromTableContextAbove(t *testing.T
 			TKNums:       intPtr(1),
 		},
 	}
-	got := mergeMarkdownUnits(units, 10, 0, "\n")
+	got := mergeMarkdownUnits(markdownSplit{chunks: units, blockStarts: []bool{true, true}, shortHeadings: []bool{true, false}}, 10, 0)
 	if len(got) != 1 {
 		t.Fatalf("chunks = %#v, want one table chunk", got)
 	}
@@ -236,7 +236,7 @@ func TestMergeMarkdownUnitsStripsOnlyFoldedHeadingFromTableContextAbove(t *testi
 			TKNums:       intPtr(1),
 		},
 	}
-	got := mergeMarkdownUnits(units, 10, 0, "\n")
+	got := mergeMarkdownUnits(markdownSplit{chunks: units, blockStarts: []bool{true, true, true}, shortHeadings: []bool{false, true, false}}, 10, 0)
 	if len(got) != 2 {
 		t.Fatalf("chunks = %#v, want intro text and table", got)
 	}
@@ -259,7 +259,7 @@ func TestMergeMarkdownUnitsKeepsImageWithMediaContextStandalone(t *testing.T) {
 		},
 		{Text: "after", DocType: "text", CKType: "text", TKNums: intPtr(1)},
 	}
-	got := mergeMarkdownUnits(units, 10, 0, "\n")
+	got := mergeMarkdownUnits(markdownSplit{chunks: units, blockStarts: []bool{true, true, true}, shortHeadings: []bool{false, false, false}}, 10, 0)
 	if len(got) != 3 {
 		t.Fatalf("chunks = %d, want text, image, text", len(got))
 	}
@@ -275,7 +275,7 @@ func TestMergeMarkdownUnitsStartsNewChunkForIncomingHeading(t *testing.T) {
 		{Text: "The architecture is split into parser and chunker.", DocType: "text", CKType: "text", TKNums: intPtr(1)},
 	}
 
-	got := mergeMarkdownUnits(units, 3, 0, "\n")
+	got := mergeMarkdownUnits(markdownSplit{chunks: units, blockStarts: []bool{true, true, true}, shortHeadings: []bool{false, true, false}}, 3, 0)
 	if texts := generalChunkTexts(got); !reflect.DeepEqual(texts, []string{
 		"Background details",
 		"## Architecture\nThe architecture is split into parser and chunker.",
@@ -327,10 +327,10 @@ func TestMergeGeneralUnitsOverlapDoesNotCopyPreviousMediaMetadata(t *testing.T)
 
 func TestTakeContextSentencesPreservesBoundarySelection(t *testing.T) {
 	text := "one!two!three!"
-	if got, want := takeContextSentences(text, tokenizeStr("two!three!"), true), "two!three!"; got != want {
+	if got, want := takeContextSentences(text, tokenizeStr("two!three!"), true, contextSentencePattern), "two!three!"; got != want {
 		t.Fatalf("suffix context = %q, want %q", got, want)
 	}
-	if got, want := takeContextSentences(text, tokenizeStr("one!two!"), false), "one!two!"; got != want {
+	if got, want := takeContextSentences(text, tokenizeStr("one!two!"), false, contextSentencePattern), "one!two!"; got != want {
 		t.Fatalf("prefix context = %q, want %q", got, want)
 	}
 }

internal/ingestion/component/chunker/general_test.go

@@ -802,10 +802,10 @@ func TestGeneralMediaContextSeparatesAdjacentSourceUnits(t *testing.T) {
 		{Text: "after two", DocType: "text", CKType: "text", TKNums: intPtr(1)},
 	}
 
-	if got, want := collectGeneralMediaContext(units, 2, 10, true), "before one\nbefore two"; got != want {
+	if got, want := collectGeneralMediaContext(units, 2, 10, true, contextSentencePattern), "before one\nbefore two"; got != want {
 		t.Fatalf("above context = %q, want %q", got, want)
 	}
-	if got, want := collectGeneralMediaContext(units, 2, 10, false), "after one\nafter two"; got != want {
+	if got, want := collectGeneralMediaContext(units, 2, 10, false, contextSentencePattern), "after one\nafter two"; got != want {
 		t.Fatalf("below context = %q, want %q", got, want)
 	}
 }

internal/ingestion/component/chunker/token.go

@@ -902,7 +902,7 @@ func collectContext(chunks []schema.ChunkDoc, i, ctxTokens int, above bool) stri
 				tk := intValue(chunks[pos].TKNums)
 				txt := chunks[pos].Text
 				if tk >= remain {
-					parts = append([]string{takeContextSentences(txt, remain, true)}, parts...)
+					parts = append([]string{takeContextSentences(txt, remain, true, contextSentencePattern)}, parts...)
 					remain = 0
 					break
 				}
@@ -918,7 +918,7 @@ func collectContext(chunks []schema.ChunkDoc, i, ctxTokens int, above bool) stri
 				tk := intValue(chunks[pos].TKNums)
 				txt := chunks[pos].Text
 				if tk >= remain {
-					parts = append(parts, takeContextSentences(txt, remain, false))
+					parts = append(parts, takeContextSentences(txt, remain, false, contextSentencePattern))
 					remain = 0
 					break
 				}

internal/ingestion/component/chunker/token_batch1_test.go

@@ -311,10 +311,10 @@ func TestTakeContextSentencesCoversBudgetOnSentenceBoundaries(t *testing.T) {
 	const text = "alpha beta. gamma delta. epsilon zeta."
 	budget := tokenizeStr("gamma delta. epsilon zeta.")
 
-	if got, want := takeContextSentences(text, budget, true), "gamma delta. epsilon zeta."; got != want {
+	if got, want := takeContextSentences(text, budget, true, contextSentencePattern), "gamma delta. epsilon zeta."; got != want {
 		t.Errorf("tail context = %q, want %q", got, want)
 	}
-	if got, want := takeContextSentences(text, budget, false), "alpha beta. gamma delta. "; got != want {
+	if got, want := takeContextSentences(text, budget, false, contextSentencePattern), "alpha beta. gamma delta. "; got != want {
 		t.Errorf("prefix context = %q, want %q", got, want)
 	}
 	// One sentence less falls short of the budget, so the walk stopped at the
@@ -330,10 +330,10 @@ func TestTakeContextSentencesCoversBudgetOnSentenceBoundaries(t *testing.T) {
 // — comes back whole.
 func TestTakeContextSentencesKeepsBoundarylessText(t *testing.T) {
 	s := strings.Repeat("中", 60)
-	if got := takeContextSentences(s, 20, true); got != s {
+	if got := takeContextSentences(s, 20, true, contextSentencePattern); got != s {
 		t.Fatalf("context = %q, want the whole boundary-less text", got)
 	}
-	if got := takeContextSentences(s, 20, false); got != s {
+	if got := takeContextSentences(s, 20, false, contextSentencePattern); got != s {
 		t.Fatalf("context = %q, want the whole boundary-less text", got)
 	}
 }