fix(chunk): number paragraph chunks by their position (#21823)

This commit is contained in:
stv authored and GitHub committed 2026-11-02 17:24:24 +08:00
1 parent 7ee683a2c2
commit 673da1e6f6
2 files changed
+78 -3

No files matched your search

+8 -3
View File
@@ -249,14 +249,19 @@ func (o *SplitOperator) splitByLength(text string) []ChunkData {
func (o *SplitOperator) splitByParagraph(text string) []ChunkData {
paragraphs := strings.Split(text, "\n")
chunks := make([]ChunkData, 0, len(paragraphs))
for i, p := range paragraphs {
for _, p := range paragraphs {
trimmed := strings.TrimSpace(p)
if trimmed == "" {
continue
}
chunks = append(chunks, ChunkData{
Content: trimmed,
Index: i,
Content: trimmed,
// Index is the chunk's position in the output, the same as for
// the other strategies. Using the paragraph's position in the
// split left gaps wherever a blank line was skipped (a two-chunk
// document came back as 0 and 3), and those gaps only disappeared
// when a postprocess stage happened to re-index the result.
Index: len(chunks),
Metadata: map[string]interface{}{"language": DetectLanguage(trimmed)},
})
}
@@ -0,0 +1,70 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package chunk
import "testing"
// runParagraph builds a SplitOperator for the "paragraph" strategy and runs it.
func runParagraph(t *testing.T, text string) []ChunkData {
t.Helper()
op, err := NewSplitOperator(map[string]interface{}{"strategy": "paragraph"})
if err != nil {
t.Fatalf("NewSplitOperator returned error: %v", err)
}
ctx := &ChunkContext{TextAfterPreprocess: text}
if err := op.Execute(ctx); err != nil {
t.Fatalf("Execute returned error: %v", err)
}
return ctx.SplitChunks
}
func TestSplitByParagraphSkipsBlankLines(t *testing.T) {
chunks := runParagraph(t, "第一段\n\n\n第二段\n \n第三段")
want := []string{"第一段", "第二段", "第三段"}
if len(chunks) != len(want) {
t.Fatalf("got %d chunks, want %d: %#v", len(chunks), len(want), chunks)
}
for i, w := range want {
if chunks[i].Content != w {
t.Errorf("chunk %d = %q, want %q", i, chunks[i].Content, w)
}
}
}
func TestSplitByParagraphIndexMatchesPosition(t *testing.T) {
// A skipped blank line used to leave a hole in the Index sequence: the
// second chunk of this document came back as 3 instead of 1.
chunks := runParagraph(t, "第一段\n\n\n第二段\n \n第三段")
if len(chunks) != 3 {
t.Fatalf("got %d chunks, want 3", len(chunks))
}
for i, c := range chunks {
if c.Index != i {
t.Errorf("chunk %d (%q) Index = %d, want %d", i, c.Content, c.Index, i)
}
}
}
func TestSplitByParagraphBlankInput(t *testing.T) {
for _, text := range []string{"", "\n\n\n", " \n\t\n "} {
if chunks := runParagraph(t, text); len(chunks) != 0 {
t.Errorf("%q produced %d chunks, want 0", text, len(chunks))
}
}
}