mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-11-10 21:32:43 +08:00
fix(chunk): number paragraph chunks by their position (#21823)
This commit is contained in:
2 files changed
+78
-3
No files matched your search
@@ -249,14 +249,19 @@ func (o *SplitOperator) splitByLength(text string) []ChunkData {
|
||||
func (o *SplitOperator) splitByParagraph(text string) []ChunkData {
|
||||
paragraphs := strings.Split(text, "\n")
|
||||
chunks := make([]ChunkData, 0, len(paragraphs))
|
||||
for i, p := range paragraphs {
|
||||
for _, p := range paragraphs {
|
||||
trimmed := strings.TrimSpace(p)
|
||||
if trimmed == "" {
|
||||
continue
|
||||
}
|
||||
chunks = append(chunks, ChunkData{
|
||||
Content: trimmed,
|
||||
Index: i,
|
||||
Content: trimmed,
|
||||
// Index is the chunk's position in the output, the same as for
|
||||
// the other strategies. Using the paragraph's position in the
|
||||
// split left gaps wherever a blank line was skipped (a two-chunk
|
||||
// document came back as 0 and 3), and those gaps only disappeared
|
||||
// when a postprocess stage happened to re-index the result.
|
||||
Index: len(chunks),
|
||||
Metadata: map[string]interface{}{"language": DetectLanguage(trimmed)},
|
||||
})
|
||||
}
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package chunk
|
||||
|
||||
import "testing"
|
||||
|
||||
// runParagraph builds a SplitOperator for the "paragraph" strategy and runs it.
|
||||
func runParagraph(t *testing.T, text string) []ChunkData {
|
||||
t.Helper()
|
||||
op, err := NewSplitOperator(map[string]interface{}{"strategy": "paragraph"})
|
||||
if err != nil {
|
||||
t.Fatalf("NewSplitOperator returned error: %v", err)
|
||||
}
|
||||
ctx := &ChunkContext{TextAfterPreprocess: text}
|
||||
if err := op.Execute(ctx); err != nil {
|
||||
t.Fatalf("Execute returned error: %v", err)
|
||||
}
|
||||
return ctx.SplitChunks
|
||||
}
|
||||
|
||||
func TestSplitByParagraphSkipsBlankLines(t *testing.T) {
|
||||
chunks := runParagraph(t, "第一段\n\n\n第二段\n \n第三段")
|
||||
|
||||
want := []string{"第一段", "第二段", "第三段"}
|
||||
if len(chunks) != len(want) {
|
||||
t.Fatalf("got %d chunks, want %d: %#v", len(chunks), len(want), chunks)
|
||||
}
|
||||
for i, w := range want {
|
||||
if chunks[i].Content != w {
|
||||
t.Errorf("chunk %d = %q, want %q", i, chunks[i].Content, w)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitByParagraphIndexMatchesPosition(t *testing.T) {
|
||||
// A skipped blank line used to leave a hole in the Index sequence: the
|
||||
// second chunk of this document came back as 3 instead of 1.
|
||||
chunks := runParagraph(t, "第一段\n\n\n第二段\n \n第三段")
|
||||
|
||||
if len(chunks) != 3 {
|
||||
t.Fatalf("got %d chunks, want 3", len(chunks))
|
||||
}
|
||||
for i, c := range chunks {
|
||||
if c.Index != i {
|
||||
t.Errorf("chunk %d (%q) Index = %d, want %d", i, c.Content, c.Index, i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitByParagraphBlankInput(t *testing.T) {
|
||||
for _, text := range []string{"", "\n\n\n", " \n\t\n "} {
|
||||
if chunks := runParagraph(t, text); len(chunks) != 0 {
|
||||
t.Errorf("%q produced %d chunks, want 0", text, len(chunks))
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user