mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-11-10 21:32:43 +08:00
feat(parser): add opt-in VLM enhancement for images (#20166)
This commit is contained in:
1 parent
01f6b9c3bc
commit
31fd92c2fa
45 files changed
+3240
-1076
No files matched your search
@@ -45,6 +45,11 @@ func (e *PDFOxideEngine) RenderPageImage(pageNum int, dpi float64) (image.Image,
|
||||
return e.Inner.RenderPageImage(pageNum, dpi)
|
||||
}
|
||||
|
||||
// PageSize returns the effective dimensions of a zero-based page in PDF points.
|
||||
func (e *PDFOxideEngine) PageSize(pageNum int) (float64, float64, error) {
|
||||
return e.Inner.PageSize(pageNum)
|
||||
}
|
||||
|
||||
func (e *PDFOxideEngine) ExtractChars(pageNum int) ([]pdf.TextChar, error) {
|
||||
chars, err := e.Inner.ExtractChars(pageNum)
|
||||
if err != nil {
|
||||
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"image"
|
||||
"image/color"
|
||||
"math"
|
||||
"strings"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
@@ -20,23 +19,63 @@ import (
|
||||
//
|
||||
// Python: pdf_parser.py:1802 RAGFlowPdfParser.crop()
|
||||
func CropSectionImage(posTag string, decodedImages map[int]image.Image, zoom float64) string {
|
||||
return encodeCroppedImage(cropSectionImageRaster(posTag, decodedImages, zoom))
|
||||
}
|
||||
|
||||
func cropSectionImageRaster(posTag string, decodedImages map[int]image.Image, zoom float64) image.Image {
|
||||
if len(decodedImages) == 0 {
|
||||
common.Warn("cropSectionImage: no page images available, skipping image generation")
|
||||
return ""
|
||||
return nil
|
||||
}
|
||||
|
||||
positions := ExtractPositions(posTag)
|
||||
if len(positions) == 0 {
|
||||
common.Warn("cropSectionImage: empty position list in tag", zap.String("posTag", posTag[:min(80, len(posTag))]))
|
||||
return ""
|
||||
return nil
|
||||
}
|
||||
return cropSectionPositionsRaster(positions, decodedImages, zoom)
|
||||
}
|
||||
|
||||
// Filter valid positions (all pages available).
|
||||
var valid []pdf.Position
|
||||
type sectionRasterSegment struct {
|
||||
page int
|
||||
x0 int
|
||||
y0 int
|
||||
x1 int
|
||||
y1 int
|
||||
isEdge bool
|
||||
}
|
||||
|
||||
type sectionRasterPlan struct {
|
||||
segments []sectionRasterSegment
|
||||
width int
|
||||
height int
|
||||
}
|
||||
|
||||
const (
|
||||
sectionRasterContextPad = 120.0
|
||||
sectionRasterGap = 6
|
||||
)
|
||||
|
||||
func cropSectionPositionsRaster(positions []pdf.Position, decodedImages map[int]image.Image, zoom float64) image.Image {
|
||||
plan, ok := buildSectionRasterPlan(positions, decodedImages, zoom)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return renderSectionRaster(plan, decodedImages)
|
||||
}
|
||||
|
||||
func buildSectionRasterPlan(positions []pdf.Position, decodedImages map[int]image.Image, zoom float64) (*sectionRasterPlan, bool) {
|
||||
if len(decodedImages) == 0 || len(positions) == 0 || zoom <= 0 || math.IsNaN(zoom) || math.IsInf(zoom, 0) {
|
||||
return nil, false
|
||||
}
|
||||
valid := make([]pdf.Position, 0, len(positions))
|
||||
for _, pos := range positions {
|
||||
if len(pos.PageNumbers) == 0 {
|
||||
continue
|
||||
}
|
||||
allValid := true
|
||||
for _, pn := range pos.PageNumbers {
|
||||
if _, ok := decodedImages[pn]; !ok {
|
||||
for _, pageNum := range pos.PageNumbers {
|
||||
if img, ok := decodedImages[pageNum]; !ok || img == nil {
|
||||
allValid = false
|
||||
break
|
||||
}
|
||||
@@ -46,196 +85,168 @@ func CropSectionImage(posTag string, decodedImages map[int]image.Image, zoom flo
|
||||
}
|
||||
}
|
||||
if len(valid) == 0 {
|
||||
common.Warn("cropSectionImage: no valid positions after filtering, skipping crop")
|
||||
return ""
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// Context padding (Python: 120px above first, 120 below last, 6px gap)
|
||||
const contextPad = 120.0
|
||||
const gap = 6
|
||||
|
||||
// Compute max width across original positions for full-width edge bands.
|
||||
maxWidth := 6.0
|
||||
for _, pos := range valid {
|
||||
w := pos.Right - pos.Left
|
||||
if w > maxWidth {
|
||||
maxWidth = w
|
||||
if width := pos.Right - pos.Left; width > maxWidth {
|
||||
maxWidth = width
|
||||
}
|
||||
}
|
||||
|
||||
// Python-style: insert synthetic context bands at edges.
|
||||
// Original positions are all middle entries (narrow width).
|
||||
// Synthetic bands are edge entries (full width + semi-transparent overlay).
|
||||
first := valid[0]
|
||||
last := valid[len(valid)-1]
|
||||
firstPageIdx := first.PageNumbers[0]
|
||||
lastPageIdx := last.PageNumbers[len(last.PageNumbers)-1]
|
||||
lastPageH := float64(decodedImages[lastPageIdx].Bounds().Dy()) / zoom
|
||||
|
||||
// topBand: 120px context above the first content position.
|
||||
topBandPos := pdf.Position{
|
||||
PageNumbers: []int{firstPageIdx},
|
||||
firstPage := first.PageNumbers[0]
|
||||
lastPage := last.PageNumbers[len(last.PageNumbers)-1]
|
||||
lastPageHeight := float64(decodedImages[lastPage].Bounds().Dy()) / zoom
|
||||
topBand := pdf.Position{
|
||||
PageNumbers: []int{firstPage},
|
||||
Left: first.Left,
|
||||
Right: first.Right,
|
||||
Top: math.Max(0, first.Top-contextPad),
|
||||
Bottom: math.Max(first.Top-gap, 0),
|
||||
Top: math.Max(0, first.Top-sectionRasterContextPad),
|
||||
Bottom: math.Max(first.Top-sectionRasterGap, 0),
|
||||
}
|
||||
// bottomBand: 120px context below the last content position.
|
||||
bottomBandPos := pdf.Position{
|
||||
PageNumbers: []int{lastPageIdx},
|
||||
bottomBand := pdf.Position{
|
||||
PageNumbers: []int{lastPage},
|
||||
Left: last.Left,
|
||||
Right: last.Right,
|
||||
Top: math.Min(lastPageH, last.Bottom+gap),
|
||||
Bottom: math.Min(lastPageH, last.Bottom+contextPad),
|
||||
Top: math.Min(lastPageHeight, last.Bottom+sectionRasterGap),
|
||||
Bottom: math.Min(lastPageHeight, last.Bottom+sectionRasterContextPad),
|
||||
}
|
||||
|
||||
// Build entry list: [topBand, original positions..., bottomBand].
|
||||
type segment struct {
|
||||
img image.Image
|
||||
isEdge bool
|
||||
type positionEntry struct {
|
||||
position pdf.Position
|
||||
isEdge bool
|
||||
}
|
||||
var segments []segment
|
||||
|
||||
allPos := make([]struct {
|
||||
pos pdf.Position
|
||||
isEdge bool
|
||||
}, 0, len(valid)+2)
|
||||
allPos = append(allPos, struct {
|
||||
pos pdf.Position
|
||||
isEdge bool
|
||||
}{topBandPos, true})
|
||||
entries := make([]positionEntry, 0, len(valid)+2)
|
||||
entries = append(entries, positionEntry{position: topBand, isEdge: true})
|
||||
for _, pos := range valid {
|
||||
allPos = append(allPos, struct {
|
||||
pos pdf.Position
|
||||
isEdge bool
|
||||
}{pos, false})
|
||||
entries = append(entries, positionEntry{position: pos})
|
||||
}
|
||||
allPos = append(allPos, struct {
|
||||
pos pdf.Position
|
||||
isEdge bool
|
||||
}{bottomBandPos, true})
|
||||
entries = append(entries, positionEntry{position: bottomBand, isEdge: true})
|
||||
|
||||
for _, entry := range allPos {
|
||||
pos := entry.pos
|
||||
isEdge := entry.isEdge
|
||||
|
||||
top := pos.Top
|
||||
bottom := pos.Bottom
|
||||
left := pos.Left
|
||||
right := pos.Right
|
||||
|
||||
// Width: edge segments are full-width, middle are narrow.
|
||||
if !isEdge {
|
||||
right = math.Max(left+10, right)
|
||||
} else {
|
||||
plan := §ionRasterPlan{}
|
||||
maxInt := int(^uint(0) >> 1)
|
||||
for _, entry := range entries {
|
||||
pos := entry.position
|
||||
left, right := pos.Left, pos.Right
|
||||
if entry.isEdge {
|
||||
right = left + maxWidth
|
||||
} else {
|
||||
right = math.Max(left+10, right)
|
||||
}
|
||||
firstPage := pos.PageNumbers[0]
|
||||
accumBottom := pos.Bottom * zoom
|
||||
for _, pageNum := range pos.PageNumbers[1:] {
|
||||
if pageNum != firstPage {
|
||||
accumBottom += float64(decodedImages[pageNum].Bounds().Dy())
|
||||
}
|
||||
}
|
||||
|
||||
pn0 := pos.PageNumbers[0]
|
||||
pageImage := decodedImages[firstPage]
|
||||
pageHeight := float64(pageImage.Bounds().Dy())
|
||||
bottomClamped := math.Min(accumBottom, pageHeight)
|
||||
plan.addCrop(firstPage, int(left*zoom), int(pos.Top*zoom), int(right*zoom), int(bottomClamped), entry.isEdge)
|
||||
|
||||
// Accumulate bottom for multi-page positions.
|
||||
accumBottom := bottom * zoom
|
||||
for _, pn := range pos.PageNumbers[1:] {
|
||||
if pn == pn0 {
|
||||
bottomRemaining := accumBottom - pageHeight
|
||||
for _, pageNum := range pos.PageNumbers[1:] {
|
||||
if pageNum == firstPage {
|
||||
continue
|
||||
}
|
||||
if img, ok := decodedImages[pn]; ok {
|
||||
accumBottom += float64(img.Bounds().Dy())
|
||||
if bottomRemaining <= 0 {
|
||||
break
|
||||
}
|
||||
pageImage := decodedImages[pageNum]
|
||||
bottomClamped := math.Min(bottomRemaining, float64(pageImage.Bounds().Dy()))
|
||||
bottomPixels := int(bottomClamped)
|
||||
if bottomPixels <= 0 {
|
||||
break
|
||||
}
|
||||
plan.addCrop(pageNum, int(left*zoom), 0, int(right*zoom), bottomPixels, entry.isEdge)
|
||||
bottomRemaining -= bottomClamped
|
||||
}
|
||||
|
||||
pageImg, ok := decodedImages[pn0]
|
||||
if !ok {
|
||||
common.Warn("cropSectionImage: page image not found", zap.Int("page", pn0))
|
||||
return ""
|
||||
}
|
||||
if len(plan.segments) == 0 {
|
||||
return nil, false
|
||||
}
|
||||
for _, segment := range plan.segments {
|
||||
width, height := cropSize(decodedImages[segment.page], segment.x0, segment.y0, segment.x1, segment.y1)
|
||||
plan.width = max(plan.width, width)
|
||||
if height > maxInt-sectionRasterGap || plan.height > maxInt-(height+sectionRasterGap) {
|
||||
return nil, false
|
||||
}
|
||||
pageH := float64(pageImg.Bounds().Dy())
|
||||
bottomClamped := math.Min(accumBottom, pageH)
|
||||
plan.height += height + sectionRasterGap
|
||||
}
|
||||
return plan, plan.width > 0 && plan.height > 0
|
||||
}
|
||||
|
||||
// Crop first page of this position.
|
||||
cropped := FastCrop(pageImg,
|
||||
int(left*zoom), int(top*zoom),
|
||||
int(right*zoom), int(bottomClamped))
|
||||
if isEdge {
|
||||
func (p *sectionRasterPlan) addCrop(page, x0, y0, x1, y1 int, isEdge bool) {
|
||||
p.segments = append(p.segments, sectionRasterSegment{
|
||||
page: page, x0: x0, y0: y0, x1: x1, y1: y1, isEdge: isEdge,
|
||||
})
|
||||
}
|
||||
|
||||
func cropSize(img image.Image, x0, y0, x1, y1 int) (int, int) {
|
||||
bounds := cropRectBounds(img, x0, y0, x1, y1)
|
||||
if bounds.Empty() {
|
||||
return 1, 1
|
||||
}
|
||||
return bounds.Dx(), bounds.Dy()
|
||||
}
|
||||
|
||||
func renderSectionRaster(plan *sectionRasterPlan, decodedImages map[int]image.Image) image.Image {
|
||||
segments := make([]image.Image, 0, len(plan.segments))
|
||||
for _, segment := range plan.segments {
|
||||
cropped := FastCrop(decodedImages[segment.page], segment.x0, segment.y0, segment.x1, segment.y1)
|
||||
if segment.isEdge {
|
||||
cropped = applyEdgeOverlay(cropped)
|
||||
}
|
||||
segments = append(segments, segment{img: cropped, isEdge: isEdge})
|
||||
|
||||
// Subsequent pages (only those different from the first page).
|
||||
bottomRemaining := accumBottom - pageH
|
||||
for _, pn := range pos.PageNumbers[1:] {
|
||||
if pn == pn0 {
|
||||
continue
|
||||
}
|
||||
pageImg2, ok := decodedImages[pn]
|
||||
if !ok {
|
||||
common.Warn("cropSectionImage: page image not found for subsequent page", zap.Int("page", pn))
|
||||
return ""
|
||||
}
|
||||
pageH2 := float64(pageImg2.Bounds().Dy())
|
||||
bottomClamped2 := math.Min(bottomRemaining, pageH2)
|
||||
cropped2 := FastCrop(pageImg2,
|
||||
int(left*zoom), 0,
|
||||
int(right*zoom), int(bottomClamped2))
|
||||
if isEdge {
|
||||
cropped2 = applyEdgeOverlay(cropped2)
|
||||
}
|
||||
segments = append(segments, segment{img: cropped2, isEdge: isEdge})
|
||||
bottomRemaining -= bottomClamped2
|
||||
}
|
||||
segments = append(segments, cropped)
|
||||
}
|
||||
|
||||
if len(segments) == 0 {
|
||||
return ""
|
||||
}
|
||||
|
||||
// Stitch vertically with gray background and 6px gaps.
|
||||
totalH := 0
|
||||
maxW := 0
|
||||
for _, seg := range segments {
|
||||
totalH += seg.img.Bounds().Dy() + gap
|
||||
maxW = max(maxW, seg.img.Bounds().Dx())
|
||||
}
|
||||
stitched := image.NewRGBA(image.Rect(0, 0, maxW, totalH))
|
||||
|
||||
// Fill background using direct Pix slice write (matching fastCrop pattern).
|
||||
// Gray 245,245,245,255 as BGRA bytes.
|
||||
for y := 0; y < totalH; y++ {
|
||||
row := stitched.Pix[stitched.PixOffset(0, y):stitched.PixOffset(maxW, y)]
|
||||
stitched := image.NewRGBA(image.Rect(0, 0, plan.width, plan.height))
|
||||
for y := 0; y < plan.height; y++ {
|
||||
row := stitched.Pix[stitched.PixOffset(0, y):stitched.PixOffset(plan.width, y)]
|
||||
for i := 0; i < len(row); i += 4 {
|
||||
row[i] = 245 // B
|
||||
row[i+1] = 245 // G
|
||||
row[i+2] = 245 // R
|
||||
row[i+3] = 255 // A
|
||||
row[i], row[i+1], row[i+2], row[i+3] = 245, 245, 245, 255
|
||||
}
|
||||
}
|
||||
|
||||
curY := 0
|
||||
for _, seg := range segments {
|
||||
srcW := seg.img.Bounds().Dx()
|
||||
srcH := seg.img.Bounds().Dy()
|
||||
if rgba, ok := seg.img.(*image.RGBA); ok {
|
||||
// Fast path: direct Pix slice copy (matching fastCrop in geometry.go).
|
||||
srcMinX := seg.img.Bounds().Min.X
|
||||
srcMinY := seg.img.Bounds().Min.Y
|
||||
for ry := 0; ry < srcH; ry++ {
|
||||
srcStart := rgba.PixOffset(srcMinX, srcMinY+ry)
|
||||
for _, segment := range segments {
|
||||
srcW, srcH := segment.Bounds().Dx(), segment.Bounds().Dy()
|
||||
if rgba, ok := segment.(*image.RGBA); ok {
|
||||
srcMinX, srcMinY := segment.Bounds().Min.X, segment.Bounds().Min.Y
|
||||
for row := 0; row < srcH; row++ {
|
||||
srcStart := rgba.PixOffset(srcMinX, srcMinY+row)
|
||||
srcRow := rgba.Pix[srcStart : srcStart+srcW*4]
|
||||
dstStart := stitched.PixOffset(0, curY+ry)
|
||||
dstStart := stitched.PixOffset(0, curY+row)
|
||||
copy(stitched.Pix[dstStart:], srcRow)
|
||||
}
|
||||
} else {
|
||||
// Fallback: pixel-by-pixel for non-RGBA images (e.g. edge overlays).
|
||||
for y := 0; y < srcH; y++ {
|
||||
for x := 0; x < srcW; x++ {
|
||||
stitched.Set(x, curY+y, seg.img.At(x+seg.img.Bounds().Min.X, y+seg.img.Bounds().Min.Y))
|
||||
stitched.Set(x, curY+y, segment.At(x+segment.Bounds().Min.X, y+segment.Bounds().Min.Y))
|
||||
}
|
||||
}
|
||||
}
|
||||
curY += srcH + gap
|
||||
curY += srcH + sectionRasterGap
|
||||
}
|
||||
return stitched
|
||||
}
|
||||
|
||||
data, err := EncodePNG(stitched)
|
||||
func rasterPlanExceedsLimit(plan *sectionRasterPlan, maxPixels int64) bool {
|
||||
if plan == nil || plan.width <= 0 || plan.height <= 0 || maxPixels <= 0 {
|
||||
return true
|
||||
}
|
||||
return int64(plan.width) > maxPixels/int64(plan.height)
|
||||
}
|
||||
|
||||
func encodeCroppedImage(img image.Image) string {
|
||||
if img == nil {
|
||||
return ""
|
||||
}
|
||||
data, err := EncodePNG(img)
|
||||
if err != nil {
|
||||
common.Warn("cropSectionImage: PNG encode failed", zap.Error(err))
|
||||
return ""
|
||||
@@ -598,25 +609,24 @@ func CropImageRegion(img image.Image, r pdf.DLARegion) (image.Image, error) {
|
||||
//
|
||||
// Python: pdf_parser.py:1802 RAGFlowPdfParser.crop()
|
||||
func CropSectionPositions(positions []pdf.Position, decodedImages map[int]image.Image, zoom float64) string {
|
||||
if len(positions) == 0 {
|
||||
return ""
|
||||
return encodeCroppedImage(CropSectionPositionsRaster(positions, decodedImages, zoom))
|
||||
}
|
||||
|
||||
// CropSectionPositionsRaster crops typed PDF positions and returns the raster
|
||||
// without encoding it. Callers that need both OCR and a VLM payload can reuse
|
||||
// this image and encode it once after OCR.
|
||||
func CropSectionPositionsRaster(positions []pdf.Position, decodedImages map[int]image.Image, zoom float64) image.Image {
|
||||
return cropSectionPositionsRaster(positions, decodedImages, zoom)
|
||||
}
|
||||
|
||||
// CropSectionPositionsRasterLimited rejects a section before rendering when
|
||||
// the planned stitched raster exceeds maxPixels.
|
||||
func CropSectionPositionsRasterLimited(positions []pdf.Position, decodedImages map[int]image.Image, zoom float64, maxPixels int64) image.Image {
|
||||
plan, ok := buildSectionRasterPlan(positions, decodedImages, zoom)
|
||||
if !ok || rasterPlanExceedsLimit(plan, maxPixels) {
|
||||
return nil
|
||||
}
|
||||
var tag strings.Builder
|
||||
for _, pos := range positions {
|
||||
if len(pos.PageNumbers) == 0 {
|
||||
continue
|
||||
}
|
||||
if len(pos.PageNumbers) == 1 {
|
||||
tag.WriteString(FormatPositionTag(pos.PageNumbers[0], pos.Left, pos.Right, pos.Top, pos.Bottom))
|
||||
} else {
|
||||
from, to := pos.PageNumbers[0], pos.PageNumbers[len(pos.PageNumbers)-1]
|
||||
tag.WriteString(FormatPositionTagRange(from, to, pos.Left, pos.Right, pos.Top, pos.Bottom))
|
||||
}
|
||||
}
|
||||
if tag.Len() == 0 {
|
||||
return ""
|
||||
}
|
||||
return CropSectionImage(tag.String(), decodedImages, zoom)
|
||||
return renderSectionRaster(plan, decodedImages)
|
||||
}
|
||||
|
||||
// PositionsFromMatrix converts the _pdf_positions / positions matrix form
|
||||
|
||||
@@ -31,6 +31,19 @@ func decodePNG(t *testing.T, data []byte) image.Image {
|
||||
return img
|
||||
}
|
||||
|
||||
func TestFastCropReturnsPlaceholderForInvertedBounds(t *testing.T) {
|
||||
src := makeTestPageImage(100, 100, color.RGBA{255, 0, 0, 255})
|
||||
for _, bounds := range [][4]int{
|
||||
{80, 10, 20, 90},
|
||||
{10, 80, 90, 20},
|
||||
} {
|
||||
got := FastCrop(src, bounds[0], bounds[1], bounds[2], bounds[3]).Bounds()
|
||||
if got != image.Rect(0, 0, 1, 1) {
|
||||
t.Errorf("FastCrop(%v) bounds = %v, want 1x1 placeholder", bounds, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCropSectionImage_SinglePage(t *testing.T) {
|
||||
pageImages := map[int]image.Image{
|
||||
0: makeTestPageImage(200, 300, color.RGBA{255, 0, 0, 255}),
|
||||
@@ -147,6 +160,35 @@ func TestCropSectionImage_ZoomScaling(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
type accessCountingImage struct {
|
||||
bounds image.Rectangle
|
||||
reads int
|
||||
}
|
||||
|
||||
func (i *accessCountingImage) ColorModel() color.Model { return color.RGBAModel }
|
||||
func (i *accessCountingImage) Bounds() image.Rectangle { return i.bounds }
|
||||
func (i *accessCountingImage) At(int, int) color.Color {
|
||||
i.reads++
|
||||
return color.RGBA{A: 255}
|
||||
}
|
||||
|
||||
func TestCropSectionPositionsRasterLimitedRejectsOversizedStitchBeforeCopy(t *testing.T) {
|
||||
page := &accessCountingImage{bounds: image.Rect(0, 0, 400, 400)}
|
||||
positions := []pdf.Position{
|
||||
{PageNumbers: []int{0}, Left: 10, Right: 30, Top: 150, Bottom: 170},
|
||||
{PageNumbers: []int{0}, Left: 10, Right: 30, Top: 150, Bottom: 170},
|
||||
{PageNumbers: []int{0}, Left: 10, Right: 30, Top: 150, Bottom: 170},
|
||||
}
|
||||
|
||||
got := CropSectionPositionsRasterLimited(positions, map[int]image.Image{0: page}, 1, 5_000)
|
||||
if got != nil {
|
||||
t.Fatalf("limited crop = %v, want nil when stitched pixels exceed the limit", got.Bounds())
|
||||
}
|
||||
if page.reads != 0 {
|
||||
t.Fatalf("source pixel reads = %d, want 0 before rejecting the oversized stitch", page.reads)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRotateImageCW(t *testing.T) {
|
||||
// Create a 3x2 image with known colors: (0,0)=red, (1,0)=green, (2,0)=blue,
|
||||
// (0,1)=white, (1,1)=black, (2,1)=gray
|
||||
@@ -301,34 +343,29 @@ func colorEqual(a, b color.Color) bool {
|
||||
return ar == br && ag == bg && ab == bb && aa == ba
|
||||
}
|
||||
|
||||
// TestCropSectionImage_MultiPage verifies the bottomRemaining fix for 3+ page
|
||||
// positions where page heights differ. Regression test for Bug #3.
|
||||
// TestCropSectionImage_MultiPage verifies that a non-positive trailing-page
|
||||
// remainder does not become an inverted crop region.
|
||||
func TestCropSectionImage_MultiPage(t *testing.T) {
|
||||
// Page 0: tall (2000px), Page 1: short (800px), Page 2: short (800px)
|
||||
// Content spans all 3 pages. The old bug subtracted full pageH2 from
|
||||
// bottomRemaining instead of the actual clamped value, causing negative
|
||||
// y1 on the last page → 1×1 placeholder crop.
|
||||
// The range stores only its endpoint page numbers. With these page heights,
|
||||
// the remaining crop height is exhausted before the trailing endpoint.
|
||||
pageImages := map[int]image.Image{
|
||||
0: makeTestPageImage(100, 2000, color.RGBA{200, 0, 0, 255}),
|
||||
1: makeTestPageImage(100, 800, color.RGBA{0, 200, 0, 255}),
|
||||
2: makeTestPageImage(100, 800, color.RGBA{0, 0, 200, 255}),
|
||||
}
|
||||
// pdf.Position spans pages 0-2, bottom reaches into page 2.
|
||||
// The position range covers pages 0-2.
|
||||
posTag := "@@1-3\t0.0\t100.0\t0.0\t500.0##"
|
||||
b64 := CropSectionImage(posTag, pageImages, 1)
|
||||
if b64 == "" {
|
||||
t.Fatal("expected non-empty result for multi-page position")
|
||||
}
|
||||
// Decode and check height: content 500pt + bottom on page 1 clamped
|
||||
// to 800 → page 1 crop 0-800, page 2 crop 0-200. Total with 2x6px gaps
|
||||
// should be ~2000 + 200 + 12 = 2212.
|
||||
// The first page and context bands should still yield a non-empty crop.
|
||||
decoded, _ := base64.StdEncoding.DecodeString(b64)
|
||||
img := decodePNG(t, decoded)
|
||||
h := img.Bounds().Dy()
|
||||
// Without the fix, page 2 gets negative y1 → 1x1 output (~100 + gap).
|
||||
// With fix, proper crop from all 3 pages.
|
||||
if h < 500 {
|
||||
t.Errorf("multi-page height too small: got %d, want >= 500 (bug: bottomRemaining over-subtraction)", h)
|
||||
t.Errorf("multi-page height too small: got %d, want >= 500", h)
|
||||
}
|
||||
t.Logf("multi-page stitch height: %d", h)
|
||||
}
|
||||
|
||||
@@ -193,7 +193,29 @@ func RectOverlap(a, b Rect) float64 {
|
||||
// Uses direct Pix slice copy for *image.RGBA sources (zero allocation per row);
|
||||
// falls back to pixel-by-pixel for other image types.
|
||||
func FastCrop(src image.Image, x0, y0, x1, y1 int) *image.RGBA {
|
||||
// Clamp to source bounds
|
||||
b := cropRectBounds(src, x0, y0, x1, y1)
|
||||
if b.Empty() {
|
||||
return image.NewRGBA(image.Rect(0, 0, 1, 1))
|
||||
}
|
||||
w, h := b.Dx(), b.Dy()
|
||||
dst := image.NewRGBA(image.Rect(0, 0, w, h))
|
||||
if rgba, ok := src.(*image.RGBA); ok {
|
||||
for y := b.Min.Y; y < b.Max.Y; y++ {
|
||||
srcRow := rgba.Pix[rgba.PixOffset(b.Min.X, y):rgba.PixOffset(b.Max.X, y)]
|
||||
dstRow := dst.Pix[dst.PixOffset(0, y-b.Min.Y):]
|
||||
copy(dstRow, srcRow)
|
||||
}
|
||||
} else {
|
||||
for y := b.Min.Y; y < b.Max.Y; y++ {
|
||||
for x := b.Min.X; x < b.Max.X; x++ {
|
||||
dst.Set(x-b.Min.X, y-b.Min.Y, src.At(x, y))
|
||||
}
|
||||
}
|
||||
}
|
||||
return dst
|
||||
}
|
||||
|
||||
func cropRectBounds(src image.Image, x0, y0, x1, y1 int) image.Rectangle {
|
||||
b := src.Bounds()
|
||||
if x0 < b.Min.X {
|
||||
x0 = b.Min.X
|
||||
@@ -208,25 +230,9 @@ func FastCrop(src image.Image, x0, y0, x1, y1 int) *image.RGBA {
|
||||
y1 = b.Max.Y
|
||||
}
|
||||
if x0 >= x1 || y0 >= y1 {
|
||||
return image.NewRGBA(image.Rect(0, 0, 1, 1))
|
||||
return image.Rectangle{}
|
||||
}
|
||||
w, h := x1-x0, y1-y0
|
||||
dst := image.NewRGBA(image.Rect(0, 0, w, h))
|
||||
if rgba, ok := src.(*image.RGBA); ok {
|
||||
for y := y0; y < y1; y++ {
|
||||
srcRow := rgba.Pix[rgba.PixOffset(x0, y):rgba.PixOffset(x1, y)]
|
||||
dstRow := dst.Pix[dst.PixOffset(0, y-y0):]
|
||||
copy(dstRow, srcRow)
|
||||
}
|
||||
|
||||
} else {
|
||||
for y := y0; y < y1; y++ {
|
||||
for x := x0; x < x1; x++ {
|
||||
dst.Set(x-x0, y-y0, src.At(x, y))
|
||||
}
|
||||
}
|
||||
}
|
||||
return dst
|
||||
return image.Rect(x0, y0, x1, y1)
|
||||
}
|
||||
|
||||
// ── Geometry helpers (pure functions, moved from type/types.go) ─────────
|
||||
|
||||
@@ -111,10 +111,8 @@ func TestChunker_PreservesLayoutField(t *testing.T) {
|
||||
}
|
||||
|
||||
// TestChunker_PassesPDFPositionsThrough pins the _pdf_positions
|
||||
// pass-through. The PDF parser path (DeepDOC) attaches bbox
|
||||
// coordinates under this key; the chunker must forward them
|
||||
// so downstream layout-aware components can rebuild the page
|
||||
// geometry.
|
||||
// pass-through. DeepDOC's PDF parser attaches bounding boxes under this key;
|
||||
// downstream layout-aware components use them to rebuild page geometry.
|
||||
func TestChunker_PassesPDFPositionsThrough(t *testing.T) {
|
||||
positions := [][]float64{{0.1, 0.2, 0.3, 0.4}, {0.5, 0.6, 0.7, 0.8}}
|
||||
items := []map[string]any{
|
||||
@@ -134,6 +132,28 @@ func TestChunker_PassesPDFPositionsThrough(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunker_PreservesVisualParentMetadata(t *testing.T) {
|
||||
items := []map[string]any{
|
||||
{
|
||||
"text": "OCR text from a table image",
|
||||
"doc_type_kwd": "image",
|
||||
"parent_table_id": "docx-table-1",
|
||||
"media_order": 2,
|
||||
},
|
||||
}
|
||||
out := invokeAsTokenChunker(t, items)
|
||||
chunks := chunksFromOutput(t, out)
|
||||
if len(chunks) == 0 {
|
||||
t.Fatal("no chunks emitted")
|
||||
}
|
||||
if got, want := chunks[0]["parent_table_id"], "docx-table-1"; got != want {
|
||||
t.Errorf("chunks[0].parent_table_id = %v, want %v", got, want)
|
||||
}
|
||||
if got, want := chunks[0]["media_order"], float64(2); got != want {
|
||||
t.Errorf("chunks[0].media_order = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestChunker_DropsContentWithWeightOnlyItems enforces the pre-index wire
|
||||
// contract: chunkers consume canonical "text" only.
|
||||
func TestChunker_DropsContentWithWeightOnlyItems(t *testing.T) {
|
||||
|
||||
@@ -830,6 +830,7 @@ func buildChunkDoc(it schema.ChunkDoc, ckType, text, ctxAbove, ctxBelow string)
|
||||
Positions: it.Positions,
|
||||
Image: it.Image,
|
||||
PageNumber: it.PageNumber,
|
||||
Extra: cloneExtra(it.Extra),
|
||||
}
|
||||
if ctxAbove != "" {
|
||||
out.ContextAbove = ctxAbove
|
||||
@@ -1642,11 +1643,17 @@ func cloneChunkDoc(in schema.ChunkDoc) schema.ChunkDoc {
|
||||
if in.Positions != nil {
|
||||
out.Positions = append(json.RawMessage(nil), in.Positions...)
|
||||
}
|
||||
if in.Extra != nil {
|
||||
out.Extra = make(map[string]json.RawMessage, len(in.Extra))
|
||||
for k, v := range in.Extra {
|
||||
out.Extra[k] = append(json.RawMessage(nil), v...)
|
||||
}
|
||||
out.Extra = cloneExtra(in.Extra)
|
||||
return out
|
||||
}
|
||||
|
||||
func cloneExtra(in map[string]json.RawMessage) map[string]json.RawMessage {
|
||||
if in == nil {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]json.RawMessage, len(in))
|
||||
for key, value := range in {
|
||||
out[key] = append(json.RawMessage(nil), value...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -47,8 +47,9 @@ var resolveModelConfig = defaultResolveModelConfig
|
||||
|
||||
// configuredMediaModelID extracts a per-call model reference from a parser setup.
|
||||
// Image parsing stores the VLM model reference in parse_method when it is not
|
||||
// "ocr" (mirroring Python rag/flow/parser/parser.py:_image). Other media
|
||||
// families use vlm.llm_id, matching the frontend parser form.
|
||||
// "ocr" (mirroring Python rag/flow/parser/parser.py:_image); "ocr" selects the
|
||||
// tenant default vision model, and Go does not run OCR for standalone images.
|
||||
// Other media families use vlm.llm_id, matching the frontend parser form.
|
||||
func configuredMediaModelID(setup schema.ParserSetup, family string) string {
|
||||
if family == "image" {
|
||||
if ref := getStringOr(setup, "parse_method", ""); ref != "" && !strings.EqualFold(ref, "ocr") {
|
||||
|
||||
@@ -14,36 +14,17 @@
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
// Media dispatch: image, audio, video parser branches that require
|
||||
// model access (OCR, IMAGE2TEXT, SPEECH2TEXT) at the component
|
||||
// layer. Mirrors Python's _image / _audio / _video methods in
|
||||
// rag/flow/parser/parser.py and rag/app/picture.py.
|
||||
//
|
||||
// These follow the maybeDispatchPDFVision pattern: they bypass
|
||||
// dispatchParse and call the model directly from the component
|
||||
// layer, returning a parser.ParseResult.
|
||||
// Media dispatch: image, audio, and video branches that require model access
|
||||
// (IMAGE2TEXT, SPEECH2TEXT) at the component layer.
|
||||
|
||||
package component
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"image"
|
||||
|
||||
// Import image decoders for common formats.
|
||||
_ "golang.org/x/image/bmp"
|
||||
_ "golang.org/x/image/tiff"
|
||||
_ "golang.org/x/image/webp"
|
||||
_ "image/gif"
|
||||
_ "image/jpeg"
|
||||
_ "image/png"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
@@ -90,13 +71,7 @@ func maybeDispatchVideo(
|
||||
fmt.Errorf("Parser: video parsing is not yet supported; underlying video analysis capability is pending")
|
||||
}
|
||||
|
||||
// Image dispatch: OCR + IMAGE2TEXT vision describe ---
|
||||
// Mirrors Python's rag/app/picture.py:chunk() image branch:
|
||||
// 1. Try PaddleOCR if layout_recognize is "@PaddleOCR"
|
||||
// 2. Fallback to local ONNX OCR (DeepDoc /predict/ocr endpoint)
|
||||
// 3. If OCR text is short (≤32 chars or ≤32 English words),
|
||||
// also call IMAGE2TEXT VLM describe()
|
||||
// 4. Returns combined text
|
||||
// Image dispatch: IMAGE2TEXT vision describe ---
|
||||
|
||||
func maybeDispatchImage(
|
||||
ctx context.Context,
|
||||
@@ -106,6 +81,7 @@ func maybeDispatchImage(
|
||||
binary []byte,
|
||||
inputs map[string]any,
|
||||
setups map[string]schema.ParserSetup,
|
||||
enableVisionEnhancement bool,
|
||||
) (parser.ParseResult, bool, error) {
|
||||
if fileType != utility.FileTypeVISUAL {
|
||||
return parser.ParseResult{}, false, nil
|
||||
@@ -115,56 +91,52 @@ func maybeDispatchImage(
|
||||
return parser.ParseResult{}, false, nil
|
||||
}
|
||||
tenantID := getStringOr(inputs, "tenant_id", "")
|
||||
if tenantID == "" {
|
||||
return parser.ParseResult{}, true,
|
||||
fmt.Errorf("parser: image requires tenant_id")
|
||||
if !enableVisionEnhancement {
|
||||
return parser.ParseResult{}, true, fmt.Errorf("parser: image has no searchable text because vision enhancement is disabled")
|
||||
}
|
||||
|
||||
// --- Phase 1: OCR ---
|
||||
var ocrText string
|
||||
|
||||
// Step 1a: Try PaddleOCR if layout_recognize is set to PaddleOCR.
|
||||
// Mirrors Python's picture.py:_try_paddleocr_image().
|
||||
layoutRecognize := getStringOr(setup, "layout_recognize", "")
|
||||
if layoutRecognize != "" {
|
||||
recognizer, _ := normalizeLayoutRecognizer(layoutRecognize)
|
||||
if recognizer == "PaddleOCR" {
|
||||
if txt, err := runPaddleOCRImage(binary, filename); err == nil && txt != "" {
|
||||
ocrText = txt
|
||||
parsed := dispatchParse(ctx, fileType, filename, binary, setups)
|
||||
if parsed.Err != nil {
|
||||
return parsed, true, parsed.Err
|
||||
}
|
||||
if len(parsed.JSON) == 0 {
|
||||
return parser.ParseResult{}, true, fmt.Errorf("parser: image parser returned no image item")
|
||||
}
|
||||
imageData, _ := parsed.JSON[0]["image"].(string)
|
||||
if imageData == "" {
|
||||
return parser.ParseResult{}, true, fmt.Errorf("parser: image parser returned no image payload")
|
||||
}
|
||||
result, handled, err := maybeDispatchImageVLM(ctx, db, imageData, tenantID, setup, inputs)
|
||||
result.Warnings = append(result.Warnings, parsed.Warnings...)
|
||||
result.File = parsed.File
|
||||
if err == nil && len(result.JSON) > 0 {
|
||||
text, _ := result.JSON[0]["text"].(string)
|
||||
if strings.TrimSpace(text) == "" {
|
||||
if len(result.Warnings) > 0 {
|
||||
return result, true, fmt.Errorf("parser: image has no searchable text: %s", result.Warnings[0])
|
||||
}
|
||||
return result, true, fmt.Errorf("parser: vision enhancement returned no searchable text")
|
||||
}
|
||||
}
|
||||
return result, handled, err
|
||||
}
|
||||
|
||||
// Step 1b: Fallback to local ONNX OCR (DeepDoc /predict/ocr).
|
||||
// Mirrors Python's picture.py:ocr(np.array(img)) from deepdoc.vision.
|
||||
if ocrText == "" {
|
||||
if txt, err := runLocalImageOCR(binary); err == nil && txt != "" {
|
||||
ocrText = txt
|
||||
}
|
||||
}
|
||||
|
||||
// The image family always emits a structured JSON item carrying the
|
||||
// image attachment (data URI) and doc_type_kwd, mirroring Python
|
||||
// rag/app/picture.py:71-72 (doc["image"]=img, doc["doc_type_kwd"]=
|
||||
// "image"). picture.py has no "text" output mode — it always returns
|
||||
// a structured doc — so output_format is hardcoded to "json" and any
|
||||
// setup override is ignored. The former behavior returned a bare Text
|
||||
// string, which dropped the image attachment, set doc_type to "text",
|
||||
// and on the default json path produced JSON=nil so downstream
|
||||
// Chunkers rejected the payload with errRequiredField{"json"}.
|
||||
imageB64 := base64.StdEncoding.EncodeToString(binary)
|
||||
dataURI := "data:" + imageMIME(filename) + ";base64," + imageB64
|
||||
|
||||
// --- Phase 2: VLM description (when OCR text is short) ---
|
||||
// Mirrors Python's check: if (eng and len(txt.split()) > 32) or len(txt) > 32
|
||||
// then use OCR text only; otherwise call cv_mdl.describe().
|
||||
func maybeDispatchImageVLM(
|
||||
ctx context.Context,
|
||||
db *gorm.DB,
|
||||
dataURI string,
|
||||
tenantID string,
|
||||
setup schema.ParserSetup,
|
||||
inputs map[string]any,
|
||||
) (parser.ParseResult, bool, error) {
|
||||
// --- Optional VLM description ---
|
||||
lang := resolveVisionLanguage(inputs, getStringOr(setup, "lang", ""))
|
||||
if vlmGateShouldSkip(ocrText, lang) {
|
||||
// OCR returned substantial text — skip VLM.
|
||||
return imageDispatchResult(ocrText, dataURI), true, nil
|
||||
if tenantID == "" {
|
||||
result := imageDispatchResult("", dataURI)
|
||||
result.Warnings = append(result.Warnings, "image VLM enhancement skipped: tenant ID is missing")
|
||||
return result, true, nil
|
||||
}
|
||||
|
||||
// Short OCR text (or no text): supplement with VLM describe.
|
||||
// Use the configured image VLM or the tenant default.
|
||||
modelRef := configuredMediaModelID(setup, "image")
|
||||
var driver modelModule.ModelDriver
|
||||
var modelName string
|
||||
@@ -180,13 +152,13 @@ func maybeDispatchImage(
|
||||
} else {
|
||||
driver, modelName, apiConfig, _, err = resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeImage2Text)
|
||||
}
|
||||
if err == nil && driver == nil {
|
||||
err = fmt.Errorf("no usable vision model")
|
||||
}
|
||||
if err != nil {
|
||||
// If VLM is unavailable, but we have OCR text, return it.
|
||||
if ocrText != "" {
|
||||
return imageDispatchResult(ocrText, dataURI), true, nil
|
||||
}
|
||||
return parser.ParseResult{}, true,
|
||||
fmt.Errorf("parser: picture image2text model: %w", err)
|
||||
result := imageDispatchResult("", dataURI)
|
||||
result.Warnings = append(result.Warnings, fmt.Sprintf("image VLM enhancement skipped: model unavailable: %v", err))
|
||||
return result, true, nil
|
||||
}
|
||||
|
||||
prompt := defaultImageVisionPrompt(lang)
|
||||
@@ -206,32 +178,20 @@ func maybeDispatchImage(
|
||||
vision := true
|
||||
resp, err := driver.ChatWithMessages(ctx, modelName, messages, apiConfig, &modelModule.ChatConfig{Vision: &vision}, nil)
|
||||
if err != nil {
|
||||
if ocrText != "" {
|
||||
return imageDispatchResult(ocrText, dataURI), true, nil
|
||||
}
|
||||
return parser.ParseResult{}, true,
|
||||
fmt.Errorf("parser: picture describe: %w", err)
|
||||
result := imageDispatchResult("", dataURI)
|
||||
result.Warnings = append(result.Warnings, fmt.Sprintf("image VLM enhancement failed: %v", err))
|
||||
return result, true, nil
|
||||
}
|
||||
vlmText := ""
|
||||
if resp != nil && resp.Answer != nil {
|
||||
vlmText = strings.TrimSpace(*resp.Answer)
|
||||
}
|
||||
|
||||
// Combine OCR + VLM text.
|
||||
// Mirrors Python: txt += "\n" + ans
|
||||
combined := ocrText
|
||||
if vlmText != "" {
|
||||
if combined != "" {
|
||||
combined += "\n" + vlmText
|
||||
} else {
|
||||
combined = vlmText
|
||||
}
|
||||
}
|
||||
return imageDispatchResult(combined, dataURI), true, nil
|
||||
return imageDispatchResult(vlmText, dataURI), true, nil
|
||||
}
|
||||
|
||||
// imageDispatchResult builds the structured JSON payload for the image
|
||||
// family: a single item carrying the combined text, the image attachment
|
||||
// family: a single item carrying the VLM description, the image attachment
|
||||
// (data URI), and doc_type_kwd "image". Mirrors Python
|
||||
// rag/app/picture.py:71-72.
|
||||
func imageDispatchResult(text, dataURI string) parser.ParseResult {
|
||||
@@ -336,59 +296,6 @@ func writeTempAudioFile(filename string, binary []byte) (string, error) {
|
||||
return tmp.Name(), nil
|
||||
}
|
||||
|
||||
// normalizeLayoutRecognizer parses layout_recognize strings like
|
||||
// "model@PaddleOCR" → ("PaddleOCR", "model@PaddleOCR").
|
||||
// Mirrors Python's common/parser_config_utils.py:normalize_layout_recognizer().
|
||||
func normalizeLayoutRecognizer(raw string) (recognizer, modelName string) {
|
||||
lowered := strings.ToLower(raw)
|
||||
if strings.HasSuffix(lowered, "@paddleocr") {
|
||||
return "PaddleOCR", raw
|
||||
}
|
||||
if strings.HasSuffix(lowered, "@mineru") {
|
||||
return "MinerU", raw
|
||||
}
|
||||
if strings.HasSuffix(lowered, "@somark") {
|
||||
return "SoMark", raw
|
||||
}
|
||||
if strings.HasSuffix(lowered, "@opendataloader") {
|
||||
return "OpenDataLoader", raw
|
||||
}
|
||||
return raw, ""
|
||||
}
|
||||
|
||||
// imageMIME maps common image filename extensions to MIME types
|
||||
// for constructing base64 data URIs.
|
||||
func imageMIME(filename string) string {
|
||||
dot := strings.LastIndex(filename, ".")
|
||||
if dot == -1 {
|
||||
return "image/png"
|
||||
}
|
||||
switch strings.ToLower(filename[dot+1:]) {
|
||||
case "jpg", "jpeg":
|
||||
return "image/jpeg"
|
||||
case "png":
|
||||
return "image/png"
|
||||
case "gif":
|
||||
return "image/gif"
|
||||
case "bmp":
|
||||
return "image/bmp"
|
||||
case "webp":
|
||||
return "image/webp"
|
||||
case "svg":
|
||||
return "image/svg+xml"
|
||||
case "tiff", "tif":
|
||||
return "image/tiff"
|
||||
case "ico":
|
||||
return "image/x-icon"
|
||||
case "avif":
|
||||
return "image/avif"
|
||||
case "heic":
|
||||
return "image/heic"
|
||||
default:
|
||||
return "image/png"
|
||||
}
|
||||
}
|
||||
|
||||
// videoMIME maps common video filename extensions to MIME types
|
||||
// for constructing base64 data URIs. Retained as a reference for the
|
||||
// future real video-parsing implementation (provider-specific frame
|
||||
@@ -422,196 +329,3 @@ func videoMIME(filename string) string {
|
||||
return "video/mp4"
|
||||
}
|
||||
}
|
||||
|
||||
// --- OCR helpers for picture dispatch ---
|
||||
|
||||
// vlmGateShouldSkip determines whether the OCR text is substantial enough
|
||||
// to skip the secondary VLM description call.
|
||||
// Mirrors Python picture.py:chunk():
|
||||
//
|
||||
// txt = txt.strip()
|
||||
// if (eng and len(txt.split()) > 32) or len(txt) > 32 -> skip VLM
|
||||
func vlmGateShouldSkip(ocrText, lang string) bool {
|
||||
ocrText = strings.TrimSpace(ocrText)
|
||||
if ocrText == "" {
|
||||
return false
|
||||
}
|
||||
eng := strings.EqualFold(lang, "english")
|
||||
wordCount := len(strings.Fields(ocrText))
|
||||
charCount := utf8.RuneCountInString(ocrText)
|
||||
return (eng && wordCount > 32) || charCount > 32
|
||||
}
|
||||
|
||||
// runPaddleOCRImage tries PaddleOCR remote API for image text extraction.
|
||||
// Mirrors Python's picture.py:_try_paddleocr_image() which creates a
|
||||
// PaddleOCRParser and calls parse_image().
|
||||
func runPaddleOCRImage(binary []byte, filename string) (string, error) {
|
||||
client := parser.NewPaddleOCRClientFromEnv()
|
||||
if !client.Enabled() {
|
||||
return "", fmt.Errorf("paddleocr: not configured (set PADDLEOCR_ACCESS_TOKEN)")
|
||||
}
|
||||
return client.ParseImage(binary, filename)
|
||||
}
|
||||
|
||||
// runLocalImageOCR detects and recognizes text in an image using the
|
||||
// in-process DeepDoc analyzer (ONNX models served locally via the native
|
||||
// backend). Mirrors Python's deepdoc.vision.OCR local ONNX pipeline; the
|
||||
// external HTTP service is no longer a backend (see parser.GetDocAnalyzer).
|
||||
//
|
||||
// Pipeline:
|
||||
// 1. Decode image bytes → image.Image
|
||||
// 2. OCRDetect → find text region boxes
|
||||
// 3. For each box: crop → OCRRecognize → text
|
||||
// 4. Sort boxes by Y, then X (reading order)
|
||||
// 5. Join all recognized text with newlines
|
||||
func runLocalImageOCR(binary []byte) (string, error) {
|
||||
analyzer, err := parser.GetDocAnalyzer()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("local OCR: %w", err)
|
||||
}
|
||||
|
||||
img, _, err := image.Decode(bytes.NewReader(binary))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("local OCR: decode image: %w", err)
|
||||
}
|
||||
|
||||
// Step 1: Detect text regions.
|
||||
ctx := context.Background()
|
||||
boxes, err := analyzer.OCRDetect(ctx, img)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("local OCR: detect: %w", err)
|
||||
}
|
||||
if len(boxes) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// Step 2: Sort boxes by Y (top to bottom), then X (left to right)
|
||||
// for reading-order text assembly.
|
||||
sort.Slice(boxes, func(i, j int) bool {
|
||||
yi := (boxes[i].Y0 + boxes[i].Y2) / 2
|
||||
yj := (boxes[j].Y0 + boxes[j].Y2) / 2
|
||||
if yi < yj {
|
||||
return true
|
||||
}
|
||||
if yi > yj {
|
||||
return false
|
||||
}
|
||||
return boxes[i].X0 < boxes[j].X0
|
||||
})
|
||||
|
||||
// Step 3: Recognize text per box.
|
||||
var texts []string
|
||||
bounds := img.Bounds()
|
||||
for _, box := range boxes {
|
||||
// Convert quad box to axis-aligned crop rect.
|
||||
x0 := int(min4(box.X0, box.X1, box.X2, box.X3))
|
||||
y0 := int(min4(box.Y0, box.Y1, box.Y2, box.Y3))
|
||||
x1 := int(max4(box.X0, box.X1, box.X2, box.X3))
|
||||
y1 := int(max4(box.Y0, box.Y1, box.Y2, box.Y3))
|
||||
|
||||
// Clamp to image bounds.
|
||||
if x0 < bounds.Min.X {
|
||||
x0 = bounds.Min.X
|
||||
}
|
||||
if y0 < bounds.Min.Y {
|
||||
y0 = bounds.Min.Y
|
||||
}
|
||||
if x1 > bounds.Max.X {
|
||||
x1 = bounds.Max.X
|
||||
}
|
||||
if y1 > bounds.Max.Y {
|
||||
y1 = bounds.Max.Y
|
||||
}
|
||||
if x1 <= x0 || y1 <= y0 {
|
||||
continue
|
||||
}
|
||||
|
||||
// Crop the region. This requires an image type that supports
|
||||
// cropping; for simplicity we recode through a sub-image.
|
||||
crop := cropImage(img, x0, y0, x1, y1)
|
||||
if crop == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
recTexts, err := analyzer.OCRRecognize(ctx, crop)
|
||||
if err != nil {
|
||||
continue // skip boxes that fail recognition
|
||||
}
|
||||
for _, t := range recTexts {
|
||||
s := strings.TrimSpace(t.Text)
|
||||
if s != "" {
|
||||
texts = append(texts, s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(texts) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
return strings.Join(texts, "\n"), nil
|
||||
}
|
||||
|
||||
// cropImage extracts a sub-rectangle from img. Works with any image.Image
|
||||
// by converting to RGBA if needed, then cropping.
|
||||
func cropImage(img image.Image, x0, y0, x1, y1 int) image.Image {
|
||||
bounds := img.Bounds()
|
||||
cropRect := image.Rect(
|
||||
bounds.Min.X+x0, bounds.Min.Y+y0,
|
||||
bounds.Min.X+x1, bounds.Min.Y+y1,
|
||||
)
|
||||
switch src := img.(type) {
|
||||
case *image.RGBA:
|
||||
return src.SubImage(cropRect)
|
||||
case *image.NRGBA:
|
||||
return src.SubImage(cropRect)
|
||||
case *image.RGBA64:
|
||||
return src.SubImage(cropRect)
|
||||
case *image.NRGBA64:
|
||||
return src.SubImage(cropRect)
|
||||
case *image.Gray:
|
||||
return src.SubImage(cropRect)
|
||||
case *image.Gray16:
|
||||
return src.SubImage(cropRect)
|
||||
case *image.YCbCr:
|
||||
return src.SubImage(cropRect)
|
||||
case *image.Paletted:
|
||||
return src.SubImage(cropRect)
|
||||
default:
|
||||
// Convert to RGBA for cropping.
|
||||
rgba := image.NewRGBA(cropRect)
|
||||
for y := cropRect.Min.Y; y < cropRect.Max.Y; y++ {
|
||||
for x := cropRect.Min.X; x < cropRect.Max.X; x++ {
|
||||
rgba.Set(x, y, img.At(x, y))
|
||||
}
|
||||
}
|
||||
return rgba
|
||||
}
|
||||
}
|
||||
|
||||
func min4(a, b, c, d float64) float64 {
|
||||
m := a
|
||||
if b < m {
|
||||
m = b
|
||||
}
|
||||
if c < m {
|
||||
m = c
|
||||
}
|
||||
if d < m {
|
||||
m = d
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func max4(a, b, c, d float64) float64 {
|
||||
m := a
|
||||
if b > m {
|
||||
m = b
|
||||
}
|
||||
if c > m {
|
||||
m = c
|
||||
}
|
||||
if d > m {
|
||||
m = d
|
||||
}
|
||||
return m
|
||||
}
|
||||
@@ -34,6 +34,17 @@ import (
|
||||
"ragflow/internal/utility"
|
||||
)
|
||||
|
||||
func TestMaybeDispatchImageWithoutVisionReportsNoContent(t *testing.T) {
|
||||
result, handled, err := maybeDispatchImage(t.Context(), dao.DB, utility.FileTypeVISUAL,
|
||||
"photo.png", []byte("image bytes"), nil, defaultSetups(), false)
|
||||
if !handled {
|
||||
t.Fatal("image was not handled")
|
||||
}
|
||||
if err == nil || !strings.Contains(err.Error(), "vision enhancement") {
|
||||
t.Fatalf("error = %v, want explicit vision requirement; result = %+v", err, result)
|
||||
}
|
||||
}
|
||||
|
||||
// imagePromptCaptureDriver embeds ModelDriver so it satisfies the interface
|
||||
// without listing every method; only ChatWithMessages is overridden to record
|
||||
// the messages the image branch selected.
|
||||
@@ -41,11 +52,13 @@ type imagePromptCaptureDriver struct {
|
||||
modelModule.ModelDriver
|
||||
mu sync.Mutex
|
||||
captured []modelModule.Message
|
||||
ctxErr error
|
||||
}
|
||||
|
||||
func (d *imagePromptCaptureDriver) ChatWithMessages(ctx context.Context, modelName string, messages []modelModule.Message, apiConfig *modelModule.APIConfig, chatModelConfig *modelModule.ChatConfig, usage *common.ModelUsage) (*modelModule.ChatResponse, error) {
|
||||
d.mu.Lock()
|
||||
d.captured = append(d.captured, messages...)
|
||||
d.ctxErr = ctx.Err()
|
||||
d.mu.Unlock()
|
||||
ans := "captured"
|
||||
return &modelModule.ChatResponse{Answer: &ans}, nil
|
||||
@@ -107,6 +120,7 @@ func TestMaybeDispatchImage_UsesSystemPrompt(t *testing.T) {
|
||||
[]byte("not-a-real-image"),
|
||||
map[string]any{"tenant_id": "t1"},
|
||||
setups,
|
||||
true,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("maybeDispatchImage: %v", err)
|
||||
@@ -158,6 +172,7 @@ func TestMaybeDispatchImage_DefaultPromptUsesDatasetLanguage(t *testing.T) {
|
||||
[]byte("not-a-real-image"),
|
||||
map[string]any{"tenant_id": "t1", "lang": "Japanese"},
|
||||
setups,
|
||||
true,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("maybeDispatchImage: %v", err)
|
||||
@@ -200,6 +215,7 @@ func TestMaybeDispatchImage_ReturnsJSONWithImage(t *testing.T) {
|
||||
[]byte("not-a-real-image"),
|
||||
map[string]any{"tenant_id": "t1"},
|
||||
setups,
|
||||
true,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("maybeDispatchImage: %v", err)
|
||||
@@ -222,7 +238,7 @@ func TestMaybeDispatchImage_ReturnsJSONWithImage(t *testing.T) {
|
||||
t.Errorf("image = %q, want a data URI (data:<mime>;base64,<b64>)", img)
|
||||
}
|
||||
if txt, _ := item["text"].(string); txt == "" {
|
||||
t.Errorf("text field empty; want non-empty combined OCR+VLM text")
|
||||
t.Errorf("text field empty; want non-empty VLM description")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -251,6 +267,7 @@ func TestMaybeDispatchImage_HardcodesJSONOutput(t *testing.T) {
|
||||
[]byte("not-a-real-image"),
|
||||
map[string]any{"tenant_id": "t1"},
|
||||
setups,
|
||||
true,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("maybeDispatchImage: %v", err)
|
||||
@@ -442,6 +459,7 @@ func TestMaybeDispatchImage_UsesConfiguredVLMModel(t *testing.T) {
|
||||
[]byte("not-a-real-image"),
|
||||
map[string]any{"tenant_id": "t1"},
|
||||
setups,
|
||||
true,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("maybeDispatchImage: %v", err)
|
||||
@@ -516,12 +534,12 @@ func TestMaybeDispatchAudio_UsesConfiguredModel(t *testing.T) {
|
||||
}
|
||||
|
||||
// TestImageDecoders_RegisteredFormats validates that image decoders for WebP, BMP,
|
||||
// TIFF, PNG, JPEG, and GIF are registered by media_dispatch.go and can decode
|
||||
// TIFF, PNG, JPEG, and GIF are registered by vision_image.go and can decode
|
||||
// their respective binary payloads via image.Decode without importing the decoder
|
||||
// packages directly in the test file.
|
||||
func TestImageDecoders_RegisteredFormats(t *testing.T) {
|
||||
// Fixed binary fixtures for image formats decoded via decoders registered in
|
||||
// media_dispatch.go (neither standard library nor x/image decoders are imported
|
||||
// vision_image.go (neither standard library nor x/image decoders are imported
|
||||
// in this test file).
|
||||
const (
|
||||
webpB64 = "UklGRiQAAABXRUJQVlA4IBgAAAAwAQCdASoBAAEAD8D+JaQAA3AA/ua1AAA="
|
||||
@@ -563,93 +581,3 @@ func TestImageDecoders_RegisteredFormats(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestVLMGateShouldSkip verifies the rune vs word count threshold
|
||||
// for skipping VLM description. Specifically:
|
||||
// - Zero-allocation rune counting via utf8.RuneCountInString correctly handles multi-byte UTF-8.
|
||||
// - Whitespace trimming matches Python txt.strip(): surrounding whitespace is trimmed before counting.
|
||||
// - CJK text is measured in unicode runes: 12 CJK characters occupy 36 bytes (>32 bytes)
|
||||
// but only 12 runes (<=32 runes), so VLM must NOT be skipped.
|
||||
// - CJK text >32 runes (e.g. 33 runes) skips VLM.
|
||||
// - CJK exact boundary text (32 runes) triggers VLM.
|
||||
// - English text >32 words skips VLM.
|
||||
// - English short text (<=32 words and <=32 chars) triggers VLM.
|
||||
// - English text with <=32 words but >32 chars skips VLM.
|
||||
func TestVLMGateShouldSkip(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
lang string
|
||||
ocrText string
|
||||
wantSkip bool
|
||||
}{
|
||||
{
|
||||
name: "empty text does not skip",
|
||||
lang: "Chinese",
|
||||
ocrText: "",
|
||||
wantSkip: false,
|
||||
},
|
||||
{
|
||||
name: "whitespace only text does not skip",
|
||||
lang: "Chinese",
|
||||
ocrText: " \n\t ",
|
||||
wantSkip: false,
|
||||
},
|
||||
{
|
||||
name: "CJK substantial text (>32 runes, >32 bytes) skips VLM",
|
||||
lang: "Chinese",
|
||||
ocrText: strings.Repeat("中", 33), // 33 runes, 99 bytes
|
||||
wantSkip: true,
|
||||
},
|
||||
{
|
||||
name: "CJK short text with >32 bytes but <=32 runes triggers VLM",
|
||||
lang: "Chinese",
|
||||
ocrText: strings.Repeat("中", 12), // 12 runes, 36 bytes (>32 bytes)
|
||||
wantSkip: false,
|
||||
},
|
||||
{
|
||||
name: "CJK exact boundary text (32 runes, 96 bytes) triggers VLM",
|
||||
lang: "Chinese",
|
||||
ocrText: strings.Repeat("中", 32), // 32 runes, 96 bytes (32 is not > 32)
|
||||
wantSkip: false,
|
||||
},
|
||||
{
|
||||
name: "CJK exact boundary text with whitespace padding trims to <=32 runes and triggers VLM",
|
||||
lang: "Chinese",
|
||||
ocrText: " " + strings.Repeat("中", 32) + " ", // 32 runes after trim
|
||||
wantSkip: false,
|
||||
},
|
||||
{
|
||||
name: "CJK substantial text with whitespace padding trims to >32 runes and skips VLM",
|
||||
lang: "Chinese",
|
||||
ocrText: " " + strings.Repeat("中", 33) + " ", // 33 runes after trim
|
||||
wantSkip: true,
|
||||
},
|
||||
{
|
||||
name: "English substantial text (>32 words) skips VLM",
|
||||
lang: "English",
|
||||
ocrText: strings.Repeat("word ", 33), // 33 words
|
||||
wantSkip: true,
|
||||
},
|
||||
{
|
||||
name: "English short text (<=32 words and <=32 chars) triggers VLM",
|
||||
lang: "English",
|
||||
ocrText: "hello world", // 2 words, 11 chars
|
||||
wantSkip: false,
|
||||
},
|
||||
{
|
||||
name: "English text with <=32 words but >32 chars skips VLM",
|
||||
lang: "English",
|
||||
ocrText: "abcdefghijklmnopqrstuvwxyz01234567", // 1 word, 34 chars (>32 chars)
|
||||
wantSkip: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := vlmGateShouldSkip(tc.ocrText, tc.lang)
|
||||
if got != tc.wantSkip {
|
||||
t.Errorf("vlmGateShouldSkip(%q, %q) = %v, want %v", tc.ocrText, tc.lang, got, tc.wantSkip)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -110,7 +110,8 @@ const pageFormFeed = '\f'
|
||||
// the goroutine that returned from Invoke. The static Param is
|
||||
// read-only after construction.
|
||||
type ParserComponent struct {
|
||||
setups map[string]schema.ParserSetup
|
||||
setups map[string]schema.ParserSetup
|
||||
enableVisionEnhancement bool
|
||||
}
|
||||
|
||||
// NewParserComponent constructs a Parser from a DSL param map.
|
||||
@@ -123,6 +124,7 @@ type ParserComponent struct {
|
||||
// Param map shape (all keys optional):
|
||||
//
|
||||
// {
|
||||
// "enable_vision_enhancement": bool,
|
||||
// "pdf": map[string]any,
|
||||
// "docx": map[string]any,
|
||||
// ...
|
||||
@@ -139,8 +141,16 @@ func NewParserComponent(params map[string]any) (runtime.Component, error) {
|
||||
// Canvases saved by the Python-era frontend nest the per-family setups
|
||||
// under a "setups" key; lift them so every family lands at the top level.
|
||||
params = schema.FlattenLegacyParserSetups(params)
|
||||
var enableVisionEnhancement bool
|
||||
if raw, exists := params["enable_vision_enhancement"]; exists {
|
||||
var ok bool
|
||||
enableVisionEnhancement, ok = raw.(bool)
|
||||
if !ok {
|
||||
return nil, errors.New("parser: enable_vision_enhancement must be a boolean")
|
||||
}
|
||||
}
|
||||
for k, raw := range params {
|
||||
if k == "outputs" || k == "allowed_output_format" {
|
||||
if k == "outputs" || k == "allowed_output_format" || k == "enable_vision_enhancement" {
|
||||
continue
|
||||
}
|
||||
ftCfg, ok := raw.(map[string]any)
|
||||
@@ -155,7 +165,7 @@ func NewParserComponent(params map[string]any) (runtime.Component, error) {
|
||||
}
|
||||
}
|
||||
normalizeParserOutputFormats(s)
|
||||
pc := &ParserComponent{setups: s}
|
||||
pc := &ParserComponent{setups: s, enableVisionEnhancement: enableVisionEnhancement}
|
||||
if err := pc.Check(); err != nil {
|
||||
return nil, fmt.Errorf("parser: %w", err)
|
||||
}
|
||||
@@ -206,11 +216,8 @@ func cloneParserSetupValue(value any) any {
|
||||
}
|
||||
}
|
||||
|
||||
// Check mirrors the applicable subset of Python ParserParam.check()
|
||||
// (rag/flow/parser/parser.py:251-321). Runs at construction time so
|
||||
// a malformed DSL surfaces as a canvas compile failure rather than a
|
||||
// mid-run error. Returns the first validation error encountered
|
||||
// (Python raises ValueError on the first failure).
|
||||
// Check validates parser methods at construction time so a malformed DSL
|
||||
// surfaces as a canvas compile failure rather than a mid-run error.
|
||||
//
|
||||
// NOT covered here (intentional):
|
||||
// - audio/video vlm.llm_id: Python's check() does not validate it
|
||||
@@ -233,11 +240,13 @@ func (c *ParserComponent) Check() error {
|
||||
}
|
||||
}
|
||||
}
|
||||
// image family (parser.py:283-287).
|
||||
// image family (parser.py:283-287). The legacy "ocr" setup value selects
|
||||
// the tenant default vision model; image OCR is not run by this path.
|
||||
if img, ok := c.setups["image"]; ok {
|
||||
pm, _ := img["parse_method"].(string)
|
||||
// OCR mode does not need a VLM language; any other value does.
|
||||
if pm != "ocr" {
|
||||
// A model selected for optional image enhancement needs a language
|
||||
// only when enhancement is enabled.
|
||||
if c.enableVisionEnhancement && pm != "ocr" {
|
||||
if lang, _ := img["lang"].(string); lang == "" {
|
||||
return errors.New("image VLM language does not support empty value")
|
||||
}
|
||||
@@ -472,9 +481,9 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[st
|
||||
}
|
||||
var handledImage bool
|
||||
if !handledVision && !handledMedia {
|
||||
// Image/Picture dispatch: OCR + IMAGE2TEXT vision describe.
|
||||
// Image/Picture dispatch: optional IMAGE2TEXT vision description.
|
||||
// Mirrors Python's rag/app/picture.py:chunk() image branch.
|
||||
dispatched, handledImage, visionErr = maybeDispatchImage(ctx, db, fileTypeExt, filename, binary, inputs, setups)
|
||||
dispatched, handledImage, visionErr = maybeDispatchImage(ctx, db, fileTypeExt, filename, binary, inputs, setups, c.enableVisionEnhancement)
|
||||
if visionErr != nil {
|
||||
return nil, visionErr
|
||||
}
|
||||
@@ -491,14 +500,11 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[st
|
||||
if !handledVision && !handledMedia && !handledImage && !handledAudio {
|
||||
dispatched = dispatchParse(ctx, fileTypeExt, filename, binary, setups)
|
||||
|
||||
// Vision figure enhancement: on the JSON output path,
|
||||
// append vision-model descriptions to embedded image and
|
||||
// table items. Mirrors Python's enhance_media_sections_with_vision
|
||||
// (rag/flow/parser/utils.py:162, called at parser.py:772/978/1115).
|
||||
// Errors (including context cancellation) are intentionally
|
||||
// discarded — enhancement is best-effort, matching Python's
|
||||
// try/except pass pattern.
|
||||
dispatched, _, _ = maybeDispatchVisionEnhancement(ctx, db, fileTypeExt, dispatched, inputs, setups)
|
||||
if c.enableVisionEnhancement {
|
||||
// Enhancement is optional; parser-provided text and image metadata
|
||||
// remain available if a vision model cannot describe an image.
|
||||
dispatched, _, _ = maybeDispatchVisionEnhancement(ctx, db, fileTypeExt, dispatched, inputs, setups)
|
||||
}
|
||||
}
|
||||
// Known/supported families must fail loudly when dispatch or
|
||||
// parsing breaks. Only unknown families keep the raw-text fallback.
|
||||
|
||||
@@ -23,8 +23,8 @@ import (
|
||||
"ragflow/internal/ingestion/component/schema"
|
||||
)
|
||||
|
||||
// TestParserComponent_Check covers the construction-time business
|
||||
// validation for parser methods that require language configuration.
|
||||
// TestParserComponent_Check covers construction-time validation for parser
|
||||
// methods that require language configuration.
|
||||
//
|
||||
// audio/video vlm.llm_id is not validated: Python's check() has no
|
||||
// such branch, and audio resolves its model at dispatch time with a
|
||||
@@ -33,6 +33,7 @@ func TestParserComponent_Check(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
setups map[string]schema.ParserSetup
|
||||
enhance bool
|
||||
wantErr string // non-empty substring expected in error; empty means no error
|
||||
}{
|
||||
// --- PDF family (parser.py:252-261) ---
|
||||
@@ -96,6 +97,7 @@ func TestParserComponent_Check(t *testing.T) {
|
||||
{
|
||||
name: "image: non-ocr without lang → error",
|
||||
setups: map[string]schema.ParserSetup{"image": {"parse_method": "vlm_xyz", "lang": ""}},
|
||||
enhance: true,
|
||||
wantErr: "image VLM language",
|
||||
},
|
||||
{
|
||||
@@ -138,7 +140,7 @@ func TestParserComponent_Check(t *testing.T) {
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
c := &ParserComponent{setups: tc.setups}
|
||||
c := &ParserComponent{setups: tc.setups, enableVisionEnhancement: tc.enhance}
|
||||
err := c.Check()
|
||||
if tc.wantErr != "" {
|
||||
if err == nil {
|
||||
@@ -183,3 +185,28 @@ func TestParserComponent_New_RunsCheck(t *testing.T) {
|
||||
t.Errorf("want nil component on error, got %T", c)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewParserComponentVisionEnhancementSetting(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
params map[string]any
|
||||
enabled bool
|
||||
}{
|
||||
{name: "absent", params: nil},
|
||||
{name: "explicit false", params: map[string]any{"enable_vision_enhancement": false}},
|
||||
{name: "explicit true", params: map[string]any{"enable_vision_enhancement": true}, enabled: true},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
component, err := NewParserComponent(test.params)
|
||||
if err != nil {
|
||||
t.Fatalf("NewParserComponent: %v", err)
|
||||
}
|
||||
if got := component.(*ParserComponent).enableVisionEnhancement; got != test.enabled {
|
||||
t.Fatalf("enableVisionEnhancement = %v, want %v", got, test.enabled)
|
||||
}
|
||||
})
|
||||
}
|
||||
if _, err := NewParserComponent(map[string]any{"enable_vision_enhancement": "true"}); err == nil || !strings.Contains(err.Error(), "must be a boolean") {
|
||||
t.Fatalf("string switch error = %v, want boolean validation", err)
|
||||
}
|
||||
}
|
||||
@@ -280,6 +280,33 @@ func TestChunkDocLegacyRowIRKeysPassThrough(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkDocVisualParentMetadataRoundTrip(t *testing.T) {
|
||||
imageItem := map[string]any{
|
||||
"text": "caption",
|
||||
"doc_type_kwd": "image",
|
||||
"image": "aGVsbG8=",
|
||||
"parent_table_id": "docx-table-1",
|
||||
"row_index": 2,
|
||||
"column_index": 3,
|
||||
"media_order": 4,
|
||||
}
|
||||
doc, err := ChunkDocFromMap(imageItem)
|
||||
if err != nil {
|
||||
t.Fatalf("ChunkDocFromMap: %v", err)
|
||||
}
|
||||
got := doc.ToMap()
|
||||
for key, want := range map[string]any{
|
||||
"parent_table_id": "docx-table-1",
|
||||
"row_index": float64(2),
|
||||
"column_index": float64(3),
|
||||
"media_order": float64(4),
|
||||
} {
|
||||
if got[key] != want {
|
||||
t.Errorf("metadata %q = %v, want %v", key, got[key], want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkerOutputsJSONRoundTrip(t *testing.T) {
|
||||
original := ChunkerOutputs{
|
||||
OutputFormat: PayloadFormatChunks,
|
||||
|
||||
@@ -14,9 +14,9 @@
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
// Package component — vision figure enhancement: enriches parsed JSON items
|
||||
// (PDF / DOCX / Markdown, OutputFormat=="json") with vision-model descriptions
|
||||
// of embedded images/tables. Mirrors Python's enhance_media_sections_with_vision
|
||||
// Package component — vision media enhancement: enriches parsed JSON items
|
||||
// with vision-model descriptions of embedded images/tables.
|
||||
// Mirrors Python's enhance_media_sections_with_vision
|
||||
// (rag/flow/parser/utils.py:162, called at parser.py:772/978/1115).
|
||||
|
||||
package component
|
||||
@@ -26,6 +26,8 @@ import (
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"image"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -48,6 +50,7 @@ import (
|
||||
var (
|
||||
figureVisionPromptBuilder func(language string) (string, error) = buildFigureVisionPrompt
|
||||
visionChatInvoker = defaultVisionChatInvoker
|
||||
visionImageCropperFactory = newVisionImageCropper
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -56,7 +59,8 @@ const (
|
||||
// visionChatTimeout bounds a single VLM call so a hung endpoint cannot
|
||||
// occupy one of the concurrency slots indefinitely. Python wraps the
|
||||
// per-image call in @timeout(30, 3) (deepdoc/parser/figure_parser.py).
|
||||
visionChatTimeout = 30 * time.Second
|
||||
visionChatTimeout = 30 * time.Second
|
||||
visionMediaItemBudget = 10 * time.Second
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -98,60 +102,63 @@ func isUsableVisionImage(raw string) bool {
|
||||
if raw == "" {
|
||||
return false
|
||||
}
|
||||
if len(raw) > maxVLMEncodedBytes+256 {
|
||||
return false
|
||||
}
|
||||
if strings.HasPrefix(raw, "data:image/") {
|
||||
idx := strings.Index(raw, "base64,")
|
||||
if idx < 0 {
|
||||
return false
|
||||
}
|
||||
return isValidBase64(raw[idx+len("base64,"):])
|
||||
return isValidBase64(raw[idx+len("base64,"):], false)
|
||||
}
|
||||
if strings.HasPrefix(raw, "http://") || strings.HasPrefix(raw, "https://") {
|
||||
return true
|
||||
}
|
||||
cleaned := strings.Map(func(r rune) rune {
|
||||
if r == '\r' || r == '\n' || r == ' ' || r == '\t' {
|
||||
return -1
|
||||
if len(raw) > maxVLMEncodedBytes {
|
||||
return false
|
||||
}
|
||||
return isValidBase64(raw, true)
|
||||
}
|
||||
|
||||
func isValidBase64(s string, stripWhitespace bool) bool {
|
||||
if s == "" || len(s) > maxVLMEncodedBytes {
|
||||
return false
|
||||
}
|
||||
for _, encoding := range []*base64.Encoding{base64.StdEncoding, base64.RawStdEncoding} {
|
||||
var source io.Reader = strings.NewReader(s)
|
||||
if stripWhitespace {
|
||||
source = &visionPayloadReader{source: s}
|
||||
}
|
||||
return r
|
||||
}, raw)
|
||||
return isValidBase64(cleaned)
|
||||
decoded := base64.NewDecoder(encoding, source)
|
||||
n, err := io.CopyN(io.Discard, decoded, int64(maxVisionImageBytes)+1)
|
||||
if n > int64(maxVisionImageBytes) {
|
||||
return false
|
||||
}
|
||||
if err == io.EOF {
|
||||
return n > 0
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isValidBase64(s string) bool {
|
||||
if s == "" {
|
||||
return false
|
||||
}
|
||||
if _, err := base64.StdEncoding.DecodeString(s); err == nil {
|
||||
return true
|
||||
}
|
||||
_, err := base64.RawStdEncoding.DecodeString(s)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// isVisionEnhancementAllowed mirrors Python's 3 call sites in
|
||||
// rag/flow/parser/parser.py:772/978/1115 (PDF/DOCX/Markdown JSON branches).
|
||||
func isVisionEnhancementAllowed(fileType utility.FileType) bool {
|
||||
switch fileType {
|
||||
case utility.FileTypePDF, utility.FileTypeDOCX, utility.FileTypeMarkdown:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// visionImageCropper yields a vision-usable base64 image for a parsed item.
|
||||
// Under cgo it crops on demand from the source PDF when the item carries
|
||||
// positions but no inlined image; under !cgo it returns the inlined image
|
||||
// (the only form available without a native renderer). Close releases any
|
||||
// re-acquired engine so native handles are not leaked.
|
||||
// visionImageCropper materializes the VLM payload for a parsed item. Under
|
||||
// cgo it crops PDF sections on demand; inline images stay encoded. Close
|
||||
// releases any re-acquired native engine.
|
||||
type visionImageCropper interface {
|
||||
Crop(item map[string]any) (string, error)
|
||||
Crop(ctx context.Context, item map[string]any) (*visionImage, error)
|
||||
Close() error
|
||||
}
|
||||
|
||||
// maybeDispatchVisionEnhancement enriches parsed JSON items with vision-model
|
||||
// descriptions of embedded images and tables (doc_type_kwd in {"image", "table"}
|
||||
// with non-empty image field).
|
||||
type visionImage struct {
|
||||
Raster image.Image
|
||||
VLMData string
|
||||
VLMDataValidated bool
|
||||
}
|
||||
|
||||
// maybeDispatchVisionEnhancement appends VLM descriptions to parsed image
|
||||
// resources and table regions. PDF OCR remains in the PDF parser; non-PDF
|
||||
// image text comes from this optional enhancement.
|
||||
// Mirrors Python's enhance_media_sections_with_vision in rag/flow/parser/utils.py:162.
|
||||
func maybeDispatchVisionEnhancement(
|
||||
ctx context.Context,
|
||||
@@ -161,159 +168,230 @@ func maybeDispatchVisionEnhancement(
|
||||
inputs map[string]any,
|
||||
setups map[string]schema.ParserSetup,
|
||||
) (parser.ParseResult, bool, error) {
|
||||
// 0. FileType allowlist guard.
|
||||
if !isVisionEnhancementAllowed(fileType) {
|
||||
return dispatched, false, nil
|
||||
}
|
||||
// Only enhance successful JSON output format containing items.
|
||||
if dispatched.Err != nil || dispatched.OutputFormat != "json" || len(dispatched.JSON) == 0 {
|
||||
return dispatched, false, nil
|
||||
}
|
||||
|
||||
tenantID := getStringOr(inputs, "tenant_id", "")
|
||||
if tenantID == "" {
|
||||
return dispatched, false, nil
|
||||
}
|
||||
// Language priority mirrors Python's enhance_media_sections_with_vision
|
||||
// (rag/flow/parser/parser.py:778): the run-level dataset language first
|
||||
// (the Parser pulls it into inputs from Globals), then the family setup's
|
||||
// lang, then English.
|
||||
family := resolveParserFamily(fileType)
|
||||
setup := setups[family]
|
||||
language := resolveVisionLanguage(inputs, getStringOr(setup, "lang", ""))
|
||||
|
||||
// 1. Collect target items (images/tables that can be described). A target
|
||||
// may carry an inlined image (docx/markdown, or any pre-inlined source) or,
|
||||
// under cgo for PDF, only PDF positions — in which case the cropper
|
||||
// re-acquires the source PDF and crops on demand (the parser no longer
|
||||
// inlines PDF media). The cropper transparently handles both forms.
|
||||
type target struct {
|
||||
idx int
|
||||
}
|
||||
var targets []target
|
||||
// Collect visual resources, including Markdown images whose type is text
|
||||
// because flatten_media_to_text is enabled.
|
||||
var items []int
|
||||
for i, item := range dispatched.JSON {
|
||||
kd, _ := item["doc_type_kwd"].(string)
|
||||
if kd != "image" && kd != "table" {
|
||||
continue
|
||||
}
|
||||
if img, _ := item["image"].(string); img != "" {
|
||||
targets = append(targets, target{idx: i})
|
||||
items = append(items, i)
|
||||
continue
|
||||
}
|
||||
if _, ok := parser.ExtractPDFPositions(item); ok {
|
||||
targets = append(targets, target{idx: i})
|
||||
kd, _ := item["doc_type_kwd"].(string)
|
||||
if kd == "image" || kd == "table" {
|
||||
if _, ok := parser.ExtractPDFPositions(item); ok {
|
||||
items = append(items, i)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(targets) == 0 {
|
||||
if len(items) == 0 {
|
||||
return dispatched, false, nil
|
||||
}
|
||||
|
||||
// Acquire the on-demand cropper (cgo: crops from storage; !cgo: returns
|
||||
// the inlined image). Best-effort: a failure here means no vision
|
||||
// enhancement, matching Python's try/except pass.
|
||||
cropper, cerr := newVisionImageCropper(ctx, db, inputs)
|
||||
if cerr != nil {
|
||||
return dispatched, false, nil
|
||||
}
|
||||
defer cropper.Close()
|
||||
|
||||
// 2. Resolve the per-call IMAGE2TEXT model, then fall back to the tenant
|
||||
// default. Mirror Python's vlm_conf["llm_id"] preference.
|
||||
modelRef := configuredMediaModelID(setup, family)
|
||||
// Resolve VLM before materializing image payloads, so disabled or
|
||||
// unavailable enhancement does not spend time loading media.
|
||||
var driver modelModule.ModelDriver
|
||||
var modelName string
|
||||
var apiConfig *modelModule.APIConfig
|
||||
var err error
|
||||
if modelRef != "" {
|
||||
driver, modelName, apiConfig, _, err = resolveModelConfig(ctx, db, tenantID, entity.ModelTypeImage2Text, modelRef)
|
||||
if err != nil {
|
||||
common.Warn("vision enhancement: per-call VLM resolve failed, falling back to tenant default",
|
||||
zap.String("family", family), zap.String("modelRef", modelRef), zap.String("tenant", tenantID), zap.Error(err))
|
||||
var prompt string
|
||||
vlmReady := false
|
||||
var resolveErr error
|
||||
if tenantID != "" {
|
||||
modelRef := configuredMediaModelID(setup, family)
|
||||
var err error
|
||||
if modelRef != "" {
|
||||
driver, modelName, apiConfig, _, err = resolveModelConfig(ctx, db, tenantID, entity.ModelTypeImage2Text, modelRef)
|
||||
if err != nil {
|
||||
common.Warn("vision enhancement: per-call VLM resolve failed, falling back to tenant default",
|
||||
zap.String("family", family), zap.String("modelRef", modelRef), zap.String("tenant", tenantID), zap.Error(err))
|
||||
driver, modelName, apiConfig, _, err = resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeImage2Text)
|
||||
}
|
||||
} else {
|
||||
driver, modelName, apiConfig, _, err = resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeImage2Text)
|
||||
}
|
||||
} else {
|
||||
driver, modelName, apiConfig, _, err = resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeImage2Text)
|
||||
if err == nil {
|
||||
prompt, err = figureVisionPromptBuilder(language)
|
||||
vlmReady = err == nil && driver != nil
|
||||
}
|
||||
resolveErr = err
|
||||
}
|
||||
if err != nil {
|
||||
// Model not available — skip vision enhancement silently, matching Python's try/except pass.
|
||||
if !vlmReady {
|
||||
warning := "vision enhancement skipped: tenant ID is missing"
|
||||
if tenantID != "" {
|
||||
warning = "vision enhancement skipped: no usable vision model"
|
||||
if resolveErr != nil {
|
||||
warning = fmt.Sprintf("vision enhancement skipped: %v", resolveErr)
|
||||
}
|
||||
}
|
||||
dispatched.Warnings = append(dispatched.Warnings, warning)
|
||||
return dispatched, false, nil
|
||||
}
|
||||
|
||||
// Hoist prompt once: language is invariant across items.
|
||||
prompt, perr := figureVisionPromptBuilder(language)
|
||||
if perr != nil {
|
||||
//nolint:nilerr // Vision enhancement is best-effort.
|
||||
// Materialize one resource at a time. The VLM semaphore is acquired before
|
||||
// materialization, so at most visionEnhancementConcurrency encoded payloads
|
||||
// remain live while model requests run.
|
||||
cropper, cerr := visionImageCropperFactory(ctx, db, inputs)
|
||||
if cerr != nil {
|
||||
dispatched.Warnings = append(dispatched.Warnings, fmt.Sprintf("vision enhancement skipped: %v", cerr))
|
||||
return dispatched, false, nil
|
||||
}
|
||||
|
||||
// 3. Concurrently invoke VLM — acquire semaphore before launching goroutine
|
||||
// so live goroutine count is bounded by visionEnhancementConcurrency.
|
||||
// Stop scheduling new VLM calls after cancellation; already-running calls
|
||||
// finish via wg.Wait() and cancellation propagates via ctx through
|
||||
// visionChatInvoker.
|
||||
descriptions := make([]string, len(targets))
|
||||
defer cropper.Close()
|
||||
modified := false
|
||||
descriptions := make([]string, len(items))
|
||||
failures := make([]string, len(items))
|
||||
var wg sync.WaitGroup
|
||||
sem := make(chan struct{}, visionEnhancementConcurrency)
|
||||
|
||||
dispatch:
|
||||
for slot, tg := range targets {
|
||||
if ctx.Err() != nil {
|
||||
break dispatch
|
||||
for slot, itemIdx := range items {
|
||||
if err := ctx.Err(); err != nil {
|
||||
break
|
||||
}
|
||||
vlmSlot := false
|
||||
select {
|
||||
case sem <- struct{}{}:
|
||||
vlmSlot = true
|
||||
case <-ctx.Done():
|
||||
break dispatch
|
||||
// The loop exits below when no slot was acquired.
|
||||
}
|
||||
if !vlmSlot {
|
||||
break
|
||||
}
|
||||
var releaseSlotOnce sync.Once
|
||||
releaseSlot := func() {
|
||||
releaseSlotOnce.Do(func() { <-sem })
|
||||
}
|
||||
var resource *visionImage
|
||||
func() {
|
||||
itemCtx, cancelItem := context.WithTimeout(ctx, visionMediaItemBudget)
|
||||
defer cancelItem()
|
||||
release, err := parser.AcquireImageMedia(itemCtx)
|
||||
if err != nil {
|
||||
if ctx.Err() == nil && vlmReady {
|
||||
if payload, _ := dispatched.JSON[itemIdx]["image"].(string); payload != "" {
|
||||
resource = &visionImage{VLMData: payload}
|
||||
}
|
||||
}
|
||||
if resource == nil && ctx.Err() == nil {
|
||||
failures[slot] = fmt.Sprintf("media admission failed: %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
defer release()
|
||||
item := dispatched.JSON[itemIdx]
|
||||
resource, err = cropper.Crop(itemCtx, item)
|
||||
if err != nil || resource == nil {
|
||||
cropErr := err
|
||||
if ctx.Err() == nil && vlmReady {
|
||||
if payload, _ := item["image"].(string); isUsableVisionImage(payload) {
|
||||
resource = &visionImage{VLMData: payload, VLMDataValidated: true}
|
||||
}
|
||||
}
|
||||
if resource == nil && ctx.Err() == nil {
|
||||
if cropErr != nil {
|
||||
failures[slot] = fmt.Sprintf("image materialization failed: %v", cropErr)
|
||||
} else {
|
||||
failures[slot] = "no usable image payload"
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
if ctx.Err() == nil && vlmReady && resource.VLMData == "" && resource.Raster != nil {
|
||||
resource.VLMData, err = encodeVisionRaster(resource.Raster)
|
||||
if err != nil {
|
||||
resource.VLMData = ""
|
||||
failures[slot] = fmt.Sprintf("image encoding failed: %v", err)
|
||||
} else {
|
||||
resource.VLMDataValidated = resource.VLMData != ""
|
||||
}
|
||||
}
|
||||
}()
|
||||
if err := ctx.Err(); err != nil {
|
||||
releaseSlot()
|
||||
break
|
||||
}
|
||||
if resource == nil {
|
||||
releaseSlot()
|
||||
continue
|
||||
}
|
||||
if !resource.VLMDataValidated && !isUsableVisionImage(resource.VLMData) {
|
||||
if failures[slot] == "" {
|
||||
failures[slot] = "image payload is invalid or exceeds configured limits"
|
||||
}
|
||||
releaseSlot()
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(slot int, itemIdx int) {
|
||||
go func(slot int, imageData string) {
|
||||
defer wg.Done()
|
||||
defer func() { <-sem }()
|
||||
defer releaseSlot()
|
||||
|
||||
img, ierr := cropper.Crop(dispatched.JSON[itemIdx])
|
||||
if ierr != nil || img == "" {
|
||||
return
|
||||
}
|
||||
if !isUsableVisionImage(img) {
|
||||
common.Warn("vision enhancement: invalid image data skipped",
|
||||
zap.Int("item", itemIdx))
|
||||
return
|
||||
}
|
||||
messages := buildVisionMessages(prompt, img)
|
||||
messages := buildVisionMessages(prompt, imageData)
|
||||
if len(messages) == 0 {
|
||||
failures[slot] = "could not build a VLM request"
|
||||
return
|
||||
}
|
||||
resp, ierr := visionChatInvoker(ctx, driver, modelName, messages, apiConfig)
|
||||
if ierr != nil {
|
||||
failures[slot] = fmt.Sprintf("VLM request failed: %v", ierr)
|
||||
return
|
||||
}
|
||||
descriptions[slot] = extractVisionAnswer(resp)
|
||||
}(slot, tg.idx)
|
||||
if strings.TrimSpace(descriptions[slot]) == "" {
|
||||
failures[slot] = "VLM returned an empty description"
|
||||
}
|
||||
}(slot, resource.VLMData)
|
||||
}
|
||||
wg.Wait()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return dispatched, false, err
|
||||
return dispatched, modified, err
|
||||
}
|
||||
failedItems := 0
|
||||
firstFailure := ""
|
||||
for _, failure := range failures {
|
||||
if failure == "" {
|
||||
continue
|
||||
}
|
||||
failedItems++
|
||||
if firstFailure == "" {
|
||||
firstFailure = failure
|
||||
}
|
||||
}
|
||||
if failedItems > 0 {
|
||||
warning := fmt.Sprintf("vision enhancement failed for %d item(s): %s", failedItems, firstFailure)
|
||||
dispatched.Warnings = append(dispatched.Warnings, warning)
|
||||
common.Warn("vision enhancement: one or more image descriptions failed",
|
||||
zap.Int("failed_items", failedItems), zap.String("first_failure", firstFailure))
|
||||
}
|
||||
|
||||
// 4. Append descriptions to item text (single newline \n, matching Python).
|
||||
modified := false
|
||||
for slot, tg := range targets {
|
||||
// Append descriptions after parser-provided text and captions.
|
||||
for slot, itemIdx := range items {
|
||||
desc := strings.TrimSpace(descriptions[slot])
|
||||
if desc == "" {
|
||||
continue
|
||||
}
|
||||
existing, _ := dispatched.JSON[tg.idx]["text"].(string)
|
||||
if existing != "" {
|
||||
dispatched.JSON[tg.idx]["text"] = existing + "\n" + desc
|
||||
} else {
|
||||
dispatched.JSON[tg.idx]["text"] = desc
|
||||
}
|
||||
appendItemText(dispatched.JSON[itemIdx], desc)
|
||||
modified = true
|
||||
}
|
||||
|
||||
return dispatched, modified, nil
|
||||
}
|
||||
|
||||
func appendItemText(item map[string]any, text string) {
|
||||
existing, _ := item["text"].(string)
|
||||
if existing == "" {
|
||||
item["text"] = text
|
||||
return
|
||||
}
|
||||
item["text"] = existing + "\n" + text
|
||||
}
|
||||
|
||||
func buildFigureVisionPrompt(language string) (string, error) {
|
||||
template, err := loadFigureVisionPromptFile(figureVisionPromptFile)
|
||||
if err != nil {
|
||||
|
||||
@@ -20,9 +20,14 @@ package component
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"image"
|
||||
"math"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
"ragflow/internal/common"
|
||||
deepdocpdf "ragflow/internal/deepdoc/parser/pdf"
|
||||
"ragflow/internal/deepdoc/parser/pdf/util"
|
||||
deepdoctype "ragflow/internal/deepdoc/parser/type"
|
||||
@@ -40,6 +45,14 @@ var (
|
||||
visionEngineOpener func(data []byte) (deepdoctype.PDFEngine, error)
|
||||
)
|
||||
|
||||
const visionPDFSourceBudget = 90 * time.Second
|
||||
|
||||
type visionPageSizer interface {
|
||||
PageSize(pageNum int) (float64, float64, error)
|
||||
}
|
||||
|
||||
var _ visionPageSizer = (*deepdocpdf.PDFOxideEngine)(nil)
|
||||
|
||||
func init() {
|
||||
visionSourceFetcher = FetchBinary
|
||||
visionEngineOpener = deepdocpdf.NewEngine
|
||||
@@ -53,43 +66,72 @@ func init() {
|
||||
// chunker does at index time. The engine is opened lazily and at most once per
|
||||
// vision-enhancement call, then released via Close.
|
||||
type visionPDFCropper struct {
|
||||
ctx context.Context
|
||||
db *gorm.DB
|
||||
inputs map[string]any
|
||||
invocationCtx context.Context
|
||||
sourceCtx context.Context
|
||||
cancelSource context.CancelFunc
|
||||
db *gorm.DB
|
||||
inputs map[string]any
|
||||
|
||||
once sync.Once
|
||||
engine deepdoctype.PDFEngine
|
||||
engErr error
|
||||
mu sync.Mutex
|
||||
initialized bool
|
||||
engine deepdoctype.PDFEngine
|
||||
engErr error
|
||||
sourceErr error
|
||||
}
|
||||
|
||||
// newVisionImageCropper builds the on-demand cropper. It never touches storage
|
||||
// here; the source PDF is re-acquired lazily on the first Crop call that needs
|
||||
// it.
|
||||
func newVisionImageCropper(ctx context.Context, db *gorm.DB, inputs map[string]any) (visionImageCropper, error) {
|
||||
return &visionPDFCropper{ctx: ctx, db: db, inputs: inputs}, nil
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
sourceCtx, cancelSource := context.WithTimeout(ctx, visionPDFSourceBudget)
|
||||
return &visionPDFCropper{
|
||||
invocationCtx: ctx,
|
||||
sourceCtx: sourceCtx,
|
||||
cancelSource: cancelSource,
|
||||
db: db,
|
||||
inputs: inputs,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *visionPDFCropper) Crop(item map[string]any) (string, error) {
|
||||
func (c *visionPDFCropper) Crop(ctx context.Context, item map[string]any) (*visionImage, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Fast path: an inlined image (docx/markdown, or any pre-inlined source)
|
||||
// is used directly — no storage access, no engine.
|
||||
if img, _ := item["image"].(string); img != "" {
|
||||
return img, nil
|
||||
return materializeInlineVisionImage(img)
|
||||
}
|
||||
matrix, ok := parser.ExtractPDFPositions(item)
|
||||
if !ok {
|
||||
return "", nil
|
||||
return nil, nil
|
||||
}
|
||||
positions := util.PositionsFromMatrix(matrix)
|
||||
if len(positions) == 0 {
|
||||
return "", nil
|
||||
return nil, nil
|
||||
}
|
||||
if err := c.ensureEngine(); err != nil {
|
||||
// Best-effort: a missing/unreadable source PDF means no vision
|
||||
// description for this item, not a hard failure.
|
||||
return "", nil
|
||||
return nil, nil
|
||||
}
|
||||
if err := c.invocationCtx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Source acquisition has its own invocation-level budget and can outlive
|
||||
// this item's materialization deadline. Once the shared source is ready,
|
||||
// give rendering a fresh per-item budget while still honoring invocation
|
||||
// cancellation.
|
||||
if err := ctx.Err(); err != nil && !errors.Is(err, context.DeadlineExceeded) {
|
||||
return nil, err
|
||||
}
|
||||
renderCtx, cancelRender := context.WithTimeout(c.invocationCtx, visionMediaItemBudget)
|
||||
defer cancelRender()
|
||||
if c.engine == nil {
|
||||
return "", nil
|
||||
return nil, nil
|
||||
}
|
||||
// Render each distinct page the positions span (1-based → 0-based is
|
||||
// handled by PositionsFromMatrix). Reuse the chunker's sliding-window
|
||||
@@ -100,8 +142,14 @@ func (c *visionPDFCropper) Crop(item map[string]any) (string, error) {
|
||||
pages[pn] = struct{}{}
|
||||
}
|
||||
}
|
||||
if !pdfPagesRasterWithinVisionLimits(c.engine, pages) {
|
||||
return nil, nil
|
||||
}
|
||||
single := make(map[int]image.Image, len(pages))
|
||||
for pn := range pages {
|
||||
if err := renderCtx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
img, rerr := deepdocpdf.RenderPageToImage(c.engine, pn)
|
||||
if rerr != nil || img == nil {
|
||||
continue
|
||||
@@ -109,50 +157,116 @@ func (c *visionPDFCropper) Crop(item map[string]any) (string, error) {
|
||||
single[pn] = img
|
||||
}
|
||||
if len(single) == 0 {
|
||||
return "", nil
|
||||
return nil, nil
|
||||
}
|
||||
return util.CropSectionPositions(positions, single, deepdoctype.DlaScale), nil
|
||||
raster := util.CropSectionPositionsRasterLimited(positions, single, deepdoctype.DlaScale, maxVisionImagePixels)
|
||||
if raster == nil {
|
||||
return nil, nil
|
||||
}
|
||||
return &visionImage{Raster: raster}, nil
|
||||
}
|
||||
|
||||
func pdfPagesRasterWithinVisionLimits(engine deepdoctype.PDFEngine, pageNums map[int]struct{}) bool {
|
||||
sizer, ok := engine.(visionPageSizer)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
var totalPixels int64
|
||||
for pageNum := range pageNums {
|
||||
widthPoints, heightPoints, err := sizer.PageSize(pageNum)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
widthPixels := math.Ceil(widthPoints * deepdoctype.DlaScale)
|
||||
heightPixels := math.Ceil(heightPoints * deepdoctype.DlaScale)
|
||||
if math.IsNaN(widthPixels) || math.IsInf(widthPixels, 0) ||
|
||||
math.IsNaN(heightPixels) || math.IsInf(heightPixels, 0) ||
|
||||
widthPixels <= 0 || heightPixels <= 0 ||
|
||||
widthPixels > maxVisionImageEdge || heightPixels > maxVisionImageEdge ||
|
||||
widthPixels*heightPixels > float64(maxVisionImagePixels) {
|
||||
return false
|
||||
}
|
||||
pagePixels := int64(widthPixels * heightPixels)
|
||||
if pagePixels > maxVisionImagePixels-totalPixels {
|
||||
return false
|
||||
}
|
||||
totalPixels += pagePixels
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (c *visionPDFCropper) ensureEngine() error {
|
||||
c.once.Do(func() {
|
||||
data, err := c.acquireSource()
|
||||
if err != nil || len(data) == 0 {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.initialized {
|
||||
return c.engErr
|
||||
}
|
||||
if err := c.sourceCtx.Err(); err != nil {
|
||||
c.initialized = true
|
||||
c.engErr = err
|
||||
c.sourceErr = err
|
||||
return err
|
||||
}
|
||||
data, err := c.acquireSource(c.sourceCtx)
|
||||
if err != nil {
|
||||
c.sourceErr = err
|
||||
if c.sourceCtx.Err() != nil {
|
||||
c.initialized = true
|
||||
c.engErr = err
|
||||
} else if !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) {
|
||||
c.initialized = true
|
||||
c.engErr = err
|
||||
return
|
||||
}
|
||||
// Only PDFs can be cropped. Guard against other binary types so a
|
||||
// docx/markdown item that happens to reach here stays a no-op.
|
||||
if len(data) < 5 || string(data[:5]) != "%PDF-" {
|
||||
return
|
||||
}
|
||||
eng, oerr := visionEngineOpener(data)
|
||||
if oerr != nil {
|
||||
c.engErr = oerr
|
||||
return
|
||||
}
|
||||
c.engine = eng
|
||||
})
|
||||
return err
|
||||
}
|
||||
if err := c.sourceCtx.Err(); err != nil {
|
||||
c.initialized = true
|
||||
c.engErr = err
|
||||
c.sourceErr = err
|
||||
return err
|
||||
}
|
||||
c.sourceErr = nil
|
||||
// Empty and non-PDF sources are deterministic no-op results for this
|
||||
// invocation. A transient fetch error can be retried by a later item while
|
||||
// the shared source context remains active.
|
||||
c.initialized = true
|
||||
if len(data) == 0 || len(data) < 5 || string(data[:5]) != "%PDF-" {
|
||||
return nil
|
||||
}
|
||||
eng, err := visionEngineOpener(data)
|
||||
if err != nil {
|
||||
c.engErr = err
|
||||
c.sourceErr = err
|
||||
return err
|
||||
}
|
||||
c.engine = eng
|
||||
return c.engErr
|
||||
}
|
||||
|
||||
func (c *visionPDFCropper) acquireSource() ([]byte, error) {
|
||||
func (c *visionPDFCropper) acquireSource(ctx context.Context) ([]byte, error) {
|
||||
if bucket, _ := getString(c.inputs, "bucket"); bucket != "" {
|
||||
if path, _ := getString(c.inputs, "path"); path != "" {
|
||||
return visionSourceFetcher(c.ctx, bucket, path)
|
||||
return visionSourceFetcher(ctx, bucket, path)
|
||||
}
|
||||
}
|
||||
if docID, _ := getString(c.inputs, "doc_id"); docID != "" {
|
||||
ref, err := ResolveDocumentStorage(c.ctx, c.db, docID)
|
||||
ref, err := ResolveDocumentStorage(ctx, c.db, docID)
|
||||
if err != nil || ref == nil {
|
||||
return nil, err
|
||||
}
|
||||
return visionSourceFetcher(c.ctx, ref.Bucket, ref.Path)
|
||||
return visionSourceFetcher(ctx, ref.Bucket, ref.Path)
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (c *visionPDFCropper) Close() error {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.cancelSource()
|
||||
if c.sourceErr != nil && c.invocationCtx.Err() == nil {
|
||||
common.Warn("vision enhancement: source PDF unavailable for VLM crop", zap.Error(c.sourceErr))
|
||||
c.sourceErr = nil
|
||||
}
|
||||
if c.engine != nil {
|
||||
return c.engine.Close()
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"context"
|
||||
"image"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
deepdoctype "ragflow/internal/deepdoc/parser/type"
|
||||
)
|
||||
@@ -13,7 +14,10 @@ import (
|
||||
// mockVisionEngine returns a fixed image for any rendered page so the
|
||||
// on-demand cropper can exercise CropSectionPositions without a real PDF.
|
||||
type mockVisionEngine struct {
|
||||
closed *bool
|
||||
closed *bool
|
||||
pageWidth float64
|
||||
pageHeight float64
|
||||
renderCalls *int
|
||||
}
|
||||
|
||||
func (m mockVisionEngine) ExtractChars(int) ([]deepdoctype.TextChar, error) {
|
||||
@@ -21,8 +25,21 @@ func (m mockVisionEngine) ExtractChars(int) ([]deepdoctype.TextChar, error) {
|
||||
}
|
||||
func (m mockVisionEngine) RenderPage(int, float64) ([]byte, error) { return nil, nil }
|
||||
func (m mockVisionEngine) RenderPageImage(int, float64) (image.Image, error) {
|
||||
if m.renderCalls != nil {
|
||||
*m.renderCalls++
|
||||
}
|
||||
return image.NewRGBA(image.Rect(0, 0, 1000, 1000)), nil
|
||||
}
|
||||
func (m mockVisionEngine) PageSize(int) (float64, float64, error) {
|
||||
width, height := m.pageWidth, m.pageHeight
|
||||
if width == 0 {
|
||||
width = 333
|
||||
}
|
||||
if height == 0 {
|
||||
height = 333
|
||||
}
|
||||
return width, height, nil
|
||||
}
|
||||
func (m mockVisionEngine) RawData() []byte { return nil }
|
||||
func (m mockVisionEngine) PageCount() (int, error) { return 1, nil }
|
||||
func (m mockVisionEngine) Outlines() ([]deepdoctype.Outline, error) { return nil, nil }
|
||||
@@ -64,18 +81,90 @@ func TestVisionCropImage_OnDemandFromPositions(t *testing.T) {
|
||||
}
|
||||
defer cropper.Close()
|
||||
|
||||
img, err := cropper.Crop(cgoPositions())
|
||||
img, err := cropper.Crop(context.Background(), cgoPositions())
|
||||
if err != nil {
|
||||
t.Fatalf("Crop: %v", err)
|
||||
}
|
||||
if img == "" {
|
||||
t.Fatal("Crop returned empty; expected an on-demand cropped base64 image")
|
||||
if img == nil || img.Raster == nil {
|
||||
t.Fatal("Crop returned no raster; expected an on-demand cropped image")
|
||||
}
|
||||
if img.Raster.Bounds().Empty() {
|
||||
t.Fatal("Crop returned an empty raster")
|
||||
}
|
||||
if closed {
|
||||
t.Fatal("engine closed before use completed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVisionCropImage_RejectsOversizedPageBeforeRendering(t *testing.T) {
|
||||
renderCalls := 0
|
||||
oldFetcher := visionSourceFetcher
|
||||
oldOpener := visionEngineOpener
|
||||
defer func() {
|
||||
visionSourceFetcher = oldFetcher
|
||||
visionEngineOpener = oldOpener
|
||||
}()
|
||||
visionSourceFetcher = func(context.Context, string, string) ([]byte, error) {
|
||||
return []byte("%PDF-fake-engine-bytes"), nil
|
||||
}
|
||||
visionEngineOpener = func([]byte) (deepdoctype.PDFEngine, error) {
|
||||
return mockVisionEngine{pageWidth: 5000, pageHeight: 1000, renderCalls: &renderCalls}, nil
|
||||
}
|
||||
|
||||
cropper, err := newVisionImageCropper(context.Background(), nil, map[string]any{"bucket": "b", "path": "p"})
|
||||
if err != nil {
|
||||
t.Fatalf("newVisionImageCropper: %v", err)
|
||||
}
|
||||
defer cropper.Close()
|
||||
|
||||
img, err := cropper.Crop(context.Background(), cgoPositions())
|
||||
if err != nil {
|
||||
t.Fatalf("Crop: %v", err)
|
||||
}
|
||||
if img != nil {
|
||||
t.Fatalf("Crop = %#v, want nil for an oversized page", img)
|
||||
}
|
||||
if renderCalls != 0 {
|
||||
t.Fatalf("render calls = %d, want page rejected before raster allocation", renderCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVisionCropImage_RejectsAggregatePagePixelsBeforeRendering(t *testing.T) {
|
||||
renderCalls := 0
|
||||
oldFetcher := visionSourceFetcher
|
||||
oldOpener := visionEngineOpener
|
||||
defer func() {
|
||||
visionSourceFetcher = oldFetcher
|
||||
visionEngineOpener = oldOpener
|
||||
}()
|
||||
visionSourceFetcher = func(context.Context, string, string) ([]byte, error) {
|
||||
return []byte("%PDF-fake-engine-bytes"), nil
|
||||
}
|
||||
visionEngineOpener = func([]byte) (deepdoctype.PDFEngine, error) {
|
||||
return mockVisionEngine{pageWidth: 2000, pageHeight: 2000, renderCalls: &renderCalls}, nil
|
||||
}
|
||||
|
||||
item := map[string]any{
|
||||
"_pdf_positions": [][]any{{[]any{1, 2}, 10.0, 100.0, 10.0, 100.0}},
|
||||
}
|
||||
cropper, err := newVisionImageCropper(context.Background(), nil, map[string]any{"bucket": "b", "path": "p"})
|
||||
if err != nil {
|
||||
t.Fatalf("newVisionImageCropper: %v", err)
|
||||
}
|
||||
defer cropper.Close()
|
||||
|
||||
img, err := cropper.Crop(context.Background(), item)
|
||||
if err != nil {
|
||||
t.Fatalf("Crop: %v", err)
|
||||
}
|
||||
if img != nil {
|
||||
t.Fatalf("Crop = %#v, want nil when combined page pixels exceed the vision budget", img)
|
||||
}
|
||||
if renderCalls != 0 {
|
||||
t.Fatalf("render calls = %d, want pages rejected before raster allocation", renderCalls)
|
||||
}
|
||||
}
|
||||
|
||||
// TestVisionCropImage_PassthroughInline proves that when the item already
|
||||
// carries an inlined image (docx/markdown, or any pre-inlined source), the
|
||||
// cropper returns it directly without touching storage.
|
||||
@@ -95,12 +184,12 @@ func TestVisionCropImage_PassthroughInline(t *testing.T) {
|
||||
defer cropper.Close()
|
||||
|
||||
const inline = "data:image/png;base64,preinlined"
|
||||
img, err := cropper.Crop(map[string]any{"image": inline})
|
||||
img, err := cropper.Crop(context.Background(), map[string]any{"image": inline})
|
||||
if err != nil {
|
||||
t.Fatalf("Crop: %v", err)
|
||||
}
|
||||
if img != inline {
|
||||
t.Fatalf("Crop = %q, want passthrough %q", img, inline)
|
||||
if img == nil || img.VLMData != inline {
|
||||
t.Fatalf("Crop = %#v, want VLM payload %q", img, inline)
|
||||
}
|
||||
if fetchCalled {
|
||||
t.Fatal("storage fetch must not be called for an inlined image")
|
||||
@@ -124,12 +213,12 @@ func TestVisionCropImage_NoImageNoPositions(t *testing.T) {
|
||||
}
|
||||
defer cropper.Close()
|
||||
|
||||
img, err := cropper.Crop(map[string]any{"doc_type_kwd": "image"})
|
||||
img, err := cropper.Crop(context.Background(), map[string]any{"doc_type_kwd": "image"})
|
||||
if err != nil {
|
||||
t.Fatalf("Crop: %v", err)
|
||||
}
|
||||
if img != "" {
|
||||
t.Fatalf("Crop = %q, want empty (no image, no positions)", img)
|
||||
if img != nil {
|
||||
t.Fatalf("Crop = %#v, want nil (no image, no positions)", img)
|
||||
}
|
||||
if fetchCalled {
|
||||
t.Fatal("storage fetch must not be called without positions")
|
||||
@@ -161,12 +250,12 @@ func TestVisionCropImage_NonPDFBytesIsNoOp(t *testing.T) {
|
||||
}
|
||||
defer cropper.Close()
|
||||
|
||||
img, err := cropper.Crop(cgoPositions())
|
||||
img, err := cropper.Crop(context.Background(), cgoPositions())
|
||||
if err != nil {
|
||||
t.Fatalf("Crop: %v", err)
|
||||
}
|
||||
if img != "" {
|
||||
t.Fatalf("Crop = %q, want empty for non-PDF bytes", img)
|
||||
if img != nil {
|
||||
t.Fatalf("Crop = %#v, want nil for non-PDF bytes", img)
|
||||
}
|
||||
if openerCalled {
|
||||
t.Fatal("engine opener must not run on non-PDF bytes")
|
||||
@@ -194,7 +283,7 @@ func TestVisionCropImage_EngineClosedAfterUse(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("newVisionImageCropper: %v", err)
|
||||
}
|
||||
if _, err := cropper.Crop(cgoPositions()); err != nil {
|
||||
if _, err := cropper.Crop(context.Background(), cgoPositions()); err != nil {
|
||||
t.Fatalf("Crop: %v", err)
|
||||
}
|
||||
if err := cropper.Close(); err != nil {
|
||||
@@ -204,3 +293,147 @@ func TestVisionCropImage_EngineClosedAfterUse(t *testing.T) {
|
||||
t.Fatal("engine should be closed after use")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVisionCropImage_RetriesAfterTransientSourceFailure(t *testing.T) {
|
||||
oldFetcher := visionSourceFetcher
|
||||
oldOpener := visionEngineOpener
|
||||
t.Cleanup(func() {
|
||||
visionSourceFetcher = oldFetcher
|
||||
visionEngineOpener = oldOpener
|
||||
})
|
||||
|
||||
fetchCalls := 0
|
||||
visionSourceFetcher = func(context.Context, string, string) ([]byte, error) {
|
||||
fetchCalls++
|
||||
if fetchCalls == 1 {
|
||||
return nil, context.DeadlineExceeded
|
||||
}
|
||||
return []byte("%PDF-fake-engine-bytes"), nil
|
||||
}
|
||||
visionEngineOpener = func([]byte) (deepdoctype.PDFEngine, error) {
|
||||
return mockVisionEngine{}, nil
|
||||
}
|
||||
|
||||
cropper, err := newVisionImageCropper(context.Background(), nil, map[string]any{"bucket": "b", "path": "p"})
|
||||
if err != nil {
|
||||
t.Fatalf("newVisionImageCropper: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = cropper.Close() })
|
||||
|
||||
if _, err := cropper.Crop(context.Background(), cgoPositions()); err != nil {
|
||||
t.Fatalf("first Crop: %v", err)
|
||||
}
|
||||
img, err := cropper.Crop(context.Background(), cgoPositions())
|
||||
if err != nil {
|
||||
t.Fatalf("second Crop: %v", err)
|
||||
}
|
||||
if img == nil || img.Raster == nil {
|
||||
t.Fatal("second Crop returned no raster after the source fetch recovered")
|
||||
}
|
||||
if fetchCalls != 2 {
|
||||
t.Fatalf("source fetch calls = %d, want 2 after transient failure", fetchCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVisionCropImage_ReusesSourceAfterItemContextExpires(t *testing.T) {
|
||||
oldFetcher := visionSourceFetcher
|
||||
oldOpener := visionEngineOpener
|
||||
t.Cleanup(func() {
|
||||
visionSourceFetcher = oldFetcher
|
||||
visionEngineOpener = oldOpener
|
||||
})
|
||||
|
||||
fetchStarted := make(chan struct{})
|
||||
finishFetch := make(chan struct{})
|
||||
fetchCalls := 0
|
||||
visionSourceFetcher = func(ctx context.Context, bucket, path string) ([]byte, error) {
|
||||
fetchCalls++
|
||||
select {
|
||||
case fetchStarted <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
select {
|
||||
case <-finishFetch:
|
||||
return []byte("%PDF-fake-engine-bytes"), nil
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
visionEngineOpener = func([]byte) (deepdoctype.PDFEngine, error) {
|
||||
return mockVisionEngine{}, nil
|
||||
}
|
||||
|
||||
invocationCtx, cancelInvocation := context.WithCancel(context.Background())
|
||||
defer cancelInvocation()
|
||||
cropper, err := newVisionImageCropper(invocationCtx, nil, map[string]any{"bucket": "b", "path": "p"})
|
||||
if err != nil {
|
||||
t.Fatalf("newVisionImageCropper: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = cropper.Close() })
|
||||
|
||||
firstItemCtx, cancelFirstItem := context.WithTimeout(invocationCtx, 20*time.Millisecond)
|
||||
defer cancelFirstItem()
|
||||
firstResult := make(chan struct {
|
||||
img *visionImage
|
||||
err error
|
||||
}, 1)
|
||||
go func() {
|
||||
img, err := cropper.Crop(firstItemCtx, cgoPositions())
|
||||
firstResult <- struct {
|
||||
img *visionImage
|
||||
err error
|
||||
}{img: img, err: err}
|
||||
}()
|
||||
<-fetchStarted
|
||||
<-firstItemCtx.Done()
|
||||
close(finishFetch)
|
||||
first := <-firstResult
|
||||
if first.err != nil {
|
||||
t.Fatalf("first Crop: %v", first.err)
|
||||
}
|
||||
if first.img == nil || first.img.Raster == nil {
|
||||
t.Fatal("first Crop returned no raster after the shared source fetch completed")
|
||||
}
|
||||
|
||||
secondItemCtx, cancelSecondItem := context.WithCancel(invocationCtx)
|
||||
defer cancelSecondItem()
|
||||
second, err := cropper.Crop(secondItemCtx, cgoPositions())
|
||||
if err != nil {
|
||||
t.Fatalf("second Crop: %v", err)
|
||||
}
|
||||
if second == nil || second.Raster == nil {
|
||||
t.Fatal("second Crop returned no raster after the shared source fetch completed")
|
||||
}
|
||||
if fetchCalls != 1 {
|
||||
t.Fatalf("source fetch calls = %d, want 1 across item contexts", fetchCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVisionCropImage_CanceledParentDoesNotRetrySource(t *testing.T) {
|
||||
oldFetcher := visionSourceFetcher
|
||||
t.Cleanup(func() { visionSourceFetcher = oldFetcher })
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
fetchCalls := 0
|
||||
visionSourceFetcher = func(context.Context, string, string) ([]byte, error) {
|
||||
fetchCalls++
|
||||
cancel()
|
||||
return nil, context.Canceled
|
||||
}
|
||||
|
||||
cropper, err := newVisionImageCropper(ctx, nil, map[string]any{"bucket": "b", "path": "p"})
|
||||
if err != nil {
|
||||
t.Fatalf("newVisionImageCropper: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = cropper.Close() })
|
||||
|
||||
if img, err := cropper.Crop(ctx, cgoPositions()); err != nil || img != nil {
|
||||
t.Fatalf("first Crop = (%#v, %v), want best-effort no result after cancellation", img, err)
|
||||
}
|
||||
if _, err := cropper.Crop(ctx, cgoPositions()); err == nil {
|
||||
t.Fatal("subsequent Crop with a canceled parent context should return its cancellation")
|
||||
}
|
||||
if fetchCalls != 1 {
|
||||
t.Fatalf("source fetch calls = %d, want one; canceled parent must prevent retries", fetchCalls)
|
||||
}
|
||||
}
|
||||
@@ -35,11 +35,11 @@ func newVisionImageCropper(_ context.Context, _ *gorm.DB, _ map[string]any) (vis
|
||||
return visionPDFCropperNoop{}, nil
|
||||
}
|
||||
|
||||
func (visionPDFCropperNoop) Crop(item map[string]any) (string, error) {
|
||||
func (visionPDFCropperNoop) Crop(_ context.Context, item map[string]any) (*visionImage, error) {
|
||||
if img, _ := item["image"].(string); img != "" {
|
||||
return img, nil
|
||||
return materializeInlineVisionImage(img)
|
||||
}
|
||||
return "", nil
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (visionPDFCropperNoop) Close() error { return nil }
|
||||
@@ -17,17 +17,24 @@
|
||||
package component
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"image"
|
||||
"image/png"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"ragflow/internal/common"
|
||||
"ragflow/internal/dao"
|
||||
deepdocpdf "ragflow/internal/deepdoc/parser/pdf"
|
||||
deepdoctype "ragflow/internal/deepdoc/parser/type"
|
||||
"ragflow/internal/entity"
|
||||
modelModule "ragflow/internal/entity/models"
|
||||
"ragflow/internal/ingestion/component/schema"
|
||||
@@ -41,6 +48,14 @@ type visionEnhanceFakeDriver struct {
|
||||
modelModule.ModelDriver
|
||||
}
|
||||
|
||||
type failingVisionImageCropper struct{ err error }
|
||||
|
||||
func (c failingVisionImageCropper) Crop(context.Context, map[string]any) (*visionImage, error) {
|
||||
return nil, c.err
|
||||
}
|
||||
|
||||
func (failingVisionImageCropper) Close() error { return nil }
|
||||
|
||||
type visionEnhanceCaptureInvoker struct {
|
||||
mu sync.Mutex
|
||||
images []string
|
||||
@@ -105,6 +120,265 @@ func fakePrompt(language string) (string, error) {
|
||||
return "describe the figure in " + language, nil
|
||||
}
|
||||
|
||||
func visionTestPNGBase64(t *testing.T) string {
|
||||
t.Helper()
|
||||
var encoded bytes.Buffer
|
||||
if err := png.Encode(&encoded, image.NewRGBA(image.Rect(0, 0, 20, 20))); err != nil {
|
||||
t.Fatalf("encode image: %v", err)
|
||||
}
|
||||
return base64.StdEncoding.EncodeToString(encoded.Bytes())
|
||||
}
|
||||
|
||||
func TestParserComponent_VisionEnhancementSwitchGatesEmbeddedImage(t *testing.T) {
|
||||
previousAnalyzer := deepdoctype.NativeDocAnalyzerFactory
|
||||
t.Cleanup(func() { deepdoctype.NativeDocAnalyzerFactory = previousAnalyzer })
|
||||
deepdoctype.SetNativeDocAnalyzerFactory(func() (deepdoctype.DocAnalyzer, bool) {
|
||||
return &deepdocpdf.MockDocAnalyzer{Healthy: true}, true
|
||||
})
|
||||
|
||||
resolverCalls := 0
|
||||
invoker := &visionEnhanceCaptureInvoker{}
|
||||
swapVisionGlobals(t, func(ctx context.Context, db *gorm.DB, tenantID string, modelType entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {
|
||||
resolverCalls++
|
||||
return fakeResolver(ctx, db, tenantID, modelType)
|
||||
}, invoker.invoke, fakePrompt)
|
||||
inputs := func() map[string]any {
|
||||
return map[string]any{
|
||||
"binary": []byte(`<p><img src="data:image/png;base64,` + visionTestPNGBase64(t) + `" alt="figure"></p>`),
|
||||
"file_type": "html",
|
||||
"name": "figure.html",
|
||||
"tenant_id": "tenant-1",
|
||||
}
|
||||
}
|
||||
|
||||
defaultComponent, err := NewParserComponent(nil)
|
||||
if err != nil {
|
||||
t.Fatalf("NewParserComponent(nil): %v", err)
|
||||
}
|
||||
withoutVision, err := defaultComponent.Invoke(t.Context(), nil, inputs())
|
||||
if err != nil {
|
||||
t.Fatalf("default Invoke: %v", err)
|
||||
}
|
||||
if resolverCalls != 0 || len(invoker.images) != 0 {
|
||||
t.Fatalf("default-off enhancement used model: %d resolves, %d VLM calls", resolverCalls, len(invoker.images))
|
||||
}
|
||||
withoutItems, ok := withoutVision["json"].([]map[string]any)
|
||||
if !ok || len(withoutItems) == 0 {
|
||||
t.Fatalf("default JSON = %T/%v", withoutVision["json"], withoutVision["json"])
|
||||
}
|
||||
|
||||
enabledComponent, err := NewParserComponent(map[string]any{"enable_vision_enhancement": true})
|
||||
if err != nil {
|
||||
t.Fatalf("NewParserComponent(enabled): %v", err)
|
||||
}
|
||||
withVision, err := enabledComponent.Invoke(t.Context(), nil, inputs())
|
||||
if err != nil {
|
||||
t.Fatalf("enabled Invoke: %v", err)
|
||||
}
|
||||
if resolverCalls != 1 || len(invoker.images) != 1 {
|
||||
t.Fatalf("enabled enhancement used model %d/%d times, want 1/1", resolverCalls, len(invoker.images))
|
||||
}
|
||||
withItems, ok := withVision["json"].([]map[string]any)
|
||||
if !ok || len(withItems) == 0 {
|
||||
t.Fatalf("enabled JSON = %T/%v", withVision["json"], withVision["json"])
|
||||
}
|
||||
withoutText, _ := withoutItems[0]["text"].(string)
|
||||
withText, _ := withItems[0]["text"].(string)
|
||||
if strings.Contains(withoutText, "a diagram of a pipeline") || !strings.Contains(withText, "a diagram of a pipeline") {
|
||||
t.Fatalf("image texts without/with enhancement = %q / %q", withoutText, withText)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVisionEnhancement_AppendsVLMToExistingText(t *testing.T) {
|
||||
invoker := &visionEnhanceCaptureInvoker{}
|
||||
swapVisionGlobals(t, fakeResolver, invoker.invoke, fakePrompt)
|
||||
|
||||
dispatched := parser.ParseResult{
|
||||
OutputFormat: "json",
|
||||
JSON: []map[string]any{{
|
||||
"text": "Existing caption",
|
||||
"image": visionTestPNGBase64(t),
|
||||
"doc_type_kwd": "image",
|
||||
}},
|
||||
}
|
||||
res, handled, err := maybeDispatchVisionEnhancement(
|
||||
t.Context(), dao.DB, utility.FileTypeXLSX, dispatched,
|
||||
map[string]any{"tenant_id": "t1"}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("maybeDispatchVisionEnhancement: %v", err)
|
||||
}
|
||||
if !handled {
|
||||
t.Fatal("handled = false, want true")
|
||||
}
|
||||
want := "Existing caption\na diagram of a pipeline"
|
||||
if got := res.JSON[0]["text"]; got != want {
|
||||
t.Errorf("enhanced text = %q, want %q", got, want)
|
||||
}
|
||||
if len(invoker.images) != 1 {
|
||||
t.Errorf("VLM calls = %d, want 1", len(invoker.images))
|
||||
}
|
||||
}
|
||||
|
||||
func TestVisionEnhancement_ReportsVLMInvocationFailure(t *testing.T) {
|
||||
swapVisionGlobals(t, fakeResolver, func(context.Context, modelModule.ModelDriver, string, []modelModule.Message, *modelModule.APIConfig) (*modelModule.ChatResponse, error) {
|
||||
return nil, errors.New("VLM unavailable")
|
||||
}, fakePrompt)
|
||||
|
||||
dispatched := parser.ParseResult{
|
||||
OutputFormat: "json",
|
||||
JSON: []map[string]any{{
|
||||
"image": visionTestPNGBase64(t),
|
||||
"doc_type_kwd": "image",
|
||||
}},
|
||||
}
|
||||
result, _, err := maybeDispatchVisionEnhancement(t.Context(), dao.DB, utility.FileTypeDOCX, dispatched,
|
||||
map[string]any{"tenant_id": "t1"}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("maybeDispatchVisionEnhancement: %v", err)
|
||||
}
|
||||
if !strings.Contains(strings.Join(result.Warnings, "\n"), "VLM unavailable") {
|
||||
t.Fatalf("warnings = %v, want the VLM failure", result.Warnings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVisionEnhancement_CropFailureFallsBackToInlineImage(t *testing.T) {
|
||||
invoker := &visionEnhanceCaptureInvoker{}
|
||||
swapVisionGlobals(t, fakeResolver, invoker.invoke, fakePrompt)
|
||||
|
||||
originalFactory := visionImageCropperFactory
|
||||
visionImageCropperFactory = func(context.Context, *gorm.DB, map[string]any) (visionImageCropper, error) {
|
||||
return failingVisionImageCropper{err: errors.New("crop failed")}, nil
|
||||
}
|
||||
t.Cleanup(func() { visionImageCropperFactory = originalFactory })
|
||||
|
||||
dispatched := parser.ParseResult{
|
||||
OutputFormat: "json",
|
||||
JSON: []map[string]any{{
|
||||
"image": visionTestPNGBase64(t),
|
||||
"doc_type_kwd": "image",
|
||||
}},
|
||||
}
|
||||
result, handled, err := maybeDispatchVisionEnhancement(
|
||||
t.Context(), dao.DB, utility.FileTypePDF, dispatched,
|
||||
map[string]any{"tenant_id": "t1"}, map[string]schema.ParserSetup{"pdf": {"parse_method": "mineru"}},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("maybeDispatchVisionEnhancement: %v", err)
|
||||
}
|
||||
if !handled {
|
||||
t.Fatal("handled = false, want inline VLM fallback")
|
||||
}
|
||||
if len(invoker.images) != 1 {
|
||||
t.Fatalf("VLM calls = %d, want 1", len(invoker.images))
|
||||
}
|
||||
if !strings.HasPrefix(invoker.images[0], "data:image/png;base64,") {
|
||||
t.Errorf("VLM image URL = %q, want inline PNG data URI", invoker.images[0])
|
||||
}
|
||||
if got, want := result.JSON[0]["text"], "a diagram of a pipeline"; got != want {
|
||||
t.Errorf("enhanced text = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVisionEnhancement_EnhancesPDFTablesRegardlessOfOCRProvider(t *testing.T) {
|
||||
for _, parseMethod := range []string{"deepdoc", "mineru"} {
|
||||
t.Run(parseMethod, func(t *testing.T) {
|
||||
invoker := &visionEnhanceCaptureInvoker{}
|
||||
swapVisionGlobals(t, fakeResolver, invoker.invoke, fakePrompt)
|
||||
|
||||
dispatched := parser.ParseResult{
|
||||
OutputFormat: "json",
|
||||
JSON: []map[string]any{{
|
||||
"text": "Existing table",
|
||||
"image": visionTestPNGBase64(t),
|
||||
"doc_type_kwd": "table",
|
||||
}},
|
||||
}
|
||||
res, handled, err := maybeDispatchVisionEnhancement(
|
||||
t.Context(), dao.DB, utility.FileTypePDF, dispatched,
|
||||
map[string]any{"tenant_id": "t1"},
|
||||
map[string]schema.ParserSetup{"pdf": {"parse_method": parseMethod}},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("maybeDispatchVisionEnhancement: %v", err)
|
||||
}
|
||||
if !handled {
|
||||
t.Fatal("handled = false, want VLM enhancement")
|
||||
}
|
||||
if got, want := res.JSON[0]["text"], "Existing table\na diagram of a pipeline"; got != want {
|
||||
t.Errorf("enhanced text = %q, want %q", got, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestVisionEnhancement_EnhancesTableAndCellImages(t *testing.T) {
|
||||
invoker := &visionEnhanceCaptureInvoker{}
|
||||
swapVisionGlobals(t, fakeResolver, invoker.invoke, fakePrompt)
|
||||
imagePayload := visionTestPNGBase64(t)
|
||||
dispatched := parser.ParseResult{
|
||||
OutputFormat: "json",
|
||||
JSON: []map[string]any{
|
||||
{"text": "<table><tr><td>figure</td></tr></table>", "image": imagePayload, "doc_type_kwd": "table"},
|
||||
{
|
||||
"text": "cell alt",
|
||||
"image": imagePayload,
|
||||
"doc_type_kwd": "image",
|
||||
"parent_table_id": "html-table-1",
|
||||
"row_index": 1,
|
||||
"column_index": 1,
|
||||
"media_order": 1,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
result, handled, err := maybeDispatchVisionEnhancement(
|
||||
t.Context(), dao.DB, utility.FileTypePDF, dispatched,
|
||||
map[string]any{"tenant_id": "t1"}, map[string]schema.ParserSetup{"pdf": {"parse_method": "deepdoc"}},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("maybeDispatchVisionEnhancement: %v", err)
|
||||
}
|
||||
if !handled {
|
||||
t.Fatal("handled = false, want table and cell image VLM processing")
|
||||
}
|
||||
if got, want := result.JSON[0]["text"], "<table><tr><td>figure</td></tr></table>\na diagram of a pipeline"; got != want {
|
||||
t.Errorf("table text = %q, want %q", got, want)
|
||||
}
|
||||
cellWant := "cell alt\na diagram of a pipeline"
|
||||
if got := result.JSON[1]["text"]; got != cellWant {
|
||||
t.Errorf("cell image text = %q, want %q", got, cellWant)
|
||||
}
|
||||
if len(invoker.images) != 2 {
|
||||
t.Errorf("VLM calls = %d, want table plus cell image", len(invoker.images))
|
||||
}
|
||||
}
|
||||
|
||||
func TestVisionEnhancement_EnhancesExternalPDFImage(t *testing.T) {
|
||||
invoker := &visionEnhanceCaptureInvoker{}
|
||||
swapVisionGlobals(t, fakeResolver, invoker.invoke, fakePrompt)
|
||||
dispatched := parser.ParseResult{
|
||||
OutputFormat: "json",
|
||||
JSON: []map[string]any{{"text": "existing", "image": visionTestPNGBase64(t), "doc_type_kwd": "image"}},
|
||||
}
|
||||
|
||||
result, handled, err := maybeDispatchVisionEnhancement(
|
||||
t.Context(), dao.DB, utility.FileTypePDF, dispatched,
|
||||
map[string]any{"tenant_id": "t1"}, map[string]schema.ParserSetup{"pdf": {"parse_method": "mineru"}},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("maybeDispatchVisionEnhancement: %v", err)
|
||||
}
|
||||
if !handled {
|
||||
t.Fatal("handled = false, want VLM enhancement")
|
||||
}
|
||||
if got, want := result.JSON[0]["text"], "existing\na diagram of a pipeline"; got != want {
|
||||
t.Errorf("image text = %q, want %q", got, want)
|
||||
}
|
||||
if len(invoker.images) != 1 {
|
||||
t.Errorf("VLM calls = %d, want 1", len(invoker.images))
|
||||
}
|
||||
}
|
||||
|
||||
func TestVisionEnhancement_EnhancesJSONImagesAndTables(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
@@ -274,11 +548,13 @@ func TestVisionEnhancement_MarkdownOutputUntouched(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestVisionEnhancement_NonAllowedFileTypeSkipped(t *testing.T) {
|
||||
func TestVisionEnhancement_UsesVisualPayloadRegardlessOfFileType(t *testing.T) {
|
||||
invoker := &visionEnhanceCaptureInvoker{}
|
||||
swapVisionGlobals(t, fakeResolver, invoker.invoke, fakePrompt)
|
||||
dispatched := parser.ParseResult{
|
||||
OutputFormat: "json",
|
||||
JSON: []map[string]any{
|
||||
{"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},
|
||||
{"text": "caption", "image": "aGVsbG8=", "doc_type_kwd": "image"},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -291,11 +567,14 @@ func TestVisionEnhancement_NonAllowedFileTypeSkipped(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if handled {
|
||||
t.Error("handled = true, want false for FileTypeOTHER")
|
||||
if !handled {
|
||||
t.Fatal("handled = false, want true for a visual payload in FileTypeOTHER")
|
||||
}
|
||||
if res.JSON[0]["text"] != "" {
|
||||
t.Errorf("text = %q, want empty", res.JSON[0]["text"])
|
||||
if got, want := res.JSON[0]["text"], "caption\na diagram of a pipeline"; got != want {
|
||||
t.Errorf("text = %q, want %q", got, want)
|
||||
}
|
||||
if len(invoker.images) != 1 {
|
||||
t.Fatalf("VLM calls = %d, want 1", len(invoker.images))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package component
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"image"
|
||||
"image/png"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
_ "golang.org/x/image/bmp"
|
||||
_ "golang.org/x/image/tiff"
|
||||
_ "golang.org/x/image/webp"
|
||||
_ "image/gif"
|
||||
_ "image/jpeg"
|
||||
|
||||
parserlib "ragflow/internal/parser/parser"
|
||||
)
|
||||
|
||||
const (
|
||||
maxVisionImageBytes = parserlib.MaxImagePayloadBytes
|
||||
maxVisionImagePixels = parserlib.MaxImagePixels
|
||||
maxVisionImageEdge = parserlib.MaxImageEdge
|
||||
maxVLMEncodedBytes = (maxVisionImageBytes+2)/3*4 + 256
|
||||
)
|
||||
|
||||
func materializeInlineVisionImage(raw string) (*visionImage, error) {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return nil, nil
|
||||
}
|
||||
materialized := &visionImage{VLMData: raw}
|
||||
if strings.HasPrefix(raw, "http://") || strings.HasPrefix(raw, "https://") {
|
||||
materialized.VLMDataValidated = isUsableVisionImage(raw)
|
||||
return materialized, nil
|
||||
}
|
||||
|
||||
isDataURI := strings.HasPrefix(strings.ToLower(raw), "data:image/")
|
||||
payload := raw
|
||||
if isDataURI {
|
||||
comma := strings.IndexByte(raw, ',')
|
||||
if comma <= 0 || !strings.HasSuffix(strings.ToLower(raw[:comma]), ";base64") {
|
||||
return materialized, nil
|
||||
}
|
||||
payload = raw[comma+1:]
|
||||
} else if strings.HasPrefix(strings.ToLower(raw), "data:") {
|
||||
return materialized, nil
|
||||
}
|
||||
format, err := inspectVisionImagePayload(payload)
|
||||
if err != nil {
|
||||
materialized.VLMDataValidated = false
|
||||
return materialized, nil
|
||||
}
|
||||
materialized.VLMDataValidated = true
|
||||
if strings.ContainsAny(payload, "\r\n \t") {
|
||||
if data, decodeErr := decodeVisionPayload(payload); decodeErr == nil {
|
||||
payload = base64.StdEncoding.EncodeToString(data)
|
||||
}
|
||||
}
|
||||
if isDataURI {
|
||||
if payload != raw[strings.IndexByte(raw, ',')+1:] {
|
||||
materialized.VLMData = raw[:strings.IndexByte(raw, ',')+1] + payload
|
||||
}
|
||||
return materialized, nil
|
||||
}
|
||||
if mimeType := imageMIMEForFormat(format); mimeType != "" {
|
||||
materialized.VLMData = "data:" + mimeType + ";base64," + payload
|
||||
}
|
||||
return materialized, nil
|
||||
}
|
||||
|
||||
func inspectVisionImagePayload(payload string) (string, error) {
|
||||
if payload == "" || len(payload) > maxVLMEncodedBytes {
|
||||
return "", fmt.Errorf("vision image: encoded payload exceeds limit")
|
||||
}
|
||||
for _, encoding := range []*base64.Encoding{base64.StdEncoding, base64.RawStdEncoding} {
|
||||
decoder := base64.NewDecoder(encoding, &visionPayloadReader{source: payload})
|
||||
config, format, err := image.DecodeConfig(io.LimitReader(decoder, maxVisionImageBytes+1))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if config.Width <= 0 || config.Height <= 0 || config.Width > maxVisionImageEdge || config.Height > maxVisionImageEdge || int64(config.Width)*int64(config.Height) > maxVisionImagePixels {
|
||||
return "", fmt.Errorf("vision image: image dimensions %dx%d exceed limits", config.Width, config.Height)
|
||||
}
|
||||
return format, nil
|
||||
}
|
||||
return "", fmt.Errorf("vision image: invalid image payload")
|
||||
}
|
||||
|
||||
func decodeVisionPayload(raw string) ([]byte, error) {
|
||||
encoded := strings.TrimSpace(raw)
|
||||
if strings.HasPrefix(strings.ToLower(encoded), "data:") {
|
||||
comma := strings.IndexByte(encoded, ',')
|
||||
if comma < 0 || !strings.Contains(strings.ToLower(encoded[:comma]), ";base64") {
|
||||
return nil, fmt.Errorf("vision image: unsupported data URI")
|
||||
}
|
||||
encoded = encoded[comma+1:]
|
||||
}
|
||||
maxEncodedBytes := (maxVisionImageBytes+2)/3*4 + 256
|
||||
if len(encoded) == 0 || len(encoded) > maxEncodedBytes {
|
||||
return nil, fmt.Errorf("vision image: encoded payload exceeds limit")
|
||||
}
|
||||
for _, encoding := range []*base64.Encoding{base64.StdEncoding, base64.RawStdEncoding} {
|
||||
decoder := base64.NewDecoder(encoding, &visionPayloadReader{source: encoded})
|
||||
data, err := io.ReadAll(io.LimitReader(decoder, maxVisionImageBytes+1))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if len(data) > maxVisionImageBytes {
|
||||
return nil, fmt.Errorf("vision image: payload exceeds %d bytes", maxVisionImageBytes)
|
||||
}
|
||||
if len(data) == 0 {
|
||||
return nil, fmt.Errorf("vision image: empty base64 payload")
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
return nil, fmt.Errorf("vision image: decode base64 failed")
|
||||
}
|
||||
|
||||
type visionPayloadReader struct {
|
||||
source string
|
||||
offset int
|
||||
}
|
||||
|
||||
func (r *visionPayloadReader) Read(dst []byte) (int, error) {
|
||||
written := 0
|
||||
for written < len(dst) && r.offset < len(r.source) {
|
||||
b := r.source[r.offset]
|
||||
r.offset++
|
||||
if b == '\r' || b == '\n' || b == ' ' || b == '\t' {
|
||||
continue
|
||||
}
|
||||
dst[written] = b
|
||||
written++
|
||||
}
|
||||
if written > 0 {
|
||||
return written, nil
|
||||
}
|
||||
if r.offset == len(r.source) {
|
||||
return 0, io.EOF
|
||||
}
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
func imageWithinVisionLimits(img image.Image) bool {
|
||||
if img == nil {
|
||||
return false
|
||||
}
|
||||
bounds := img.Bounds()
|
||||
width, height := bounds.Dx(), bounds.Dy()
|
||||
return width > 0 && height > 0 && width <= maxVisionImageEdge && height <= maxVisionImageEdge && int64(width)*int64(height) <= maxVisionImagePixels
|
||||
}
|
||||
|
||||
func imageMIMEForFormat(format string) string {
|
||||
switch strings.ToLower(format) {
|
||||
case "jpeg":
|
||||
return "image/jpeg"
|
||||
case "png":
|
||||
return "image/png"
|
||||
case "gif":
|
||||
return "image/gif"
|
||||
case "bmp":
|
||||
return "image/bmp"
|
||||
case "tiff":
|
||||
return "image/tiff"
|
||||
case "webp":
|
||||
return "image/webp"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func encodeVisionRaster(img image.Image) (string, error) {
|
||||
if img == nil {
|
||||
return "", nil
|
||||
}
|
||||
if !imageWithinVisionLimits(img) {
|
||||
return "", fmt.Errorf("vision image: raster dimensions exceed limits")
|
||||
}
|
||||
var encoded bytes.Buffer
|
||||
if err := png.Encode(&encoded, img); err != nil {
|
||||
return "", fmt.Errorf("vision image: encode raster: %w", err)
|
||||
}
|
||||
if encoded.Len() > maxVisionImageBytes {
|
||||
return "", fmt.Errorf("vision image: encoded raster exceeds %d bytes", maxVisionImageBytes)
|
||||
}
|
||||
return "data:image/png;base64," + base64.StdEncoding.EncodeToString(encoded.Bytes()), nil
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package component
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"image"
|
||||
"image/png"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestMaterializeInlineVisionImageKeepsVLMForImageDimensionLimit(t *testing.T) {
|
||||
var encoded bytes.Buffer
|
||||
large := image.NewRGBA(image.Rect(0, 0, maxVisionImageEdge+1, 1))
|
||||
if err := png.Encode(&encoded, large); err != nil {
|
||||
t.Fatalf("encode oversized image: %v", err)
|
||||
}
|
||||
dataURI := "data:image/png;base64," + base64.StdEncoding.EncodeToString(encoded.Bytes())
|
||||
|
||||
materialized, err := materializeInlineVisionImage(dataURI)
|
||||
if err != nil {
|
||||
t.Fatalf("materializeInlineVisionImage: %v", err)
|
||||
}
|
||||
if materialized == nil || materialized.Raster != nil {
|
||||
t.Fatalf("raster = %#v, want OCR to skip the oversized image", materialized)
|
||||
}
|
||||
if materialized.VLMData != dataURI || !isUsableVisionImage(materialized.VLMData) {
|
||||
t.Fatal("oversized OCR input should preserve its valid VLM payload")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsUsableVisionImageRejectsPayloadOverVLMByteBudget(t *testing.T) {
|
||||
encodedLength := base64.StdEncoding.EncodedLen(maxVisionImageBytes + 3)
|
||||
payload := strings.Repeat("A", encodedLength)
|
||||
if isUsableVisionImage(payload) {
|
||||
t.Fatal("base64 payload above the VLM byte budget must be rejected")
|
||||
}
|
||||
}
|
||||
@@ -27,6 +27,7 @@ package parser
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"html"
|
||||
"strings"
|
||||
)
|
||||
@@ -204,6 +205,81 @@ func docxIRTableToHTML(el docxIRElement) string {
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
type docxTableImage struct {
|
||||
data []byte
|
||||
row int
|
||||
column int
|
||||
included bool
|
||||
}
|
||||
|
||||
func forEachDOCXTableImage(el docxIRElement, visit func(docxTableImage) bool) bool {
|
||||
for rowIndex, row := range el.Rows {
|
||||
for columnIndex, cell := range row.Cells {
|
||||
completed := forEachDOCXIRImage(cell.Content, func(data []byte) bool {
|
||||
return visit(docxTableImage{data: data, row: rowIndex + 1, column: columnIndex + 1})
|
||||
})
|
||||
if !completed {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// forEachDOCXIRImage visits image payloads without first collecting a second
|
||||
// document-sized slice of image references. Returning false from visit stops
|
||||
// traversal as soon as the caller's media budget is exhausted.
|
||||
func forEachDOCXIRImage(elements []docxIRElement, visit func([]byte) bool) bool {
|
||||
for _, el := range elements {
|
||||
switch el.Type {
|
||||
case "image":
|
||||
if len(el.Data) > 0 && !visit(el.Data) {
|
||||
return false
|
||||
}
|
||||
case "paragraph", "heading":
|
||||
for _, run := range el.contentRuns() {
|
||||
if run.Type == "image" && len(run.Data) > 0 && !visit(run.Data) {
|
||||
return false
|
||||
}
|
||||
if !forEachDOCXIRImage(run.Content, visit) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
case "table":
|
||||
for _, row := range el.Rows {
|
||||
for _, cell := range row.Cells {
|
||||
if !forEachDOCXIRImage(cell.Content, visit) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
case "list":
|
||||
if !forEachDOCXIRListImage(docxIRList{Items: el.Items}, visit) {
|
||||
return false
|
||||
}
|
||||
case "text_box":
|
||||
if !forEachDOCXIRImage(el.contentBlocks(), visit) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func forEachDOCXIRListImage(list docxIRList, visit func([]byte) bool) bool {
|
||||
for _, item := range list.Items {
|
||||
if !forEachDOCXIRImage(item.Content, visit) {
|
||||
return false
|
||||
}
|
||||
if item.Nested != nil {
|
||||
if !forEachDOCXIRListImage(*item.Nested, visit) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// docxElementText returns the plain-text rendering of any supported
|
||||
// IR element type. Used by buildPPTXJSONSections to flatten one slide
|
||||
// section, and by extractDOCXFiguresFromIR so that tables, lists, and
|
||||
@@ -253,60 +329,86 @@ func docxElementText(el docxIRElement, cellSep string) string {
|
||||
// buildDOCXJSONSections converts an office_oxide IR JSON string into a
|
||||
// slice of structured items compatible with the chunker's JSON input
|
||||
// contract. Each item carries at least text and doc_type_kwd.
|
||||
func buildDOCXJSONSections(irJSON string) []map[string]any {
|
||||
func buildDOCXJSONSections(irJSON string, budget *embeddedMediaBudget) []map[string]any {
|
||||
var ir docxIRDocument
|
||||
if err := json.Unmarshal([]byte(irJSON), &ir); err != nil {
|
||||
return nil
|
||||
}
|
||||
if budget == nil {
|
||||
budget = newEmbeddedMediaBudget()
|
||||
}
|
||||
var sections []map[string]any
|
||||
tableSequence := 0
|
||||
for _, sec := range ir.Sections {
|
||||
for _, el := range sec.Elements {
|
||||
switch el.Type {
|
||||
case "paragraph", "heading":
|
||||
sections = appendDOCXParagraphSections(sections, el, el.Type == "heading")
|
||||
sections = appendDOCXParagraphSections(sections, el, el.Type == "heading", budget)
|
||||
|
||||
case "image":
|
||||
b64 := base64.StdEncoding.EncodeToString(el.Data)
|
||||
sections = append(sections, map[string]any{
|
||||
"text": "",
|
||||
"image": b64,
|
||||
"doc_type_kwd": "image",
|
||||
})
|
||||
sections, _ = appendDOCXImageSection(sections, el.Data, budget, nil)
|
||||
|
||||
case "table":
|
||||
tableSequence++
|
||||
html := docxIRTableToHTML(el)
|
||||
if html == "<table></table>" {
|
||||
continue
|
||||
}
|
||||
sections = append(sections, map[string]any{
|
||||
table := map[string]any{
|
||||
"text": html,
|
||||
"image": nil,
|
||||
"doc_type_kwd": "table",
|
||||
}
|
||||
tableID := ""
|
||||
foundImage := false
|
||||
var tableImages []docxTableImage
|
||||
forEachDOCXTableImage(el, func(tableImage docxTableImage) bool {
|
||||
foundImage = true
|
||||
included, keepWalking := budget.include(tableImage.data)
|
||||
if included || keepWalking {
|
||||
tableImage.included = included
|
||||
tableImages = append(tableImages, tableImage)
|
||||
}
|
||||
return keepWalking
|
||||
})
|
||||
if foundImage {
|
||||
tableID = fmt.Sprintf("docx-table-%d", tableSequence)
|
||||
table["source_table_id"] = tableID
|
||||
}
|
||||
sections = append(sections, table)
|
||||
for mediaOrder, tableImage := range tableImages {
|
||||
metadata := map[string]any{
|
||||
"parent_table_id": tableID,
|
||||
"row_index": tableImage.row,
|
||||
"column_index": tableImage.column,
|
||||
"media_order": mediaOrder + 1,
|
||||
}
|
||||
sections = appendDOCXImagePayload(sections, tableImage.data, tableImage.included, metadata)
|
||||
}
|
||||
|
||||
case "list":
|
||||
for _, item := range el.Items {
|
||||
text := extractTextFromListItem(item)
|
||||
if text == "" {
|
||||
continue
|
||||
if text != "" {
|
||||
sections = append(sections, map[string]any{
|
||||
"text": text,
|
||||
"image": nil,
|
||||
"doc_type_kwd": "text",
|
||||
})
|
||||
}
|
||||
sections = appendDOCXListImageSections(sections, docxIRList{Items: []docxIRListItem{item}}, budget)
|
||||
}
|
||||
|
||||
case "text_box":
|
||||
text := extractTextFromBlockElements(el.contentBlocks())
|
||||
if text != "" {
|
||||
sections = append(sections, map[string]any{
|
||||
"text": text,
|
||||
"image": nil,
|
||||
"doc_type_kwd": "text",
|
||||
})
|
||||
}
|
||||
|
||||
case "text_box":
|
||||
text := extractTextFromBlockElements(el.contentBlocks())
|
||||
if text == "" {
|
||||
continue
|
||||
}
|
||||
sections = append(sections, map[string]any{
|
||||
"text": text,
|
||||
"image": nil,
|
||||
"doc_type_kwd": "text",
|
||||
})
|
||||
sections = appendDOCXIRImageSections(sections, el.contentBlocks(), budget)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -317,7 +419,7 @@ func buildDOCXJSONSections(irJSON string) []map[string]any {
|
||||
// order. A paragraph is not necessarily a single text unit: office_oxide can
|
||||
// place an image between two text runs, and dropping that run would make the
|
||||
// downstream DOCX chunker lose both media order and adjacency context.
|
||||
func appendDOCXParagraphSections(sections []map[string]any, el docxIRElement, heading bool) []map[string]any {
|
||||
func appendDOCXParagraphSections(sections []map[string]any, el docxIRElement, heading bool, budget *embeddedMediaBudget) []map[string]any {
|
||||
var text strings.Builder
|
||||
flushText := func() {
|
||||
value := strings.TrimSpace(text.String())
|
||||
@@ -336,14 +438,7 @@ func appendDOCXParagraphSections(sections []map[string]any, el docxIRElement, he
|
||||
sections = append(sections, item)
|
||||
}
|
||||
appendImage := func(data []byte) {
|
||||
if len(data) == 0 {
|
||||
return
|
||||
}
|
||||
sections = append(sections, map[string]any{
|
||||
"text": "",
|
||||
"image": base64.StdEncoding.EncodeToString(data),
|
||||
"doc_type_kwd": "image",
|
||||
})
|
||||
sections, _ = appendDOCXImageSection(sections, data, budget, nil)
|
||||
}
|
||||
for _, run := range el.contentRuns() {
|
||||
switch run.Type {
|
||||
@@ -365,28 +460,77 @@ func appendDOCXParagraphSections(sections []map[string]any, el docxIRElement, he
|
||||
return sections
|
||||
}
|
||||
|
||||
func appendDOCXImageSection(sections []map[string]any, data []byte, budget *embeddedMediaBudget, metadata map[string]any) ([]map[string]any, bool) {
|
||||
if len(data) == 0 {
|
||||
return sections, true
|
||||
}
|
||||
included, keepWalking := budget.include(data)
|
||||
if !included && !keepWalking {
|
||||
return sections, false
|
||||
}
|
||||
return appendDOCXImagePayload(sections, data, included, metadata), keepWalking
|
||||
}
|
||||
|
||||
func appendDOCXListImageSections(sections []map[string]any, list docxIRList, budget *embeddedMediaBudget) []map[string]any {
|
||||
for _, item := range list.Items {
|
||||
sections = appendDOCXIRImageSections(sections, item.Content, budget)
|
||||
if item.Nested != nil {
|
||||
sections = appendDOCXListImageSections(sections, *item.Nested, budget)
|
||||
}
|
||||
}
|
||||
return sections
|
||||
}
|
||||
|
||||
func appendDOCXIRImageSections(sections []map[string]any, elements []docxIRElement, budget *embeddedMediaBudget) []map[string]any {
|
||||
forEachDOCXIRImage(elements, func(data []byte) bool {
|
||||
var keepWalking bool
|
||||
sections, keepWalking = appendDOCXImageSection(sections, data, budget, nil)
|
||||
return keepWalking
|
||||
})
|
||||
return sections
|
||||
}
|
||||
|
||||
func appendDOCXImagePayload(sections []map[string]any, data []byte, included bool, metadata map[string]any) []map[string]any {
|
||||
item := map[string]any{
|
||||
"text": "",
|
||||
"image": nil,
|
||||
"doc_type_kwd": "image",
|
||||
}
|
||||
if included {
|
||||
item["image"] = base64.StdEncoding.EncodeToString(data)
|
||||
} else {
|
||||
item["media_omitted"] = true
|
||||
}
|
||||
for key, value := range metadata {
|
||||
item[key] = value
|
||||
}
|
||||
return append(sections, item)
|
||||
}
|
||||
|
||||
// --- figure extraction (used by the cgo parser path) ---
|
||||
|
||||
// extractDOCXFiguresFromIR parses the office_oxide IR JSON and
|
||||
// returns every embedded image block together with the plain text
|
||||
// immediately surrounding it. The context matches what Python's
|
||||
// returns the images admitted by budget together with the plain text
|
||||
// immediately surrounding each image. The context matches what Python's
|
||||
// naive_merge_docx attaches as context_above / context_below on
|
||||
// each chunk that carries an image.
|
||||
//
|
||||
// Reuses the IR already obtained from the doc handle in
|
||||
// ParseWithResult so the binary is not opened twice.
|
||||
func extractDOCXFiguresFromIR(irJSON string) []DOCXFigure {
|
||||
func extractDOCXFiguresFromIR(irJSON string, budget *embeddedMediaBudget) []DOCXFigure {
|
||||
var ir docxIRDocument
|
||||
if err := json.Unmarshal([]byte(irJSON), &ir); err != nil {
|
||||
return nil
|
||||
}
|
||||
if budget == nil {
|
||||
budget = newEmbeddedMediaBudget()
|
||||
}
|
||||
|
||||
var flat []flatBlock
|
||||
for _, sec := range ir.Sections {
|
||||
for _, el := range sec.Elements {
|
||||
if el.Type == "image" {
|
||||
b64 := base64.StdEncoding.EncodeToString(el.Data)
|
||||
flat = append(flat, flatBlock{image: b64})
|
||||
flat = append(flat, flatBlock{imageData: el.Data})
|
||||
continue
|
||||
}
|
||||
if el.Type == "paragraph" || el.Type == "heading" {
|
||||
@@ -400,10 +544,17 @@ func extractDOCXFiguresFromIR(irJSON string) []DOCXFigure {
|
||||
|
||||
var figures []DOCXFigure
|
||||
for i, block := range flat {
|
||||
if block.image == "" {
|
||||
if len(block.imageData) == 0 {
|
||||
continue
|
||||
}
|
||||
fig := DOCXFigure{Image: block.image}
|
||||
included, keepWalking := budget.include(block.imageData)
|
||||
if !included && !keepWalking {
|
||||
break
|
||||
}
|
||||
fig := DOCXFigure{}
|
||||
if included {
|
||||
fig.Image = base64.StdEncoding.EncodeToString(block.imageData)
|
||||
}
|
||||
|
||||
// Collect text above (backward scan up to docxContextWindow
|
||||
// chars, or until another image is hit).
|
||||
@@ -426,6 +577,9 @@ func extractDOCXFiguresFromIR(irJSON string) []DOCXFigure {
|
||||
}
|
||||
|
||||
figures = append(figures, fig)
|
||||
if !keepWalking {
|
||||
break
|
||||
}
|
||||
}
|
||||
return figures
|
||||
}
|
||||
@@ -447,7 +601,7 @@ func docxInlineFlatBlocks(el docxIRElement) []flatBlock {
|
||||
if len(data) == 0 {
|
||||
return
|
||||
}
|
||||
blocks = append(blocks, flatBlock{image: base64.StdEncoding.EncodeToString(data)})
|
||||
blocks = append(blocks, flatBlock{imageData: data})
|
||||
}
|
||||
for _, run := range el.contentRuns() {
|
||||
switch run.Type {
|
||||
@@ -474,8 +628,8 @@ func docxInlineFlatBlocks(el docxIRElement) []flatBlock {
|
||||
// flatBlock is a flattened IR element used internally to collect
|
||||
// text / image context around embedded figures.
|
||||
type flatBlock struct {
|
||||
text string
|
||||
image string // base64-encoded image data (empty for non-image)
|
||||
text string
|
||||
imageData []byte
|
||||
}
|
||||
|
||||
const docxContextWindow = 512
|
||||
@@ -484,7 +638,7 @@ func collectDOCXPrevText(flat []flatBlock, idx, maxLen int) string {
|
||||
var parts []string
|
||||
remaining := maxLen
|
||||
for i := idx - 1; i >= 0 && remaining > 0; i-- {
|
||||
if flat[i].image != "" {
|
||||
if len(flat[i].imageData) > 0 {
|
||||
break // stop at previous image
|
||||
}
|
||||
if flat[i].text == "" {
|
||||
@@ -512,7 +666,7 @@ func collectDOCXNextText(flat []flatBlock, idx, maxLen int) string {
|
||||
var parts []string
|
||||
remaining := maxLen
|
||||
for i := idx + 1; i < len(flat) && remaining > 0; i++ {
|
||||
if flat[i].image != "" {
|
||||
if len(flat[i].imageData) > 0 {
|
||||
break // stop at next image
|
||||
}
|
||||
if flat[i].text == "" {
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
package parser
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
@@ -22,7 +25,7 @@ func TestBuildDOCXJSONSections_FromJSON(t *testing.T) {
|
||||
{"type":"list","items":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"item1"}]}]}]}
|
||||
]}]}`
|
||||
|
||||
got := buildDOCXJSONSections(irJSON)
|
||||
got := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())
|
||||
want := []map[string]any{
|
||||
{"text": "Title", "image": nil, "doc_type_kwd": "text", "ck_type": "heading"},
|
||||
{"text": "Hello", "image": nil, "doc_type_kwd": "text"},
|
||||
@@ -43,7 +46,7 @@ func TestBuildDOCXJSONSections_EmptyTableSkipped(t *testing.T) {
|
||||
{"type":"paragraph","content":[{"type":"text","text":"keep"}]},
|
||||
{"type":"table","rows":[]}
|
||||
]}]}`
|
||||
got := buildDOCXJSONSections(irJSON)
|
||||
got := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())
|
||||
if len(got) != 1 || got[0]["text"] != "keep" {
|
||||
t.Fatalf("expected only the paragraph to survive, got %+v", got)
|
||||
}
|
||||
@@ -57,7 +60,7 @@ func TestBuildDOCXJSONSections_PreservesInlineImageOrder(t *testing.T) {
|
||||
{"type":"text","text":"after"}
|
||||
]}
|
||||
]}]}`
|
||||
got := buildDOCXJSONSections(irJSON)
|
||||
got := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())
|
||||
if len(got) != 3 {
|
||||
t.Fatalf("sections = %+v, want text/image/text sequence", got)
|
||||
}
|
||||
@@ -72,6 +75,189 @@ func TestBuildDOCXJSONSections_PreservesInlineImageOrder(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildDOCXJSONSections_ExtractsTableCellImages(t *testing.T) {
|
||||
irJSON := `{"sections":[{"elements":[
|
||||
{"type":"table","rows":[{"cells":[
|
||||
{"content":[{"type":"paragraph","content":[
|
||||
{"type":"text","text":"before"},
|
||||
{"type":"image","data":"aGVsbG8="},
|
||||
{"type":"text","text":"after"}
|
||||
]}]},
|
||||
{"content":[{"type":"image","data":"aW1hZ2U="}]}
|
||||
]}]}
|
||||
]}]}`
|
||||
|
||||
got := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())
|
||||
if len(got) != 3 {
|
||||
t.Fatalf("sections = %+v, want table plus two cell image items", got)
|
||||
}
|
||||
if got[0]["doc_type_kwd"] != "table" || got[0]["text"] != "<table><tr><td>beforeafter</td><td></td></tr></table>" || got[0]["source_table_id"] != "docx-table-1" {
|
||||
t.Fatalf("table item = %+v", got[0])
|
||||
}
|
||||
for i, want := range []string{"aGVsbG8=", "aW1hZ2U="} {
|
||||
item := got[i+1]
|
||||
if item["doc_type_kwd"] != "image" || item["image"] != want {
|
||||
t.Errorf("image item %d = %+v, want payload %q", i, item, want)
|
||||
}
|
||||
if item["parent_table_id"] != "docx-table-1" || item["row_index"] != 1 || item["column_index"] != i+1 || item["media_order"] != i+1 {
|
||||
t.Errorf("image item %d metadata = %+v", i, item)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildDOCXJSONSections_BudgetKeepsInlineTextAroundOmittedImage(t *testing.T) {
|
||||
irJSON := `{"sections":[{"elements":[{"type":"paragraph","content":[
|
||||
{"type":"text","text":"before"},
|
||||
{"type":"image","data":"YWJjZA=="},
|
||||
{"type":"text","text":"after"}
|
||||
]}]}]}`
|
||||
budget := &embeddedMediaBudget{maxImageBytes: 3, maxTotalBytes: 5, maxItems: 10}
|
||||
|
||||
items := buildDOCXJSONSections(irJSON, budget)
|
||||
if len(items) != 3 {
|
||||
t.Fatalf("items = %+v, want text/image/text with omitted image metadata", items)
|
||||
}
|
||||
if items[0]["text"] != "before" || items[2]["text"] != "after" {
|
||||
t.Fatalf("inline text was lost around omitted image: %+v", items)
|
||||
}
|
||||
if items[1]["doc_type_kwd"] != "image" || items[1]["image"] != nil || items[1]["media_omitted"] != true {
|
||||
t.Fatalf("omitted inline image item = %+v, want metadata without payload", items[1])
|
||||
}
|
||||
if len(budget.warnings()) == 0 {
|
||||
t.Fatal("omitted image should produce a parser warning")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildDOCXJSONSections_BudgetKeepsTableImageMetadata(t *testing.T) {
|
||||
irJSON := `{"sections":[{"elements":[{"type":"table","rows":[{"cells":[
|
||||
{"content":[{"type":"image","data":"YWJj"}]},
|
||||
{"content":[{"type":"image","data":"ZGVm"}]}
|
||||
]}]}]}]}`
|
||||
budget := &embeddedMediaBudget{maxImageBytes: 4, maxTotalBytes: 5, maxItems: 10}
|
||||
|
||||
items := buildDOCXJSONSections(irJSON, budget)
|
||||
if len(items) != 3 {
|
||||
t.Fatalf("items = %+v, want table and two image metadata items", items)
|
||||
}
|
||||
if items[1]["image"] != "YWJj" || items[1]["media_omitted"] == true {
|
||||
t.Fatalf("first table image = %+v, want accepted payload", items[1])
|
||||
}
|
||||
omitted := items[2]
|
||||
if omitted["image"] != nil || omitted["media_omitted"] != true {
|
||||
t.Fatalf("second table image = %+v, want omitted payload", omitted)
|
||||
}
|
||||
if omitted["parent_table_id"] != items[0]["source_table_id"] || omitted["row_index"] != 1 || omitted["column_index"] != 2 || omitted["media_order"] != 2 {
|
||||
t.Fatalf("omitted table image lost location metadata: %+v", omitted)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildDOCXJSONSections_BudgetCapsMediaCount(t *testing.T) {
|
||||
var ir strings.Builder
|
||||
ir.WriteString(`{"sections":[{"elements":[`)
|
||||
for i := 0; i < 257; i++ {
|
||||
if i > 0 {
|
||||
ir.WriteByte(',')
|
||||
}
|
||||
ir.WriteString(`{"type":"image","data":"YQ=="}`)
|
||||
}
|
||||
ir.WriteString(`]}]}`)
|
||||
|
||||
budget := newEmbeddedMediaBudget()
|
||||
items := buildDOCXJSONSections(ir.String(), budget)
|
||||
imageCount := 0
|
||||
for _, item := range items {
|
||||
if item["doc_type_kwd"] == "image" {
|
||||
imageCount++
|
||||
}
|
||||
}
|
||||
if imageCount != budget.maxItems {
|
||||
t.Fatalf("image items = %d, want count limit %d", imageCount, budget.maxItems)
|
||||
}
|
||||
if len(budget.warnings()) == 0 {
|
||||
t.Fatal("truncated media count should produce a parser warning")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildPPTXJSONSections_BudgetKeepsSlideMetadata(t *testing.T) {
|
||||
irJSON := `{"sections":[{"elements":[
|
||||
{"type":"paragraph","content":[{"type":"text","text":"slide text"}]},
|
||||
{"type":"image","data":"YWJjZA=="}
|
||||
]}]}`
|
||||
budget := &embeddedMediaBudget{maxImageBytes: 3, maxTotalBytes: 5, maxItems: 10}
|
||||
|
||||
items, err := buildPPTXJSONSections(irJSON, budget)
|
||||
if err != nil {
|
||||
t.Fatalf("buildPPTXJSONSections: %v", err)
|
||||
}
|
||||
if len(items) != 2 || items[0]["text"] != "slide text" {
|
||||
t.Fatalf("items = %+v, want text and omitted image items", items)
|
||||
}
|
||||
omitted := items[1]
|
||||
if omitted["image"] != nil || omitted["media_omitted"] != true || omitted["slide_number"] != 1 || omitted["media_order"] != 1 {
|
||||
t.Fatalf("omitted slide image lost metadata: %+v", omitted)
|
||||
}
|
||||
if itemsAllEmpty([]map[string]any{omitted}) {
|
||||
t.Fatal("an image-only slide with a budget-omitted image must not be replaced by whole-deck fallback")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildPPTXJSONSections_BudgetCapsMediaCount(t *testing.T) {
|
||||
var ir strings.Builder
|
||||
ir.WriteString(`{"sections":[{"elements":[`)
|
||||
for i := 0; i < 257; i++ {
|
||||
if i > 0 {
|
||||
ir.WriteByte(',')
|
||||
}
|
||||
ir.WriteString(`{"type":"image","data":"YQ=="}`)
|
||||
}
|
||||
ir.WriteString(`]}]}`)
|
||||
|
||||
budget := newEmbeddedMediaBudget()
|
||||
items, err := buildPPTXJSONSections(ir.String(), budget)
|
||||
if err != nil {
|
||||
t.Fatalf("buildPPTXJSONSections: %v", err)
|
||||
}
|
||||
imageCount := 0
|
||||
for _, item := range items {
|
||||
if item["doc_type_kwd"] == "image" {
|
||||
imageCount++
|
||||
}
|
||||
}
|
||||
if imageCount != budget.maxItems {
|
||||
t.Fatalf("image items = %d, want count limit %d", imageCount, budget.maxItems)
|
||||
}
|
||||
if len(budget.warnings()) == 0 {
|
||||
t.Fatal("truncated media count should produce a parser warning")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmbeddedMediaBudgetRejectsPerImageAndDocumentByteLimits(t *testing.T) {
|
||||
budget := &embeddedMediaBudget{maxImageBytes: 4, maxTotalBytes: 5, maxItems: 4}
|
||||
if included, keepGoing := budget.include([]byte("abc")); !included || !keepGoing {
|
||||
t.Fatalf("first image admission = (%v, %v), want (true, true)", included, keepGoing)
|
||||
}
|
||||
if included, keepGoing := budget.include([]byte("def")); included || !keepGoing {
|
||||
t.Fatalf("aggregate overflow admission = (%v, %v), want (false, true)", included, keepGoing)
|
||||
}
|
||||
if included, keepGoing := budget.include([]byte("12345")); included || !keepGoing {
|
||||
t.Fatalf("per-image overflow admission = (%v, %v), want (false, true)", included, keepGoing)
|
||||
}
|
||||
if len(budget.warnings()) == 0 {
|
||||
t.Fatal("omitted byte payloads should produce a parser warning")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildDOCXJSONSectionsEncodesOnlyAcceptedImageData(t *testing.T) {
|
||||
data := []byte("text is not a valid image, but the parser preserves it")
|
||||
encoded := base64.StdEncoding.EncodeToString(data)
|
||||
irJSON := fmt.Sprintf(`{"sections":[{"elements":[{"type":"image","data":%q}]}]}`, encoded)
|
||||
budget := &embeddedMediaBudget{maxImageBytes: 8, maxTotalBytes: 8, maxItems: 4}
|
||||
items := buildDOCXJSONSections(irJSON, budget)
|
||||
if len(items) != 1 || items[0]["image"] != nil || items[0]["media_omitted"] != true {
|
||||
t.Fatalf("oversized image items = %+v, want metadata without payload", items)
|
||||
}
|
||||
}
|
||||
|
||||
// TestJoinDOCXIRRuns pins that only text-type runs are concatenated;
|
||||
// non-text runs (e.g. nested image runs) are skipped.
|
||||
func TestJoinDOCXIRRuns(t *testing.T) {
|
||||
@@ -111,7 +297,7 @@ func TestExtractDOCXFiguresFromIR(t *testing.T) {
|
||||
{"type":"paragraph","content":[{"type":"text","text":"after"}]}
|
||||
]}]}`
|
||||
|
||||
figs := extractDOCXFiguresFromIR(irJSON)
|
||||
figs := extractDOCXFiguresFromIR(irJSON, newEmbeddedMediaBudget())
|
||||
if len(figs) != 1 {
|
||||
t.Fatalf("expected 1 figure, got %d", len(figs))
|
||||
}
|
||||
@@ -135,7 +321,7 @@ func TestExtractDOCXFiguresFromIR_InlineImage(t *testing.T) {
|
||||
{"type":"text","text":"after"}
|
||||
]}
|
||||
]}]}`
|
||||
figs := extractDOCXFiguresFromIR(irJSON)
|
||||
figs := extractDOCXFiguresFromIR(irJSON, newEmbeddedMediaBudget())
|
||||
if len(figs) != 1 {
|
||||
t.Fatalf("expected 1 inline figure, got %d", len(figs))
|
||||
}
|
||||
@@ -144,13 +330,30 @@ func TestExtractDOCXFiguresFromIR_InlineImage(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractDOCXFiguresFromIR_BudgetKeepsContextWithoutPayload(t *testing.T) {
|
||||
irJSON := `{"sections":[{"elements":[
|
||||
{"type":"paragraph","content":[{"type":"text","text":"before"}]},
|
||||
{"type":"image","data":"YWJjZA=="},
|
||||
{"type":"paragraph","content":[{"type":"text","text":"after"}]}
|
||||
]}]}`
|
||||
budget := &embeddedMediaBudget{maxImageBytes: 3, maxTotalBytes: 5, maxItems: 10}
|
||||
|
||||
figures := extractDOCXFiguresFromIR(irJSON, budget)
|
||||
if len(figures) != 1 {
|
||||
t.Fatalf("figures = %+v, want one metadata record", figures)
|
||||
}
|
||||
if figures[0].Image != "" || figures[0].ContextAbove != "before" || figures[0].ContextBelow != "after" || figures[0].Marker != "before" {
|
||||
t.Fatalf("budgeted figure = %+v, want context metadata without payload", figures[0])
|
||||
}
|
||||
}
|
||||
|
||||
// TestExtractDOCXFiguresFromIR_NoImage returns nil when the IR has no
|
||||
// image blocks.
|
||||
func TestExtractDOCXFiguresFromIR_NoImage(t *testing.T) {
|
||||
irJSON := `{"sections":[{"elements":[
|
||||
{"type":"paragraph","content":[{"type":"text","text":"only text"}]}
|
||||
]}]}`
|
||||
if figs := extractDOCXFiguresFromIR(irJSON); figs != nil {
|
||||
if figs := extractDOCXFiguresFromIR(irJSON, newEmbeddedMediaBudget()); figs != nil {
|
||||
t.Fatalf("expected nil, got %+v", figs)
|
||||
}
|
||||
}
|
||||
@@ -158,7 +361,7 @@ func TestExtractDOCXFiguresFromIR_NoImage(t *testing.T) {
|
||||
// TestExtractDOCXFiguresFromIR_BadJSON returns nil on unparseable IR,
|
||||
// mirroring the json.Unmarshal error guard.
|
||||
func TestExtractDOCXFiguresFromIR_BadJSON(t *testing.T) {
|
||||
if figs := extractDOCXFiguresFromIR("{not json"); figs != nil {
|
||||
if figs := extractDOCXFiguresFromIR("{not json", newEmbeddedMediaBudget()); figs != nil {
|
||||
t.Fatalf("expected nil for bad JSON, got %+v", figs)
|
||||
}
|
||||
}
|
||||
@@ -176,7 +379,7 @@ func TestExtractDOCXFiguresFromIR_NonParagraphContext(t *testing.T) {
|
||||
{"type":"list","items":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"list item"}]}]}]},
|
||||
{"type":"text_box","content":[{"type":"paragraph","content":[{"type":"text","text":"box text"}]}]}
|
||||
]}]}`
|
||||
figs := extractDOCXFiguresFromIR(irJSON)
|
||||
figs := extractDOCXFiguresFromIR(irJSON, newEmbeddedMediaBudget())
|
||||
if len(figs) != 1 {
|
||||
t.Fatalf("expected 1 figure, got %d", len(figs))
|
||||
}
|
||||
@@ -203,7 +406,7 @@ func TestCollectDOCXText_BoundedByMaxLen(t *testing.T) {
|
||||
flat = append(flat, flatBlock{text: "x"})
|
||||
}
|
||||
imgIdx := len(flat)
|
||||
flat = append(flat, flatBlock{image: "img"})
|
||||
flat = append(flat, flatBlock{imageData: []byte("img")})
|
||||
for i := 0; i < 600; i++ {
|
||||
flat = append(flat, flatBlock{text: "x"})
|
||||
}
|
||||
@@ -278,7 +481,7 @@ func TestExtractTextFromListItem_NestedNull(t *testing.T) {
|
||||
{"content":[{"type":"paragraph","content":[{"type":"text","text":"only item"}]}],"nested":null}
|
||||
]}
|
||||
]}]}`
|
||||
sections := buildDOCXJSONSections(irJSON)
|
||||
sections := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())
|
||||
if len(sections) != 1 || sections[0]["text"] != "only item" {
|
||||
t.Fatalf("expected single item \"only item\", got %+v", sections)
|
||||
}
|
||||
@@ -301,7 +504,7 @@ func TestExtractTextFromListItem_NestedTable(t *testing.T) {
|
||||
]}
|
||||
]}
|
||||
]}]}`
|
||||
sections := buildDOCXJSONSections(irJSON)
|
||||
sections := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())
|
||||
if len(sections) != 1 {
|
||||
t.Fatalf("expected 1 section, got %d: %+v", len(sections), sections)
|
||||
}
|
||||
@@ -352,7 +555,7 @@ func TestBuildDOCXJSONSections_TextBoxTable(t *testing.T) {
|
||||
]}
|
||||
]}
|
||||
]}]}`
|
||||
sections := buildDOCXJSONSections(irJSON)
|
||||
sections := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())
|
||||
if len(sections) != 1 {
|
||||
t.Fatalf("expected 1 section, got %d: %+v", len(sections), sections)
|
||||
}
|
||||
@@ -373,7 +576,7 @@ func TestBuildDOCXJSONSections_NestedList(t *testing.T) {
|
||||
]}}
|
||||
]}
|
||||
]}]}`
|
||||
sections := buildDOCXJSONSections(irJSON)
|
||||
sections := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())
|
||||
if len(sections) != 1 {
|
||||
t.Fatalf("expected 1 section, got %d: %+v", len(sections), sections)
|
||||
}
|
||||
@@ -381,3 +584,48 @@ func TestBuildDOCXJSONSections_NestedList(t *testing.T) {
|
||||
t.Errorf("nested list text = %q, want %q", got, "L1\nL2a\nL2b")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildDOCXJSONSections_EmitsListAndTextBoxImages(t *testing.T) {
|
||||
irJSON := `{"sections":[{"elements":[
|
||||
{"type":"list","items":[
|
||||
{"content":[{"type":"paragraph","content":[
|
||||
{"type":"text","text":"list text"},
|
||||
{"type":"image","data":"aGVsbG8="}
|
||||
]}],"nested":{"items":[
|
||||
{"content":[{"type":"paragraph","content":[
|
||||
{"type":"image","data":"d29ybGQ="}
|
||||
]}]}
|
||||
]}}
|
||||
]},
|
||||
{"type":"text_box","content":[{"type":"paragraph","content":[
|
||||
{"type":"text","text":"text box text"},
|
||||
{"type":"image","data":"cGljdHVyZQ=="}
|
||||
]}]}
|
||||
]}]}`
|
||||
sections := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())
|
||||
want := []struct {
|
||||
docType string
|
||||
text string
|
||||
image string
|
||||
}{
|
||||
{docType: "text", text: "list text"},
|
||||
{docType: "image", image: "aGVsbG8="},
|
||||
{docType: "image", image: "d29ybGQ="},
|
||||
{docType: "text", text: "text box text"},
|
||||
{docType: "image", image: "cGljdHVyZQ=="},
|
||||
}
|
||||
if len(sections) != len(want) {
|
||||
t.Fatalf("sections = %+v, want %d items", sections, len(want))
|
||||
}
|
||||
for i, expected := range want {
|
||||
if got := sections[i]["doc_type_kwd"]; got != expected.docType {
|
||||
t.Errorf("section %d type = %v, want %q", i, got, expected.docType)
|
||||
}
|
||||
if expected.text != "" && sections[i]["text"] != expected.text {
|
||||
t.Errorf("section %d text = %v, want %q", i, sections[i]["text"], expected.text)
|
||||
}
|
||||
if expected.image != "" && sections[i]["image"] != expected.image {
|
||||
t.Errorf("section %d image = %v, want %q", i, sections[i]["image"], expected.image)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -68,8 +68,8 @@ func (p *DOCXParser) ConfigureFromSetup(setup map[string]any) {
|
||||
|
||||
// ParseWithResult produces structured JSON items (when
|
||||
// p.outputFormat == "json") or markdown (default) from a
|
||||
// docx document. Embedded images are extracted in both paths
|
||||
// for downstream vision-figure dispatch.
|
||||
// docx document. The JSON path emits bounded image items; the
|
||||
// Markdown path attaches bounded figure metadata to the file result.
|
||||
//
|
||||
// JSON path mirrors python parser.py:_docx() output_format == "json".
|
||||
// Markdown path mirrors python naive.py: Docx() → naive_merge_docx().
|
||||
@@ -106,22 +106,18 @@ func (p *DOCXParser) ParseWithResult(ctx context.Context, filename string, data
|
||||
}
|
||||
|
||||
// Extract IR JSON for section building (JSON path) and
|
||||
// embedded-image extraction (both paths).
|
||||
// embedded-image extraction on the Markdown path.
|
||||
irJSON, irErr := doc.ToIRJSON()
|
||||
var figures []DOCXFigure
|
||||
if irErr == nil {
|
||||
figures = extractDOCXFiguresFromIR(irJSON)
|
||||
}
|
||||
if len(figures) > 0 {
|
||||
fileMeta["figures"] = buildFiguresMap(figures)
|
||||
}
|
||||
|
||||
if p.outputFormat == "json" {
|
||||
if irErr != nil {
|
||||
return ParseResult{Err: fmt.Errorf("docx to-ir-json: %w", irErr)}
|
||||
}
|
||||
var sections []map[string]any
|
||||
sections = buildDOCXJSONSections(irJSON)
|
||||
budget := newEmbeddedMediaBudget()
|
||||
sections := buildDOCXJSONSections(irJSON, budget)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return ParseResult{Err: err}
|
||||
}
|
||||
// remove_header_footer: drop sections whose normalized text
|
||||
// matches a docx header/footer entry (mirrors Python
|
||||
// parser.py:889-891 extract_docx_header_footer_texts +
|
||||
@@ -143,6 +139,15 @@ func (p *DOCXParser) ParseWithResult(ctx context.Context, filename string, data
|
||||
OutputFormat: "json",
|
||||
File: fileMeta,
|
||||
JSON: sections,
|
||||
Warnings: budget.warnings(),
|
||||
}
|
||||
}
|
||||
|
||||
budget := newEmbeddedMediaBudget()
|
||||
if irErr == nil {
|
||||
figures := extractDOCXFiguresFromIR(irJSON, budget)
|
||||
if len(figures) > 0 {
|
||||
fileMeta["figures"] = buildFiguresMap(figures)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -187,6 +192,7 @@ func (p *DOCXParser) ParseWithResult(ctx context.Context, filename string, data
|
||||
OutputFormat: "markdown",
|
||||
File: fileMeta,
|
||||
Markdown: md,
|
||||
Warnings: budget.warnings(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -29,7 +29,7 @@ func TestBuildDOCXJSONSections_Paragraphs(t *testing.T) {
|
||||
ir := `{"sections":[{"title":"","elements":[
|
||||
{"type":"paragraph","content":[{"type":"text","text":"Hello world"}],"style":"Normal"}
|
||||
]}]}`
|
||||
sections := buildDOCXJSONSections(ir)
|
||||
sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())
|
||||
if len(sections) != 1 {
|
||||
t.Fatalf("got %d sections, want 1", len(sections))
|
||||
}
|
||||
@@ -46,7 +46,7 @@ func TestBuildDOCXJSONSections_Headings(t *testing.T) {
|
||||
ir := `{"sections":[{"title":"","elements":[
|
||||
{"type":"heading","level":1,"content":[{"type":"text","text":"Chapter 1"}]}
|
||||
]}]}`
|
||||
sections := buildDOCXJSONSections(ir)
|
||||
sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())
|
||||
if len(sections) != 1 {
|
||||
t.Fatalf("got %d sections, want 1", len(sections))
|
||||
}
|
||||
@@ -67,7 +67,7 @@ func TestBuildDOCXJSONSections_Images(t *testing.T) {
|
||||
ir := `{"sections":[{"title":"","elements":[
|
||||
{"type":"image","data":"` + b64 + `"}
|
||||
]}]}`
|
||||
sections := buildDOCXJSONSections(ir)
|
||||
sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())
|
||||
if len(sections) != 1 {
|
||||
t.Fatalf("got %d sections, want 1", len(sections))
|
||||
}
|
||||
@@ -90,7 +90,7 @@ func TestBuildDOCXJSONSections_Tables(t *testing.T) {
|
||||
{"cells":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"A2"}]}]},{"content":[{"type":"paragraph","content":[{"type":"text","text":"B2"}]}]}]}
|
||||
]}
|
||||
]}]}`
|
||||
sections := buildDOCXJSONSections(ir)
|
||||
sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())
|
||||
if len(sections) != 1 {
|
||||
t.Fatalf("got %d sections, want 1", len(sections))
|
||||
}
|
||||
@@ -120,7 +120,7 @@ func TestBuildDOCXJSONSections_MixedContent(t *testing.T) {
|
||||
{"type":"table","rows":[{"cells":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"cell"}]}]}]}]},
|
||||
{"type":"heading","level":2,"content":[{"type":"text","text":"Sub title"}]}
|
||||
]}]}`
|
||||
sections := buildDOCXJSONSections(ir)
|
||||
sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())
|
||||
if len(sections) != 4 {
|
||||
t.Fatalf("got %d sections, want 4", len(sections))
|
||||
}
|
||||
@@ -153,7 +153,7 @@ func TestBuildDOCXJSONSections_List(t *testing.T) {
|
||||
{"content":[{"type":"paragraph","content":[{"type":"text","text":"Item 3"}]}]}
|
||||
],"level":0}
|
||||
]}]}`
|
||||
sections := buildDOCXJSONSections(ir)
|
||||
sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())
|
||||
if len(sections) != 3 {
|
||||
t.Fatalf("got %d sections, want 3 (each list item should be a section)", len(sections))
|
||||
}
|
||||
@@ -171,7 +171,7 @@ func TestBuildDOCXJSONSections_TextBox(t *testing.T) {
|
||||
ir := `{"sections":[{"title":"","elements":[
|
||||
{"type":"text_box","content":[{"type":"paragraph","content":[{"type":"text","text":"Boxed paragraph"}]}],"width_emu":null}
|
||||
]}]}`
|
||||
sections := buildDOCXJSONSections(ir)
|
||||
sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())
|
||||
if len(sections) != 1 {
|
||||
t.Fatalf("got %d sections, want 1 (text_box content should become a section)", len(sections))
|
||||
}
|
||||
@@ -192,7 +192,7 @@ func TestBuildDOCXJSONSections_MixedWithList(t *testing.T) {
|
||||
],"level":0},
|
||||
{"type":"paragraph","content":[{"type":"text","text":"Trailer"}],"style":"Normal"}
|
||||
]}]}`
|
||||
sections := buildDOCXJSONSections(ir)
|
||||
sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())
|
||||
if len(sections) != 4 {
|
||||
t.Fatalf("got %d sections, want 4 (para + 2 list items + para)", len(sections))
|
||||
}
|
||||
@@ -210,7 +210,7 @@ func TestBuildDOCXJSONSections_EmptySkipped(t *testing.T) {
|
||||
{"type":"paragraph","content":[{"type":"text","text":"Only this matters"}]},
|
||||
{"type":"table","rows":[]}
|
||||
]}]}`
|
||||
sections := buildDOCXJSONSections(ir)
|
||||
sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())
|
||||
if len(sections) != 1 {
|
||||
t.Fatalf("got %d sections, want 1 (only non-empty paragraph)", len(sections))
|
||||
}
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
package parser
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const htmlMediaDataURI = "data:image/png;base64,aGVsbG8="
|
||||
|
||||
func TestHTMLParser_EmitsInlineImagesInDocumentOrder(t *testing.T) {
|
||||
input := `<p>before <img alt="figure alt" src="` + htmlMediaDataURI + `"> after</p>`
|
||||
result := NewHTMLParser().ParseWithResult(context.Background(), "inline.html", []byte(input))
|
||||
if result.Err != nil {
|
||||
t.Fatalf("ParseWithResult: %v", result.Err)
|
||||
}
|
||||
if len(result.Warnings) != 0 {
|
||||
t.Fatalf("warnings = %v, want none for a valid inline image", result.Warnings)
|
||||
}
|
||||
if len(result.JSON) != 3 {
|
||||
t.Fatalf("items = %+v, want text/image/text", result.JSON)
|
||||
}
|
||||
if result.JSON[0]["text"] != "before" || result.JSON[0]["doc_type_kwd"] != "text" {
|
||||
t.Errorf("before item = %+v", result.JSON[0])
|
||||
}
|
||||
imageItem := result.JSON[1]
|
||||
if imageItem["doc_type_kwd"] != "image" || imageItem["image"] != htmlMediaDataURI || imageItem["text"] != "figure alt" || imageItem["media_order"] != 1 {
|
||||
t.Errorf("image item = %+v", imageItem)
|
||||
}
|
||||
if result.JSON[2]["text"] != "after" || result.JSON[2]["doc_type_kwd"] != "text" {
|
||||
t.Errorf("after item = %+v", result.JSON[2])
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTMLParser_EmitsTableCellImagesWithParentAndCellOrder(t *testing.T) {
|
||||
input := `<table><tr><td>A<img alt="first" src="` + htmlMediaDataURI + `"></td><td><img src="https://example.com/second.png"></td></tr></table>`
|
||||
result := NewHTMLParser().ParseWithResult(context.Background(), "table.html", []byte(input))
|
||||
if result.Err != nil {
|
||||
t.Fatalf("ParseWithResult: %v", result.Err)
|
||||
}
|
||||
if len(result.JSON) != 2 {
|
||||
t.Fatalf("items = %+v, want table and one inline image item", result.JSON)
|
||||
}
|
||||
if result.JSON[0]["doc_type_kwd"] != "table" || result.JSON[0]["source_table_id"] != "html-table-1" {
|
||||
t.Errorf("table item = %+v", result.JSON[0])
|
||||
}
|
||||
tableText, _ := result.JSON[0]["text"].(string)
|
||||
if strings.Contains(tableText, "data:image/") {
|
||||
t.Error("table text duplicates the separately emitted inline image payload")
|
||||
}
|
||||
if !strings.Contains(tableText, `alt="first"`) {
|
||||
t.Errorf("table text lost the image alt text: %q", tableText)
|
||||
}
|
||||
if result.JSON[1]["image"] != htmlMediaDataURI {
|
||||
t.Errorf("inline image payload = %v, want data URI", result.JSON[1]["image"])
|
||||
}
|
||||
item := result.JSON[1]
|
||||
if item["doc_type_kwd"] != "image" || item["parent_table_id"] != "html-table-1" {
|
||||
t.Errorf("inline image item = %+v", item)
|
||||
}
|
||||
if item["row_index"] != 1 || item["column_index"] != 1 || item["media_order"] != 1 {
|
||||
t.Errorf("inline image position = %+v", item)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTMLParser_RejectsOversizedInlineImagePayload(t *testing.T) {
|
||||
payload := base64.StdEncoding.EncodeToString(make([]byte, maxEmbeddedImageBytes+1))
|
||||
input := "<p><img alt=\"large\" src=\"data:image/png;base64," + payload + "\"></p>"
|
||||
result := NewHTMLParser().ParseWithResult(context.Background(), "large.html", []byte(input))
|
||||
if result.Err != nil {
|
||||
t.Fatalf("ParseWithResult: %v", result.Err)
|
||||
}
|
||||
for _, item := range result.JSON {
|
||||
if item["doc_type_kwd"] == "image" {
|
||||
t.Fatal("oversized data URI emitted as an image item")
|
||||
}
|
||||
}
|
||||
if len(result.Warnings) == 0 {
|
||||
t.Fatal("oversized inline image should produce a parser warning")
|
||||
}
|
||||
}
|
||||
@@ -19,7 +19,9 @@ package parser
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/net/html"
|
||||
@@ -53,19 +55,8 @@ func (p *HTMLParser) ConfigureFromSetup(setup map[string]any) {
|
||||
}
|
||||
}
|
||||
|
||||
// ParseWithResult emits one item per block-level HTML element
|
||||
// (headings, paragraphs, lists, pre blocks). The walker is a
|
||||
// pure-Go replacement for the previous `fmt.Printf` debug output:
|
||||
// it descends the html.Parse tree, joins the leaf text of each
|
||||
// block-level element, and emits the python-compatible
|
||||
// `{text, doc_type_kwd:"text"}` shape.
|
||||
//
|
||||
// Phase 2.5 (Slice 1) of port-rag-flow-pipeline-to-go.md makes
|
||||
// HTMLParser a ParseResultProducer so the dispatch seam routes
|
||||
// the html family through the structured path. Inline formatting
|
||||
// (bold / links / images) is intentionally NOT surfaced as a
|
||||
// separate ck_type — the python HtmlParser collapses inline
|
||||
// formatting into the parent block's text.
|
||||
// ParseWithResult emits normalized text items for block-level HTML elements
|
||||
// and independent image items for valid inline image payloads.
|
||||
func (p *HTMLParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {
|
||||
// x/net/html assumes UTF-8 input, so a GBK/Big5/Shift-JIS page would
|
||||
// otherwise surface as U+FFFD mojibake. Decode first, mirroring the
|
||||
@@ -87,7 +78,11 @@ func (p *HTMLParser) ParseWithResult(ctx context.Context, filename string, data
|
||||
return ParseResult{Err: fmt.Errorf("html parse: %w", err)}
|
||||
}
|
||||
var items []map[string]any
|
||||
walkHTMLBlocks(doc, &items)
|
||||
state := &htmlWalkState{}
|
||||
walkHTMLBlocksWithState(doc, &items, state)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return ParseResult{Err: err}
|
||||
}
|
||||
// remove_toc: post-parse text heuristic (mirrors Python
|
||||
// parser.py:1087-1088 remove_toc → remove_contents_table).
|
||||
if p.RemoveTOC {
|
||||
@@ -102,7 +97,8 @@ func (p *HTMLParser) ParseWithResult(ctx context.Context, filename string, data
|
||||
"name": filename,
|
||||
"encoding": encName,
|
||||
},
|
||||
JSON: items,
|
||||
JSON: items,
|
||||
Warnings: state.warnings(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -112,12 +108,23 @@ func decodeHTMLToUTF8(data []byte) ([]byte, string) {
|
||||
return DecodeToUTF8(data, "text/html")
|
||||
}
|
||||
|
||||
// walkHTMLBlocks emits one normalized item per block-level
|
||||
// descendant of root. Inline elements (b, i, a, span, …) are
|
||||
// collapsed into the parent's text via leafText. <script>,
|
||||
// <style>, and <noscript> blocks are skipped entirely so they
|
||||
// don't pollute the downstream chunker input.
|
||||
func walkHTMLBlocks(root *html.Node, out *[]map[string]any) {
|
||||
type htmlWalkState struct {
|
||||
tableSequence int
|
||||
mediaOrder int
|
||||
skippedInlineImages int
|
||||
}
|
||||
|
||||
func (s *htmlWalkState) warnings() []string {
|
||||
if s.skippedInlineImages == 0 {
|
||||
return nil
|
||||
}
|
||||
return []string{fmt.Sprintf("HTML parser skipped %d invalid or oversized inline image(s)", s.skippedInlineImages)}
|
||||
}
|
||||
|
||||
// walkHTMLBlocksWithState emits normalized text and image items in document
|
||||
// order. Inline formatting is collapsed into its parent text; script, style,
|
||||
// and noscript subtrees are skipped.
|
||||
func walkHTMLBlocksWithState(root *html.Node, out *[]map[string]any, state *htmlWalkState) {
|
||||
for child := root.FirstChild; child != nil; child = child.NextSibling {
|
||||
if child.Type == html.TextNode {
|
||||
if emitsLooseHTMLText(root) {
|
||||
@@ -138,7 +145,7 @@ func walkHTMLBlocks(root *html.Node, out *[]map[string]any) {
|
||||
continue
|
||||
case "html", "body":
|
||||
// Wrapper elements: descend into their children.
|
||||
walkHTMLBlocks(child, out)
|
||||
walkHTMLBlocksWithState(child, out, state)
|
||||
continue
|
||||
case "table":
|
||||
// Emit the <table> as a SINGLE structured doc_type_kwd:"table"
|
||||
@@ -149,19 +156,12 @@ func walkHTMLBlocks(root *html.Node, out *[]map[string]any) {
|
||||
// table context). We emit ONLY this item — no duplicate
|
||||
// doc_type_kwd:"text" copy — so the table is embedded once and
|
||||
// its markup does not pollute neighbouring prose chunks.
|
||||
markup := renderTableHTML(child)
|
||||
if strings.TrimSpace(markup) != "" {
|
||||
*out = append(*out, map[string]any{
|
||||
"text": markup,
|
||||
"doc_type_kwd": "table",
|
||||
"ck_type": "table",
|
||||
})
|
||||
}
|
||||
emitHTMLTable(child, out, state)
|
||||
continue
|
||||
}
|
||||
ckType := htmlTagToCkType(tag)
|
||||
trim := tag != "pre" && tag != "textarea"
|
||||
htmlLeafText(child, out, ckType, trim)
|
||||
htmlLeafTextWithState(child, out, ckType, trim, state)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -285,7 +285,7 @@ func (w *leafWriter) hardBreak() {
|
||||
w.endsNL = true
|
||||
}
|
||||
|
||||
// htmlLeafText joins the visible text of an HTML node and its
|
||||
// htmlLeafTextWithState joins the visible text of an HTML node and its
|
||||
// descendants and emits items directly into out. <script>/<style>/<noscript>
|
||||
// subtrees are skipped. Whitespace is folded per CSS rules (so
|
||||
// "<h1>Hello world</h1>" becomes "Hello world" and "<br>" survives as a
|
||||
@@ -297,10 +297,10 @@ func (w *leafWriter) hardBreak() {
|
||||
// doc_type_kwd:"table" item at its document position (see walkHTMLLeaf's
|
||||
// "table" case) — the prose around it is flushed as ordinary text items, so
|
||||
// the table is never relocated to the end and never duplicated.
|
||||
func htmlLeafText(n *html.Node, out *[]map[string]any, ckType string, trim bool) {
|
||||
func htmlLeafTextWithState(n *html.Node, out *[]map[string]any, ckType string, trim bool, state *htmlWalkState) {
|
||||
var b bytes.Buffer
|
||||
w := &leafWriter{b: &b}
|
||||
walkHTMLLeaf(n, w, out)
|
||||
walkHTMLLeaf(n, w, out, ckType, trim, state)
|
||||
flushLeafText(w, out, ckType, trim)
|
||||
}
|
||||
|
||||
@@ -325,7 +325,7 @@ func flushLeafText(w *leafWriter, out *[]map[string]any, ckType string, trim boo
|
||||
w.endsNL = false
|
||||
}
|
||||
|
||||
func walkHTMLLeaf(n *html.Node, w *leafWriter, out *[]map[string]any) {
|
||||
func walkHTMLLeaf(n *html.Node, w *leafWriter, out *[]map[string]any, ckType string, trim bool, state *htmlWalkState) {
|
||||
switch n.Type {
|
||||
case html.TextNode:
|
||||
w.writeText(n.Data)
|
||||
@@ -337,11 +337,23 @@ func walkHTMLLeaf(n *html.Node, w *leafWriter, out *[]map[string]any) {
|
||||
w.hardBreak()
|
||||
return
|
||||
}
|
||||
if n.Data == "img" {
|
||||
src := htmlAttribute(n, "src")
|
||||
if !usableHTMLImageSource(src) {
|
||||
if isHTMLDataImageSource(src) {
|
||||
state.skippedInlineImages++
|
||||
}
|
||||
return
|
||||
}
|
||||
flushLeafText(w, out, ckType, trim)
|
||||
appendHTMLImageItem(out, state, src, htmlAttribute(n, "alt"), "", 0, 0)
|
||||
return
|
||||
}
|
||||
if n.Data == "pre" || n.Data == "textarea" {
|
||||
// Verbatim: no folding, no injected block breaks.
|
||||
w.pre = true
|
||||
for child := n.FirstChild; child != nil; child = child.NextSibling {
|
||||
walkHTMLLeaf(child, w, out)
|
||||
walkHTMLLeaf(child, w, out, ckType, trim, state)
|
||||
}
|
||||
w.pre = false
|
||||
return
|
||||
@@ -355,15 +367,8 @@ func walkHTMLLeaf(n *html.Node, w *leafWriter, out *[]map[string]any) {
|
||||
// row/column structure survives; we do NOT also inline the markup
|
||||
// into the parent's text, which would duplicate the table and
|
||||
// pollute the prose chunk with raw tags.
|
||||
markup := renderTableHTML(n)
|
||||
if strings.TrimSpace(markup) != "" {
|
||||
flushLeafText(w, out, "text", true)
|
||||
*out = append(*out, map[string]any{
|
||||
"text": markup,
|
||||
"doc_type_kwd": "table",
|
||||
"ck_type": "table",
|
||||
})
|
||||
}
|
||||
flushLeafText(w, out, "text", true)
|
||||
emitHTMLTable(n, out, state)
|
||||
return
|
||||
}
|
||||
// Add a line break between block children so headings, paragraphs,
|
||||
@@ -378,7 +383,7 @@ func walkHTMLLeaf(n *html.Node, w *leafWriter, out *[]map[string]any) {
|
||||
}
|
||||
}
|
||||
for child := n.FirstChild; child != nil; child = child.NextSibling {
|
||||
walkHTMLLeaf(child, w, out)
|
||||
walkHTMLLeaf(child, w, out, ckType, trim, state)
|
||||
}
|
||||
if !w.pre && isBlockTag(n.Data) && w.b.Len() > 0 && !w.endsNL {
|
||||
w.hardBreak()
|
||||
@@ -386,6 +391,201 @@ func walkHTMLLeaf(n *html.Node, w *leafWriter, out *[]map[string]any) {
|
||||
}
|
||||
}
|
||||
|
||||
type htmlTableImage struct {
|
||||
src string
|
||||
alt string
|
||||
row int
|
||||
column int
|
||||
}
|
||||
|
||||
func emitHTMLTable(n *html.Node, out *[]map[string]any, state *htmlWalkState) {
|
||||
images, skippedInlineImages := htmlTableImages(n)
|
||||
state.skippedInlineImages += skippedInlineImages
|
||||
markup := renderTableHTML(n)
|
||||
if strings.TrimSpace(markup) == "" {
|
||||
return
|
||||
}
|
||||
table := map[string]any{
|
||||
"text": markup,
|
||||
"doc_type_kwd": "table",
|
||||
"ck_type": "table",
|
||||
}
|
||||
tableID := ""
|
||||
if len(images) > 0 {
|
||||
state.tableSequence++
|
||||
tableID = fmt.Sprintf("html-table-%d", state.tableSequence)
|
||||
table["source_table_id"] = tableID
|
||||
}
|
||||
*out = append(*out, table)
|
||||
for _, media := range images {
|
||||
appendHTMLImageItem(out, state, media.src, media.alt, tableID, media.row, media.column)
|
||||
}
|
||||
}
|
||||
|
||||
func htmlTableImages(table *html.Node) ([]htmlTableImage, int) {
|
||||
var images []htmlTableImage
|
||||
skippedInlineImages := 0
|
||||
rowCount := 0
|
||||
columnCounts := make(map[int]int)
|
||||
var walk func(node *html.Node, row, column int)
|
||||
walk = func(node *html.Node, row, column int) {
|
||||
if node.Type == html.ElementNode {
|
||||
switch node.Data {
|
||||
case "tr":
|
||||
rowCount++
|
||||
row = rowCount
|
||||
column = 0
|
||||
case "td", "th":
|
||||
columnCounts[row]++
|
||||
column = columnCounts[row]
|
||||
case "img":
|
||||
src := htmlAttribute(node, "src")
|
||||
if !usableHTMLImageSource(src) {
|
||||
if isHTMLDataImageSource(src) {
|
||||
skippedInlineImages++
|
||||
removeHTMLImageSource(node)
|
||||
}
|
||||
} else {
|
||||
images = append(images, htmlTableImage{
|
||||
src: src,
|
||||
alt: htmlAttribute(node, "alt"),
|
||||
row: row,
|
||||
column: column,
|
||||
})
|
||||
// The image is emitted as its own item below, so leave its
|
||||
// alt text and table cell in the markup without duplicating
|
||||
// the (potentially large) data URI.
|
||||
removeHTMLImageSource(node)
|
||||
}
|
||||
}
|
||||
}
|
||||
for child := node.FirstChild; child != nil; child = child.NextSibling {
|
||||
walk(child, row, column)
|
||||
}
|
||||
}
|
||||
walk(table, 0, 0)
|
||||
return images, skippedInlineImages
|
||||
}
|
||||
|
||||
func removeHTMLImageSource(n *html.Node) {
|
||||
attrs := n.Attr[:0]
|
||||
for _, attr := range n.Attr {
|
||||
if !strings.EqualFold(attr.Key, "src") {
|
||||
attrs = append(attrs, attr)
|
||||
}
|
||||
}
|
||||
n.Attr = attrs
|
||||
}
|
||||
|
||||
func appendHTMLImageItem(out *[]map[string]any, state *htmlWalkState, src, alt, parentTableID string, row, column int) {
|
||||
if comma := strings.IndexByte(src, ','); comma > 0 && len(src) >= len("data:image/") && strings.EqualFold(src[:len("data:image/")], "data:image/") {
|
||||
src = strings.ToLower(src[:comma]) + src[comma:]
|
||||
}
|
||||
state.mediaOrder++
|
||||
item := map[string]any{
|
||||
"text": strings.TrimSpace(alt),
|
||||
"doc_type_kwd": "image",
|
||||
"ck_type": "image",
|
||||
"media_order": state.mediaOrder,
|
||||
}
|
||||
item["image"] = src
|
||||
if parentTableID != "" {
|
||||
item["parent_table_id"] = parentTableID
|
||||
item["row_index"] = row
|
||||
item["column_index"] = column
|
||||
}
|
||||
*out = append(*out, item)
|
||||
}
|
||||
|
||||
func usableHTMLImageSource(src string) bool {
|
||||
src = strings.TrimSpace(src)
|
||||
if !isHTMLDataImageSource(src) {
|
||||
return false
|
||||
}
|
||||
separator := strings.IndexByte(src, ',')
|
||||
if separator <= len("data:image/") || separator > 128 || !strings.EqualFold(src[separator-len(";base64"):separator], ";base64") {
|
||||
return false
|
||||
}
|
||||
return validBase64ImagePayload(src[separator+1:], maxEmbeddedImageBytes)
|
||||
}
|
||||
|
||||
func isHTMLDataImageSource(src string) bool {
|
||||
const dataPrefix = "data:image/"
|
||||
src = strings.TrimSpace(src)
|
||||
return len(src) >= len(dataPrefix) && strings.EqualFold(src[:len(dataPrefix)], dataPrefix)
|
||||
}
|
||||
|
||||
func validBase64ImagePayload(payload string, maxDecodedBytes int) bool {
|
||||
if maxDecodedBytes < 0 {
|
||||
return false
|
||||
}
|
||||
maxEncodedBytes := base64.StdEncoding.EncodedLen(maxDecodedBytes)
|
||||
encodedBytes := 0
|
||||
for i := 0; i < len(payload); i++ {
|
||||
if payload[i] == '\r' || payload[i] == '\n' {
|
||||
continue
|
||||
}
|
||||
encodedBytes++
|
||||
if encodedBytes > maxEncodedBytes {
|
||||
return false
|
||||
}
|
||||
}
|
||||
if encodedBytes == 0 {
|
||||
return false
|
||||
}
|
||||
decodedSize, ok := base64DecodedSize(payload, encodedBytes)
|
||||
if !ok || decodedSize > int64(maxDecodedBytes) {
|
||||
return false
|
||||
}
|
||||
|
||||
for _, encoding := range []*base64.Encoding{base64.StdEncoding, base64.RawStdEncoding} {
|
||||
decoder := base64.NewDecoder(encoding, strings.NewReader(payload))
|
||||
decodedBytes, err := io.CopyN(io.Discard, decoder, int64(maxDecodedBytes)+1)
|
||||
if decodedBytes > int64(maxDecodedBytes) {
|
||||
return false
|
||||
}
|
||||
if err == io.EOF {
|
||||
return decodedBytes > 0
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func base64DecodedSize(payload string, encodedBytes int) (int64, bool) {
|
||||
padding := 0
|
||||
sawPadding := false
|
||||
for i := 0; i < len(payload); i++ {
|
||||
if payload[i] == '\r' || payload[i] == '\n' {
|
||||
continue
|
||||
}
|
||||
if payload[i] == '=' {
|
||||
sawPadding = true
|
||||
padding++
|
||||
if padding > 2 {
|
||||
return 0, false
|
||||
}
|
||||
continue
|
||||
}
|
||||
if sawPadding {
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
if encodedBytes%4 == 1 || (padding > 0 && encodedBytes%4 != 0) {
|
||||
return 0, false
|
||||
}
|
||||
decodedSize := int64(encodedBytes/4)*3 + int64(encodedBytes%4)*3/4 - int64(padding)
|
||||
return decodedSize, decodedSize >= 0
|
||||
}
|
||||
|
||||
func htmlAttribute(n *html.Node, key string) string {
|
||||
for _, attr := range n.Attr {
|
||||
if strings.EqualFold(attr.Key, key) {
|
||||
return attr.Val
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func isBlockTag(tag string) bool {
|
||||
switch tag {
|
||||
case "h1", "h2", "h3", "h4", "h5", "h6", "p", "li", "pre",
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package parser
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
|
||||
deepdocpdf "ragflow/internal/deepdoc/parser/pdf"
|
||||
)
|
||||
|
||||
const (
|
||||
// MaxImagePayloadBytes bounds compressed image data accepted by media paths.
|
||||
MaxImagePayloadBytes = 32 << 20
|
||||
// MaxImagePixels bounds decoded image dimensions before allocating a raster.
|
||||
MaxImagePixels = 40_000_000
|
||||
// MaxImageEdge bounds either decoded image dimension before raster allocation.
|
||||
MaxImageEdge = 12_000
|
||||
)
|
||||
|
||||
// AcquireImageMedia limits concurrent image materialization across Parser
|
||||
// payload encoding and ingestion VLM crop paths. The native analyzer keeps
|
||||
// its own inference limit.
|
||||
func AcquireImageMedia(ctx context.Context) (func(), error) {
|
||||
return sharedImageMediaAdmission().acquire(ctx)
|
||||
}
|
||||
|
||||
type imageMediaAdmission struct {
|
||||
slots chan struct{}
|
||||
}
|
||||
|
||||
func newImageMediaAdmission(limit int) *imageMediaAdmission {
|
||||
if limit < 1 {
|
||||
limit = 1
|
||||
}
|
||||
return &imageMediaAdmission{slots: make(chan struct{}, limit)}
|
||||
}
|
||||
|
||||
func (a *imageMediaAdmission) acquire(ctx context.Context) (func(), error) {
|
||||
select {
|
||||
case a.slots <- struct{}{}:
|
||||
var once sync.Once
|
||||
return func() { once.Do(func() { <-a.slots }) }, nil
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
imageMediaAdmissionOnce sync.Once
|
||||
imageMediaAdmissionPool *imageMediaAdmission
|
||||
)
|
||||
|
||||
func sharedImageMediaAdmission() *imageMediaAdmission {
|
||||
imageMediaAdmissionOnce.Do(func() {
|
||||
imageMediaAdmissionPool = newImageMediaAdmission(deepdocpdf.DeepDocConcurrency())
|
||||
})
|
||||
return imageMediaAdmissionPool
|
||||
}
|
||||
@@ -108,16 +108,30 @@ func (p *MarkdownParser) ParseWithResult(ctx context.Context, filename string, d
|
||||
doc := markdownNew().Parse([]byte(rendered))
|
||||
|
||||
var items []map[string]any
|
||||
walkMarkdownBlocksWithImages(doc, &items, p.FlattenMediaToText, p.FetchRemoteImages)
|
||||
mediaBudget := newEmbeddedMediaBudget()
|
||||
imageCtx, cancel := context.WithTimeout(ctx, 60*time.Second)
|
||||
defer cancel()
|
||||
unresolvedImages := walkMarkdownBlocksWithImages(imageCtx, doc, &items, p.FlattenMediaToText, p.FetchRemoteImages, mediaBudget)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return ParseResult{Err: err}
|
||||
}
|
||||
if items == nil {
|
||||
items = []map[string]any{{"text": "", "doc_type_kwd": "text"}}
|
||||
}
|
||||
warnings := mediaBudget.warnings()
|
||||
if mediaBudget.items > 0 && imageCtx.Err() == context.DeadlineExceeded && ctx.Err() == nil {
|
||||
warnings = append(warnings, "Markdown image processing reached the document time budget")
|
||||
}
|
||||
if unresolvedImages > 0 {
|
||||
warnings = append(warnings, fmt.Sprintf("Markdown parser could not resolve %d image source(s)", unresolvedImages))
|
||||
}
|
||||
return ParseResult{
|
||||
OutputFormat: "json",
|
||||
File: map[string]any{
|
||||
"name": filename,
|
||||
},
|
||||
JSON: items,
|
||||
JSON: items,
|
||||
Warnings: warnings,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -376,7 +390,8 @@ func markdownTableCells(line string) []string {
|
||||
// doc_type_kwd:"table" to keep the table whole and attach table context
|
||||
// to neighbouring chunks (chunker/token.go). Non-table HTML blocks
|
||||
// (<div>, <style>, …) are emitted as ordinary text with no ck_type.
|
||||
func walkMarkdownBlocksWithImages(doc ast.Node, out *[]map[string]any, flatten, fetchRemoteImages bool) {
|
||||
func walkMarkdownBlocksWithImages(ctx context.Context, doc ast.Node, out *[]map[string]any, flatten, fetchRemoteImages bool, budget *embeddedMediaBudget) int {
|
||||
unresolvedImages := 0
|
||||
for _, child := range doc.GetChildren() {
|
||||
var ckType string
|
||||
var docTypeKwd string
|
||||
@@ -457,8 +472,36 @@ func walkMarkdownBlocksWithImages(doc ast.Node, out *[]map[string]any, flatten,
|
||||
if imgURL, ok := findBlockImage(child); ok {
|
||||
isRemote := strings.HasPrefix(imgURL, "http://") || strings.HasPrefix(imgURL, "https://")
|
||||
if fetchRemoteImages || !isRemote {
|
||||
if imgData, resolved := resolveImageURL(imgURL); resolved && imgData != "" {
|
||||
item["image"] = imgData
|
||||
switch {
|
||||
case isRemote && ctx.Err() != nil:
|
||||
item["media_omitted"] = true
|
||||
case !budget.reserveItem():
|
||||
item["media_omitted"] = true
|
||||
default:
|
||||
encoded, raw, resolved := resolveImageURL(ctx, imgURL)
|
||||
if !resolved {
|
||||
unresolvedImages++
|
||||
break
|
||||
}
|
||||
size := len(raw)
|
||||
if raw == nil {
|
||||
encodedBytes := len(encoded) - strings.Count(encoded, "\r") - strings.Count(encoded, "\n")
|
||||
decodedSize, ok := base64DecodedSize(encoded, encodedBytes)
|
||||
if !ok {
|
||||
unresolvedImages++
|
||||
break
|
||||
}
|
||||
size = int(decodedSize)
|
||||
}
|
||||
if !budget.includeReservedSize(size) {
|
||||
item["media_omitted"] = true
|
||||
break
|
||||
}
|
||||
if raw != nil {
|
||||
item["image"] = base64.StdEncoding.EncodeToString(raw)
|
||||
} else {
|
||||
item["image"] = encoded
|
||||
}
|
||||
if !flatten {
|
||||
item["doc_type_kwd"] = "image"
|
||||
}
|
||||
@@ -468,6 +511,7 @@ func walkMarkdownBlocksWithImages(doc ast.Node, out *[]map[string]any, flatten,
|
||||
|
||||
*out = append(*out, item)
|
||||
}
|
||||
return unresolvedImages
|
||||
}
|
||||
|
||||
// isTableHTML reports whether block text is an outer <table> element (the
|
||||
@@ -502,44 +546,53 @@ func findBlockImage(n ast.Node) (string, bool) {
|
||||
return url, found
|
||||
}
|
||||
|
||||
// resolveImageURL resolves a Markdown image URL to its base64-encoded data.
|
||||
// resolveImageURL resolves a Markdown image URL to an encoded payload or raw
|
||||
// bytes.
|
||||
// Supports:
|
||||
// - data:image/... URIs → decoded directly
|
||||
// - data:image/... URIs → validated encoded payload
|
||||
// - http:// / https:// URLs → fetched (with basic SSRF filtering)
|
||||
//
|
||||
// Local / relative paths are not fetched (security). Returns
|
||||
// (base64String, true) on success, ("", false) when resolution fails.
|
||||
func resolveImageURL(imageURL string) (string, bool) {
|
||||
if strings.HasPrefix(imageURL, dataURIPrefix) {
|
||||
// exactly one of encoded and raw populated on success.
|
||||
func resolveImageURL(ctx context.Context, imageURL string) (encoded string, raw []byte, ok bool) {
|
||||
if strings.HasPrefix(strings.ToLower(imageURL), dataURIPrefix) {
|
||||
// data:image/png;base64,xxxx
|
||||
idx := strings.Index(imageURL, "base64,")
|
||||
if idx < 0 {
|
||||
return "", false
|
||||
idx = strings.Index(strings.ToLower(imageURL), "base64,")
|
||||
}
|
||||
return imageURL[idx+len("base64,"):], true
|
||||
if idx < 0 {
|
||||
return "", nil, false
|
||||
}
|
||||
encoded = imageURL[idx+len("base64,"):]
|
||||
if !validBase64ImagePayload(encoded, MaxImagePayloadBytes) {
|
||||
return "", nil, false
|
||||
}
|
||||
return encoded, nil, true
|
||||
}
|
||||
if strings.HasPrefix(imageURL, "http://") || strings.HasPrefix(imageURL, "https://") {
|
||||
b64, err := fetchImageAsBase64(imageURL)
|
||||
raw, err := fetchImage(ctx, imageURL)
|
||||
if err != nil {
|
||||
return "", false
|
||||
return "", nil, false
|
||||
}
|
||||
return b64, true
|
||||
return "", raw, true
|
||||
}
|
||||
// Local / relative paths — not fetched for security.
|
||||
return "", false
|
||||
return "", nil, false
|
||||
}
|
||||
|
||||
// fetchImageAsBase64 fetches an HTTP(S) image URL and returns its
|
||||
// content as a base64-encoded string. Local/private addresses and
|
||||
// redirects to them are rejected (SSRF guard). Hostnames are resolved
|
||||
// once and the validated IP is pinned in a custom DialContext to
|
||||
// prevent DNS-rebinding TOCTOU attacks.
|
||||
func fetchImageAsBase64(rawURL string) (string, error) {
|
||||
// fetchImage fetches a bounded HTTP(S) image payload. Local/private addresses
|
||||
// and redirects to them are rejected (SSRF guard). Hostnames are resolved
|
||||
// once and the validated IP is pinned in a custom DialContext to prevent
|
||||
// DNS-rebinding TOCTOU attacks.
|
||||
func fetchImage(ctx context.Context, rawURL string) ([]byte, error) {
|
||||
rawURL = strings.TrimSpace(rawURL)
|
||||
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
parsed, err := url.Parse(rawURL)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("markdown: invalid image URL: %w", err)
|
||||
return nil, fmt.Errorf("markdown: invalid image URL: %w", err)
|
||||
}
|
||||
|
||||
// pinned maps hostname (without port) → validated IP. The hostname
|
||||
@@ -549,7 +602,7 @@ func fetchImageAsBase64(rawURL string) (string, error) {
|
||||
pinned := make(map[string]net.IP)
|
||||
|
||||
pinHost := func(host string) error {
|
||||
ip, err := resolveAndValidateHost(host)
|
||||
ip, err := resolveAndValidateHost(ctx, host)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -564,7 +617,7 @@ func fetchImageAsBase64(rawURL string) (string, error) {
|
||||
}
|
||||
|
||||
if err := pinHost(parsed.Host); err != nil {
|
||||
return "", err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
transport := &http.Transport{
|
||||
@@ -594,32 +647,36 @@ func fetchImageAsBase64(rawURL string) (string, error) {
|
||||
},
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("markdown: create image request: %w", err)
|
||||
return nil, fmt.Errorf("markdown: create image request: %w", err)
|
||||
}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("markdown: fetch image %s: %w", rawURL, err)
|
||||
return nil, fmt.Errorf("markdown: fetch image %s: %w", rawURL, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return "", fmt.Errorf("markdown: fetch image %s: HTTP %d", rawURL, resp.StatusCode)
|
||||
return nil, fmt.Errorf("markdown: fetch image %s: HTTP %d", rawURL, resp.StatusCode)
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 32*1024*1024)) // 32 MiB cap
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, MaxImagePayloadBytes+1))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("markdown: read image %s: %w", rawURL, err)
|
||||
return nil, fmt.Errorf("markdown: read image %s: %w", rawURL, err)
|
||||
}
|
||||
return base64.StdEncoding.EncodeToString(body), nil
|
||||
if len(body) > MaxImagePayloadBytes {
|
||||
return nil, fmt.Errorf("markdown: image %s exceeds %d-byte limit", rawURL, MaxImagePayloadBytes)
|
||||
}
|
||||
if len(body) == 0 {
|
||||
return nil, fmt.Errorf("markdown: image %s is empty", rawURL)
|
||||
}
|
||||
return body, nil
|
||||
}
|
||||
|
||||
// resolveAndValidateHost resolves a host (which may include a port),
|
||||
// validates none of its IPs are internal/private, and returns the
|
||||
// first public IP for connection pinning.
|
||||
func resolveAndValidateHost(host string) (net.IP, error) {
|
||||
func resolveAndValidateHost(ctx context.Context, host string) (net.IP, error) {
|
||||
hostname := host
|
||||
if h, _, err := net.SplitHostPort(host); err == nil {
|
||||
hostname = h
|
||||
@@ -634,7 +691,7 @@ func resolveAndValidateHost(host string) (net.IP, error) {
|
||||
return ip, nil
|
||||
}
|
||||
|
||||
addrs, err := net.DefaultResolver.LookupIPAddr(context.Background(), hostname)
|
||||
addrs, err := net.DefaultResolver.LookupIPAddr(ctx, hostname)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("markdown: cannot resolve image host: %s", hostname)
|
||||
}
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
package parser
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
)
|
||||
|
||||
@@ -438,18 +440,18 @@ func TestMarkdownParser_FlattenMediaToText(t *testing.T) {
|
||||
|
||||
func TestResolveImageURL_DataURI(t *testing.T) {
|
||||
b64 := base64.StdEncoding.EncodeToString([]byte("fakeimage"))
|
||||
result, found := resolveImageURL("data:image/png;base64," + b64)
|
||||
result, raw, found := resolveImageURL(context.Background(), "data:image/png;base64,"+b64)
|
||||
if !found {
|
||||
t.Fatal("expected image found for data URI")
|
||||
}
|
||||
if result != b64 {
|
||||
if result != b64 || raw != nil {
|
||||
t.Fatalf("got %q, want %q", result, b64)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveImageURL_LocalPathNotFetched(t *testing.T) {
|
||||
// Local / relative paths are not fetched (security); resolution fails.
|
||||
if _, found := resolveImageURL("./local/image.png"); found {
|
||||
if _, _, found := resolveImageURL(context.Background(), "./local/image.png"); found {
|
||||
t.Fatal("expected no image resolved for a local path")
|
||||
}
|
||||
}
|
||||
@@ -468,13 +470,52 @@ func TestResolveImageURL_HTTPImage(t *testing.T) {
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
result, found := resolveImageURL(ts.URL + "/image.png")
|
||||
encoded, result, found := resolveImageURL(context.Background(), ts.URL+"/image.png")
|
||||
if !found {
|
||||
t.Fatal("expected image found for HTTP URL")
|
||||
}
|
||||
expectedB64 := base64.StdEncoding.EncodeToString([]byte("fake-png-bytes"))
|
||||
if result != expectedB64 {
|
||||
t.Fatalf("got %q, want %q", result, expectedB64)
|
||||
if encoded != "" || string(result) != "fake-png-bytes" {
|
||||
t.Fatalf("got encoded=%q raw=%q, want raw image bytes", encoded, result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarkdownImageBudgetSkipsFetchAfterCountLimit(t *testing.T) {
|
||||
previous := ssrfAllowLoopback
|
||||
ssrfAllowLoopback = true
|
||||
t.Cleanup(func() { ssrfAllowLoopback = previous })
|
||||
|
||||
var fetches atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
fetches.Add(1)
|
||||
w.Header().Set("Content-Type", "image/png")
|
||||
_, _ = w.Write([]byte("abc"))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
markdown := "\n\n" +
|
||||
"\n\n" +
|
||||
"\n"
|
||||
doc := markdownNew().Parse([]byte(markdown))
|
||||
budget := &embeddedMediaBudget{maxImageBytes: 4, maxTotalBytes: 4, maxItems: 2}
|
||||
var items []map[string]any
|
||||
if unresolved := walkMarkdownBlocksWithImages(t.Context(), doc, &items, false, true, budget); unresolved != 0 {
|
||||
t.Fatalf("unresolved images = %d, want 0", unresolved)
|
||||
}
|
||||
if got := fetches.Load(); got != 2 {
|
||||
t.Fatalf("remote fetches = %d, want 2 before the item limit", got)
|
||||
}
|
||||
if len(items) != 3 || items[0]["image"] != base64.StdEncoding.EncodeToString([]byte("abc")) {
|
||||
t.Fatalf("first image was not retained: %+v", items)
|
||||
}
|
||||
for _, index := range []int{1, 2} {
|
||||
if items[index]["image"] != nil || items[index]["media_omitted"] != true {
|
||||
t.Fatalf("image %d = %+v, want omitted payload", index, items[index])
|
||||
}
|
||||
}
|
||||
warnings := strings.Join(budget.warnings(), "\n")
|
||||
if !strings.Contains(warnings, "omitted 1 embedded image payload(s) after reaching the 4-byte document image budget") ||
|
||||
!strings.Contains(warnings, "stopped extracting embedded images after the 2-item document limit") {
|
||||
t.Fatalf("warnings = %q, want document byte and item limit warnings", warnings)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -517,21 +558,21 @@ func TestFindBlockImage(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchImageAsBase64_RejectsCredentials(t *testing.T) {
|
||||
_, err := fetchImageAsBase64("https://user:pass@example.com/img.png")
|
||||
func TestFetchImage_RejectsCredentials(t *testing.T) {
|
||||
_, err := fetchImage(context.Background(), "https://user:pass@example.com/img.png")
|
||||
if err == nil {
|
||||
t.Fatal("expected error for URL with credentials")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchImageAsBase64_InvalidURL(t *testing.T) {
|
||||
func TestFetchImage_InvalidURL(t *testing.T) {
|
||||
withSSRFBypass(t)
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
_, err := fetchImageAsBase64(ts.URL + "/nonexistent.png")
|
||||
_, err := fetchImage(context.Background(), ts.URL+"/nonexistent.png")
|
||||
if err == nil {
|
||||
t.Fatal("expected error for 404 response")
|
||||
}
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package parser
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
)
|
||||
|
||||
const (
|
||||
maxEmbeddedImageBytes = MaxImagePayloadBytes
|
||||
maxEmbeddedDocumentImageBytes = 64 << 20
|
||||
maxEmbeddedMediaItems = 256
|
||||
)
|
||||
|
||||
type embeddedMediaBudget struct {
|
||||
maxImageBytes int
|
||||
maxTotalBytes int
|
||||
maxItems int
|
||||
|
||||
items int
|
||||
totalBytes int
|
||||
oversizedImages int
|
||||
documentOverflows int
|
||||
countExhausted bool
|
||||
}
|
||||
|
||||
func newEmbeddedMediaBudget() *embeddedMediaBudget {
|
||||
return &embeddedMediaBudget{
|
||||
maxImageBytes: maxEmbeddedImageBytes,
|
||||
maxTotalBytes: maxEmbeddedDocumentImageBytes,
|
||||
maxItems: maxEmbeddedMediaItems,
|
||||
}
|
||||
}
|
||||
|
||||
// include reserves budget for one image payload. The second result is false
|
||||
// when callers should stop walking images because the document item limit has
|
||||
// been reached. Byte-limit omissions can keep walking so later text and media
|
||||
// metadata remain available.
|
||||
func (b *embeddedMediaBudget) include(data []byte) (bool, bool) {
|
||||
if !b.reserveItem() {
|
||||
return false, false
|
||||
}
|
||||
return b.includeReservedSize(len(data)), true
|
||||
}
|
||||
|
||||
// reserveItem lets a parser cap image source attempts before fetching bytes.
|
||||
func (b *embeddedMediaBudget) reserveItem() bool {
|
||||
if b.countExhausted {
|
||||
return false
|
||||
}
|
||||
if b.items >= b.maxItems {
|
||||
b.countExhausted = true
|
||||
return false
|
||||
}
|
||||
b.items++
|
||||
return true
|
||||
}
|
||||
|
||||
// includeReservedSize charges a previously reserved image without retaining
|
||||
// or decoding its payload just to measure its size.
|
||||
func (b *embeddedMediaBudget) includeReservedSize(size int) bool {
|
||||
if size > b.maxImageBytes {
|
||||
b.oversizedImages++
|
||||
return false
|
||||
}
|
||||
if size > b.maxTotalBytes-b.totalBytes {
|
||||
b.documentOverflows++
|
||||
return false
|
||||
}
|
||||
b.totalBytes += size
|
||||
return true
|
||||
}
|
||||
|
||||
func (b *embeddedMediaBudget) warnings() []string {
|
||||
if b == nil {
|
||||
return nil
|
||||
}
|
||||
var warnings []string
|
||||
if b.oversizedImages > 0 {
|
||||
warnings = append(warnings, fmt.Sprintf(
|
||||
"omitted %d embedded image payload(s) larger than the %d-byte per-image limit",
|
||||
b.oversizedImages,
|
||||
b.maxImageBytes,
|
||||
))
|
||||
}
|
||||
if b.documentOverflows > 0 {
|
||||
warnings = append(warnings, fmt.Sprintf(
|
||||
"omitted %d embedded image payload(s) after reaching the %d-byte document image budget",
|
||||
b.documentOverflows,
|
||||
b.maxTotalBytes,
|
||||
))
|
||||
}
|
||||
if b.countExhausted {
|
||||
warnings = append(warnings, fmt.Sprintf(
|
||||
"stopped extracting embedded images after the %d-item document limit",
|
||||
b.maxItems,
|
||||
))
|
||||
}
|
||||
return warnings
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
package parser
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestEmbeddedMediaBudgetReservedItemsAndWarnings(t *testing.T) {
|
||||
budget := &embeddedMediaBudget{maxImageBytes: 4, maxTotalBytes: 5, maxItems: 3}
|
||||
|
||||
if !budget.reserveItem() || !budget.includeReservedSize(3) {
|
||||
t.Fatal("first image should fit")
|
||||
}
|
||||
if !budget.reserveItem() || budget.includeReservedSize(3) {
|
||||
t.Fatal("second image should exceed the document byte limit")
|
||||
}
|
||||
if !budget.reserveItem() || budget.includeReservedSize(5) {
|
||||
t.Fatal("third image should exceed the per-image byte limit")
|
||||
}
|
||||
if budget.reserveItem() {
|
||||
t.Fatal("fourth image should be rejected before its source is read")
|
||||
}
|
||||
if included, keepWalking := budget.include([]byte("x")); included || keepWalking {
|
||||
t.Fatalf("include after count exhaustion = (%v, %v), want (false, false)", included, keepWalking)
|
||||
}
|
||||
if budget.items != 3 || budget.totalBytes != 3 {
|
||||
t.Fatalf("budget counters = (%d items, %d bytes), want (3, 3)", budget.items, budget.totalBytes)
|
||||
}
|
||||
want := []string{
|
||||
"omitted 1 embedded image payload(s) larger than the 4-byte per-image limit",
|
||||
"omitted 1 embedded image payload(s) after reaching the 5-byte document image budget",
|
||||
"stopped extracting embedded images after the 3-item document limit",
|
||||
}
|
||||
if got := budget.warnings(); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("warnings = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmbeddedMediaBudgetIncludesExactByteLimits(t *testing.T) {
|
||||
budget := &embeddedMediaBudget{maxImageBytes: 4, maxTotalBytes: 4, maxItems: 2}
|
||||
if included, keepWalking := budget.include([]byte("abcd")); !included || !keepWalking {
|
||||
t.Fatalf("exact-limit image = (%v, %v), want (true, true)", included, keepWalking)
|
||||
}
|
||||
if included, keepWalking := budget.include([]byte("e")); included || !keepWalking {
|
||||
t.Fatalf("image after exact document limit = (%v, %v), want (false, true)", included, keepWalking)
|
||||
}
|
||||
if budget.totalBytes != 4 {
|
||||
t.Fatalf("totalBytes = %d, want 4", budget.totalBytes)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
//go:build cgo
|
||||
|
||||
package parser
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/png"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
officeOxide "github.com/yfedoseev/office_oxide/go"
|
||||
deepdocpdf "ragflow/internal/deepdoc/parser/pdf"
|
||||
deepdoctype "ragflow/internal/deepdoc/parser/type"
|
||||
)
|
||||
|
||||
func useFailingOCRAnalyzer(t *testing.T) {
|
||||
t.Helper()
|
||||
previous := deepdoctype.NativeDocAnalyzerFactory
|
||||
t.Cleanup(func() { deepdoctype.NativeDocAnalyzerFactory = previous })
|
||||
deepdoctype.SetNativeDocAnalyzerFactory(func() (deepdoctype.DocAnalyzer, bool) {
|
||||
return &deepdocpdf.MockDocAnalyzer{Healthy: true, OCRDetectErr: errors.New("image parser must not invoke OCR")}, true
|
||||
})
|
||||
}
|
||||
|
||||
func TestDOCXParser_RealTableImage(t *testing.T) {
|
||||
useFailingOCRAnalyzer(t)
|
||||
data, err := os.ReadFile(filepath.Join("testdata", "docx_table_image.docx"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p := NewDOCXParser()
|
||||
p.ConfigureFromSetup(map[string]any{"output_format": "json"})
|
||||
result := p.ParseWithResult(t.Context(), "docx_table_image.docx", data)
|
||||
if result.Err != nil {
|
||||
t.Fatal(result.Err)
|
||||
}
|
||||
if len(result.Warnings) != 0 {
|
||||
t.Fatalf("warnings = %v, want none for an admitted image", result.Warnings)
|
||||
}
|
||||
var tableID string
|
||||
for _, item := range result.JSON {
|
||||
if item["doc_type_kwd"] == "table" {
|
||||
tableID, _ = item["source_table_id"].(string)
|
||||
}
|
||||
}
|
||||
if tableID == "" {
|
||||
t.Fatalf("table with image not found: %+v", result.JSON)
|
||||
}
|
||||
for _, item := range result.JSON {
|
||||
if item["doc_type_kwd"] != "image" || item["parent_table_id"] != tableID {
|
||||
continue
|
||||
}
|
||||
encoded, _ := item["image"].(string)
|
||||
decoded, err := base64.StdEncoding.DecodeString(encoded)
|
||||
if err != nil {
|
||||
t.Fatalf("table image payload: %v", err)
|
||||
}
|
||||
if _, err := png.Decode(bytes.NewReader(decoded)); err != nil {
|
||||
t.Fatalf("table image is not PNG: %v", err)
|
||||
}
|
||||
if item["row_index"] != 1 || item["column_index"] != 1 || item["media_order"] != 1 {
|
||||
t.Fatalf("table image location = %+v", item)
|
||||
}
|
||||
return
|
||||
}
|
||||
t.Fatalf("no image linked to table %q: %+v", tableID, result.JSON)
|
||||
}
|
||||
|
||||
func TestPPTXParser_RealSlideImage(t *testing.T) {
|
||||
useFailingOCRAnalyzer(t)
|
||||
raster := image.NewRGBA(image.Rect(0, 0, 32, 16))
|
||||
for y := 0; y < 16; y++ {
|
||||
for x := 0; x < 32; x++ {
|
||||
raster.Set(x, y, color.RGBA{R: 255, A: 255})
|
||||
}
|
||||
}
|
||||
var imageData bytes.Buffer
|
||||
if err := png.Encode(&imageData, raster); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w := officeOxide.NewPptxWriter()
|
||||
defer w.Close()
|
||||
slide := w.AddSlide()
|
||||
w.AddSlideText(slide, "slide with image")
|
||||
w.AddSlideImage(slide, imageData.Bytes(), "png", 914400, 914400, 914400, 457200)
|
||||
data, err := w.ToBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result := NewPPTXParser().ParseWithResult(t.Context(), "slide_image.pptx", data)
|
||||
if result.Err != nil {
|
||||
t.Fatal(result.Err)
|
||||
}
|
||||
if len(result.Warnings) != 0 {
|
||||
t.Fatalf("warnings = %v, want none for an admitted image", result.Warnings)
|
||||
}
|
||||
for _, item := range result.JSON {
|
||||
if item["doc_type_kwd"] != "image" {
|
||||
continue
|
||||
}
|
||||
encoded, _ := item["image"].(string)
|
||||
decoded, err := base64.StdEncoding.DecodeString(encoded)
|
||||
if err != nil {
|
||||
t.Fatalf("slide image payload: %v", err)
|
||||
}
|
||||
if _, err := png.Decode(bytes.NewReader(decoded)); err != nil {
|
||||
t.Fatalf("slide image is not PNG: %v", err)
|
||||
}
|
||||
if item["slide_number"] != 1 || item["media_order"] != 1 {
|
||||
t.Fatalf("slide image location = %+v", item)
|
||||
}
|
||||
return
|
||||
}
|
||||
t.Fatalf("slide image not emitted: %+v", result.JSON)
|
||||
}
|
||||
@@ -297,7 +297,7 @@ func deprecatedHTML4Excel(setup map[string]any, parserName string) {
|
||||
// worksheet as structured parser items. Excelize exposes both kinds through
|
||||
// GetPictureCells/GetPictures; walking the reported anchor cells avoids
|
||||
// scanning the worksheet's entire coordinate space.
|
||||
func extractXLSXImages(f *excelize.File, sheet string) ([]map[string]any, []string) {
|
||||
func extractXLSXImages(f *excelize.File, sheet string, budget *embeddedMediaBudget) ([]map[string]any, []string) {
|
||||
cells, err := f.GetPictureCells(sheet)
|
||||
if err != nil {
|
||||
return nil, []string{fmt.Sprintf("XLSX image discovery failed for sheet %q: %v", sheet, err)}
|
||||
@@ -308,6 +308,7 @@ func extractXLSXImages(f *excelize.File, sheet string) ([]map[string]any, []stri
|
||||
|
||||
items := make([]map[string]any, 0, len(cells))
|
||||
warnings := make([]string, 0)
|
||||
cellLoop:
|
||||
for _, cell := range cells {
|
||||
pictures, err := f.GetPictures(sheet, cell)
|
||||
if err != nil {
|
||||
@@ -323,24 +324,37 @@ func extractXLSXImages(f *excelize.File, sheet string) ([]map[string]any, []stri
|
||||
warnings = append(warnings, fmt.Sprintf("XLSX image skipped for sheet %q cell %s: unsupported extension %q", sheet, cell, picture.Extension))
|
||||
continue
|
||||
}
|
||||
encoded := base64.StdEncoding.EncodeToString(picture.File)
|
||||
included, keepWalking := true, true
|
||||
if budget != nil {
|
||||
included, keepWalking = budget.include(picture.File)
|
||||
}
|
||||
if !keepWalking {
|
||||
break cellLoop
|
||||
}
|
||||
alt := cell
|
||||
if picture.Format != nil && picture.Format.AltText != "" {
|
||||
alt = picture.Format.AltText
|
||||
}
|
||||
row, col := axisToRC(cell)
|
||||
items = append(items, map[string]any{
|
||||
item := map[string]any{
|
||||
"text": alt,
|
||||
"doc_type_kwd": "image",
|
||||
"ck_type": "image",
|
||||
"image": "data:" + mimeType + ";base64," + encoded,
|
||||
"sheet": sheet,
|
||||
"cell": cell,
|
||||
"row_start": row,
|
||||
"row_end": row,
|
||||
"col_start": col,
|
||||
"col_end": col,
|
||||
})
|
||||
}
|
||||
if included {
|
||||
encoded := base64.StdEncoding.EncodeToString(picture.File)
|
||||
item["image"] = "data:" + mimeType + ";base64," + encoded
|
||||
} else {
|
||||
item["image"] = nil
|
||||
item["media_omitted"] = true
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
}
|
||||
return items, warnings
|
||||
|
||||
@@ -367,7 +367,7 @@ func TestXLSXImageMIMEType(t *testing.T) {
|
||||
func TestExtractXLSXImagesWarningForInvalidSheet(t *testing.T) {
|
||||
f := excelize.NewFile()
|
||||
defer f.Close()
|
||||
_, warnings := extractXLSXImages(f, "MissingSheet")
|
||||
_, warnings := extractXLSXImages(f, "MissingSheet", nil)
|
||||
if len(warnings) != 1 || !strings.Contains(warnings[0], "image discovery failed") {
|
||||
t.Fatalf("warnings = %v, want image discovery warning", warnings)
|
||||
}
|
||||
|
||||
@@ -62,8 +62,8 @@ type ParseResult struct {
|
||||
// `[]map[string]any` with `text` + `doc_type_kwd` keys (and
|
||||
// optional `image` / `layout` / `positions` fields);
|
||||
// Markdown / HTML / text emit normalized
|
||||
// `{text, doc_type_kwd}` items; image emits OCR/VLM result
|
||||
// items.
|
||||
// `{text, doc_type_kwd}` items; image emits an image payload and, after
|
||||
// ingestion enhancement, its VLM description.
|
||||
JSON []map[string]any
|
||||
|
||||
// Markdown is a backend response awaiting normalization.
|
||||
|
||||
@@ -241,12 +241,13 @@ Tail text`)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWalkHTMLBlocks_SkipsHeadLooseText(t *testing.T) {
|
||||
func TestWalkHTMLBlocksWithState_SkipsHeadLooseText(t *testing.T) {
|
||||
head := &html.Node{Type: html.ElementNode, Data: "head"}
|
||||
head.AppendChild(&html.Node{Type: html.TextNode, Data: "Head metadata"})
|
||||
|
||||
var items []map[string]any
|
||||
walkHTMLBlocks(head, &items)
|
||||
state := &htmlWalkState{}
|
||||
walkHTMLBlocksWithState(head, &items, state)
|
||||
if len(items) != 0 {
|
||||
t.Fatalf("JSON len = %d, want 0: %#v", len(items), items)
|
||||
}
|
||||
|
||||
@@ -380,8 +380,8 @@ func resolveDocAnalyzer(factory func() (deepdoctype.DocAnalyzer, bool)) (deepdoc
|
||||
|
||||
// GetDocAnalyzer returns the configured in-process DeepDoc analyzer. It is the
|
||||
// single production entry point now that the external HTTP service is no longer
|
||||
// a backend. Callers outside the parser package (e.g. standalone image OCR in
|
||||
// the ingestion component) use this instead of constructing a client.
|
||||
// a backend. PDF and non-PDF image parsers use it instead of constructing a
|
||||
// client.
|
||||
func GetDocAnalyzer() (deepdoctype.DocAnalyzer, error) {
|
||||
return deepDocAnalyzerFromEnv()
|
||||
}
|
||||
|
||||
@@ -14,23 +14,14 @@
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
// PictureParser validates and stores configuration for image files.
|
||||
// The actual OCR and VLM description is performed by the
|
||||
// component-layer maybeDispatchImage, which mirrors Python's
|
||||
// rag/app/picture.py:chunk() image branch.
|
||||
//
|
||||
// Python reference:
|
||||
// - Image is opened with PIL, converted to RGB
|
||||
// - PaddleOCR tried first (if layout_recognize == "@PaddleOCR")
|
||||
// - Falls back to local deepdoc.vision.OCR (ONNX text detection)
|
||||
// - If OCR text is short (≤32 chars / ≤32 words for English),
|
||||
// calls IMAGE2TEXT VLM describe() for a natural-language description
|
||||
// - Returns tokenized text with media context attached
|
||||
// PictureParser validates image files and returns their image payload for
|
||||
// optional VLM enhancement in the ingestion component.
|
||||
|
||||
package parser
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -50,7 +41,7 @@ var imageExtensions = map[string]bool{
|
||||
"ai": true, "raw": true, "wmf": true,
|
||||
}
|
||||
|
||||
// PictureParser handles image files for OCR and VLM description.
|
||||
// PictureParser emits image files for optional vision enhancement.
|
||||
type PictureParser struct {
|
||||
OutputFormat string
|
||||
}
|
||||
@@ -71,10 +62,7 @@ func (p *PictureParser) ConfigureFromSetup(setup map[string]any) {
|
||||
}
|
||||
}
|
||||
|
||||
// ParseWithResult implements ParseResultProducer. It validates the
|
||||
// file extension against the image extension whitelist. The actual
|
||||
// OCR and VLM description happens via maybeDispatchImage at the
|
||||
// component layer (mirrors Python's picture.py:chunk()).
|
||||
// ParseWithResult validates the image extension and returns its image payload.
|
||||
func (p *PictureParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {
|
||||
ext := strings.ToLower(filepath.Ext(filename))
|
||||
if len(ext) > 1 && ext[0] == '.' {
|
||||
@@ -94,6 +82,9 @@ func (p *PictureParser) ParseWithResult(ctx context.Context, filename string, da
|
||||
Err: fmt.Errorf("picture: unsupported extension %q (filename: %s); accepted: .jpg/.jpeg/.png/.gif/.bmp/.tiff/.tif/.webp/.svg/.ico/.avif/.heic/...", ext, filename),
|
||||
}
|
||||
}
|
||||
if len(data) == 0 || len(data) > MaxImagePayloadBytes {
|
||||
return ParseResult{Err: fmt.Errorf("picture: image payload size %d is outside the %d-byte limit", len(data), MaxImagePayloadBytes)}
|
||||
}
|
||||
|
||||
// Parser output is normalized to JSON at the component boundary, so an
|
||||
// absent backend format defaults to JSON here as well.
|
||||
@@ -102,13 +93,55 @@ func (p *PictureParser) ParseWithResult(ctx context.Context, filename string, da
|
||||
outFmt = "json"
|
||||
}
|
||||
|
||||
release, err := AcquireImageMedia(ctx)
|
||||
if err != nil {
|
||||
return ParseResult{Err: err}
|
||||
}
|
||||
imagePayload := "data:" + pictureMIME(ext) + ";base64," + base64.StdEncoding.EncodeToString(data)
|
||||
release()
|
||||
item := map[string]any{
|
||||
"text": "",
|
||||
"image": imagePayload,
|
||||
"doc_type_kwd": DocTypeImage,
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return ParseResult{Err: err}
|
||||
}
|
||||
return ParseResult{
|
||||
OutputFormat: outFmt,
|
||||
File: map[string]any{
|
||||
"name": filename,
|
||||
"size": len(data),
|
||||
"doc_type_kwd": "image",
|
||||
"doc_type_kwd": DocTypeImage,
|
||||
},
|
||||
JSON: []map[string]any{item},
|
||||
}
|
||||
}
|
||||
|
||||
func pictureMIME(ext string) string {
|
||||
switch ext {
|
||||
case "jpg", "jpeg":
|
||||
return "image/jpeg"
|
||||
case "png":
|
||||
return "image/png"
|
||||
case "gif":
|
||||
return "image/gif"
|
||||
case "bmp":
|
||||
return "image/bmp"
|
||||
case "webp":
|
||||
return "image/webp"
|
||||
case "svg":
|
||||
return "image/svg+xml"
|
||||
case "tiff", "tif":
|
||||
return "image/tiff"
|
||||
case "ico":
|
||||
return "image/x-icon"
|
||||
case "avif":
|
||||
return "image/avif"
|
||||
case "heic":
|
||||
return "image/heic"
|
||||
default:
|
||||
return "image/png"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -17,10 +17,29 @@
|
||||
package parser
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPictureParserKeepsImageWithoutOCR(t *testing.T) {
|
||||
data := []byte("picture bytes")
|
||||
res := NewPictureParser().ParseWithResult(t.Context(), "photo.png", data)
|
||||
if res.Err != nil {
|
||||
t.Fatal(res.Err)
|
||||
}
|
||||
if len(res.JSON) != 1 || res.JSON[0]["text"] != "" {
|
||||
t.Fatalf("picture text = %+v, want empty text", res.JSON)
|
||||
}
|
||||
wantImage := "data:image/png;base64," + base64.StdEncoding.EncodeToString(data)
|
||||
if res.JSON[0]["image"] != wantImage {
|
||||
t.Fatalf("picture image = %v, want %q", res.JSON[0]["image"], wantImage)
|
||||
}
|
||||
if len(res.Warnings) != 0 {
|
||||
t.Fatalf("picture warnings = %v, want none", res.Warnings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewPictureParser(t *testing.T) {
|
||||
p := NewPictureParser()
|
||||
if p == nil {
|
||||
|
||||
@@ -17,16 +17,17 @@
|
||||
package parser
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// buildPPTXJSONSections converts office_oxide presentation IR JSON into
|
||||
// one JSON item per slide. The IR carries exactly one section per slide,
|
||||
// so section index + 1 is the slide number. Per-slide text is flattened
|
||||
// with the shared IR walker (docxElementText), which covers paragraphs,
|
||||
// headings, text boxes, tables, and (nested) lists.
|
||||
// buildPPTXJSONSections converts office_oxide presentation IR JSON into one
|
||||
// text item per slide followed by that slide's image items. The IR carries
|
||||
// exactly one section per slide, so section index + 1 is the slide number.
|
||||
// Per-slide text is flattened with the shared IR walker (docxElementText),
|
||||
// which covers paragraphs, headings, text boxes, tables, and (nested) lists.
|
||||
//
|
||||
// A slide without extractable text (blank layout, image-only) still
|
||||
// yields an item with an empty text field, so slide_number stays
|
||||
@@ -34,11 +35,14 @@ import (
|
||||
//
|
||||
// Pure Go (no cgo) so the IR → items transform is unit-testable without
|
||||
// the office_oxide native library.
|
||||
func buildPPTXJSONSections(irJSON string) ([]map[string]any, error) {
|
||||
func buildPPTXJSONSections(irJSON string, budget *embeddedMediaBudget) ([]map[string]any, error) {
|
||||
var ir docxIRDocument
|
||||
if err := json.Unmarshal([]byte(irJSON), &ir); err != nil {
|
||||
return nil, fmt.Errorf("presentation ir-json decode: %w", err)
|
||||
}
|
||||
if budget == nil {
|
||||
budget = newEmbeddedMediaBudget()
|
||||
}
|
||||
items := make([]map[string]any, 0, len(ir.Sections))
|
||||
for i, sec := range ir.Sections {
|
||||
var lines []string
|
||||
@@ -58,19 +62,47 @@ func buildPPTXJSONSections(irJSON string) ([]map[string]any, error) {
|
||||
"doc_type_kwd": "text",
|
||||
"slide_number": i + 1,
|
||||
})
|
||||
mediaOrder := 0
|
||||
forEachDOCXIRImage(sec.Elements, func(data []byte) bool {
|
||||
mediaOrder++
|
||||
imageItem := map[string]any{
|
||||
"text": "",
|
||||
"image": nil,
|
||||
"doc_type_kwd": "image",
|
||||
"ck_type": "image",
|
||||
"slide_number": i + 1,
|
||||
"media_order": mediaOrder,
|
||||
}
|
||||
included, keepWalking := budget.include(data)
|
||||
if included {
|
||||
imageItem["image"] = base64.StdEncoding.EncodeToString(data)
|
||||
} else if keepWalking {
|
||||
imageItem["media_omitted"] = true
|
||||
} else {
|
||||
return false
|
||||
}
|
||||
items = append(items, imageItem)
|
||||
return keepWalking
|
||||
})
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// itemsAllEmpty reports whether every item carries no extractable text.
|
||||
// A deck whose IR sections all flatten to "" (e.g. text the IR walker does
|
||||
// not yet cover) must fall back to PlainText instead of emitting empty
|
||||
// chunks that the Tokenizer later filters, silently yielding 0 chunks.
|
||||
// itemsAllEmpty reports whether every item lacks both text and image payloads.
|
||||
// A deck whose IR sections all flatten to "" and contain no images must fall
|
||||
// back to PlainText instead of emitting empty chunks that the Tokenizer later
|
||||
// filters, silently yielding 0 chunks.
|
||||
func itemsAllEmpty(items []map[string]any) bool {
|
||||
for _, it := range items {
|
||||
if text, _ := it["text"].(string); strings.TrimSpace(text) != "" {
|
||||
return false
|
||||
}
|
||||
if image, _ := it["image"].(string); strings.TrimSpace(image) != "" {
|
||||
return false
|
||||
}
|
||||
if omitted, _ := it["media_omitted"].(bool); omitted {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -103,11 +103,6 @@ func TestBuildPPTXJSONSections(t *testing.T) {
|
||||
irJSON: `{"sections":[{"elements":[{"type":"text","text":"loose text"}]}]}`,
|
||||
wantTexts: []string{"loose text"},
|
||||
},
|
||||
{
|
||||
name: "image-only slide yields empty text",
|
||||
irJSON: `{"sections":[{"elements":[{"type":"image","data":"aGVsbG8="}]}]}`,
|
||||
wantTexts: []string{""},
|
||||
},
|
||||
{
|
||||
// bwbd.pptx regression: office_oxide wraps grouped slide shapes
|
||||
// (graphicFrame tables) in a text_box, so the slide's text_box
|
||||
@@ -176,7 +171,7 @@ func TestBuildPPTXJSONSections(t *testing.T) {
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
items, err := buildPPTXJSONSections(tt.irJSON)
|
||||
items, err := buildPPTXJSONSections(tt.irJSON, newEmbeddedMediaBudget())
|
||||
if tt.wantErr {
|
||||
if err == nil {
|
||||
t.Fatalf("buildPPTXJSONSections: want error, got items %+v", items)
|
||||
@@ -204,6 +199,47 @@ func TestBuildPPTXJSONSections(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildPPTXJSONSections_ExtractsSlideImagesInOrder(t *testing.T) {
|
||||
irJSON := `{"sections":[{"elements":[
|
||||
{"type":"paragraph","content":[{"type":"text","text":"slide text"}]},
|
||||
{"type":"image","data":"aGVsbG8="},
|
||||
{"type":"table","rows":[{"cells":[{"content":[{"type":"image","data":"aW1hZ2U="}]}]}]}
|
||||
]},{"elements":[{"type":"image","data":"c2Vjb25k"}]}]}`
|
||||
|
||||
items, err := buildPPTXJSONSections(irJSON, newEmbeddedMediaBudget())
|
||||
if err != nil {
|
||||
t.Fatalf("buildPPTXJSONSections: %v", err)
|
||||
}
|
||||
want := []struct {
|
||||
docType string
|
||||
text string
|
||||
image string
|
||||
slide int
|
||||
order int
|
||||
}{
|
||||
{docType: "text", text: "slide text", slide: 1},
|
||||
{docType: "image", image: "aGVsbG8=", slide: 1, order: 1},
|
||||
{docType: "image", image: "aW1hZ2U=", slide: 1, order: 2},
|
||||
{docType: "text", slide: 2},
|
||||
{docType: "image", image: "c2Vjb25k", slide: 2, order: 1},
|
||||
}
|
||||
if len(items) != len(want) {
|
||||
t.Fatalf("items = %+v, want %d slide/text and image items", items, len(want))
|
||||
}
|
||||
for i, w := range want {
|
||||
item := items[i]
|
||||
if item["doc_type_kwd"] != w.docType || item["slide_number"] != w.slide {
|
||||
t.Errorf("item %d identity = %+v, want type %q slide %d", i, item, w.docType, w.slide)
|
||||
}
|
||||
if w.docType == "image" && (item["image"] != w.image || item["media_order"] != w.order) {
|
||||
t.Errorf("item %d image = %+v, want payload %q order %d", i, item, w.image, w.order)
|
||||
}
|
||||
if w.docType == "text" && item["text"] != w.text {
|
||||
t.Errorf("item %d text = %v, want %q", i, item["text"], w.text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestItemsAllEmpty pins the all-empty fallback gate: empty and whitespace
|
||||
// items trigger the PlainText salvage, any non-empty item does not.
|
||||
func TestItemsAllEmpty(t *testing.T) {
|
||||
@@ -224,6 +260,11 @@ func TestItemsAllEmpty(t *testing.T) {
|
||||
items: []map[string]any{{"text": ""}, {"text": "hello"}},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "image payload",
|
||||
items: []map[string]any{{"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"}},
|
||||
want: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
|
||||
@@ -164,10 +164,14 @@ func (p *PPTXParser) ParseWithResult(ctx context.Context, filename string, data
|
||||
JSON: itemsFromPlainText(text),
|
||||
}
|
||||
}
|
||||
items, err := buildPPTXJSONSections(irJSON)
|
||||
budget := newEmbeddedMediaBudget()
|
||||
items, err := buildPPTXJSONSections(irJSON, budget)
|
||||
if err != nil {
|
||||
return ParseResult{Err: err}
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return ParseResult{Err: err}
|
||||
}
|
||||
if len(items) == 0 || itemsAllEmpty(items) {
|
||||
// A deck whose IR carries no sections at all, or whose sections
|
||||
// all flatten to empty text (text the IR walker does not cover):
|
||||
@@ -186,5 +190,6 @@ func (p *PPTXParser) ParseWithResult(ctx context.Context, filename string, data
|
||||
OutputFormat: "json",
|
||||
File: map[string]any{"name": filename, "format": effFormat},
|
||||
JSON: items,
|
||||
Warnings: budget.warnings(),
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@@ -85,10 +85,15 @@ func (p *XLSParser) ParseWithResult(ctx context.Context, filename string, data [
|
||||
}
|
||||
}
|
||||
|
||||
items, warnings, sheetsCount, err := parseXLSXBytes(data)
|
||||
mediaBudget := newEmbeddedMediaBudget()
|
||||
items, warnings, sheetsCount, err := parseXLSXBytes(data, mediaBudget)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return ParseResult{Err: err}
|
||||
}
|
||||
if err != nil {
|
||||
return ParseResult{Err: fmt.Errorf("xls parse: %w", err)}
|
||||
}
|
||||
warnings = append(warnings, mediaBudget.warnings()...)
|
||||
|
||||
return ParseResult{
|
||||
OutputFormat: spreadsheetOutputFormat,
|
||||
|
||||
@@ -3,10 +3,16 @@ package parser
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"image"
|
||||
"image/png"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/xuri/excelize/v2"
|
||||
deepdocpdf "ragflow/internal/deepdoc/parser/pdf"
|
||||
deepdoctype "ragflow/internal/deepdoc/parser/type"
|
||||
)
|
||||
|
||||
func createTestExcelBytes(t *testing.T) []byte {
|
||||
@@ -112,3 +118,100 @@ func TestXLSXParser_JSONOutput(t *testing.T) {
|
||||
t.Errorf("resHTML.JSON should have items")
|
||||
}
|
||||
}
|
||||
|
||||
func TestXLSParser_ImagesDoNotRunOCR(t *testing.T) {
|
||||
var imageData bytes.Buffer
|
||||
if err := png.Encode(&imageData, image.NewRGBA(image.Rect(0, 0, 2, 2))); err != nil {
|
||||
t.Fatalf("encode fixture: %v", err)
|
||||
}
|
||||
imageBytes := imageData.Bytes()
|
||||
f := excelize.NewFile()
|
||||
defer f.Close()
|
||||
if err := f.AddPictureFromBytes("Sheet1", "C3", &excelize.Picture{
|
||||
Extension: ".png",
|
||||
File: imageBytes,
|
||||
Format: &excelize.GraphicOptions{AltText: "sheet image"},
|
||||
}); err != nil {
|
||||
t.Fatalf("AddPictureFromBytes: %v", err)
|
||||
}
|
||||
var workbook bytes.Buffer
|
||||
if err := f.Write(&workbook); err != nil {
|
||||
t.Fatalf("write workbook: %v", err)
|
||||
}
|
||||
|
||||
previous := deepdoctype.NativeDocAnalyzerFactory
|
||||
t.Cleanup(func() { deepdoctype.NativeDocAnalyzerFactory = previous })
|
||||
deepdoctype.SetNativeDocAnalyzerFactory(func() (deepdoctype.DocAnalyzer, bool) {
|
||||
return &deepdocpdf.MockDocAnalyzer{Healthy: true, OCRDetectErr: errors.New("xls OCR sentinel")}, true
|
||||
})
|
||||
|
||||
p, err := NewXLSParser("excelize")
|
||||
if err != nil {
|
||||
t.Fatalf("NewXLSParser: %v", err)
|
||||
}
|
||||
result := p.ParseWithResult(t.Context(), "with-image.xls", workbook.Bytes())
|
||||
if result.Err != nil {
|
||||
t.Fatalf("ParseWithResult: %v", result.Err)
|
||||
}
|
||||
var imageItem map[string]any
|
||||
for _, item := range result.JSON {
|
||||
if item["doc_type_kwd"] == "image" {
|
||||
imageItem = item
|
||||
break
|
||||
}
|
||||
}
|
||||
if imageItem == nil || imageItem["image"] != "data:image/png;base64,"+base64.StdEncoding.EncodeToString(imageBytes) {
|
||||
t.Fatalf("image item = %+v, want retained picture payload", imageItem)
|
||||
}
|
||||
if len(result.Warnings) != 0 {
|
||||
t.Fatalf("warnings = %v, want none for an admitted image", result.Warnings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestXLSParser_CapsEmbeddedImageCount(t *testing.T) {
|
||||
var imageData bytes.Buffer
|
||||
if err := png.Encode(&imageData, image.NewRGBA(image.Rect(0, 0, 2, 2))); err != nil {
|
||||
t.Fatalf("encode fixture: %v", err)
|
||||
}
|
||||
f := excelize.NewFile()
|
||||
defer f.Close()
|
||||
for i := 0; i <= maxEmbeddedMediaItems; i++ {
|
||||
cell, err := excelize.CoordinatesToCellName(3, i+1)
|
||||
if err != nil {
|
||||
t.Fatalf("cell %d: %v", i, err)
|
||||
}
|
||||
if err := f.AddPictureFromBytes("Sheet1", cell, &excelize.Picture{Extension: ".png", File: imageData.Bytes()}); err != nil {
|
||||
t.Fatalf("AddPictureFromBytes(%s): %v", cell, err)
|
||||
}
|
||||
}
|
||||
var workbook bytes.Buffer
|
||||
if err := f.Write(&workbook); err != nil {
|
||||
t.Fatalf("write workbook: %v", err)
|
||||
}
|
||||
|
||||
previous := deepdoctype.NativeDocAnalyzerFactory
|
||||
t.Cleanup(func() { deepdoctype.NativeDocAnalyzerFactory = previous })
|
||||
deepdoctype.SetNativeDocAnalyzerFactory(func() (deepdoctype.DocAnalyzer, bool) {
|
||||
return &deepdocpdf.MockDocAnalyzer{Healthy: true}, true
|
||||
})
|
||||
p, err := NewXLSParser("excelize")
|
||||
if err != nil {
|
||||
t.Fatalf("NewXLSParser: %v", err)
|
||||
}
|
||||
result := p.ParseWithResult(t.Context(), "many-images.xls", workbook.Bytes())
|
||||
if result.Err != nil {
|
||||
t.Fatalf("ParseWithResult: %v", result.Err)
|
||||
}
|
||||
imageCount := 0
|
||||
for _, item := range result.JSON {
|
||||
if item["doc_type_kwd"] == "image" {
|
||||
imageCount++
|
||||
}
|
||||
}
|
||||
if imageCount != maxEmbeddedMediaItems {
|
||||
t.Fatalf("image items = %d, want %d", imageCount, maxEmbeddedMediaItems)
|
||||
}
|
||||
if warnings := strings.Join(result.Warnings, "\n"); !strings.Contains(warnings, "stopped extracting embedded images after the 256-item document limit") {
|
||||
t.Fatalf("warnings = %q, want image-count limit warning", warnings)
|
||||
}
|
||||
}
|
||||
@@ -107,10 +107,18 @@ func (p *XLSXParser) ParseWithResult(ctx context.Context, filename string, data
|
||||
// for spreadsheet processing.
|
||||
}
|
||||
|
||||
items, warnings, sheets, err := parseXLSXBytes(data)
|
||||
mediaBudget := newEmbeddedMediaBudget()
|
||||
items, warnings, sheets, err := parseXLSXBytes(data, mediaBudget)
|
||||
if err == nil {
|
||||
if ctx.Err() != nil {
|
||||
return ParseResult{Err: ctx.Err()}
|
||||
}
|
||||
warnings = append(warnings, mediaBudget.warnings()...)
|
||||
return xlsxParseResult(filename, items, warnings, sheets)
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return ParseResult{Err: ctx.Err()}
|
||||
}
|
||||
|
||||
normalized, normalizeWarnings, changed, normalizeErr := normalizeXLSXForRead(data)
|
||||
if normalizeErr != nil {
|
||||
@@ -119,15 +127,22 @@ func (p *XLSXParser) ParseWithResult(ctx context.Context, filename string, data
|
||||
if !changed {
|
||||
return ParseResult{Err: fmt.Errorf("xlsx parse: %w", err)}
|
||||
}
|
||||
items, warnings, sheets, retryErr := parseXLSXBytes(normalized)
|
||||
// The first parse attempt is discarded, so its media reservations must not
|
||||
// affect the normalized retry's output.
|
||||
mediaBudget = newEmbeddedMediaBudget()
|
||||
items, warnings, sheets, retryErr := parseXLSXBytes(normalized, mediaBudget)
|
||||
if retryErr != nil {
|
||||
return ParseResult{Err: fmt.Errorf("xlsx parse: %w; retry after normalization: %v", err, retryErr)}
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return ParseResult{Err: ctx.Err()}
|
||||
}
|
||||
warnings = append(normalizeWarnings, warnings...)
|
||||
warnings = append(warnings, mediaBudget.warnings()...)
|
||||
return xlsxParseResult(filename, items, warnings, sheets)
|
||||
}
|
||||
|
||||
func parseXLSXBytes(data []byte) ([]map[string]any, []string, int, error) {
|
||||
func parseXLSXBytes(data []byte, mediaBudget *embeddedMediaBudget) ([]map[string]any, []string, int, error) {
|
||||
f, err := excelize.OpenReader(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return nil, nil, 0, fmt.Errorf("open XLSX: %w", err)
|
||||
@@ -143,7 +158,7 @@ func parseXLSXBytes(data []byte) ([]map[string]any, []string, int, error) {
|
||||
return nil, warnings, len(sheets), err
|
||||
}
|
||||
warnings = append(warnings, sheetWarnings...)
|
||||
images, imageWarnings := extractXLSXImages(f, sheet)
|
||||
images, imageWarnings := extractXLSXImages(f, sheet, mediaBudget)
|
||||
for _, image := range images {
|
||||
row, _ := numericItemInt(image["row_start"])
|
||||
col, _ := numericItemInt(image["col_start"])
|
||||
|
||||
Reference in new issue
Block a user