ragflow/internal/parser/parser/pdf_parser_remote_common.go

package parser

import (
	"context"
	"strings"
)

func pdfFileMeta(filename string, pageCount int) map[string]any {
	if pageCount < 0 {
		pageCount = 0
	}
	return map[string]any{
		"name":       filename,
		"page_count": pageCount,
		"outline":    []map[string]any{},
	}
}

func pdfItemsToResult(filename string, items []map[string]any, pageCount int) ParseResult {
	if len(items) == 0 {
		items = []map[string]any{{"text": "", "doc_type_kwd": "text"}}
	}
	pageCount = normalizePDFPageCount(pageCount, items)
	return ParseResult{
		File: pdfFileMeta(filename, pageCount),
		JSON: MapsToItems(items),
	}
}

func normalizePDFPageCount(fallback int, items []map[string]any) int {
	if inferred := inferPDFPageCountFromItems(items); inferred > fallback {
		return inferred
	}
	return fallback
}

func inferPDFPageCountFromItems(items []map[string]any) int {
	pages := map[int]struct{}{}
	for _, item := range items {
		for page := range collectPDFPageNumbers(item) {
			pages[page] = struct{}{}
		}
	}
	return len(pages)
}

func collectPDFPageNumbers(raw any) map[int]struct{} {
	pages := map[int]struct{}{}
	var walk func(any)
	walk = func(node any) {
		switch v := node.(type) {
		case map[string]any:
			for _, key := range []string{"page number", "page_number", "page_num", "page_no", "page_index", "page_idx", "page"} {
				if page := int(numberValue(v[key])); page > 0 {
					pages[page] = struct{}{}
				}
			}
			for _, key := range []string{"_pdf_positions", "positions"} {
				if positions, ok := v[key].([][]any); ok {
					for _, pos := range positions {
						if len(pos) > 0 {
							if page := int(numberValue(pos[0])); page > 0 {
								pages[page] = struct{}{}
							}
						}
					}
				}
			}
			for _, child := range v {
				walk(child)
			}
		case []any:
			for _, child := range v {
				walk(child)
			}
		case []map[string]any:
			for _, child := range v {
				walk(child)
			}
		}
	}
	walk(raw)
	return pages
}

// pdfTableDocType labels a table region "table" only when its text is <table>
// markup, because consumers of that label read the text as HTML rows. Free
// text recognized inside a table region is labelled "text".
func pdfTableDocType(text string) string {
	if isTableHTML(text) {
		return "table"
	}
	return "text"
}

// rowsHaveText reports whether any cell holds non-whitespace text. Blank rows
// must not replace the content of a table region with an empty <table>.
func rowsHaveText(rows [][]string) bool {
	for _, row := range rows {
		for _, cell := range row {
			if strings.TrimSpace(cell) != "" {
				return true
			}
		}
	}
	return false
}

func pdfMarkdownToResult(ctx context.Context, filename, markdown string, pageCount int) ParseResult {
	return (&PDFParser{}).normalizePDFMarkdown(ctx, filename, markdown, pdfFileMeta(filename, pageCount))
}