diff --git a/internal/agent/runtime/stub.go b/internal/agent/runtime/stub.go new file mode 100644 index 0000000000..b6929967a0 --- /dev/null +++ b/internal/agent/runtime/stub.go @@ -0,0 +1,19 @@ +//go:build !ee + +package runtime + +import "context" + +// WithIOProgress is the community-edition stub for enterprise runtime I/O +// progress reporting. +func WithIOProgress(ctx context.Context) (context.Context, func()) { + return ctx, func() {} +} + +// TrackIO is a no-op in the community edition. +func TrackIO(context.Context, string, string, string) func() { + return func() {} +} + +// ReportIOProgress is a no-op in the community edition. +func ReportIOProgress(context.Context, string, string) {} diff --git a/internal/deepdoc/parser/pdf/parser_concurrency.go b/internal/deepdoc/parser/pdf/parser_concurrency.go index 798eabe435..d255eebbf7 100644 --- a/internal/deepdoc/parser/pdf/parser_concurrency.go +++ b/internal/deepdoc/parser/pdf/parser_concurrency.go @@ -220,9 +220,18 @@ func (p *Parser) inferDLA(ctx context.Context, doc pdf.DocAnalyzer, pageImg imag if doc == nil || !doc.Health() { return nil, nil } + finish := p.trackModelCall(ctx, "DLA") + defer finish() return doc.DLA(ctx, pageImg) } +func (p *Parser) trackModelCall(ctx context.Context, operation string) func() { + if p == nil || p.Config.OnModelCall == nil { + return func() {} + } + return p.Config.OnModelCall(ctx, operation) +} + // reportPageInferenceFailure logs one page-local inference failure (DLA, TSR or // OCR). A failure raised while the parse context is cancelled is the stop path, // not a fault: cancelling terminates every in-flight ONNX Run, and the native @@ -242,6 +251,8 @@ func (p *Parser) inferTSR(ctx context.Context, tb pdf.TableBuilder, cropped imag if tb == nil { return nil, nil } + finish := p.trackModelCall(ctx, "TSR") + defer finish() return tb.DetectCells(ctx, cropped) } @@ -251,6 +262,8 @@ func (p *Parser) inferOCRDetect(ctx context.Context, doc pdf.DocAnalyzer, pageIm if doc == nil || !doc.Health() { return nil, nil } + finish := p.trackModelCall(ctx, "OCRDetect") + defer finish() return doc.OCRDetect(ctx, pageImg) } @@ -260,6 +273,8 @@ func (p *Parser) inferOCRRecognize(ctx context.Context, doc pdf.DocAnalyzer, cro if doc == nil || !doc.Health() { return nil, nil } + finish := p.trackModelCall(ctx, "OCRRecognize") + defer finish() return doc.OCRRecognize(ctx, cropped) } @@ -289,6 +304,8 @@ func (p *Parser) inferOCRRecognizeBatch(ctx context.Context, doc pdf.DocAnalyzer if doc == nil || !doc.Health() { return nil, nil } + finish := p.trackModelCall(ctx, "OCRRecognize") + defer finish() return br.OCRRecognizeBatch(ctx, crops) } diff --git a/internal/deepdoc/parser/type/types.go b/internal/deepdoc/parser/type/types.go index 000d4f93cb..b9a7b144ff 100644 --- a/internal/deepdoc/parser/type/types.go +++ b/internal/deepdoc/parser/type/types.go @@ -286,6 +286,9 @@ type ParserConfig struct { // surface parse progress without the parser knowing about any progress // sink; nil disables the callback at zero cost. OnPageDone func(done, total int) + // OnModelCall wraps one DeepDOC model operation. It lets the caller report + // long-running inference without coupling the parser to a progress sink. + OnModelCall func(ctx context.Context, operation string) func() } // DefaultParserConfig returns a ParserConfig with sensible defaults. diff --git a/internal/ingestion/component/knowledge_compiler/pool_wiring.go b/internal/ingestion/component/knowledge_compiler/pool_wiring.go index 13edf9cf90..23cdadd45f 100644 --- a/internal/ingestion/component/knowledge_compiler/pool_wiring.go +++ b/internal/ingestion/component/knowledge_compiler/pool_wiring.go @@ -33,7 +33,7 @@ import ( // core count regardless of which stage is running. func init() { submit := func(ctx context.Context, jobs []func() error) error { - return knowledge_compile.SubmitCompilerJobs(ctx, jobs) + return knowledge_compile.SubmitCompilerLLMJobs(ctx, jobs) } structure.SetBatchSubmitter(submit) mindmap.SetBatchSubmitter(submit) diff --git a/internal/ingestion/component/media_dispatch.go b/internal/ingestion/component/media_dispatch.go index 7fc30d88d2..acf6e14a03 100644 --- a/internal/ingestion/component/media_dispatch.go +++ b/internal/ingestion/component/media_dispatch.go @@ -32,6 +32,7 @@ import ( "go.uber.org/zap" + agentruntime "ragflow/internal/agent/runtime" "ragflow/internal/common" imageparser "ragflow/internal/deepdoc/parser/image" deepdoctype "ragflow/internal/deepdoc/parser/type" @@ -139,7 +140,9 @@ func maybeDispatchImage( var analyzer deepdoctype.DocAnalyzer analyzer, ocrErr = parser.GetDocAnalyzer() if ocrErr == nil { + finish := agentruntime.TrackIO(ctx, "DeepDOC", "image OCR", "media=image") blocks, ocrErr = imageparser.Parse(ctx, img, analyzer) + finish() if ocrErr == nil { text = imageBlockText(blocks) } @@ -148,7 +151,9 @@ func maybeDispatchImage( // Selected OCR uses original bytes, so media admission need not // cover database resolution or remote inference. release() + finish := agentruntime.TrackIO(ctx, "Parser", "OCR model", "media=image backend=selected") text, ocrErr = extractSelectedImageOCR(ctx, db, getStringOr(inputs, "tenant_id", ""), ocrModelID, binary) + finish() } } } @@ -380,7 +385,9 @@ func describeImage( }} visionFlag := true chatModel := modelModule.NewChatModel(driver, &modelName, apiConfig) + finish := agentruntime.TrackIO(ctx, "Parser", "VLM", "media=image") resp, err := chatModel.ChatWithMessages(ctx, messages, &modelModule.ChatConfig{Vision: &visionFlag}, nil) + finish() if err != nil { return "", []string{fmt.Sprintf("image VLM enhancement failed: %v", err)} } @@ -448,7 +455,9 @@ func maybeDispatchAudio( defer os.Remove(tmpFile) asrModel := modelModule.NewASRModel(driver, &modelName, apiConfig) + finish := agentruntime.TrackIO(ctx, "Parser", "ASR", "media=audio") resp, err := asrModel.Transcribe(ctx, &tmpFile, nil, nil) + finish() if err != nil { return parser.ParseResult{}, true, fmt.Errorf("Parser: audio transcription: %w", err) diff --git a/internal/ingestion/component/parser_dispatch.go b/internal/ingestion/component/parser_dispatch.go index f946b87a98..9fe4cb41e2 100644 --- a/internal/ingestion/component/parser_dispatch.go +++ b/internal/ingestion/component/parser_dispatch.go @@ -28,6 +28,7 @@ import ( "fmt" "strings" + agentruntime "ragflow/internal/agent/runtime" "ragflow/internal/ingestion/component/progress" "ragflow/internal/ingestion/component/schema" "ragflow/internal/parser/parser" @@ -91,6 +92,13 @@ func dispatchParse(ctx context.Context, p parser.ParseResultProducer, fileType u configureParserFromSetups(p, fileType, setups) attachPDFPageProgress(ctx, p, window) + var finishParserIO func() + if method := strings.ToLower(strings.TrimSpace(parseMethod)); method != "" && method != "deepdoc" && method != "plain_text" { + finishParserIO = agentruntime.TrackIO(ctx, "Parser", "remote parser", "backend="+method) + } else { + finishParserIO = func() {} + } + defer finishParserIO() res := p.ParseWithResult(ctx, filename, data) if res.Err != nil { @@ -135,7 +143,13 @@ func pdfPageProgressSupported(fileType utility.FileType, setups map[string]schem // the fraction stays at zero. func attachPDFPageProgress(ctx context.Context, p parser.ParseResultProducer, window progress.Window) { pdfParser, ok := p.(*parser.PDFParser) - if !ok || window.Span <= 0 { + if !ok { + return + } + pdfParser.OnModelCall = func(callCtx context.Context, operation string) func() { + return agentruntime.TrackIO(callCtx, "DeepDOC", operation, "file_type=pdf") + } + if window.Span <= 0 { return } pdfParser.OnPageDone = func(done, total int) { diff --git a/internal/ingestion/component/pdf_vision_dispatch.go b/internal/ingestion/component/pdf_vision_dispatch.go index 7e880331d5..983875d588 100644 --- a/internal/ingestion/component/pdf_vision_dispatch.go +++ b/internal/ingestion/component/pdf_vision_dispatch.go @@ -20,6 +20,7 @@ import ( "strings" "time" + agentruntime "ragflow/internal/agent/runtime" "ragflow/internal/common" "ragflow/internal/entity" modelModule "ragflow/internal/entity/models" @@ -390,7 +391,9 @@ func monkeyOCRv2ParseRequest(ctx context.Context, apiURL, filename string, binar req.Header.Set("Content-Type", writer.FormDataContentType()) client := *http.DefaultClient client.Timeout = timeout + finish := agentruntime.TrackIO(ctx, "Parser", "remote parser", "backend=MonkeyOCRv2") resp, err := client.Do(req) + finish() if err != nil { return nil, fmt.Errorf("MonkeyOCRv2 request: %w", err) } @@ -646,14 +649,18 @@ func dispatchMinerUPDF( } cloudCtx, cloudCancel := context.WithTimeout(ctx, timeout) defer cloudCancel() + finish := agentruntime.TrackIO(cloudCtx, "Parser", "remote parser", "backend=MinerU") result, err := modelModule.ParseMinerUCloud(cloudCtx, baseURL, apiKeyRaw, filename, binary, backend, parseMethod, mineruLang) + finish() if err != nil { return parser.ParseResult{}, fmt.Errorf("parser: MinerU cloud: %w", err) } return finishMinerURemoteResult(ctx, filename, binary, result) } + finishProbe := agentruntime.TrackIO(ctx, "Parser", "remote parser", "backend=MinerU phase=probe") v1, err := modelModule.MinerUSupportsV1(ctx, baseURL, apiKeyRaw) + finishProbe() if err != nil { return parser.ParseResult{}, fmt.Errorf("parser: MinerU probe: %w", err) } @@ -667,7 +674,9 @@ func dispatchMinerUPDF( if !extraction.formulaEnable || !extraction.tableEnable { return parser.ParseResult{}, fmt.Errorf("parser: MinerU V1 does not support disabling formula or table extraction; use a /file_parse server to configure these switches") } + finish := agentruntime.TrackIO(ctx, "Parser", "remote parser", "backend=MinerU") result, err := modelModule.ParseMinerUV1(ctx, baseURL, apiKeyRaw, filename, binary, backend, parseMethod, timeout) + finish() if err != nil { return parser.ParseResult{}, fmt.Errorf("parser: MinerU V1: %w", err) } @@ -1145,7 +1154,9 @@ func mineruStreamParse(ctx context.Context, apiURL string, apiKey string, binary req.Header.Set("Authorization", "Bearer "+token) } + finish := agentruntime.TrackIO(ctx, "Parser", "remote parser", "backend=MinerU") resp, err := http.DefaultClient.Do(req) + finish() if err != nil { return nil, fmt.Errorf("send request: %w", err) } @@ -2044,7 +2055,10 @@ func defaultPDFVisionChatInvoker( ) (*modelModule.ChatResponse, error) { vision := true chatModel := modelModule.NewChatModel(driver, &modelName, apiConfig) - return chatModel.ChatWithMessages(ctx, messages, &modelModule.ChatConfig{Vision: &vision}, nil) + finish := agentruntime.TrackIO(ctx, "Parser", "VLM", "media=pdf") + resp, err := chatModel.ChatWithMessages(ctx, messages, &modelModule.ChatConfig{Vision: &vision}, nil) + finish() + return resp, err } // renderPDFVisionPrompt only renders page metadata. The full-page PDF vision diff --git a/internal/ingestion/component/tokenizer.go b/internal/ingestion/component/tokenizer.go index b14345754b..48a3ef8dbf 100644 --- a/internal/ingestion/component/tokenizer.go +++ b/internal/ingestion/component/tokenizer.go @@ -605,7 +605,9 @@ func (c *TokenizerComponent) embedChunks(ctx context.Context, tenantID, kbID, na // tokenizer.py:95 which passes name verbatim to embedding. The // empty-name guard above still uses TrimSpace, matching Python's // `.strip()==""` check at tokenizer.py:200. + finish := runtime.TrackIO(ctx, "Tokenizer", "Embedding title", "chunks=1") titleResults, err := embedder.Encode(ctx, []string{name}) + finish() if err != nil { return nil, 0, fmt.Errorf("tokenizer: encode title: %w", err) } @@ -628,7 +630,11 @@ func (c *TokenizerComponent) embedChunks(ctx context.Context, tenantID, kbID, na if end > len(texts) { end = len(texts) } + batch := start/batchSize + 1 + batchTotal := (len(texts) + batchSize - 1) / batchSize + finish := runtime.TrackIO(ctx, "Tokenizer", "Embedding chunks", fmt.Sprintf("batch=%d/%d chunks=%d", batch, batchTotal, end-start)) batchResults, err := embedder.Encode(ctx, texts[start:end]) + finish() if err != nil { return nil, 0, fmt.Errorf("tokenizer: encode: %w", err) } diff --git a/internal/ingestion/component/vision_enhancement.go b/internal/ingestion/component/vision_enhancement.go index cf72e7fb97..8b024f4619 100644 --- a/internal/ingestion/component/vision_enhancement.go +++ b/internal/ingestion/component/vision_enhancement.go @@ -35,6 +35,7 @@ import ( "go.uber.org/zap" + agentruntime "ragflow/internal/agent/runtime" "ragflow/internal/common" pdflayout "ragflow/internal/deepdoc/parser/pdf/layout" "ragflow/internal/entity" @@ -552,5 +553,8 @@ func defaultVisionChatInvoker( config.Thinking = &thinking } chatModel := modelModule.NewChatModel(driver, &modelName, apiConfig) - return chatModel.ChatWithMessages(chatCtx, messages, config, nil) + finish := agentruntime.TrackIO(chatCtx, "Parser", "VLM", "operation=vision_enhancement") + resp, err := chatModel.ChatWithMessages(chatCtx, messages, config, nil) + finish() + return resp, err } diff --git a/internal/ingestion/knowledge_compile/consumer.go b/internal/ingestion/knowledge_compile/consumer.go index e0462188e4..1014e98d9b 100644 --- a/internal/ingestion/knowledge_compile/consumer.go +++ b/internal/ingestion/knowledge_compile/consumer.go @@ -1144,18 +1144,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e Score: g.score, }) } - llmCtx, finishLLMProgress := withKnowledgeCompileLLMProgress(ctx, func(progressCtx context.Context, component, message string) { - c.reportProgress( - progressCtx, - tenant, - kb, - token, - taskTypes, - 0.82, - "llm_merge", - fmt.Sprintf("%s: %s", component, message), - ) - }) + llmCtx, finishLLMProgress := withKnowledgeCompileLLMProgress(ctx, tenant, kb) batched, err = deduper.DecideBatch(llmCtx, batched) finishLLMProgress() if err != nil { @@ -1397,7 +1386,7 @@ func rewriteMergedWikiPages(ctx context.Context, tenant string, pages []kccommon return nil }) } - return SubmitCompilerJobs(ctx, jobs) + return SubmitCompilerLLMJobs(ctx, jobs) } func floatPtr(value float64) *float64 { return &value } diff --git a/internal/ingestion/knowledge_compile/dedup.go b/internal/ingestion/knowledge_compile/dedup.go index 0b167c1071..e99c26bc11 100644 --- a/internal/ingestion/knowledge_compile/dedup.go +++ b/internal/ingestion/knowledge_compile/dedup.go @@ -79,7 +79,7 @@ func NewLLMDeduper(chat kccommon.ChatInvoker, embed kccommon.Embedder, llmID str // token-bounded sub-batches run concurrently with the rest of the pipeline // (LLM-bounded), all under one concurrency limit. decider.SetSubmitter(func(ctx context.Context, jobs []func() error) error { - return SubmitCompilerJobs(ctx, jobs) + return SubmitCompilerLLMJobs(ctx, jobs) }) return &llmDeduper{group: structure.NewGroupedDeduper(decider), decider: decider, embed: embed} } diff --git a/internal/ingestion/knowledge_compile/stub.go b/internal/ingestion/knowledge_compile/stub.go index f58d15bcaa..8a5ebcd644 100644 --- a/internal/ingestion/knowledge_compile/stub.go +++ b/internal/ingestion/knowledge_compile/stub.go @@ -6,7 +6,12 @@ import "context" func withKnowledgeCompileLLMProgress( ctx context.Context, - _ func(context.Context, string, string), + _, _ string, ) (context.Context, func()) { return ctx, func() {} } + +// SubmitCompilerLLMJobs preserves pool submission without enterprise progress reporting. +func SubmitCompilerLLMJobs(ctx context.Context, jobs []CompilerJob) error { + return SubmitCompilerJobs(ctx, jobs) +} diff --git a/internal/ingestion/task/knowledge_compiler_wiring.go b/internal/ingestion/task/knowledge_compiler_wiring.go index 365c86fe6b..9344eba81e 100644 --- a/internal/ingestion/task/knowledge_compiler_wiring.go +++ b/internal/ingestion/task/knowledge_compiler_wiring.go @@ -602,7 +602,7 @@ func newKnowledgeCompilerDepsResolver() kc.DepsResolver { }) return kc.Deps{ - Chat: &kcChatInvoker{factory: modelFactory, tenantID: tenantID, llmID: llmID, llmProgress: newKCLLMProgressReporter()}, + Chat: &kcChatInvoker{factory: modelFactory, tenantID: tenantID, llmID: llmID}, Embed: embedder, WikiPages: &kcWikiPageStore{docEngine: kcWiringDeps.Engine}, WikiMapVersions: mapStore, @@ -617,10 +617,9 @@ func newKnowledgeCompilerDepsResolver() kc.DepsResolver { // kcChatInvoker adapts a factory-created ChatModel to the knowledge compiler. type kcChatInvoker struct { - factory *service.ModelFactory - tenantID string - llmID string - llmProgress kcLLMProgressReporter + factory *service.ModelFactory + tenantID string + llmID string } func (c *kcChatInvoker) Chat(ctx context.Context, req kc.ChatRequest) (*kc.ChatResponse, error) { @@ -630,8 +629,8 @@ func (c *kcChatInvoker) Chat(ctx context.Context, req kc.ChatRequest) (*kc.ChatR } chatModel, err := c.factory.NewChatModel(permission.WithTenantID(ctx, c.tenantID), llmID) if err != nil { - if !req.DisableRetry && c.llmProgress != nil { - c.llmProgress.report(ctx, "[ERROR] LLM call failed: "+kc.CompactError(err)) + if !req.DisableRetry { + runtime.ReportProgressMessage(ctx, "Compiler", "[ERROR] LLM call failed: "+kc.CompactError(err)) } return nil, err } @@ -690,10 +689,8 @@ func (c *kcChatInvoker) Chat(ctx context.Context, req kc.ChatRequest) (*kc.ChatR attemptCtx, cancel := context.WithTimeout(ctx, kcChatAttemptTimeout) defer cancel() r, err := func() (*models.ChatResponse, error) { - if c.llmProgress == nil { - return chatModel.ChatWithMessages(attemptCtx, msgs, config, nil) - } - defer c.llmProgress.begin(ctx)() + finishIO := runtime.TrackIO(ctx, "Compiler", "LLM call", "") + defer finishIO() return chatModel.ChatWithMessages(attemptCtx, msgs, config, nil) }() if err != nil { diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.go index acfe9d98de..4c367844e6 100644 --- a/internal/ingestion/task/pipeline_executor.go +++ b/internal/ingestion/task/pipeline_executor.go @@ -25,6 +25,7 @@ import ( "strings" "time" + agentruntime "ragflow/internal/agent/runtime" "ragflow/internal/common" "ragflow/internal/dao" "ragflow/internal/engine" @@ -229,6 +230,10 @@ func (s *PipelineExecutor) Doc() *entity.Document { return &s.taskCtx.Doc } func (s *PipelineExecutor) Tenant() *entity.Tenant { return &s.taskCtx.Tenant } func (s *PipelineExecutor) Execute(ctx context.Context) (*PipelineResult, error) { + ctx = s.withProgressMessageReporter(ctx) + ctx, finishIOProgress := agentruntime.WithIOProgress(ctx) + defer finishIOProgress() + start := time.Now() if err := ctx.Err(); err != nil { return nil, err @@ -279,6 +284,36 @@ func (s *PipelineExecutor) Execute(ctx context.Context) (*PipelineResult, error) return result, nil } +type pipelineMessageReporter interface { + OnComponentMessage(ctx context.Context, taskID, documentID, component, message string) +} + +func (s *PipelineExecutor) withProgressMessageReporter(ctx context.Context) context.Context { + if s.taskCtx == nil { + return ctx + } + sink, hasSink := s.progressSink.(pipelineMessageReporter) + taskID := "" + if s.taskCtx.IngestionTask != nil { + taskID = s.taskCtx.IngestionTask.ID + } + documentID := s.taskCtx.Doc.ID + return agentruntime.WithProgressMessageCallback(ctx, func(component, message string) { + common.Info("component progress detail", + zap.String("component", component), + zap.String("task_id", taskID), + zap.String("document_id", documentID), + zap.String("message", message), + ) + if !hasSink { + return + } + sinkCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) + defer cancel() + sink.OnComponentMessage(sinkCtx, taskID, documentID, component, message) + }) +} + // collectDebugOutput builds a PipelineResult for a debug (dry-run) run. // It surfaces the pipeline's chunks so a debug endpoint can render them, but // performs no DB/index writes — the embedding vectors already computed by the @@ -360,11 +395,20 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput map } indexChunks := append(chunks, parentChunks...) indexdoc.StampDatasetPagerank(indexChunks, s.taskCtx.KB.Pagerank) - if err := s.indexWriter.Write(ctx, indexChunks); err != nil { + indexWriteStart := time.Now() + finishIndexWrite := agentruntime.TrackIO(ctx, "Indexer", "Write chunks", fmt.Sprintf("count=%d", len(indexChunks))) + indexWriteErr := s.indexWriter.Write(ctx, indexChunks) + finishIndexWrite() + indexMessage := fmt.Sprintf("Write chunks | count=%d elapsed=%s", len(indexChunks), time.Since(indexWriteStart).Round(time.Millisecond)) + if indexWriteErr != nil { + indexMessage += " error=" + kccommon.CompactError(indexWriteErr) + } + agentruntime.ReportIOProgress(ctx, "Indexer", indexMessage) + if indexWriteErr != nil { if cleanupErr := s.compensateFailedIndexWrite(ctx, indexChunks); cleanupErr != nil { - return nil, fmt.Errorf("write chunks: %w; compensate partial index write: %v", err, cleanupErr) + return nil, fmt.Errorf("write chunks: %w; compensate partial index write: %v", indexWriteErr, cleanupErr) } - return nil, fmt.Errorf("write chunks: %w", err) + return nil, fmt.Errorf("write chunks: %w", indexWriteErr) } if err := s.reconcileDocumentCompiledProducts(ctx, oldCompiledProductIDs, chunks); err != nil { return nil, err @@ -461,12 +505,20 @@ func (s *PipelineExecutor) compensateFailedIndexWrite(ctx context.Context, chunk } cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) defer cancel() + started := time.Now() + finish := agentruntime.TrackIO(ctx, "Indexer", "Delete partial write", fmt.Sprintf("count=%d", len(ids))) _, err := s.deleteChunksFunc( cleanupCtx, map[string]any{"id": ids, "kb_id": s.taskCtx.Doc.KbID}, s.indexWriter.baseName, s.taskCtx.Doc.KbID, ) + finish() + message := fmt.Sprintf("Delete partial write | count=%d elapsed=%s", len(ids), time.Since(started).Round(time.Millisecond)) + if err != nil { + message += " error=" + kccommon.CompactError(err) + } + agentruntime.ReportIOProgress(ctx, "Indexer", message) return err } @@ -575,6 +627,8 @@ func (s *PipelineExecutor) loadDocumentCompiledState(ctx context.Context) ([]str oldIDs := make([]string, 0) oldProducts := make([]map[string]any, 0) for offset := 0; ; offset += pageSize { + started := time.Now() + finish := agentruntime.TrackIO(ctx, "Indexer", "Search compiled products", fmt.Sprintf("offset=%d limit=%d", offset, pageSize)) result, err := docEngine.Search(ctx, &enginetypes.SearchRequest{ IndexNames: []string{indexName}, KbIDs: []string{s.taskCtx.Doc.KbID}, @@ -583,6 +637,16 @@ func (s *PipelineExecutor) loadDocumentCompiledState(ctx context.Context) ([]str SelectFields: []string{"id", "compile_kwd", "compilation_template_kind_kwd", "type_kwd"}, Filter: map[string]any{"doc_id": []string{s.taskCtx.Doc.ID}}, }) + finish() + hits := 0 + if result != nil { + hits = len(result.Chunks) + } + message := fmt.Sprintf("Search compiled products | offset=%d limit=%d hits=%d elapsed=%s", offset, pageSize, hits, time.Since(started).Round(time.Millisecond)) + if err != nil { + message += " error=" + kccommon.CompactError(err) + } + agentruntime.ReportIOProgress(ctx, "Indexer", message) if err != nil { return nil, nil, nil, fmt.Errorf("load document compiler products: %w", err) } @@ -634,11 +698,20 @@ func (s *PipelineExecutor) reconcileDocumentCompiledProducts(ctx context.Context indexName := fmt.Sprintf("ragflow_%s", s.taskCtx.Tenant.ID) for start := 0; start < len(staleIDs); start += pageSize { end := min(start+pageSize, len(staleIDs)) - if _, err := docEngine.DeleteChunks(ctx, map[string]any{ + started := time.Now() + finish := agentruntime.TrackIO(ctx, "Indexer", "Delete stale products", fmt.Sprintf("batch=%d/%d count=%d", start/pageSize+1, (len(staleIDs)+pageSize-1)/pageSize, end-start)) + _, deleteErr := docEngine.DeleteChunks(ctx, map[string]any{ "id": staleIDs[start:end], "kb_id": s.taskCtx.Doc.KbID, - }, indexName, s.taskCtx.Doc.KbID); err != nil { - return fmt.Errorf("delete stale document compiler products: %w", err) + }, indexName, s.taskCtx.Doc.KbID) + finish() + message := fmt.Sprintf("Delete stale products | count=%d elapsed=%s", end-start, time.Since(started).Round(time.Millisecond)) + if deleteErr != nil { + message += " error=" + kccommon.CompactError(deleteErr) + } + agentruntime.ReportIOProgress(ctx, "Indexer", message) + if deleteErr != nil { + return fmt.Errorf("delete stale document compiler products: %w", deleteErr) } } return nil diff --git a/internal/ingestion/task/stub.go b/internal/ingestion/task/stub.go deleted file mode 100644 index be4b4d5f31..0000000000 --- a/internal/ingestion/task/stub.go +++ /dev/null @@ -1,22 +0,0 @@ -//go:build !ee - -package task - -import "context" - -type kcLLMProgressReporter interface { - begin(ctx context.Context) func() - report(ctx context.Context, message string) -} - -type noopKCLLMProgressReporter struct{} - -func newKCLLMProgressReporter() kcLLMProgressReporter { - return noopKCLLMProgressReporter{} -} - -func (noopKCLLMProgressReporter) begin(context.Context) func() { - return func() {} -} - -func (noopKCLLMProgressReporter) report(context.Context, string) {} diff --git a/internal/parser/parser/pdf_parser_common.go b/internal/parser/parser/pdf_parser_common.go index 52fc835017..22a06bccf7 100644 --- a/internal/parser/parser/pdf_parser_common.go +++ b/internal/parser/parser/pdf_parser_common.go @@ -76,7 +76,10 @@ type PDFParser struct { // caller observes per-page parse progress (done/total). Only the deepdoc // backend invokes it; remote engines poll opaque HTTP jobs and never call // it, so the fraction simply stays where it was. - OnPageDone func(done, total int) + OnPageDone func(done, total int) + // OnModelCall reports the start and end of an individual DeepDOC model call. + // It is runtime-only and is wired by the ingestion parser component. + OnModelCall func(ctx context.Context, operation string) func() MinerUAPIServer string MinerUAPIKey string MinerUBackend string @@ -399,6 +402,7 @@ func (p *PDFParser) parseWithResult(ctx context.Context, filename string, data [ cfg.RemoveTOC = p.RemoveTOC cfg.RemoveHeaderFooter = p.RemoveHeaderFooter cfg.OnPageDone = p.OnPageDone + cfg.OnModelCall = p.OnModelCall res := parsePDFWithDeepDocOptions(ctx, filename, data, pdfPostProcessOptions{ zoom: cfg.Zoom, enableMultiColumn: p.EnableMultiColumn,