feat: report ingestion runtime I/O progress (#22194)

internal/agent/runtime/stub.go

@@ -0,0 +1,19 @@
+//go:build !ee
+
+package runtime
+
+import "context"
+
+// WithIOProgress is the community-edition stub for enterprise runtime I/O
+// progress reporting.
+func WithIOProgress(ctx context.Context) (context.Context, func()) {
+	return ctx, func() {}
+}
+
+// TrackIO is a no-op in the community edition.
+func TrackIO(context.Context, string, string, string) func() {
+	return func() {}
+}
+
+// ReportIOProgress is a no-op in the community edition.
+func ReportIOProgress(context.Context, string, string) {}

internal/deepdoc/parser/pdf/parser_concurrency.go

@@ -220,9 +220,18 @@ func (p *Parser) inferDLA(ctx context.Context, doc pdf.DocAnalyzer, pageImg imag
 	if doc == nil || !doc.Health() {
 		return nil, nil
 	}
+	finish := p.trackModelCall(ctx, "DLA")
+	defer finish()
 	return doc.DLA(ctx, pageImg)
 }
 
+func (p *Parser) trackModelCall(ctx context.Context, operation string) func() {
+	if p == nil || p.Config.OnModelCall == nil {
+		return func() {}
+	}
+	return p.Config.OnModelCall(ctx, operation)
+}
+
 // reportPageInferenceFailure logs one page-local inference failure (DLA, TSR or
 // OCR). A failure raised while the parse context is cancelled is the stop path,
 // not a fault: cancelling terminates every in-flight ONNX Run, and the native
@@ -242,6 +251,8 @@ func (p *Parser) inferTSR(ctx context.Context, tb pdf.TableBuilder, cropped imag
 	if tb == nil {
 		return nil, nil
 	}
+	finish := p.trackModelCall(ctx, "TSR")
+	defer finish()
 	return tb.DetectCells(ctx, cropped)
 }
 
@@ -251,6 +262,8 @@ func (p *Parser) inferOCRDetect(ctx context.Context, doc pdf.DocAnalyzer, pageIm
 	if doc == nil || !doc.Health() {
 		return nil, nil
 	}
+	finish := p.trackModelCall(ctx, "OCRDetect")
+	defer finish()
 	return doc.OCRDetect(ctx, pageImg)
 }
 
@@ -260,6 +273,8 @@ func (p *Parser) inferOCRRecognize(ctx context.Context, doc pdf.DocAnalyzer, cro
 	if doc == nil || !doc.Health() {
 		return nil, nil
 	}
+	finish := p.trackModelCall(ctx, "OCRRecognize")
+	defer finish()
 	return doc.OCRRecognize(ctx, cropped)
 }
 
@@ -289,6 +304,8 @@ func (p *Parser) inferOCRRecognizeBatch(ctx context.Context, doc pdf.DocAnalyzer
 	if doc == nil || !doc.Health() {
 		return nil, nil
 	}
+	finish := p.trackModelCall(ctx, "OCRRecognize")
+	defer finish()
 	return br.OCRRecognizeBatch(ctx, crops)
 }
 

internal/deepdoc/parser/type/types.go

@@ -286,6 +286,9 @@ type ParserConfig struct {
 	// surface parse progress without the parser knowing about any progress
 	// sink; nil disables the callback at zero cost.
 	OnPageDone func(done, total int)
+	// OnModelCall wraps one DeepDOC model operation. It lets the caller report
+	// long-running inference without coupling the parser to a progress sink.
+	OnModelCall func(ctx context.Context, operation string) func()
 }
 
 // DefaultParserConfig returns a ParserConfig with sensible defaults.

internal/ingestion/component/knowledge_compiler/pool_wiring.go

@@ -33,7 +33,7 @@ import (
 // core count regardless of which stage is running.
 func init() {
 	submit := func(ctx context.Context, jobs []func() error) error {
-		return knowledge_compile.SubmitCompilerJobs(ctx, jobs)
+		return knowledge_compile.SubmitCompilerLLMJobs(ctx, jobs)
 	}
 	structure.SetBatchSubmitter(submit)
 	mindmap.SetBatchSubmitter(submit)

internal/ingestion/component/media_dispatch.go

@@ -32,6 +32,7 @@ import (
 
 	"go.uber.org/zap"
 
+	agentruntime "ragflow/internal/agent/runtime"
 	"ragflow/internal/common"
 	imageparser "ragflow/internal/deepdoc/parser/image"
 	deepdoctype "ragflow/internal/deepdoc/parser/type"
@@ -139,7 +140,9 @@ func maybeDispatchImage(
 				var analyzer deepdoctype.DocAnalyzer
 				analyzer, ocrErr = parser.GetDocAnalyzer()
 				if ocrErr == nil {
+					finish := agentruntime.TrackIO(ctx, "DeepDOC", "image OCR", "media=image")
 					blocks, ocrErr = imageparser.Parse(ctx, img, analyzer)
+					finish()
 					if ocrErr == nil {
 						text = imageBlockText(blocks)
 					}
@@ -148,7 +151,9 @@ func maybeDispatchImage(
 				// Selected OCR uses original bytes, so media admission need not
 				// cover database resolution or remote inference.
 				release()
+				finish := agentruntime.TrackIO(ctx, "Parser", "OCR model", "media=image backend=selected")
 				text, ocrErr = extractSelectedImageOCR(ctx, db, getStringOr(inputs, "tenant_id", ""), ocrModelID, binary)
+				finish()
 			}
 		}
 	}
@@ -380,7 +385,9 @@ func describeImage(
 	}}
 	visionFlag := true
 	chatModel := modelModule.NewChatModel(driver, &modelName, apiConfig)
+	finish := agentruntime.TrackIO(ctx, "Parser", "VLM", "media=image")
 	resp, err := chatModel.ChatWithMessages(ctx, messages, &modelModule.ChatConfig{Vision: &visionFlag}, nil)
+	finish()
 	if err != nil {
 		return "", []string{fmt.Sprintf("image VLM enhancement failed: %v", err)}
 	}
@@ -448,7 +455,9 @@ func maybeDispatchAudio(
 	defer os.Remove(tmpFile)
 
 	asrModel := modelModule.NewASRModel(driver, &modelName, apiConfig)
+	finish := agentruntime.TrackIO(ctx, "Parser", "ASR", "media=audio")
 	resp, err := asrModel.Transcribe(ctx, &tmpFile, nil, nil)
+	finish()
 	if err != nil {
 		return parser.ParseResult{}, true,
 			fmt.Errorf("Parser: audio transcription: %w", err)

internal/ingestion/component/parser_dispatch.go

@@ -28,6 +28,7 @@ import (
 	"fmt"
 	"strings"
 
+	agentruntime "ragflow/internal/agent/runtime"
 	"ragflow/internal/ingestion/component/progress"
 	"ragflow/internal/ingestion/component/schema"
 	"ragflow/internal/parser/parser"
@@ -91,6 +92,13 @@ func dispatchParse(ctx context.Context, p parser.ParseResultProducer, fileType u
 
 	configureParserFromSetups(p, fileType, setups)
 	attachPDFPageProgress(ctx, p, window)
+	var finishParserIO func()
+	if method := strings.ToLower(strings.TrimSpace(parseMethod)); method != "" && method != "deepdoc" && method != "plain_text" {
+		finishParserIO = agentruntime.TrackIO(ctx, "Parser", "remote parser", "backend="+method)
+	} else {
+		finishParserIO = func() {}
+	}
+	defer finishParserIO()
 
 	res := p.ParseWithResult(ctx, filename, data)
 	if res.Err != nil {
@@ -135,7 +143,13 @@ func pdfPageProgressSupported(fileType utility.FileType, setups map[string]schem
 // the fraction stays at zero.
 func attachPDFPageProgress(ctx context.Context, p parser.ParseResultProducer, window progress.Window) {
 	pdfParser, ok := p.(*parser.PDFParser)
-	if !ok || window.Span <= 0 {
+	if !ok {
+		return
+	}
+	pdfParser.OnModelCall = func(callCtx context.Context, operation string) func() {
+		return agentruntime.TrackIO(callCtx, "DeepDOC", operation, "file_type=pdf")
+	}
+	if window.Span <= 0 {
 		return
 	}
 	pdfParser.OnPageDone = func(done, total int) {

internal/ingestion/component/pdf_vision_dispatch.go

@@ -20,6 +20,7 @@ import (
 	"strings"
 	"time"
 
+	agentruntime "ragflow/internal/agent/runtime"
 	"ragflow/internal/common"
 	"ragflow/internal/entity"
 	modelModule "ragflow/internal/entity/models"
@@ -390,7 +391,9 @@ func monkeyOCRv2ParseRequest(ctx context.Context, apiURL, filename string, binar
 	req.Header.Set("Content-Type", writer.FormDataContentType())
 	client := *http.DefaultClient
 	client.Timeout = timeout
+	finish := agentruntime.TrackIO(ctx, "Parser", "remote parser", "backend=MonkeyOCRv2")
 	resp, err := client.Do(req)
+	finish()
 	if err != nil {
 		return nil, fmt.Errorf("MonkeyOCRv2 request: %w", err)
 	}
@@ -646,14 +649,18 @@ func dispatchMinerUPDF(
 		}
 		cloudCtx, cloudCancel := context.WithTimeout(ctx, timeout)
 		defer cloudCancel()
+		finish := agentruntime.TrackIO(cloudCtx, "Parser", "remote parser", "backend=MinerU")
 		result, err := modelModule.ParseMinerUCloud(cloudCtx, baseURL, apiKeyRaw, filename, binary, backend, parseMethod, mineruLang)
+		finish()
 		if err != nil {
 			return parser.ParseResult{}, fmt.Errorf("parser: MinerU cloud: %w", err)
 		}
 		return finishMinerURemoteResult(ctx, filename, binary, result)
 	}
 
+	finishProbe := agentruntime.TrackIO(ctx, "Parser", "remote parser", "backend=MinerU phase=probe")
 	v1, err := modelModule.MinerUSupportsV1(ctx, baseURL, apiKeyRaw)
+	finishProbe()
 	if err != nil {
 		return parser.ParseResult{}, fmt.Errorf("parser: MinerU probe: %w", err)
 	}
@@ -667,7 +674,9 @@ func dispatchMinerUPDF(
 		if !extraction.formulaEnable || !extraction.tableEnable {
 			return parser.ParseResult{}, fmt.Errorf("parser: MinerU V1 does not support disabling formula or table extraction; use a /file_parse server to configure these switches")
 		}
+		finish := agentruntime.TrackIO(ctx, "Parser", "remote parser", "backend=MinerU")
 		result, err := modelModule.ParseMinerUV1(ctx, baseURL, apiKeyRaw, filename, binary, backend, parseMethod, timeout)
+		finish()
 		if err != nil {
 			return parser.ParseResult{}, fmt.Errorf("parser: MinerU V1: %w", err)
 		}
@@ -1145,7 +1154,9 @@ func mineruStreamParse(ctx context.Context, apiURL string, apiKey string, binary
 		req.Header.Set("Authorization", "Bearer "+token)
 	}
 
+	finish := agentruntime.TrackIO(ctx, "Parser", "remote parser", "backend=MinerU")
 	resp, err := http.DefaultClient.Do(req)
+	finish()
 	if err != nil {
 		return nil, fmt.Errorf("send request: %w", err)
 	}
@@ -2044,7 +2055,10 @@ func defaultPDFVisionChatInvoker(
 ) (*modelModule.ChatResponse, error) {
 	vision := true
 	chatModel := modelModule.NewChatModel(driver, &modelName, apiConfig)
-	return chatModel.ChatWithMessages(ctx, messages, &modelModule.ChatConfig{Vision: &vision}, nil)
+	finish := agentruntime.TrackIO(ctx, "Parser", "VLM", "media=pdf")
+	resp, err := chatModel.ChatWithMessages(ctx, messages, &modelModule.ChatConfig{Vision: &vision}, nil)
+	finish()
+	return resp, err
 }
 
 // renderPDFVisionPrompt only renders page metadata. The full-page PDF vision

internal/ingestion/component/tokenizer.go

@@ -605,7 +605,9 @@ func (c *TokenizerComponent) embedChunks(ctx context.Context, tenantID, kbID, na
 		// tokenizer.py:95 which passes name verbatim to embedding. The
 		// empty-name guard above still uses TrimSpace, matching Python's
 		// `.strip()==""` check at tokenizer.py:200.
+		finish := runtime.TrackIO(ctx, "Tokenizer", "Embedding title", "chunks=1")
 		titleResults, err := embedder.Encode(ctx, []string{name})
+		finish()
 		if err != nil {
 			return nil, 0, fmt.Errorf("tokenizer: encode title: %w", err)
 		}
@@ -628,7 +630,11 @@ func (c *TokenizerComponent) embedChunks(ctx context.Context, tenantID, kbID, na
 		if end > len(texts) {
 			end = len(texts)
 		}
+		batch := start/batchSize + 1
+		batchTotal := (len(texts) + batchSize - 1) / batchSize
+		finish := runtime.TrackIO(ctx, "Tokenizer", "Embedding chunks", fmt.Sprintf("batch=%d/%d chunks=%d", batch, batchTotal, end-start))
 		batchResults, err := embedder.Encode(ctx, texts[start:end])
+		finish()
 		if err != nil {
 			return nil, 0, fmt.Errorf("tokenizer: encode: %w", err)
 		}

internal/ingestion/component/vision_enhancement.go

@@ -35,6 +35,7 @@ import (
 
 	"go.uber.org/zap"
 
+	agentruntime "ragflow/internal/agent/runtime"
 	"ragflow/internal/common"
 	pdflayout "ragflow/internal/deepdoc/parser/pdf/layout"
 	"ragflow/internal/entity"
@@ -552,5 +553,8 @@ func defaultVisionChatInvoker(
 		config.Thinking = &thinking
 	}
 	chatModel := modelModule.NewChatModel(driver, &modelName, apiConfig)
-	return chatModel.ChatWithMessages(chatCtx, messages, config, nil)
+	finish := agentruntime.TrackIO(chatCtx, "Parser", "VLM", "operation=vision_enhancement")
+	resp, err := chatModel.ChatWithMessages(chatCtx, messages, config, nil)
+	finish()
+	return resp, err
 }

internal/ingestion/knowledge_compile/consumer.go

@@ -1144,18 +1144,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e
 				Score:      g.score,
 			})
 		}
-		llmCtx, finishLLMProgress := withKnowledgeCompileLLMProgress(ctx, func(progressCtx context.Context, component, message string) {
-			c.reportProgress(
-				progressCtx,
-				tenant,
-				kb,
-				token,
-				taskTypes,
-				0.82,
-				"llm_merge",
-				fmt.Sprintf("%s: %s", component, message),
-			)
-		})
+		llmCtx, finishLLMProgress := withKnowledgeCompileLLMProgress(ctx, tenant, kb)
 		batched, err = deduper.DecideBatch(llmCtx, batched)
 		finishLLMProgress()
 		if err != nil {
@@ -1397,7 +1386,7 @@ func rewriteMergedWikiPages(ctx context.Context, tenant string, pages []kccommon
 			return nil
 		})
 	}
-	return SubmitCompilerJobs(ctx, jobs)
+	return SubmitCompilerLLMJobs(ctx, jobs)
 }
 
 func floatPtr(value float64) *float64 { return &value }

internal/ingestion/knowledge_compile/dedup.go

@@ -79,7 +79,7 @@ func NewLLMDeduper(chat kccommon.ChatInvoker, embed kccommon.Embedder, llmID str
 	// token-bounded sub-batches run concurrently with the rest of the pipeline
 	// (LLM-bounded), all under one concurrency limit.
 	decider.SetSubmitter(func(ctx context.Context, jobs []func() error) error {
-		return SubmitCompilerJobs(ctx, jobs)
+		return SubmitCompilerLLMJobs(ctx, jobs)
 	})
 	return &llmDeduper{group: structure.NewGroupedDeduper(decider), decider: decider, embed: embed}
 }

internal/ingestion/knowledge_compile/stub.go

@@ -6,7 +6,12 @@ import "context"
 
 func withKnowledgeCompileLLMProgress(
 	ctx context.Context,
-	_ func(context.Context, string, string),
+	_, _ string,
 ) (context.Context, func()) {
 	return ctx, func() {}
 }
+
+// SubmitCompilerLLMJobs preserves pool submission without enterprise progress reporting.
+func SubmitCompilerLLMJobs(ctx context.Context, jobs []CompilerJob) error {
+	return SubmitCompilerJobs(ctx, jobs)
+}

internal/ingestion/task/knowledge_compiler_wiring.go

@@ -602,7 +602,7 @@ func newKnowledgeCompilerDepsResolver() kc.DepsResolver {
 		})
 
 		return kc.Deps{
-			Chat:            &kcChatInvoker{factory: modelFactory, tenantID: tenantID, llmID: llmID, llmProgress: newKCLLMProgressReporter()},
+			Chat:            &kcChatInvoker{factory: modelFactory, tenantID: tenantID, llmID: llmID},
 			Embed:           embedder,
 			WikiPages:       &kcWikiPageStore{docEngine: kcWiringDeps.Engine},
 			WikiMapVersions: mapStore,
@@ -617,10 +617,9 @@ func newKnowledgeCompilerDepsResolver() kc.DepsResolver {
 
 // kcChatInvoker adapts a factory-created ChatModel to the knowledge compiler.
 type kcChatInvoker struct {
-	factory     *service.ModelFactory
-	tenantID    string
-	llmID       string
-	llmProgress kcLLMProgressReporter
+	factory  *service.ModelFactory
+	tenantID string
+	llmID    string
 }
 
 func (c *kcChatInvoker) Chat(ctx context.Context, req kc.ChatRequest) (*kc.ChatResponse, error) {
@@ -630,8 +629,8 @@ func (c *kcChatInvoker) Chat(ctx context.Context, req kc.ChatRequest) (*kc.ChatR
 	}
 	chatModel, err := c.factory.NewChatModel(permission.WithTenantID(ctx, c.tenantID), llmID)
 	if err != nil {
-		if !req.DisableRetry && c.llmProgress != nil {
-			c.llmProgress.report(ctx, "[ERROR] LLM call failed: "+kc.CompactError(err))
+		if !req.DisableRetry {
+			runtime.ReportProgressMessage(ctx, "Compiler", "[ERROR] LLM call failed: "+kc.CompactError(err))
 		}
 		return nil, err
 	}
@@ -690,10 +689,8 @@ func (c *kcChatInvoker) Chat(ctx context.Context, req kc.ChatRequest) (*kc.ChatR
 		attemptCtx, cancel := context.WithTimeout(ctx, kcChatAttemptTimeout)
 		defer cancel()
 		r, err := func() (*models.ChatResponse, error) {
-			if c.llmProgress == nil {
-				return chatModel.ChatWithMessages(attemptCtx, msgs, config, nil)
-			}
-			defer c.llmProgress.begin(ctx)()
+			finishIO := runtime.TrackIO(ctx, "Compiler", "LLM call", "")
+			defer finishIO()
 			return chatModel.ChatWithMessages(attemptCtx, msgs, config, nil)
 		}()
 		if err != nil {

internal/ingestion/task/pipeline_executor.go

@@ -25,6 +25,7 @@ import (
 	"strings"
 	"time"
 
+	agentruntime "ragflow/internal/agent/runtime"
 	"ragflow/internal/common"
 	"ragflow/internal/dao"
 	"ragflow/internal/engine"
@@ -229,6 +230,10 @@ func (s *PipelineExecutor) Doc() *entity.Document     { return &s.taskCtx.Doc }
 func (s *PipelineExecutor) Tenant() *entity.Tenant    { return &s.taskCtx.Tenant }
 
 func (s *PipelineExecutor) Execute(ctx context.Context) (*PipelineResult, error) {
+	ctx = s.withProgressMessageReporter(ctx)
+	ctx, finishIOProgress := agentruntime.WithIOProgress(ctx)
+	defer finishIOProgress()
+
 	start := time.Now()
 	if err := ctx.Err(); err != nil {
 		return nil, err
@@ -279,6 +284,36 @@ func (s *PipelineExecutor) Execute(ctx context.Context) (*PipelineResult, error)
 	return result, nil
 }
 
+type pipelineMessageReporter interface {
+	OnComponentMessage(ctx context.Context, taskID, documentID, component, message string)
+}
+
+func (s *PipelineExecutor) withProgressMessageReporter(ctx context.Context) context.Context {
+	if s.taskCtx == nil {
+		return ctx
+	}
+	sink, hasSink := s.progressSink.(pipelineMessageReporter)
+	taskID := ""
+	if s.taskCtx.IngestionTask != nil {
+		taskID = s.taskCtx.IngestionTask.ID
+	}
+	documentID := s.taskCtx.Doc.ID
+	return agentruntime.WithProgressMessageCallback(ctx, func(component, message string) {
+		common.Info("component progress detail",
+			zap.String("component", component),
+			zap.String("task_id", taskID),
+			zap.String("document_id", documentID),
+			zap.String("message", message),
+		)
+		if !hasSink {
+			return
+		}
+		sinkCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
+		defer cancel()
+		sink.OnComponentMessage(sinkCtx, taskID, documentID, component, message)
+	})
+}
+
 // collectDebugOutput builds a PipelineResult for a debug (dry-run) run.
 // It surfaces the pipeline's chunks so a debug endpoint can render them, but
 // performs no DB/index writes — the embedding vectors already computed by the
@@ -360,11 +395,20 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput map
 	}
 	indexChunks := append(chunks, parentChunks...)
 	indexdoc.StampDatasetPagerank(indexChunks, s.taskCtx.KB.Pagerank)
-	if err := s.indexWriter.Write(ctx, indexChunks); err != nil {
+	indexWriteStart := time.Now()
+	finishIndexWrite := agentruntime.TrackIO(ctx, "Indexer", "Write chunks", fmt.Sprintf("count=%d", len(indexChunks)))
+	indexWriteErr := s.indexWriter.Write(ctx, indexChunks)
+	finishIndexWrite()
+	indexMessage := fmt.Sprintf("Write chunks | count=%d elapsed=%s", len(indexChunks), time.Since(indexWriteStart).Round(time.Millisecond))
+	if indexWriteErr != nil {
+		indexMessage += " error=" + kccommon.CompactError(indexWriteErr)
+	}
+	agentruntime.ReportIOProgress(ctx, "Indexer", indexMessage)
+	if indexWriteErr != nil {
 		if cleanupErr := s.compensateFailedIndexWrite(ctx, indexChunks); cleanupErr != nil {
-			return nil, fmt.Errorf("write chunks: %w; compensate partial index write: %v", err, cleanupErr)
+			return nil, fmt.Errorf("write chunks: %w; compensate partial index write: %v", indexWriteErr, cleanupErr)
 		}
-		return nil, fmt.Errorf("write chunks: %w", err)
+		return nil, fmt.Errorf("write chunks: %w", indexWriteErr)
 	}
 	if err := s.reconcileDocumentCompiledProducts(ctx, oldCompiledProductIDs, chunks); err != nil {
 		return nil, err
@@ -461,12 +505,20 @@ func (s *PipelineExecutor) compensateFailedIndexWrite(ctx context.Context, chunk
 	}
 	cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
 	defer cancel()
+	started := time.Now()
+	finish := agentruntime.TrackIO(ctx, "Indexer", "Delete partial write", fmt.Sprintf("count=%d", len(ids)))
 	_, err := s.deleteChunksFunc(
 		cleanupCtx,
 		map[string]any{"id": ids, "kb_id": s.taskCtx.Doc.KbID},
 		s.indexWriter.baseName,
 		s.taskCtx.Doc.KbID,
 	)
+	finish()
+	message := fmt.Sprintf("Delete partial write | count=%d elapsed=%s", len(ids), time.Since(started).Round(time.Millisecond))
+	if err != nil {
+		message += " error=" + kccommon.CompactError(err)
+	}
+	agentruntime.ReportIOProgress(ctx, "Indexer", message)
 	return err
 }
 
@@ -575,6 +627,8 @@ func (s *PipelineExecutor) loadDocumentCompiledState(ctx context.Context) ([]str
 	oldIDs := make([]string, 0)
 	oldProducts := make([]map[string]any, 0)
 	for offset := 0; ; offset += pageSize {
+		started := time.Now()
+		finish := agentruntime.TrackIO(ctx, "Indexer", "Search compiled products", fmt.Sprintf("offset=%d limit=%d", offset, pageSize))
 		result, err := docEngine.Search(ctx, &enginetypes.SearchRequest{
 			IndexNames:   []string{indexName},
 			KbIDs:        []string{s.taskCtx.Doc.KbID},
@@ -583,6 +637,16 @@ func (s *PipelineExecutor) loadDocumentCompiledState(ctx context.Context) ([]str
 			SelectFields: []string{"id", "compile_kwd", "compilation_template_kind_kwd", "type_kwd"},
 			Filter:       map[string]any{"doc_id": []string{s.taskCtx.Doc.ID}},
 		})
+		finish()
+		hits := 0
+		if result != nil {
+			hits = len(result.Chunks)
+		}
+		message := fmt.Sprintf("Search compiled products | offset=%d limit=%d hits=%d elapsed=%s", offset, pageSize, hits, time.Since(started).Round(time.Millisecond))
+		if err != nil {
+			message += " error=" + kccommon.CompactError(err)
+		}
+		agentruntime.ReportIOProgress(ctx, "Indexer", message)
 		if err != nil {
 			return nil, nil, nil, fmt.Errorf("load document compiler products: %w", err)
 		}
@@ -634,11 +698,20 @@ func (s *PipelineExecutor) reconcileDocumentCompiledProducts(ctx context.Context
 	indexName := fmt.Sprintf("ragflow_%s", s.taskCtx.Tenant.ID)
 	for start := 0; start < len(staleIDs); start += pageSize {
 		end := min(start+pageSize, len(staleIDs))
-		if _, err := docEngine.DeleteChunks(ctx, map[string]any{
+		started := time.Now()
+		finish := agentruntime.TrackIO(ctx, "Indexer", "Delete stale products", fmt.Sprintf("batch=%d/%d count=%d", start/pageSize+1, (len(staleIDs)+pageSize-1)/pageSize, end-start))
+		_, deleteErr := docEngine.DeleteChunks(ctx, map[string]any{
 			"id":    staleIDs[start:end],
 			"kb_id": s.taskCtx.Doc.KbID,
-		}, indexName, s.taskCtx.Doc.KbID); err != nil {
-			return fmt.Errorf("delete stale document compiler products: %w", err)
+		}, indexName, s.taskCtx.Doc.KbID)
+		finish()
+		message := fmt.Sprintf("Delete stale products | count=%d elapsed=%s", end-start, time.Since(started).Round(time.Millisecond))
+		if deleteErr != nil {
+			message += " error=" + kccommon.CompactError(deleteErr)
+		}
+		agentruntime.ReportIOProgress(ctx, "Indexer", message)
+		if deleteErr != nil {
+			return fmt.Errorf("delete stale document compiler products: %w", deleteErr)
 		}
 	}
 	return nil

internal/ingestion/task/stub.go

@@ -1,22 +0,0 @@
-//go:build !ee
-
-package task
-
-import "context"
-
-type kcLLMProgressReporter interface {
-	begin(ctx context.Context) func()
-	report(ctx context.Context, message string)
-}
-
-type noopKCLLMProgressReporter struct{}
-
-func newKCLLMProgressReporter() kcLLMProgressReporter {
-	return noopKCLLMProgressReporter{}
-}
-
-func (noopKCLLMProgressReporter) begin(context.Context) func() {
-	return func() {}
-}
-
-func (noopKCLLMProgressReporter) report(context.Context, string) {}

internal/parser/parser/pdf_parser_common.go

@@ -76,7 +76,10 @@ type PDFParser struct {
 	// caller observes per-page parse progress (done/total). Only the deepdoc
 	// backend invokes it; remote engines poll opaque HTTP jobs and never call
 	// it, so the fraction simply stays where it was.
-	OnPageDone                        func(done, total int)
+	OnPageDone func(done, total int)
+	// OnModelCall reports the start and end of an individual DeepDOC model call.
+	// It is runtime-only and is wired by the ingestion parser component.
+	OnModelCall                       func(ctx context.Context, operation string) func()
 	MinerUAPIServer                   string
 	MinerUAPIKey                      string
 	MinerUBackend                     string
@@ -399,6 +402,7 @@ func (p *PDFParser) parseWithResult(ctx context.Context, filename string, data [
 	cfg.RemoveTOC = p.RemoveTOC
 	cfg.RemoveHeaderFooter = p.RemoveHeaderFooter
 	cfg.OnPageDone = p.OnPageDone
+	cfg.OnModelCall = p.OnModelCall
 	res := parsePDFWithDeepDocOptions(ctx, filename, data, pdfPostProcessOptions{
 		zoom:              cfg.Zoom,
 		enableMultiColumn: p.EnableMultiColumn,