diff --git a/internal/entity/models/types.go b/internal/entity/models/types.go index c27e5f82e3..c5f3622c85 100644 --- a/internal/entity/models/types.go +++ b/internal/entity/models/types.go @@ -417,6 +417,50 @@ func (r *RerankModel) Rerank(ctx context.Context, request RerankRequest, apiConf return r.ModelDriver.Rerank(ctx, r.ModelName, request, apiConfig, rerankConfig, modelUsage) } +// ASRModel wraps a ModelDriver with speech-to-text configuration. +type ASRModel struct { + ModelDriver ModelDriver + ModelName *string + APIConfig *APIConfig +} + +// NewASRModel creates a new ASRModel. +func NewASRModel(driver ModelDriver, modelName *string, apiConfig *APIConfig) *ASRModel { + return &ASRModel{ModelDriver: driver, ModelName: modelName, APIConfig: apiConfig} +} + +// Transcribe converts audio to text. +func (m *ASRModel) Transcribe(ctx context.Context, audioFile *string, config *ASRConfig, usage *common.ModelUsage) (*ASRResponse, error) { + return m.ModelDriver.TranscribeAudio(ctx, m.ModelName, audioFile, m.APIConfig, config, usage) +} + +// TranscribeWithSender streams transcription results through sender. +func (m *ASRModel) TranscribeWithSender(ctx context.Context, audioFile *string, config *ASRConfig, usage *common.ModelUsage, sender func(*string, *string) error) error { + return m.ModelDriver.TranscribeAudioWithSender(ctx, m.ModelName, audioFile, m.APIConfig, config, usage, sender) +} + +// TTSModel wraps a ModelDriver with text-to-speech configuration. +type TTSModel struct { + ModelDriver ModelDriver + ModelName *string + APIConfig *APIConfig +} + +// NewTTSModel creates a new TTSModel. +func NewTTSModel(driver ModelDriver, modelName *string, apiConfig *APIConfig) *TTSModel { + return &TTSModel{ModelDriver: driver, ModelName: modelName, APIConfig: apiConfig} +} + +// Speech converts text to audio. +func (m *TTSModel) Speech(ctx context.Context, audioContent *string, config *TTSConfig, usage *common.ModelUsage) (*TTSResponse, error) { + return m.ModelDriver.AudioSpeech(ctx, m.ModelName, audioContent, m.APIConfig, config, usage) +} + +// SpeechWithSender streams synthesized audio through sender. +func (m *TTSModel) SpeechWithSender(ctx context.Context, audioContent *string, config *TTSConfig, usage *common.ModelUsage, sender func(*string, *string) error) error { + return m.ModelDriver.AudioSpeechWithSender(ctx, m.ModelName, audioContent, m.APIConfig, config, usage, sender) +} + // ToolConfig bundles tool-calling configuration for a ChatModel. type ToolConfig struct { Tools string // JSON-encoded tools list diff --git a/internal/handler/chat_audio.go b/internal/handler/chat_audio.go index c281ab0562..0743081d7b 100644 --- a/internal/handler/chat_audio.go +++ b/internal/handler/chat_audio.go @@ -82,6 +82,7 @@ func (h *ChatHandler) ChatAudioSpeech(c *gin.Context) { driver := target.Driver modelName := target.ModelName apiConfig := target.APIConfig + ttsModel := modelModule.NewTTSModel(driver, &modelName, apiConfig) writeAudioHeaders := func(mediaType string) { c.Header("Content-Type", mediaType) @@ -108,7 +109,7 @@ func (h *ChatHandler) ChatAudioSpeech(c *gin.Context) { if seg == "" { continue } - resp, err := driver.AudioSpeech(ctx, &modelName, &seg, apiConfig, &modelModule.TTSConfig{Format: "mp3"}, nil) + resp, err := ttsModel.Speech(ctx, &seg, &modelModule.TTSConfig{Format: "mp3"}, nil) if err != nil { if firstSynthErr == nil { firstSynthErr = err @@ -276,6 +277,7 @@ func (h *ChatHandler) ChatAudioTranscription(c *gin.Context) { driver := target.Driver modelName := target.ModelName apiConfig := target.APIConfig + asrModel := modelModule.NewASRModel(driver, &modelName, apiConfig) streamMode := strings.ToLower(c.PostForm("stream")) == "true" if streamMode { @@ -306,7 +308,7 @@ func (h *ChatHandler) ChatAudioTranscription(c *gin.Context) { return nil } - if err = driver.TranscribeAudioWithSender(ctx, &modelName, &tmpPath, apiConfig, &modelModule.ASRConfig{}, nil, sender); err != nil { + if err = asrModel.TranscribeWithSender(ctx, &tmpPath, &modelModule.ASRConfig{}, nil, sender); err != nil { errEvent := map[string]interface{}{"event": "error", "text": err.Error()} data, _ := json.Marshal(errEvent) _, _ = c.Writer.WriteString(fmt.Sprintf("data: %s\n\n", data)) @@ -324,7 +326,7 @@ func (h *ChatHandler) ChatAudioTranscription(c *gin.Context) { return } - resp, err := driver.TranscribeAudio(ctx, &modelName, &tmpPath, apiConfig, &modelModule.ASRConfig{}, nil) + resp, err := asrModel.Transcribe(ctx, &tmpPath, &modelModule.ASRConfig{}, nil) if err != nil { common.ErrorWithCode(c, common.CodeServerError, err.Error()) return diff --git a/internal/ingestion/component/media_dispatch.go b/internal/ingestion/component/media_dispatch.go index 2ba8240b38..8511798f06 100644 --- a/internal/ingestion/component/media_dispatch.go +++ b/internal/ingestion/component/media_dispatch.go @@ -260,7 +260,8 @@ func maybeDispatchAudio( } defer os.Remove(tmpFile) - resp, err := driver.TranscribeAudio(ctx, &modelName, &tmpFile, apiConfig, nil, nil) + asrModel := modelModule.NewASRModel(driver, &modelName, apiConfig) + resp, err := asrModel.Transcribe(ctx, &tmpFile, nil, nil) if err != nil { return parser.ParseResult{}, true, fmt.Errorf("Parser: audio transcription: %w", err) diff --git a/internal/service/chat_pipeline.go b/internal/service/chat_pipeline.go index aa3e7230d9..2e0f987f41 100644 --- a/internal/service/chat_pipeline.go +++ b/internal/service/chat_pipeline.go @@ -1662,7 +1662,7 @@ func (s *ChatPipelineService) AsyncChatSolo( chatModel := modelModule.NewChatModel(target.Driver, &target.ModelName, target.APIConfig) // 5. Resolve TTS model. Best-effort: warn and proceed without TTS on lookup failure. - var ttsModel *modelModule.ChatModel + var ttsModel *modelModule.TTSModel if promptConfig != nil { if useTTS, _ := promptConfig["tts"].(bool); useTTS { target, ttsErr := s.ModelProviderSvc.modelSolver().ResolveDefaultModelConfig(ctx, chat.TenantID, entity.ModelTypeTTS) @@ -1671,7 +1671,7 @@ func (s *ChatPipelineService) AsyncChatSolo( zap.String("tenant_id", chat.TenantID), zap.Error(ttsErr)) } else { - ttsModel = modelModule.NewChatModel(target.Driver, &target.ModelName, target.APIConfig) + ttsModel = modelModule.NewTTSModel(target.Driver, &target.ModelName, target.APIConfig) } } } @@ -2298,7 +2298,7 @@ func (s *ChatPipelineService) getModels(ctx context.Context, chat *entity.Chat) *modelModule.EmbeddingModel, *modelModule.RerankModel, *modelModule.ChatModel, - *modelModule.ChatModel, // TTS model + *modelModule.TTSModel, // TTS model error, ) { kbDAO := dao.NewKnowledgebaseDAO() @@ -2368,12 +2368,12 @@ func (s *ChatPipelineService) getModels(ctx context.Context, chat *entity.Chat) } // TTS model. - var ttsModel *modelModule.ChatModel + var ttsModel *modelModule.TTSModel if chat.PromptConfig != nil { if useTTS, _ := chat.PromptConfig["tts"].(bool); useTTS { target, err := s.ModelProviderSvc.modelSolver().ResolveDefaultModelConfig(ctx, chat.TenantID, entity.ModelTypeTTS) if err == nil { - ttsModel = modelModule.NewChatModel(target.Driver, &target.ModelName, target.APIConfig) + ttsModel = modelModule.NewTTSModel(target.Driver, &target.ModelName, target.APIConfig) } } } @@ -2596,7 +2596,7 @@ func cleanTTSText(text string) string { // synthesizeTTS calls the TTS model to convert text to audio. // Mirrors dialog_service.py:1426-1432. -func (s *ChatPipelineService) synthesizeTTS(ctx context.Context, ttsModel *modelModule.ChatModel, text string) interface{} { +func (s *ChatPipelineService) synthesizeTTS(ctx context.Context, ttsModel *modelModule.TTSModel, text string) interface{} { if ttsModel == nil || text == "" { return nil } @@ -2604,9 +2604,7 @@ func (s *ChatPipelineService) synthesizeTTS(ctx context.Context, ttsModel *model if text == "" { return nil } - ttsResp, err := ttsModel.ModelDriver.AudioSpeech( - ctx, ttsModel.ModelName, &text, ttsModel.APIConfig, &modelModule.TTSConfig{Format: "mp3"}, nil, - ) + ttsResp, err := ttsModel.Speech(ctx, &text, &modelModule.TTSConfig{Format: "mp3"}, nil) if err != nil { common.Warn("TTS synthesis failed", zap.Error(err)) return nil @@ -3117,7 +3115,7 @@ func (s *ChatPipelineService) decorateAnswer( embModel *modelModule.EmbeddingModel, vectorSimilarityWeight float64, quote bool, - ttsModel *modelModule.ChatModel, + ttsModel *modelModule.TTSModel, langfuseTraceID string, llmModelConfig map[string]interface{}, tenantID string, diff --git a/internal/service/model_call.go b/internal/service/model_call.go index 17c3f39cfe..453728a5a7 100644 --- a/internal/service/model_call.go +++ b/internal/service/model_call.go @@ -173,7 +173,8 @@ func (s *ModelCallService) TranscribeAudio(ctx context.Context, modelRef, userID } modelName := target.ModelName - response, err := target.Driver.TranscribeAudio(ctx, &modelName, audioFile, target.APIConfig, config, nil) + asrModel := modelModule.NewASRModel(target.Driver, &modelName, target.APIConfig) + response, err := asrModel.Transcribe(ctx, audioFile, config, nil) if err != nil { return nil, common.CodeServerError, err } @@ -195,7 +196,8 @@ func (s *ModelCallService) TranscribeAudioStream(ctx context.Context, modelRef, } modelName := target.ModelName - if err := target.Driver.TranscribeAudioWithSender(ctx, &modelName, audioFile, target.APIConfig, config, nil, sender); err != nil { + asrModel := modelModule.NewASRModel(target.Driver, &modelName, target.APIConfig) + if err := asrModel.TranscribeWithSender(ctx, audioFile, config, nil, sender); err != nil { return common.CodeServerError, err } return common.CodeSuccess, nil @@ -231,7 +233,8 @@ func (s *ModelCallService) AudioSpeech(ctx context.Context, modelRef, userID str } modelName := target.ModelName - response, err := target.Driver.AudioSpeech(ctx, &modelName, audioContent, target.APIConfig, config, nil) + ttsModel := modelModule.NewTTSModel(target.Driver, &modelName, target.APIConfig) + response, err := ttsModel.Speech(ctx, audioContent, config, nil) if err != nil { return nil, common.CodeServerError, err } @@ -263,7 +266,8 @@ func (s *ModelCallService) AudioSpeechForTenant(ctx context.Context, modelRef, t } modelName := target.ModelName - response, err := target.Driver.AudioSpeech(ctx, &modelName, audioContent, target.APIConfig, config, nil) + ttsModel := modelModule.NewTTSModel(target.Driver, &modelName, target.APIConfig) + response, err := ttsModel.Speech(ctx, audioContent, config, nil) if err != nil { return nil, common.CodeServerError, err } @@ -285,7 +289,8 @@ func (s *ModelCallService) AudioSpeechStream(ctx context.Context, modelRef, user } modelName := target.ModelName - if err := target.Driver.AudioSpeechWithSender(ctx, &modelName, audioContent, target.APIConfig, config, nil, sender); err != nil { + ttsModel := modelModule.NewTTSModel(target.Driver, &modelName, target.APIConfig) + if err := ttsModel.SpeechWithSender(ctx, audioContent, config, nil, sender); err != nil { return common.CodeServerError, err } return common.CodeSuccess, nil diff --git a/internal/service/model_service.go b/internal/service/model_service.go index 750daee019..803dd03d95 100644 --- a/internal/service/model_service.go +++ b/internal/service/model_service.go @@ -977,7 +977,8 @@ func verifyProviderModel(ctx context.Context, driver modelModule.ModelDriver, pr _, err = driver.Rerank(ctx, &modelName, rerankRequest, apiConfig, &modelModule.RerankConfig{}, nil) case "tts": content := "hello" - _, err = driver.AudioSpeech(ctx, &modelName, &content, apiConfig, nil, nil) + ttsModel := modelModule.NewTTSModel(driver, &modelName, apiConfig) + _, err = ttsModel.Speech(ctx, &content, nil, nil) case "asr": err = verifyASRModel(ctx, driver, modelName, apiConfig) case "ocr": @@ -1099,7 +1100,8 @@ func verifyASRModel(ctx context.Context, driver modelModule.ModelDriver, modelNa } tmpFile.Close() - resp, err := driver.TranscribeAudio(ctx, &modelName, &tmpPath, apiConfig, nil, nil) + asrModel := modelModule.NewASRModel(driver, &modelName, apiConfig) + resp, err := asrModel.Transcribe(ctx, &tmpPath, nil, nil) if err != nil { return err }