Add C-API SetVoiceSample for VoxLock PCM enroll (#4151)

sherpa-onnx/c-api/c-api.cc

@@ -3078,6 +3078,24 @@ int32_t SherpaOnnxOfflineSpeechDenoiserGetSampleRate(
   return sd->impl->GetSampleRate();
 }
 
+int32_t SherpaOnnxOfflineSpeechDenoiserSetVoiceSample(
+    const SherpaOnnxOfflineSpeechDenoiser *sd, const float *samples, int32_t n,
+    int32_t sample_rate) {
+  if (sd == nullptr || sd->impl == nullptr) {
+    return -1;
+  }
+  if (samples == nullptr && n > 0) {
+    return -1;
+  }
+
+  try {
+    return sd->impl->SetVoiceSample(samples, n, sample_rate) ? 0 : -1;
+  } catch (const std::exception &e) {
+    SHERPA_ONNX_LOGE("Failed to set voice sample: %s", e.what());
+    return -1;
+  }
+}
+
 const SherpaOnnxDenoisedAudio *SherpaOnnxOfflineSpeechDenoiserRun(
     const SherpaOnnxOfflineSpeechDenoiser *sd, const float *samples, int32_t n,
     int32_t sample_rate) {
@@ -3352,6 +3370,24 @@ void SherpaOnnxOnlineSpeechDenoiserReset(
   sd->impl->Reset();
 }
 
+int32_t SherpaOnnxOnlineSpeechDenoiserSetVoiceSample(
+    const SherpaOnnxOnlineSpeechDenoiser *sd, const float *samples, int32_t n,
+    int32_t sample_rate) {
+  if (sd == nullptr || sd->impl == nullptr) {
+    return -1;
+  }
+  if (samples == nullptr && n > 0) {
+    return -1;
+  }
+
+  try {
+    return sd->impl->SetVoiceSample(samples, n, sample_rate) ? 0 : -1;
+  } catch (const std::exception &e) {
+    SHERPA_ONNX_LOGE("Failed to set voice sample: %s", e.what());
+    return -1;
+  }
+}
+
 #if SHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION == 1
 
 struct SherpaOnnxOfflineSpeakerDiarization {

sherpa-onnx/c-api/c-api.h

@@ -4372,6 +4372,23 @@ SHERPA_ONNX_API void SherpaOnnxDestroyOfflineSpeechDenoiser(
 SHERPA_ONNX_API int32_t SherpaOnnxOfflineSpeechDenoiserGetSampleRate(
     const SherpaOnnxOfflineSpeechDenoiser *sd);
 
+/**
+ * @brief Set a PCM voice sample for VoxLock.
+ *
+ * After resampling to the model rate, it keeps the loudest 3 seconds
+ * within the first 30 seconds. Returns -1 if the clip is shorter than
+ * 3 seconds after resampling; the previous voice sample is left unchanged.
+ *
+ * @param sd A pointer returned by SherpaOnnxCreateOfflineSpeechDenoiser().
+ * @param samples Input mono PCM samples normalized to [-1, 1].
+ * @param n Number of input samples.
+ * @param sample_rate Input sample rate in Hz.
+ * @return 0 on success, -1 on error.
+ */
+SHERPA_ONNX_API int32_t SherpaOnnxOfflineSpeechDenoiserSetVoiceSample(
+    const SherpaOnnxOfflineSpeechDenoiser *sd, const float *samples, int32_t n,
+    int32_t sample_rate);
+
 /**
  * @brief Denoised audio returned by offline or online speech enhancement APIs.
  *
@@ -4511,6 +4528,22 @@ SherpaOnnxOnlineSpeechDenoiserFlush(const SherpaOnnxOnlineSpeechDenoiser *sd);
 SHERPA_ONNX_API void SherpaOnnxOnlineSpeechDenoiserReset(
     const SherpaOnnxOnlineSpeechDenoiser *sd);
 
+/**
+ * @brief Set a PCM voice sample for VoxLock.
+ *
+ * See SherpaOnnxOfflineSpeechDenoiserSetVoiceSample(). On success the
+ * denoiser is reset so it uses the new clip.
+ *
+ * @param sd A pointer returned by SherpaOnnxCreateOnlineSpeechDenoiser().
+ * @param samples Input mono PCM samples normalized to [-1, 1].
+ * @param n Number of input samples.
+ * @param sample_rate Input sample rate in Hz.
+ * @return 0 on success, -1 on error.
+ */
+SHERPA_ONNX_API int32_t SherpaOnnxOnlineSpeechDenoiserSetVoiceSample(
+    const SherpaOnnxOnlineSpeechDenoiser *sd, const float *samples, int32_t n,
+    int32_t sample_rate);
+
 // =========================================================================
 // Source separation
 // =========================================================================

sherpa-onnx/c-api/cxx-api.cc

@@ -876,6 +876,15 @@ int32_t OfflineSpeechDenoiser::GetSampleRate() const {
   return SherpaOnnxOfflineSpeechDenoiserGetSampleRate(p_);
 }
 
+bool OfflineSpeechDenoiser::SetVoiceSample(const float *samples, int32_t n,
+                                           int32_t sample_rate) const {
+  if (!p_) {
+    return false;
+  }
+  return SherpaOnnxOfflineSpeechDenoiserSetVoiceSample(p_, samples, n,
+                                                       sample_rate) == 0;
+}
+
 OnlineSpeechDenoiser OnlineSpeechDenoiser::Create(
     const OnlineSpeechDenoiserConfig &config) {
   struct SherpaOnnxOnlineSpeechDenoiserConfig c;
@@ -925,6 +934,15 @@ void OnlineSpeechDenoiser::Reset() const {
   SherpaOnnxOnlineSpeechDenoiserReset(p_);
 }
 
+bool OnlineSpeechDenoiser::SetVoiceSample(const float *samples, int32_t n,
+                                          int32_t sample_rate) const {
+  if (!p_) {
+    return false;
+  }
+  return SherpaOnnxOnlineSpeechDenoiserSetVoiceSample(p_, samples, n,
+                                                      sample_rate) == 0;
+}
+
 int32_t OnlineSpeechDenoiser::GetSampleRate() const {
   return SherpaOnnxOnlineSpeechDenoiserGetSampleRate(p_);
 }

sherpa-onnx/c-api/cxx-api.h

@@ -1360,6 +1360,10 @@ class SHERPA_ONNX_API OfflineSpeechDenoiser
   /** @brief Return the expected input sample rate. */
   int32_t GetSampleRate() const;
 
+  /** @brief Set a PCM voice sample for VoxLock. */
+  bool SetVoiceSample(const float *samples, int32_t n,
+                      int32_t sample_rate) const;
+
  private:
   explicit OfflineSpeechDenoiser(const SherpaOnnxOfflineSpeechDenoiser *p);
 };
@@ -1389,6 +1393,10 @@ class SHERPA_ONNX_API OnlineSpeechDenoiser
   /** @brief Reset the denoiser for a new stream. */
   void Reset() const;
 
+  /** @brief Set a PCM voice sample for VoxLock. */
+  bool SetVoiceSample(const float *samples, int32_t n,
+                      int32_t sample_rate) const;
+
   /** @brief Return the expected input sample rate. */
   int32_t GetSampleRate() const;
 

sherpa-onnx/csrc/offline-speech-denoiser-impl.h

@@ -26,6 +26,11 @@ class OfflineSpeechDenoiserImpl {
                             int32_t sample_rate) const = 0;
 
   virtual int32_t GetSampleRate() const = 0;
+
+  virtual bool SetVoiceSample(const float * /*samples*/, int32_t /*n*/,
+                              int32_t /*sample_rate*/) {
+    return false;
+  }
 };
 
 }  // namespace sherpa_onnx

sherpa-onnx/csrc/offline-speech-denoiser-voxlock-impl.h

@@ -88,6 +88,18 @@ class OfflineSpeechDenoiserVoxLockImpl : public OfflineSpeechDenoiserImpl {
     return model_.GetMetaData().sample_rate;
   }
 
+  bool SetVoiceSample(const float *samples, int32_t n,
+                      int32_t sample_rate) override {
+    const auto &meta = model_.GetMetaData();
+    auto prepared = PrepareVoxLockVoiceSampleFromPcm(
+        samples, n, sample_rate, meta.sample_rate, meta.voice_sample_seconds);
+    if (prepared.empty()) {
+      return false;
+    }
+    voice_sample_ = std::move(prepared);
+    return true;
+  }
+
  private:
   void Init(const OfflineSpeechDenoiserConfig &config) {
     const auto &meta = model_.GetMetaData();

sherpa-onnx/csrc/offline-speech-denoiser.cc

@@ -53,6 +53,14 @@ int32_t OfflineSpeechDenoiser::GetSampleRate() const {
   return impl_->GetSampleRate();
 }
 
+bool OfflineSpeechDenoiser::SetVoiceSample(const float *samples, int32_t n,
+                                           int32_t sample_rate) {
+  if (!impl_) {
+    return false;
+  }
+  return impl_->SetVoiceSample(samples, n, sample_rate);
+}
+
 #if __ANDROID_API__ >= 9
 template OfflineSpeechDenoiser::OfflineSpeechDenoiser(
     AAssetManager *mgr, const OfflineSpeechDenoiserConfig &config);

sherpa-onnx/csrc/offline-speech-denoiser.h

@@ -52,6 +52,19 @@ class OfflineSpeechDenoiser {
    */
   int32_t GetSampleRate() const;
 
+  /*
+   * Set a PCM voice sample for VoxLock.
+   *
+   * @param samples 1-D array of audio samples. Each sample is in the
+   *                range [-1, 1].
+   * @param n Number of samples
+   * @param sample_rate Sample rate of the input samples
+   * @return Return true on success. Return false if this is not a VoxLock
+   *         denoiser or if the clip is shorter than 3 seconds after
+   *         resampling.
+   */
+  bool SetVoiceSample(const float *samples, int32_t n, int32_t sample_rate);
+
  private:
   std::unique_ptr<OfflineSpeechDenoiserImpl> impl_;
 };

sherpa-onnx/csrc/online-speech-denoiser-impl.h

@@ -28,6 +28,11 @@ class OnlineSpeechDenoiserImpl {
   virtual void Reset() = 0;
   virtual int32_t GetSampleRate() const = 0;
   virtual int32_t GetFrameShiftInSamples() const = 0;
+
+  virtual bool SetVoiceSample(const float * /*samples*/, int32_t /*n*/,
+                              int32_t /*sample_rate*/) {
+    return false;
+  }
 };
 
 }  // namespace sherpa_onnx

sherpa-onnx/csrc/online-speech-denoiser-voxlock-impl.h

@@ -132,17 +132,7 @@ class OnlineSpeechDenoiserVoxLockImpl : public OnlineSpeechDenoiserImpl {
   }
 
   void Reset() override {
-    resampler_.reset();
-    input_sample_rate_ = -1;
-    total_input_samples_ = 0;
-    total_output_samples_ = 0;
-
-    voiced_.clear();
-    energy_.clear();
-    rest_.clear();
-    recent_.clear();
-    after_.clear();
-    num_to_skip_ = 0;
+    ClearState();
 
     if (voice_sample_.empty()) {
       stream_.reset();
@@ -162,7 +152,38 @@ class OnlineSpeechDenoiserVoxLockImpl : public OnlineSpeechDenoiserImpl {
     return model_.GetMetaData().hop_length;
   }
 
+  bool SetVoiceSample(const float *samples, int32_t n,
+                      int32_t sample_rate) override {
+    const auto &meta = model_.GetMetaData();
+    auto prepared = PrepareVoxLockVoiceSampleFromPcm(
+        samples, n, sample_rate, meta.sample_rate, meta.voice_sample_seconds);
+    if (prepared.empty()) {
+      return false;
+    }
+    auto stream = std::make_unique<SpeechDenoiserVoxLockStream>(
+        &model_, prepared.data(), prepared.size());
+    voice_sample_ = std::move(prepared);
+    ClearState();
+    stream_ = std::move(stream);
+    num_faded_ = fade_length_;
+    return true;
+  }
+
  private:
+  void ClearState() {
+    resampler_.reset();
+    input_sample_rate_ = -1;
+    total_input_samples_ = 0;
+    total_output_samples_ = 0;
+
+    voiced_.clear();
+    energy_.clear();
+    rest_.clear();
+    recent_.clear();
+    after_.clear();
+    num_to_skip_ = 0;
+  }
+
   void Init(const OnlineSpeechDenoiserConfig &config) {
     const auto &meta = model_.GetMetaData();
     const auto &filename = config.model.voxlock.voice_sample;

sherpa-onnx/csrc/online-speech-denoiser.cc

@@ -62,6 +62,14 @@ int32_t OnlineSpeechDenoiser::GetFrameShiftInSamples() const {
   return impl_->GetFrameShiftInSamples();
 }
 
+bool OnlineSpeechDenoiser::SetVoiceSample(const float *samples, int32_t n,
+                                          int32_t sample_rate) {
+  if (!impl_) {
+    return false;
+  }
+  return impl_->SetVoiceSample(samples, n, sample_rate);
+}
+
 #if __ANDROID_API__ >= 9
 template OnlineSpeechDenoiser::OnlineSpeechDenoiser(
     AAssetManager *mgr, const OnlineSpeechDenoiserConfig &config);

sherpa-onnx/csrc/online-speech-denoiser.h

@@ -49,6 +49,14 @@ class OnlineSpeechDenoiser {
   int32_t GetSampleRate() const;
   int32_t GetFrameShiftInSamples() const;
 
+  /*
+   * Set a PCM voice sample for VoxLock.
+   *
+   * See OfflineSpeechDenoiser::SetVoiceSample. On success the denoiser is
+   * reset so it uses the new clip.
+   */
+  bool SetVoiceSample(const float *samples, int32_t n, int32_t sample_rate);
+
  private:
   std::unique_ptr<OnlineSpeechDenoiserImpl> impl_;
 };

sherpa-onnx/csrc/speech-denoiser-voxlock-stream.h

@@ -131,6 +131,45 @@ inline std::vector<float> PrepareVoxLockVoiceSample(
                                   kVoxLockMaxVoiceSampleSeconds);
 }
 
+// Resample PCM to sample_rate and return exactly `seconds` seconds of it.
+// Return an empty vector if it is shorter than `seconds` seconds.
+inline std::vector<float> PrepareVoxLockVoiceSampleFromPcm(
+    const float *samples, int32_t n, int32_t in_sample_rate,
+    int32_t sample_rate, int32_t seconds) {
+  if (samples == nullptr || n <= 0 || in_sample_rate <= 0 || sample_rate <= 0 ||
+      seconds <= 0) {
+    return {};
+  }
+
+  std::vector<float> resampled;
+  if (in_sample_rate == sample_rate) {
+    resampled.assign(samples, samples + n);
+  } else {
+    CreateVoxLockResampler(in_sample_rate, sample_rate)
+        ->Resample(samples, n, true, &resampled);
+  }
+
+  const int32_t rn = static_cast<int32_t>(resampled.size());
+  if (rn < seconds * sample_rate) {
+    SHERPA_ONNX_LOGE(
+        "The voxlock voice sample is %.3f seconds long, shorter than %d "
+        "seconds.",
+        static_cast<float>(rn) / sample_rate, seconds);
+    return {};
+  }
+
+  if (rn > seconds * sample_rate) {
+    SHERPA_ONNX_LOGE(
+        "The voxlock voice sample is %.3f seconds long. Use its loudest "
+        "%d seconds within its first %d seconds.",
+        static_cast<float>(rn) / sample_rate, seconds,
+        kVoxLockMaxVoiceSampleSeconds);
+  }
+
+  return SelectVoxLockVoiceSample(resampled.data(), rn, sample_rate, seconds,
+                                  kVoxLockMaxVoiceSampleSeconds);
+}
+
 // Read a voice sample from a wave file and return exactly `seconds` seconds
 // of it. Return an empty vector if it cannot be read or is too short.
 inline std::vector<float> ReadVoxLockVoiceSample(const std::string &filename,