mirror of
https://github.com/k2-fsa/sherpa-onnx.git
synced 2026-11-10 21:11:03 +08:00
Add Java and Kotlin API for VoxLock (#4107)
This commit is contained in:
1 parent
42e1e0d077
commit
687a26126c
20 files changed
+922
No files matched your search
@@ -78,3 +78,29 @@ jobs:
|
|||||||
with:
|
with:
|
||||||
name: tts-files-${{ matrix.os }}
|
name: tts-files-${{ matrix.os }}
|
||||||
path: kotlin-api-examples/test-*.wav
|
path: kotlin-api-examples/test-*.wav
|
||||||
|
|
||||||
|
- uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: test-offline-speech-denoiser-voxlock-${{ matrix.os }}
|
||||||
|
# run.sh keeps the input and output wavs of each test in a folder
|
||||||
|
# named after the test file
|
||||||
|
path: kotlin-api-examples/test_offline_speech_denoiser_voxlock
|
||||||
|
if-no-files-found: warn
|
||||||
|
|
||||||
|
- uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: test-offline-speech-denoiser-voxlock-with-voice-sample-${{ matrix.os }}
|
||||||
|
path: kotlin-api-examples/test_offline_speech_denoiser_voxlock_with_voice_sample
|
||||||
|
if-no-files-found: warn
|
||||||
|
|
||||||
|
- uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: test-online-speech-denoiser-voxlock-${{ matrix.os }}
|
||||||
|
path: kotlin-api-examples/test_online_speech_denoiser_voxlock
|
||||||
|
if-no-files-found: warn
|
||||||
|
|
||||||
|
- uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: test-online-speech-denoiser-voxlock-with-voice-sample-${{ matrix.os }}
|
||||||
|
path: kotlin-api-examples/test_online_speech_denoiser_voxlock_with_voice_sample
|
||||||
|
if-no-files-found: warn
|
||||||
@@ -158,6 +158,41 @@ jobs:
|
|||||||
cd ./java-api-examples
|
cd ./java-api-examples
|
||||||
./run-version-test.sh
|
./run-version-test.sh
|
||||||
|
|
||||||
|
- name: Run speech enhancement (VoxLock)
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
cd ./java-api-examples
|
||||||
|
./run-non-streaming-speech-enhancement-voxlock.sh
|
||||||
|
./run-non-streaming-speech-enhancement-voxlock-with-voice-sample.sh
|
||||||
|
./run-streaming-speech-enhancement-voxlock.sh
|
||||||
|
./run-streaming-speech-enhancement-voxlock-with-voice-sample.sh
|
||||||
|
|
||||||
|
- uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: NonStreamingSpeechEnhancementVoxLock-${{ matrix.os }}
|
||||||
|
# each run-*.sh keeps its input and output wavs in a folder named
|
||||||
|
# after the example
|
||||||
|
path: java-api-examples/NonStreamingSpeechEnhancementVoxLock
|
||||||
|
if-no-files-found: warn
|
||||||
|
|
||||||
|
- uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: NonStreamingSpeechEnhancementVoxLockWithVoiceSample-${{ matrix.os }}
|
||||||
|
path: java-api-examples/NonStreamingSpeechEnhancementVoxLockWithVoiceSample
|
||||||
|
if-no-files-found: warn
|
||||||
|
|
||||||
|
- uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: StreamingSpeechEnhancementVoxLock-${{ matrix.os }}
|
||||||
|
path: java-api-examples/StreamingSpeechEnhancementVoxLock
|
||||||
|
if-no-files-found: warn
|
||||||
|
|
||||||
|
- uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: StreamingSpeechEnhancementVoxLockWithVoiceSample-${{ matrix.os }}
|
||||||
|
path: java-api-examples/StreamingSpeechEnhancementVoxLockWithVoiceSample
|
||||||
|
if-no-files-found: warn
|
||||||
|
|
||||||
- name: Run java test (Non-Streaming TTS)
|
- name: Run java test (Non-Streaming TTS)
|
||||||
shell: bash
|
shell: bash
|
||||||
run: |
|
run: |
|
||||||
|
|||||||
@@ -142,6 +142,11 @@ README-DEV.txt
|
|||||||
*.jit
|
*.jit
|
||||||
##clion
|
##clion
|
||||||
.idea
|
.idea
|
||||||
|
|
||||||
|
## Eclipse and other IDEs
|
||||||
|
.classpath
|
||||||
|
.project
|
||||||
|
.settings/
|
||||||
sherpa-onnx-dolphin-base-ctc-multi-lang-int8-2025-04-02
|
sherpa-onnx-dolphin-base-ctc-multi-lang-int8-2025-04-02
|
||||||
dict
|
dict
|
||||||
*.npz
|
*.npz
|
||||||
|
|||||||
@@ -0,0 +1,57 @@
|
|||||||
|
// Copyright 2026 Xiaomi Corporation
|
||||||
|
|
||||||
|
// This file shows how to use VoxLock speech enhancement models in sherpa-onnx.
|
||||||
|
//
|
||||||
|
// VoxLock is a target-speaker voice isolation model. It keeps the voice of one
|
||||||
|
// speaker, e.g., the caller in a phone call, and removes the other voices.
|
||||||
|
//
|
||||||
|
// The model is from https://github.com/arbazsiddiqui/voxlock
|
||||||
|
//
|
||||||
|
// This example does not provide a voice sample, so VoxLock takes the loudest
|
||||||
|
// 3 seconds in the first 15 seconds of the input as the speaker to keep. See
|
||||||
|
// NonStreamingSpeechEnhancementVoxLockWithVoiceSample.java if you have a voice
|
||||||
|
// sample of the speaker.
|
||||||
|
//
|
||||||
|
// Download models and sample test waves from:
|
||||||
|
// https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models
|
||||||
|
|
||||||
|
import com.k2fsa.sherpa.onnx.*;
|
||||||
|
|
||||||
|
public class NonStreamingSpeechEnhancementVoxLock {
|
||||||
|
public static void main(String[] args) {
|
||||||
|
OfflineSpeechDenoiserModelConfig modelConfig =
|
||||||
|
OfflineSpeechDenoiserModelConfig.builder()
|
||||||
|
.setNumThreads(1)
|
||||||
|
.setDebug(true)
|
||||||
|
.setProvider("cpu")
|
||||||
|
.setVoxlock(
|
||||||
|
OfflineSpeechDenoiserVoxLockModelConfig.builder()
|
||||||
|
.setEnrollModel("./sherpa-onnx-voxlock-int8/enroll.onnx")
|
||||||
|
.setChunkModel("./sherpa-onnx-voxlock-int8/chunk_int8.onnx")
|
||||||
|
.build())
|
||||||
|
.build();
|
||||||
|
OfflineSpeechDenoiserConfig config =
|
||||||
|
OfflineSpeechDenoiserConfig.builder().setModel(modelConfig).build();
|
||||||
|
|
||||||
|
OfflineSpeechDenoiser speechDenoiser = new OfflineSpeechDenoiser(config);
|
||||||
|
|
||||||
|
WaveReader reader = new WaveReader("./sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav");
|
||||||
|
|
||||||
|
long begin = System.currentTimeMillis();
|
||||||
|
DenoisedAudio denoised = speechDenoiser.run(reader.getSamples(), reader.getSampleRate());
|
||||||
|
long elapsedMs = System.currentTimeMillis() - begin;
|
||||||
|
|
||||||
|
String outFilename = "voxlock-enhanced-16k.wav";
|
||||||
|
WaveWriter.write(outFilename, denoised.getSamples(), denoised.getSampleRate());
|
||||||
|
|
||||||
|
float elapsedSeconds = elapsedMs / 1000.0f;
|
||||||
|
float duration = reader.getSamples().length / (float) reader.getSampleRate();
|
||||||
|
System.out.printf("Saved to %s\n", outFilename);
|
||||||
|
System.out.printf("Elapsed seconds: %.3f\n", elapsedSeconds);
|
||||||
|
System.out.printf("Audio duration in seconds: %.3f\n", duration);
|
||||||
|
System.out.printf(
|
||||||
|
"RTF: %.3f/%.3f = %.3f\n", elapsedSeconds, duration, elapsedSeconds / duration);
|
||||||
|
|
||||||
|
speechDenoiser.release();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,60 @@
|
|||||||
|
// Copyright 2026 Xiaomi Corporation
|
||||||
|
|
||||||
|
// This file shows how to use VoxLock speech enhancement models in sherpa-onnx
|
||||||
|
// with a voice sample.
|
||||||
|
//
|
||||||
|
// VoxLock is a target-speaker voice isolation model. It keeps the voice of one
|
||||||
|
// speaker, e.g., the caller in a phone call, and removes the other voices.
|
||||||
|
//
|
||||||
|
// The model is from https://github.com/arbazsiddiqui/voxlock
|
||||||
|
//
|
||||||
|
// This example provides a voice sample of the speaker to keep, so VoxLock
|
||||||
|
// isolates that speaker from the first sample. The voice sample must have at
|
||||||
|
// least 3 seconds of the speaker's voice; from a longer file, VoxLock uses the
|
||||||
|
// loudest 3 seconds within its first 30 seconds. See
|
||||||
|
// NonStreamingSpeechEnhancementVoxLock.java if you don't have a voice sample.
|
||||||
|
//
|
||||||
|
// Download models and sample test waves from:
|
||||||
|
// https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models
|
||||||
|
|
||||||
|
import com.k2fsa.sherpa.onnx.*;
|
||||||
|
|
||||||
|
public class NonStreamingSpeechEnhancementVoxLockWithVoiceSample {
|
||||||
|
public static void main(String[] args) {
|
||||||
|
OfflineSpeechDenoiserModelConfig modelConfig =
|
||||||
|
OfflineSpeechDenoiserModelConfig.builder()
|
||||||
|
.setNumThreads(1)
|
||||||
|
.setDebug(true)
|
||||||
|
.setProvider("cpu")
|
||||||
|
.setVoxlock(
|
||||||
|
OfflineSpeechDenoiserVoxLockModelConfig.builder()
|
||||||
|
.setEnrollModel("./sherpa-onnx-voxlock-int8/enroll.onnx")
|
||||||
|
.setChunkModel("./sherpa-onnx-voxlock-int8/chunk_int8.onnx")
|
||||||
|
.setVoiceSample("./sherpa-onnx-voxlock-int8/test_wavs/en-voice-sample-f.wav")
|
||||||
|
.build())
|
||||||
|
.build();
|
||||||
|
OfflineSpeechDenoiserConfig config =
|
||||||
|
OfflineSpeechDenoiserConfig.builder().setModel(modelConfig).build();
|
||||||
|
|
||||||
|
OfflineSpeechDenoiser speechDenoiser = new OfflineSpeechDenoiser(config);
|
||||||
|
|
||||||
|
WaveReader reader = new WaveReader("./sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav");
|
||||||
|
|
||||||
|
long begin = System.currentTimeMillis();
|
||||||
|
DenoisedAudio denoised = speechDenoiser.run(reader.getSamples(), reader.getSampleRate());
|
||||||
|
long elapsedMs = System.currentTimeMillis() - begin;
|
||||||
|
|
||||||
|
String outFilename = "voxlock-enhanced-with-voice-sample-16k.wav";
|
||||||
|
WaveWriter.write(outFilename, denoised.getSamples(), denoised.getSampleRate());
|
||||||
|
|
||||||
|
float elapsedSeconds = elapsedMs / 1000.0f;
|
||||||
|
float duration = reader.getSamples().length / (float) reader.getSampleRate();
|
||||||
|
System.out.printf("Saved to %s\n", outFilename);
|
||||||
|
System.out.printf("Elapsed seconds: %.3f\n", elapsedSeconds);
|
||||||
|
System.out.printf("Audio duration in seconds: %.3f\n", duration);
|
||||||
|
System.out.printf(
|
||||||
|
"RTF: %.3f/%.3f = %.3f\n", elapsedSeconds, duration, elapsedSeconds / duration);
|
||||||
|
|
||||||
|
speechDenoiser.release();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,85 @@
|
|||||||
|
// Copyright 2026 Xiaomi Corporation
|
||||||
|
//
|
||||||
|
// This file shows how to use streaming VoxLock speech enhancement models in
|
||||||
|
// sherpa-onnx.
|
||||||
|
//
|
||||||
|
// VoxLock is a target-speaker voice isolation model. It keeps the voice of one
|
||||||
|
// speaker, e.g., the caller in a phone call, and removes the other voices.
|
||||||
|
//
|
||||||
|
// The model is from https://github.com/arbazsiddiqui/voxlock
|
||||||
|
//
|
||||||
|
// This example does not provide a voice sample, so VoxLock passes the audio
|
||||||
|
// through unchanged until it has heard 3 seconds of speech, takes that speech
|
||||||
|
// as the speaker to keep, and isolates from then on. See
|
||||||
|
// StreamingSpeechEnhancementVoxLockWithVoiceSample.java if you have a voice
|
||||||
|
// sample of the speaker.
|
||||||
|
//
|
||||||
|
// Download models and sample test waves from:
|
||||||
|
// https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models
|
||||||
|
|
||||||
|
import com.k2fsa.sherpa.onnx.*;
|
||||||
|
|
||||||
|
public class StreamingSpeechEnhancementVoxLock {
|
||||||
|
private static void appendSamples(java.util.ArrayList<Float> dst, float[] src) {
|
||||||
|
for (float v : src) {
|
||||||
|
dst.add(v);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static float[] toFloatArray(java.util.ArrayList<Float> src) {
|
||||||
|
float[] ans = new float[src.size()];
|
||||||
|
for (int i = 0; i != src.size(); ++i) {
|
||||||
|
ans[i] = src.get(i);
|
||||||
|
}
|
||||||
|
return ans;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void main(String[] args) {
|
||||||
|
OfflineSpeechDenoiserModelConfig modelConfig =
|
||||||
|
OfflineSpeechDenoiserModelConfig.builder()
|
||||||
|
.setNumThreads(1)
|
||||||
|
.setDebug(true)
|
||||||
|
.setProvider("cpu")
|
||||||
|
.setVoxlock(
|
||||||
|
OfflineSpeechDenoiserVoxLockModelConfig.builder()
|
||||||
|
.setEnrollModel("./sherpa-onnx-voxlock-int8/enroll.onnx")
|
||||||
|
.setChunkModel("./sherpa-onnx-voxlock-int8/chunk_int8.onnx")
|
||||||
|
.build())
|
||||||
|
.build();
|
||||||
|
|
||||||
|
OnlineSpeechDenoiserConfig config =
|
||||||
|
OnlineSpeechDenoiserConfig.builder().setModel(modelConfig).build();
|
||||||
|
|
||||||
|
OnlineSpeechDenoiser speechDenoiser = new OnlineSpeechDenoiser(config);
|
||||||
|
|
||||||
|
WaveReader reader = new WaveReader("./sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav");
|
||||||
|
int frameShift = speechDenoiser.getFrameShiftInSamples();
|
||||||
|
java.util.ArrayList<Float> output = new java.util.ArrayList<>();
|
||||||
|
|
||||||
|
float[] samples = reader.getSamples();
|
||||||
|
long begin = System.currentTimeMillis();
|
||||||
|
for (int start = 0; start < samples.length; start += frameShift) {
|
||||||
|
int end = Math.min(start + frameShift, samples.length);
|
||||||
|
float[] chunk = java.util.Arrays.copyOfRange(samples, start, end);
|
||||||
|
DenoisedAudio denoised = speechDenoiser.run(chunk, reader.getSampleRate());
|
||||||
|
appendSamples(output, denoised.getSamples());
|
||||||
|
}
|
||||||
|
|
||||||
|
DenoisedAudio denoised = speechDenoiser.flush();
|
||||||
|
appendSamples(output, denoised.getSamples());
|
||||||
|
long elapsedMs = System.currentTimeMillis() - begin;
|
||||||
|
|
||||||
|
String outFilename = "voxlock-enhanced-online-16k.wav";
|
||||||
|
WaveWriter.write(outFilename, toFloatArray(output), speechDenoiser.getSampleRate());
|
||||||
|
|
||||||
|
float elapsedSeconds = elapsedMs / 1000.0f;
|
||||||
|
float duration = samples.length / (float) reader.getSampleRate();
|
||||||
|
System.out.printf("Saved to %s\n", outFilename);
|
||||||
|
System.out.printf("Elapsed seconds: %.3f\n", elapsedSeconds);
|
||||||
|
System.out.printf("Audio duration in seconds: %.3f\n", duration);
|
||||||
|
System.out.printf(
|
||||||
|
"RTF: %.3f/%.3f = %.3f\n", elapsedSeconds, duration, elapsedSeconds / duration);
|
||||||
|
|
||||||
|
speechDenoiser.release();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,86 @@
|
|||||||
|
// Copyright 2026 Xiaomi Corporation
|
||||||
|
//
|
||||||
|
// This file shows how to use streaming VoxLock speech enhancement models in
|
||||||
|
// sherpa-onnx with a voice sample.
|
||||||
|
//
|
||||||
|
// VoxLock is a target-speaker voice isolation model. It keeps the voice of one
|
||||||
|
// speaker, e.g., the caller in a phone call, and removes the other voices.
|
||||||
|
//
|
||||||
|
// The model is from https://github.com/arbazsiddiqui/voxlock
|
||||||
|
//
|
||||||
|
// This example provides a voice sample of the speaker to keep, so VoxLock
|
||||||
|
// isolates that speaker from the first sample. The voice sample must have at
|
||||||
|
// least 3 seconds of the speaker's voice; from a longer file, VoxLock uses the
|
||||||
|
// loudest 3 seconds within its first 30 seconds. See
|
||||||
|
// StreamingSpeechEnhancementVoxLock.java if you don't have a voice sample.
|
||||||
|
//
|
||||||
|
// Download models and sample test waves from:
|
||||||
|
// https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models
|
||||||
|
|
||||||
|
import com.k2fsa.sherpa.onnx.*;
|
||||||
|
|
||||||
|
public class StreamingSpeechEnhancementVoxLockWithVoiceSample {
|
||||||
|
private static void appendSamples(java.util.ArrayList<Float> dst, float[] src) {
|
||||||
|
for (float v : src) {
|
||||||
|
dst.add(v);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static float[] toFloatArray(java.util.ArrayList<Float> src) {
|
||||||
|
float[] ans = new float[src.size()];
|
||||||
|
for (int i = 0; i != src.size(); ++i) {
|
||||||
|
ans[i] = src.get(i);
|
||||||
|
}
|
||||||
|
return ans;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void main(String[] args) {
|
||||||
|
OfflineSpeechDenoiserModelConfig modelConfig =
|
||||||
|
OfflineSpeechDenoiserModelConfig.builder()
|
||||||
|
.setNumThreads(1)
|
||||||
|
.setDebug(true)
|
||||||
|
.setProvider("cpu")
|
||||||
|
.setVoxlock(
|
||||||
|
OfflineSpeechDenoiserVoxLockModelConfig.builder()
|
||||||
|
.setEnrollModel("./sherpa-onnx-voxlock-int8/enroll.onnx")
|
||||||
|
.setChunkModel("./sherpa-onnx-voxlock-int8/chunk_int8.onnx")
|
||||||
|
.setVoiceSample("./sherpa-onnx-voxlock-int8/test_wavs/en-voice-sample-f.wav")
|
||||||
|
.build())
|
||||||
|
.build();
|
||||||
|
|
||||||
|
OnlineSpeechDenoiserConfig config =
|
||||||
|
OnlineSpeechDenoiserConfig.builder().setModel(modelConfig).build();
|
||||||
|
|
||||||
|
OnlineSpeechDenoiser speechDenoiser = new OnlineSpeechDenoiser(config);
|
||||||
|
|
||||||
|
WaveReader reader = new WaveReader("./sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav");
|
||||||
|
int frameShift = speechDenoiser.getFrameShiftInSamples();
|
||||||
|
java.util.ArrayList<Float> output = new java.util.ArrayList<>();
|
||||||
|
|
||||||
|
float[] samples = reader.getSamples();
|
||||||
|
long begin = System.currentTimeMillis();
|
||||||
|
for (int start = 0; start < samples.length; start += frameShift) {
|
||||||
|
int end = Math.min(start + frameShift, samples.length);
|
||||||
|
float[] chunk = java.util.Arrays.copyOfRange(samples, start, end);
|
||||||
|
DenoisedAudio denoised = speechDenoiser.run(chunk, reader.getSampleRate());
|
||||||
|
appendSamples(output, denoised.getSamples());
|
||||||
|
}
|
||||||
|
|
||||||
|
DenoisedAudio denoised = speechDenoiser.flush();
|
||||||
|
appendSamples(output, denoised.getSamples());
|
||||||
|
long elapsedMs = System.currentTimeMillis() - begin;
|
||||||
|
|
||||||
|
String outFilename = "voxlock-enhanced-online-with-voice-sample-16k.wav";
|
||||||
|
WaveWriter.write(outFilename, toFloatArray(output), speechDenoiser.getSampleRate());
|
||||||
|
|
||||||
|
float elapsedSeconds = elapsedMs / 1000.0f;
|
||||||
|
float duration = samples.length / (float) reader.getSampleRate();
|
||||||
|
System.out.printf("Saved to %s\n", outFilename);
|
||||||
|
System.out.printf("Elapsed seconds: %.3f\n", elapsedSeconds);
|
||||||
|
System.out.printf("Audio duration in seconds: %.3f\n", duration);
|
||||||
|
System.out.printf(
|
||||||
|
"RTF: %.3f/%.3f = %.3f\n", elapsedSeconds, duration, elapsedSeconds / duration);
|
||||||
|
|
||||||
|
speechDenoiser.release();
|
||||||
|
}
|
||||||
|
}
|
||||||
+20
@@ -0,0 +1,20 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
|
set -ex
|
||||||
|
|
||||||
|
source ./setup.sh
|
||||||
|
|
||||||
|
if [ ! -d ./sherpa-onnx-voxlock-int8 ]; then
|
||||||
|
curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/sherpa-onnx-voxlock-int8.tar.bz2
|
||||||
|
tar xvf sherpa-onnx-voxlock-int8.tar.bz2
|
||||||
|
rm sherpa-onnx-voxlock-int8.tar.bz2
|
||||||
|
fi
|
||||||
|
|
||||||
|
java \
|
||||||
|
-Dsherpa_onnx.native.path=$PWD/../build/lib \
|
||||||
|
-cp ../sherpa-onnx/java-api/target/sherpa-onnx-jvm-*.jar \
|
||||||
|
NonStreamingSpeechEnhancementVoxLockWithVoiceSample.java
|
||||||
|
|
||||||
|
mkdir -p NonStreamingSpeechEnhancementVoxLockWithVoiceSample
|
||||||
|
mv voxlock-enhanced-with-voice-sample-16k.wav NonStreamingSpeechEnhancementVoxLockWithVoiceSample/
|
||||||
|
cp sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav sherpa-onnx-voxlock-int8/test_wavs/en-voice-sample-f.wav NonStreamingSpeechEnhancementVoxLockWithVoiceSample/
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
|
set -ex
|
||||||
|
|
||||||
|
source ./setup.sh
|
||||||
|
|
||||||
|
if [ ! -d ./sherpa-onnx-voxlock-int8 ]; then
|
||||||
|
curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/sherpa-onnx-voxlock-int8.tar.bz2
|
||||||
|
tar xvf sherpa-onnx-voxlock-int8.tar.bz2
|
||||||
|
rm sherpa-onnx-voxlock-int8.tar.bz2
|
||||||
|
fi
|
||||||
|
|
||||||
|
java \
|
||||||
|
-Dsherpa_onnx.native.path=$PWD/../build/lib \
|
||||||
|
-cp ../sherpa-onnx/java-api/target/sherpa-onnx-jvm-*.jar \
|
||||||
|
NonStreamingSpeechEnhancementVoxLock.java
|
||||||
|
|
||||||
|
mkdir -p NonStreamingSpeechEnhancementVoxLock
|
||||||
|
mv voxlock-enhanced-16k.wav NonStreamingSpeechEnhancementVoxLock/
|
||||||
|
cp sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav NonStreamingSpeechEnhancementVoxLock/
|
||||||
+20
@@ -0,0 +1,20 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
|
set -ex
|
||||||
|
|
||||||
|
source ./setup.sh
|
||||||
|
|
||||||
|
if [ ! -d ./sherpa-onnx-voxlock-int8 ]; then
|
||||||
|
curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/sherpa-onnx-voxlock-int8.tar.bz2
|
||||||
|
tar xvf sherpa-onnx-voxlock-int8.tar.bz2
|
||||||
|
rm sherpa-onnx-voxlock-int8.tar.bz2
|
||||||
|
fi
|
||||||
|
|
||||||
|
java \
|
||||||
|
-Dsherpa_onnx.native.path=$PWD/../build/lib \
|
||||||
|
-cp ../sherpa-onnx/java-api/target/sherpa-onnx-jvm-*.jar \
|
||||||
|
StreamingSpeechEnhancementVoxLockWithVoiceSample.java
|
||||||
|
|
||||||
|
mkdir -p StreamingSpeechEnhancementVoxLockWithVoiceSample
|
||||||
|
mv voxlock-enhanced-online-with-voice-sample-16k.wav StreamingSpeechEnhancementVoxLockWithVoiceSample/
|
||||||
|
cp sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav sherpa-onnx-voxlock-int8/test_wavs/en-voice-sample-f.wav StreamingSpeechEnhancementVoxLockWithVoiceSample/
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
|
set -ex
|
||||||
|
|
||||||
|
source ./setup.sh
|
||||||
|
|
||||||
|
if [ ! -d ./sherpa-onnx-voxlock-int8 ]; then
|
||||||
|
curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/sherpa-onnx-voxlock-int8.tar.bz2
|
||||||
|
tar xvf sherpa-onnx-voxlock-int8.tar.bz2
|
||||||
|
rm sherpa-onnx-voxlock-int8.tar.bz2
|
||||||
|
fi
|
||||||
|
|
||||||
|
java \
|
||||||
|
-Dsherpa_onnx.native.path=$PWD/../build/lib \
|
||||||
|
-cp ../sherpa-onnx/java-api/target/sherpa-onnx-jvm-*.jar \
|
||||||
|
StreamingSpeechEnhancementVoxLock.java
|
||||||
|
|
||||||
|
mkdir -p StreamingSpeechEnhancementVoxLock
|
||||||
|
mv voxlock-enhanced-online-16k.wav StreamingSpeechEnhancementVoxLock/
|
||||||
|
cp sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav StreamingSpeechEnhancementVoxLock/
|
||||||
@@ -930,7 +930,113 @@ function testOfflineZipformerCtcHlg() {
|
|||||||
java -Djava.library.path=../build/lib -jar $out_filename
|
java -Djava.library.path=../build/lib -jar $out_filename
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function testOfflineSpeechDenoiserVoxLock() {
|
||||||
|
if [ ! -d ./sherpa-onnx-voxlock-int8 ]; then
|
||||||
|
curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/sherpa-onnx-voxlock-int8.tar.bz2
|
||||||
|
tar xvf sherpa-onnx-voxlock-int8.tar.bz2
|
||||||
|
rm sherpa-onnx-voxlock-int8.tar.bz2
|
||||||
|
fi
|
||||||
|
|
||||||
|
out_filename=test_offline_speech_denoiser_voxlock.jar
|
||||||
|
kotlinc-jvm -include-runtime -d $out_filename \
|
||||||
|
test_offline_speech_denoiser_voxlock.kt \
|
||||||
|
OfflineSpeechDenoiser.kt \
|
||||||
|
DenoisedAudio.kt \
|
||||||
|
WaveReader.kt \
|
||||||
|
faked-asset-manager.kt \
|
||||||
|
faked-log.kt
|
||||||
|
|
||||||
|
ls -lh $out_filename
|
||||||
|
|
||||||
|
java -Djava.library.path=../build/lib -jar $out_filename
|
||||||
|
|
||||||
|
mkdir -p test_offline_speech_denoiser_voxlock
|
||||||
|
mv voxlock-enhanced-16k.wav test_offline_speech_denoiser_voxlock/
|
||||||
|
cp sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav test_offline_speech_denoiser_voxlock/
|
||||||
|
}
|
||||||
|
|
||||||
|
function testOfflineSpeechDenoiserVoxLockWithVoiceSample() {
|
||||||
|
if [ ! -d ./sherpa-onnx-voxlock-int8 ]; then
|
||||||
|
curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/sherpa-onnx-voxlock-int8.tar.bz2
|
||||||
|
tar xvf sherpa-onnx-voxlock-int8.tar.bz2
|
||||||
|
rm sherpa-onnx-voxlock-int8.tar.bz2
|
||||||
|
fi
|
||||||
|
|
||||||
|
out_filename=test_offline_speech_denoiser_voxlock_with_voice_sample.jar
|
||||||
|
kotlinc-jvm -include-runtime -d $out_filename \
|
||||||
|
test_offline_speech_denoiser_voxlock_with_voice_sample.kt \
|
||||||
|
OfflineSpeechDenoiser.kt \
|
||||||
|
DenoisedAudio.kt \
|
||||||
|
WaveReader.kt \
|
||||||
|
faked-asset-manager.kt \
|
||||||
|
faked-log.kt
|
||||||
|
|
||||||
|
ls -lh $out_filename
|
||||||
|
|
||||||
|
java -Djava.library.path=../build/lib -jar $out_filename
|
||||||
|
|
||||||
|
mkdir -p test_offline_speech_denoiser_voxlock_with_voice_sample
|
||||||
|
mv voxlock-enhanced-with-voice-sample-16k.wav test_offline_speech_denoiser_voxlock_with_voice_sample/
|
||||||
|
cp sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav sherpa-onnx-voxlock-int8/test_wavs/en-voice-sample-f.wav test_offline_speech_denoiser_voxlock_with_voice_sample/
|
||||||
|
}
|
||||||
|
|
||||||
|
function testOnlineSpeechDenoiserVoxLock() {
|
||||||
|
if [ ! -d ./sherpa-onnx-voxlock-int8 ]; then
|
||||||
|
curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/sherpa-onnx-voxlock-int8.tar.bz2
|
||||||
|
tar xvf sherpa-onnx-voxlock-int8.tar.bz2
|
||||||
|
rm sherpa-onnx-voxlock-int8.tar.bz2
|
||||||
|
fi
|
||||||
|
|
||||||
|
out_filename=test_online_speech_denoiser_voxlock.jar
|
||||||
|
kotlinc-jvm -include-runtime -d $out_filename \
|
||||||
|
test_online_speech_denoiser_voxlock.kt \
|
||||||
|
OnlineSpeechDenoiser.kt \
|
||||||
|
OfflineSpeechDenoiser.kt \
|
||||||
|
DenoisedAudio.kt \
|
||||||
|
WaveReader.kt \
|
||||||
|
faked-asset-manager.kt \
|
||||||
|
faked-log.kt
|
||||||
|
|
||||||
|
ls -lh $out_filename
|
||||||
|
|
||||||
|
java -Djava.library.path=../build/lib -jar $out_filename
|
||||||
|
|
||||||
|
mkdir -p test_online_speech_denoiser_voxlock
|
||||||
|
mv voxlock-enhanced-online-16k.wav test_online_speech_denoiser_voxlock/
|
||||||
|
cp sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav test_online_speech_denoiser_voxlock/
|
||||||
|
}
|
||||||
|
|
||||||
|
function testOnlineSpeechDenoiserVoxLockWithVoiceSample() {
|
||||||
|
if [ ! -d ./sherpa-onnx-voxlock-int8 ]; then
|
||||||
|
curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/sherpa-onnx-voxlock-int8.tar.bz2
|
||||||
|
tar xvf sherpa-onnx-voxlock-int8.tar.bz2
|
||||||
|
rm sherpa-onnx-voxlock-int8.tar.bz2
|
||||||
|
fi
|
||||||
|
|
||||||
|
out_filename=test_online_speech_denoiser_voxlock_with_voice_sample.jar
|
||||||
|
kotlinc-jvm -include-runtime -d $out_filename \
|
||||||
|
test_online_speech_denoiser_voxlock_with_voice_sample.kt \
|
||||||
|
OnlineSpeechDenoiser.kt \
|
||||||
|
OfflineSpeechDenoiser.kt \
|
||||||
|
DenoisedAudio.kt \
|
||||||
|
WaveReader.kt \
|
||||||
|
faked-asset-manager.kt \
|
||||||
|
faked-log.kt
|
||||||
|
|
||||||
|
ls -lh $out_filename
|
||||||
|
|
||||||
|
java -Djava.library.path=../build/lib -jar $out_filename
|
||||||
|
|
||||||
|
mkdir -p test_online_speech_denoiser_voxlock_with_voice_sample
|
||||||
|
mv voxlock-enhanced-online-with-voice-sample-16k.wav test_online_speech_denoiser_voxlock_with_voice_sample/
|
||||||
|
cp sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav sherpa-onnx-voxlock-int8/test_wavs/en-voice-sample-f.wav test_online_speech_denoiser_voxlock_with_voice_sample/
|
||||||
|
}
|
||||||
|
|
||||||
testVersion
|
testVersion
|
||||||
|
testOfflineSpeechDenoiserVoxLock
|
||||||
|
testOfflineSpeechDenoiserVoxLockWithVoiceSample
|
||||||
|
testOnlineSpeechDenoiserVoxLock
|
||||||
|
testOnlineSpeechDenoiserVoxLockWithVoiceSample
|
||||||
testPocketTts
|
testPocketTts
|
||||||
testZipVoiceTtsLexicon
|
testZipVoiceTtsLexicon
|
||||||
testSupertonicTts
|
testSupertonicTts
|
||||||
|
|||||||
@@ -0,0 +1,60 @@
|
|||||||
|
package com.k2fsa.sherpa.onnx
|
||||||
|
|
||||||
|
// VoxLock is a target-speaker voice isolation model. It keeps the voice of one
|
||||||
|
// speaker, e.g., the caller in a phone call, and removes the other voices.
|
||||||
|
//
|
||||||
|
// The model is from https://github.com/arbazsiddiqui/voxlock
|
||||||
|
//
|
||||||
|
// This example does not provide a voice sample, so VoxLock takes the loudest
|
||||||
|
// 3 seconds in the first 15 seconds of the input as the speaker to keep. See
|
||||||
|
// test_offline_speech_denoiser_voxlock_with_voice_sample.kt if you have a
|
||||||
|
// voice sample of the speaker.
|
||||||
|
//
|
||||||
|
// Please download test files in this script from
|
||||||
|
// https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models
|
||||||
|
|
||||||
|
fun main() {
|
||||||
|
test()
|
||||||
|
}
|
||||||
|
|
||||||
|
fun test() {
|
||||||
|
val denoiser = createOfflineSpeechDenoiser()
|
||||||
|
|
||||||
|
val waveFilename = "./sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav"
|
||||||
|
val waveData = WaveReader.readWaveFromFile(filename = waveFilename)
|
||||||
|
|
||||||
|
val begin = System.currentTimeMillis()
|
||||||
|
val denoised = denoiser.run(waveData.samples, waveData.sampleRate)
|
||||||
|
val elapsedMs = System.currentTimeMillis() - begin
|
||||||
|
|
||||||
|
denoised.save(filename = "./voxlock-enhanced-16k.wav")
|
||||||
|
|
||||||
|
val elapsedSeconds = elapsedMs / 1000.0f
|
||||||
|
val duration = waveData.samples.size / waveData.sampleRate.toFloat()
|
||||||
|
println("Saved to ./voxlock-enhanced-16k.wav")
|
||||||
|
println("Elapsed seconds: " + "%.3f".format(elapsedSeconds))
|
||||||
|
println("Audio duration in seconds: " + "%.3f".format(duration))
|
||||||
|
println(
|
||||||
|
"RTF: " + "%.3f".format(elapsedSeconds) + "/" + "%.3f".format(duration) + " = " +
|
||||||
|
"%.3f".format(elapsedSeconds / duration)
|
||||||
|
)
|
||||||
|
|
||||||
|
denoiser.release()
|
||||||
|
}
|
||||||
|
|
||||||
|
fun createOfflineSpeechDenoiser(): OfflineSpeechDenoiser {
|
||||||
|
val config = OfflineSpeechDenoiserConfig(
|
||||||
|
model = OfflineSpeechDenoiserModelConfig(
|
||||||
|
voxlock = OfflineSpeechDenoiserVoxLockModelConfig(
|
||||||
|
enrollModel = "./sherpa-onnx-voxlock-int8/enroll.onnx",
|
||||||
|
chunkModel = "./sherpa-onnx-voxlock-int8/chunk_int8.onnx",
|
||||||
|
),
|
||||||
|
provider = "cpu",
|
||||||
|
numThreads = 1,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
println(config)
|
||||||
|
|
||||||
|
return OfflineSpeechDenoiser(config = config)
|
||||||
|
}
|
||||||
@@ -0,0 +1,62 @@
|
|||||||
|
package com.k2fsa.sherpa.onnx
|
||||||
|
|
||||||
|
// VoxLock is a target-speaker voice isolation model. It keeps the voice of one
|
||||||
|
// speaker, e.g., the caller in a phone call, and removes the other voices.
|
||||||
|
//
|
||||||
|
// The model is from https://github.com/arbazsiddiqui/voxlock
|
||||||
|
//
|
||||||
|
// This example provides a voice sample of the speaker to keep, so VoxLock
|
||||||
|
// isolates that speaker from the first sample. The voice sample must have at
|
||||||
|
// least 3 seconds of the speaker's voice; from a longer file, VoxLock uses the
|
||||||
|
// loudest 3 seconds within its first 30 seconds. See
|
||||||
|
// test_offline_speech_denoiser_voxlock.kt if you don't have a voice sample.
|
||||||
|
//
|
||||||
|
// Please download test files in this script from
|
||||||
|
// https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models
|
||||||
|
|
||||||
|
fun main() {
|
||||||
|
test()
|
||||||
|
}
|
||||||
|
|
||||||
|
fun test() {
|
||||||
|
val denoiser = createOfflineSpeechDenoiser()
|
||||||
|
|
||||||
|
val waveFilename = "./sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav"
|
||||||
|
val waveData = WaveReader.readWaveFromFile(filename = waveFilename)
|
||||||
|
|
||||||
|
val begin = System.currentTimeMillis()
|
||||||
|
val denoised = denoiser.run(waveData.samples, waveData.sampleRate)
|
||||||
|
val elapsedMs = System.currentTimeMillis() - begin
|
||||||
|
|
||||||
|
denoised.save(filename = "./voxlock-enhanced-with-voice-sample-16k.wav")
|
||||||
|
|
||||||
|
val elapsedSeconds = elapsedMs / 1000.0f
|
||||||
|
val duration = waveData.samples.size / waveData.sampleRate.toFloat()
|
||||||
|
println("Saved to ./voxlock-enhanced-with-voice-sample-16k.wav")
|
||||||
|
println("Elapsed seconds: " + "%.3f".format(elapsedSeconds))
|
||||||
|
println("Audio duration in seconds: " + "%.3f".format(duration))
|
||||||
|
println(
|
||||||
|
"RTF: " + "%.3f".format(elapsedSeconds) + "/" + "%.3f".format(duration) + " = " +
|
||||||
|
"%.3f".format(elapsedSeconds / duration)
|
||||||
|
)
|
||||||
|
|
||||||
|
denoiser.release()
|
||||||
|
}
|
||||||
|
|
||||||
|
fun createOfflineSpeechDenoiser(): OfflineSpeechDenoiser {
|
||||||
|
val config = OfflineSpeechDenoiserConfig(
|
||||||
|
model = OfflineSpeechDenoiserModelConfig(
|
||||||
|
voxlock = OfflineSpeechDenoiserVoxLockModelConfig(
|
||||||
|
enrollModel = "./sherpa-onnx-voxlock-int8/enroll.onnx",
|
||||||
|
chunkModel = "./sherpa-onnx-voxlock-int8/chunk_int8.onnx",
|
||||||
|
voiceSample = "./sherpa-onnx-voxlock-int8/test_wavs/en-voice-sample-f.wav",
|
||||||
|
),
|
||||||
|
provider = "cpu",
|
||||||
|
numThreads = 1,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
println(config)
|
||||||
|
|
||||||
|
return OfflineSpeechDenoiser(config = config)
|
||||||
|
}
|
||||||
@@ -0,0 +1,70 @@
|
|||||||
|
package com.k2fsa.sherpa.onnx
|
||||||
|
|
||||||
|
// VoxLock is a target-speaker voice isolation model. It keeps the voice of one
|
||||||
|
// speaker, e.g., the caller in a phone call, and removes the other voices.
|
||||||
|
//
|
||||||
|
// The model is from https://github.com/arbazsiddiqui/voxlock
|
||||||
|
//
|
||||||
|
// This example does not provide a voice sample, so VoxLock passes the audio
|
||||||
|
// through unchanged until it has heard 3 seconds of speech, takes that speech
|
||||||
|
// as the speaker to keep, and isolates from then on. See
|
||||||
|
// test_online_speech_denoiser_voxlock_with_voice_sample.kt if you have a voice
|
||||||
|
// sample of the speaker.
|
||||||
|
//
|
||||||
|
// Please download test files in this script from
|
||||||
|
// https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models
|
||||||
|
|
||||||
|
fun main() {
|
||||||
|
test()
|
||||||
|
}
|
||||||
|
|
||||||
|
fun test() {
|
||||||
|
val denoiser = createOnlineSpeechDenoiser()
|
||||||
|
val waveData = WaveReader.readWaveFromFile("./sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav")
|
||||||
|
val output = mutableListOf<Float>()
|
||||||
|
val frameShift = denoiser.frameShiftInSamples
|
||||||
|
|
||||||
|
val begin = System.currentTimeMillis()
|
||||||
|
var start = 0
|
||||||
|
while (start < waveData.samples.size) {
|
||||||
|
val end = minOf(start + frameShift, waveData.samples.size)
|
||||||
|
val chunk = waveData.samples.copyOfRange(start, end)
|
||||||
|
val denoised = denoiser.run(chunk, waveData.sampleRate)
|
||||||
|
output.addAll(denoised.samples.asList())
|
||||||
|
start = end
|
||||||
|
}
|
||||||
|
|
||||||
|
output.addAll(denoiser.flush().samples.asList())
|
||||||
|
val elapsedMs = System.currentTimeMillis() - begin
|
||||||
|
|
||||||
|
DenoisedAudio(output.toFloatArray(), denoiser.sampleRate).save(
|
||||||
|
filename = "./voxlock-enhanced-online-16k.wav"
|
||||||
|
)
|
||||||
|
|
||||||
|
val elapsedSeconds = elapsedMs / 1000.0f
|
||||||
|
val duration = waveData.samples.size / waveData.sampleRate.toFloat()
|
||||||
|
println("Saved to ./voxlock-enhanced-online-16k.wav")
|
||||||
|
println("Elapsed seconds: " + "%.3f".format(elapsedSeconds))
|
||||||
|
println("Audio duration in seconds: " + "%.3f".format(duration))
|
||||||
|
println(
|
||||||
|
"RTF: " + "%.3f".format(elapsedSeconds) + "/" + "%.3f".format(duration) + " = " +
|
||||||
|
"%.3f".format(elapsedSeconds / duration)
|
||||||
|
)
|
||||||
|
|
||||||
|
denoiser.release()
|
||||||
|
}
|
||||||
|
|
||||||
|
fun createOnlineSpeechDenoiser(): OnlineSpeechDenoiser {
|
||||||
|
val config = OnlineSpeechDenoiserConfig(
|
||||||
|
model = OfflineSpeechDenoiserModelConfig(
|
||||||
|
voxlock = OfflineSpeechDenoiserVoxLockModelConfig(
|
||||||
|
enrollModel = "./sherpa-onnx-voxlock-int8/enroll.onnx",
|
||||||
|
chunkModel = "./sherpa-onnx-voxlock-int8/chunk_int8.onnx",
|
||||||
|
),
|
||||||
|
provider = "cpu",
|
||||||
|
numThreads = 1,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
return OnlineSpeechDenoiser(config = config)
|
||||||
|
}
|
||||||
@@ -0,0 +1,71 @@
|
|||||||
|
package com.k2fsa.sherpa.onnx
|
||||||
|
|
||||||
|
// VoxLock is a target-speaker voice isolation model. It keeps the voice of one
|
||||||
|
// speaker, e.g., the caller in a phone call, and removes the other voices.
|
||||||
|
//
|
||||||
|
// The model is from https://github.com/arbazsiddiqui/voxlock
|
||||||
|
//
|
||||||
|
// This example provides a voice sample of the speaker to keep, so VoxLock
|
||||||
|
// isolates that speaker from the first sample. The voice sample must have at
|
||||||
|
// least 3 seconds of the speaker's voice; from a longer file, VoxLock uses the
|
||||||
|
// loudest 3 seconds within its first 30 seconds. See
|
||||||
|
// test_online_speech_denoiser_voxlock.kt if you don't have a voice sample.
|
||||||
|
//
|
||||||
|
// Please download test files in this script from
|
||||||
|
// https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models
|
||||||
|
|
||||||
|
fun main() {
|
||||||
|
test()
|
||||||
|
}
|
||||||
|
|
||||||
|
fun test() {
|
||||||
|
val denoiser = createOnlineSpeechDenoiser()
|
||||||
|
val waveData = WaveReader.readWaveFromFile("./sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav")
|
||||||
|
val output = mutableListOf<Float>()
|
||||||
|
val frameShift = denoiser.frameShiftInSamples
|
||||||
|
|
||||||
|
val begin = System.currentTimeMillis()
|
||||||
|
var start = 0
|
||||||
|
while (start < waveData.samples.size) {
|
||||||
|
val end = minOf(start + frameShift, waveData.samples.size)
|
||||||
|
val chunk = waveData.samples.copyOfRange(start, end)
|
||||||
|
val denoised = denoiser.run(chunk, waveData.sampleRate)
|
||||||
|
output.addAll(denoised.samples.asList())
|
||||||
|
start = end
|
||||||
|
}
|
||||||
|
|
||||||
|
output.addAll(denoiser.flush().samples.asList())
|
||||||
|
val elapsedMs = System.currentTimeMillis() - begin
|
||||||
|
|
||||||
|
DenoisedAudio(output.toFloatArray(), denoiser.sampleRate).save(
|
||||||
|
filename = "./voxlock-enhanced-online-with-voice-sample-16k.wav"
|
||||||
|
)
|
||||||
|
|
||||||
|
val elapsedSeconds = elapsedMs / 1000.0f
|
||||||
|
val duration = waveData.samples.size / waveData.sampleRate.toFloat()
|
||||||
|
println("Saved to ./voxlock-enhanced-online-with-voice-sample-16k.wav")
|
||||||
|
println("Elapsed seconds: " + "%.3f".format(elapsedSeconds))
|
||||||
|
println("Audio duration in seconds: " + "%.3f".format(duration))
|
||||||
|
println(
|
||||||
|
"RTF: " + "%.3f".format(elapsedSeconds) + "/" + "%.3f".format(duration) + " = " +
|
||||||
|
"%.3f".format(elapsedSeconds / duration)
|
||||||
|
)
|
||||||
|
|
||||||
|
denoiser.release()
|
||||||
|
}
|
||||||
|
|
||||||
|
fun createOnlineSpeechDenoiser(): OnlineSpeechDenoiser {
|
||||||
|
val config = OnlineSpeechDenoiserConfig(
|
||||||
|
model = OfflineSpeechDenoiserModelConfig(
|
||||||
|
voxlock = OfflineSpeechDenoiserVoxLockModelConfig(
|
||||||
|
enrollModel = "./sherpa-onnx-voxlock-int8/enroll.onnx",
|
||||||
|
chunkModel = "./sherpa-onnx-voxlock-int8/chunk_int8.onnx",
|
||||||
|
voiceSample = "./sherpa-onnx-voxlock-int8/test_wavs/en-voice-sample-f.wav",
|
||||||
|
),
|
||||||
|
provider = "cpu",
|
||||||
|
numThreads = 1,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
return OnlineSpeechDenoiser(config = config)
|
||||||
|
}
|
||||||
+8
@@ -4,6 +4,7 @@ package com.k2fsa.sherpa.onnx;
|
|||||||
public class OfflineSpeechDenoiserModelConfig {
|
public class OfflineSpeechDenoiserModelConfig {
|
||||||
private final OfflineSpeechDenoiserGtcrnModelConfig gtcrn;
|
private final OfflineSpeechDenoiserGtcrnModelConfig gtcrn;
|
||||||
private final OfflineSpeechDenoiserDpdfNetModelConfig dpdfnet;
|
private final OfflineSpeechDenoiserDpdfNetModelConfig dpdfnet;
|
||||||
|
private final OfflineSpeechDenoiserVoxLockModelConfig voxlock;
|
||||||
private final int numThreads;
|
private final int numThreads;
|
||||||
private final boolean debug;
|
private final boolean debug;
|
||||||
private final String provider;
|
private final String provider;
|
||||||
@@ -11,6 +12,7 @@ public class OfflineSpeechDenoiserModelConfig {
|
|||||||
private OfflineSpeechDenoiserModelConfig(Builder builder) {
|
private OfflineSpeechDenoiserModelConfig(Builder builder) {
|
||||||
this.gtcrn = builder.gtcrn;
|
this.gtcrn = builder.gtcrn;
|
||||||
this.dpdfnet = builder.dpdfnet;
|
this.dpdfnet = builder.dpdfnet;
|
||||||
|
this.voxlock = builder.voxlock;
|
||||||
this.numThreads = builder.numThreads;
|
this.numThreads = builder.numThreads;
|
||||||
this.debug = builder.debug;
|
this.debug = builder.debug;
|
||||||
this.provider = builder.provider;
|
this.provider = builder.provider;
|
||||||
@@ -23,6 +25,7 @@ public class OfflineSpeechDenoiserModelConfig {
|
|||||||
public static class Builder {
|
public static class Builder {
|
||||||
private OfflineSpeechDenoiserGtcrnModelConfig gtcrn = OfflineSpeechDenoiserGtcrnModelConfig.builder().build();
|
private OfflineSpeechDenoiserGtcrnModelConfig gtcrn = OfflineSpeechDenoiserGtcrnModelConfig.builder().build();
|
||||||
private OfflineSpeechDenoiserDpdfNetModelConfig dpdfnet = OfflineSpeechDenoiserDpdfNetModelConfig.builder().build();
|
private OfflineSpeechDenoiserDpdfNetModelConfig dpdfnet = OfflineSpeechDenoiserDpdfNetModelConfig.builder().build();
|
||||||
|
private OfflineSpeechDenoiserVoxLockModelConfig voxlock = OfflineSpeechDenoiserVoxLockModelConfig.builder().build();
|
||||||
private int numThreads = 1;
|
private int numThreads = 1;
|
||||||
private boolean debug = true;
|
private boolean debug = true;
|
||||||
private String provider = "cpu";
|
private String provider = "cpu";
|
||||||
@@ -41,6 +44,11 @@ public class OfflineSpeechDenoiserModelConfig {
|
|||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public Builder setVoxlock(OfflineSpeechDenoiserVoxLockModelConfig voxlock) {
|
||||||
|
this.voxlock = voxlock;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
public Builder setNumThreads(int numThreads) {
|
public Builder setNumThreads(int numThreads) {
|
||||||
this.numThreads = numThreads;
|
this.numThreads = numThreads;
|
||||||
return this;
|
return this;
|
||||||
|
|||||||
+80
@@ -0,0 +1,80 @@
|
|||||||
|
// Copyright 2026 Xiaomi Corporation
|
||||||
|
package com.k2fsa.sherpa.onnx;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* VoxLock target-speaker voice isolation model config.
|
||||||
|
*
|
||||||
|
* <p>VoxLock keeps the voice of one speaker (the caller) and removes the other voices. The model is
|
||||||
|
* from https://github.com/arbazsiddiqui/voxlock
|
||||||
|
*/
|
||||||
|
public class OfflineSpeechDenoiserVoxLockModelConfig {
|
||||||
|
private final String enrollModel;
|
||||||
|
private final String chunkModel;
|
||||||
|
private final String voiceSample;
|
||||||
|
|
||||||
|
private OfflineSpeechDenoiserVoxLockModelConfig(Builder builder) {
|
||||||
|
this.enrollModel = builder.enrollModel;
|
||||||
|
this.chunkModel = builder.chunkModel;
|
||||||
|
this.voiceSample = builder.voiceSample;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Builder builder() {
|
||||||
|
return new Builder();
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getEnrollModel() {
|
||||||
|
return enrollModel;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getChunkModel() {
|
||||||
|
return chunkModel;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getVoiceSample() {
|
||||||
|
return voiceSample;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static class Builder {
|
||||||
|
private String enrollModel = "";
|
||||||
|
private String chunkModel = "";
|
||||||
|
private String voiceSample = "";
|
||||||
|
|
||||||
|
public OfflineSpeechDenoiserVoxLockModelConfig build() {
|
||||||
|
return new OfflineSpeechDenoiserVoxLockModelConfig(this);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Path to the VoxLock enroll model, e.g., enroll.onnx. It runs once on a voice sample of the
|
||||||
|
* speaker to keep.
|
||||||
|
*/
|
||||||
|
public Builder setEnrollModel(String enrollModel) {
|
||||||
|
this.enrollModel = enrollModel;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Path to the VoxLock chunk model, e.g., chunk.onnx or chunk_int8.onnx. It runs on each chunk
|
||||||
|
* of the audio.
|
||||||
|
*/
|
||||||
|
public Builder setChunkModel(String chunkModel) {
|
||||||
|
this.chunkModel = chunkModel;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Optional. Path to a wave file with at least 3 seconds of the caller's voice. VoxLock keeps
|
||||||
|
* this voice and removes the others. From a longer file, it uses the loudest 3 seconds within
|
||||||
|
* the first 30 seconds. A file that is shorter or cannot be read is ignored, as if it were
|
||||||
|
* empty.
|
||||||
|
*
|
||||||
|
* <p>If empty or ignored, the voice sample is taken from the input audio: the offline denoiser
|
||||||
|
* uses the loudest 3 seconds in the first 15 seconds of the input, and the online denoiser
|
||||||
|
* passes the audio through unchanged until 3 seconds of speech have been heard, then uses that
|
||||||
|
* speech.
|
||||||
|
*/
|
||||||
|
public Builder setVoiceSample(String voiceSample) {
|
||||||
|
this.voiceSample = voiceSample;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -33,6 +33,19 @@ OfflineSpeechDenoiserModelConfig GetOfflineSpeechDenoiserModelConfig(
|
|||||||
SHERPA_ONNX_JNI_READ_FLOAT(ans.dpdfnet.attenuation_limit_db,
|
SHERPA_ONNX_JNI_READ_FLOAT(ans.dpdfnet.attenuation_limit_db,
|
||||||
attenuationLimitDb, dpdfnet_cls, dpdfnet);
|
attenuationLimitDb, dpdfnet_cls, dpdfnet);
|
||||||
|
|
||||||
|
fid = env->GetFieldID(
|
||||||
|
model_config_cls, "voxlock",
|
||||||
|
"Lcom/k2fsa/sherpa/onnx/OfflineSpeechDenoiserVoxLockModelConfig;");
|
||||||
|
jobject voxlock = env->GetObjectField(model, fid);
|
||||||
|
jclass voxlock_cls = env->GetObjectClass(voxlock);
|
||||||
|
|
||||||
|
SHERPA_ONNX_JNI_READ_STRING(ans.voxlock.enroll_model, enrollModel, voxlock_cls,
|
||||||
|
voxlock);
|
||||||
|
SHERPA_ONNX_JNI_READ_STRING(ans.voxlock.chunk_model, chunkModel, voxlock_cls,
|
||||||
|
voxlock);
|
||||||
|
SHERPA_ONNX_JNI_READ_STRING(ans.voxlock.voice_sample, voiceSample, voxlock_cls,
|
||||||
|
voxlock);
|
||||||
|
|
||||||
SHERPA_ONNX_JNI_READ_INT(ans.num_threads, numThreads, model_config_cls,
|
SHERPA_ONNX_JNI_READ_INT(ans.num_threads, numThreads, model_config_cls,
|
||||||
model);
|
model);
|
||||||
SHERPA_ONNX_JNI_READ_BOOL(ans.debug, debug, model_config_cls, model);
|
SHERPA_ONNX_JNI_READ_BOOL(ans.debug, debug, model_config_cls, model);
|
||||||
|
|||||||
@@ -11,9 +11,27 @@ data class OfflineSpeechDenoiserDpdfNetModelConfig(
|
|||||||
var attenuationLimitDb: Float = 0.0f,
|
var attenuationLimitDb: Float = 0.0f,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// VoxLock keeps the voice of one speaker (the caller) and removes the other
|
||||||
|
// voices. The model is from https://github.com/arbazsiddiqui/voxlock
|
||||||
|
data class OfflineSpeechDenoiserVoxLockModelConfig(
|
||||||
|
var enrollModel: String = "",
|
||||||
|
var chunkModel: String = "",
|
||||||
|
// Optional. A wave file with at least 3 seconds of the caller's voice.
|
||||||
|
// From a longer file, the loudest 3 seconds within its first 30 seconds
|
||||||
|
// are used. A file that is shorter or cannot be read is ignored, as if it
|
||||||
|
// were empty. If empty or ignored, the voice sample is taken from the
|
||||||
|
// input audio: the offline denoiser uses the loudest 3 seconds in the
|
||||||
|
// first 15 seconds of the input, and the online denoiser passes audio
|
||||||
|
// through unchanged until 3 seconds of speech have been heard, then uses
|
||||||
|
// that speech.
|
||||||
|
var voiceSample: String = "",
|
||||||
|
)
|
||||||
|
|
||||||
data class OfflineSpeechDenoiserModelConfig(
|
data class OfflineSpeechDenoiserModelConfig(
|
||||||
var gtcrn: OfflineSpeechDenoiserGtcrnModelConfig = OfflineSpeechDenoiserGtcrnModelConfig(),
|
var gtcrn: OfflineSpeechDenoiserGtcrnModelConfig = OfflineSpeechDenoiserGtcrnModelConfig(),
|
||||||
var dpdfnet: OfflineSpeechDenoiserDpdfNetModelConfig = OfflineSpeechDenoiserDpdfNetModelConfig(),
|
var dpdfnet: OfflineSpeechDenoiserDpdfNetModelConfig = OfflineSpeechDenoiserDpdfNetModelConfig(),
|
||||||
|
var voxlock: OfflineSpeechDenoiserVoxLockModelConfig =
|
||||||
|
OfflineSpeechDenoiserVoxLockModelConfig(),
|
||||||
var numThreads: Int = 1,
|
var numThreads: Int = 1,
|
||||||
var debug: Boolean = false,
|
var debug: Boolean = false,
|
||||||
var provider: String = "cpu",
|
var provider: String = "cpu",
|
||||||
|
|||||||
Reference in new issue
Block a user