From 2fc8934b5f6f15cff18b2bcbb79a0b3f1dcc71ef Mon Sep 17 00:00:00 2001 From: Jacob Mink <152457076+jacob-vincent-mink@users.noreply.github.com> Date: Tue, 8 Sep 2026 03:46:39 -0500 Subject: [PATCH] Add Intel NPU support through OpenVINO (#3910) --- CHANGELOG.md | 1 + OPENVINO.md | 114 ++++++++++++++++++ README.md | 8 +- .../csrc/online-conformer-transducer-model.cc | 23 ++-- .../csrc/online-conformer-transducer-model.h | 4 +- .../csrc/online-lstm-transducer-model.cc | 23 ++-- .../csrc/online-lstm-transducer-model.h | 4 +- .../csrc/online-zipformer-transducer-model.cc | 23 ++-- .../csrc/online-zipformer-transducer-model.h | 4 +- sherpa-onnx/csrc/provider-config.cc | 3 +- sherpa-onnx/csrc/provider.cc | 2 + sherpa-onnx/csrc/provider.h | 1 + sherpa-onnx/csrc/session-test.cc | 14 +++ sherpa-onnx/csrc/session.cc | 76 +++++++++++- .../csrc/sherpa-onnx-keyword-spotter.cc | 2 +- sherpa-onnx/csrc/vad-model-config.cc | 2 +- 16 files changed, 268 insertions(+), 36 deletions(-) create mode 100644 OPENVINO.md diff --git a/CHANGELOG.md b/CHANGELOG.md index cb43863a1..d4d1ffe88 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ## 1.13.7 +* Add Intel NPU support through the OpenVINO Execution Provider * Add iOS support for Rust and add hello world tauri example (#3901) * Remove caret prefix from sherpa_onnx flutter dependencies (#3905) * Fix Flutter packages for iOS (#3887) diff --git a/OPENVINO.md b/OPENVINO.md new file mode 100644 index 000000000..81078cd05 --- /dev/null +++ b/OPENVINO.md @@ -0,0 +1,114 @@ +# Intel NPU support with OpenVINO + +Sherpa-onnx can run ONNX models on Intel NPUs through ONNX Runtime's OpenVINO +Execution Provider. This requires ONNX Runtime 1.17 or newer built with the +OpenVINO provider; the standard ONNX Runtime package bundled by sherpa-onnx +does not include it. + +A standalone OpenVINO installation is not sufficient by itself: the ONNX +Runtime library must also be built with the OpenVINO Execution Provider, and +that build must be compatible with the installed OpenVINO runtime. + +## Build + +Install an OpenVINO-enabled ONNX Runtime and initialize the OpenVINO runtime +environment. Then point sherpa-onnx at its C/C++ headers and libraries: + +```bash +source /path/to/openvino/setupvars.sh +export SHERPA_ONNXRUNTIME_INCLUDE_DIR=/path/to/onnxruntime/include +export SHERPA_ONNXRUNTIME_LIB_DIR=/path/to/onnxruntime/lib + +cmake -S . -B build-openvino \ + -DBUILD_SHARED_LIBS=ON \ + -DSHERPA_ONNX_USE_PRE_INSTALLED_ONNXRUNTIME_IF_AVAILABLE=ON +cmake --build build-openvino --parallel +``` + +`OpenVINOExecutionProvider` should appear in the available-provider list when +the OpenVINO-enabled ONNX Runtime and its dependencies are discoverable. + +## Use the Intel NPU for wake-word detection + +Set the existing sherpa-onnx provider option to `openvino`. The default +OpenVINO device is `NPU`: + +```bash +build-openvino/bin/sherpa-onnx-keyword-spotter \ + --provider=openvino \ + --tokens=/path/to/tokens.txt \ + --encoder=/path/to/encoder.onnx \ + --decoder=/path/to/decoder.onnx \ + --joiner=/path/to/joiner.onnx \ + --keywords-file=/path/to/keywords.txt \ + /path/to/audio.wav +``` + +For transducer keyword-spotting models, sherpa-onnx runs the compute-heavy +encoder on OpenVINO and keeps the small decoder and joiner on CPU. This avoids +unnecessary NPU dispatch overhead and preserves decoder accuracy. The Silero +VAD model tested for this integration ran entirely on OpenVINO. Other VAD +models may be partitioned by ONNX Runtime, with unsupported nodes assigned to +the CPU provider. + +For a microphone, use `sherpa-onnx-keyword-spotter-microphone` (PortAudio) or +`sherpa-onnx-keyword-spotter-alsa` with the same `--provider` value. + +## Use the Intel NPU for VAD + +VAD uses the `--vad-provider` option: + +```bash +build-openvino/bin/sherpa-onnx-vad \ + --vad-provider=openvino \ + --silero-vad-model=/path/to/silero_vad.onnx \ + /path/to/input.wav \ + /path/to/speech-only.wav +``` + +The microphone VAD binaries accept the same `--vad-provider` value. + +The same provider string works through the C++, C, Python, Java, Kotlin, Dart, +Go, Rust, Swift, C#, and Node.js APIs. For example, Python model configs accept +`provider="openvino"`. + +If OpenVINO is unavailable or cannot initialize, sherpa-onnx logs the reason +and falls back to the ONNX Runtime CPU provider. + +## Provider options + +OpenVINO options use sherpa-onnx's existing `provider:config-file` syntax. The +file contains one `key=value` entry per line: + +```ini +# openvino-npu.config +device_type=NPU +enable_qdq_optimizer=True +``` + +Pass it by appending the path to the provider name. Use `--provider` for +keyword spotting and `--vad-provider` for VAD: + +```bash +--provider=openvino:openvino-npu.config +--vad-provider=openvino:openvino-npu.config +``` + +Any OpenVINO V2 provider option can be used. `device_type` may also be `CPU`, +`GPU`, or `AUTO`. Multi-device modes require at least two devices; for example, +use `HETERO:GPU,CPU` or `MULTI:GPU,CPU`. Bare `HETERO` and `MULTI` values are +incomplete device selections. `AUTO:GPU,NPU,CPU` can be used to give automatic +selection an explicit device priority. If `GraphOptimizationLevel` is omitted, +sherpa-onnx disables ONNX Runtime graph optimizations as recommended by the +OpenVINO Execution Provider. Set it explicitly in the config file to override +that behavior. + +OpenVINO's NPU support depends on the model's operators and shapes. Dynamic +speech-model inputs may need `disable_dynamic_shapes` and `reshape_input` +bounds tailored to the selected model; unsupported work can execute on the CPU +fallback provider. + +See the [ONNX Runtime OpenVINO Execution Provider documentation][ort-openvino] +for installation packages, supported devices, options, and compatibility. + +[ort-openvino]: https://onnxruntime.ai/docs/execution-providers/OpenVINO-ExecutionProvider.html diff --git a/README.md b/README.md index e4e7ac507..30b416c04 100644 --- a/README.md +++ b/README.md @@ -85,9 +85,11 @@ It also supports WebAssembly. |-------------------------------------|-----------------------------------|-----------------------------| | ✔️ | ✔️ | ✔️ | -| [4. Axera NPU][axera-npu] | -|---------------------------| -| ✔️ | +| [4. Axera NPU][axera-npu] | [5. Intel NPU (OpenVINO)][openvino-npu] | +|---------------------------|------------------------------------------| +| ✔️ | ✔️ | + +[openvino-npu]: OPENVINO.md [Join our discord](https://discord.gg/fJdxzg2VbG) diff --git a/sherpa-onnx/csrc/online-conformer-transducer-model.cc b/sherpa-onnx/csrc/online-conformer-transducer-model.cc index deedb98a3..db4800069 100644 --- a/sherpa-onnx/csrc/online-conformer-transducer-model.cc +++ b/sherpa-onnx/csrc/online-conformer-transducer-model.cc @@ -37,18 +37,23 @@ OnlineConformerTransducerModel::OnlineConformerTransducerModel( const OnlineModelConfig &config) : env_(ORT_LOGGING_LEVEL_ERROR), config_(config), - sess_opts_(GetSessionOptions(config)), + encoder_sess_opts_(GetSessionOptions(config)), + decoder_sess_opts_(GetSessionOptions(config, "decoder")), + joiner_sess_opts_(GetSessionOptions(config, "joiner")), allocator_{} { encoder_sess_ = std::make_unique( - env_, SHERPA_ONNX_TO_ORT_PATH(config.transducer.encoder), sess_opts_); + env_, SHERPA_ONNX_TO_ORT_PATH(config.transducer.encoder), + encoder_sess_opts_); InitEncoder(nullptr, 0); decoder_sess_ = std::make_unique( - env_, SHERPA_ONNX_TO_ORT_PATH(config.transducer.decoder), sess_opts_); + env_, SHERPA_ONNX_TO_ORT_PATH(config.transducer.decoder), + decoder_sess_opts_); InitDecoder(nullptr, 0); joiner_sess_ = std::make_unique( - env_, SHERPA_ONNX_TO_ORT_PATH(config.transducer.joiner), sess_opts_); + env_, SHERPA_ONNX_TO_ORT_PATH(config.transducer.joiner), + joiner_sess_opts_); InitJoiner(nullptr, 0); } @@ -57,7 +62,9 @@ OnlineConformerTransducerModel::OnlineConformerTransducerModel( Manager *mgr, const OnlineModelConfig &config) : env_(ORT_LOGGING_LEVEL_ERROR), config_(config), - sess_opts_(GetSessionOptions(config)), + encoder_sess_opts_(GetSessionOptions(config)), + decoder_sess_opts_(GetSessionOptions(config, "decoder")), + joiner_sess_opts_(GetSessionOptions(config, "joiner")), allocator_{} { { auto buf = ReadFile(mgr, config.transducer.encoder); @@ -79,7 +86,7 @@ void OnlineConformerTransducerModel::InitEncoder(void *model_data, size_t model_data_length) { if (model_data) { encoder_sess_ = std::make_unique( - env_, model_data, model_data_length, sess_opts_); + env_, model_data, model_data_length, encoder_sess_opts_); } else if (!encoder_sess_) { SHERPA_ONNX_LOGE( "Please pass model data or initialize the encoder outside of " @@ -120,7 +127,7 @@ void OnlineConformerTransducerModel::InitDecoder(void *model_data, size_t model_data_length) { if (model_data) { decoder_sess_ = std::make_unique( - env_, model_data, model_data_length, sess_opts_); + env_, model_data, model_data_length, decoder_sess_opts_); } else if (!decoder_sess_) { SHERPA_ONNX_LOGE( "Please pass model data or initialize the decoder outside of " @@ -156,7 +163,7 @@ void OnlineConformerTransducerModel::InitJoiner(void *model_data, size_t model_data_length) { if (model_data) { joiner_sess_ = std::make_unique( - env_, model_data, model_data_length, sess_opts_); + env_, model_data, model_data_length, joiner_sess_opts_); } else if (!joiner_sess_) { SHERPA_ONNX_LOGE( "Please pass model data or initialize the joiner outside of " diff --git a/sherpa-onnx/csrc/online-conformer-transducer-model.h b/sherpa-onnx/csrc/online-conformer-transducer-model.h index b03ada0d5..a0a08811b 100644 --- a/sherpa-onnx/csrc/online-conformer-transducer-model.h +++ b/sherpa-onnx/csrc/online-conformer-transducer-model.h @@ -55,7 +55,9 @@ class OnlineConformerTransducerModel : public OnlineTransducerModel { private: Ort::Env env_; - Ort::SessionOptions sess_opts_; + Ort::SessionOptions encoder_sess_opts_; + Ort::SessionOptions decoder_sess_opts_; + Ort::SessionOptions joiner_sess_opts_; Ort::AllocatorWithDefaultOptions allocator_; std::unique_ptr encoder_sess_; diff --git a/sherpa-onnx/csrc/online-lstm-transducer-model.cc b/sherpa-onnx/csrc/online-lstm-transducer-model.cc index 382dbd8d3..02d179f1a 100644 --- a/sherpa-onnx/csrc/online-lstm-transducer-model.cc +++ b/sherpa-onnx/csrc/online-lstm-transducer-model.cc @@ -36,18 +36,23 @@ OnlineLstmTransducerModel::OnlineLstmTransducerModel( const OnlineModelConfig &config) : env_(ORT_LOGGING_LEVEL_ERROR), config_(config), - sess_opts_(GetSessionOptions(config)), + encoder_sess_opts_(GetSessionOptions(config)), + decoder_sess_opts_(GetSessionOptions(config, "decoder")), + joiner_sess_opts_(GetSessionOptions(config, "joiner")), allocator_{} { encoder_sess_ = std::make_unique( - env_, SHERPA_ONNX_TO_ORT_PATH(config.transducer.encoder), sess_opts_); + env_, SHERPA_ONNX_TO_ORT_PATH(config.transducer.encoder), + encoder_sess_opts_); InitEncoder(nullptr, 0); decoder_sess_ = std::make_unique( - env_, SHERPA_ONNX_TO_ORT_PATH(config.transducer.decoder), sess_opts_); + env_, SHERPA_ONNX_TO_ORT_PATH(config.transducer.decoder), + decoder_sess_opts_); InitDecoder(nullptr, 0); joiner_sess_ = std::make_unique( - env_, SHERPA_ONNX_TO_ORT_PATH(config.transducer.joiner), sess_opts_); + env_, SHERPA_ONNX_TO_ORT_PATH(config.transducer.joiner), + joiner_sess_opts_); InitJoiner(nullptr, 0); } @@ -56,7 +61,9 @@ OnlineLstmTransducerModel::OnlineLstmTransducerModel( Manager *mgr, const OnlineModelConfig &config) : env_(ORT_LOGGING_LEVEL_ERROR), config_(config), - sess_opts_(GetSessionOptions(config)), + encoder_sess_opts_(GetSessionOptions(config)), + decoder_sess_opts_(GetSessionOptions(config, "decoder")), + joiner_sess_opts_(GetSessionOptions(config, "joiner")), allocator_{} { { auto buf = ReadFile(mgr, config.transducer.encoder); @@ -78,7 +85,7 @@ void OnlineLstmTransducerModel::InitEncoder(void *model_data, size_t model_data_length) { if (model_data) { encoder_sess_ = std::make_unique( - env_, model_data, model_data_length, sess_opts_); + env_, model_data, model_data_length, encoder_sess_opts_); } else if (!encoder_sess_) { SHERPA_ONNX_LOGE( "Please pass model data or initialize the encoder outside of " @@ -117,7 +124,7 @@ void OnlineLstmTransducerModel::InitDecoder(void *model_data, size_t model_data_length) { if (model_data) { decoder_sess_ = std::make_unique( - env_, model_data, model_data_length, sess_opts_); + env_, model_data, model_data_length, decoder_sess_opts_); } else if (!decoder_sess_) { SHERPA_ONNX_LOGE( "Please pass model data or initialize the decoder outside of " @@ -149,7 +156,7 @@ void OnlineLstmTransducerModel::InitJoiner(void *model_data, size_t model_data_length) { if (model_data) { joiner_sess_ = std::make_unique( - env_, model_data, model_data_length, sess_opts_); + env_, model_data, model_data_length, joiner_sess_opts_); } else if (!joiner_sess_) { SHERPA_ONNX_LOGE( "Please pass model data or initialize the joiner outside of " diff --git a/sherpa-onnx/csrc/online-lstm-transducer-model.h b/sherpa-onnx/csrc/online-lstm-transducer-model.h index 64ade36d0..e825781bb 100644 --- a/sherpa-onnx/csrc/online-lstm-transducer-model.h +++ b/sherpa-onnx/csrc/online-lstm-transducer-model.h @@ -54,7 +54,9 @@ class OnlineLstmTransducerModel : public OnlineTransducerModel { private: Ort::Env env_; - Ort::SessionOptions sess_opts_; + Ort::SessionOptions encoder_sess_opts_; + Ort::SessionOptions decoder_sess_opts_; + Ort::SessionOptions joiner_sess_opts_; Ort::AllocatorWithDefaultOptions allocator_; std::unique_ptr encoder_sess_; diff --git a/sherpa-onnx/csrc/online-zipformer-transducer-model.cc b/sherpa-onnx/csrc/online-zipformer-transducer-model.cc index a5a9d46c6..aa04bd271 100644 --- a/sherpa-onnx/csrc/online-zipformer-transducer-model.cc +++ b/sherpa-onnx/csrc/online-zipformer-transducer-model.cc @@ -37,18 +37,23 @@ OnlineZipformerTransducerModel::OnlineZipformerTransducerModel( const OnlineModelConfig &config) : env_(ORT_LOGGING_LEVEL_ERROR), config_(config), - sess_opts_(GetSessionOptions(config)), + encoder_sess_opts_(GetSessionOptions(config)), + decoder_sess_opts_(GetSessionOptions(config, "decoder")), + joiner_sess_opts_(GetSessionOptions(config, "joiner")), allocator_{} { encoder_sess_ = std::make_unique( - env_, SHERPA_ONNX_TO_ORT_PATH(config.transducer.encoder), sess_opts_); + env_, SHERPA_ONNX_TO_ORT_PATH(config.transducer.encoder), + encoder_sess_opts_); InitEncoder(nullptr, 0); decoder_sess_ = std::make_unique( - env_, SHERPA_ONNX_TO_ORT_PATH(config.transducer.decoder), sess_opts_); + env_, SHERPA_ONNX_TO_ORT_PATH(config.transducer.decoder), + decoder_sess_opts_); InitDecoder(nullptr, 0); joiner_sess_ = std::make_unique( - env_, SHERPA_ONNX_TO_ORT_PATH(config.transducer.joiner), sess_opts_); + env_, SHERPA_ONNX_TO_ORT_PATH(config.transducer.joiner), + joiner_sess_opts_); InitJoiner(nullptr, 0); } @@ -57,7 +62,9 @@ OnlineZipformerTransducerModel::OnlineZipformerTransducerModel( Manager *mgr, const OnlineModelConfig &config) : env_(ORT_LOGGING_LEVEL_ERROR), config_(config), - sess_opts_(GetSessionOptions(config)), + encoder_sess_opts_(GetSessionOptions(config)), + decoder_sess_opts_(GetSessionOptions(config, "decoder")), + joiner_sess_opts_(GetSessionOptions(config, "joiner")), allocator_{} { { auto buf = ReadFile(mgr, config.transducer.encoder); @@ -79,7 +86,7 @@ void OnlineZipformerTransducerModel::InitEncoder(void *model_data, size_t model_data_length) { if (model_data) { encoder_sess_ = std::make_unique( - env_, model_data, model_data_length, sess_opts_); + env_, model_data, model_data_length, encoder_sess_opts_); } else if (!encoder_sess_) { SHERPA_ONNX_LOGE( "Please pass model data or initialize the encoder outside of " @@ -148,7 +155,7 @@ void OnlineZipformerTransducerModel::InitDecoder(void *model_data, size_t model_data_length) { if (model_data) { decoder_sess_ = std::make_unique( - env_, model_data, model_data_length, sess_opts_); + env_, model_data, model_data_length, decoder_sess_opts_); } else if (!decoder_sess_) { SHERPA_ONNX_LOGE( "Please pass model data or initialize the decoder outside of " @@ -184,7 +191,7 @@ void OnlineZipformerTransducerModel::InitJoiner(void *model_data, size_t model_data_length) { if (model_data) { joiner_sess_ = std::make_unique( - env_, model_data, model_data_length, sess_opts_); + env_, model_data, model_data_length, joiner_sess_opts_); } else if (!joiner_sess_) { SHERPA_ONNX_LOGE( "Please pass model data or initialize the joiner outside of " diff --git a/sherpa-onnx/csrc/online-zipformer-transducer-model.h b/sherpa-onnx/csrc/online-zipformer-transducer-model.h index 9e4368a69..e01439b08 100644 --- a/sherpa-onnx/csrc/online-zipformer-transducer-model.h +++ b/sherpa-onnx/csrc/online-zipformer-transducer-model.h @@ -54,7 +54,9 @@ class OnlineZipformerTransducerModel : public OnlineTransducerModel { private: Ort::Env env_; - Ort::SessionOptions sess_opts_; + Ort::SessionOptions encoder_sess_opts_; + Ort::SessionOptions decoder_sess_opts_; + Ort::SessionOptions joiner_sess_opts_; Ort::AllocatorWithDefaultOptions allocator_; std::unique_ptr encoder_sess_; diff --git a/sherpa-onnx/csrc/provider-config.cc b/sherpa-onnx/csrc/provider-config.cc index 518b3a111..0dcdfbf43 100644 --- a/sherpa-onnx/csrc/provider-config.cc +++ b/sherpa-onnx/csrc/provider-config.cc @@ -109,7 +109,8 @@ void ProviderConfig::Register(ParseOptions *po) { po->Register("device", &device, "GPU device index for CUDA and Trt EP"); po->Register("provider", &provider, - "Specify a provider to use: cpu, cuda, coreml, trt, rknn, qnn"); + "Specify a provider to use: cpu, cuda, coreml, openvino, trt, " + "rknn, qnn"); } bool ProviderConfig::Validate() const { diff --git a/sherpa-onnx/csrc/provider.cc b/sherpa-onnx/csrc/provider.cc index 8f7fe3175..bc83dcee2 100644 --- a/sherpa-onnx/csrc/provider.cc +++ b/sherpa-onnx/csrc/provider.cc @@ -31,6 +31,8 @@ Provider StringToProvider(std::string s) { return Provider::kDirectML; } else if (s == "spacemit") { return Provider::kSpacemiT; + } else if (s == "openvino") { + return Provider::kOpenVINO; } else { SHERPA_ONNX_LOGE("Unsupported string: %s. Fallback to cpu", s.c_str()); return Provider::kCPU; diff --git a/sherpa-onnx/csrc/provider.h b/sherpa-onnx/csrc/provider.h index 8842c797b..0ad5be284 100644 --- a/sherpa-onnx/csrc/provider.h +++ b/sherpa-onnx/csrc/provider.h @@ -22,6 +22,7 @@ enum class Provider { kTRT = 5, // TensorRTExecutionProvider kDirectML = 6, // DmlExecutionProvider kSpacemiT = 7, // SpacemiTExecutionProvider + kOpenVINO = 8, // OpenVINOExecutionProvider }; /** diff --git a/sherpa-onnx/csrc/session-test.cc b/sherpa-onnx/csrc/session-test.cc index 747818a15..024d7e2ee 100644 --- a/sherpa-onnx/csrc/session-test.cc +++ b/sherpa-onnx/csrc/session-test.cc @@ -15,6 +15,7 @@ #endif #include "gtest/gtest.h" +#include "sherpa-onnx/csrc/provider.h" namespace sherpa_onnx { @@ -123,4 +124,17 @@ TEST(SessionConfigPassthrough, ForwardsPrefixedKeysAndIgnoresOthers) { EXPECT_FALSE(HasConfigEntry(sess_opts, "SessionConfig.")); } +TEST(Provider, ConvertsOpenVINOCaseInsensitively) { + EXPECT_EQ(StringToProvider("openvino"), Provider::kOpenVINO); + EXPECT_EQ(StringToProvider("OpenVINO"), Provider::kOpenVINO); +} + +TEST(Provider, OpenVINOSessionOptionsDoNotThrow) { + EXPECT_NO_THROW({ + Ort::Env env(ORT_LOGGING_LEVEL_WARNING, "openvino-session-test"); + auto sess_opts = GetSessionOptionsImpl(1, "openvino"); + (void)sess_opts; + }); +} + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/session.cc b/sherpa-onnx/csrc/session.cc index ee81d7528..e5c220c79 100755 --- a/sherpa-onnx/csrc/session.cc +++ b/sherpa-onnx/csrc/session.cc @@ -103,7 +103,8 @@ static void SplitProviderAndConfig( std::unordered_map &config) { // provider string format: provider_name[:config_path], e.g., // tensorrt:trt_config.config or spacemit:spacemit_config.config or cpu. The - // config file is optional. If it is not provided, default config will be used. + // config file is optional. If it is not provided, default config will be + // used. provider_str = Trim(s); config.clear(); @@ -157,7 +158,9 @@ Ort::SessionOptions GetSessionOptionsImpl( } // Other possible options - if (config.find("GraphOptimizationLevel") != config.end()) { + bool has_graph_optimization_level = + config.find("GraphOptimizationLevel") != config.end(); + if (has_graph_optimization_level) { int32_t graph_optimization_level = ToIntOrDefault( config["GraphOptimizationLevel"], static_cast(GraphOptimizationLevel::ORT_ENABLE_ALL)); @@ -176,7 +179,8 @@ Ort::SessionOptions GetSessionOptionsImpl( } if (config.find("ProfilingFilePrefix") != config.end()) { - sess_opts.EnableProfiling(SHERPA_ONNX_TO_ORT_PATH(config["ProfilingFilePrefix"])); + sess_opts.EnableProfiling( + SHERPA_ONNX_TO_ORT_PATH(config["ProfilingFilePrefix"])); config.erase("ProfilingFilePrefix"); } @@ -449,6 +453,61 @@ Ort::SessionOptions GetSessionOptionsImpl( SHERPA_ONNX_LOGE( "SpacemiT Execution Provider is for SpacemiT AI-CPUs only. Fallback " "to cpu!"); +#endif + break; + } + case Provider::kOpenVINO: { +#if ORT_API_VERSION >= 17 + if (std::find(available_providers.begin(), available_providers.end(), + "OpenVINOExecutionProvider") == available_providers.end()) { + SHERPA_ONNX_LOGE( + "OpenVINOExecutionProvider is not available. Install an " + "OpenVINO-enabled ONNX Runtime. Available providers: %s. " + "Fallback to cpu!", + os.str().c_str()); + break; + } + + std::unordered_map provider_options(config); + provider_options.erase("DEBUG"); + if (provider_options.find("device_type") == provider_options.end()) { + provider_options.emplace("device_type", "NPU"); + } + + std::vector option_keys; + std::vector option_values; + option_keys.reserve(provider_options.size()); + option_values.reserve(provider_options.size()); + for (const auto &kv : provider_options) { + option_keys.push_back(kv.first.c_str()); + option_values.push_back(kv.second.c_str()); + } + + const auto &api = Ort::GetApi(); + OrtStatus *status = api.SessionOptionsAppendExecutionProvider_OpenVINO_V2( + sess_opts, option_keys.data(), option_values.data(), + option_keys.size()); + if (status) { + const char *msg = api.GetErrorMessage(status); + SHERPA_ONNX_LOGE( + "Failed to enable OpenVINO Execution Provider: %s. Available " + "providers: %s. Fallback to cpu", + msg, os.str().c_str()); + api.ReleaseStatus(status); + } else if (!has_graph_optimization_level) { + // OpenVINO performs its own graph optimizations and recommends + // receiving the original ONNX graph. Only disable ORT optimizations + // after successfully enabling OpenVINO so that a CPU fallback keeps + // its normal optimization level. Users can override this in the + // config file. + sess_opts.SetGraphOptimizationLevel( + GraphOptimizationLevel::ORT_DISABLE_ALL); + } +#else + SHERPA_ONNX_LOGE( + "OpenVINO requires ONNX Runtime 1.17 or newer. Current API version: " + "%d. Fallback to cpu!", + static_cast(ORT_API_VERSION)); #endif break; } @@ -473,6 +532,17 @@ Ort::SessionOptions GetSessionOptions(const OnlineModelConfig &config, return GetSessionOptionsImpl(config.num_threads, "cuda", &config.provider_config); } + + // The transducer decoder and joiner are tiny, latency-sensitive graphs. + // Keep them on CPU for OpenVINO: the encoder contains nearly all of the + // compute and is the graph that benefits from NPU acceleration. + std::string provider_name = config.provider_config.provider.substr( + 0, config.provider_config.provider.find(':')); + if (ToLowerAscii(provider_name) == "openvino" && + (model_type == "decoder" || model_type == "joiner")) { + return GetSessionOptionsImpl(config.num_threads, "cpu", + &config.provider_config); + } return GetSessionOptionsImpl(config.num_threads, config.provider_config.provider, &config.provider_config); diff --git a/sherpa-onnx/csrc/sherpa-onnx-keyword-spotter.cc b/sherpa-onnx/csrc/sherpa-onnx-keyword-spotter.cc index 74d47eadb..d5da72140 100644 --- a/sherpa-onnx/csrc/sherpa-onnx-keyword-spotter.cc +++ b/sherpa-onnx/csrc/sherpa-onnx-keyword-spotter.cc @@ -42,7 +42,7 @@ Usage: Note: It supports decoding multiple files in batches Default value for num_threads is 2. -Valid values for provider: cpu (default), cuda, coreml. +Valid values for provider: cpu (default), cuda, coreml, openvino. foo.wav should be of single channel, 16-bit PCM encoded wave file; its sampling rate can be arbitrary and does not need to be 16kHz. diff --git a/sherpa-onnx/csrc/vad-model-config.cc b/sherpa-onnx/csrc/vad-model-config.cc index 4ca45b15c..001517cda 100644 --- a/sherpa-onnx/csrc/vad-model-config.cc +++ b/sherpa-onnx/csrc/vad-model-config.cc @@ -24,7 +24,7 @@ void VadModelConfig::Register(ParseOptions *po) { po->Register("vad-provider", &provider, "Specify a provider to run the VAD model. Supported values: " - "cpu, cuda, coreml"); + "cpu, cuda, coreml, openvino"); po->Register("vad-debug", &debug, "true to display debug information when loading vad models");