Add two Piper Chinese TTS models (#3546)

This pull request introduces support for Chinese Piper TTS models using G2PW pinyin instead of espeak-ng. Key changes include updating metadata generation to include a has_g2pw flag, adding scripts for lexicon generation, and modifying the C++ core to handle BOS/EOS tokens and rule-based FSTs.

.github/workflows/export-piper.yaml

@@ -43,8 +43,13 @@ jobs:
       - name: Install Python dependencies
         shell: bash
         run: |
-          python3 -m pip install --upgrade pip jinja2 iso639-lang onnx==1.17.0 onnxruntime==1.17.1 sherpa-onnx onnxmltools==1.13.0
-          python3 -m pip install "numpy<2" soundfile
+          python3 -m pip install --upgrade \
+            pip jinja2 iso639-lang \
+            onnx==1.17.0 onnxruntime==1.17.1 \
+            onnxmltools==1.13.0 \
+            "numpy<2" soundfile requests
+
+          python3 -m pip install --verbose sherpa_onnx_bin sherpa_onnx_core sherpa_onnx --no-index -f https://k2-fsa.github.io/sherpa/onnx/cpu.html
 
       - name: Generate script
         env:
@@ -176,8 +181,13 @@ jobs:
               vits-piper-ar_JO-SA_miro_V2-high-int8
               vits-piper-ar_JO-SA_miro_V2-high-fp16
               vits-piper-ar_JO-SA_miro_V2-high
-              vits-piper-ar_JO-SA_miro_V2-high
               vits-piper-sq_AL-edon-medium
+              vits-piper-zh_CN-xiao_ya-medium
+              vits-piper-zh_CN-xiao_ya-medium-int8
+              vits-piper-zh_CN-xiao_ya-medium-fp16
+              vits-piper-zh_CN-chaowen-medium
+              vits-piper-zh_CN-chaowen-medium-int8
+              vits-piper-zh_CN-chaowen-medium-fp16
             )
             for d in ${dirs[@]}; do
               src=scripts/piper/release/$d

scripts/apk/generate-tts-apk-script.py

@@ -277,10 +277,19 @@ def get_piper_models() -> List[TtsModel]:
         TtsModel(model_dir="vits-piper-vi_VN-vais1000-medium"),
         TtsModel(model_dir="vits-piper-vi_VN-vivos-x_low"),
         TtsModel(model_dir="vits-piper-zh_CN-huayan-medium"),
+        TtsModel(model_dir="vits-piper-zh_CN-xiao_ya-medium"),
+        TtsModel(model_dir="vits-piper-zh_CN-chaowen-medium"),
     ]
 
     for m in models:
-        m.data_dir = m.model_dir + "/" + "espeak-ng-data"
+        if "zh_CN" in m.model_dir and (
+            "xiao_ya" in m.model_dir or "chaowen" in m.model_dir
+        ):
+            m.lexicon = f"{m.model_dir}/lexicon.txt"
+            m.rule_fsts = f"{m.model_dir}/phone.fst,{m.model_dir}/date.fst,{m.model_dir}/number.fst"
+        else:
+            m.data_dir = m.model_dir + "/" + "espeak-ng-data"
+
         m.model_name = m.model_dir[len("vits-piper-") :] + ".onnx"
         m.lang = m.model_dir.split("-")[2][:2]
 

scripts/piper/add_meta_data.py

@@ -8,6 +8,11 @@ from typing import Any, Dict
 import onnx
 from iso639 import Lang
 
+# For the following model,
+# https://huggingface.co/rhasspy/piper-voices/blob/main/zh/zh_CN/xiao_ya/medium/zh_CN-xiao_ya-medium.onnx.json
+# it uses g2pw, not espeak-ng.
+# To handle that, we use has_g2pw = 1 in the meta_data
+
 
 def get_args():
     # For en_GB-semaine-medium
@@ -110,13 +115,22 @@ def main():
     else:
         voice = config["espeak"]["voice"]
 
+    has_g2pw = 0
+    has_espeak = 1
+
+    if config["phoneme_type"] == "pinyin" and voice == "zh":
+        has_espeak = 0
+        has_g2pw = 1
+
     print("add model metadata")
     meta_data = {
         "model_type": "vits",
         "comment": "piper",  # must be piper for models from piper
         "language": lang_iso.name,
         "voice": voice,  # e.g., en-us
-        "has_espeak": 1,
+        "version": 1,
+        "has_espeak": has_espeak,
+        "has_g2pw": has_g2pw,
         "n_speakers": config["num_speakers"],
         "sample_rate": sample_rate,
     }

scripts/piper/generate.py

@@ -1647,6 +1647,37 @@ def get_vi_models():
     return ans
 
 
+# Chinese
+def get_zh_models():
+    zh_CN = [
+        PiperModel(name="xiao_ya", kind="medium", sr=22050, ns=1),
+        PiperModel(name="chaowen", kind="medium", sr=22050, ns=1),
+    ]
+
+    for m in zh_CN:
+        m.lang = "zh_CN"
+
+    ans = zh_CN
+
+    for m in ans:
+        m.text = "某某银行的副行长和一些行政领导表示，他们去过长江和长白山; 经济不断增长。2024年12月31号，拨打110或者18920240511。123456块钱。当夜幕降临，星光点点，伴随着微风拂面，我在静谧中感受着时光的流转，思念如涟漪荡漾，梦境如画卷展开，我与自然融为一体，沉静在这片宁静的美丽之中，感受着生命的奇迹与温柔."
+
+        if m.model_name == "":
+            m.model_name = f"{m.lang}-{m.name}-{m.kind}.onnx"
+
+        code = m.lang[:2]
+        if m.cmd == "":
+            m.cmd = f"""
+            wget -qq https://huggingface.co/rhasspy/piper-voices/resolve/main/{code}/{m.lang}/{m.name}/{m.kind}/{m.model_name}
+            wget -qq https://huggingface.co/rhasspy/piper-voices/resolve/main/{code}/{m.lang}/{m.name}/{m.kind}/{m.model_name}.json
+            wget -qq https://huggingface.co/rhasspy/piper-voices/resolve/main/{code}/{m.lang}/{m.name}/{m.kind}/MODEL_CARD
+            """
+        if m.url == "":
+            m.url = f"https://huggingface.co/rhasspy/piper-voices/tree/main/{code}/{m.lang}/{m.name}/{m.kind}"
+
+    return ans
+
+
 # Indonesian
 def get_id_models():
     id_ID = [
@@ -2114,6 +2145,7 @@ def get_all_models():
     ans += get_tr_models()
     ans += get_uk_models()
     ans += get_vi_models()
+    ans += get_zh_models()
 
     for i, m in enumerate(ans):
         m.index = i
@@ -2182,8 +2214,21 @@ def main():
                 "model": f"{model_dir}/{m.model_name}",
                 "data_dir": f"{model_dir}/espeak-ng-data",
                 "tokens": f"{model_dir}/tokens.txt",
+                "lexicon": "",
                 "text": m.text,
+                "rule_fsts": "",
             }
+
+            if m.lang == "zh_CN" and m.name in ("xiao_ya", "chaowen"):
+                d = {
+                    "model": f"{model_dir}/{m.model_name}",
+                    "data_dir": "",
+                    "tokens": f"{model_dir}/tokens.txt",
+                    "lexicon": f"{model_dir}/lexicon.txt",
+                    "text": m.text,
+                    "rule_fsts": f"{model_dir}/phone.fst,{model_dir}/date.fst,{model_dir}/number.fst",
+                }
+
             for i in range(m.ns):
                 s = template.render(
                     **d,

scripts/piper/generate.sh.in

@@ -12,8 +12,15 @@ log() {
 }
 
 wget -qq https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/espeak-ng-data.tar.bz2
+wget -qq https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-zh-g2pw.txt
+
+wget -qq https://huggingface.co/csukuangfj/icefall-tts-aishell3-vits-low-2024-04-06/resolve/main/data/date.fst
+wget -qq https://huggingface.co/csukuangfj/icefall-tts-aishell3-vits-low-2024-04-06/resolve/main/data/number.fst
+wget -qq https://huggingface.co/csukuangfj/icefall-tts-aishell3-vits-low-2024-04-06/resolve/main/data/phone.fst
+
 tar xf espeak-ng-data.tar.bz2
 rm espeak-ng-data.tar.bz2
+ls -lh lexicon-zh-g2pw.txt
 
 mkdir -p release
 
@@ -50,7 +57,16 @@ mv -v README $dst/ || true
 mv -v README.md $dst/ || true
 mv -v LICENSE.txt $dst/ || true
 mv -v *.json  $dst/
-cp -a ./espeak-ng-data $dst/
+if [[ ! $model_name =~ zh_CN-(xiao_ya|chaowen)-medium ]]; then
+  cp -a ./espeak-ng-data $dst/
+else
+  cp lexicon-zh-g2pw.txt $dst/lexicon.txt
+  cp date.fst $dst/
+  cp number.fst $dst/
+  cp phone.fst $dst/
+fi
+
+ls -lh $dst/
 
 cp -a $dst $dst_int8
 cp -a $dst $dst_fp16
@@ -79,6 +95,7 @@ if [ -d hf ]; then
   mkdir -p hf/piper/mp3/$lang/vits-piper-$lang-$name-$kind
   for i in $(seq $num_speakers); do
     i=$((i-1))
+    cat ./generate_samples-$dst-$i.py
     python3 ./generate_samples-$dst-$i.py
   done
   ls -lh hf/piper/mp3/$lang/vits-piper-$lang-$name-$kind

scripts/piper/generate_lexicon_zh.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+# Copyright    2026  Xiaomi Corp.        (authors: Fangjun Kuang)
+
+# It may take 100 minutes to generate the lexicon.
+#
+# You can download a pre-generated one from
+# https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models
+
+from typing import List, Tuple
+
+from pypinyin import phrases_dict, pinyin_dict
+
+from piper.phonemize_chinese import ChinesePhonemizer
+
+
+def generate_chinese_lexicon():
+    word_dict = pinyin_dict.pinyin_dict
+    phrases = phrases_dict.phrases_dict
+
+    phonemizer = ChinesePhonemizer(model_dir="./abc")
+
+    lexicon = []
+    for key in word_dict:
+        if not (0x4E00 <= key <= 0x9FFF):
+            continue
+        w = chr(key)
+
+        phonemes = phonemizer.phonemize(w)[0]
+        tokens = []
+        for p in phonemes:
+            tokens.append(p)
+            if p in {"1", "2", "3", "4", "5"}:
+                tokens.append("_")
+
+        lexicon.append((w, tokens))
+
+    for key in phrases:
+        phonemes = phonemizer.phonemize(key)[0]
+        tokens = []
+        for p in phonemes:
+            tokens.append(p)
+            if p in {"1", "2", "3", "4", "5"}:
+                tokens.append("_")
+
+        lexicon.append((key, tokens))
+
+    return lexicon
+
+
+def save(filename: str, lexicon: List[Tuple[str, List[str]]]):
+    with open(filename, "w", encoding="utf-8") as f:
+        for word, phones in lexicon:
+            tokens = " ".join(phones)
+            f.write(f"{word} {tokens}\n")
+
+
+def main():
+    zh = generate_chinese_lexicon()
+
+    save("lexicon-zh-g2pw.txt", zh)
+
+
+if __name__ == "__main__":
+    main()

scripts/piper/generate_samples.py.in

@@ -5,12 +5,13 @@ config = sherpa_onnx.OfflineTtsConfig(
     model=sherpa_onnx.OfflineTtsModelConfig(
         vits=sherpa_onnx.OfflineTtsVitsModelConfig(
             model="{{ model }}",
-            lexicon="",
+            lexicon="{{ lexicon }}",
             data_dir="{{ data_dir }}",
             tokens="{{ tokens }}",
         ),
         num_threads=1,
     ),
+    rule_fsts="{{ rule_fsts }}",
 )
 
 if not config.validate():

sherpa-onnx/csrc/character-lexicon.cc

@@ -35,8 +35,9 @@ namespace sherpa_onnx {
 
 class CharacterLexicon::Impl {
  public:
-  Impl(const std::string &lexicon, const std::string &tokens, bool debug)
-      : debug_(debug) {
+  Impl(const std::string &lexicon, const std::string &tokens, bool debug,
+       bool use_g2pw)
+      : debug_(debug), use_g2pw_(use_g2pw) {
     if (lexicon.empty()) {
       SHERPA_ONNX_LOGE("Please provide lexicon.txt for this model");
       SHERPA_ONNX_EXIT(-1);
@@ -55,8 +56,8 @@ class CharacterLexicon::Impl {
 
   template <typename Manager>
   Impl(Manager *mgr, const std::string &lexicon, const std::string &tokens,
-       bool debug)
-      : debug_(debug) {
+       bool debug, bool use_g2pw)
+      : debug_(debug), use_g2pw_(use_g2pw) {
     if (lexicon.empty()) {
       SHERPA_ONNX_LOGE("Please provide lexicon.txt for this model");
       SHERPA_ONNX_EXIT(-1);
@@ -161,6 +162,14 @@ class CharacterLexicon::Impl {
     std::vector<TokenIDs> ans;
     std::vector<int64_t> this_sentence;
 
+    if (use_g2pw_) {
+      // please see
+      // https://github.com/OHF-Voice/piper1-gpl/blob/main/src/piper/phonemize_chinese.py#L296
+      // it adds bos at the beginning of each sentence and adds eos at the end
+      // of each sentence
+      this_sentence.push_back(bos_id_);
+    }
+
     PhraseMatcher matcher(&all_words_, words, debug_);
 
     for (const std::string &w : matcher) {
@@ -177,12 +186,22 @@ class CharacterLexicon::Impl {
       this_sentence.insert(this_sentence.end(), ids.begin(), ids.end());
 
       if (IsPunct(w)) {
+        if (use_g2pw_) {
+          this_sentence.push_back(eos_id_);
+        }
         ans.emplace_back(std::move(this_sentence));
-        this_sentence = {};
+        if (use_g2pw_) {
+          this_sentence = {bos_id_};
+        } else {
+          this_sentence.clear();
+        }
       }
     }  // for (const std::string &w : matcher)
 
     if (!this_sentence.empty()) {
+      if (use_g2pw_) {
+        this_sentence.push_back(eos_id_);
+      }
       ans.emplace_back(std::move(this_sentence));
     }
 
@@ -254,6 +273,12 @@ class CharacterLexicon::Impl {
         id2token_[p.second] = p.first;
       }
     }
+
+    if (use_g2pw_) {
+      pad_id_ = token2id_.at("_");
+      bos_id_ = token2id_.at("^");
+      eos_id_ = token2id_.at("$");
+    }
   }
 
   void InitLexicon(std::istream &is) {
@@ -325,18 +350,25 @@ class CharacterLexicon::Impl {
   std::unordered_map<int32_t, std::string> id2token_;
 
   bool debug_ = false;
+  bool use_g2pw_ = false;
+
+  int32_t bos_id_ = 0;
+  int32_t eos_id_ = 0;
+  int32_t pad_id_ = 0;
 };
 
 CharacterLexicon::~CharacterLexicon() = default;
 
 CharacterLexicon::CharacterLexicon(const std::string &lexicon,
-                                   const std::string &tokens, bool debug)
-    : impl_(std::make_unique<Impl>(lexicon, tokens, debug)) {}
+                                   const std::string &tokens, bool debug,
+                                   bool use_g2pw /*= false*/)
+    : impl_(std::make_unique<Impl>(lexicon, tokens, debug, use_g2pw)) {}
 
 template <typename Manager>
 CharacterLexicon::CharacterLexicon(Manager *mgr, const std::string &lexicon,
-                                   const std::string &tokens, bool debug)
-    : impl_(std::make_unique<Impl>(mgr, lexicon, tokens, debug)) {}
+                                   const std::string &tokens, bool debug,
+                                   bool use_g2pw /*= false*/)
+    : impl_(std::make_unique<Impl>(mgr, lexicon, tokens, debug, use_g2pw)) {}
 
 std::vector<TokenIDs> CharacterLexicon::ConvertTextToTokenIds(
     const std::string &text, const std::string & /*unused_voice = ""*/) const {
@@ -347,14 +379,16 @@ std::vector<TokenIDs> CharacterLexicon::ConvertTextToTokenIds(
 template CharacterLexicon::CharacterLexicon(AAssetManager *mgr,
                                             const std::string &lexicon,
                                             const std::string &tokens,
-                                            bool debug);
+                                            bool debug, bool use_g2pw);
+
 #endif
 
 #if __OHOS__
 template CharacterLexicon::CharacterLexicon(NativeResourceManager *mgr,
                                             const std::string &lexicon,
                                             const std::string &tokens,
-                                            bool debug);
+                                            bool debug, bool use_g2pw);
+
 #endif
 
 }  // namespace sherpa_onnx

sherpa-onnx/csrc/character-lexicon.h

@@ -19,11 +19,12 @@ class CharacterLexicon : public OfflineTtsFrontend {
   ~CharacterLexicon() override;
 
   CharacterLexicon(const std::string &lexicon, const std::string &tokens,
-                   bool debug);
+                   bool debug, bool use_g2pw = false);
 
   template <typename Manager>
   CharacterLexicon(Manager *mgr, const std::string &lexicon,
-                   const std::string &tokens, bool debug);
+                   const std::string &tokens, bool debug,
+                   bool use_g2pw = false);
 
   std::vector<TokenIDs> ConvertTextToTokenIds(
       const std::string &text,

sherpa-onnx/csrc/offline-tts-vits-impl.h

@@ -5,8 +5,8 @@
 #define SHERPA_ONNX_CSRC_OFFLINE_TTS_VITS_IMPL_H_
 
 #include <memory>
-#include <string>
 #include <sstream>
+#include <string>
 #include <utility>
 #include <vector>
 
@@ -130,8 +130,8 @@ class OfflineTtsVitsImpl : public OfflineTtsImpl {
           tn_list_.push_back(
               std::make_unique<kaldifst::TextNormalizer>(std::move(r)));
         }
-      }    // for (const auto &f : files)
-    }      // if (!config.rule_fars.empty())
+      }  // for (const auto &f : files)
+    }  // if (!config.rule_fars.empty())
   }
 
   int32_t SampleRate() const override {
@@ -305,8 +305,8 @@ class OfflineTtsVitsImpl : public OfflineTtsImpl {
         }
       }
 
-      auto audio = Process(batch_x, batch_tones, sid, speed,
-                           gen_config.silence_scale);
+      auto audio =
+          Process(batch_x, batch_tones, sid, speed, gen_config.silence_scale);
       ans.sample_rate = audio.sample_rate;
       ans.samples.insert(ans.samples.end(), audio.samples.begin(),
                          audio.samples.end());
@@ -370,10 +370,10 @@ class OfflineTtsVitsImpl : public OfflineTtsImpl {
       frontend_ = std::make_unique<MeloTtsLexicon>(
           mgr, config_.model.vits.lexicon, config_.model.vits.tokens,
           model_->GetMetaData(), config_.model.debug);
-    } else if (meta_data.jieba) {
+    } else if (meta_data.jieba || meta_data.use_g2pw) {
       frontend_ = std::make_unique<CharacterLexicon>(
           mgr, config_.model.vits.lexicon, config_.model.vits.tokens,
-          config_.model.debug);
+          config_.model.debug, meta_data.use_g2pw);
     } else if (meta_data.is_melo_tts && meta_data.language == "English") {
       frontend_ = std::make_unique<MeloTtsLexicon>(
           mgr, config_.model.vits.lexicon, config_.model.vits.tokens,
@@ -412,10 +412,10 @@ class OfflineTtsVitsImpl : public OfflineTtsImpl {
       frontend_ = std::make_unique<MeloTtsLexicon>(
           config_.model.vits.lexicon, config_.model.vits.tokens,
           model_->GetMetaData(), config_.model.debug);
-    } else if (meta_data.jieba) {
-      frontend_ = std::make_unique<CharacterLexicon>(config_.model.vits.lexicon,
-                                                     config_.model.vits.tokens,
-                                                     config_.model.debug);
+    } else if (meta_data.jieba || meta_data.use_g2pw) {
+      frontend_ = std::make_unique<CharacterLexicon>(
+          config_.model.vits.lexicon, config_.model.vits.tokens,
+          config_.model.debug, meta_data.use_g2pw);
     } else if ((meta_data.is_piper || meta_data.is_coqui ||
                 meta_data.is_icefall) &&
                !config_.model.vits.data_dir.empty()) {
@@ -437,8 +437,7 @@ class OfflineTtsVitsImpl : public OfflineTtsImpl {
 
   GeneratedAudio Process(const std::vector<std::vector<int64_t>> &tokens,
                          const std::vector<std::vector<int64_t>> &tones,
-                         int32_t sid, float speed,
-                         float silence_scale) const {
+                         int32_t sid, float speed, float silence_scale) const {
     int32_t num_tokens = 0;
     for (const auto &k : tokens) {
       num_tokens += k.size();

sherpa-onnx/csrc/offline-tts-vits-model-meta-data.h

@@ -23,6 +23,9 @@ struct OfflineTtsVitsModelMetaData {
   bool is_icefall = false;
   bool is_melo_tts = false;
 
+  // for piper zh models using pinyin instead of espeak-ng
+  bool use_g2pw = false;
+
   // for Chinese TTS models from
   // https://github.com/Plachtaa/VITS-fast-fine-tuning
   int32_t jieba = 0;

sherpa-onnx/csrc/offline-tts-vits-model.cc

@@ -122,8 +122,8 @@ class OfflineTtsVitsModel::Impl {
  private:
   void Init(void *model_data, size_t model_data_length) {
     if (model_data) {
-      sess_ = std::make_unique<Ort::Session>(
-          env_, model_data, model_data_length, sess_opts_);
+      sess_ = std::make_unique<Ort::Session>(env_, model_data,
+                                             model_data_length, sess_opts_);
     } else if (!sess_) {
       SHERPA_ONNX_LOGE(
           "Please pass model data or initialize the session outside of "
@@ -187,6 +187,7 @@ class OfflineTtsVitsModel::Impl {
     SHERPA_ONNX_READ_META_DATA_WITH_DEFAULT(meta_data_.use_eos_bos,
                                             "use_eos_bos", 1);
     SHERPA_ONNX_READ_META_DATA_WITH_DEFAULT(meta_data_.pad_id, "pad_id", 0);
+    SHERPA_ONNX_READ_META_DATA_WITH_DEFAULT(meta_data_.use_g2pw, "has_g2pw", 0);
 
     std::string comment;
     SHERPA_ONNX_READ_META_DATA_STR(comment, "comment");