jni: Fix strings returned to Java for characters outside the BMP (#4064)

sherpa-onnx/csrc/text-utils-test.cc

@@ -429,18 +429,39 @@ TEST(ParseLexiconFile, MultiCodepointPhonemes) {
   EXPECT_EQ(entries[0].phonemes[5], "ɪ");
 }
 
-TEST(RemoveInvalidUtf8Sequences, ByteLevelBpeTokenFragment) {
-    // Whisper byte-level-BPE tokens can split a multi-byte UTF-8 character
-    // across two tokens. Each fragment alone is invalid UTF-8 and is
-    // stripped; the character only survives when the fragments are joined
-    // first. Recognizer implementations must therefore concatenate tokens
-    // before calling this function (see the whisper Convert()).
-    std::string frag1 = "\xd7";  // leading byte of a Hebrew letter
-    std::string frag2 = "\xa8";  // continuation byte (frag1+frag2 is a letter)
-    EXPECT_EQ(RemoveInvalidUtf8Sequences(frag1), "");
-    EXPECT_EQ(RemoveInvalidUtf8Sequences(frag2), "");
-    EXPECT_EQ(RemoveInvalidUtf8Sequences(frag1 + frag2), "\xd7\xa8");
-  }
+TEST(Utf8ToUtf16, Bmp) {
+  // a, U+00E9, U+20AC
+  std::u16string expected = {u'a', 0x00E9, 0x20AC};
+  EXPECT_EQ(Utf8ToUtf16("a\xc3\xa9\xe2\x82\xac"), expected);
+}
+
+TEST(Utf8ToUtf16, SurrogatePairs) {
+  // U+1F600 and U+20000 become surrogate pairs; the text after them is kept
+  std::u16string expected = {u'a', 0xD83D, 0xDE00, u'b', 0xD840, 0xDC00, u'c'};
+  EXPECT_EQ(Utf8ToUtf16("a\xf0\x9f\x98\x80"
+                        "b\xf0\xa0\x80\x80"
+                        "c"),
+            expected);
+
+  std::u16string max = {0xDBFF, 0xDFFF};
+  EXPECT_EQ(Utf8ToUtf16("\xf4\x8f\xbf\xbf"), max);  // U+10FFFF
+}
+
+TEST(Utf8ToUtf16, InvalidBytes) {
+  // an encoded surrogate (modified UTF-8, ed a0 bd) is not valid UTF-8
+  std::u16string expected = {0xFFFD, 0xFFFD, 0xFFFD, u'a'};
+  EXPECT_EQ(Utf8ToUtf16("\xed\xa0\xbd"
+                        "a"),
+            expected);
+}
+
+TEST(Utf8ToUtf16, EmbeddedNull) {
+  // the text after U+0000 is kept (modified UTF-8 would use c0 80)
+  std::u16string expected = {u'a', 0, u'b'};
+  EXPECT_EQ(Utf8ToUtf16(std::string("a\0b", 3)), expected);
+}
+
+TEST(Utf8ToUtf16, Empty) { EXPECT_EQ(Utf8ToUtf16(""), u""); }
 
 TEST(Utf16ToUtf8, Bmp) {
   // A, é, €, →

sherpa-onnx/csrc/text-utils.cc

@@ -1019,6 +1019,24 @@ std::u32string Utf8ToUtf32(const std::string &str) {
   return out;
 }
 
+std::u16string Utf8ToUtf16(const std::string &str) {
+  std::u16string out;
+  out.reserve(str.size());
+
+  // Utf8ToUtf32 never returns surrogates or code points above U+10FFFF
+  for (char32_t cp : Utf8ToUtf32(str)) {
+    if (cp >= 0x10000) {
+      cp -= 0x10000;
+      out.push_back(static_cast<char16_t>(0xD800 + (cp >> 10)));
+      out.push_back(static_cast<char16_t>(0xDC00 + (cp & 0x3FF)));
+    } else {
+      out.push_back(static_cast<char16_t>(cp));
+    }
+  }
+
+  return out;
+}
+
 std::string Utf32ToUtf8(const std::u32string &str) {
   std::string out;
   out.reserve(str.size() * 2);  // rough estimate

sherpa-onnx/csrc/text-utils.h

@@ -169,6 +169,10 @@ std::string Join(const std::vector<std::string> &ss,
 // Converts a UTF-8 std::string to a UTF-32 std::u32string
 std::u32string Utf8ToUtf32(const std::string &str);
 
+// Converts a UTF-8 std::string to a UTF-16 std::u16string. A code point
+// above U+FFFF becomes a surrogate pair; invalid bytes become U+FFFD.
+std::u16string Utf8ToUtf16(const std::string &str);
+
 // Converts a UTF-32 std::u32string to a UTF-8 std::string
 std::string Utf32ToUtf8(const std::u32string &str);
 

sherpa-onnx/jni/common.h

@@ -146,11 +146,20 @@
 jobject NewInteger(JNIEnv *env, int32_t value);
 jobject NewFloat(JNIEnv *env, float value);
 
-// Wrapper for NewStringUTF that strips invalid UTF-8 byte sequences first.
-// This prevents JNI crashes when the ASR model outputs invalid UTF-8
-// (e.g., orphaned UTF-16 surrogate halves from the Arabic moonshine model).
+// Create a Java string from UTF-8 text. Invalid UTF-8 byte sequences are
+// stripped first. This prevents JNI crashes when the ASR model outputs
+// invalid UTF-8 (e.g., orphaned UTF-16 surrogate halves from the Arabic
+// moonshine model).
+//
+// NewStringUTF is not used since it expects modified UTF-8. Given a 4-byte
+// sequence (a code point above U+FFFF, e.g., an emoji), it returns garbage
+// and drops characters at the end of the string.
 inline jstring SafeNewStringUTF(JNIEnv *env, const std::string &s) {
-  return env->NewStringUTF(sherpa_onnx::RemoveInvalidUtf8Sequences(s).c_str());
+  std::u16string u16 =
+      sherpa_onnx::Utf8ToUtf16(sherpa_onnx::RemoveInvalidUtf8Sequences(s));
+  static_assert(sizeof(jchar) == sizeof(char16_t), "jchar must be 16-bit");
+  return env->NewString(reinterpret_cast<const jchar *>(u16.data()),
+                        static_cast<jsize>(u16.size()));
 }
 
 inline jstring SafeNewStringUTF(JNIEnv *env, const char *s) {