diff --git a/CMakeLists.txt b/CMakeLists.txt index 52637f9..adef58d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.12) project(libime VERSION 1.1.14) set(LibIME_VERSION ${PROJECT_VERSION}) -set(REQUIRED_FCITX_VERSION 5.1.13) +set(REQUIRED_FCITX_VERSION 5.1.20) find_package(ECM 1.0 REQUIRED) set(CMAKE_MODULE_PATH ${ECM_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake" ${CMAKE_MODULE_PATH}) diff --git a/src/libime/pinyin/pinyindictionary.cpp b/src/libime/pinyin/pinyindictionary.cpp index 9220473..3a8a870 100644 --- a/src/libime/pinyin/pinyindictionary.cpp +++ b/src/libime/pinyin/pinyindictionary.cpp @@ -239,9 +239,16 @@ PinyinDictionary::TrieType loadTextImpl(std::istream &in) { } lineNo++; - auto line = fcitx::stringutils::trimView(lineBuf); - std::vector tokens = - fcitx::stringutils::split(line, FCITX_WHITESPACE); + std::string_view line = lineBuf; + std::vector tokens; + while (!line.empty()) { + std::string token; + auto consumed = fcitx::stringutils::consumeMaybeEscapedValue( + line, FCITX_WHITESPACE, &token); + if (!consumed.empty()) { + tokens.push_back(std::string(token)); + } + } if (tokens.size() == 3 || tokens.size() == 2) { const std::string &hanzi = tokens[0]; std::string_view pinyin = tokens[1]; @@ -899,10 +906,10 @@ void PinyinDictionary::saveText(size_t idx, std::ostream &out) { if (sep == std::string::npos) { return true; } + auto fullPinyin = PinyinEncoder::decodeFullPinyin(buf.data(), sep); std::string_view ref(buf); - auto fullPinyin = PinyinEncoder::decodeFullPinyin(ref.data(), sep); - out << ref.substr(sep + 1) << " " << fullPinyin << " " - << std::setprecision(16) << value << '\n'; + out << fcitx::stringutils::escapeForValue(ref.substr(sep + 1)) << " " + << fullPinyin << " " << std::setprecision(16) << value << '\n'; return true; }); out.copyfmt(state); diff --git a/test/testpinyindictionary.cpp b/test/testpinyindictionary.cpp index 7eb9a88..f0da8bf 100644 --- a/test/testpinyindictionary.cpp +++ b/test/testpinyindictionary.cpp @@ -16,6 +16,8 @@ using namespace libime; +namespace { + constexpr char testPinyin1[] = "ni'hui"; constexpr char testHanzi1[] = "倪辉"; @@ -64,7 +66,7 @@ bool searchWordPrefix(const PinyinDictionary &dict, const char *data, return seenWord; } -int main() { +void testBasic() { PinyinDictionary dict; dict.load(PinyinDictionary::SystemDict, LIBIME_BINARY_DIR "/data/dict_sc.txt", PinyinDictFormat::Text); @@ -93,5 +95,29 @@ int main() { dict.save(0, LIBIME_BINARY_DIR "/test/testpinyindictionary.dict", PinyinDictFormat::Binary); +} + +void testEscape() { + std::stringstream ss; + constexpr std::string_view input = R"( +"你\n好 不\"" ni +)"; + + ss << input; + PinyinDictionary dict; + dict.load(PinyinDictionary::SystemDict, ss, PinyinDictFormat::Text); + FCITX_ASSERT( + dict.lookupWord(PinyinDictionary::SystemDict, "ni", "你\n好 不\"")); + std::stringstream dump; + dict.save(PinyinDictionary::SystemDict, dump, PinyinDictFormat::Text); + FCITX_ASSERT(dump.str() == "\"你\\n好 不\\\"\" ni 0\n") + << "dump: " << dump.str(); +} + +} // namespace + +int main() { + testBasic(); + testEscape(); return 0; }