mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-08-08 00:18:12 +08:00
Fix tokenizer in cpp (#13735)
### What problem does this PR solve? Tokenzier in Infinity is modified in https://github.com/infiniflow/infinity/pull/3330, sync the code change to cpp files in ragflow ### Type of change - [x] Bug Fix (non-breaking change which fixes an issue)
This commit is contained in:
@@ -136,7 +136,7 @@ public:
|
||||
[[nodiscard]] std::vector<std::pair<std::vector<std::string_view>, double>> GetBestTokensTopN(
|
||||
std::string_view chars, uint32_t n) const;
|
||||
|
||||
static const size_t term_string_buffer_limit_ = 4096 * 3;
|
||||
static constexpr size_t term_string_buffer_limit_ = 4096 * 3;
|
||||
|
||||
std::string dict_path_;
|
||||
|
||||
@@ -152,6 +152,8 @@ public:
|
||||
|
||||
OpenCC* opencc_{nullptr};
|
||||
|
||||
mutable std::vector<char> lowercase_string_buffer_;
|
||||
|
||||
bool fine_grained_{false};
|
||||
|
||||
bool enable_position_{false};
|
||||
|
||||
Reference in New Issue
Block a user