zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-07-23 17:06:42 +08:00
Code Issues Packages Projects Releases Wiki Activity

Port PR14140 and PR16881 to GO (#17102)

Browse Source 
### Summary

Port
https://github.com/infiniflow/ragflow/pull/14140/
https://github.com/infiniflow/ragflow/pull/16881
This commit is contained in:
qinling0210
2026-07-21 17:46:23 +08:00
committed by GitHub
parent 20b760f266
commit 0189ca3700
19 changed files with 570 additions and 152 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
internal/binding/cpp/rag_analyzer.h
View File

@@ -50,7 +50,9 @@ public:
~RAGAnalyzer();
void InitStemmer(Language language) { stemmer_->Init(language); }
void InitStemmer(Language language);
void SetLanguage(const std::string& language);
int32_t Load();
@@ -59,38 +61,41 @@ public:
void SetEnablePosition(bool enable_position) { enable_position_ = enable_position; }
std::pair<std::vector<std::string>, std::vector<std::pair<unsigned, unsigned>>> TokenizeWithPosition(
const std::string& line) const;
std::string Tokenize(const std::string& line) const;
const std::string& line);
std::string Tokenize(const std::string& line);
void FineGrainedTokenize(const std::string& tokens, std::vector<std::string>& result) const;
void FineGrainedTokenize(const std::string& tokens, std::vector<std::string>& result);
void TokenizeInnerWithPosition(const std::string& L,
std::vector<std::string>& tokens,
std::vector<std::pair<unsigned, unsigned>>& positions,
unsigned base_pos,
const std::vector<unsigned>* pos_mapping = nullptr) const;
const std::vector<unsigned>* pos_mapping = nullptr);
void FineGrainedTokenizeWithPosition(const std::string& tokens_str,
const std::vector<std::pair<unsigned, unsigned>>& positions,
std::vector<std::string>& fine_tokens,
std::vector<std::pair<unsigned, unsigned>>& fine_positions) const;
std::vector<std::pair<unsigned, unsigned>>& fine_positions);
void EnglishNormalizeWithPosition(const std::vector<std::string>& tokens,
const std::vector<std::pair<unsigned, unsigned>>& positions,
std::vector<std::string>& normalize_tokens,
std::vector<std::pair<unsigned, unsigned>>& normalize_positions) const;
std::vector<std::pair<unsigned, unsigned>>& normalize_positions);
unsigned MapToOriginalPosition(unsigned processed_pos,
const std::vector<std::pair<unsigned, unsigned>>& mapping) const;
const std::vector<std::pair<unsigned, unsigned>>& mapping);
void MergeWithPosition(const std::vector<std::string>& tokens,
const std::vector<std::pair<unsigned, unsigned>>& positions,
std::vector<std::string>& merged_tokens,
std::vector<std::pair<unsigned, unsigned>>& merged_positions) const;
std::vector<std::pair<unsigned, unsigned>>& merged_positions);
void SplitByLang(const std::string& line, std::vector<std::pair<std::string, bool>>& txt_lang_pairs) const;
int32_t Freq(std::string_view key) const;
std::string Tag(std::string_view key) const;
static bool IsStopword(const std::string& term);
protected:
int AnalyzeImpl(const Term& input, void* data, bool fine_grained, bool enable_position, HookType func) const;
int AnalyzeImpl(const Term& input, void* data, HookType func);
private:
static constexpr float DENOMINATOR = 1000000;
@@ -130,7 +135,7 @@ private:
[[nodiscard]] std::string Merge(const std::string& tokens) const;
void EnglishNormalize(const std::vector<std::string>& tokens, std::vector<std::string>& res) const;
void EnglishNormalize(const std::vector<std::string>& tokens, std::vector<std::string>& res);
public:
[[nodiscard]] std::vector<std::pair<std::vector<std::string_view>, double>> GetBestTokensTopN(
@@ -154,6 +159,8 @@ public:
mutable std::vector<char> lowercase_string_buffer_;
bool use_lemmatizer_{true};
bool fine_grained_{false};
bool enable_position_{false};