zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-07-23 17:06:42 +08:00
Code Issues Packages Projects Releases Wiki Activity

Port PR14140 and PR16881 to GO (#17102)

Browse Source 
### Summary

Port
https://github.com/infiniflow/ragflow/pull/14140/
https://github.com/infiniflow/ragflow/pull/16881
This commit is contained in:
qinling0210
2026-07-21 17:46:23 +08:00
committed by GitHub
parent 20b760f266
commit 0189ca3700
19 changed files with 570 additions and 152 deletions
Download Patch File Download Diff File Expand all files Collapse all files

175
internal/binding/cpp/rag_analyzer.cpp
View File

@@ -28,6 +28,8 @@
#include <iostream>
#include <cmath>
#include <fstream>
#include <algorithm>
#include <unordered_set>
// import :term;
// import :stemmer;
// import :analyzer;
@@ -47,6 +49,28 @@ static const std::string WORDNET_PATH = "wordnet";
static const std::string OPENCC_PATH = "opencc";
// Map language names (lowercase) to Stemmer Language enum.
// Used by SetLanguage() to configure language-specific stemming.
static const std::pair<std::string, Language> SNOWBALL_LANGUAGE_MAP[] = {
{"english", STEM_LANG_ENGLISH},
{"dutch", STEM_LANG_DUTCH},
{"german", STEM_LANG_GERMAN},
{"french", STEM_LANG_FRENCH},
{"spanish", STEM_LANG_SPANISH},
{"italian", STEM_LANG_ITALIAN},
{"portuguese", STEM_LANG_PORTUGUESE},
{"portuguese br", STEM_LANG_PORTUGUESE},
{"russian", STEM_LANG_RUSSIAN},
{"arabic", STEM_LANG_UNKNOWN},
{"danish", STEM_LANG_DANISH},
{"finnish", STEM_LANG_FINNISH},
{"hungarian", STEM_LANG_HUNGARIAN},
{"norwegian", STEM_LANG_NORWEGIAN},
{"romanian", STEM_LANG_ROMANIAN},
{"swedish", STEM_LANG_SWEDISH},
{"turkish", STEM_LANG_TURKISH},
};
static const std::string REGEX_SPLIT_CHAR =
R"#(([ ,\.<>/?;'\[\]\`!@#$%^&*$$\{\}\|_+=《》，。？、；‘’：“”【】~！￥%……（）——-]+|[a-zA-Z\.-]+|[0-9,\.-]+))#";
@@ -79,11 +103,6 @@ static inline int32_t DecodeFreq(int32_t value) {
return v1;
}
static inline int32_t DecodePOSIndex(int32_t value) {
// POS index is stored in the high 8 bits (bits 24-31)
return static_cast<int32_t>(static_cast<uint32_t>(value) >> 24);
}
void Split(const std::string &input, const std::string &split_pattern, std::vector<std::string> &result, bool keep_delim = false) {
re2::RE2 pattern(split_pattern);
re2::StringPiece leftover(input.data());
@@ -364,17 +383,16 @@ struct CompiledRegex {
class NLTKWordTokenizer {
MacIntyreContractions contractions_;
// Static singleton instance
static std::unique_ptr<NLTKWordTokenizer> instance_;
static std::once_flag init_flag_;
public:
// Static method to get the singleton instance
// Magic Static singleton
static NLTKWordTokenizer &GetInstance() {
std::call_once(init_flag_, []() { instance_ = std::make_unique<NLTKWordTokenizer>(); });
return *instance_;
static NLTKWordTokenizer instance;
return instance;
}
NLTKWordTokenizer(const NLTKWordTokenizer &) = delete;
NLTKWordTokenizer &operator=(const NLTKWordTokenizer &) = delete;
// Starting quotes.
std::vector<std::pair<std::string, std::string>> STARTING_QUOTES = {
{std::string(R"(([«“‘„]|[`]+))"), std::string(R"( $1 )")},
@@ -618,10 +636,6 @@ private:
}
};
// Static member definitions for NLTKWordTokenizer singleton
std::unique_ptr<NLTKWordTokenizer> NLTKWordTokenizer::instance_ = nullptr;
std::once_flag NLTKWordTokenizer::init_flag_;
void SentenceSplitter(const std::string &text, std::vector<std::string> &result) {
int error_code;
PCRE2_SIZE error_offset;
@@ -682,6 +696,31 @@ RAGAnalyzer::~RAGAnalyzer() {
}
}
void RAGAnalyzer::InitStemmer(Language language) {
stemmer_->Init(language);
use_lemmatizer_ = (language == STEM_LANG_ENGLISH);
}
void RAGAnalyzer::SetLanguage(const std::string &language) {
std::string lang_key = language;
std::transform(lang_key.begin(), lang_key.end(), lang_key.begin(), [](unsigned char c) { return std::tolower(c); });
lang_key.erase(lang_key.find_last_not_of(" \t") + 1);
lang_key.erase(0, lang_key.find_first_not_of(" \t"));
Language stem_lang = STEM_LANG_UNKNOWN;
for (const auto &pair : SNOWBALL_LANGUAGE_MAP) {
if (pair.first == lang_key) {
stem_lang = pair.second;
break;
}
}
if (stem_lang != STEM_LANG_UNKNOWN) {
stemmer_->Init(stem_lang);
use_lemmatizer_ = (stem_lang == STEM_LANG_ENGLISH);
}
}
int32_t RAGAnalyzer::Load() {
fs::path root(dict_path_);
@@ -733,12 +772,6 @@ int32_t RAGAnalyzer::Load() {
int32_t freq = std::stoi(results[1]);
freq = int32_t(std::log(float(freq) / DENOMINATOR) + 0.5);
int32_t pos_idx = pos_table_->GetPOSIndex(results[2]);
// If the POS tag is not in the POS table (e.g. "eng" for
// English words), default to index 0. Using -1 would cause
// Encode() to produce a negative int32_t which the darts
// trie rejects.
if (pos_idx < 0)
pos_idx = 0;
int value = Encode(freq, pos_idx);
trie_->Add(results[0], value);
std::string rkey = RKey(results[0]);
@@ -860,17 +893,14 @@ int32_t RAGAnalyzer::Freq(const std::string_view key) const {
return static_cast<int32_t>(std::exp(v) * DENOMINATOR + 0.5);
}
std::string RAGAnalyzer::Tag(std::string_view key) const {
std::string lower_key = Key(std::string(key));
int32_t encoded_value = trie_->Get(lower_key);
if (encoded_value == -1) {
return "";
}
int32_t pos_idx = DecodePOSIndex(encoded_value);
if (pos_table_ == nullptr) {
return "";
}
const char* pos_tag = pos_table_->GetPOS(pos_idx);
std::string RAGAnalyzer::Tag(const std::string_view key) const {
std::string lower_key = Key(key);
int32_t v = trie_->Get(lower_key);
if (v == -1) return "";
uint32_t pos_idx = (static_cast<uint32_t>(v) >> 24) & 0xFF;
if (pos_idx == 0) return "";
if (pos_table_ == nullptr) return "";
const char *pos_tag = pos_table_->GetPOS(static_cast<int32_t>(pos_idx));
return pos_tag ? std::string(pos_tag) : "";
}
@@ -1347,7 +1377,7 @@ std::string RAGAnalyzer::Merge(const std::string &tks_str) const {
void RAGAnalyzer::MergeWithPosition(const std::vector<std::string> &tokens,
const std::vector<std::pair<unsigned, unsigned>> &positions,
std::vector<std::string> &merged_tokens,
std::vector<std::pair<unsigned, unsigned>> &merged_positions) const {
std::vector<std::pair<unsigned, unsigned>> &merged_positions) {
// Filter out empty tokens first (like spaces) to match Merge behavior
std::vector<std::string> filtered_tokens;
std::vector<std::pair<unsigned, unsigned>> filtered_positions;
@@ -1391,15 +1421,19 @@ void RAGAnalyzer::MergeWithPosition(const std::vector<std::string> &tokens,
merged_positions = std::move(res_positions);
}
void RAGAnalyzer::EnglishNormalize(const std::vector<std::string> &tokens, std::vector<std::string> &res) const {
void RAGAnalyzer::EnglishNormalize(const std::vector<std::string> &tokens, std::vector<std::string> &res) {
for (auto &t : tokens) {
if (re2::RE2::PartialMatch(t, pattern1_)) { //"[a-zA-Z_-]+$"
// Apply lowercase before lemmatization to match Python NLTK behavior
char *lowercase_term = lowercase_string_buffer_.data();
ToLower(t.c_str(), t.size(), lowercase_term, term_string_buffer_limit_);
std::string lemma_term = wordnet_lemma_->Lemmatize(lowercase_term);
std::string term_to_stem;
if (use_lemmatizer_) {
term_to_stem = wordnet_lemma_->Lemmatize(lowercase_term);
} else {
term_to_stem = lowercase_term;
}
std::string stem_term;
stemmer_->Stem(lemma_term, stem_term);
stemmer_->Stem(term_to_stem, stem_term);
res.push_back(stem_term);
} else {
res.push_back(t);
@@ -1733,7 +1767,7 @@ std::string PCRE2GlobalReplace(const std::string &text, const std::string &patte
return result;
}
std::string RAGAnalyzer::Tokenize(const std::string &line) const {
std::string RAGAnalyzer::Tokenize(const std::string &line) {
// Python-style simple tokenization: re.sub(r"\\W+", " ", line)
std::string processed_line = PCRE2GlobalReplace(line, R"#(\W+)#", " ");
std::string str1 = StrQ2B(processed_line);
@@ -1756,12 +1790,16 @@ std::string RAGAnalyzer::Tokenize(const std::string &line) const {
NLTKWordTokenizer::GetInstance().Tokenize(sentence, term_list);
}
for (unsigned i = 0; i < term_list.size(); ++i) {
// Apply lowercase before lemmatization to match Python NLTK behavior
char *lowercase_term = lowercase_string_buffer_.data();
ToLower(term_list[i].c_str(), term_list[i].size(), lowercase_term, term_string_buffer_limit_);
std::string lemma_term = wordnet_lemma_->Lemmatize(lowercase_term);
std::string term_to_stem;
if (use_lemmatizer_) {
term_to_stem = wordnet_lemma_->Lemmatize(lowercase_term);
} else {
term_to_stem = lowercase_term;
}
std::string stem_term;
stemmer_->Stem(lemma_term, stem_term);
stemmer_->Stem(term_to_stem, stem_term);
res.push_back(stem_term);
}
continue;
@@ -1793,7 +1831,7 @@ std::string RAGAnalyzer::Tokenize(const std::string &line) const {
return ret;
}
std::pair<std::vector<std::string>, std::vector<std::pair<unsigned, unsigned>>> RAGAnalyzer::TokenizeWithPosition(const std::string &line) const {
std::pair<std::vector<std::string>, std::vector<std::pair<unsigned, unsigned>>> RAGAnalyzer::TokenizeWithPosition(const std::string &line) {
// Python-style simple tokenization: re.sub(r"\W+", " ", line)
// Get processed line and position mapping from PCRE2GlobalReplace
auto [processed_line, pcre2_pos_mapping] = PCRE2GlobalReplaceWithPosition(line, R"#(\W+)#", " ");
@@ -1877,9 +1915,14 @@ std::pair<std::vector<std::string>, std::vector<std::pair<unsigned, unsigned>>>
// Apply lowercase before lemmatization to match Python NLTK behavior
char *lowercase_term = lowercase_string_buffer_.data();
ToLower(term.c_str(), term.size(), lowercase_term, term_string_buffer_limit_);
std::string lemma_term = wordnet_lemma_->Lemmatize(lowercase_term);
std::string term_to_stem;
if (use_lemmatizer_) {
term_to_stem = wordnet_lemma_->Lemmatize(lowercase_term);
} else {
term_to_stem = lowercase_term;
}
std::string stem_term;
stemmer_->Stem(lemma_term, stem_term);
stemmer_->Stem(term_to_stem, stem_term);
tokens.push_back(stem_term);
@@ -1943,7 +1986,7 @@ std::pair<std::vector<std::string>, std::vector<std::pair<unsigned, unsigned>>>
return {std::move(tokens), std::move(positions)};
}
unsigned RAGAnalyzer::MapToOriginalPosition(unsigned processed_pos, const std::vector<std::pair<unsigned, unsigned>> &mapping) const {
unsigned RAGAnalyzer::MapToOriginalPosition(unsigned processed_pos, const std::vector<std::pair<unsigned, unsigned>> &mapping) {
for (const auto &[orig, proc] : mapping) {
if (proc == processed_pos) {
return orig;
@@ -1964,7 +2007,7 @@ void RAGAnalyzer::TokenizeInnerWithPosition(const std::string &L,
std::vector<std::string> &tokens,
std::vector<std::pair<unsigned, unsigned>> &positions,
unsigned base_pos,
const std::vector<unsigned> *pos_mapping) const {
const std::vector<unsigned> *pos_mapping) {
auto [tks, s] = MaxForward(L);
auto [tks1, s1] = MaxBackward(L);
@@ -2193,18 +2236,22 @@ void RAGAnalyzer::TokenizeInnerWithPosition(const std::string &L,
void RAGAnalyzer::EnglishNormalizeWithPosition(const std::vector<std::string> &tokens,
const std::vector<std::pair<unsigned, unsigned>> &positions,
std::vector<std::string> &normalize_tokens,
std::vector<std::pair<unsigned, unsigned>> &normalize_positions) const {
std::vector<std::pair<unsigned, unsigned>> &normalize_positions) {
for (size_t i = 0; i < tokens.size(); ++i) {
const auto &token = tokens[i];
const auto &[start_pos, end_pos] = positions[i];
if (re2::RE2::PartialMatch(token, pattern1_)) { //"[a-zA-Z_-]+$"
// Apply lowercase before lemmatization to match Python NLTK behavior
char *lowercase_term = lowercase_string_buffer_.data();
ToLower(token.c_str(), token.size(), lowercase_term, term_string_buffer_limit_);
std::string lemma_term = wordnet_lemma_->Lemmatize(lowercase_term);
std::string term_to_stem;
if (use_lemmatizer_) {
term_to_stem = wordnet_lemma_->Lemmatize(lowercase_term);
} else {
term_to_stem = lowercase_term;
}
std::string stem_term;
stemmer_->Stem(lemma_term, stem_term);
stemmer_->Stem(term_to_stem, stem_term);
normalize_tokens.push_back(stem_term);
normalize_positions.emplace_back(start_pos, end_pos);
@@ -2218,7 +2265,7 @@ void RAGAnalyzer::EnglishNormalizeWithPosition(const std::vector<std::string> &t
void RAGAnalyzer::FineGrainedTokenizeWithPosition(const std::string &tokens_str,
const std::vector<std::pair<unsigned, unsigned>> &positions,
std::vector<std::string> &fine_tokens,
std::vector<std::pair<unsigned, unsigned>> &fine_positions) const {
std::vector<std::pair<unsigned, unsigned>> &fine_positions) {
std::vector<std::string> tks;
Split(tokens_str, blank_pattern_, tks);
@@ -2331,7 +2378,7 @@ void RAGAnalyzer::FineGrainedTokenizeWithPosition(const std::string &tokens_str,
// fine_tokens already contains the correct Chinese tokens
}
void RAGAnalyzer::FineGrainedTokenize(const std::string &tokens, std::vector<std::string> &result) const {
void RAGAnalyzer::FineGrainedTokenize(const std::string &tokens, std::vector<std::string> &result) {
std::vector<std::string> tks;
Split(tokens, blank_pattern_, tks);
std::vector<std::string> res;
@@ -2422,11 +2469,11 @@ void RAGAnalyzer::FineGrainedTokenize(const std::string &tokens, std::vector<std
// return ret;
}
int RAGAnalyzer::AnalyzeImpl(const Term &input, void *data, bool fine_grained, bool enable_position, HookType func) const {
if (enable_position) {
int RAGAnalyzer::AnalyzeImpl(const Term &input, void *data, HookType func) {
if (enable_position_) {
auto [tokens, positions] = TokenizeWithPosition(input.text_);
if (fine_grained) {
if (fine_grained_) {
std::vector<std::string> fine_tokens;
std::vector<std::pair<unsigned, unsigned>> fine_positions;
FineGrainedTokenizeWithPosition(Join(tokens, 0), positions, fine_tokens, fine_positions);
@@ -2437,13 +2484,15 @@ int RAGAnalyzer::AnalyzeImpl(const Term &input, void *data, bool fine_grained, b
for (size_t i = 0; i < tokens.size(); ++i) {
if (tokens[i].empty())
continue;
if (IsStopword(tokens[i]))
continue;
const auto &[start_pos, end_pos] = positions[i];
func(data, tokens[i].c_str(), tokens[i].size(), start_pos, end_pos, false, 0);
}
} else {
std::string result = Tokenize(input.text_);
std::vector<std::string> tokens;
if (fine_grained) {
if (fine_grained_) {
FineGrainedTokenize(result, tokens);
} else {
Split(result, blank_pattern_, tokens);
@@ -2452,8 +2501,20 @@ int RAGAnalyzer::AnalyzeImpl(const Term &input, void *data, bool fine_grained, b
for (auto &t : tokens) {
if (t.empty())
continue;
if (IsStopword(t))
continue;
func(data, t.c_str(), t.size(), offset++, 0, false, 0);
}
}
return 0;
}
bool RAGAnalyzer::IsStopword(const std::string &term) {
// Mirrors rag/nlp/term_weight.py Dealer.stop_words in RAGFlow.
// Kept hard-coded (not loaded from a resource file) for parity with RAGFlow.
static const std::unordered_set<std::string> kStopwords = {
"请问", "您", "你", "我", "他", "是", "的", "就", "有", "于", "及", "即", "在", "为", "最",
"从", "以", "了", "将", "与", "吗", "吧", "中", "#", "什么", "怎么", "哪个", "哪些", "啥", "相关",
};
return kStopwords.find(term) != kStopwords.end();
}

29
internal/binding/cpp/rag_analyzer.h
View File

@@ -50,7 +50,9 @@ public:
~RAGAnalyzer();
void InitStemmer(Language language) { stemmer_->Init(language); }
void InitStemmer(Language language);
void SetLanguage(const std::string& language);
int32_t Load();
@@ -59,38 +61,41 @@ public:
void SetEnablePosition(bool enable_position) { enable_position_ = enable_position; }
std::pair<std::vector<std::string>, std::vector<std::pair<unsigned, unsigned>>> TokenizeWithPosition(
const std::string& line) const;
std::string Tokenize(const std::string& line) const;
const std::string& line);
std::string Tokenize(const std::string& line);
void FineGrainedTokenize(const std::string& tokens, std::vector<std::string>& result) const;
void FineGrainedTokenize(const std::string& tokens, std::vector<std::string>& result);
void TokenizeInnerWithPosition(const std::string& L,
std::vector<std::string>& tokens,
std::vector<std::pair<unsigned, unsigned>>& positions,
unsigned base_pos,
const std::vector<unsigned>* pos_mapping = nullptr) const;
const std::vector<unsigned>* pos_mapping = nullptr);
void FineGrainedTokenizeWithPosition(const std::string& tokens_str,
const std::vector<std::pair<unsigned, unsigned>>& positions,
std::vector<std::string>& fine_tokens,
std::vector<std::pair<unsigned, unsigned>>& fine_positions) const;
std::vector<std::pair<unsigned, unsigned>>& fine_positions);
void EnglishNormalizeWithPosition(const std::vector<std::string>& tokens,
const std::vector<std::pair<unsigned, unsigned>>& positions,
std::vector<std::string>& normalize_tokens,
std::vector<std::pair<unsigned, unsigned>>& normalize_positions) const;
std::vector<std::pair<unsigned, unsigned>>& normalize_positions);
unsigned MapToOriginalPosition(unsigned processed_pos,
const std::vector<std::pair<unsigned, unsigned>>& mapping) const;
const std::vector<std::pair<unsigned, unsigned>>& mapping);
void MergeWithPosition(const std::vector<std::string>& tokens,
const std::vector<std::pair<unsigned, unsigned>>& positions,
std::vector<std::string>& merged_tokens,
std::vector<std::pair<unsigned, unsigned>>& merged_positions) const;
std::vector<std::pair<unsigned, unsigned>>& merged_positions);
void SplitByLang(const std::string& line, std::vector<std::pair<std::string, bool>>& txt_lang_pairs) const;
int32_t Freq(std::string_view key) const;
std::string Tag(std::string_view key) const;
static bool IsStopword(const std::string& term);
protected:
int AnalyzeImpl(const Term& input, void* data, bool fine_grained, bool enable_position, HookType func) const;
int AnalyzeImpl(const Term& input, void* data, HookType func);
private:
static constexpr float DENOMINATOR = 1000000;
@@ -130,7 +135,7 @@ private:
[[nodiscard]] std::string Merge(const std::string& tokens) const;
void EnglishNormalize(const std::vector<std::string>& tokens, std::vector<std::string>& res) const;
void EnglishNormalize(const std::vector<std::string>& tokens, std::vector<std::string>& res);
public:
[[nodiscard]] std::vector<std::pair<std::vector<std::string_view>, double>> GetBestTokensTopN(
@@ -154,6 +159,8 @@ public:
mutable std::vector<char> lowercase_string_buffer_;
bool use_lemmatizer_{true};
bool fine_grained_{false};
bool enable_position_{false};

52
internal/binding/cpp/rag_analyzer_c_api.cpp
View File

@@ -44,6 +44,12 @@ void RAGAnalyzer_SetEnablePosition(RAGAnalyzerHandle handle, bool enable_positio
analyzer->SetEnablePosition(enable_position);
}
void RAGAnalyzer_SetLanguage(RAGAnalyzerHandle handle, const char* language) {
if (!handle || !language) return;
RAGAnalyzer* analyzer = static_cast<RAGAnalyzer*>(handle);
analyzer->SetLanguage(std::string(language));
}
int RAGAnalyzer_Analyze(RAGAnalyzerHandle handle, const char* text, RAGTokenCallback callback) {
if (!handle || !text || !callback) return -1;
@@ -88,43 +94,47 @@ RAGTokenList* RAGAnalyzer_TokenizeWithPosition(RAGAnalyzerHandle handle, const c
RAGAnalyzer* analyzer = static_cast<RAGAnalyzer*>(handle);
Term input;
input.text_ = std::string(text);
auto [tokens, positions] = analyzer->TokenizeWithPosition(std::string(text));
if (analyzer->fine_grained_) {
std::string joined_tokens;
for (size_t i = 0; i < tokens.size(); ++i) {
if (i > 0) joined_tokens += " ";
joined_tokens += tokens[i];
}
TermList output;
// Pass fine_grained and enable_position=true to get position information
analyzer->Analyze(input, output, analyzer->fine_grained_, true);
std::vector<std::string> fine_tokens;
std::vector<std::pair<unsigned, unsigned>> fine_positions;
analyzer->FineGrainedTokenizeWithPosition(joined_tokens, positions, fine_tokens, fine_positions);
tokens = std::move(fine_tokens);
positions = std::move(fine_positions);
}
// Allocate memory for the token list structure
RAGTokenList* token_list = static_cast<RAGTokenList*>(malloc(sizeof(RAGTokenList)));
if (!token_list) {
return nullptr;
}
// Allocate memory for the tokens array
token_list->tokens = nullptr;
token_list->count = static_cast<uint32_t>(tokens.size());
if (tokens.empty()) {
return token_list;
}
token_list->tokens = static_cast<RAGTokenWithPosition*>(
malloc(sizeof(RAGTokenWithPosition) * output.size())
malloc(sizeof(RAGTokenWithPosition) * tokens.size())
);
if (!token_list->tokens) {
free(token_list);
return nullptr;
}
token_list->count = static_cast<uint32_t>(output.size());
// Fill in the tokens
for (size_t i = 0; i < output.size(); ++i) {
// Allocate memory for the text and copy it
token_list->tokens[i].text = static_cast<char*>(
malloc(output[i].text_.size() + 1)
);
for (size_t i = 0; i < tokens.size(); ++i) {
token_list->tokens[i].text = static_cast<char*>(malloc(tokens[i].size() + 1));
if (token_list->tokens[i].text) {
std::memcpy(token_list->tokens[i].text,
output[i].text_.c_str(),
output[i].text_.size() + 1);
std::memcpy(token_list->tokens[i].text, tokens[i].c_str(), tokens[i].size() + 1);
}
token_list->tokens[i].offset = output[i].word_offset_;
token_list->tokens[i].end_offset = output[i].end_offset_;
token_list->tokens[i].offset = positions[i].first;
token_list->tokens[i].end_offset = positions[i].second;
}
return token_list;

3
internal/binding/cpp/rag_analyzer_c_api.h
View File

@@ -40,6 +40,9 @@ void RAGAnalyzer_SetFineGrained(RAGAnalyzerHandle handle, bool fine_grained);
// Set enable position tracking
void RAGAnalyzer_SetEnablePosition(RAGAnalyzerHandle handle, bool enable_position);
// Set the language for stemming (e.g. "English", "Dutch", "German")
void RAGAnalyzer_SetLanguage(RAGAnalyzerHandle handle, const char* language);
// Analyze text and call callback for each token
// Returns: 0 on success, negative value on failure
int RAGAnalyzer_Analyze(

7
internal/binding/cpp/rag_analyzer_c_api_debug.cpp
View File

@@ -112,6 +112,13 @@ void RAGAnalyzer_SetEnablePosition(RAGAnalyzerHandle handle, bool enable_positio
fprintf(stderr, "[C_API] SetEnablePosition: %d\n", enable_position);
}
void RAGAnalyzer_SetLanguage(RAGAnalyzerHandle handle, const char* language) {
if (!handle || !language) return;
RAGAnalyzer* analyzer = static_cast<RAGAnalyzer*>(handle);
analyzer->SetLanguage(std::string(language));
fprintf(stderr, "[C_API] SetLanguage: %s\n", language);
}
int RAGAnalyzer_Analyze(RAGAnalyzerHandle handle, const char* text, RAGTokenCallback callback) {
if (!handle || !text || !callback) return -1;

11
internal/binding/rag_analyzer.go
View File

@@ -96,6 +96,17 @@ func (a *Analyzer) SetEnablePosition(enablePosition bool) {
C.RAGAnalyzer_SetEnablePosition(a.handle, C.bool(enablePosition))
}
// SetLanguage configures the Snowball stemmer for the given language (e.g. "English", "Dutch").
// Falls back to the English Porter stemmer for unmapped languages.
func (a *Analyzer) SetLanguage(language string) {
if a.handle == nil {
return
}
cLang := C.CString(language)
defer C.free(unsafe.Pointer(cLang))
C.RAGAnalyzer_SetLanguage(a.handle, cLang)
}
// Analyze analyzes the input text and returns all tokens
func (a *Analyzer) Analyze(text string) ([]Token, error) {
if a.handle == nil {

2
internal/binding/rag_analyzer_nocgo.go
View File

@@ -48,6 +48,8 @@ func (a *Analyzer) SetFineGrained(bool) {}
func (a *Analyzer) SetEnablePosition(bool) {}
func (a *Analyzer) SetLanguage(string) {}
func (a *Analyzer) Analyze(text string) ([]Token, error) {
return nil, fmt.Errorf("rag_analyzer: cgo required for Analyze(%q)", text)
}