zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-07-11 14:15:40 +08:00
Code Issues Packages Projects Releases Wiki Activity

Go: format code (#16813)

Browse Source 
### Summary

Aligned to EE

Signed-off-by: Jin Hai <haijin.chn@gmail.com>
This commit is contained in:
Jin Hai
2026-07-10 20:30:00 +08:00
committed by GitHub
parent 106c2d0a41
commit 0aa2993d55
27 changed files with 172 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
internal/binding/cpp/darts_trie.cpp
View File

@@ -107,4 +107,4 @@ int DartsTrie::Get(std::string_view key) const {
key = empty_null_terminated_sv;
}
return darts_->exactMatchSearch<DartsCore::value_type>(key.data(), key.size());
}
}

2
internal/binding/cpp/main.cpp
View File

@@ -439,4 +439,4 @@ int main() {
// test_fine_grained_tokenize_consistency_with_python();
test_tokenize_text("在本研究中，我们提出了一种novel的neural network架构，用于解决multi-modal learning问题。我们的方法结合了CNN(Convolutional Neural Networks)和Transformer的优势，在ImageNet数据集上达到了state-of-the-art性能。实验结果表明，在batch size为256、learning rate为0.001的条件下，我们的模型在validation set上的accuracy达到了95.7%，比baseline方法提高了3.2%。此外，我们还进行了ablation study来分析不同components的contribution。所有代码已在GitHub上开源，地址是https://github.com/example/our-project。未来工作将集中在model compression和real-time inference optimization上。");
return 0;
}
}

2
internal/binding/cpp/pcre2.h
View File

@@ -43,7 +43,7 @@ POSSIBILITY OF SUCH DAMAGE.
#define PCRE2_MAJOR 10
#define PCRE2_MINOR 47
#define PCRE2_PRERELEASE
#define PCRE2_PRERELEASE
#define PCRE2_DATE 2025-10-21
/* When an application links to a PCRE2 DLL in Windows, the symbols that are

2
internal/binding/cpp/rag_analyzer.cpp
View File

@@ -2450,4 +2450,4 @@ int RAGAnalyzer::AnalyzeImpl(const Term &input, void *data, bool fine_grained, b
}
}
return 0;
}
}

24
internal/binding/cpp/rag_analyzer_c_api_debug.cpp
View File

@@ -114,28 +114,28 @@ void RAGAnalyzer_SetEnablePosition(RAGAnalyzerHandle handle, bool enable_positio
int RAGAnalyzer_Analyze(RAGAnalyzerHandle handle, const char* text, RAGTokenCallback callback) {
if (!handle || !text || !callback) return -1;
fprintf(stderr, "[C_API] Analyze called with text length: %zu\n", strlen(text));
RAGAnalyzer* analyzer = static_cast<RAGAnalyzer*>(handle);
Term input;
input.text_ = std::string(text);
TermList output;
int ret = analyzer->Analyze(input, output);
fprintf(stderr, "[C_API] Analyze returned: %d, tokens: %zu\n", ret, output.size());
if (ret != 0) {
return ret;
}
// Call callback for each token
for (const auto& term : output) {
callback(term.text_.c_str(), term.text_.length(), term.word_offset_, term.end_offset_);
}
return 0;
}
@@ -144,13 +144,13 @@ char* RAGAnalyzer_Tokenize(RAGAnalyzerHandle handle, const char* text) {
fprintf(stderr, "[C_API] Tokenize called with null handle or text\n");
return nullptr;
}
fprintf(stderr, "[C_API] Tokenize called with text length: %zu\n", strlen(text));
RAGAnalyzer* analyzer = static_cast<RAGAnalyzer*>(handle);
std::string result = analyzer->Tokenize(std::string(text));
// Allocate memory for C string
char* c_result = static_cast<char*>(DEBUG_MALLOC(result.size() + 1));
if (c_result) {

40
internal/binding/cpp/rag_analyzer_c_test.cpp
View File

@@ -8,38 +8,38 @@
// Test case 1: Single thread, loop 1000 times
void test_single_thread() {
std::cout << "Test 1: Single thread, 1000 iterations..." << std::endl;
// Create analyzer instance
RAGAnalyzerHandle handle = RAGAnalyzer_Create("./resource");
assert(handle != nullptr && "Failed to create RAGAnalyzer");
// Load the analyzer
int result = RAGAnalyzer_Load(handle);
if (result != 0) {
printf("Failed to load RAGAnalyzer: %d\n", result);
}
assert(result == 0 && "Failed to load RAGAnalyzer");
const char* input = "rag";
bool all_passed = true;
for (int i = 0; i < 1000; ++i) {
char* tokens = RAGAnalyzer_Tokenize(handle, input);
if (tokens == nullptr || strlen(tokens) == 0) {
std::cerr << "Iteration " << i << ": Failed - returned empty or null string" << std::endl;
all_passed = false;
}
// Free the returned string
if (tokens != nullptr) {
free(tokens);
}
}
// Destroy analyzer instance
RAGAnalyzer_Destroy(handle);
if (all_passed) {
std::cout << "Test 1: PASSED" << std::endl;
} else {
@@ -51,32 +51,32 @@ void test_single_thread() {
// Test case 2: 16 threads, each loop 1000 times
void test_multi_thread() {
std::cout << "Test 2: 32 threads, each 100000 iterations..." << std::endl;
// Create analyzer instance (shared across threads)
RAGAnalyzerHandle handle = RAGAnalyzer_Create(".");
assert(handle != nullptr && "Failed to create RAGAnalyzer");
// Load the analyzer
int result = RAGAnalyzer_Load(handle);
assert(result == 0 && "Failed to load RAGAnalyzer");
const char* input = "rag";
const int num_threads = 32;
const int iterations_per_thread = 100000;
std::vector<std::thread> threads;
std::vector<bool> thread_results(num_threads, true);
for (int t = 0; t < num_threads; ++t) {
threads.emplace_back([&, t]() {
for (int i = 0; i < iterations_per_thread; ++i) {
char* tokens = RAGAnalyzer_Tokenize(handle, input);
if (tokens == nullptr || strlen(tokens) == 0) {
std::cerr << "Thread " << t << " Iteration " << i << ": Failed - returned empty or null string" << std::endl;
thread_results[t] = false;
}
// Free the returned string
if (tokens != nullptr) {
free(tokens);
@@ -84,15 +84,15 @@ void test_multi_thread() {
}
});
}
// Wait for all threads to complete
for (auto& t : threads) {
t.join();
}
// Destroy analyzer instance
RAGAnalyzer_Destroy(handle);
bool all_passed = true;
for (int t = 0; t < num_threads; ++t) {
if (!thread_results[t]) {
@@ -100,7 +100,7 @@ void test_multi_thread() {
break;
}
}
if (all_passed) {
std::cout << "Test 2: PASSED" << std::endl;
} else {
@@ -115,7 +115,7 @@ int main() {
test_single_thread();
// test_multi_thread();
std::cout << "=== All tests PASSED ===" << std::endl;
return 0;
}

2
internal/binding/cpp/term.cpp
View File

@@ -21,4 +21,4 @@ void Term::Reset() {
word_offset_ = 0;
}
Term TermList::global_temporary_;
Term TermList::global_temporary_;

46
internal/binding/cpp/thinc_parser.cpp
View File

@@ -165,7 +165,7 @@ struct Tok2vecModel {
int po_nO=96,po_nP=3,po_nI=576;
struct ResBlk{bool has=false;std::vector<float>W,b,lnG,lnb;};
ResBlk res[4]; int n_res=0;
bool load(const std::string& dir) {
std::ifstream cf(dir+"/model.ckpt"); if(!cf)return false;
std::stringstream cb;cb<<cf.rdbuf();
@@ -206,7 +206,7 @@ struct Tok2vecModel {
if(ld(pk+"W",&rb.W,&r0,&r1,&r2)){ld(pk+"B",&rb.b);ld(pk+"lnG",&rb.lnG);ld(pk+"lnb",&rb.lnb);rb.has=true;n_res++;}}
return!embeds.empty();
}
// Run tok2vec → (n_tokens, 96)
void forward(const std::vector<std::string>& tokens, float* out) {
int n=(int)tokens.size(),D=96,NE=(int)embeds.size(),EC=NE*D;
@@ -241,7 +241,7 @@ struct ParserModel {
std::vector<float> pW_pre,pb_pre,pad_pre; // preaffine
std::vector<float> pW_cls,pb_cls; // classifier
std::vector<std::string> move_names;
bool load(const std::string& dir) {
std::ifstream cf(dir+"/model.ckpt"); if(!cf)return false;
std::stringstream cb;cb<<cf.rdbuf();
@@ -275,7 +275,7 @@ struct ParserModel {
if(mn&&mn->type==JVal::ARR)for(auto& v:mn->arr)move_names.push_back(v.str);}
return!pW_hid.empty() && !pW_pre.empty() && !pW_cls.empty();
}
// Run parser forward + state machine → (heads, labels)
void parse(const float* tokvecs, int n_tokens,
std::vector<int>& out_heads, std::vector<std::string>& out_labels) {
@@ -283,7 +283,7 @@ struct ParserModel {
std::vector<float> hidden((size_t)n_tokens*nO,0);
for(int i=0;i<n_tokens;i++) linear(hidden.data()+(size_t)i*nO, tokvecs+(size_t)i*96,
pW_hid.data(), pb_hid.data(), nO, 96);
// 2. Pre-compute features
std::vector<float> precomp((size_t)(n_tokens+1)*nP*nO*nI,0);
// Pad token (index 0)
@@ -304,20 +304,20 @@ struct ParserModel {
}
}
}
// Helper: get feature value for a token at piece p, window w, output dim o
auto feat = [&](int idx, int p, int w, int o) -> float {
int ri = (idx < 0 || idx >= n_tokens) ? 0 : idx + 1;
return precomp[(size_t)ri*nP*nO*nI + (size_t)p*nO*nI + (size_t)w + (size_t)o*nI];
};
// 3. Arc-hybrid state machine
out_heads.assign(n_tokens, -1);
out_labels.assign(n_tokens, "");
std::vector<int> stack;
std::vector<int> buffer(n_tokens);
for(int i=0;i<n_tokens;i++) buffer[i]=i;
// Validate move_names covers all actions before indexing
if((int)move_names.size()!=n_actions){out_heads.clear();out_labels.clear();return;}
int act_S=-1, act_D=-1;
@@ -325,7 +325,7 @@ struct ParserModel {
if(move_names[i]=="S") act_S=i;
if(move_names[i]=="D") act_D=i;
}
auto leftmost = [&](int idx)->int{
for(int i=0;i<n_tokens;i++) if(out_heads[i]==idx) return i;
return -1;
@@ -334,21 +334,21 @@ struct ParserModel {
int r=-1; for(int i=0;i<n_tokens;i++) if(out_heads[i]==idx) r=i;
return r;
};
std::vector<float> scores(n_actions,0);
std::vector<float> feats(nP*nI*nO,0);
for(int step=0; step<n_tokens*4 && !(buffer.empty()&&stack.size()<=1); step++){
int s0=stack.empty()?-1:stack.back();
int s1=stack.size()<2?-1:stack[stack.size()-2];
int s2=stack.size()<3?-1:stack[stack.size()-3];
int b0=buffer.empty()?-1:buffer[0];
int b1=buffer.size()<2?-1:buffer[1];
// Build feature indices (same as verified Python implementation)
int idxs[16]={s0,s1, b0,s0, s0,leftmost(s0), s0,rightmost(s0),
s1,leftmost(s1), s1,rightmost(s1), s2,b1, b0,b1};
// Build feature vector: sum of precomputed features at each (idx, piece, window)
for(int o=0;o<nO;o++) feats[o]=0;
for(int p=0;p<nP;p++){
@@ -359,7 +359,7 @@ struct ParserModel {
}
}
}
// Classify
for(int a=0;a<n_actions;a++){
float s=pb_cls[a];
@@ -368,7 +368,7 @@ struct ParserModel {
for(int j=0;j<nO;j++) s += pW_cls[(size_t)a*nO+j] * hidden[(size_t)cls_idx*nO+j];
scores[a]=s;
}
// Pick best VALID action
int best=-1; float best_sc=-1e30f;
for(int a=0;a<n_actions;a++){
@@ -380,7 +380,7 @@ struct ParserModel {
if(valid && scores[a]>best_sc){best_sc=scores[a];best=a;}
}
if(best<0) break;
const std::string& act=move_names[best];
if(act=="S"){stack.push_back(buffer[0]);buffer.erase(buffer.begin());}
else if(act=="D"){stack.pop_back();}
@@ -432,23 +432,23 @@ void ThincParser_Destroy(ThincParserHandle h) { delete (ParserState*)h; }
char* ThincParser_Predict(ThincParserHandle h, const char* tokens_json) {
auto* s=(ParserState*)h;
if(!s||!s->loaded||!tokens_json) return strdup("[]");
auto j=JParser().parse(std::string(tokens_json));
if(j.type!=JVal::ARR) return strdup("[]");
std::vector<std::string> tokens;
for(auto& v:j.arr) tokens.push_back(v.str);
int n=(int)tokens.size();
if(!n) return strdup("[]");
// Run tok2vec
std::vector<float> tokvecs((size_t)n*96,0);
s->tok2vec.forward(tokens, tokvecs.data());
// Run parser
std::vector<int> heads;
std::vector<std::string> labels;
s->parser.parse(tokvecs.data(), n, heads, labels);
// Build JSON output
std::string r="[";
for(int i=0;i<n;i++){
@@ -507,11 +507,11 @@ char* ThincTagger_Predict(ThincTaggerHandle h, const char* tokens_json) {
for(auto& v:j.arr) tokens.push_back(v.str);
int n=(int)tokens.size(), n_tags=(int)s->tW.size()/96;
if(!n||!n_tags)return strdup("[]");
// Run tok2vec to get 96-dim embeddings, then softmax + argmax
std::vector<float> tokvecs((size_t)n*96,0);
s->tok2vec.forward(tokens, tokvecs.data());
std::vector<int> best_tags(n, 0);
for(int i=0;i<n;i++){
float best_sc=-1e30f;
@@ -521,7 +521,7 @@ char* ThincTagger_Predict(ThincTaggerHandle h, const char* tokens_json) {
if(sc>best_sc){best_sc=sc;best_tags[i]=t;}
}
}
// Strip morphologizer output to just POS (e.g. "Gender=Masc|Number=Sing|POS=NOUN" → "NOUN")
// For non-morphologizer models the tag string is used as-is.
auto pos_only = [](const std::string& t) -> std::string {

2
internal/binding/cpp/tokenizer.cpp
View File

@@ -312,4 +312,4 @@ bool Tokenizer::TokenizeWhite(const std::string &input_string, TermList &raw_ter
}
return true;
}
}

2
internal/binding/cpp/wordnet_lemmatizer.cpp
View File

@@ -229,4 +229,4 @@ std::string WordNetLemmatizer::Lemmatize(const std::string &form, const std::str
}
return form;
}
}