zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-08-25 17:42:24 +08:00
Code Issues Packages Projects Releases Wiki Activity

Refactor: reformat all code for lefthook using ruff and gofmt (#16585)

Browse Source
This commit is contained in:
Wang Qi
2026-07-03 12:53:39 +08:00
committed by GitHub
parent 19fcb4a981
commit 6a4b9be426
588 changed files with 11123 additions and 15412 deletions
Download Patch File Download Diff File Expand all files Collapse all files

139
agent/component/message.py
View File

@@ -14,8 +14,10 @@
# limitations under the License.
#
import asyncio
try:
import nest_asyncio
nest_asyncio.apply()
except Exception:
pass
@@ -45,20 +47,14 @@ class MessageParam(ComponentParamBase):
"""
Define the Message component parameters.
"""
def __init__(self):
super().__init__()
self.content = []
self.stream = True
self.output_format = None # default output format
self.auto_play = False
self.outputs = {
"content": {
"type": "str"
},
"downloads": {
"type": "list"
}
}
self.outputs = {"content": {"type": "str"}, "downloads": {"type": "list"}}
def check(self):
self.check_empty(self.content, "[Message] Content")
@@ -71,9 +67,7 @@ class Message(ComponentBase):
@staticmethod
def _is_download_info(value: Any) -> bool:
return isinstance(value, dict) and all(
key in value for key in ("doc_id", "filename", "mime_type")
)
return isinstance(value, dict) and all(key in value for key in ("doc_id", "filename", "mime_type"))
@staticmethod
def _download_info_includes_content(value: Any) -> bool:
@@ -157,7 +151,7 @@ class Message(ComponentBase):
delimiter: str = None,
downloads: list[dict[str, Any]] | None = None,
) -> tuple[str, dict[str, str | list | Any]]:
for k,v in self.get_input_elements_from_text(script).items():
for k, v in self.get_input_elements_from_text(script).items():
if k in kwargs:
continue
v = v["value"]
@@ -191,7 +185,7 @@ class Message(ComponentBase):
buf += t
return buf
async def _stream(self, rand_cnt:str):
async def _stream(self, rand_cnt: str):
s = 0
all_content = ""
cache = {}
@@ -200,8 +194,8 @@ class Message(ComponentBase):
if self.check_if_canceled("Message streaming"):
return
all_content += rand_cnt[s: r.start()]
yield rand_cnt[s: r.start()]
all_content += rand_cnt[s : r.start()]
yield rand_cnt[s : r.start()]
s = r.end()
exp = r.group(1)
if exp in cache:
@@ -235,9 +229,7 @@ class Message(ComponentBase):
continue
elif inspect.isawaitable(v):
v = await v
v = self._stringify_message_value(
v, downloads=downloads, fallback_to_str=True
)
v = self._stringify_message_value(v, downloads=downloads, fallback_to_str=True)
yield v
self.set_input_value(exp, v)
all_content += v
@@ -247,21 +239,19 @@ class Message(ComponentBase):
if self.check_if_canceled("Message streaming"):
return
all_content += rand_cnt[s: ]
yield rand_cnt[s: ]
all_content += rand_cnt[s:]
yield rand_cnt[s:]
self.set_output("downloads", downloads)
self.set_output("content", all_content)
self._convert_content(all_content)
await self._save_to_memory(all_content)
def _is_jinjia2(self, content:str) -> bool:
patt = [
r"\{%.*%\}", "{{", "}}"
]
def _is_jinjia2(self, content: str) -> bool:
patt = [r"\{%.*%\}", "{{", "}}"]
return any([re.search(p, content) for p in patt])
@timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10*60)))
@timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10 * 60)))
def _invoke(self, **kwargs):
if self.check_if_canceled("Message processing"):
return
@@ -303,19 +293,19 @@ class Message(ComponentBase):
def _parse_markdown_table_lines(self, table_lines: list):
"""
Parse a list of Markdown table lines into a pandas DataFrame.
Args:
table_lines: List of strings, each representing a row in the Markdown table
(excluding separator lines like |---|---|)
Returns:
pandas DataFrame with the table data, or None if parsing fails
"""
import pandas as pd
if not table_lines:
return None
rows = []
headers = None
@@ -350,29 +340,29 @@ class Message(ComponentBase):
return cell
return cell
for line in table_lines:
# Split by | and clean up
cells = [cell.strip() for cell in line.split('|')]
cells = [cell.strip() for cell in line.split("|")]
# Remove empty first and last elements from split (caused by leading/trailing |)
cells = [c for c in cells if c]
if headers is None:
headers = cells
else:
cells = [_coerce_excel_cell_type(c) for c in cells]
rows.append(cells)
if headers and rows:
# Ensure all rows have same number of columns as headers
normalized_rows = []
for row in rows:
while len(row) < len(headers):
row.append('')
normalized_rows.append(row[:len(headers)])
row.append("")
normalized_rows.append(row[: len(headers)])
return pd.DataFrame(normalized_rows, columns=headers)
return None
def _convert_content(self, content):
@@ -380,6 +370,7 @@ class Message(ComponentBase):
return
import pypandoc
doc_id = get_uuid()
if self._param.output_format.lower() not in {"markdown", "html", "pdf", "docx", "xlsx"}:
@@ -408,49 +399,47 @@ class Message(ComponentBase):
# Debug: log the content being parsed
logging.info(f"XLSX Parser: Content length={len(content) if content else 0}, first 500 chars: {content[:500] if content else 'None'}")
# Try to parse ALL Markdown tables from the content
# Each table will be written to a separate sheet
tables = [] # List of (sheet_name, dataframe)
if isinstance(content, str):
lines = content.strip().split('\n')
lines = content.strip().split("\n")
logging.info(f"XLSX Parser: Total lines={len(lines)}, lines starting with '|': {sum(1 for line in lines if line.strip().startswith('|'))}")
current_table_lines = []
current_table_title = None
pending_title = None
in_table = False
table_count = 0
for i, line in enumerate(lines):
stripped = line.strip()
# Check for potential table title (lines before a table)
# Look for patterns like "Table 1:", "## Table", or markdown headers
if not in_table and stripped and not stripped.startswith('|'):
if not in_table and stripped and not stripped.startswith("|"):
# Check if this could be a table title
lower_stripped = stripped.lower()
if (lower_stripped.startswith('table') or
stripped.startswith('#') or
':' in stripped):
pending_title = stripped.lstrip('#').strip()
if stripped.startswith('|') and '|' in stripped[1:]:
if lower_stripped.startswith("table") or stripped.startswith("#") or ":" in stripped:
pending_title = stripped.lstrip("#").strip()
if stripped.startswith("|") and "|" in stripped[1:]:
# Check if this is a separator line (|---|---|)
cleaned = stripped.replace(' ', '').replace('|', '').replace('-', '').replace(':', '')
if cleaned == '':
cleaned = stripped.replace(" ", "").replace("|", "").replace("-", "").replace(":", "")
if cleaned == "":
continue # Skip separator line
if not in_table:
# Starting a new table
in_table = True
current_table_lines = []
current_table_title = pending_title
pending_title = None
current_table_lines.append(stripped)
elif in_table and not stripped.startswith('|'):
elif in_table and not stripped.startswith("|"):
# End of current table - save it
if current_table_lines:
df = self._parse_markdown_table_lines(current_table_lines)
@@ -460,24 +449,22 @@ class Message(ComponentBase):
if current_table_title:
# Clean and truncate title for sheet name
sheet_name = current_table_title[:31]
sheet_name = sheet_name.replace('/', '_').replace('\\', '_').replace('*', '').replace('?', '').replace('[', '').replace(']', '').replace(':', '')
sheet_name = sheet_name.replace("/", "_").replace("\\", "_").replace("*", "").replace("?", "").replace("[", "").replace("]", "").replace(":", "")
else:
sheet_name = f"Table_{table_count}"
tables.append((sheet_name, df))
# Reset for next table
in_table = False
current_table_lines = []
current_table_title = None
# Check if this line could be a title for the next table
if stripped:
lower_stripped = stripped.lower()
if (lower_stripped.startswith('table') or
stripped.startswith('#') or
':' in stripped):
pending_title = stripped.lstrip('#').strip()
if lower_stripped.startswith("table") or stripped.startswith("#") or ":" in stripped:
pending_title = stripped.lstrip("#").strip()
# Don't forget the last table if content ends with a table
if in_table and current_table_lines:
df = self._parse_markdown_table_lines(current_table_lines)
@@ -485,11 +472,11 @@ class Message(ComponentBase):
table_count += 1
if current_table_title:
sheet_name = current_table_title[:31]
sheet_name = sheet_name.replace('/', '_').replace('\\', '_').replace('*', '').replace('?', '').replace('[', '').replace(']', '').replace(':', '')
sheet_name = sheet_name.replace("/", "_").replace("\\", "_").replace("*", "").replace("?", "").replace("[", "").replace("]", "").replace(":", "")
else:
sheet_name = f"Table_{table_count}"
tables.append((sheet_name, df))
# Fallback: if no tables found, create single sheet with content
if not tables:
df = pd.DataFrame({"Content": [content if content else ""]})
@@ -497,7 +484,7 @@ class Message(ComponentBase):
# Write all tables to Excel, each in a separate sheet
excel_io = BytesIO()
with pd.ExcelWriter(excel_io, engine='openpyxl') as writer:
with pd.ExcelWriter(excel_io, engine="openpyxl") as writer:
used_names = set()
for sheet_name, df in tables:
# Ensure unique sheet names
@@ -505,14 +492,14 @@ class Message(ComponentBase):
counter = 1
while sheet_name in used_names:
suffix = f"_{counter}"
sheet_name = original_name[:31-len(suffix)] + suffix
sheet_name = original_name[: 31 - len(suffix)] + suffix
counter += 1
used_names.add(sheet_name)
df.to_excel(writer, sheet_name=sheet_name, index=False)
excel_io.seek(0)
binary_content = excel_io.read()
logging.info(f"Generated Excel with {len(tables)} sheet(s): {[t[0] for t in tables]}")
else: # pdf, docx
@@ -543,10 +530,7 @@ class Message(ComponentBase):
os.remove(tmp_name)
settings.STORAGE_IMPL.put(self._canvas._tenant_id, doc_id, binary_content)
self.set_output("attachment", {
"doc_id":doc_id,
"format":self._param.output_format,
"file_name":f"{doc_id[:8]}.{self._param.output_format}"})
self.set_output("attachment", {"doc_id": doc_id, "format": self._param.output_format, "file_name": f"{doc_id[:8]}.{self._param.output_format}"})
logging.info(f"Converted content uploaded as {doc_id} (format={self._param.output_format})")
@@ -560,15 +544,10 @@ class Message(ComponentBase):
user_id = self._param.user_id if hasattr(self._param, "user_id") else ""
if user_id:
import re
# is variable
if re.match(r"^{.*}$", user_id):
user_id = self._canvas.get_variable_value(user_id)
message_dict = {
"user_id": user_id,
"agent_id": self._canvas._id,
"session_id": self._canvas.task_id,
"user_input": self._canvas.get_sys_query(),
"agent_response": content
}
message_dict = {"user_id": user_id, "agent_id": self._canvas._id, "session_id": self._canvas.task_id, "user_input": self._canvas.get_sys_query(), "agent_response": content}
return await queue_save_to_memory_task(self._param.memory_ids, message_dict)