diff --git a/rag/app/tag.py b/rag/app/tag.py index 2c3a27785e..656226a792 100644 --- a/rag/app/tag.py +++ b/rag/app/tag.py @@ -96,12 +96,16 @@ def chunk(filename, binary=None, lang="Chinese", callback=None, **kwargs): fails = [] content = "" res = [] - reader = csv.reader(lines) + reader = csv.reader((line + "\n" for line in lines)) + prev_line_num = 0 + # line_num tracks the physical span when quoted fields cross lines. for i, row in enumerate(reader): + raw = "\n".join(lines[prev_line_num : reader.line_num]) + prev_line_num = reader.line_num row = [r.strip() for r in row if r.strip()] if len(row) != 2: - content += "\n" + lines[i] + content += "\n" + raw elif len(row) == 2: content += "\n" + row[0] res.append(beAdoc(deepcopy(doc), content, row[1], eng, i))