From 20a7c7d17a25844653d8103fdadf5e033aeca3a6 Mon Sep 17 00:00:00 2001
From: Yash Raj Pandey <55940078+devYRPauli@users.noreply.github.com>
Date: Sat, 18 Jul 2026 06:31:43 -0400
Subject: [PATCH] Fix: Q&A CSV parser splices wrong text when a field opens
 with a quote (#16881)

### Problem

Parsing a Q&A `.csv` can splice unrelated text into the wrong answers
(reported in #16791).

### Root cause

The `.csv` branch of `rag/app/qa.py`'s `chunk()` builds records with
`csv.reader(lines, delimiter=delimiter)` (default `quotechar='"'`), but
then indexes `lines[i]` by the reader's *record* index in `answer +=
"\n" + lines[i]`. When a line's field opens with a `"`, `csv.reader`
treats it as an unclosed quoted field and merges several physical lines
into one record. From there the record index permanently desyncs from
the physical line numbers, so `lines[i]` returns the wrong line and
unrelated Q&A content gets appended to the wrong answer.

### Reproduction (stdlib only)

```python
import csv
lines = 'Q1,A1\n"quoted answer start\ncontinues here,extra\nQ2,A2\n'.split("\n")
list(csv.reader(lines, delimiter=","))
# record 1 swallows 3 physical lines: ['quoted answer startcontinues here,extraQ2,A2']
# -> the reader index no longer matches lines[i]

list(csv.reader(lines, delimiter=",", quoting=csv.QUOTE_NONE))
# one physical line per record; index stays aligned
```

### Fix

Pass `quoting=csv.QUOTE_NONE` so one physical line maps to one record,
keeping the reader index aligned with `lines[i]` (the surrounding code
already relies on that 1:1 mapping).

Fixes #16791.

---------

Signed-off-by: Yash Raj Pandey <yashpn62@gmail.com>
Co-authored-by: Yingfeng <yingfeng.zhang@gmail.com>
---
 rag/app/qa.py                         |  8 ++++--
 test/unit_test/rag/app/test_qa_csv.py | 39 +++++++++++++++++++++++++++
 2 files changed, 45 insertions(+), 2 deletions(-)

diff --git a/rag/app/qa.py b/rag/app/qa.py
index f663d7142e..156ddda9f6 100644
--- a/rag/app/qa.py
+++ b/rag/app/qa.py
@@ -353,12 +353,16 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
         fails = []
         question, answer = "", ""
         res = []
-        reader = csv.reader(lines, delimiter=delimiter)
+        reader = csv.reader((line + "\n" for line in lines), delimiter=delimiter)
+        prev_line_num = 0
 
+        # line_num tracks the physical span when quoted fields cross lines.
         for i, row in enumerate(reader):
+            raw = "\n".join(lines[prev_line_num : reader.line_num])
+            prev_line_num = reader.line_num
             if len(row) != 2:
                 if question:
-                    answer += "\n" + lines[i]
+                    answer += "\n" + raw
                 else:
                     fails.append(str(i + 1))
             elif len(row) == 2:
diff --git a/test/unit_test/rag/app/test_qa_csv.py b/test/unit_test/rag/app/test_qa_csv.py
index 04897fe65b..b23ec81bf7 100644
--- a/test/unit_test/rag/app/test_qa_csv.py
+++ b/test/unit_test/rag/app/test_qa_csv.py
@@ -52,3 +52,42 @@ def test_csv_final_pair_uses_last_line_number():
     assert len(chunks) == 2
     assert chunks[0]["top_int"] == [1]
     assert chunks[1]["top_int"] == [2]
+
+
+@pytest.mark.p2
+def test_csv_quoted_comma_stays_in_one_field():
+    chunks = qa.chunk(
+        "qa.csv",
+        binary=b'"Question, one",Answer 1\nQuestion 2,Answer 2',
+        lang="English",
+        callback=_noop_callback,
+    )
+
+    assert len(chunks) == 2
+    assert chunks[0]["content_with_weight"] == "Question: Question, one\tAnswer: 1"
+
+
+@pytest.mark.p2
+def test_csv_quoted_field_preserves_embedded_newline():
+    chunks = qa.chunk(
+        "qa.csv",
+        binary=b'"first line\nsecond line",some answer\n',
+        lang="English",
+        callback=_noop_callback,
+    )
+
+    assert len(chunks) == 1
+    assert chunks[0]["content_with_weight"] == "Question: first line\nsecond line\tAnswer: some answer"
+
+
+@pytest.mark.p2
+def test_csv_multiline_quote_uses_physical_continuation_line():
+    chunks = qa.chunk(
+        "qa.csv",
+        binary=b'first,one\n"second\nquestion",two\ntwo continued\nthird,three',
+        lang="English",
+        callback=_noop_callback,
+    )
+
+    assert len(chunks) == 3
+    assert chunks[1]["content_with_weight"] == "Question: second\nquestion\tAnswer: two\ntwo continued"