From e098fcf6ad5e195e6b141f418f8049489e47ebc8 Mon Sep 17 00:00:00 2001
From: Kevin Hu <kevinhu.sh@gmail.com>
Date: Mon, 13 Jan 2025 12:03:18 +0800
Subject: [PATCH] Fix csv for TAG. (#4454)

### What problem does this PR solve?


### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)
---
 rag/app/tag.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/rag/app/tag.py b/rag/app/tag.py
index cd59ff7bd99..c1a15fafa3c 100644
--- a/rag/app/tag.py
+++ b/rag/app/tag.py
@@ -91,14 +91,14 @@ def chunk(filename, binary=None, lang="Chinese", callback=None, **kwargs):
         callback(0.1, "Start to parse.")
         txt = get_text(filename, binary)
         lines = txt.split("\n")
-        delimiter = "\t" if any("\t" in line for line in lines) else ","
 
         fails = []
         content = ""
         res = []
-        reader = csv.reader(lines, delimiter=delimiter)
+        reader = csv.reader(lines)
 
         for i, row in enumerate(reader):
+            row = [r.strip() for r in row if r.strip()]
             if len(row) != 2:
                 content += "\n" + lines[i]
             elif len(row) == 2: