31 lines
1.0 KiB
Python
31 lines
1.0 KiB
Python
|
import unittest
|
||
|
|
|
||
|
|
from rag.cleaning import clean_document_text
|
||
|
|
|
||
|
|
|
||
|
|
class CleaningTests(unittest.TestCase):
|
||
|
|
def test_normalizes_bom_line_endings_whitespace_and_blank_lines(self):
|
||
|
|
result = clean_document_text("\ufeff标题\r\n\r\n\r\n段落\t 内容\x00")
|
||
|
|
|
||
|
|
self.assertEqual(result.text, "标题\n\n段落 内容")
|
||
|
|
self.assertTrue(result.changed)
|
||
|
|
self.assertTrue(result.warnings)
|
||
|
|
|
||
|
|
def test_removes_standalone_page_markers_but_keeps_business_numbers(self):
|
||
|
|
result = clean_document_text("产品代码 110011\n第 1 页\n年化收益率 3.5%\nPage 2")
|
||
|
|
|
||
|
|
self.assertEqual(result.text, "产品代码 110011\n年化收益率 3.5%")
|
||
|
|
self.assertIn("110011", result.text)
|
||
|
|
|
||
|
|
def test_does_not_remove_meaningful_content(self):
|
||
|
|
text = "基金名称：示例基金\n风险等级：R3\n申购费率：1.2%"
|
||
|
|
|
||
|
|
result = clean_document_text(text)
|
||
|
|
|
||
|
|
self.assertEqual(result.text, text)
|
||
|
|
self.assertFalse(result.changed)
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
unittest.main()
|