import unittest

from rag.cleaning import clean_document_text


class CleaningTests(unittest.TestCase):
    def test_normalizes_bom_line_endings_whitespace_and_blank_lines(self):
        result = clean_document_text("\ufeff标题\r\n\r\n\r\n段落\t  内容\x00")

        self.assertEqual(result.text, "标题\n\n段落 内容")
        self.assertTrue(result.changed)
        self.assertTrue(result.warnings)

    def test_removes_standalone_page_markers_but_keeps_business_numbers(self):
        result = clean_document_text("产品代码 110011\n第 1 页\n年化收益率 3.5%\nPage 2")

        self.assertEqual(result.text, "产品代码 110011\n年化收益率 3.5%")
        self.assertIn("110011", result.text)

    def test_does_not_remove_meaningful_content(self):
        text = "基金名称：示例基金\n风险等级：R3\n申购费率：1.2%"

        result = clean_document_text(text)

        self.assertEqual(result.text, text)
        self.assertFalse(result.changed)


if __name__ == "__main__":
    unittest.main()