first commit

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-07-16 11:12:17 +08:00
commit 4003624b8c
80 changed files with 9990 additions and 0 deletions
@@ -0,0 +1,98 @@
+"""Tests for PDF text merge and heading classification."""
+
+from __future__ import annotations
+
+import unittest
+
+from rag_cut.layout_meta import enrich_layout_metadata
+from rag_cut.models import Block, BlockType
+from rag_cut.parsers.pdf.layout import TextLine, _merge_lines_to_paragraphs
+from rag_cut.parsers.pdf.text_extract import MAX_HEADING_CHARS, _font_heading_level, _looks_like_ui_label
+from rag_cut.parsers.pdf.noise_filter import is_margin_noise_block
+from rag_cut.splitters.heading_splitter import is_heading_block, normalize_heading_block, numbered_heading_level
+
+
+class ParagraphMergeTest(unittest.TestCase):
+    def test_does_not_merge_title_and_body_with_different_font_sizes(self) -> None:
+        lines = [
+            TextLine(x0=90, y0=90, x1=400, y1=110, text="Master Collection/Pattern Scanning", font_size=18),
+            TextLine(x0=90, y0=115, x1=480, y1=132, text="及所有大師/形態/策略指標的位置", font_size=17.5),
+            TextLine(
+                x0=90,
+                y0=145,
+                x1=500,
+                y1=200,
+                text="頁面左側【篩選】欄下方即是【Master Collection】文件夾。點擊此檔夾即可出現下拉頁面。",
+                font_size=11,
+            ),
+        ]
+        merged = _merge_lines_to_paragraphs(lines, page_width=600)
+        texts = [ln.text for ln in merged]
+        self.assertEqual(len(merged), 2)
+        self.assertIn("Master Collection", texts[0])
+        self.assertTrue(texts[1].startswith("頁面左側"))
+        self.assertNotIn("頁面左側", texts[0])
+
+
+class FontHeadingTest(unittest.TestCase):
+    def test_short_large_text_is_heading(self) -> None:
+        level = _font_heading_level(18, 11, "Master Collection 位置說明")
+        self.assertIsNotNone(level)
+
+    def test_title_plus_body_blob_is_not_heading(self) -> None:
+        blob = (
+            "Master Collection/Pattern Scanning/Strategy Scanning 及所有大師/形態/策略指標的位置 "
+            "頁面左側【篩選】欄下方即是【Master Collection】文件夾。點擊此檔夾即可出現下拉頁面，"
+            "其中包含了十五個大師的選股策略。如圖所示："
+        )
+        self.assertGreater(len(blob), MAX_HEADING_CHARS)
+        self.assertIsNone(_font_heading_level(18, 11, blob))
+
+    def test_compact_numbered_chinese_title_is_heading(self) -> None:
+        # FAQ style: "10.上升三角形態" with no space after the dot.
+        level = _font_heading_level(16, 12, "10.上升三角形態")
+        self.assertEqual(level, 1)
+        self.assertFalse(_looks_like_ui_label("10.上升三角形態"))
+
+    def test_cjk_section_banner_is_level1_not_ui_label(self) -> None:
+        self.assertFalse(_looks_like_ui_label("形態指標"))
+        self.assertEqual(_font_heading_level(16, 12, "形態指標"), 1)
+
+
+
+class HeadingDemoteTest(unittest.TestCase):
+    def test_normalize_demotes_overlong_heading(self) -> None:
+        blob = "T" * (MAX_HEADING_CHARS + 20)
+        block = Block(type=BlockType.HEADING, text=blob, level=1, meta={"page": 1})
+        normalized = normalize_heading_block(block)
+        self.assertEqual(normalized.type, BlockType.PARAGRAPH)
+        self.assertFalse(is_heading_block(normalized))
+
+    def test_enrich_demotes_overlong_heading_block(self) -> None:
+        blob = (
+            "Master Collection/Pattern Scanning/Strategy Scanning Master Collection/"
+            "Pattern Scanning/Strategy Scanning 及所有大師 /形態/策略指標的位置 "
+            "頁面左側【篩選】欄下方即是【Master Collection】文件夾。點擊此檔夾即可出 "
+            "現下拉頁面，其中包含了十五個大師的選股策略。"
+        )
+        blocks = [
+            Block(type=BlockType.HEADING, text=blob, level=1, meta={"page": 1, "bbox": [90, 90, 500, 240]}),
+            Block(type=BlockType.IMAGE, image_id="a.png", image_path="a.png", meta={"page": 1, "bbox": [90, 250, 500, 480]}),
+        ]
+        enriched = enrich_layout_metadata(blocks)
+        self.assertEqual(enriched[0].type, BlockType.PARAGRAPH)
+        self.assertNotEqual(enriched[1].meta.get("bound_heading"), blob)
+
+    def test_top_of_page_numbered_title_not_margin_noise(self) -> None:
+        block = Block(
+            type=BlockType.PARAGRAPH,
+            text="10.上升三角形態",
+            meta={"page": 50, "bbox": [90, 74, 209, 90], "page_height": 842},
+        )
+        self.assertFalse(is_margin_noise_block(block, 842))
+        self.assertEqual(numbered_heading_level("10.上升三角形態"), 1)
+        self.assertTrue(is_heading_block(normalize_heading_block(block)))
+
+
+if __name__ == "__main__":
+    unittest.main()