#
#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#

"""Unit tests for PDF garbled text detection and layout garbage filtering.

Tests cover:
- RAGFlowPdfParser static methods: _is_garbled_char, _is_garbled_text,
  _has_subset_font_prefix, _is_garbled_by_font_encoding
- layout_recognizer.__is_garbage: CID pattern filtering
"""

import re
import sys
import os
import importlib.util
from unittest import mock

# Import RAGFlowPdfParser directly by file path to avoid triggering
# deepdoc/parser/__init__.py which pulls in heavy dependencies
# (pdfplumber, xgboost, etc.) that may not be available in test environments.
#
# We mock the heavy third-party modules so that pdf_parser.py can be loaded
# purely for its static detection methods.
_MOCK_MODULES = [
    "numpy",
    "np",
    "pdfplumber",
    "xgboost",
    "xgb",
    "huggingface_hub",
    "PIL",
    "PIL.Image",
    "pypdf",
    "sklearn",
    "sklearn.cluster",
    "sklearn.metrics",
    "common",
    "common.file_utils",
    "common.misc_utils",
    "common.settings",
    "common.token_utils",
    "deepdoc",
    "deepdoc.vision",
    "deepdoc.parser",
    "rag",
    "rag.nlp",
    "rag.prompts",
    "rag.prompts.generator",
]
for _m in _MOCK_MODULES:
    if _m not in sys.modules:
        sys.modules[_m] = mock.MagicMock()


def _find_project_root(marker="pyproject.toml"):
    """Walk up from this file until a directory containing *marker* is found."""
    cur = os.path.dirname(os.path.abspath(__file__))
    while True:
        if os.path.exists(os.path.join(cur, marker)):
            return cur
        parent = os.path.dirname(cur)
        if parent == cur:
            raise FileNotFoundError(f"Could not locate project root (missing {marker})")
        cur = parent


_MODULE_PATH = os.path.join(_find_project_root(), "deepdoc", "parser", "pdf_parser.py")
_spec = importlib.util.spec_from_file_location("pdf_parser", _MODULE_PATH)
_mod = importlib.util.module_from_spec(_spec)
_spec.loader.exec_module(_mod)

_Parser = _mod.RAGFlowPdfParser
is_garbled_char = _Parser._is_garbled_char
is_garbled_text = _Parser._is_garbled_text
has_subset_font_prefix = _Parser._has_subset_font_prefix
is_garbled_by_font_encoding = _Parser._is_garbled_by_font_encoding


# ---------------------------------------------------------------------------
# Tests for is_garbled_char
# ---------------------------------------------------------------------------


class TestIsGarbledChar:
    """Tests for the is_garbled_char function."""

    def test_normal_ascii_chars(self):
        for ch in "Hello World 123 !@#":
            assert is_garbled_char(ch) is False

    def test_normal_chinese_chars(self):
        for ch in "中文测试你好世界":
            assert is_garbled_char(ch) is False

    def test_normal_japanese_chars(self):
        for ch in "日本語テスト":
            assert is_garbled_char(ch) is False

    def test_normal_korean_chars(self):
        for ch in "한국어테스트":
            assert is_garbled_char(ch) is False

    def test_common_whitespace_not_garbled(self):
        assert is_garbled_char("\t") is False
        assert is_garbled_char("\n") is False
        assert is_garbled_char("\r") is False
        assert is_garbled_char(" ") is False

    def test_pua_chars_are_garbled(self):
        assert is_garbled_char("\ue000") is True
        assert is_garbled_char("\uf000") is True
        assert is_garbled_char("\uf8ff") is True

    def test_supplementary_pua_a(self):
        assert is_garbled_char(chr(0xF0000)) is True
        assert is_garbled_char(chr(0xFFFFF)) is True

    def test_supplementary_pua_b(self):
        assert is_garbled_char(chr(0x100000)) is True
        assert is_garbled_char(chr(0x10FFFF)) is True

    def test_replacement_char(self):
        assert is_garbled_char("\ufffd") is True

    def test_c0_control_chars(self):
        assert is_garbled_char("\x00") is True
        assert is_garbled_char("\x01") is True
        assert is_garbled_char("\x1f") is True

    def test_c1_control_chars(self):
        assert is_garbled_char("\x80") is True
        assert is_garbled_char("\x8f") is True
        assert is_garbled_char("\x9f") is True

    def test_empty_string(self):
        assert is_garbled_char("") is False

    def test_common_punctuation(self):
        for ch in ".,;:!?()[]{}\"'-/\\@#$%^&*+=<>~`|":
            assert is_garbled_char(ch) is False

    def test_unicode_symbols(self):
        for ch in "©®™°±²³µ¶·¹º»¼½¾":
            assert is_garbled_char(ch) is False


# ---------------------------------------------------------------------------
# Tests for is_garbled_text
# ---------------------------------------------------------------------------


class TestIsGarbledText:
    """Tests for the is_garbled_text function."""

    def test_normal_chinese_text(self):
        assert is_garbled_text("这是一段正常的中文文本") is False

    def test_normal_english_text(self):
        assert is_garbled_text("This is normal English text.") is False

    def test_mixed_normal_text(self):
        assert is_garbled_text("Hello 你好 World 世界 123") is False

    def test_empty_text(self):
        assert is_garbled_text("") is False
        assert is_garbled_text("   ") is False

    def test_none_text(self):
        assert is_garbled_text(None) is False

    def test_all_pua_chars(self):
        text = "\ue000\ue001\ue002\ue003\ue004"
        assert is_garbled_text(text) is True

    def test_mostly_garbled(self):
        text = "\ue000\ue001\ue002好"
        assert is_garbled_text(text, threshold=0.5) is True

    def test_few_garbled_below_threshold(self):
        text = "这是正常文本\ue000"
        assert is_garbled_text(text, threshold=0.5) is False

    def test_cid_pattern_detected(self):
        assert is_garbled_text("Hello (cid:123) World") is True
        assert is_garbled_text("(cid : 45)") is True
        assert is_garbled_text("(cid:0)") is True

    def test_cid_like_but_not_matching(self):
        assert is_garbled_text("This is a valid cid reference") is False

    def test_whitespace_only_text(self):
        assert is_garbled_text("   \t\n  ") is False

    def test_custom_threshold(self):
        text = "\ue000正常"
        assert is_garbled_text(text, threshold=0.3) is True
        assert is_garbled_text(text, threshold=0.5) is False

    def test_replacement_chars_in_text(self):
        text = "文档\ufffd\ufffd解析"
        assert is_garbled_text(text, threshold=0.5) is False
        assert is_garbled_text(text, threshold=0.3) is True

    def test_real_world_garbled_pattern(self):
        text = "\ue000\ue001\ue002\ue003\ue004\ue005\ue006\ue007"
        assert is_garbled_text(text) is True

    def test_mixed_garbled_and_normal_at_boundary(self):
        text = "AB\ue000\ue001"
        assert is_garbled_text(text, threshold=0.5) is True
        text2 = "ABC\ue000"
        assert is_garbled_text(text2, threshold=0.5) is False


# ---------------------------------------------------------------------------
# Tests for has_subset_font_prefix
# ---------------------------------------------------------------------------


class TestHasSubsetFontPrefix:
    """Tests for the has_subset_font_prefix function."""

    def test_standard_subset_prefix(self):
        assert has_subset_font_prefix("ABCDEF+Arial") is True
        assert has_subset_font_prefix("XYZABC+TimesNewRoman") is True

    def test_short_subset_prefix(self):
        assert has_subset_font_prefix("DY1+ZLQDm1-1") is True
        assert has_subset_font_prefix("AB+Font") is True

    def test_alphanumeric_prefix(self):
        assert has_subset_font_prefix("DY2+ZLQDnC-2") is True
        assert has_subset_font_prefix("A1B2C3+MyFont") is True

    def test_no_prefix(self):
        assert has_subset_font_prefix("Arial") is False
        assert has_subset_font_prefix("TimesNewRoman") is False

    def test_empty_or_none(self):
        assert has_subset_font_prefix("") is False
        assert has_subset_font_prefix(None) is False

    def test_plus_in_middle_not_prefix(self):
        assert has_subset_font_prefix("Font+Name") is False

    def test_lowercase_not_prefix(self):
        assert has_subset_font_prefix("abc+Font") is False


# ---------------------------------------------------------------------------
# Tests for is_garbled_by_font_encoding
# ---------------------------------------------------------------------------


def _make_chars(texts, fontname="DY1+ZLQDm1-1"):
    """Helper to create a list of pdfplumber-like char dicts."""
    return [{"text": t, "fontname": fontname} for t in texts]


class TestIsGarbledByFontEncoding:
    """Tests for font-encoding garbled text detection.

    This covers the scenario where PDF fonts with broken ToUnicode
    mappings cause CJK characters to be extracted as ASCII
    punctuation/symbols (e.g. GB.18067-2000.pdf).
    """

    def test_ascii_punct_from_subset_font_is_garbled(self):
        """Simulates GB.18067-2000.pdf: all chars are ASCII punct from subset fonts."""
        chars = _make_chars(
            list("!\"#$%&'('&)'\"*$!\"#$%&''()*+,$-"),
            fontname="DY1+ZLQDm1-1",
        )
        assert is_garbled_by_font_encoding(chars) is True

    def test_normal_cjk_text_not_garbled(self):
        """Normal Chinese text from subset fonts should not be flagged."""
        chars = _make_chars(
            list("这是一段正常的中文文本用于测试的示例内容没有问题"),
            fontname="ABCDEF+SimSun",
        )
        assert is_garbled_by_font_encoding(chars) is False

    def test_mixed_cjk_and_ascii_not_garbled(self):
        """Mixed CJK and ASCII content should not be flagged."""
        chars = _make_chars(
            list("GB18067-2000居住区大气中酚卫生标准"),
            fontname="DY1+ZLQDm1-1",
        )
        assert is_garbled_by_font_encoding(chars) is False

    def test_non_subset_font_not_flagged(self):
        """ASCII punct from non-subset fonts should not be flagged."""
        chars = _make_chars(
            list("!\"#$%&'()*+,-./!\"#$%&'()*+,-./"),
            fontname="Arial",
        )
        assert is_garbled_by_font_encoding(chars) is False

    def test_too_few_chars_not_flagged(self):
        """Pages with very few chars should not trigger detection."""
        chars = _make_chars(list('!"#$'), fontname="DY1+ZLQDm1-1")
        assert is_garbled_by_font_encoding(chars) is False

    def test_mostly_digits_not_garbled(self):
        """Pages with lots of digits (like data tables) should not be flagged."""
        chars = _make_chars(
            list("1234567890" * 3),
            fontname="DY1+ZLQDm1-1",
        )
        assert is_garbled_by_font_encoding(chars) is False

    def test_english_letters_not_garbled(self):
        """Pages with English letters should not be flagged."""
        chars = _make_chars(
            list("The quick brown fox jumps over the lazy dog"),
            fontname="ABCDEF+Arial",
        )
        assert is_garbled_by_font_encoding(chars) is False

    def test_real_world_gb18067_page1(self):
        """Simulate actual GB.18067-2000.pdf Page 1 character distribution."""
        page_text = "!\"#$%&'('&)'\"*$!\"#$%&''()*+,$-"
        chars = _make_chars(list(page_text), fontname="DY1+ZLQDm1-1")
        assert is_garbled_by_font_encoding(chars) is True

    def test_real_world_gb18067_page3(self):
        """Simulate actual GB.18067-2000.pdf Page 3 character distribution."""
        page_text = "!\"#$%&'()*+,-.*+/0+123456789:;<"
        chars = _make_chars(list(page_text), fontname="DY1+ZLQDnC-1")
        assert is_garbled_by_font_encoding(chars) is True

    def test_empty_chars(self):
        assert is_garbled_by_font_encoding([]) is False
        assert is_garbled_by_font_encoding(None) is False

    def test_only_spaces(self):
        chars = _make_chars([" "] * 30, fontname="DY1+ZLQDm1-1")
        assert is_garbled_by_font_encoding(chars) is False

    def test_small_min_chars_threshold(self):
        """With reduced min_chars, even small boxes can be detected."""
        chars = _make_chars(list('!"#$%&'), fontname="DY1+ZLQDm1-1")
        assert is_garbled_by_font_encoding(chars, min_chars=5) is True
        assert is_garbled_by_font_encoding(chars, min_chars=20) is False

    def test_boundary_cjk_ratio(self):
        """Just below 5% CJK threshold should still be flagged."""
        # 1 CJK out of 25 chars = 4% CJK, rest are punct from subset font
        chars = _make_chars(list("!\"#$%&'()*+,-./!@#$%^&*"), fontname="DY1+Font")
        chars.append({"text": "中", "fontname": "DY1+Font"})
        assert is_garbled_by_font_encoding(chars, min_chars=5) is True

    def test_boundary_above_cjk_threshold(self):
        """Above 5% CJK ratio should NOT be flagged."""
        # 3 CJK out of 23 chars = ~13% CJK
        chars = _make_chars(list("!\"#$%&'()*+,-./!@#$"), fontname="DY1+Font")
        for ch in "中文字":
            chars.append({"text": ch, "fontname": "DY1+Font"})
        assert is_garbled_by_font_encoding(chars, min_chars=5) is False

    def test_low_subset_ratio_not_flagged(self):
        """When only a few chars come from subset fonts, should not be flagged.

        Addresses reviewer feedback: a single subset font should not cause
        the entire page to be flagged as garbled.
        """
        # 5 chars from subset font, 20 from normal font -> 20% subset ratio < 30%
        chars = _make_chars(list('!"#$%'), fontname="DY1+Font")
        chars.extend(_make_chars(list("!\"#$%&'()*+,-./!@#$%"), fontname="Arial"))
        assert is_garbled_by_font_encoding(chars, min_chars=5) is False

    def test_high_subset_ratio_flagged(self):
        """When most chars come from subset fonts, detection should trigger."""
        # All 30 chars from subset font with punct -> garbled
        chars = _make_chars(
            list("!\"#$%&'()*+,-./!@#$%^&*()[]{}"),
            fontname="BCDGEE+R0015",
        )
        assert is_garbled_by_font_encoding(chars) is True


# ---------------------------------------------------------------------------
# Tests for layout_recognizer.__is_garbage
# ---------------------------------------------------------------------------


def _is_garbage(b):
    """Reproduce LayoutRecognizer.__is_garbage for unit testing.

    The original is a closure nested inside LayoutRecognizer.__call__
    (deepdoc/vision/layout_recognizer.py). We replicate it here because
    it cannot be directly imported.
    """
    patt = [r"\(cid\s*:\s*\d+\s*\)"]
    return any([re.search(p, b.get("text", "")) for p in patt])


class TestLayoutRecognizerIsGarbage:
    """Tests for the layout_recognizer __is_garbage function.

    This function filters out text boxes containing CID patterns like
    (cid:123) which indicate unmapped characters in PDF fonts.
    """

    def test_cid_pattern_simple(self):
        assert _is_garbage({"text": "(cid:123)"}) is True

    def test_cid_pattern_with_spaces(self):
        assert _is_garbage({"text": "(cid : 45)"}) is True
        assert _is_garbage({"text": "(cid :  0)"}) is True

    def test_cid_pattern_embedded_in_text(self):
        assert _is_garbage({"text": "Hello (cid:99) World"}) is True

    def test_cid_pattern_multiple(self):
        assert _is_garbage({"text": "(cid:1)(cid:2)(cid:3)"}) is True

    def test_normal_text_not_garbage(self):
        assert _is_garbage({"text": "This is normal text."}) is False

    def test_chinese_text_not_garbage(self):
        assert _is_garbage({"text": "这是正常的中文内容"}) is False

    def test_empty_text_not_garbage(self):
        assert _is_garbage({"text": ""}) is False

    def test_missing_text_key_not_garbage(self):
        assert _is_garbage({}) is False

    def test_parentheses_without_cid_not_garbage(self):
        assert _is_garbage({"text": "(hello:123)"}) is False
        assert _is_garbage({"text": "cid:123"}) is False

    def test_partial_cid_not_garbage(self):
        assert _is_garbage({"text": "(cid:)"}) is False
        assert _is_garbage({"text": "(cid)"}) is False

    def test_cid_with_zero(self):
        assert _is_garbage({"text": "(cid:0)"}) is True

    def test_cid_with_large_number(self):
        assert _is_garbage({"text": "(cid:99999)"}) is True


# ---------------------------------------------------------------------------
# Tests for _ocr_can_represent
# ---------------------------------------------------------------------------


class TestOcrCanRepresent:
    """The OCR fallback is skipped for a script the recogniser cannot spell.

    ocr.res is CJK+Latin, so it covers ~100% of an English page but only ~6% of a
    Cyrillic one; re-OCRing a script it cannot spell only produces garbage.
    """

    _LATIN = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 .,-"

    def test_english_is_representable(self, monkeypatch):
        monkeypatch.setattr(_Parser, "_OCR_ALPHABET", set(self._LATIN))
        assert _Parser._ocr_can_represent("Minimum edit distance") is True

    def test_russian_is_not_representable(self, monkeypatch):
        monkeypatch.setattr(_Parser, "_OCR_ALPHABET", set(self._LATIN))
        assert _Parser._ocr_can_represent("О книге как-то иначе") is False

    def test_empty_text_is_representable(self, monkeypatch):
        monkeypatch.setattr(_Parser, "_OCR_ALPHABET", set(self._LATIN))
        assert _Parser._ocr_can_represent("") is True
        assert _Parser._ocr_can_represent("   ") is True

    def test_unknown_alphabet_is_representable(self, monkeypatch):
        monkeypatch.setattr(_Parser, "_OCR_ALPHABET", set())
        assert _Parser._ocr_can_represent("О книге") is True

    def test_coverage_threshold_boundary(self, monkeypatch):
        monkeypatch.setattr(_Parser, "_OCR_ALPHABET", set("abcd"))
        assert _Parser._ocr_can_represent("abcdx", min_coverage=0.8) is True  # 4/5
        assert _Parser._ocr_can_represent("abcxx", min_coverage=0.8) is False  # 3/5


# ---------------------------------------------------------------------------
# Tests for _insert_word_spaces (geometric word-boundary recovery)
# ---------------------------------------------------------------------------


def _line(words, char_w=5.0, word_gap=2.0, kern=0.0, x0=0.0):
    """Build a spaceless pdfplumber-style char stream: fixed-width glyphs,
    ``word_gap`` pt between words, ``kern`` pt between glyphs inside a word."""
    chars = []
    x = x0
    for wi, w in enumerate(words):
        if wi:
            x += word_gap
        for ci, ch in enumerate(w):
            if ci:
                x += kern
            chars.append({"text": ch, "x0": x, "x1": x + char_w, "width": char_w})
            x += char_w
    return chars


def _spaced(chars):
    _Parser._insert_word_spaces(chars)
    return "".join(c["text"] for c in chars)


class TestInsertWordSpaces:
    def test_cyrillic_words_are_split(self):
        # mean width 5 -> threshold 0.25*5 = 1.25pt; the 2.0pt word gap exceeds it.
        assert _spaced(_line(["Окниге", "както"])).split() == ["Окниге", "както"]

    def test_latin_word_gap_becomes_a_space(self):
        assert _spaced(_line(["Hello", "World"])) == "Hello World"

    def test_intra_word_kern_is_not_a_space(self):
        # A 0.3pt kern inside a word is well below the threshold.
        assert _spaced(_line(["Large"], kern=0.3)) == "Large"

    def test_cjk_gap_is_not_a_word_boundary(self):
        # CJK does not separate words with spaces: a wide gap between CJK glyphs
        # is ordinary tracking, not a boundary.
        assert _spaced(_line(["乙", "丙"])) == "乙丙"

    def test_cjk_adjacent_to_latin_gets_no_invented_space(self):
        assert _spaced(_line(["乙", "A"])) == "乙A"

    def test_japanese_and_korean_gaps_are_not_boundaries(self):
        assert _spaced(_line(["ア", "イ"])) == "アイ"
        assert _spaced(_line(["한", "글"])) == "한글"
