127 lines
4.9 KiB
Python
127 lines
4.9 KiB
Python
"""Tests for the adversarial fact-checker helpers in press_release.py."""
|
|
|
|
from cheddahbot.tools.press_release import _apply_fact_check, _build_fact_check_prompt
|
|
|
|
|
|
class TestApplyFactCheck:
|
|
"""Tests for _apply_fact_check output parsing."""
|
|
|
|
ORIGINAL = (
|
|
"Acme Corp Delivers Advanced Widget Solutions\n\n"
|
|
"Acme Corp, a leading manufacturer of widgets, today highlighted "
|
|
"its expanded product line. The company, based in Milwaukee, Wisconsin, "
|
|
"produces over 500 widget variants for industrial applications."
|
|
)
|
|
|
|
def test_no_errors_returns_original(self):
|
|
text, status, changes = _apply_fact_check("[NO_ERRORS]", self.ORIGINAL)
|
|
assert status == "clean"
|
|
assert text == self.ORIGINAL
|
|
assert changes == ""
|
|
|
|
def test_no_errors_with_trailing_whitespace(self):
|
|
text, status, changes = _apply_fact_check("[NO_ERRORS] \n", self.ORIGINAL)
|
|
assert status == "clean"
|
|
assert text == self.ORIGINAL
|
|
|
|
def test_corrected_with_changes(self):
|
|
corrected_pr = self.ORIGINAL.replace("500 widget", "300 widget")
|
|
raw = (
|
|
f"[CORRECTED]\n{corrected_pr}\n\n"
|
|
"CHANGES:\n1. Changed '500 widget variants' to '300 widget variants' "
|
|
"-- company website lists 300."
|
|
)
|
|
text, status, changes = _apply_fact_check(raw, self.ORIGINAL)
|
|
assert status == "corrected"
|
|
assert "300 widget" in text
|
|
assert "500" not in text
|
|
assert "300 widget variants" in changes
|
|
|
|
def test_corrected_without_changes_section(self):
|
|
corrected_pr = self.ORIGINAL.replace("500", "300")
|
|
raw = f"[CORRECTED]\n{corrected_pr}"
|
|
text, status, changes = _apply_fact_check(raw, self.ORIGINAL)
|
|
assert status == "corrected"
|
|
assert "300" in text
|
|
assert changes == ""
|
|
|
|
def test_empty_output_returns_skipped(self):
|
|
text, status, changes = _apply_fact_check("", self.ORIGINAL)
|
|
assert status == "skipped"
|
|
assert text == self.ORIGINAL
|
|
|
|
def test_none_like_output_returns_skipped(self):
|
|
text, status, changes = _apply_fact_check(" \n ", self.ORIGINAL)
|
|
assert status == "skipped"
|
|
assert text == self.ORIGINAL
|
|
|
|
def test_garbage_output_returns_skipped(self):
|
|
text, status, changes = _apply_fact_check(
|
|
"I reviewed the press release and it looks good overall.", self.ORIGINAL
|
|
)
|
|
assert status == "skipped"
|
|
assert text == self.ORIGINAL
|
|
|
|
def test_rejects_oversized_rewrite(self):
|
|
"""If fact-checker rewrites too much (>15% word count delta), reject."""
|
|
# Double the content -- way more than 15%
|
|
bloated = self.ORIGINAL + "\n\n" + self.ORIGINAL + "\n\nExtra content here."
|
|
raw = f"[CORRECTED]\n{bloated}\n\nCHANGES:\n1. Added more detail."
|
|
text, status, changes = _apply_fact_check(raw, self.ORIGINAL)
|
|
assert status == "skipped"
|
|
assert text == self.ORIGINAL
|
|
assert "word count delta" in changes
|
|
|
|
def test_accepts_minor_word_count_change(self):
|
|
"""Small changes (within 15%) should be accepted."""
|
|
# Change one word -- well within 15%
|
|
minor_edit = self.ORIGINAL.replace("500 widget variants", "480 widget variants")
|
|
raw = (
|
|
f"[CORRECTED]\n{minor_edit}\n\n"
|
|
"CHANGES:\n1. Corrected variant count from 500 to 480."
|
|
)
|
|
text, status, changes = _apply_fact_check(raw, self.ORIGINAL)
|
|
assert status == "corrected"
|
|
assert "480" in text
|
|
|
|
def test_corrected_but_empty_body_returns_skipped(self):
|
|
text, status, changes = _apply_fact_check("[CORRECTED]\n", self.ORIGINAL)
|
|
assert status == "skipped"
|
|
assert text == self.ORIGINAL
|
|
|
|
|
|
class TestBuildFactCheckPrompt:
|
|
"""Tests for _build_fact_check_prompt structure."""
|
|
|
|
def test_includes_ground_truth_data(self):
|
|
prompt = _build_fact_check_prompt(
|
|
"Some PR text here.",
|
|
company_name="Acme Corp",
|
|
url="https://acme.com",
|
|
topic="widgets",
|
|
keyword="industrial widgets",
|
|
)
|
|
assert "Acme Corp" in prompt
|
|
assert "https://acme.com" in prompt
|
|
assert "widgets" in prompt
|
|
assert "industrial widgets" in prompt
|
|
assert "ground truth" in prompt.lower() or "GROUND TRUTH" in prompt
|
|
|
|
def test_includes_pr_text(self):
|
|
prompt = _build_fact_check_prompt(
|
|
"The quick brown fox.",
|
|
company_name="Test",
|
|
url="https://test.com",
|
|
topic="foxes",
|
|
keyword="brown fox",
|
|
)
|
|
assert "The quick brown fox." in prompt
|
|
|
|
def test_output_format_instructions(self):
|
|
prompt = _build_fact_check_prompt(
|
|
"Text.", company_name="X", url="u", topic="t", keyword="k"
|
|
)
|
|
assert "[NO_ERRORS]" in prompt
|
|
assert "[CORRECTED]" in prompt
|
|
assert "CHANGES:" in prompt
|