"""Tests for evidence packer - deduplication, credibility scoring, circuit breaker.""" from unittest.mock import AsyncMock, patch import httpx import pytest from web.config import WebSettings from web.evidence.packer import EvidencePacker from web.schemas.common import PageContent from web.schemas.evidence import EvidencePackRequest @pytest.fixture def test_settings() -> WebSettings: """Create test settings.""" return WebSettings( searxng_base_url="http://localhost:55100", llm_base_url="http://localhost:14011", external_url="http://localhost:51100", ) @pytest.fixture def packer(test_settings: WebSettings) -> EvidencePacker: """Create an EvidencePacker instance.""" return EvidencePacker(test_settings) class TestDeduplication: """Tests for content deduplication.""" def test_identical_texts_high_similarity(self, packer: EvidencePacker) -> None: """Test that identical texts have very high similarity.""" text = "This is a test document about climate change effects." score = packer._calculate_similarity(text, text) assert score >= 0.95 def test_different_texts_low_similarity(self, packer: EvidencePacker) -> None: """Test that very different texts have low similarity.""" text1 = "Climate change is affecting global temperatures." text2 = "The stock market rallied yesterday on positive earnings." score = packer._calculate_similarity(text1, text2) assert score < 0.5 def test_empty_texts_zero_similarity(self, packer: EvidencePacker) -> None: """Test that empty texts return zero similarity.""" assert packer._calculate_similarity("", "") == 0.0 assert packer._calculate_similarity("text", "") == 0.0 assert packer._calculate_similarity("", "text") == 0.0 def test_length_ratio_prefilter(self, packer: EvidencePacker) -> None: """Test that very different length texts get low similarity via pre-filter.""" short = "Hello" long = "This is a very long text " * 100 score = packer._calculate_similarity(short, long) assert score < 0.5 class TestCredibilityScoring: """Tests for credibility scoring.""" def test_high_credibility_domain(self, packer: EvidencePacker) -> None: """Test that known credible domains get high scores.""" score = packer._score_credibility_simple("https://reuters.com/article/123") assert score == 0.9 def test_high_credibility_with_www(self, packer: EvidencePacker) -> None: """Test that www prefix is stripped.""" score = packer._score_credibility_simple("https://www.bbc.com/news/article") assert score == 0.9 def test_medium_credibility_domain(self, packer: EvidencePacker) -> None: """Test that medium-credibility domains get appropriate scores.""" score = packer._score_credibility_simple("https://cnn.com/article") assert score == 0.7 def test_unknown_domain_default_score(self, packer: EvidencePacker) -> None: """Test that unknown domains get default score.""" score = packer._score_credibility_simple("https://random-blog.com/post") assert score == 0.5 def test_subdomain_does_not_match(self, packer: EvidencePacker) -> None: """Test that 'notreuters.com' does not match 'reuters.com'.""" score = packer._score_credibility_simple("https://notreuters.com/article") assert score == 0.5 def test_subdomain_matches(self, packer: EvidencePacker) -> None: """Test that 'news.bbc.co.uk' matches 'bbc.co.uk'.""" score = packer._score_credibility_simple("https://news.bbc.co.uk/article") assert score == 0.9 class TestThinkingTagStripping: """Tests for LLM thinking tag removal.""" def test_strips_complete_think_tags(self, packer: EvidencePacker) -> None: """Test stripping complete ... blocks.""" text = "Let me analyze this...The answer is 42." result = packer._strip_thinking_tags(text) assert result == "The answer is 42." def test_strips_truncated_think_tags(self, packer: EvidencePacker) -> None: """Test stripping truncated without closing tag.""" text = "Still thinking about this" result = packer._strip_thinking_tags(text) assert result == "" def test_preserves_text_without_tags(self, packer: EvidencePacker) -> None: """Test that text without think tags is preserved.""" text = "No thinking tags here." result = packer._strip_thinking_tags(text) assert result == "No thinking tags here." class TestLLMCircuitBreaker: """Tests for LLM availability probe / circuit breaker.""" @pytest.mark.asyncio async def test_probe_marks_unavailable_on_connection_error( self, packer: EvidencePacker ) -> None: """Test that a connection error marks LLM as unavailable.""" with patch("web.evidence.packer.httpx.AsyncClient") as mock_cls: mock_client = AsyncMock() mock_client.get.side_effect = httpx.ConnectError("refused") mock_client.__aenter__ = AsyncMock(return_value=mock_client) mock_client.__aexit__ = AsyncMock(return_value=False) mock_cls.return_value = mock_client result = await packer._is_llm_available() assert result is False assert packer._llm_available is False @pytest.mark.asyncio async def test_probe_marks_available_on_success( self, packer: EvidencePacker ) -> None: """Test that a successful response marks LLM as available.""" with patch("web.evidence.packer.httpx.AsyncClient") as mock_cls: mock_client = AsyncMock() mock_resp = AsyncMock() mock_resp.raise_for_status = lambda: None mock_client.get.return_value = mock_resp mock_client.__aenter__ = AsyncMock(return_value=mock_client) mock_client.__aexit__ = AsyncMock(return_value=False) mock_cls.return_value = mock_client result = await packer._is_llm_available() assert result is True assert packer._llm_available is True @pytest.mark.asyncio async def test_pack_disables_llm_when_unreachable( self, packer: EvidencePacker ) -> None: """Test that pack() disables LLM features when probe fails.""" page = PageContent( url="https://example.com/article", title="Test", text="Enough content to process for evidence packing test." * 5, text_hash="abc123", extraction_method="http", retrieved_at="2025-01-01T00:00:00Z", extraction_time_ms=100.0, ) request = EvidencePackRequest( pages=[page], claim="Test claim for circuit breaker verification", score_relevance=True, extract_snippets=False, ) # Make probe fail with patch.object( packer, "_is_llm_available", new_callable=AsyncMock, return_value=False ): response = await packer.pack(request) # Should succeed without LLM calls (no errors) assert len(response.evidence) == 1 # No LLM tokens used since LLM was disabled assert response.stats.tokens_used == 0 @pytest.mark.asyncio async def test_negative_cache_prevents_repeated_probes( self, packer: EvidencePacker ) -> None: """Test that negative cache prevents repeated probe calls.""" with patch("web.evidence.packer.httpx.AsyncClient") as mock_cls: mock_client = AsyncMock() mock_client.get.side_effect = httpx.ConnectError("refused") mock_client.__aenter__ = AsyncMock(return_value=mock_client) mock_client.__aexit__ = AsyncMock(return_value=False) mock_cls.return_value = mock_client # First call: probes and caches negative result result1 = await packer._is_llm_available() assert result1 is False # Second call: should use cached result (no new probe) mock_cls.reset_mock() result2 = await packer._is_llm_available() assert result2 is False mock_cls.assert_not_called() class TestSummarize: """Tests for LLM summarization feature.""" def test_parse_batch_summary_extracts_correct_summary( self, packer: EvidencePacker ) -> None: """Test parsing individual summaries from batch response.""" response = ( "SUMMARY 1: This is the first summary about economics.\n" "SUMMARY 2: This is the second summary about politics." ) result1 = packer._parse_batch_summary(response, 1, 800) result2 = packer._parse_batch_summary(response, 2, 800) assert result1 is not None assert "first summary" in result1 assert result2 is not None assert "second summary" in result2 def test_parse_batch_summary_returns_none_for_missing( self, packer: EvidencePacker ) -> None: """Test that missing summary number returns None.""" response = "SUMMARY 1: Only one summary here." result = packer._parse_batch_summary(response, 3, 800) assert result is None def test_parse_batch_summary_respects_max_length( self, packer: EvidencePacker ) -> None: """Test that summaries are truncated to max_length.""" long_text = "A" * 1000 response = f"SUMMARY 1: {long_text}" result = packer._parse_batch_summary(response, 1, 200) assert result is not None assert len(result) <= 200 @pytest.mark.asyncio async def test_pack_disables_summarize_when_llm_unreachable( self, packer: EvidencePacker ) -> None: """Test that pack() disables summarize when LLM probe fails.""" page = PageContent( url="https://example.com/article", title="Test", text="Content about economic growth in Romania." * 10, text_hash="summ123", extraction_method="http", retrieved_at="2025-01-01T00:00:00Z", extraction_time_ms=100.0, ) request = EvidencePackRequest( pages=[page], claim="Romania had the highest economic growth in the EU in 2024", summarize=True, ) with patch.object( packer, "_is_llm_available", new_callable=AsyncMock, return_value=False ): response = await packer.pack(request) assert len(response.evidence) == 1 # Summary should be None since LLM was disabled assert response.evidence[0].summary is None assert response.stats.tokens_used == 0 @pytest.mark.asyncio async def test_summarize_produces_summary_field( self, packer: EvidencePacker ) -> None: """Test that summarize=True produces a summary field via LLM.""" page = PageContent( url="https://example.com/article", title="Test Article", text="Romania's GDP grew by 4.1% in 2024, the highest in the EU." * 5, text_hash="summ456", extraction_method="http", retrieved_at="2025-01-01T00:00:00Z", extraction_time_ms=100.0, ) request = EvidencePackRequest( pages=[page], claim="Romania had the highest economic growth in the EU in 2024", summarize=True, ) mock_summary = "Romania achieved 4.1% GDP growth in 2024, leading the EU." with ( patch.object( packer, "_is_llm_available", new_callable=AsyncMock, return_value=True ), patch.object( packer, "_summarize_single", new_callable=AsyncMock, return_value=(mock_summary, 150), ), ): response = await packer.pack(request) assert len(response.evidence) == 1 assert response.evidence[0].summary == mock_summary assert response.stats.tokens_used == 150