test: add fetcher unit tests (IDs 27, 28, 33, 34)

SepineTam · SepineTam · commit febebfc341db · 2026-06-09T15:14:53.000+08:00
Add tests for parse_page, _extract_abstract, _extract_published_version,
_normalize_date, and _parse_search_result.

Also fix a bug in _parse_search_result where authors=None caused
TypeError: 'NoneType' object is not iterable.
diff --git a/src/nber_cli/fetcher.py b/src/nber_cli/fetcher.py
@@ -276,7 +276,8 @@ def _parse_search_result(raw_result: dict[str, Any]) -> NBER:
     paper_id_match = re.search(r"/papers/w(\d+)", url)
     paper_id = int(paper_id_match.group(1)) if paper_id_match else 0
     full_url = f"{_NBER_BASE_URL}{url}" if url.startswith("/") else url
-    authors = [_clean_html_text(author) for author in raw_result.get("authors", [])]
+    authors_raw = raw_result.get("authors", []) or []
+    authors = [_clean_html_text(author) for author in authors_raw]
 
     return NBER(
         paper_id=paper_id,
diff --git a/tests/test_fetcher.py b/tests/test_fetcher.py
@@ -136,3 +136,190 @@ def fake_urlopen(request, timeout, context=None):
 
         assert text == "ok"
         assert mock_sleep.call_count == 2
+
+
+class TestParsePage:
+    def test_parses_full_page(self):
+        page = """<html><head>
+<meta name="citation_title" content="Test Paper Title">
+<meta name="citation_author" content="Author One">
+<meta name="citation_author" content="Author Two">
+<meta name="citation_publication_date" content="2024/01/15">
+<meta name="citation_technical_report_number" content="w32000">
+</head><body>
+<div class="page-header__intro-inner">  <p>This is the <em>abstract</em> of the paper.</p></div>
+<h2>Published Versions</h2>  <p>Published in <em>Journal of Economics</em>, 2024.</p>
+</body></html>"""
+        from nber_cli.fetcher import parse_page
+        result = parse_page(page)
+        assert result.paper_id == 32000
+        assert result.title == "Test Paper Title"
+        assert result.authors == ["Author One", "Author Two"]
+        assert result.date == "2024/01/15"
+        assert result.abstract == "This is the abstract of the paper."
+        assert result.published_version == "Published in Journal of Economics, 2024."
+
+    def test_handles_missing_fields(self):
+        page = "<html><head></head><body></body></html>"
+        from nber_cli.fetcher import parse_page
+        result = parse_page(page)
+        assert result.paper_id == 0
+        assert result.title == ""
+        assert result.authors == []
+        assert result.date == ""
+        assert result.abstract == ""
+        assert result.published_version is None
+
+    def test_parses_paper_id_with_leading_zeros(self):
+        page = '<meta name="citation_technical_report_number" content="w00123">'
+        from nber_cli.fetcher import parse_page
+        result = parse_page(page)
+        assert result.paper_id == 123
+
+    def test_parses_paper_id_without_w_prefix(self):
+        page = '<meta name="citation_technical_report_number" content="456">'
+        from nber_cli.fetcher import parse_page
+        result = parse_page(page)
+        assert result.paper_id == 456
+
+
+class TestExtractAbstract:
+    def test_extracts_abstract(self):
+        from nber_cli.fetcher import _extract_abstract
+        page = '<div class="page-header__intro-inner">\n<p>This is the abstract.</p>\n</div>'
+        assert _extract_abstract(page) == "This is the abstract."
+
+    def test_strips_html_tags(self):
+        from nber_cli.fetcher import _extract_abstract
+        page = '<div class="page-header__intro-inner"><p>Abstract with <em>markup</em> and <a href="#">links</a>.</p></div>'
+        assert _extract_abstract(page) == "Abstract with markup and links."
+
+    def test_collapses_whitespace(self):
+        from nber_cli.fetcher import _extract_abstract
+        page = '<div class="page-header__intro-inner"><p>Abstract\n\nwith   spaces.</p></div>'
+        assert _extract_abstract(page) == "Abstract with spaces."
+
+    def test_returns_empty_when_missing(self):
+        from nber_cli.fetcher import _extract_abstract
+        assert _extract_abstract("<html></html>") == ""
+
+
+class TestExtractPublishedVersion:
+    def test_extracts_published_version(self):
+        from nber_cli.fetcher import _extract_published_version
+        page = '<h2>Published Versions</h2>\n<p>Published in Journal, 2024.</p>'
+        assert _extract_published_version(page) == "Published in Journal, 2024."
+
+    def test_strips_html_tags(self):
+        from nber_cli.fetcher import _extract_published_version
+        page = '<h2>Published Versions</h2><p>Published in <em>Journal</em>.</p>'
+        assert _extract_published_version(page) == "Published in Journal."
+
+    def test_returns_none_when_missing(self):
+        from nber_cli.fetcher import _extract_published_version
+        assert _extract_published_version("<html></html>") is None
+
+
+class TestNormalizeDate:
+    def test_none_returns_none(self):
+        from nber_cli.fetcher import _normalize_date
+        assert _normalize_date(None) is None
+
+    def test_date_object_returns_isoformat(self):
+        from nber_cli.fetcher import _normalize_date
+        assert _normalize_date(date(2024, 1, 15)) == "2024-01-15"
+
+    def test_valid_string_returns_isoformat(self):
+        from nber_cli.fetcher import _normalize_date
+        assert _normalize_date("2024-01-15") == "2024-01-15"
+
+    def test_rejects_invalid_format(self):
+        from nber_cli.fetcher import _normalize_date
+        with pytest.raises(ValueError, match="expected YYYY-MM-DD"):
+            _normalize_date("01/15/2024")
+
+    def test_rejects_malformed_date(self):
+        from nber_cli.fetcher import _normalize_date
+        with pytest.raises(ValueError, match="expected YYYY-MM-DD"):
+            _normalize_date("2024-13-01")
+
+    def test_rejects_empty_string(self):
+        from nber_cli.fetcher import _normalize_date
+        with pytest.raises(ValueError, match="expected YYYY-MM-DD"):
+            _normalize_date("")
+
+
+class TestParseSearchResult:
+    def test_parses_complete_result(self):
+        from nber_cli.fetcher import _parse_search_result
+        raw = {
+            "url": "/papers/w12345",
+            "title": "A Paper",
+            "authors": ['<a href="/people/a">Author A</a>'],
+            "displaydate": "Jan 2024",
+            "abstract": "Abstract text.",
+        }
+        result = _parse_search_result(raw)
+        assert result.paper_id == 12345
+        assert result.title == "A Paper"
+        assert result.authors == ["Author A"]
+        assert result.date == "Jan 2024"
+        assert result.abstract == "Abstract text."
+        assert result.url == "https://www.nber.org/papers/w12345"
+
+    def test_handles_missing_url(self):
+        from nber_cli.fetcher import _parse_search_result
+        result = _parse_search_result({})
+        assert result.paper_id == 0
+        assert result.url == ""
+
+    def test_handles_missing_fields_gracefully(self):
+        from nber_cli.fetcher import _parse_search_result
+        raw = {"url": "/papers/w99999"}
+        result = _parse_search_result(raw)
+        assert result.paper_id == 99999
+        assert result.title == ""
+        assert result.authors == []
+        assert result.date == ""
+        assert result.abstract == ""
+
+    def test_skips_empty_authors(self):
+        from nber_cli.fetcher import _parse_search_result
+        raw = {"authors": ["", "Author A", ""]}
+        result = _parse_search_result(raw)
+        assert result.authors == ["Author A"]
+
+    def test_handles_external_url(self):
+        from nber_cli.fetcher import _parse_search_result
+        raw = {"url": "https://example.com/papers/w1"}
+        result = _parse_search_result(raw)
+        assert result.url == "https://example.com/papers/w1"
+
+    def test_handles_none_values(self):
+        from nber_cli.fetcher import _parse_search_result
+        raw = {
+            "url": None,
+            "title": None,
+            "authors": None,
+            "displaydate": None,
+            "abstract": None,
+        }
+        result = _parse_search_result(raw)
+        assert result.paper_id == 0
+        assert result.title == ""
+        assert result.authors == []
+        assert result.date == ""
+        assert result.abstract == ""
+
+    def test_cleans_html_in_all_fields(self):
+        from nber_cli.fetcher import _parse_search_result
+        raw = {
+            "title": "Title with <em>markup</em>",
+            "authors": ['<a href="#">A &amp; B</a>'],
+            "displaydate": "Jan 2024",
+            "abstract": "Abstract with <br>line break.",
+        }
+        result = _parse_search_result(raw)
+        assert result.title == "Title with markup"
+        assert result.authors == ["A & B"]
+        assert result.abstract == "Abstract with line break."
