fix: LinkExtractor never filters compound extensions like .tar.gz

scrapling/spiders/links.py

@@ -149,12 +149,14 @@ def _compile_patterns(patterns: Union[str, Pattern[str], PatternInput, None]) ->
     return tuple(p if isinstance(p, re.Pattern) else re.compile(p) for p in patterns)
 
 
-def _url_extension(url: str) -> str:
+def _url_extension(url: str) -> Set[str]:
     path = urlsplit(url).path
     _, _, last = path.rpartition("/")
     if "." not in last:
-        return ""
-    return last.rsplit(".", 1)[1].lower()
+        return set()
+    parts = last.lower().split(".")
+    # Return all dot-suffixes so compound extensions like "tar.gz" are matched
+    return {".".join(parts[i:]) for i in range(1, len(parts))}
 
 
 def _filler(x):
@@ -277,8 +279,8 @@ def _url_passes(self, url: str) -> bool:
         if url.split("://", 1)[0] not in valid_schemas:
             return False
 
-        ext = _url_extension(url)
-        if ext and ext in self.deny_extensions:
+        exts = _url_extension(url)
+        if exts and exts & self.deny_extensions:
             return False
 
         if self.allow and not any(p.search(url) for p in self.allow):

tests/spiders/test_links.py

@@ -196,6 +196,24 @@ def test_empty_deny_extensions_allows_everything(self):
         urls = LinkExtractor(deny_extensions=()).extract(resp)
         assert urls == ["https://example.com/a.pdf"]
 
+    def test_default_deny_extensions_drops_compound_tar_gz(self):
+        # tar.gz is in IGNORED_EXTENSIONS but the last-dot-only suffix is "gz"
+        html = '<a href="/d/dataset.tar.gz">tgz</a><a href="/d/data.tar.bz2">tbz</a><a href="/d">ok</a>'
+        resp = _make_response(html)
+        urls = LinkExtractor().extract(resp)
+        assert urls == ["https://example.com/d"]
+
+    def test_custom_compound_deny_extension(self):
+        html = '<a href="/dataset.tar.gz">tgz</a><a href="/plain.gz">gz</a>'
+        resp = _make_response(html)
+        # only the compound ext is denied; a plain .gz must still pass
+        urls = LinkExtractor(deny_extensions=["tar.gz"]).extract(resp)
+        assert urls == ["https://example.com/plain.gz"]
+
+    def test_compound_deny_is_case_insensitive(self):
+        ex = LinkExtractor()
+        assert ex.matches("https://example.com/archive.TAR.GZ") is False
+
 
 class TestStrip:
     def test_strip_removes_whitespace(self):