autometadata/websearch.py at main · Seglats/autometadata · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
"""
websearch.py — SearXNG fallback search for Audible ASINs.

Searches the web via a local SearXNG instance to find Audible book pages
when Audible's own search fails.

.env:
    SEARXNG_URL=http://localhost:8888
"""

import re
from typing import Optional

import requests


def find_audible_asins(
    query: str,
    searxng_url: str = "http://localhost:8888",
    max_results: int = 5,
) -> list[dict]:
  """
  Search SearXNG for Audible pages matching query.
  Returns list of {"asin": str, "url": str, "title": str, "region": str}.
  """
  try:
    r = requests.get(
        f"{searxng_url.rstrip('/')}/search",
        params={
            "q": f"site:audible.com OR site:audible.co.uk {query}",
            "format": "json",
            "engines": "google,bing,duckduckgo",
            "categories": "general",
        },
        timeout=15,
    )
    r.raise_for_status()
    data = r.json()
  except Exception:
    return []

  results = []
  seen_asins = set()

  for item in data.get("results", [])[:max_results * 2]:
    url = item.get("url", "")
    title = item.get("title", "")

    # Extract ASIN from Audible URL
    asin_match = re.search(r'/([A-Z0-9]{10})(?:\?|$)', url)
    if not asin_match:
      continue

    # Must be an audible domain
    if "audible.com" not in url and "audible.co.uk" not in url:
      continue

    asin = asin_match.group(1)
    if asin in seen_asins:
      continue
    seen_asins.add(asin)

    # Determine region from URL
    region = "us"
    if "audible.co.uk" in url:
      region = "uk"
    elif "audible.de" in url:
      region = "de"
    elif "audible.fr" in url:
      region = "fr"

    # Clean up title (remove " | Audible.com" suffixes)
    title = re.sub(r'\s*\|.*$', '', title).strip()
    title = re.sub(r'\s*[-–].*?[Aa]udible.*$', '', title).strip()
    title = re.sub(r'\s*[Aa]udiobook\s*$', '', title).strip()

    results.append({
        "asin": asin,
        "url": url,
        "title": title,
        "region": region,
    })

    if len(results) >= max_results:
      break

  return results


def search_general(
    query: str,
    searxng_url: str = "http://localhost:8888",
    max_results: int = 5,
) -> list[dict]:
  """
  General SearXNG search. Returns list of {"url": str, "title": str, "content": str}.
  Useful for finding alternate titles, ISBNs, etc.
  """
  try:
    r = requests.get(
        f"{searxng_url.rstrip('/')}/search",
        params={
            "q": query,
            "format": "json",
            "categories": "general",
        },
        timeout=15,
    )
    r.raise_for_status()
    data = r.json()
  except Exception:
    return []

  return [
      {
          "url": item.get("url", ""),
          "title": item.get("title", ""),
          "content": item.get("content", ""),
      }
      for item in data.get("results", [])[:max_results]
  ]