extract_archive_pages.py

#!/usr/bin/env python3
"""
Download page images from an Internet Archive book (e.g. Stand on Zanzibar).

Usage:
  python extract_archive_pages.py [--start N] [--end N] [--output DIR] [--cookies FILE]

For access-restricted books you must:
  1. Go to https://archive.org/details/standonzanzibar0000unse
  2. Borrow the book (e.g. "Borrow for 1 hour") and open it in the BookReader
  3. Export cookies (e.g. with a "cookies.txt" browser extension in Netscape format)
  4. Run: python extract_archive_pages.py --cookies cookies.txt

References:
  - https://archive.readme.io/reference/retrieving-book-pages
  - https://github.com/opemvbs/archiveOrgImageDownloader (for borrowed books)
"""

import argparse
import re
import sys
from pathlib import Path
from typing import Optional

try:
    import requests
except ImportError:
    print("Install requests: pip install requests", file=sys.stderr)
    sys.exit(1)

DEFAULT_IDENTIFIER = "standonzanzibar0000unse"
# Page count from BookReader (674 leaves)
DEFAULT_PAGE_COUNT = 674
BASE_URL = "https://archive.org/download/{identifier}/page/page_{n}.jpg"
USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"


def get_session(cookies_file: Optional[Path], identifier: str):
    s = requests.Session()
    s.headers["User-Agent"] = USER_AGENT
    s.headers["Referer"] = f"https://archive.org/details/{identifier}"
    s.headers["Accept"] = "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8"
    s.max_redirects = 10
    if cookies_file and cookies_file.exists():
        # Netscape cookies.txt: domain, include_subdomains, path, secure, expires, name, value
        with open(cookies_file) as f:
            for line in f:
                line = line.strip()
                if not line or line.startswith("#"):
                    continue
                parts = line.split("\t")
                if len(parts) >= 7:
                    domain, _include_sub, path, _secure, _expires, name, value = parts[:7]
                    if "archive.org" not in domain:
                        continue
                    # Ensure subdomains get the cookie (e.g. ia800303.us.archive.org)
                    if not domain.startswith("."):
                        domain = "." + domain
                    s.cookies.set(name, value, domain=domain, path=path)
    return s


def main():
    p = argparse.ArgumentParser(description="Extract page images from an Archive.org book.")
    p.add_argument("--id", default=DEFAULT_IDENTIFIER, help="Archive.org item identifier")
    p.add_argument("--start", type=int, default=1, help="First page number (1-based)")
    p.add_argument("--end", type=int, default=None, help="Last page number (default: --pages)")
    p.add_argument("--pages", type=int, default=DEFAULT_PAGE_COUNT, help="Total pages (used for --end if --end not set)")
    p.add_argument("--output", "-o", type=Path, default=None, help="Output directory (default: stand-on-zanzibar-pages)")
    p.add_argument("--cookies", type=Path, default=None, help="Netscape cookies.txt for restricted/borrowed books")
    p.add_argument("--dry-run", action="store_true", help="Only print URLs, do not download")
    args = p.parse_args()

    end = args.end if args.end is not None else args.pages
    if args.start < 1 or end < args.start:
        p.error("Require 1 <= --start <= --end")

    out_dir = args.output or Path("stand-on-zanzibar-pages")
    if not args.dry_run:
        out_dir.mkdir(parents=True, exist_ok=True)

    session = get_session(args.cookies, args.id)

    for n in range(args.start, end + 1):
        url = BASE_URL.format(identifier=args.id, n=n)
        if args.dry_run:
            print(url)
            continue
        out_path = out_dir / f"page_{n:04d}.jpg"
        if out_path.exists():
            print(f"Skip (exists): {out_path.name}")
            continue
        try:
            r = session.get(url, timeout=30)
            r.raise_for_status()
            out_path.write_bytes(r.content)
            print(f"Saved: {out_path.name}")
        except requests.RequestException as e:
            print(f"Error page {n}: {e}", file=sys.stderr)
            if getattr(e.response, "status_code", None) == 403:
                print("If the book is access-restricted, borrow it and use --cookies cookies.txt", file=sys.stderr)


if __name__ == "__main__":
    main()