|
| 1 | +#!/usr/bin/env python3 |
| 2 | +""" |
| 3 | +Fortress live-scraping demos — reproduces the GIFs in the README. |
| 4 | +
|
| 5 | +Each demo drives the real Fortress engine over CDP and overlays a "verification" |
| 6 | +HUD (green highlights + value tags) so you can *see* the scrape happen, then |
| 7 | +writes an animated GIF. |
| 8 | +
|
| 9 | + pip install tilion-fortress playwright pillow |
| 10 | + python examples/scrape_demos.py structured # -> fortress-scrape-structured.gif |
| 11 | + python examples/scrape_demos.py paginated | detail | js | all |
| 12 | +
|
| 13 | +Patterns shown: |
| 14 | + structured books.toscrape.com -> typed records build into a live JSON panel |
| 15 | + paginated quotes.toscrape.com -> auto-pagination across pages 1..3 |
| 16 | + detail books.toscrape.com -> deep detail-page crawl (UPC/price/tax/stock/reviews) |
| 17 | + js quotes.toscrape.com/js -> client-side-rendered DOM captured over CDP |
| 18 | +
|
| 19 | +Fortress spoofs the fingerprint in the engine's C++, so nothing here adds JS stealth. |
| 20 | +If a site still blocks you it's the IP (datacenter) — route egress through a residential proxy. |
| 21 | +""" |
| 22 | +from __future__ import annotations |
| 23 | +import sys, time |
| 24 | +from pathlib import Path |
| 25 | + |
| 26 | +OUT = Path(__file__).resolve().parent |
| 27 | +VIEWPORT = {"width": 1180, "height": 820} |
| 28 | +FPS = 9 |
| 29 | + |
| 30 | +# One declarative overlay controller, re-injected after every navigation. |
| 31 | +FX_SETUP = r""" |
| 32 | +window.FX = (function(){ |
| 33 | + const A = '#39d353'; |
| 34 | + function ensureBar(title){ |
| 35 | + let bar = document.getElementById('fx-bar'); |
| 36 | + if(!bar){ bar=document.createElement('div'); bar.id='fx-bar'; |
| 37 | + bar.style.cssText=`position:fixed;top:0;left:0;right:0;height:44px;z-index:2147483000; |
| 38 | + background:rgba(13,17,23,.94);color:#e6edf3;display:flex;align-items:center;gap:12px; |
| 39 | + padding:0 16px;font:600 15px system-ui,Segoe UI,sans-serif;box-shadow:0 2px 18px rgba(0,0,0,.4)`; |
| 40 | + document.body.appendChild(bar); } |
| 41 | + bar.innerHTML=`<span style="font-size:18px">🏰</span><span style="letter-spacing:.5px">FORTRESS</span>`+ |
| 42 | + `<span style="color:${A};font-weight:700">● ${title}</span>`+ |
| 43 | + `<span id="fx-status" style="margin-left:auto;color:#9da7b3;font-weight:500"></span>`; |
| 44 | + } |
| 45 | + return { render(s){ |
| 46 | + ensureBar(s.title||'stealth engine'); |
| 47 | + const st=document.getElementById('fx-status'); if(st) st.textContent=s.status||''; |
| 48 | + let root=document.getElementById('fx-root'); |
| 49 | + if(!root){ root=document.createElement('div'); root.id='fx-root'; document.body.appendChild(root); } |
| 50 | + root.innerHTML=''; |
| 51 | + (s.boxes||[]).forEach(b=>{ |
| 52 | + let el=null; try{ el=document.querySelectorAll(b.sel)[b.idx||0]; }catch(e){} |
| 53 | + if(!el) return; const r=el.getBoundingClientRect(); |
| 54 | + const isA=!!b.active, pop=b.pop==null?1:b.pop; |
| 55 | + const box=document.createElement('div'); |
| 56 | + box.style.cssText=`position:fixed;left:${r.left-5}px;top:${r.top-5}px;width:${r.width+10}px; |
| 57 | + height:${r.height+10}px;z-index:2147482000;border-radius:9px;border:3px solid ${A}; |
| 58 | + box-shadow:0 0 ${isA?22*pop:7}px ${A}${isA?'':'55'};pointer-events:none`; |
| 59 | + root.appendChild(box); |
| 60 | + if(b.label){ const t=document.createElement('div'); const op=isA?pop:1; |
| 61 | + t.style.cssText=`position:fixed;left:${r.left-5}px;top:${r.top+r.height+7}px;z-index:2147482100; |
| 62 | + background:${A};color:#08160c;font:700 12px system-ui,Segoe UI,sans-serif;padding:4px 9px; |
| 63 | + border-radius:7px;opacity:${op};transform:translateY(${(1-op)*6}px); |
| 64 | + box-shadow:0 3px 10px rgba(0,0,0,.35);white-space:nowrap;max-width:340px;overflow:hidden;text-overflow:ellipsis`; |
| 65 | + t.textContent=b.label; root.appendChild(t); } |
| 66 | + }); |
| 67 | + if(s.panel){ const p=document.createElement('div'); |
| 68 | + p.style.cssText=`position:fixed;right:14px;top:58px;width:340px;max-height:82vh;overflow:hidden; |
| 69 | + z-index:2147482200;background:rgba(13,17,23,.93);border:1px solid ${A}55;border-radius:12px; |
| 70 | + padding:12px 14px;font:600 12px ui-monospace,Consolas,monospace;color:#e6edf3;box-shadow:0 10px 34px rgba(0,0,0,.45)`; |
| 71 | + let html=`<div style="color:${A};font-weight:700;margin-bottom:8px;font-family:system-ui">${s.panel.title}</div>`; |
| 72 | + (s.panel.rows||[]).forEach(r=>{ html+=`<div style="padding:3px 0;border-top:1px solid #ffffff10;color:#c9d3de">${r}</div>`; }); |
| 73 | + p.innerHTML=html; root.appendChild(p); } |
| 74 | + if(s.done){ const d=document.createElement('div'); |
| 75 | + d.style.cssText=`position:fixed;left:50%;top:52%;transform:translate(-50%,-50%);z-index:2147483100; |
| 76 | + background:rgba(13,17,23,.96);color:#e6edf3;padding:18px 26px;border-radius:14px;border:1px solid ${A}; |
| 77 | + box-shadow:0 10px 40px rgba(0,0,0,.5);text-align:center;font:600 15px system-ui,Segoe UI,sans-serif`; |
| 78 | + d.innerHTML=`<div style="font-size:21px;color:${A};margin-bottom:6px">${s.done.title}</div>`+ |
| 79 | + `<div style="color:#9da7b3;font-weight:500">${s.done.sub}</div>`; root.appendChild(d); } |
| 80 | + }}; |
| 81 | +})(); |
| 82 | +""" |
| 83 | + |
| 84 | + |
| 85 | +class Cap: |
| 86 | + def __init__(self, pg, frames_dir): |
| 87 | + self.pg, self.dir, self.n = pg, frames_dir, 0 |
| 88 | + frames_dir.mkdir(parents=True, exist_ok=True) |
| 89 | + for f in frames_dir.glob("*.png"): |
| 90 | + f.unlink() |
| 91 | + |
| 92 | + def fx(self): |
| 93 | + self.pg.evaluate(FX_SETUP) |
| 94 | + |
| 95 | + def frame(self, state, reps=1): |
| 96 | + for _ in range(reps): |
| 97 | + self.pg.evaluate("(s)=>window.FX.render(s)", state) |
| 98 | + self.pg.screenshot(path=str(self.dir / f"f{self.n:04d}.png")) |
| 99 | + self.n += 1 |
| 100 | + |
| 101 | + def pops(self, base, box, reps=(0.34, 0.7, 1.0)): |
| 102 | + for pv in reps: |
| 103 | + b = dict(box); b["active"] = True; b["pop"] = pv |
| 104 | + self.frame({**base, "boxes": base.get("boxes", []) + [b]}) |
| 105 | + |
| 106 | + |
| 107 | +# ---------------- demos ---------------- |
| 108 | +def d_paginated(pg, cap): |
| 109 | + total = 0 |
| 110 | + for page in (1, 2, 3): |
| 111 | + pg.goto(f"https://quotes.toscrape.com/page/{page}/", wait_until="domcontentloaded", timeout=30000) |
| 112 | + pg.evaluate("window.scrollTo(0,0)"); cap.fx() |
| 113 | + cnt = pg.evaluate("()=>document.querySelectorAll('div.quote').length") |
| 114 | + data = pg.evaluate("""()=>[...document.querySelectorAll('div.quote')].slice(0,3).map(q=> |
| 115 | + ({a:q.querySelector('.author')?.textContent, t:[...q.querySelectorAll('a.tag')].slice(0,2).map(x=>x.textContent).join(', ')}))""") |
| 116 | + for i, q in enumerate(data): |
| 117 | + base = {"title": "paginated scrape", "status": f"page {page}/3 · {total+i+1} quotes", |
| 118 | + "boxes": [{"sel": "div.quote", "idx": j, "active": False} for j in range(i)]} |
| 119 | + cap.pops(base, {"sel": "div.quote", "idx": i, "label": f"✓ {q['a']} · #{q['t']}"}) |
| 120 | + total += cnt |
| 121 | + cap.frame({"title": "paginated scrape", "status": f"page {page}/3 · {total} quotes", |
| 122 | + "boxes": [{"sel": "div.quote", "idx": j, "active": False} for j in range(3)]}, reps=2) |
| 123 | + for _ in range(10): |
| 124 | + cap.frame({"title": "paginated scrape", "status": "done · 30 quotes", |
| 125 | + "done": {"title": "30 quotes · 3 pages", "sub": "authors + tags · auto-pagination · 0 blocks"}}) |
| 126 | + |
| 127 | + |
| 128 | +def d_structured(pg, cap): |
| 129 | + pg.goto("https://books.toscrape.com/", wait_until="networkidle", timeout=30000) |
| 130 | + pg.evaluate("window.scrollTo(0,90)"); cap.fx() |
| 131 | + books = pg.evaluate("""()=>[...document.querySelectorAll('article.product_pod')].slice(0,8).map(a=> |
| 132 | + ({t:a.querySelector('h3 a')?.getAttribute('title'),p:a.querySelector('.price_color')?.textContent?.trim(), |
| 133 | + r:(a.querySelector('p.star-rating')?.className||'').replace('star-rating','').trim(), |
| 134 | + s:a.querySelector('.instock')?.textContent?.trim()?'In stock':'-'}))""") |
| 135 | + rows = [] |
| 136 | + for i, bk in enumerate(books): |
| 137 | + t = (bk['t'] or '')[:22] |
| 138 | + base = {"title": "structured extraction", "status": f"{i+1}/8 records", |
| 139 | + "panel": {"title": "records[] → JSON", "rows": list(rows)}, |
| 140 | + "boxes": [{"sel": "article.product_pod", "idx": j, "active": False} for j in range(i)]} |
| 141 | + cap.pops(base, {"sel": "article.product_pod", "idx": i, "label": f"✓ {bk['p']} · ★{bk['r']}"}) |
| 142 | + rows.append(f'{{ "{t}", "{bk["p"]}", "★{bk["r"]}", "{bk["s"]}" }}') |
| 143 | + cap.frame({"title": "structured extraction", "status": f"{i+1}/8 records", |
| 144 | + "panel": {"title": "records[] → JSON", "rows": list(rows)}, |
| 145 | + "boxes": [{"sel": "article.product_pod", "idx": j, "active": False} for j in range(i+1)]}) |
| 146 | + for _ in range(10): |
| 147 | + cap.frame({"title": "structured extraction", "status": "done", |
| 148 | + "panel": {"title": "records[] → JSON", "rows": list(rows)}, |
| 149 | + "done": {"title": "20 records → structured", "sub": "title · price · rating · stock · typed"}}) |
| 150 | + |
| 151 | + |
| 152 | +def d_detail(pg, cap): |
| 153 | + pg.goto("https://books.toscrape.com/catalogue/a-light-in-the-attic_1000/index.html", |
| 154 | + wait_until="domcontentloaded", timeout=30000) |
| 155 | + pg.evaluate("window.scrollTo(0,220)"); cap.fx() |
| 156 | + fields = pg.evaluate(r"""()=>{ |
| 157 | + const rows=[...document.querySelectorAll('table.table-striped tr')]; |
| 158 | + const get=(l)=>{const tr=rows.find(r=>r.querySelector('th')?.textContent.trim()===l);return tr?tr.querySelector('td').textContent.trim():'';}; |
| 159 | + return [ |
| 160 | + {sel:'div.product_main h1', idx:0, label:'title: '+(document.querySelector('div.product_main h1')?.textContent||'').slice(0,20)}, |
| 161 | + {sel:'table.table-striped tr', idx:0, label:'UPC: '+get('UPC')}, |
| 162 | + {sel:'table.table-striped tr', idx:3, label:'Price (incl tax): '+get('Price (incl. tax)')}, |
| 163 | + {sel:'table.table-striped tr', idx:5, label:'Availability: '+get('Availability')}, |
| 164 | + {sel:'table.table-striped tr', idx:6, label:'Reviews: '+get('Number of reviews')}, |
| 165 | + ]; |
| 166 | + }""") |
| 167 | + for i, f in enumerate(fields): |
| 168 | + base = {"title": "deep detail crawl", "status": f"extracting field {i+1}/{len(fields)}", |
| 169 | + "boxes": [{"sel": fields[j]['sel'], "idx": fields[j]['idx'], "label": fields[j]['label'], "active": False} for j in range(i)]} |
| 170 | + cap.pops(base, {"sel": f['sel'], "idx": f['idx'], "label": f['label']}) |
| 171 | + hold = {"title": "deep detail crawl", "status": "done", |
| 172 | + "boxes": [{"sel": f['sel'], "idx": f['idx'], "label": f['label'], "active": False} for f in fields]} |
| 173 | + for _ in range(10): |
| 174 | + cap.frame({**hold, "done": {"title": f"{len(fields)} fields · deep crawl", "sub": "UPC · price · tax · stock · reviews"}}) |
| 175 | + |
| 176 | + |
| 177 | +def d_js(pg, cap): |
| 178 | + pg.goto("https://quotes.toscrape.com/js/", wait_until="domcontentloaded", timeout=30000); cap.fx() |
| 179 | + for _ in range(3): |
| 180 | + cap.frame({"title": "JS-rendered (CDP)", "status": "client-side render — waiting for JS…"}) |
| 181 | + pg.wait_for_selector("div.quote", timeout=15000); time.sleep(0.5); pg.evaluate("window.scrollTo(0,0)") |
| 182 | + data = pg.evaluate("()=>[...document.querySelectorAll('div.quote')].slice(0,4).map(q=>({a:q.querySelector('.author')?.textContent}))") |
| 183 | + for i, q in enumerate(data): |
| 184 | + base = {"title": "JS-rendered (CDP)", "status": f"rendered by V8 · {i+1} captured", |
| 185 | + "boxes": [{"sel": "div.quote", "idx": j, "active": False} for j in range(i)]} |
| 186 | + cap.pops(base, {"sel": "div.quote", "idx": i, "label": f"✓ {q['a']}"}) |
| 187 | + for _ in range(10): |
| 188 | + cap.frame({"title": "JS-rendered (CDP)", "status": "done · full browser render", |
| 189 | + "done": {"title": "JS-rendered · captured", "sub": "client-side DOM · real V8 · over raw CDP"}}) |
| 190 | + |
| 191 | + |
| 192 | +DEMOS = {"paginated": d_paginated, "structured": d_structured, "detail": d_detail, "js": d_js} |
| 193 | + |
| 194 | + |
| 195 | +def build_gif(frames_dir: Path, out_gif: Path): |
| 196 | + from PIL import Image |
| 197 | + files = sorted(frames_dir.glob("f*.png")) |
| 198 | + imgs = [Image.open(f).convert("RGB").resize((760, int(760 * Image.open(f).height / Image.open(f).width))) for f in files] |
| 199 | + pal = imgs[len(imgs) // 2].quantize(colors=256) |
| 200 | + frames = [im.quantize(palette=pal, dither=Image.Dither.FLOYDSTEINBERG) for im in imgs] |
| 201 | + frames[0].save(out_gif, save_all=True, append_images=frames[1:], duration=int(1000 / FPS), loop=0, optimize=True) |
| 202 | + print(f"[*] wrote {out_gif} ({out_gif.stat().st_size // 1024} KB, {len(frames)} frames)") |
| 203 | + |
| 204 | + |
| 205 | +def run_one(name: str): |
| 206 | + from tilion_fortress import Fortress |
| 207 | + from playwright.sync_api import sync_playwright |
| 208 | + frames_dir = OUT / f".frames_{name}" |
| 209 | + print(f"[*] demo '{name}': launching Fortress...") |
| 210 | + with Fortress() as f: |
| 211 | + with sync_playwright() as p: |
| 212 | + b = p.chromium.connect_over_cdp(f.cdp_url) |
| 213 | + ctx = b.contexts[0] if b.contexts else b.new_context() |
| 214 | + pg = ctx.pages[0] if ctx.pages else ctx.new_page() |
| 215 | + pg.set_viewport_size(VIEWPORT) |
| 216 | + cap = Cap(pg, frames_dir) |
| 217 | + DEMOS[name](pg, cap) |
| 218 | + print(f"[*] captured {cap.n} frames") |
| 219 | + b.close() |
| 220 | + build_gif(frames_dir, OUT / f"fortress-scrape-{name}.gif") |
| 221 | + |
| 222 | + |
| 223 | +def main(): |
| 224 | + which = sys.argv[1] if len(sys.argv) > 1 else "structured" |
| 225 | + names = list(DEMOS) if which == "all" else [which] |
| 226 | + for n in names: |
| 227 | + if n not in DEMOS: |
| 228 | + print(f"unknown demo '{n}'. choose: {', '.join(DEMOS)} | all"); return 2 |
| 229 | + run_one(n) |
| 230 | + return 0 |
| 231 | + |
| 232 | + |
| 233 | +if __name__ == "__main__": |
| 234 | + raise SystemExit(main()) |
0 commit comments