|
| 1 | +"""TagPacks: the attribution format GraphSense publishes, read as a source. |
| 2 | +
|
| 3 | +Two projects arrived at the same shape independently. A TagPack tag carries a |
| 4 | +label, a **mandatory** source, a confidence and a category --- which is this |
| 5 | +package's `Attribution` with different field names. That convergence is why |
| 6 | +reading them is a mapping rather than a translation, and it is the strongest |
| 7 | +argument for using their format instead of inventing a third. |
| 8 | +
|
| 9 | +**What this buys.** `graphsense-tagpacks` publishes 523,988 attributed |
| 10 | +addresses under MIT, across Bitcoin, Ethereum and others: exchange wallets, |
| 11 | +mining pools, mixers, sextortion campaigns, and services identified by INTERPOL. |
| 12 | +That is an order of magnitude more than every other source here combined, and |
| 13 | +none of it had to be collected. |
| 14 | +
|
| 15 | +**Their confidence model is better than ours and this file says so.** Ours is an |
| 16 | +abstract ladder (SPECULATIVE..CERTAIN) with a separate `Method`. Theirs is one |
| 17 | +vocabulary keyed on *how the tag was obtained* --- `ownership` (the creator holds |
| 18 | +the key) scores 100, `authority_data` (OFAC and the like) scores 60, |
| 19 | +`web_crawl` 20, `heuristic` 10. Confidence and provenance cannot drift apart |
| 20 | +because they are the same field. |
| 21 | +
|
| 22 | +The `authority_data` level is the interesting one. This package rates OFAC |
| 23 | +`CERTAIN`; GraphSense rates it 60. They are right. A sanctions listing is an |
| 24 | +authoritative *claim*, but the address-to-entity mapping inside it is still |
| 25 | +somebody's research and has been wrong before. `_CONFIDENCE` below preserves |
| 26 | +their judgement rather than flattening it into ours. |
| 27 | +
|
| 28 | +Format: https://github.com/graphsense/graphsense-tagpacks |
| 29 | +Taxonomy: https://github.com/graphsense/DW-VA-Taxonomy |
| 30 | +""" |
| 31 | + |
| 32 | +from __future__ import annotations |
| 33 | + |
| 34 | +from datetime import datetime, timezone |
| 35 | +from pathlib import Path |
| 36 | +from typing import Any |
| 37 | + |
| 38 | +from ...core.attribution import Attribution, Category, Confidence, Method |
| 39 | +from ...core.chainid import ChainId |
| 40 | +from ..base import Source, SourceError, SourceMeta |
| 41 | + |
| 42 | +__all__ = ["DEFAULT_DIR", "REPO", "TagPackSource"] |
| 43 | + |
| 44 | +REPO = "https://github.com/graphsense/graphsense-tagpacks" |
| 45 | +DEFAULT_DIR = "data/labels/tagpacks" |
| 46 | + |
| 47 | +#: Their confidence ids, with the level each carries and what this package |
| 48 | +#: makes of it. Their levels are 0-100; ours is a five-step ladder, so the |
| 49 | +#: mapping loses resolution --- the original id travels in the rationale so |
| 50 | +#: nothing is thrown away. |
| 51 | +#: |
| 52 | +#: Deliberately NOT collapsing `authority_data` to CERTAIN. See the module |
| 53 | +#: docstring: a sanctions listing is an authoritative claim about an entity, |
| 54 | +#: and the address-to-entity mapping inside it is still research. |
| 55 | +_CONFIDENCE: dict[str, tuple[int, Confidence]] = { |
| 56 | + "override": (100, Confidence.HIGH), |
| 57 | + "ownership": (100, Confidence.CERTAIN), |
| 58 | + "ledger_immanent": (100, Confidence.CERTAIN), |
| 59 | + "manual_transaction": (90, Confidence.HIGH), |
| 60 | + "service_api": (70, Confidence.HIGH), |
| 61 | + "forensic_investigation": (70, Confidence.HIGH), |
| 62 | + "authority_data": (60, Confidence.HIGH), |
| 63 | + "trusted_provider": (50, Confidence.MEDIUM), |
| 64 | + "service_data": (50, Confidence.MEDIUM), |
| 65 | + "forensic": (50, Confidence.MEDIUM), |
| 66 | + "untrusted_transaction": (40, Confidence.MEDIUM), |
| 67 | + "web_crawl": (20, Confidence.LOW), |
| 68 | + "heuristic": (10, Confidence.SPECULATIVE), |
| 69 | +} |
| 70 | + |
| 71 | +#: Their concept taxonomy onto ours. Unmapped concepts become `UNKNOWN` rather |
| 72 | +#: than being guessed at --- a wrong category is a claim nobody made, and their |
| 73 | +#: vocabulary is larger than ours by design. |
| 74 | +_CATEGORY: dict[str, Category] = { |
| 75 | + "exchange": Category.CEX, |
| 76 | + "decentralized_exchange": Category.DEX, |
| 77 | + "defi": Category.DEX, |
| 78 | + "mixing_service": Category.MIXER, |
| 79 | + "bridge": Category.BRIDGE, |
| 80 | + "miner": Category.MINER, |
| 81 | + "mining_pool": Category.MINER, |
| 82 | + "gambling": Category.SERVICE, |
| 83 | + "wallet_service": Category.SERVICE, |
| 84 | + "hosted_wallet": Category.SERVICE, |
| 85 | + "payment_processor": Category.SERVICE, |
| 86 | + "merchant_service": Category.SERVICE, |
| 87 | + "marketplace": Category.SERVICE, |
| 88 | + "atm": Category.SERVICE, |
| 89 | + "scam": Category.SCAM, |
| 90 | + "ponzi_scheme": Category.SCAM, |
| 91 | + "sextortion": Category.SCAM, |
| 92 | + "phishing": Category.SCAM, |
| 93 | + "ransomware": Category.ILLICIT, |
| 94 | + "darknet_market": Category.ILLICIT, |
| 95 | + "stolen_funds": Category.ILLICIT, |
| 96 | + "theft": Category.ILLICIT, |
| 97 | + "malware": Category.ILLICIT, |
| 98 | + "sanctions": Category.SANCTIONED, |
| 99 | + "terrorism_financing": Category.SANCTIONED, |
| 100 | +} |
| 101 | + |
| 102 | +#: Their `currency` codes onto CAIP-2. Only what the corpus actually contains; |
| 103 | +#: an unknown code yields a chain-agnostic claim rather than a guessed chain, |
| 104 | +#: because attaching a claim to the wrong chain is worse than attaching it to |
| 105 | +#: none. |
| 106 | +_CHAIN: dict[str, str] = { |
| 107 | + "BTC": "bip122:000000000019d6689c085ae165831e93", |
| 108 | + "ETH": "eip155:1", |
| 109 | + "BCH": "bip122:000000000000000000651ef99cb9fcbe", |
| 110 | + "LTC": "bip122:12a765e31ffd4059bada1e25190f6e98", |
| 111 | + "ZEC": "bip122:0000000000196a45a4f0a1b0e5a0d4b6", |
| 112 | + "TRX": "tron:mainnet", |
| 113 | +} |
| 114 | + |
| 115 | + |
| 116 | +class TagPackSource(Source): |
| 117 | + """Attribution tags from a local checkout of the public TagPacks. |
| 118 | +
|
| 119 | + Header fields are inherited by every tag in the pack and overridden per |
| 120 | + tag, which is how the format keeps 50,000-address files readable. Both |
| 121 | + levels are honoured here; a tag that sets its own `confidence` wins over |
| 122 | + the pack's. |
| 123 | + """ |
| 124 | + |
| 125 | + name = "tagpack" |
| 126 | + |
| 127 | + def __init__(self, path: Path | str = DEFAULT_DIR) -> None: |
| 128 | + self.path = Path(path) |
| 129 | + self.meta = SourceMeta( |
| 130 | + publisher="GraphSense / Iknaio Cryptoasset Analytics GmbH and contributors", |
| 131 | + license="MIT", |
| 132 | + redistributable=True, |
| 133 | + url=REPO, |
| 134 | + ) |
| 135 | + self._index: dict[str, list[dict[str, Any]]] | None = None |
| 136 | + |
| 137 | + def ready(self) -> bool: |
| 138 | + """Whether a checkout is present. |
| 139 | +
|
| 140 | + Separate from `lookup` returning nothing, and the separation is the |
| 141 | + point: a source that answers "no tags" because its directory is missing |
| 142 | + looks exactly like a clean screening result. |
| 143 | + """ |
| 144 | + return self.path.is_dir() and any(self.path.rglob("*.yaml")) |
| 145 | + |
| 146 | + def _load(self) -> dict[str, list[dict[str, Any]]]: |
| 147 | + if self._index is not None: |
| 148 | + return self._index |
| 149 | + if not self.ready(): |
| 150 | + raise SourceError( |
| 151 | + f"no tagpacks at {self.path}. Clone {REPO} there " |
| 152 | + f"(`chainscope labels fetch tagpack`). Until then this source " |
| 153 | + f"reports nothing, and nothing is not the same as clean" |
| 154 | + ) |
| 155 | + try: |
| 156 | + import yaml |
| 157 | + except ImportError as exc: |
| 158 | + # Optional on purpose: the corpus is a separate half-million-address |
| 159 | + # download, and somebody who never fetches it should not carry a |
| 160 | + # YAML parser. Naming the extra matters --- "no module named yaml" |
| 161 | + # sends a reader to pip install yaml, which is a different package. |
| 162 | + raise SourceError( |
| 163 | + "reading tagpacks needs PyYAML, which is an optional extra " |
| 164 | + "here: `pip install 'chainscope[tagpacks]'`" |
| 165 | + ) from exc |
| 166 | + |
| 167 | + index: dict[str, list[dict[str, Any]]] = {} |
| 168 | + for file in sorted(self.path.rglob("*.yaml")): |
| 169 | + try: |
| 170 | + pack = yaml.safe_load(file.read_text(errors="replace")) |
| 171 | + except Exception: |
| 172 | + # One malformed pack must not cost the other seventy-six. |
| 173 | + continue |
| 174 | + if not isinstance(pack, dict): |
| 175 | + continue |
| 176 | + header = {k: v for k, v in pack.items() if k != "tags"} |
| 177 | + header["_pack"] = file.name |
| 178 | + for tag in pack.get("tags") or (): |
| 179 | + if not isinstance(tag, dict): |
| 180 | + continue |
| 181 | + address = str(tag.get("address") or "").strip() |
| 182 | + if not address: |
| 183 | + continue |
| 184 | + merged = {**header, **tag} |
| 185 | + index.setdefault(_fold(address), []).append(merged) |
| 186 | + self._index = index |
| 187 | + return index |
| 188 | + |
| 189 | + def lookup(self, address: str, chain: ChainId | None = None) -> list[Attribution]: |
| 190 | + """Tags for this address. Empty means *not in this corpus*. |
| 191 | +
|
| 192 | + Not "clean". 523,988 addresses is large but it is a curated collection, |
| 193 | + heavily weighted towards Bitcoin services and a few campaigns. |
| 194 | + """ |
| 195 | + rows = self._load().get(_fold(address), []) |
| 196 | + out: list[Attribution] = [] |
| 197 | + for row in rows: |
| 198 | + claim = _to_attribution(address, row) |
| 199 | + if claim is None: |
| 200 | + continue |
| 201 | + if chain is not None and claim.chain is not None and claim.chain != chain: |
| 202 | + continue |
| 203 | + out.append(claim) |
| 204 | + return out |
| 205 | + |
| 206 | + |
| 207 | +def _fold(address: str) -> str: |
| 208 | + """Fold an EVM address, leave anything else exactly as written. |
| 209 | +
|
| 210 | + The corpus is mostly base58 Bitcoin addresses, where lowercasing both |
| 211 | + invents a match against an address nobody listed and loses the one that |
| 212 | + was. Same rule as every other source here. |
| 213 | + """ |
| 214 | + text = address.strip() |
| 215 | + if text.startswith(("0x", "0X")) and len(text) == 42: |
| 216 | + return text.lower() |
| 217 | + return text |
| 218 | + |
| 219 | + |
| 220 | +def _to_attribution(address: str, row: dict[str, Any]) -> Attribution | None: |
| 221 | + label = str(row.get("label") or "").strip() |
| 222 | + if not label: |
| 223 | + return None |
| 224 | + |
| 225 | + level, confidence = _CONFIDENCE.get( |
| 226 | + str(row.get("confidence") or "").strip(), (0, Confidence.SPECULATIVE) |
| 227 | + ) |
| 228 | + category = _CATEGORY.get(str(row.get("category") or "").strip().lower(), Category.UNKNOWN) |
| 229 | + chain = _chain_of(row) |
| 230 | + |
| 231 | + # Their id and numeric level travel in the rationale. Our five-step ladder |
| 232 | + # is coarser than their hundred-point scale, so the mapping loses |
| 233 | + # resolution --- keeping the original means a reader can recover it, and a |
| 234 | + # future version can use it directly. |
| 235 | + parts = [f"tagpack {row.get('_pack')}"] |
| 236 | + if row.get("confidence"): |
| 237 | + parts.append(f"confidence={row['confidence']} ({level}/100)") |
| 238 | + if row.get("actor"): |
| 239 | + parts.append(f"actor={row['actor']}") |
| 240 | + if row.get("context"): |
| 241 | + parts.append(str(row["context"])[:200]) |
| 242 | + |
| 243 | + return Attribution( |
| 244 | + address=address, |
| 245 | + chain=chain, |
| 246 | + label=label, |
| 247 | + category=category, |
| 248 | + confidence=confidence, |
| 249 | + method=Method.LIST, |
| 250 | + source=f"{row.get('title') or 'TagPack'} via {row.get('creator') or 'unknown creator'}", |
| 251 | + rationale="; ".join(parts), |
| 252 | + observed_at=_when(row.get("lastmod")), |
| 253 | + ) |
| 254 | + |
| 255 | + |
| 256 | +def _chain_of(row: dict[str, Any]) -> ChainId | None: |
| 257 | + """The CAIP-2 chain, or None when the code is unrecognised. |
| 258 | +
|
| 259 | + None means "applies everywhere", which is how a chain-agnostic claim is |
| 260 | + represented here --- and it is the safe answer, because attaching a tag to |
| 261 | + the wrong chain asserts something about twenty bytes on a network the |
| 262 | + tagger never looked at. |
| 263 | + """ |
| 264 | + code = str(row.get("network") or row.get("currency") or "").strip().upper() |
| 265 | + caip = _CHAIN.get(code) |
| 266 | + if caip is None: |
| 267 | + return None |
| 268 | + try: |
| 269 | + return ChainId.parse(caip) |
| 270 | + except Exception: |
| 271 | + return None |
| 272 | + |
| 273 | + |
| 274 | +def _when(value: Any) -> datetime | None: |
| 275 | + """The tag's date, or None. Never "now" --- see `darklist._parse_date`.""" |
| 276 | + if not value: |
| 277 | + return None |
| 278 | + if isinstance(value, datetime): |
| 279 | + return value if value.tzinfo else value.replace(tzinfo=timezone.utc) |
| 280 | + text = str(value).strip().replace("Z", "+00:00") |
| 281 | + try: |
| 282 | + parsed = datetime.fromisoformat(text) |
| 283 | + except ValueError: |
| 284 | + try: |
| 285 | + parsed = datetime.strptime(text, "%Y-%m-%d") |
| 286 | + except ValueError: |
| 287 | + return None |
| 288 | + return parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc) |
0 commit comments