feat: base model switched to roberta rebase

.gitattributes

@@ -1 +1,2 @@
 model/quantized/** filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text

model/dataset/huggingface/upload_dataset_to_hf.py

@@ -7,7 +7,10 @@
 
 import json
 import os
+import shutil
+import subprocess
 import sys
+import tempfile
 from collections import Counter
 from pathlib import Path
 
@@ -277,6 +280,79 @@ def _generate_dataset_card(
     return card.strip() + "\n"
 
 
+def _upload_binary_via_git(
+    file_path: Path,
+    path_in_repo: str,
+    repo_id: str,
+    token: str,
+) -> None:
+    """Upload a binary file to HuggingFace via git clone + LFS push.
+
+    The HF Hub API rejects binary files and requires Xet/LFS storage.
+    This clones the repo, adds the file with LFS tracking, and pushes.
+    """
+    tmpdir = Path(tempfile.mkdtemp())
+    try:
+        repo_url = f"https://x-access-token:{token}@huggingface.co/datasets/{repo_id}"
+        subprocess.run(
+            ["git", "clone", "--depth", "1", repo_url, str(tmpdir / "repo")],
+            check=True,
+            capture_output=True,
+            env={**os.environ, "GIT_LFS_SKIP_SMUDGE": "1"},
+        )
+        repo_dir = tmpdir / "repo"
+
+        # Ensure the file extension is tracked by LFS
+        ext = file_path.suffix  # e.g. ".tsv"
+        gitattributes = repo_dir / ".gitattributes"
+        lfs_pattern = f"*{ext} filter=lfs diff=lfs merge=lfs -text"
+        if gitattributes.exists():
+            content = gitattributes.read_text()
+            if lfs_pattern not in content:
+                with gitattributes.open("a") as f:
+                    f.write(f"\n{lfs_pattern}\n")
+        else:
+            gitattributes.write_text(f"{lfs_pattern}\n")
+
+        shutil.copy(file_path, repo_dir / path_in_repo)
+
+        git_env = {**os.environ}
+        subprocess.run(
+            ["git", "add", ".gitattributes", path_in_repo],
+            cwd=repo_dir,
+            check=True,
+            capture_output=True,
+            env=git_env,
+        )
+        subprocess.run(
+            [
+                "git",
+                "-c",
+                "user.name=kiji",
+                "-c",
+                "user.email=kiji@575.ai",
+                "commit",
+                "-m",
+                f"Add {path_in_repo}",
+            ],
+            cwd=repo_dir,
+            check=True,
+            capture_output=True,
+            env=git_env,
+        )
+        subprocess.run(
+            ["git", "push"],
+            cwd=repo_dir,
+            check=True,
+            capture_output=True,
+            text=True,
+            env=git_env,
+        )
+        print(f"  Pushed {path_in_repo} via git LFS")
+    finally:
+        shutil.rmtree(tmpdir, ignore_errors=True)
+
+
 def upload_to_huggingface(
     samples_dir: str = "model/dataset/data_samples/training_samples",
     repo_id: str | None = None,
@@ -343,7 +419,7 @@ def upload_to_huggingface(
     print(f"\nPushing to {repo_id} (private={private})...")
     dataset_dict.push_to_hub(repo_id, token=token)
 
-    # Generate and upload dataset card
+    # Generate dataset card
     print("Generating dataset card...")
     stats = _compute_dataset_stats(samples)
     card = _generate_dataset_card(
@@ -352,13 +428,29 @@ def upload_to_huggingface(
         train_count=len(dataset_dict["train"]),
         test_count=len(dataset_dict["test"]),
     )
+
+    # Upload dataset card (text file, via API)
     api.upload_file(
         path_or_fileobj=card.encode(),
         path_in_repo="README.md",
         repo_id=repo_id,
         repo_type="dataset",
     )
 
+    # Upload audit ledger via git clone + LFS (HF rejects binary files via API,
+    # requiring Xet/LFS storage via git push)
+    audit_ledger_path = Path(samples_dir) / "audit_ledger.tsv"
+    if audit_ledger_path.exists():
+        print(f"Uploading audit ledger via git LFS: {audit_ledger_path}")
+        _upload_binary_via_git(
+            file_path=audit_ledger_path,
+            path_in_repo="audit_ledger.tsv",
+            repo_id=repo_id,
+            token=token,
+        )
+    else:
+        print(f"Warning: audit_ledger.tsv not found at {audit_ledger_path}, skipping")
+
     print(f"Done! Dataset available at: https://huggingface.co/datasets/{repo_id}")
     print("\nUsage:")
     print("  from datasets import load_dataset")

model/flows/training_config.toml

@@ -39,7 +39,7 @@ subsample_count = 0
 num_ai4privacy_samples = -1
 # Path to audit_allowlist.txt to filter training samples (empty = no filtering)
 # Generate with: uv run python model/dataset/audit_dataset.py --samples-dir <dir>
-audit_allowlist = ""
+audit_allowlist = "/home/hannes/kiji-proxy/model/dataset/data_samples/training_samples/audit_ledger.tsv"
 
 [pipeline]
 # Skip Label Studio export and use existing data from paths.training_samples_dir

model/quantized/added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "[MASK]": 128000
+}

model/quantized/label_mappings.json

@@ -1,3 +1,115 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:25751d603bba29f7a3269f38f7d3e402770ea31cc0ceff68a9eeda79efce3aa6
-size 2933
+{
+  "pii": {
+    "label2id": {
+      "O": 0,
+      "B-SURNAME": 1,
+      "I-SURNAME": 2,
+      "B-FIRSTNAME": 3,
+      "I-FIRSTNAME": 4,
+      "B-BUILDINGNUM": 5,
+      "I-BUILDINGNUM": 6,
+      "B-DATEOFBIRTH": 7,
+      "I-DATEOFBIRTH": 8,
+      "B-EMAIL": 9,
+      "I-EMAIL": 10,
+      "B-PHONENUMBER": 11,
+      "I-PHONENUMBER": 12,
+      "B-CITY": 13,
+      "I-CITY": 14,
+      "B-URL": 15,
+      "I-URL": 16,
+      "B-COMPANYNAME": 17,
+      "I-COMPANYNAME": 18,
+      "B-STATE": 19,
+      "I-STATE": 20,
+      "B-ZIP": 21,
+      "I-ZIP": 22,
+      "B-STREET": 23,
+      "I-STREET": 24,
+      "B-COUNTRY": 25,
+      "I-COUNTRY": 26,
+      "B-SSN": 27,
+      "I-SSN": 28,
+      "B-DRIVERLICENSENUM": 29,
+      "I-DRIVERLICENSENUM": 30,
+      "B-PASSPORTID": 31,
+      "I-PASSPORTID": 32,
+      "B-NATIONALID": 33,
+      "I-NATIONALID": 34,
+      "B-IDCARDNUM": 35,
+      "I-IDCARDNUM": 36,
+      "B-TAXNUM": 37,
+      "I-TAXNUM": 38,
+      "B-LICENSEPLATENUM": 39,
+      "I-LICENSEPLATENUM": 40,
+      "B-PASSWORD": 41,
+      "I-PASSWORD": 42,
+      "B-IBAN": 43,
+      "I-IBAN": 44,
+      "B-AGE": 45,
+      "I-AGE": 46,
+      "B-SECURITYTOKEN": 47,
+      "I-SECURITYTOKEN": 48,
+      "B-CREDITCARDNUMBER": 49,
+      "I-CREDITCARDNUMBER": 50,
+      "B-USERNAME": 51,
+      "I-USERNAME": 52
+    },
+    "id2label": {
+      "0": "O",
+      "1": "B-SURNAME",
+      "2": "I-SURNAME",
+      "3": "B-FIRSTNAME",
+      "4": "I-FIRSTNAME",
+      "5": "B-BUILDINGNUM",
+      "6": "I-BUILDINGNUM",
+      "7": "B-DATEOFBIRTH",
+      "8": "I-DATEOFBIRTH",
+      "9": "B-EMAIL",
+      "10": "I-EMAIL",
+      "11": "B-PHONENUMBER",
+      "12": "I-PHONENUMBER",
+      "13": "B-CITY",
+      "14": "I-CITY",
+      "15": "B-URL",
+      "16": "I-URL",
+      "17": "B-COMPANYNAME",
+      "18": "I-COMPANYNAME",
+      "19": "B-STATE",
+      "20": "I-STATE",
+      "21": "B-ZIP",
+      "22": "I-ZIP",
+      "23": "B-STREET",
+      "24": "I-STREET",
+      "25": "B-COUNTRY",
+      "26": "I-COUNTRY",
+      "27": "B-SSN",
+      "28": "I-SSN",
+      "29": "B-DRIVERLICENSENUM",
+      "30": "I-DRIVERLICENSENUM",
+      "31": "B-PASSPORTID",
+      "32": "I-PASSPORTID",
+      "33": "B-NATIONALID",
+      "34": "I-NATIONALID",
+      "35": "B-IDCARDNUM",
+      "36": "I-IDCARDNUM",
+      "37": "B-TAXNUM",
+      "38": "I-TAXNUM",
+      "39": "B-LICENSEPLATENUM",
+      "40": "I-LICENSEPLATENUM",
+      "41": "B-PASSWORD",
+      "42": "I-PASSWORD",
+      "43": "B-IBAN",
+      "44": "I-IBAN",
+      "45": "B-AGE",
+      "46": "I-AGE",
+      "47": "B-SECURITYTOKEN",
+      "48": "I-SECURITYTOKEN",
+      "49": "B-CREDITCARDNUMBER",
+      "50": "I-CREDITCARDNUMBER",
+      "51": "B-USERNAME",
+      "52": "I-USERNAME",
+      "-100": "IGNORE"
+    }
+  }
+}