docs: update notebooks for release (#218)

Author: lipikaramaswamy

docs/notebook_source/01_your_first_anonymization.py

@@ -1,6 +1,3 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
 # ---
 # jupyter:
 #   jupytext:
@@ -15,6 +12,10 @@
 # ---
 
 # %% [markdown]
+# <!--
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+# -->
 # # 🕵️ Your First Anonymization
 #
 # Detect sensitive entities and replace them with LLM-generated substitutes --

docs/notebook_source/02_inspecting_detected_entities.py

@@ -1,6 +1,3 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
 # ---
 # jupyter:
 #   jupytext:
@@ -15,6 +12,10 @@
 # ---
 
 # %% [markdown]
+# <!--
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+# -->
 # # 🕵️ Inspecting Detected Entities
 #
 # Dig into the entity detection pipeline output -- what was detected,
@@ -26,6 +27,10 @@
 # We use **Annotate** mode because it preserves the original text while tagging each entity
 # with its label, making it ideal for reviewing detection quality.
 #
+# > **Privacy warning:** `Annotate` does not anonymize the text. Sensitive values
+# > remain in the output, so use it only for inspection -- not as a privacy-safe
+# > production strategy.
+#
 # #### 📚 What you'll learn
 #
 # - Run the detection pipeline and inspect its output using Annotate mode
@@ -100,13 +105,15 @@
 # %% [markdown]
 # ## 📋 Columns
 #
-# - `trace_dataframe` contains all internal columns from the pipeline
-#   (detection, validation, replacement, etc.).
+# - `result.dataframe["final_entities"]` is the stable, public entity output.
+# - `trace_dataframe` contains internal pipeline columns for deeper debugging;
+#   those underscore-prefixed columns may change between releases.
 
 # %%
-df = result.trace_dataframe
-print(f"Records: {len(df)}")
-print(f"Columns: {list(df.columns)}")
+trace_df = result.trace_dataframe
+final_entities = result.dataframe["final_entities"]
+print(f"Records: {len(trace_df)}")
+print(f"Columns: {list(trace_df.columns)}")
 
 # %% [markdown]
 # ## 🎯 Detected entities
@@ -116,7 +123,7 @@
 
 # %%
 row_idx = 0
-raw = df.loc[row_idx, "_detected_entities"]
+raw = final_entities.iloc[row_idx]
 entities = raw["entities"] if isinstance(raw, dict) else raw
 print(f"Record {row_idx}: {len(entities)} entities detected\n")
 
@@ -132,7 +139,7 @@
 
 # %%
 label_counts = Counter()
-for raw in df["_detected_entities"]:
+for raw in final_entities:
     entity_list = raw["entities"] if isinstance(raw, dict) else raw
     for entity in entity_list:
         label_counts[entity["label"]] += 1
@@ -152,7 +159,7 @@
 
 # %%
 source_counts = Counter()
-for raw in df["_detected_entities"]:
+for raw in final_entities:
     entity_list = raw["entities"] if isinstance(raw, dict) else raw
     for entity in entity_list:
         source_counts[entity.get("source", "unknown")] += 1
@@ -168,7 +175,7 @@
 
 # %%
 row_idx = 0
-raw_bv = df.loc[row_idx, "_entities_by_value"]
+raw_bv = trace_df.loc[row_idx, "_entities_by_value"]
 by_value = raw_bv["entities_by_value"] if isinstance(raw_bv, dict) else raw_bv
 print(f"Record {row_idx}: {len(by_value)} unique entity values\n")
 

docs/notebook_source/03_choosing_a_replacement_strategy.py

@@ -1,6 +1,3 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
 # ---
 # jupyter:
 #   jupytext:
@@ -15,6 +12,10 @@
 # ---
 
 # %% [markdown]
+# <!--
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+# -->
 # # 🕵️ Choosing a Replacement Strategy
 #
 # Four [replace mode](../../concepts/replace/) strategies compared side-by-side on the same data.
@@ -25,6 +26,7 @@
 # | **Redact** | Label-based markers (`[REDACTED_FIRST_NAME]`) |
 # | **Annotate** | Tags entities but keeps original text |
 # | **Hash** | Deterministic hash digest |
+#
 # #### 📚 What you'll learn
 #
 # - Compare **Redact**, **Annotate**, **Hash**, and **Substitute** on the same input

docs/notebook_source/04_rewriting_biographies.py

@@ -1,6 +1,3 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
 # ---
 # jupyter:
 #   jupytext:
@@ -15,16 +12,22 @@
 # ---
 
 # %% [markdown]
+# <!--
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+# -->
 # # 🕵️ Rewriting Biographies
 #
 # Instead of replacing entities with tokens, rewrite mode generates a
-# privacy-safe transformation of the entire text. The pipeline:
+# privacy-safe transformation of the entire text. The `run()` / `preview()` pipeline:
 #
 # 1. Detects entities (same as replace mode, plus latent entity detection)
 # 2. Classifies the domain and assigns sensitivity dispositions
 # 3. Generates a rewritten version that obscures sensitive entities
 # 4. Evaluates quality (utility) and privacy (leakage) with an automated repair loop
-# 5. Runs a final optional LLM judge for informational scores
+#
+# Afterward, a separate optional `evaluate()` call runs LLM judges for
+# detection validity and holistic privacy, quality, and style scores.
 #
 #
 # #### 📚 What you'll learn
@@ -103,6 +106,8 @@
             protect="All direct identifiers and quasi-identifier combinations (names, locations, employers, dates)",
             preserve="Career trajectory, educational background, and professional accomplishments",
         ),
+        risk_tolerance="low",
+        max_repair_iterations=3,
     ),
 )
 
@@ -125,6 +130,11 @@
 preview.display_record(1)
 
 # %% [markdown]
+# > **How to interpret leakage:** Leakage is measured against the sensitivity
+# > disposition. Details marked `leave_as_is` may remain without increasing
+# > `leakage_mass`. If an output retains something you expected the privacy goal
+# > to protect, inspect the Entity Disposition table.
+#
 # ## 🚀 Full run
 #
 # - `result.dataframe` has user-facing columns: rewritten text, scores, and the review flag.
@@ -145,6 +155,8 @@
 # ## 🚩 Filter by review flag
 #
 # - Records where automated metrics exceed thresholds are flagged for manual review.
+# - `needs_human_review` is threshold-based, so a record can have small nonzero
+#   leakage without being flagged.
 # - Use this to prioritize human attention on the records that need it most.
 # - See [Working with flagged records](../../concepts/rewrite/#working-with-flagged-records)
 #   for guidance on diagnosing and resolving flagged records.
@@ -159,6 +171,9 @@
 # ## 🔬 Evaluate (optional)
 #
 # Call `evaluate()` to run LLM-as-judge scoring on the rewrite result — detection validity and three quality rubrics (privacy, quality, style).
+# Evaluation makes additional LLM calls per record. For larger datasets, evaluate
+# a preview first; this tutorial evaluates all 25 rows to demonstrate the complete workflow.
+# This holistic judge is independent of pipeline leakage scoring, so their assessments may differ.
 # See [Evaluation](../../concepts/evaluation/#rewrite-evaluation) for details.
 
 # %%

docs/notebook_source/05_rewriting_legal_documents.py

@@ -1,6 +1,3 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
 # ---
 # jupyter:
 #   jupytext:
@@ -15,6 +12,10 @@
 # ---
 
 # %% [markdown]
+# <!--
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+# -->
 # # 🕵️ Rewriting Legal Documents
 #
 # Rewriting legal text (TAB dataset) with a domain-specific privacy goal
@@ -115,6 +116,8 @@
 # ## 🎛️ Configure
 #
 # - `Detect(entity_labels=...)` overrides the default entity set with legal-specific labels.
+#   The explicit list is a strict allowlist for both detection and LLM augmentation:
+#   labels not included here are filtered out, so include every entity type you need.
 # - `PrivacyGoal` tells the rewriter what to **protect** (identifiers, case numbers,
 #   institutional references) and what to **preserve** (legal reasoning, statutory references,
 #   ruling structure).
@@ -153,9 +156,16 @@
 preview.display_record(1)
 
 # %% [markdown]
+# > **How to interpret leakage:** Leakage is measured against the sensitivity
+# > disposition. Details marked `leave_as_is` may remain without increasing
+# > `leakage_mass`. If an output retains something you expected the privacy goal
+# > to protect, inspect the Entity Disposition table.
+#
 # ## 🚀 Full run
 #
 # - `result.dataframe` has user-facing columns: rewritten text, scores, and the review flag.
+# - This notebook uses `risk_tolerance="minimal"`, which applies stricter repair
+#   and review thresholds than notebook 04.
 
 # %%
 result = anonymizer.run(config=config, data=input_data)
@@ -169,6 +179,8 @@
 # ## 🚩 Filter by review flag
 #
 # - Records where automated metrics exceed thresholds are flagged for manual review.
+# - The repair loop stops after `max_repair_iterations`; records that still need
+#   repair remain flagged for human review but are not pipeline failures.
 # - Use this to prioritize human attention on the records that need it most.
 # - See [Working with flagged records](../../concepts/rewrite/#working-with-flagged-records)
 #   for guidance on diagnosing and resolving flagged records.
@@ -183,6 +195,9 @@
 # ## 🔬 Evaluate (optional)
 #
 # Call `evaluate()` to run LLM-as-judge scoring on the rewrite result — detection validity and three quality rubrics (privacy, quality, style).
+# Evaluation makes additional LLM calls per record. For larger datasets, evaluate
+# a preview first; this tutorial evaluates all 25 rows to demonstrate the complete workflow.
+# This holistic judge is independent of pipeline leakage scoring, so their assessments may differ.
 # See [Evaluation](../../concepts/evaluation/#rewrite-evaluation) for details.
 
 # %%