Skip to content

Commit a856ee4

Browse files
loreylorey
committed
refactor: Extract plots into own module
1 parent baf6afd commit a856ee4

2 files changed

Lines changed: 144 additions & 137 deletions

File tree

‎apistemic/benchmarks/plots.py‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
import numpy as np
2+
import matplotlib.pyplot as plt
3+
from .models import EvaluationMetrics
4+
5+
6+
def create_box_plot(all_results: dict[str, list[EvaluationMetrics]]) -> None:
7+
"""Create box plot of R² scores by model."""
8+
models = list(all_results.keys())
9+
r2_scores = []
10+
11+
for model in models:
12+
model_r2_scores = [metrics.r2 for metrics in all_results[model]]
13+
r2_scores.append(model_r2_scores)
14+
15+
plt.style.use("grayscale")
16+
plt.figure(figsize=(10, 6))
17+
plt.boxplot(r2_scores, tick_labels=models, patch_artist=False)
18+
19+
plt.title("How Well Embedding Models Understand Companies")
20+
plt.xlabel("Embedding Model (applied to company name)")
21+
plt.ylabel("R² Score (based on embedded company name only)")
22+
plt.grid(True, alpha=0.3)
23+
plt.xticks(rotation=45)
24+
plt.tight_layout()
25+
26+
# Save the plot
27+
plt.savefig(".data/plots/r2-scores-boxplot.png", dpi=300, bbox_inches="tight")
28+
29+
# Print summary statistics
30+
print("\n" + "=" * 60)
31+
print("SUMMARY STATISTICS")
32+
print("=" * 60)
33+
for model, model_r2_scores in zip(models, r2_scores):
34+
print(f"\n{model}:")
35+
print(f" Mean R²: {np.mean(model_r2_scores):.4f}")
36+
print(f" Std R²: {np.std(model_r2_scores):.4f}")
37+
print(f" Min R²: {np.min(model_r2_scores):.4f}")
38+
print(f" Max R²: {np.max(model_r2_scores):.4f}")
39+
40+
41+
def create_r2_plot(results: dict[str, EvaluationMetrics]) -> None:
42+
"""Create bar plot of R² scores by LLM model."""
43+
# Sort models by R² score in descending order
44+
models = sorted(results.keys(), key=lambda x: results[x].r2, reverse=True)
45+
r2_scores = [results[model].r2 for model in models]
46+
47+
# Clean up model names for display
48+
display_names = []
49+
for model in models:
50+
if "__" in model:
51+
provider, model_name = model.split("__", 1)
52+
display_names.append(f"{provider}\n{model_name}")
53+
else:
54+
display_names.append(model)
55+
56+
plt.style.use("grayscale")
57+
plt.figure(figsize=(12, 6))
58+
bars = plt.bar(range(len(models)), r2_scores)
59+
60+
plt.title("LLM Performance on Competitiveness Rating Task")
61+
plt.xlabel("LLM Model")
62+
plt.ylabel("R² Score")
63+
plt.xticks(range(len(models)), display_names, rotation=45, ha="right")
64+
plt.grid(True, alpha=0.3, axis="y")
65+
plt.ylim(-1.0, 1.0)
66+
67+
# Add value labels on bars
68+
for bar, score in zip(bars, r2_scores):
69+
y_pos = bar.get_height() + 0.01 if score >= 0 else bar.get_height() - 0.01
70+
va = "bottom" if score >= 0 else "top"
71+
plt.text(
72+
bar.get_x() + bar.get_width() / 2,
73+
y_pos,
74+
f"{score:.3f}",
75+
ha="center",
76+
va=va,
77+
)
78+
79+
plt.tight_layout()
80+
81+
# Save the plot
82+
plt.savefig(".data/plots/r2-scores-barplot.png", dpi=300, bbox_inches="tight")
83+
84+
85+
def create_spearman_plot(results: dict[str, EvaluationMetrics]) -> None:
86+
"""Create bar plot of Spearman correlations by LLM model."""
87+
# Sort models by Spearman correlation in descending order
88+
models = sorted(
89+
results.keys(), key=lambda x: results[x].spearman_corr, reverse=True
90+
)
91+
spearman_corrs = [results[model].spearman_corr for model in models]
92+
93+
# Clean up model names for display
94+
display_names = []
95+
for model in models:
96+
if "__" in model:
97+
provider, model_name = model.split("__", 1)
98+
display_names.append(f"{provider}\n{model_name}")
99+
else:
100+
display_names.append(model)
101+
102+
plt.style.use("grayscale")
103+
plt.figure(figsize=(12, 6))
104+
bars = plt.bar(range(len(models)), spearman_corrs)
105+
106+
plt.title("LLM Performance on Competitiveness Rating Task")
107+
plt.xlabel("LLM Model")
108+
plt.ylabel("Spearman Correlation")
109+
plt.xticks(range(len(models)), display_names, rotation=45, ha="right")
110+
plt.grid(True, alpha=0.3, axis="y")
111+
plt.ylim(0, 1.0)
112+
113+
# Add value labels on bars
114+
for bar, corr in zip(bars, spearman_corrs):
115+
plt.text(
116+
bar.get_x() + bar.get_width() / 2,
117+
bar.get_height() + 0.01,
118+
f"{corr:.3f}",
119+
ha="center",
120+
va="bottom",
121+
)
122+
123+
plt.tight_layout()
124+
125+
# Save the plot
126+
plt.savefig(
127+
".data/plots/spearman-correlations-barplot.png", dpi=300, bbox_inches="tight"
128+
)
129+
130+
# Print summary statistics
131+
print("\n" + "=" * 60)
132+
print("LLM SPEARMAN CORRELATION RESULTS")
133+
print("=" * 60)
134+
for model, metrics in results.items():
135+
print(f"\n{model}:")
136+
print(f" Spearman ρ: {metrics.spearman_corr:.4f} (p={metrics.spearman_p:.4f})")
137+
print(f" R²: {metrics.r2:.4f}")
138+
print(f" RMSE: {metrics.rmse:.4f}")

‎cli.py‎

Lines changed: 6 additions & 137 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@
2525
from sklearn.metrics import r2_score, mean_squared_error
2626
from scipy.stats import spearmanr
2727
import numpy as np
28-
import matplotlib.pyplot as plt
2928
from apistemic.benchmarks.datasets.companies import fetch_companies_df
3029
from apistemic.benchmarks.datasets.competitors import fetch_competitor_votes
3130
from apistemic.benchmarks.models import CompetitvenessRatingAnswer
@@ -35,6 +34,11 @@
3534
CompanyTupleTransformer,
3635
EmbeddingDiffTransformer,
3736
)
37+
from apistemic.benchmarks.plots import (
38+
create_box_plot,
39+
create_r2_plot,
40+
create_spearman_plot,
41+
)
3842

3943

4044
class ModelType(Enum):
@@ -77,141 +81,6 @@ def evaluate_similarity_predictions(y_true, y_pred):
7781
)
7882

7983

80-
def create_box_plot(all_results: dict[str, list[EvaluationMetrics]]) -> None:
81-
"""Create box plot of R² scores by model."""
82-
models = list(all_results.keys())
83-
r2_scores = []
84-
85-
for model in models:
86-
model_r2_scores = [metrics.r2 for metrics in all_results[model]]
87-
r2_scores.append(model_r2_scores)
88-
89-
plt.style.use("grayscale")
90-
plt.figure(figsize=(10, 6))
91-
plt.boxplot(r2_scores, tick_labels=models, patch_artist=False)
92-
93-
plt.title("How Well Embedding Models Understand Companies")
94-
plt.xlabel("Embedding Model (applied to company name)")
95-
plt.ylabel("R² Score (based on embedded company name only)")
96-
plt.grid(True, alpha=0.3)
97-
plt.xticks(rotation=45)
98-
plt.tight_layout()
99-
100-
# Save the plot
101-
plt.savefig(".data/plots/r2-scores-boxplot.png", dpi=300, bbox_inches="tight")
102-
103-
# Print summary statistics
104-
print("\n" + "=" * 60)
105-
print("SUMMARY STATISTICS")
106-
print("=" * 60)
107-
for model, model_r2_scores in zip(models, r2_scores):
108-
print(f"\n{model}:")
109-
print(f" Mean R²: {np.mean(model_r2_scores):.4f}")
110-
print(f" Std R²: {np.std(model_r2_scores):.4f}")
111-
print(f" Min R²: {np.min(model_r2_scores):.4f}")
112-
print(f" Max R²: {np.max(model_r2_scores):.4f}")
113-
114-
115-
def create_r2_plot(results: dict[str, EvaluationMetrics]) -> None:
116-
"""Create bar plot of R² scores by LLM model."""
117-
# Sort models by R² score in descending order
118-
models = sorted(results.keys(), key=lambda x: results[x].r2, reverse=True)
119-
r2_scores = [results[model].r2 for model in models]
120-
121-
# Clean up model names for display
122-
display_names = []
123-
for model in models:
124-
if "__" in model:
125-
provider, model_name = model.split("__", 1)
126-
display_names.append(f"{provider}\n{model_name}")
127-
else:
128-
display_names.append(model)
129-
130-
plt.style.use("grayscale")
131-
plt.figure(figsize=(12, 6))
132-
bars = plt.bar(range(len(models)), r2_scores)
133-
134-
plt.title("LLM Performance on Competitiveness Rating Task")
135-
plt.xlabel("LLM Model")
136-
plt.ylabel("R² Score")
137-
plt.xticks(range(len(models)), display_names, rotation=45, ha="right")
138-
plt.grid(True, alpha=0.3, axis="y")
139-
plt.ylim(-1.0, 1.0)
140-
141-
# Add value labels on bars
142-
for bar, score in zip(bars, r2_scores):
143-
y_pos = bar.get_height() + 0.01 if score >= 0 else bar.get_height() - 0.01
144-
va = "bottom" if score >= 0 else "top"
145-
plt.text(
146-
bar.get_x() + bar.get_width() / 2,
147-
y_pos,
148-
f"{score:.3f}",
149-
ha="center",
150-
va=va,
151-
)
152-
153-
plt.tight_layout()
154-
155-
# Save the plot
156-
plt.savefig(".data/plots/r2-scores-barplot.png", dpi=300, bbox_inches="tight")
157-
158-
159-
def create_spearman_plot(results: dict[str, EvaluationMetrics]) -> None:
160-
"""Create bar plot of Spearman correlations by LLM model."""
161-
# Sort models by Spearman correlation in descending order
162-
models = sorted(
163-
results.keys(), key=lambda x: results[x].spearman_corr, reverse=True
164-
)
165-
spearman_corrs = [results[model].spearman_corr for model in models]
166-
167-
# Clean up model names for display
168-
display_names = []
169-
for model in models:
170-
if "__" in model:
171-
provider, model_name = model.split("__", 1)
172-
display_names.append(f"{provider}\n{model_name}")
173-
else:
174-
display_names.append(model)
175-
176-
plt.style.use("grayscale")
177-
plt.figure(figsize=(12, 6))
178-
bars = plt.bar(range(len(models)), spearman_corrs)
179-
180-
plt.title("LLM Performance on Competitiveness Rating Task")
181-
plt.xlabel("LLM Model")
182-
plt.ylabel("Spearman Correlation")
183-
plt.xticks(range(len(models)), display_names, rotation=45, ha="right")
184-
plt.grid(True, alpha=0.3, axis="y")
185-
plt.ylim(0, 1.0)
186-
187-
# Add value labels on bars
188-
for bar, corr in zip(bars, spearman_corrs):
189-
plt.text(
190-
bar.get_x() + bar.get_width() / 2,
191-
bar.get_height() + 0.01,
192-
f"{corr:.3f}",
193-
ha="center",
194-
va="bottom",
195-
)
196-
197-
plt.tight_layout()
198-
199-
# Save the plot
200-
plt.savefig(
201-
".data/plots/spearman-correlations-barplot.png", dpi=300, bbox_inches="tight"
202-
)
203-
204-
# Print summary statistics
205-
print("\n" + "=" * 60)
206-
print("LLM SPEARMAN CORRELATION RESULTS")
207-
print("=" * 60)
208-
for model, metrics in results.items():
209-
print(f"\n{model}:")
210-
print(f" Spearman ρ: {metrics.spearman_corr:.4f} (p={metrics.spearman_p:.4f})")
211-
print(f" R²: {metrics.r2:.4f}")
212-
print(f" RMSE: {metrics.rmse:.4f}")
213-
214-
21584
def run_embedding_classification(run_count: int = 5):
21685
df = fetch_competitor_votes()
21786
df_agg = df.groupby(["from_organization_id", "to_organization_id"]).mean(
@@ -378,7 +247,7 @@ def run_scoring():
378247
store = FilesystemStore(".cache")
379248

380249
results = {}
381-
for llm_identifier, llm in sorted(llms.items(), key=lambda x: random.random()):
250+
for llm_identifier, llm in sorted(llms.items(), key=lambda _: random.random()):
382251
print("#" * 50)
383252
print(f"Running {llm}")
384253
print("#" * 50)

0 commit comments

Comments
 (0)