Skip to content

Commit a1a0500

Browse files
margostinomargostino
committed
Add metrics JSON, switch simulation LLMs to OSS/deepseek, enable env‑based OpenAI API key/base, and adjust AI processor and dev script accordingly.
1 parent 55f2014 commit a1a0500

6 files changed

Lines changed: 240 additions & 72 deletions

File tree

‎examples/metrics/gift_choice_2.json‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
[
2+
{
3+
"name": "initial_preference_alignment",
4+
"description": "Measures the initial alignment of Ana and John's gift preferences.",
5+
"type": "GAUGE",
6+
"unit": "percentage",
7+
"tags": [
8+
{
9+
"tag": "agent_name",
10+
"description": "The name of the agent (Ana or John)."
11+
}
12+
]
13+
},
14+
{
15+
"name": "final_agreement_score",
16+
"description": "Evaluates how much the final gift decision aligns with each agent's preferences.",
17+
"type": "GAUGE",
18+
"unit": "percentage",
19+
"tags": [
20+
{
21+
"tag": "agent_name",
22+
"description": "The name of the agent (Ana or John)."
23+
}
24+
]
25+
},
26+
{
27+
"name": "decision_iterations",
28+
"description": "Tracks the number of iterations or discussions needed to reach a decision.",
29+
"type": "COUNTER",
30+
"unit": "iterations",
31+
"tags": []
32+
},
33+
{
34+
"name": "influence_score",
35+
"description": "Measures the influence each agent had on the final gift decision.",
36+
"type": "GAUGE",
37+
"unit": "percentage",
38+
"tags": [
39+
{
40+
"tag": "agent_name",
41+
"description": "The name of the agent (Ana or John)."
42+
}
43+
]
44+
},
45+
{
46+
"name": "compromise_index",
47+
"description": "Measures how much each agent had to compromise from their initial preferences.",
48+
"type": "GAUGE",
49+
"unit": "percentage",
50+
"tags": [
51+
{
52+
"tag": "agent_name",
53+
"description": "The name of the agent (Ana or John)."
54+
}
55+
]
56+
},
57+
{
58+
"name": "satisfaction_score",
59+
"description": "Measures each agent's satisfaction with the final gift decision.",
60+
"type": "GAUGE",
61+
"unit": "score",
62+
"tags": [
63+
{
64+
"tag": "agent_name",
65+
"description": "The name of the agent (Ana or John)."
66+
}
67+
]
68+
},
69+
{
70+
"name": "flexibility_score",
71+
"description": "Measures the willingness of each agent to adapt their preferences during the decision process.",
72+
"type": "GAUGE",
73+
"unit": "score",
74+
"tags": [
75+
{
76+
"tag": "agent_name",
77+
"description": "The name of the agent (Ana or John)."
78+
}
79+
]
80+
},
81+
{
82+
"name": "consensus_level",
83+
"description": "Indicates the level of agreement between Ana and John on the final gift decision.",
84+
"type": "GAUGE",
85+
"unit": "percentage",
86+
"tags": []
87+
}
88+
]

‎examples/simulations/gift_choice.json‎

Lines changed: 68 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -1,75 +1,75 @@
11
{
2-
"name": "Gift choice",
3-
"max_steps": 150,
4-
"timeout_seconds": 120,
5-
"shutdown_grace_period_seconds": 5,
6-
"description": "This simulation is about two friends that need to decide together a gift for a special occasion.",
7-
"task": "The task is VERY simple and should be completed in seconds: Ana and John need to pick a gift for the 30th birthday of another friend called Maria who likes books and reading, mainly fantasy books. She has preferences for Tokien and one book she has never read is 'The Hobbit'. Maria will be happy with any gift, but she will be happier with whatever gift. Ana and John should pick between 'The Hobbit: Illustrated Deluxe Edition, HarperCollins, Alan Lee' and 'The Hobbit, or There and Back Again, J.R.R. Tolkien'. Budget, delivery time, availability and NOTHING else matters, they ONLY need to pick one of the two options. There must not be other discussion, they should pick and that's it.",
8-
"evaluator": {
9-
"name": "EVALUATOR",
10-
"mailbox": {
11-
"max_size": 400
12-
},
13-
"llm": {
14-
"model": "gpt-5-nano"
15-
}
2+
"name": "Gift choice",
3+
"max_steps": 150,
4+
"timeout_seconds": 120,
5+
"shutdown_grace_period_seconds": 5,
6+
"description": "This simulation is about two friends that need to decide together a gift for a special occasion.",
7+
"task": "The task is VERY simple and should be completed in seconds: Ana and John need to pick a gift for the 30th birthday of another friend called Maria who likes books and reading, mainly fantasy books. She has preferences for Tokien and one book she has never read is 'The Hobbit'. Maria will be happy with any gift, but she will be happier with whatever gift. Ana and John should pick between 'The Hobbit: Illustrated Deluxe Edition, HarperCollins, Alan Lee' and 'The Hobbit, or There and Back Again, J.R.R. Tolkien'. Budget, delivery time, availability and NOTHING else matters, they ONLY need to pick one of the two options. There must not be other discussion, they should pick and that's it.",
8+
"evaluator": {
9+
"name": "EVALUATOR",
10+
"mailbox": {
11+
"max_size": 400
1612
},
17-
"reporter": {
18-
"name": "REPORTER",
19-
"mailbox": {
20-
"max_size": 400
21-
},
22-
"llm": {
23-
"model": "gpt-5-nano"
24-
}
13+
"llm": {
14+
"model": "gpt-oss:20b"
15+
}
16+
},
17+
"reporter": {
18+
"name": "REPORTER",
19+
"mailbox": {
20+
"max_size": 400
2521
},
26-
"planner": {
27-
"name": "PLANNER",
28-
"mailbox": {
29-
"max_size": 400
30-
},
31-
"llm": {
32-
"model": "gpt-5-nano"
33-
}
22+
"llm": {
23+
"model": "gpt-oss:20b"
24+
}
25+
},
26+
"planner": {
27+
"name": "PLANNER",
28+
"mailbox": {
29+
"max_size": 400
3430
},
35-
"orchestrator": {
36-
"name": "ORCHESTRATOR",
37-
"mailbox": {
38-
"max_size": 400
39-
},
40-
"llm": {
41-
"model": "gpt-5-nano"
42-
}
31+
"llm": {
32+
"model": "gpt-oss:20b"
33+
}
34+
},
35+
"orchestrator": {
36+
"name": "ORCHESTRATOR",
37+
"mailbox": {
38+
"max_size": 400
39+
},
40+
"llm": {
41+
"model": "gpt-oss:20b"
42+
}
43+
},
44+
"workers": [
45+
{
46+
"name": "ANA",
47+
"description": "this is ana agent",
48+
"role": "Ana is Maria's friend. Pick quick no much thinking. She will buy.",
49+
"backstory": "She is passionate about reading and books.",
50+
"llm": {
51+
"model": "gpt-oss:20b"
52+
},
53+
"mailbox": {
54+
"max_size": 100
55+
}
4356
},
44-
"workers": [
45-
{
46-
"name": "ANA",
47-
"description": "this is ana agent",
48-
"role": "Ana is Maria's friend. Pick quick no much thinking. She will buy.",
49-
"backstory": "She is passionate about reading and books.",
50-
"llm": {
51-
"model": "gpt-5-nano"
52-
},
53-
"mailbox": {
54-
"max_size": 100
55-
}
56-
},
57-
{
58-
"name": "JOHN",
59-
"description": "this is john agent",
60-
"role": "John is Maria's friend. Pick quick no much thinking.",
61-
"backstory": "He is passionate about books of any kind.",
62-
"llm": {
63-
"model": "gpt-5-nano"
64-
},
65-
"mailbox": {
66-
"max_size": 100
67-
}
68-
}
69-
],
70-
"logging": {
71-
"verbose": false,
72-
"log_path": "logs",
73-
"log_file": "gift_choice.log"
57+
{
58+
"name": "JOHN",
59+
"description": "this is john agent",
60+
"role": "John is Maria's friend. Pick quick no much thinking.",
61+
"backstory": "He is passionate about books of any kind.",
62+
"llm": {
63+
"model": "gpt-oss:20b"
64+
},
65+
"mailbox": {
66+
"max_size": 100
67+
}
7468
}
69+
],
70+
"logging": {
71+
"verbose": false,
72+
"log_path": "logs",
73+
"log_file": "gift_choice.log"
74+
}
7575
}

‎examples/simulations/gift_choice_2.json‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
{
2+
"name": "Gift choice",
3+
"max_steps": 150,
4+
"timeout_seconds": 120,
5+
"shutdown_grace_period_seconds": 5,
6+
"description": "This simulation is about two friends that need to decide together a gift for a special occasion.",
7+
"task": "The task is VERY simple and should be completed in seconds: Ana and John need to pick a gift for the 30th birthday of another friend called Maria who likes books and reading, mainly fantasy books. She has preferences for Tokien and one book she has never read is 'The Hobbit'. Maria will be happy with any gift, but she will be happier with whatever gift. Ana and John should pick between 'The Hobbit: Illustrated Deluxe Edition, HarperCollins, Alan Lee' and 'The Hobbit, or There and Back Again, J.R.R. Tolkien'. Budget, delivery time, availability and NOTHING else matters, they ONLY need to pick one of the two options. There must not be other discussion, they should pick and that's it.",
8+
"evaluator": {
9+
"name": "EVALUATOR",
10+
"mailbox": {
11+
"max_size": 400
12+
},
13+
"llm": {
14+
"model": "deepseek-r1:7b"
15+
}
16+
},
17+
"reporter": {
18+
"name": "REPORTER",
19+
"mailbox": {
20+
"max_size": 400
21+
},
22+
"llm": {
23+
"model": "deepseek-r1:7b"
24+
}
25+
},
26+
"planner": {
27+
"name": "PLANNER",
28+
"mailbox": {
29+
"max_size": 400
30+
},
31+
"llm": {
32+
"model": "deepseek-r1:7b"
33+
}
34+
},
35+
"orchestrator": {
36+
"name": "ORCHESTRATOR",
37+
"mailbox": {
38+
"max_size": 400
39+
},
40+
"llm": {
41+
"model": "deepseek-r1:7b"
42+
}
43+
},
44+
"workers": [
45+
{
46+
"name": "ANA",
47+
"description": "this is ana agent",
48+
"role": "Ana is Maria's friend. Pick quick no much thinking. She will buy.",
49+
"backstory": "She is passionate about reading and books.",
50+
"llm": {
51+
"model": "deepseek-r1:7b"
52+
},
53+
"mailbox": {
54+
"max_size": 100
55+
}
56+
},
57+
{
58+
"name": "JOHN",
59+
"description": "this is john agent",
60+
"role": "John is Maria's friend. Pick quick no much thinking.",
61+
"backstory": "He is passionate about books of any kind.",
62+
"llm": {
63+
"model": "deepseek-r1:7b"
64+
},
65+
"mailbox": {
66+
"max_size": 100
67+
}
68+
}
69+
],
70+
"logging": {
71+
"verbose": false,
72+
"log_path": "logs",
73+
"log_file": "gift_choice.log"
74+
}
75+
}

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
"description": "TypeScript implementation of Autobox simulation runtime",
55
"main": "dist/index.js",
66
"scripts": {
7-
"dev": "tsx watch --clear-screen=true src/index.ts --config=./examples",
7+
"dev": "tsx watch --clear-screen=true src/index.ts --config=./examples --simulation-name=gift_choice",
88
"build": "tsc",
99
"start": "node dist/index.js",
1010
"start:cli": "node dist/index.js --cli",

‎src/core/llm/createAiProcessor.ts‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import {
55
ChatCompletionMessageParam,
66
} from 'openai/resources';
77
import { z } from 'zod';
8-
import { DEFAULT_OPEN_AI_MODEL, logger } from '../../config';
8+
import { DEFAULT_OPEN_AI_MODEL, env, logger } from '../../config';
99

1010
export const createAiProcessor = ({
1111
systemPrompt,
@@ -16,7 +16,10 @@ export const createAiProcessor = ({
1616
model?: string;
1717
schema?: z.ZodType<any>;
1818
}) => {
19-
const openai: OpenAI = new OpenAI();
19+
const openai: OpenAI = new OpenAI({
20+
apiKey: env.OPENAI_API_KEY,
21+
baseURL: env.OPENAI_BASE_URL,
22+
});
2023

2124
const think = async ({
2225
name,
@@ -25,7 +28,7 @@ export const createAiProcessor = ({
2528
name: string;
2629
messages: ChatCompletionMessageParam[];
2730
}): Promise<unknown> => {
28-
logger.info(`[${name}] thinking...`);
31+
logger.info(`[${name}] thinking (${model})...`);
2932

3033
const completionMessages = [
3134
{ role: 'system', content: systemPrompt } as ChatCompletionMessageParam,

‎src/schemas/internal/environment.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ export const EnvironmentSchema = z.object({
1919
JWT_EXPIRES_IN: z.string().min(1),
2020
CONFIG_PATH: z.string().default('/autobox/config'),
2121
LOG_FORMAT: z.enum(['pretty', 'json']).default('pretty'),
22+
OPENAI_API_KEY: z.string().min(1),
23+
OPENAI_BASE_URL: z.string().url().default('https://api.openai.com/v1'),
2224
});
2325

2426
export type Environment = z.infer<typeof EnvironmentSchema>;

0 commit comments

Comments
 (0)