feat: add octicon eval to mcp experiment

joshblack · joshblack · commit a111608c2d7e · 2026-05-18T13:35:08.000-05:00
diff --git a/evals/002-agent-uses-octicon-from-primer/tsconfig.json b/evals/002-agent-uses-octicon-from-primer/tsconfig.json
@@ -1,13 +1,25 @@
 {
-  "extends": "../../tsconfig.base.json",
   "compilerOptions": {
-    "jsx": "preserve",
+    "target": "ES2017",
+    "lib": ["dom", "dom.iterable", "esnext"],
+    "allowJs": true,
+    "skipLibCheck": true,
+    "strict": false,
+    "noEmit": true,
+    "incremental": true,
+    "module": "esnext",
+    "esModuleInterop": true,
+    "moduleResolution": "bundler",
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "jsx": "react-jsx",
     "plugins": [
       {
         "name": "next"
       }
     ]
   },
-  "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
+  "include": ["next-env.d.ts", ".next/types/**/*.ts", ".next/dev/types/**/*.ts", "**/*.mts", "**/*.ts", "**/*.tsx"],
   "exclude": ["node_modules"]
 }
+
diff --git a/packages/experiments/src/experiments/mcp.ts b/packages/experiments/src/experiments/mcp.ts
@@ -4,7 +4,7 @@ export const experiment: ExperimentConfig = {
   name: 'MCP',
   description: 'Compare MCP versus local instructions performance for Primer usage.',
   models: ['gpt-5.5', 'claude-opus-4.7', 'claude-sonnet-4.6'],
-  evals: ['001-agent-uses-button-from-primer'],
+  evals: ['001-agent-uses-button-from-primer', '002-agent-uses-octicon-from-primer'],
   treatments: [
     {
       name: 'MCP with local instructions',

Original file line number	Diff line number	Diff line change
`@@ -1,13 +1,25 @@`
`1`	`1`	`{`
`2`		`- "extends": "../../tsconfig.base.json",`
`3`	`2`	`"compilerOptions": {`
`4`		`- "jsx": "preserve",`
	`3`	`+ "target": "ES2017",`
	`4`	`+ "lib": ["dom", "dom.iterable", "esnext"],`
	`5`	`+ "allowJs": true,`
	`6`	`+ "skipLibCheck": true,`
	`7`	`+ "strict": false,`
	`8`	`+ "noEmit": true,`
	`9`	`+ "incremental": true,`
	`10`	`+ "module": "esnext",`
	`11`	`+ "esModuleInterop": true,`
	`12`	`+ "moduleResolution": "bundler",`
	`13`	`+ "resolveJsonModule": true,`
	`14`	`+ "isolatedModules": true,`
	`15`	`+ "jsx": "react-jsx",`
`5`	`16`	`"plugins": [`
`6`	`17`	`{`
`7`	`18`	`"name": "next"`
`8`	`19`	`}`
`9`	`20`	`]`
`10`	`21`	`},`
`11`		`- "include": ["next-env.d.ts", "*/.ts", "*/.tsx", ".next/types/*/.ts"],`
	`22`	`+ "include": ["next-env.d.ts", ".next/types/*/.ts", ".next/dev/types/*/.ts", "*/.mts", "*/.ts", "*/.tsx"],`
`12`	`23`	`"exclude": ["node_modules"]`
`13`	`24`	`}`
	`25`	`+`
Original file line number	Diff line number	Diff line change
`@@ -4,7 +4,7 @@ export const experiment: ExperimentConfig = {`
`4`	`4`	`name: 'MCP',`
`5`	`5`	`description: 'Compare MCP versus local instructions performance for Primer usage.',`
`6`	`6`	`models: ['gpt-5.5', 'claude-opus-4.7', 'claude-sonnet-4.6'],`
`7`		`- evals: ['001-agent-uses-button-from-primer'],`
	`7`	`+ evals: ['001-agent-uses-button-from-primer', '002-agent-uses-octicon-from-primer'],`
`8`	`8`	`treatments: [`
`9`	`9`	`{`
`10`	`10`	`name: 'MCP with local instructions',`