refactor: normalize directory options

Author: joshblack

packages/agent-eval/README.md

@@ -81,8 +81,9 @@ APIs:
 ```ts
 import {findScenario, listScenarios, loadExperimentConfigs, run, type Model} from '@primer/agent-eval'
 
-const scenarios = await listScenarios({scenariosDirectory: './scenarios'})
+const experiments = await loadExperimentConfigs({directory: './experiments'})
+const scenarios = await listScenarios({directory: './scenarios'})
 const scenario = await findScenario('001-agent-uses-button-from-primer', {
-  scenariosDirectory: './scenarios',
+  directory: './scenarios',
 })
 ```

packages/agent-eval/src/cli.ts

@@ -66,14 +66,14 @@ if (!existsSync(ARTIFACTS_DIR)) {
 if (values.experiment) {
   experimentConfigs = await loadExperimentConfigs({
     experiment: values.experiment,
-    experimentsDirectory: values.experiments,
+    directory: values.experiments,
   })
   if (experimentConfigs.length === 0) {
     console.log('Experiments:')
     console.log(
       (
         await listExperiments({
-          experimentsDirectory: values.experiments,
+          directory: values.experiments,
         })
       )
         .map(([name]) => name)
@@ -82,7 +82,7 @@ if (values.experiment) {
   }
 } else {
   experimentConfigs = await loadExperimentConfigs({
-    experimentsDirectory: values.experiments,
+    directory: values.experiments,
   })
 }
 
@@ -392,7 +392,7 @@ for (const config of experimentConfigs) {
   const scenarios = await Promise.all(
     config.scenarios.map(scenarioConfig => {
       return resolveExperimentScenario(scenarioConfig, {
-        scenariosDirectory: values.scenarios,
+        directory: values.scenarios,
       })
     }),
   )

packages/agent-eval/src/experiments.test.ts

@@ -34,7 +34,7 @@ describe('local experiment loading', () => {
   test('lists experiments from a local directory', async () => {
     const directory = await createExperimentsDirectory()
 
-    await expect(listExperiments({experimentsDirectory: directory})).resolves.toEqual([
+    await expect(listExperiments({directory})).resolves.toEqual([
       ['default-export', expect.objectContaining({name: 'Default export'})],
       ['example', expect.objectContaining({name: 'Example'})],
     ])
@@ -43,9 +43,7 @@ describe('local experiment loading', () => {
   test('finds a named experiment from a local directory', async () => {
     const directory = await createExperimentsDirectory()
 
-    await expect(findExperiment('example', {experimentsDirectory: directory})).resolves.toEqual(
-      expect.objectContaining({name: 'Example'}),
-    )
+    await expect(findExperiment('example', {directory})).resolves.toEqual(expect.objectContaining({name: 'Example'}))
   })
 
   test('finds an experiment from a local file path', async () => {
@@ -67,6 +65,6 @@ describe('local experiment loading', () => {
   test('loads all experiments when no experiment is specified', async () => {
     const directory = await createExperimentsDirectory()
 
-    await expect(loadExperimentConfigs({experimentsDirectory: directory})).resolves.toHaveLength(2)
+    await expect(loadExperimentConfigs({directory})).resolves.toHaveLength(2)
   })
 })

packages/agent-eval/src/experiments.ts

@@ -11,7 +11,7 @@ type ExperimentModule = {
 }
 
 type ExperimentSourceOptions = {
-  experimentsDirectory?: string
+  directory?: string
 }
 
 type LoadExperimentOptions = ExperimentSourceOptions & {
@@ -64,8 +64,8 @@ async function getLocalExperimentEntries(experimentsDirectory: string): Promise<
 }
 
 async function listExperiments(options: ExperimentSourceOptions = {}): Promise<Array<[string, ExperimentConfig]>> {
-  if (options.experimentsDirectory) {
-    return getLocalExperimentEntries(options.experimentsDirectory)
+  if (options.directory) {
+    return getLocalExperimentEntries(options.directory)
   }
 
   return listPackagedExperiments()
@@ -75,8 +75,8 @@ async function findExperiment(
   id: string,
   options: ExperimentSourceOptions = {},
 ): Promise<ExperimentConfig | undefined> {
-  if (options.experimentsDirectory) {
-    const experiments = await getLocalExperimentEntries(options.experimentsDirectory)
+  if (options.directory) {
+    const experiments = await getLocalExperimentEntries(options.directory)
     return experiments.find(([name]) => name === id)?.[1]
   }
 

packages/agent-eval/src/scenario.test.ts

@@ -30,7 +30,7 @@ describe(resolveExperimentScenario, () => {
 
     await expect(
       resolveExperimentScenario('button-scenario', {
-        scenariosDirectory,
+        directory: scenariosDirectory,
       }),
     ).resolves.toEqual({
       id: 'button-scenario',

packages/agent-eval/src/scenario.ts

@@ -12,11 +12,11 @@ async function resolveExperimentScenario(
 ): Promise<ResolvedScenario> {
   if (typeof scenarioConfig === 'string') {
     const scenario = await findScenario(scenarioConfig, {
-      scenariosDirectory: options.scenariosDirectory,
+      directory: options.directory,
     })
     if (!scenario) {
       throw new Error(
-        `Scenario "${scenarioConfig}" was not found in: ${path.resolve(options.scenariosDirectory ?? 'scenarios')}`,
+        `Scenario "${scenarioConfig}" was not found in: ${path.resolve(options.directory ?? 'scenarios')}`,
       )
     }
     return scenario

packages/agent-eval/src/scenarios.test.ts

@@ -34,7 +34,7 @@ describe('scenario loading', () => {
     await createScenario(scenariosDirectory, 'second', 'Second prompt')
     await createScenario(scenariosDirectory, 'first', 'First prompt')
 
-    await expect(listScenarios({scenariosDirectory})).resolves.toEqual([
+    await expect(listScenarios({directory: scenariosDirectory})).resolves.toEqual([
       expect.objectContaining({id: 'first', config: {prompt: 'First prompt'}}),
       expect.objectContaining({id: 'second', config: {prompt: 'Second prompt'}}),
     ])
@@ -44,7 +44,7 @@ describe('scenario loading', () => {
     const scenariosDirectory = await createScenariosDirectory()
     const directory = await createScenario(scenariosDirectory, 'example', 'Example prompt')
 
-    await expect(findScenario('example', {scenariosDirectory})).resolves.toEqual({
+    await expect(findScenario('example', {directory: scenariosDirectory})).resolves.toEqual({
       id: 'example',
       directory,
       config: {prompt: 'Example prompt'},
@@ -55,6 +55,6 @@ describe('scenario loading', () => {
   test('returns undefined when a scenario is not found', async () => {
     const scenariosDirectory = await createScenariosDirectory()
 
-    await expect(findScenario('missing-scenario', {scenariosDirectory})).resolves.toBeUndefined()
+    await expect(findScenario('missing-scenario', {directory: scenariosDirectory})).resolves.toBeUndefined()
   })
 })

packages/agent-eval/src/scenarios.ts

@@ -11,11 +11,11 @@ type ResolvedScenario = {
 }
 
 type ScenarioSourceOptions = {
-  scenariosDirectory?: string
+  directory?: string
 }
 
 function resolveScenariosDirectory(options: ScenarioSourceOptions): string {
-  return path.resolve(options.scenariosDirectory ?? 'scenarios')
+  return path.resolve(options.directory ?? 'scenarios')
 }
 
 async function assertScenariosDirectory(directory: string) {