diff --git a/js/src/evaluation/runner.ts b/js/src/evaluation/runner.ts index b40b6912b..b10c28c05 100644 --- a/js/src/evaluation/runner.ts +++ b/js/src/evaluation/runner.ts @@ -367,7 +367,10 @@ class _ExperimentManager extends _ExperimentManagerMixin { get evaluationResults(): AsyncIterable { if (this._evaluationResults === undefined) { return async function* (this: _ExperimentManager) { - for await (const _ of this.examples) { + // In this case, we need evaluationResults to have the same + // number of items as examples. Make a copy of this.examples + // with `asyncTee` and yield empty results for each example. + for await (const _ of asyncTee(this.examples, 1)) { yield { results: [] }; } }.call(this); @@ -494,18 +497,23 @@ class _ExperimentManager extends _ExperimentManagerMixin { // const evaluationResultsIter = // this.evaluationResults[Symbol.asyncIterator](); - let runs = []; - let examples = []; - let evaluationResults = []; - for await (const run of this.runs) { - runs.push(run); - } + let runs: Run[] = []; + let examples: Example[] = []; + let evaluationResults: EvaluationResults[] = []; + for await (const example of this.examples) { examples.push(example); + evaluationResults.push( + (await (this.evaluationResults as AsyncGenerator).next()).value + ); + runs.push((await (this.runs as AsyncGenerator).next()).value); } - for await (const evaluationResult of this.evaluationResults) { - evaluationResults.push(evaluationResult); - } + // for await (const example of this.examples) { + // examples.push(example); + // } + // for await (const run of this.runs) { + // runs.push(run); + // } // return an array of objects with run, example, and evaluationResults for (let i = 0; i < runs.length; i++) { diff --git a/js/src/tests/evaluate.int.test.ts b/js/src/tests/evaluate.int.test.ts index c05ba3672..5be1c2ec8 100644 --- a/js/src/tests/evaluate.int.test.ts +++ b/js/src/tests/evaluate.int.test.ts @@ -10,18 +10,18 @@ test("evaluate can evaluate", async () => { }; const evalRes = await evaluate(evalFunc, { data: dummyDatasetName }); - + // console.log(evalRes.results) expect(evalRes.results).toHaveLength(2); expect(evalRes.results[0].run).toBeDefined(); expect(evalRes.results[0].example).toBeDefined(); - // expect(evalRes.results[0].evaluationResults).toBeDefined(); + expect(evalRes.results[0].evaluationResults).toBeDefined(); const firstRun = evalRes.results[0].run; expect(firstRun.outputs).toEqual({ foo: 3 }); expect(evalRes.results[1].run).toBeDefined(); expect(evalRes.results[1].example).toBeDefined(); - // expect(evalRes.results[1].evaluationResults).toBeDefined(); + expect(evalRes.results[1].evaluationResults).toBeDefined(); const secondRun = evalRes.results[1].run; expect(secondRun.outputs).toEqual({ foo: 2 }); });