A/B Comparison

import { Agent, Cogitator } from '@cogitator-ai/core';
import { Dataset, EvalComparison, contains, exactMatch } from '@cogitator-ai/evals';

const apiKey = process.env.GOOGLE_API_KEY;
if (!apiKey) throw new Error('Set GOOGLE_API_KEY');

const dataset = Dataset.from([
  { input: 'What is 7 * 8?', expected: '56' },
  { input: 'What is 15% of 200?', expected: '30' },
  { input: 'What is 1000 - 373?', expected: '627' },
  { input: 'What is 3^4?', expected: '81' },
]);

const cog = new Cogitator({ llm: { providers: { google: { apiKey } } } });

const precise = new Agent({
  name: 'precise',
  model: 'google/gemini-3.5-flash-lite',
  instructions: 'You are a calculator. Reply with ONLY the numeric answer.',
  temperature: 0,
  maxIterations: 1,
});

const verbose = new Agent({
  name: 'verbose',
  model: 'google/gemini-3.5-flash-lite',
  instructions: 'Explain your reasoning step by step, then give the answer.',
  temperature: 0.7,
  maxIterations: 1,
});

const comparison = new EvalComparison({
  dataset,
  targets: {
    baseline: { agent: precise, cogitator: cog },
    challenger: { agent: verbose, cogitator: cog },
  },
  metrics: [exactMatch(), contains()],
  concurrency: 1,
});

const result = await comparison.run();
for (const [metric, scores] of Object.entries(result.summary.metrics)) {
  console.log(
    `${metric}: baseline ${scores.baseline.toFixed(2)} vs challenger ${scores.challenger.toFixed(2)} ` +
      `(p=${scores.pValue.toFixed(3)}, winner: ${scores.winner})`
  );
}
console.log('Overall winner:', result.summary.winner);

await cog.close();

pnpm add @cogitator-ai/core @cogitator-ai/evals

GOOGLE_API_KEY=your-key npx tsx ab-comparison.ts

npx tsx examples/evals/03-ab-comparison.ts