Web Scraping

import { Agent, Cogitator } from '@cogitator-ai/core';
import { BrowserSession, browserTools } from '@cogitator-ai/browser';

const apiKey = process.env.GOOGLE_API_KEY;
if (!apiKey) throw new Error('Set GOOGLE_API_KEY');

const session = new BrowserSession({ headless: true });
await session.start();

const scraper = new Agent({
  name: 'web-scraper',
  model: 'google/gemini-3.5-flash-lite',
  instructions: `You are a web scraping agent. Navigate to the URL first,
then use the extraction tools to collect structured data.`,
  tools: browserTools(session, { modules: ['navigation', 'extraction'] }),
  temperature: 0.2,
  maxIterations: 10,
});

const cog = new Cogitator({ llm: { providers: { google: { apiKey } } } });

try {
  const result = await cog.run(scraper, {
    input: 'Go to https://news.ycombinator.com and extract the top 5 stories with titles, URLs and scores.',
  });
  console.log(result.output);
  console.log('Tools:', result.toolCalls.map((call) => call.name).join(', '));
} finally {
  await session.close();
  await cog.close();
}

pnpm add @cogitator-ai/core @cogitator-ai/browser playwright
npx playwright install chromium

GOOGLE_API_KEY=your-key npx tsx web-scraping.ts

npx tsx examples/browser/01-web-scraping.ts