Chunking Strategies

import { GoogleEmbeddingService } from '@cogitator-ai/memory';
import { FixedSizeChunker, RecursiveChunker, SemanticChunker } from '@cogitator-ai/rag';

const apiKey = process.env.GOOGLE_API_KEY;
if (!apiKey) throw new Error('Set GOOGLE_API_KEY');

const text = `Early AI systems relied on hand-crafted rules and symbolic reasoning.

The shift to machine learning in the 1990s changed everything. Researchers trained algorithms on data instead of programming rules.

The transformer architecture, introduced in 2017, revolutionized natural language processing with self-attention.

Retrieval-augmented generation retrieves relevant documents at inference time, which reduces hallucinations.`;

function describe(name: string, chunks: { content: string }[]) {
  const sizes = chunks.map((chunk) => chunk.content.length);
  console.log(`${name}: ${chunks.length} chunks, ${Math.min(...sizes)}–${Math.max(...sizes)} chars`);
}

describe('fixed', new FixedSizeChunker({ chunkSize: 200, chunkOverlap: 30 }).chunk(text, 'doc-1'));
describe('recursive', new RecursiveChunker({ chunkSize: 200, chunkOverlap: 30 }).chunk(text, 'doc-1'));

const semantic = new SemanticChunker({
  embeddingService: new GoogleEmbeddingService({ apiKey }),
  breakpointThreshold: 0.5,
  minChunkSize: 100,
  maxChunkSize: 600,
});
describe('semantic', await semantic.chunk(text, 'doc-1'));

pnpm add @cogitator-ai/memory @cogitator-ai/rag

GOOGLE_API_KEY=your-key npx tsx chunking.ts

npx tsx examples/rag/02-chunking-strategies.ts