Hanzo Dev
0b6f26d0bf
- Implement feature parity between Python and TypeScript MCP versions - Add comprehensive test coverage for all orthogonal tools - Consolidate tools to follow single-tool-multiple-actions pattern - Add bash, git-search, critic, and mode tools - Create multi-platform build system for VS Code, Claude Desktop, and DXT - Add test files for filesystem, search, shell, process, web-fetch, mode, and git-search tools - Fix TypeScript compilation issues and update build scripts - Update package.json with hanzo-ai publisher ID - Successfully build all distribution formats: - VS Code Extension (.vsix) - Claude Desktop MCP package - Claude Code DXT file - Standalone MCP server
499 lines
17 KiB
JavaScript
Executable File
499 lines
17 KiB
JavaScript
Executable File
#!/usr/bin/env node
|
|
|
|
/**
|
|
* Advanced benchmarks for Graph Database, Vector Store, and AST operations
|
|
*/
|
|
|
|
const { performance } = require('perf_hooks');
|
|
const path = require('path');
|
|
const fs = require('fs').promises;
|
|
const Module = require('module');
|
|
|
|
// Hook require for vscode mock
|
|
const originalRequire = Module.prototype.require;
|
|
Module.prototype.require = function(id) {
|
|
if (id === 'vscode') {
|
|
return require('../scripts/vscode-mock');
|
|
}
|
|
return originalRequire.apply(this, arguments);
|
|
};
|
|
|
|
// Load implementations
|
|
const { GraphDatabase } = require('../out/core/graph-db');
|
|
const { VectorStore } = require('../out/core/vector-store');
|
|
const { ASTIndex } = require('../out/core/ast-index');
|
|
const { DocumentStore } = require('../out/core/document-store');
|
|
|
|
class AdvancedBenchmark {
|
|
constructor() {
|
|
this.results = {};
|
|
}
|
|
|
|
async run() {
|
|
console.log('🚀 Advanced Benchmark Suite\n');
|
|
console.log('Testing Graph Database, Vector Store, AST Index, and Document Store\n');
|
|
|
|
await this.benchmarkGraphDatabase();
|
|
await this.benchmarkVectorStore();
|
|
await this.benchmarkASTIndex();
|
|
await this.benchmarkDocumentStore();
|
|
await this.benchmarkIntegration();
|
|
|
|
this.printResults();
|
|
}
|
|
|
|
async benchmark(name, fn, iterations = 100) {
|
|
console.log(`📊 ${name}`);
|
|
|
|
const times = [];
|
|
|
|
// Warm-up
|
|
for (let i = 0; i < 5; i++) {
|
|
try {
|
|
await fn();
|
|
} catch (e) {
|
|
// Ignore warm-up errors
|
|
}
|
|
}
|
|
|
|
// Actual benchmark
|
|
for (let i = 0; i < iterations; i++) {
|
|
const start = performance.now();
|
|
try {
|
|
await fn();
|
|
times.push(performance.now() - start);
|
|
} catch (e) {
|
|
// Count errors but continue
|
|
}
|
|
}
|
|
|
|
if (times.length === 0) {
|
|
console.log(' ❌ All iterations failed\n');
|
|
return;
|
|
}
|
|
|
|
const avg = times.reduce((a, b) => a + b, 0) / times.length;
|
|
const sorted = times.sort((a, b) => a - b);
|
|
const median = sorted[Math.floor(sorted.length / 2)];
|
|
const p95 = sorted[Math.floor(sorted.length * 0.95)] || sorted[sorted.length - 1];
|
|
|
|
this.results[name] = { avg, median, p95, iterations: times.length };
|
|
console.log(` ✅ Avg: ${avg.toFixed(2)}ms | Median: ${median.toFixed(2)}ms | P95: ${p95.toFixed(2)}ms\n`);
|
|
}
|
|
|
|
async benchmarkGraphDatabase() {
|
|
console.log('\n🔷 Graph Database Benchmarks\n');
|
|
|
|
const db = new GraphDatabase();
|
|
|
|
// Prepare test data
|
|
const nodes = [];
|
|
const edges = [];
|
|
|
|
for (let i = 0; i < 1000; i++) {
|
|
nodes.push({
|
|
id: `node_${i}`,
|
|
type: i % 3 === 0 ? 'class' : i % 3 === 1 ? 'function' : 'variable',
|
|
properties: {
|
|
name: `Item_${i}`,
|
|
value: i,
|
|
category: `cat_${i % 10}`
|
|
}
|
|
});
|
|
}
|
|
|
|
for (let i = 0; i < 500; i++) {
|
|
edges.push({
|
|
id: `edge_${i}`,
|
|
from: `node_${i}`,
|
|
to: `node_${(i + 1) % 1000}`,
|
|
type: i % 2 === 0 ? 'calls' : 'references'
|
|
});
|
|
}
|
|
|
|
// Add nodes benchmark
|
|
await this.benchmark('Graph: Add 1000 nodes', async () => {
|
|
const tempDb = new GraphDatabase();
|
|
for (const node of nodes) {
|
|
tempDb.addNode(node);
|
|
}
|
|
}, 10);
|
|
|
|
// Setup for other benchmarks
|
|
for (const node of nodes) {
|
|
db.addNode(node);
|
|
}
|
|
for (const edge of edges) {
|
|
db.addEdge(edge);
|
|
}
|
|
|
|
// Query benchmarks
|
|
await this.benchmark('Graph: Query by type', async () => {
|
|
db.queryNodes({ type: 'class' });
|
|
});
|
|
|
|
await this.benchmark('Graph: Query by properties', async () => {
|
|
db.queryNodes({ properties: { category: 'cat_5' } });
|
|
});
|
|
|
|
await this.benchmark('Graph: Find path', async () => {
|
|
db.findPath('node_0', 'node_50', 10);
|
|
});
|
|
|
|
await this.benchmark('Graph: Get node edges', async () => {
|
|
db.getNodeEdges('node_100', 'both');
|
|
});
|
|
|
|
await this.benchmark('Graph: Get connected components', async () => {
|
|
db.getConnectedComponents();
|
|
}, 10);
|
|
|
|
// Complex query
|
|
await this.benchmark('Graph: Complex query with connections', async () => {
|
|
db.queryNodes({
|
|
type: 'function',
|
|
connected: { type: 'calls', direction: 'out' }
|
|
});
|
|
});
|
|
}
|
|
|
|
async benchmarkVectorStore() {
|
|
console.log('\n🔷 Vector Store Benchmarks\n');
|
|
|
|
const store = new VectorStore();
|
|
|
|
// Prepare test documents
|
|
const documents = [];
|
|
const sampleTexts = [
|
|
'The quick brown fox jumps over the lazy dog',
|
|
'Machine learning is a subset of artificial intelligence',
|
|
'JavaScript is a programming language for web development',
|
|
'Data structures and algorithms are fundamental concepts',
|
|
'Cloud computing provides on-demand computing resources'
|
|
];
|
|
|
|
for (let i = 0; i < 1000; i++) {
|
|
documents.push({
|
|
content: sampleTexts[i % sampleTexts.length] + ` Document ${i}`,
|
|
metadata: {
|
|
type: i % 3 === 0 ? 'code' : i % 3 === 1 ? 'documentation' : 'comment',
|
|
language: i % 2 === 0 ? 'javascript' : 'python',
|
|
project: `project_${i % 5}`
|
|
}
|
|
});
|
|
}
|
|
|
|
// Add documents benchmark
|
|
await this.benchmark('Vector: Add 1000 documents', async () => {
|
|
const tempStore = new VectorStore();
|
|
for (const doc of documents) {
|
|
await tempStore.addDocument(doc.content, doc.metadata);
|
|
}
|
|
}, 5);
|
|
|
|
// Setup for search benchmarks
|
|
for (const doc of documents) {
|
|
await store.addDocument(doc.content, doc.metadata);
|
|
}
|
|
|
|
// Search benchmarks
|
|
await this.benchmark('Vector: Simple search', async () => {
|
|
await store.search('programming language');
|
|
});
|
|
|
|
await this.benchmark('Vector: Search with filters', async () => {
|
|
await store.search('machine learning', {
|
|
filter: { type: 'documentation' }
|
|
});
|
|
});
|
|
|
|
await this.benchmark('Vector: Search top 50', async () => {
|
|
await store.search('data', { topK: 50 });
|
|
});
|
|
|
|
// Get similar documents
|
|
const firstDoc = store.getDocument(documents[0].content);
|
|
if (firstDoc) {
|
|
await this.benchmark('Vector: Find similar documents', async () => {
|
|
await store.getSimilar(firstDoc.id, 20);
|
|
});
|
|
}
|
|
|
|
// Metadata search
|
|
await this.benchmark('Vector: Search by metadata', async () => {
|
|
store.searchByMetadata({ language: 'javascript', project: 'project_2' });
|
|
});
|
|
}
|
|
|
|
async benchmarkASTIndex() {
|
|
console.log('\n🔷 AST Index Benchmarks\n');
|
|
|
|
const ast = new ASTIndex();
|
|
|
|
// Create test TypeScript files
|
|
const testDir = path.join(__dirname, 'test-ast');
|
|
await fs.mkdir(testDir, { recursive: true });
|
|
|
|
// Generate test files
|
|
for (let i = 0; i < 20; i++) {
|
|
const content = `
|
|
export class TestClass${i} {
|
|
private value: number = ${i};
|
|
|
|
constructor() {
|
|
this.initialize();
|
|
}
|
|
|
|
private initialize(): void {
|
|
console.log('Initializing TestClass${i}');
|
|
}
|
|
|
|
public getValue(): number {
|
|
return this.value;
|
|
}
|
|
|
|
public process(data: string): string {
|
|
return data.toUpperCase();
|
|
}
|
|
}
|
|
|
|
export function helperFunction${i}(input: any): boolean {
|
|
return input !== null && input !== undefined;
|
|
}
|
|
|
|
export interface TestInterface${i} {
|
|
id: number;
|
|
name: string;
|
|
process(): void;
|
|
}
|
|
|
|
export type TestType${i} = string | number | boolean;
|
|
|
|
const testVariable${i} = new TestClass${i}();
|
|
testVariable${i}.process('test');
|
|
`;
|
|
await fs.writeFile(path.join(testDir, `test${i}.ts`), content);
|
|
}
|
|
|
|
// Index files benchmark
|
|
await this.benchmark('AST: Index 20 TypeScript files', async () => {
|
|
const tempAst = new ASTIndex();
|
|
for (let i = 0; i < 20; i++) {
|
|
await tempAst.indexFile(path.join(testDir, `test${i}.ts`));
|
|
}
|
|
}, 10);
|
|
|
|
// Setup for search benchmarks
|
|
for (let i = 0; i < 20; i++) {
|
|
await ast.indexFile(path.join(testDir, `test${i}.ts`));
|
|
}
|
|
|
|
// Search benchmarks
|
|
await this.benchmark('AST: Search symbols', async () => {
|
|
ast.searchSymbols('TestClass');
|
|
});
|
|
|
|
await this.benchmark('AST: Search exact match', async () => {
|
|
ast.searchSymbols('TestClass5', { exact: true });
|
|
});
|
|
|
|
await this.benchmark('AST: Find references', async () => {
|
|
ast.findReferences('helperFunction10');
|
|
});
|
|
|
|
await this.benchmark('AST: Get call hierarchy', async () => {
|
|
ast.getCallHierarchy('process');
|
|
});
|
|
|
|
await this.benchmark('AST: Get type hierarchy', async () => {
|
|
ast.getTypeHierarchy('TestInterface10');
|
|
});
|
|
|
|
// Cleanup
|
|
await fs.rm(testDir, { recursive: true, force: true });
|
|
}
|
|
|
|
async benchmarkDocumentStore() {
|
|
console.log('\n🔷 Document Store Benchmarks\n');
|
|
|
|
const storePath = path.join(__dirname, 'test-docs');
|
|
await fs.mkdir(storePath, { recursive: true });
|
|
const store = new DocumentStore(storePath);
|
|
await store.initialize();
|
|
|
|
// Prepare test data
|
|
const chatId = 'chat_benchmark';
|
|
const documents = [];
|
|
|
|
for (let i = 0; i < 100; i++) {
|
|
documents.push({
|
|
content: `This is document ${i} with some sample content for testing`,
|
|
type: i % 3 === 0 ? 'code' : i % 3 === 1 ? 'markdown' : 'text',
|
|
metadata: {
|
|
tags: [`tag${i % 5}`, `category${i % 3}`],
|
|
author: `user${i % 10}`
|
|
}
|
|
});
|
|
}
|
|
|
|
// Add documents benchmark
|
|
await this.benchmark('DocStore: Add 100 documents', async () => {
|
|
for (const doc of documents.slice(0, 10)) {
|
|
await store.addDocument(chatId, doc.content, doc.type, doc.metadata);
|
|
}
|
|
}, 10);
|
|
|
|
// Search benchmarks
|
|
await this.benchmark('DocStore: Search documents', async () => {
|
|
await store.searchDocuments('document sample content');
|
|
});
|
|
|
|
await this.benchmark('DocStore: Search with filters', async () => {
|
|
await store.searchDocuments('content', {
|
|
type: 'code',
|
|
tags: ['tag2']
|
|
});
|
|
});
|
|
|
|
// Session operations
|
|
const session = {
|
|
id: 'session_test',
|
|
title: 'Test Session',
|
|
created: new Date(),
|
|
updated: new Date(),
|
|
messages: Array(50).fill(null).map((_, i) => ({
|
|
id: `msg_${i}`,
|
|
role: i % 2 === 0 ? 'user' : 'assistant',
|
|
content: `Message ${i}`,
|
|
timestamp: new Date()
|
|
})),
|
|
documents: [],
|
|
tags: ['benchmark', 'test']
|
|
};
|
|
|
|
await this.benchmark('DocStore: Save session', async () => {
|
|
await store.saveSession(session);
|
|
});
|
|
|
|
// Cleanup
|
|
await fs.rm(storePath, { recursive: true, force: true });
|
|
}
|
|
|
|
async benchmarkIntegration() {
|
|
console.log('\n🔷 Integration Benchmarks\n');
|
|
|
|
// Combined operations that use multiple systems
|
|
const db = new GraphDatabase();
|
|
const vector = new VectorStore();
|
|
const ast = new ASTIndex();
|
|
|
|
// Simulate indexing a codebase with all systems
|
|
await this.benchmark('Integration: Index codebase (Graph + Vector + AST)', async () => {
|
|
// Add file nodes to graph
|
|
for (let i = 0; i < 10; i++) {
|
|
db.addNode({
|
|
id: `file_${i}`,
|
|
type: 'file',
|
|
properties: { name: `file${i}.ts`, size: 1000 + i * 100 }
|
|
});
|
|
|
|
// Add to vector store
|
|
await vector.addDocument(`File ${i} content with code`, {
|
|
filePath: `file${i}.ts`,
|
|
type: 'code'
|
|
});
|
|
}
|
|
|
|
// Add relationships
|
|
for (let i = 0; i < 5; i++) {
|
|
db.addEdge({
|
|
id: `import_${i}`,
|
|
from: `file_${i}`,
|
|
to: `file_${i + 1}`,
|
|
type: 'imports'
|
|
});
|
|
}
|
|
}, 10);
|
|
|
|
// Complex search across systems
|
|
await this.benchmark('Integration: Multi-system search', async () => {
|
|
// Search in vector store
|
|
const vectorResults = await vector.search('code content');
|
|
|
|
// For each result, find in graph
|
|
for (const result of vectorResults.slice(0, 5)) {
|
|
const fileId = result.document.metadata.filePath?.replace('.ts', '');
|
|
if (fileId) {
|
|
db.getNodeEdges(`file_${fileId}`, 'both');
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
printResults() {
|
|
console.log('\n' + '='.repeat(80));
|
|
console.log('\n📈 Benchmark Results Summary\n');
|
|
console.log('='.repeat(80));
|
|
|
|
// Group results
|
|
const categories = {
|
|
'Graph Database': Object.keys(this.results).filter(k => k.includes('Graph:')),
|
|
'Vector Store': Object.keys(this.results).filter(k => k.includes('Vector:')),
|
|
'AST Index': Object.keys(this.results).filter(k => k.includes('AST:')),
|
|
'Document Store': Object.keys(this.results).filter(k => k.includes('DocStore:')),
|
|
'Integration': Object.keys(this.results).filter(k => k.includes('Integration:'))
|
|
};
|
|
|
|
for (const [category, tests] of Object.entries(categories)) {
|
|
if (tests.length === 0) continue;
|
|
|
|
console.log(`\n${category}:`);
|
|
console.log('-'.repeat(70));
|
|
console.log('Operation | Avg (ms) | Median | P95 |');
|
|
console.log('-'.repeat(70));
|
|
|
|
for (const test of tests) {
|
|
const result = this.results[test];
|
|
const name = test.replace(/^[^:]+:\s*/, '').padEnd(36);
|
|
console.log(`${name} | ${result.avg.toFixed(2).padStart(8)} | ${result.median.toFixed(2).padStart(6)} | ${result.p95.toFixed(2).padStart(6)} |`);
|
|
}
|
|
}
|
|
|
|
// Performance insights
|
|
console.log('\n📊 Performance Insights:\n');
|
|
|
|
// Graph DB insights
|
|
const graphAddNodes = this.results['Graph: Add 1000 nodes'];
|
|
if (graphAddNodes) {
|
|
const nodesPerMs = 1000 / graphAddNodes.avg;
|
|
console.log(`• Graph DB: Can add ${nodesPerMs.toFixed(0)} nodes/ms`);
|
|
}
|
|
|
|
// Vector store insights
|
|
const vectorAdd = this.results['Vector: Add 1000 documents'];
|
|
if (vectorAdd) {
|
|
const docsPerMs = 1000 / vectorAdd.avg;
|
|
console.log(`• Vector Store: Can index ${docsPerMs.toFixed(0)} documents/ms`);
|
|
}
|
|
|
|
const vectorSearch = this.results['Vector: Simple search'];
|
|
if (vectorSearch) {
|
|
console.log(`• Vector Search: ${vectorSearch.avg < 50 ? '🟢 Fast' : vectorSearch.avg < 200 ? '🟡 Moderate' : '🔴 Slow'} (${vectorSearch.avg.toFixed(2)}ms avg)`);
|
|
}
|
|
|
|
// AST insights
|
|
const astIndex = this.results['AST: Index 20 TypeScript files'];
|
|
if (astIndex) {
|
|
const filesPerSecond = (20 / astIndex.avg) * 1000;
|
|
console.log(`• AST Indexing: ~${filesPerSecond.toFixed(0)} files/second`);
|
|
}
|
|
|
|
console.log('\n✨ Benchmark complete!');
|
|
}
|
|
}
|
|
|
|
// Run benchmark
|
|
if (require.main === module) {
|
|
const benchmark = new AdvancedBenchmark();
|
|
benchmark.run().catch(console.error);
|
|
} |