diff --git a/scripts/run-benchmark.js b/scripts/run-benchmark.js index 5b5bedd..c8afefe 100644 --- a/scripts/run-benchmark.js +++ b/scripts/run-benchmark.js @@ -2,21 +2,26 @@ import { promises as fs } from 'fs'; import path from 'path'; import { pathToFileURL } from 'url'; import axios from 'axios'; +import { performance } from 'perf_hooks'; const CWD = process.cwd(); -const README_PATH = path.join(CWD, 'README'); +const README_PATH = path.join(CWD, 'README.md'); const TESTS_DIR = path.join(CWD, 'tests'); +const RESULTS_PATH = path.join(CWD, 'results.json'); const getLlmCode = async (prompt, model, functionName) => { + const start = performance.now(); try { const res = await axios.post( 'https://openrouter.ai/api/v1/chat/completions', { model, messages: [{ role: 'user', content: prompt }] }, { headers: { Authorization: `Bearer ${process.env.OPENROUTER_KEY}` } } ); + const duration = (performance.now() - start) / 1000; const content = res.data.choices[0].message.content; const code = content.match(/```(?:javascript|js)?\n([\s\S]+?)\n```/)?.[1].trim() ?? content.trim(); - return `${code.replace(/^export\s+(default\s+)?/, '')}\nexport default ${functionName};`; + const finalCode = `${code.replace(/^export\s+(default\s+)?/, '')}\nexport default ${functionName};`; + return { code: finalCode, duration }; } catch (error) { console.error(`API Error for ${model}: ${error.message}`); return null; @@ -35,22 +40,27 @@ const main = async () => { ); const testsToRun = allTestDirs.slice(0, Math.ceil(allTestDirs.length * (percentage / 100))); + const genData = {}; for (const model of models) { + genData[model] = {}; for (const dir of testsToRun) { const testModule = await import(pathToFileURL(path.join(TESTS_DIR, dir, 'test.js'))); const { prompt, functionName } = testModule.default; console.log(`Generating ${dir} for ${model}...`); - const llmCode = await getLlmCode(prompt, model, functionName); - if (!llmCode) continue; + const result = await getLlmCode(prompt, model, functionName); + + genData[model][dir] = result?.duration ?? null; + if (!result) continue; const outDir = path.join(TESTS_DIR, dir, 'outputs'); await fs.mkdir(outDir, { recursive: true }); const fname = `${model.replace(/[\/:]/g, '_')}.js`; - await fs.writeFile(path.join(outDir, fname), llmCode); + await fs.writeFile(path.join(outDir, fname), result.code); } } - console.log('Code generation complete.'); + await fs.writeFile(RESULTS_PATH, JSON.stringify(genData, null, 2)); + console.log('Code generation and results.json update complete.'); }; main().catch(console.error);