multipleof4/lynchmark
1
0
Fork 0
mirror of https://github.com/multipleof4/lynchmark.git synced 2026-01-14 08:37:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactor: Record generation time and output results.json

Browse Source
This commit is contained in:
multipleof4
2025-10-13 10:33:48 -07:00
parent ac363ef174
commit 27db7da944
1 changed files with 16 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
scripts/run-benchmark.js
View File

@@ -2,21 +2,26 @@ import { promises as fs } from 'fs';
import path from 'path'; import path from 'path';
import { pathToFileURL } from 'url'; import { pathToFileURL } from 'url';
import axios from 'axios'; import axios from 'axios';
import { performance } from 'perf_hooks';
const CWD = process.cwd(); const CWD = process.cwd();
const README_PATH = path.join(CWD, 'README'); const README_PATH = path.join(CWD, 'README.md');
const TESTS_DIR = path.join(CWD, 'tests'); const TESTS_DIR = path.join(CWD, 'tests');
const RESULTS_PATH = path.join(CWD, 'results.json');
const getLlmCode = async (prompt, model, functionName) => { const getLlmCode = async (prompt, model, functionName) => {
const start = performance.now();
try { try {
const res = await axios.post( const res = await axios.post(
'https://openrouter.ai/api/v1/chat/completions', 'https://openrouter.ai/api/v1/chat/completions',
{ model, messages: [{ role: 'user', content: prompt }] }, { model, messages: [{ role: 'user', content: prompt }] },
{ headers: { Authorization: `Bearer ${process.env.OPENROUTER_KEY}` } } { headers: { Authorization: `Bearer ${process.env.OPENROUTER_KEY}` } }
); );
const duration = (performance.now() - start) / 1000;
const content = res.data.choices[0].message.content; const content = res.data.choices[0].message.content;
const code = content.match(/```(?:javascript|js)?\n([\s\S]+?)\n```/)?.[1].trim() ?? content.trim(); const code = content.match(/```(?:javascript|js)?\n([\s\S]+?)\n```/)?.[1].trim() ?? content.trim();
return `${code.replace(/^export\s+(default\s+)?/, '')}\nexport default ${functionName};`; const finalCode = `${code.replace(/^export\s+(default\s+)?/, '')}\nexport default ${functionName};`;
return { code: finalCode, duration };
} catch (error) { } catch (error) {
console.error(`API Error for ${model}: ${error.message}`); console.error(`API Error for ${model}: ${error.message}`);
return null; return null;
@@ -35,22 +40,27 @@ const main = async () => {
); );
const testsToRun = allTestDirs.slice(0, Math.ceil(allTestDirs.length * (percentage / 100))); const testsToRun = allTestDirs.slice(0, Math.ceil(allTestDirs.length * (percentage / 100)));
const genData = {};
for (const model of models) { for (const model of models) {
genData[model] = {};
for (const dir of testsToRun) { for (const dir of testsToRun) {
const testModule = await import(pathToFileURL(path.join(TESTS_DIR, dir, 'test.js'))); const testModule = await import(pathToFileURL(path.join(TESTS_DIR, dir, 'test.js')));
const { prompt, functionName } = testModule.default; const { prompt, functionName } = testModule.default;
console.log(`Generating ${dir} for ${model}...`); console.log(`Generating ${dir} for ${model}...`);
const llmCode = await getLlmCode(prompt, model, functionName); const result = await getLlmCode(prompt, model, functionName);
if (!llmCode) continue;
genData[model][dir] = result?.duration ?? null;
if (!result) continue;
const outDir = path.join(TESTS_DIR, dir, 'outputs'); const outDir = path.join(TESTS_DIR, dir, 'outputs');
await fs.mkdir(outDir, { recursive: true }); await fs.mkdir(outDir, { recursive: true });
const fname = `${model.replace(/[\/:]/g, '_')}.js`; const fname = `${model.replace(/[\/:]/g, '_')}.js`;
await fs.writeFile(path.join(outDir, fname), llmCode); await fs.writeFile(path.join(outDir, fname), result.code);
} }
} }
console.log('Code generation complete.'); await fs.writeFile(RESULTS_PATH, JSON.stringify(genData, null, 2));
console.log('Code generation and results.json update complete.');
}; };
main().catch(console.error); main().catch(console.error);