multipleof4/lynchmark
1
0
Fork 0
mirror of https://github.com/multipleof4/lynchmark.git synced 2026-01-14 00:27:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactor: Support test folders and save LLM outputs

Browse Source
This commit is contained in:
multipleof4
2025-10-13 05:50:26 -07:00
parent 43d17e9b0f
commit b9b01d6bfa
1 changed files with 28 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
scripts/run-benchmark.js
View File

@@ -5,7 +5,7 @@ const { exec } = require('child_process');
const util = require('util'); const util = require('util');
const execPromise = util.promisify(exec); const execPromise = util.promisify(exec);
const README_PATH = path.join(__dirname, '..', 'README.md'); const README_PATH = path.join(__dirname, '..', 'README');
const TESTS_DIR = path.join(__dirname, '..', 'tests'); const TESTS_DIR = path.join(__dirname, '..', 'tests');
const TEMP_FILE = path.join(__dirname, 'temp_test.mjs'); const TEMP_FILE = path.join(__dirname, 'temp_test.mjs');
@@ -17,8 +17,7 @@ const getLlmCode = async (prompt, model) => {
{ headers: { Authorization: `Bearer ${process.env.OPENROUTER_KEY}` } } { headers: { Authorization: `Bearer ${process.env.OPENROUTER_KEY}` } }
); );
const content = res.data.choices[0].message.content; const content = res.data.choices[0].message.content;
const match = content.match(/```(?:javascript|js)?\n([\s\S]+?)\n```/); return content.match(/```(?:javascript|js)?\n([\s\S]+?)\n```/)?.[1].trim() ?? content.trim();
return match ? match[1].trim() : content.trim();
} catch (error) { } catch (error) {
console.error(`API Error for ${model}: ${error.message}`); console.error(`API Error for ${model}: ${error.message}`);
return null; return null;
@@ -27,7 +26,7 @@ const getLlmCode = async (prompt, model) => {
const runTest = async (code) => { const runTest = async (code) => {
try { try {
await fs.writeFile(TEMP_FILE, code, 'utf-8'); await fs.writeFile(TEMP_FILE, code);
await execPromise(`node ${TEMP_FILE}`); await execPromise(`node ${TEMP_FILE}`);
return true; return true;
} catch (error) { } catch (error) {
@@ -40,22 +39,39 @@ const runTest = async (code) => {
const main = async () => { const main = async () => {
const readme = await fs.readFile(README_PATH, 'utf-8'); const readme = await fs.readFile(README_PATH, 'utf-8');
const models = readme.match(/<!-- MODELS_START -->\n([\s\S]+?)\n<!-- MODELS_END -->/)[1].trim().split('\n'); const models = readme.match(/<!-- MODELS_START -->\n([\s\S]+?)\n<!-- MODELS_END -->/)[1].trim().split('\n');
const testFiles = (await fs.readdir(TESTS_DIR)).sort(); const percentage = parseInt(readme.match(/RUN_PERCENTAGE:\s*(\d+)/)?.[1] ?? '100', 10);
const allTestDirs = (await fs.readdir(TESTS_DIR, { withFileTypes: true }))
.filter(d => d.isDirectory()).map(d => d.name).sort();
const testsToRun = allTestDirs.slice(0, Math.ceil(allTestDirs.length * (percentage / 100)));
let resultsTable = '| Model | ' + testFiles.map(f => f.replace('.js','')).join(' | ') + ' |\n'; let resultsTable = '| Model | ' + allTestDirs.join(' | ') + ' |\n';
resultsTable += '|' + ' --- |'.repeat(testFiles.length + 1) + '\n'; resultsTable += '|' + ' --- |'.repeat(allTestDirs.length + 1) + '\n';
for (const model of models) { for (const model of models) {
resultsTable += `| ${model} |`; resultsTable += `| ${model} |`;
for (const file of testFiles) { for (const dir of allTestDirs) {
const { prompt, harness } = require(path.join(TESTS_DIR, file)); if (!testsToRun.includes(dir)) {
console.log(`Running ${file} for ${model}...`); resultsTable += ' ⚪ Not Run |';
continue;
}
const { prompt, harness } = require(path.join(TESTS_DIR, dir, 'test.js'));
console.log(`Running ${dir} for ${model}...`);
const llmCode = await getLlmCode(prompt, model); const llmCode = await getLlmCode(prompt, model);
if (!llmCode) { if (!llmCode) {
resultsTable += ' ❌ API Error |'; resultsTable += ' ❌ API Error |';
continue; continue;
} }
const passed = await runTest(llmCode + '\n' + harness);
const outDir = path.join(TESTS_DIR, dir, 'outputs');
await fs.mkdir(outDir, { recursive: true });
const fname = `${model.replace(/[\/:]/g, '_')}_${new Date().toISOString().replace(/:/g, '-')}.js`;
await fs.writeFile(path.join(outDir, fname), llmCode);
const passed = await runTest(`${llmCode}\n${harness}`);
resultsTable += ` ${passed ? '✅ Pass' : '❌ Fail'} |`; resultsTable += ` ${passed ? '✅ Pass' : '❌ Fail'} |`;
} }
resultsTable += '\n'; resultsTable += '\n';
@@ -65,7 +81,7 @@ const main = async () => {
/<!-- RESULTS_START -->[\s\S]*<!-- RESULTS_END -->/, /<!-- RESULTS_START -->[\s\S]*<!-- RESULTS_END -->/,
`<!-- RESULTS_START -->\n${resultsTable}\n<!-- RESULTS_END -->` `<!-- RESULTS_START -->\n${resultsTable}\n<!-- RESULTS_END -->`
); );
await fs.writeFile(README_PATH, newReadme, 'utf-8'); await fs.writeFile(README_PATH, newReadme);
console.log('Benchmark complete. README updated.'); console.log('Benchmark complete. README updated.');
}; };