multipleof4/lynchmark
1
0
Fork 0
mirror of https://github.com/multipleof4/lynchmark.git synced 2026-01-14 16:47:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Feat: Add test execution timing and format results as list

Browse Source
This commit is contained in:
multipleof4
2025-10-13 06:01:06 -07:00
parent c0f6dec7f4
commit daa8806226
1 changed files with 14 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
scripts/run-benchmark.js
View File

@@ -3,6 +3,7 @@ const path = require('path');
const axios = require('axios'); const axios = require('axios');
const { exec } = require('child_process'); const { exec } = require('child_process');
const util = require('util'); const util = require('util');
const { performance } = require('perf_hooks');
const execPromise = util.promisify(exec); const execPromise = util.promisify(exec);
const README_PATH = path.join(__dirname, '..', 'README'); const README_PATH = path.join(__dirname, '..', 'README');
@@ -25,15 +26,17 @@ const getLlmCode = async (prompt, model) => {
}; };
const runTest = async (code) => { const runTest = async (code) => {
const start = performance.now();
let passed = false;
try { try {
await fs.writeFile(TEMP_FILE, code); await fs.writeFile(TEMP_FILE, code);
await execPromise(`node ${TEMP_FILE}`); await execPromise(`node ${TEMP_FILE}`);
return true; passed = true;
} catch (error) { } catch (error) { // eslint-disable-line no-empty
return false;
} finally { } finally {
await fs.unlink(TEMP_FILE).catch(() => {}); await fs.unlink(TEMP_FILE).catch(() => {});
} }
return { passed, duration: (performance.now() - start) / 1000 };
}; };
const main = async () => { const main = async () => {
@@ -44,12 +47,12 @@ const main = async () => {
.filter(d => d.isDirectory()).map(d => d.name).sort(); .filter(d => d.isDirectory()).map(d => d.name).sort();
const testsToRun = allTestDirs.slice(0, Math.ceil(allTestDirs.length * (percentage / 100))); const testsToRun = allTestDirs.slice(0, Math.ceil(allTestDirs.length * (percentage / 100)));
const resultsData = []; const results = [];
for (const model of models) { for (const model of models) {
const row = { Model: model }; results.push(`**${model}**`);
for (const dir of allTestDirs) { for (const dir of allTestDirs) {
if (!testsToRun.includes(dir)) { if (!testsToRun.includes(dir)) {
row[dir] = '⚪ Not Run'; results.push(`- ${dir}: ⚪ Not Run`);
continue; continue;
} }
const { prompt, harness } = require(path.join(TESTS_DIR, dir, 'test.js')); const { prompt, harness } = require(path.join(TESTS_DIR, dir, 'test.js'));
@@ -60,32 +63,18 @@ const main = async () => {
await fs.mkdir(outDir, { recursive: true }); await fs.mkdir(outDir, { recursive: true });
const fname = `${model.replace(/[\/:]/g, '_')}_${new Date().toISOString().replace(/:/g, '-')}.js`; const fname = `${model.replace(/[\/:]/g, '_')}_${new Date().toISOString().replace(/:/g, '-')}.js`;
await fs.writeFile(path.join(outDir, fname), llmCode); await fs.writeFile(path.join(outDir, fname), llmCode);
row[dir] = (await runTest(`${llmCode}\n${harness}`)) ? '✅ Pass' : '❌ Fail'; const { passed, duration } = await runTest(`${llmCode}\n${harness}`);
results.push(`- ${dir}: ${passed ? '✅ Pass' : '❌ Fail'} (${duration.toFixed(3)}s)`);
} else { } else {
row[dir] = '❌ API Error'; results.push(`- ${dir}: ❌ API Error`);
} }
} }
resultsData.push(row); results.push('');
} }
const headers = ['Model', ...allTestDirs];
const colWidths = Object.fromEntries(headers.map(h =>
[h, Math.max(h.length, ...resultsData.map(r => r[h]?.length ?? 0))]
));
const formatRow = (data, padChar = ' ') =>
headers.map(h => (data[h] ?? '').padEnd(colWidths[h], padChar)).join(' | ');
const headerObj = Object.fromEntries(headers.map(h => [h,h]));
const resultsTable = [
formatRow(headerObj),
formatRow({}, '-'),
...resultsData.map(row => formatRow(row))
].map(line => ' ' + line).join('\n');
const newReadme = readme.replace( const newReadme = readme.replace(
/<!-- RESULTS_START -->[\s\S]*<!-- RESULTS_END -->/, /<!-- RESULTS_START -->[\s\S]*<!-- RESULTS_END -->/,
`<!-- RESULTS_START -->\n${resultsTable}\n<!-- RESULTS_END -->` `<!-- RESULTS_START -->\n${results.join('\n').trim()}\n<!-- RESULTS_END -->`
); );
await fs.writeFile(README_PATH, newReadme); await fs.writeFile(README_PATH, newReadme);
console.log('Benchmark complete. README updated.'); console.log('Benchmark complete. README updated.');