multipleof4/lynchmark
1
0
Fork 0
mirror of https://github.com/multipleof4/lynchmark.git synced 2026-01-14 00:27:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactor: Generate indented text block for README

Browse Source
This commit is contained in:
multipleof4
2025-10-13 05:56:09 -07:00
parent 881269bb0b
commit 9c773c493c
1 changed files with 27 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
scripts/run-benchmark.js
View File

@@ -40,42 +40,48 @@ const main = async () => {
const readme = await fs.readFile(README_PATH, 'utf-8'); const readme = await fs.readFile(README_PATH, 'utf-8');
const models = readme.match(/<!-- MODELS_START -->\n([\s\S]+?)\n<!-- MODELS_END -->/)[1].trim().split('\n'); const models = readme.match(/<!-- MODELS_START -->\n([\s\S]+?)\n<!-- MODELS_END -->/)[1].trim().split('\n');
const percentage = parseInt(readme.match(/RUN_PERCENTAGE:\s*(\d+)/)?.[1] ?? '100', 10); const percentage = parseInt(readme.match(/RUN_PERCENTAGE:\s*(\d+)/)?.[1] ?? '100', 10);
const allTestDirs = (await fs.readdir(TESTS_DIR, { withFileTypes: true })) const allTestDirs = (await fs.readdir(TESTS_DIR, { withFileTypes: true }))
.filter(d => d.isDirectory()).map(d => d.name).sort(); .filter(d => d.isDirectory()).map(d => d.name).sort();
const testsToRun = allTestDirs.slice(0, Math.ceil(allTestDirs.length * (percentage / 100))); const testsToRun = allTestDirs.slice(0, Math.ceil(allTestDirs.length * (percentage / 100)));
let resultsTable = '| Model | ' + allTestDirs.join(' | ') + ' |\n'; const resultsData = [];
resultsTable += '|' + ' --- |'.repeat(allTestDirs.length + 1) + '\n';
for (const model of models) { for (const model of models) {
resultsTable += `| ${model} |`; const row = { Model: model };
for (const dir of allTestDirs) { for (const dir of allTestDirs) {
if (!testsToRun.includes(dir)) { if (!testsToRun.includes(dir)) {
resultsTable += ' ⚪ Not Run |'; row[dir] = '⚪ Not Run';
continue; continue;
} }
const { prompt, harness } = require(path.join(TESTS_DIR, dir, 'test.js')); const { prompt, harness } = require(path.join(TESTS_DIR, dir, 'test.js'));
console.log(`Running ${dir} for ${model}...`); console.log(`Running ${dir} for ${model}...`);
const llmCode = await getLlmCode(prompt, model); const llmCode = await getLlmCode(prompt, model);
if (llmCode) {
if (!llmCode) {
resultsTable += ' ❌ API Error |';
continue;
}
const outDir = path.join(TESTS_DIR, dir, 'outputs'); const outDir = path.join(TESTS_DIR, dir, 'outputs');
await fs.mkdir(outDir, { recursive: true }); await fs.mkdir(outDir, { recursive: true });
const fname = `${model.replace(/[\/:]/g, '_')}_${new Date().toISOString().replace(/:/g, '-')}.js`; const fname = `${model.replace(/[\/:]/g, '_')}_${new Date().toISOString().replace(/:/g, '-')}.js`;
await fs.writeFile(path.join(outDir, fname), llmCode); await fs.writeFile(path.join(outDir, fname), llmCode);
row[dir] = (await runTest(`${llmCode}\n${harness}`)) ? '✅ Pass' : '❌ Fail';
} else {
row[dir] = '❌ API Error';
}
}
resultsData.push(row);
}
const passed = await runTest(`${llmCode}\n${harness}`); const headers = ['Model', ...allTestDirs];
resultsTable += ` ${passed ? '✅ Pass' : '❌ Fail'} |`; const colWidths = Object.fromEntries(headers.map(h =>
} [h, Math.max(h.length, ...resultsData.map(r => r[h]?.length ?? 0))]
resultsTable += '\n'; ));
}
const formatRow = (data, padChar = ' ') =>
headers.map(h => (data[h] ?? '').padEnd(colWidths[h], padChar)).join(' | ');
const headerObj = Object.fromEntries(headers.map(h => [h,h]));
const resultsTable = [
formatRow(headerObj),
formatRow({}, '-'),
...resultsData.map(row => formatRow(row))
].map(line => ' ' + line).join('\n');
const newReadme = readme.replace( const newReadme = readme.replace(
/<!-- RESULTS_START -->[\s\S]*<!-- RESULTS_END -->/, /<!-- RESULTS_START -->[\s\S]*<!-- RESULTS_END -->/,