multipleof4/lynchmark
1
0
Fork 0
mirror of https://github.com/multipleof4/lynchmark.git synced 2026-01-13 16:17:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactor: Generate indented text block for README

Browse Source
This commit is contained in:
multipleof4
2025-10-13 05:56:09 -07:00
parent 881269bb0b
commit 9c773c493c
1 changed files with 27 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
scripts/run-benchmark.js
View File

@@ -40,43 +40,49 @@ const main = async () => {
const readme = await fs.readFile(README_PATH, 'utf-8');
const models = readme.match(/<!-- MODELS_START -->\n([\s\S]+?)\n<!-- MODELS_END -->/)[1].trim().split('\n');
const percentage = parseInt(readme.match(/RUN_PERCENTAGE:\s*(\d+)/)?.[1] ?? '100', 10);
const allTestDirs = (await fs.readdir(TESTS_DIR, { withFileTypes: true }))
.filter(d => d.isDirectory()).map(d => d.name).sort();
const testsToRun = allTestDirs.slice(0, Math.ceil(allTestDirs.length * (percentage / 100)));
let resultsTable = '| Model | ' + allTestDirs.join(' | ') + ' |\n';
resultsTable += '|' + ' --- |'.repeat(allTestDirs.length + 1) + '\n';
const resultsData = [];
for (const model of models) {
resultsTable += `| ${model} |`;
const row = { Model: model };
for (const dir of allTestDirs) {
if (!testsToRun.includes(dir)) {
resultsTable += ' ⚪ Not Run |';
row[dir] = '⚪ Not Run';
continue;
}
const { prompt, harness } = require(path.join(TESTS_DIR, dir, 'test.js'));
console.log(`Running ${dir} for ${model}...`);
const llmCode = await getLlmCode(prompt, model);
if (!llmCode) {
resultsTable += ' ❌ API Error |';
continue;
if (llmCode) {
const outDir = path.join(TESTS_DIR, dir, 'outputs');
await fs.mkdir(outDir, { recursive: true });
const fname = `${model.replace(/[\/:]/g, '_')}_${new Date().toISOString().replace(/:/g, '-')}.js`;
await fs.writeFile(path.join(outDir, fname), llmCode);
row[dir] = (await runTest(`${llmCode}\n${harness}`)) ? '✅ Pass' : '❌ Fail';
} else {
row[dir] = '❌ API Error';
}
const outDir = path.join(TESTS_DIR, dir, 'outputs');
await fs.mkdir(outDir, { recursive: true });
const fname = `${model.replace(/[\/:]/g, '_')}_${new Date().toISOString().replace(/:/g, '-')}.js`;
await fs.writeFile(path.join(outDir, fname), llmCode);
const passed = await runTest(`${llmCode}\n${harness}`);
resultsTable += ` ${passed ? '✅ Pass' : '❌ Fail'} |`;
}
resultsTable += '\n';
resultsData.push(row);
}
const headers = ['Model', ...allTestDirs];
const colWidths = Object.fromEntries(headers.map(h =>
[h, Math.max(h.length, ...resultsData.map(r => r[h]?.length ?? 0))]
));
const formatRow = (data, padChar = ' ') =>
headers.map(h => (data[h] ?? '').padEnd(colWidths[h], padChar)).join(' | ');
const headerObj = Object.fromEntries(headers.map(h => [h,h]));
const resultsTable = [
formatRow(headerObj),
formatRow({}, '-'),
...resultsData.map(row => formatRow(row))
].map(line => ' ' + line).join('\n');
const newReadme = readme.replace(
/<!-- RESULTS_START -->[\s\S]*<!-- RESULTS_END -->/,
`<!-- RESULTS_START -->\n${resultsTable}\n<!-- RESULTS_END -->`