multipleof4/lynchmark
1
0
Fork 0
mirror of https://github.com/multipleof4/lynchmark.git synced 2026-01-14 00:27:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Revert: Update run-benchmark.js

Browse Source
This commit is contained in:
multipleof4
2025-10-13 06:15:02 -07:00
parent 39f49566ee
commit c1aaadeb7a
1 changed files with 6 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
scripts/run-benchmark.js
View File

@@ -8,7 +8,7 @@ import { performance } from 'perf_hooks';
const execPromise = promisify(exec); const execPromise = promisify(exec);
const __dirname = path.dirname(fileURLToPath(import.meta.url)); const __dirname = path.dirname(fileURLToPath(import.meta.url));
const README_PATH = path.join(__dirname, '..', 'README.md'); const README_PATH = path.join(__dirname, '..', 'README');
const TESTS_DIR = path.join(__dirname, '..', 'tests'); const TESTS_DIR = path.join(__dirname, '..', 'tests');
const TEMP_FILE = path.join(__dirname, 'temp_test.js'); const TEMP_FILE = path.join(__dirname, 'temp_test.js');
@@ -47,14 +47,7 @@ const main = async () => {
const models = readme.match(/<!-- MODELS_START -->\n([\s\S]+?)\n<!-- MODELS_END -->/)[1].trim().split('\n'); const models = readme.match(/<!-- MODELS_START -->\n([\s\S]+?)\n<!-- MODELS_END -->/)[1].trim().split('\n');
const percentage = parseInt(readme.match(/RUN_PERCENTAGE:\s*(\d+)/)?.[1] ?? '100', 10); const percentage = parseInt(readme.match(/RUN_PERCENTAGE:\s*(\d+)/)?.[1] ?? '100', 10);
const allTestDirs = (await fs.readdir(TESTS_DIR, { withFileTypes: true })) const allTestDirs = (await fs.readdir(TESTS_DIR, { withFileTypes: true }))
.filter(d => d.isDirectory()).map(d => d.name); .filter(d => d.isDirectory()).map(d => d.name).sort();
console.log('Clearing old test outputs...');
await Promise.all(allTestDirs.map(dir =>
fs.rm(path.join(TESTS_DIR, dir, 'outputs'), { recursive: true, force: true })
));
allTestDirs.sort();
const testsToRun = allTestDirs.slice(0, Math.ceil(allTestDirs.length * (percentage / 100))); const testsToRun = allTestDirs.slice(0, Math.ceil(allTestDirs.length * (percentage / 100)));
const results = []; const results = [];
@@ -66,15 +59,15 @@ const main = async () => {
continue; continue;
} }
const testModule = await import(pathToFileURL(path.join(TESTS_DIR, dir, 'test.js'))); const testModule = await import(pathToFileURL(path.join(TESTS_DIR, dir, 'test.js')));
const { prompt, imports, harness } = testModule.default; const { prompt, harness } = testModule.default;
console.log(`Running ${dir} for ${model}...`); console.log(`Running ${dir} for ${model}...`);
const llmCode = await getLlmCode(prompt, model); const llmCode = await getLlmCode(prompt, model);
if (llmCode) { if (llmCode) {
const outDir = path.join(TESTS_DIR, dir, 'outputs'); const outDir = path.join(TESTS_DIR, dir, 'outputs');
await fs.mkdir(outDir, { recursive: true }); await fs.mkdir(outDir, { recursive: true });
const fname = `${model.replace(/[\/:]/g, '_')}.js`; const fname = `${model.replace(/[\/:]/g, '_')}_${new Date().toISOString().replace(/:/g, '-')}.js`;
await fs.writeFile(path.join(outDir, fname), llmCode); await fs.writeFile(path.join(outDir, fname), llmCode);
const { passed, duration } = await runTest(`${imports || ''}\n${llmCode}\n${harness}`); const { passed, duration } = await runTest(`${llmCode}\n${harness}`);
results.push(`- ${dir}: ${passed ? '✅ Pass' : '❌ Fail'} (${duration.toFixed(3)}s)`); results.push(`- ${dir}: ${passed ? '✅ Pass' : '❌ Fail'} (${duration.toFixed(3)}s)`);
} else { } else {
results.push(`- ${dir}: ❌ API Error`); results.push(`- ${dir}: ❌ API Error`);
@@ -92,3 +85,4 @@ const main = async () => {
}; };
main().catch(console.error); main().catch(console.error);