multipleof4/lynchmark
1
0
Fork 0
mirror of https://github.com/multipleof4/lynchmark.git synced 2026-01-14 08:37:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Feat: Support Google API and Gemini mode

Browse Source
This commit is contained in:
multipleof4
2025-11-18 10:28:01 -08:00
parent 5decb550ab
commit 7420f7d2cb
1 changed files with 51 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

111
scripts/run-benchmark.js
View File

@@ -5,84 +5,75 @@ import axios from 'axios';
import { performance } from 'perf_hooks'; import { performance } from 'perf_hooks';
const CWD = process.cwd(); const CWD = process.cwd();
const README_PATH = path.join(CWD, 'README'); const [README, TESTS, RESULTS] = ['README', 'tests', 'results.json'].map(f => path.join(CWD, f));
const TESTS_DIR = path.join(CWD, 'tests'); const getArg = n => { const i = process.argv.indexOf(n); return i > -1 ? process.argv[i + 1] : null; };
const RESULTS_PATH = path.join(CWD, 'results.json'); const isGemini = process.argv.includes('--gemini');
const getArg = name => { const i = process.argv.indexOf(name); return i > -1 ? process.argv[i + 1] : null; };
const getLlmCode = async (prompt, model, functionName, temperature) => { const apiCall = async (prompt, model, temp) => {
if (isGemini) {
const url = `https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent?key=${process.env.GOOGLE_KEY}`;
const body = { contents: [{ parts: [{ text: prompt }] }], generationConfig: { temperature: temp } };
const res = await axios.post(url, body);
return res.data.candidates[0].content.parts[0].text;
}
const res = await axios.post(
'https://openrouter.ai/api/v1/chat/completions',
{ model, messages: [{ role: 'user', content: prompt }], ...(temp !== undefined && { temperature: temp }) },
{ headers: { Authorization: `Bearer ${process.env.OPENROUTER_KEY}` } }
);
return res.data.choices[0].message.content;
};
const getLlmCode = async (prompt, model, funcName, temp) => {
const start = performance.now(); const start = performance.now();
try { try {
const res = await axios.post( const content = await apiCall(prompt, model, temp);
'https://openrouter.ai/api/v1/chat/completions',
{ model, messages: [{ role: 'user', content: prompt }], ...(temperature !== undefined && { temperature }) },
{ headers: { Authorization: `Bearer ${process.env.OPENROUTER_KEY}` } }
);
const duration = (performance.now() - start) / 1000; const duration = (performance.now() - start) / 1000;
const content = res.data.choices[0].message.content;
const code = content.match(/```(?:javascript|js)?\n([\s\S]+?)\n```/)?.[1].trim() ?? content.trim(); const code = content.match(/```(?:javascript|js)?\n([\s\S]+?)\n```/)?.[1].trim() ?? content.trim();
const finalCode = `${code.replace(/export\s+default\s+.*$/m, '')}\nexport default ${functionName};`; return { code: `${code.replace(/export\s+default\s+.*$/m, '')}\nexport default ${funcName};`, duration };
return { code: finalCode, duration }; } catch (e) { console.error(`API Error ${model}: ${e.message}`); return null; }
} catch (error) {
console.error(`API Error for ${model}: ${error.message}`);
return null;
}
}; };
const main = async () => { const main = async () => {
const readme = await fs.readFile(README_PATH, 'utf-8'); const readme = await fs.readFile(README, 'utf-8');
const allModels = readme.match(/<!-- MODELS_START -->\n([\s\S]+?)\n<!-- MODELS_END -->/)[1].trim().split('\n'); const regex = isGemini ? /<!-- GEMINI_START -->\n([\s\S]+?)\n<!-- MODELS_END -->/ : /<!-- MODELS_START -->\n([\s\S]+?)\n<!-- MODELS_END -->/;
const percentage = parseInt(readme.match(/RUN_PERCENTAGE:\s*(\d+)/)?.[1] ?? '100', 10); const allModels = readme.match(regex)[1].trim().split('\n').filter(Boolean);
const sharedPrompt = readme.match(/SHARED_PROMPT:\s*"([\s\S]+?)"/)?.[1] ?? ''; const pct = parseInt(readme.match(/RUN_PERCENTAGE:\s*(\d+)/)?.[1] ?? '100', 10);
const shared = readme.match(/SHARED_PROMPT:\s*"([\s\S]+?)"/)?.[1] ?? '';
const singleModel = getArg('--model'); const sModel = getArg('--model');
if (singleModel && !allModels.includes(singleModel)) { if (sModel && !allModels.includes(sModel)) throw new Error(`Model "${sModel}" not found in README.`);
throw new Error(`Model "${singleModel}" not found in README.md.`); const models = sModel ? [sModel] : allModels;
}
const modelsToRun = singleModel ? [singleModel] : allModels;
const allTestDirs = (await fs.readdir(TESTS_DIR, { withFileTypes: true })) const testDirs = (await fs.readdir(TESTS, { withFileTypes: true })).filter(d => d.isDirectory()).map(d => d.name).sort();
.filter(d => d.isDirectory()).map(d => d.name).sort(); const targetTests = testDirs.slice(0, Math.ceil(testDirs.length * (pct / 100)));
if (singleModel) { const clean = dir => fs.rm(path.join(TESTS, dir, 'outputs', sModel ? `${sModel.replace(/[\/:]/g, '_')}.js` : ''), { recursive: !sModel, force: true });
const sModel = singleModel.replace(/[\/:]/g, '_'); await Promise.all(testDirs.map(clean));
await Promise.all(allTestDirs.map(dir =>
fs.rm(path.join(TESTS_DIR, dir, 'outputs', `${sModel}.js`), { force: true })
));
} else {
await Promise.all(allTestDirs.map(dir =>
fs.rm(path.join(TESTS_DIR, dir, 'outputs'), { recursive: true, force: true })
));
}
const testsToRun = allTestDirs.slice(0, Math.ceil(allTestDirs.length * (percentage / 100))); let data = {};
try { data = JSON.parse(await fs.readFile(RESULTS, 'utf-8')); } catch {}
let genData = {};
if (singleModel) {
try { genData = JSON.parse(await fs.readFile(RESULTS_PATH, 'utf-8')); } catch {}
}
for (const modelSpec of modelsToRun) { for (const mSpec of models) {
const [model, tempStr] = modelSpec.split(' TEMP:'); const [model, tStr] = mSpec.split(' TEMP:');
const temperature = tempStr ? parseFloat(tempStr) : undefined; const temp = tStr ? parseFloat(tStr) : undefined;
data[mSpec] ||= {};
genData[modelSpec] = {};
for (const dir of testsToRun) { for (const dir of targetTests) {
const { prompt, functionName } = (await import(pathToFileURL(path.join(TESTS_DIR, dir, 'test.js')))).default; const { prompt, functionName } = (await import(pathToFileURL(path.join(TESTS, dir, 'test.js')))).default;
console.log(`Generating ${dir} for ${modelSpec}...`); console.log(`Gen ${dir} for ${mSpec}...`);
const result = await getLlmCode(`${sharedPrompt}\n\n${prompt.trim()}`, model, functionName, temperature); const res = await getLlmCode(`${shared}\n\n${prompt.trim()}`, model, functionName, temp);
genData[modelSpec][dir] = result?.duration ?? null; data[mSpec][dir] = res?.duration ?? null;
if (!result) continue; if (!res) continue;
const outDir = path.join(TESTS_DIR, dir, 'outputs'); const outDir = path.join(TESTS, dir, 'outputs');
await fs.mkdir(outDir, { recursive: true }); await fs.mkdir(outDir, { recursive: true });
const fname = `${modelSpec.replace(/[\/:]/g, '_')}.js`; await fs.writeFile(path.join(outDir, `${mSpec.replace(/[\/:]/g, '_')}.js`), res.code);
await fs.writeFile(path.join(outDir, fname), result.code);
} }
} }
await fs.writeFile(RESULTS_PATH, JSON.stringify(genData, null, 2)); await fs.writeFile(RESULTS, JSON.stringify(data, null, 2));
console.log('Code generation and results.json update complete.'); console.log('Done.');
}; };
main().catch(console.error); main().catch(console.error);