multipleof4/lynchmark
1
0
Fork 0
mirror of https://github.com/multipleof4/lynchmark.git synced 2026-01-14 00:27:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Feat: Use temperature from README model config

Browse Source
This commit is contained in:
multipleof4
2025-10-13 11:21:35 -07:00
parent 3a91f0e250
commit eaf29ad08d
1 changed files with 16 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
scripts/run-benchmark.js
View File

@@ -9,12 +9,12 @@ const README_PATH = path.join(CWD, 'README');
const TESTS_DIR = path.join(CWD, 'tests'); const TESTS_DIR = path.join(CWD, 'tests');
const RESULTS_PATH = path.join(CWD, 'results.json'); const RESULTS_PATH = path.join(CWD, 'results.json');
const getLlmCode = async (prompt, model, functionName) => { const getLlmCode = async (prompt, model, functionName, temperature) => {
const start = performance.now(); const start = performance.now();
try { try {
const res = await axios.post( const res = await axios.post(
'https://openrouter.ai/api/v1/chat/completions', 'https://openrouter.ai/api/v1/chat/completions',
{ model, messages: [{ role: 'user', content: prompt }] }, { model, messages: [{ role: 'user', content: prompt }], ...(temperature !== undefined && { temperature }) },
{ headers: { Authorization: `Bearer ${process.env.OPENROUTER_KEY}` } } { headers: { Authorization: `Bearer ${process.env.OPENROUTER_KEY}` } }
); );
const duration = (performance.now() - start) / 1000; const duration = (performance.now() - start) / 1000;
@@ -43,19 +43,27 @@ const main = async () => {
const testsToRun = allTestDirs.slice(0, Math.ceil(allTestDirs.length * (percentage / 100))); const testsToRun = allTestDirs.slice(0, Math.ceil(allTestDirs.length * (percentage / 100)));
const genData = {}; const genData = {};
for (const model of models) { for (const modelSpec of models) {
genData[model] = {}; const [model, tempStr] = modelSpec.split(' TEMP:');
const temperature = tempStr ? parseFloat(tempStr) : undefined;
genData[modelSpec] = {};
for (const dir of testsToRun) { for (const dir of testsToRun) {
const { prompt, functionName } = (await import(pathToFileURL(path.join(TESTS_DIR, dir, 'test.js')))).default; const { prompt, functionName } = (await import(pathToFileURL(path.join(TESTS_DIR, dir, 'test.js')))).default;
console.log(`Generating ${dir} for ${model}...`); console.log(`Generating ${dir} for ${modelSpec}...`);
const result = await getLlmCode(`${sharedPrompt}\n\n${prompt.trim()}`, model, functionName); const result = await getLlmCode(
`${sharedPrompt}\n\n${prompt.trim()}`,
model,
functionName,
temperature
);
genData[model][dir] = result?.duration ?? null; genData[modelSpec][dir] = result?.duration ?? null;
if (!result) continue; if (!result) continue;
const outDir = path.join(TESTS_DIR, dir, 'outputs'); const outDir = path.join(TESTS_DIR, dir, 'outputs');
await fs.mkdir(outDir, { recursive: true }); await fs.mkdir(outDir, { recursive: true });
const fname = `${model.replace(/[\/:]/g, '_')}.js`; const fname = `${modelSpec.replace(/[\/:]/g, '_')}.js`;
await fs.writeFile(path.join(outDir, fname), result.code); await fs.writeFile(path.join(outDir, fname), result.code);
} }
} }