multipleof4/lynchmark
1
0
Fork 0
mirror of https://github.com/multipleof4/lynchmark.git synced 2026-01-13 16:17:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Feat: Support EFF:xhigh syntax for reasoning effort

Browse Source
This commit is contained in:
multipleof4
2025-12-11 13:06:14 -08:00
parent e56e9e9ad9
commit 4d3beabb98
1 changed files with 14 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
scripts/run-benchmark.js
View File

@@ -11,7 +11,7 @@ const getArg = n => { const i = process.argv.indexOf(n); return i > -1 ? process
const isGemini = process.argv.includes('--gemini'); const isGemini = process.argv.includes('--gemini');
const OUT_DIR_NAME = isGemini ? 'outputs_gemini' : 'outputs'; const OUT_DIR_NAME = isGemini ? 'outputs_gemini' : 'outputs';
const apiCall = async (prompt, model, temp) => { const apiCall = async (prompt, model, temp, effort) => {
if (isGemini) { if (isGemini) {
const url = `https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent?key=${process.env.GOOGLE_KEY}`; const url = `https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent?key=${process.env.GOOGLE_KEY}`;
const body = { contents: [{ parts: [{ text: prompt }] }], generationConfig: { temperature: temp } }; const body = { contents: [{ parts: [{ text: prompt }] }], generationConfig: { temperature: temp } };
@@ -20,16 +20,21 @@ const apiCall = async (prompt, model, temp) => {
} }
const res = await axios.post( const res = await axios.post(
'https://openrouter.ai/api/v1/chat/completions', 'https://openrouter.ai/api/v1/chat/completions',
{ model, messages: [{ role: 'user', content: prompt }], ...(temp !== undefined && { temperature: temp }) }, {
model,
messages: [{ role: 'user', content: prompt }],
...(temp !== undefined && { temperature: temp }),
...(effort && { reasoning: { effort } })
},
{ headers: { Authorization: `Bearer ${process.env.OPENROUTER_KEY}` } } { headers: { Authorization: `Bearer ${process.env.OPENROUTER_KEY}` } }
); );
return res.data.choices[0].message.content; return res.data.choices[0].message.content;
}; };
const getLlmCode = async (prompt, model, funcName, temp) => { const getLlmCode = async (prompt, model, funcName, temp, effort) => {
const start = performance.now(); const start = performance.now();
try { try {
const content = await apiCall(prompt, model, temp); const content = await apiCall(prompt, model, temp, effort);
const duration = (performance.now() - start) / 1000; const duration = (performance.now() - start) / 1000;
const code = content.match(/```(?:javascript|js)?\n([\s\S]+?)\n```/)?.[1].trim() ?? content.trim(); const code = content.match(/```(?:javascript|js)?\n([\s\S]+?)\n```/)?.[1].trim() ?? content.trim();
const clean = code.replace(/export\s+default\s+.*$/m, ''); const clean = code.replace(/export\s+default\s+.*$/m, '');
@@ -88,7 +93,10 @@ const main = async () => {
const page = await browser.newPage(); const page = await browser.newPage();
for (const mSpec of models) { for (const mSpec of models) {
const [model, tStr] = mSpec.split(' TEMP:'); const parts = mSpec.split(/\s+/);
const model = parts[0];
const tStr = parts.find(p => p.startsWith('TEMP:'))?.split(':')[1];
const eStr = parts.find(p => p.startsWith('EFF:'))?.split(':')[1];
const temp = tStr ? parseFloat(tStr) : undefined; const temp = tStr ? parseFloat(tStr) : undefined;
for (const dir of targetTests) { for (const dir of targetTests) {
@@ -97,7 +105,7 @@ const main = async () => {
const testCode = await fs.readFile(testPath, 'utf-8'); const testCode = await fs.readFile(testPath, 'utf-8');
console.log(`Gen ${dir} for ${mSpec} in ${OUT_DIR_NAME}...`); console.log(`Gen ${dir} for ${mSpec} in ${OUT_DIR_NAME}...`);
const res = await getLlmCode(`${shared}\n\n${prompt.trim()}`, model, functionName, temp); const res = await getLlmCode(`${shared}\n\n${prompt.trim()}`, model, functionName, temp, eStr);
if (!res) continue; if (!res) continue;