From 7420f7d2cb82edc461ec104ba5c2ccd9b11fc493 Mon Sep 17 00:00:00 2001 From: multipleof4 Date: Tue, 18 Nov 2025 10:28:01 -0800 Subject: [PATCH] Feat: Support Google API and Gemini mode --- scripts/run-benchmark.js | 111 ++++++++++++++++++--------------------- 1 file changed, 51 insertions(+), 60 deletions(-) diff --git a/scripts/run-benchmark.js b/scripts/run-benchmark.js index 00bfc97..0155054 100644 --- a/scripts/run-benchmark.js +++ b/scripts/run-benchmark.js @@ -5,84 +5,75 @@ import axios from 'axios'; import { performance } from 'perf_hooks'; const CWD = process.cwd(); -const README_PATH = path.join(CWD, 'README'); -const TESTS_DIR = path.join(CWD, 'tests'); -const RESULTS_PATH = path.join(CWD, 'results.json'); -const getArg = name => { const i = process.argv.indexOf(name); return i > -1 ? process.argv[i + 1] : null; }; +const [README, TESTS, RESULTS] = ['README', 'tests', 'results.json'].map(f => path.join(CWD, f)); +const getArg = n => { const i = process.argv.indexOf(n); return i > -1 ? process.argv[i + 1] : null; }; +const isGemini = process.argv.includes('--gemini'); -const getLlmCode = async (prompt, model, functionName, temperature) => { +const apiCall = async (prompt, model, temp) => { + if (isGemini) { + const url = `https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent?key=${process.env.GOOGLE_KEY}`; + const body = { contents: [{ parts: [{ text: prompt }] }], generationConfig: { temperature: temp } }; + const res = await axios.post(url, body); + return res.data.candidates[0].content.parts[0].text; + } + const res = await axios.post( + 'https://openrouter.ai/api/v1/chat/completions', + { model, messages: [{ role: 'user', content: prompt }], ...(temp !== undefined && { temperature: temp }) }, + { headers: { Authorization: `Bearer ${process.env.OPENROUTER_KEY}` } } + ); + return res.data.choices[0].message.content; +}; + +const getLlmCode = async (prompt, model, funcName, temp) => { const start = performance.now(); try { - const res = await axios.post( - 'https://openrouter.ai/api/v1/chat/completions', - { model, messages: [{ role: 'user', content: prompt }], ...(temperature !== undefined && { temperature }) }, - { headers: { Authorization: `Bearer ${process.env.OPENROUTER_KEY}` } } - ); + const content = await apiCall(prompt, model, temp); const duration = (performance.now() - start) / 1000; - const content = res.data.choices[0].message.content; const code = content.match(/```(?:javascript|js)?\n([\s\S]+?)\n```/)?.[1].trim() ?? content.trim(); - const finalCode = `${code.replace(/export\s+default\s+.*$/m, '')}\nexport default ${functionName};`; - return { code: finalCode, duration }; - } catch (error) { - console.error(`API Error for ${model}: ${error.message}`); - return null; - } + return { code: `${code.replace(/export\s+default\s+.*$/m, '')}\nexport default ${funcName};`, duration }; + } catch (e) { console.error(`API Error ${model}: ${e.message}`); return null; } }; const main = async () => { - const readme = await fs.readFile(README_PATH, 'utf-8'); - const allModels = readme.match(/\n([\s\S]+?)\n/)[1].trim().split('\n'); - const percentage = parseInt(readme.match(/RUN_PERCENTAGE:\s*(\d+)/)?.[1] ?? '100', 10); - const sharedPrompt = readme.match(/SHARED_PROMPT:\s*"([\s\S]+?)"/)?.[1] ?? ''; + const readme = await fs.readFile(README, 'utf-8'); + const regex = isGemini ? /\n([\s\S]+?)\n/ : /\n([\s\S]+?)\n/; + const allModels = readme.match(regex)[1].trim().split('\n').filter(Boolean); + const pct = parseInt(readme.match(/RUN_PERCENTAGE:\s*(\d+)/)?.[1] ?? '100', 10); + const shared = readme.match(/SHARED_PROMPT:\s*"([\s\S]+?)"/)?.[1] ?? ''; - const singleModel = getArg('--model'); - if (singleModel && !allModels.includes(singleModel)) { - throw new Error(`Model "${singleModel}" not found in README.md.`); - } - const modelsToRun = singleModel ? [singleModel] : allModels; + const sModel = getArg('--model'); + if (sModel && !allModels.includes(sModel)) throw new Error(`Model "${sModel}" not found in README.`); + const models = sModel ? [sModel] : allModels; - const allTestDirs = (await fs.readdir(TESTS_DIR, { withFileTypes: true })) - .filter(d => d.isDirectory()).map(d => d.name).sort(); + const testDirs = (await fs.readdir(TESTS, { withFileTypes: true })).filter(d => d.isDirectory()).map(d => d.name).sort(); + const targetTests = testDirs.slice(0, Math.ceil(testDirs.length * (pct / 100))); - if (singleModel) { - const sModel = singleModel.replace(/[\/:]/g, '_'); - await Promise.all(allTestDirs.map(dir => - fs.rm(path.join(TESTS_DIR, dir, 'outputs', `${sModel}.js`), { force: true }) - )); - } else { - await Promise.all(allTestDirs.map(dir => - fs.rm(path.join(TESTS_DIR, dir, 'outputs'), { recursive: true, force: true }) - )); - } + const clean = dir => fs.rm(path.join(TESTS, dir, 'outputs', sModel ? `${sModel.replace(/[\/:]/g, '_')}.js` : ''), { recursive: !sModel, force: true }); + await Promise.all(testDirs.map(clean)); - const testsToRun = allTestDirs.slice(0, Math.ceil(allTestDirs.length * (percentage / 100))); - - let genData = {}; - if (singleModel) { - try { genData = JSON.parse(await fs.readFile(RESULTS_PATH, 'utf-8')); } catch {} - } + let data = {}; + try { data = JSON.parse(await fs.readFile(RESULTS, 'utf-8')); } catch {} - for (const modelSpec of modelsToRun) { - const [model, tempStr] = modelSpec.split(' TEMP:'); - const temperature = tempStr ? parseFloat(tempStr) : undefined; - - genData[modelSpec] = {}; - for (const dir of testsToRun) { - const { prompt, functionName } = (await import(pathToFileURL(path.join(TESTS_DIR, dir, 'test.js')))).default; - console.log(`Generating ${dir} for ${modelSpec}...`); - const result = await getLlmCode(`${sharedPrompt}\n\n${prompt.trim()}`, model, functionName, temperature); + for (const mSpec of models) { + const [model, tStr] = mSpec.split(' TEMP:'); + const temp = tStr ? parseFloat(tStr) : undefined; + data[mSpec] ||= {}; + + for (const dir of targetTests) { + const { prompt, functionName } = (await import(pathToFileURL(path.join(TESTS, dir, 'test.js')))).default; + console.log(`Gen ${dir} for ${mSpec}...`); + const res = await getLlmCode(`${shared}\n\n${prompt.trim()}`, model, functionName, temp); - genData[modelSpec][dir] = result?.duration ?? null; - if (!result) continue; + data[mSpec][dir] = res?.duration ?? null; + if (!res) continue; - const outDir = path.join(TESTS_DIR, dir, 'outputs'); + const outDir = path.join(TESTS, dir, 'outputs'); await fs.mkdir(outDir, { recursive: true }); - const fname = `${modelSpec.replace(/[\/:]/g, '_')}.js`; - await fs.writeFile(path.join(outDir, fname), result.code); + await fs.writeFile(path.join(outDir, `${mSpec.replace(/[\/:]/g, '_')}.js`), res.code); } } - await fs.writeFile(RESULTS_PATH, JSON.stringify(genData, null, 2)); - console.log('Code generation and results.json update complete.'); + await fs.writeFile(RESULTS, JSON.stringify(data, null, 2)); + console.log('Done.'); }; main().catch(console.error);