diff --git a/README.md b/README.md index dfc5d13..28f881f 100644 --- a/README.md +++ b/README.md @@ -305,6 +305,7 @@ Every request runs the Worker, which counts toward the free plan's 100,000 reque - The top 19,900 with at least `MIN_VIEWS` (20) are promoted, up to `MAX_BYTES` (4 GB) in total. The free plan allows 20,000 static files per deploy - Only JPEG, PNG, GIF and WebP are promoted. Static files get their type from their extension, which these have none of, so `public/_headers` declares them all `image/jpeg`, and browsers detect these four formats from the bytes anyway - Images are cached between runs (Actions cache), so only newly promoted ones download from R2 +- An image is dropped once it's 90 days old, the same as the R2 lifecycle rule, so a popular query (e.g. `current us president`) is still searched again instead of staying static forever. It's promoted again the next day if it's still popular Static files ignore query strings, so the zone's rules (`scripts/rules.js`) fold the site and `?i=` into the path first. The visitor's URL doesn't change: diff --git a/scripts/notify.js b/scripts/notify.js index bf783a3..2fd32fb 100644 --- a/scripts/notify.js +++ b/scripts/notify.js @@ -1,6 +1,6 @@ // Sends the deploy's outcome to ntfy: what got promoted, how close the account is to the free plan's limits, and what // to do when something needs attention. Runs last in .github/workflows/deploy.yml, even when an earlier step failed. -import { readFile, appendFile } from "node:fs/promises"; +import { readFile } from "node:fs/promises"; const env = process.env; const r = JSON.parse(await readFile("promoted/report.json", "utf8").catch(() => "{}")); @@ -31,21 +31,17 @@ if (r.workers) { } if (r.workersError) lines.push(`**Worker requests:** couldn't read (${esc(r.workersError)})`); -// The top images, with thumbnails, go on the run's summary page, which the message links to. free/2/orange+cat → -// https://free.direct-img.link/orange+cat?i=2 (a static file, so viewing the page doesn't cost Worker requests). -if (r.top?.length && env.GITHUB_STEP_SUMMARY) { - const zone = env.ZONE || "direct-img.link"; - const rows = r.top.map(({ id, views }) => { - const [f, i, name] = id.split("/"), url = `https://${f === "web" ? "" : `${f}.`}${zone}/${name}${i > 1 ? `?i=${i}` : ""}`; - return `| | [${esc(name.replaceAll("+", " "))}](${url}) | ${f === "web" ? "main" : f} | ${i} | ${fmt(views)} |`; - }); - await appendFile(env.GITHUB_STEP_SUMMARY, `## Top static images (views in the last ${r.days} days)\n\n| | Query | Site | i | Views |\n|---|---|---|---|---|\n${rows.join("\n")}\n`); -} +// web/2/orange+cat → [orange cat](https://direct-img.link/orange+cat?i=2), so each opens on its site +const zone = env.ZONE || "direct-img.link"; +const link = id => { + const [f, i, name] = id.split("/"); + return `[${esc(name.replaceAll("+", " "))}](https://${f === "web" ? "" : `${f}.`}${zone}/${name}${i > 1 ? `?i=${i}` : ""})${i > 1 ? ` i=${i}` : ""}${f === "web" ? "" : ` (${f})`}`; +}; const failed = env.STATUS !== "success"; const body = [ lines.join(" \n"), - r.top?.length && env.RUN_URL && `[See top images](${env.RUN_URL})`, + r.top?.length && `**Top images**\n${r.top.map((t, n) => `${n + 1}. ${link(t.id)} — ${fmt(t.views)}`).join("\n")}`, actions.length && `**⚠ Needs attention**\n${actions.map(a => `- ${a}`).join("\n")}`, ].filter(Boolean).join("\n\n") || "No report was written."; console.log(body); diff --git a/scripts/promote.js b/scripts/promote.js index 72528d7..f5152a7 100644 --- a/scripts/promote.js +++ b/scripts/promote.js @@ -4,7 +4,7 @@ // promoted/report.json for scripts/notify.js. promoted/ is kept between runs (Actions cache), so only new images download. import { mkdir, readFile, writeFile, link, rm, readdir } from "node:fs/promises"; import { dirname } from "node:path"; -import { MAX_INDEX, siteOf, folder, normalizeQuery, cacheKey, r2Key } from "../src/_utils/keys.js"; +import { TTL_SECONDS, MAX_INDEX, siteOf, folder, normalizeQuery, cacheKey, r2Key } from "../src/_utils/keys.js"; import { parseFreeSource } from "../src/_utils/free.js"; import { parseAiSource } from "../src/_utils/ai.js"; @@ -83,20 +83,24 @@ async function workerRequests(day) { return { total: Object.values(byScript).reduce((t, r) => t + r, 0), byScript }; } -// The image from the last run's cache, or from R2. Skips a failed search's empty marker and formats browsers can't detect. +// The image from the last run's cache, or from R2. Skips a failed search's empty marker, formats browsers can't detect, +// and images past TTL_SECONDS, so a popular query gets searched again like any other instead of staying static forever. +const expired = uploaded => Date.now() - uploaded >= TTL_SECONDS * 1000; async function fetchImage(img, prev) { - const file = `${CACHE}/files/${img.id}`; - if (prev[img.id] && await readFile(file).then(() => true, () => false)) return { ...prev[img.id], cached: true }; + const file = `${CACHE}/files/${img.id}`, p = prev[img.id]; + if (p?.uploaded && await readFile(file).then(() => true, () => false)) return expired(p.uploaded) ? { skip: "expired" } : { ...p, cached: true }; const res = await api(`/accounts/${env.CLOUDFLARE_ACCOUNT_ID}/r2/buckets/${BUCKET}/objects/${img.r2}`); if (res.status === 404) return { skip: "missing" }; if (!res.ok) return { skip: "failed", error: `${img.id}: HTTP ${res.status}` }; const type = res.headers.get("content-type")?.split(";")[0].trim().toLowerCase(), buf = Buffer.from(await res.arrayBuffer()); + const uploaded = Date.parse(res.headers.get("last-modified")); if (!buf.length) return { skip: "missing" }; + if (expired(uploaded)) return { skip: "expired" }; if (!TYPES.includes(type)) return { skip: "type" }; if (buf.length > 25 * 2 ** 20) return { skip: "size" }; await mkdir(dirname(file), { recursive: true }); await writeFile(file, buf); - return { bytes: buf.length, type }; + return { bytes: buf.length, type, uploaded }; } const report = { ok: false, minViews: MIN_VIEWS, days: DAYS, maxFiles: MAX_FILES, maxBytes: MAX_BYTES, skipped: {} }; @@ -140,7 +144,7 @@ for (let k = 0; k < ranked.length; k += 4) { } else if (n >= MAX_FILES) report.notFitted = (report.notFitted || 0) + 1; else if (bytes + g.bytes > MAX_BYTES) report.skipped.budget = (report.skipped.budget || 0) + 1; else { - chosen[img.id] = { views: img.views, bytes: g.bytes, type: g.type, r2: img.r2 }; + chosen[img.id] = { views: img.views, bytes: g.bytes, type: g.type, r2: img.r2, uploaded: g.uploaded }; bytes += g.bytes; n++; } @@ -167,7 +171,7 @@ Object.assign(report, { ok: true, promoted: ids.length, bytes, added: ids.filter(id => !prev[id]).length, removed: Object.keys(prev).filter(id => !chosen[id]).length, promotedViews: ids.reduce((a, id) => a + chosen[id].views, 0), - top: ids.slice(0, 50).map(id => ({ id, views: chosen[id].views })), + top: ids.slice(0, 10).map(id => ({ id, views: chosen[id].views })), }); await writeFile(`${CACHE}/report.json`, JSON.stringify(report, null, 2)); console.log(JSON.stringify(report, null, 2)); diff --git a/src/_utils/keys.js b/src/_utils/keys.js index 210106e..48421c5 100644 --- a/src/_utils/keys.js +++ b/src/_utils/keys.js @@ -1,5 +1,9 @@ // Shared by the worker and scripts/promote.js, so a promoted image's R2 key always matches the one the worker stored. +// How long an image is kept: the R2 bucket's lifecycle rule deletes it then, so the query is searched again, and +// promote.js stops serving it as a static file +export const TTL_SECONDS = 90 * 24 * 60 * 60; + // Highest ?i= per site: 40 on the main site (100 Brave results; a Firecrawl query's ?i= past FIRECRAWL_RESULTS gets // an AI image), 20 on free.* (~50 Openverse + Wikimedia results) and 1000 on ai.* (each is a new image, so there's no // list to run out). diff --git a/src/worker.js b/src/worker.js index cfb439a..eb02c20 100644 --- a/src/worker.js +++ b/src/worker.js @@ -4,10 +4,9 @@ import { freeImageUrls, parseFreeSource, FREE_UA } from "./_utils/free.js"; import { isFlickrPlaceholder } from "./_utils/flickr.js"; import { aiImage, parseAiSource, DEFAULT_AI_SOURCE } from "./_utils/ai.js"; import { surreal } from "./_utils/surreal.js"; -import { MAX_INDEX, siteOf, folder, normalizeQuery, cacheKey as keyFor, r2Key as r2KeyFor, sha256 } from "./_utils/keys.js"; +import { TTL_SECONDS, MAX_INDEX, siteOf, folder, normalizeQuery, cacheKey as keyFor, r2Key as r2KeyFor, sha256 } from "./_utils/keys.js"; import * as LIMITS from "../public/assets/rate-limit.js"; -const TTL_SECONDS = 90 * 24 * 60 * 60; // How long a failed search keeps serving bad.webp before the query is searched again const ERR_SECONDS = 24 * 60 * 60; // How long a search may hold its query's lock; longer than the 20s image deadline plus the search itself