mirror of
https://github.com/direct-img/direct-img.link.git
synced 2026-10-29 07:24:52 +00:00
Feat: Move from Pages to a Worker and serve popular images as static files
Static assets don't count toward the free plan's 100,000 requests a day, so a daily GitHub Action promotes the most viewed images (by Cloudflare analytics) from R2 into public/ before deploying, and reports to ntfy. Zone rules fold the site and ?i= into the path, since static files ignore query strings. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -0,0 +1,50 @@
|
||||
// Sends the deploy's outcome to ntfy: what got promoted, how close the account is to the free plan's limits, and what
|
||||
// to do when something needs attention. Runs last in .github/workflows/deploy.yml, even when an earlier step failed.
|
||||
import { readFile } from "node:fs/promises";
|
||||
|
||||
const env = process.env;
|
||||
const r = JSON.parse(await readFile("promoted/report.json", "utf8").catch(() => "{}"));
|
||||
const fmt = n => Math.round(n).toLocaleString("en-US");
|
||||
const gb = b => `${(b / 1e9).toFixed(2)} GB`;
|
||||
const pct = (a, b) => `${Math.round((100 * a) / b)}%`;
|
||||
const lines = [], actions = [];
|
||||
|
||||
if (env.STATUS !== "success") {
|
||||
actions.push(`${env.FAILED_STEP || "A step"} failed, so the site still runs the previous deploy. Open the run for the error.`);
|
||||
}
|
||||
if (r.ok) {
|
||||
lines.push(`Static images: ${fmt(r.promoted)} / ${fmt(r.maxFiles)} slots, ${gb(r.bytes)} / ${gb(r.maxBytes)}`);
|
||||
lines.push(`+${fmt(r.added)} promoted, −${fmt(r.removed)} dropped (min ${r.minViews} views in ${r.days} days)`);
|
||||
if (r.views?.total) lines.push(`They got ${pct(r.promotedViews, r.views.total)} of ${fmt(r.views.total)} image views in ${r.days} days`);
|
||||
// web/1/orange+cat → orange cat, free/2/orange+cat → orange cat i=2 [free]
|
||||
const label = id => id.replace(/^(\w+)\/(\d+)\/(.*)$/, (_, f, i, q) => `${q.replaceAll("+", " ")}${i > 1 ? ` i=${i}` : ""}${f !== "web" ? ` [${f}]` : ""}`);
|
||||
if (r.top?.length) lines.push(`Top: ${r.top.map(t => `${label(t.id)} (${fmt(t.views)})`).join(", ")}`);
|
||||
if (r.notFitted) actions.push(`All ${fmt(r.maxFiles)} static slots are full and ~${fmt(r.notFitted)} more images qualified. Workers Paid ($5/mo) allows 100,000 files.`);
|
||||
if (r.skipped.budget) actions.push(`Size budget full: ${fmt(r.skipped.budget)} images left out. Raise MAX_BYTES in deploy.yml (runner disk ~14 GB, Actions cache 10 GB).`);
|
||||
if (r.skipped.failed) actions.push(`${fmt(r.skipped.failed)} downloads from R2 failed, e.g. ${r.firstError}. They're retried next run.`);
|
||||
if (r.analyticsError) actions.push(`Couldn't read Cloudflare analytics (${r.analyticsError}), so the last run's images were kept. Check the API token's Analytics permission.`);
|
||||
}
|
||||
if (r.workers) {
|
||||
const { total, byScript } = r.workers;
|
||||
const top = Object.entries(byScript).sort((a, b) => b[1] - a[1]).slice(0, 3).map(([s, n]) => `${s} ${fmt(n)}`).join(", ");
|
||||
lines.push(`Worker requests yesterday: ${fmt(total)} / 100,000 (${pct(total, 1e5)})${top && `: ${top}`}`);
|
||||
if (total >= 8e4) actions.push(`Worker requests hit ${pct(total, 1e5)} of the free 100,000/day; past it, images that aren't static files break until 00:00 UTC. Lower MIN_VIEWS to promote more, or move to Workers Paid ($5/mo).`);
|
||||
}
|
||||
if (r.workersError) lines.push(`Couldn't read Worker usage: ${r.workersError}`);
|
||||
|
||||
const failed = env.STATUS !== "success";
|
||||
const body = [...lines, ...actions.map(a => `⚠ ${a}`)].join("\n") || "No report was written.";
|
||||
console.log(body);
|
||||
if (!env.NTFY_URL) process.exit(0);
|
||||
const res = await fetch(env.NTFY_URL.startsWith("http") ? env.NTFY_URL : `https://${env.NTFY_URL}`, {
|
||||
method: "POST",
|
||||
body,
|
||||
headers: {
|
||||
Title: failed ? "direct-img deploy failed" : actions.length ? "direct-img deploy needs attention" : `direct-img deployed (${env.TRIGGER || "manual"})`,
|
||||
Priority: failed ? "5" : actions.length ? "4" : "2",
|
||||
Tags: failed ? "rotating_light,x" : actions.length ? "warning" : "white_check_mark",
|
||||
...(env.RUN_URL && { Click: env.RUN_URL }),
|
||||
...(env.NTFY_TOKEN && { Authorization: `Bearer ${env.NTFY_TOKEN}` }),
|
||||
},
|
||||
});
|
||||
console.log(`ntfy: HTTP ${res.status}`);
|
||||
@@ -0,0 +1,173 @@
|
||||
// Copies the most viewed images from R2 into public/, so the next deploy serves them as static files: free, unlimited
|
||||
// and without running the worker. Views come from Cloudflare's analytics, which count static files too, so a promoted
|
||||
// image that stops being viewed drops out. Run before every deploy (.github/workflows/deploy.yml); writes
|
||||
// promoted/report.json for scripts/notify.js. promoted/ is kept between runs (Actions cache), so only new images download.
|
||||
import { mkdir, readFile, writeFile, link, rm, readdir } from "node:fs/promises";
|
||||
import { dirname } from "node:path";
|
||||
import { MAX_INDEX, siteOf, folder, normalizeQuery, cacheKey, r2Key } from "../src/_utils/keys.js";
|
||||
import { parseFreeSource } from "../src/_utils/free.js";
|
||||
import { parseAiSource } from "../src/_utils/ai.js";
|
||||
|
||||
const env = process.env;
|
||||
const ZONE = env.ZONE || "direct-img.link", BUCKET = "direct-img-store";
|
||||
const HOSTS = [ZONE, `free.${ZONE}`, `ai.${ZONE}`];
|
||||
// The free plan allows 20,000 static files per deploy; the rest is room for index.html, assets/ etc.
|
||||
const MAX_FILES = +(env.MAX_FILES || 19900);
|
||||
// Views an image needs over the last DAYS to be promoted
|
||||
const MIN_VIEWS = +(env.MIN_VIEWS || 20), DAYS = +(env.DAYS || 7);
|
||||
// Total size of promoted images, which the runner restores from cache (or downloads) and uploads on every deploy
|
||||
const MAX_BYTES = +(env.MAX_BYTES || 4e9);
|
||||
// Formats browsers detect from the bytes, since every promoted image is declared image/jpeg (public/_headers)
|
||||
const TYPES = ["image/jpeg", "image/jpg", "image/pjpeg", "image/png", "image/gif", "image/webp"];
|
||||
// Queries whose file name the rules or the asset lookup would change, so a promoted file would never match
|
||||
const BAD_NAME = /[+.%\/\\?#\x00-\x1f\x7f]/;
|
||||
const CACHE = "promoted", OUT = "public";
|
||||
const sleep = ms => new Promise(r => setTimeout(r, ms));
|
||||
|
||||
// Cloudflare's API, retrying rate limits (1,200 requests per 5 minutes) and server errors
|
||||
async function api(path, init = {}) {
|
||||
for (let attempt = 1; ; attempt++) {
|
||||
const res = await fetch(`https://api.cloudflare.com/client/v4${path}`, { ...init, headers: { Authorization: `Bearer ${env.CLOUDFLARE_API_TOKEN}`, "Content-Type": "application/json" } });
|
||||
if ((res.status !== 429 && res.status < 500) || attempt === 5) return res;
|
||||
await sleep((+res.headers.get("retry-after") || 2 ** attempt) * 1000);
|
||||
}
|
||||
}
|
||||
|
||||
async function gql(query) {
|
||||
const r = await (await api("/graphql", { method: "POST", body: JSON.stringify({ query: `{ viewer { ${query} } }` }) })).json();
|
||||
if (r.errors?.length) throw new Error(r.errors[0].message);
|
||||
return r.data.viewer;
|
||||
}
|
||||
|
||||
// The image a request was for: a folded path (/web/2/orange+cat) or a bare query (/orange+cat), which is ?i=1 since
|
||||
// the rules redirect every ?i= to its folded path. null for anything else (the homepage, assets, another site's folder).
|
||||
async function parse(host, path) {
|
||||
const site = siteOf(host), m = path.match(/^\/(?:(web|free|ai)\/(\d+)\/)?([^/.]+)$/);
|
||||
if (!m || (m[1] && m[1] !== folder(site))) return null;
|
||||
// A promoted image's +s come back as %2B from the asset redirect, while the rules lowercase a query's own %2b
|
||||
const i = +(m[2] || 1), query = normalizeQuery(m[1] ? m[3].replaceAll("%2B", "+") : m[3]);
|
||||
if (!query || query.length > 200 || i < 1 || i > MAX_INDEX[site]) return null;
|
||||
const name = query.replaceAll(" ", "+");
|
||||
if (BAD_NAME.test(query) || Buffer.byteLength(name) > 200) return null;
|
||||
const src = site === "free" ? parseFreeSource(new URLSearchParams()) : site === "ai" ? parseAiSource(new URLSearchParams()) : null;
|
||||
return { id: `${folder(site)}/${i}/${name}`, r2: await r2Key(site, cacheKey(site, src, i, query)) };
|
||||
}
|
||||
|
||||
// Views per image over the last DAYS: one query per host and day, since a query returns at most 10,000 rows. Only 200s
|
||||
// count, so a promoted image isn't counted again for the redirect that encodes its + as %2B.
|
||||
async function views(zoneTag) {
|
||||
const totals = new Map(), now = Date.now();
|
||||
let total = 0;
|
||||
for (const host of HOSTS) for (let d = 0; d < DAYS; d++) {
|
||||
const range = `datetime_geq: "${new Date(now - (d + 1) * 864e5).toISOString()}", datetime_lt: "${new Date(now - d * 864e5).toISOString()}"`;
|
||||
const { zones } = await gql(`zones(filter: {zoneTag: "${zoneTag}"}) { httpRequestsAdaptiveGroups(limit: 10000, orderBy: [count_DESC], filter: {${range}, clientRequestHTTPHost: "${host}", edgeResponseStatus: 200}) { count dimensions { clientRequestPath } } }`);
|
||||
for (const { count, dimensions } of zones[0].httpRequestsAdaptiveGroups) {
|
||||
const img = await parse(host, dimensions.clientRequestPath);
|
||||
if (!img) continue;
|
||||
total += count;
|
||||
totals.set(img.id, { ...img, views: (totals.get(img.id)?.views || 0) + count });
|
||||
}
|
||||
}
|
||||
return { list: [...totals.values()], total };
|
||||
}
|
||||
|
||||
// Requests per Worker over a UTC day, which is what the free plan's 100,000 a day counts. Every Pages project's Functions
|
||||
// count too, as one entry. (A limit over 100 silently returns nothing.)
|
||||
async function workerRequests(day) {
|
||||
const filter = `limit: 100, filter: {datetime_geq: "${day}", datetime_lt: "${new Date(Date.parse(day) + 864e5).toISOString()}"}) { sum { requests } dimensions { scriptName } }`;
|
||||
const [a] = (await gql(`accounts(filter: {accountTag: "${env.CLOUDFLARE_ACCOUNT_ID}"}) { w: workersInvocationsAdaptive(${filter} p: pagesFunctionsInvocationsAdaptiveGroups(${filter} }`)).accounts;
|
||||
const byScript = {};
|
||||
for (const { sum, dimensions } of a.w) byScript[dimensions.scriptName] = (byScript[dimensions.scriptName] || 0) + sum.requests;
|
||||
const pages = a.p.reduce((t, r) => t + r.sum.requests, 0);
|
||||
if (pages) byScript["Pages Functions"] = pages;
|
||||
return { total: Object.values(byScript).reduce((t, r) => t + r, 0), byScript };
|
||||
}
|
||||
|
||||
// The image from the last run's cache, or from R2. Skips a failed search's empty marker and formats browsers can't detect.
|
||||
async function fetchImage(img, prev) {
|
||||
const file = `${CACHE}/files/${img.id}`;
|
||||
if (prev[img.id] && await readFile(file).then(() => true, () => false)) return { ...prev[img.id], cached: true };
|
||||
const res = await api(`/accounts/${env.CLOUDFLARE_ACCOUNT_ID}/r2/buckets/${BUCKET}/objects/${img.r2}`);
|
||||
if (res.status === 404) return { skip: "missing" };
|
||||
if (!res.ok) return { skip: "failed", error: `${img.id}: HTTP ${res.status}` };
|
||||
const type = res.headers.get("content-type")?.split(";")[0].trim().toLowerCase(), buf = Buffer.from(await res.arrayBuffer());
|
||||
if (!buf.length) return { skip: "missing" };
|
||||
if (!TYPES.includes(type)) return { skip: "type" };
|
||||
if (buf.length > 25 * 2 ** 20) return { skip: "size" };
|
||||
await mkdir(dirname(file), { recursive: true });
|
||||
await writeFile(file, buf);
|
||||
return { bytes: buf.length, type };
|
||||
}
|
||||
|
||||
const report = { ok: false, minViews: MIN_VIEWS, days: DAYS, maxFiles: MAX_FILES, maxBytes: MAX_BYTES, skipped: {} };
|
||||
const prev = JSON.parse(await readFile(`${CACHE}/manifest.json`, "utf8").catch(() => "{}"));
|
||||
const yesterday = new Date(Date.now() - 864e5).toISOString().slice(0, 10) + "T00:00:00Z";
|
||||
|
||||
let ranked;
|
||||
try {
|
||||
const zone = (await (await api(`/zones?name=${ZONE}`)).json()).result[0].id;
|
||||
const { list, total } = await views(zone);
|
||||
ranked = list.filter(c => c.views >= MIN_VIEWS).sort((a, b) => b.views - a.views);
|
||||
report.views = { total };
|
||||
} catch (err) {
|
||||
// Keeps the last run's images rather than dropping them all because analytics had a bad moment
|
||||
report.analyticsError = err.message;
|
||||
ranked = Object.entries(prev).map(([id, p]) => ({ id, ...p })).sort((a, b) => b.views - a.views);
|
||||
}
|
||||
report.qualified = ranked.length;
|
||||
|
||||
try {
|
||||
report.workers = await workerRequests(yesterday);
|
||||
} catch (err) {
|
||||
report.workersError = err.message;
|
||||
}
|
||||
|
||||
// Most viewed first, until the slots or the size budget run out. 4 at a time keeps downloads under the API's rate limit.
|
||||
const chosen = {};
|
||||
let bytes = 0, n = 0;
|
||||
for (let k = 0; k < ranked.length; k += 4) {
|
||||
if (n >= MAX_FILES) {
|
||||
report.notFitted = ranked.length - k;
|
||||
break;
|
||||
}
|
||||
const batch = ranked.slice(k, k + 4), started = Date.now();
|
||||
const got = await Promise.all(batch.map(img => fetchImage(img, prev).catch(err => ({ skip: "failed", error: `${img.id}: ${err.message}` }))));
|
||||
got.forEach((g, j) => {
|
||||
const img = batch[j];
|
||||
if (g.skip) {
|
||||
report.skipped[g.skip] = (report.skipped[g.skip] || 0) + 1;
|
||||
report.firstError ||= g.error;
|
||||
} else if (n >= MAX_FILES) report.notFitted = (report.notFitted || 0) + 1;
|
||||
else if (bytes + g.bytes > MAX_BYTES) report.skipped.budget = (report.skipped.budget || 0) + 1;
|
||||
else {
|
||||
chosen[img.id] = { views: img.views, bytes: g.bytes, type: g.type, r2: img.r2 };
|
||||
bytes += g.bytes;
|
||||
n++;
|
||||
}
|
||||
});
|
||||
if (got.some(g => !g.cached)) await sleep(1200 - (Date.now() - started));
|
||||
}
|
||||
|
||||
// public/<folder>/ holds only this run's images, hard-linked from the cache so they aren't stored twice
|
||||
for (const f of ["web", "free", "ai"]) await rm(`${OUT}/${f}`, { recursive: true, force: true });
|
||||
for (const id of Object.keys(chosen)) {
|
||||
await mkdir(dirname(`${OUT}/${id}`), { recursive: true });
|
||||
await link(`${CACHE}/files/${id}`, `${OUT}/${id}`);
|
||||
}
|
||||
// Images no longer promoted leave the cache too, so it doesn't grow past the size budget
|
||||
for (const e of await readdir(`${CACHE}/files`, { recursive: true, withFileTypes: true }).catch(() => [])) {
|
||||
const id = `${e.parentPath.slice(`${CACHE}/files/`.length)}/${e.name}`;
|
||||
if (e.isFile() && !chosen[id]) await rm(`${CACHE}/files/${id}`);
|
||||
}
|
||||
await mkdir(CACHE, { recursive: true });
|
||||
await writeFile(`${CACHE}/manifest.json`, JSON.stringify(chosen));
|
||||
|
||||
const ids = Object.keys(chosen);
|
||||
Object.assign(report, {
|
||||
ok: true, promoted: ids.length, bytes,
|
||||
added: ids.filter(id => !prev[id]).length, removed: Object.keys(prev).filter(id => !chosen[id]).length,
|
||||
promotedViews: ids.reduce((a, id) => a + chosen[id].views, 0),
|
||||
top: ids.slice(0, 5).map(id => ({ id, views: chosen[id].views })),
|
||||
});
|
||||
await writeFile(`${CACHE}/report.json`, JSON.stringify(report, null, 2));
|
||||
console.log(JSON.stringify(report, null, 2));
|
||||
@@ -0,0 +1,38 @@
|
||||
// Sets the zone's rules that fold the site and ?i= into the path, so promoted images can be static files (which ignore
|
||||
// query strings): a URL Rewrite turns /orange+cat into /web/1/orange+cat, and a Redirect sends /orange+cat?i=2 to
|
||||
// /web/2/orange+cat, which also makes ?i= show up in analytics. Replaces only the rules it made earlier.
|
||||
// node scripts/rules.js direct-img.link free.direct-img.link ai.direct-img.link
|
||||
// With no hostnames it removes them. Needs CLOUDFLARE_API_TOKEN with Zone > Transform Rules and Single Redirect edit.
|
||||
import { siteOf, folder } from "../src/_utils/keys.js";
|
||||
|
||||
const env = process.env, hosts = process.argv.slice(2), ZONE = env.ZONE || "direct-img.link", TAG = "direct-img fold:";
|
||||
const api = (path, init) => fetch(`https://api.cloudflare.com/client/v4${path}`, { ...init, headers: { Authorization: `Bearer ${env.CLOUDFLARE_API_TOKEN}`, "Content-Type": "application/json" } }).then(r => r.json());
|
||||
// A bare query: not the homepage, assets/, robots.txt or an already folded path (which a redirect leads to). A literal +
|
||||
// (%2B) is left to the worker, since a promoted file's + stands for a space.
|
||||
const FLAT = 'http.request.uri.path ne "/" and not http.request.uri.path contains "." and not http.request.uri.path wildcard "/*/*" and not lower(http.request.uri.path) contains "%2b"';
|
||||
|
||||
const bySite = Object.groupBy(hosts, h => folder(siteOf(h)));
|
||||
const rules = {
|
||||
http_request_transform: Object.entries(bySite).map(([f, hs]) => ({
|
||||
description: `${TAG} ${f} bare query → /${f}/1/`,
|
||||
expression: `http.host in {${hs.map(h => JSON.stringify(h)).join(" ")}} and http.request.uri.query eq "" and ${FLAT}`,
|
||||
action: "rewrite",
|
||||
action_parameters: { uri: { path: { expression: `concat("/${f}/1", lower(http.request.uri.path))` } } },
|
||||
})),
|
||||
http_request_dynamic_redirect: Object.entries(bySite).map(([f, hs]) => ({
|
||||
description: `${TAG} ${f} ?i= → /${f}/<i>/`,
|
||||
expression: `http.host in {${hs.map(h => JSON.stringify(h)).join(" ")}} and http.request.uri.query strict wildcard "i=*" and not http.request.uri.query contains "&" and ${FLAT}`,
|
||||
action: "redirect",
|
||||
action_parameters: { from_value: { status_code: 302, preserve_query_string: false, target_url: { expression: `concat("https://", http.host, "/${f}/", http.request.uri.args["i"][0], lower(http.request.uri.path))` } } },
|
||||
})),
|
||||
};
|
||||
|
||||
const zone = (await api(`/zones?name=${ZONE}`)).result[0].id;
|
||||
for (const [phase, mine] of Object.entries(rules)) {
|
||||
const others = ((await api(`/zones/${zone}/rulesets/phases/${phase}/entrypoint`)).result?.rules || [])
|
||||
.filter(r => !r.description?.startsWith(TAG))
|
||||
.map(({ id, version, last_updated, ref, ...r }) => r);
|
||||
const res = await api(`/zones/${zone}/rulesets/phases/${phase}/entrypoint`, { method: "PUT", body: JSON.stringify({ rules: [...others, ...mine] }) });
|
||||
console.log(phase, res.success ? `${mine.length} set, ${others.length} others kept` : JSON.stringify(res.errors));
|
||||
if (!res.success) process.exitCode = 1;
|
||||
}
|
||||
Reference in New Issue
Block a user