mirror of
https://github.com/direct-img/direct-img.link.git
synced 2026-10-29 07:24:52 +00:00
Feat: Promote every ?i= and ?src=, measured by a WAF logging rule
Analytics has no query string, so a WAF custom rule logs image requests with theirs and promote.js splits each path's views across its variants. The rewrite names static files after the param values (/orange%2Bcat/i2-), since the static file server redirects names with = or &. Drops the legacy /free/5/x paths and the /1/ segment. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
+5
-3
@@ -31,10 +31,12 @@ if (r.workers) {
|
||||
}
|
||||
if (r.workersError) lines.push(`**Worker requests:** couldn't read (${esc(r.workersError)})`);
|
||||
|
||||
// free.direct-img.link/1/orange+cat → [orange cat](https://free.direct-img.link/orange+cat) (free), so each opens on its site
|
||||
// free.direct-img.link/orange+cat/i2-wikimedia → [orange cat](https://free.direct-img.link/orange+cat?i=2&src=wikimedia)
|
||||
// i=2 src=wikimedia (free), so each opens on its site
|
||||
const link = id => {
|
||||
const [host, , name] = id.split("/"), site = host.split(".").length > 2 && host.split(".")[0];
|
||||
return `[${esc(name.replaceAll("+", " "))}](https://${host}/${name})${site ? ` (${site})` : ""}`;
|
||||
const [host, name, file] = id.split("/"), [, i, src] = file.match(/^i([^-]*)-(.*)$/), site = host.split(".").length > 2 && host.split(".")[0];
|
||||
const qs = new URLSearchParams(Object.entries({ i, src }).filter(([, v]) => v)).toString();
|
||||
return `[${esc(name.replaceAll("+", " "))}](https://${host}/${name}${qs && `?${qs}`})${qs && ` ${esc(qs.replace("&", " "))}`}${site ? ` (${site})` : ""}`;
|
||||
};
|
||||
|
||||
const failed = env.STATUS !== "success";
|
||||
|
||||
+44
-24
@@ -1,10 +1,11 @@
|
||||
// Copies the most viewed images from R2 into public/, so the next deploy serves them as static files: free, unlimited
|
||||
// and without running the worker. Views come from Cloudflare's analytics, which count static files too, so a promoted
|
||||
// image that stops being viewed drops out. Run before every deploy (.github/workflows/deploy.yml); writes
|
||||
// promoted/report.json for scripts/notify.js. promoted/ is kept between runs (Actions cache), so only new images download.
|
||||
// Copies the most viewed images (any ?i= and ?src=) from R2 into public/, so the next deploy serves them as static
|
||||
// files: free, unlimited and without running the worker. Views come from Cloudflare's analytics and the WAF rule's logs,
|
||||
// which count static files too, so a promoted image that stops being viewed drops out. Run before every deploy
|
||||
// (.github/workflows/deploy.yml); writes promoted/report.json for scripts/notify.js. promoted/ is kept between runs
|
||||
// (Actions cache), so only new images download.
|
||||
import { mkdir, readFile, writeFile, link, rm, readdir } from "node:fs/promises";
|
||||
import { dirname } from "node:path";
|
||||
import { TTL_SECONDS, MAX_PLUSES, siteOf, normalizeQuery, cacheKey, r2Key } from "../src/_utils/keys.js";
|
||||
import { TTL_SECONDS, MAX_INDEX, MAX_PLUSES, parseIndex, siteOf, normalizeQuery, cacheKey, r2Key } from "../src/_utils/keys.js";
|
||||
import { parseFreeSource } from "../src/_utils/free.js";
|
||||
import { parseAiSource } from "../src/_utils/ai.js";
|
||||
|
||||
@@ -21,6 +22,8 @@ const MAX_BYTES = +(env.MAX_BYTES || 4e9);
|
||||
const TYPES = ["image/jpeg", "image/jpg", "image/pjpeg", "image/png", "image/gif", "image/webp"];
|
||||
// Queries whose file name the rules or the asset lookup would change, so a promoted file would never match
|
||||
const BAD_NAME = /[+.%\/\\?#\x00-\x1f\x7f]/;
|
||||
// Logged requests read per hour, enough to split a busy path's views across its ?i= and ?src=
|
||||
const PER_HOUR = 1000;
|
||||
const CACHE = "promoted", OUT = "public";
|
||||
const sleep = ms => new Promise(r => setTimeout(r, ms));
|
||||
|
||||
@@ -39,31 +42,48 @@ async function gql(query) {
|
||||
return r.data.viewer;
|
||||
}
|
||||
|
||||
// The image a request for a bare query (/orange+cat) was for. Analytics has no query string, so a ?i= view counts toward
|
||||
// its query's ?i=1 too, and only ?i=1 is promoted (other ?i= always run the worker). null for anything else (the
|
||||
// homepage, assets, probes like /wp-admin/x.php).
|
||||
async function parse(host, path) {
|
||||
const site = siteOf(host), m = path.match(/^\/([^/.]+)$/), query = m && normalizeQuery(m[1]);
|
||||
if (!query || query.length > 200) return null;
|
||||
const name = query.replaceAll(" ", "+");
|
||||
if (BAD_NAME.test(query) || Buffer.byteLength(name) > 200 || name.split("+").length > MAX_PLUSES + 1) return null;
|
||||
const src = site === "free" ? parseFreeSource(new URLSearchParams()) : site === "ai" ? parseAiSource(new URLSearchParams()) : null;
|
||||
return { id: `${host}/1/${name}`, r2: await r2Key(site, cacheKey(site, src, 1, query)) };
|
||||
// The image a request was for, from its path (/orange+cat) and query string (?i=2&src=wikimedia), checked like the
|
||||
// worker checks them. Its id is its static file's path, which the rules build the same way
|
||||
// (free.direct-img.link/orange+cat/i2-wikimedia). null for anything else (the homepage, assets, probes like
|
||||
// /wp-admin/x.php, params the worker rejects).
|
||||
async function image(host, path, qs) {
|
||||
const site = siteOf(host), m = path.match(/^\/([^/.]+)$/), query = m && normalizeQuery(m[1]), p = new URLSearchParams(qs);
|
||||
if (!query || query.length > 200 || [...p.keys()].some(k => k !== "i" && !(site && k === "src"))) return null;
|
||||
const i = parseIndex(p), src = site === "free" ? parseFreeSource(p) : site === "ai" ? parseAiSource(p) : null;
|
||||
if (!i || i > MAX_INDEX[site] || (site && !src)) return null;
|
||||
const name = query.replaceAll(" ", "+"), file = `i${p.get("i") ?? ""}-${p.get("src") ?? ""}`;
|
||||
if (BAD_NAME.test(query) || /[^\w-]/.test(file) || Buffer.byteLength(name) > 200 || name.split("+").length > MAX_PLUSES + 1) return null;
|
||||
return { id: `${host}/${name}/${file}`, r2: await r2Key(site, cacheKey(site, src, i, query)) };
|
||||
}
|
||||
|
||||
// Views per image over the last DAYS: one query per host and day, since a query returns at most 10,000 rows. Only 200s
|
||||
// count, so redirects (e.g. /favicon.ico) aren't.
|
||||
// Views per image over the last DAYS. Analytics counts requests per path (one query per host and day, at most 10,000
|
||||
// rows each; only 200s, so redirects like /favicon.ico aren't) but has no query string, so each path's count is split
|
||||
// across its ?i= and ?src= by that day's logged requests (the WAF rule from scripts/rules.js), read an hour at a time
|
||||
// so a busy day is sampled evenly. A path with none logged that day counts as the bare query.
|
||||
async function views(zoneTag) {
|
||||
const totals = new Map(), now = Date.now();
|
||||
const now = Date.now(), at = h => new Date(now - h * 36e5).toISOString();
|
||||
const logged = Array.from({ length: DAYS }, () => new Map());
|
||||
for (let h = 0; h < DAYS * 24; h++) {
|
||||
const { zones } = await gql(`zones(filter: {zoneTag: "${zoneTag}"}) { firewallEventsAdaptive(limit: ${PER_HOUR}, filter: {datetime_geq: "${at(h + 1)}", datetime_lt: "${at(h)}", action: "skip", source: "firewallCustom"}) { clientRequestHTTPHost clientRequestPath clientRequestQuery sampleInterval } }`);
|
||||
for (const e of zones[0].firewallEventsAdaptive) {
|
||||
const day = logged[Math.floor(h / 24)], key = e.clientRequestHTTPHost + e.clientRequestPath;
|
||||
const qs = day.get(key) || day.set(key, new Map()).get(key);
|
||||
qs.set(e.clientRequestQuery, (qs.get(e.clientRequestQuery) || 0) + e.sampleInterval);
|
||||
}
|
||||
}
|
||||
const totals = new Map();
|
||||
let total = 0;
|
||||
for (const host of HOSTS) for (let d = 0; d < DAYS; d++) {
|
||||
const range = `datetime_geq: "${new Date(now - (d + 1) * 864e5).toISOString()}", datetime_lt: "${new Date(now - d * 864e5).toISOString()}"`;
|
||||
const range = `datetime_geq: "${at((d + 1) * 24)}", datetime_lt: "${at(d * 24)}"`;
|
||||
const { zones } = await gql(`zones(filter: {zoneTag: "${zoneTag}"}) { httpRequestsAdaptiveGroups(limit: 10000, orderBy: [count_DESC], filter: {${range}, clientRequestHTTPHost: "${host}", edgeResponseStatus: 200}) { count dimensions { clientRequestPath } } }`);
|
||||
for (const { count, dimensions } of zones[0].httpRequestsAdaptiveGroups) {
|
||||
const img = await parse(host, dimensions.clientRequestPath);
|
||||
if (!img) continue;
|
||||
total += count;
|
||||
totals.set(img.id, { ...img, views: (totals.get(img.id)?.views || 0) + count });
|
||||
for (const { count, dimensions: { clientRequestPath: path } } of zones[0].httpRequestsAdaptiveGroups) {
|
||||
const qs = logged[d].get(host + path) || new Map([["", 1]]), weight = [...qs.values()].reduce((a, b) => a + b, 0);
|
||||
for (const [q, w] of qs) {
|
||||
const img = await image(host, path, q);
|
||||
if (!img) continue;
|
||||
total += (count * w) / weight;
|
||||
totals.set(img.id, { ...img, views: (totals.get(img.id)?.views || 0) + (count * w) / weight });
|
||||
}
|
||||
}
|
||||
}
|
||||
return { list: [...totals.values()], total };
|
||||
|
||||
+40
-23
@@ -1,32 +1,49 @@
|
||||
// Sets the zone's URL Rewrite Rules that turn a bare query into its static file's path, under its hostname
|
||||
// (free.direct-img.link/Orange+Cat → /free.direct-img.link/1/orange%2Bcat), so a promoted image is served without
|
||||
// running the worker and the visitor's URL doesn't change. +s become %2B (the static file's own encoding), since a +
|
||||
// would get a redirect to it. Replaces only the rules it made earlier.
|
||||
// Sets the zone's rules for static images, replacing only the ones it made earlier:
|
||||
// - URL Rewrite Rules give each image request a static file path under its hostname, named after its ?i= and ?src=
|
||||
// (free.direct-img.link/Orange+Cat?i=2 → /free.direct-img.link/orange%2Bcat/i2-), so a promoted image is served
|
||||
// without running the worker and the visitor's URL doesn't change. +s become %2B and the params skip = and &, since
|
||||
// the static file server redirects to those encodings otherwise.
|
||||
// - A WAF custom rule logs every image request with its query string (analytics has no query string), which is how
|
||||
// scripts/promote.js tells ?i= and ?src= apart. It's a skip that skips nothing: it's kept last in its ruleset.
|
||||
// node scripts/rules.js direct-img.link free.direct-img.link ai.direct-img.link
|
||||
// With no hostnames it removes them. Needs CLOUDFLARE_API_TOKEN with Zone > Transform Rules edit.
|
||||
// With no hostnames it removes them. Needs CLOUDFLARE_API_TOKEN with Zone > Transform Rules and Zone WAF edit.
|
||||
import { MAX_PLUSES } from "../src/_utils/keys.js";
|
||||
|
||||
const env = process.env, hosts = process.argv.slice(2), ZONE = env.ZONE || "direct-img.link", TAG = "direct-img fold:";
|
||||
const PHASE = "http_request_transform", PATH = "http.request.uri.path";
|
||||
const PATH = "http.request.uri.path", ARG = n => `http.request.uri.args["${n}"][0]`;
|
||||
const api = (path, init) => fetch(`https://api.cloudflare.com/client/v4${path}`, { ...init, headers: { Authorization: `Bearer ${env.CLOUDFLARE_API_TOKEN}`, "Content-Type": "application/json" } }).then(r => r.json());
|
||||
// A bare query with no params: not the homepage, assets/, robots.txt or an already folded path. A literal + (%2B) is
|
||||
// left to the worker, since a static file's %2B stands for a space.
|
||||
const BARE = `http.host in {${hosts.map(h => JSON.stringify(h)).join(" ")}} and http.request.uri.query eq "" and ${PATH} ne "/" and not ${PATH} contains "." and not ${PATH} wildcard "/*/*" and not lower(${PATH}) contains "%2b"`;
|
||||
// An image request: not the homepage, assets/, robots.txt, probes like /wp-admin/x.php or an already rewritten path
|
||||
const IMAGE = `http.host in {${hosts.map(h => JSON.stringify(h)).join(" ")}} and ${PATH} ne "/" and not ${PATH} contains "." and not ${PATH} wildcard "/*/*"`;
|
||||
// Folded only with no params but i and src (others get bad.webp from the worker), and without a literal + (%2B), since
|
||||
// a static file's %2B stands for a space
|
||||
const FOLDABLE = `${IMAGE} and (http.request.uri.query eq "" or all(http.request.uri.args.names[*] in {"i" "src"})) and not lower(${PATH}) contains "%2b"`;
|
||||
const pluses = n => `"${"*+".repeat(n)}*"`;
|
||||
|
||||
// A rule's condition sees the original path, and the free plan has no regex_replace and allows one wildcard_replace per
|
||||
// rule, so each rule handles an exact number of +s; a query with more always runs the worker
|
||||
const mine = hosts.length ? Array.from({ length: MAX_PLUSES + 1 }, (_, n) => ({
|
||||
description: `${TAG} bare query with ${n} +`,
|
||||
expression: `${BARE} and ${n ? `${PATH} wildcard ${pluses(n)}` : `not ${PATH} contains "+"`} and not ${PATH} wildcard ${pluses(n + 1)}`,
|
||||
action: "rewrite",
|
||||
action_parameters: { uri: { path: { expression: `concat("/", http.host, "/1", ${n ? `wildcard_replace(lower(${PATH}), ${pluses(n)}, "${Array.from({ length: n }, (_, k) => `\${${k + 1}}%2B`).join("")}\${${n + 1}}")` : `lower(${PATH})`})` } } },
|
||||
})) : [];
|
||||
const rules = {
|
||||
// A rule's condition sees the original path, and the free plan has no regex_replace and allows one wildcard_replace
|
||||
// per rule, so each rule handles an exact number of +s; a query with more always runs the worker. A missing param is
|
||||
// empty, so a bare query is i-.
|
||||
http_request_transform: hosts.length ? Array.from({ length: MAX_PLUSES + 1 }, (_, n) => ({
|
||||
description: `${TAG} query with ${n} +`,
|
||||
expression: `${FOLDABLE} and ${n ? `${PATH} wildcard ${pluses(n)}` : `not ${PATH} contains "+"`} and not ${PATH} wildcard ${pluses(n + 1)}`,
|
||||
action: "rewrite",
|
||||
action_parameters: { uri: { path: { expression: `concat("/", http.host, ${n ? `wildcard_replace(lower(${PATH}), ${pluses(n)}, "${Array.from({ length: n }, (_, k) => `\${${k + 1}}%2B`).join("")}\${${n + 1}}")` : `lower(${PATH})`}, "/i", ${ARG("i")}, "-", ${ARG("src")})` } } },
|
||||
})) : [],
|
||||
http_request_firewall_custom: hosts.length ? [{
|
||||
description: `${TAG} log image requests`,
|
||||
expression: IMAGE,
|
||||
action: "skip",
|
||||
action_parameters: { ruleset: "current" },
|
||||
logging: { enabled: true },
|
||||
}] : [],
|
||||
};
|
||||
|
||||
const zone = (await api(`/zones?name=${ZONE}`)).result[0].id;
|
||||
const others = ((await api(`/zones/${zone}/rulesets/phases/${PHASE}/entrypoint`)).result?.rules || [])
|
||||
.filter(r => !r.description?.startsWith(TAG))
|
||||
.map(({ id, version, last_updated, ref, ...r }) => r);
|
||||
const res = await api(`/zones/${zone}/rulesets/phases/${PHASE}/entrypoint`, { method: "PUT", body: JSON.stringify({ rules: [...others, ...mine] }) });
|
||||
console.log(res.success ? `${mine.length} set, ${others.length} others kept` : JSON.stringify(res.errors));
|
||||
if (!res.success) process.exitCode = 1;
|
||||
for (const [phase, mine] of Object.entries(rules)) {
|
||||
const others = ((await api(`/zones/${zone}/rulesets/phases/${phase}/entrypoint`)).result?.rules || [])
|
||||
.filter(r => !r.description?.startsWith(TAG))
|
||||
.map(({ id, version, last_updated, ref, ...r }) => r);
|
||||
const res = await api(`/zones/${zone}/rulesets/phases/${phase}/entrypoint`, { method: "PUT", body: JSON.stringify({ rules: [...others, ...mine] }) });
|
||||
console.log(phase, res.success ? `${mine.length} set, ${others.length} others kept` : JSON.stringify(res.errors));
|
||||
if (!res.success) process.exitCode = 1;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user