diff --git a/.gitignore b/.gitignore index 56473d9..287c4dc 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,4 @@ node_modules/ .wrangler/ # Filled by scripts/promote.js before each deploy promoted/ -public/web/ -public/free/ -public/ai/ +public/*direct-img.link/ diff --git a/README.md b/README.md index 674244f..b82bb81 100644 --- a/README.md +++ b/README.md @@ -303,18 +303,20 @@ Every request runs the Worker, which counts toward the free plan's 100,000 reque - Views come from Cloudflare's zone analytics over the last 7 days, which count static files too, so a promoted image that stops being viewed drops out - The top 19,900 with at least `MIN_VIEWS` (20) are promoted, up to `MAX_BYTES` (4 GB) in total. The free plan allows 20,000 static files per deploy -- Only JPEG, PNG, GIF and WebP are promoted. Static files get their type from their extension, which these have none of, so `public/_headers` declares them all `image/jpeg`, and browsers detect these four formats from the bytes anyway +- Only JPEG, PNG, GIF and WebP are promoted, and only queries of up to 8 words (see below). Static files get their type from their extension, which these have none of, so `public/_headers` declares them all `image/jpeg`, and browsers detect these four formats from the bytes anyway - Images are cached between runs (Actions cache), so only newly promoted ones download from R2 - An image is dropped once it's 90 days old, the same as the R2 lifecycle rule, so a popular query (e.g. `current us president`) is still searched again instead of staying static forever. It's promoted again the next day if it's still popular -Static files ignore query strings, so a URL Rewrite Rule (`scripts/rules.js`) turns each site's bare queries into their static file's path first. The visitor's URL doesn't change: +Static files ignore query strings and are shared by all three hostnames, so URL Rewrite Rules (`scripts/rules.js`) turn each bare query into its static file's path, under its hostname, first. The visitor's URL doesn't change: | Request | Looked up as | |---|---| -| `direct-img.link/orange+cat` | `/web/1/orange%2Bcat` (lowercased, `+` encoded like the file's name) | -| `free.direct-img.link/Orange+Cat` | `/free/1/orange%2Bcat` | +| `direct-img.link/orange+cat` | `/direct-img.link/1/orange%2Bcat` (lowercased, `+` encoded like the file's name) | +| `free.direct-img.link/Orange+Cat` | `/free.direct-img.link/1/orange%2Bcat` | -A promoted file there serves it; otherwise the Worker gets the request, unfolds the path and handles it as before. Requests with any params (`?i=`, `?src=`), a literal `+` (`%2B`) or a trailing slash aren't rewritten and always go to the Worker. Analytics has no query string, so `?i=` views count toward the query's `?i=1`, and only `?i=1` is promoted. The free plan has no `regex_replace`, so the rule encodes up to 10 `+`s; a longer query's static file answers with one extra (free) redirect. +A promoted file there serves it; otherwise the Worker gets the request, unfolds the path and handles it as before. Requests with any params (`?i=`, `?src=`), a literal `+` (`%2B`) or a trailing slash aren't rewritten and always go to the Worker. Analytics has no query string, so `?i=` views count toward the query's `?i=1`, and only `?i=1` is promoted. + +A rule's condition sees the original path, and the free plan has no `regex_replace` and allows one `wildcard_replace` per rule, so there's one rule per number of `+`s (0–7, 8 of the plan's 10). Queries of 9+ words are never promoted. ### KV: `DIRECT_IMG_CACHE` diff --git a/public/_headers b/public/_headers index 6f2fa00..4fda655 100644 --- a/public/_headers +++ b/public/_headers @@ -1,14 +1,14 @@ # Promoted images (scripts/promote.js) have no file extension, so they'd go out without a Content-Type. Browsers detect # JPEG, PNG, GIF and WebP from the bytes whatever type is declared, so promote.js promotes only those. -/web/* +/direct-img.link/* Content-Type: image/jpeg Cache-Control: public, max-age=604800 Access-Control-Allow-Origin: * -/free/* +/free.direct-img.link/* Content-Type: image/jpeg Cache-Control: public, max-age=604800 Access-Control-Allow-Origin: * -/ai/* +/ai.direct-img.link/* Content-Type: image/jpeg Cache-Control: public, max-age=604800 Access-Control-Allow-Origin: * diff --git a/scripts/notify.js b/scripts/notify.js index 2fd32fb..b8cb684 100644 --- a/scripts/notify.js +++ b/scripts/notify.js @@ -31,11 +31,10 @@ if (r.workers) { } if (r.workersError) lines.push(`**Worker requests:** couldn't read (${esc(r.workersError)})`); -// web/2/orange+cat → [orange cat](https://direct-img.link/orange+cat?i=2), so each opens on its site -const zone = env.ZONE || "direct-img.link"; +// free.direct-img.link/1/orange+cat → [orange cat](https://free.direct-img.link/orange+cat) (free), so each opens on its site const link = id => { - const [f, i, name] = id.split("/"); - return `[${esc(name.replaceAll("+", " "))}](https://${f === "web" ? "" : `${f}.`}${zone}/${name}${i > 1 ? `?i=${i}` : ""})${i > 1 ? ` i=${i}` : ""}${f === "web" ? "" : ` (${f})`}`; + const [host, , name] = id.split("/"), site = host.split(".").length > 2 && host.split(".")[0]; + return `[${esc(name.replaceAll("+", " "))}](https://${host}/${name})${site ? ` (${site})` : ""}`; }; const failed = env.STATUS !== "success"; diff --git a/scripts/promote.js b/scripts/promote.js index 7c64aed..3672db5 100644 --- a/scripts/promote.js +++ b/scripts/promote.js @@ -4,7 +4,7 @@ // promoted/report.json for scripts/notify.js. promoted/ is kept between runs (Actions cache), so only new images download. import { mkdir, readFile, writeFile, link, rm, readdir } from "node:fs/promises"; import { dirname } from "node:path"; -import { TTL_SECONDS, siteOf, folder, normalizeQuery, cacheKey, r2Key } from "../src/_utils/keys.js"; +import { TTL_SECONDS, MAX_PLUSES, siteOf, normalizeQuery, cacheKey, r2Key } from "../src/_utils/keys.js"; import { parseFreeSource } from "../src/_utils/free.js"; import { parseAiSource } from "../src/_utils/ai.js"; @@ -46,9 +46,9 @@ async function parse(host, path) { const site = siteOf(host), m = path.match(/^\/([^/.]+)$/), query = m && normalizeQuery(m[1]); if (!query || query.length > 200) return null; const name = query.replaceAll(" ", "+"); - if (BAD_NAME.test(query) || Buffer.byteLength(name) > 200) return null; + if (BAD_NAME.test(query) || Buffer.byteLength(name) > 200 || name.split("+").length > MAX_PLUSES + 1) return null; const src = site === "free" ? parseFreeSource(new URLSearchParams()) : site === "ai" ? parseAiSource(new URLSearchParams()) : null; - return { id: `${folder(site)}/1/${name}`, r2: await r2Key(site, cacheKey(site, src, 1, query)) }; + return { id: `${host}/1/${name}`, r2: await r2Key(site, cacheKey(site, src, 1, query)) }; } // Views per image over the last DAYS: one query per host and day, since a query returns at most 10,000 rows. Only 200s @@ -150,8 +150,8 @@ for (let k = 0; k < ranked.length; k += 4) { if (got.some(g => !g.cached)) await sleep(1200 - (Date.now() - started)); } -// public// holds only this run's images, hard-linked from the cache so they aren't stored twice -for (const f of ["web", "free", "ai"]) await rm(`${OUT}/${f}`, { recursive: true, force: true }); +// public// holds only this run's images, hard-linked from the cache so they aren't stored twice +for (const h of HOSTS) await rm(`${OUT}/${h}`, { recursive: true, force: true }); for (const id of Object.keys(chosen)) { await mkdir(dirname(`${OUT}/${id}`), { recursive: true }); await link(`${CACHE}/files/${id}`, `${OUT}/${id}`); diff --git a/scripts/rules.js b/scripts/rules.js index 7967c07..088e010 100644 --- a/scripts/rules.js +++ b/scripts/rules.js @@ -1,25 +1,27 @@ -// Sets the zone's URL Rewrite Rules that turn a bare query into its site's static file path (/orange+cat → -// /web/1/orange%2Bcat), so a promoted image is served without running the worker and the visitor's URL doesn't change. -// +s become %2B (the static file's own encoding), since a + would get a redirect to it. Replaces only rules it made. +// Sets the zone's URL Rewrite Rules that turn a bare query into its static file's path, under its hostname +// (free.direct-img.link/Orange+Cat → /free.direct-img.link/1/orange%2Bcat), so a promoted image is served without +// running the worker and the visitor's URL doesn't change. +s become %2B (the static file's own encoding), since a + +// would get a redirect to it. Replaces only the rules it made earlier. // node scripts/rules.js direct-img.link free.direct-img.link ai.direct-img.link // With no hostnames it removes them. Needs CLOUDFLARE_API_TOKEN with Zone > Transform Rules edit. -import { siteOf, folder } from "../src/_utils/keys.js"; +import { MAX_PLUSES } from "../src/_utils/keys.js"; const env = process.env, hosts = process.argv.slice(2), ZONE = env.ZONE || "direct-img.link", TAG = "direct-img fold:"; -const PHASE = "http_request_transform"; +const PHASE = "http_request_transform", PATH = "http.request.uri.path"; const api = (path, init) => fetch(`https://api.cloudflare.com/client/v4${path}`, { ...init, headers: { Authorization: `Bearer ${env.CLOUDFLARE_API_TOKEN}`, "Content-Type": "application/json" } }).then(r => r.json()); // A bare query with no params: not the homepage, assets/, robots.txt or an already folded path. A literal + (%2B) is // left to the worker, since a static file's %2B stands for a space. -const BARE = 'http.request.uri.query eq "" and http.request.uri.path ne "/" and not http.request.uri.path contains "." and not http.request.uri.path wildcard "/*/*" and not lower(http.request.uri.path) contains "%2b"'; -// Each wildcard_replace encodes one + (the free plan has no regex_replace); a query with more keeps a redirect -const path = Array.from({ length: 10 }).reduce(e => `wildcard_replace(${e}, "*+*", "\${1}%2B\${2}")`, "lower(http.request.uri.path)"); +const BARE = `http.host in {${hosts.map(h => JSON.stringify(h)).join(" ")}} and http.request.uri.query eq "" and ${PATH} ne "/" and not ${PATH} contains "." and not ${PATH} wildcard "/*/*" and not lower(${PATH}) contains "%2b"`; +const pluses = n => `"${"*+".repeat(n)}*"`; -const mine = Object.entries(Object.groupBy(hosts, h => folder(siteOf(h)))).map(([f, hs]) => ({ - description: `${TAG} ${f} bare query → /${f}/1/`, - expression: `http.host in {${hs.map(h => JSON.stringify(h)).join(" ")}} and ${BARE}`, +// A rule's condition sees the original path, and the free plan has no regex_replace and allows one wildcard_replace per +// rule, so each rule handles an exact number of +s; a query with more always runs the worker +const mine = hosts.length ? Array.from({ length: MAX_PLUSES + 1 }, (_, n) => ({ + description: `${TAG} bare query with ${n} +`, + expression: `${BARE} and ${n ? `${PATH} wildcard ${pluses(n)}` : `not ${PATH} contains "+"`} and not ${PATH} wildcard ${pluses(n + 1)}`, action: "rewrite", - action_parameters: { uri: { path: { expression: `concat("/${f}/1", ${path})` } } }, -})); + action_parameters: { uri: { path: { expression: `concat("/", http.host, "/1", ${n ? `wildcard_replace(lower(${PATH}), ${pluses(n)}, "${Array.from({ length: n }, (_, k) => `\${${k + 1}}%2B`).join("")}\${${n + 1}}")` : `lower(${PATH})`})` } } }, +})) : []; const zone = (await api(`/zones?name=${ZONE}`)).result[0].id; const others = ((await api(`/zones/${zone}/rulesets/phases/${PHASE}/entrypoint`)).result?.rules || []) diff --git a/src/_utils/keys.js b/src/_utils/keys.js index 48421c5..b78f02a 100644 --- a/src/_utils/keys.js +++ b/src/_utils/keys.js @@ -13,9 +13,8 @@ export const MAX_INDEX = { "": 40, free: 20, ai: 1000 }; // "free", "ai" or "" (main site), from the hostname's first label export const siteOf = hostname => ["free", "ai"].find(s => hostname.startsWith(`${s}.`)) || ""; -// Path prefix each site's ?i= is folded into by the zone's URL Rewrite and Redirect Rules (scripts/rules.js), e.g. -// /orange+cat?i=2 → /web/2/orange+cat, so promoted images can be static files, which ignore query strings -export const folder = site => site || "web"; +// Most +s in a query the zone's URL Rewrite Rules (scripts/rules.js) can turn into a static file's path, one rule each +export const MAX_PLUSES = 7; export function normalizeQuery(path) { const clean = s => s.toLowerCase().trim().replace(/[\x00-\x1f]/g, "").replace(/\/+$/, "").replace(/\s+/g, " "); diff --git a/src/worker.js b/src/worker.js index 3a9754f..ef97f90 100644 --- a/src/worker.js +++ b/src/worker.js @@ -4,7 +4,7 @@ import { freeImageUrls, parseFreeSource, FREE_UA } from "./_utils/free.js"; import { isFlickrPlaceholder } from "./_utils/flickr.js"; import { aiImage, parseAiSource, DEFAULT_AI_SOURCE } from "./_utils/ai.js"; import { surreal } from "./_utils/surreal.js"; -import { TTL_SECONDS, MAX_INDEX, siteOf, folder, normalizeQuery, cacheKey as keyFor, r2Key as r2KeyFor, sha256 } from "./_utils/keys.js"; +import { TTL_SECONDS, MAX_INDEX, siteOf, normalizeQuery, cacheKey as keyFor, r2Key as r2KeyFor, sha256 } from "./_utils/keys.js"; import * as LIMITS from "../public/assets/rate-limit.js"; // How long a failed search keeps serving bad.webp before the query is searched again @@ -365,13 +365,14 @@ async function countHit(env, request, query, site, params) { } catch {} } -// The zone's URL Rewrite (scripts/rules.js) turns a bare query into its site's static file path (/orange+cat → -// /web/1/orange%2Bcat) so a promoted image is served without running the worker. Unfolded back here, so the rest of the -// worker sees the original request: the rule's %2B were +s, since it leaves a query's own %2B unfolded. Other ?i= in the -// path are still accepted (links from when ?i= was folded too). A folded path for another site is rejected like any slash. +// The zone's URL Rewrite Rules (scripts/rules.js) turn a bare query into its static file's path, under its hostname +// (free.direct-img.link/orange+cat → /free.direct-img.link/1/orange%2Bcat), so a promoted image is served without +// running the worker. Unfolded back here, so the rest of the worker sees the original request: the rules' %2B were +s, +// since they leave a query's own %2B alone. Links from when ?i= was folded too (/free/2/orange+cat) still work. A folded +// path for another site is rejected like any slash. function unfold(url, site) { - const m = url.pathname.match(/^\/(web|free|ai)\/([^/]*)\/([^/]+)$/); - if (!m || m[1] !== folder(site)) return url; + const m = url.pathname.match(/^\/([^/]+)\/([^/]*)\/([^/]+)$/); + if (!m || ![url.hostname, site || "web"].includes(m[1])) return url; const u = new URL(url); u.pathname = `/${m[3].replace(/%2B/gi, "+")}`; if (m[2] && m[2] !== "1") u.searchParams.set("i", m[2]);