diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..600ce2e --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,72 @@ +# Deploys the worker on every push to main and once a day. Each run first promotes the most viewed images to static +# files (scripts/promote.js) and ends with an ntfy message on what changed and anything that needs attention. +name: Deploy + +on: + push: + branches: [main] + schedule: + # Just after 00:00 UTC, when the free plan's daily request count resets, so the report covers a full day + - cron: "23 0 * * *" + workflow_dispatch: + +concurrency: + group: deploy + +permissions: + contents: read + actions: write # deletes the previous run's image cache + +jobs: + deploy: + runs-on: ubuntu-latest + timeout-minutes: 120 + env: + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + # Promotion limits (see scripts/promote.js) + MIN_VIEWS: 20 + MAX_BYTES: 4000000000 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 22 + + # Last run's promoted images, so only newly promoted ones download from R2 + - uses: actions/cache/restore@v4 + with: + path: promoted + key: promoted-${{ github.run_id }} + restore-keys: promoted- + + - id: promote + run: node scripts/promote.js + + - id: deploy + run: npx -y wrangler@4 deploy + + - if: steps.promote.outcome == 'success' + uses: actions/cache/save@v4 + with: + path: promoted + key: promoted-${{ github.run_id }} + + # Each run saves a new cache, so older ones go to stay under the repo's 10 GB + - if: steps.promote.outcome == 'success' + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + run: gh cache list --key promoted- --json id,key --jq '.[] | select(.key != "promoted-${{ github.run_id }}") | .id' | xargs -r -n1 gh cache delete + + - if: always() + env: + NTFY_URL: ${{ secrets.NTFY_URL }} + NTFY_TOKEN: ${{ secrets.NTFY_TOKEN }} + STATUS: ${{ job.status }} + FAILED_STEP: ${{ steps.promote.outcome == 'failure' && 'Promoting images' || steps.deploy.outcome == 'failure' && 'wrangler deploy' || '' }} + TRIGGER: ${{ github.event_name }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: node scripts/notify.js diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..56473d9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +node_modules/ +.wrangler/ +# Filled by scripts/promote.js before each deploy +promoted/ +public/web/ +public/free/ +public/ai/ diff --git a/README.md b/README.md index f78d742..adb12ae 100644 --- a/README.md +++ b/README.md @@ -209,7 +209,7 @@ curl -X POST -H "Content-Type: application/json" -d '{"name":"" 2. Click the verification link Openverse emails you (until then, anonymous limits apply) 3. Add them as the `OPENVERSE_CLIENT_ID` and `OPENVERSE_CLIENT_SECRET` secrets below -The function exchanges them for an access token (valid ~12h) and caches it in KV, fetching a new one automatically when it expires. +The worker exchanges them for an access token (valid ~12h) and caches it in KV, fetching a new one automatically when it expires. ### 4. Cloudflare Resources @@ -220,9 +220,9 @@ Create in your Cloudflare dashboard: | R2 Bucket | `direct-img-store` | Stores cached images | | KV Namespace | `DIRECT_IMG_CACHE` | Openverse and SurrealDB access tokens | -### 5. Pages Bindings +### 5. Worker Bindings -**Settings → Functions → Bindings:** +Set in `wrangler.jsonc` (put your KV namespace's ID there): | Type | Variable | Resource | |---|---|---| @@ -231,7 +231,7 @@ Create in your Cloudflare dashboard: ### 6. Secrets -**Settings → Environment variables:** +**Worker secrets** (`npx wrangler secret put `; `GOATCOUNTER_URL` is a plain var in `wrangler.jsonc`): | Variable | Description | Required | |---|---|---| @@ -256,11 +256,25 @@ Create in your Cloudflare dashboard: ### 8. Deploy -Fork this repo, connect to Cloudflare Pages, deploy. +Fork this repo and add these **GitHub Actions secrets** (Settings → Secrets and variables → Actions): -### 9. Free and AI Images Domains +| Secret | Description | +|---|---| +| `CLOUDFLARE_API_TOKEN` | API token with Account: Workers Scripts Edit, Workers R2 Storage Read, Account Analytics Read, and Zone (your domain): Workers Routes Edit, Analytics Read, Zone Read | +| `CLOUDFLARE_ACCOUNT_ID` | Your Cloudflare account ID | +| `NTFY_URL`, `NTFY_TOKEN` | Where each deploy's report goes (optional) | -**Pages project → Custom domains → Set up a domain:** add `free.` and `ai.`. Any hostname starting with `free.` serves free images, and any starting with `ai.` serves AI images; everything else serves regular search. +`.github/workflows/deploy.yml` deploys on every push to `main` and daily just after 00:00 UTC (also runnable by hand from the Actions tab). Each run promotes popular images first (see [Static Images](#static-images)) and ends with an ntfy message: how many images are static, how many Worker requests the account used yesterday out of the free 100,000, and what to do when something needs attention (failed deploy, requests near the limit, static slots or size budget full, analytics unreadable). + +### 9. Domains and Rules + +Change the three `routes` in `wrangler.jsonc` to your domain, `free.` and `ai.` (each needs a proxied DNS record). Any hostname starting with `free.` serves free images, and any starting with `ai.` serves AI images; everything else serves regular search. + +Then add the rules that let static images work (needs a token with Zone: Transform Rules Edit and Single Redirect Edit): + +```bash +CLOUDFLARE_API_TOKEN=... node scripts/rules.js free. ai. +``` --- @@ -283,6 +297,25 @@ An AI image served for an `?i=` past a search's working images is stored under t Add an object lifecycle rule in **R2 → direct-img-store → Settings** that applies to all prefixes and deletes uploaded objects after **90 days**. It's required: nothing else expires cached images. +### Static Images + +Every request runs the Worker, which counts toward the free plan's 100,000 requests a day, except requests for files in `public/`: those are free, unlimited and keep working past the limit. So before each deploy, `scripts/promote.js` copies the most viewed images from R2 into `public/`: + +- Views come from Cloudflare's zone analytics over the last 7 days, which count static files too, so a promoted image that stops being viewed drops out +- The top 19,900 with at least `MIN_VIEWS` (20) are promoted, up to `MAX_BYTES` (4 GB) in total. The free plan allows 20,000 static files per deploy +- Only JPEG, PNG, GIF and WebP are promoted. Static files get their type from their extension, which these have none of, so `public/_headers` declares them all `image/jpeg`, and browsers detect these four formats from the bytes anyway +- Images are cached between runs (Actions cache), so only newly promoted ones download from R2 + +Static files ignore query strings, so the zone's rules (`scripts/rules.js`) fold the site and `?i=` into the path first. The visitor's URL doesn't change: + +| Request | Looked up as | +|---|---| +| `direct-img.link/orange+cat` | `/web/1/orange+cat` (URL Rewrite, lowercased) | +| `direct-img.link/orange+cat?i=2` | 302 to `/web/2/orange+cat` (Redirect, so `?i=` shows up in analytics) | +| `free.direct-img.link/orange+cat` | `/free/1/orange+cat` | + +A promoted file named `/web/2/orange+cat` serves those; anything else falls through to the Worker, which unfolds the path and handles it as before. Requests with other params (`?src=`), a literal `+` (`%2B`) or a trailing slash aren't folded and always go to the Worker. Static files with a `+` in the name answer with one extra redirect (to `%2B`), which is still free. + ### KV: `DIRECT_IMG_CACHE` Only holds access tokens: @@ -308,7 +341,7 @@ After deploying, visit `https:///_setup` once. It creates the `dire ## Stack -- **Cloudflare Pages** — hosting + edge functions +- **Cloudflare Workers** — hosting, with popular images as static assets - **Cloudflare R2** — image storage - **Cloudflare KV** — API access tokens - **SurrealDB (v2.6.5)** — atomic rate limiting, search locks + cached result lists diff --git a/_routes.json b/_routes.json deleted file mode 100644 index b46035a..0000000 --- a/_routes.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "version": 1, - "include": ["/*"], - "exclude": ["/", "/index.html", "/assets/*"] -} diff --git a/public/_headers b/public/_headers new file mode 100644 index 0000000..6f2fa00 --- /dev/null +++ b/public/_headers @@ -0,0 +1,14 @@ +# Promoted images (scripts/promote.js) have no file extension, so they'd go out without a Content-Type. Browsers detect +# JPEG, PNG, GIF and WebP from the bytes whatever type is declared, so promote.js promotes only those. +/web/* + Content-Type: image/jpeg + Cache-Control: public, max-age=604800 + Access-Control-Allow-Origin: * +/free/* + Content-Type: image/jpeg + Cache-Control: public, max-age=604800 + Access-Control-Allow-Origin: * +/ai/* + Content-Type: image/jpeg + Cache-Control: public, max-age=604800 + Access-Control-Allow-Origin: * diff --git a/public/_redirects b/public/_redirects new file mode 100644 index 0000000..b459846 --- /dev/null +++ b/public/_redirects @@ -0,0 +1 @@ +/favicon.ico /assets/favicon.ico 301 diff --git a/assets/bad.webp b/public/assets/bad.webp similarity index 100% rename from assets/bad.webp rename to public/assets/bad.webp diff --git a/assets/donate.webp b/public/assets/donate.webp similarity index 100% rename from assets/donate.webp rename to public/assets/donate.webp diff --git a/assets/favicon.ico b/public/assets/favicon.ico similarity index 100% rename from assets/favicon.ico rename to public/assets/favicon.ico diff --git a/assets/limit.webp b/public/assets/limit.webp similarity index 100% rename from assets/limit.webp rename to public/assets/limit.webp diff --git a/assets/logo.jpg b/public/assets/logo.jpg similarity index 100% rename from assets/logo.jpg rename to public/assets/logo.jpg diff --git a/assets/logo.svg b/public/assets/logo.svg similarity index 100% rename from assets/logo.svg rename to public/assets/logo.svg diff --git a/assets/max-i.webp b/public/assets/max-i.webp similarity index 100% rename from assets/max-i.webp rename to public/assets/max-i.webp diff --git a/assets/rate-limit.js b/public/assets/rate-limit.js similarity index 100% rename from assets/rate-limit.js rename to public/assets/rate-limit.js diff --git a/index.html b/public/index.html similarity index 100% rename from index.html rename to public/index.html diff --git a/robots.txt b/public/robots.txt similarity index 100% rename from robots.txt rename to public/robots.txt diff --git a/scripts/notify.js b/scripts/notify.js new file mode 100644 index 0000000..7c0185f --- /dev/null +++ b/scripts/notify.js @@ -0,0 +1,50 @@ +// Sends the deploy's outcome to ntfy: what got promoted, how close the account is to the free plan's limits, and what +// to do when something needs attention. Runs last in .github/workflows/deploy.yml, even when an earlier step failed. +import { readFile } from "node:fs/promises"; + +const env = process.env; +const r = JSON.parse(await readFile("promoted/report.json", "utf8").catch(() => "{}")); +const fmt = n => Math.round(n).toLocaleString("en-US"); +const gb = b => `${(b / 1e9).toFixed(2)} GB`; +const pct = (a, b) => `${Math.round((100 * a) / b)}%`; +const lines = [], actions = []; + +if (env.STATUS !== "success") { + actions.push(`${env.FAILED_STEP || "A step"} failed, so the site still runs the previous deploy. Open the run for the error.`); +} +if (r.ok) { + lines.push(`Static images: ${fmt(r.promoted)} / ${fmt(r.maxFiles)} slots, ${gb(r.bytes)} / ${gb(r.maxBytes)}`); + lines.push(`+${fmt(r.added)} promoted, −${fmt(r.removed)} dropped (min ${r.minViews} views in ${r.days} days)`); + if (r.views?.total) lines.push(`They got ${pct(r.promotedViews, r.views.total)} of ${fmt(r.views.total)} image views in ${r.days} days`); + // web/1/orange+cat → orange cat, free/2/orange+cat → orange cat i=2 [free] + const label = id => id.replace(/^(\w+)\/(\d+)\/(.*)$/, (_, f, i, q) => `${q.replaceAll("+", " ")}${i > 1 ? ` i=${i}` : ""}${f !== "web" ? ` [${f}]` : ""}`); + if (r.top?.length) lines.push(`Top: ${r.top.map(t => `${label(t.id)} (${fmt(t.views)})`).join(", ")}`); + if (r.notFitted) actions.push(`All ${fmt(r.maxFiles)} static slots are full and ~${fmt(r.notFitted)} more images qualified. Workers Paid ($5/mo) allows 100,000 files.`); + if (r.skipped.budget) actions.push(`Size budget full: ${fmt(r.skipped.budget)} images left out. Raise MAX_BYTES in deploy.yml (runner disk ~14 GB, Actions cache 10 GB).`); + if (r.skipped.failed) actions.push(`${fmt(r.skipped.failed)} downloads from R2 failed, e.g. ${r.firstError}. They're retried next run.`); + if (r.analyticsError) actions.push(`Couldn't read Cloudflare analytics (${r.analyticsError}), so the last run's images were kept. Check the API token's Analytics permission.`); +} +if (r.workers) { + const { total, byScript } = r.workers; + const top = Object.entries(byScript).sort((a, b) => b[1] - a[1]).slice(0, 3).map(([s, n]) => `${s} ${fmt(n)}`).join(", "); + lines.push(`Worker requests yesterday: ${fmt(total)} / 100,000 (${pct(total, 1e5)})${top && `: ${top}`}`); + if (total >= 8e4) actions.push(`Worker requests hit ${pct(total, 1e5)} of the free 100,000/day; past it, images that aren't static files break until 00:00 UTC. Lower MIN_VIEWS to promote more, or move to Workers Paid ($5/mo).`); +} +if (r.workersError) lines.push(`Couldn't read Worker usage: ${r.workersError}`); + +const failed = env.STATUS !== "success"; +const body = [...lines, ...actions.map(a => `⚠ ${a}`)].join("\n") || "No report was written."; +console.log(body); +if (!env.NTFY_URL) process.exit(0); +const res = await fetch(env.NTFY_URL.startsWith("http") ? env.NTFY_URL : `https://${env.NTFY_URL}`, { + method: "POST", + body, + headers: { + Title: failed ? "direct-img deploy failed" : actions.length ? "direct-img deploy needs attention" : `direct-img deployed (${env.TRIGGER || "manual"})`, + Priority: failed ? "5" : actions.length ? "4" : "2", + Tags: failed ? "rotating_light,x" : actions.length ? "warning" : "white_check_mark", + ...(env.RUN_URL && { Click: env.RUN_URL }), + ...(env.NTFY_TOKEN && { Authorization: `Bearer ${env.NTFY_TOKEN}` }), + }, +}); +console.log(`ntfy: HTTP ${res.status}`); diff --git a/scripts/promote.js b/scripts/promote.js new file mode 100644 index 0000000..0223520 --- /dev/null +++ b/scripts/promote.js @@ -0,0 +1,173 @@ +// Copies the most viewed images from R2 into public/, so the next deploy serves them as static files: free, unlimited +// and without running the worker. Views come from Cloudflare's analytics, which count static files too, so a promoted +// image that stops being viewed drops out. Run before every deploy (.github/workflows/deploy.yml); writes +// promoted/report.json for scripts/notify.js. promoted/ is kept between runs (Actions cache), so only new images download. +import { mkdir, readFile, writeFile, link, rm, readdir } from "node:fs/promises"; +import { dirname } from "node:path"; +import { MAX_INDEX, siteOf, folder, normalizeQuery, cacheKey, r2Key } from "../src/_utils/keys.js"; +import { parseFreeSource } from "../src/_utils/free.js"; +import { parseAiSource } from "../src/_utils/ai.js"; + +const env = process.env; +const ZONE = env.ZONE || "direct-img.link", BUCKET = "direct-img-store"; +const HOSTS = [ZONE, `free.${ZONE}`, `ai.${ZONE}`]; +// The free plan allows 20,000 static files per deploy; the rest is room for index.html, assets/ etc. +const MAX_FILES = +(env.MAX_FILES || 19900); +// Views an image needs over the last DAYS to be promoted +const MIN_VIEWS = +(env.MIN_VIEWS || 20), DAYS = +(env.DAYS || 7); +// Total size of promoted images, which the runner restores from cache (or downloads) and uploads on every deploy +const MAX_BYTES = +(env.MAX_BYTES || 4e9); +// Formats browsers detect from the bytes, since every promoted image is declared image/jpeg (public/_headers) +const TYPES = ["image/jpeg", "image/jpg", "image/pjpeg", "image/png", "image/gif", "image/webp"]; +// Queries whose file name the rules or the asset lookup would change, so a promoted file would never match +const BAD_NAME = /[+.%\/\\?#\x00-\x1f\x7f]/; +const CACHE = "promoted", OUT = "public"; +const sleep = ms => new Promise(r => setTimeout(r, ms)); + +// Cloudflare's API, retrying rate limits (1,200 requests per 5 minutes) and server errors +async function api(path, init = {}) { + for (let attempt = 1; ; attempt++) { + const res = await fetch(`https://api.cloudflare.com/client/v4${path}`, { ...init, headers: { Authorization: `Bearer ${env.CLOUDFLARE_API_TOKEN}`, "Content-Type": "application/json" } }); + if ((res.status !== 429 && res.status < 500) || attempt === 5) return res; + await sleep((+res.headers.get("retry-after") || 2 ** attempt) * 1000); + } +} + +async function gql(query) { + const r = await (await api("/graphql", { method: "POST", body: JSON.stringify({ query: `{ viewer { ${query} } }` }) })).json(); + if (r.errors?.length) throw new Error(r.errors[0].message); + return r.data.viewer; +} + +// The image a request was for: a folded path (/web/2/orange+cat) or a bare query (/orange+cat), which is ?i=1 since +// the rules redirect every ?i= to its folded path. null for anything else (the homepage, assets, another site's folder). +async function parse(host, path) { + const site = siteOf(host), m = path.match(/^\/(?:(web|free|ai)\/(\d+)\/)?([^/.]+)$/); + if (!m || (m[1] && m[1] !== folder(site))) return null; + // A promoted image's +s come back as %2B from the asset redirect, while the rules lowercase a query's own %2b + const i = +(m[2] || 1), query = normalizeQuery(m[1] ? m[3].replaceAll("%2B", "+") : m[3]); + if (!query || query.length > 200 || i < 1 || i > MAX_INDEX[site]) return null; + const name = query.replaceAll(" ", "+"); + if (BAD_NAME.test(query) || Buffer.byteLength(name) > 200) return null; + const src = site === "free" ? parseFreeSource(new URLSearchParams()) : site === "ai" ? parseAiSource(new URLSearchParams()) : null; + return { id: `${folder(site)}/${i}/${name}`, r2: await r2Key(site, cacheKey(site, src, i, query)) }; +} + +// Views per image over the last DAYS: one query per host and day, since a query returns at most 10,000 rows. Only 200s +// count, so a promoted image isn't counted again for the redirect that encodes its + as %2B. +async function views(zoneTag) { + const totals = new Map(), now = Date.now(); + let total = 0; + for (const host of HOSTS) for (let d = 0; d < DAYS; d++) { + const range = `datetime_geq: "${new Date(now - (d + 1) * 864e5).toISOString()}", datetime_lt: "${new Date(now - d * 864e5).toISOString()}"`; + const { zones } = await gql(`zones(filter: {zoneTag: "${zoneTag}"}) { httpRequestsAdaptiveGroups(limit: 10000, orderBy: [count_DESC], filter: {${range}, clientRequestHTTPHost: "${host}", edgeResponseStatus: 200}) { count dimensions { clientRequestPath } } }`); + for (const { count, dimensions } of zones[0].httpRequestsAdaptiveGroups) { + const img = await parse(host, dimensions.clientRequestPath); + if (!img) continue; + total += count; + totals.set(img.id, { ...img, views: (totals.get(img.id)?.views || 0) + count }); + } + } + return { list: [...totals.values()], total }; +} + +// Requests per Worker over a UTC day, which is what the free plan's 100,000 a day counts. Every Pages project's Functions +// count too, as one entry. (A limit over 100 silently returns nothing.) +async function workerRequests(day) { + const filter = `limit: 100, filter: {datetime_geq: "${day}", datetime_lt: "${new Date(Date.parse(day) + 864e5).toISOString()}"}) { sum { requests } dimensions { scriptName } }`; + const [a] = (await gql(`accounts(filter: {accountTag: "${env.CLOUDFLARE_ACCOUNT_ID}"}) { w: workersInvocationsAdaptive(${filter} p: pagesFunctionsInvocationsAdaptiveGroups(${filter} }`)).accounts; + const byScript = {}; + for (const { sum, dimensions } of a.w) byScript[dimensions.scriptName] = (byScript[dimensions.scriptName] || 0) + sum.requests; + const pages = a.p.reduce((t, r) => t + r.sum.requests, 0); + if (pages) byScript["Pages Functions"] = pages; + return { total: Object.values(byScript).reduce((t, r) => t + r, 0), byScript }; +} + +// The image from the last run's cache, or from R2. Skips a failed search's empty marker and formats browsers can't detect. +async function fetchImage(img, prev) { + const file = `${CACHE}/files/${img.id}`; + if (prev[img.id] && await readFile(file).then(() => true, () => false)) return { ...prev[img.id], cached: true }; + const res = await api(`/accounts/${env.CLOUDFLARE_ACCOUNT_ID}/r2/buckets/${BUCKET}/objects/${img.r2}`); + if (res.status === 404) return { skip: "missing" }; + if (!res.ok) return { skip: "failed", error: `${img.id}: HTTP ${res.status}` }; + const type = res.headers.get("content-type")?.split(";")[0].trim().toLowerCase(), buf = Buffer.from(await res.arrayBuffer()); + if (!buf.length) return { skip: "missing" }; + if (!TYPES.includes(type)) return { skip: "type" }; + if (buf.length > 25 * 2 ** 20) return { skip: "size" }; + await mkdir(dirname(file), { recursive: true }); + await writeFile(file, buf); + return { bytes: buf.length, type }; +} + +const report = { ok: false, minViews: MIN_VIEWS, days: DAYS, maxFiles: MAX_FILES, maxBytes: MAX_BYTES, skipped: {} }; +const prev = JSON.parse(await readFile(`${CACHE}/manifest.json`, "utf8").catch(() => "{}")); +const yesterday = new Date(Date.now() - 864e5).toISOString().slice(0, 10) + "T00:00:00Z"; + +let ranked; +try { + const zone = (await (await api(`/zones?name=${ZONE}`)).json()).result[0].id; + const { list, total } = await views(zone); + ranked = list.filter(c => c.views >= MIN_VIEWS).sort((a, b) => b.views - a.views); + report.views = { total }; +} catch (err) { + // Keeps the last run's images rather than dropping them all because analytics had a bad moment + report.analyticsError = err.message; + ranked = Object.entries(prev).map(([id, p]) => ({ id, ...p })).sort((a, b) => b.views - a.views); +} +report.qualified = ranked.length; + +try { + report.workers = await workerRequests(yesterday); +} catch (err) { + report.workersError = err.message; +} + +// Most viewed first, until the slots or the size budget run out. 4 at a time keeps downloads under the API's rate limit. +const chosen = {}; +let bytes = 0, n = 0; +for (let k = 0; k < ranked.length; k += 4) { + if (n >= MAX_FILES) { + report.notFitted = ranked.length - k; + break; + } + const batch = ranked.slice(k, k + 4), started = Date.now(); + const got = await Promise.all(batch.map(img => fetchImage(img, prev).catch(err => ({ skip: "failed", error: `${img.id}: ${err.message}` })))); + got.forEach((g, j) => { + const img = batch[j]; + if (g.skip) { + report.skipped[g.skip] = (report.skipped[g.skip] || 0) + 1; + report.firstError ||= g.error; + } else if (n >= MAX_FILES) report.notFitted = (report.notFitted || 0) + 1; + else if (bytes + g.bytes > MAX_BYTES) report.skipped.budget = (report.skipped.budget || 0) + 1; + else { + chosen[img.id] = { views: img.views, bytes: g.bytes, type: g.type, r2: img.r2 }; + bytes += g.bytes; + n++; + } + }); + if (got.some(g => !g.cached)) await sleep(1200 - (Date.now() - started)); +} + +// public// holds only this run's images, hard-linked from the cache so they aren't stored twice +for (const f of ["web", "free", "ai"]) await rm(`${OUT}/${f}`, { recursive: true, force: true }); +for (const id of Object.keys(chosen)) { + await mkdir(dirname(`${OUT}/${id}`), { recursive: true }); + await link(`${CACHE}/files/${id}`, `${OUT}/${id}`); +} +// Images no longer promoted leave the cache too, so it doesn't grow past the size budget +for (const e of await readdir(`${CACHE}/files`, { recursive: true, withFileTypes: true }).catch(() => [])) { + const id = `${e.parentPath.slice(`${CACHE}/files/`.length)}/${e.name}`; + if (e.isFile() && !chosen[id]) await rm(`${CACHE}/files/${id}`); +} +await mkdir(CACHE, { recursive: true }); +await writeFile(`${CACHE}/manifest.json`, JSON.stringify(chosen)); + +const ids = Object.keys(chosen); +Object.assign(report, { + ok: true, promoted: ids.length, bytes, + added: ids.filter(id => !prev[id]).length, removed: Object.keys(prev).filter(id => !chosen[id]).length, + promotedViews: ids.reduce((a, id) => a + chosen[id].views, 0), + top: ids.slice(0, 5).map(id => ({ id, views: chosen[id].views })), +}); +await writeFile(`${CACHE}/report.json`, JSON.stringify(report, null, 2)); +console.log(JSON.stringify(report, null, 2)); diff --git a/scripts/rules.js b/scripts/rules.js new file mode 100644 index 0000000..8f9c83f --- /dev/null +++ b/scripts/rules.js @@ -0,0 +1,38 @@ +// Sets the zone's rules that fold the site and ?i= into the path, so promoted images can be static files (which ignore +// query strings): a URL Rewrite turns /orange+cat into /web/1/orange+cat, and a Redirect sends /orange+cat?i=2 to +// /web/2/orange+cat, which also makes ?i= show up in analytics. Replaces only the rules it made earlier. +// node scripts/rules.js direct-img.link free.direct-img.link ai.direct-img.link +// With no hostnames it removes them. Needs CLOUDFLARE_API_TOKEN with Zone > Transform Rules and Single Redirect edit. +import { siteOf, folder } from "../src/_utils/keys.js"; + +const env = process.env, hosts = process.argv.slice(2), ZONE = env.ZONE || "direct-img.link", TAG = "direct-img fold:"; +const api = (path, init) => fetch(`https://api.cloudflare.com/client/v4${path}`, { ...init, headers: { Authorization: `Bearer ${env.CLOUDFLARE_API_TOKEN}`, "Content-Type": "application/json" } }).then(r => r.json()); +// A bare query: not the homepage, assets/, robots.txt or an already folded path (which a redirect leads to). A literal + +// (%2B) is left to the worker, since a promoted file's + stands for a space. +const FLAT = 'http.request.uri.path ne "/" and not http.request.uri.path contains "." and not http.request.uri.path wildcard "/*/*" and not lower(http.request.uri.path) contains "%2b"'; + +const bySite = Object.groupBy(hosts, h => folder(siteOf(h))); +const rules = { + http_request_transform: Object.entries(bySite).map(([f, hs]) => ({ + description: `${TAG} ${f} bare query → /${f}/1/`, + expression: `http.host in {${hs.map(h => JSON.stringify(h)).join(" ")}} and http.request.uri.query eq "" and ${FLAT}`, + action: "rewrite", + action_parameters: { uri: { path: { expression: `concat("/${f}/1", lower(http.request.uri.path))` } } }, + })), + http_request_dynamic_redirect: Object.entries(bySite).map(([f, hs]) => ({ + description: `${TAG} ${f} ?i= → /${f}//`, + expression: `http.host in {${hs.map(h => JSON.stringify(h)).join(" ")}} and http.request.uri.query strict wildcard "i=*" and not http.request.uri.query contains "&" and ${FLAT}`, + action: "redirect", + action_parameters: { from_value: { status_code: 302, preserve_query_string: false, target_url: { expression: `concat("https://", http.host, "/${f}/", http.request.uri.args["i"][0], lower(http.request.uri.path))` } } }, + })), +}; + +const zone = (await api(`/zones?name=${ZONE}`)).result[0].id; +for (const [phase, mine] of Object.entries(rules)) { + const others = ((await api(`/zones/${zone}/rulesets/phases/${phase}/entrypoint`)).result?.rules || []) + .filter(r => !r.description?.startsWith(TAG)) + .map(({ id, version, last_updated, ref, ...r }) => r); + const res = await api(`/zones/${zone}/rulesets/phases/${phase}/entrypoint`, { method: "PUT", body: JSON.stringify({ rules: [...others, ...mine] }) }); + console.log(phase, res.success ? `${mine.length} set, ${others.length} others kept` : JSON.stringify(res.errors)); + if (!res.success) process.exitCode = 1; +} diff --git a/functions/_utils/ai.js b/src/_utils/ai.js similarity index 100% rename from functions/_utils/ai.js rename to src/_utils/ai.js diff --git a/functions/_utils/brave.js b/src/_utils/brave.js similarity index 100% rename from functions/_utils/brave.js rename to src/_utils/brave.js diff --git a/functions/_utils/firecrawl.js b/src/_utils/firecrawl.js similarity index 100% rename from functions/_utils/firecrawl.js rename to src/_utils/firecrawl.js diff --git a/functions/_utils/flickr.js b/src/_utils/flickr.js similarity index 100% rename from functions/_utils/flickr.js rename to src/_utils/flickr.js diff --git a/functions/_utils/free.js b/src/_utils/free.js similarity index 100% rename from functions/_utils/free.js rename to src/_utils/free.js diff --git a/src/_utils/keys.js b/src/_utils/keys.js new file mode 100644 index 0000000..210106e --- /dev/null +++ b/src/_utils/keys.js @@ -0,0 +1,33 @@ +// Shared by the worker and scripts/promote.js, so a promoted image's R2 key always matches the one the worker stored. + +// Highest ?i= per site: 40 on the main site (100 Brave results; a Firecrawl query's ?i= past FIRECRAWL_RESULTS gets +// an AI image), 20 on free.* (~50 Openverse + Wikimedia results) and 1000 on ai.* (each is a new image, so there's no +// list to run out). +// Higher values serve max-i.webp, which lists these, so redraw it from design/max-i.html when they change. +export const MAX_INDEX = { "": 40, free: 20, ai: 1000 }; + +// "free", "ai" or "" (main site), from the hostname's first label +export const siteOf = hostname => ["free", "ai"].find(s => hostname.startsWith(`${s}.`)) || ""; + +// Path prefix each site's ?i= is folded into by the zone's URL Rewrite and Redirect Rules (scripts/rules.js), e.g. +// /orange+cat?i=2 → /web/2/orange+cat, so promoted images can be static files, which ignore query strings +export const folder = site => site || "web"; + +export function normalizeQuery(path) { + const clean = s => s.toLowerCase().trim().replace(/[\x00-\x1f]/g, "").replace(/\/+$/, "").replace(/\s+/g, " "); + try { + return clean(decodeURIComponent(path.replace(/\+/g, " "))); + } catch { + return clean(path); + } +} + +// Main-site i=1 keeps the bare query so existing cache entries stay valid. +// Other keys use uppercase prefixes, which can't collide with (always lowercased) queries. +export const cacheKey = (site, src, i, query) => site ? `${site.toUpperCase()}:${src}:${i}:${query}` : i === 1 ? query : `WEB:${i}:${query}`; +export const r2Key = async (site, key) => `${site ? `${site}/` : ""}${await sha256(key)}`; + +export async function sha256(str) { + const buf = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(str)); + return [...new Uint8Array(buf)].map(b => b.toString(16).padStart(2, "0")).join(""); +} diff --git a/functions/_utils/openverse.js b/src/_utils/openverse.js similarity index 100% rename from functions/_utils/openverse.js rename to src/_utils/openverse.js diff --git a/functions/_utils/surreal.js b/src/_utils/surreal.js similarity index 100% rename from functions/_utils/surreal.js rename to src/_utils/surreal.js diff --git a/functions/_utils/wikimedia.js b/src/_utils/wikimedia.js similarity index 100% rename from functions/_utils/wikimedia.js rename to src/_utils/wikimedia.js diff --git a/functions/[[path]].js b/src/worker.js similarity index 92% rename from functions/[[path]].js rename to src/worker.js index a628953..cfb439a 100644 --- a/functions/[[path]].js +++ b/src/worker.js @@ -4,16 +4,12 @@ import { freeImageUrls, parseFreeSource, FREE_UA } from "./_utils/free.js"; import { isFlickrPlaceholder } from "./_utils/flickr.js"; import { aiImage, parseAiSource, DEFAULT_AI_SOURCE } from "./_utils/ai.js"; import { surreal } from "./_utils/surreal.js"; -import * as LIMITS from "../assets/rate-limit.js"; +import { MAX_INDEX, siteOf, folder, normalizeQuery, cacheKey as keyFor, r2Key as r2KeyFor, sha256 } from "./_utils/keys.js"; +import * as LIMITS from "../public/assets/rate-limit.js"; const TTL_SECONDS = 90 * 24 * 60 * 60; // How long a failed search keeps serving bad.webp before the query is searched again const ERR_SECONDS = 24 * 60 * 60; -// Highest ?i= per site: 40 on the main site (100 Brave results; a Firecrawl query's ?i= past FIRECRAWL_RESULTS gets -// an AI image), 20 on free.* (~50 Openverse + Wikimedia results) and 1000 on ai.* (each is a new image, so there's no -// list to run out). -// Higher values serve max-i.webp, which lists these, so redraw it from design/max-i.html when they change. -const MAX_INDEX = { "": 40, free: 20, ai: 1000 }; // How long a search may hold its query's lock; longer than the 20s image deadline plus the search itself const LOCK_SECONDS = 45; // How long a search's results are kept for the query's other ?i=, which then don't search again or count toward limits @@ -25,20 +21,22 @@ const PARALLEL = 4; // gets the same provider and its ?i= keep their order. const FIRECRAWL_SHARE = 500 / 1500; -export async function onRequest(context) { - const { request, env, params } = context; - const url = new URL(request.url); - const path = params.path?.join("/") || ""; +export default { fetch: handle }; + +async function handle(request, env, context) { + const site = siteOf(new URL(request.url).hostname); + const url = unfold(new URL(request.url), site); + const path = url.pathname.slice(1); const asset = f => env.ASSETS.fetch(new Request(new URL(`/assets/${f}`, url.origin))); // Every alert ends with the full request URL (site, query, ?i= and ?src=), so it's clear which request triggered it const note = n => context.waitUntil(notify(env, { ...n, message: `${n.message}\n${url.origin}/${path}${url.search}` })); // free.* images are public domain or CC0 and its searches cost no credits, so search engines may crawl them (e.g. to // index images embedded in articles). The main site and ai.* get robots.txt, which allows only the homepage. - if (path === "robots.txt" && url.hostname.startsWith("free.")) { + if (path === "robots.txt" && site === "free") { return new Response("User-agent: *\nAllow: /\n", { headers: { "Content-Type": "text/plain; charset=utf-8" } }); } - // _routes.json serves these (except robots.txt) statically without invoking this function; kept as a fallback + // Static assets are served without invoking this worker (except robots.txt, see run_worker_first); kept as a fallback if (!path || path === "index.html" || path === "robots.txt" || path.startsWith("assets/")) { return env.ASSETS.fetch(request); } @@ -56,7 +54,6 @@ export async function onRequest(context) { // ?i= picks which result to serve. free.direct-img.link serves only unrestricted images and ai.direct-img.link only // AI-generated ones; both also take ?src=. Any other param (e.g. scanner probes like ?path=../../.env) is rejected // before it can cost a search. - const site = ["free", "ai"].find(s => url.hostname.startsWith(`${s}.`)) || ""; const free = site === "free", ai = site === "ai"; const label = { free: "Free ", ai: "AI " }[site] || ""; const i = parseIndex(url.searchParams); @@ -69,11 +66,9 @@ export async function onRequest(context) { const counted = new URLSearchParams(Object.entries({ i, src }).filter(([k]) => url.searchParams.has(k))).toString(); context.waitUntil(countHit(env, request, query, site, counted)); - // Main-site i=1 keeps the bare query so existing cache entries stay valid. - // Other keys use uppercase prefixes, which can't collide with (always lowercased) queries. - const cacheKey = site ? `${site.toUpperCase()}:${src}:${i}:${query}` : i === 1 ? query : `WEB:${i}:${query}`; + const cacheKey = keyFor(site, src, i, query); const keyHash = await sha256(cacheKey); - const r2Key = `${site ? `${site}/` : ""}${keyHash}`; + const r2Key = await r2KeyFor(site, cacheKey); // A search's results are kept for LIST_SECONDS, so other ?i= of the same query skip the search. ai.* has no list. const listHash = !ai && await sha256(["LIST", site, src, query].join(":")); @@ -371,17 +366,16 @@ async function countHit(env, request, query, site, params) { } catch {} } -function normalizeQuery(path) { - try { - return decodeURIComponent(path.replace(/\+/g, " ")).toLowerCase().trim().replace(/[\x00-\x1f]/g, "").replace(/\/+$/, "").replace(/\s+/g, " "); - } catch { - return path.toLowerCase().trim().replace(/[\x00-\x1f]/g, "").replace(/\/+$/, "").replace(/\s+/g, " "); - } -} - -async function sha256(str) { - const buf = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(str)); - return [...new Uint8Array(buf)].map(b => b.toString(16).padStart(2, "0")).join(""); +// The zone's rules fold the site and ?i= into the path (/orange+cat?i=2 → /web/2/orange+cat, and a bare query gets i 1) +// so promoted images can be served as static files, which ignore query strings. Unfolded back here, so the rest of the +// worker sees the original request. A folded path for another site is left as is, so it's rejected like any slash. +function unfold(url, site) { + const m = url.pathname.match(/^\/(web|free|ai)\/([^/]*)\/([^/]+)$/); + if (!m || m[1] !== folder(site)) return url; + const u = new URL(url); + u.pathname = `/${m[3]}`; + if (m[2] && m[2] !== "1") u.searchParams.set("i", m[2]); + return u; } // abort cancels a download that's no longer needed (another candidate already won) diff --git a/wrangler.jsonc b/wrangler.jsonc new file mode 100644 index 0000000..2c7019f --- /dev/null +++ b/wrangler.jsonc @@ -0,0 +1,20 @@ +{ + "name": "direct-img", + "main": "src/worker.js", + "compatibility_date": "2026-02-13", + // Files in public/ are served without running the worker (free, unlimited and not counted toward the 100,000 requests + // a day), including the images scripts/promote.js adds before each deploy. robots.txt differs on free.*, so the + // worker answers it. + "assets": { "directory": "public", "binding": "ASSETS", "run_worker_first": ["/robots.txt"] }, + "r2_buckets": [{ "binding": "R2_IMAGES", "bucket_name": "direct-img-store" }], + "kv_namespaces": [{ "binding": "DIRECT_IMG_CACHE", "id": "3d7301ab34db464cb568f31cbea7ce8c" }], + "vars": { "GOATCOUNTER_URL": "https://direct-img.goatcounter.com" }, + // Routes rather than custom domains, so the old Pages project stays behind them as the origin to roll back to + "routes": [ + { "pattern": "direct-img.link/*", "zone_name": "direct-img.link" }, + { "pattern": "free.direct-img.link/*", "zone_name": "direct-img.link" }, + { "pattern": "ai.direct-img.link/*", "zone_name": "direct-img.link" } + ], + "workers_dev": false, + "preview_urls": false +}