diff --git a/PRIVACY.md b/PRIVACY.md index 93c5023..1ce5a0b 100644 --- a/PRIVACY.md +++ b/PRIVACY.md @@ -12,7 +12,7 @@ direct-img.link has no accounts and doesn't sell data. This page explains what i ## Search queries and images -- When a query isn't already cached, it's sent to Brave Search, and to Bing if Brave returns nothing. On free.direct-img.link, it's sent to Openverse and/or Wikimedia Commons instead. Searches and image downloads are made from our servers, so your IP address isn't shared with these search providers or the sites images come from. +- When a query isn't already cached, it's sent to Brave Search, and to Bing if Brave returns nothing or is unavailable. On free.direct-img.link, it's sent to Openverse and/or Wikimedia Commons instead. Searches and image downloads are made from our servers, so your IP address isn't shared with these search providers or the sites images come from. - Each query (with its `i` and `src` options) and the image it returned are cached for up to 90 days. Failed searches are remembered for 24 hours. - Cached images are shared: anyone requesting the same query and options gets the same image. diff --git a/README.md b/README.md index dc18a70..67edf69 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,7 @@ Literal slashes (`/`) and dots (`.`) in the URL path are **rejected** to prevent - **Double-encoded values** are decoded once — `%2520` becomes `%20` (literal), not a space - Two queries that normalize to the same string share the same cached image - **Nothing found?** If a search finds no image (or fewer than `i` working ones), an [AI image](#ai-images) of the query is served instead of `bad.webp`, and cached like any other image. `bad.webp` is only served if the AI image fails too +- **Out of searches?** If Brave refuses (e.g. its monthly spending limit is reached), Bing is searched instead. If Bing is blocking us too, `donate.webp` is served instead of an AI image, and nothing is cached, so the query works again once either is back ## Free Images @@ -151,6 +152,7 @@ Limits change over time, so they aren't listed here. **Check the Rate Limits sec - **New searches (cache misses)** are limited per IP per day, then `limit.webp` is served. `direct-img.link`, `free.direct-img.link` and `ai.direct-img.link` have separate counters, all reset at midnight UTC - **Cache hits are unlimited** (within the burst limit) - Other `?i=` values of a query searched in the last hour reuse its results, so they don't count (except on `ai.`, where each `i` is a new image) +- If Brave's spending limit is hit, searches use Bing; if Bing also blocks us, `donate.webp` is served (not cached) - Searches that find nothing count toward the limit. If the AI image served instead fails too, the failure is remembered for 24h (`bad.webp`) - A Cloudflare WAF burst limit applies to all requests (HTTP 429). Its counters are approximate (not shared perfectly across Cloudflare servers), so bursts may slip slightly past it @@ -182,6 +184,7 @@ Free community service. Donations help cover API and infrastructure costs. 4. Subscribe — you get **$5 in free monthly credits** (covers 1,000 queries/month) 5. Go to your [API dashboard](https://api.search.brave.com/app/#/subscriptions) 6. Copy your **API key** (starts with `BSA...`) +7. Optional: set a monthly spending limit there so you never pay past the credit. Once it's reached, searches fall back to Bing ### 2. Openverse API Credentials (Optional) @@ -292,7 +295,7 @@ After deploying, visit `https:///_setup` once. It creates the `dire - **Cloudflare KV** — generic lookups - **SurrealDB (v2.6.5)** — atomic rate limiting, search locks + cached result lists - **Cloudflare WAF** — layer 7 mitigation -- **Brave Image Search API** — image sourcing +- **Brave Image Search API** — image sourcing, with Bing as the fallback - **Openverse API + Wikimedia Commons API** — free image sourcing - **[AuroraIMG-6M](https://github.com/direct-img/AuroraIMG-6M) on Dokploy** — AI images and the nothing-found fallback diff --git a/assets/donate.webp b/assets/donate.webp new file mode 100644 index 0000000..3328c01 Binary files /dev/null and b/assets/donate.webp differ diff --git a/functions/[[path]].js b/functions/[[path]].js index 133c8fc..9fc0bec 100644 --- a/functions/[[path]].js +++ b/functions/[[path]].js @@ -215,6 +215,11 @@ COMMIT TRANSACTION;`; // The query's cached results, or a fresh search. pos holds the indexes of the working images found so far and next // the first URL not tried yet, so a known ?i= downloads only its own image and a new one carries on from next. let { urls, pos, next } = list || { urls: await (free ? freeImageUrls(query, src, env) : webImageUrls(query, env, note)), pos: [], next: 0 }; + // Brave and Bing both failed: ask for donations instead of an AI image, and cache nothing since it's temporary + if (!urls) { + release(); + return asset("donate.webp"); + } // Starting at a known image skips the ones before it, so it can't add to pos const known = pos.length >= i; let at = known ? pos[i - 1] : next, found = known ? i - 1 : pos.length, imgResult = null, timedOut = false; @@ -279,12 +284,18 @@ function parseIndex(searchParams) { return /^\d+$/.test(i) && +i >= 1 && +i <= MAX_INDEX ? +i : null; } -// Brave results, or Bing's if Brave has none +// Brave results, or Bing's if Brave has none or refuses (e.g. once its spending limit is hit). null if Bing fails too, +// which means we're out of searches rather than that nothing exists. async function webImageUrls(query, env, note) { - const urls = await braveImageSearch(query, env.BRAVE_API_KEY); - if (urls?.length) return urls; - note({ title: "Brave Search Empty", message: `No results for: ${query}. Trying Bing Fallback.`, tags: "warning,mag", priority: 3 }); - return (await bingImageSearchFallback(query)) || []; + const brave = await braveImageSearch(query, env.BRAVE_API_KEY); + if (brave.urls?.length) return brave.urls; + note(brave.error + ? { title: "Brave Unavailable", message: `Brave failed for: ${query}. Trying Bing Fallback.\nBrave: ${brave.error}`, tags: "warning,mag", priority: 3 } + : { title: "Brave Search Empty", message: `No results for: ${query}. Trying Bing Fallback.`, tags: "warning,mag", priority: 3 }); + const bing = await bingImageSearchFallback(query); + if (bing.urls) return bing.urls; + note({ title: "Bing Blocked, Served donate.webp", message: `Bing failed for: ${query}\nBing: ${bing.error}\nBrave: ${brave.error || "no results"}`, tags: "rotating_light,x", priority: 4 }); + return null; } async function setupSurreal(env) { diff --git a/functions/_utils/bing.js b/functions/_utils/bing.js index 3466aa4..9499cd7 100644 --- a/functions/_utils/bing.js +++ b/functions/_utils/bing.js @@ -1,3 +1,5 @@ +// { urls }, or { error } if Bing failed. Bing finds something for nearly any query (even gibberish), so a page without +// results almost always means it's blocking us (e.g. with a CAPTCHA) or changed its page format. export async function bingImageSearchFallback(query) { try { // SafeSearch off, like Brave. The cookie is what actually sets it; adlt= alone is ignored. @@ -8,24 +10,14 @@ export async function bingImageSearchFallback(query) { "Cookie": "SRCHHPGUSR=ADLT=OFF" } }); - if (!res.ok) return null; + if (!res.ok) return { error: `HTTP ${res.status}` }; const html = await res.text(); - const urls = []; - + // Bing encodes the image payload inside HTML entity attributes and script tags - const regex1 = /"murl":"(https:[^&"]+)"/g; - const regex2 = /"murl":"(https:[^"]+)"/g; - - let match; - while ((match = regex1.exec(html)) !== null) { - urls.push(match[1].replace(/\\\//g, '/')); - } - while ((match = regex2.exec(html)) !== null) { - urls.push(match[1].replace(/\\\//g, '/')); - } - - return urls.length > 0 ? [...new Set(urls)] : null; + const urls = [...html.matchAll(/"murl":"(https:[^&"]+)"/g), ...html.matchAll(/"murl":"(https:[^"]+)"/g)] + .map(m => m[1].replace(/\\\//g, '/')); + return urls.length ? { urls: [...new Set(urls)] } : { error: `No results in page (${html.length} bytes, ${res.url})` }; } catch (err) { - return null; + return { error: err.message }; } } diff --git a/functions/_utils/brave.js b/functions/_utils/brave.js index b35d9e6..0ee2292 100644 --- a/functions/_utils/brave.js +++ b/functions/_utils/brave.js @@ -1,8 +1,13 @@ +// { urls } ([] if none found), or { error } if Brave refused or failed, e.g. once our rate or spending limit is hit export async function braveImageSearch(query, apiKey) { - const res = await fetch(`https://api.search.brave.com/res/v1/images/search?q=${encodeURIComponent(query)}&count=50&safesearch=off`, { - headers: { "Accept": "application/json", "X-Subscription-Token": apiKey }, - }); - if (!res.ok) return null; - const data = await res.json(); - return data.results?.map(r => r.properties?.url || r.thumbnail?.src).filter(url => !!url) || null; + try { + const res = await fetch(`https://api.search.brave.com/res/v1/images/search?q=${encodeURIComponent(query)}&count=50&safesearch=off`, { + headers: { "Accept": "application/json", "X-Subscription-Token": apiKey }, + }); + if (!res.ok) return { error: `HTTP ${res.status}: ${(await res.text()).slice(0, 300)}` }; + const data = await res.json(); + return { urls: data.results?.map(r => r.properties?.url || r.thumbnail?.src).filter(url => !!url) || [] }; + } catch (err) { + return { error: err.message }; + } }