From 43a054153c9fb34dc8ec8126825bf6c938491849 Mon Sep 17 00:00:00 2001 From: "planetrenox@protonmail.com" Date: Tue, 29 Sep 2026 17:04:08 +0000 Subject: [PATCH] Copy GitHub releases onto Gitea mirrors Gitea drops releases for pull mirrors, so a new cron/manual workflow creates releases for tags already synced to Gitea, updates edited notes, and streams missing assets. Adds a README. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/sync-releases.yml | 34 ++++++ README.md | 37 +++++++ mirror.js | 1 - notify.js | 9 ++ sync-releases.js | 161 ++++++++++++++++++++++++++++ 5 files changed, 241 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/sync-releases.yml create mode 100644 README.md create mode 100644 sync-releases.js diff --git a/.github/workflows/sync-releases.yml b/.github/workflows/sync-releases.yml new file mode 100644 index 0000000..497aaa4 --- /dev/null +++ b/.github/workflows/sync-releases.yml @@ -0,0 +1,34 @@ +name: Sync releases to Gitea + +on: + workflow_dispatch: + schedule: + - cron: '30 */4 * * *' # Every 4 hours, 30 min after the mirror run + +concurrency: + group: sync-releases + +jobs: + releases: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 24 + - name: Copy GitHub releases to Gitea mirrors + id: sync + run: node sync-releases.js + env: + GH_PAT: ${{ secrets.GH_PAT }} + GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }} + GITEA_URL: ${{ secrets.GITEA_URL }} + + - name: Notify via ntfy + if: ${{ always() }} + run: node notify.js + env: + NTFY_TOPIC_URL: ${{ secrets.NTFY_TOPIC_URL }} + WORKFLOW_KIND: releases + STEP_OUTCOME: ${{ steps.sync.outcome }} + RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} diff --git a/README.md b/README.md new file mode 100644 index 0000000..46a7190 --- /dev/null +++ b/README.md @@ -0,0 +1,37 @@ +# gitea-github-cron + +GitHub Actions that back up every GitHub repo I own (personal and org) to a self-hosted Gitea as pull mirrors. + +**Gitea version: 1.27.\*** + +## Workflows + +| Workflow | Schedule (UTC) | Script | Does | +| --- | --- | --- | --- | +| Mirror to Gitea | every 4h at :00 | `mirror.js` | Creates missing orgs and pull mirrors, syncs repo/org visibility and org avatars | +| Sync releases to Gitea | every 4h at :30 | `sync-releases.js` | Copies GitHub releases (title, notes, assets) onto the mirrors | +| Rename deleted GitHub mirrors | daily 08:00 | `rename-gone-mirrors.js` | Renames mirrors whose GitHub repo was deleted or moved to `-goneN` and freezes them | + +All three can also be run by hand from the Actions tab. Each run posts a summary to ntfy through `notify.js`. + +## Releases on mirrors + +Gitea's migrate API drops `releases: true` for pull mirrors, so a mirror only gets bare tag entries. `sync-releases.js` fills in the real releases: + +- Matches releases by tag, creates missing ones, updates changed title/notes/prerelease, uploads missing assets by name. +- Skips GitHub drafts. +- Skips a release until its tag has reached Gitea through a mirror sync. Otherwise Gitea would create the tag on the default branch. +- Streams each asset from GitHub straight into Gitea. Uploads are capped by `[repository.release] FILE_MAX_SIZE` (default 2048 MB, which matches GitHub's 2 GiB per-file limit) and need `[attachment] ENABLED`. +- Skips frozen mirrors (sync interval 0). +- Never deletes anything on Gitea. + +This relies on Gitea ≥ 1.21.5, where a mirror sync only touches tag-only entries and keeps real releases ([go-gitea/gitea#28817](https://github.com/go-gitea/gitea/pull/28817)). + +## Secrets + +| Secret | Used for | +| --- | --- | +| `GH_PAT` | Classic GitHub token with `repo` scope | +| `GITEA_TOKEN` | Gitea token for the account that owns the mirrors | +| `GITEA_URL` | Gitea host, with or without `https://` | +| `NTFY_TOPIC_URL` | ntfy topic that gets run summaries | diff --git a/mirror.js b/mirror.js index 195e4ff..351b227 100644 --- a/mirror.js +++ b/mirror.js @@ -102,7 +102,6 @@ const getPages = async (url) => { private: r.private, wiki: true, lfs: true, - releases: true, issues: true, pull_requests: true, labels: true, diff --git a/notify.js b/notify.js index 7ea2f98..4fcba9d 100644 --- a/notify.js +++ b/notify.js @@ -18,6 +18,15 @@ const labels = { ['renamed mirror', 'renamed'], ['skipped org', 'skippedOrgs'], ['failure', 'failures'] + ]], + releases: ['Release sync', 'release-facts.json', [ + ['mirror', 'mirrorsChecked'], + ['GitHub release', 'releasesFound'], + ['created release', 'created'], + ['updated release', 'updated'], + ['uploaded asset', 'assetsUploaded'], + ['unsynced tag', 'unsyncedTags'], + ['failure', 'failures'] ]] }; const md = text => String(text).replace(/[\\`*_[\]#]/g, '\\$&'); diff --git a/sync-releases.js b/sync-releases.js new file mode 100644 index 0000000..c6bc373 --- /dev/null +++ b/sync-releases.js @@ -0,0 +1,161 @@ +const { writeFileSync } = require('node:fs'); +const { GH_PAT, GITEA_TOKEN, GITEA_URL } = process.env; +if (![GH_PAT, GITEA_TOKEN, GITEA_URL].every(Boolean)) { + throw new Error('GH_PAT, GITEA_TOKEN, and GITEA_URL are required'); +} + +const gitea = `https://${GITEA_URL.replace(/^https?:\/\//, '').replace(/\/$/, '')}/api/v1`; +const github = 'https://api.github.com'; +const ghHeaders = { Authorization: `token ${GH_PAT}`, Accept: 'application/vnd.github+json' }; +const gtHeaders = { Authorization: `token ${GITEA_TOKEN}`, 'Content-Type': 'application/json' }; +const facts = { + mirrorsChecked: 0, releasesFound: 0, created: 0, updated: 0, assetsUploaded: 0, + unsyncedTags: 0, failures: 0, changes: [] +}; +const key = value => value.toLowerCase(); +const enc = (...parts) => parts.map(encodeURIComponent).join('/'); +const transferTimeout = () => AbortSignal.timeout(30 * 60_000); + +const request = async (url, headers, method = 'GET', body, missingOK = false) => { + const res = await fetch(url, { + method, headers, body: body && JSON.stringify(body), signal: AbortSignal.timeout(30_000) + }); + if (res.status === 404 && missingOK) return null; + if (!res.ok) throw new Error(`${method} ${url}: ${res.status} ${(await res.text()).slice(0, 300)}`); + return res.json(); +}; + +const pages = async (url, headers, sizeKey, missingOK = false) => { + const all = []; + for (let page = 1; page <= 1000; page++) { + const next = new URL(url); + next.searchParams.set(sizeKey, '100'); + next.searchParams.set('page', String(page)); + const batch = await request(next, headers, 'GET', null, missingOK); + if (batch === null) return null; + if (!Array.isArray(batch)) throw new Error(`Expected an array from ${next}`); + if (!batch.length) return all; + all.push(...batch); + } + throw new Error(`Pagination limit reached for ${url}`); +}; + +const source = repo => { + try { + const url = new URL(repo.original_url.replace(/^git@github\.com:/i, 'https://github.com/')); + const parts = url.pathname.replace(/\.git\/?$/i, '').split('/').filter(Boolean); + return url.hostname === 'github.com' && parts.length === 2 ? { owner: parts[0], name: parts[1] } : null; + } catch { + return null; + } +}; + +// Pipe the GitHub download straight into Gitea's raw upload, so assets never touch disk or memory +const copyAsset = async (gh, gtRepo, releaseId, asset) => { + const download = await fetch(`${github}/repos/${enc(gh.owner, gh.name)}/releases/assets/${asset.id}`, { + headers: { ...ghHeaders, Accept: 'application/octet-stream' }, signal: transferTimeout() + }); + if (!download.ok) throw new Error(`download ${asset.name}: ${download.status}`); + const upload = await fetch(`${gtRepo}/releases/${releaseId}/assets?name=${encodeURIComponent(asset.name)}`, { + method: 'POST', body: download.body, duplex: 'half', signal: transferTimeout(), + headers: { Authorization: gtHeaders.Authorization, 'Content-Type': 'application/octet-stream' } + }); + if (upload.status === 413) throw new Error(`${asset.name} (${asset.size} bytes) is over Gitea's [repository.release] FILE_MAX_SIZE`); + if (!upload.ok) throw new Error(`upload ${asset.name}: ${upload.status} ${(await upload.text()).slice(0, 300)}`); +}; + +const syncRelease = async ({ repo, gh }, rel, existing) => { + const gtRepo = `${gitea}/repos/${enc(repo.owner.login, repo.name)}`; + const where = `${repo.owner.login}/${repo.name}@${rel.tag_name}`; + const want = { name: rel.name || rel.tag_name, body: rel.body || '', prerelease: rel.prerelease }; + let gt = existing.get(rel.tag_name), action; + + if (!gt) { + // Creating a release for a tag Gitea doesn't have yet would tag the default branch instead + const tag = await request(`${gtRepo}/tags/${rel.tag_name.split('/').map(encodeURIComponent).join('/')}`, + gtHeaders, 'GET', null, true); + if (!tag) { + facts.unsyncedTags++; + return console.log(`Waiting for mirror sync: ${where}`); + } + gt = await request(`${gtRepo}/releases`, gtHeaders, 'POST', { tag_name: rel.tag_name, ...want }); + facts.created++; + action = 'Created'; + } else if (Object.entries(want).some(([k, v]) => gt[k] !== v)) { + gt = await request(`${gtRepo}/releases/${gt.id}`, gtHeaders, 'PATCH', want); + facts.updated++; + action = 'Updated'; + } + + const have = new Set(gt.assets.map(asset => asset.name)); + let uploaded = 0; + for (const asset of rel.assets.filter(asset => !have.has(asset.name))) { + await copyAsset(gh, gtRepo, gt.id, asset); + uploaded++; + facts.assetsUploaded++; + } + + if (action || uploaded) { + const assets = uploaded ? ` (+${uploaded} asset${uploaded === 1 ? '' : 's'})` : ''; + const line = `${action || 'Added assets to'} ${where}${assets}`; + facts.changes.push(line); + console.log(line); + } +}; + +(async () => { + const [gtUser, gtOrgs] = await Promise.all([ + request(`${gitea}/user`, gtHeaders), + pages(`${gitea}/user/orgs`, gtHeaders, 'limit') + ]); + const owners = [gtUser.login, ...gtOrgs.map(org => org.username || org.name)]; + const repos = (await Promise.all(owners.map((owner, i) => + pages(i ? `${gitea}/orgs/${enc(owner)}/repos` : `${gitea}/user/repos`, gtHeaders, 'limit') + ))).flat(); + const ownerKeys = new Set(owners.map(key)); + + // Frozen mirrors (sync interval 0, e.g. renamed -goneN ones) never get new tags + const mirrors = [...new Map(repos.map(repo => [repo.id, repo])).values()] + .filter(repo => repo.mirror && repo.original_url && !repo.archived && repo.mirror_interval !== '0s') + .filter(repo => ownerKeys.has(key(repo.owner.login))) + .map(repo => ({ repo, gh: source(repo) })) + .filter(mirror => mirror.gh); + facts.mirrorsChecked = mirrors.length; + console.log(`Checking releases for ${mirrors.length} GitHub mirror(s)...`); + + for (const mirror of mirrors) { + const { repo, gh } = mirror; + let releases, existing; + try { + releases = await pages(`${github}/repos/${enc(gh.owner, gh.name)}/releases`, ghHeaders, 'per_page', true); + releases = (releases || []).filter(rel => !rel.draft).reverse(); + if (!releases.length) continue; + const gtReleases = await pages(`${gitea}/repos/${enc(repo.owner.login, repo.name)}/releases`, gtHeaders, 'limit'); + existing = new Map(gtReleases.map(rel => [rel.tag_name, rel])); + } catch (error) { + facts.failures++; + console.error(`Failed to list releases for ${repo.owner.login}/${repo.name}: ${error.message}`); + continue; + } + + facts.releasesFound += releases.length; + for (const rel of releases) { + try { + await syncRelease(mirror, rel, existing); + } catch (error) { + facts.failures++; + console.error(`Failed to sync ${repo.owner.login}/${repo.name}@${rel.tag_name}: ${error.message}`); + } + } + } + + console.log(`Created ${facts.created}, updated ${facts.updated}, uploaded ${facts.assetsUploaded} asset(s); ${facts.failures} failure(s).`); + if (facts.unsyncedTags) console.log(`${facts.unsyncedTags} release(s) wait for their tag to reach Gitea.`); + if (facts.failures) process.exitCode = 1; +})().catch(error => { + console.error(`Fatal: ${error.message}`); + facts.fatal = true; + process.exitCode = 1; +}).finally(() => { + if (process.env.GITHUB_ACTIONS) writeFileSync('release-facts.json', JSON.stringify(facts)); +});