From e027cf96f08cd0e26615572f75abe151a4328346 Mon Sep 17 00:00:00 2001 From: "planetrenox@protonmail.com" Date: Mon, 12 Oct 2026 18:57:48 +0000 Subject: [PATCH] Add video attachments with in-browser compression and playback Videos are compressed with Mediabunny to 360p at 4 fps (HEVC when the device can encode it, H.264 otherwise) with mono 32 kbps AAC, then stored as video_url data URLs. Frames are resized after the frame-rate drop, and rotation is baked in during that same step, so phone videos stay on the fast path. The attach badge shows compression progress, sending waits for pending compressions, and "Clear all attachments" cancels them. Messages show video thumbnails that open a fullscreen player. Co-Authored-By: Claude Opus 5.5 --- package.json | 4 +++ src/attachments.js | 7 +++- src/main.js | 21 ++++++----- src/parts/footer.html | 2 +- src/video.js | 81 +++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 104 insertions(+), 11 deletions(-) create mode 100644 src/video.js diff --git a/package.json b/package.json index 4cedc83..c1004c0 100644 --- a/package.json +++ b/package.json @@ -12,5 +12,9 @@ "vite": "8.0.*", "vite-plugin-html-inject": "1.1.*", "vite-plugin-pwa": "1.2.*" + }, + "dependencies": { + "@mediabunny/aac-encoder": "1.61.*", + "mediabunny": "1.61.*" } } diff --git a/src/attachments.js b/src/attachments.js index 71f7e88..363a102 100644 --- a/src/attachments.js +++ b/src/attachments.js @@ -1,9 +1,11 @@ import { asDataURL, imgToWebp, b64 } from './utils.js'; +import { isVideo, compressVideo } from './video.js'; -export async function toAttach(file) { +export async function toAttach(file, opts) { if (!file) return null; if (file instanceof File) { const name = file.name || 'file', mime = (file.type || 'application/octet-stream').toLowerCase(); + if (isVideo(name, mime)) return compressVideo(file, opts); if (/^image\//.test(mime) || /\.(png|jpe?g|webp|gif)$/i.test(name)) { const data = mime === 'image/webp' || /\.webp$/i.test(name) ? await asDataURL(file) : await imgToWebp(file, 2048, 94); return { type: 'image_url', image_url: { url: data } }; @@ -28,6 +30,9 @@ export async function toAttach(file) { if (mime === 'application/pdf') { return { type: 'file', file: { filename: name, file_data: file.data } }; } + if (/^video\//.test(mime)) { + return { type: 'video_url', video_url: { url: `data:${mime};base64,${file.data}` } }; + } if (/^audio\//.test(mime)) { const fmt = /mp3/.test(mime) ? 'mp3' : 'wav'; return { type: 'input_audio', input_audio: { data: file.data, format: fmt } }; diff --git a/src/main.js b/src/main.js index 2077693..49188cb 100644 --- a/src/main.js +++ b/src/main.js @@ -9,6 +9,7 @@ import { USER } from './user.js'; import { md, enhanceCodeBlocks, renderMarkdown } from './markdown.js'; import { kbUpdate, kbBind } from './keyboard.js'; import { toAttach } from './attachments.js'; +import { videoStrip, revokeVideoUrls } from './video.js'; import { titleFrom, serializeThreadName, deserializeThreadName } from './threads-utils.js'; import { resolveSuneSrc, processSuneIncludes, renderSuneHTML } from './sune-html.js'; @@ -69,11 +70,11 @@ const suneRow=a=>`
`}else{el.userMenuAvatar.className='h-6 w-6 rounded-full bg-gray-900 text-white flex items-center justify-center shrink-0 text-xs';el.userMenuAvatar.textContent='👤'}} const getSuneLabel=m=>{const name=(m&&m.sune_name)||SUNE.name,modelShort=getModelShort(m&&m.model);return `${name} · ${modelShort}`} -function _createMessageRow(m){const role=typeof m==='string'?m:(m&&m.role)||'assistant',meta=typeof m==='string'?{}:m||{},isUser=role==='user',$row=$('
'),$head=$('
'),$avatar=$('
');const uAva=isUser?USER.avatar:meta.avatar;uAva?$avatar.attr('class','msg-avatar shrink-0 h-7 w-7 rounded-full overflow-hidden').html(``):$avatar.attr('class',`${isUser?'bg-gray-900 text-white':'bg-gray-200 text-gray-900'} msg-avatar shrink-0 h-7 w-7 rounded-full flex items-center justify-center`).text(isUser?'👤':'✺');const $name=$('
').text(isUser?USER.name:getSuneLabel(meta));const $deleteBtn=$('').on('click',async e=>{e.stopPropagation();state.messages=state.messages.filter(msg=>msg.id!==m.id);$row.remove();await THREAD.persist()});const $copyBtn=$('').on('click',async function(e){e.stopPropagation();const cur=state.messages.find(x=>x.id===m.id)||m;if(await copyToClipboard(partsToText(cur,true))){$(this).html('');icons();setTimeout(()=>{$(this).html('');icons()},1200)}});$head.append($avatar,$name,$copyBtn,$deleteBtn);const $bubble=$(`
`);$row.append($head,$bubble);return $row} +function _createMessageRow(m){const role=typeof m==='string'?m:(m&&m.role)||'assistant',meta=typeof m==='string'?{}:m||{},isUser=role==='user',$row=$('
'),$head=$('
'),$avatar=$('
');const uAva=isUser?USER.avatar:meta.avatar;uAva?$avatar.attr('class','msg-avatar shrink-0 h-7 w-7 rounded-full overflow-hidden').html(``):$avatar.attr('class',`${isUser?'bg-gray-900 text-white':'bg-gray-200 text-gray-900'} msg-avatar shrink-0 h-7 w-7 rounded-full flex items-center justify-center`).text(isUser?'👤':'✺');const $name=$('
').text(isUser?USER.name:getSuneLabel(meta));const $deleteBtn=$('').on('click',async e=>{e.stopPropagation();state.messages=state.messages.filter(msg=>msg.id!==m.id);$row.remove();await THREAD.persist()});const $copyBtn=$('').on('click',async function(e){e.stopPropagation();const cur=state.messages.find(x=>x.id===m.id)||m;if(await copyToClipboard(partsToText(cur,true))){$(this).html('');icons();setTimeout(()=>{$(this).html('');icons()},1200)}});$head.append($avatar,$name,$copyBtn,$deleteBtn);const $bubble=$(`
`);$row.append($head,$bubble);const vids=(Array.isArray(meta.content)?meta.content:[]).filter(p=>p?.type==='video_url');if(vids.length)$row.append(videoStrip(vids));return $row} function msgRow(m){const $row=_createMessageRow(m);$(el.messages).append($row);queueMicrotask(()=>{el.chat.scrollTo({top:el.chat.scrollHeight,behavior:'smooth'});icons()});return $row.find('.msg-bubble')[0]} const addMessage=window.addMessage=function(m,track=true){m.id=m.id||gid();if(!Array.isArray(m.content)&&m.content!=null){m.content=[{type:'text',text:String(m.content)}]}const bubble=msgRow(m);bubble.dataset.mid=m.id;renderMarkdown(bubble,partsToText(m));if(track)state.messages.push(m);if(m.role==='assistant')el.composer.dispatchEvent(new CustomEvent('sune:newSuneResponse',{detail:{message:m}}));return bubble} const addSuneBubbleStreaming=(meta,id)=>msgRow(Object.assign({role:'assistant',id},meta)) -const clearChat=()=>{el.suneHtml.dispatchEvent(new CustomEvent('sune:unmount'));state.messages=[];el.messages.innerHTML='';state.attachments=[];updateAttachBadge();el.fileInput.value=''} +const clearChat=()=>{el.suneHtml.dispatchEvent(new CustomEvent('sune:unmount'));state.messages=[];el.messages.innerHTML='';clearAttachments();revokeVideoUrls()} const payloadWithSampling=b=>{const o=Object.assign({},b),s=SUNE,p={temperature:num(s.temperature,null),top_p:num(s.top_p,null),top_k:int(s.top_k,null),frequency_penalty:num(s.frequency_penalty,null),repetition_penalty:num(s.repetition_penalty,null),min_p:num(s.min_p,null),top_a:num(s.top_a,null),max_tokens:int(s.max_tokens,null)>0?int(s.max_tokens,null):null};Object.keys(p).forEach(k=>{const v=p[k];if(v!==null)o[k]=v});return o} function setBtnStop(){const b=el.sendBtn;b.dataset.mode='stop';b.type='button';b.setAttribute('aria-label','Stop');b.innerHTML='';icons();b.onclick=()=>{state.abortRequested=true;state.controller?.abort?.();state.busy=false;setBtnSend()}} function setBtnSend(){const b=el.sendBtn;b.dataset.mode='send';b.type='submit';b.setAttribute('aria-label','Send');b.innerHTML='';icons();b.onclick=null} @@ -118,23 +119,25 @@ $(el.suneList).on('click',async e=>{const menuBtn=e.target.closest('[data-sune-m $(el.sunePopover).on('click',async e=>{const act=e.target.closest('[data-action]')?.getAttribute('data-action');if(!act||!menuSuneId)return;const s=SUNE.get(menuSuneId);if(!s)return;const updateAndRender=async()=>{s.updatedAt=Date.now();SUNE.save();renderSidebar();await reflectActiveSune();markLocalDirty()};if(act==='pin'){s.pinned=!s.pinned;await updateAndRender()}else if(act==='rename'){const n=prompt('Rename sune to:',s.name);if(n!=null){s.name=n.trim();await updateAndRender()}}else if(act==='pfp'){const i=document.createElement('input');i.type='file';i.accept='image/*';i.onchange=async()=>{const f=i.files?.[0];if(!f)return;try{s.avatar=await imgToWebp(f);await updateAndRender()}catch{}};i.click()}else if(act==='export')dl(`sune-${(s.name||'sune').replace(/\W/g,'_')}-${ts()}.sune`,[s]);hideSunePopover()}) $(el.sunesSyncUploadBtn).on('click',()=>{hideSunesSyncPopover();performSuneUpload()}); $(el.sunesSyncDownloadBtn).on('click',()=>{hideSunesSyncPopover();performSuneDownload(false)}); -function updateAttachBadge(){const n=state.attachments.length;el.attachBadge.textContent=String(n);el.attachBadge.classList.toggle('hidden',n===0)} -const addAttachments=async files=>{for(const f of files){const at=await toAttach(f).catch(()=>null);if(at)state.attachments.push(at)}updateAttachBadge()} +let attachGen=0,attachJob=null,attachQueue=Promise.resolve(),attachWait=false +function updateAttachBadge(){const n=state.attachments.length;el.attachBadge.textContent=attachJob?Math.floor(attachJob.p*100)+'%':String(n);el.attachBadge.classList.toggle('hidden',!n&&!attachJob)} +function clearAttachments(){attachGen++;attachJob?.ac.abort();attachJob=null;state.attachments=[];updateAttachBadge();el.fileInput.value=''} +const addAttachments=files=>{const g=attachGen;attachQueue=attachQueue.then(async()=>{for(const f of files){if(g!==attachGen)return;const job=attachJob={ac:new AbortController,p:0};updateAttachBadge();const at=await toAttach(f,{signal:job.ac.signal,onProgress:p=>{job.p=p;updateAttachBadge()}}).catch(e=>{if(g===attachGen)alert(`Couldn't attach ${f.name||'file'}: ${e?.message||e}`)});if(g!==attachGen)return;attachJob=null;if(at)state.attachments.push(at);updateAttachBadge()}}).catch(()=>{})} const hideAttachPopover=()=>el.attachPopover.classList.add('hidden') -function showAttachPopover(){if(!state.attachments.length||!el.attachPopover.classList.contains('hidden'))return;haptic();el.attachPopover.classList.remove('hidden');positionPopover(el.attachBtn,el.attachPopover);icons()} +function showAttachPopover(){if((!state.attachments.length&&!attachJob)||!el.attachPopover.classList.contains('hidden'))return;haptic();el.attachPopover.classList.remove('hidden');positionPopover(el.attachBtn,el.attachPopover);icons()} let attachHold=null,attachHeld=false;const holdAttach=()=>{attachHeld=true;showAttachPopover()} $(el.attachBtn).on('pointerdown',e=>{attachHeld=false;if(e.pointerType!=='mouse')attachHold=setTimeout(holdAttach,500)}).on('pointerup pointercancel pointerleave',()=>clearTimeout(attachHold)).on('contextmenu',e=>{e.preventDefault();holdAttach()}).on('click',()=>{if(attachHeld)return attachHeld=false;hideAttachPopover();el.fileInput.click()}) -$(el.attachPopover).on('click',e=>{if(!e.target.closest('[data-action="clear"]'))return;state.attachments=[];updateAttachBadge();hideAttachPopover()}) +$(el.attachPopover).on('click',e=>{if(!e.target.closest('[data-action="clear"]'))return;clearAttachments();hideAttachPopover()}) $(document).on('click',e=>{if(!el.attachPopover.contains(e.target)&&!el.attachBtn.contains(e.target))hideAttachPopover()}) $(window).on('resize',()=>{if(!el.attachPopover.classList.contains('hidden'))positionPopover(el.attachBtn,el.attachPopover)}) -$(el.fileInput).on('change',async()=>{const files=[...(el.fileInput.files||[])];el.fileInput.value='';await addAttachments(files)}) +$(el.fileInput).on('change',()=>{const files=[...(el.fileInput.files||[])];el.fileInput.value='';addAttachments(files)}) $(el.composer).on('paste',async e=>{ const cbd=e.clipboardData||e.originalEvent?.clipboardData,files=[...(cbd?.files||[])]; if(!files.length)return; e.preventDefault(); - await addAttachments(files); + addAttachments(files); }) -$(el.composer).on('submit',async e=>{e.preventDefault();if(state.busy)return;const text=el.input.value.trim();if(!text&&!state.attachments.length)return SUNE.infer();await ensureThreadOnFirstUser(text||'(attachments)');const th=THREAD.active,shouldGenTitle=th&&!th.title;el.input.value='';const parts=[];if(text)parts.push({type:'text',text});parts.push(...state.attachments);state.attachments=[];updateAttachBadge();el.fileInput.value='';const userMsg={role:'user',content:parts.length?parts:[{type:'text',text:text||'(sent attachments)'}]};addMessage(userMsg);el.composer.dispatchEvent(new CustomEvent('user:send',{detail:{message:userMsg}}));if(shouldGenTitle)(async()=>{const title=await generateTitleWithAI(state.messages)||partsToText(state.messages.find(m=>m.role==='user')).replace(/!\[\]\(data:[^\)]+\)/g,'[Image]')||'Untitled';await THREAD.setTitle(th.id,title)})();if(!SUNE.model)return;state.busy=true;setBtnStop();const a=SUNE.active,suneMeta={sune_name:a.name,model:SUNE.model,avatar:a.avatar||''},streamId=sid(),suneBubble=addSuneBubbleStreaming(suneMeta,streamId);suneBubble.dataset.mid=streamId;suneBubble.innerHTML=SUNE_LOGO_SVG;const assistantMsg=Object.assign({id:streamId,role:'assistant',content:[{type:'text',text:''}]},suneMeta);state.messages.push(assistantMsg);THREAD.persist(false);state.stream={rid:streamId,bubble:suneBubble,meta:suneMeta,text:'',done:false};let buf='',completed=false;const onDelta=(delta,done,imgs)=>{if(imgs){if(!assistantMsg.images)assistantMsg.images=[];assistantMsg.images.push(...imgs)}buf+=delta;state.stream.text=buf;assistantMsg.content[0].text=buf;renderMarkdown(suneBubble,partsToText(assistantMsg),{enhance:false});if(done&&!completed){completed=true;setBtnSend();state.busy=false;enhanceCodeBlocks(suneBubble,true);THREAD.persist(true);el.composer.dispatchEvent(new CustomEvent('sune:newSuneResponse',{detail:{message:assistantMsg}}));state.stream={rid:null,bubble:null,meta:null,text:'',done:false}}else if(!done)THREAD.persist(false)};await streamChat(onDelta,streamId)}) +$(el.composer).on('submit',async e=>{e.preventDefault();if(state.busy||attachWait)return;attachWait=true;await attachQueue;attachWait=false;const text=el.input.value.trim();if(!text&&!state.attachments.length)return SUNE.infer();await ensureThreadOnFirstUser(text||'(attachments)');const th=THREAD.active,shouldGenTitle=th&&!th.title;el.input.value='';const parts=[];if(text)parts.push({type:'text',text});parts.push(...state.attachments);state.attachments=[];updateAttachBadge();el.fileInput.value='';const userMsg={role:'user',content:parts.length?parts:[{type:'text',text:text||'(sent attachments)'}]};addMessage(userMsg);el.composer.dispatchEvent(new CustomEvent('user:send',{detail:{message:userMsg}}));if(shouldGenTitle)(async()=>{const title=await generateTitleWithAI(state.messages)||partsToText(state.messages.find(m=>m.role==='user')).replace(/!\[\]\(data:[^\)]+\)/g,'[Image]')||'Untitled';await THREAD.setTitle(th.id,title)})();if(!SUNE.model)return;state.busy=true;setBtnStop();const a=SUNE.active,suneMeta={sune_name:a.name,model:SUNE.model,avatar:a.avatar||''},streamId=sid(),suneBubble=addSuneBubbleStreaming(suneMeta,streamId);suneBubble.dataset.mid=streamId;suneBubble.innerHTML=SUNE_LOGO_SVG;const assistantMsg=Object.assign({id:streamId,role:'assistant',content:[{type:'text',text:''}]},suneMeta);state.messages.push(assistantMsg);THREAD.persist(false);state.stream={rid:streamId,bubble:suneBubble,meta:suneMeta,text:'',done:false};let buf='',completed=false;const onDelta=(delta,done,imgs)=>{if(imgs){if(!assistantMsg.images)assistantMsg.images=[];assistantMsg.images.push(...imgs)}buf+=delta;state.stream.text=buf;assistantMsg.content[0].text=buf;renderMarkdown(suneBubble,partsToText(assistantMsg),{enhance:false});if(done&&!completed){completed=true;setBtnSend();state.busy=false;enhanceCodeBlocks(suneBubble,true);THREAD.persist(true);el.composer.dispatchEvent(new CustomEvent('sune:newSuneResponse',{detail:{message:assistantMsg}}));state.stream={rid:null,bubble:null,meta:null,text:'',done:false}}else if(!done)THREAD.persist(false)};await streamChat(onDelta,streamId)}) let jars={html:null,extension:null}; const ensureJars=async()=>{ if(jars.html&&jars.extension)return jars; diff --git a/src/parts/footer.html b/src/parts/footer.html index 9cf8f66..c05967e 100644 --- a/src/parts/footer.html +++ b/src/parts/footer.html @@ -6,7 +6,7 @@
- + diff --git a/src/video.js b/src/video.js new file mode 100644 index 0000000..4a9cfba --- /dev/null +++ b/src/video.js @@ -0,0 +1,81 @@ +import { asDataURL } from './utils.js'; + +// Videos are shrunk to 360p @ 4 fps before attaching; HEVC when the device can encode it, H.264 otherwise. +const SHORT_SIDE = 360, FPS = 4, VIDEO_BPS = { hevc: 130e3, avc: 160e3 }, AUDIO_BPS = 32e3; +const even = n => Math.max(2, Math.round(n / 2) * 2); +let mb; +const loadMb = async () => { + if (mb) return mb; + const m = await import('mediabunny'); + if (!(await m.canEncodeAudio('aac', { numberOfChannels: 1, sampleRate: 48000, bitrate: AUDIO_BPS }))) (await import('@mediabunny/aac-encoder')).registerAacEncoder(); + return mb = m; +}; + +export const isVideo = (name, mime) => /^video\//.test(mime) || /\.(mp4|m4v|mov|webm|mkv|3gp)$/i.test(name); + +export async function compressVideo(file, { onProgress, signal } = {}) { + const { Input, Output, Conversion, ALL_FORMATS, BlobSource, BufferTarget, Mp4OutputFormat, Quality, canEncodeVideo } = await loadMb(); + const input = new Input({ source: new BlobSource(file), formats: ALL_FORMATS }), track = await input.getPrimaryVideoTrack(); + if (!track) throw new Error('no video track'); + const w = await track.getDisplayWidth(), h = await track.getDisplayHeight(), s = Math.min(1, SHORT_SIDE / Math.min(w, h)), width = even(w * s), height = even(h * s); + let codec = null; + for (const c of ['hevc', 'avc']) if (await canEncodeVideo(c, { width, height, frameRate: FPS, bitrate: VIDEO_BPS[c] })) { codec = c; break; } + if (!codec) throw new Error('this device cannot encode HEVC or H.264'); + const output = new Output({ format: new Mp4OutputFormat({ fastStart: 'in-memory' }), target: new BufferTarget() }); + const video = { frameRate: FPS, codec, quality: new Quality({ bitrate: VIDEO_BPS[codec] }), forceTranscode: true }; + // Mediabunny resizes every decoded frame before dropping to FPS, so resize in `process` (runs after the drop) instead. + // `draw` bakes in each frame's rotation, so cancelling the track rotation skips Mediabunny's per-frame re-render too. + if (await track.getFlip()) Object.assign(video, { width, height, fit: 'fill' }); + else { + const cx = new OffscreenCanvas(width, height).getContext('2d'), rot = await track.getRotation(); + cx.imageSmoothingQuality = 'high'; + Object.assign(video, { process: f => (f.draw(cx, 0, 0, width, height), cx.canvas), processedWidth: width, processedHeight: height, rotate: (360 - rot) % 360 }); + } + const conv = await Conversion.init({ + input, output, tracks: 'primary', video, + audio: { numberOfChannels: 1, codec: 'aac', quality: new Quality({ bitrate: AUDIO_BPS }), forceTranscode: true } + }); + if (!conv.isValid) throw new Error(conv.discardedTracks.map(t => t.reason).join(', ') || 'unsupported video'); + if (signal) { if (signal.aborted) return null; signal.addEventListener('abort', () => conv.cancel(), { once: true }); } + conv.onProgress = p => onProgress?.(p); + try { await conv.execute(); } catch (e) { if (signal?.aborted) return null; throw e; } + return { type: 'video_url', video_url: { url: await asDataURL(new Blob([output.target.buffer], { type: 'video/mp4' })) } }; +} + +const blobUrls = new Set(); +const toBlobUrl = async u => { const o = URL.createObjectURL(await (await fetch(u)).blob()); blobUrls.add(o); return o; }; +export const revokeVideoUrls = () => { blobUrls.forEach(u => URL.revokeObjectURL(u)); blobUrls.clear(); }; + +function openPlayer(src) { + const o = document.createElement('div'); + o.className = 'fixed inset-0 z-[70] bg-black flex items-center justify-center'; + o.innerHTML = ''; + const v = o.querySelector('video'), done = () => { v.pause(); o.remove(); removeEventListener('keydown', esc); document.fullscreenElement && document.exitFullscreen().catch(() => {}); }, + esc = e => e.key === 'Escape' && history.back(); + v.src = src; + history.pushState({ video: 1 }, ''); + addEventListener('popstate', done, { once: true }); + addEventListener('keydown', esc); + o.querySelector('button').onclick = () => history.back(); + o.addEventListener('fullscreenchange', () => !document.fullscreenElement && o.isConnected && history.back()); + document.body.append(o); + window.lucide?.createIcons(); + o.requestFullscreen?.().catch(() => {}); +} + +export function videoStrip(parts) { + const wrap = document.createElement('div'); + wrap.className = 'flex gap-2 overflow-x-auto no-scrollbar px-4'; + for (const p of parts) { + const b = document.createElement('button'); + b.type = 'button'; + b.setAttribute('aria-label', 'Play video'); + b.className = 'relative shrink-0 h-40 rounded-xl overflow-hidden bg-black'; + b.innerHTML = ''; + const src = toBlobUrl(p.video_url?.url || ''); + src.then(u => b.querySelector('video').src = u + '#t=0.1').catch(() => {}); + b.onclick = async () => openPlayer(await src); + wrap.append(b); + } + return wrap; +}