sune-org/sune
1
0
Fork 0
mirror of https://github.com/sune-org/sune.git synced 2026-10-15 14:57:53 +00:00
Code Issues Packages Projects Releases 3 Wiki Activity

Add video attachments with in-browser compression and playback

Browse Source 
Videos are compressed with Mediabunny to 360p at 4 fps (HEVC when the
device can encode it, H.264 otherwise) with mono 32 kbps AAC, then
stored as video_url data URLs. Frames are resized after the frame-rate
drop, and rotation is baked in during that same step, so phone videos
stay on the fast path.

The attach badge shows compression progress, sending waits for pending
compressions, and "Clear all attachments" cancels them. Messages show
video thumbnails that open a fullscreen player.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
multipleof4
2026-10-12 18:57:48 +00:00
co-authored by Claude Opus 5.5
parent 3c26bda123
commit e027cf96f0
5 changed files with 104 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main.js
+12 -9
View File
@@ -9,6 +9,7 @@ import { USER } from './user.js';
import { md, enhanceCodeBlocks, renderMarkdown } from './markdown.js';
import { kbUpdate, kbBind } from './keyboard.js';
import { toAttach } from './attachments.js';
import { videoStrip, revokeVideoUrls } from './video.js';
import { titleFrom, serializeThreadName, deserializeThreadName } from './threads-utils.js';
import { resolveSuneSrc, processSuneIncludes, renderSuneHTML } from './sune-html.js';
@@ -69,11 +70,11 @@ const suneRow=a=>`<div class="relative flex items-center gap-2 px-3 py-2 ${a.pin
const renderSidebar=window.renderSidebar=()=>{const list=[...SUNE.list].sort((a,b)=>(b.pinned-a.pinned));el.suneList.innerHTML=list.map(suneRow).join('');icons()}
const renderUserUI=window.renderUserUI=()=>{if(!el.userMenuAvatar)return;const a=USER.avatar;if(a){el.userMenuAvatar.className='h-6 w-6 rounded-full overflow-hidden shrink-0';el.userMenuAvatar.innerHTML=`<img src="${esc(a)}" class="h-full w-full object-cover"/>`}else{el.userMenuAvatar.className='h-6 w-6 rounded-full bg-gray-900 text-white flex items-center justify-center shrink-0 text-xs';el.userMenuAvatar.textContent='👤'}}
const getSuneLabel=m=>{const name=(m&&m.sune_name)||SUNE.name,modelShort=getModelShort(m&&m.model);return `${name} · ${modelShort}`}
function _createMessageRow(m){const role=typeof m==='string'?m:(m&&m.role)||'assistant',meta=typeof m==='string'?{}:m||{},isUser=role==='user',$row=$('<div class="flex flex-col gap-2"></div>'),$head=$('<div class="flex items-center gap-2 px-4"></div>'),$avatar=$('<div></div>');const uAva=isUser?USER.avatar:meta.avatar;uAva?$avatar.attr('class','msg-avatar shrink-0 h-7 w-7 rounded-full overflow-hidden').html(`<img src="${esc(uAva)}" class="h-full w-full object-cover">`):$avatar.attr('class',`${isUser?'bg-gray-900 text-white':'bg-gray-200 text-gray-900'} msg-avatar shrink-0 h-7 w-7 rounded-full flex items-center justify-center`).text(isUser?'👤':'✺');const $name=$('<div class="text-xs font-medium text-gray-500"></div>').text(isUser?USER.name:getSuneLabel(meta));const $deleteBtn=$('<button class="p-1.5 rounded-lg hover:bg-gray-200 text-gray-400 hover:text-red-500" title="Delete message"><i data-lucide="x" class="h-4 w-4"></i></button>').on('click',async e=>{e.stopPropagation();state.messages=state.messages.filter(msg=>msg.id!==m.id);$row.remove();await THREAD.persist()});const $copyBtn=$('<button class="ml-auto p-1.5 rounded-lg hover:bg-gray-200 text-gray-400 hover:text-gray-600" title="Copy message"><i data-lucide="copy" class="h-4 w-4"></i></button>').on('click',async function(e){e.stopPropagation();const cur=state.messages.find(x=>x.id===m.id)||m;if(await copyToClipboard(partsToText(cur,true))){$(this).html('<i data-lucide="check" class="h-4 w-4 text-green-500"></i>');icons();setTimeout(()=>{$(this).html('<i data-lucide="copy" class="h-4 w-4"></i>');icons()},1200)}});$head.append($avatar,$name,$copyBtn,$deleteBtn);const $bubble=$(`<div class="${(isUser?'bg-gray-50 border border-gray-200':'bg-gray-100')+' msg-bubble markdown-body rounded-none px-4 py-3 w-full'}"></div>`);$row.append($head,$bubble);return $row}
function _createMessageRow(m){const role=typeof m==='string'?m:(m&&m.role)||'assistant',meta=typeof m==='string'?{}:m||{},isUser=role==='user',$row=$('<div class="flex flex-col gap-2"></div>'),$head=$('<div class="flex items-center gap-2 px-4"></div>'),$avatar=$('<div></div>');const uAva=isUser?USER.avatar:meta.avatar;uAva?$avatar.attr('class','msg-avatar shrink-0 h-7 w-7 rounded-full overflow-hidden').html(`<img src="${esc(uAva)}" class="h-full w-full object-cover">`):$avatar.attr('class',`${isUser?'bg-gray-900 text-white':'bg-gray-200 text-gray-900'} msg-avatar shrink-0 h-7 w-7 rounded-full flex items-center justify-center`).text(isUser?'👤':'✺');const $name=$('<div class="text-xs font-medium text-gray-500"></div>').text(isUser?USER.name:getSuneLabel(meta));const $deleteBtn=$('<button class="p-1.5 rounded-lg hover:bg-gray-200 text-gray-400 hover:text-red-500" title="Delete message"><i data-lucide="x" class="h-4 w-4"></i></button>').on('click',async e=>{e.stopPropagation();state.messages=state.messages.filter(msg=>msg.id!==m.id);$row.remove();await THREAD.persist()});const $copyBtn=$('<button class="ml-auto p-1.5 rounded-lg hover:bg-gray-200 text-gray-400 hover:text-gray-600" title="Copy message"><i data-lucide="copy" class="h-4 w-4"></i></button>').on('click',async function(e){e.stopPropagation();const cur=state.messages.find(x=>x.id===m.id)||m;if(await copyToClipboard(partsToText(cur,true))){$(this).html('<i data-lucide="check" class="h-4 w-4 text-green-500"></i>');icons();setTimeout(()=>{$(this).html('<i data-lucide="copy" class="h-4 w-4"></i>');icons()},1200)}});$head.append($avatar,$name,$copyBtn,$deleteBtn);const $bubble=$(`<div class="${(isUser?'bg-gray-50 border border-gray-200':'bg-gray-100')+' msg-bubble markdown-body rounded-none px-4 py-3 w-full'}"></div>`);$row.append($head,$bubble);const vids=(Array.isArray(meta.content)?meta.content:[]).filter(p=>p?.type==='video_url');if(vids.length)$row.append(videoStrip(vids));return $row}
function msgRow(m){const $row=_createMessageRow(m);$(el.messages).append($row);queueMicrotask(()=>{el.chat.scrollTo({top:el.chat.scrollHeight,behavior:'smooth'});icons()});return $row.find('.msg-bubble')[0]}
const addMessage=window.addMessage=function(m,track=true){m.id=m.id||gid();if(!Array.isArray(m.content)&&m.content!=null){m.content=[{type:'text',text:String(m.content)}]}const bubble=msgRow(m);bubble.dataset.mid=m.id;renderMarkdown(bubble,partsToText(m));if(track)state.messages.push(m);if(m.role==='assistant')el.composer.dispatchEvent(new CustomEvent('sune:newSuneResponse',{detail:{message:m}}));return bubble}
const addSuneBubbleStreaming=(meta,id)=>msgRow(Object.assign({role:'assistant',id},meta))
const clearChat=()=>{el.suneHtml.dispatchEvent(new CustomEvent('sune:unmount'));state.messages=[];el.messages.innerHTML='';state.attachments=[];updateAttachBadge();el.fileInput.value=''}
const clearChat=()=>{el.suneHtml.dispatchEvent(new CustomEvent('sune:unmount'));state.messages=[];el.messages.innerHTML='';clearAttachments();revokeVideoUrls()}
const payloadWithSampling=b=>{const o=Object.assign({},b),s=SUNE,p={temperature:num(s.temperature,null),top_p:num(s.top_p,null),top_k:int(s.top_k,null),frequency_penalty:num(s.frequency_penalty,null),repetition_penalty:num(s.repetition_penalty,null),min_p:num(s.min_p,null),top_a:num(s.top_a,null),max_tokens:int(s.max_tokens,null)>0?int(s.max_tokens,null):null};Object.keys(p).forEach(k=>{const v=p[k];if(v!==null)o[k]=v});return o}
function setBtnStop(){const b=el.sendBtn;b.dataset.mode='stop';b.type='button';b.setAttribute('aria-label','Stop');b.innerHTML='<i data-lucide="square" class="h-5 w-5"></i>';icons();b.onclick=()=>{state.abortRequested=true;state.controller?.abort?.();state.busy=false;setBtnSend()}}
function setBtnSend(){const b=el.sendBtn;b.dataset.mode='send';b.type='submit';b.setAttribute('aria-label','Send');b.innerHTML='<i data-lucide="sparkles" class="h-5 w-5"></i>';icons();b.onclick=null}
@@ -118,23 +119,25 @@ $(el.suneList).on('click',async e=>{const menuBtn=e.target.closest('[data-sune-m
$(el.sunePopover).on('click',async e=>{const act=e.target.closest('[data-action]')?.getAttribute('data-action');if(!act||!menuSuneId)return;const s=SUNE.get(menuSuneId);if(!s)return;const updateAndRender=async()=>{s.updatedAt=Date.now();SUNE.save();renderSidebar();await reflectActiveSune();markLocalDirty()};if(act==='pin'){s.pinned=!s.pinned;await updateAndRender()}else if(act==='rename'){const n=prompt('Rename sune to:',s.name);if(n!=null){s.name=n.trim();await updateAndRender()}}else if(act==='pfp'){const i=document.createElement('input');i.type='file';i.accept='image/*';i.onchange=async()=>{const f=i.files?.[0];if(!f)return;try{s.avatar=await imgToWebp(f);await updateAndRender()}catch{}};i.click()}else if(act==='export')dl(`sune-${(s.name||'sune').replace(/\W/g,'_')}-${ts()}.sune`,[s]);hideSunePopover()})
$(el.sunesSyncUploadBtn).on('click',()=>{hideSunesSyncPopover();performSuneUpload()});
$(el.sunesSyncDownloadBtn).on('click',()=>{hideSunesSyncPopover();performSuneDownload(false)});
function updateAttachBadge(){const n=state.attachments.length;el.attachBadge.textContent=String(n);el.attachBadge.classList.toggle('hidden',n===0)}
const addAttachments=async files=>{for(const f of files){const at=await toAttach(f).catch(()=>null);if(at)state.attachments.push(at)}updateAttachBadge()}
let attachGen=0,attachJob=null,attachQueue=Promise.resolve(),attachWait=false
function updateAttachBadge(){const n=state.attachments.length;el.attachBadge.textContent=attachJob?Math.floor(attachJob.p*100)+'%':String(n);el.attachBadge.classList.toggle('hidden',!n&&!attachJob)}
function clearAttachments(){attachGen++;attachJob?.ac.abort();attachJob=null;state.attachments=[];updateAttachBadge();el.fileInput.value=''}
const addAttachments=files=>{const g=attachGen;attachQueue=attachQueue.then(async()=>{for(const f of files){if(g!==attachGen)return;const job=attachJob={ac:new AbortController,p:0};updateAttachBadge();const at=await toAttach(f,{signal:job.ac.signal,onProgress:p=>{job.p=p;updateAttachBadge()}}).catch(e=>{if(g===attachGen)alert(`Couldn't attach ${f.name||'file'}: ${e?.message||e}`)});if(g!==attachGen)return;attachJob=null;if(at)state.attachments.push(at);updateAttachBadge()}}).catch(()=>{})}
const hideAttachPopover=()=>el.attachPopover.classList.add('hidden')
function showAttachPopover(){if(!state.attachments.length||!el.attachPopover.classList.contains('hidden'))return;haptic();el.attachPopover.classList.remove('hidden');positionPopover(el.attachBtn,el.attachPopover);icons()}
function showAttachPopover(){if((!state.attachments.length&&!attachJob)||!el.attachPopover.classList.contains('hidden'))return;haptic();el.attachPopover.classList.remove('hidden');positionPopover(el.attachBtn,el.attachPopover);icons()}
let attachHold=null,attachHeld=false;const holdAttach=()=>{attachHeld=true;showAttachPopover()}
$(el.attachBtn).on('pointerdown',e=>{attachHeld=false;if(e.pointerType!=='mouse')attachHold=setTimeout(holdAttach,500)}).on('pointerup pointercancel pointerleave',()=>clearTimeout(attachHold)).on('contextmenu',e=>{e.preventDefault();holdAttach()}).on('click',()=>{if(attachHeld)return attachHeld=false;hideAttachPopover();el.fileInput.click()})
$(el.attachPopover).on('click',e=>{if(!e.target.closest('[data-action="clear"]'))return;state.attachments=[];updateAttachBadge();hideAttachPopover()})
$(el.attachPopover).on('click',e=>{if(!e.target.closest('[data-action="clear"]'))return;clearAttachments();hideAttachPopover()})
$(document).on('click',e=>{if(!el.attachPopover.contains(e.target)&&!el.attachBtn.contains(e.target))hideAttachPopover()})
$(window).on('resize',()=>{if(!el.attachPopover.classList.contains('hidden'))positionPopover(el.attachBtn,el.attachPopover)})
$(el.fileInput).on('change',async()=>{const files=[...(el.fileInput.files||[])];el.fileInput.value='';await addAttachments(files)})
$(el.fileInput).on('change',()=>{const files=[...(el.fileInput.files||[])];el.fileInput.value='';addAttachments(files)})
$(el.composer).on('paste',async e=>{
const cbd=e.clipboardData||e.originalEvent?.clipboardData,files=[...(cbd?.files||[])];
if(!files.length)return;
e.preventDefault();
await addAttachments(files);
addAttachments(files);
})
$(el.composer).on('submit',async e=>{e.preventDefault();if(state.busy)return;const text=el.input.value.trim();if(!text&&!state.attachments.length)return SUNE.infer();await ensureThreadOnFirstUser(text||'(attachments)');const th=THREAD.active,shouldGenTitle=th&&!th.title;el.input.value='';const parts=[];if(text)parts.push({type:'text',text});parts.push(...state.attachments);state.attachments=[];updateAttachBadge();el.fileInput.value='';const userMsg={role:'user',content:parts.length?parts:[{type:'text',text:text||'(sent attachments)'}]};addMessage(userMsg);el.composer.dispatchEvent(new CustomEvent('user:send',{detail:{message:userMsg}}));if(shouldGenTitle)(async()=>{const title=await generateTitleWithAI(state.messages)||partsToText(state.messages.find(m=>m.role==='user')).replace(/!\[\]\(data:[^\)]+\)/g,'[Image]')||'Untitled';await THREAD.setTitle(th.id,title)})();if(!SUNE.model)return;state.busy=true;setBtnStop();const a=SUNE.active,suneMeta={sune_name:a.name,model:SUNE.model,avatar:a.avatar||''},streamId=sid(),suneBubble=addSuneBubbleStreaming(suneMeta,streamId);suneBubble.dataset.mid=streamId;suneBubble.innerHTML=SUNE_LOGO_SVG;const assistantMsg=Object.assign({id:streamId,role:'assistant',content:[{type:'text',text:''}]},suneMeta);state.messages.push(assistantMsg);THREAD.persist(false);state.stream={rid:streamId,bubble:suneBubble,meta:suneMeta,text:'',done:false};let buf='',completed=false;const onDelta=(delta,done,imgs)=>{if(imgs){if(!assistantMsg.images)assistantMsg.images=[];assistantMsg.images.push(...imgs)}buf+=delta;state.stream.text=buf;assistantMsg.content[0].text=buf;renderMarkdown(suneBubble,partsToText(assistantMsg),{enhance:false});if(done&&!completed){completed=true;setBtnSend();state.busy=false;enhanceCodeBlocks(suneBubble,true);THREAD.persist(true);el.composer.dispatchEvent(new CustomEvent('sune:newSuneResponse',{detail:{message:assistantMsg}}));state.stream={rid:null,bubble:null,meta:null,text:'',done:false}}else if(!done)THREAD.persist(false)};await streamChat(onDelta,streamId)})
$(el.composer).on('submit',async e=>{e.preventDefault();if(state.busy||attachWait)return;attachWait=true;await attachQueue;attachWait=false;const text=el.input.value.trim();if(!text&&!state.attachments.length)return SUNE.infer();await ensureThreadOnFirstUser(text||'(attachments)');const th=THREAD.active,shouldGenTitle=th&&!th.title;el.input.value='';const parts=[];if(text)parts.push({type:'text',text});parts.push(...state.attachments);state.attachments=[];updateAttachBadge();el.fileInput.value='';const userMsg={role:'user',content:parts.length?parts:[{type:'text',text:text||'(sent attachments)'}]};addMessage(userMsg);el.composer.dispatchEvent(new CustomEvent('user:send',{detail:{message:userMsg}}));if(shouldGenTitle)(async()=>{const title=await generateTitleWithAI(state.messages)||partsToText(state.messages.find(m=>m.role==='user')).replace(/!\[\]\(data:[^\)]+\)/g,'[Image]')||'Untitled';await THREAD.setTitle(th.id,title)})();if(!SUNE.model)return;state.busy=true;setBtnStop();const a=SUNE.active,suneMeta={sune_name:a.name,model:SUNE.model,avatar:a.avatar||''},streamId=sid(),suneBubble=addSuneBubbleStreaming(suneMeta,streamId);suneBubble.dataset.mid=streamId;suneBubble.innerHTML=SUNE_LOGO_SVG;const assistantMsg=Object.assign({id:streamId,role:'assistant',content:[{type:'text',text:''}]},suneMeta);state.messages.push(assistantMsg);THREAD.persist(false);state.stream={rid:streamId,bubble:suneBubble,meta:suneMeta,text:'',done:false};let buf='',completed=false;const onDelta=(delta,done,imgs)=>{if(imgs){if(!assistantMsg.images)assistantMsg.images=[];assistantMsg.images.push(...imgs)}buf+=delta;state.stream.text=buf;assistantMsg.content[0].text=buf;renderMarkdown(suneBubble,partsToText(assistantMsg),{enhance:false});if(done&&!completed){completed=true;setBtnSend();state.busy=false;enhanceCodeBlocks(suneBubble,true);THREAD.persist(true);el.composer.dispatchEvent(new CustomEvent('sune:newSuneResponse',{detail:{message:assistantMsg}}));state.stream={rid:null,bubble:null,meta:null,text:'',done:false}}else if(!done)THREAD.persist(false)};await streamChat(onDelta,streamId)})
let jars={html:null,extension:null};
const ensureJars=async()=>{
if(jars.html&&jars.extension)return jars;