diff --git a/package.json b/package.json index 4cedc83..c1004c0 100644 --- a/package.json +++ b/package.json @@ -12,5 +12,9 @@ "vite": "8.0.*", "vite-plugin-html-inject": "1.1.*", "vite-plugin-pwa": "1.2.*" + }, + "dependencies": { + "@mediabunny/aac-encoder": "1.61.*", + "mediabunny": "1.61.*" } } diff --git a/src/attachments.js b/src/attachments.js index 71f7e88..363a102 100644 --- a/src/attachments.js +++ b/src/attachments.js @@ -1,9 +1,11 @@ import { asDataURL, imgToWebp, b64 } from './utils.js'; +import { isVideo, compressVideo } from './video.js'; -export async function toAttach(file) { +export async function toAttach(file, opts) { if (!file) return null; if (file instanceof File) { const name = file.name || 'file', mime = (file.type || 'application/octet-stream').toLowerCase(); + if (isVideo(name, mime)) return compressVideo(file, opts); if (/^image\//.test(mime) || /\.(png|jpe?g|webp|gif)$/i.test(name)) { const data = mime === 'image/webp' || /\.webp$/i.test(name) ? await asDataURL(file) : await imgToWebp(file, 2048, 94); return { type: 'image_url', image_url: { url: data } }; @@ -28,6 +30,9 @@ export async function toAttach(file) { if (mime === 'application/pdf') { return { type: 'file', file: { filename: name, file_data: file.data } }; } + if (/^video\//.test(mime)) { + return { type: 'video_url', video_url: { url: `data:${mime};base64,${file.data}` } }; + } if (/^audio\//.test(mime)) { const fmt = /mp3/.test(mime) ? 'mp3' : 'wav'; return { type: 'input_audio', input_audio: { data: file.data, format: fmt } }; diff --git a/src/main.js b/src/main.js index 2077693..49188cb 100644 --- a/src/main.js +++ b/src/main.js @@ -9,6 +9,7 @@ import { USER } from './user.js'; import { md, enhanceCodeBlocks, renderMarkdown } from './markdown.js'; import { kbUpdate, kbBind } from './keyboard.js'; import { toAttach } from './attachments.js'; +import { videoStrip, revokeVideoUrls } from './video.js'; import { titleFrom, serializeThreadName, deserializeThreadName } from './threads-utils.js'; import { resolveSuneSrc, processSuneIncludes, renderSuneHTML } from './sune-html.js'; @@ -69,11 +70,11 @@ const suneRow=a=>`
`}else{el.userMenuAvatar.className='h-6 w-6 rounded-full bg-gray-900 text-white flex items-center justify-center shrink-0 text-xs';el.userMenuAvatar.textContent='👤'}} const getSuneLabel=m=>{const name=(m&&m.sune_name)||SUNE.name,modelShort=getModelShort(m&&m.model);return `${name} · ${modelShort}`} -function _createMessageRow(m){const role=typeof m==='string'?m:(m&&m.role)||'assistant',meta=typeof m==='string'?{}:m||{},isUser=role==='user',$row=$('
'),$head=$('
'),$avatar=$('
');const uAva=isUser?USER.avatar:meta.avatar;uAva?$avatar.attr('class','msg-avatar shrink-0 h-7 w-7 rounded-full overflow-hidden').html(``):$avatar.attr('class',`${isUser?'bg-gray-900 text-white':'bg-gray-200 text-gray-900'} msg-avatar shrink-0 h-7 w-7 rounded-full flex items-center justify-center`).text(isUser?'👤':'✺');const $name=$('
').text(isUser?USER.name:getSuneLabel(meta));const $deleteBtn=$('').on('click',async e=>{e.stopPropagation();state.messages=state.messages.filter(msg=>msg.id!==m.id);$row.remove();await THREAD.persist()});const $copyBtn=$('').on('click',async function(e){e.stopPropagation();const cur=state.messages.find(x=>x.id===m.id)||m;if(await copyToClipboard(partsToText(cur,true))){$(this).html('');icons();setTimeout(()=>{$(this).html('');icons()},1200)}});$head.append($avatar,$name,$copyBtn,$deleteBtn);const $bubble=$(`
`);$row.append($head,$bubble);return $row} +function _createMessageRow(m){const role=typeof m==='string'?m:(m&&m.role)||'assistant',meta=typeof m==='string'?{}:m||{},isUser=role==='user',$row=$('
'),$head=$('
'),$avatar=$('
');const uAva=isUser?USER.avatar:meta.avatar;uAva?$avatar.attr('class','msg-avatar shrink-0 h-7 w-7 rounded-full overflow-hidden').html(``):$avatar.attr('class',`${isUser?'bg-gray-900 text-white':'bg-gray-200 text-gray-900'} msg-avatar shrink-0 h-7 w-7 rounded-full flex items-center justify-center`).text(isUser?'👤':'✺');const $name=$('
').text(isUser?USER.name:getSuneLabel(meta));const $deleteBtn=$('').on('click',async e=>{e.stopPropagation();state.messages=state.messages.filter(msg=>msg.id!==m.id);$row.remove();await THREAD.persist()});const $copyBtn=$('').on('click',async function(e){e.stopPropagation();const cur=state.messages.find(x=>x.id===m.id)||m;if(await copyToClipboard(partsToText(cur,true))){$(this).html('');icons();setTimeout(()=>{$(this).html('');icons()},1200)}});$head.append($avatar,$name,$copyBtn,$deleteBtn);const $bubble=$(`
`);$row.append($head,$bubble);const vids=(Array.isArray(meta.content)?meta.content:[]).filter(p=>p?.type==='video_url');if(vids.length)$row.append(videoStrip(vids));return $row} function msgRow(m){const $row=_createMessageRow(m);$(el.messages).append($row);queueMicrotask(()=>{el.chat.scrollTo({top:el.chat.scrollHeight,behavior:'smooth'});icons()});return $row.find('.msg-bubble')[0]} const addMessage=window.addMessage=function(m,track=true){m.id=m.id||gid();if(!Array.isArray(m.content)&&m.content!=null){m.content=[{type:'text',text:String(m.content)}]}const bubble=msgRow(m);bubble.dataset.mid=m.id;renderMarkdown(bubble,partsToText(m));if(track)state.messages.push(m);if(m.role==='assistant')el.composer.dispatchEvent(new CustomEvent('sune:newSuneResponse',{detail:{message:m}}));return bubble} const addSuneBubbleStreaming=(meta,id)=>msgRow(Object.assign({role:'assistant',id},meta)) -const clearChat=()=>{el.suneHtml.dispatchEvent(new CustomEvent('sune:unmount'));state.messages=[];el.messages.innerHTML='';state.attachments=[];updateAttachBadge();el.fileInput.value=''} +const clearChat=()=>{el.suneHtml.dispatchEvent(new CustomEvent('sune:unmount'));state.messages=[];el.messages.innerHTML='';clearAttachments();revokeVideoUrls()} const payloadWithSampling=b=>{const o=Object.assign({},b),s=SUNE,p={temperature:num(s.temperature,null),top_p:num(s.top_p,null),top_k:int(s.top_k,null),frequency_penalty:num(s.frequency_penalty,null),repetition_penalty:num(s.repetition_penalty,null),min_p:num(s.min_p,null),top_a:num(s.top_a,null),max_tokens:int(s.max_tokens,null)>0?int(s.max_tokens,null):null};Object.keys(p).forEach(k=>{const v=p[k];if(v!==null)o[k]=v});return o} function setBtnStop(){const b=el.sendBtn;b.dataset.mode='stop';b.type='button';b.setAttribute('aria-label','Stop');b.innerHTML='';icons();b.onclick=()=>{state.abortRequested=true;state.controller?.abort?.();state.busy=false;setBtnSend()}} function setBtnSend(){const b=el.sendBtn;b.dataset.mode='send';b.type='submit';b.setAttribute('aria-label','Send');b.innerHTML='';icons();b.onclick=null} @@ -118,23 +119,25 @@ $(el.suneList).on('click',async e=>{const menuBtn=e.target.closest('[data-sune-m $(el.sunePopover).on('click',async e=>{const act=e.target.closest('[data-action]')?.getAttribute('data-action');if(!act||!menuSuneId)return;const s=SUNE.get(menuSuneId);if(!s)return;const updateAndRender=async()=>{s.updatedAt=Date.now();SUNE.save();renderSidebar();await reflectActiveSune();markLocalDirty()};if(act==='pin'){s.pinned=!s.pinned;await updateAndRender()}else if(act==='rename'){const n=prompt('Rename sune to:',s.name);if(n!=null){s.name=n.trim();await updateAndRender()}}else if(act==='pfp'){const i=document.createElement('input');i.type='file';i.accept='image/*';i.onchange=async()=>{const f=i.files?.[0];if(!f)return;try{s.avatar=await imgToWebp(f);await updateAndRender()}catch{}};i.click()}else if(act==='export')dl(`sune-${(s.name||'sune').replace(/\W/g,'_')}-${ts()}.sune`,[s]);hideSunePopover()}) $(el.sunesSyncUploadBtn).on('click',()=>{hideSunesSyncPopover();performSuneUpload()}); $(el.sunesSyncDownloadBtn).on('click',()=>{hideSunesSyncPopover();performSuneDownload(false)}); -function updateAttachBadge(){const n=state.attachments.length;el.attachBadge.textContent=String(n);el.attachBadge.classList.toggle('hidden',n===0)} -const addAttachments=async files=>{for(const f of files){const at=await toAttach(f).catch(()=>null);if(at)state.attachments.push(at)}updateAttachBadge()} +let attachGen=0,attachJob=null,attachQueue=Promise.resolve(),attachWait=false +function updateAttachBadge(){const n=state.attachments.length;el.attachBadge.textContent=attachJob?Math.floor(attachJob.p*100)+'%':String(n);el.attachBadge.classList.toggle('hidden',!n&&!attachJob)} +function clearAttachments(){attachGen++;attachJob?.ac.abort();attachJob=null;state.attachments=[];updateAttachBadge();el.fileInput.value=''} +const addAttachments=files=>{const g=attachGen;attachQueue=attachQueue.then(async()=>{for(const f of files){if(g!==attachGen)return;const job=attachJob={ac:new AbortController,p:0};updateAttachBadge();const at=await toAttach(f,{signal:job.ac.signal,onProgress:p=>{job.p=p;updateAttachBadge()}}).catch(e=>{if(g===attachGen)alert(`Couldn't attach ${f.name||'file'}: ${e?.message||e}`)});if(g!==attachGen)return;attachJob=null;if(at)state.attachments.push(at);updateAttachBadge()}}).catch(()=>{})} const hideAttachPopover=()=>el.attachPopover.classList.add('hidden') -function showAttachPopover(){if(!state.attachments.length||!el.attachPopover.classList.contains('hidden'))return;haptic();el.attachPopover.classList.remove('hidden');positionPopover(el.attachBtn,el.attachPopover);icons()} +function showAttachPopover(){if((!state.attachments.length&&!attachJob)||!el.attachPopover.classList.contains('hidden'))return;haptic();el.attachPopover.classList.remove('hidden');positionPopover(el.attachBtn,el.attachPopover);icons()} let attachHold=null,attachHeld=false;const holdAttach=()=>{attachHeld=true;showAttachPopover()} $(el.attachBtn).on('pointerdown',e=>{attachHeld=false;if(e.pointerType!=='mouse')attachHold=setTimeout(holdAttach,500)}).on('pointerup pointercancel pointerleave',()=>clearTimeout(attachHold)).on('contextmenu',e=>{e.preventDefault();holdAttach()}).on('click',()=>{if(attachHeld)return attachHeld=false;hideAttachPopover();el.fileInput.click()}) -$(el.attachPopover).on('click',e=>{if(!e.target.closest('[data-action="clear"]'))return;state.attachments=[];updateAttachBadge();hideAttachPopover()}) +$(el.attachPopover).on('click',e=>{if(!e.target.closest('[data-action="clear"]'))return;clearAttachments();hideAttachPopover()}) $(document).on('click',e=>{if(!el.attachPopover.contains(e.target)&&!el.attachBtn.contains(e.target))hideAttachPopover()}) $(window).on('resize',()=>{if(!el.attachPopover.classList.contains('hidden'))positionPopover(el.attachBtn,el.attachPopover)}) -$(el.fileInput).on('change',async()=>{const files=[...(el.fileInput.files||[])];el.fileInput.value='';await addAttachments(files)}) +$(el.fileInput).on('change',()=>{const files=[...(el.fileInput.files||[])];el.fileInput.value='';addAttachments(files)}) $(el.composer).on('paste',async e=>{ const cbd=e.clipboardData||e.originalEvent?.clipboardData,files=[...(cbd?.files||[])]; if(!files.length)return; e.preventDefault(); - await addAttachments(files); + addAttachments(files); }) -$(el.composer).on('submit',async e=>{e.preventDefault();if(state.busy)return;const text=el.input.value.trim();if(!text&&!state.attachments.length)return SUNE.infer();await ensureThreadOnFirstUser(text||'(attachments)');const th=THREAD.active,shouldGenTitle=th&&!th.title;el.input.value='';const parts=[];if(text)parts.push({type:'text',text});parts.push(...state.attachments);state.attachments=[];updateAttachBadge();el.fileInput.value='';const userMsg={role:'user',content:parts.length?parts:[{type:'text',text:text||'(sent attachments)'}]};addMessage(userMsg);el.composer.dispatchEvent(new CustomEvent('user:send',{detail:{message:userMsg}}));if(shouldGenTitle)(async()=>{const title=await generateTitleWithAI(state.messages)||partsToText(state.messages.find(m=>m.role==='user')).replace(/!\[\]\(data:[^\)]+\)/g,'[Image]')||'Untitled';await THREAD.setTitle(th.id,title)})();if(!SUNE.model)return;state.busy=true;setBtnStop();const a=SUNE.active,suneMeta={sune_name:a.name,model:SUNE.model,avatar:a.avatar||''},streamId=sid(),suneBubble=addSuneBubbleStreaming(suneMeta,streamId);suneBubble.dataset.mid=streamId;suneBubble.innerHTML=SUNE_LOGO_SVG;const assistantMsg=Object.assign({id:streamId,role:'assistant',content:[{type:'text',text:''}]},suneMeta);state.messages.push(assistantMsg);THREAD.persist(false);state.stream={rid:streamId,bubble:suneBubble,meta:suneMeta,text:'',done:false};let buf='',completed=false;const onDelta=(delta,done,imgs)=>{if(imgs){if(!assistantMsg.images)assistantMsg.images=[];assistantMsg.images.push(...imgs)}buf+=delta;state.stream.text=buf;assistantMsg.content[0].text=buf;renderMarkdown(suneBubble,partsToText(assistantMsg),{enhance:false});if(done&&!completed){completed=true;setBtnSend();state.busy=false;enhanceCodeBlocks(suneBubble,true);THREAD.persist(true);el.composer.dispatchEvent(new CustomEvent('sune:newSuneResponse',{detail:{message:assistantMsg}}));state.stream={rid:null,bubble:null,meta:null,text:'',done:false}}else if(!done)THREAD.persist(false)};await streamChat(onDelta,streamId)}) +$(el.composer).on('submit',async e=>{e.preventDefault();if(state.busy||attachWait)return;attachWait=true;await attachQueue;attachWait=false;const text=el.input.value.trim();if(!text&&!state.attachments.length)return SUNE.infer();await ensureThreadOnFirstUser(text||'(attachments)');const th=THREAD.active,shouldGenTitle=th&&!th.title;el.input.value='';const parts=[];if(text)parts.push({type:'text',text});parts.push(...state.attachments);state.attachments=[];updateAttachBadge();el.fileInput.value='';const userMsg={role:'user',content:parts.length?parts:[{type:'text',text:text||'(sent attachments)'}]};addMessage(userMsg);el.composer.dispatchEvent(new CustomEvent('user:send',{detail:{message:userMsg}}));if(shouldGenTitle)(async()=>{const title=await generateTitleWithAI(state.messages)||partsToText(state.messages.find(m=>m.role==='user')).replace(/!\[\]\(data:[^\)]+\)/g,'[Image]')||'Untitled';await THREAD.setTitle(th.id,title)})();if(!SUNE.model)return;state.busy=true;setBtnStop();const a=SUNE.active,suneMeta={sune_name:a.name,model:SUNE.model,avatar:a.avatar||''},streamId=sid(),suneBubble=addSuneBubbleStreaming(suneMeta,streamId);suneBubble.dataset.mid=streamId;suneBubble.innerHTML=SUNE_LOGO_SVG;const assistantMsg=Object.assign({id:streamId,role:'assistant',content:[{type:'text',text:''}]},suneMeta);state.messages.push(assistantMsg);THREAD.persist(false);state.stream={rid:streamId,bubble:suneBubble,meta:suneMeta,text:'',done:false};let buf='',completed=false;const onDelta=(delta,done,imgs)=>{if(imgs){if(!assistantMsg.images)assistantMsg.images=[];assistantMsg.images.push(...imgs)}buf+=delta;state.stream.text=buf;assistantMsg.content[0].text=buf;renderMarkdown(suneBubble,partsToText(assistantMsg),{enhance:false});if(done&&!completed){completed=true;setBtnSend();state.busy=false;enhanceCodeBlocks(suneBubble,true);THREAD.persist(true);el.composer.dispatchEvent(new CustomEvent('sune:newSuneResponse',{detail:{message:assistantMsg}}));state.stream={rid:null,bubble:null,meta:null,text:'',done:false}}else if(!done)THREAD.persist(false)};await streamChat(onDelta,streamId)}) let jars={html:null,extension:null}; const ensureJars=async()=>{ if(jars.html&&jars.extension)return jars; diff --git a/src/parts/footer.html b/src/parts/footer.html index 9cf8f66..c05967e 100644 --- a/src/parts/footer.html +++ b/src/parts/footer.html @@ -6,7 +6,7 @@
- + diff --git a/src/video.js b/src/video.js new file mode 100644 index 0000000..4a9cfba --- /dev/null +++ b/src/video.js @@ -0,0 +1,81 @@ +import { asDataURL } from './utils.js'; + +// Videos are shrunk to 360p @ 4 fps before attaching; HEVC when the device can encode it, H.264 otherwise. +const SHORT_SIDE = 360, FPS = 4, VIDEO_BPS = { hevc: 130e3, avc: 160e3 }, AUDIO_BPS = 32e3; +const even = n => Math.max(2, Math.round(n / 2) * 2); +let mb; +const loadMb = async () => { + if (mb) return mb; + const m = await import('mediabunny'); + if (!(await m.canEncodeAudio('aac', { numberOfChannels: 1, sampleRate: 48000, bitrate: AUDIO_BPS }))) (await import('@mediabunny/aac-encoder')).registerAacEncoder(); + return mb = m; +}; + +export const isVideo = (name, mime) => /^video\//.test(mime) || /\.(mp4|m4v|mov|webm|mkv|3gp)$/i.test(name); + +export async function compressVideo(file, { onProgress, signal } = {}) { + const { Input, Output, Conversion, ALL_FORMATS, BlobSource, BufferTarget, Mp4OutputFormat, Quality, canEncodeVideo } = await loadMb(); + const input = new Input({ source: new BlobSource(file), formats: ALL_FORMATS }), track = await input.getPrimaryVideoTrack(); + if (!track) throw new Error('no video track'); + const w = await track.getDisplayWidth(), h = await track.getDisplayHeight(), s = Math.min(1, SHORT_SIDE / Math.min(w, h)), width = even(w * s), height = even(h * s); + let codec = null; + for (const c of ['hevc', 'avc']) if (await canEncodeVideo(c, { width, height, frameRate: FPS, bitrate: VIDEO_BPS[c] })) { codec = c; break; } + if (!codec) throw new Error('this device cannot encode HEVC or H.264'); + const output = new Output({ format: new Mp4OutputFormat({ fastStart: 'in-memory' }), target: new BufferTarget() }); + const video = { frameRate: FPS, codec, quality: new Quality({ bitrate: VIDEO_BPS[codec] }), forceTranscode: true }; + // Mediabunny resizes every decoded frame before dropping to FPS, so resize in `process` (runs after the drop) instead. + // `draw` bakes in each frame's rotation, so cancelling the track rotation skips Mediabunny's per-frame re-render too. + if (await track.getFlip()) Object.assign(video, { width, height, fit: 'fill' }); + else { + const cx = new OffscreenCanvas(width, height).getContext('2d'), rot = await track.getRotation(); + cx.imageSmoothingQuality = 'high'; + Object.assign(video, { process: f => (f.draw(cx, 0, 0, width, height), cx.canvas), processedWidth: width, processedHeight: height, rotate: (360 - rot) % 360 }); + } + const conv = await Conversion.init({ + input, output, tracks: 'primary', video, + audio: { numberOfChannels: 1, codec: 'aac', quality: new Quality({ bitrate: AUDIO_BPS }), forceTranscode: true } + }); + if (!conv.isValid) throw new Error(conv.discardedTracks.map(t => t.reason).join(', ') || 'unsupported video'); + if (signal) { if (signal.aborted) return null; signal.addEventListener('abort', () => conv.cancel(), { once: true }); } + conv.onProgress = p => onProgress?.(p); + try { await conv.execute(); } catch (e) { if (signal?.aborted) return null; throw e; } + return { type: 'video_url', video_url: { url: await asDataURL(new Blob([output.target.buffer], { type: 'video/mp4' })) } }; +} + +const blobUrls = new Set(); +const toBlobUrl = async u => { const o = URL.createObjectURL(await (await fetch(u)).blob()); blobUrls.add(o); return o; }; +export const revokeVideoUrls = () => { blobUrls.forEach(u => URL.revokeObjectURL(u)); blobUrls.clear(); }; + +function openPlayer(src) { + const o = document.createElement('div'); + o.className = 'fixed inset-0 z-[70] bg-black flex items-center justify-center'; + o.innerHTML = ''; + const v = o.querySelector('video'), done = () => { v.pause(); o.remove(); removeEventListener('keydown', esc); document.fullscreenElement && document.exitFullscreen().catch(() => {}); }, + esc = e => e.key === 'Escape' && history.back(); + v.src = src; + history.pushState({ video: 1 }, ''); + addEventListener('popstate', done, { once: true }); + addEventListener('keydown', esc); + o.querySelector('button').onclick = () => history.back(); + o.addEventListener('fullscreenchange', () => !document.fullscreenElement && o.isConnected && history.back()); + document.body.append(o); + window.lucide?.createIcons(); + o.requestFullscreen?.().catch(() => {}); +} + +export function videoStrip(parts) { + const wrap = document.createElement('div'); + wrap.className = 'flex gap-2 overflow-x-auto no-scrollbar px-4'; + for (const p of parts) { + const b = document.createElement('button'); + b.type = 'button'; + b.setAttribute('aria-label', 'Play video'); + b.className = 'relative shrink-0 h-40 rounded-xl overflow-hidden bg-black'; + b.innerHTML = ''; + const src = toBlobUrl(p.video_url?.url || ''); + src.then(u => b.querySelector('video').src = u + '#t=0.1').catch(() => {}); + b.onclick = async () => openPlayer(await src); + wrap.append(b); + } + return wrap; +}