This commit is contained in:
Timothy Jaeryang Baek
2026-06-12 18:07:24 +01:00
Unverified
parent daa5554c0e
commit 7f4a05373d
10 changed files with 207 additions and 49 deletions
+11
View File
@@ -5,6 +5,17 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [0.3.2] - 2026-06-12
### Added
- 🎙️ **Voice memos.** Record audio memos directly from the "+" menu or ⌘⇧M. Recordings are saved to your workspace as audio files with an auto-generated markdown transcript. Uses a save-first architecture with IndexedDB backup so your recording is never lost, even if the network drops. Transcription is powered by any OpenAI-compatible STT API (Whisper, etc.), configurable in Settings > Audio.
- 🔊 **Audio admin settings.** New Audio tab in admin settings with separate controls for enabling voice memos, toggling auto-transcription, and configuring STT credentials (base URL, API key, model).
### Changed
- 🎨 **Consistent admin input styling.** Standardized all input fields in the Configuration admin panel to match the Browser settings design. Uniform height, background, focus states, and label sizing across all admin tabs.
## [0.3.1] - 2026-06-12
### Added
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "frontend",
"private": true,
"version": "0.3.1",
"version": "0.3.2",
"type": "module",
"scripts": {
"dev": "vite dev",
@@ -11,7 +11,7 @@
let saving = $state(false);
// Config state
let voiceNotesEnabled = $state(false);
let voiceMemosEnabled = $state(false);
let transcribeEnabled = $state(true);
let sttBaseUrl = $state('https://api.openai.com/v1');
let sttApiKey = $state('');
@@ -21,7 +21,7 @@
onMount(async () => {
try {
const config = await getAdminConfig();
voiceNotesEnabled = config['audio.voice_notes_enabled'] === true;
voiceMemosEnabled = config['audio.voice_memos_enabled'] === true;
transcribeEnabled = config['audio.transcribe_enabled'] !== false;
sttBaseUrl = (config['audio.stt_base_url'] as string) || 'https://api.openai.com/v1';
sttModel = (config['audio.stt_model'] as string) || 'whisper-1';
@@ -34,7 +34,7 @@
saving = true;
try {
const cfg: Record<string, unknown> = {
'audio.voice_notes_enabled': voiceNotesEnabled,
'audio.voice_memos_enabled': voiceMemosEnabled,
'audio.transcribe_enabled': transcribeEnabled,
'audio.stt_base_url': sttBaseUrl,
'audio.stt_model': sttModel
@@ -61,12 +61,12 @@
<div class="flex justify-center py-8"><Spinner size={16} /></div>
{:else}
<!-- Voice Notes -->
<h3 class="text-xs text-gray-400 dark:text-gray-600 mb-2">Voice Notes</h3>
<h3 class="text-xs text-gray-400 dark:text-gray-600 mb-2">Voice Memos</h3>
<div class="flex flex-col gap-2.5">
<label class="flex items-center justify-between cursor-pointer">
<span class="text-xs text-gray-600 dark:text-gray-400">Enable Voice Notes</span>
<ToggleSwitch value={voiceNotesEnabled} onchange={(v) => { voiceNotesEnabled = v; }} />
<span class="text-xs text-gray-600 dark:text-gray-400">Enable Voice Memos</span>
<ToggleSwitch value={voiceMemosEnabled} onchange={(v) => { voiceMemosEnabled = v; }} />
</label>
<p class="text-[11px] text-gray-400 dark:text-gray-600 -mt-1">
Record voice memos from the "+" menu.
@@ -21,14 +21,14 @@
} from '$lib/stores';
import { openChatTab } from '$lib/stores';
import { chatEnabled, streamingChatTabs } from '$lib/stores/chat';
import { voiceNotesEnabled, showVoiceNote } from '$lib/stores/audio';
import { voiceMemosEnabled, showVoiceMemo } from '$lib/stores/audio';
import { keybindings, formatChord } from '$lib/stores/keybindings';
import Icon from './Icon.svelte';
import Spinner from './common/Spinner.svelte';
import DropdownMenu from './DropdownMenu.svelte';
import { tooltip } from '$lib/tooltip';
import { t } from '$lib/i18n';
import VoiceNoteModal from './VoiceNoteModal.svelte';
import VoiceMemoModal from './VoiceMemoModal.svelte';
interface Props {
group: EditorGroup;
@@ -162,14 +162,14 @@
openTerminalTab(group.id);
}
},
...($voiceNotesEnabled
...($voiceMemosEnabled
? [
{
label: 'Voice Note',
label: 'Voice Memo',
icon: 'microphone',
shortcut: formatChord($keybindings.voiceNote),
shortcut: formatChord($keybindings.voiceMemo),
onclick: () => {
showVoiceNote.set(true);
showVoiceMemo.set(true);
}
}
]
@@ -405,11 +405,11 @@
/>
{/if}
{#if $showVoiceNote}
<VoiceNoteModal
{#if $showVoiceMemo}
<VoiceMemoModal
workspace={$activeWorkspace?.path ?? ''}
directory={$activeWorkspace?.fileBrowserCwd ?? $activeWorkspace?.path ?? ''}
onclose={() => showVoiceNote.set(false)}
onclose={() => showVoiceMemo.set(false)}
/>
{/if}
+5 -5
View File
@@ -1,21 +1,21 @@
/**
* Audio state: voice notes feature flag, transcription toggle, and modal visibility.
* Audio state: voice memos feature flag, transcription toggle, and modal visibility.
*/
import { writable } from 'svelte/store';
import { fetchJSON } from '$lib/apis';
export const voiceNotesEnabled = writable<boolean>(false);
export const voiceMemosEnabled = writable<boolean>(false);
export const transcribeEnabled = writable<boolean>(true);
export const showVoiceNote = writable<boolean>(false);
export const showVoiceMemo = writable<boolean>(false);
export async function refreshAudioState() {
try {
const data = await fetchJSON<{ config: Record<string, unknown> }>(
'/api/admin/config/audio'
);
voiceNotesEnabled.set(data.config?.['audio.voice_notes_enabled'] === true);
voiceMemosEnabled.set(data.config?.['audio.voice_memos_enabled'] === true);
transcribeEnabled.set(data.config?.['audio.transcribe_enabled'] !== false);
} catch {
voiceNotesEnabled.set(false);
voiceMemosEnabled.set(false);
}
}
+6 -6
View File
@@ -39,7 +39,7 @@ export const ACTION_IDS = [
'openSettings',
'toggleSplit',
'toggleSidebar',
'voiceNote'
'voiceMemo'
] as const;
export type ActionId = (typeof ACTION_IDS)[number];
@@ -58,7 +58,7 @@ export const ACTION_LABELS: Record<ActionId, string> = {
openSettings: 'Open Settings',
toggleSplit: 'Toggle Split',
toggleSidebar: 'Toggle Sidebar',
voiceNote: 'Voice Note'
voiceMemo: 'Voice Memo'
};
// ── Default bindings (browser-safe) ─────────────────────────────
@@ -75,7 +75,7 @@ export const DEFAULT_KEYBINDINGS: Record<ActionId, string> = {
openSettings: 'Cmd+.',
toggleSplit: 'Cmd+\\',
toggleSidebar: 'Cmd+Shift+S',
voiceNote: 'Cmd+Shift+M'
voiceMemo: 'Cmd+Shift+M'
};
// ── Store ───────────────────────────────────────────────────────
@@ -238,7 +238,7 @@ export function executeAction(
toggleQuickOpen?: () => void;
toggleSettings?: () => void;
toggleSearch?: () => void;
toggleVoiceNote?: () => void;
toggleVoiceMemo?: () => void;
}
): boolean {
switch (action) {
@@ -305,8 +305,8 @@ export function executeAction(
sidebarOpen.update((v) => !v);
return true;
case 'voiceNote':
callbacks?.toggleVoiceNote?.();
case 'voiceMemo':
callbacks?.toggleVoiceMemo?.();
return true;
default:
+3 -3
View File
@@ -225,10 +225,10 @@
toggleSearch: () => {
showSearch.update((v) => !v);
},
toggleVoiceNote: () => {
import('$lib/stores/audio').then(({ voiceNotesEnabled, showVoiceNote }) => {
toggleVoiceMemo: () => {
import('$lib/stores/audio').then(({ voiceMemosEnabled, showVoiceMemo }) => {
import('svelte/store').then(({ get }) => {
if (get(voiceNotesEnabled)) showVoiceNote.update((v) => !v);
if (get(voiceMemosEnabled)) showVoiceMemo.update((v) => !v);
});
});
}
+165 -18
View File
@@ -1,8 +1,16 @@
"""Audio router: STT transcription via Whisper-compatible API."""
"""Audio router: STT transcription via Whisper-compatible API.
Supports optional pydub/ffmpeg for compressing and splitting long recordings.
If pydub is not installed, raw audio is sent directly (works for < 25MB files).
"""
from __future__ import annotations
import asyncio
import logging
import os
import tempfile
from pathlib import Path
import httpx
from fastapi import APIRouter, HTTPException, Request, UploadFile
@@ -13,9 +21,18 @@ from cptr.utils.crypto import decrypt_key
logger = logging.getLogger(__name__)
# Optional pydub for long audio handling
try:
from pydub import AudioSegment # type: ignore[import-untyped]
HAS_PYDUB = True
except ImportError:
HAS_PYDUB = False
router = APIRouter(prefix="/api/audio", tags=["audio"])
COOKIE_NAME = "cptr_session"
MAX_FILE_SIZE = 25 * 1024 * 1024 # 25MB Whisper API limit
def _get_user(request: Request) -> str:
@@ -28,11 +45,96 @@ def _get_user(request: Request) -> str:
return auth.user_id
# ── Audio processing (optional, requires pydub + ffmpeg) ─────────
def compress_audio(file_path: str) -> str:
"""Compress audio to 16kHz mono 32kbps MP3 if larger than MAX_FILE_SIZE."""
if not HAS_PYDUB or os.path.getsize(file_path) <= MAX_FILE_SIZE:
return file_path
audio = AudioSegment.from_file(file_path)
audio = audio.set_frame_rate(16000).set_channels(1)
base, _ = os.path.splitext(file_path)
compressed_path = f"{base}_compressed.mp3"
audio.export(compressed_path, format="mp3", bitrate="32k")
return compressed_path
def split_audio(file_path: str, max_bytes: int) -> list[str]:
"""Split audio into chunks not exceeding max_bytes.
Returns a list of chunk file paths. If audio fits, returns [file_path].
"""
file_size = os.path.getsize(file_path)
if file_size <= max_bytes:
return [file_path]
if not HAS_PYDUB:
return [file_path] # Can't split without pydub
audio = AudioSegment.from_file(file_path)
duration_ms = len(audio)
approx_chunk_ms = max(int(duration_ms * (max_bytes / file_size)) - 1000, 1000)
chunks: list[str] = []
start = 0
i = 0
base, _ = os.path.splitext(file_path)
while start < duration_ms:
end = min(start + approx_chunk_ms, duration_ms)
chunk = audio[start:end]
chunk_path = f"{base}_chunk_{i}.mp3"
chunk.export(chunk_path, format="mp3", bitrate="32k")
# Halve chunk duration if still too large
while os.path.getsize(chunk_path) > max_bytes and (end - start) > 5000:
end = start + ((end - start) // 2)
chunk = audio[start:end]
chunk.export(chunk_path, format="mp3", bitrate="32k")
if os.path.getsize(chunk_path) > max_bytes:
os.remove(chunk_path)
raise RuntimeError("Audio chunk cannot be reduced below max file size.")
chunks.append(chunk_path)
start = end
i += 1
return chunks
# ── Transcription ────────────────────────────────────────────────
async def _transcribe_chunk(
data: bytes,
filename: str,
content_type: str,
base_url: str,
api_key: str,
model: str,
) -> str:
"""Send a single audio chunk to the STT API."""
async with httpx.AsyncClient(timeout=httpx.Timeout(120)) as client:
resp = await client.post(
f"{base_url}/audio/transcriptions",
headers={"Authorization": f"Bearer {api_key}"},
files={"file": (filename, data, content_type)},
data={"model": model},
)
resp.raise_for_status()
return resp.json().get("text", "")
@router.post("/transcribe")
async def transcribe(file: UploadFile, request: Request):
"""Send audio to a Whisper-compatible STT API, return transcript text.
Reads credentials from audio.stt_* config keys (set in Admin → Audio).
If pydub + ffmpeg are available and the file is large, it will be
compressed and split into chunks that are transcribed concurrently.
"""
_get_user(request)
@@ -47,27 +149,72 @@ async def transcribe(file: UploadFile, request: Request):
base_url = (await Config.get("audio.stt_base_url")) or "https://api.openai.com/v1"
model = (await Config.get("audio.stt_model")) or "whisper-1"
data = await file.read()
raw_data = await file.read()
filename = file.filename or "audio.webm"
content_type = file.content_type or "audio/webm"
# Small file: send directly, no temp files needed
if len(raw_data) <= MAX_FILE_SIZE or not HAS_PYDUB:
try:
text = await _transcribe_chunk(raw_data, filename, content_type, base_url, api_key, model)
return {"text": text}
except httpx.HTTPStatusError as exc:
detail = f"STT API error: {exc.response.status_code}"
if len(raw_data) > MAX_FILE_SIZE and not HAS_PYDUB:
detail += ". Recording is too large. Install ffmpeg and pydub for automatic splitting."
logger.warning("[transcribe] %s: %s", detail, exc.response.text[:500])
raise HTTPException(502, detail)
except httpx.ConnectError:
raise HTTPException(502, "Could not connect to STT API")
# Large file: compress → split → transcribe chunks concurrently
tmp_dir = tempfile.mkdtemp(prefix="cptr-stt-")
file_path = os.path.join(tmp_dir, filename)
Path(file_path).write_bytes(raw_data)
chunk_paths: list[str] = []
try:
async with httpx.AsyncClient(timeout=httpx.Timeout(60)) as client:
resp = await client.post(
f"{base_url}/audio/transcriptions",
headers={"Authorization": f"Bearer {api_key}"},
files={"file": (filename, data, content_type)},
data={"model": model},
)
resp.raise_for_status()
# Compress
compressed = await asyncio.to_thread(compress_audio, file_path)
# Split
chunk_paths = await asyncio.to_thread(split_audio, compressed, MAX_FILE_SIZE)
# Transcribe all chunks concurrently
async def _do_chunk(path: str) -> str:
chunk_data = await asyncio.to_thread(Path(path).read_bytes)
chunk_name = os.path.basename(path)
return await _transcribe_chunk(chunk_data, chunk_name, "audio/mpeg", base_url, api_key, model)
tasks = [_do_chunk(p) for p in chunk_paths]
# Use gather to preserve order (as_completed doesn't guarantee it)
results = await asyncio.gather(*tasks)
return {"text": " ".join(r for r in results if r)}
except httpx.HTTPStatusError as exc:
logger.warning(
"[transcribe] STT API error %s: %s",
exc.response.status_code,
exc.response.text[:500],
)
logger.warning("[transcribe] STT API error %s: %s", exc.response.status_code, exc.response.text[:500])
raise HTTPException(502, f"STT API error: {exc.response.status_code}")
except httpx.ConnectError:
raise HTTPException(502, "Could not connect to STT API")
return {"text": resp.json().get("text", "")}
except RuntimeError as exc:
raise HTTPException(400, str(exc))
finally:
# Clean up temp files
for p in chunk_paths:
if p != file_path and os.path.isfile(p):
try:
os.remove(p)
except OSError:
pass
# Clean up compressed file if different from original
if "compressed" in locals() and compressed != file_path and os.path.isfile(compressed):
try:
os.remove(compressed)
except OSError:
pass
try:
os.remove(file_path)
os.rmdir(tmp_dir)
except OSError:
pass
+1 -1
View File
@@ -1,6 +1,6 @@
[project]
name = "cptr"
version = "0.3.1"
version = "0.3.2"
description = "Your computer, from anywhere. Code, manage, and control your machine from the web."
license = {file = "LICENSE"}
readme = "README.md"