From fcc1f8ac9e082d2c61af3ae200ccb29d51732c50 Mon Sep 17 00:00:00 2001 From: yibo Date: Wed, 3 Jun 2026 00:22:34 +0800 Subject: [PATCH] feat(lifelog-recorder): v0.4 — 段播放 + bottom sheet + 声纹回填 --- lifelog-recorder/HANDOFF.md | 10 ++++++---- lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/MainActivity.kt | 98 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++------ lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/asr/SpeakerMatcher.kt | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++ lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/audio/SegmentPlayer.kt | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 219 insertions(+), 10 deletions(-) create mode 100644 lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/asr/SpeakerMatcher.kt create mode 100644 lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/audio/SegmentPlayer.kt diff --git a/lifelog-recorder/HANDOFF.md b/lifelog-recorder/HANDOFF.md index 715e3e0..6f6e913 100644 --- a/lifelog-recorder/HANDOFF.md +++ b/lifelog-recorder/HANDOFF.md @@ -47,10 +47,12 @@ app/src/main/kotlin/com/lemon/lifelog/ │ └── BootReceiver.kt — BOOT_COMPLETED 自启 (Android 14 有 caveat 见 §七) ├── audio/ │ ├── AudioCapture.kt — AudioRecord wrapper，emits FloatArray frames -│ └── SegmentWriter.kt — Opus encode via MediaCodec + Ogg muxer，~17× 压缩 +│ ├── SegmentWriter.kt — Opus encode via MediaCodec + Ogg muxer，~17× 压缩 +│ └── SegmentPlayer.kt — MediaPlayer 包装，Opus 段原音播放（系统原生支持） ├── asr/ │ ├── ParaformerEngine.kt — sherpa-onnx paraformer 包装 ←★ 主 ASR │ ├── SpeakerEmbedder.kt — sherpa-onnx 3dspeaker campplus 192-d 嵌入 + cosineSim +│ ├── SpeakerMatcher.kt — 回填工具：用当前 profile 集重算所有段的 speakerId │ ├── ModelDownloader.kt — 首次启动下模型（paraformer 232 MB + speaker 28 MB） │ └── XfyunIatClient.kt — 讯飞大模型 WS 客户端，仅 fallback / 云复核用 └── data/ @@ -155,15 +157,15 @@ VAD 参数（在 `RecorderService.runPipeline` 里调）： ✅ **已完成**： - v0.2 (2026-06)：Opus 编码 / BootReceiver / 电池白名单 - v0.3 (2026-06)：声纹识别（3dspeaker campplus）/ FTS4 中文搜索 +- v0.4 (2026-06)：段播放 (MediaPlayer Opus) / bottom sheet 操作菜单 / enroll 后自动回填老段 剩下： 1. **LLM 纪要 worker**：WorkManager cron。每小时 / 静音 30min 触发，按时间窗 group segments → POST 到 hermes gateway `http://127.0.0.1:8642/v1/chat/completions`（需要先把 hermes api_server platform 配置起来，见 `~/.hermes/config.yaml`）。 2. **存储 TTL**：删超过 N 天的 .opus，文字永留。Periodic WorkManager job。 3. **设置页**：存储用量、模型切换、provider 切换、暂停时段配置（v1 定的 MVP 是「一键暂停就够」，但 v2 可能要细化）。 -4. **段播放**：时间轴每条点开能直接播原音（Opus 系统原生支持，MediaPlayer 就能放）。当前点开是「记录说话人」对话框，需要分离 tap vs long-press 或加二级菜单。 -5. **多 anchor 平均 / 声纹回填**：现在 enroll 是单段单 anchor，可以「连选多段都标 我」自动算平均 embedding；并对老段做一次性 backfill 匹配。 -6. **回链 lemonPCRM**：Transcript 提到的人名 / 项目 → GraphStore `MENTIONED_IN` 边。 +4. **多 anchor 平均**：现在 enroll 是单段单 anchor，可以「连选多段都标 我」自动算平均 embedding 提高匹配率。 +5. **回链 lemonPCRM**：Transcript 提到的人名 / 项目 → GraphStore `MENTIONED_IN` 边。 ### 存储预算（v0.2 后） diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/MainActivity.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/MainActivity.kt index bab3b1c..3977f31 100644 --- a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/MainActivity.kt +++ b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/MainActivity.kt @@ -30,13 +30,16 @@ import androidx.compose.foundation.text.KeyboardOptions import androidx.compose.foundation.verticalScroll import androidx.compose.material3.AlertDialog import androidx.compose.material3.Button +import androidx.compose.material3.ExperimentalMaterial3Api import androidx.compose.material3.HorizontalDivider import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.ModalBottomSheet import androidx.compose.material3.OutlinedTextField import androidx.compose.material3.Surface import androidx.compose.material3.Text import androidx.compose.material3.TextButton import androidx.compose.material3.TextField +import androidx.compose.material3.rememberModalBottomSheetState import androidx.compose.runtime.Composable import androidx.compose.runtime.collectAsState import androidx.compose.runtime.DisposableEffect @@ -54,10 +57,12 @@ import androidx.core.content.ContextCompat import com.lemon.lifelog.asr.ModelDownloader import com.lemon.lifelog.asr.ParaformerEngine import com.lemon.lifelog.asr.SpeakerEmbedder +import com.lemon.lifelog.asr.SpeakerMatcher import com.lemon.lifelog.asr.XfyunIatClient import com.lemon.lifelog.asr.toFloatArrayLe import com.lemon.lifelog.asr.toLeBytes import com.lemon.lifelog.asr.transcribeWithStats +import com.lemon.lifelog.audio.SegmentPlayer import com.lemon.lifelog.data.Repository import com.lemon.lifelog.data.Segment import com.lemon.lifelog.data.SegmentRow @@ -112,8 +117,12 @@ private fun MainScreen() { val timeline by timelineFlow.collectAsState(initial = emptyList()) val speakers by repo.speakers.observeAll().collectAsState(initial = emptyList()) - // Enrollment dialog state. + // Sheet / dialog state. + var actionRow by remember { mutableStateOf(null) } var enrollFor by remember { mutableStateOf(null) } + val player = remember { SegmentPlayer() } + var nowPlayingId by remember { mutableStateOf(null) } + DisposableEffect(player) { onDispose { player.stop() } } // Battery optimisation exemption — recompute whenever activity resumes // (user may have just returned from the system settings dialog). @@ -210,7 +219,11 @@ private fun MainScreen() { verticalArrangement = Arrangement.spacedBy(4.dp), ) { items(timeline, key = { it.id }) { row -> - TimelineRow(row, onClick = { enrollFor = row }) + TimelineRow( + row = row, + isPlaying = nowPlayingId == row.id, + onClick = { actionRow = row }, + ) } } @@ -247,6 +260,29 @@ private fun MainScreen() { Text(asrLog, style = MaterialTheme.typography.bodySmall) } + // Bottom sheet with row actions. + actionRow?.let { row -> + SegmentActionSheet( + row = row, + isPlaying = nowPlayingId == row.id, + onDismiss = { actionRow = null }, + onPlay = { + if (nowPlayingId == row.id) { + player.stop() + nowPlayingId = null + } else { + nowPlayingId = row.id + player.play(row.audioPath) { nowPlayingId = null } + } + actionRow = null + }, + onEnroll = { + enrollFor = row + actionRow = null + }, + ) + } + // Speaker enrollment dialog. enrollFor?.let { row -> EnrollSpeakerDialog( @@ -258,7 +294,9 @@ private fun MainScreen() { enrollFor = null if (target != null && name.isNotBlank()) { scope.launch { - enrollSpeakerForSegment(activity, repo, target.id, name) + enrollSpeakerForSegment(repo, target.id, name) + // Backfill all older segments with embeddings against the new profile. + SpeakerMatcher.backfillAll(repo) } } }, @@ -266,6 +304,54 @@ private fun MainScreen() { } } +@OptIn(ExperimentalMaterial3Api::class) +@Composable +private fun SegmentActionSheet( + row: SegmentRow, + isPlaying: Boolean, + onDismiss: () -> Unit, + onPlay: () -> Unit, + onEnroll: () -> Unit, +) { + val sheetState = rememberModalBottomSheetState() + ModalBottomSheet( + onDismissRequest = onDismiss, + sheetState = sheetState, + ) { + Column( + modifier = Modifier.fillMaxWidth().padding(16.dp), + verticalArrangement = Arrangement.spacedBy(8.dp), + ) { + Text( + "段 #${row.id} · ${"%.1fs".format(row.durationMs / 1000.0)}", + style = MaterialTheme.typography.titleMedium, + ) + row.text?.let { + Text(it, style = MaterialTheme.typography.bodyMedium) + } + row.speakerName?.let { name -> + Text( + "已识别：$name (sim=${"%.2f".format(row.speakerSim ?: 0f)})", + style = MaterialTheme.typography.labelMedium, + ) + } + Spacer(Modifier.height(8.dp)) + Button( + onClick = onPlay, + modifier = Modifier.fillMaxWidth(), + ) { + Text(if (isPlaying) "■ 停止" else "▶ 播放原音") + } + Button( + onClick = onEnroll, + modifier = Modifier.fillMaxWidth(), + ) { + Text("记为说话人") + } + } + } +} + @Composable private fun EnrollSpeakerDialog( row: SegmentRow, @@ -317,7 +403,6 @@ private fun EnrollSpeakerDialog( * Future segments will be auto-matched against it (see RecorderService.refreshProfiles). */ private suspend fun enrollSpeakerForSegment( - ctx: Context, repo: Repository, segmentId: Long, name: String, @@ -373,7 +458,7 @@ private fun requestBatteryOptExemption(ctx: Context) { private val TIMELINE_FMT = SimpleDateFormat("HH:mm:ss", Locale.US) @Composable -private fun TimelineRow(row: SegmentRow, onClick: () -> Unit) { +private fun TimelineRow(row: SegmentRow, isPlaying: Boolean, onClick: () -> Unit) { val time = TIMELINE_FMT.format(Date(row.startTs)) val durSec = "%.1fs".format(row.durationMs / 1000.0) val speakerLabel = when { @@ -382,6 +467,7 @@ private fun TimelineRow(row: SegmentRow, onClick: () -> Unit) { row.speakerSim != null -> "未匹配 (${"%.2f".format(row.speakerSim)})" else -> "—" } + val playMarker = if (isPlaying) "▶ " else "" Column( modifier = Modifier .fillMaxWidth() @@ -389,7 +475,7 @@ private fun TimelineRow(row: SegmentRow, onClick: () -> Unit) { .padding(vertical = 4.dp), ) { Text( - "$time · $durSec · #${row.id} · $speakerLabel", + "$playMarker$time · $durSec · #${row.id} · $speakerLabel", style = MaterialTheme.typography.labelSmall, ) Text( diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/asr/SpeakerMatcher.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/asr/SpeakerMatcher.kt new file mode 100644 index 0000000..d8cb4fe --- /dev/null +++ b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/asr/SpeakerMatcher.kt @@ -0,0 +1,52 @@ +package com.lemon.lifelog.asr + +import android.util.Log +import com.lemon.lifelog.data.Repository +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext + +/** + * Backfill speakerId / speakerSim for all segments that already have a stored embedding, + * using the current SpeakerProfile set. Cheap enough to run synchronously after + * any enroll / delete profile event (192-d cosine × O(segments × profiles)). + */ +object SpeakerMatcher { + private const val TAG = "SpeakerMatcher" + const val MATCH_THRESHOLD = 0.5f + + suspend fun backfillAll(repo: Repository): BackfillStats = withContext(Dispatchers.IO) { + val profiles = repo.speakers.all().map { p -> + Cached(p.id, p.name, p.embedding.toFloatArrayLe()) + } + val segments = repo.segments.allWithEmbedding() + var assigned = 0 + var unassigned = 0 + for (seg in segments) { + val emb = seg.embedding?.toFloatArrayLe() ?: continue + val (newId, newSim) = if (profiles.isEmpty()) { + null to null + } else { + var bestId: Long? = null + var bestSim = -1f + for (p in profiles) { + val sim = cosineSim(emb, p.embedding) + if (sim > bestSim) { + bestSim = sim + bestId = p.id + } + } + if (bestSim >= MATCH_THRESHOLD) bestId to bestSim else null to bestSim + } + if (newId != seg.speakerId || newSim != seg.speakerSim) { + repo.segments.updateSpeaker(seg.id, newId, newSim) + } + if (newId != null) assigned++ else unassigned++ + } + Log.i(TAG, "backfill: ${segments.size} segments, $assigned matched, $unassigned unmatched") + BackfillStats(total = segments.size, matched = assigned, unmatched = unassigned) + } + + private data class Cached(val id: Long, val name: String, val embedding: FloatArray) + + data class BackfillStats(val total: Int, val matched: Int, val unmatched: Int) +} diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/audio/SegmentPlayer.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/audio/SegmentPlayer.kt new file mode 100644 index 0000000..c30d286 --- /dev/null +++ b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/audio/SegmentPlayer.kt @@ -0,0 +1,69 @@ +package com.lemon.lifelog.audio + +import android.media.AudioAttributes +import android.media.MediaPlayer +import android.util.Log + +/** + * One-at-a-time MediaPlayer for Opus segments. + * Android MediaPlayer natively decodes Ogg-Opus (API 21+). + * + * Lifecycle: call [play] to start a new clip (any previous one is released); + * [stop] when navigating away. + */ +class SegmentPlayer { + companion object { + private const val TAG = "SegmentPlayer" + } + + private var player: MediaPlayer? = null + private var currentPath: String? = null + + val nowPlaying: String? get() = currentPath + + fun play(path: String, onComplete: () -> Unit) { + stop() + currentPath = path + player = MediaPlayer().apply { + setAudioAttributes( + AudioAttributes.Builder() + .setUsage(AudioAttributes.USAGE_MEDIA) + .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) + .build(), + ) + try { + setDataSource(path) + setOnCompletionListener { + currentPath = null + onComplete() + } + setOnErrorListener { _, what, extra -> + Log.w(TAG, "MediaPlayer error what=$what extra=$extra path=$path") + currentPath = null + onComplete() + true + } + prepare() + start() + Log.i(TAG, "playing $path duration=${duration}ms") + } catch (t: Throwable) { + Log.e(TAG, "failed to play $path", t) + release() + currentPath = null + onComplete() + } + } + } + + fun stop() { + player?.let { mp -> + try { + if (mp.isPlaying) mp.stop() + } catch (_: IllegalStateException) { + } + mp.release() + } + player = null + currentPath = null + } +} -- libgit2 0.21.4