From 894eb7c04081630a6164e257518d3a3f5a5ace38 Mon Sep 17 00:00:00 2001 From: yibo Date: Wed, 3 Jun 2026 00:00:51 +0800 Subject: [PATCH] feat(lifelog-recorder): v0.3 — 声纹识别 + FTS4 中文搜索 --- lifelog-recorder/HANDOFF.md | 28 +++++++++++++++------------- lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/MainActivity.kt | 172 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++------- lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/asr/ModelDownloader.kt | 46 +++++++++++++++++++++++++++++++++++++++++----- lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/asr/SpeakerEmbedder.kt | 114 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/AppDatabase.kt | 17 ++++++++++++++--- lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/Daos.kt | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/Entities.kt | 27 ++++++++++++++++++++++++++- lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/Repository.kt | 1 + lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/service/RecorderService.kt | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-------- 9 files changed, 512 insertions(+), 38 deletions(-) create mode 100644 lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/asr/SpeakerEmbedder.kt diff --git a/lifelog-recorder/HANDOFF.md b/lifelog-recorder/HANDOFF.md index 5d90c6b..715e3e0 100644 --- a/lifelog-recorder/HANDOFF.md +++ b/lifelog-recorder/HANDOFF.md @@ -40,7 +40,7 @@ APK 大小: 62 MB（含 34 MB sherpa-onnx .so + 0.6 MB silero VAD + 3 MB samp ``` app/src/main/kotlin/com/lemon/lifelog/ ├── LifelogApp.kt — Application，建 NotificationChannel -├── MainActivity.kt — Compose UI，时间轴 + 电池豁免 UI + 测试按钮 +├── MainActivity.kt — Compose UI，时间轴 + 搜索 + 声纹 enroll dialog ├── service/ │ ├── RecorderService.kt — 前台服务，pipeline 主体 ←★ 改 ASR 流程改这里 │ ├── RecorderState.kt — StateFlow 把服务状态推到 UI @@ -50,12 +50,13 @@ app/src/main/kotlin/com/lemon/lifelog/ │ └── SegmentWriter.kt — Opus encode via MediaCodec + Ogg muxer，~17× 压缩 ├── asr/ │ ├── ParaformerEngine.kt — sherpa-onnx paraformer 包装 ←★ 主 ASR -│ ├── ModelDownloader.kt — 首次启动下模型 + 断点续传 +│ ├── SpeakerEmbedder.kt — sherpa-onnx 3dspeaker campplus 192-d 嵌入 + cosineSim +│ ├── ModelDownloader.kt — 首次启动下模型（paraformer 232 MB + speaker 28 MB） │ └── XfyunIatClient.kt — 讯飞大模型 WS 客户端，仅 fallback / 云复核用 └── data/ - ├── Entities.kt — Segment / Transcript / Summary - ├── Daos.kt — 含 observeTimeline() 给 UI 流 - ├── AppDatabase.kt — Room 1.0 + ├── Entities.kt — Segment / Transcript / Summary / SpeakerProfile / TranscriptFts + ├── Daos.kt — observeTimeline + searchTimeline (FTS4) + SpeakerDao + ├── AppDatabase.kt — Room v2，destructive migration（v0.3 schema 大改） └── Repository.kt ``` @@ -151,17 +152,18 @@ VAD 参数（在 `RecorderService.runPipeline` 里调）： ## §五 §六 待办（按优先级） -✅ **已完成（v0.2，2026-06）：Opus 编码 / BootReceiver / 电池白名单** +✅ **已完成**： +- v0.2 (2026-06)：Opus 编码 / BootReceiver / 电池白名单 +- v0.3 (2026-06)：声纹识别（3dspeaker campplus）/ FTS4 中文搜索 剩下： -1. **声纹识别 / speaker diarization**（🔥 用户当面提过 3 次）：sherpa-onnx 已有 `Speaker.kt` + `SpeakerEmbeddingExtractorConfig.kt`。流程：(a)「这是我」录音 → 建 anchor embedding 存 SQLite；(b) 每段算 embedding 算 cosine 找最近邻；(c) Transcript 加 `speakerHint` 列。需要下 `sherpa-onnx-3dspeaker-zh.onnx`（~38 MB）。 -2. **FTS5 中文搜索**：Room + sqlite-jsqlite-with-icu。Schema 加 `TranscriptFts` 虚表用 ICU tokenizer。UI 加搜索框 + 高亮命中段。 -3. **LLM 纪要 worker**：WorkManager cron。每小时 / 静音 30min 触发，按时间窗 group segments → POST 到 hermes gateway `http://127.0.0.1:8642/v1/chat/completions`（需要先把 hermes api_server platform 配置起来，见 `~/.hermes/config.yaml`）。 -4. **存储 TTL**：删超过 N 天的 .opus，文字永留。Periodic WorkManager job。 -5. **设置页**：存储用量、模型切换、provider 切换、暂停时段配置（v1 定的 MVP 是「一键暂停就够」，但 v2 可能要细化）。 -6. **段播放**：时间轴每条点开能直接播原音（Opus 系统原生支持，MediaPlayer 就能放）。 -7. **回链 lemonPCRM**：Transcript 提到的人名 / 项目 → GraphStore `MENTIONED_IN` 边。 +1. **LLM 纪要 worker**：WorkManager cron。每小时 / 静音 30min 触发，按时间窗 group segments → POST 到 hermes gateway `http://127.0.0.1:8642/v1/chat/completions`（需要先把 hermes api_server platform 配置起来，见 `~/.hermes/config.yaml`）。 +2. **存储 TTL**：删超过 N 天的 .opus，文字永留。Periodic WorkManager job。 +3. **设置页**：存储用量、模型切换、provider 切换、暂停时段配置（v1 定的 MVP 是「一键暂停就够」，但 v2 可能要细化）。 +4. **段播放**：时间轴每条点开能直接播原音（Opus 系统原生支持，MediaPlayer 就能放）。当前点开是「记录说话人」对话框，需要分离 tap vs long-press 或加二级菜单。 +5. **多 anchor 平均 / 声纹回填**：现在 enroll 是单段单 anchor，可以「连选多段都标 我」自动算平均 embedding；并对老段做一次性 backfill 匹配。 +6. **回链 lemonPCRM**：Transcript 提到的人名 / 项目 → GraphStore `MENTIONED_IN` 边。 ### 存储预算（v0.2 后） diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/MainActivity.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/MainActivity.kt index da6c197..bab3b1c 100644 --- a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/MainActivity.kt +++ b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/MainActivity.kt @@ -13,6 +13,7 @@ import androidx.activity.ComponentActivity import androidx.activity.compose.rememberLauncherForActivityResult import androidx.activity.compose.setContent import androidx.activity.result.contract.ActivityResultContracts +import androidx.compose.foundation.clickable import androidx.compose.foundation.layout.Arrangement import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.Row @@ -25,12 +26,17 @@ import androidx.compose.foundation.layout.padding import androidx.compose.foundation.lazy.LazyColumn import androidx.compose.foundation.lazy.items import androidx.compose.foundation.rememberScrollState +import androidx.compose.foundation.text.KeyboardOptions import androidx.compose.foundation.verticalScroll +import androidx.compose.material3.AlertDialog import androidx.compose.material3.Button import androidx.compose.material3.HorizontalDivider import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.OutlinedTextField import androidx.compose.material3.Surface import androidx.compose.material3.Text +import androidx.compose.material3.TextButton +import androidx.compose.material3.TextField import androidx.compose.runtime.Composable import androidx.compose.runtime.collectAsState import androidx.compose.runtime.DisposableEffect @@ -47,10 +53,15 @@ import androidx.compose.ui.unit.dp import androidx.core.content.ContextCompat import com.lemon.lifelog.asr.ModelDownloader import com.lemon.lifelog.asr.ParaformerEngine +import com.lemon.lifelog.asr.SpeakerEmbedder import com.lemon.lifelog.asr.XfyunIatClient +import com.lemon.lifelog.asr.toFloatArrayLe +import com.lemon.lifelog.asr.toLeBytes import com.lemon.lifelog.asr.transcribeWithStats import com.lemon.lifelog.data.Repository +import com.lemon.lifelog.data.Segment import com.lemon.lifelog.data.SegmentRow +import com.lemon.lifelog.data.SpeakerProfile import com.lemon.lifelog.service.RecorderService import com.lemon.lifelog.service.RecorderState import kotlinx.coroutines.flow.Flow @@ -90,9 +101,19 @@ private fun MainScreen() { val activity = androidx.compose.ui.platform.LocalContext.current val scope = rememberCoroutineScope() - // Timeline: observe segments + transcripts from Room. + // Timeline + search. val repo = remember(activity) { Repository(activity) } - val timeline by repo.segments.observeTimeline(200).collectAsState(initial = emptyList()) + var searchQuery by remember { mutableStateOf("") } + val effectiveSearch = remember(searchQuery) { sanitizeFtsQuery(searchQuery) } + val timelineFlow = remember(effectiveSearch) { + if (effectiveSearch.isEmpty()) repo.segments.observeTimeline(200) + else repo.segments.searchTimeline(effectiveSearch, 200) + } + val timeline by timelineFlow.collectAsState(initial = emptyList()) + val speakers by repo.speakers.observeAll().collectAsState(initial = emptyList()) + + // Enrollment dialog state. + var enrollFor by remember { mutableStateOf(null) } // Battery optimisation exemption — recompute whenever activity resumes // (user may have just returned from the system settings dialog). @@ -165,14 +186,32 @@ private fun MainScreen() { HorizontalDivider() - Text("时间轴（${timeline.size}）", style = MaterialTheme.typography.titleMedium) + Row(verticalAlignment = androidx.compose.ui.Alignment.CenterVertically, + horizontalArrangement = Arrangement.spacedBy(8.dp)) { + Text( + if (searchQuery.isEmpty()) "时间轴（${timeline.size}）" + else "搜索结果（${timeline.size}）", + style = MaterialTheme.typography.titleMedium, + ) + Text("· 声纹库 ${speakers.size}", + style = MaterialTheme.typography.bodySmall) + } + OutlinedTextField( + value = searchQuery, + onValueChange = { searchQuery = it }, + modifier = Modifier.fillMaxWidth(), + placeholder = { Text("中文搜索…") }, + singleLine = true, + ) LazyColumn( modifier = Modifier .fillMaxWidth() .heightIn(min = 200.dp, max = 600.dp), verticalArrangement = Arrangement.spacedBy(4.dp), ) { - items(timeline, key = { it.id }) { row -> TimelineRow(row) } + items(timeline, key = { it.id }) { row -> + TimelineRow(row, onClick = { enrollFor = row }) + } } HorizontalDivider() @@ -207,6 +246,114 @@ private fun MainScreen() { Spacer(Modifier.height(8.dp)) Text(asrLog, style = MaterialTheme.typography.bodySmall) } + + // Speaker enrollment dialog. + enrollFor?.let { row -> + EnrollSpeakerDialog( + row = row, + existingNames = speakers.map { it.name }, + onDismiss = { enrollFor = null }, + onConfirm = { name -> + val target = enrollFor + enrollFor = null + if (target != null && name.isNotBlank()) { + scope.launch { + enrollSpeakerForSegment(activity, repo, target.id, name) + } + } + }, + ) + } +} + +@Composable +private fun EnrollSpeakerDialog( + row: SegmentRow, + existingNames: List, + onDismiss: () -> Unit, + onConfirm: (String) -> Unit, +) { + var name by remember { mutableStateOf("") } + AlertDialog( + onDismissRequest = onDismiss, + title = { Text("记录说话人") }, + text = { + Column(verticalArrangement = Arrangement.spacedBy(8.dp)) { + Text( + "用这一段的声音建一个声纹 anchor。之后说话风格相近的段会自动打上同名标签。", + style = MaterialTheme.typography.bodySmall, + ) + Text( + "段 #${row.id}: ${row.text?.take(40) ?: "(无转写)"}", + style = MaterialTheme.typography.bodyMedium, + ) + OutlinedTextField( + value = name, + onValueChange = { name = it }, + label = { Text("说话人名字（如：我、老婆、张三）") }, + singleLine = true, + ) + if (existingNames.isNotEmpty()) { + Text( + "已存声纹：${existingNames.joinToString("、")}", + style = MaterialTheme.typography.labelSmall, + ) + } + } + }, + confirmButton = { + TextButton(onClick = { onConfirm(name) }, enabled = name.isNotBlank()) { + Text("保存") + } + }, + dismissButton = { + TextButton(onClick = onDismiss) { Text("取消") } + }, + ) +} + +/** + * Read embedding from segment row, save as new SpeakerProfile. + * Future segments will be auto-matched against it (see RecorderService.refreshProfiles). + */ +private suspend fun enrollSpeakerForSegment( + ctx: Context, + repo: Repository, + segmentId: Long, + name: String, +) { + withContext(Dispatchers.IO) { + val seg = repo.segments.get(segmentId) ?: return@withContext + val embedding = seg.embedding ?: return@withContext + repo.speakers.insert( + SpeakerProfile( + name = name.trim(), + embedding = embedding, + sampleCount = 1, + ), + ) + android.util.Log.i("MainActivity", "enrolled speaker '$name' from segment #$segmentId") + } +} + +/** + * FTS4 MATCH expects a query string. For Chinese character-level search with the + * unicode61 tokenizer, append `*` to each token so substring/prefix lookup works. + * Strip FTS metacharacters that would otherwise crash the parser. + */ +private fun sanitizeFtsQuery(raw: String): String { + val cleaned = raw.trim() + .replace("\"", "") + .replace("'", "") + .replace("(", "") + .replace(")", "") + .replace(":", "") + if (cleaned.isEmpty()) return "" + // Treat whole input as one prefix term. For longer queries the user can put spaces + // for AND-of-terms; we add * suffix to each so partial matches work. + return cleaned.split(Regex("\\s+")) + .filter { it.isNotEmpty() } + .joinToString(" ") { "$it*" } } private fun isBatteryOptExempt(ctx: Context): Boolean { @@ -226,12 +373,23 @@ private fun requestBatteryOptExemption(ctx: Context) { private val TIMELINE_FMT = SimpleDateFormat("HH:mm:ss", Locale.US) @Composable -private fun TimelineRow(row: SegmentRow) { +private fun TimelineRow(row: SegmentRow, onClick: () -> Unit) { val time = TIMELINE_FMT.format(Date(row.startTs)) val durSec = "%.1fs".format(row.durationMs / 1000.0) - Column(modifier = Modifier.fillMaxWidth().padding(vertical = 2.dp)) { + val speakerLabel = when { + row.speakerName != null && row.speakerSim != null -> + "${row.speakerName} (${"%.2f".format(row.speakerSim)})" + row.speakerSim != null -> "未匹配 (${"%.2f".format(row.speakerSim)})" + else -> "—" + } + Column( + modifier = Modifier + .fillMaxWidth() + .clickable(onClick = onClick) + .padding(vertical = 4.dp), + ) { Text( - "$time · $durSec · #${row.id}", + "$time · $durSec · #${row.id} · $speakerLabel", style = MaterialTheme.typography.labelSmall, ) Text( diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/asr/ModelDownloader.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/asr/ModelDownloader.kt index 0263a06..692d48c 100644 --- a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/asr/ModelDownloader.kt +++ b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/asr/ModelDownloader.kt @@ -24,6 +24,8 @@ class ModelDownloader(private val ctx: Context) { val present: Boolean, ) + data class SpeakerModelFile(val path: String, val present: Boolean) + fun paths(): ModelFiles { val dir = File(ctx.filesDir, "models/paraformer-zh").apply { mkdirs() } val model = File(dir, "model.int8.onnx") @@ -32,16 +34,21 @@ class ModelDownloader(private val ctx: Context) { return ModelFiles(model.absolutePath, tokens.absolutePath, present) } + fun speakerPath(): SpeakerModelFile { + val dir = File(ctx.filesDir, "models/3dspeaker").apply { mkdirs() } + val model = File(dir, "campplus_zh.onnx") + // expected ~28 MB, accept anything > 20 MB as valid + val present = model.exists() && model.length() > 20_000_000L + return SpeakerModelFile(model.absolutePath, present) + } + suspend fun ensure(progress: (String) -> Unit): ModelFiles = withContext(Dispatchers.IO) { val state = paths() if (state.present) { - progress("model present (${File(state.modelPath).length() / 1_000_000}MB), skip") + progress("paraformer model present (${File(state.modelPath).length() / 1_000_000}MB), skip") return@withContext state } - val client = OkHttpClient.Builder() - .connectTimeout(15, TimeUnit.SECONDS) - .readTimeout(120, TimeUnit.SECONDS) - .build() + val client = http() val tasks = listOf( DownloadTask( @@ -68,6 +75,35 @@ class ModelDownloader(private val ctx: Context) { paths() } + /** + * 3dspeaker CAMPPlus 16k speaker embedding (192 dim) from sherpa-onnx releases. + * ~28 MB on-device. + */ + suspend fun ensureSpeakerModel(progress: (String) -> Unit): SpeakerModelFile = + withContext(Dispatchers.IO) { + val state = speakerPath() + if (state.present) { + progress("speaker model present (${File(state.path).length() / 1_000_000}MB), skip") + return@withContext state + } + val task = DownloadTask( + File(state.path), + listOf( + // hf-mirror works without login in China; HF直连为备份 + "https://hf-mirror.com/csukuangfj/speaker-embedding-models/resolve/main/3dspeaker_speech_campplus_sv_zh-cn_16k-common.onnx", + "https://huggingface.co/csukuangfj/speaker-embedding-models/resolve/main/3dspeaker_speech_campplus_sv_zh-cn_16k-common.onnx", + ), + expectedSize = 28_281_138L, + ) + downloadWithResume(http(), task, progress) + speakerPath() + } + + private fun http() = OkHttpClient.Builder() + .connectTimeout(15, TimeUnit.SECONDS) + .readTimeout(120, TimeUnit.SECONDS) + .build() + private data class DownloadTask( val target: File, val urls: List, diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/asr/SpeakerEmbedder.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/asr/SpeakerEmbedder.kt new file mode 100644 index 0000000..b997f57 --- /dev/null +++ b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/asr/SpeakerEmbedder.kt @@ -0,0 +1,114 @@ +package com.lemon.lifelog.asr + +import android.util.Log +import com.k2fsa.sherpa.onnx.SpeakerEmbeddingExtractor +import com.k2fsa.sherpa.onnx.SpeakerEmbeddingExtractorConfig +import java.io.File +import java.nio.ByteBuffer +import java.nio.ByteOrder +import kotlin.math.sqrt + +/** + * On-device speaker embedding via sherpa-onnx (3dspeaker CAMPPlus, 192-dim). + * + * Use case (v0.3 MVP): + * - For every VAD segment compute a 192-d float embedding. + * - User enrolls 1+ "anchor profiles" (e.g. {name: "我", embedding}). + * - Match new segments by cosine similarity; if max > threshold → assign that profile. + * + * Thread-unsafe per stream. Reuse the extractor across segments. + */ +class SpeakerEmbedder private constructor( + private val extractor: SpeakerEmbeddingExtractor, + val dim: Int, +) { + + /** Compute embedding for a PCM segment. Returns null if not enough audio. */ + fun embed(samples: FloatArray, sampleRate: Int = 16000): FloatArray? { + if (samples.isEmpty()) return null + val stream = extractor.createStream() + return try { + stream.acceptWaveform(samples, sampleRate) + stream.inputFinished() + if (!extractor.isReady(stream)) { + Log.w(TAG, "extractor not ready for ${samples.size} samples") + return null + } + extractor.compute(stream) + } finally { + stream.release() + } + } + + fun release() { + extractor.release() + } + + companion object { + private const val TAG = "SpeakerEmbedder" + const val DIM = 192 + + fun load(modelPath: String, numThreads: Int = 1): SpeakerEmbedder { + require(File(modelPath).exists()) { "speaker model not found: $modelPath" } + val t0 = System.nanoTime() + val extractor = SpeakerEmbeddingExtractor( + config = SpeakerEmbeddingExtractorConfig( + model = modelPath, + numThreads = numThreads, + debug = false, + provider = "cpu", + ), + ) + val dim = extractor.dim() + Log.i(TAG, "loaded in ${(System.nanoTime() - t0) / 1_000_000}ms dim=$dim threads=$numThreads") + return SpeakerEmbedder(extractor, dim) + } + } +} + +// -------- helpers -------------------------------------------------------- + +/** Pack a FloatArray as little-endian bytes for Room BLOB column. */ +fun FloatArray.toLeBytes(): ByteArray { + val buf = ByteBuffer.allocate(size * 4).order(ByteOrder.LITTLE_ENDIAN) + for (f in this) buf.putFloat(f) + return buf.array() +} + +/** Inverse of [toLeBytes]. */ +fun ByteArray.toFloatArrayLe(): FloatArray { + val n = size / 4 + val out = FloatArray(n) + val buf = ByteBuffer.wrap(this).order(ByteOrder.LITTLE_ENDIAN).asFloatBuffer() + buf.get(out) + return out +} + +/** Cosine similarity ∈ [-1,1]. Higher = more similar. */ +fun cosineSim(a: FloatArray, b: FloatArray): Float { + if (a.size != b.size || a.isEmpty()) return 0f + var dot = 0.0 + var na = 0.0 + var nb = 0.0 + for (i in a.indices) { + dot += a[i] * b[i] + na += a[i] * a[i] + nb += b[i] * b[i] + } + if (na == 0.0 || nb == 0.0) return 0f + return (dot / (sqrt(na) * sqrt(nb))).toFloat() +} + +/** Average a list of embeddings into one (per-dim mean), normalised. */ +fun averageEmbeddings(embeddings: List): FloatArray { + require(embeddings.isNotEmpty()) { "empty list" } + val dim = embeddings.first().size + val sum = FloatArray(dim) + for (e in embeddings) { + require(e.size == dim) { "dim mismatch" } + for (i in 0 until dim) sum[i] += e[i] + } + val n = embeddings.size.toFloat() + for (i in 0 until dim) sum[i] /= n + return sum +} diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/AppDatabase.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/AppDatabase.kt index 65bc026..b9f9d62 100644 --- a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/AppDatabase.kt +++ b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/AppDatabase.kt @@ -6,13 +6,21 @@ import androidx.room.Room import androidx.room.RoomDatabase @Database( - entities = [Segment::class, Transcript::class, Summary::class], - version = 1, + entities = [ + Segment::class, + Transcript::class, + Summary::class, + SpeakerProfile::class, + TranscriptFts::class, + ], + version = 2, + exportSchema = false, ) abstract class AppDatabase : RoomDatabase() { abstract fun segmentDao(): SegmentDao abstract fun transcriptDao(): TranscriptDao abstract fun summaryDao(): SummaryDao + abstract fun speakerDao(): SpeakerDao companion object { @Volatile @@ -24,7 +32,10 @@ abstract class AppDatabase : RoomDatabase() { ctx.applicationContext, AppDatabase::class.java, "lifelog.db", - ).build().also { instance = it } + ) + .fallbackToDestructiveMigration() // v0.3 dev: schema 大改，旧测试数据可丢 + .build() + .also { instance = it } } } } diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/Daos.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/Daos.kt index 983e42f..7131d37 100644 --- a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/Daos.kt +++ b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/Daos.kt @@ -11,6 +11,9 @@ data class SegmentRow( val durationMs: Long, val text: String?, val audioPath: String, + val speakerId: Long?, + val speakerSim: Float?, + val speakerName: String?, ) @Dao @@ -24,17 +27,52 @@ interface SegmentDao { @Query("SELECT * FROM segments ORDER BY startTs DESC") fun observeAll(): Flow> + @Query("SELECT * FROM segments WHERE id = :id") + suspend fun get(id: Long): Segment? + + @Query("UPDATE segments SET speakerId = :speakerId, speakerSim = :sim WHERE id = :id") + suspend fun updateSpeaker(id: Long, speakerId: Long?, sim: Float?) + + /** + * Backfill: re-assign speakerId for all segments that have a stored embedding, + * using the latest profile set. Called once after a new profile is enrolled. + * (Actual cosine math is done in Kotlin — Room can't do it.) + */ + @Query("SELECT * FROM segments WHERE embedding IS NOT NULL") + suspend fun allWithEmbedding(): List + @Query( """ SELECT s.id AS id, s.startTs AS startTs, s.durationMs AS durationMs, - t.text AS text, s.audioPath AS audioPath + t.text AS text, s.audioPath AS audioPath, + s.speakerId AS speakerId, s.speakerSim AS speakerSim, + sp.name AS speakerName FROM segments s LEFT JOIN transcripts t ON t.segmentId = s.id + LEFT JOIN speaker_profiles sp ON sp.id = s.speakerId ORDER BY s.startTs DESC LIMIT :limit """, ) fun observeTimeline(limit: Int = 100): Flow> + + /** Full-text search via FTS4 virtual table. Falls back to LIKE if MATCH query is empty. */ + @Query( + """ + SELECT s.id AS id, s.startTs AS startTs, s.durationMs AS durationMs, + t.text AS text, s.audioPath AS audioPath, + s.speakerId AS speakerId, s.speakerSim AS speakerSim, + sp.name AS speakerName + FROM segments s + INNER JOIN transcripts t ON t.segmentId = s.id + INNER JOIN transcript_fts fts ON fts.docid = t.segmentId + LEFT JOIN speaker_profiles sp ON sp.id = s.speakerId + WHERE transcript_fts MATCH :query + ORDER BY s.startTs DESC + LIMIT :limit + """, + ) + fun searchTimeline(query: String, limit: Int = 200): Flow> } @Dao @@ -54,3 +92,18 @@ interface SummaryDao { @Query("SELECT * FROM summaries ORDER BY windowStartTs DESC LIMIT :n") fun recent(n: Int): Flow> } + +@Dao +interface SpeakerDao { + @Insert + suspend fun insert(p: SpeakerProfile): Long + + @Query("SELECT * FROM speaker_profiles ORDER BY createdAt ASC") + fun observeAll(): Flow> + + @Query("SELECT * FROM speaker_profiles ORDER BY createdAt ASC") + suspend fun all(): List + + @Query("DELETE FROM speaker_profiles WHERE id = :id") + suspend fun delete(id: Long) +} diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/Entities.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/Entities.kt index d99199c..d019871 100644 --- a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/Entities.kt +++ b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/Entities.kt @@ -1,6 +1,7 @@ package com.lemon.lifelog.data import androidx.room.Entity +import androidx.room.Fts4 import androidx.room.PrimaryKey @Entity(tableName = "segments") @@ -11,6 +12,12 @@ data class Segment( val audioPath: String, val durationMs: Long, val energyDb: Float = 0f, + /** 192-dim float embedding packed as little-endian bytes (192*4=768 B). null = 还没算 */ + val embedding: ByteArray? = null, + /** 匹配到的 SpeakerProfile.id，null = 没匹配上 / 还没匹配 */ + val speakerId: Long? = null, + /** 与 matched profile 的余弦相似度，null = 没匹配 */ + val speakerSim: Float? = null, ) @Entity(tableName = "transcripts") @@ -18,7 +25,7 @@ data class Transcript( @PrimaryKey val segmentId: Long, val text: String, val lang: String = "zh", - val engine: String = "xfyun", + val engine: String = "paraformer-zh-on-device", val confidence: Float = 1f, val reviewedAt: Long? = null, ) @@ -32,3 +39,21 @@ data class Summary( val model: String, val createdAt: Long, ) + +@Entity(tableName = "speaker_profiles") +data class SpeakerProfile( + @PrimaryKey(autoGenerate = true) val id: Long = 0, + val name: String, + /** 192-dim float embedding packed as LE bytes，平均或单段 */ + val embedding: ByteArray, + /** 用了多少段平均出来的 */ + val sampleCount: Int = 1, + val createdAt: Long = System.currentTimeMillis(), +) + +/** FTS4 全文索引，内容跟随 Transcript 表。Room 自动建表 + 触发器同步。 */ +@Fts4(contentEntity = Transcript::class, tokenizer = "unicode61") +@Entity(tableName = "transcript_fts") +data class TranscriptFts( + val text: String, +) diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/Repository.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/Repository.kt index 0e96cf5..81a9c46 100644 --- a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/Repository.kt +++ b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/Repository.kt @@ -7,4 +7,5 @@ class Repository(ctx: Context) { val segments = db.segmentDao() val transcripts = db.transcriptDao() val summaries = db.summaryDao() + val speakers = db.speakerDao() } diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/service/RecorderService.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/service/RecorderService.kt index 25dab14..b1bff8d 100644 --- a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/service/RecorderService.kt +++ b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/service/RecorderService.kt @@ -16,10 +16,15 @@ import com.k2fsa.sherpa.onnx.VadModelConfig import com.lemon.lifelog.MainActivity import com.lemon.lifelog.asr.ModelDownloader import com.lemon.lifelog.asr.ParaformerEngine +import com.lemon.lifelog.asr.SpeakerEmbedder +import com.lemon.lifelog.asr.cosineSim +import com.lemon.lifelog.asr.toFloatArrayLe +import com.lemon.lifelog.asr.toLeBytes import com.lemon.lifelog.audio.AudioCapture import com.lemon.lifelog.audio.SegmentWriter import com.lemon.lifelog.data.Repository import com.lemon.lifelog.data.Segment +import com.lemon.lifelog.data.SpeakerProfile import com.lemon.lifelog.data.Transcript import java.io.File import kotlinx.coroutines.CoroutineScope @@ -50,6 +55,15 @@ class RecorderService : Service() { private lateinit var segWriter: SegmentWriter private var vad: Vad? = null private var engine: ParaformerEngine? = null + private var speakerEmbedder: SpeakerEmbedder? = null + + /** Anchor profiles cached in memory; refreshed when DB changes. */ + @Volatile + private var profiles: List = emptyList() + private data class CachedProfile(val id: Long, val name: String, val embedding: FloatArray) + + /** Cosine threshold above which we assign a profile to a segment. Tuned for CAMPPlus. */ + private val speakerMatchThreshold = 0.5f override fun onCreate() { super.onCreate() @@ -72,6 +86,7 @@ class RecorderService : Service() { capture.stop() vad?.release() engine?.release() + speakerEmbedder?.release() scope.cancel() wakeLock?.takeIf { it.isHeld }?.release() wakeLock = null @@ -130,10 +145,30 @@ class RecorderService : Service() { ), ) + // 2c. Speaker embedder (optional; download is 28 MB, may take a bit on cold start). + try { + RecorderState.setStatus("loading speaker model…") + val speakerFile = downloader.ensureSpeakerModel { msg -> + Log.i(TAG, "speaker dl: $msg") + RecorderState.setStatus(msg) + } + if (speakerFile.present) { + speakerEmbedder = SpeakerEmbedder.load(speakerFile.path, numThreads = 2) + } + } catch (t: Throwable) { + Log.w(TAG, "speaker embedder not available; segments will skip embedding", t) + } + + // 2d. Load anchor profiles (cosine matching). + refreshProfiles() + scope.launch { + repo.speakers.observeAll().collect { _ -> refreshProfiles() } + } + // 3. Start capturing. capture.start(scope) RecorderState.setStatus("listening") - Log.i(TAG, "pipeline running") + Log.i(TAG, "pipeline running speakers=${profiles.size}") capture.frames.collect { samples -> val v = vad ?: return@collect @@ -153,12 +188,12 @@ class RecorderService : Service() { val now = System.currentTimeMillis() val startTs = now - audioMs - // a. persist audio (off CPU thread) + // a. persist audio (off CPU thread, Opus encode). val audioPath = kotlinx.coroutines.withContext(Dispatchers.IO) { segWriter.write(samples, startTs, sr) } - // b. paraformer + // b. paraformer. val wallStart = System.nanoTime() val text = try { engine!!.transcribeFloat(samples, sr).text @@ -168,7 +203,33 @@ class RecorderService : Service() { } val wallMs = (System.nanoTime() - wallStart) / 1_000_000 - // c. Room + // c. speaker embedding + nearest-profile match. + val embedding: FloatArray? = speakerEmbedder?.let { embedder -> + try { + embedder.embed(samples, sr) + } catch (t: Throwable) { + Log.w(TAG, "embed failed", t) + null + } + } + val (matchedSpeakerId: Long?, matchedSim: Float?, matchedName: String?) = + if (embedding != null && profiles.isNotEmpty()) { + var bestId: Long? = null + var bestName: String? = null + var bestSim = -1f + for (p in profiles) { + val sim = cosineSim(embedding, p.embedding) + if (sim > bestSim) { + bestSim = sim + bestId = p.id + bestName = p.name + } + } + if (bestSim >= speakerMatchThreshold) Triple(bestId, bestSim, bestName) + else Triple(null, bestSim, null) + } else Triple(null, null, null) + + // d. Room insert. val segmentId = kotlinx.coroutines.withContext(Dispatchers.IO) { val id = repo.segments.insert( Segment( @@ -177,6 +238,9 @@ class RecorderService : Service() { audioPath = audioPath, durationMs = audioMs, energyDb = 0f, + embedding = embedding?.toLeBytes(), + speakerId = matchedSpeakerId, + speakerSim = matchedSim, ), ) if (text.isNotEmpty()) { @@ -184,19 +248,29 @@ class RecorderService : Service() { Transcript( segmentId = id, text = text, - lang = "zh", - engine = "paraformer-zh-on-device", - confidence = 1f, ), ) } id } - Log.i(TAG, "segment#$segmentId audio=${audioMs}ms wall=${wallMs}ms text=$text") + Log.i( + TAG, + "segment#$segmentId audio=${audioMs}ms wall=${wallMs}ms " + + "speaker=${matchedName ?: "—"} sim=${matchedSim?.let { "%.2f".format(it) } ?: "—"} " + + "text=$text", + ) RecorderState.onSegment(text, audioMs, wallMs) } + private suspend fun refreshProfiles() { + val rows = kotlinx.coroutines.withContext(Dispatchers.IO) { repo.speakers.all() } + profiles = rows.map { + CachedProfile(it.id, it.name, it.embedding.toFloatArrayLe()) + } + Log.i(TAG, "loaded ${profiles.size} speaker profiles") + } + private fun startForegroundCompat() { val notification = buildNotification() if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.UPSIDE_DOWN_CAKE) { -- libgit2 0.21.4