Commit 3512a84f758e79ab750db56c506e979f3820598d

Authored by yibo
1 parent c3220da9

feat(lifelog): 录音改为 block-based diarization-first 管线(攒段→整块分轮转写)

弃掉「录一句、转一句、实时聚类」——短句(0.3-30s) embedding 噪声大,实时滑动
聚类把噪声平均进 anchor,是声纹串人的根因。改为:

录音端(RecorderService):VAD 只负责存语音 + 检测块边界,删掉实时 transcribe/
embed/clusterer.assign。块关闭条件:静音≥5min 或 块时长≥10min(长会议保活)。
  - BlockBuffer:speech-only PCM 追加 + manifest(run 偏移→真实时间);崩溃重启
    listPending 续处理。
处理端(BlockProcessor):整块 pyannote 分轮 → 每 turn 切片 paraformer 转写 +
CAMPPlus 嵌入 + 全局 anchor 匹配 → 写 turn 行(turn = 时间轴一行)。pyannote
缺失则降级整块单 turn。块处理与采集解耦(Channel 顺序消费),不阻塞录音。

embedding 从源头是单说话人干净切片,不再被实时逐句污染。SpeakerClusterer.assign
也加 L1 冻结(命名 anchor 不漂移)。废弃旧的周期 DiarizationWorker(块创建即
diarize),删 DiarizationScheduler/Worker;纪要按钮不再触发它。

实测(95s 真实会议块):13 段语音 → 24 个干净不重叠 turn,逐 turn 带说话人+文字。

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/LifelogApp.kt
@@ -3,7 +3,6 @@ package com.lemon.lifelog @@ -3,7 +3,6 @@ package com.lemon.lifelog
3 import android.app.Application 3 import android.app.Application
4 import android.app.NotificationChannel 4 import android.app.NotificationChannel
5 import android.app.NotificationManager 5 import android.app.NotificationManager
6 -import com.lemon.lifelog.diarize.DiarizationScheduler  
7 import com.lemon.lifelog.service.RecorderService 6 import com.lemon.lifelog.service.RecorderService
8 import com.lemon.lifelog.summary.SummaryScheduler 7 import com.lemon.lifelog.summary.SummaryScheduler
9 8
@@ -18,9 +17,10 @@ class LifelogApp : Application() { @@ -18,9 +17,10 @@ class LifelogApp : Application() {
18 NotificationManager.IMPORTANCE_LOW, 17 NotificationManager.IMPORTANCE_LOW,
19 ) 18 )
20 ) 19 )
21 - // 纪要 + diarization periodic worker(各自 15min,UPDATE)。 20 + // 纪要 periodic worker(15min,UPDATE)。
  21 + // 注:旧的周期 DiarizationWorker 已废弃——diarization-first 管线在「块」关闭时
  22 + // 当场分轮+转写,不再需要事后整体重跑。声纹纠偏走「重新聚类」按钮(reclusterAll)。
22 SummaryScheduler.schedulePeriodic(this) 23 SummaryScheduler.schedulePeriodic(this)
23 - DiarizationScheduler.schedulePeriodic(this)  
24 // 相册:未处理照片的周期后台处理(人脸+VL),空闲时跑,不阻塞导入。 24 // 相册:未处理照片的周期后台处理(人脸+VL),空闲时跑,不阻塞导入。
25 com.lemon.lifelog.vision.PhotoScheduler.schedulePeriodic(this) 25 com.lemon.lifelog.vision.PhotoScheduler.schedulePeriodic(this)
26 } 26 }
lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/MainActivity.kt
@@ -83,7 +83,6 @@ import com.lemon.lifelog.asr.XfyunIatClient @@ -83,7 +83,6 @@ import com.lemon.lifelog.asr.XfyunIatClient
83 import com.lemon.lifelog.asr.transcribeWithStats 83 import com.lemon.lifelog.asr.transcribeWithStats
84 import com.lemon.lifelog.audio.SegmentPlayer 84 import com.lemon.lifelog.audio.SegmentPlayer
85 import com.lemon.lifelog.chat.ChatConfig 85 import com.lemon.lifelog.chat.ChatConfig
86 -import com.lemon.lifelog.diarize.DiarizationScheduler  
87 import com.lemon.lifelog.summary.SummaryProgress 86 import com.lemon.lifelog.summary.SummaryProgress
88 import com.lemon.lifelog.summary.SummaryRunLog 87 import com.lemon.lifelog.summary.SummaryRunLog
89 import com.lemon.lifelog.data.Repository 88 import com.lemon.lifelog.data.Repository
@@ -905,7 +904,7 @@ private fun SummariesTab(repo: Repository, activity: Context, onJumpToTime: (Lon @@ -905,7 +904,7 @@ private fun SummariesTab(repo: Repository, activity: Context, onJumpToTime: (Lon
905 Column(modifier = Modifier.fillMaxSize().padding(16.dp)) { 904 Column(modifier = Modifier.fillMaxSize().padding(16.dp)) {
906 Row(verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(8.dp)) { 905 Row(verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(8.dp)) {
907 Text("纪要(${summaries.size})", style = MaterialTheme.typography.titleMedium) 906 Text("纪要(${summaries.size})", style = MaterialTheme.typography.titleMedium)
908 - Button(onClick = { DiarizationScheduler.runNow(activity); SummaryScheduler.runNow(activity) }, enabled = !running) { 907 + Button(onClick = { SummaryScheduler.runNow(activity) }, enabled = !running) {
909 if (running) { 908 if (running) {
910 CircularProgressIndicator(modifier = Modifier.size(16.dp), strokeWidth = 2.dp) 909 CircularProgressIndicator(modifier = Modifier.size(16.dp), strokeWidth = 2.dp)
911 Spacer(Modifier.width(6.dp)) 910 Spacer(Modifier.width(6.dp))
lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/asr/SpeakerClusterer.kt
@@ -59,7 +59,12 @@ class SpeakerClusterer( @@ -59,7 +59,12 @@ class SpeakerClusterer(
59 } 59 }
60 60
61 if (best != null && bestSim >= matchThreshold) { 61 if (best != null && bestSim >= matchThreshold) {
62 - // capped running mean:上限保证质心保持自适应、不被早期样本钉死。 62 + if (autoIndexOf(best.name) == null) {
  63 + // 人工命名 anchor:只归属、**冻结质心不漂移**(同 reclusterAll 的 L1)。
  64 + // 否则 anchor 吸入相邻说话人后会朝其漂移、再吞更多 → 雪球式串人。
  65 + return@withLock Assignment(best.id, bestSim, best.name)
  66 + }
  67 + // 自动簇:capped running mean,质心保持自适应、不被早期样本钉死。
63 val n = minOf(best.count, maxSamplesPerCentroid) 68 val n = minOf(best.count, maxSamplesPerCentroid)
64 val merged = FloatArray(emb.size) { i -> (best.centroid[i] * n + emb[i]) / (n + 1) } 69 val merged = FloatArray(emb.size) { i -> (best.centroid[i] * n + emb[i]) / (n + 1) }
65 l2normalize(merged) 70 l2normalize(merged)
lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/diarize/DiarizationScheduler.kt deleted
1 -package com.lemon.lifelog.diarize  
2 -  
3 -import android.content.Context  
4 -import androidx.work.Constraints  
5 -import androidx.work.ExistingPeriodicWorkPolicy  
6 -import androidx.work.ExistingWorkPolicy  
7 -import androidx.work.NetworkType  
8 -import androidx.work.OneTimeWorkRequestBuilder  
9 -import androidx.work.PeriodicWorkRequestBuilder  
10 -import androidx.work.WorkManager  
11 -import java.util.concurrent.TimeUnit  
12 -  
13 -object DiarizationScheduler {  
14 - private const val PERIODIC_TAG = "lifelog-diarize-periodic"  
15 - private const val ONE_SHOT_TAG = "lifelog-diarize-once"  
16 -  
17 - fun schedulePeriodic(ctx: Context) {  
18 - val constraints = Constraints.Builder()  
19 - .setRequiredNetworkType(NetworkType.NOT_REQUIRED)  
20 - .setRequiresBatteryNotLow(true)  
21 - .build()  
22 - val request = PeriodicWorkRequestBuilder<DiarizationWorker>(15, TimeUnit.MINUTES)  
23 - .setConstraints(constraints)  
24 - .build()  
25 - WorkManager.getInstance(ctx).enqueueUniquePeriodicWork(  
26 - PERIODIC_TAG,  
27 - ExistingPeriodicWorkPolicy.UPDATE,  
28 - request,  
29 - )  
30 - }  
31 -  
32 - /** 手动跑一次(KEEP:已在跑/排队则忽略)。 */  
33 - fun runNow(ctx: Context) {  
34 - val request = OneTimeWorkRequestBuilder<DiarizationWorker>().addTag(ONE_SHOT_TAG).build()  
35 - WorkManager.getInstance(ctx).enqueueUniqueWork(ONE_SHOT_TAG, ExistingWorkPolicy.KEEP, request)  
36 - }  
37 -}  
lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/diarize/DiarizationWorker.kt deleted
1 -package com.lemon.lifelog.diarize  
2 -  
3 -import android.content.Context  
4 -import android.util.Log  
5 -import androidx.work.CoroutineWorker  
6 -import androidx.work.WorkerParameters  
7 -import com.lemon.lifelog.asr.DiarizationEngine  
8 -import com.lemon.lifelog.asr.ModelDownloader  
9 -import com.lemon.lifelog.asr.SpeakerClusterer  
10 -import com.lemon.lifelog.asr.SpeakerEmbedder  
11 -import com.lemon.lifelog.asr.toLeBytes  
12 -import com.lemon.lifelog.audio.AudioDecoder  
13 -import com.lemon.lifelog.data.Repository  
14 -import com.lemon.lifelog.data.Segment  
15 -import com.lemon.lifelog.summary.SummaryWorker  
16 -import java.io.File  
17 -import java.util.concurrent.atomic.AtomicBoolean  
18 -import kotlinx.coroutines.Dispatchers  
19 -import kotlinx.coroutines.withContext  
20 -  
21 -/**  
22 - * 会话级 diarization 重跑(方案 A)。对每个**已结束、未 diarize**的会话:  
23 - * 1. 解码 + 按序拼接该会话所有段音频(speech-only 16k mono)。  
24 - * 2. pyannote 分割 + CAMPPlus + FastClustering → 准确的 [start,end]→局部说话人 turns  
25 - * (能把「几秒多人」的段拆开)。  
26 - * 3. 每个局部说话人 → 取其最长 turn 的 embedding → 经 SpeakerClusterer 桥接到  
27 - * 全局 SpeakerProfile(跨会话/跨天「说话人N」一致)。  
28 - * 4. 每个 VAD 段 → 找重叠最大的 turn → 重标 speakerId(段内多说话人先归主导者,v1)。  
29 - *  
30 - * 与逐段实时聚类相比准确得多;实时标签只作即时占位,这里批处理纠正。  
31 - */  
32 -class DiarizationWorker(ctx: Context, params: WorkerParameters) : CoroutineWorker(ctx, params) {  
33 -  
34 - companion object {  
35 - const val TAG = "DiarizationWorker"  
36 - private val isRunning = AtomicBoolean(false)  
37 - // WorkManager 给每个 worker 硬上限 10min,超时直接 kill。预算压到 5min,  
38 - // 留足余量让最后一块 diarize 收尾 + 全局 recluster + 写 cursor,绝不撞红线。  
39 - const val RUN_BUDGET_MS = 5 * 60 * 1000L  
40 - const val MIN_SESSION_SEGMENTS = 2 // 单段会话无需 diarize  
41 - // 单块 ≤3min speech:一次 diarize ~3min,远小于 10min 红线,一轮能跑完多块、  
42 - // 多次写 cursor 推进;块内说话人 index 不跨块——身份交全局 reclusterAll。  
43 - const val CHUNK_SAMPLES = 3 * 60 * 16000  
44 - private const val KEY_DONE = "diarized_sessions" // prefs StringSet of windowStartTs  
45 - private const val KEY_CURSOR = "diar_cursor_" // + key → 已处理段数(断点续跑长会议)  
46 - }  
47 -  
48 - private val sr = 16000  
49 -  
50 - override suspend fun doWork(): Result {  
51 - if (!isRunning.compareAndSet(false, true)) {  
52 - Log.i(TAG, "another diarization in progress, skip"); return Result.success()  
53 - }  
54 - return try {  
55 - run()  
56 - } catch (t: Throwable) {  
57 - Log.w(TAG, "diarization run failed", t); Result.success()  
58 - } finally {  
59 - isRunning.set(false)  
60 - }  
61 - }  
62 -  
63 - private suspend fun run(): Result {  
64 - val runStart = System.currentTimeMillis()  
65 - val now = System.currentTimeMillis()  
66 - val prefs = applicationContext.getSharedPreferences(SummaryWorker.PREFS, Context.MODE_PRIVATE)  
67 - val gapMs = prefs.getInt(SummaryWorker.KEY_GAP_MIN, SummaryWorker.DEFAULT_GAP_MIN) * 60_000L  
68 - val threshold = SpeakerClusterer.thresholdFromPrefs(applicationContext)  
69 -  
70 - val repo = Repository(applicationContext)  
71 - val segments = repo.segments.page(limit = 5000, offset = 0).sortedBy { it.startTs }  
72 - if (segments.size < MIN_SESSION_SEGMENTS) return Result.success()  
73 -  
74 - val sessions = SummaryWorker.groupSessions(segments, gapMs)  
75 - val done = prefs.getStringSet(KEY_DONE, emptySet())!!.toMutableSet()  
76 -  
77 - // 模型路径  
78 - val downloader = ModelDownloader(applicationContext)  
79 - val speakerFile = downloader.ensureSpeakerModel { }  
80 - if (!speakerFile.present) { Log.w(TAG, "speaker model missing"); return Result.success() }  
81 - val pyannote = downloader.ensurePyannoteModel { Log.i(TAG, "pyannote: $it") }  
82 - if (!pyannote.present) { Log.w(TAG, "pyannote seg model missing/download failed"); return Result.success() }  
83 -  
84 - var engine: DiarizationEngine? = null  
85 - var embedder: SpeakerEmbedder? = null  
86 - var processed = 0  
87 - var attempted = false  
88 - try {  
89 - // 最新会话优先(用户更关心最近的;旧会话后续轮次补)。  
90 - for (session in sessions.reversed()) {  
91 - val start = session.first().startTs  
92 - val end = session.last().endTs  
93 - val key = start.toString()  
94 -  
95 - if (now - end <= gapMs) continue // 未结束  
96 - if (session.size < MIN_SESSION_SEGMENTS) continue  
97 - if (done.contains(key)) continue // 已 diarize  
98 - if (System.currentTimeMillis() - runStart > RUN_BUDGET_MS) {  
99 - Log.i(TAG, "time budget hit, rest next round"); break  
100 - }  
101 -  
102 - // 懒加载模型(有活才加载)  
103 - if (engine == null) {  
104 - engine = DiarizationEngine.load(pyannote.path, speakerFile.path, numThreads = 4, threshold = threshold)  
105 - embedder = SpeakerEmbedder.load(speakerFile.path, numThreads = 2)  
106 - }  
107 -  
108 - attempted = true  
109 - val startIdx = prefs.getInt(KEY_CURSOR + key, 0)  
110 - val reached = diarizeSession(repo, session, engine!!, embedder!!, runStart, startIdx)  
111 - if (reached >= session.size) {  
112 - done.add(key); processed++  
113 - prefs.edit().remove(KEY_CURSOR + key).apply()  
114 - } else {  
115 - // 预算中途用完:记录断点,下轮从此处续跑(不重头),然后退出本轮。  
116 - prefs.edit().putInt(KEY_CURSOR + key, reached).apply()  
117 - Log.i(TAG, "session $key partial: cursor=$reached/${session.size}, resume next round"); break  
118 - }  
119 - }  
120 - } finally {  
121 - engine?.release()  
122 - embedder?.release()  
123 - }  
124 -  
125 - // 全局重聚类:用 diarization 产出的"干净"per-turn embedding 统一聚类  
126 - //(reclusterAll 本就全局 + 保留人工命名锚点)。只要这轮碰过会话就跑。  
127 - if (attempted) {  
128 - val n = SpeakerClusterer.reclusterAll(repo, matchThreshold = threshold)  
129 - Log.i(TAG, "global recluster → $n speakers")  
130 - }  
131 - prefs.edit().putStringSet(KEY_DONE, done).apply()  
132 - Log.i(TAG, "done: ${sessions.size} sessions, $processed diarized")  
133 - return Result.success()  
134 - }  
135 -  
136 - /**  
137 - * 分块 diarize 整个会话(长会议不再只取前 N min),从 [startIdx] 段断点续跑。  
138 - * 返回**已处理到的段下标**(== session.size 表示整场跑完;< 表示预算中途用完,  
139 - * 外层据此存断点、下轮从这里续跑——块内说话人 index 不跨块,身份交全局 reclusterAll)。  
140 - */  
141 - private suspend fun diarizeSession(  
142 - repo: Repository,  
143 - session: List<Segment>,  
144 - engine: DiarizationEngine,  
145 - embedder: SpeakerEmbedder,  
146 - runStart: Long,  
147 - startIdx: Int,  
148 - ): Int = withContext(Dispatchers.Default) {  
149 - var idx = startIdx.coerceIn(0, session.size)  
150 - var nChunks = 0  
151 - var totalTurns = 0  
152 - var embedded = 0  
153 - while (idx < session.size) {  
154 - if (System.currentTimeMillis() - runStart > RUN_BUDGET_MS) {  
155 - Log.i(TAG, "session ${session.first().startTs}: budget hit at seg $idx/${session.size}")  
156 - return@withContext idx // 断点,外层续跑  
157 - }  
158 - // 攒一块(≤CHUNK_SAMPLES),解码即拼,记录每段在块内的 [from,to]  
159 - val segs = ArrayList<Segment>()  
160 - val froms = ArrayList<Int>()  
161 - val tos = ArrayList<Int>()  
162 - val pcms = ArrayList<FloatArray>()  
163 - var len = 0  
164 - while (idx < session.size && len < CHUNK_SAMPLES) {  
165 - val seg = session[idx]; idx++  
166 - val pcm = AudioDecoder.decodeToFloat(seg.audioPath)  
167 - if (pcm.isEmpty()) continue  
168 - segs.add(seg); froms.add(len); pcms.add(pcm); len += pcm.size; tos.add(len)  
169 - }  
170 - if (segs.isEmpty()) continue  
171 - val samples = FloatArray(len)  
172 - var off = 0  
173 - for (p in pcms) { p.copyInto(samples, off); off += p.size }  
174 - pcms.clear()  
175 -  
176 - val turns = engine.diarize(samples)  
177 - if (turns.isEmpty()) continue  
178 - nChunks++; totalTurns += turns.size  
179 -  
180 - val turnEmb = HashMap<Int, FloatArray?>() // turn 下标 → embedding(多段共用同一 turn 缓存)  
181 - for (k in segs.indices) {  
182 - val s0 = froms[k].toFloat() / sr  
183 - val s1 = tos[k].toFloat() / sr  
184 - var bestIdx = -1  
185 - var bestOv = 0f  
186 - turns.forEachIndexed { i, t ->  
187 - val ov = minOf(s1, t.endSec) - maxOf(s0, t.startSec)  
188 - if (ov > bestOv) { bestOv = ov; bestIdx = i }  
189 - }  
190 - if (bestIdx < 0) continue  
191 - val emb = turnEmb.getOrPut(bestIdx) {  
192 - val t = turns[bestIdx]  
193 - val a = (t.startSec * sr).toInt().coerceIn(0, samples.size)  
194 - val b = (t.endSec * sr).toInt().coerceIn(a, samples.size)  
195 - if (b - a < sr / 4) null else embedder.embed(samples.copyOfRange(a, b), sr)  
196 - }  
197 - if (emb != null) {  
198 - withContext(Dispatchers.IO) { repo.segments.updateEmbedding(segs[k].id, emb.toLeBytes()) }  
199 - embedded++  
200 - }  
201 - }  
202 - }  
203 - Log.i(TAG, "session ${session.first().startTs}: $startIdx→${session.size} segs, $nChunks chunks, $totalTurns turns, $embedded clean-embedded")  
204 - idx // == session.size,整场跑完  
205 - }  
206 -}  
lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/service/BlockBuffer.kt 0 → 100644
  1 +package com.lemon.lifelog.service
  2 +
  3 +import android.util.Log
  4 +import java.io.File
  5 +import java.nio.ByteBuffer
  6 +import java.nio.ByteOrder
  7 +import org.json.JSONArray
  8 +import org.json.JSONObject
  9 +
  10 +/**
  11 + * 当前录音「块」的累积器(diarization-first 管线的录音端)。
  12 + *
  13 + * VAD 每产出一段语音就 [append] 进块的 `audio.pcm`(raw PCM16 LE,16k mono),并把该段的
  14 + * `{wallTs, sampleOffset, sampleLen}` 记进 manifest(用于之后把 turn 的样本偏移还原成真实时间)。
  15 + *
  16 + * 关块条件([shouldClose],由 RecorderService 每帧/每段检查):
  17 + * - 距上次语音静音 ≥ [SILENCE_MS](5min) 或
  18 + * - 块时长 ≥ [MAX_MS](10min,长会议保活,避免全程不出文字)
  19 + *
  20 + * 块目录:`filesDir/blocks/<blockStartTs>/{audio.pcm, manifest.json}`。
  21 + * [close] 落 manifest(closed=true) 后交给 BlockProcessor;崩溃时块目录仍在,重启 [listPending] 重扫。
  22 + * 只存 speech(静音不落盘)→ 内存/磁盘按语音时长计,不按墙钟。
  23 + */
  24 +class BlockBuffer(private val root: File, private val sampleRate: Int = 16000) {
  25 +
  26 + data class Run(val ts: Long, val off: Int, val len: Int)
  27 +
  28 + private var dir: File? = null
  29 + private var out: java.io.BufferedOutputStream? = null
  30 + private val runs = ArrayList<Run>()
  31 + private var sampleCount = 0
  32 + private var blockStartTs = 0L
  33 + private var lastSpeechEndTs = 0L
  34 +
  35 + val isOpen get() = dir != null
  36 +
  37 + init {
  38 + root.mkdirs()
  39 + }
  40 +
  41 + /** 追加一段语音(startTs = 该段真实起始墙钟)。块未开则自动开。 */
  42 + fun append(samples: FloatArray, startTs: Long) {
  43 + if (samples.isEmpty()) return
  44 + if (!isOpen) open(startTs)
  45 + val bytes = ByteArray(samples.size * 2)
  46 + val bb = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN)
  47 + for (s in samples) bb.putShort((s.coerceIn(-1f, 1f) * 32767f).toInt().toShort())
  48 + out!!.write(bytes)
  49 + out!!.flush()
  50 + runs.add(Run(startTs, sampleCount, samples.size))
  51 + sampleCount += samples.size
  52 + lastSpeechEndTs = startTs + samples.size.toLong() * 1000 / sampleRate
  53 + writeManifest(dir!!, closed = false)
  54 + }
  55 +
  56 + /** 在 [ts] 这一刻是否该关块(静音超时或块超长)。块未开恒 false。 */
  57 + fun shouldClose(ts: Long): Boolean {
  58 + if (!isOpen) return false
  59 + return (ts - lastSpeechEndTs >= SILENCE_MS) || (ts - blockStartTs >= MAX_MS)
  60 + }
  61 +
  62 + /** 关块:落 manifest(closed=true),返回块目录;空块直接删并返回 null。 */
  63 + fun close(): File? {
  64 + if (!isOpen) return null
  65 + runCatching { out?.flush(); out?.close() }
  66 + out = null
  67 + val d = dir!!
  68 + dir = null
  69 + if (sampleCount == 0) {
  70 + d.deleteRecursively()
  71 + return null
  72 + }
  73 + writeManifest(d, closed = true)
  74 + Log.i(TAG, "block close ${d.name}: ${runs.size} runs / $sampleCount samples")
  75 + return d
  76 + }
  77 +
  78 + private fun open(ts: Long) {
  79 + blockStartTs = ts
  80 + lastSpeechEndTs = ts
  81 + runs.clear()
  82 + sampleCount = 0
  83 + val d = File(root, ts.toString()).apply { mkdirs() }
  84 + dir = d
  85 + out = File(d, "audio.pcm").outputStream().buffered()
  86 + Log.i(TAG, "block open @$ts -> ${d.name}")
  87 + }
  88 +
  89 + private fun writeManifest(d: File, closed: Boolean) {
  90 + val arr = JSONArray()
  91 + for (r in runs) arr.put(JSONObject().put("ts", r.ts).put("off", r.off).put("len", r.len))
  92 + val obj = JSONObject()
  93 + .put("start", blockStartTs)
  94 + .put("sr", sampleRate)
  95 + .put("closed", closed)
  96 + .put("runs", arr)
  97 + runCatching { File(d, "manifest.json").writeText(obj.toString()) }
  98 + }
  99 +
  100 + companion object {
  101 + private const val TAG = "BlockBuffer"
  102 + const val SILENCE_MS = 5 * 60 * 1000L // 静音 ≥5min 切块
  103 + const val MAX_MS = 10 * 60 * 1000L // 块 ≥10min 强切(长会议保活)
  104 +
  105 + data class Loaded(val startTs: Long, val sampleRate: Int, val samples: FloatArray, val runs: List<Run>)
  106 +
  107 + /** 启动时列出所有未处理块目录(崩溃恢复:处理后目录会被删除)。 */
  108 + fun listPending(root: File): List<File> =
  109 + root.listFiles()
  110 + ?.filter { it.isDirectory && File(it, "audio.pcm").exists() }
  111 + ?.sortedBy { it.name.toLongOrNull() ?: 0L }
  112 + ?: emptyList()
  113 +
  114 + /** 读块:manifest + audio.pcm → 整块 float + runs。 */
  115 + fun load(dir: File): Loaded? {
  116 + val af = File(dir, "audio.pcm")
  117 + if (!af.exists()) return null
  118 + val mf = File(dir, "manifest.json")
  119 + var start = dir.name.toLongOrNull() ?: 0L
  120 + var sr = 16000
  121 + val runs = ArrayList<Run>()
  122 + if (mf.exists()) {
  123 + runCatching {
  124 + val o = JSONObject(mf.readText())
  125 + start = o.optLong("start", start)
  126 + sr = o.optInt("sr", 16000)
  127 + o.optJSONArray("runs")?.let { a ->
  128 + for (i in 0 until a.length()) {
  129 + val r = a.getJSONObject(i)
  130 + runs.add(Run(r.getLong("ts"), r.getInt("off"), r.getInt("len")))
  131 + }
  132 + }
  133 + }
  134 + }
  135 + val bytes = af.readBytes()
  136 + val n = bytes.size / 2
  137 + val floats = FloatArray(n)
  138 + val bb = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN)
  139 + for (i in 0 until n) floats[i] = bb.short / 32768f
  140 + // 崩溃时 manifest 可能没记全 runs → 合成一个整块 run(时间退化为连续)。
  141 + if (runs.isEmpty()) runs.add(Run(start, 0, n))
  142 + return Loaded(start, sr, floats, runs)
  143 + }
  144 + }
  145 +}
lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/service/BlockProcessor.kt 0 → 100644
  1 +package com.lemon.lifelog.service
  2 +
  3 +import android.util.Log
  4 +import com.lemon.lifelog.asr.DiarizationEngine
  5 +import com.lemon.lifelog.asr.ParaformerEngine
  6 +import com.lemon.lifelog.asr.SpeakerClusterer
  7 +import com.lemon.lifelog.asr.SpeakerEmbedder
  8 +import com.lemon.lifelog.asr.toLeBytes
  9 +import com.lemon.lifelog.audio.SegmentWriter
  10 +import com.lemon.lifelog.data.Repository
  11 +import com.lemon.lifelog.data.Segment
  12 +import com.lemon.lifelog.data.Transcript
  13 +import java.io.File
  14 +import kotlinx.coroutines.Dispatchers
  15 +import kotlinx.coroutines.withContext
  16 +
  17 +/**
  18 + * 把一个关闭的录音块处理成时间轴 turns(diarization-first 管线的处理端):
  19 + *
  20 + * pyannote 分轮 → 每个 turn 切片 → paraformer 转写 + CAMPPlus 嵌入 + 全局 anchor 匹配
  21 + * → 写 turn WAV + `segments` 行 + `transcripts` 行。
  22 + *
  23 + * 一个 **turn = 时间轴一行**(一人一段连续发言)。embedding 是单说话人的干净切片嵌入,
  24 + * 从源头避免了实时逐句聚类的噪声污染。处理完删块目录(= 已处理标记)。
  25 + *
  26 + * [diar] 为 null(pyannote 模型缺失)时降级:整块当一个 turn 转写。
  27 + */
  28 +class BlockProcessor(
  29 + private val repo: Repository,
  30 + private val segWriter: SegmentWriter,
  31 + private val diar: DiarizationEngine?,
  32 + private val asr: ParaformerEngine,
  33 + private val embedder: SpeakerEmbedder?,
  34 + private val clusterer: SpeakerClusterer,
  35 +) {
  36 + suspend fun process(dir: File) {
  37 + val loaded = withContext(Dispatchers.IO) { BlockBuffer.load(dir) }
  38 + if (loaded == null || loaded.samples.isEmpty()) {
  39 + withContext(Dispatchers.IO) { dir.deleteRecursively() }
  40 + return
  41 + }
  42 + val sr = loaded.sampleRate
  43 + val samples = loaded.samples
  44 + val runs = loaded.runs
  45 +
  46 + // 样本下标 → 真实墙钟(speech-only 拼接,用 run 偏移还原)。runs 按 off 升序。
  47 + fun wallAt(sample: Int): Long {
  48 + var r = runs.first()
  49 + for (run in runs) if (run.off <= sample) r = run else break
  50 + val within = (sample - r.off).coerceIn(0, r.len)
  51 + return r.ts + within.toLong() * 1000 / sr
  52 + }
  53 +
  54 + var turns = diar?.diarize(samples) ?: emptyList()
  55 + if (turns.isEmpty()) {
  56 + // 降级 / 极短块:整块一个 turn。
  57 + turns = listOf(DiarizationEngine.Turn(0f, samples.size.toFloat() / sr, 0))
  58 + }
  59 +
  60 + var written = 0
  61 + for (t in turns) {
  62 + val a = (t.startSec * sr).toInt().coerceIn(0, samples.size)
  63 + val b = (t.endSec * sr).toInt().coerceIn(a, samples.size)
  64 + if (b - a < sr / 5) continue // <200ms 跳过
  65 + val slice = samples.copyOfRange(a, b)
  66 + val startTs = wallAt(a)
  67 + val durMs = (b - a).toLong() * 1000 / sr
  68 +
  69 + val text = try {
  70 + asr.transcribeFloat(slice, sr).text
  71 + } catch (e: Throwable) {
  72 + Log.w(TAG, "asr failed", e); ""
  73 + }
  74 + val emb = embedder?.let {
  75 + try { it.embed(slice, sr) } catch (e: Throwable) { Log.w(TAG, "embed failed", e); null }
  76 + }
  77 + val assign = emb?.let { clusterer.assign(it, durMs) }
  78 + ?: SpeakerClusterer.Assignment(null, null, null)
  79 +
  80 + val audioPath = withContext(Dispatchers.IO) { segWriter.write(slice, startTs, sr) }
  81 + withContext(Dispatchers.IO) {
  82 + val id = repo.segments.insert(
  83 + Segment(
  84 + startTs = startTs,
  85 + endTs = startTs + durMs,
  86 + audioPath = audioPath,
  87 + durationMs = durMs,
  88 + energyDb = 0f,
  89 + embedding = emb?.toLeBytes(),
  90 + speakerId = assign.speakerId,
  91 + speakerSim = assign.sim,
  92 + ),
  93 + )
  94 + if (text.isNotBlank()) repo.transcripts.insert(Transcript(segmentId = id, text = text))
  95 + }
  96 + written++
  97 + }
  98 + Log.i(TAG, "block ${dir.name}: ${turns.size} turns -> $written rows")
  99 + withContext(Dispatchers.IO) { dir.deleteRecursively() }
  100 + }
  101 +
  102 + companion object {
  103 + private const val TAG = "BlockProcessor"
  104 + }
  105 +}
lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/service/RecorderService.kt
@@ -14,23 +14,21 @@ import com.k2fsa.sherpa.onnx.SileroVadModelConfig @@ -14,23 +14,21 @@ import com.k2fsa.sherpa.onnx.SileroVadModelConfig
14 import com.k2fsa.sherpa.onnx.Vad 14 import com.k2fsa.sherpa.onnx.Vad
15 import com.k2fsa.sherpa.onnx.VadModelConfig 15 import com.k2fsa.sherpa.onnx.VadModelConfig
16 import com.lemon.lifelog.MainActivity 16 import com.lemon.lifelog.MainActivity
  17 +import com.lemon.lifelog.asr.DiarizationEngine
17 import com.lemon.lifelog.asr.ModelDownloader 18 import com.lemon.lifelog.asr.ModelDownloader
18 import com.lemon.lifelog.asr.ParaformerEngine 19 import com.lemon.lifelog.asr.ParaformerEngine
19 import com.lemon.lifelog.asr.SpeakerClusterer 20 import com.lemon.lifelog.asr.SpeakerClusterer
20 import com.lemon.lifelog.asr.SpeakerEmbedder 21 import com.lemon.lifelog.asr.SpeakerEmbedder
21 -import com.lemon.lifelog.asr.toLeBytes  
22 import com.lemon.lifelog.audio.AudioCapture 22 import com.lemon.lifelog.audio.AudioCapture
23 import com.lemon.lifelog.audio.SegmentWriter 23 import com.lemon.lifelog.audio.SegmentWriter
24 import com.lemon.lifelog.data.Repository 24 import com.lemon.lifelog.data.Repository
25 -import com.lemon.lifelog.data.Segment  
26 -import com.lemon.lifelog.data.SpeakerProfile  
27 -import com.lemon.lifelog.data.Transcript  
28 import java.io.File 25 import java.io.File
29 import kotlinx.coroutines.CoroutineScope 26 import kotlinx.coroutines.CoroutineScope
30 import kotlinx.coroutines.Dispatchers 27 import kotlinx.coroutines.Dispatchers
31 import kotlinx.coroutines.Job 28 import kotlinx.coroutines.Job
32 import kotlinx.coroutines.SupervisorJob 29 import kotlinx.coroutines.SupervisorJob
33 import kotlinx.coroutines.cancel 30 import kotlinx.coroutines.cancel
  31 +import kotlinx.coroutines.channels.Channel
34 import kotlinx.coroutines.flow.collect 32 import kotlinx.coroutines.flow.collect
35 import kotlinx.coroutines.launch 33 import kotlinx.coroutines.launch
36 34
@@ -70,10 +68,17 @@ class RecorderService : Service() { @@ -70,10 +68,17 @@ class RecorderService : Service() {
70 private var vad: Vad? = null 68 private var vad: Vad? = null
71 private var engine: ParaformerEngine? = null 69 private var engine: ParaformerEngine? = null
72 private var speakerEmbedder: SpeakerEmbedder? = null 70 private var speakerEmbedder: SpeakerEmbedder? = null
  71 + private var diarEngine: DiarizationEngine? = null
73 72
74 /** 在线说话人聚类器(方案 B):自动发现说话人并持久化为 speaker_profiles。 */ 73 /** 在线说话人聚类器(方案 B):自动发现说话人并持久化为 speaker_profiles。 */
75 private var clusterer: SpeakerClusterer? = null 74 private var clusterer: SpeakerClusterer? = null
76 75
  76 + /** diarization-first 管线:VAD 段攒进当前块,块关闭后整块 diarize+转写。 */
  77 + private lateinit var blockBuffer: BlockBuffer
  78 +
  79 + /** 关闭的块排队顺序处理(与采集解耦,不阻塞录音)。 */
  80 + private val pendingBlocks = Channel<File>(Channel.UNLIMITED)
  81 +
77 override fun onCreate() { 82 override fun onCreate() {
78 super.onCreate() 83 super.onCreate()
79 startForegroundCompat() 84 startForegroundCompat()
@@ -83,6 +88,7 @@ class RecorderService : Service() { @@ -83,6 +88,7 @@ class RecorderService : Service() {
83 RecorderState.reset() 88 RecorderState.reset()
84 repo = Repository(this) 89 repo = Repository(this)
85 segWriter = SegmentWriter(File(filesDir, "segments")) 90 segWriter = SegmentWriter(File(filesDir, "segments"))
  91 + blockBuffer = BlockBuffer(File(filesDir, "blocks"))
86 capture = AudioCapture() 92 capture = AudioCapture()
87 93
88 pipelineJob = scope.launch { runPipeline() } 94 pipelineJob = scope.launch { runPipeline() }
@@ -113,7 +119,7 @@ class RecorderService : Service() { @@ -113,7 +119,7 @@ class RecorderService : Service() {
113 acquireWakeLock() 119 acquireWakeLock()
114 capture.start(scope) 120 capture.start(scope)
115 RecorderState.setPaused(false) 121 RecorderState.setPaused(false)
116 - RecorderState.setStatus("listening") 122 + RecorderState.setStatus("录音中")
117 notify("录音中") 123 notify("录音中")
118 Log.i(TAG, "resumed") 124 Log.i(TAG, "resumed")
119 } 125 }
@@ -126,9 +132,13 @@ class RecorderService : Service() { @@ -126,9 +132,13 @@ class RecorderService : Service() {
126 override fun onDestroy() { 132 override fun onDestroy() {
127 pipelineJob?.cancel() 133 pipelineJob?.cancel()
128 capture.stop() 134 capture.stop()
  135 + // 关掉当前块(落 manifest 留在磁盘,下次启动 listPending 续处理)。
  136 + runCatching { blockBuffer.close() }
  137 + pendingBlocks.close()
129 vad?.release() 138 vad?.release()
130 engine?.release() 139 engine?.release()
131 speakerEmbedder?.release() 140 speakerEmbedder?.release()
  141 + diarEngine?.release()
132 scope.cancel() 142 scope.cancel()
133 wakeLock?.takeIf { it.isHeld }?.release() 143 wakeLock?.takeIf { it.isHeld }?.release()
134 wakeLock = null 144 wakeLock = null
@@ -210,94 +220,73 @@ class RecorderService : Service() { @@ -210,94 +220,73 @@ class RecorderService : Service() {
210 repo.speakers.observeAll().collect { clusterer?.load() } 220 repo.speakers.observeAll().collect { clusterer?.load() }
211 } 221 }
212 222
  223 + // 2e. Diarization 引擎(pyannote 分割 + CAMPPlus),块级整段分轮用。缺失则降级单 turn。
  224 + try {
  225 + RecorderState.setStatus("loading diarization…")
  226 + val pyannote = downloader.ensurePyannoteModel { Log.i(TAG, "pyannote dl: $it") }
  227 + val speaker = downloader.speakerPath()
  228 + if (pyannote.present && speaker.present) {
  229 + diarEngine = DiarizationEngine.load(
  230 + pyannote.path, speaker.path, numThreads = 4,
  231 + threshold = SpeakerClusterer.thresholdFromPrefs(this),
  232 + )
  233 + } else {
  234 + Log.w(TAG, "pyannote/speaker model missing; block will fall back to single-turn")
  235 + }
  236 + } catch (t: Throwable) {
  237 + Log.w(TAG, "diarization engine unavailable; single-turn fallback", t)
  238 + }
  239 +
  240 + // 2f. 块处理器 + 顺序处理协程(与采集解耦)。
  241 + val processor = BlockProcessor(repo, segWriter, diarEngine, engine!!, speakerEmbedder, clusterer!!)
  242 + scope.launch {
  243 + for (dir in pendingBlocks) {
  244 + RecorderState.setStatus("处理段落…")
  245 + try {
  246 + processor.process(dir)
  247 + } catch (t: Throwable) {
  248 + Log.w(TAG, "block ${dir.name} process failed", t)
  249 + }
  250 + RecorderState.setStatus(if (paused) "已暂停" else "录音中")
  251 + }
  252 + }
  253 + // 崩溃恢复:把上次没处理完的块重新入队。
  254 + BlockBuffer.listPending(File(filesDir, "blocks")).forEach { pendingBlocks.trySend(it) }
  255 +
213 // 3. Start capturing (unless paused arrived during model load). 256 // 3. Start capturing (unless paused arrived during model load).
214 if (!paused) { 257 if (!paused) {
215 capture.start(scope) 258 capture.start(scope)
216 - RecorderState.setStatus("listening") 259 + RecorderState.setStatus("录音中")
217 } 260 }
218 - Log.i(TAG, "pipeline running, clusterer loaded") 261 + Log.i(TAG, "pipeline running (block-based diarization-first)")
219 262
220 capture.frames.collect { samples -> 263 capture.frames.collect { samples ->
221 val v = vad ?: return@collect 264 val v = vad ?: return@collect
222 v.acceptWaveform(samples) 265 v.acceptWaveform(samples)
  266 + val now = System.currentTimeMillis()
223 while (!v.empty()) { 267 while (!v.empty()) {
224 val seg = v.front() 268 val seg = v.front()
225 v.pop() 269 v.pop()
226 if (seg.samples.isEmpty()) continue 270 if (seg.samples.isEmpty()) continue
227 - handleSegment(seg.samples) 271 + appendSpeech(seg.samples)
228 } 272 }
  273 + // 静音超时 / 块超长 → 关块入队(即使本帧没有新语音也要检查)。
  274 + if (blockBuffer.shouldClose(now)) closeBlock()
229 } 275 }
230 } 276 }
231 277
232 - private suspend fun handleSegment(samples: FloatArray) { 278 + /** VAD 段 → 攒进当前块(不再实时转写/聚类)。跨静音/超长时先关旧块。 */
  279 + private fun appendSpeech(samples: FloatArray) {
233 val sr = AudioCapture.SAMPLE_RATE 280 val sr = AudioCapture.SAMPLE_RATE
234 val audioMs = samples.size.toLong() * 1000 / sr 281 val audioMs = samples.size.toLong() * 1000 / sr
235 - val now = System.currentTimeMillis()  
236 - val startTs = now - audioMs  
237 -  
238 - // a. persist audio (off CPU thread, Opus encode).  
239 - val audioPath = kotlinx.coroutines.withContext(Dispatchers.IO) {  
240 - segWriter.write(samples, startTs, sr)  
241 - }  
242 -  
243 - // b. paraformer.  
244 - val wallStart = System.nanoTime()  
245 - val text = try {  
246 - engine!!.transcribeFloat(samples, sr).text  
247 - } catch (t: Throwable) {  
248 - Log.w(TAG, "transcribe failed", t)  
249 - ""  
250 - }  
251 - val wallMs = (System.nanoTime() - wallStart) / 1_000_000  
252 -  
253 - // c. speaker embedding + online clustering (auto-discovers 说话人N).  
254 - val embedding: FloatArray? = speakerEmbedder?.let { embedder ->  
255 - try {  
256 - embedder.embed(samples, sr)  
257 - } catch (t: Throwable) {  
258 - Log.w(TAG, "embed failed", t)  
259 - null  
260 - }  
261 - }  
262 - val assignment = embedding?.let { emb ->  
263 - clusterer?.assign(emb, audioMs)  
264 - } ?: SpeakerClusterer.Assignment(null, null, null)  
265 - val matchedSpeakerId = assignment.speakerId  
266 - val matchedSim = assignment.sim  
267 - val matchedName = assignment.name  
268 -  
269 - // d. Room insert.  
270 - val segmentId = kotlinx.coroutines.withContext(Dispatchers.IO) {  
271 - val id = repo.segments.insert(  
272 - Segment(  
273 - startTs = startTs,  
274 - endTs = now,  
275 - audioPath = audioPath,  
276 - durationMs = audioMs,  
277 - energyDb = 0f,  
278 - embedding = embedding?.toLeBytes(),  
279 - speakerId = matchedSpeakerId,  
280 - speakerSim = matchedSim,  
281 - ),  
282 - )  
283 - if (text.isNotEmpty()) {  
284 - repo.transcripts.insert(  
285 - Transcript(  
286 - segmentId = id,  
287 - text = text,  
288 - ),  
289 - )  
290 - }  
291 - id  
292 - } 282 + val startTs = System.currentTimeMillis() - audioMs
  283 + if (blockBuffer.isOpen && blockBuffer.shouldClose(startTs)) closeBlock()
  284 + blockBuffer.append(samples, startTs)
  285 + }
293 286
294 - Log.i(  
295 - TAG,  
296 - "segment#$segmentId audio=${audioMs}ms wall=${wallMs}ms " +  
297 - "speaker=${matchedName ?: "—"} sim=${matchedSim?.let { "%.2f".format(it) } ?: "—"} " +  
298 - "text=$text",  
299 - )  
300 - RecorderState.onSegment(text, audioMs, wallMs) 287 + private fun closeBlock() {
  288 + val dir = blockBuffer.close() ?: return
  289 + pendingBlocks.trySend(dir)
301 } 290 }
302 291
303 private fun startForegroundCompat() { 292 private fun startForegroundCompat() {