Commit 3512a84f758e79ab750db56c506e979f3820598d

Authored by yibo
1 parent c3220da9

feat(lifelog): 录音改为 block-based diarization-first 管线(攒段→整块分轮转写)

弃掉「录一句、转一句、实时聚类」——短句(0.3-30s) embedding 噪声大,实时滑动
聚类把噪声平均进 anchor,是声纹串人的根因。改为:

录音端(RecorderService):VAD 只负责存语音 + 检测块边界,删掉实时 transcribe/
embed/clusterer.assign。块关闭条件:静音≥5min 或 块时长≥10min(长会议保活)。
  - BlockBuffer:speech-only PCM 追加 + manifest(run 偏移→真实时间);崩溃重启
    listPending 续处理。
处理端(BlockProcessor):整块 pyannote 分轮 → 每 turn 切片 paraformer 转写 +
CAMPPlus 嵌入 + 全局 anchor 匹配 → 写 turn 行(turn = 时间轴一行)。pyannote
缺失则降级整块单 turn。块处理与采集解耦(Channel 顺序消费),不阻塞录音。

embedding 从源头是单说话人干净切片,不再被实时逐句污染。SpeakerClusterer.assign
也加 L1 冻结(命名 anchor 不漂移)。废弃旧的周期 DiarizationWorker(块创建即
diarize),删 DiarizationScheduler/Worker;纪要按钮不再触发它。

实测(95s 真实会议块):13 段语音 → 24 个干净不重叠 turn,逐 turn 带说话人+文字。

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/LifelogApp.kt
... ... @@ -3,7 +3,6 @@ package com.lemon.lifelog
3 3 import android.app.Application
4 4 import android.app.NotificationChannel
5 5 import android.app.NotificationManager
6   -import com.lemon.lifelog.diarize.DiarizationScheduler
7 6 import com.lemon.lifelog.service.RecorderService
8 7 import com.lemon.lifelog.summary.SummaryScheduler
9 8  
... ... @@ -18,9 +17,10 @@ class LifelogApp : Application() {
18 17 NotificationManager.IMPORTANCE_LOW,
19 18 )
20 19 )
21   - // 纪要 + diarization periodic worker(各自 15min,UPDATE)。
  20 + // 纪要 periodic worker(15min,UPDATE)。
  21 + // 注:旧的周期 DiarizationWorker 已废弃——diarization-first 管线在「块」关闭时
  22 + // 当场分轮+转写,不再需要事后整体重跑。声纹纠偏走「重新聚类」按钮(reclusterAll)。
22 23 SummaryScheduler.schedulePeriodic(this)
23   - DiarizationScheduler.schedulePeriodic(this)
24 24 // 相册:未处理照片的周期后台处理(人脸+VL),空闲时跑,不阻塞导入。
25 25 com.lemon.lifelog.vision.PhotoScheduler.schedulePeriodic(this)
26 26 }
... ...
lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/MainActivity.kt
... ... @@ -83,7 +83,6 @@ import com.lemon.lifelog.asr.XfyunIatClient
83 83 import com.lemon.lifelog.asr.transcribeWithStats
84 84 import com.lemon.lifelog.audio.SegmentPlayer
85 85 import com.lemon.lifelog.chat.ChatConfig
86   -import com.lemon.lifelog.diarize.DiarizationScheduler
87 86 import com.lemon.lifelog.summary.SummaryProgress
88 87 import com.lemon.lifelog.summary.SummaryRunLog
89 88 import com.lemon.lifelog.data.Repository
... ... @@ -905,7 +904,7 @@ private fun SummariesTab(repo: Repository, activity: Context, onJumpToTime: (Lon
905 904 Column(modifier = Modifier.fillMaxSize().padding(16.dp)) {
906 905 Row(verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(8.dp)) {
907 906 Text("纪要(${summaries.size})", style = MaterialTheme.typography.titleMedium)
908   - Button(onClick = { DiarizationScheduler.runNow(activity); SummaryScheduler.runNow(activity) }, enabled = !running) {
  907 + Button(onClick = { SummaryScheduler.runNow(activity) }, enabled = !running) {
909 908 if (running) {
910 909 CircularProgressIndicator(modifier = Modifier.size(16.dp), strokeWidth = 2.dp)
911 910 Spacer(Modifier.width(6.dp))
... ...
lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/asr/SpeakerClusterer.kt
... ... @@ -59,7 +59,12 @@ class SpeakerClusterer(
59 59 }
60 60  
61 61 if (best != null && bestSim >= matchThreshold) {
62   - // capped running mean:上限保证质心保持自适应、不被早期样本钉死。
  62 + if (autoIndexOf(best.name) == null) {
  63 + // 人工命名 anchor:只归属、**冻结质心不漂移**(同 reclusterAll 的 L1)。
  64 + // 否则 anchor 吸入相邻说话人后会朝其漂移、再吞更多 → 雪球式串人。
  65 + return@withLock Assignment(best.id, bestSim, best.name)
  66 + }
  67 + // 自动簇:capped running mean,质心保持自适应、不被早期样本钉死。
63 68 val n = minOf(best.count, maxSamplesPerCentroid)
64 69 val merged = FloatArray(emb.size) { i -> (best.centroid[i] * n + emb[i]) / (n + 1) }
65 70 l2normalize(merged)
... ...
lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/diarize/DiarizationScheduler.kt deleted
1   -package com.lemon.lifelog.diarize
2   -
3   -import android.content.Context
4   -import androidx.work.Constraints
5   -import androidx.work.ExistingPeriodicWorkPolicy
6   -import androidx.work.ExistingWorkPolicy
7   -import androidx.work.NetworkType
8   -import androidx.work.OneTimeWorkRequestBuilder
9   -import androidx.work.PeriodicWorkRequestBuilder
10   -import androidx.work.WorkManager
11   -import java.util.concurrent.TimeUnit
12   -
13   -object DiarizationScheduler {
14   - private const val PERIODIC_TAG = "lifelog-diarize-periodic"
15   - private const val ONE_SHOT_TAG = "lifelog-diarize-once"
16   -
17   - fun schedulePeriodic(ctx: Context) {
18   - val constraints = Constraints.Builder()
19   - .setRequiredNetworkType(NetworkType.NOT_REQUIRED)
20   - .setRequiresBatteryNotLow(true)
21   - .build()
22   - val request = PeriodicWorkRequestBuilder<DiarizationWorker>(15, TimeUnit.MINUTES)
23   - .setConstraints(constraints)
24   - .build()
25   - WorkManager.getInstance(ctx).enqueueUniquePeriodicWork(
26   - PERIODIC_TAG,
27   - ExistingPeriodicWorkPolicy.UPDATE,
28   - request,
29   - )
30   - }
31   -
32   - /** 手动跑一次(KEEP:已在跑/排队则忽略)。 */
33   - fun runNow(ctx: Context) {
34   - val request = OneTimeWorkRequestBuilder<DiarizationWorker>().addTag(ONE_SHOT_TAG).build()
35   - WorkManager.getInstance(ctx).enqueueUniqueWork(ONE_SHOT_TAG, ExistingWorkPolicy.KEEP, request)
36   - }
37   -}
lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/diarize/DiarizationWorker.kt deleted
1   -package com.lemon.lifelog.diarize
2   -
3   -import android.content.Context
4   -import android.util.Log
5   -import androidx.work.CoroutineWorker
6   -import androidx.work.WorkerParameters
7   -import com.lemon.lifelog.asr.DiarizationEngine
8   -import com.lemon.lifelog.asr.ModelDownloader
9   -import com.lemon.lifelog.asr.SpeakerClusterer
10   -import com.lemon.lifelog.asr.SpeakerEmbedder
11   -import com.lemon.lifelog.asr.toLeBytes
12   -import com.lemon.lifelog.audio.AudioDecoder
13   -import com.lemon.lifelog.data.Repository
14   -import com.lemon.lifelog.data.Segment
15   -import com.lemon.lifelog.summary.SummaryWorker
16   -import java.io.File
17   -import java.util.concurrent.atomic.AtomicBoolean
18   -import kotlinx.coroutines.Dispatchers
19   -import kotlinx.coroutines.withContext
20   -
21   -/**
22   - * 会话级 diarization 重跑(方案 A)。对每个**已结束、未 diarize**的会话:
23   - * 1. 解码 + 按序拼接该会话所有段音频(speech-only 16k mono)。
24   - * 2. pyannote 分割 + CAMPPlus + FastClustering → 准确的 [start,end]→局部说话人 turns
25   - * (能把「几秒多人」的段拆开)。
26   - * 3. 每个局部说话人 → 取其最长 turn 的 embedding → 经 SpeakerClusterer 桥接到
27   - * 全局 SpeakerProfile(跨会话/跨天「说话人N」一致)。
28   - * 4. 每个 VAD 段 → 找重叠最大的 turn → 重标 speakerId(段内多说话人先归主导者,v1)。
29   - *
30   - * 与逐段实时聚类相比准确得多;实时标签只作即时占位,这里批处理纠正。
31   - */
32   -class DiarizationWorker(ctx: Context, params: WorkerParameters) : CoroutineWorker(ctx, params) {
33   -
34   - companion object {
35   - const val TAG = "DiarizationWorker"
36   - private val isRunning = AtomicBoolean(false)
37   - // WorkManager 给每个 worker 硬上限 10min,超时直接 kill。预算压到 5min,
38   - // 留足余量让最后一块 diarize 收尾 + 全局 recluster + 写 cursor,绝不撞红线。
39   - const val RUN_BUDGET_MS = 5 * 60 * 1000L
40   - const val MIN_SESSION_SEGMENTS = 2 // 单段会话无需 diarize
41   - // 单块 ≤3min speech:一次 diarize ~3min,远小于 10min 红线,一轮能跑完多块、
42   - // 多次写 cursor 推进;块内说话人 index 不跨块——身份交全局 reclusterAll。
43   - const val CHUNK_SAMPLES = 3 * 60 * 16000
44   - private const val KEY_DONE = "diarized_sessions" // prefs StringSet of windowStartTs
45   - private const val KEY_CURSOR = "diar_cursor_" // + key → 已处理段数(断点续跑长会议)
46   - }
47   -
48   - private val sr = 16000
49   -
50   - override suspend fun doWork(): Result {
51   - if (!isRunning.compareAndSet(false, true)) {
52   - Log.i(TAG, "another diarization in progress, skip"); return Result.success()
53   - }
54   - return try {
55   - run()
56   - } catch (t: Throwable) {
57   - Log.w(TAG, "diarization run failed", t); Result.success()
58   - } finally {
59   - isRunning.set(false)
60   - }
61   - }
62   -
63   - private suspend fun run(): Result {
64   - val runStart = System.currentTimeMillis()
65   - val now = System.currentTimeMillis()
66   - val prefs = applicationContext.getSharedPreferences(SummaryWorker.PREFS, Context.MODE_PRIVATE)
67   - val gapMs = prefs.getInt(SummaryWorker.KEY_GAP_MIN, SummaryWorker.DEFAULT_GAP_MIN) * 60_000L
68   - val threshold = SpeakerClusterer.thresholdFromPrefs(applicationContext)
69   -
70   - val repo = Repository(applicationContext)
71   - val segments = repo.segments.page(limit = 5000, offset = 0).sortedBy { it.startTs }
72   - if (segments.size < MIN_SESSION_SEGMENTS) return Result.success()
73   -
74   - val sessions = SummaryWorker.groupSessions(segments, gapMs)
75   - val done = prefs.getStringSet(KEY_DONE, emptySet())!!.toMutableSet()
76   -
77   - // 模型路径
78   - val downloader = ModelDownloader(applicationContext)
79   - val speakerFile = downloader.ensureSpeakerModel { }
80   - if (!speakerFile.present) { Log.w(TAG, "speaker model missing"); return Result.success() }
81   - val pyannote = downloader.ensurePyannoteModel { Log.i(TAG, "pyannote: $it") }
82   - if (!pyannote.present) { Log.w(TAG, "pyannote seg model missing/download failed"); return Result.success() }
83   -
84   - var engine: DiarizationEngine? = null
85   - var embedder: SpeakerEmbedder? = null
86   - var processed = 0
87   - var attempted = false
88   - try {
89   - // 最新会话优先(用户更关心最近的;旧会话后续轮次补)。
90   - for (session in sessions.reversed()) {
91   - val start = session.first().startTs
92   - val end = session.last().endTs
93   - val key = start.toString()
94   -
95   - if (now - end <= gapMs) continue // 未结束
96   - if (session.size < MIN_SESSION_SEGMENTS) continue
97   - if (done.contains(key)) continue // 已 diarize
98   - if (System.currentTimeMillis() - runStart > RUN_BUDGET_MS) {
99   - Log.i(TAG, "time budget hit, rest next round"); break
100   - }
101   -
102   - // 懒加载模型(有活才加载)
103   - if (engine == null) {
104   - engine = DiarizationEngine.load(pyannote.path, speakerFile.path, numThreads = 4, threshold = threshold)
105   - embedder = SpeakerEmbedder.load(speakerFile.path, numThreads = 2)
106   - }
107   -
108   - attempted = true
109   - val startIdx = prefs.getInt(KEY_CURSOR + key, 0)
110   - val reached = diarizeSession(repo, session, engine!!, embedder!!, runStart, startIdx)
111   - if (reached >= session.size) {
112   - done.add(key); processed++
113   - prefs.edit().remove(KEY_CURSOR + key).apply()
114   - } else {
115   - // 预算中途用完:记录断点,下轮从此处续跑(不重头),然后退出本轮。
116   - prefs.edit().putInt(KEY_CURSOR + key, reached).apply()
117   - Log.i(TAG, "session $key partial: cursor=$reached/${session.size}, resume next round"); break
118   - }
119   - }
120   - } finally {
121   - engine?.release()
122   - embedder?.release()
123   - }
124   -
125   - // 全局重聚类:用 diarization 产出的"干净"per-turn embedding 统一聚类
126   - //(reclusterAll 本就全局 + 保留人工命名锚点)。只要这轮碰过会话就跑。
127   - if (attempted) {
128   - val n = SpeakerClusterer.reclusterAll(repo, matchThreshold = threshold)
129   - Log.i(TAG, "global recluster → $n speakers")
130   - }
131   - prefs.edit().putStringSet(KEY_DONE, done).apply()
132   - Log.i(TAG, "done: ${sessions.size} sessions, $processed diarized")
133   - return Result.success()
134   - }
135   -
136   - /**
137   - * 分块 diarize 整个会话(长会议不再只取前 N min),从 [startIdx] 段断点续跑。
138   - * 返回**已处理到的段下标**(== session.size 表示整场跑完;< 表示预算中途用完,
139   - * 外层据此存断点、下轮从这里续跑——块内说话人 index 不跨块,身份交全局 reclusterAll)。
140   - */
141   - private suspend fun diarizeSession(
142   - repo: Repository,
143   - session: List<Segment>,
144   - engine: DiarizationEngine,
145   - embedder: SpeakerEmbedder,
146   - runStart: Long,
147   - startIdx: Int,
148   - ): Int = withContext(Dispatchers.Default) {
149   - var idx = startIdx.coerceIn(0, session.size)
150   - var nChunks = 0
151   - var totalTurns = 0
152   - var embedded = 0
153   - while (idx < session.size) {
154   - if (System.currentTimeMillis() - runStart > RUN_BUDGET_MS) {
155   - Log.i(TAG, "session ${session.first().startTs}: budget hit at seg $idx/${session.size}")
156   - return@withContext idx // 断点,外层续跑
157   - }
158   - // 攒一块(≤CHUNK_SAMPLES),解码即拼,记录每段在块内的 [from,to]
159   - val segs = ArrayList<Segment>()
160   - val froms = ArrayList<Int>()
161   - val tos = ArrayList<Int>()
162   - val pcms = ArrayList<FloatArray>()
163   - var len = 0
164   - while (idx < session.size && len < CHUNK_SAMPLES) {
165   - val seg = session[idx]; idx++
166   - val pcm = AudioDecoder.decodeToFloat(seg.audioPath)
167   - if (pcm.isEmpty()) continue
168   - segs.add(seg); froms.add(len); pcms.add(pcm); len += pcm.size; tos.add(len)
169   - }
170   - if (segs.isEmpty()) continue
171   - val samples = FloatArray(len)
172   - var off = 0
173   - for (p in pcms) { p.copyInto(samples, off); off += p.size }
174   - pcms.clear()
175   -
176   - val turns = engine.diarize(samples)
177   - if (turns.isEmpty()) continue
178   - nChunks++; totalTurns += turns.size
179   -
180   - val turnEmb = HashMap<Int, FloatArray?>() // turn 下标 → embedding(多段共用同一 turn 缓存)
181   - for (k in segs.indices) {
182   - val s0 = froms[k].toFloat() / sr
183   - val s1 = tos[k].toFloat() / sr
184   - var bestIdx = -1
185   - var bestOv = 0f
186   - turns.forEachIndexed { i, t ->
187   - val ov = minOf(s1, t.endSec) - maxOf(s0, t.startSec)
188   - if (ov > bestOv) { bestOv = ov; bestIdx = i }
189   - }
190   - if (bestIdx < 0) continue
191   - val emb = turnEmb.getOrPut(bestIdx) {
192   - val t = turns[bestIdx]
193   - val a = (t.startSec * sr).toInt().coerceIn(0, samples.size)
194   - val b = (t.endSec * sr).toInt().coerceIn(a, samples.size)
195   - if (b - a < sr / 4) null else embedder.embed(samples.copyOfRange(a, b), sr)
196   - }
197   - if (emb != null) {
198   - withContext(Dispatchers.IO) { repo.segments.updateEmbedding(segs[k].id, emb.toLeBytes()) }
199   - embedded++
200   - }
201   - }
202   - }
203   - Log.i(TAG, "session ${session.first().startTs}: $startIdx→${session.size} segs, $nChunks chunks, $totalTurns turns, $embedded clean-embedded")
204   - idx // == session.size,整场跑完
205   - }
206   -}
lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/service/BlockBuffer.kt 0 → 100644
  1 +package com.lemon.lifelog.service
  2 +
  3 +import android.util.Log
  4 +import java.io.File
  5 +import java.nio.ByteBuffer
  6 +import java.nio.ByteOrder
  7 +import org.json.JSONArray
  8 +import org.json.JSONObject
  9 +
  10 +/**
  11 + * 当前录音「块」的累积器(diarization-first 管线的录音端)。
  12 + *
  13 + * VAD 每产出一段语音就 [append] 进块的 `audio.pcm`(raw PCM16 LE,16k mono),并把该段的
  14 + * `{wallTs, sampleOffset, sampleLen}` 记进 manifest(用于之后把 turn 的样本偏移还原成真实时间)。
  15 + *
  16 + * 关块条件([shouldClose],由 RecorderService 每帧/每段检查):
  17 + * - 距上次语音静音 ≥ [SILENCE_MS](5min) 或
  18 + * - 块时长 ≥ [MAX_MS](10min,长会议保活,避免全程不出文字)
  19 + *
  20 + * 块目录:`filesDir/blocks/<blockStartTs>/{audio.pcm, manifest.json}`。
  21 + * [close] 落 manifest(closed=true) 后交给 BlockProcessor;崩溃时块目录仍在,重启 [listPending] 重扫。
  22 + * 只存 speech(静音不落盘)→ 内存/磁盘按语音时长计,不按墙钟。
  23 + */
  24 +class BlockBuffer(private val root: File, private val sampleRate: Int = 16000) {
  25 +
  26 + data class Run(val ts: Long, val off: Int, val len: Int)
  27 +
  28 + private var dir: File? = null
  29 + private var out: java.io.BufferedOutputStream? = null
  30 + private val runs = ArrayList<Run>()
  31 + private var sampleCount = 0
  32 + private var blockStartTs = 0L
  33 + private var lastSpeechEndTs = 0L
  34 +
  35 + val isOpen get() = dir != null
  36 +
  37 + init {
  38 + root.mkdirs()
  39 + }
  40 +
  41 + /** 追加一段语音(startTs = 该段真实起始墙钟)。块未开则自动开。 */
  42 + fun append(samples: FloatArray, startTs: Long) {
  43 + if (samples.isEmpty()) return
  44 + if (!isOpen) open(startTs)
  45 + val bytes = ByteArray(samples.size * 2)
  46 + val bb = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN)
  47 + for (s in samples) bb.putShort((s.coerceIn(-1f, 1f) * 32767f).toInt().toShort())
  48 + out!!.write(bytes)
  49 + out!!.flush()
  50 + runs.add(Run(startTs, sampleCount, samples.size))
  51 + sampleCount += samples.size
  52 + lastSpeechEndTs = startTs + samples.size.toLong() * 1000 / sampleRate
  53 + writeManifest(dir!!, closed = false)
  54 + }
  55 +
  56 + /** 在 [ts] 这一刻是否该关块(静音超时或块超长)。块未开恒 false。 */
  57 + fun shouldClose(ts: Long): Boolean {
  58 + if (!isOpen) return false
  59 + return (ts - lastSpeechEndTs >= SILENCE_MS) || (ts - blockStartTs >= MAX_MS)
  60 + }
  61 +
  62 + /** 关块:落 manifest(closed=true),返回块目录;空块直接删并返回 null。 */
  63 + fun close(): File? {
  64 + if (!isOpen) return null
  65 + runCatching { out?.flush(); out?.close() }
  66 + out = null
  67 + val d = dir!!
  68 + dir = null
  69 + if (sampleCount == 0) {
  70 + d.deleteRecursively()
  71 + return null
  72 + }
  73 + writeManifest(d, closed = true)
  74 + Log.i(TAG, "block close ${d.name}: ${runs.size} runs / $sampleCount samples")
  75 + return d
  76 + }
  77 +
  78 + private fun open(ts: Long) {
  79 + blockStartTs = ts
  80 + lastSpeechEndTs = ts
  81 + runs.clear()
  82 + sampleCount = 0
  83 + val d = File(root, ts.toString()).apply { mkdirs() }
  84 + dir = d
  85 + out = File(d, "audio.pcm").outputStream().buffered()
  86 + Log.i(TAG, "block open @$ts -> ${d.name}")
  87 + }
  88 +
  89 + private fun writeManifest(d: File, closed: Boolean) {
  90 + val arr = JSONArray()
  91 + for (r in runs) arr.put(JSONObject().put("ts", r.ts).put("off", r.off).put("len", r.len))
  92 + val obj = JSONObject()
  93 + .put("start", blockStartTs)
  94 + .put("sr", sampleRate)
  95 + .put("closed", closed)
  96 + .put("runs", arr)
  97 + runCatching { File(d, "manifest.json").writeText(obj.toString()) }
  98 + }
  99 +
  100 + companion object {
  101 + private const val TAG = "BlockBuffer"
  102 + const val SILENCE_MS = 5 * 60 * 1000L // 静音 ≥5min 切块
  103 + const val MAX_MS = 10 * 60 * 1000L // 块 ≥10min 强切(长会议保活)
  104 +
  105 + data class Loaded(val startTs: Long, val sampleRate: Int, val samples: FloatArray, val runs: List<Run>)
  106 +
  107 + /** 启动时列出所有未处理块目录(崩溃恢复:处理后目录会被删除)。 */
  108 + fun listPending(root: File): List<File> =
  109 + root.listFiles()
  110 + ?.filter { it.isDirectory && File(it, "audio.pcm").exists() }
  111 + ?.sortedBy { it.name.toLongOrNull() ?: 0L }
  112 + ?: emptyList()
  113 +
  114 + /** 读块:manifest + audio.pcm → 整块 float + runs。 */
  115 + fun load(dir: File): Loaded? {
  116 + val af = File(dir, "audio.pcm")
  117 + if (!af.exists()) return null
  118 + val mf = File(dir, "manifest.json")
  119 + var start = dir.name.toLongOrNull() ?: 0L
  120 + var sr = 16000
  121 + val runs = ArrayList<Run>()
  122 + if (mf.exists()) {
  123 + runCatching {
  124 + val o = JSONObject(mf.readText())
  125 + start = o.optLong("start", start)
  126 + sr = o.optInt("sr", 16000)
  127 + o.optJSONArray("runs")?.let { a ->
  128 + for (i in 0 until a.length()) {
  129 + val r = a.getJSONObject(i)
  130 + runs.add(Run(r.getLong("ts"), r.getInt("off"), r.getInt("len")))
  131 + }
  132 + }
  133 + }
  134 + }
  135 + val bytes = af.readBytes()
  136 + val n = bytes.size / 2
  137 + val floats = FloatArray(n)
  138 + val bb = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN)
  139 + for (i in 0 until n) floats[i] = bb.short / 32768f
  140 + // 崩溃时 manifest 可能没记全 runs → 合成一个整块 run(时间退化为连续)。
  141 + if (runs.isEmpty()) runs.add(Run(start, 0, n))
  142 + return Loaded(start, sr, floats, runs)
  143 + }
  144 + }
  145 +}
... ...
lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/service/BlockProcessor.kt 0 → 100644
  1 +package com.lemon.lifelog.service
  2 +
  3 +import android.util.Log
  4 +import com.lemon.lifelog.asr.DiarizationEngine
  5 +import com.lemon.lifelog.asr.ParaformerEngine
  6 +import com.lemon.lifelog.asr.SpeakerClusterer
  7 +import com.lemon.lifelog.asr.SpeakerEmbedder
  8 +import com.lemon.lifelog.asr.toLeBytes
  9 +import com.lemon.lifelog.audio.SegmentWriter
  10 +import com.lemon.lifelog.data.Repository
  11 +import com.lemon.lifelog.data.Segment
  12 +import com.lemon.lifelog.data.Transcript
  13 +import java.io.File
  14 +import kotlinx.coroutines.Dispatchers
  15 +import kotlinx.coroutines.withContext
  16 +
  17 +/**
  18 + * 把一个关闭的录音块处理成时间轴 turns(diarization-first 管线的处理端):
  19 + *
  20 + * pyannote 分轮 → 每个 turn 切片 → paraformer 转写 + CAMPPlus 嵌入 + 全局 anchor 匹配
  21 + * → 写 turn WAV + `segments` 行 + `transcripts` 行。
  22 + *
  23 + * 一个 **turn = 时间轴一行**(一人一段连续发言)。embedding 是单说话人的干净切片嵌入,
  24 + * 从源头避免了实时逐句聚类的噪声污染。处理完删块目录(= 已处理标记)。
  25 + *
  26 + * [diar] 为 null(pyannote 模型缺失)时降级:整块当一个 turn 转写。
  27 + */
  28 +class BlockProcessor(
  29 + private val repo: Repository,
  30 + private val segWriter: SegmentWriter,
  31 + private val diar: DiarizationEngine?,
  32 + private val asr: ParaformerEngine,
  33 + private val embedder: SpeakerEmbedder?,
  34 + private val clusterer: SpeakerClusterer,
  35 +) {
  36 + suspend fun process(dir: File) {
  37 + val loaded = withContext(Dispatchers.IO) { BlockBuffer.load(dir) }
  38 + if (loaded == null || loaded.samples.isEmpty()) {
  39 + withContext(Dispatchers.IO) { dir.deleteRecursively() }
  40 + return
  41 + }
  42 + val sr = loaded.sampleRate
  43 + val samples = loaded.samples
  44 + val runs = loaded.runs
  45 +
  46 + // 样本下标 → 真实墙钟(speech-only 拼接,用 run 偏移还原)。runs 按 off 升序。
  47 + fun wallAt(sample: Int): Long {
  48 + var r = runs.first()
  49 + for (run in runs) if (run.off <= sample) r = run else break
  50 + val within = (sample - r.off).coerceIn(0, r.len)
  51 + return r.ts + within.toLong() * 1000 / sr
  52 + }
  53 +
  54 + var turns = diar?.diarize(samples) ?: emptyList()
  55 + if (turns.isEmpty()) {
  56 + // 降级 / 极短块:整块一个 turn。
  57 + turns = listOf(DiarizationEngine.Turn(0f, samples.size.toFloat() / sr, 0))
  58 + }
  59 +
  60 + var written = 0
  61 + for (t in turns) {
  62 + val a = (t.startSec * sr).toInt().coerceIn(0, samples.size)
  63 + val b = (t.endSec * sr).toInt().coerceIn(a, samples.size)
  64 + if (b - a < sr / 5) continue // <200ms 跳过
  65 + val slice = samples.copyOfRange(a, b)
  66 + val startTs = wallAt(a)
  67 + val durMs = (b - a).toLong() * 1000 / sr
  68 +
  69 + val text = try {
  70 + asr.transcribeFloat(slice, sr).text
  71 + } catch (e: Throwable) {
  72 + Log.w(TAG, "asr failed", e); ""
  73 + }
  74 + val emb = embedder?.let {
  75 + try { it.embed(slice, sr) } catch (e: Throwable) { Log.w(TAG, "embed failed", e); null }
  76 + }
  77 + val assign = emb?.let { clusterer.assign(it, durMs) }
  78 + ?: SpeakerClusterer.Assignment(null, null, null)
  79 +
  80 + val audioPath = withContext(Dispatchers.IO) { segWriter.write(slice, startTs, sr) }
  81 + withContext(Dispatchers.IO) {
  82 + val id = repo.segments.insert(
  83 + Segment(
  84 + startTs = startTs,
  85 + endTs = startTs + durMs,
  86 + audioPath = audioPath,
  87 + durationMs = durMs,
  88 + energyDb = 0f,
  89 + embedding = emb?.toLeBytes(),
  90 + speakerId = assign.speakerId,
  91 + speakerSim = assign.sim,
  92 + ),
  93 + )
  94 + if (text.isNotBlank()) repo.transcripts.insert(Transcript(segmentId = id, text = text))
  95 + }
  96 + written++
  97 + }
  98 + Log.i(TAG, "block ${dir.name}: ${turns.size} turns -> $written rows")
  99 + withContext(Dispatchers.IO) { dir.deleteRecursively() }
  100 + }
  101 +
  102 + companion object {
  103 + private const val TAG = "BlockProcessor"
  104 + }
  105 +}
... ...
lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/service/RecorderService.kt
... ... @@ -14,23 +14,21 @@ import com.k2fsa.sherpa.onnx.SileroVadModelConfig
14 14 import com.k2fsa.sherpa.onnx.Vad
15 15 import com.k2fsa.sherpa.onnx.VadModelConfig
16 16 import com.lemon.lifelog.MainActivity
  17 +import com.lemon.lifelog.asr.DiarizationEngine
17 18 import com.lemon.lifelog.asr.ModelDownloader
18 19 import com.lemon.lifelog.asr.ParaformerEngine
19 20 import com.lemon.lifelog.asr.SpeakerClusterer
20 21 import com.lemon.lifelog.asr.SpeakerEmbedder
21   -import com.lemon.lifelog.asr.toLeBytes
22 22 import com.lemon.lifelog.audio.AudioCapture
23 23 import com.lemon.lifelog.audio.SegmentWriter
24 24 import com.lemon.lifelog.data.Repository
25   -import com.lemon.lifelog.data.Segment
26   -import com.lemon.lifelog.data.SpeakerProfile
27   -import com.lemon.lifelog.data.Transcript
28 25 import java.io.File
29 26 import kotlinx.coroutines.CoroutineScope
30 27 import kotlinx.coroutines.Dispatchers
31 28 import kotlinx.coroutines.Job
32 29 import kotlinx.coroutines.SupervisorJob
33 30 import kotlinx.coroutines.cancel
  31 +import kotlinx.coroutines.channels.Channel
34 32 import kotlinx.coroutines.flow.collect
35 33 import kotlinx.coroutines.launch
36 34  
... ... @@ -70,10 +68,17 @@ class RecorderService : Service() {
70 68 private var vad: Vad? = null
71 69 private var engine: ParaformerEngine? = null
72 70 private var speakerEmbedder: SpeakerEmbedder? = null
  71 + private var diarEngine: DiarizationEngine? = null
73 72  
74 73 /** 在线说话人聚类器(方案 B):自动发现说话人并持久化为 speaker_profiles。 */
75 74 private var clusterer: SpeakerClusterer? = null
76 75  
  76 + /** diarization-first 管线:VAD 段攒进当前块,块关闭后整块 diarize+转写。 */
  77 + private lateinit var blockBuffer: BlockBuffer
  78 +
  79 + /** 关闭的块排队顺序处理(与采集解耦,不阻塞录音)。 */
  80 + private val pendingBlocks = Channel<File>(Channel.UNLIMITED)
  81 +
77 82 override fun onCreate() {
78 83 super.onCreate()
79 84 startForegroundCompat()
... ... @@ -83,6 +88,7 @@ class RecorderService : Service() {
83 88 RecorderState.reset()
84 89 repo = Repository(this)
85 90 segWriter = SegmentWriter(File(filesDir, "segments"))
  91 + blockBuffer = BlockBuffer(File(filesDir, "blocks"))
86 92 capture = AudioCapture()
87 93  
88 94 pipelineJob = scope.launch { runPipeline() }
... ... @@ -113,7 +119,7 @@ class RecorderService : Service() {
113 119 acquireWakeLock()
114 120 capture.start(scope)
115 121 RecorderState.setPaused(false)
116   - RecorderState.setStatus("listening")
  122 + RecorderState.setStatus("录音中")
117 123 notify("录音中")
118 124 Log.i(TAG, "resumed")
119 125 }
... ... @@ -126,9 +132,13 @@ class RecorderService : Service() {
126 132 override fun onDestroy() {
127 133 pipelineJob?.cancel()
128 134 capture.stop()
  135 + // 关掉当前块(落 manifest 留在磁盘,下次启动 listPending 续处理)。
  136 + runCatching { blockBuffer.close() }
  137 + pendingBlocks.close()
129 138 vad?.release()
130 139 engine?.release()
131 140 speakerEmbedder?.release()
  141 + diarEngine?.release()
132 142 scope.cancel()
133 143 wakeLock?.takeIf { it.isHeld }?.release()
134 144 wakeLock = null
... ... @@ -210,94 +220,73 @@ class RecorderService : Service() {
210 220 repo.speakers.observeAll().collect { clusterer?.load() }
211 221 }
212 222  
  223 + // 2e. Diarization 引擎(pyannote 分割 + CAMPPlus),块级整段分轮用。缺失则降级单 turn。
  224 + try {
  225 + RecorderState.setStatus("loading diarization…")
  226 + val pyannote = downloader.ensurePyannoteModel { Log.i(TAG, "pyannote dl: $it") }
  227 + val speaker = downloader.speakerPath()
  228 + if (pyannote.present && speaker.present) {
  229 + diarEngine = DiarizationEngine.load(
  230 + pyannote.path, speaker.path, numThreads = 4,
  231 + threshold = SpeakerClusterer.thresholdFromPrefs(this),
  232 + )
  233 + } else {
  234 + Log.w(TAG, "pyannote/speaker model missing; block will fall back to single-turn")
  235 + }
  236 + } catch (t: Throwable) {
  237 + Log.w(TAG, "diarization engine unavailable; single-turn fallback", t)
  238 + }
  239 +
  240 + // 2f. 块处理器 + 顺序处理协程(与采集解耦)。
  241 + val processor = BlockProcessor(repo, segWriter, diarEngine, engine!!, speakerEmbedder, clusterer!!)
  242 + scope.launch {
  243 + for (dir in pendingBlocks) {
  244 + RecorderState.setStatus("处理段落…")
  245 + try {
  246 + processor.process(dir)
  247 + } catch (t: Throwable) {
  248 + Log.w(TAG, "block ${dir.name} process failed", t)
  249 + }
  250 + RecorderState.setStatus(if (paused) "已暂停" else "录音中")
  251 + }
  252 + }
  253 + // 崩溃恢复:把上次没处理完的块重新入队。
  254 + BlockBuffer.listPending(File(filesDir, "blocks")).forEach { pendingBlocks.trySend(it) }
  255 +
213 256 // 3. Start capturing (unless paused arrived during model load).
214 257 if (!paused) {
215 258 capture.start(scope)
216   - RecorderState.setStatus("listening")
  259 + RecorderState.setStatus("录音中")
217 260 }
218   - Log.i(TAG, "pipeline running, clusterer loaded")
  261 + Log.i(TAG, "pipeline running (block-based diarization-first)")
219 262  
220 263 capture.frames.collect { samples ->
221 264 val v = vad ?: return@collect
222 265 v.acceptWaveform(samples)
  266 + val now = System.currentTimeMillis()
223 267 while (!v.empty()) {
224 268 val seg = v.front()
225 269 v.pop()
226 270 if (seg.samples.isEmpty()) continue
227   - handleSegment(seg.samples)
  271 + appendSpeech(seg.samples)
228 272 }
  273 + // 静音超时 / 块超长 → 关块入队(即使本帧没有新语音也要检查)。
  274 + if (blockBuffer.shouldClose(now)) closeBlock()
229 275 }
230 276 }
231 277  
232   - private suspend fun handleSegment(samples: FloatArray) {
  278 + /** VAD 段 → 攒进当前块(不再实时转写/聚类)。跨静音/超长时先关旧块。 */
  279 + private fun appendSpeech(samples: FloatArray) {
233 280 val sr = AudioCapture.SAMPLE_RATE
234 281 val audioMs = samples.size.toLong() * 1000 / sr
235   - val now = System.currentTimeMillis()
236   - val startTs = now - audioMs
237   -
238   - // a. persist audio (off CPU thread, Opus encode).
239   - val audioPath = kotlinx.coroutines.withContext(Dispatchers.IO) {
240   - segWriter.write(samples, startTs, sr)
241   - }
242   -
243   - // b. paraformer.
244   - val wallStart = System.nanoTime()
245   - val text = try {
246   - engine!!.transcribeFloat(samples, sr).text
247   - } catch (t: Throwable) {
248   - Log.w(TAG, "transcribe failed", t)
249   - ""
250   - }
251   - val wallMs = (System.nanoTime() - wallStart) / 1_000_000
252   -
253   - // c. speaker embedding + online clustering (auto-discovers 说话人N).
254   - val embedding: FloatArray? = speakerEmbedder?.let { embedder ->
255   - try {
256   - embedder.embed(samples, sr)
257   - } catch (t: Throwable) {
258   - Log.w(TAG, "embed failed", t)
259   - null
260   - }
261   - }
262   - val assignment = embedding?.let { emb ->
263   - clusterer?.assign(emb, audioMs)
264   - } ?: SpeakerClusterer.Assignment(null, null, null)
265   - val matchedSpeakerId = assignment.speakerId
266   - val matchedSim = assignment.sim
267   - val matchedName = assignment.name
268   -
269   - // d. Room insert.
270   - val segmentId = kotlinx.coroutines.withContext(Dispatchers.IO) {
271   - val id = repo.segments.insert(
272   - Segment(
273   - startTs = startTs,
274   - endTs = now,
275   - audioPath = audioPath,
276   - durationMs = audioMs,
277   - energyDb = 0f,
278   - embedding = embedding?.toLeBytes(),
279   - speakerId = matchedSpeakerId,
280   - speakerSim = matchedSim,
281   - ),
282   - )
283   - if (text.isNotEmpty()) {
284   - repo.transcripts.insert(
285   - Transcript(
286   - segmentId = id,
287   - text = text,
288   - ),
289   - )
290   - }
291   - id
292   - }
  282 + val startTs = System.currentTimeMillis() - audioMs
  283 + if (blockBuffer.isOpen && blockBuffer.shouldClose(startTs)) closeBlock()
  284 + blockBuffer.append(samples, startTs)
  285 + }
293 286  
294   - Log.i(
295   - TAG,
296   - "segment#$segmentId audio=${audioMs}ms wall=${wallMs}ms " +
297   - "speaker=${matchedName ?: "—"} sim=${matchedSim?.let { "%.2f".format(it) } ?: "—"} " +
298   - "text=$text",
299   - )
300   - RecorderState.onSegment(text, audioMs, wallMs)
  287 + private fun closeBlock() {
  288 + val dir = blockBuffer.close() ?: return
  289 + pendingBlocks.trySend(dir)
301 290 }
302 291  
303 292 private fun startForegroundCompat() {
... ...