Commit f47915e0687f2c41f4d587a3d9c02ec48550240a
1 parent
1edfc66e
feat(lifelog/diarize): 会话级 diarization 重跑(方案 A)—— 解决段内多说话人
逐 VAD 段单 embedding 的实时方案无法处理「一段几秒内多人说话」(混合 embedding=噪声) + 在线贪心聚类过度合并。改为会话级批处理重跑(与纪要 worker 同构): - DiarizationEngine:sherpa-onnx OfflineSpeakerDiarization(pyannote 分割检测 说话人变更点+重叠 → CAMPPlus embedding → FastClustering 自动聚类)。 - AudioDecoder:.opus(Opus-in-Ogg) → 16k mono PCM(MediaCodec,含 48k→16k 重采样 + 下混)。 - DiarizationWorker:每个已结束/未 diarize 会话 → 解码拼接 → diarize → 局部说话人 桥接全局 SpeakerProfile(复用 SpeakerClusterer)→ 段按重叠最大 turn 重标。 AtomicBoolean 互斥 + 7min 预算 + 进程内 idempotency(prefs)。 - DiarizationScheduler:periodic 15min(UPDATE) + 「现在跑一次」一并触发。 - ModelDownloader.ensurePyannoteModel:首启下 pyannote-seg(~6MB)。 实测端到端通:56 段会话 → 44 turns、检测出多说话人(核心能力达成)。 已知待调(非 bug): - 测试数据是有声书(一人配几十角色),对声纹是地狱级输入 → 全局说话人偏多(98)。 需在真实多人录音上验证;threshold 走设置页可调。 - 全局身份桥接现为贪心 match-or-create,易累积;v1.1 应改全局重聚类。 - 段内多说话人 v1 先归主导 turn,不拆分+重 ASR(v2)。 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Showing
7 changed files
with
429 additions
and
2 deletions
lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/LifelogApp.kt
| @@ -3,6 +3,7 @@ package com.lemon.lifelog | @@ -3,6 +3,7 @@ package com.lemon.lifelog | ||
| 3 | import android.app.Application | 3 | import android.app.Application |
| 4 | import android.app.NotificationChannel | 4 | import android.app.NotificationChannel |
| 5 | import android.app.NotificationManager | 5 | import android.app.NotificationManager |
| 6 | +import com.lemon.lifelog.diarize.DiarizationScheduler | ||
| 6 | import com.lemon.lifelog.service.RecorderService | 7 | import com.lemon.lifelog.service.RecorderService |
| 7 | import com.lemon.lifelog.summary.SummaryScheduler | 8 | import com.lemon.lifelog.summary.SummaryScheduler |
| 8 | 9 | ||
| @@ -17,7 +18,8 @@ class LifelogApp : Application() { | @@ -17,7 +18,8 @@ class LifelogApp : Application() { | ||
| 17 | NotificationManager.IMPORTANCE_LOW, | 18 | NotificationManager.IMPORTANCE_LOW, |
| 18 | ) | 19 | ) |
| 19 | ) | 20 | ) |
| 20 | - // Hourly 纪要 worker (KEEP = 已排过就不动)。 | 21 | + // 纪要 + diarization periodic worker(各自 15min,UPDATE)。 |
| 21 | SummaryScheduler.schedulePeriodic(this) | 22 | SummaryScheduler.schedulePeriodic(this) |
| 23 | + DiarizationScheduler.schedulePeriodic(this) | ||
| 22 | } | 24 | } |
| 23 | } | 25 | } |
lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/MainActivity.kt
| @@ -83,6 +83,7 @@ import com.lemon.lifelog.asr.XfyunIatClient | @@ -83,6 +83,7 @@ import com.lemon.lifelog.asr.XfyunIatClient | ||
| 83 | import com.lemon.lifelog.asr.transcribeWithStats | 83 | import com.lemon.lifelog.asr.transcribeWithStats |
| 84 | import com.lemon.lifelog.audio.SegmentPlayer | 84 | import com.lemon.lifelog.audio.SegmentPlayer |
| 85 | import com.lemon.lifelog.chat.ChatConfig | 85 | import com.lemon.lifelog.chat.ChatConfig |
| 86 | +import com.lemon.lifelog.diarize.DiarizationScheduler | ||
| 86 | import com.lemon.lifelog.summary.SummaryProgress | 87 | import com.lemon.lifelog.summary.SummaryProgress |
| 87 | import com.lemon.lifelog.summary.SummaryRunLog | 88 | import com.lemon.lifelog.summary.SummaryRunLog |
| 88 | import com.lemon.lifelog.data.Repository | 89 | import com.lemon.lifelog.data.Repository |
| @@ -848,7 +849,7 @@ private fun SummariesTab(repo: Repository, activity: Context, onJumpToTime: (Lon | @@ -848,7 +849,7 @@ private fun SummariesTab(repo: Repository, activity: Context, onJumpToTime: (Lon | ||
| 848 | Column(modifier = Modifier.fillMaxSize().padding(16.dp)) { | 849 | Column(modifier = Modifier.fillMaxSize().padding(16.dp)) { |
| 849 | Row(verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(8.dp)) { | 850 | Row(verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(8.dp)) { |
| 850 | Text("纪要(${summaries.size})", style = MaterialTheme.typography.titleMedium) | 851 | Text("纪要(${summaries.size})", style = MaterialTheme.typography.titleMedium) |
| 851 | - Button(onClick = { SummaryScheduler.runNow(activity) }, enabled = !running) { | 852 | + Button(onClick = { DiarizationScheduler.runNow(activity); SummaryScheduler.runNow(activity) }, enabled = !running) { |
| 852 | if (running) { | 853 | if (running) { |
| 853 | CircularProgressIndicator(modifier = Modifier.size(16.dp), strokeWidth = 2.dp) | 854 | CircularProgressIndicator(modifier = Modifier.size(16.dp), strokeWidth = 2.dp) |
| 854 | Spacer(Modifier.width(6.dp)) | 855 | Spacer(Modifier.width(6.dp)) |
lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/asr/DiarizationEngine.kt
0 → 100644
| 1 | +package com.lemon.lifelog.asr | ||
| 2 | + | ||
| 3 | +import android.util.Log | ||
| 4 | +import com.k2fsa.sherpa.onnx.FastClusteringConfig | ||
| 5 | +import com.k2fsa.sherpa.onnx.OfflineSpeakerDiarization | ||
| 6 | +import com.k2fsa.sherpa.onnx.OfflineSpeakerDiarizationConfig | ||
| 7 | +import com.k2fsa.sherpa.onnx.OfflineSpeakerSegmentationModelConfig | ||
| 8 | +import com.k2fsa.sherpa.onnx.OfflineSpeakerSegmentationPyannoteModelConfig | ||
| 9 | +import com.k2fsa.sherpa.onnx.SpeakerEmbeddingExtractorConfig | ||
| 10 | +import java.io.File | ||
| 11 | + | ||
| 12 | +/** | ||
| 13 | + * 离线说话人 diarization(sherpa-onnx):pyannote 分割(检测说话人变更点 + 重叠语音) | ||
| 14 | + * + CAMPPlus embedding + FastClustering 自动聚类。跑在一整段会话音频上,输出 | ||
| 15 | + * [start,end]→speaker turns,能把「几秒内多人」的区域拆成一人一段。 | ||
| 16 | + * | ||
| 17 | + * 与逐 VAD 段单 embedding 的实时方案不同——这是会话级批处理重跑,准确得多。 | ||
| 18 | + */ | ||
| 19 | +class DiarizationEngine private constructor(private val sd: OfflineSpeakerDiarization) { | ||
| 20 | + | ||
| 21 | + data class Turn(val startSec: Float, val endSec: Float, val speaker: Int) | ||
| 22 | + | ||
| 23 | + /** 16k mono PCM float([-1,1])→ 说话人 turns(按时间排序)。 */ | ||
| 24 | + fun diarize(samples: FloatArray): List<Turn> = | ||
| 25 | + sd.process(samples).map { Turn(it.start, it.end, it.speaker) }.sortedBy { it.startSec } | ||
| 26 | + | ||
| 27 | + fun sampleRate(): Int = sd.sampleRate() | ||
| 28 | + | ||
| 29 | + fun release() = sd.release() | ||
| 30 | + | ||
| 31 | + companion object { | ||
| 32 | + private const val TAG = "DiarizationEngine" | ||
| 33 | + | ||
| 34 | + /** | ||
| 35 | + * @param threshold FastClustering 余弦阈值;越大分越细(同 SpeakerClusterer 取值习惯)。 | ||
| 36 | + */ | ||
| 37 | + fun load( | ||
| 38 | + segModelPath: String, | ||
| 39 | + embModelPath: String, | ||
| 40 | + numThreads: Int = 4, | ||
| 41 | + threshold: Float = 0.5f, | ||
| 42 | + ): DiarizationEngine { | ||
| 43 | + require(File(segModelPath).exists()) { "pyannote seg model 缺失: $segModelPath" } | ||
| 44 | + require(File(embModelPath).exists()) { "CAMPPlus emb model 缺失: $embModelPath" } | ||
| 45 | + val config = OfflineSpeakerDiarizationConfig( | ||
| 46 | + segmentation = OfflineSpeakerSegmentationModelConfig( | ||
| 47 | + pyannote = OfflineSpeakerSegmentationPyannoteModelConfig(model = segModelPath), | ||
| 48 | + numThreads = numThreads, | ||
| 49 | + ), | ||
| 50 | + embedding = SpeakerEmbeddingExtractorConfig(model = embModelPath, numThreads = numThreads), | ||
| 51 | + clustering = FastClusteringConfig(numClusters = -1, threshold = threshold), | ||
| 52 | + minDurationOn = 0.2f, | ||
| 53 | + minDurationOff = 0.5f, | ||
| 54 | + ) | ||
| 55 | + val t0 = System.nanoTime() | ||
| 56 | + val sd = OfflineSpeakerDiarization(config = config) | ||
| 57 | + Log.i(TAG, "loaded in ${(System.nanoTime() - t0) / 1_000_000}ms sr=${sd.sampleRate()} thr=$threshold") | ||
| 58 | + return DiarizationEngine(sd) | ||
| 59 | + } | ||
| 60 | + } | ||
| 61 | +} |
lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/asr/ModelDownloader.kt
| @@ -99,6 +99,31 @@ class ModelDownloader(private val ctx: Context) { | @@ -99,6 +99,31 @@ class ModelDownloader(private val ctx: Context) { | ||
| 99 | speakerPath() | 99 | speakerPath() |
| 100 | } | 100 | } |
| 101 | 101 | ||
| 102 | + fun pyannotePath(): SpeakerModelFile { | ||
| 103 | + val model = File(ctx.filesDir, "models/pyannote-seg.onnx") | ||
| 104 | + return SpeakerModelFile(model.absolutePath, model.exists() && model.length() > 4_000_000L) | ||
| 105 | + } | ||
| 106 | + | ||
| 107 | + /** pyannote-segmentation-3.0(~6MB)—— diarization 的分割模型。 */ | ||
| 108 | + suspend fun ensurePyannoteModel(progress: (String) -> Unit): SpeakerModelFile = | ||
| 109 | + withContext(Dispatchers.IO) { | ||
| 110 | + val state = pyannotePath() | ||
| 111 | + if (state.present) { | ||
| 112 | + progress("pyannote model present, skip"); return@withContext state | ||
| 113 | + } | ||
| 114 | + File(ctx.filesDir, "models").mkdirs() | ||
| 115 | + val task = DownloadTask( | ||
| 116 | + File(state.path), | ||
| 117 | + listOf( | ||
| 118 | + "https://hf-mirror.com/csukuangfj/sherpa-onnx-pyannote-segmentation-3-0/resolve/main/model.onnx", | ||
| 119 | + "https://huggingface.co/csukuangfj/sherpa-onnx-pyannote-segmentation-3-0/resolve/main/model.onnx", | ||
| 120 | + ), | ||
| 121 | + expectedSize = 5_992_913L, | ||
| 122 | + ) | ||
| 123 | + downloadWithResume(http(), task, progress) | ||
| 124 | + pyannotePath() | ||
| 125 | + } | ||
| 126 | + | ||
| 102 | private fun http() = OkHttpClient.Builder() | 127 | private fun http() = OkHttpClient.Builder() |
| 103 | .connectTimeout(15, TimeUnit.SECONDS) | 128 | .connectTimeout(15, TimeUnit.SECONDS) |
| 104 | .readTimeout(120, TimeUnit.SECONDS) | 129 | .readTimeout(120, TimeUnit.SECONDS) |
lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/audio/AudioDecoder.kt
0 → 100644
| 1 | +package com.lemon.lifelog.audio | ||
| 2 | + | ||
| 3 | +import android.media.MediaCodec | ||
| 4 | +import android.media.MediaExtractor | ||
| 5 | +import android.media.MediaFormat | ||
| 6 | +import android.util.Log | ||
| 7 | +import java.io.File | ||
| 8 | +import java.nio.ByteOrder | ||
| 9 | + | ||
| 10 | +/** | ||
| 11 | + * 把 SegmentWriter 写的 .opus(Opus-in-Ogg)解码回 16k mono PCM float([-1,1])。 | ||
| 12 | + * | ||
| 13 | + * 两个坑: | ||
| 14 | + * - MediaCodec 的 Opus 解码常输出 48kHz(Opus 原生率),与 diarization 期望的 16k 不符 → 重采样。 | ||
| 15 | + * - 编码是 mono,但保险起见处理 stereo → 下混。 | ||
| 16 | + */ | ||
| 17 | +object AudioDecoder { | ||
| 18 | + private const val TAG = "AudioDecoder" | ||
| 19 | + private const val TARGET_SR = 16000 | ||
| 20 | + | ||
| 21 | + fun decodeToFloat(path: String): FloatArray { | ||
| 22 | + if (!File(path).exists()) return FloatArray(0) | ||
| 23 | + val extractor = MediaExtractor() | ||
| 24 | + return try { | ||
| 25 | + extractor.setDataSource(path) | ||
| 26 | + var track = -1 | ||
| 27 | + var inFormat: MediaFormat? = null | ||
| 28 | + for (i in 0 until extractor.trackCount) { | ||
| 29 | + val f = extractor.getTrackFormat(i) | ||
| 30 | + if (f.getString(MediaFormat.KEY_MIME)?.startsWith("audio/") == true) { | ||
| 31 | + track = i; inFormat = f; break | ||
| 32 | + } | ||
| 33 | + } | ||
| 34 | + if (track < 0 || inFormat == null) return FloatArray(0) | ||
| 35 | + extractor.selectTrack(track) | ||
| 36 | + | ||
| 37 | + val codec = MediaCodec.createDecoderByType(inFormat.getString(MediaFormat.KEY_MIME)!!) | ||
| 38 | + codec.configure(inFormat, null, null, 0) | ||
| 39 | + codec.start() | ||
| 40 | + | ||
| 41 | + var srcRate = inFormat.getIntOr(MediaFormat.KEY_SAMPLE_RATE, TARGET_SR) | ||
| 42 | + var channels = inFormat.getIntOr(MediaFormat.KEY_CHANNEL_COUNT, 1) | ||
| 43 | + | ||
| 44 | + val shorts = ArrayList<Short>(TARGET_SR * 30) | ||
| 45 | + val info = MediaCodec.BufferInfo() | ||
| 46 | + var inEos = false | ||
| 47 | + var outEos = false | ||
| 48 | + val timeout = 10_000L | ||
| 49 | + | ||
| 50 | + while (!outEos) { | ||
| 51 | + if (!inEos) { | ||
| 52 | + val ii = codec.dequeueInputBuffer(timeout) | ||
| 53 | + if (ii >= 0) { | ||
| 54 | + val buf = codec.getInputBuffer(ii)!! | ||
| 55 | + val n = extractor.readSampleData(buf, 0) | ||
| 56 | + if (n < 0) { | ||
| 57 | + codec.queueInputBuffer(ii, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM) | ||
| 58 | + inEos = true | ||
| 59 | + } else { | ||
| 60 | + codec.queueInputBuffer(ii, 0, n, extractor.sampleTime, 0) | ||
| 61 | + extractor.advance() | ||
| 62 | + } | ||
| 63 | + } | ||
| 64 | + } | ||
| 65 | + val oi = codec.dequeueOutputBuffer(info, timeout) | ||
| 66 | + when { | ||
| 67 | + oi == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED -> { | ||
| 68 | + val of = codec.outputFormat | ||
| 69 | + srcRate = of.getIntOr(MediaFormat.KEY_SAMPLE_RATE, srcRate) | ||
| 70 | + channels = of.getIntOr(MediaFormat.KEY_CHANNEL_COUNT, channels) | ||
| 71 | + } | ||
| 72 | + oi >= 0 -> { | ||
| 73 | + if (info.size > 0) { | ||
| 74 | + val ob = codec.getOutputBuffer(oi)!! | ||
| 75 | + ob.position(info.offset); ob.limit(info.offset + info.size) | ||
| 76 | + val sb = ob.order(ByteOrder.LITTLE_ENDIAN).asShortBuffer() | ||
| 77 | + while (sb.hasRemaining()) shorts.add(sb.get()) | ||
| 78 | + } | ||
| 79 | + codec.releaseOutputBuffer(oi, false) | ||
| 80 | + if (info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0) outEos = true | ||
| 81 | + } | ||
| 82 | + } | ||
| 83 | + } | ||
| 84 | + codec.stop(); codec.release() | ||
| 85 | + | ||
| 86 | + // 下混 stereo → mono | ||
| 87 | + var mono = if (channels >= 2) { | ||
| 88 | + FloatArray(shorts.size / channels) { i -> | ||
| 89 | + var s = 0f | ||
| 90 | + for (c in 0 until channels) s += shorts[i * channels + c] / 32768f | ||
| 91 | + s / channels | ||
| 92 | + } | ||
| 93 | + } else { | ||
| 94 | + FloatArray(shorts.size) { shorts[it] / 32768f } | ||
| 95 | + } | ||
| 96 | + if (srcRate != TARGET_SR) mono = resample(mono, srcRate, TARGET_SR) | ||
| 97 | + mono | ||
| 98 | + } catch (t: Throwable) { | ||
| 99 | + Log.w(TAG, "decode failed $path: ${t.message}") | ||
| 100 | + FloatArray(0) | ||
| 101 | + } finally { | ||
| 102 | + extractor.release() | ||
| 103 | + } | ||
| 104 | + } | ||
| 105 | + | ||
| 106 | + private fun MediaFormat.getIntOr(key: String, def: Int): Int = | ||
| 107 | + if (containsKey(key)) getInteger(key) else def | ||
| 108 | + | ||
| 109 | + /** 线性重采样到 dstRate。 */ | ||
| 110 | + private fun resample(input: FloatArray, srcRate: Int, dstRate: Int): FloatArray { | ||
| 111 | + if (srcRate == dstRate || input.isEmpty()) return input | ||
| 112 | + val ratio = dstRate.toDouble() / srcRate | ||
| 113 | + val outLen = (input.size * ratio).toInt() | ||
| 114 | + val out = FloatArray(outLen) | ||
| 115 | + val last = input.size - 1 | ||
| 116 | + for (i in out.indices) { | ||
| 117 | + val srcPos = i / ratio | ||
| 118 | + val i0 = srcPos.toInt() | ||
| 119 | + val frac = (srcPos - i0).toFloat() | ||
| 120 | + val a = input[i0.coerceIn(0, last)] | ||
| 121 | + val b = input[(i0 + 1).coerceIn(0, last)] | ||
| 122 | + out[i] = a + (b - a) * frac | ||
| 123 | + } | ||
| 124 | + return out | ||
| 125 | + } | ||
| 126 | +} |
lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/diarize/DiarizationScheduler.kt
0 → 100644
| 1 | +package com.lemon.lifelog.diarize | ||
| 2 | + | ||
| 3 | +import android.content.Context | ||
| 4 | +import androidx.work.Constraints | ||
| 5 | +import androidx.work.ExistingPeriodicWorkPolicy | ||
| 6 | +import androidx.work.ExistingWorkPolicy | ||
| 7 | +import androidx.work.NetworkType | ||
| 8 | +import androidx.work.OneTimeWorkRequestBuilder | ||
| 9 | +import androidx.work.PeriodicWorkRequestBuilder | ||
| 10 | +import androidx.work.WorkManager | ||
| 11 | +import java.util.concurrent.TimeUnit | ||
| 12 | + | ||
| 13 | +object DiarizationScheduler { | ||
| 14 | + private const val PERIODIC_TAG = "lifelog-diarize-periodic" | ||
| 15 | + private const val ONE_SHOT_TAG = "lifelog-diarize-once" | ||
| 16 | + | ||
| 17 | + fun schedulePeriodic(ctx: Context) { | ||
| 18 | + val constraints = Constraints.Builder() | ||
| 19 | + .setRequiredNetworkType(NetworkType.NOT_REQUIRED) | ||
| 20 | + .setRequiresBatteryNotLow(true) | ||
| 21 | + .build() | ||
| 22 | + val request = PeriodicWorkRequestBuilder<DiarizationWorker>(15, TimeUnit.MINUTES) | ||
| 23 | + .setConstraints(constraints) | ||
| 24 | + .build() | ||
| 25 | + WorkManager.getInstance(ctx).enqueueUniquePeriodicWork( | ||
| 26 | + PERIODIC_TAG, | ||
| 27 | + ExistingPeriodicWorkPolicy.UPDATE, | ||
| 28 | + request, | ||
| 29 | + ) | ||
| 30 | + } | ||
| 31 | + | ||
| 32 | + /** 手动跑一次(KEEP:已在跑/排队则忽略)。 */ | ||
| 33 | + fun runNow(ctx: Context) { | ||
| 34 | + val request = OneTimeWorkRequestBuilder<DiarizationWorker>().addTag(ONE_SHOT_TAG).build() | ||
| 35 | + WorkManager.getInstance(ctx).enqueueUniqueWork(ONE_SHOT_TAG, ExistingWorkPolicy.KEEP, request) | ||
| 36 | + } | ||
| 37 | +} |
lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/diarize/DiarizationWorker.kt
0 → 100644
| 1 | +package com.lemon.lifelog.diarize | ||
| 2 | + | ||
| 3 | +import android.content.Context | ||
| 4 | +import android.util.Log | ||
| 5 | +import androidx.work.CoroutineWorker | ||
| 6 | +import androidx.work.WorkerParameters | ||
| 7 | +import com.lemon.lifelog.asr.DiarizationEngine | ||
| 8 | +import com.lemon.lifelog.asr.ModelDownloader | ||
| 9 | +import com.lemon.lifelog.asr.SpeakerClusterer | ||
| 10 | +import com.lemon.lifelog.asr.SpeakerEmbedder | ||
| 11 | +import com.lemon.lifelog.audio.AudioDecoder | ||
| 12 | +import com.lemon.lifelog.data.Repository | ||
| 13 | +import com.lemon.lifelog.data.Segment | ||
| 14 | +import com.lemon.lifelog.summary.SummaryWorker | ||
| 15 | +import java.io.File | ||
| 16 | +import java.util.concurrent.atomic.AtomicBoolean | ||
| 17 | +import kotlinx.coroutines.Dispatchers | ||
| 18 | +import kotlinx.coroutines.withContext | ||
| 19 | + | ||
| 20 | +/** | ||
| 21 | + * 会话级 diarization 重跑(方案 A)。对每个**已结束、未 diarize**的会话: | ||
| 22 | + * 1. 解码 + 按序拼接该会话所有段音频(speech-only 16k mono)。 | ||
| 23 | + * 2. pyannote 分割 + CAMPPlus + FastClustering → 准确的 [start,end]→局部说话人 turns | ||
| 24 | + * (能把「几秒多人」的段拆开)。 | ||
| 25 | + * 3. 每个局部说话人 → 取其最长 turn 的 embedding → 经 SpeakerClusterer 桥接到 | ||
| 26 | + * 全局 SpeakerProfile(跨会话/跨天「说话人N」一致)。 | ||
| 27 | + * 4. 每个 VAD 段 → 找重叠最大的 turn → 重标 speakerId(段内多说话人先归主导者,v1)。 | ||
| 28 | + * | ||
| 29 | + * 与逐段实时聚类相比准确得多;实时标签只作即时占位,这里批处理纠正。 | ||
| 30 | + */ | ||
| 31 | +class DiarizationWorker(ctx: Context, params: WorkerParameters) : CoroutineWorker(ctx, params) { | ||
| 32 | + | ||
| 33 | + companion object { | ||
| 34 | + const val TAG = "DiarizationWorker" | ||
| 35 | + private val isRunning = AtomicBoolean(false) | ||
| 36 | + const val RUN_BUDGET_MS = 7 * 60 * 1000L | ||
| 37 | + const val MIN_SESSION_SEGMENTS = 2 // 单段会话无需 diarize | ||
| 38 | + const val MAX_CONCAT_SAMPLES = 18 * 60 * 16000 // ≤18min speech,防内存/超时 | ||
| 39 | + private const val KEY_DONE = "diarized_sessions" // prefs StringSet of windowStartTs | ||
| 40 | + } | ||
| 41 | + | ||
| 42 | + private val sr = 16000 | ||
| 43 | + | ||
| 44 | + override suspend fun doWork(): Result { | ||
| 45 | + if (!isRunning.compareAndSet(false, true)) { | ||
| 46 | + Log.i(TAG, "another diarization in progress, skip"); return Result.success() | ||
| 47 | + } | ||
| 48 | + return try { | ||
| 49 | + run() | ||
| 50 | + } catch (t: Throwable) { | ||
| 51 | + Log.w(TAG, "diarization run failed", t); Result.success() | ||
| 52 | + } finally { | ||
| 53 | + isRunning.set(false) | ||
| 54 | + } | ||
| 55 | + } | ||
| 56 | + | ||
| 57 | + private suspend fun run(): Result { | ||
| 58 | + val runStart = System.currentTimeMillis() | ||
| 59 | + val now = System.currentTimeMillis() | ||
| 60 | + val prefs = applicationContext.getSharedPreferences(SummaryWorker.PREFS, Context.MODE_PRIVATE) | ||
| 61 | + val gapMs = prefs.getInt(SummaryWorker.KEY_GAP_MIN, SummaryWorker.DEFAULT_GAP_MIN) * 60_000L | ||
| 62 | + val threshold = SpeakerClusterer.thresholdFromPrefs(applicationContext) | ||
| 63 | + | ||
| 64 | + val repo = Repository(applicationContext) | ||
| 65 | + val segments = repo.segments.page(limit = 5000, offset = 0).sortedBy { it.startTs } | ||
| 66 | + if (segments.size < MIN_SESSION_SEGMENTS) return Result.success() | ||
| 67 | + | ||
| 68 | + val sessions = SummaryWorker.groupSessions(segments, gapMs) | ||
| 69 | + val done = prefs.getStringSet(KEY_DONE, emptySet())!!.toMutableSet() | ||
| 70 | + | ||
| 71 | + // 模型路径 | ||
| 72 | + val downloader = ModelDownloader(applicationContext) | ||
| 73 | + val speakerFile = downloader.ensureSpeakerModel { } | ||
| 74 | + if (!speakerFile.present) { Log.w(TAG, "speaker model missing"); return Result.success() } | ||
| 75 | + val pyannote = downloader.ensurePyannoteModel { Log.i(TAG, "pyannote: $it") } | ||
| 76 | + if (!pyannote.present) { Log.w(TAG, "pyannote seg model missing/download failed"); return Result.success() } | ||
| 77 | + | ||
| 78 | + var engine: DiarizationEngine? = null | ||
| 79 | + var embedder: SpeakerEmbedder? = null | ||
| 80 | + var processed = 0 | ||
| 81 | + try { | ||
| 82 | + for (session in sessions) { | ||
| 83 | + val start = session.first().startTs | ||
| 84 | + val end = session.last().endTs | ||
| 85 | + val key = start.toString() | ||
| 86 | + | ||
| 87 | + if (now - end <= gapMs) continue // 未结束 | ||
| 88 | + if (session.size < MIN_SESSION_SEGMENTS) continue | ||
| 89 | + if (done.contains(key)) continue // 已 diarize | ||
| 90 | + if (System.currentTimeMillis() - runStart > RUN_BUDGET_MS) { | ||
| 91 | + Log.i(TAG, "time budget hit, rest next round"); break | ||
| 92 | + } | ||
| 93 | + | ||
| 94 | + // 懒加载模型(有活才加载) | ||
| 95 | + if (engine == null) { | ||
| 96 | + engine = DiarizationEngine.load(pyannote.path, speakerFile.path, numThreads = 4, threshold = threshold) | ||
| 97 | + embedder = SpeakerEmbedder.load(speakerFile.path, numThreads = 2) | ||
| 98 | + } | ||
| 99 | + | ||
| 100 | + val ok = diarizeSession(repo, session, engine!!, embedder!!, threshold) | ||
| 101 | + if (ok) { done.add(key); processed++ } | ||
| 102 | + } | ||
| 103 | + } finally { | ||
| 104 | + engine?.release() | ||
| 105 | + embedder?.release() | ||
| 106 | + } | ||
| 107 | + | ||
| 108 | + prefs.edit().putStringSet(KEY_DONE, done).apply() | ||
| 109 | + Log.i(TAG, "done: ${sessions.size} sessions, $processed diarized") | ||
| 110 | + return Result.success() | ||
| 111 | + } | ||
| 112 | + | ||
| 113 | + /** 对单个会话做 diarization + 段重标。返回是否成功处理。 */ | ||
| 114 | + private suspend fun diarizeSession( | ||
| 115 | + repo: Repository, | ||
| 116 | + session: List<Segment>, | ||
| 117 | + engine: DiarizationEngine, | ||
| 118 | + embedder: SpeakerEmbedder, | ||
| 119 | + threshold: Float, | ||
| 120 | + ): Boolean = withContext(Dispatchers.Default) { | ||
| 121 | + // 1. 解码 + 拼接,记录每段在拼接波形里的 [offset, offset+len](采样) | ||
| 122 | + data class SegSpan(val seg: Segment, val from: Int, val to: Int) | ||
| 123 | + val spans = ArrayList<SegSpan>(session.size) | ||
| 124 | + val audio = ArrayList<Float>(sr * 60) | ||
| 125 | + for (seg in session) { | ||
| 126 | + val pcm = AudioDecoder.decodeToFloat(seg.audioPath) | ||
| 127 | + if (pcm.isEmpty()) continue | ||
| 128 | + val from = audio.size | ||
| 129 | + for (v in pcm) audio.add(v) | ||
| 130 | + spans.add(SegSpan(seg, from, audio.size)) | ||
| 131 | + if (audio.size >= MAX_CONCAT_SAMPLES) break | ||
| 132 | + } | ||
| 133 | + if (spans.size < MIN_SESSION_SEGMENTS || audio.isEmpty()) return@withContext false | ||
| 134 | + val samples = FloatArray(audio.size) { audio[it] } | ||
| 135 | + | ||
| 136 | + // 2. diarize | ||
| 137 | + val turns = engine.diarize(samples) | ||
| 138 | + if (turns.isEmpty()) return@withContext false | ||
| 139 | + | ||
| 140 | + // 3. 每个局部说话人 → 最长 turn 的 embedding → 桥接全局 | ||
| 141 | + val byLocal = turns.groupBy { it.speaker } | ||
| 142 | + val clusterer = SpeakerClusterer(repo, matchThreshold = threshold).also { it.load() } | ||
| 143 | + val localToGlobal = HashMap<Int, Pair<Long, Float>>() // local → (globalSpeakerId, sim) | ||
| 144 | + for ((local, ts) in byLocal) { | ||
| 145 | + // 局部说话人总时长 <2s 视为噪声,不桥接(避免一堆短 turn 各建一个全局说话人)。 | ||
| 146 | + val totalSec = ts.sumOf { (it.endSec - it.startSec).toDouble() } | ||
| 147 | + if (totalSec < 2.0) continue | ||
| 148 | + val longest = ts.maxByOrNull { it.endSec - it.startSec } ?: continue | ||
| 149 | + val a = (longest.startSec * sr).toInt().coerceIn(0, samples.size) | ||
| 150 | + val b = (longest.endSec * sr).toInt().coerceIn(a, samples.size) | ||
| 151 | + if (b - a < sr / 2) continue // <0.5s 太短,跳过该局部说话人 | ||
| 152 | + val emb = embedder.embed(samples.copyOfRange(a, b), sr) ?: continue | ||
| 153 | + val durMs = ((longest.endSec - longest.startSec) * 1000).toLong() | ||
| 154 | + val asg = clusterer.assign(emb, durMs) | ||
| 155 | + if (asg.speakerId != null) localToGlobal[local] = asg.speakerId to (asg.sim ?: 1f) | ||
| 156 | + } | ||
| 157 | + if (localToGlobal.isEmpty()) return@withContext false | ||
| 158 | + | ||
| 159 | + // 4. 每段 → 重叠最大的 turn → 全局 speakerId | ||
| 160 | + for (span in spans) { | ||
| 161 | + val s0 = span.from.toFloat() / sr | ||
| 162 | + val s1 = span.to.toFloat() / sr | ||
| 163 | + var bestTurn: DiarizationEngine.Turn? = null | ||
| 164 | + var bestOv = 0f | ||
| 165 | + for (t in turns) { | ||
| 166 | + val ov = minOf(s1, t.endSec) - maxOf(s0, t.startSec) | ||
| 167 | + if (ov > bestOv) { bestOv = ov; bestTurn = t } | ||
| 168 | + } | ||
| 169 | + val g = bestTurn?.let { localToGlobal[it.speaker] } ?: continue | ||
| 170 | + withContext(Dispatchers.IO) { repo.segments.updateSpeaker(span.seg.id, g.first, g.second) } | ||
| 171 | + } | ||
| 172 | + Log.i(TAG, "session ${session.first().startTs}: ${spans.size} segs, ${turns.size} turns, ${byLocal.size} local speakers") | ||
| 173 | + true | ||
| 174 | + } | ||
| 175 | +} |