From 9beb325a9c788a525b1aa44cc61eca26d06c858f Mon Sep 17 00:00:00 2001 From: yibo Date: Mon, 8 Jun 2026 19:17:05 +0800 Subject: [PATCH] feat(lifelog/diarize): 凝聚式合并收尾，治声纹过度切分（同人被拆成多簇） --- lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/MainActivity.kt | 2 ++ lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/SettingsScreen.kt | 7 ++++++- lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/asr/SpeakerClusterer.kt | 87 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 92 insertions(+), 4 deletions(-) diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/MainActivity.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/MainActivity.kt index 316c663..14872ff 100644 --- a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/MainActivity.kt +++ b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/MainActivity.kt @@ -864,6 +864,7 @@ private fun SpeakersTab( repo, matchThreshold = SpeakerClusterer.thresholdFromPrefs(ctx), minRecognizeMs = SpeakerClusterer.minRecognizeMsFromPrefs(ctx), + mergeThreshold = SpeakerClusterer.mergeThresholdFromPrefs(ctx), ) } reclustering = false @@ -975,6 +976,7 @@ private fun SpeakersTab( repo, matchThreshold = SpeakerClusterer.thresholdFromPrefs(ctx), minRecognizeMs = SpeakerClusterer.minRecognizeMsFromPrefs(ctx), + mergeThreshold = SpeakerClusterer.mergeThresholdFromPrefs(ctx), ) } reclustering = false diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/SettingsScreen.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/SettingsScreen.kt index 14e2b79..fd5c839 100644 --- a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/SettingsScreen.kt +++ b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/SettingsScreen.kt @@ -51,6 +51,7 @@ fun SettingsScreen(activity: Context) { var minSpeech by remember { mutableStateOf(prefs.getInt(SummaryWorker.KEY_MIN_SPEECH_SEC, SummaryWorker.DEFAULT_MIN_SPEECH_SEC)) } var backendId by remember { mutableStateOf(prefs.getString(SummaryWorker.KEY_BACKEND, ChatConfig.ID_2B) ?: ChatConfig.ID_2B) } var threshold by remember { mutableStateOf(SpeakerClusterer.thresholdFromPrefs(activity)) } + var mergeThreshold by remember { mutableStateOf(SpeakerClusterer.mergeThresholdFromPrefs(activity)) } var minRecognizeSec by remember { mutableStateOf(prefs.getInt(SpeakerClusterer.KEY_MIN_RECOGNIZE_SEC, (SpeakerClusterer.MIN_RECOGNIZE_MS / 1000).toInt())) } @@ -98,11 +99,15 @@ fun SettingsScreen(activity: Context) { listOf(0.35f, 0.40f, 0.45f, 0.50f, 0.55f, 0.60f, 0.65f).map { "%.2f".format(it) to it }) { threshold = it; prefs.edit().putFloat(SpeakerClusterer.KEY_THRESHOLD, it).apply() } + DropdownSetting("合并阈值（越小越爱并同人）", "%.2f".format(mergeThreshold), + listOf(0.45f, 0.50f, 0.55f, 0.60f, 0.65f).map { "%.2f".format(it) to it }) { + mergeThreshold = it; prefs.edit().putFloat(SpeakerClusterer.KEY_MERGE_THRESHOLD, it).apply() + } DropdownSetting("最短识别时长（短于此不认声纹）", "$minRecognizeSec 秒", listOf(1, 2, 3, 5, 8).map { "$it 秒" to it }) { minRecognizeSec = it; prefs.edit().putInt(SpeakerClusterer.KEY_MIN_RECOGNIZE_SEC, it).apply() } - Text("改完到「声纹」tab 点「重新聚类」对历史段生效；新录音自动用新值。", + Text("重聚类会先贪心分簇、再把质心很近的簇按「合并阈值」并回（治同人被拆成多个）。改完到「声纹」tab 点「重新聚类」生效。", style = MaterialTheme.typography.labelSmall) HorizontalDivider() diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/asr/SpeakerClusterer.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/asr/SpeakerClusterer.kt index adc3ed8..82dcd8c 100644 --- a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/asr/SpeakerClusterer.kt +++ b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/asr/SpeakerClusterer.kt @@ -114,11 +114,21 @@ class SpeakerClusterer( const val PREFS = "lifelog_prefs" const val KEY_THRESHOLD = "speaker_threshold" const val KEY_MIN_RECOGNIZE_SEC = "speaker_min_recognize_sec" + const val KEY_MERGE_THRESHOLD = "speaker_merge_threshold" fun thresholdFromPrefs(ctx: android.content.Context): Float = ctx.getSharedPreferences(PREFS, android.content.Context.MODE_PRIVATE) .getFloat(KEY_THRESHOLD, MATCH_THRESHOLD) + /** 合并阈值（设置页可调）：重聚类后，质心余弦 ≥ 此值的两簇判为同一人并合并。 */ + fun mergeThresholdFromPrefs(ctx: android.content.Context): Float = + ctx.getSharedPreferences(PREFS, android.content.Context.MODE_PRIVATE) + .getFloat(KEY_MERGE_THRESHOLD, MERGE_THRESHOLD) + + /** 合并阈值默认值：贪心聚类易把同一人切成多簇（早期噪声段另起簇后互相吸段）， + * 收尾做凝聚式合并把质心很近的簇并回来。0.55 偏保守，只并高置信对；调低并得更狠。 */ + const val MERGE_THRESHOLD = 0.55f + /** 最短识别时长（设置页可调，秒）：短于此的段完全跳过声纹识别。 */ fun minRecognizeMsFromPrefs(ctx: android.content.Context): Long = ctx.getSharedPreferences(PREFS, android.content.Context.MODE_PRIVATE) @@ -170,6 +180,7 @@ class SpeakerClusterer( matchThreshold: Float = MATCH_THRESHOLD, minNewClusterMs: Long = MIN_NEW_CLUSTER_MS, minRecognizeMs: Long = MIN_RECOGNIZE_MS, + mergeThreshold: Float = MERGE_THRESHOLD, maxSamplesPerCentroid: Int = MAX_SAMPLES_PER_CENTROID, ): Int = withContext(Dispatchers.IO) { // 1. 删掉自动簇，保留人工命名簇作种子。 @@ -234,8 +245,77 @@ class SpeakerClusterer( repo.segments.updateSpeaker(seg.id, assignment.first, assignment.second) } - Log.i(TAG, "reclusterAll: ${segs.size} segments → ${clusters.size} clusters") - clusters.size + Log.i(TAG, "reclusterAll greedy: ${segs.size} segments → ${clusters.size} clusters") + // 3. 凝聚式合并：把过度切分的同人簇并回来。 + val merges = mergeOverSplit(repo, mergeThreshold) + val finalCount = repo.speakers.all().size + Log.i(TAG, "reclusterAll: merged $merges pairs → $finalCount clusters") + finalCount + } + + /** + * 凝聚式合并（agglomerative）：解决贪心聚类的过度切分——同一人因早期噪声段另起簇后， + * 后续段按「最近质心」各自归簇，再不会并回。这里反复找质心余弦最大的一对簇， + * ≥ [mergeThreshold] 就合并（质心用两簇**全部成员段**重算），直到没有可并的对。 + * + * 约束：不合并两个**人工命名**簇（会毁掉用户标定）；一人工一自动则并入人工（保留名字）； + * 都自动则并入成员多的那个。合并后刷新被并段的 speakerId 与相似度。 + * @return 合并发生的次数 + */ + suspend fun mergeOverSplit( + repo: Repository, + mergeThreshold: Float = MERGE_THRESHOLD, + ): Int = withContext(Dispatchers.IO) { + class WC(val id: Long, val manual: Boolean, val members: MutableList, var centroid: FloatArray) + fun meanNorm(ms: List): FloatArray { + val dim = ms[0].size + val m = FloatArray(dim) + for (e in ms) for (i in 0 until dim) m[i] += e[i] + for (i in 0 until dim) m[i] /= ms.size + l2normalize(m); return m + } + val wcs = ArrayList() + for (p in repo.speakers.all()) { + val mem = repo.segments.embeddingsForSpeaker(p.id).map { it.toFloatArrayLe() } + if (mem.isEmpty()) continue + wcs += WC(p.id, autoIndexOf(p.name) == null, mem.toMutableList(), meanNorm(mem)) + } + var merges = 0 + while (true) { + var bi = -1; var bj = -1; var best = mergeThreshold + for (i in wcs.indices) for (j in i + 1 until wcs.size) { + if (wcs[i].manual && wcs[j].manual) continue // 两个人工名不并 + val c = cosineSim(wcs[i].centroid, wcs[j].centroid) + if (c >= best) { best = c; bi = i; bj = j } + } + if (bi < 0) break + val a = wcs[bi]; val b = wcs[bj] + val keep: WC; val drop: WC + when { + a.manual && !b.manual -> { keep = a; drop = b } + b.manual && !a.manual -> { keep = b; drop = a } + a.members.size >= b.members.size -> { keep = a; drop = b } + else -> { keep = b; drop = a } + } + keep.members.addAll(drop.members) + keep.centroid = meanNorm(keep.members) + repo.segments.reassignSpeaker(drop.id, keep.id) + repo.speakers.delete(drop.id) + wcs.remove(drop) + merges++ + } + // 持久化保留簇的质心 + 刷新各段相似度（合并后质心变了）。 + for (w in wcs) { + repo.speakers.updateEmbedding(w.id, w.centroid.toLeBytes(), minOf(w.members.size, MAX_SAMPLES_PER_CENTROID)) + } + val byId = wcs.associateBy { it.id } + for (seg in repo.segments.allWithEmbedding()) { + val w = byId[seg.speakerId] ?: continue + val sim = cosineSim(seg.embedding!!.toFloatArrayLe(), w.centroid) + repo.segments.updateSpeaker(seg.id, w.id, sim) + } + Log.i(TAG, "mergeOverSplit: $merges merges, ${wcs.size} clusters remain (threshold=$mergeThreshold)") + merges } /** @@ -248,11 +328,12 @@ class SpeakerClusterer( matchThreshold: Float = MATCH_THRESHOLD, minNewClusterMs: Long = MIN_NEW_CLUSTER_MS, minRecognizeMs: Long = MIN_RECOGNIZE_MS, + mergeThreshold: Float = MERGE_THRESHOLD, ): Int = withContext(Dispatchers.IO) { repo.segments.clearAllSpeakerAssignments() for (p in repo.speakers.all()) repo.speakers.delete(p.id) Log.i(TAG, "clearAllAndRecluster: wiped all profiles + assignments") - reclusterAll(repo, matchThreshold, minNewClusterMs, minRecognizeMs) + reclusterAll(repo, matchThreshold, minNewClusterMs, minRecognizeMs, mergeThreshold) } } } -- libgit2 0.21.4