diff --git a/lifelog-recorder/HANDOFF.md b/lifelog-recorder/HANDOFF.md index 6f6e913..5530a59 100644 --- a/lifelog-recorder/HANDOFF.md +++ b/lifelog-recorder/HANDOFF.md @@ -158,14 +158,46 @@ VAD 参数（在 `RecorderService.runPipeline` 里调）： - v0.2 (2026-06)：Opus 编码 / BootReceiver / 电池白名单 - v0.3 (2026-06)：声纹识别（3dspeaker campplus）/ FTS4 中文搜索 - v0.4 (2026-06)：段播放 (MediaPlayer Opus) / bottom sheet 操作菜单 / enroll 后自动回填老段 +- v0.5 (2026-06)：hermes api_server 端点 + SummaryWorker + 纪要 UI section 剩下： -1. **LLM 纪要 worker**：WorkManager cron。每小时 / 静音 30min 触发，按时间窗 group segments → POST 到 hermes gateway `http://127.0.0.1:8642/v1/chat/completions`（需要先把 hermes api_server platform 配置起来，见 `~/.hermes/config.yaml`）。 -2. **存储 TTL**：删超过 N 天的 .opus，文字永留。Periodic WorkManager job。 -3. **设置页**：存储用量、模型切换、provider 切换、暂停时段配置（v1 定的 MVP 是「一键暂停就够」，但 v2 可能要细化）。 -4. **多 anchor 平均**：现在 enroll 是单段单 anchor，可以「连选多段都标 我」自动算平均 embedding 提高匹配率。 -5. **回链 lemonPCRM**：Transcript 提到的人名 / 项目 → GraphStore `MENTIONED_IN` 边。 +1. **存储 TTL**：删超过 N 天的 .opus，文字永留。Periodic WorkManager job。 +2. **设置页**：存储用量、模型切换、provider 切换、暂停时段配置（v1 定的 MVP 是「一键暂停就够」，但 v2 可能要细化）。 +3. **多 anchor 平均**：现在 enroll 是单段单 anchor，可以「连选多段都标 我」自动算平均 embedding 提高匹配率。 +4. **回链 lemonPCRM**：Transcript 提到的人名 / 项目 → GraphStore `MENTIONED_IN` 边。 +5. **hermes gateway 自启**：当前需用户手动跑 `bash ~/start_gw.sh`，可写到 Termux:Boot 启动脚本。 + +### hermes gateway 设置（v0.5 关键前置） + +```bash +# 一次性：~/.hermes/.env 末尾追加 +API_SERVER_ENABLED=true +API_SERVER_HOST=127.0.0.1 +API_SERVER_PORT=8642 +API_SERVER_KEY= +API_SERVER_MODEL_NAME=kimi-for-coding + +# 需要 anthropic SDK（kimi-for-coding 协议层走 Anthropic-compat） +~/hermes-agent/venv/bin/pip install -i https://pypi.tuna.tsinghua.edu.cn/simple/ 'anthropic>=0.39.0' + +# 启动：必须用 setsid + disown 才能在 adb shell 之外存活 +cat > ~/start_gw.sh <<'EOF' +#!/data/data/com.termux/files/usr/bin/bash +export PATH=/data/data/com.termux/files/usr/bin:$PATH +export HOME=/data/data/com.termux/files/home +pkill -f 'hermes.*gateway' 2>/dev/null; sleep 2 +set -a; . $HOME/.hermes/.env; set +a +: > $HOME/.hermes/logs/gateway.log +setsid hermes gateway run $HOME/.hermes/logs/gateway.log 2>&1 & +disown $! +EOF +chmod 755 ~/start_gw.sh +bash ~/start_gw.sh +``` + +lifelog 的 `keys.properties` 里 `hermes.apiUrl` / `hermes.apiKey` 要跟这里 +对得上才能调通。`network_security_config.xml` 已开 127.0.0.1 cleartext 豁免。 ### 存储预算（v0.2 后） diff --git a/lifelog-recorder/app/build.gradle.kts b/lifelog-recorder/app/build.gradle.kts index f98defb..ce78a34 100644 --- a/lifelog-recorder/app/build.gradle.kts +++ b/lifelog-recorder/app/build.gradle.kts @@ -35,6 +35,9 @@ android { buildConfigField("String", "XFYUN_API_SECRET", "\"${key("xfyun.apiSecret")}\"") buildConfigField("String", "XFYUN_IAT_HOST", "\"${key("xfyun.iatHost").ifEmpty { "iat.xf-yun.com" }}\"") buildConfigField("String", "XFYUN_IAT_PATH", "\"${key("xfyun.iatPath").ifEmpty { "/v1" }}\"") + buildConfigField("String", "HERMES_API_URL", "\"${key("hermes.apiUrl").ifEmpty { "http://127.0.0.1:8642" }}\"") + buildConfigField("String", "HERMES_API_KEY", "\"${key("hermes.apiKey")}\"") + buildConfigField("String", "HERMES_MODEL", "\"${key("hermes.model").ifEmpty { "kimi-for-coding" }}\"") } packaging { diff --git a/lifelog-recorder/app/src/main/AndroidManifest.xml b/lifelog-recorder/app/src/main/AndroidManifest.xml index 7c1dcc3..e8c1a7c 100644 --- a/lifelog-recorder/app/src/main/AndroidManifest.xml +++ b/lifelog-recorder/app/src/main/AndroidManifest.xml @@ -17,6 +17,7 @@ android:label="Lifelog" android:allowBackup="false" android:supportsRtl="true" + android:networkSecurityConfig="@xml/network_security_config" android:theme="@style/Theme.Lifelog" tools:targetApi="35"> diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/LifelogApp.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/LifelogApp.kt index 1439d25..399ecde 100644 --- a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/LifelogApp.kt +++ b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/LifelogApp.kt @@ -4,6 +4,7 @@ import android.app.Application import android.app.NotificationChannel import android.app.NotificationManager import com.lemon.lifelog.service.RecorderService +import com.lemon.lifelog.summary.SummaryScheduler class LifelogApp : Application() { override fun onCreate() { @@ -16,5 +17,7 @@ class LifelogApp : Application() { NotificationManager.IMPORTANCE_LOW, ) ) + // Hourly 纪要 worker (KEEP = 已排过就不动)。 + SummaryScheduler.schedulePeriodic(this) } } diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/MainActivity.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/MainActivity.kt index 3977f31..0890f8b 100644 --- a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/MainActivity.kt +++ b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/MainActivity.kt @@ -63,6 +63,8 @@ import com.lemon.lifelog.asr.toFloatArrayLe import com.lemon.lifelog.asr.toLeBytes import com.lemon.lifelog.asr.transcribeWithStats import com.lemon.lifelog.audio.SegmentPlayer +import com.lemon.lifelog.data.Summary +import com.lemon.lifelog.summary.SummaryScheduler import com.lemon.lifelog.data.Repository import com.lemon.lifelog.data.Segment import com.lemon.lifelog.data.SegmentRow @@ -116,6 +118,7 @@ private fun MainScreen() { } val timeline by timelineFlow.collectAsState(initial = emptyList()) val speakers by repo.speakers.observeAll().collectAsState(initial = emptyList()) + val summaries by repo.summaries.recent(20).collectAsState(initial = emptyList()) // Sheet / dialog state. var actionRow by remember { mutableStateOf(null) } @@ -229,6 +232,27 @@ private fun MainScreen() { HorizontalDivider() + Row(verticalAlignment = androidx.compose.ui.Alignment.CenterVertically, + horizontalArrangement = Arrangement.spacedBy(8.dp)) { + Text("纪要（${summaries.size}）", style = MaterialTheme.typography.titleMedium) + Button(onClick = { SummaryScheduler.runNow(activity) }) { + Text("现在跑一次") + } + } + if (summaries.isEmpty()) { + Text("还没有纪要。点上面按钮触发；或等 hourly worker（要求 hermes 在 127.0.0.1:8642 跑着）。", + style = MaterialTheme.typography.bodySmall) + } else { + Column( + modifier = Modifier.fillMaxWidth(), + verticalArrangement = Arrangement.spacedBy(4.dp), + ) { + summaries.take(5).forEach { s -> SummaryCard(s) } + } + } + + HorizontalDivider() + Text("ASR Bench (95s TTS sample)", style = MaterialTheme.typography.titleMedium) Text(asrStatus, style = MaterialTheme.typography.bodyMedium) @@ -456,6 +480,20 @@ private fun requestBatteryOptExemption(ctx: Context) { } private val TIMELINE_FMT = SimpleDateFormat("HH:mm:ss", Locale.US) +private val SUMMARY_FMT = SimpleDateFormat("MM-dd HH:mm", Locale.US) + +@Composable +private fun SummaryCard(s: Summary) { + Column( + modifier = Modifier.fillMaxWidth().padding(vertical = 4.dp), + ) { + Text( + "${SUMMARY_FMT.format(Date(s.windowStartTs))} → ${SUMMARY_FMT.format(Date(s.windowEndTs))} · ${s.model}", + style = MaterialTheme.typography.labelSmall, + ) + Text(s.markdown, style = MaterialTheme.typography.bodySmall) + } +} @Composable private fun TimelineRow(row: SegmentRow, isPlaying: Boolean, onClick: () -> Unit) { diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/asr/HermesClient.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/asr/HermesClient.kt new file mode 100644 index 0000000..ac247bd --- /dev/null +++ b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/asr/HermesClient.kt @@ -0,0 +1,109 @@ +package com.lemon.lifelog.asr + +import android.util.Log +import com.lemon.lifelog.BuildConfig +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.RequestBody.Companion.toRequestBody +import org.json.JSONArray +import org.json.JSONObject +import java.io.IOException +import java.util.concurrent.TimeUnit + +/** + * Minimal OpenAI-compatible client targeting the hermes api_server endpoint + * (default http://127.0.0.1:8642/v1). + * + * Used by SummaryWorker to produce 纪要 from transcribed segments. + * Hermes routes to Kimi (kimi-for-coding) underneath; protocol stays OpenAI-shaped. + */ +class HermesClient( + private val baseUrl: String = BuildConfig.HERMES_API_URL, + private val apiKey: String = BuildConfig.HERMES_API_KEY, + private val model: String = BuildConfig.HERMES_MODEL, +) { + companion object { + private const val TAG = "HermesClient" + private val JSON = "application/json; charset=utf-8".toMediaType() + } + + private val http: OkHttpClient = OkHttpClient.Builder() + .connectTimeout(10, TimeUnit.SECONDS) + .readTimeout(120, TimeUnit.SECONDS) // Kimi 大段总结可以慢 + .writeTimeout(30, TimeUnit.SECONDS) + .build() + + data class ChatResult( + val text: String, + val promptTokens: Int, + val completionTokens: Int, + val model: String, + ) + + @Throws(IOException::class) + fun chat( + systemPrompt: String, + userPrompt: String, + maxTokens: Int = 800, + ): ChatResult { + val payload = JSONObject().apply { + put("model", model) + put("max_tokens", maxTokens) + put("messages", JSONArray().apply { + if (systemPrompt.isNotBlank()) { + put(JSONObject().apply { + put("role", "system") + put("content", systemPrompt) + }) + } + put(JSONObject().apply { + put("role", "user") + put("content", userPrompt) + }) + }) + } + val req = Request.Builder() + .url("$baseUrl/v1/chat/completions") + .header("Authorization", "Bearer $apiKey") + .header("Content-Type", "application/json") + .post(payload.toString().toRequestBody(JSON)) + .build() + + http.newCall(req).execute().use { resp -> + val body = resp.body?.string().orEmpty() + if (!resp.isSuccessful) { + throw IOException("hermes ${resp.code}: ${body.take(200)}") + } + return parseChat(body) + } + } + + private fun parseChat(body: String): ChatResult { + val json = JSONObject(body) + val choices = json.optJSONArray("choices") ?: throw IOException("no choices in response") + if (choices.length() == 0) throw IOException("empty choices") + val message = choices.getJSONObject(0).optJSONObject("message") + ?: throw IOException("no message in choice") + val text = message.optString("content", "") + val usage = json.optJSONObject("usage") + return ChatResult( + text = text, + promptTokens = usage?.optInt("prompt_tokens", 0) ?: 0, + completionTokens = usage?.optInt("completion_tokens", 0) ?: 0, + model = json.optString("model", model), + ) + } + + /** Cheap health check used by UI status pill. */ + fun ping(): Boolean = try { + val req = Request.Builder() + .url("$baseUrl/v1/models") + .header("Authorization", "Bearer $apiKey") + .build() + http.newCall(req).execute().use { it.isSuccessful } + } catch (t: Throwable) { + Log.w(TAG, "ping failed: ${t.message}") + false + } +} diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/summary/SummaryScheduler.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/summary/SummaryScheduler.kt new file mode 100644 index 0000000..570a461 --- /dev/null +++ b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/summary/SummaryScheduler.kt @@ -0,0 +1,42 @@ +package com.lemon.lifelog.summary + +import android.content.Context +import androidx.work.Constraints +import androidx.work.Data +import androidx.work.ExistingPeriodicWorkPolicy +import androidx.work.NetworkType +import androidx.work.OneTimeWorkRequestBuilder +import androidx.work.PeriodicWorkRequestBuilder +import androidx.work.WorkManager +import java.util.concurrent.TimeUnit + +object SummaryScheduler { + private const val PERIODIC_TAG = "lifelog-summary-hourly" + private const val ONE_SHOT_TAG = "lifelog-summary-once" + + fun schedulePeriodic(ctx: Context) { + val constraints = Constraints.Builder() + .setRequiredNetworkType(NetworkType.NOT_REQUIRED) // hermes 在本机 127.0.0.1 + .setRequiresBatteryNotLow(true) + .build() + val request = PeriodicWorkRequestBuilder(1, TimeUnit.HOURS) + .setConstraints(constraints) + .build() + WorkManager.getInstance(ctx).enqueueUniquePeriodicWork( + PERIODIC_TAG, + ExistingPeriodicWorkPolicy.KEEP, + request, + ) + } + + fun runNow(ctx: Context, windowOverride: Long? = null) { + val data = if (windowOverride != null) { + Data.Builder().putLong(SummaryWorker.INPUT_NOW_TS, windowOverride).build() + } else Data.EMPTY + val request = OneTimeWorkRequestBuilder() + .setInputData(data) + .addTag(ONE_SHOT_TAG) + .build() + WorkManager.getInstance(ctx).enqueue(request) + } +} diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/summary/SummaryWorker.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/summary/SummaryWorker.kt new file mode 100644 index 0000000..b3bc683 --- /dev/null +++ b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/summary/SummaryWorker.kt @@ -0,0 +1,112 @@ +package com.lemon.lifelog.summary + +import android.content.Context +import android.util.Log +import androidx.work.CoroutineWorker +import androidx.work.WorkerParameters +import com.lemon.lifelog.asr.HermesClient +import com.lemon.lifelog.data.Repository +import com.lemon.lifelog.data.Summary +import java.text.SimpleDateFormat +import java.util.Date +import java.util.Locale + +/** + * Periodic / on-demand 纪要 worker. + * + * Strategy: + * - Pull transcripts in a sliding window (default: last [WINDOW_MS] ms). + * - If the window covers < [MIN_DURATION_MS] of actual speech, skip. + * - If a Summary already exists that overlaps the window with the same or + * larger span, skip (idempotent). + * - Compose prompt with transcripts (+ speaker labels if known) and POST to + * hermes /v1/chat/completions; save the markdown reply as a Summary row. + * + * Failure modes: + * - hermes gateway down → IOException → Worker returns retry(). + * - Empty window → success() no-op. + */ +class SummaryWorker( + ctx: Context, + params: WorkerParameters, +) : CoroutineWorker(ctx, params) { + + companion object { + const val TAG = "SummaryWorker" + + /** 总结窗口大小：默认最近 1 小时。 */ + const val WINDOW_MS = 60L * 60 * 1000 + + /** 窗口内必须至少有这么多毫秒的语音才触发，否则没必要总结。 */ + const val MIN_DURATION_MS = 60_000L // 60s + + const val INPUT_NOW_TS = "now_ts" // override "now" for tests / manual run + } + + private val dateFmt = SimpleDateFormat("yyyy-MM-dd HH:mm", Locale.US) + + override suspend fun doWork(): Result { + val nowTs = inputData.getLong(INPUT_NOW_TS, System.currentTimeMillis()) + val windowStart = nowTs - WINDOW_MS + val repo = Repository(applicationContext) + + val segments = repo.segments.page(limit = 500, offset = 0) + .filter { it.startTs in windowStart..nowTs } + .sortedBy { it.startTs } + + val totalSpeechMs = segments.sumOf { it.durationMs } + if (totalSpeechMs < MIN_DURATION_MS) { + Log.i(TAG, "skip: total speech ${totalSpeechMs}ms < ${MIN_DURATION_MS}ms threshold") + return Result.success() + } + + // Build the user prompt: time range + each segment's text (+ speaker label if any). + val sb = StringBuilder() + sb.appendLine("以下是 ${dateFmt.format(Date(windowStart))} – ${dateFmt.format(Date(nowTs))} 的语音转写片段（按时间顺序）。") + sb.appendLine() + for (seg in segments) { + val transcript = repo.transcripts.get(seg.id)?.text ?: continue + if (transcript.isBlank()) continue + val speakerLabel = repo.segments.get(seg.id)?.speakerId + ?.let { repo.speakers.all().firstOrNull { p -> p.id == it }?.name } + ?: "未知" + val timeStr = SimpleDateFormat("HH:mm:ss", Locale.US).format(Date(seg.startTs)) + sb.appendLine("[$timeStr] $speakerLabel: $transcript") + } + + val systemPrompt = """ + 你是一个 lifelog 纪要助手。给定一段时间内的语音转写片段（含时间戳和说话人）， + 请输出一份简洁的 markdown 纪要，包含： + 1. 一句话主题概括 + 2. 关键信息要点（bullet 列表，5-10 条） + 3. 如果有待办事项 / 决定 / 提到的人名地名，单独列「待办」「人物」「地点」三个小节 + 纪要语言保持中文。不要复述全文，只摘核心。 + """.trimIndent() + + val client = HermesClient() + val response = try { + client.chat(systemPrompt, sb.toString()) + } catch (t: Throwable) { + Log.w(TAG, "hermes call failed: ${t.message}; will retry") + return Result.retry() + } + + repo.summaries.insert( + Summary( + windowStartTs = windowStart, + windowEndTs = nowTs, + markdown = response.text, + model = response.model, + createdAt = System.currentTimeMillis(), + ), + ) + + Log.i( + TAG, + "summary saved span=${WINDOW_MS / 60_000}min " + + "speech=${totalSpeechMs / 1000}s segments=${segments.size} " + + "tokens(prompt=${response.promptTokens}, completion=${response.completionTokens})", + ) + return Result.success() + } +} diff --git a/lifelog-recorder/app/src/main/res/xml/network_security_config.xml b/lifelog-recorder/app/src/main/res/xml/network_security_config.xml new file mode 100644 index 0000000..ae186f0 --- /dev/null +++ b/lifelog-recorder/app/src/main/res/xml/network_security_config.xml @@ -0,0 +1,9 @@ + + + + + 127.0.0.1 + localhost + + diff --git a/lifelog-recorder/keys.properties.example b/lifelog-recorder/keys.properties.example index aba37d6..813e57d 100644 --- a/lifelog-recorder/keys.properties.example +++ b/lifelog-recorder/keys.properties.example @@ -3,3 +3,8 @@ xfyun.apiKey=YOUR_API_KEY xfyun.apiSecret=YOUR_API_SECRET xfyun.iatHost=iat.xf-yun.com xfyun.iatPath=/v1 + +# hermes gateway (OpenAI-compat /v1 endpoint) +hermes.apiUrl=http://127.0.0.1:8642 +hermes.apiKey=GENERATE_WITH_openssl_rand_hex_16 +hermes.model=kimi-for-coding