From d67ce3d2c2a599ea7759a4b36be095994d5e1ef2 Mon Sep 17 00:00:00 2001 From: yibo Date: Tue, 2 Jun 2026 23:37:43 +0800 Subject: [PATCH] feat(lifelog-recorder): v0.2 — Opus 编码 + 自启动 + 电池白名单 --- lifelog-recorder/HANDOFF.md | 38 ++++++++++++++++++++++++++++---------- lifelog-recorder/app/src/main/AndroidManifest.xml | 12 ++++++++++++ lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/MainActivity.kt | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++ lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/audio/SegmentWriter.kt | 154 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++--------------------------------- lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/service/BootReceiver.kt | 40 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 253 insertions(+), 43 deletions(-) create mode 100644 lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/service/BootReceiver.kt diff --git a/lifelog-recorder/HANDOFF.md b/lifelog-recorder/HANDOFF.md index 3d09f1b..5d90c6b 100644 --- a/lifelog-recorder/HANDOFF.md +++ b/lifelog-recorder/HANDOFF.md @@ -40,13 +40,14 @@ APK 大小: 62 MB（含 34 MB sherpa-onnx .so + 0.6 MB silero VAD + 3 MB samp ``` app/src/main/kotlin/com/lemon/lifelog/ ├── LifelogApp.kt — Application，建 NotificationChannel -├── MainActivity.kt — Compose UI，时间轴 LazyColumn + 测试按钮 +├── MainActivity.kt — Compose UI，时间轴 + 电池豁免 UI + 测试按钮 ├── service/ │ ├── RecorderService.kt — 前台服务，pipeline 主体 ←★ 改 ASR 流程改这里 -│ └── RecorderState.kt — StateFlow 把服务状态推到 UI +│ ├── RecorderState.kt — StateFlow 把服务状态推到 UI +│ └── BootReceiver.kt — BOOT_COMPLETED 自启 (Android 14 有 caveat 见 §七) ├── audio/ │ ├── AudioCapture.kt — AudioRecord wrapper，emits FloatArray frames -│ └── SegmentWriter.kt — 写 16k mono WAV 到 filesDir（待换 Opus） +│ └── SegmentWriter.kt — Opus encode via MediaCodec + Ogg muxer，~17× 压缩 ├── asr/ │ ├── ParaformerEngine.kt — sherpa-onnx paraformer 包装 ←★ 主 ASR │ ├── ModelDownloader.kt — 首次启动下模型 + 断点续传 @@ -150,13 +151,30 @@ VAD 参数（在 `RecorderService.runPipeline` 里调）： ## §五 §六 待办（按优先级） -1. **Opus 编码**（🔥 必须）：WAV 在 24/7 跑下 ~2.8 GB/天。换 `MediaCodec OMX.google.opus.encoder` 12 kbps voice mode，缩到 ~130 MB/天。改 `SegmentWriter.kt`。 -2. **自启动 + 电池白名单**（🔥 必须）：BOOT_COMPLETED Receiver 拉起 RecorderService；首启弹 REQUEST_IGNORE_BATTERY_OPTIMIZATIONS。否则一重启就死、Doze 一介入就死。 -3. **声纹识别 / speaker diarization**（用户当面提过 3 次）：sherpa-onnx 已有 `Speaker.kt` + `SpeakerEmbeddingExtractorConfig.kt`。流程：(a) 「这是我」录音 → 建 anchor embedding 存 SQLite；(b) 每段算 embedding 算 cosine 找最近邻；(c) Transcript 加 `speakerHint` 列。需要下 `sherpa-onnx-3dspeaker-zh.onnx`（~38 MB）。 -4. **FTS5 中文搜索**：Room + sqlite-jsqlite-with-icu。Schema 加 `TranscriptFts` 虚表用 ICU tokenizer。UI 加搜索框 + 高亮命中段。 -5. **LLM 纪要 worker**：WorkManager cron。每小时 / 静音 30min 触发，按时间窗 group segments → POST 到 hermes gateway `http://127.0.0.1:8642/v1/chat/completions`（需要先把 hermes api_server platform 配置起来，见 `~/.hermes/config.yaml`）。 -6. **存储 TTL**：删超过 N 天的 WAV，文字永留。Periodic WorkManager job。 -7. **设置页**：存储用量、模型切换、provider 切换、暂停时段配置（这次定的 MVP 是「一键暂停就够」，但用户后面可能要细化）。 +✅ **已完成（v0.2，2026-06）：Opus 编码 / BootReceiver / 电池白名单** + +剩下： + +1. **声纹识别 / speaker diarization**（🔥 用户当面提过 3 次）：sherpa-onnx 已有 `Speaker.kt` + `SpeakerEmbeddingExtractorConfig.kt`。流程：(a)「这是我」录音 → 建 anchor embedding 存 SQLite；(b) 每段算 embedding 算 cosine 找最近邻；(c) Transcript 加 `speakerHint` 列。需要下 `sherpa-onnx-3dspeaker-zh.onnx`（~38 MB）。 +2. **FTS5 中文搜索**：Room + sqlite-jsqlite-with-icu。Schema 加 `TranscriptFts` 虚表用 ICU tokenizer。UI 加搜索框 + 高亮命中段。 +3. **LLM 纪要 worker**：WorkManager cron。每小时 / 静音 30min 触发，按时间窗 group segments → POST 到 hermes gateway `http://127.0.0.1:8642/v1/chat/completions`（需要先把 hermes api_server platform 配置起来，见 `~/.hermes/config.yaml`）。 +4. **存储 TTL**：删超过 N 天的 .opus，文字永留。Periodic WorkManager job。 +5. **设置页**：存储用量、模型切换、provider 切换、暂停时段配置（v1 定的 MVP 是「一键暂停就够」，但 v2 可能要细化）。 +6. **段播放**：时间轴每条点开能直接播原音（Opus 系统原生支持，MediaPlayer 就能放）。 +7. **回链 lemonPCRM**：Transcript 提到的人名 / 项目 → GraphStore `MENTIONED_IN` 边。 + +### 存储预算（v0.2 后） + +- Opus 16 kbps mono = ~2 KB/s +- VAD 后实际有效语音 ≈ 8 h/天 +- → ~57 MB/天，~1.7 GB/月 +- 转写文字（UTF-8 中文）≈ 30 字/秒 × 8h × 3600 ≈ 不到 1 MB/月 + +### Android 14+ BOOT_COMPLETED FG service caveat + +`RecorderService` 的 `foregroundServiceType="microphone"` 是受限 FG type。从 BOOT_COMPLETED 启动**不保证总是成功**——Android 14+ 的 `ForegroundServiceStartNotAllowedException` 可能扔出来（特别是首次重启、还没用过 app 的情况）。`BootReceiver.onReceive` 已 try/catch 兜底，失败时 log + no-op，用户开 app 手动点开始就能恢复。 + +实测 Pixel 8 Pro Android 16 BP2A 上**重启后能自动续上**（前提：app 已经至少打开过一次 + RECORD_AUDIO 已授）。 --- diff --git a/lifelog-recorder/app/src/main/AndroidManifest.xml b/lifelog-recorder/app/src/main/AndroidManifest.xml index e066f04..7c1dcc3 100644 --- a/lifelog-recorder/app/src/main/AndroidManifest.xml +++ b/lifelog-recorder/app/src/main/AndroidManifest.xml @@ -8,6 +8,7 @@ + @@ -34,5 +35,16 @@ android:name=".service.RecorderService" android:exported="false" android:foregroundServiceType="microphone" /> + + + + + + + + diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/MainActivity.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/MainActivity.kt index 1d0f01d..da6c197 100644 --- a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/MainActivity.kt +++ b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/MainActivity.kt @@ -1,9 +1,14 @@ package com.lemon.lifelog import android.Manifest +import android.annotation.SuppressLint +import android.content.Context import android.content.Intent +import android.net.Uri import android.os.Build import android.os.Bundle +import android.os.PowerManager +import android.provider.Settings import androidx.activity.ComponentActivity import androidx.activity.compose.rememberLauncherForActivityResult import androidx.activity.compose.setContent @@ -28,11 +33,15 @@ import androidx.compose.material3.Surface import androidx.compose.material3.Text import androidx.compose.runtime.Composable import androidx.compose.runtime.collectAsState +import androidx.compose.runtime.DisposableEffect import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember import androidx.compose.runtime.rememberCoroutineScope import androidx.compose.runtime.setValue +import androidx.lifecycle.Lifecycle +import androidx.lifecycle.LifecycleEventObserver +import androidx.lifecycle.compose.LocalLifecycleOwner import androidx.compose.ui.Modifier import androidx.compose.ui.unit.dp import androidx.core.content.ContextCompat @@ -85,6 +94,20 @@ private fun MainScreen() { val repo = remember(activity) { Repository(activity) } val timeline by repo.segments.observeTimeline(200).collectAsState(initial = emptyList()) + // Battery optimisation exemption — recompute whenever activity resumes + // (user may have just returned from the system settings dialog). + var batteryExempt by remember { mutableStateOf(isBatteryOptExempt(activity)) } + val lifecycleOwner = LocalLifecycleOwner.current + DisposableEffect(lifecycleOwner) { + val observer = LifecycleEventObserver { _, event -> + if (event == Lifecycle.Event.ON_RESUME) { + batteryExempt = isBatteryOptExempt(activity) + } + } + lifecycleOwner.lifecycle.addObserver(observer) + onDispose { lifecycleOwner.lifecycle.removeObserver(observer) } + } + val permissionLauncher = rememberLauncherForActivityResult( ActivityResultContracts.RequestMultiplePermissions() ) { results -> @@ -117,6 +140,21 @@ private fun MainScreen() { } }) { Text(if (running) "停止录音" else "开始录音") } + // Battery optimisation exemption row. + Row(verticalAlignment = androidx.compose.ui.Alignment.CenterVertically, + horizontalArrangement = Arrangement.spacedBy(8.dp)) { + Text( + if (batteryExempt) "✓ 已加入电池白名单" + else "⚠ 未加入电池白名单（Doze 会关掉录音）", + style = MaterialTheme.typography.bodySmall, + ) + if (!batteryExempt) { + Button(onClick = { requestBatteryOptExemption(activity) }) { + Text("申请豁免") + } + } + } + if (running && lastAudioMs > 0) { val rtSpeed = if (lastWallMs > 0) lastAudioMs.toDouble() / lastWallMs else 0.0 Text( @@ -171,6 +209,20 @@ private fun MainScreen() { } } +private fun isBatteryOptExempt(ctx: Context): Boolean { + val pm = ctx.getSystemService(Context.POWER_SERVICE) as PowerManager + return pm.isIgnoringBatteryOptimizations(ctx.packageName) +} + +@SuppressLint("BatteryLife") +private fun requestBatteryOptExemption(ctx: Context) { + val intent = Intent(Settings.ACTION_REQUEST_IGNORE_BATTERY_OPTIMIZATIONS).apply { + data = Uri.parse("package:${ctx.packageName}") + addFlags(Intent.FLAG_ACTIVITY_NEW_TASK) + } + runCatching { ctx.startActivity(intent) } +} + private val TIMELINE_FMT = SimpleDateFormat("HH:mm:ss", Locale.US) @Composable diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/audio/SegmentWriter.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/audio/SegmentWriter.kt index b008daf..f934793 100644 --- a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/audio/SegmentWriter.kt +++ b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/audio/SegmentWriter.kt @@ -1,7 +1,11 @@ package com.lemon.lifelog.audio +import android.media.MediaCodec +import android.media.MediaCodecInfo +import android.media.MediaFormat +import android.media.MediaMuxer +import android.util.Log import java.io.File -import java.io.RandomAccessFile import java.nio.ByteBuffer import java.nio.ByteOrder import java.text.SimpleDateFormat @@ -9,13 +13,23 @@ import java.util.Date import java.util.Locale /** - * Writes a finished VAD segment to disk as 16-bit PCM WAV. - * File layout: `/YYYY-MM-DD/HH-MM-SS-NNNN.wav` + * Writes a finished VAD segment to disk as Opus-in-Ogg (.opus, ~16 kbps voice mode). + * File layout: `/YYYY-MM-DD/HH-MM-SS-MS.opus` * - * (Opus encoding via MediaCodec is a future optimization; raw WAV at 16k mono is - * ~32 kB/s = ~1 MB/min, still bounded for 24/7 use after VAD trimming.) + * Bitrate budget: 16 kHz mono PCM16 = 32 kB/s WAV → ~2 kB/s Opus = ~16× compression. + * Per 24/7 use (≈8 h voice/day after VAD): ~57 MB/day, ~1.7 GB/month. + * + * Implementation: MediaCodec (audio/opus encoder) + MediaMuxer (OGG container). + * Each segment uses its own encoder + muxer pair — startup overhead is small (a few ms) + * and avoids cross-segment state bugs. */ class SegmentWriter(private val root: File) { + companion object { + private const val TAG = "SegmentWriter" + private const val MIME = MediaFormat.MIMETYPE_AUDIO_OPUS // "audio/opus" + private const val BITRATE = 16_000 // ~16 kbps, plenty for speech + } + private val dayFmt = SimpleDateFormat("yyyy-MM-dd", Locale.US) private val timeFmt = SimpleDateFormat("HH-mm-ss-SSS", Locale.US) @@ -23,42 +37,116 @@ class SegmentWriter(private val root: File) { root.mkdirs() } - /** Writes samples to a new file, returns absolute path. */ + /** Writes samples to a new .opus file, returns absolute path. */ fun write(samplesFloat: FloatArray, ts: Long, sampleRate: Int = 16000): String { val date = Date(ts) val dir = File(root, dayFmt.format(date)).apply { mkdirs() } - val file = File(dir, "${timeFmt.format(date)}.wav") - writeWav(file, samplesFloat, sampleRate) + val file = File(dir, "${timeFmt.format(date)}.opus") + encodeToOgg(samplesFloat, file, sampleRate) return file.absolutePath } - private fun writeWav(file: File, samples: FloatArray, sampleRate: Int) { - val pcmBytes = samples.size * 2 - RandomAccessFile(file, "rw").use { raf -> - raf.setLength(0) - // RIFF header - raf.writeBytes("RIFF") - raf.writeInt(Integer.reverseBytes(36 + pcmBytes)) - raf.writeBytes("WAVE") - // fmt chunk - raf.writeBytes("fmt ") - raf.writeInt(Integer.reverseBytes(16)) // chunk size - raf.writeShort(java.lang.Short.reverseBytes(1).toInt()) // PCM - raf.writeShort(java.lang.Short.reverseBytes(1).toInt()) // channels - raf.writeInt(Integer.reverseBytes(sampleRate)) - raf.writeInt(Integer.reverseBytes(sampleRate * 2)) // byte rate - raf.writeShort(java.lang.Short.reverseBytes(2).toInt()) // block align - raf.writeShort(java.lang.Short.reverseBytes(16).toInt())// bits per sample - // data chunk - raf.writeBytes("data") - raf.writeInt(Integer.reverseBytes(pcmBytes)) - // samples: scale float [-1,1] → int16 LE - val buf = ByteBuffer.allocate(pcmBytes).order(ByteOrder.LITTLE_ENDIAN) + private fun encodeToOgg(samples: FloatArray, outFile: File, sampleRate: Int) { + if (samples.isEmpty()) return + + // 1. PCM16 LE buffer (mono). + val pcm = ByteArray(samples.size * 2) + ByteBuffer.wrap(pcm).order(ByteOrder.LITTLE_ENDIAN).apply { for (s in samples) { - val clamped = (s * 32767f).coerceIn(-32768f, 32767f) - buf.putShort(clamped.toInt().toShort()) + putShort((s.coerceIn(-1f, 1f) * 32767f).toInt().toShort()) + } + } + + // 2. Set up encoder. + val codec = MediaCodec.createEncoderByType(MIME) + val format = MediaFormat.createAudioFormat(MIME, sampleRate, 1).apply { + setInteger(MediaFormat.KEY_BIT_RATE, BITRATE) + // Default frame size for Opus encoder is sane; explicit max-input-size is a hint. + setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, 8192) + } + try { + codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE) + } catch (t: Throwable) { + Log.e(TAG, "configure failed", t) + codec.release() + throw t + } + codec.start() + + // 3. Muxer (OGG container). + val muxer = MediaMuxer(outFile.absolutePath, MediaMuxer.OutputFormat.MUXER_OUTPUT_OGG) + var trackIndex = -1 + var muxerStarted = false + + val info = MediaCodec.BufferInfo() + var bytesFed = 0 + var ptUs = 0L + var inputDone = false + var outputDone = false + + try { + while (!outputDone) { + // Feed PCM. + if (!inputDone) { + val idx = codec.dequeueInputBuffer(50_000) + if (idx >= 0) { + val buf = codec.getInputBuffer(idx)!! + buf.clear() + val cap = buf.capacity() + if (bytesFed < pcm.size) { + val n = minOf(cap, pcm.size - bytesFed) + buf.put(pcm, bytesFed, n) + codec.queueInputBuffer(idx, 0, n, ptUs, 0) + // 2 bytes/sample mono → frames = n/2 → us = frames * 1e6 / sr + ptUs += n.toLong() / 2L * 1_000_000L / sampleRate + bytesFed += n + } else { + codec.queueInputBuffer( + idx, 0, 0, ptUs, + MediaCodec.BUFFER_FLAG_END_OF_STREAM, + ) + inputDone = true + } + } + } + + // Drain encoded output. + drain@ while (true) { + val oIdx = codec.dequeueOutputBuffer(info, 50_000) + when { + oIdx == MediaCodec.INFO_TRY_AGAIN_LATER -> break@drain + oIdx == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED -> { + if (!muxerStarted) { + trackIndex = muxer.addTrack(codec.outputFormat) + muxer.start() + muxerStarted = true + } + } + oIdx >= 0 -> { + val out = codec.getOutputBuffer(oIdx)!! + if (info.size > 0 && muxerStarted && + (info.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0 + ) { + out.position(info.offset) + out.limit(info.offset + info.size) + muxer.writeSampleData(trackIndex, out, info) + } + codec.releaseOutputBuffer(oIdx, false) + if ((info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { + outputDone = true + break@drain + } + } + } + } + } + } finally { + runCatching { codec.stop() } + codec.release() + if (muxerStarted) { + runCatching { muxer.stop() } } - raf.write(buf.array()) + muxer.release() } } } diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/service/BootReceiver.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/service/BootReceiver.kt new file mode 100644 index 0000000..a51144e --- /dev/null +++ b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/service/BootReceiver.kt @@ -0,0 +1,40 @@ +package com.lemon.lifelog.service + +import android.content.BroadcastReceiver +import android.content.Context +import android.content.Intent +import android.util.Log +import androidx.core.content.ContextCompat + +/** + * Resumes recording on device boot. + * + * Android 14+ note: starting a `foregroundServiceType=microphone` service from a + * BOOT_COMPLETED broadcast is allowed because RECORD_AUDIO is a "while-in-use" permission + * that survives across reboots once granted. If the start fails (e.g. user has revoked + * the runtime permission), Android will throw [ForegroundServiceStartNotAllowedException] + * and we just no-op; the user opens the app and taps "开始录音" to resume. + */ +class BootReceiver : BroadcastReceiver() { + override fun onReceive(context: Context, intent: Intent) { + val action = intent.action ?: return + if (action != Intent.ACTION_BOOT_COMPLETED && + action != Intent.ACTION_LOCKED_BOOT_COMPLETED && + action != "android.intent.action.QUICKBOOT_POWERON" + ) return + + Log.i(TAG, "boot action=$action → start RecorderService") + try { + ContextCompat.startForegroundService( + context, + Intent(context, RecorderService::class.java), + ) + } catch (t: Throwable) { + Log.w(TAG, "could not start service from boot", t) + } + } + + companion object { + private const val TAG = "BootReceiver" + } +} -- libgit2 0.21.4