From 950edbab3295eb9ac27c94fd56ed3761a48356a4 Mon Sep 17 00:00:00 2001 From: yibo Date: Tue, 9 Jun 2026 15:31:54 +0800 Subject: [PATCH] feat(lifelog/photolog): 全相册自动索引 — 分层管线 + 搜图 + VL 暂停/停止 --- lifelog-recorder/HANDOFF.md | 65 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++---- lifelog-recorder/app/src/main/AndroidManifest.xml | 6 ++++++ lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/LifelogApp.kt | 1 + lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/PersonProfileScreen.kt | 2 +- lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/PhotologScreen.kt | 229 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++----------------- lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/PhotologViewModel.kt | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/SettingsScreen.kt | 6 +++++- lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/chat/ChatConfig.kt | 9 ++++++++- lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/AppDatabase.kt | 22 ++++++++++++++++++++-- lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/Daos.kt | 116 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++------------- lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/Entities.kt | 22 +++++++++++++++++++--- lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/EmbedClient.kt | 65 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/GalleryScanner.kt | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/ImageUtil.kt | 68 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++- lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/PhotoFaceWorker.kt | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/PhotoProcessor.kt | 66 +++++++++++++++++++++++++++++++++++++++++++++++------------------- lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/PhotoScheduler.kt | 71 +++++++++++++++++++++++++++++++++++++++++++++++++---------------------- lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/PhotoVlState.kt | 40 ++++++++++++++++++++++++++++++++++++++++ lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/PhotoVlWorker.kt | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/PhotoWorker.kt | 61 ------------------------------------------------------------- 20 files changed, 967 insertions(+), 145 deletions(-) create mode 100644 lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/EmbedClient.kt create mode 100644 lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/GalleryScanner.kt create mode 100644 lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/PhotoFaceWorker.kt create mode 100644 lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/PhotoVlState.kt create mode 100644 lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/PhotoVlWorker.kt delete mode 100644 lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/PhotoWorker.kt diff --git a/lifelog-recorder/HANDOFF.md b/lifelog-recorder/HANDOFF.md index c4940e0..ee853a2 100644 --- a/lifelog-recorder/HANDOFF.md +++ b/lifelog-recorder/HANDOFF.md @@ -70,10 +70,26 @@ app/src/main/kotlin/com/lemon/lifelog/ │ ├── HermesClient.kt — OpenAI 兼容（纪要 worker 用，指本地 llama-swap；readTimeout 600s） │ ├── ModelDownloader.kt — 首次启动下模型（paraformer 232 MB + speaker 28 MB） │ └── XfyunIatClient.kt — 讯飞大模型 WS 客户端，仅 fallback / 云复核用 +├── PhotologScreen.kt — 「人物」+「相册」一级 tab（通讯录/档案/照片网格/全相册索引+搜索+VL 控制） +├── PhotologViewModel.kt — photolog 总线（导入/标注/合并/绑声纹/AI 印象/搜图/VL 控制） +├── PersonProfileScreen.kt — 联系人档案（照片/录音/微信聊天/朋友圈/AI 印象，tab 式 + 记录搜索） +├── vision/ — ★ photolog 视觉层（人脸 + 全相册自动索引，见 §四下半） +│ ├── GalleryScanner.kt — 全相册 MediaStore 游标扫描，content:// uri 原地登记（不拷文件），mediaId 去重增量 +│ ├── PhotoFaceWorker.kt — Tier-0：扫描 + ML Kit 人脸 + 512d emb + GPS（快，battery-not-low） +│ ├── PhotoVlWorker.kt — Tier-1：Qwen-VL 场景描述 + caption 语义向量（慢，充电时；尊重暂停标志） +│ ├── PhotoVlState.kt — ★ VL 运行态 singleton（running/currentPhotoId/paused 持久化），UI 控制按钮 + 单张转圈 +│ ├── PhotoScheduler.kt — 两条周期 worker（face 15min / vl 15min+charging）+ runNow/runVlNow/cancelVl +│ ├── PhotoProcessor.kt — faceStage（Tier-0）/ captionStage（Tier-1），按来源解码 +│ ├── FaceEngine.kt — ML Kit 检测+5点对齐 → TFLite MobileFaceNet 512d（非 onnx，避与 sherpa 冲突） +│ ├── FaceMatcher.kt — 人脸归属（人物锚点+已标注脸 gallery，余弦 0.45）+ backfill +│ ├── VisionClient.kt — llama-swap "vl" 模型出场景描述（不报人名） +│ ├── EmbedClient.kt — llama-swap "embed" 模型出 caption 文本向量（语义搜；未配置则优雅降级） +│ ├── ImageUtil.kt — EXIF/解码/缩放/embedding 打包；decode 按 source 走 file 或 content uri +│ └── WeChatEnrollWorker.kt — 微信头像 → 人脸锚点入库 └── data/ - ├── Entities.kt — Segment / Transcript / Summary / SpeakerProfile / TranscriptFts / ChatConversation / ChatMessage - ├── Daos.kt — observeTimeline + searchTimeline(FTS4) + Speaker/Summary/ChatDao + 范围查询 - ├── AppDatabase.kt — Room v3，addMigrations(MIGRATION_2_3)，★ 已删 fallbackToDestructiveMigration（改 schema 不写迁移会崩，绝不静默清库） + ├── Entities.kt — Segment / Transcript / Summary / SpeakerProfile / TranscriptFts / Chat* / Person / Photo / Face / Moment / WechatMessage + ├── Daos.kt — observeTimeline + searchTimeline(FTS4) + Photo/Face/Person/... DAO；★ photos 分层队列(pendingFace/pendingCaption) + observeProgress + searchByKeyword + captionVecs + ├── AppDatabase.kt — Room **v14**，addMigrations(... MIGRATION_13_14)，★ 已删 fallbackToDestructiveMigration（改 schema 不写迁移会崩，绝不静默清库） └── Repository.kt ``` @@ -188,6 +204,43 @@ VAD 参数（在 `RecorderService.runPipeline` 里调）： --- +### photolog 全相册自动索引（v0.8，2026-06-09） + +把手机**整个相册**自动跑成可搜索内容。核心矛盾：本地 VL（Qwen2.5-VL-3B）真机 **~30-40s/张**，几千张全量 VL 是按天计的长任务 → **分层（tiered）**：快的全量先铺，慢的 VL 排队慢慢补。 + +``` +GalleryScanner.scan() ── MediaStore 游标遍历全部图片 + └─ 只存 content:// uri（不拷文件！全相册拷一份会撑爆私有目录）+ mediaId 去重 + DATE_TAKEN + → photos 行 (source='gallery', scanState=0) + │ + ┌──────────────────────────┴───────────────────────────┐ +Tier-0 PhotoFaceWorker (15min, battery-not-low) Tier-1 PhotoVlWorker (15min, +charging) + 每轮先 scan() 增量登记，再处理 scanState=0： 处理 scanState=1，优先级队列(有人脸>近期)： + ├─ ImageUtil.decode(ctx,photo) ← 按 source 解码 ├─ VisionClient.caption(jpeg 448px) → 场景描述 + ├─ FaceEngine 人脸+512d emb + FaceMatcher 认人 ├─ EmbedClient.embed(caption) → 语义向量(可选) + ├─ ImageUtil.readGps (ACCESS_MEDIA_LOCATION) └─ markCaptionDone → scanState=2 + processedAt + └─ markFaceDone → scanState=1 失败 markCaptionSkipped(=2,caption 空) 不卡队列 + 秒级，几小时铺满全库 → 立刻可按人/时间搜 每张 7min 预算 + retry，避开 WM 10min 上限 +``` + +**scanState 状态机**：`0 待索引 → 1 人脸/时间已铺 → 2 已出场景描述`。迁移 13→14 把老的已处理照片（`processedAt` 非空）直接标 2 不重跑；imported 源（手动导入，拷进私有目录）走同一管线，只是 `source='imported'` 且 `decode` 走 file path。 + +**搜索**：① 关键词 `searchByKeyword` = caption + 人名 LIKE 子串（中文子串比 FTS4 unicode61 靠谱，跟 transcript 的 searchTextLike 一致）；② 语义 `searchSemantic` = embed query → 全库 `captionVecs()` 余弦 top-K（Kotlin 算，floor 0.20）。语义要 llama-swap 配 `embed` 模型（`--embedding` 的中文 GGUF，如 bge-small-zh），没配 `EmbedClient` 抛 IOException，UI 提示「未配置 embed 端点」，不影响关键词搜。 + +**VL 控制（PhotoVlState）**：worker 推 `running`/`currentPhotoId`，相册页据此： +- 运行中 → 顶部「⏸ 暂停」「⏹ 停止」+ 正在 VL 的那格半透明遮罩转圈； +- 空闲 → 「⚡ 立即理解」（暂停且未跑完 → 「▶ 继续理解」）； +- **暂停** = `paused` 持久化到 prefs，worker 每张前 `isPaused()` 检查 break，**连充电时的周期 worker 也尊重**，需「继续」才恢复；**停止** = cancelVl 取消手动这轮 + 清 paused（之后充电仍自动补完）。 +- 格子角标：`👤N` 人脸数 / `📝` 已出描述(可搜) / `待索引·待理解·无描述` 未处理态。 + +> ⚠️ 关键坑（已修）：`ImageUtil.decodeUri` 里 bounds 预解码用 `inJustDecodeBounds=true`，`BitmapFactory.decodeStream` **按设计返回 null**（只填 bounds）——不能拿它的返回值判失败，否则**每张都提前 return null**，人脸全跑空。`?:` 要判 `openInputStream` 的 null。 + +> ⚠️ WM worker 互斥：periodic + 手动 + retry 可能并发，多 VL worker 打同一 llama-server 会 502 → `PhotoVlWorker`/`PhotoFaceWorker` 各有 `AtomicBoolean isRunning` 守。`PhotoVlState.running` 只由拿到锁那个置位。 + +> 全相册扫描没有 sqlite3 在设备上：dev 改 photos 表用 `adb exec-out run-as com.lemon.lifelog cat databases/lifelog.db{,-wal,-shm}` 拉到 Mac，`sqlite3 'PRAGMA wal_checkpoint(TRUNCATE);'` 合并 WAL 后改，再 `run-as sh -c 'cat > databases/lifelog.db'` 推回 + 删 wal/shm。 + +--- + ## §五 §六 待办（按优先级） ✅ **已完成**： @@ -196,7 +249,8 @@ VAD 参数（在 `RecorderService.runPipeline` 里调）： - v0.4 (2026-06)：段播放 (MediaPlayer Opus) / bottom sheet 操作菜单 / enroll 后自动回填老段 - v0.5 (2026-06)：hermes api_server 端点 + SummaryWorker + 纪要 UI section - v0.6 (2026-06-04)：**声纹在线聚类**（SpeakerClusterer，自动「说话人N」+ reclusterAll）/ **5-tab UI**（录音/对话/时间轴/声纹/纪要，时间轴全 CRUD + 批量删导出 + 按说话人/时间/未匹配筛选 + 排序）/ **真暂停**（停采集留模型）/ **会话级纪要**（按时间间隔切会话、可调阈值、要点带 [HH:mm:ss] 点击跳回时间轴、折叠头显示说话人名单）/ **App 内对话**（本地 2B/4B + Hermes 多端点流式）/ **本地 LLM 栈**（llama.cpp HEAD + llama-swap 8080，纪要+对话全本地，见 §六）/ **Room v3 迁移**（去 destructive 改 fail-loud） -- v0.7 (2026-06-07)：**block-based diarization-first 录音管线**（攒块→整块 pyannote 分轮→逐 turn 转写+嵌入+认人，turn=时间轴一行；去实时字幕；BlockBuffer/BlockProcessor；崩溃 listPending 续跑；commit 07f3d22）/ **声纹纠偏 L1+L2**（L1=reclusterAll & assign 冻结命名 anchor 不漂移止串人；L2=重命名/合并时 rebuildAnchor 用该人全部段均值重建多 anchor）/ 阈值默认 0.45、设置页 0.35-0.65 可调 +- v0.7 (2026-06-07)：**block-based diarization-first 录音管线**（攒块→整块 pyannote 分轮→逐 turn 转写+嵌入+认人，turn=时间轴一行；去实时字幕；BlockBuffer/BlockProcessor；崩溃 listPending 续跑；commit 07f3d22）/ **声纹纠偏 L1+L2**（L1=reclusterAll & assign 冻结命名 anchor 不漂移止串人；L2=重命名/合并时 rebuildAnchor 用该人全部段均值重建多 anchor）/ 阈值默认 0.45、设置页 0.35-0.65 可调 / **photolog 人脉 CRM 层**（人物档案 + 相册人脸聚类 + 微信通讯录/朋友圈/AI 印象） +- v0.8 (2026-06-09)：**全相册自动索引 + 分层管线**（GalleryScanner 扫 MediaStore 全量、content uri 原地登记不拷文件；Tier-0 人脸/时间秒级全量铺、Tier-1 VL 充电时排队补；Room v14；详见 §四 photolog 段）/ **搜图**（关键词 caption+人名 LIKE / 语义 caption 向量余弦，embed 走 llama-swap）/ **VL 控制**（PhotoVlState：暂停/继续/停止切换 + 单张转圈 + 格子状态角标） 剩下： @@ -315,6 +369,9 @@ openrouter/nous metadata 超时），对开机服务可接受。 | hf-mirror.com 偶尔 truncate | HF mirror 限速 / 中断 | `ModelDownloader` 已带断点续传 | | `am start-foreground-service` 返回 "Requires permission not exported" | 服务 `android:exported="false"`，adb shell 没权 | 用 `su -c "am ..."` 提权 | | `adb shell 'su 10288 -c "..."` 网络全断 | Magisk su 没传 supplementary groups | 加 `-G 3003 -G 9997 -G 20 -G 50` | +| 全相册照片人脸全 0、`decode failed (face)` 刷屏 | `decodeUri` bounds 预解码处 `decodeStream(inJustDecodeBounds=true)` **按设计返回 null**，被 `?:` 当失败提前 return | `?:` 改判 `openInputStream` 的 null，不是 `decodeStream` 返回值（已修，见 `ImageUtil.decodeUri`） | +| 语义搜 `embed 404: no router for requested model` | llama-swap 没配 `embed` 模型 | config.yaml 加一个 `embed` model（`llama-server --embedding` + 中文 GGUF）；不配则只用关键词搜 | +| VL 进度不动 / 点「立即理解」像没反应 | 单张 VL ~30-40s（含 llama-swap 热加载 vl 模型），且要 battery-not-low | 正常现象；看「场景描述 N/总」进度条；想自动补完插充电（周期 worker 要 charging） | --- diff --git a/lifelog-recorder/app/src/main/AndroidManifest.xml b/lifelog-recorder/app/src/main/AndroidManifest.xml index 04b9a78..909d2f0 100644 --- a/lifelog-recorder/app/src/main/AndroidManifest.xml +++ b/lifelog-recorder/app/src/main/AndroidManifest.xml @@ -12,6 +12,12 @@ + + + + + AsyncImage( - model = File(ph.path), contentDescription = ph.caption, contentScale = ContentScale.Crop, + model = photoModel(ph.source, ph.path), contentDescription = ph.caption, contentScale = ContentScale.Crop, modifier = Modifier.aspectRatio(1f).clip(RoundedCornerShape(6.dp)).clickable { onOpenPhoto(ph.id) }, ) } diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/PhotologScreen.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/PhotologScreen.kt index cdc54a8..871b5a5 100644 --- a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/PhotologScreen.kt +++ b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/PhotologScreen.kt @@ -2,6 +2,8 @@ package com.lemon.lifelog import android.Manifest import android.content.pm.PackageManager +import android.net.Uri +import android.os.Build import androidx.activity.compose.rememberLauncherForActivityResult import androidx.activity.result.PickVisualMediaRequest import androidx.activity.result.contract.ActivityResultContracts @@ -22,6 +24,7 @@ import androidx.compose.foundation.layout.fillMaxWidth import androidx.compose.foundation.layout.height import androidx.compose.foundation.layout.padding import androidx.compose.foundation.layout.size +import androidx.compose.foundation.layout.width import androidx.compose.foundation.shape.CircleShape import androidx.compose.ui.draw.clip import androidx.compose.ui.window.Dialog @@ -41,6 +44,7 @@ import androidx.compose.material3.CircularProgressIndicator import androidx.compose.material3.ExperimentalMaterial3Api import androidx.compose.material3.FilterChip import androidx.compose.material3.HorizontalDivider +import androidx.compose.material3.LinearProgressIndicator import androidx.compose.material3.MaterialTheme import androidx.compose.material3.ModalBottomSheet import androidx.compose.material3.OutlinedButton @@ -70,9 +74,12 @@ import androidx.lifecycle.viewmodel.compose.viewModel import coil.compose.AsyncImage import com.lemon.lifelog.data.Face import com.lemon.lifelog.data.Person +import com.lemon.lifelog.data.PhotoProgress import com.lemon.lifelog.data.PhotoRow import com.lemon.lifelog.data.Repository +import com.lemon.lifelog.vision.PhotoVlState import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.delay import kotlinx.coroutines.flow.flowOf import kotlinx.coroutines.withContext import java.io.File @@ -122,38 +129,121 @@ internal fun ContactsTab(repo: Repository) { } } -/** 「相册」一级 tab：照片网格（导入 + 详情 + 标注）。 */ +/** PhotoRow → Coil model：gallery 源是 content uri，imported 源是私有文件。 */ +internal fun photoModel(source: String, path: String): Any = + if (source == "gallery") Uri.parse(path) else File(path) + +/** 「相册」一级 tab：全相册自动索引 + 进度 + 搜索 + 网格（详情 + 标注）。 */ @OptIn(ExperimentalMaterial3Api::class) @Composable internal fun AlbumTab(repo: Repository) { val vm: PhotologViewModel = viewModel() + val ctx = LocalContext.current var detailPhotoId by remember { mutableStateOf(null) } val persons by vm.repo.persons.observeAll().collectAsState(initial = emptyList()) - val pendingCount by vm.repo.photos.observeUnprocessedCount().collectAsState(initial = 0) + val progress by vm.repo.photos.observeProgress().collectAsState(initial = PhotoProgress(0, 0, 0)) val photos by remember { vm.repo.photos.observeGrid(1000) }.collectAsState(initial = emptyList()) + // VL 运行态：控制按钮切换 + 单张处理指示。 + val vlRunning by PhotoVlState.running.collectAsState() + val vlPaused by PhotoVlState.paused.collectAsState() + val vlCurrentId by PhotoVlState.currentPhotoId.collectAsState() + + // 读相册权限（13+ READ_MEDIA_IMAGES；12 及以下 READ_EXTERNAL_STORAGE）。授予即扫全相册。 + val mediaPerm = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) + Manifest.permission.READ_MEDIA_IMAGES else Manifest.permission.READ_EXTERNAL_STORAGE + var granted by remember { + mutableStateOf(ContextCompat.checkSelfPermission(ctx, mediaPerm) == PackageManager.PERMISSION_GRANTED) + } + val permLauncher = rememberLauncherForActivityResult( + ActivityResultContracts.RequestMultiplePermissions(), + ) { res -> granted = res[mediaPerm] == true } + LaunchedEffect(granted) { if (granted) vm.scanGallery() } + // 读图片 + 读 EXIF GPS（ACCESS_MEDIA_LOCATION 是独立的危险权限，要一起请求）。 + val requestPerms = remember(mediaPerm) { + arrayOf(mediaPerm, Manifest.permission.ACCESS_MEDIA_LOCATION) + } + + // 搜索：results=null 显示全部网格；非 null 显示搜索结果。 + var query by remember { mutableStateOf("") } + var results by remember { mutableStateOf?>(null) } + var searchMsg by remember { mutableStateOf(null) } + // 关键词即时搜（去抖 300ms）。语义搜走显式按钮。 + LaunchedEffect(query) { + if (query.isBlank()) { results = null; searchMsg = null; return@LaunchedEffect } + delay(300) + vm.searchKeyword(query) { results = it; searchMsg = "关键词命中 ${it.size}" } + } val picker = rememberLauncherForActivityResult( ActivityResultContracts.PickMultipleVisualMedia(30), ) { uris -> vm.importUris(uris) } + val shown = results ?: photos + Column(modifier = Modifier.fillMaxSize()) { + // 权限 + 进度。 + if (!granted) { + Row( + modifier = Modifier.fillMaxWidth().padding(horizontal = 12.dp, vertical = 6.dp), + verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(8.dp), + ) { + Button(onClick = { permLauncher.launch(requestPerms) }) { Text("开启全相册索引") } + Text("授予读相册权限后自动扫描全部照片", style = MaterialTheme.typography.labelSmall) + } + } else { + ProgressHeader( + p = progress, running = vlRunning, paused = vlPaused, + onRun = { + vm.scanGallery(); vm.runVlNow() + android.widget.Toast.makeText( + ctx, "已开始场景理解（每张约 30–40 秒，看「场景描述」进度）", android.widget.Toast.LENGTH_LONG, + ).show() + }, + onPause = { vm.pauseVl() }, + onStop = { vm.stopVl() }, + ) + } + // 搜索框 + 语义按钮。 Row( - modifier = Modifier.fillMaxWidth().padding(horizontal = 12.dp, vertical = 6.dp), - verticalAlignment = Alignment.CenterVertically, - horizontalArrangement = Arrangement.spacedBy(8.dp), + modifier = Modifier.fillMaxWidth().padding(horizontal = 12.dp, vertical = 4.dp), + verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(8.dp), ) { - Button(onClick = { + OutlinedTextField( + value = query, onValueChange = { query = it }, + placeholder = { Text("搜照片：场景 / 人名 / 物体…") }, singleLine = true, + modifier = Modifier.weight(1f), + ) + Button( + enabled = query.isNotBlank(), + onClick = { + searchMsg = "语义搜索中…" + vm.searchSemantic( + query, + onResult = { results = it; searchMsg = "语义命中 ${it.size}" }, + onError = { searchMsg = "⚠ $it" }, + ) + }, + ) { Text("🧠语义") } + } + Row( + modifier = Modifier.fillMaxWidth().padding(horizontal = 12.dp).padding(bottom = 4.dp), + verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(8.dp), + ) { + OutlinedButton(onClick = { picker.launch(PickVisualMediaRequest(ActivityResultContracts.PickVisualMedia.ImageOnly)) - }) { Text("+ 导入照片") } - if (pendingCount > 0) { - CircularProgressIndicator(modifier = Modifier.height(18.dp), strokeWidth = 2.dp) - Text("待处理 $pendingCount（后台识别中）", style = MaterialTheme.typography.labelSmall) - } + }) { Text("+ 手动导入") } + searchMsg?.let { Text(it, style = MaterialTheme.typography.labelSmall) } } - if (photos.isEmpty()) { + HorizontalDivider() + if (shown.isEmpty()) { Box(Modifier.fillMaxSize().padding(24.dp), contentAlignment = Alignment.Center) { Text( - if (pendingCount > 0) "已导入 $pendingCount 张，后台识别中…稍后刷新" else "还没有照片。点「+ 导入照片」加入相册。\n导入瞬时完成，人脸+场景识别后台定时处理。", + when { + results != null -> "没搜到。换个词，或等场景描述（VL）跑得更多再搜。" + !granted -> "点上方「开启全相册索引」，自动把手机相册所有照片纳入。" + progress.total > 0 -> "正在索引 ${progress.total} 张…稍后刷新。" + else -> "还没有照片。开启全相册索引，或「+ 手动导入」。" + }, style = MaterialTheme.typography.bodyMedium, ) } @@ -164,7 +254,9 @@ internal fun AlbumTab(repo: Repository) { verticalArrangement = Arrangement.spacedBy(4.dp), horizontalArrangement = Arrangement.spacedBy(4.dp), ) { - gridItems(photos, key = { it.id }) { p -> PhotoCell(p) { detailPhotoId = p.id } } + gridItems(shown, key = { it.id }) { p -> + PhotoCell(p, processing = p.id == vlCurrentId) { detailPhotoId = p.id } + } } } } @@ -173,15 +265,118 @@ internal fun AlbumTab(repo: Repository) { } } +/** + * 分层进度：人脸（Tier-0）+ 场景描述（Tier-1 VL）两条进度条 + VL 控制。 + * 运行中显示「暂停 / 停止」，空闲显示「立即理解」（暂停且未跑完时显示「继续」）。 + */ +@Composable +private fun ProgressHeader( + p: PhotoProgress, + running: Boolean, + paused: Boolean, + onRun: () -> Unit, + onPause: () -> Unit, + onStop: () -> Unit, +) { + if (p.total == 0) { + Row( + modifier = Modifier.fillMaxWidth().padding(horizontal = 12.dp, vertical = 6.dp), + verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(8.dp), + ) { + CircularProgressIndicator(modifier = Modifier.height(16.dp), strokeWidth = 2.dp) + Text("等待扫描相册…", style = MaterialTheme.typography.labelSmall) + } + return + } + Column(modifier = Modifier.fillMaxWidth().padding(horizontal = 12.dp, vertical = 6.dp)) { + Row(verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(8.dp)) { + Text("人脸/时间 ${p.faceDone}/${p.total}", style = MaterialTheme.typography.labelSmall, modifier = Modifier.width(120.dp)) + LinearProgressIndicator( + progress = { if (p.total > 0) p.faceDone.toFloat() / p.total else 0f }, + modifier = Modifier.weight(1f), + ) + } + Spacer(Modifier.height(4.dp)) + Row(verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(8.dp)) { + Text("场景描述 ${p.captionDone}/${p.total}", style = MaterialTheme.typography.labelSmall, modifier = Modifier.width(120.dp)) + LinearProgressIndicator( + progress = { if (p.total > 0) p.captionDone.toFloat() / p.total else 0f }, + modifier = Modifier.weight(1f), + ) + } + Spacer(Modifier.height(6.dp)) + if (running) { + Row(verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(8.dp)) { + CircularProgressIndicator(modifier = Modifier.size(16.dp), strokeWidth = 2.dp) + Button(onClick = onPause) { Text("⏸ 暂停") } + OutlinedButton(onClick = onStop) { Text("⏹ 停止") } + Text("理解中 ${p.captionDone}/${p.total}", style = MaterialTheme.typography.labelSmall) + } + } else if (p.captionDone < p.total) { + Row(verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(8.dp)) { + Button(onClick = onRun) { Text(if (paused) "▶ 继续理解" else "⚡ 立即理解") } + if (paused) OutlinedButton(onClick = onStop) { Text("⏹ 停止") } + Text(if (paused) "已暂停" else "默认充电时跑", style = MaterialTheme.typography.labelSmall) + } + } else { + Text("✓ 全部已理解", style = MaterialTheme.typography.labelSmall) + } + } +} + +/** 角标：按 scanState/处理态显示「转圈 / 待理解 / 待索引」。 */ @Composable -private fun PhotoCell(p: PhotoRow, onClick: () -> Unit) { +private fun PhotoCell(p: PhotoRow, processing: Boolean, onClick: () -> Unit) { Box(modifier = Modifier.aspectRatio(1f).clickable(onClick = onClick)) { AsyncImage( - model = File(p.path), + model = photoModel(p.source, p.path), contentDescription = null, contentScale = ContentScale.Crop, modifier = Modifier.fillMaxSize(), ) + // 正在 VL 这张：半透明遮罩 + 转圈。 + if (processing) { + Box( + modifier = Modifier.fillMaxSize().background(MaterialTheme.colorScheme.scrim.copy(alpha = 0.45f)), + contentAlignment = Alignment.Center, + ) { + CircularProgressIndicator(modifier = Modifier.size(22.dp), strokeWidth = 2.dp, color = MaterialTheme.colorScheme.onPrimary) + } + } + // 左下角状态角标：0 待索引 / 1 待理解 / 2(无 caption) 无描述；已理解(有 caption)不标。 + val statusLabel = when { + processing -> null + p.scanState == 0 -> "待索引" + p.scanState == 1 -> "待理解" + p.scanState >= 2 && p.caption.isNullOrBlank() -> "无描述" + else -> null + } + statusLabel?.let { lbl -> + Surface( + color = MaterialTheme.colorScheme.surface.copy(alpha = 0.75f), + shape = RoundedCornerShape(6.dp), + modifier = Modifier.align(Alignment.BottomStart).padding(4.dp), + ) { + Text( + lbl, style = MaterialTheme.typography.labelSmall, + color = MaterialTheme.colorScheme.onSurface, + modifier = Modifier.padding(horizontal = 5.dp, vertical = 1.dp), + ) + } + } + // 已出场景描述：左下角一个绿点提示「可搜」。 + if (!processing && p.scanState >= 2 && !p.caption.isNullOrBlank()) { + Surface( + color = MaterialTheme.colorScheme.primary.copy(alpha = 0.85f), + shape = RoundedCornerShape(8.dp), + modifier = Modifier.align(Alignment.BottomStart).padding(4.dp), + ) { + Text( + "📝", style = MaterialTheme.typography.labelSmall, + modifier = Modifier.padding(horizontal = 4.dp, vertical = 1.dp), + ) + } + } if (p.faceCount > 0) { Surface( color = MaterialTheme.colorScheme.primary.copy(alpha = 0.8f), @@ -214,7 +409,7 @@ private fun PhotoDetailSheet(vm: PhotologViewModel, photoId: Long, persons: List Column(modifier = Modifier.fillMaxWidth().padding(16.dp)) { photo?.let { p -> AsyncImage( - model = File(p.path), + model = photoModel(p.source, p.path), contentDescription = null, contentScale = ContentScale.Fit, modifier = Modifier.fillMaxWidth().height(280.dp), diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/PhotologViewModel.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/PhotologViewModel.kt index f0dd756..64f72c8 100644 --- a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/PhotologViewModel.kt +++ b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/PhotologViewModel.kt @@ -7,6 +7,7 @@ import androidx.lifecycle.AndroidViewModel import androidx.lifecycle.viewModelScope import com.lemon.lifelog.data.Person import com.lemon.lifelog.data.Photo +import com.lemon.lifelog.data.PhotoRow import com.lemon.lifelog.data.Repository import com.lemon.lifelog.data.SpeakerProfile import androidx.work.ExistingWorkPolicy @@ -21,9 +22,12 @@ import com.lemon.lifelog.chat.ChatClient import com.lemon.lifelog.chat.ChatConfig import com.lemon.lifelog.service.RecorderService import com.lemon.lifelog.service.RecorderState +import com.lemon.lifelog.vision.EmbedClient +import com.lemon.lifelog.vision.FaceEngine import com.lemon.lifelog.vision.FaceMatcher import com.lemon.lifelog.vision.ImageUtil import com.lemon.lifelog.vision.PhotoScheduler +import com.lemon.lifelog.vision.PhotoVlState import com.lemon.lifelog.vision.WeChatEnrollWorker import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Job @@ -69,6 +73,66 @@ class PhotologViewModel(app: Application) : AndroidViewModel(app) { } } + /** 进相册页 / 拿到读相册权限后催一次全相册扫描 + Tier-0（幂等，已在跑则忽略）。 */ + fun scanGallery() = PhotoScheduler.scanNow(getApplication()) + + /** 「立即理解」/「继续」：清暂停标志 + 催一次 VL（绕开充电约束）。 */ + fun runVlNow() { + PhotoVlState.setPaused(getApplication(), false) + PhotoScheduler.runVlNow(getApplication()) + } + + /** 暂停：worker 处理完当前张就停，标志持久化（周期/充电触发也不跑），可「继续」。 */ + fun pauseVl() = PhotoVlState.setPaused(getApplication(), true) + + /** 停止：取消手动 VL + 清运行态；不留暂停标志（之后充电时周期照常推进）。 */ + fun stopVl() { + val app = getApplication() + PhotoScheduler.cancelVl(app) + PhotoVlState.setPaused(app, false) + PhotoVlState.running.value = false + PhotoVlState.currentPhotoId.value = null + } + + /** 关键词搜图（caption + 人名 LIKE 子串），即时。 */ + fun searchKeyword(kw: String, onResult: (List) -> Unit) { + viewModelScope.launch { + val r = withContext(Dispatchers.IO) { repo.photos.searchByKeyword(kw.trim()) } + onResult(r) + } + } + + /** + * 语义搜图：query → 本地 embed 端点出向量 → 跟所有 caption 向量算余弦 → top-K。 + * embed 端点没配/挂了会走 onError（UI 提示），不影响关键词搜。 + */ + fun searchSemantic(query: String, onResult: (List) -> Unit, onError: (String) -> Unit) { + viewModelScope.launch { + val app = getApplication() + try { + val rows = withContext(Dispatchers.IO) { + val embed = EmbedClient(ChatConfig.localBase(app), ChatConfig.modelEmbed(app)) + val q = embed.embed(query.trim()) + val vecs = repo.photos.captionVecs() + if (vecs.isEmpty()) return@withContext emptyList() + val scored = vecs.asSequence() + .map { it.id to FaceEngine.cosine(q, ImageUtil.unpackEmbedding(it.captionEmbedding)) } + .filter { it.second >= SEMANTIC_FLOOR } + .sortedByDescending { it.second } + .take(60) + .map { it.first } + .toList() + if (scored.isEmpty()) return@withContext emptyList() + val byId = repo.photos.rowsByIds(scored).associateBy { it.id } + scored.mapNotNull { byId[it] } // 保持相似度降序 + } + onResult(rows) + } catch (t: Throwable) { + onError(t.message ?: "语义搜索失败（检查设置里的 embed 端点）") + } + } + } + fun createPersonAndTag(faceId: Long, name: String) { viewModelScope.launch { // 手动建的人 = 我关心的 → 直接 managed。 @@ -371,6 +435,8 @@ class PhotologViewModel(app: Application) : AndroidViewModel(app) { } } +private const val SEMANTIC_FLOOR = 0.20f // 语义余弦下限，低于此基本不相关；主要靠 top-K 排序 + /** 手机系统通讯录里的一条（内存态，未入库）。phoneDisplay 留原格式给 UI，phone 是纯数字去重键。 */ data class PhoneContact( val id: Long, diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/SettingsScreen.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/SettingsScreen.kt index f1b5f7c..95b64ba 100644 --- a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/SettingsScreen.kt +++ b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/SettingsScreen.kt @@ -76,6 +76,7 @@ fun SettingsScreen(activity: Context) { var localBase by remember { mutableStateOf(ChatConfig.localBase(activity)) } var m2b by remember { mutableStateOf(ChatConfig.model2b(activity)) } var m4b by remember { mutableStateOf(ChatConfig.model4b(activity)) } + var mEmbed by remember { mutableStateOf(ChatConfig.modelEmbed(activity)) } var endpointMsg by remember { mutableStateOf("") } var batteryExempt by remember { mutableStateOf(isBatteryExempt(activity)) } @@ -133,9 +134,12 @@ fun SettingsScreen(activity: Context) { OutlinedTextField(value = m4b, onValueChange = { m4b = it }, label = { Text("4B model 名") }, singleLine = true, modifier = Modifier.weight(1f)) } + OutlinedTextField(value = mEmbed, onValueChange = { mEmbed = it }, + label = { Text("embed model 名（语义搜图，llama-swap 里配 --embedding）") }, + singleLine = true, modifier = Modifier.fillMaxWidth()) Row(verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(8.dp)) { Button(onClick = { - ChatConfig.save(activity, localBase, m2b, m4b) + ChatConfig.save(activity, localBase, m2b, m4b, mEmbed) endpointMsg = "已保存" }) { Text("保存端点") } if (endpointMsg.isNotEmpty()) Text(endpointMsg, style = MaterialTheme.typography.labelSmall) diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/chat/ChatConfig.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/chat/ChatConfig.kt index ca90687..c0a17e3 100644 --- a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/chat/ChatConfig.kt +++ b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/chat/ChatConfig.kt @@ -26,10 +26,13 @@ object ChatConfig { private const val K_LOCAL_BASE = "local_base" private const val K_MODEL_2B = "model_2b" private const val K_MODEL_4B = "model_4b" + private const val K_MODEL_EMBED = "model_embed" const val DEFAULT_LOCAL_BASE = "http://127.0.0.1:8080" const val DEFAULT_MODEL_2B = "2b" const val DEFAULT_MODEL_4B = "4b" + /** 语义搜索的文本 embedding model 名（llama-swap 里配 --embedding 的 GGUF，如 bge-small-zh）。 */ + const val DEFAULT_MODEL_EMBED = "embed" const val ID_2B = "local-2b" const val ID_4B = "local-4b" @@ -46,11 +49,15 @@ object ChatConfig { fun model4b(ctx: Context): String = prefs(ctx).getString(K_MODEL_4B, DEFAULT_MODEL_4B)?.ifBlank { DEFAULT_MODEL_4B } ?: DEFAULT_MODEL_4B - fun save(ctx: Context, localBase: String, model2b: String, model4b: String) { + fun modelEmbed(ctx: Context): String = + prefs(ctx).getString(K_MODEL_EMBED, DEFAULT_MODEL_EMBED)?.ifBlank { DEFAULT_MODEL_EMBED } ?: DEFAULT_MODEL_EMBED + + fun save(ctx: Context, localBase: String, model2b: String, model4b: String, modelEmbed: String = DEFAULT_MODEL_EMBED) { prefs(ctx).edit() .putString(K_LOCAL_BASE, localBase.trim()) .putString(K_MODEL_2B, model2b.trim()) .putString(K_MODEL_4B, model4b.trim()) + .putString(K_MODEL_EMBED, modelEmbed.trim()) .apply() } diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/AppDatabase.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/AppDatabase.kt index 4cef3ee..cc200b3 100644 --- a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/AppDatabase.kt +++ b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/AppDatabase.kt @@ -22,7 +22,7 @@ import androidx.sqlite.db.SupportSQLiteDatabase WechatMessage::class, Moment::class, ], - version = 13, + version = 14, exportSchema = false, ) abstract class AppDatabase : RoomDatabase() { @@ -132,6 +132,24 @@ abstract class AppDatabase : RoomDatabase() { } } + /** + * v0.8 全相册自动索引：photos 加来源/MediaStore 去重键/分层进度/caption 语义向量。 + * 老的「导入照片」(source 默认 imported)；已处理过的(processedAt 非空)直接标 scanState=2，不重跑。 + */ + private val MIGRATION_13_14 = object : Migration(13, 14) { + override fun migrate(db: SupportSQLiteDatabase) { + db.execSQL("ALTER TABLE `photos` ADD COLUMN `source` TEXT NOT NULL DEFAULT 'imported'") + db.execSQL("ALTER TABLE `photos` ADD COLUMN `mediaId` INTEGER") + db.execSQL("ALTER TABLE `photos` ADD COLUMN `scanState` INTEGER NOT NULL DEFAULT 0") + db.execSQL("ALTER TABLE `photos` ADD COLUMN `captionEmbedding` BLOB") + // 已处理过的老照片不重跑（人脸+VL 都做完了）。 + db.execSQL("UPDATE `photos` SET `scanState` = 2 WHERE `processedAt` IS NOT NULL") + // 加速 Tier-0/Tier-1 取队列 + MediaStore 去重。 + db.execSQL("CREATE INDEX IF NOT EXISTS `index_photos_scanState` ON `photos` (`scanState`)") + db.execSQL("CREATE INDEX IF NOT EXISTS `index_photos_mediaId` ON `photos` (`mediaId`)") + } + } + fun get(ctx: Context): AppDatabase = instance ?: synchronized(this) { instance ?: Room.databaseBuilder( @@ -139,7 +157,7 @@ abstract class AppDatabase : RoomDatabase() { AppDatabase::class.java, "lifelog.db", ) - .addMigrations(MIGRATION_2_3, MIGRATION_3_4, MIGRATION_4_5, MIGRATION_5_6, MIGRATION_6_7, MIGRATION_7_9, MIGRATION_9_10, MIGRATION_10_11, MIGRATION_11_12, MIGRATION_12_13) + .addMigrations(MIGRATION_2_3, MIGRATION_3_4, MIGRATION_4_5, MIGRATION_5_6, MIGRATION_6_7, MIGRATION_7_9, MIGRATION_9_10, MIGRATION_10_11, MIGRATION_11_12, MIGRATION_12_13, MIGRATION_13_14) // 不再 fallbackToDestructiveMigration：以后改 schema 必须写 Migration， // 否则启动直接抛 IllegalStateException（fail-loud），绝不静默清掉真实录音。 .build() diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/Daos.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/Daos.kt index 9423df0..2483a56 100644 --- a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/Daos.kt +++ b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/Daos.kt @@ -287,13 +287,32 @@ interface SummaryDao { data class PhotoRow( val id: Long, val path: String, + /** 'imported' = File(path)；'gallery' = Uri.parse(path)。UI 据此选解码方式。 */ + val source: String, val takenTs: Long, val caption: String?, val faceCount: Int, + /** 0=待索引 1=人脸已铺待理解 2=已出场景描述。格子角标据此显示状态。 */ + val scanState: Int, val lat: Double?, val lon: Double?, ) +/** 语义搜索用的轻投影：只取 id + caption 向量，cosine 在 Kotlin 里算（Room 算不了）。 */ +data class PhotoVec( + val id: Long, + val captionEmbedding: ByteArray, +) + +/** 相册分层进度（进度条用）。 */ +data class PhotoProgress( + val total: Int, + /** Tier-0 已铺（人脸+时间，scanState≥1） */ + val faceDone: Int, + /** Tier-1 已出场景描述（scanState≥2） */ + val captionDone: Int, +) + @Dao interface PersonDao { @Insert @@ -400,33 +419,75 @@ interface PhotoDao { @Query("SELECT * FROM photos WHERE path = :path LIMIT 1") suspend fun getByPath(path: String): Photo? - /** 还没跑过人脸+VL 的照片（处理 worker 取这个）。 */ - @Query("SELECT * FROM photos WHERE processedAt IS NULL ORDER BY takenTs DESC") - suspend fun unprocessed(): List + // ----- 全相册扫描：MediaStore 去重 + 增量 ----- + + /** 已登记的所有 MediaStore _ID（gallery 源去重）。 */ + @Query("SELECT mediaId FROM photos WHERE mediaId IS NOT NULL") + suspend fun knownMediaIds(): List + + /** 已登记的最大 MediaStore _ID（增量扫描可只看更大的，但默认仍全量去重）。 */ + @Query("SELECT MAX(mediaId) FROM photos WHERE source = 'gallery'") + suspend fun maxMediaId(): Long? + + // ----- 分层管线队列 ----- + + /** Tier-0 队列：还没铺人脸（scanState=0），近期优先。 */ + @Query("SELECT * FROM photos WHERE scanState = 0 ORDER BY takenTs DESC LIMIT :limit") + suspend fun pendingFace(limit: Int = 5000): List + + /** + * Tier-1 VL 队列：人脸已铺待出场景描述（scanState=1）。 + * 优先级：有人脸的优先、再近期优先（合照/有人的更值得先理解）。 + */ + @Query( + "SELECT * FROM photos WHERE scanState = 1 ORDER BY (faceCount > 0) DESC, takenTs DESC LIMIT :limit", + ) + suspend fun pendingCaption(limit: Int = 5000): List + + // ----- 分层进度 ----- + + @Query( + """ + SELECT COUNT(*) AS total, + COALESCE(SUM(CASE WHEN scanState >= 1 THEN 1 ELSE 0 END), 0) AS faceDone, + COALESCE(SUM(CASE WHEN scanState >= 2 THEN 1 ELSE 0 END), 0) AS captionDone + FROM photos + """, + ) + fun observeProgress(): Flow + + // ----- 分层写回 ----- - /** 未处理照片数（UI 显示"待处理 N"）。 */ - @Query("SELECT COUNT(*) FROM photos WHERE processedAt IS NULL") - fun observeUnprocessedCount(): Flow + /** Tier-0 完成：写人脸数 + GPS，进度推到 1。 */ + @Query("UPDATE photos SET faceCount = :n, lat = :lat, lon = :lon, scanState = 1 WHERE id = :id") + suspend fun markFaceDone(id: Long, n: Int, lat: Double?, lon: Double?) - @Query("UPDATE photos SET caption = :caption, captionModel = :model WHERE id = :id") - suspend fun updateCaption(id: Long, caption: String, model: String) + /** Tier-1 完成：写 caption + 语义向量，进度推到 2 + processedAt。 */ + @Query( + """ + UPDATE photos SET caption = :caption, captionModel = :model, + captionEmbedding = :emb, scanState = 2, processedAt = :ts WHERE id = :id + """, + ) + suspend fun markCaptionDone(id: Long, caption: String?, model: String?, emb: ByteArray?, ts: Long) - @Query("UPDATE photos SET faceCount = :n, processedAt = :ts WHERE id = :id") - suspend fun markProcessed(id: Long, n: Int, ts: Long) + /** Tier-1 失败也别卡队列：标 scanState=2（caption 留空），下轮不再取。 */ + @Query("UPDATE photos SET scanState = 2, processedAt = :ts WHERE id = :id") + suspend fun markCaptionSkipped(id: Long, ts: Long) @Query("UPDATE photos SET lat = :lat, lon = :lon WHERE id = :id") suspend fun updateGeo(id: Long, lat: Double?, lon: Double?) @Query( - "SELECT id, path, takenTs, caption, faceCount, lat, lon FROM photos ORDER BY takenTs DESC LIMIT :limit", + "SELECT id, path, source, takenTs, caption, faceCount, scanState, lat, lon FROM photos ORDER BY takenTs DESC LIMIT :limit", ) fun observeGrid(limit: Int = 500): Flow> /** 某个人出现的所有照片（faces.personId join）。 */ @Query( """ - SELECT DISTINCT p.id AS id, p.path AS path, p.takenTs AS takenTs, - p.caption AS caption, p.faceCount AS faceCount, p.lat AS lat, p.lon AS lon + SELECT DISTINCT p.id AS id, p.path AS path, p.source AS source, p.takenTs AS takenTs, + p.caption AS caption, p.faceCount AS faceCount, p.scanState AS scanState, p.lat AS lat, p.lon AS lon FROM photos p INNER JOIN faces f ON f.photoId = p.id WHERE f.personId = :personId ORDER BY p.takenTs DESC @@ -434,6 +495,35 @@ interface PhotoDao { ) fun observePhotosOfPerson(personId: Long): Flow> + // ----- 搜索（P3 关键词 LIKE：caption + 人名；中文子串比 FTS4 unicode61 更靠谱） ----- + + @Query( + """ + SELECT DISTINCT p.id AS id, p.path AS path, p.source AS source, p.takenTs AS takenTs, + p.caption AS caption, p.faceCount AS faceCount, p.scanState AS scanState, p.lat AS lat, p.lon AS lon + FROM photos p + LEFT JOIN faces f ON f.photoId = p.id + LEFT JOIN persons pe ON pe.id = f.personId + WHERE p.caption LIKE '%' || :kw || '%' OR pe.name LIKE '%' || :kw || '%' + ORDER BY p.takenTs DESC + LIMIT :limit + """, + ) + suspend fun searchByKeyword(kw: String, limit: Int = 500): List + + // ----- 搜索（P4 语义向量：取全部 caption 向量，cosine 在 Kotlin 里算） ----- + + @Query("SELECT id, captionEmbedding FROM photos WHERE captionEmbedding IS NOT NULL") + suspend fun captionVecs(): List + + @Query( + """ + SELECT id, path, source, takenTs, caption, faceCount, scanState, lat, lon + FROM photos WHERE id IN (:ids) + """, + ) + suspend fun rowsByIds(ids: List): List + @Query("DELETE FROM photos WHERE id = :id") suspend fun delete(id: Long) } diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/Entities.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/Entities.kt index de6e9f9..9dbf7e1 100644 --- a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/Entities.kt +++ b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/Entities.kt @@ -2,6 +2,7 @@ package com.lemon.lifelog.data import androidx.room.Entity import androidx.room.Fts4 +import androidx.room.Index import androidx.room.PrimaryKey @Entity(tableName = "segments") @@ -121,11 +122,19 @@ data class Person( ) /** 一张照片 + Qwen-VL 场景描述 + EXIF 时间/GPS。 */ -@Entity(tableName = "photos") +@Entity( + tableName = "photos", + indices = [Index("scanState"), Index("mediaId")], +) data class Photo( @PrimaryKey(autoGenerate = true) val id: Long = 0, + /** imported 源 = app 私有目录文件绝对路径；gallery 源 = MediaStore content:// uri 字符串（原地读，不拷贝）。 */ val path: String, - /** 拍摄时间（EXIF DateTimeOriginal，回退文件时间） */ + /** 来源：'imported'（手动选进来，拷进私有目录）| 'gallery'（全相册自动扫描，原地引用）。 */ + val source: String = "imported", + /** MediaStore _ID（gallery 源用于去重 + 增量扫描）；imported 源为 null。 */ + val mediaId: Long? = null, + /** 拍摄时间（EXIF DateTimeOriginal / MediaStore DATE_TAKEN，回退文件时间） */ val takenTs: Long, val lat: Double? = null, val lon: Double? = null, @@ -133,7 +142,14 @@ data class Photo( val caption: String? = null, val captionModel: String? = null, val faceCount: Int = 0, - /** 人脸+VL 处理完成时间，null = 还没处理 */ + /** + * 分层管线进度：0 = 已登记未处理；1 = Tier-0（人脸 + EXIF）已铺；2 = Tier-1（VL 场景描述）已出。 + * 全相册几千张，Tier-0 秒级全量铺开先可按人/时间搜，Tier-1 VL 慢、按优先级队列慢慢补。 + */ + val scanState: Int = 0, + /** caption 的本地文本 embedding（LE float bytes），用于语义搜索；null = 还没算/无 caption。 */ + val captionEmbedding: ByteArray? = null, + /** Tier-1 VL 处理完成时间，null = 还没出 caption。 */ val processedAt: Long? = null, ) diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/EmbedClient.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/EmbedClient.kt new file mode 100644 index 0000000..479499a --- /dev/null +++ b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/EmbedClient.kt @@ -0,0 +1,65 @@ +package com.lemon.lifelog.vision + +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.RequestBody.Companion.toRequestBody +import org.json.JSONArray +import org.json.JSONObject +import java.io.IOException +import java.util.concurrent.TimeUnit +import kotlin.math.sqrt + +/** + * 文本 embedding（P4 语义搜索）：打本地 llama-swap 的 OpenAI 兼容 /v1/embeddings。 + * + * 部署：llama-swap config.yaml 加一个 model（默认名 "embed"），用 llama-server --embedding 起一个 + * 中文 embedding GGUF（如 bge-small-zh / gte-small-zh）。没配这个 model 时调用会抛 IOException， + * 语义搜索优雅降级（UI 提示「未配置 embed 端点」），不影响关键词搜。 + * + * 返回向量统一 L2 归一化 → 余弦相似度 = 点积（与人脸/声纹 embedding 同约定）。 + */ +class EmbedClient( + private val baseUrl: String, + private val model: String = "embed", +) { + companion object { + private val JSON = "application/json; charset=utf-8".toMediaType() + } + + private val http = OkHttpClient.Builder() + .connectTimeout(10, TimeUnit.SECONDS) + .readTimeout(120, TimeUnit.SECONDS) // 含 llama-swap 切模型 + .build() + + @Throws(IOException::class) + fun embed(text: String): FloatArray { + val payload = JSONObject().apply { + put("model", model) + put("input", text) + } + val req = Request.Builder() + .url("$baseUrl/v1/embeddings") + .header("Content-Type", "application/json") + .post(payload.toString().toRequestBody(JSON)) + .build() + http.newCall(req).execute().use { resp -> + val body = resp.body?.string().orEmpty() + if (!resp.isSuccessful) throw IOException("embed ${resp.code}: ${body.take(200)}") + val data = JSONObject(body).optJSONArray("data") ?: throw IOException("no data") + if (data.length() == 0) throw IOException("empty data") + val arr: JSONArray = data.getJSONObject(0).optJSONArray("embedding") + ?: throw IOException("no embedding") + val e = FloatArray(arr.length()) { arr.getDouble(it).toFloat() } + return l2norm(e) + } + } + + private fun l2norm(e: FloatArray): FloatArray { + var n = 0f + for (v in e) n += v * v + n = sqrt(n) + if (n > 0f) for (i in e.indices) e[i] /= n + return e + } +} diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/GalleryScanner.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/GalleryScanner.kt new file mode 100644 index 0000000..a15bafe --- /dev/null +++ b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/GalleryScanner.kt @@ -0,0 +1,64 @@ +package com.lemon.lifelog.vision + +import android.content.ContentUris +import android.content.Context +import android.provider.MediaStore +import android.util.Log +import com.lemon.lifelog.data.Photo +import com.lemon.lifelog.data.Repository + +/** + * 全相册扫描：游标遍历 MediaStore 所有图片，把还没登记的注册成 photos 行（scanState=0）。 + * + * 关键取舍：**不拷贝文件**——gallery 源只存 content:// uri，原地引用（全相册几千上万张， + * 拷一份会撑爆私有目录）。拍摄时间直接取 MediaStore DATE_TAKEN（秒级全量铺好时间轴）， + * GPS 留到 Tier-0 解码时按需补（要 ACCESS_MEDIA_LOCATION）。 + * + * 去重按 MediaStore _ID（mediaId）：已登记的跳过，所以可反复跑、增量补新图。 + */ +object GalleryScanner { + private const val TAG = "GalleryScanner" + + /** 扫一遍相册，登记新图片。返回本轮新增数。 */ + suspend fun scan(ctx: Context, repo: Repository): Int { + val known = repo.photos.knownMediaIds().toHashSet() + val collection = MediaStore.Images.Media.getContentUri(MediaStore.VOLUME_EXTERNAL) + val proj = arrayOf( + MediaStore.Images.Media._ID, + MediaStore.Images.Media.DATE_TAKEN, // ms（可能为 0/null） + MediaStore.Images.Media.DATE_ADDED, // 秒 + ) + var added = 0 + try { + ctx.contentResolver.query( + collection, proj, null, null, + "${MediaStore.Images.Media._ID} DESC", + )?.use { c -> + val idIdx = c.getColumnIndexOrThrow(MediaStore.Images.Media._ID) + val takenIdx = c.getColumnIndexOrThrow(MediaStore.Images.Media.DATE_TAKEN) + val addedIdx = c.getColumnIndexOrThrow(MediaStore.Images.Media.DATE_ADDED) + while (c.moveToNext()) { + val id = c.getLong(idIdx) + if (!known.add(id)) continue // 已登记 + val takenMs = if (!c.isNull(takenIdx)) c.getLong(takenIdx) else 0L + val ts = if (takenMs > 0) takenMs else c.getLong(addedIdx) * 1000L + val uri = ContentUris.withAppendedId(collection, id) + repo.photos.insert( + Photo( + path = uri.toString(), + source = "gallery", + mediaId = id, + takenTs = ts, + scanState = 0, + ), + ) + added++ + } + } + } catch (t: Throwable) { + Log.w(TAG, "scan failed: ${t.message}") + } + if (added > 0) Log.i(TAG, "registered $added new gallery photos") + return added + } +} diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/ImageUtil.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/ImageUtil.kt index 58c9e3d..8675210 100644 --- a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/ImageUtil.kt +++ b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/ImageUtil.kt @@ -1,18 +1,23 @@ package com.lemon.lifelog.vision +import android.content.Context import android.graphics.Bitmap import android.graphics.BitmapFactory import android.graphics.Matrix +import android.net.Uri +import android.provider.MediaStore import androidx.exifinterface.media.ExifInterface +import com.lemon.lifelog.data.Photo import java.io.ByteArrayOutputStream import java.io.File +import java.io.InputStream import java.nio.ByteBuffer import java.nio.ByteOrder import java.text.SimpleDateFormat import java.util.Locale import kotlin.math.min -/** 照片 EXIF + 解码 + 缩放工具。 */ +/** 照片 EXIF + 解码 + 缩放工具。imported 源读私有文件，gallery 源原地读 MediaStore content uri。 */ object ImageUtil { data class Meta(val takenTs: Long, val lat: Double?, val lon: Double?) @@ -31,6 +36,67 @@ object ImageUtil { } } + /** 按来源解码：gallery 源走 content uri（原地），imported 源走文件路径。 */ + fun decode(ctx: Context, photo: Photo, reqMax: Int = 2048): Bitmap? = + if (photo.source == "gallery") decodeUri(ctx, Uri.parse(photo.path), reqMax) + else decodeOriented(photo.path, reqMax) + + /** + * 读 GPS（Tier-0 用）。gallery 源需 ACCESS_MEDIA_LOCATION + setRequireOriginal 拿未脱敏 EXIF； + * 没权限/失败就返回 (null,null)，不阻断流程。takenTs 在扫描时已从 MediaStore 取，这里只补经纬度。 + */ + fun readGps(ctx: Context, photo: Photo): Pair = try { + val stream: InputStream? = if (photo.source == "gallery") { + val orig = runCatching { MediaStore.setRequireOriginal(Uri.parse(photo.path)) } + .getOrDefault(Uri.parse(photo.path)) + ctx.contentResolver.openInputStream(orig) + } else { + File(photo.path).inputStream() + } + stream?.use { + val ll = ExifInterface(it).latLong + if (ll != null) ll[0] to ll[1] else null to null + } ?: (null to null) + } catch (t: Throwable) { + null to null + } + + /** 从 content uri 解码 + 按 EXIF 方向摆正（gallery 源）。 */ + private fun decodeUri(ctx: Context, uri: Uri, reqMax: Int): Bitmap? { + val cr = ctx.contentResolver + val bounds = BitmapFactory.Options().apply { inJustDecodeBounds = true } + // 注意：inJustDecodeBounds 模式下 decodeStream 本就返回 null（只填 bounds）， + // 不能拿它的返回值判失败——要判的是 openInputStream。 + (cr.openInputStream(uri) ?: return null).use { BitmapFactory.decodeStream(it, null, bounds) } + var sample = 1 + val longest = maxOf(bounds.outWidth, bounds.outHeight) + while (longest / sample > reqMax) sample *= 2 + val decoded = cr.openInputStream(uri)?.use { + BitmapFactory.decodeStream(it, null, BitmapFactory.Options().apply { inSampleSize = sample }) + } ?: return null + val orientation = try { + cr.openInputStream(uri)?.use { + ExifInterface(it).getAttributeInt(ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL) + } ?: ExifInterface.ORIENTATION_NORMAL + } catch (t: Throwable) { + ExifInterface.ORIENTATION_NORMAL + } + return applyOrientation(decoded, orientation) + } + + private fun applyOrientation(decoded: Bitmap, orientation: Int): Bitmap { + val m = Matrix() + when (orientation) { + ExifInterface.ORIENTATION_ROTATE_90 -> m.postRotate(90f) + ExifInterface.ORIENTATION_ROTATE_180 -> m.postRotate(180f) + ExifInterface.ORIENTATION_ROTATE_270 -> m.postRotate(270f) + else -> return decoded + } + val rotated = Bitmap.createBitmap(decoded, 0, 0, decoded.width, decoded.height, m, true) + if (rotated != decoded) decoded.recycle() + return rotated + } + /** 解码并按 EXIF 方向摆正。可选下采样 reqMax(最长边)以省内存。 */ fun decodeOriented(path: String, reqMax: Int = 2048): Bitmap? { val opts = BitmapFactory.Options() diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/PhotoFaceWorker.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/PhotoFaceWorker.kt new file mode 100644 index 0000000..96731c1 --- /dev/null +++ b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/PhotoFaceWorker.kt @@ -0,0 +1,64 @@ +package com.lemon.lifelog.vision + +import android.content.Context +import android.util.Log +import androidx.work.CoroutineWorker +import androidx.work.WorkerParameters +import com.lemon.lifelog.data.Repository +import java.util.concurrent.atomic.AtomicBoolean + +/** + * Tier-0 worker：① 扫全相册登记新图 ② 给未铺人脸的照片（scanState=0）跑 ML Kit 人脸 + GPS。 + * + * 快（<1s/张），不依赖 llama-server，battery-not-low 即可跑——目标是几小时内把整个相册 + * 的人脸/时间线铺满，先能「按人/按时间」搜。VL 场景描述是另一条慢队列（PhotoVlWorker）。 + * + * 单轮 7min 预算 + retry，避开 WorkManager ~10min 执行上限。 + */ +class PhotoFaceWorker(ctx: Context, params: WorkerParameters) : CoroutineWorker(ctx, params) { + + override suspend fun doWork(): Result { + if (!isRunning.compareAndSet(false, true)) return Result.success() + try { + val repo = Repository(applicationContext) + // 先增量登记全相册新图（需要已授予读相册权限；没权限就只处理已登记的）。 + runCatching { GalleryScanner.scan(applicationContext, repo) } + + val pending = repo.photos.pendingFace() + if (pending.isEmpty()) return Result.success() + + val engine = FaceEngine.load(applicationContext) + val proc = PhotoProcessor(applicationContext, repo) + val deadline = System.currentTimeMillis() + RUN_BUDGET_MS + var done = 0 + try { + for (p in pending) { + if (System.currentTimeMillis() > deadline) break + try { + proc.faceStage(p, engine) + } catch (t: Throwable) { + Log.w(TAG, "faceStage failed ${p.path}: ${t.message}") + repo.photos.markFaceDone(p.id, 0, null, null) // 别卡队列 + } + done++ + } + } finally { + engine.close() + } + Log.i(TAG, "tier0 face: $done / ${pending.size}") + return if (done < pending.size) Result.retry() else Result.success() + } catch (t: Throwable) { + Log.e(TAG, "worker failed", t) + return Result.retry() + } finally { + isRunning.set(false) + } + } + + companion object { + const val TAG = "PhotoFaceWorker" + /** 单轮预算 7min，避开 WorkManager ~10min 上限。 */ + const val RUN_BUDGET_MS = 7 * 60 * 1000L + private val isRunning = AtomicBoolean(false) + } +} diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/PhotoProcessor.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/PhotoProcessor.kt index f3cf162..d8a7e40 100644 --- a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/PhotoProcessor.kt +++ b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/PhotoProcessor.kt @@ -1,44 +1,40 @@ package com.lemon.lifelog.vision +import android.content.Context import android.util.Log import com.lemon.lifelog.data.Face import com.lemon.lifelog.data.Photo import com.lemon.lifelog.data.Repository import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext -import java.io.File /** - * 单张照片处理：人脸(FaceEngine) + 场景描述(VisionClient) → 更新已登记的 photos 行 + 写 faces。 - * 导入时只登记 photos 行(processedAt=null)；这里由后台 PhotoWorker 闲时调用，不阻塞 UI。 + * 单张照片的分层处理： + * - Tier-0 [faceStage]：人脸检测 + 512d embedding + GPS。秒级，全相册先铺满（按人/时间可搜）。 + * - Tier-1 [captionStage]：Qwen-VL 场景描述 + caption 语义向量。慢，按优先级队列充电时跑。 + * + * 两段都从来源（imported 私有文件 / gallery content uri）解码，互不依赖，可在不同时机/约束下跑。 */ class PhotoProcessor( + private val ctx: Context, private val repo: Repository, - private val face: FaceEngine, - private val vision: VisionClient, ) { - /** 处理一张已登记但未处理的照片(processedAt=null)。 */ - suspend fun process(photo: Photo) { + /** Tier-0：人脸 + GPS → faces 行 + 自动认人，scanState 推到 1。 */ + suspend fun faceStage(photo: Photo, face: FaceEngine) { withContext(Dispatchers.IO) { - val bmp = ImageUtil.decodeOriented(photo.path) ?: run { - Log.w(TAG, "decode failed: ${photo.path}") - repo.photos.markProcessed(photo.id, 0, System.currentTimeMillis()) + val bmp = ImageUtil.decode(ctx, photo) ?: run { + Log.w(TAG, "decode failed (face): ${photo.path}") + repo.photos.markFaceDone(photo.id, 0, null, null) return@withContext } - val faces = try { face.analyze(bmp) } catch (t: Throwable) { Log.w(TAG, "face analyze failed: ${t.message}"); emptyList() } - val caption = try { - vision.caption(ImageUtil.toJpeg(bmp)) - } catch (t: Throwable) { - Log.w(TAG, "caption failed: ${t.message}"); null - } bmp.recycle() - if (caption != null) repo.photos.updateCaption(photo.id, caption, "vl") + val (lat, lon) = ImageUtil.readGps(ctx, photo) // 匹配 gallery = 微信头像锚点 + 已标注脸样本。 val gallery = FaceMatcher.gallery(repo) @@ -55,8 +51,38 @@ class PhotoProcessor( ), ) } - repo.photos.markProcessed(photo.id, faces.size, System.currentTimeMillis()) - Log.i(TAG, "✓ ${File(photo.path).name}: ${faces.size} 脸, caption=${caption?.take(30)}") + repo.photos.markFaceDone(photo.id, faces.size, lat, lon) + } + } + + /** Tier-1：VL 场景描述 + caption 语义向量 → caption/captionEmbedding，scanState 推到 2。 */ + suspend fun captionStage(photo: Photo, vision: VisionClient, embed: EmbedClient?) { + withContext(Dispatchers.IO) { + val bmp = ImageUtil.decode(ctx, photo) ?: run { + Log.w(TAG, "decode failed (caption): ${photo.path}") + repo.photos.markCaptionSkipped(photo.id, System.currentTimeMillis()) + return@withContext + } + val caption = try { + vision.caption(ImageUtil.toJpeg(bmp, maxDim = VL_MAX_DIM)).takeIf { it.isNotBlank() } + } catch (t: Throwable) { + Log.w(TAG, "caption failed: ${t.message}"); null + } + bmp.recycle() + + // caption 出来了就顺手算语义向量（embed 端点没配/失败就只存文本，不卡进度）。 + val emb = if (caption != null && embed != null) { + try { + ImageUtil.packEmbedding(embed.embed(caption)) + } catch (t: Throwable) { + Log.w(TAG, "embed failed: ${t.message}"); null + } + } else { + null + } + + repo.photos.markCaptionDone(photo.id, caption, caption?.let { "vl" }, emb, System.currentTimeMillis()) + Log.i(TAG, "✓ photo#${photo.id}: caption=${caption?.take(30)} emb=${emb != null}") } } @@ -64,5 +90,7 @@ class PhotoProcessor( private const val TAG = "PhotoProcessor" /** 同人余弦阈值（实测同人 0.96 / 异人 0.13，取 0.45 稳）。 */ const val THRESH = 0.45f + /** 喂 VL 的最长边：448 比 512 再省些 vision token，粗粒度场景描述精度几乎不掉。 */ + const val VL_MAX_DIM = 448 } } diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/PhotoScheduler.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/PhotoScheduler.kt index d8213d9..4f8094c 100644 --- a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/PhotoScheduler.kt +++ b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/PhotoScheduler.kt @@ -12,40 +12,67 @@ import androidx.work.WorkManager import java.util.concurrent.TimeUnit /** - * 相册照片处理调度（与纪要 SummaryScheduler 同构）： - * - schedulePeriodic：每 15min（WorkManager 硬下限）扫一遍未处理照片，闲时跑。 - * - runNow：导入后触发一次，让新照片尽快在后台开始处理（不阻塞导入 UI）。 + * 相册分层处理调度： + * - Tier-0（人脸+扫描，PhotoFaceWorker）：每 15min，只要不低电就跑——快，尽快铺满全相册人脸/时间线。 + * - Tier-1（VL 场景描述，PhotoVlWorker）：每 15min，**充电 + 不低电**才跑——慢、耗电，长期后台补。 + * - runNow / scanNow：导入或进相册页时手动催一次（不阻塞 UI）。 */ object PhotoScheduler { - private const val PERIODIC_TAG = "lifelog-photo-periodic" - private const val ONE_SHOT_TAG = "lifelog-photo-once" + private const val FACE_PERIODIC = "lifelog-photo-face-periodic" + private const val VL_PERIODIC = "lifelog-photo-vl-periodic" + private const val FACE_ONCE = "lifelog-photo-face-once" + private const val VL_ONCE = "lifelog-photo-vl-once" fun schedulePeriodic(ctx: Context) { - val constraints = Constraints.Builder() - .setRequiredNetworkType(NetworkType.NOT_REQUIRED) // VL 在本机 - .setRequiresBatteryNotLow(true) + val wm = WorkManager.getInstance(ctx) + + val faceReq = PeriodicWorkRequestBuilder(15, TimeUnit.MINUTES) + .setConstraints( + Constraints.Builder() + .setRequiredNetworkType(NetworkType.NOT_REQUIRED) + .setRequiresBatteryNotLow(true) + .build(), + ) + .setBackoffCriteria(BackoffPolicy.LINEAR, 1, TimeUnit.MINUTES) .build() - val request = PeriodicWorkRequestBuilder(15, TimeUnit.MINUTES) - .setConstraints(constraints) + wm.enqueueUniquePeriodicWork(FACE_PERIODIC, ExistingPeriodicWorkPolicy.UPDATE, faceReq) + + val vlReq = PeriodicWorkRequestBuilder(15, TimeUnit.MINUTES) + .setConstraints( + Constraints.Builder() + .setRequiredNetworkType(NetworkType.NOT_REQUIRED) + .setRequiresBatteryNotLow(true) + .setRequiresCharging(true) // 全量 VL 很烧电，只在充电时推进 + .build(), + ) .setBackoffCriteria(BackoffPolicy.LINEAR, 1, TimeUnit.MINUTES) .build() - WorkManager.getInstance(ctx).enqueueUniquePeriodicWork( - PERIODIC_TAG, ExistingPeriodicWorkPolicy.UPDATE, request, - ) + wm.enqueueUniquePeriodicWork(VL_PERIODIC, ExistingPeriodicWorkPolicy.UPDATE, vlReq) } - /** 导入后调一次，让后台尽快开工。已在跑则忽略(KEEP)。 */ + /** 导入后催一次：先铺人脸（不要求充电），VL 留给周期/充电时。 */ fun runNow(ctx: Context) { - val constraints = Constraints.Builder() - .setRequiresBatteryNotLow(true) + val req = OneTimeWorkRequestBuilder() + .setConstraints(Constraints.Builder().setRequiresBatteryNotLow(true).build()) + .setBackoffCriteria(BackoffPolicy.LINEAR, 1, TimeUnit.MINUTES) .build() - val request = OneTimeWorkRequestBuilder() - .setConstraints(constraints) - .addTag(ONE_SHOT_TAG) + WorkManager.getInstance(ctx).enqueueUniqueWork(FACE_ONCE, ExistingWorkPolicy.KEEP, req) + } + + /** 进相册页催一次全相册扫描 + Tier-0（同 runNow，仅语义更明确）。 */ + fun scanNow(ctx: Context) = runNow(ctx) + + /** 手动催一次 VL（设置/相册页「立即理解」按钮用，绕开充电约束）。 */ + fun runVlNow(ctx: Context) { + val req = OneTimeWorkRequestBuilder() + .setConstraints(Constraints.Builder().setRequiresBatteryNotLow(true).build()) .setBackoffCriteria(BackoffPolicy.LINEAR, 1, TimeUnit.MINUTES) .build() - WorkManager.getInstance(ctx).enqueueUniqueWork( - ONE_SHOT_TAG, ExistingWorkPolicy.KEEP, request, - ) + WorkManager.getInstance(ctx).enqueueUniqueWork(VL_ONCE, ExistingWorkPolicy.KEEP, req) + } + + /** 停止 VL：取消手动的一次性 VL（周期由 paused 标志另行 gate）。 */ + fun cancelVl(ctx: Context) { + WorkManager.getInstance(ctx).cancelUniqueWork(VL_ONCE) } } diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/PhotoVlState.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/PhotoVlState.kt new file mode 100644 index 0000000..5b0ffbf --- /dev/null +++ b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/PhotoVlState.kt @@ -0,0 +1,40 @@ +package com.lemon.lifelog.vision + +import android.content.Context +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow + +/** + * Tier-1 VL 处理的运行态（给相册 UI：进度/暂停/停止 + 单张处理指示）。 + * 跟 RecorderState 同形——纯进程内 StateFlow，worker 推、UI 收。paused 持久化到 prefs， + * 这样周期/充电时的自动 worker 也能尊重「暂停」。 + */ +object PhotoVlState { + /** VL worker 是否正在循环处理（拿到锁那个）。 */ + val running = MutableStateFlow(false) + + /** 当前正在出场景描述的照片 id（null = 没有）。UI 在对应格上转圈。 */ + val currentPhotoId = MutableStateFlow(null) + + private val _paused = MutableStateFlow(false) + + /** 用户「暂停」标志（持久化）。worker 每张前检查；为 true 时不取新照片。 */ + val paused: StateFlow = _paused + + private const val PREFS = "photo_vl" + private const val KEY_PAUSED = "paused" + + /** App 启动时把持久化的暂停标志载入 StateFlow（在 LifelogApp.onCreate 调）。 */ + fun init(ctx: Context) { + _paused.value = ctx.getSharedPreferences(PREFS, Context.MODE_PRIVATE).getBoolean(KEY_PAUSED, false) + } + + fun setPaused(ctx: Context, v: Boolean) { + ctx.getSharedPreferences(PREFS, Context.MODE_PRIVATE).edit().putBoolean(KEY_PAUSED, v).apply() + _paused.value = v + } + + /** worker 读这个（持久化值为准，跨进程/重启可靠）。 */ + fun isPaused(ctx: Context): Boolean = + ctx.getSharedPreferences(PREFS, Context.MODE_PRIVATE).getBoolean(KEY_PAUSED, false) +} diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/PhotoVlWorker.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/PhotoVlWorker.kt new file mode 100644 index 0000000..addcace --- /dev/null +++ b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/PhotoVlWorker.kt @@ -0,0 +1,69 @@ +package com.lemon.lifelog.vision + +import android.content.Context +import android.util.Log +import androidx.work.CoroutineWorker +import androidx.work.WorkerParameters +import com.lemon.lifelog.chat.ChatConfig +import com.lemon.lifelog.data.Repository +import java.util.concurrent.atomic.AtomicBoolean + +/** + * Tier-1 worker：给已铺人脸的照片（scanState=1）按优先级跑 Qwen-VL 场景描述 + caption 语义向量。 + * + * 慢（~10-40s/张，打本地 llama-server），所以：充电 + 不低电才跑、优先级队列（有人脸>近期）、 + * 单轮 7min 预算 + retry。全相册全量 VL 是个长期后台任务，进度在相册页可见。 + * + * 并发互斥：periodic + 手动 + retry 可能并发，多 worker 打同一 llama-server 会 502。 + */ +class PhotoVlWorker(ctx: Context, params: WorkerParameters) : CoroutineWorker(ctx, params) { + + override suspend fun doWork(): Result { + // 用户暂停时彻底不跑（含周期/充电触发）。 + if (PhotoVlState.isPaused(applicationContext)) return Result.success() + if (!isRunning.compareAndSet(false, true)) return Result.success() + PhotoVlState.running.value = true + try { + val repo = Repository(applicationContext) + val pending = repo.photos.pendingCaption() + if (pending.isEmpty()) return Result.success() + + val base = ChatConfig.localBase(applicationContext) + val vision = VisionClient(base) + val embed = EmbedClient(base, ChatConfig.modelEmbed(applicationContext)) + val proc = PhotoProcessor(applicationContext, repo) + val deadline = System.currentTimeMillis() + RUN_BUDGET_MS + var done = 0 + for (p in pending) { + if (System.currentTimeMillis() > deadline) break + if (PhotoVlState.isPaused(applicationContext)) break // 暂停：当前张不再开新的 + PhotoVlState.currentPhotoId.value = p.id + try { + proc.captionStage(p, vision, embed) + } catch (t: Throwable) { + Log.w(TAG, "captionStage failed ${p.path}: ${t.message}") + repo.photos.markCaptionSkipped(p.id, System.currentTimeMillis()) // 别卡队列 + } + done++ + } + Log.i(TAG, "tier1 vl: $done / ${pending.size}") + // 暂停时别 retry（否则一直重排）；正常没跑完才 retry。 + val paused = PhotoVlState.isPaused(applicationContext) + return if (!paused && done < pending.size) Result.retry() else Result.success() + } catch (t: Throwable) { + Log.e(TAG, "worker failed", t) + return Result.retry() + } finally { + PhotoVlState.currentPhotoId.value = null + PhotoVlState.running.value = false + isRunning.set(false) + } + } + + companion object { + const val TAG = "PhotoVlWorker" + /** 单轮预算 7min（VL ~10-40s/张），避开 WorkManager ~10min 上限。 */ + const val RUN_BUDGET_MS = 7 * 60 * 1000L + private val isRunning = AtomicBoolean(false) + } +} diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/PhotoWorker.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/PhotoWorker.kt deleted file mode 100644 index 2d5a474..0000000 --- a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/PhotoWorker.kt +++ /dev/null @@ -1,61 +0,0 @@ -package com.lemon.lifelog.vision - -import android.content.Context -import android.util.Log -import androidx.work.CoroutineWorker -import androidx.work.WorkerParameters -import com.lemon.lifelog.chat.ChatConfig -import com.lemon.lifelog.data.Repository -import java.util.concurrent.atomic.AtomicBoolean - -/** - * 后台批处理未处理照片(processedAt=null)：人脸 + Qwen-VL 场景描述。 - * 导入只登记照片行，处理交给这里——闲时/定时跑，不让用户在导入时干等。 - * 单轮有时间预算，超了下轮(retry/periodic)继续，避开 WorkManager 执行上限。 - */ -class PhotoWorker(ctx: Context, params: WorkerParameters) : CoroutineWorker(ctx, params) { - - override suspend fun doWork(): Result { - if (!isRunning.compareAndSet(false, true)) return Result.success() - try { - val repo = Repository(applicationContext) - val pending = repo.photos.unprocessed() - if (pending.isEmpty()) return Result.success() - - val engine = FaceEngine.load(applicationContext) - val vision = VisionClient(ChatConfig.localBase(applicationContext)) - val proc = PhotoProcessor(repo, engine, vision) - val deadline = System.currentTimeMillis() + RUN_BUDGET_MS - var done = 0 - try { - for (p in pending) { - if (System.currentTimeMillis() > deadline) break - try { - proc.process(p) - } catch (t: Throwable) { - Log.w(TAG, "process failed ${p.path}: ${t.message}") - } - done++ - } - } finally { - engine.close() - } - Log.i(TAG, "processed $done / ${pending.size} pending") - // 还有剩的就 retry 继续（带退避），否则结束等下个周期。 - return if (done < pending.size) Result.retry() else Result.success() - } catch (t: Throwable) { - Log.e(TAG, "worker failed", t) - return Result.retry() - } finally { - isRunning.set(false) - } - } - - companion object { - const val TAG = "PhotoWorker" - /** 单轮预算 7min（VL ~45s/张，约 9 张/轮），避开 WorkManager ~10min 上限。 */ - const val RUN_BUDGET_MS = 7 * 60 * 1000L - /** 防并发：periodic + one-shot + retry 可能同时触发，多 worker 打同一 llama-server 会 502。 */ - private val isRunning = AtomicBoolean(false) - } -} -- libgit2 0.21.4