From 906306c39e98b0de87d12ca035b46d3a1351f366 Mon Sep 17 00:00:00 2001 From: yibo Date: Fri, 5 Jun 2026 16:36:15 +0800 Subject: [PATCH] feat(lifelog/photolog): v0 引擎+管线 — 端侧人脸识别 + Qwen-VL 描述 --- lifelog-recorder/app/build.gradle.kts | 13 ++++++++++++- lifelog-recorder/app/src/main/assets/face/w600k_mbf.tflite | Bin 0 -> 6848236 bytes lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/LifelogApp.kt | 2 ++ lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/AppDatabase.kt | 25 +++++++++++++++++++++++-- lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/Daos.kt | 120 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/Entities.kt | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/Repository.kt | 3 +++ lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/FaceEngine.kt | 186 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/FaceMatcher.kt | 38 ++++++++++++++++++++++++++++++++++++++ lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/ImageUtil.kt | 86 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/PhotoProcessor.kt | 76 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/PhotoScheduler.kt | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/PhotoWorker.kt | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/VisionClient.kt | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 14 files changed, 773 insertions(+), 3 deletions(-) create mode 100644 lifelog-recorder/app/src/main/assets/face/w600k_mbf.tflite create mode 100644 lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/FaceEngine.kt create mode 100644 lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/FaceMatcher.kt create mode 100644 lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/ImageUtil.kt create mode 100644 lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/PhotoProcessor.kt create mode 100644 lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/PhotoScheduler.kt create mode 100644 lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/PhotoWorker.kt create mode 100644 lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/VisionClient.kt diff --git a/lifelog-recorder/app/build.gradle.kts b/lifelog-recorder/app/build.gradle.kts index ce78a34..7763583 100644 --- a/lifelog-recorder/app/build.gradle.kts +++ b/lifelog-recorder/app/build.gradle.kts @@ -48,8 +48,9 @@ android { } androidResources { - // Keep ONNX models uncompressed so sherpa-onnx can mmap them directly from assets. + // 模型保持不压缩，sherpa-onnx / TFLite 才能直接从 assets mmap。 noCompress.add("onnx") + noCompress.add("tflite") } buildTypes { @@ -103,4 +104,14 @@ dependencies { implementation(libs.work.runtime.ktx) implementation(libs.okhttp) implementation(libs.coroutines.android) + + // 人脸：ML Kit 检测+5关键点（端侧，与 sherpa 的 onnxruntime 零冲突） + // + TFLite MobileFaceNet(ArcFace) embedding。不能用 onnxruntime-android—— + // 它和 sherpa 自带的 libonnxruntime.so(自定义符号版本)无法共存。 + implementation("com.google.mlkit:face-detection:16.1.7") + implementation("org.tensorflow:tensorflow-lite:2.16.1") + // EXIF：照片拍摄时间 + GPS。 + implementation("androidx.exifinterface:exifinterface:1.3.7") + // 相册图片异步加载/缓存。 + implementation("io.coil-kt:coil-compose:2.7.0") } diff --git a/lifelog-recorder/app/src/main/assets/face/w600k_mbf.tflite b/lifelog-recorder/app/src/main/assets/face/w600k_mbf.tflite new file mode 100644 index 0000000..2c63543 Binary files /dev/null and b/lifelog-recorder/app/src/main/assets/face/w600k_mbf.tflite differ diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/LifelogApp.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/LifelogApp.kt index 44a3d7b..db44ede 100644 --- a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/LifelogApp.kt +++ b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/LifelogApp.kt @@ -21,5 +21,7 @@ class LifelogApp : Application() { // 纪要 + diarization periodic worker（各自 15min，UPDATE）。 SummaryScheduler.schedulePeriodic(this) DiarizationScheduler.schedulePeriodic(this) + // 相册：未处理照片的周期后台处理（人脸+VL），空闲时跑，不阻塞导入。 + com.lemon.lifelog.vision.PhotoScheduler.schedulePeriodic(this) } } diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/AppDatabase.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/AppDatabase.kt index ab58a2e..e5d872c 100644 --- a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/AppDatabase.kt +++ b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/AppDatabase.kt @@ -16,8 +16,11 @@ import androidx.sqlite.db.SupportSQLiteDatabase TranscriptFts::class, ChatConversation::class, ChatMessage::class, + Person::class, + Photo::class, + Face::class, ], - version = 3, + version = 4, exportSchema = false, ) abstract class AppDatabase : RoomDatabase() { @@ -26,6 +29,9 @@ abstract class AppDatabase : RoomDatabase() { abstract fun summaryDao(): SummaryDao abstract fun speakerDao(): SpeakerDao abstract fun chatDao(): ChatDao + abstract fun personDao(): PersonDao + abstract fun photoDao(): PhotoDao + abstract fun faceDao(): FaceDao companion object { @Volatile @@ -43,6 +49,21 @@ abstract class AppDatabase : RoomDatabase() { } } + /** v0.7 photolog v0：新增 persons / photos / faces 三表。纯增量。 */ + private val MIGRATION_3_4 = object : Migration(3, 4) { + override fun migrate(db: SupportSQLiteDatabase) { + db.execSQL( + "CREATE TABLE IF NOT EXISTS `persons` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `name` TEXT NOT NULL, `voiceSpeakerId` INTEGER, `contactRef` TEXT, `avatarPath` TEXT, `createdAt` INTEGER NOT NULL)", + ) + db.execSQL( + "CREATE TABLE IF NOT EXISTS `photos` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `path` TEXT NOT NULL, `takenTs` INTEGER NOT NULL, `lat` REAL, `lon` REAL, `caption` TEXT, `captionModel` TEXT, `faceCount` INTEGER NOT NULL, `processedAt` INTEGER)", + ) + db.execSQL( + "CREATE TABLE IF NOT EXISTS `faces` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `photoId` INTEGER NOT NULL, `left` REAL NOT NULL, `top` REAL NOT NULL, `right` REAL NOT NULL, `bottom` REAL NOT NULL, `detScore` REAL NOT NULL, `embedding` BLOB, `personId` INTEGER, `personSim` REAL)", + ) + } + } + fun get(ctx: Context): AppDatabase = instance ?: synchronized(this) { instance ?: Room.databaseBuilder( @@ -50,7 +71,7 @@ abstract class AppDatabase : RoomDatabase() { AppDatabase::class.java, "lifelog.db", ) - .addMigrations(MIGRATION_2_3) + .addMigrations(MIGRATION_2_3, MIGRATION_3_4) // 不再 fallbackToDestructiveMigration：以后改 schema 必须写 Migration， // 否则启动直接抛 IllegalStateException（fail-loud），绝不静默清掉真实录音。 .build() diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/Daos.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/Daos.kt index 35e9b40..7e81c2a 100644 --- a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/Daos.kt +++ b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/Daos.kt @@ -38,6 +38,10 @@ interface SegmentDao { @Query("UPDATE segments SET speakerId = :speakerId, speakerSim = :sim WHERE id = :id") suspend fun updateSpeaker(id: Long, speakerId: Long?, sim: Float?) + /** diarization 用干净的 dominant-turn embedding 覆盖实时的混合 embedding。 */ + @Query("UPDATE segments SET embedding = :embedding WHERE id = :id") + suspend fun updateEmbedding(id: Long, embedding: ByteArray) + @Query("DELETE FROM segments WHERE id = :id") suspend fun deleteSegment(id: Long) @@ -192,6 +196,122 @@ interface SummaryDao { suspend fun delete(id: Long) } +/** 照片网格/人物页用的投影行。 */ +data class PhotoRow( + val id: Long, + val path: String, + val takenTs: Long, + val caption: String?, + val faceCount: Int, + val lat: Double?, + val lon: Double?, +) + +@Dao +interface PersonDao { + @Insert + suspend fun insert(p: Person): Long + + @Query("SELECT * FROM persons ORDER BY name ASC") + fun observeAll(): Flow> + + @Query("SELECT * FROM persons ORDER BY name ASC") + suspend fun all(): List + + @Query("SELECT * FROM persons WHERE id = :id") + suspend fun get(id: Long): Person? + + @Query("UPDATE persons SET name = :name WHERE id = :id") + suspend fun updateName(id: Long, name: String) + + /** 融合：把一个声纹 SpeakerProfile 绑到这个人。 */ + @Query("UPDATE persons SET voiceSpeakerId = :speakerId WHERE id = :id") + suspend fun linkVoice(id: Long, speakerId: Long?) + + @Query("DELETE FROM persons WHERE id = :id") + suspend fun delete(id: Long) +} + +@Dao +interface PhotoDao { + @Insert + suspend fun insert(p: Photo): Long + + @Query("SELECT * FROM photos WHERE id = :id") + suspend fun get(id: Long): Photo? + + @Query("SELECT * FROM photos WHERE path = :path LIMIT 1") + suspend fun getByPath(path: String): Photo? + + /** 还没跑过人脸+VL 的照片（处理 worker 取这个）。 */ + @Query("SELECT * FROM photos WHERE processedAt IS NULL ORDER BY takenTs DESC") + suspend fun unprocessed(): List + + /** 未处理照片数（UI 显示"待处理 N"）。 */ + @Query("SELECT COUNT(*) FROM photos WHERE processedAt IS NULL") + fun observeUnprocessedCount(): Flow + + @Query("UPDATE photos SET caption = :caption, captionModel = :model WHERE id = :id") + suspend fun updateCaption(id: Long, caption: String, model: String) + + @Query("UPDATE photos SET faceCount = :n, processedAt = :ts WHERE id = :id") + suspend fun markProcessed(id: Long, n: Int, ts: Long) + + @Query("UPDATE photos SET lat = :lat, lon = :lon WHERE id = :id") + suspend fun updateGeo(id: Long, lat: Double?, lon: Double?) + + @Query( + "SELECT id, path, takenTs, caption, faceCount, lat, lon FROM photos ORDER BY takenTs DESC LIMIT :limit", + ) + fun observeGrid(limit: Int = 500): Flow> + + /** 某个人出现的所有照片（faces.personId join）。 */ + @Query( + """ + SELECT DISTINCT p.id AS id, p.path AS path, p.takenTs AS takenTs, + p.caption AS caption, p.faceCount AS faceCount, p.lat AS lat, p.lon AS lon + FROM photos p INNER JOIN faces f ON f.photoId = p.id + WHERE f.personId = :personId + ORDER BY p.takenTs DESC + """, + ) + fun observePhotosOfPerson(personId: Long): Flow> + + @Query("DELETE FROM photos WHERE id = :id") + suspend fun delete(id: Long) +} + +@Dao +interface FaceDao { + @Insert + suspend fun insert(f: Face): Long + + @Query("SELECT * FROM faces WHERE photoId = :photoId") + suspend fun byPhoto(photoId: Long): List + + @Query("SELECT * FROM faces WHERE photoId = :photoId") + fun observeByPhoto(photoId: Long): Flow> + + /** 所有带 embedding 的脸——matcher/backfill 在 Kotlin 里做余弦（Room 算不了）。 */ + @Query("SELECT * FROM faces WHERE embedding IS NOT NULL") + suspend fun allWithEmbedding(): List + + @Query("SELECT * FROM faces WHERE personId IS NULL AND embedding IS NOT NULL") + suspend fun unassigned(): List + + @Query("UPDATE faces SET personId = :personId, personSim = :sim WHERE id = :id") + suspend fun assignPerson(id: Long, personId: Long?, sim: Float?) + + @Query("SELECT COUNT(*) FROM faces WHERE personId = :personId") + suspend fun countForPerson(personId: Long): Int + + @Query("UPDATE faces SET personId = NULL, personSim = NULL WHERE personId = :personId") + suspend fun clearPersonAssignments(personId: Long) + + @Query("DELETE FROM faces WHERE photoId = :photoId") + suspend fun deleteByPhoto(photoId: Long) +} + @Dao interface ChatDao { @Insert diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/Entities.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/Entities.kt index a0157f8..8c5be18 100644 --- a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/Entities.kt +++ b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/Entities.kt @@ -80,3 +80,54 @@ data class ChatMessage( val model: String? = null, val createdAt: Long, ) + +// ============================================================ photolog (v0) +// 以"人"为中心的身份融合：Person 绑名字 + 声纹(链 SpeakerProfile) + 联系人 + 人脸(faces.personId)。 +// 跟声纹那套同形——人脸只是另一个模态的 embedding。 + +/** 一个"人"。voiceSpeakerId 把声纹和人脸融合到同一身份上。 */ +@Entity(tableName = "persons") +data class Person( + @PrimaryKey(autoGenerate = true) val id: Long = 0, + val name: String, + /** 融合声纹：链到 SpeakerProfile.id，null = 还没绑声音 */ + val voiceSpeakerId: Long? = null, + /** 微信/通讯录联系人引用（如 "wechat:wxid_xxx"），null = 手动建 */ + val contactRef: String? = null, + val avatarPath: String? = null, + val createdAt: Long = System.currentTimeMillis(), +) + +/** 一张照片 + Qwen-VL 场景描述 + EXIF 时间/GPS。 */ +@Entity(tableName = "photos") +data class Photo( + @PrimaryKey(autoGenerate = true) val id: Long = 0, + val path: String, + /** 拍摄时间（EXIF DateTimeOriginal，回退文件时间） */ + val takenTs: Long, + val lat: Double? = null, + val lon: Double? = null, + /** Qwen-VL 场景/活动描述（不含人名——人名由人脸给） */ + val caption: String? = null, + val captionModel: String? = null, + val faceCount: Int = 0, + /** 人脸+VL 处理完成时间，null = 还没处理 */ + val processedAt: Long? = null, +) + +/** 照片里检测到的一张脸：bbox + ArcFace embedding + 归属 Person。结构同 SpeakerProfile 的脸版。 */ +@Entity(tableName = "faces") +data class Face( + @PrimaryKey(autoGenerate = true) val id: Long = 0, + val photoId: Long, + val left: Float, + val top: Float, + val right: Float, + val bottom: Float, + val detScore: Float = 0f, + /** 512维 ArcFace embedding，packed LE bytes (512*4=2048B)，null = 还没算 */ + val embedding: ByteArray? = null, + /** 匹配到的 Person.id，null = 未归属 / 还没匹配 */ + val personId: Long? = null, + val personSim: Float? = null, +) diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/Repository.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/Repository.kt index 962d0da..a4fdc87 100644 --- a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/Repository.kt +++ b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/data/Repository.kt @@ -9,4 +9,7 @@ class Repository(ctx: Context) { val summaries = db.summaryDao() val speakers = db.speakerDao() val chats = db.chatDao() + val persons = db.personDao() + val photos = db.photoDao() + val faces = db.faceDao() } diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/FaceEngine.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/FaceEngine.kt new file mode 100644 index 0000000..5cc8718 --- /dev/null +++ b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/FaceEngine.kt @@ -0,0 +1,186 @@ +package com.lemon.lifelog.vision + +import android.content.Context +import android.graphics.Bitmap +import android.graphics.Canvas +import android.graphics.Matrix +import android.graphics.Paint +import com.google.android.gms.tasks.Tasks +import com.google.mlkit.vision.common.InputImage +import com.google.mlkit.vision.face.FaceDetection +import com.google.mlkit.vision.face.FaceDetector +import com.google.mlkit.vision.face.FaceDetectorOptions +import com.google.mlkit.vision.face.FaceLandmark +import org.tensorflow.lite.Interpreter +import java.io.FileInputStream +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.nio.channels.FileChannel +import kotlin.math.sqrt + +/** 一张检测到的脸：原图坐标 bbox + 5 关键点(已按图像左右排好序)。 */ +data class DetectedFace( + val x1: Float, val y1: Float, val x2: Float, val y2: Float, + val score: Float, + /** 10 个 float，原图坐标，顺序：[左眼,右眼,鼻,左嘴角,右嘴角]（左右=图像左右） */ + val kps: FloatArray, +) + +/** 一张脸 + 它的 512 维 ArcFace embedding(已 L2 归一化)。 */ +data class FaceResult( + val face: DetectedFace, + val embedding: FloatArray, +) + +/** + * On-device 人脸引擎：ML Kit 检测+5关键点 → 5点相似变换对齐到 ArcFace 112×112 模板 + * → TFLite MobileFaceNet(w600k_mbf) 出 512 维 embedding。 + * + * 为什么不用 onnxruntime：sherpa-onnx 自带 libonnxruntime.so 用了自定义符号版本， + * 和微软 onnxruntime-android 无法共存。ML Kit + TFLite 是独立运行时，零冲突。 + * + * embedding 预处理：112×112 对齐脸, (px-127.5)/127.5, RGB, NHWC。TFLite 与 onnx 实测 cos=1.0。 + */ +class FaceEngine private constructor( + private val detector: FaceDetector, + private val rec: Interpreter, +) { + /** 检测 + 给每张脸算 embedding（阻塞，跑在后台线程）。 */ + fun analyze(bitmap: Bitmap): List = + detect(bitmap).map { FaceResult(it, embed(bitmap, it.kps)) } + + /** ML Kit 检测，返回带 5 关键点的脸。阻塞等待，调用方需在后台线程。 */ + fun detect(bitmap: Bitmap): List { + val faces = Tasks.await(detector.process(InputImage.fromBitmap(bitmap, 0))) + val out = ArrayList() + for (f in faces) { + val le = f.getLandmark(FaceLandmark.LEFT_EYE)?.position + val re = f.getLandmark(FaceLandmark.RIGHT_EYE)?.position + val nose = f.getLandmark(FaceLandmark.NOSE_BASE)?.position + val ml = f.getLandmark(FaceLandmark.MOUTH_LEFT)?.position + val mr = f.getLandmark(FaceLandmark.MOUTH_RIGHT)?.position + if (le == null || re == null || nose == null || ml == null || mr == null) continue + // 按 x 排序，杜绝 ML Kit 命名(subject 左右) vs 模板(图像左右)的镜像歧义。 + val eyeL = if (le.x <= re.x) le else re + val eyeR = if (le.x <= re.x) re else le + val mouthL = if (ml.x <= mr.x) ml else mr + val mouthR = if (ml.x <= mr.x) mr else ml + val b = f.boundingBox + out.add( + DetectedFace( + b.left.toFloat(), b.top.toFloat(), b.right.toFloat(), b.bottom.toFloat(), + 1f, + floatArrayOf( + eyeL.x, eyeL.y, eyeR.x, eyeR.y, nose.x, nose.y, + mouthL.x, mouthL.y, mouthR.x, mouthR.y, + ), + ), + ) + } + return out + } + + /** 对齐脸 → MobileFaceNet → 512 维 L2 归一化 embedding。 */ + fun embed(bitmap: Bitmap, kps: FloatArray): FloatArray { + val aligned = align112(bitmap, kps) + val input = ByteBuffer.allocateDirect(112 * 112 * 3 * 4).order(ByteOrder.nativeOrder()) + val px = IntArray(112 * 112) + aligned.getPixels(px, 0, 112, 0, 0, 112, 112) + aligned.recycle() + for (p in px) { + input.putFloat((((p shr 16) and 0xFF) - 127.5f) / 127.5f) // R + input.putFloat((((p shr 8) and 0xFF) - 127.5f) / 127.5f) // G + input.putFloat(((p and 0xFF) - 127.5f) / 127.5f) // B + } + input.rewind() + val out = Array(1) { FloatArray(512) } + rec.run(input, out) + val e = out[0] + var norm = 0f + for (v in e) norm += v * v + norm = sqrt(norm) + if (norm > 0f) for (i in e.indices) e[i] /= norm + return e + } + + /** 5点相似变换对齐到 ArcFace 112×112 模板。 */ + private fun align112(bitmap: Bitmap, kps: FloatArray): Bitmap { + val m = similarityTransform(kps, ARCFACE_DST) + val out = Bitmap.createBitmap(112, 112, Bitmap.Config.ARGB_8888) + val mat = Matrix() + mat.setValues(floatArrayOf(m[0], m[1], m[2], m[3], m[4], m[5], 0f, 0f, 1f)) + Canvas(out).drawBitmap(bitmap, mat, Paint(Paint.FILTER_BITMAP_FLAG)) + return out + } + + fun close() { + runCatching { detector.close() } + runCatching { rec.close() } + } + + companion object { + /** ArcFace 112×112 标准 5 点模板（左眼,右眼,鼻,左嘴角,右嘴角，图像左右）。 */ + private val ARCFACE_DST = floatArrayOf( + 38.2946f, 51.6963f, + 73.5318f, 51.5014f, + 56.0252f, 71.7366f, + 41.5493f, 92.3655f, + 70.7299f, 92.2041f, + ) + + fun load(ctx: Context): FaceEngine { + val opts = FaceDetectorOptions.Builder() + .setPerformanceMode(FaceDetectorOptions.PERFORMANCE_MODE_ACCURATE) + .setLandmarkMode(FaceDetectorOptions.LANDMARK_MODE_ALL) + .setMinFaceSize(0.05f) + .build() + val detector = FaceDetection.getClient(opts) + val model = mapAsset(ctx, "face/w600k_mbf.tflite") + val rec = Interpreter(model, Interpreter.Options().apply { numThreads = 4 }) + return FaceEngine(detector, rec) + } + + /** 余弦相似度（两个已归一化向量直接点积）。 */ + fun cosine(a: FloatArray, b: FloatArray): Float { + var s = 0f + for (i in a.indices) s += a[i] * b[i] + return s + } + + private fun mapAsset(ctx: Context, path: String): ByteBuffer { + val afd = ctx.assets.openFd(path) + FileInputStream(afd.fileDescriptor).use { fis -> + return fis.channel.map(FileChannel.MapMode.READ_ONLY, afd.startOffset, afd.declaredLength) + } + } + } +} + +/** + * 闭式 2D 相似变换(旋转+均匀缩放+平移，无反射) src→dst 最小二乘。 + * 等价 skimage SimilarityTransform / Umeyama。返回 [a,-b,tx, b,a,ty]（2×3 行优先）。 + */ +private fun similarityTransform(src: FloatArray, dst: FloatArray): FloatArray { + val n = 5 + var sx = 0f; var sy = 0f; var dx = 0f; var dy = 0f + for (i in 0 until n) { + sx += src[i * 2]; sy += src[i * 2 + 1] + dx += dst[i * 2]; dy += dst[i * 2 + 1] + } + sx /= n; sy /= n; dx /= n; dy /= n + var num1 = 0f; var num2 = 0f; var den = 0f + for (i in 0 until n) { + val xp = src[i * 2] - sx + val yp = src[i * 2 + 1] - sy + val xP = dst[i * 2] - dx + val yP = dst[i * 2 + 1] - dy + num1 += xP * xp + yP * yp + num2 += yP * xp - xP * yp + den += xp * xp + yp * yp + } + val a = num1 / den + val b = num2 / den + val tx = dx - (a * sx - b * sy) + val ty = dy - (b * sx + a * sy) + return floatArrayOf(a, -b, tx, b, a, ty) +} diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/FaceMatcher.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/FaceMatcher.kt new file mode 100644 index 0000000..9dd4d6d --- /dev/null +++ b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/FaceMatcher.kt @@ -0,0 +1,38 @@ +package com.lemon.lifelog.vision + +import com.lemon.lifelog.data.Repository + +/** + * 人脸归属匹配（与声纹的 SpeakerMatcher 同构）。 + * 标注一张脸到某 Person 后，把所有未归属的脸跟已命名的脸库比余弦，自动回填。 + */ +object FaceMatcher { + private const val THRESH = PhotoProcessor.THRESH + + /** 把 [faceId] 指派给 [personId]，然后回填其它未归属的脸。 */ + suspend fun assignAndBackfill(repo: Repository, faceId: Long, personId: Long) { + repo.faces.assignPerson(faceId, personId, 1f) + backfillAll(repo) + } + + /** + * 对所有未归属的脸：跟已命名脸库做最近邻，余弦 ≥ THRESH 则归属。 + * 反复一轮即可（新归属的脸不会再扩大已命名集合的语义）。 + */ + suspend fun backfillAll(repo: Repository) { + val gallery = repo.faces.allWithEmbedding() + .filter { it.personId != null && it.embedding != null } + .map { it.personId!! to ImageUtil.unpackEmbedding(it.embedding!!) } + if (gallery.isEmpty()) return + for (f in repo.faces.unassigned()) { + val emb = ImageUtil.unpackEmbedding(f.embedding!!) + var bestId: Long? = null + var bestSim = THRESH + for ((pid, g) in gallery) { + val c = FaceEngine.cosine(emb, g) + if (c > bestSim) { bestSim = c; bestId = pid } + } + if (bestId != null) repo.faces.assignPerson(f.id, bestId, bestSim) + } + } +} diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/ImageUtil.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/ImageUtil.kt new file mode 100644 index 0000000..58c9e3d --- /dev/null +++ b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/ImageUtil.kt @@ -0,0 +1,86 @@ +package com.lemon.lifelog.vision + +import android.graphics.Bitmap +import android.graphics.BitmapFactory +import android.graphics.Matrix +import androidx.exifinterface.media.ExifInterface +import java.io.ByteArrayOutputStream +import java.io.File +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.text.SimpleDateFormat +import java.util.Locale +import kotlin.math.min + +/** 照片 EXIF + 解码 + 缩放工具。 */ +object ImageUtil { + data class Meta(val takenTs: Long, val lat: Double?, val lon: Double?) + + private val EXIF_FMT = SimpleDateFormat("yyyy:MM:dd HH:mm:ss", Locale.US) + + fun readMeta(path: String): Meta { + val f = File(path) + return try { + val ex = ExifInterface(path) + val ts = ex.getAttribute(ExifInterface.TAG_DATETIME_ORIGINAL) + ?.let { runCatching { EXIF_FMT.parse(it)?.time }.getOrNull() } ?: f.lastModified() + val ll = ex.latLong // DoubleArray[lat,lon] 或 null + Meta(ts, ll?.get(0), ll?.get(1)) + } catch (t: Throwable) { + Meta(f.lastModified(), null, null) + } + } + + /** 解码并按 EXIF 方向摆正。可选下采样 reqMax(最长边)以省内存。 */ + fun decodeOriented(path: String, reqMax: Int = 2048): Bitmap? { + val opts = BitmapFactory.Options() + opts.inJustDecodeBounds = true + BitmapFactory.decodeFile(path, opts) + var sample = 1 + val longest = maxOf(opts.outWidth, opts.outHeight) + while (longest / sample > reqMax) sample *= 2 + val decoded = BitmapFactory.decodeFile(path, BitmapFactory.Options().apply { inSampleSize = sample }) + ?: return null + val orientation = try { + ExifInterface(path).getAttributeInt(ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL) + } catch (t: Throwable) { + ExifInterface.ORIENTATION_NORMAL + } + val m = Matrix() + when (orientation) { + ExifInterface.ORIENTATION_ROTATE_90 -> m.postRotate(90f) + ExifInterface.ORIENTATION_ROTATE_180 -> m.postRotate(180f) + ExifInterface.ORIENTATION_ROTATE_270 -> m.postRotate(270f) + else -> return decoded + } + val rotated = Bitmap.createBitmap(decoded, 0, 0, decoded.width, decoded.height, m, true) + if (rotated != decoded) decoded.recycle() + return rotated + } + + /** 把 bitmap 压成最长边 ≤ maxDim 的 JPEG（喂 VL，512 左右最快）。 */ + fun toJpeg(bmp: Bitmap, maxDim: Int = 512, quality: Int = 85): ByteArray { + val scale = min(1f, maxDim.toFloat() / maxOf(bmp.width, bmp.height)) + val src = if (scale < 1f) { + Bitmap.createScaledBitmap(bmp, (bmp.width * scale).toInt(), (bmp.height * scale).toInt(), true) + } else { + bmp + } + val out = ByteArrayOutputStream() + src.compress(Bitmap.CompressFormat.JPEG, quality, out) + if (src != bmp) src.recycle() + return out.toByteArray() + } + + /** 512维 float embedding ↔ LE bytes（与声纹一致的打包方式）。 */ + fun packEmbedding(e: FloatArray): ByteArray { + val b = ByteBuffer.allocate(e.size * 4).order(ByteOrder.LITTLE_ENDIAN) + for (v in e) b.putFloat(v) + return b.array() + } + + fun unpackEmbedding(bytes: ByteArray): FloatArray { + val bb = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN) + return FloatArray(bytes.size / 4) { bb.float } + } +} diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/PhotoProcessor.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/PhotoProcessor.kt new file mode 100644 index 0000000..6760171 --- /dev/null +++ b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/PhotoProcessor.kt @@ -0,0 +1,76 @@ +package com.lemon.lifelog.vision + +import android.util.Log +import com.lemon.lifelog.data.Face +import com.lemon.lifelog.data.Photo +import com.lemon.lifelog.data.Repository +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import java.io.File + +/** + * 单张照片处理：人脸(FaceEngine) + 场景描述(VisionClient) → 更新已登记的 photos 行 + 写 faces。 + * 导入时只登记 photos 行(processedAt=null)；这里由后台 PhotoWorker 闲时调用，不阻塞 UI。 + */ +class PhotoProcessor( + private val repo: Repository, + private val face: FaceEngine, + private val vision: VisionClient, +) { + /** 处理一张已登记但未处理的照片(processedAt=null)。 */ + suspend fun process(photo: Photo) { + withContext(Dispatchers.IO) { + val bmp = ImageUtil.decodeOriented(photo.path) ?: run { + Log.w(TAG, "decode failed: ${photo.path}") + repo.photos.markProcessed(photo.id, 0, System.currentTimeMillis()) + return@withContext + } + + val faces = try { + face.analyze(bmp) + } catch (t: Throwable) { + Log.w(TAG, "face analyze failed: ${t.message}"); emptyList() + } + val caption = try { + vision.caption(ImageUtil.toJpeg(bmp)) + } catch (t: Throwable) { + Log.w(TAG, "caption failed: ${t.message}"); null + } + bmp.recycle() + + if (caption != null) repo.photos.updateCaption(photo.id, caption, "vl") + + // 已命名 Person 的人脸样本（personId != null）→ 匹配新脸。 + val gallery = repo.faces.allWithEmbedding() + .filter { it.personId != null && it.embedding != null } + .map { it.personId!! to ImageUtil.unpackEmbedding(it.embedding!!) } + + for (fr in faces) { + var bestId: Long? = null + var bestSim = THRESH + for ((pid, emb) in gallery) { + val c = FaceEngine.cosine(fr.embedding, emb) + if (c > bestSim) { bestSim = c; bestId = pid } + } + repo.faces.insert( + Face( + photoId = photo.id, + left = fr.face.x1, top = fr.face.y1, right = fr.face.x2, bottom = fr.face.y2, + detScore = fr.face.score, + embedding = ImageUtil.packEmbedding(fr.embedding), + personId = bestId, + personSim = if (bestId != null) bestSim else null, + ), + ) + } + repo.photos.markProcessed(photo.id, faces.size, System.currentTimeMillis()) + Log.i(TAG, "✓ ${File(photo.path).name}: ${faces.size} 脸, caption=${caption?.take(30)}") + } + } + + companion object { + private const val TAG = "PhotoProcessor" + /** 同人余弦阈值（实测同人 0.96 / 异人 0.13，取 0.45 稳）。 */ + const val THRESH = 0.45f + } +} diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/PhotoScheduler.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/PhotoScheduler.kt new file mode 100644 index 0000000..d8213d9 --- /dev/null +++ b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/PhotoScheduler.kt @@ -0,0 +1,51 @@ +package com.lemon.lifelog.vision + +import android.content.Context +import androidx.work.BackoffPolicy +import androidx.work.Constraints +import androidx.work.ExistingPeriodicWorkPolicy +import androidx.work.ExistingWorkPolicy +import androidx.work.NetworkType +import androidx.work.OneTimeWorkRequestBuilder +import androidx.work.PeriodicWorkRequestBuilder +import androidx.work.WorkManager +import java.util.concurrent.TimeUnit + +/** + * 相册照片处理调度（与纪要 SummaryScheduler 同构）： + * - schedulePeriodic：每 15min（WorkManager 硬下限）扫一遍未处理照片，闲时跑。 + * - runNow：导入后触发一次，让新照片尽快在后台开始处理（不阻塞导入 UI）。 + */ +object PhotoScheduler { + private const val PERIODIC_TAG = "lifelog-photo-periodic" + private const val ONE_SHOT_TAG = "lifelog-photo-once" + + fun schedulePeriodic(ctx: Context) { + val constraints = Constraints.Builder() + .setRequiredNetworkType(NetworkType.NOT_REQUIRED) // VL 在本机 + .setRequiresBatteryNotLow(true) + .build() + val request = PeriodicWorkRequestBuilder(15, TimeUnit.MINUTES) + .setConstraints(constraints) + .setBackoffCriteria(BackoffPolicy.LINEAR, 1, TimeUnit.MINUTES) + .build() + WorkManager.getInstance(ctx).enqueueUniquePeriodicWork( + PERIODIC_TAG, ExistingPeriodicWorkPolicy.UPDATE, request, + ) + } + + /** 导入后调一次，让后台尽快开工。已在跑则忽略(KEEP)。 */ + fun runNow(ctx: Context) { + val constraints = Constraints.Builder() + .setRequiresBatteryNotLow(true) + .build() + val request = OneTimeWorkRequestBuilder() + .setConstraints(constraints) + .addTag(ONE_SHOT_TAG) + .setBackoffCriteria(BackoffPolicy.LINEAR, 1, TimeUnit.MINUTES) + .build() + WorkManager.getInstance(ctx).enqueueUniqueWork( + ONE_SHOT_TAG, ExistingWorkPolicy.KEEP, request, + ) + } +} diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/PhotoWorker.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/PhotoWorker.kt new file mode 100644 index 0000000..2d5a474 --- /dev/null +++ b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/PhotoWorker.kt @@ -0,0 +1,61 @@ +package com.lemon.lifelog.vision + +import android.content.Context +import android.util.Log +import androidx.work.CoroutineWorker +import androidx.work.WorkerParameters +import com.lemon.lifelog.chat.ChatConfig +import com.lemon.lifelog.data.Repository +import java.util.concurrent.atomic.AtomicBoolean + +/** + * 后台批处理未处理照片(processedAt=null)：人脸 + Qwen-VL 场景描述。 + * 导入只登记照片行，处理交给这里——闲时/定时跑，不让用户在导入时干等。 + * 单轮有时间预算，超了下轮(retry/periodic)继续，避开 WorkManager 执行上限。 + */ +class PhotoWorker(ctx: Context, params: WorkerParameters) : CoroutineWorker(ctx, params) { + + override suspend fun doWork(): Result { + if (!isRunning.compareAndSet(false, true)) return Result.success() + try { + val repo = Repository(applicationContext) + val pending = repo.photos.unprocessed() + if (pending.isEmpty()) return Result.success() + + val engine = FaceEngine.load(applicationContext) + val vision = VisionClient(ChatConfig.localBase(applicationContext)) + val proc = PhotoProcessor(repo, engine, vision) + val deadline = System.currentTimeMillis() + RUN_BUDGET_MS + var done = 0 + try { + for (p in pending) { + if (System.currentTimeMillis() > deadline) break + try { + proc.process(p) + } catch (t: Throwable) { + Log.w(TAG, "process failed ${p.path}: ${t.message}") + } + done++ + } + } finally { + engine.close() + } + Log.i(TAG, "processed $done / ${pending.size} pending") + // 还有剩的就 retry 继续（带退避），否则结束等下个周期。 + return if (done < pending.size) Result.retry() else Result.success() + } catch (t: Throwable) { + Log.e(TAG, "worker failed", t) + return Result.retry() + } finally { + isRunning.set(false) + } + } + + companion object { + const val TAG = "PhotoWorker" + /** 单轮预算 7min（VL ~45s/张，约 9 张/轮），避开 WorkManager ~10min 上限。 */ + const val RUN_BUDGET_MS = 7 * 60 * 1000L + /** 防并发：periodic + one-shot + retry 可能同时触发，多 worker 打同一 llama-server 会 502。 */ + private val isRunning = AtomicBoolean(false) + } +} diff --git a/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/VisionClient.kt b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/VisionClient.kt new file mode 100644 index 0000000..05c86ff --- /dev/null +++ b/lifelog-recorder/app/src/main/kotlin/com/lemon/lifelog/vision/VisionClient.kt @@ -0,0 +1,64 @@ +package com.lemon.lifelog.vision + +import android.util.Base64 +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.RequestBody.Companion.toRequestBody +import org.json.JSONArray +import org.json.JSONObject +import java.io.IOException +import java.util.concurrent.TimeUnit + +/** + * 经 llama-swap(8080) 的 "vl" 模型(Qwen2.5-VL-3B)给一张照片出场景描述。 + * 只描述场景/地点/活动，**不报人名**(人名由人脸识别给)。非流式，批处理用。 + */ +class VisionClient( + private val baseUrl: String, + private val model: String = "vl", +) { + companion object { + private val JSON = "application/json; charset=utf-8".toMediaType() + const val DEFAULT_PROMPT = + "用一两句中文描述这张照片：场景/地点是哪里、在做什么、有哪些主要物体。" + + "不要猜测或编造人物的姓名。" + } + + private val http = OkHttpClient.Builder() + .connectTimeout(10, TimeUnit.SECONDS) + .readTimeout(600, TimeUnit.SECONDS) // 含 llama-swap 切模型 + VL 推理，可能几十秒 + .build() + + @Throws(IOException::class) + fun caption(jpeg: ByteArray, prompt: String = DEFAULT_PROMPT, maxTokens: Int = 220): String { + val b64 = Base64.encodeToString(jpeg, Base64.NO_WRAP) + val payload = JSONObject().apply { + put("model", model) + put("max_tokens", maxTokens) + put("temperature", 0.2) + put("messages", JSONArray().put(JSONObject().apply { + put("role", "user") + put("content", JSONArray().apply { + put(JSONObject().put("type", "text").put("text", prompt)) + put(JSONObject().apply { + put("type", "image_url") + put("image_url", JSONObject().put("url", "data:image/jpeg;base64,$b64")) + }) + }) + })) + } + val req = Request.Builder() + .url("$baseUrl/v1/chat/completions") + .header("Content-Type", "application/json") + .post(payload.toString().toRequestBody(JSON)) + .build() + http.newCall(req).execute().use { resp -> + val body = resp.body?.string().orEmpty() + if (!resp.isSuccessful) throw IOException("vl ${resp.code}: ${body.take(200)}") + val choices = JSONObject(body).optJSONArray("choices") ?: throw IOException("no choices") + if (choices.length() == 0) throw IOException("empty choices") + return choices.getJSONObject(0).optJSONObject("message")?.optString("content", "").orEmpty().trim() + } + } +} -- libgit2 0.21.4