Commit f8729bee6a8c128f1b85e78b18ce706b02fb35e9
1 parent
bf83fe61
feat(lifelog/scripts): 历史录音一次性重处理脚本(端上本地模型重跑)
reprocess-history.sh：把所有旧逐句 .opus 按静音5min/10min封顶合并成「块」
(ffmpeg 解码→blocks/<ts>/{audio.pcm,manifest})，清空旧 segments/transcripts/
speaker_profiles，推回设备。重启 RecorderService 后由端上本地模型(pyannote+
paraformer+campplus)经现有 block 崩溃恢复管线整体重跑声纹+文字——识别全程不碰云/Claude，
脚本只做机械重打包+清库。自动备份原 DB。
实测：1078 段 → 89 块。
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Showing
1 changed file
with
104 additions
and
0 deletions
lifelog-recorder/scripts/reprocess-history.sh
0 → 100755
| 1 | +#!/usr/bin/env bash | |
| 2 | +# 一次性：把所有旧的逐句 .opus 重新合并成「块」，清空旧识别(segments/transcripts/ | |
| 3 | +# speaker_profiles)，让设备上 RecorderService 用**本地模型**(pyannote+paraformer+campplus) | |
| 4 | +# 按新 block 管线整体重跑声纹+文字。 | |
| 5 | +# | |
| 6 | +# 本脚本只做机械活：解码+重打包成 blocks/<ts>/{audio.pcm,manifest.json} + 清 DB。 | |
| 7 | +# 真正的识别 100% 在 Pixel 端上模型跑，不用任何云/Claude。 | |
| 8 | +# | |
| 9 | +# 用法： bash reprocess-history.sh | |
| 10 | +# 跑完后：打开 app → 录音 → 开始录音，RecorderService 会自动处理所有块。 | |
| 11 | +set -euo pipefail | |
| 12 | + | |
| 13 | +PKG=com.lemon.lifelog | |
| 14 | +DBR=/data/data/$PKG/databases/lifelog.db | |
| 15 | +SEGR=/data/data/$PKG/files/segments | |
| 16 | +BLKR=/data/data/$PKG/files/blocks | |
| 17 | +GAP_MS=$((5*60*1000)) # 静音≥5min 切块 | |
| 18 | +CAP_MS=$((10*60*1000)) # 块≥10min 封顶(控内存/单次 diarize 时长) | |
| 19 | +SR=16000 | |
| 20 | + | |
| 21 | +WORK=$(mktemp -d /tmp/reprocess.XXXXXX) | |
| 22 | +echo "工作目录: $WORK" | |
| 23 | + | |
| 24 | +echo "[1/8] 停 app" | |
| 25 | +adb shell am force-stop $PKG | |
| 26 | + | |
| 27 | +echo "[2/8] 备份 + 拉 DB" | |
| 28 | +TS=$(date +%Y%m%d-%H%M%S) | |
| 29 | +adb shell "su -c 'cp $DBR /sdcard/_bk.db'"; adb pull /sdcard/_bk.db "$WORK/backup-$TS.db" >/dev/null; adb shell "su -c 'rm /sdcard/_bk.db'" | |
| 30 | +adb shell "su -c 'cp ${DBR}* /sdcard/ 2>/dev/null'" | |
| 31 | +for f in lifelog.db lifelog.db-wal lifelog.db-shm; do adb pull /sdcard/$f "$WORK/$f" >/dev/null 2>&1 || true; done | |
| 32 | +adb shell "su -c 'rm /sdcard/lifelog.db* 2>/dev/null'" | |
| 33 | +sqlite3 "$WORK/lifelog.db" "PRAGMA wal_checkpoint(TRUNCATE);" >/dev/null 2>&1 || true | |
| 34 | +echo " 备份: $WORK/backup-$TS.db" | |
| 35 | + | |
| 36 | +echo "[3/8] 拉所有 .opus" | |
| 37 | +adb shell "su -c 'rm -rf /sdcard/_seg && cp -r $SEGR /sdcard/_seg'" | |
| 38 | +adb pull /sdcard/_seg "$WORK/segments" >/dev/null 2>&1 | |
| 39 | +adb shell "su -c 'rm -rf /sdcard/_seg'" | |
| 40 | + | |
| 41 | +echo "[4/8] 分块 + 解码重打包(ffmpeg)" | |
| 42 | +python3 - "$WORK/lifelog.db" "$WORK/segments" "$WORK/blocks" "$GAP_MS" "$CAP_MS" "$SR" <<'PY' | |
| 43 | +import sqlite3, subprocess, os, json, sys | |
| 44 | +db, segdir, outdir, GAP, CAP, SR = sys.argv[1], sys.argv[2], sys.argv[3], int(sys.argv[4]), int(sys.argv[5]), int(sys.argv[6]) | |
| 45 | +con = sqlite3.connect(db) | |
| 46 | +rows = con.execute("SELECT id,startTs,durationMs,audioPath FROM segments ORDER BY startTs").fetchall() | |
| 47 | +# 按静音间隔 + 时长封顶分块 | |
| 48 | +blocks, cur = [], [] | |
| 49 | +for r in rows: | |
| 50 | + if cur: | |
| 51 | + prev = cur[-1] | |
| 52 | + prev_end = prev[1] + prev[2] | |
| 53 | + span = r[1] - cur[0][1] | |
| 54 | + if (r[1] - prev_end) >= GAP or span >= CAP: | |
| 55 | + blocks.append(cur); cur = [] | |
| 56 | + cur.append(r) | |
| 57 | +if cur: blocks.append(cur) | |
| 58 | + | |
| 59 | +os.makedirs(outdir, exist_ok=True) | |
| 60 | +made = 0 | |
| 61 | +for b in blocks: | |
| 62 | + start = b[0][1] | |
| 63 | + bdir = os.path.join(outdir, str(start)); os.makedirs(bdir, exist_ok=True) | |
| 64 | + pcm = bytearray(); runs = []; off = 0 | |
| 65 | + for seg in b: | |
| 66 | + rel = seg[3].split('/files/segments/')[-1] | |
| 67 | + local = os.path.join(segdir, rel) | |
| 68 | + if not os.path.exists(local): | |
| 69 | + continue | |
| 70 | + raw = subprocess.run(["ffmpeg","-v","quiet","-i",local,"-ar",str(SR),"-ac","1","-f","s16le","-"], | |
| 71 | + capture_output=True).stdout | |
| 72 | + n = len(raw)//2 | |
| 73 | + if n == 0: | |
| 74 | + continue | |
| 75 | + runs.append({"ts": seg[1], "off": off, "len": n}); off += n; pcm += raw | |
| 76 | + if off == 0: | |
| 77 | + os.rmdir(bdir); continue | |
| 78 | + open(os.path.join(bdir, "audio.pcm"), "wb").write(pcm) | |
| 79 | + json.dump({"start": start, "sr": SR, "closed": True, "runs": runs}, open(os.path.join(bdir, "manifest.json"), "w")) | |
| 80 | + made += 1 | |
| 81 | + print(f" block {start}: {len(b)} segs -> {off/SR:.0f}s speech") | |
| 82 | +print(f" => {made} blocks from {len(rows)} segments") | |
| 83 | +PY | |
| 84 | + | |
| 85 | +echo "[5/8] 清空旧识别(segments/transcripts/speaker_profiles)" | |
| 86 | +sqlite3 "$WORK/lifelog.db" "DELETE FROM transcripts; DELETE FROM segments; DELETE FROM speaker_profiles; PRAGMA wal_checkpoint(TRUNCATE); VACUUM;" | |
| 87 | +sqlite3 "$WORK/lifelog.db" "PRAGMA integrity_check;" | head -1 | |
| 88 | + | |
| 89 | +echo "[6/8] 推块到设备 + 清旧 .opus + 清旧 blocks" | |
| 90 | +APP_UID=$(adb shell "su -c 'stat -c %u /data/data/$PKG'" | tr -d '\r') | |
| 91 | +echo " app uid=$APP_UID" | |
| 92 | +adb shell "su -c 'rm -rf $BLKR; mkdir -p $BLKR'" | |
| 93 | +adb shell "su -c 'rm -rf $SEGR'" # 旧逐句 .opus 全删(内容已并进块)；RecorderService 会重建并写新块 .opus | |
| 94 | +adb push "$WORK/blocks" /sdcard/_blk >/dev/null | |
| 95 | +adb shell "su -c 'cp -r /sdcard/_blk/* $BLKR/ && chown -R $APP_UID:$APP_UID $BLKR && chcon -R u:object_r:app_data_file:s0 $BLKR; rm -rf /sdcard/_blk'" | |
| 96 | +echo " pushed $(adb shell "su -c 'ls $BLKR | wc -l'" | tr -d '\r') blocks" | |
| 97 | + | |
| 98 | +echo "[7/8] 推回清空后的 DB" | |
| 99 | +adb push "$WORK/lifelog.db" /sdcard/_db >/dev/null | |
| 100 | +adb shell "su -c 'cp /sdcard/_db $DBR && rm -f ${DBR}-wal ${DBR}-shm && chown $APP_UID:$APP_UID $DBR && chcon u:object_r:app_data_file:s0 $DBR; rm -f /sdcard/_db'" | |
| 101 | + | |
| 102 | +echo "[8/8] 完成。现在打开 app → 录音 → 开始录音，端上本地模型会自动重跑所有块。" | |
| 103 | +echo " 监控： adb logcat -s BlockProcessor:I SpeakerClusterer:I" | |
| 104 | +echo " 备份在： $WORK/backup-$TS.db" | ... | ... |