extract_resumes.py
5.61 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
#!/usr/bin/env python3
"""从BOSS直聘本地DB提取简历为JSONL
用法:
python3 extract_resumes.py --job "戏文" --output data/xiwen_resumes.jsonl
python3 extract_resumes.py --job "全栈" --output data/fullstack_resumes.jsonl
python3 extract_resumes.py --account 5889492 --job "戏文" --output data/xiwen_resumes.jsonl
"""
import sqlite3, javaobj, json, argparse, sys, os
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
DATA_DIR = os.path.join('/sdcard/bosszhipin', 'data')
DB_PATH = os.path.join(DATA_DIR, 'hpbr_bosszhipin_client.db')
def flatten(x, depth=0):
if depth > 10: return str(x)[:200]
if isinstance(x, (str, int, float, bool)) or x is None: return x
if isinstance(x, (list, tuple)): return [flatten(v, depth+1) for v in x]
if isinstance(x, dict): return {k: flatten(v, depth+1) for k, v in x.items()}
if hasattr(x, '__dict__'):
return {k: flatten(v, depth+1) for k, v in vars(x).items()
if k not in ('classdesc', 'annotations')}
return str(x)[:200]
def main():
parser = argparse.ArgumentParser(description='提取BOSS直聘简历')
parser.add_argument('--job', default=None, help='岗位关键词(模糊匹配 bossJobPosition)')
parser.add_argument('--job-id', default=None, help='精确按 jobId 提取(优先于 --job)')
parser.add_argument('--account', default=None, help='指定账号 myId(不指定则查所有账号)')
parser.add_argument('--output', required=True, help='输出 JSONL 文件路径')
args = parser.parse_args()
if not args.job and not args.job_id:
print('错误: --job 或 --job-id 至少传一个', file=sys.stderr)
sys.exit(1)
if not os.path.exists(DB_PATH):
print(f'错误: DB不存在 {DB_PATH}', file=sys.stderr)
print('请先运行: bash pull_dbs_local.sh', file=sys.stderr)
sys.exit(1)
con = sqlite3.connect(DB_PATH)
con.row_factory = sqlite3.Row
# 构建查询
if args.job_id:
# 精确 jobId 匹配
query = '''
SELECT m.fromUid, c.friendName, c.bossJobPosition, c.jobId, m.resume
FROM MessageTable m
JOIN Contact c ON c.friendId = m.fromUid
WHERE c.jobId = ?
AND m.resume IS NOT NULL
ORDER BY c.bossJobPosition, c.friendName
'''
params = [args.job_id]
if args.account:
query = '''
SELECT m.fromUid, c.friendName, c.bossJobPosition, c.jobId, m.resume
FROM MessageTable m
JOIN Contact c ON c.friendId = m.fromUid
WHERE c.myId = ? AND c.jobId = ?
AND m.resume IS NOT NULL
ORDER BY c.bossJobPosition, c.friendName
'''
params = [args.account, args.job_id]
rows = con.execute(query, params).fetchall()
elif args.account:
query = '''
SELECT m.fromUid, c.friendName, c.bossJobPosition, c.jobId, m.resume
FROM MessageTable m
JOIN Contact c ON c.friendId = m.fromUid
WHERE c.myId = ? AND c.bossJobPosition LIKE ?
AND m.resume IS NOT NULL
ORDER BY c.bossJobPosition, c.friendName
'''
rows = con.execute(query, (args.account, f'%{args.job}%')).fetchall()
else:
query = '''
SELECT m.fromUid, c.friendName, c.bossJobPosition, c.jobId, m.resume
FROM MessageTable m
JOIN Contact c ON c.friendId = m.fromUid
WHERE c.bossJobPosition LIKE ?
AND m.resume IS NOT NULL
ORDER BY c.bossJobPosition, c.friendName
'''
rows = con.execute(query, (f'%{args.job}%',)).fetchall()
os.makedirs(os.path.dirname(args.output) or '.', exist_ok=True)
with open(args.output, 'w', encoding='utf-8') as f:
for row in rows:
obj = javaobj.loads(row['resume'])
resume = flatten(obj)
if not isinstance(resume, dict):
continue
record = {
'friendId': row['fromUid'],
'name': row['friendName'] or resume.get('userInfo', {}).get('name', ''),
'jobPosition': row['bossJobPosition'],
'jobId': row['jobId'],
'age': resume.get('age', ''),
'education': resume.get('education', ''),
'applyStatus': resume.get('applyStatus', ''),
'city': resume.get('city', ''),
'expSalary': resume.get('expSalary', ''),
'workAge': resume.get('workAge', ''),
'firstText': resume.get('firstText', ''),
'secondText': resume.get('secondText', ''),
'positionName': resume.get('positionName', ''),
'jobSalary': resume.get('jobSalary', ''),
'experiences': [],
}
for exp in resume.get('experiences', []):
if isinstance(exp, dict):
record['experiences'].append({
'type': exp.get('type', 0),
'startDate': exp.get('startDate', ''),
'endDate': exp.get('endDate', ''),
'organization': exp.get('organization', ''),
'occupation': exp.get('occupation', ''),
})
f.write(json.dumps(record, ensure_ascii=False) + '\n')
print(f'导出 {len(rows)} 份简历到 {args.output}')
# 统计
from collections import Counter
edu = Counter(r['education'] for r in [json.loads(l) for l in open(args.output, encoding='utf-8')])
print(f'学历分布: {edu.most_common()}')
con.close()
if __name__ == '__main__':
main()