api_config.md
8.39 KB
数据采集配置
各平台 API 配置和示例代码
1. 抖音 Open API 配置
1.1 应用注册
平台: https://open.douyin.com
应用名称: LemonAI Data Harvester
权限申请:
- user_info.baseinfo (用户基本信息)
- video.list (视频列表)
- video.data (视频数据)
- video.comment (评论数据)
1.2 OAuth 流程
// 配置文件
module.exports = {
douyin: {
client_id: process.env.DOUYIN_CLIENT_ID,
client_secret: process.env.DOUYIN_CLIENT_SECRET,
redirect_uri: 'https://lemonai.com/oauth/douyin/callback',
// 授权Scope
scopes: [
'user_info.baseinfo',
'video.list',
'video.data',
'video.comment'
],
// API 基础地址
base_url: 'https://open.douyin.com',
// 端点
endpoints: {
authorize: '/oauth/authorize/',
token: '/oauth/access_token/',
video_list: '/video/list/',
video_data: '/video/data/',
comment_list: '/video/comment/list/'
}
}
};
1.3 示例代码
class DouyinCollector {
constructor(config) {
this.client_id = config.client_id;
this.client_secret = config.client_secret;
this.access_token = null;
}
// 获取授权URL
getAuthUrl() {
const params = new URLSearchParams({
client_key: this.client_id,
redirect_uri: this.redirect_uri,
scope: this.scopes.join(','),
response_type: 'code',
state: 'random_state'
});
return `${this.base_url}/oauth/authorize/?${params}`;
}
// 换取访问令牌
async getAccessToken(code) {
const response = await fetch(`${this.base_url}/oauth/access_token/`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
client_key: this.client_id,
client_secret: this.client_secret,
code: code,
grant_type: 'authorization_code'
})
});
const data = await response.json();
this.access_token = data.data.access_token;
return this.access_token;
}
// 获取视频列表
async getVideoList(page = 1, page_size = 20) {
const response = await fetch(`${this.base_url}/video/list/`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${this.access_token}`
},
body: JSON.stringify({
page: page,
page_size: page_size
})
});
return response.json();
}
// 获取视频详情
async getVideoData(video_id) {
const response = await fetch(`${this.base_url}/video/data/`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${this.access_token}`
},
body: JSON.stringify({
video_id: video_id
})
});
return response.json();
}
// 获取评论列表
async getCommentList(video_id, cursor = 0, count = 20) {
const response = await fetch(`${this.base_url}/video/comment/list/`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${this.access_token}`
},
body: JSON.stringify({
video_id: video_id,
cursor: cursor,
count: count
})
});
return response.json();
}
}
2. B站 Open API 配置
2.1 API 列表
module.exports = {
bilibili: {
base_url: 'https://api.bilibili.com',
// 不需要登录的接口
public_endpoints: {
view: '/x/web-interface/view',
stat: '/x/web-interface/stat',
view_icon: '/x/view/icon'
},
// 需要 cookie 的接口
private_endpoints: {
myinfo: '/x/web-interface/myinfo',
dash: '/x/web-interface/dash',
replies: '/x/v2/reply'
},
// 接口示例
examples: {
// 获取视频信息
videoInfo: 'https://api.bilibili.com/x/web-interface/view?aid=170001'
// 获取视频统计
videoStat: 'https://api.bilibili.com/x/web-interface/stat?aid=170001'
}
}
};
2.2 示例代码
class BilibiliCollector {
constructor(config) {
this.base_url = config.base_url;
thiscookie = config.cookie; // SESSDATA
}
// 获取视频信息
async getVideoInfo(aid) {
const response = await fetch(`${this.base_url}/x/web-interface/view?aid=${aid}`);
return response.json();
}
// 获取视频统计
async getStat(aid) {
const response = await fetch(`${this.base_url}/x/web-interface/stat?aid=${aid}`);
return response.json();
}
// 获取评论列表
async getReplies(aid, page = 1, type = 1) {
const response = await fetch(
`${this.base_url}/x/v2/reply?type=${type}&oid=${aid}&pn=${page}&ps=20`
);
return response.json();
}
// 批量获取 (带缓存)
async batchGetVideoInfo(aidList) {
const results = [];
for (const aid of aidList) {
const info = await this.getVideoInfo(aid);
const stat = await this.getStat(aid);
results.push({ ...info.data, ...stat.data });
await this.sleep(1000); // 避免限流
}
return results;
}
}
3. 定时任务配置
3.1 采集频率
| 平台 | Cron 表达式 | 说明 |
|---|---|---|
| 抖音 | 0 2 * * * |
每日凌晨2点 |
| B站 | 0 3 * * * |
每日凌晨3点 |
| 快手 | 0 4 * * 0 |
每周日凌晨4点 |
3.2 配置示例
// config/cron.json
{
"tasks": [
{
"name": "douyin_daily",
"schedule": "0 2 * * *",
"handler": "collect_douyin",
"enabled": true
},
{
"name": "bilibili_daily",
"schedule": "0 3 * * *",
"handler": "collect_bilibili",
"enabled": true
},
{
"name": "sentiment_analysis",
"schedule": "0 5 * * *",
"handler": "analyze_sentiment",
"enabled": true
}
]
}
4. 数据存储模型
4.1 SQLite 表结构
-- 视频数据表
CREATE TABLE videos (
id INTEGER PRIMARY KEY AUTOINCREMENT,
platform TEXT NOT NULL,
video_id TEXT NOT NULL,
video_title TEXT,
author_id TEXT,
publish_time INTEGER,
views INTEGER DEFAULT 0,
likes INTEGER DEFAULT 0,
comments INTEGER DEFAULT 0,
shares INTEGER DEFAULT 0,
favorites INTEGER DEFAULT 0,
completion_rate REAL,
avg_watch_time REAL,
created_at INTEGER DEFAULT (strftime('%s', 'now')),
updated_at INTEGER DEFAULT (strftime('%s', 'now')),
UNIQUE(platform, video_id)
);
-- 评论数据表
CREATE TABLE comments (
id INTEGER PRIMARY KEY AUTOINCREMENT,
platform TEXT NOT NULL,
video_id TEXT NOT NULL,
comment_id TEXT NOT NULL,
user_id TEXT,
content TEXT,
like_count INTEGER DEFAULT 0,
sentiment TEXT,
sentiment_score REAL,
created_at INTEGER DEFAULT (strftime('%s', 'now')),
UNIQUE(platform, comment_id)
);
-- 采集日志表
CREATE TABLE collect_logs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
platform TEXT NOT NULL,
status TEXT NOT NULL,
count INTEGER DEFAULT 0,
error_message TEXT,
created_at INTEGER DEFAULT (strftime('%s', 'now'))
);
4.2 API 路由
// routes/api.js
router.get('/api/data/videos', async (ctx) => {
const { platform, days = 7 } = ctx.query;
const videos = db.prepare(`
SELECT * FROM videos
WHERE platform = ?
AND created_at > ?
ORDER BY views DESC
LIMIT 100
`).all(platform, Date.now() / 1000 - days * 86400);
ctx.body = { success: true, data: videos };
});
router.get('/api/data/comments', async (ctx) => {
const { video_id, sentiment } = ctx.query;
let query = 'SELECT * FROM comments WHERE video_id = ?';
const params = [video_id];
if (sentiment) {
query += ' AND sentiment = ?';
params.push(sentiment);
}
const comments = db.prepare(query).all(...params);
ctx.body = { success: true, data: comments };
});
5. 环境变量配置
# .env.example
# 抖音开放平台
DOUYIN_CLIENT_ID=your_client_id
DOUYIN_CLIENT_SECRET=your_client_secret
# B站
BILIBILI_COOKIE=your_sessdata_cookie
# 快手
KUAISHOU_PHONE=your_phone
KUAISHOU_PASSWORD=your_password
# 数据库
DB_PATH=./data/harvester.db
# 定时任务
ENABLE_CRON=true
CRON_TIMEZONE=Asia/Shanghai
6. 部署检查清单
- [ ] 注册抖音开放平台开发者
- [ ] 创建应用并获取 client_id/secret
- [ ] 申请相关权限 scope
- [ ] 配置 B站 cookie (SESSDATA)
- [ ] 安装依赖: npm install axios better-sqlite3 node-cron
- [ ] 初始化数据库
- [ ] 配置 .env 文件
- [ ] 测试单次采集
- [ ] 开启定时任务
- [ ] 监控采集日志