# 数据采集配置 > 各平台 API 配置和示例代码 --- ## 1. 抖音 Open API 配置 ### 1.1 应用注册 ``` 平台: https://open.douyin.com 应用名称: LemonAI Data Harvester 权限申请: - user_info.baseinfo (用户基本信息) - video.list (视频列表) - video.data (视频数据) - video.comment (评论数据) ``` ### 1.2 OAuth 流程 ```javascript // 配置文件 module.exports = { douyin: { client_id: process.env.DOUYIN_CLIENT_ID, client_secret: process.env.DOUYIN_CLIENT_SECRET, redirect_uri: 'https://lemonai.com/oauth/douyin/callback', // 授权Scope scopes: [ 'user_info.baseinfo', 'video.list', 'video.data', 'video.comment' ], // API 基础地址 base_url: 'https://open.douyin.com', // 端点 endpoints: { authorize: '/oauth/authorize/', token: '/oauth/access_token/', video_list: '/video/list/', video_data: '/video/data/', comment_list: '/video/comment/list/' } } }; ``` ### 1.3 示例代码 ```javascript class DouyinCollector { constructor(config) { this.client_id = config.client_id; this.client_secret = config.client_secret; this.access_token = null; } // 获取授权URL getAuthUrl() { const params = new URLSearchParams({ client_key: this.client_id, redirect_uri: this.redirect_uri, scope: this.scopes.join(','), response_type: 'code', state: 'random_state' }); return `${this.base_url}/oauth/authorize/?${params}`; } // 换取访问令牌 async getAccessToken(code) { const response = await fetch(`${this.base_url}/oauth/access_token/`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ client_key: this.client_id, client_secret: this.client_secret, code: code, grant_type: 'authorization_code' }) }); const data = await response.json(); this.access_token = data.data.access_token; return this.access_token; } // 获取视频列表 async getVideoList(page = 1, page_size = 20) { const response = await fetch(`${this.base_url}/video/list/`, { method: 'POST', headers: { 'Content-Type': 'application/json', 'Authorization': `Bearer ${this.access_token}` }, body: JSON.stringify({ page: page, page_size: page_size }) }); return response.json(); } // 获取视频详情 async getVideoData(video_id) { const response = await fetch(`${this.base_url}/video/data/`, { method: 'POST', headers: { 'Content-Type': 'application/json', 'Authorization': `Bearer ${this.access_token}` }, body: JSON.stringify({ video_id: video_id }) }); return response.json(); } // 获取评论列表 async getCommentList(video_id, cursor = 0, count = 20) { const response = await fetch(`${this.base_url}/video/comment/list/`, { method: 'POST', headers: { 'Content-Type': 'application/json', 'Authorization': `Bearer ${this.access_token}` }, body: JSON.stringify({ video_id: video_id, cursor: cursor, count: count }) }); return response.json(); } } ``` --- ## 2. B站 Open API 配置 ### 2.1 API 列表 ```javascript module.exports = { bilibili: { base_url: 'https://api.bilibili.com', // 不需要登录的接口 public_endpoints: { view: '/x/web-interface/view', stat: '/x/web-interface/stat', view_icon: '/x/view/icon' }, // 需要 cookie 的接口 private_endpoints: { myinfo: '/x/web-interface/myinfo', dash: '/x/web-interface/dash', replies: '/x/v2/reply' }, // 接口示例 examples: { // 获取视频信息 videoInfo: 'https://api.bilibili.com/x/web-interface/view?aid=170001' // 获取视频统计 videoStat: 'https://api.bilibili.com/x/web-interface/stat?aid=170001' } } }; ``` ### 2.2 示例代码 ```javascript class BilibiliCollector { constructor(config) { this.base_url = config.base_url; thiscookie = config.cookie; // SESSDATA } // 获取视频信息 async getVideoInfo(aid) { const response = await fetch(`${this.base_url}/x/web-interface/view?aid=${aid}`); return response.json(); } // 获取视频统计 async getStat(aid) { const response = await fetch(`${this.base_url}/x/web-interface/stat?aid=${aid}`); return response.json(); } // 获取评论列表 async getReplies(aid, page = 1, type = 1) { const response = await fetch( `${this.base_url}/x/v2/reply?type=${type}&oid=${aid}&pn=${page}&ps=20` ); return response.json(); } // 批量获取 (带缓存) async batchGetVideoInfo(aidList) { const results = []; for (const aid of aidList) { const info = await this.getVideoInfo(aid); const stat = await this.getStat(aid); results.push({ ...info.data, ...stat.data }); await this.sleep(1000); // 避免限流 } return results; } } ``` --- ## 3. 定时任务配置 ### 3.1 采集频率 | 平台 | Cron 表达式 | 说明 | |------|-------------|------| | 抖音 | `0 2 * * *` | 每日凌晨2点 | | B站 | `0 3 * * *` | 每日凌晨3点 | | 快手 | `0 4 * * 0` | 每周日凌晨4点 | ### 3.2 配置示例 ```javascript // config/cron.json { "tasks": [ { "name": "douyin_daily", "schedule": "0 2 * * *", "handler": "collect_douyin", "enabled": true }, { "name": "bilibili_daily", "schedule": "0 3 * * *", "handler": "collect_bilibili", "enabled": true }, { "name": "sentiment_analysis", "schedule": "0 5 * * *", "handler": "analyze_sentiment", "enabled": true } ] } ``` --- ## 4. 数据存储模型 ### 4.1 SQLite 表结构 ```sql -- 视频数据表 CREATE TABLE videos ( id INTEGER PRIMARY KEY AUTOINCREMENT, platform TEXT NOT NULL, video_id TEXT NOT NULL, video_title TEXT, author_id TEXT, publish_time INTEGER, views INTEGER DEFAULT 0, likes INTEGER DEFAULT 0, comments INTEGER DEFAULT 0, shares INTEGER DEFAULT 0, favorites INTEGER DEFAULT 0, completion_rate REAL, avg_watch_time REAL, created_at INTEGER DEFAULT (strftime('%s', 'now')), updated_at INTEGER DEFAULT (strftime('%s', 'now')), UNIQUE(platform, video_id) ); -- 评论数据表 CREATE TABLE comments ( id INTEGER PRIMARY KEY AUTOINCREMENT, platform TEXT NOT NULL, video_id TEXT NOT NULL, comment_id TEXT NOT NULL, user_id TEXT, content TEXT, like_count INTEGER DEFAULT 0, sentiment TEXT, sentiment_score REAL, created_at INTEGER DEFAULT (strftime('%s', 'now')), UNIQUE(platform, comment_id) ); -- 采集日志表 CREATE TABLE collect_logs ( id INTEGER PRIMARY KEY AUTOINCREMENT, platform TEXT NOT NULL, status TEXT NOT NULL, count INTEGER DEFAULT 0, error_message TEXT, created_at INTEGER DEFAULT (strftime('%s', 'now')) ); ``` ### 4.2 API 路由 ```javascript // routes/api.js router.get('/api/data/videos', async (ctx) => { const { platform, days = 7 } = ctx.query; const videos = db.prepare(` SELECT * FROM videos WHERE platform = ? AND created_at > ? ORDER BY views DESC LIMIT 100 `).all(platform, Date.now() / 1000 - days * 86400); ctx.body = { success: true, data: videos }; }); router.get('/api/data/comments', async (ctx) => { const { video_id, sentiment } = ctx.query; let query = 'SELECT * FROM comments WHERE video_id = ?'; const params = [video_id]; if (sentiment) { query += ' AND sentiment = ?'; params.push(sentiment); } const comments = db.prepare(query).all(...params); ctx.body = { success: true, data: comments }; }); ``` --- ## 5. 环境变量配置 ```bash # .env.example # 抖音开放平台 DOUYIN_CLIENT_ID=your_client_id DOUYIN_CLIENT_SECRET=your_client_secret # B站 BILIBILI_COOKIE=your_sessdata_cookie # 快手 KUAISHOU_PHONE=your_phone KUAISHOU_PASSWORD=your_password # 数据库 DB_PATH=./data/harvester.db # 定时任务 ENABLE_CRON=true CRON_TIMEZONE=Asia/Shanghai ``` --- ## 6. 部署检查清单 - [ ] 注册抖音开放平台开发者 - [ ] 创建应用并获取 client_id/secret - [ ] 申请相关权限 scope - [ ] 配置 B站 cookie (SESSDATA) - [ ] 安装依赖: npm install axios better-sqlite3 node-cron - [ ] 初始化数据库 - [ ] 配置 .env 文件 - [ ] 测试单次采集 - [ ] 开启定时任务 - [ ] 监控采集日志