api_config.md 8.39 KB

数据采集配置

各平台 API 配置和示例代码


1. 抖音 Open API 配置

1.1 应用注册

平台: https://open.douyin.com
应用名称: LemonAI Data Harvester
权限申请:
  - user_info.baseinfo (用户基本信息)
  - video.list (视频列表)
  - video.data (视频数据)
  - video.comment (评论数据)

1.2 OAuth 流程

// 配置文件
module.exports = {
  douyin: {
    client_id: process.env.DOUYIN_CLIENT_ID,
    client_secret: process.env.DOUYIN_CLIENT_SECRET,
    redirect_uri: 'https://lemonai.com/oauth/douyin/callback',

    // 授权Scope
    scopes: [
      'user_info.baseinfo',
      'video.list',
      'video.data',
      'video.comment'
    ],

    // API 基础地址
    base_url: 'https://open.douyin.com',

    // 端点
    endpoints: {
      authorize: '/oauth/authorize/',
      token: '/oauth/access_token/',
      video_list: '/video/list/',
      video_data: '/video/data/',
      comment_list: '/video/comment/list/'
    }
  }
};

1.3 示例代码

class DouyinCollector {
  constructor(config) {
    this.client_id = config.client_id;
    this.client_secret = config.client_secret;
    this.access_token = null;
  }

  // 获取授权URL
  getAuthUrl() {
    const params = new URLSearchParams({
      client_key: this.client_id,
      redirect_uri: this.redirect_uri,
      scope: this.scopes.join(','),
      response_type: 'code',
      state: 'random_state'
    });
    return `${this.base_url}/oauth/authorize/?${params}`;
  }

  // 换取访问令牌
  async getAccessToken(code) {
    const response = await fetch(`${this.base_url}/oauth/access_token/`, {
      method: 'POST',
      headers: { 'Content-Type': 'application/json' },
      body: JSON.stringify({
        client_key: this.client_id,
        client_secret: this.client_secret,
        code: code,
        grant_type: 'authorization_code'
      })
    });
    const data = await response.json();
    this.access_token = data.data.access_token;
    return this.access_token;
  }

  // 获取视频列表
  async getVideoList(page = 1, page_size = 20) {
    const response = await fetch(`${this.base_url}/video/list/`, {
      method: 'POST',
      headers: {
        'Content-Type': 'application/json',
        'Authorization': `Bearer ${this.access_token}`
      },
      body: JSON.stringify({
        page: page,
        page_size: page_size
      })
    });
    return response.json();
  }

  // 获取视频详情
  async getVideoData(video_id) {
    const response = await fetch(`${this.base_url}/video/data/`, {
      method: 'POST',
      headers: {
        'Content-Type': 'application/json',
        'Authorization': `Bearer ${this.access_token}`
      },
      body: JSON.stringify({
        video_id: video_id
      })
    });
    return response.json();
  }

  // 获取评论列表
  async getCommentList(video_id, cursor = 0, count = 20) {
    const response = await fetch(`${this.base_url}/video/comment/list/`, {
      method: 'POST',
      headers: {
        'Content-Type': 'application/json',
        'Authorization': `Bearer ${this.access_token}`
      },
      body: JSON.stringify({
        video_id: video_id,
        cursor: cursor,
        count: count
      })
    });
    return response.json();
  }
}

2. B站 Open API 配置

2.1 API 列表

module.exports = {
  bilibili: {
    base_url: 'https://api.bilibili.com',

    // 不需要登录的接口
    public_endpoints: {
      view: '/x/web-interface/view',
      stat: '/x/web-interface/stat',
      view_icon: '/x/view/icon'
    },

    // 需要 cookie 的接口
    private_endpoints: {
      myinfo: '/x/web-interface/myinfo',
      dash: '/x/web-interface/dash',
      replies: '/x/v2/reply'
    },

    // 接口示例
    examples: {
      // 获取视频信息
      videoInfo: 'https://api.bilibili.com/x/web-interface/view?aid=170001'

      // 获取视频统计
      videoStat: 'https://api.bilibili.com/x/web-interface/stat?aid=170001'
    }
  }
};

2.2 示例代码

class BilibiliCollector {
  constructor(config) {
    this.base_url = config.base_url;
    thiscookie = config.cookie; // SESSDATA
  }

  // 获取视频信息
  async getVideoInfo(aid) {
    const response = await fetch(`${this.base_url}/x/web-interface/view?aid=${aid}`);
    return response.json();
  }

  // 获取视频统计
  async getStat(aid) {
    const response = await fetch(`${this.base_url}/x/web-interface/stat?aid=${aid}`);
    return response.json();
  }

  // 获取评论列表
  async getReplies(aid, page = 1, type = 1) {
    const response = await fetch(
      `${this.base_url}/x/v2/reply?type=${type}&oid=${aid}&pn=${page}&ps=20`
    );
    return response.json();
  }

  // 批量获取 (带缓存)
  async batchGetVideoInfo(aidList) {
    const results = [];
    for (const aid of aidList) {
      const info = await this.getVideoInfo(aid);
      const stat = await this.getStat(aid);
      results.push({ ...info.data, ...stat.data });
      await this.sleep(1000); // 避免限流
    }
    return results;
  }
}

3. 定时任务配置

3.1 采集频率

平台 Cron 表达式 说明
抖音 0 2 * * * 每日凌晨2点
B站 0 3 * * * 每日凌晨3点
快手 0 4 * * 0 每周日凌晨4点

3.2 配置示例

// config/cron.json
{
  "tasks": [
    {
      "name": "douyin_daily",
      "schedule": "0 2 * * *",
      "handler": "collect_douyin",
      "enabled": true
    },
    {
      "name": "bilibili_daily",
      "schedule": "0 3 * * *",
      "handler": "collect_bilibili",
      "enabled": true
    },
    {
      "name": "sentiment_analysis",
      "schedule": "0 5 * * *",
      "handler": "analyze_sentiment",
      "enabled": true
    }
  ]
}

4. 数据存储模型

4.1 SQLite 表结构

-- 视频数据表
CREATE TABLE videos (
  id INTEGER PRIMARY KEY AUTOINCREMENT,
  platform TEXT NOT NULL,
  video_id TEXT NOT NULL,
  video_title TEXT,
  author_id TEXT,
  publish_time INTEGER,
  views INTEGER DEFAULT 0,
  likes INTEGER DEFAULT 0,
  comments INTEGER DEFAULT 0,
  shares INTEGER DEFAULT 0,
  favorites INTEGER DEFAULT 0,
  completion_rate REAL,
  avg_watch_time REAL,
  created_at INTEGER DEFAULT (strftime('%s', 'now')),
  updated_at INTEGER DEFAULT (strftime('%s', 'now')),
  UNIQUE(platform, video_id)
);

-- 评论数据表
CREATE TABLE comments (
  id INTEGER PRIMARY KEY AUTOINCREMENT,
  platform TEXT NOT NULL,
  video_id TEXT NOT NULL,
  comment_id TEXT NOT NULL,
  user_id TEXT,
  content TEXT,
  like_count INTEGER DEFAULT 0,
  sentiment TEXT,
  sentiment_score REAL,
  created_at INTEGER DEFAULT (strftime('%s', 'now')),
  UNIQUE(platform, comment_id)
);

-- 采集日志表
CREATE TABLE collect_logs (
  id INTEGER PRIMARY KEY AUTOINCREMENT,
  platform TEXT NOT NULL,
  status TEXT NOT NULL,
  count INTEGER DEFAULT 0,
  error_message TEXT,
  created_at INTEGER DEFAULT (strftime('%s', 'now'))
);

4.2 API 路由

// routes/api.js
router.get('/api/data/videos', async (ctx) => {
  const { platform, days = 7 } = ctx.query;

  const videos = db.prepare(`
    SELECT * FROM videos
    WHERE platform = ?
    AND created_at > ?
    ORDER BY views DESC
    LIMIT 100
  `).all(platform, Date.now() / 1000 - days * 86400);

  ctx.body = { success: true, data: videos };
});

router.get('/api/data/comments', async (ctx) => {
  const { video_id, sentiment } = ctx.query;

  let query = 'SELECT * FROM comments WHERE video_id = ?';
  const params = [video_id];

  if (sentiment) {
    query += ' AND sentiment = ?';
    params.push(sentiment);
  }

  const comments = db.prepare(query).all(...params);
  ctx.body = { success: true, data: comments };
});

5. 环境变量配置

# .env.example

# 抖音开放平台
DOUYIN_CLIENT_ID=your_client_id
DOUYIN_CLIENT_SECRET=your_client_secret

# B站
BILIBILI_COOKIE=your_sessdata_cookie

# 快手
KUAISHOU_PHONE=your_phone
KUAISHOU_PASSWORD=your_password

# 数据库
DB_PATH=./data/harvester.db

# 定时任务
ENABLE_CRON=true
CRON_TIMEZONE=Asia/Shanghai

6. 部署检查清单

  • [ ] 注册抖音开放平台开发者
  • [ ] 创建应用并获取 client_id/secret
  • [ ] 申请相关权限 scope
  • [ ] 配置 B站 cookie (SESSDATA)
  • [ ] 安装依赖: npm install axios better-sqlite3 node-cron
  • [ ] 初始化数据库
  • [ ] 配置 .env 文件
  • [ ] 测试单次采集
  • [ ] 开启定时任务
  • [ ] 监控采集日志