train.jsonl
13.6 KB
{"input":"请为抖音创作者后台设计一个完整的数据采集流程，目标是获取最近30天内所有视频的播放量、完播率、CTR和用户评论情感。请说明登录方式、Session管理、抓取路径、输出字段结构以及风控措施。","expected_output":{"platform":"douyin","login_method":"qr_code_or_sms","session_management":"persistent_profile_with_cookie_reuse","entry_url":"https://creator.douyin.com/creator-micro/content/manage","data_fields":["video_id","title","publish_time","play_count","completion_rate","avg_play_duration","ctr","like_count","comment_count","share_count","collect_count","follower_growth","sentiment_summary","source_url","crawl_time"],"extraction_strategy":"intercept XHR aweme/v1/web/data/fetch or parse window._SSR_HYDRATED_DATA","sentiment_analysis":"extract comments and run BERT sentiment classification with keyword extraction","anti_detection":{"request_interval_sec":5,"daily_request_limit":200,"proxy":"residential_proxy_recommended","fingerprint_rotation":true},"compliance":"only_collect_own_account_data"},"dimensions":{"D1":15,"D2":20,"D3":14,"D4":19,"D5":14,"D6":13},"score":95,"grader_hints":"评估是否覆盖抖音登录态持久化、XHR拦截、核心字段完整性、频率控制≤200次/天、隐私合规声明"}
{"input":"小红书创作者中心的数据采集需要哪些关键步骤？请详细说明 X-Sign 签名的处理方式，以及如何保证长时任务（超过30分钟）的稳定性。","expected_output":{"platform":"xiaohongshu","login_method":"qr_code","session_management":"cookie_reuse_with_xsign_refresh","entry_url":"https://creator.xiaohongshu.com/statistics/video","xsign_handling":"intercept xhr to extract X-Sign from page JS or use proxy to capture signed requests; refresh every 25 minutes","data_fields":["note_id","title","publish_time","read_count","like_count","collect_count","comment_count","share_count","completion_rate","follower_growth","product_click_count","sentiment_summary","source_url","crawl_time"],"stability_measures":["refresh_page_every_25min","renew_cookie_before_expiry","retry_with_backoff_on_401"],"anti_detection":{"request_interval_sec":4,"proxy_recommended":true}},"dimensions":{"D1":14,"D2":19,"D3":14,"D4":18,"D5":13,"D6":13},"score":91,"grader_hints":"检查 X-Sign 处理是否可行、长时任务稳定性方案是否具体、字段覆盖是否完整"}
{"input":"B站创作中心的数据抓取适合用什么技术方案？请给出推荐方案并解释为什么，同时列出需要采集的数据字段和API端点。","expected_output":{"platform":"bilibili","recommended_method":"api_direct_requests","reason":"B站提供半公开API，无需浏览器渲染，效率高且稳定","api_endpoints":{"archive_stat":"GET https://member.bilibili.com/x/web/archive/stat","overview":"GET https://api.bilibili.com/x/creator/archives/revenue/overview","fan_stat":"GET https://api.bilibili.com/x/relation/stat?vmid={mid}"},"headers":{"SESSDATA":"from_cookie","Referer":"https://member.bilibili.com"},"data_fields":["bvid","title","publish_time","view_count","danmaku_count","comment_count","coin_count","collect_count","share_count","like_count","completion_rate","play_source_distribution","fan_active_hours","source_url","crawl_time"],"rate_limit":"interval >= 2 seconds, daily <= 500 requests","anti_detection":{"fingerprint_rotation":false,"browser_fallback":"only_when_api_fails"}},"dimensions":{"D1":13,"D2":18,"D3":15,"D4":18,"D5":13,"D6":12},"score":89,"grader_hints":"确认推荐使用API直调而非Playwright优先、API端点准确、SESSDATA和CSRF处理提及"}
{"input":"作为 data-harvester，你被要求采集 YouTube Studio 的 Analytics 数据（Views、Watch time、CTR、Subscribers gained/lost）。请设计一个兼顾稳定性与成本效益的方案。","expected_output":{"platform":"youtube","recommended_method":"oauth2_official_api_first","reason":"official_API_is_most_stable_and_cost_effective","api_endpoints":{"data_api":"GET https://www.googleapis.com/youtube/v3/videos?part=statistics,snippet","analytics_api":"GET https://youtubeanalytics.googleapis.com/v2/reports?ids=channel=={channel_id}&metrics=views,estimatedMinutesWatched,averageViewDuration,subscribersGained"},"quota_management":"10,000_units_per_day","browser_automation_role":"supplemental_only_for_uncovered_fields","data_fields":["video_id","title","publish_time","views","watch_time_hours","average_view_duration","ctr","impressions","unique_viewers","subscribers_gained","subscribers_lost","likes","comments","shares","source_url","crawl_time"],"oauth2_setup":"service_account_or_user_consent","fallback":"use_studio_page_automation_if_api_quota_exhausted_or_field_missing"},"dimensions":{"D1":14,"D2":19,"D3":15,"D4":18,"D5":14,"D6":13},"score":93,"grader_hints":"必须优先推荐OAuth2官方API而非纯Playwright、提及配额管理10,000 units、说明页面自动化仅作为补充"}
{"input":"请描述微信视频号助手的数据采集策略。重点说明登录风控、Session 维持、以及可采集的字段。","expected_output":{"platform":"wechat_channels","login_method":"wechat_qr_code","risk_control":"extremely_strict;异地登录/新设备易触发24h冻结","session_maintenance":"persistent_browser_context_with_heartbeat_every_15min","entry_url":"https://channels.weixin.qq.com/platform/post/list","data_fields":["post_id","title","publish_time","play_count","like_count","comment_count","share_count","collect_count","completion_rate","follow_conversion_count","product_click_count","sentiment_summary","source_url","crawl_time"],"extraction_strategy":"XHR_intercept_plus_DOM_extraction","anti_detection":{"delay_between_pages_ms":6000,"proxy":"residential_proxy_mandatory","fingerprint_rotation":true,"profile_persistence":"essential"},"compliance":"only_collect_authorized_account_data"},"dimensions":{"D1":13,"D2":18,"D3":13,"D4":17,"D5":14,"D6":12},"score":87,"grader_hints":"检查是否提及24h冻结风险、15分钟心跳轮询、住宅代理必要性、字段覆盖完整"}
{"input":"快手创作者服务平台的数据采集需要注意哪些反爬和风控措施？请给出字段清单和推荐工具。","expected_output":{"platform":"kuaishou","login_method":"qr_code_or_sms","risk_control":"sensitive_to_IP_and_User_Agent","session_management":"cookie_persistence_after_qr_login","entry_url":"https://cp.kuaishou.com/article/manage/video","data_fields":["photo_id","title","publish_time","play_count","play_completion_rate","like_count","comment_count","share_count","follower_growth","live_revenue","source_url","crawl_time"],"extraction_strategy":"XHR_intercept_or_REST_API_after_login","anti_detection":{"delay_between_pages_ms":5000,"proxy":"residential_proxy_recommended","fingerprint_rotation":true,"ua_rotation":true},"recommended_tools":["Playwright","Bright_Data/Oxylabs"],"rate_limit":"moderate_frequency_avoid_burst"},"dimensions":{"D1":13,"D2":17,"D3":13,"D4":17,"D5":14,"D6":12},"score":86,"grader_hints":"确认快手IP/UA敏感性、住宅代理推荐、Cookie持久化、live_revenue字段提及"}
{"input":"你正在执行一个多平台批量采集任务（抖音+小红书+B站），需要输出统一格式的 JSON 报告。请定义输出 Schema，并说明如何处理各平台字段差异和缺失值。","expected_output":{"output_format":"unified_json_per_video","required_fields":["content_id","platform","title","publish_time","play_count","completion_rate","ctr","source_url","crawl_time"],"recommended_fields":["like_count","comment_count","share_count","collect_count","avg_view_duration","sentiment_summary","follower_growth"],"platform_mapping":{"douyin":{"play_count":"play_count","completion_rate":"completion_rate","ctr":"ctr","comment_sentiment":"sentiment_summary"},"xiaohongshu":{"play_count":"read_count","completion_rate":"completion_rate","ctr":"implicit_or_null","product_clicks":"product_click_count"},"bilibili":{"play_count":"view_count","completion_rate":"completion_rate","ctr":"implicit_or_null","danmaku":"danmaku_count","coins":"coin_count"}},"missing_value_handling":{"numeric_missing":0,"ratio_missing":null,"text_missing":"","sentiment_missing":{"positive":null,"neutral":null,"negative":null,"top_keywords":[]}},"consistency_rules":["all_timestamps_in_ISO8601_UTC","platform_field_must_be_lowercase","content_id_prefixed_with_platform_identifier_if_needed"]},"dimensions":{"D1":12,"D2":18,"D3":12,"D4":20,"D5":13,"D6":12},"score":87,"grader_hints":"检查统一schema是否包含所有必填字段、缺失值处理是否明确、平台字段映射是否合理"}
{"input":"在采集过程中遇到登录态失效，系统返回 401/需要重新扫码。请描述你的错误处理流程，包括自动恢复步骤和上报机制。","expected_output":{"error_type":"auth_session_expired","auto_recovery_steps":["attempt_cookie_refresh_if_refresh_token_exists","re_navigate_to_login_page_and_check_session","if_relogin_fails_mark_AUTH_REQUIRED"],"escalation":"notify_supervisor_or_ops_channel_with_account_id_and_timestamp","degradation":"pause_further_requests_for_this_account_to_avoid_lockout","logging":{"screenshot":"save_to_workspace","last_successful_cookie_time":"record","error_code":"AUTH_EXPIRED"},"prevention":["heartbeat_for_wechat_channels","cookie_renewal_before_expiry_for_xiaohongshu","persistent_context_for_all_platforms"]},"dimensions":{"D1":15,"D2":10,"D3":10,"D4":12,"D5":13,"D6":15},"score":85,"grader_hints":"评估错误分类是否准确、恢复步骤是否合理、是否包含暂停请求避免锁号的保护机制"}
{"input":"某平台突然弹出滑块验证码，Playwright 无法自动通过。请给出完整的应对策略，包括降级路径和是否需要人工介入。","expected_output":{"error_type":"captcha_challenge_slider","immediate_actions":["reduce_request_frequency_by_2x","switch_to_backup_proxy_if_available","take_screenshot_and_log_DOM"],"automation_fallbacks":["enable_playwright_manual_mode_if_supported","call_external_captcha_service_2captcha_as_last_resort"],"human_escalation":"if_automation_fails_within_3_attempts_notify_ops_with_screenshot_and_account_info","degradation":"queue_remaining_tasks_and_pause_current_account","post_resolution":"gradually_restore_frequency_and_monitor_for_rechallenge"},"dimensions":{"D1":12,"D2":10,"D3":11,"D4":11,"D5":14,"D6":14},"score":82,"grader_hints":"检查是否包含降频、切代理、截图、3次尝试后人工介入的合理流程"}
{"input":"你需要对一批视频的评论进行情感分析并输出结构化结果。请说明情感指标定义、分析流程、以及输出格式。","expected_output":{"sentiment_metrics":{"polarity":"Positive / Neutral / Negative","sentiment_score":"mapped_to_-1_to_1","top_keywords":"TF-IDF_weighted_top_N_after_stopword_removal","pain_points":"clustered_themes_from_negative_comments"},"analysis_pipeline":["collect_raw_comments_per_video","clean_and_deduplicate","segment_with_jieba_or_pkuseg","run_bert_sentiment_classification","aggregate_proportions_and_extract_keywords","identify_pain_points_from_negative_cluster"],"output_format":{"sentiment_summary":{"positive":0.62,"neutral":0.28,"negative":0.10,"top_keywords":["干货","副业","AI工具"],"pain_points":["信息过载","广告太多"]}}},"privacy_compliance":"hash_or_anonymize_user_nicknames_and_avatars_before_output","dimensions":{"D1":10,"D2":18,"D3":12,"D4":18,"D5":14,"D6":11},"score":83,"grader_hints":"确认情感分析流程是否完整、输出格式是否结构化、是否提及用户昵称脱敏"}
{"input":"一个 MCN 机构委托你采集旗下 50 个账号的数据（抖音+B站+YouTube）。请设计一个可扩展的采集架构，重点说明账号分级、频率控制、代理策略和异常隔离。","expected_output":{"architecture":"distributed_session_pool_with_account_isolation","account_tiering":{"tier1":"high_value_accounts_use_dedicated_sessions_and_conservative_frequency","tier2":"harvest_accounts_use_shared_proxy_pool_and_standard_frequency","tier3":"fallback_accounts_for_redundancy"},"frequency_control":{"douyin":"max_200_req_day_per_account_interval_5s","bilibili":"max_500_req_day_per_account_interval_2s","youtube":"api_quota_10000_units_day_shared_across_tier1"},"proxy_strategy":{"residential_proxy":"mandatory_for_douyin_kuaishou_wechat","data_center_proxy":"acceptable_for_youtube_api_only"},"error_isolation":{"account_lock":"isolate_affected_account_without_impacting_others","session_rollover":"auto_switch_to_backup_session","quota_exhaustion":"fallback_to_browser_automation_for_youtube"},"output_aggregation":"unified_json_lines_per_video_with_account_id_and_platform_tags","monitoring":"daily_success_rate_and_block_rate_dashboard"},"dimensions":{"D1":14,"D2":18,"D3":14,"D4":17,"D5":15,"D6":14},"score":92,"grader_hints":"评估账号分级是否合理、频率控制是否量化、代理策略是否区分平台、异常隔离是否完善"}
{"input":"你在采集某账号评论数据时，发现平台返回的用户信息包含昵称和头像URL。为了保证合规，你应该如何处理这些原始数据？","expected_output":{"raw_data_contains":"user_nicknames_and_avatar_urls","compliance_principle":"only_collect_own_content_data_and_anonymize_user_pii","processing_steps":["hash_user_nickname_with_sha256_or_replace_with_anonymous_id","strip_avatar_urls_or_replace_with_placeholder","do_not_persist_raw_commenter_profiles_in_long_term_storage","if_benchmark_needed_use_synthetic_or_consented_examples_only"],"output_schema":{"comment_samples":{"user_id":"anon_7a3f...","text":"评论内容","sentiment":"positive"}},"documentation":"record_anonymization_method_in_metadata_for_audit"},"dimensions":{"D1":10,"D2":12,"D3":10,"D4":15,"D5":15,"D6":10},"score":82,"grader_hints":"必须强调仅采集自有数据、对用户昵称哈希化或ID化、头像脱敏、不长期存储原始用户信息"}