crawl_config.json
1.71 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
{
"skill_name": "ceo_leadership",
"target_agent": "ceo",
"data_sources": [
{
"name": "OKR Best Practices",
"type": "rss",
"url": "https://www.whatmatters.com/feed",
"relevance": "elite",
"crawl_frequency": "weekly"
},
{
"name": "Multi-Agent Research Papers",
"type": "api",
"url": "https://arxiv.org/search/?query=multi+agent+management&format=rss",
"relevance": "elite",
"crawl_frequency": "daily"
},
{
"name": "CEO Management Articles",
"type": "web_scrape",
"url": "https://hbr.org/topic/leadership",
"selector": {
"title": "h3.hed",
"summary": "p.dek",
"author": "span.byline",
"date": "time"
},
"relevance": "average",
"crawl_frequency": "weekly"
}
],
"extraction_rules": {
"strategy_planning": {
"keywords": ["vision", "strategy", "planning", "roadmap", "market analysis"],
"weight": 0.25
},
"team_building": {
"keywords": ["team building", "org design", "hiring", "management"],
"weight": 0.25
},
"okr_management": {
"keywords": ["OKR", "objective", "key result", "goal setting"],
"weight": 0.25
},
"task_delegation": {
"keywords": ["delegation", "task assignment", "dispatch", "execution"],
"weight": 0.25
}
},
"quality_filters": {
"min_content_length": 500,
"required_fields": ["title", "content"],
"deduplicate_by": "title"
},
"output_format": {
"file_type": "json",
"schema": {
"title": "string",
"content": "string",
"source": "string",
"date": "datetime",
"relevance": "enum[elite,average,broad]",
"category": "string"
}
}
}