test_scrapling.py
1.25 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
import os
import json
import traceback
from scrapling import StealthyFetcher
def fetch_hn():
print("Initiating scrapling StealthyFetcher request to Hacker News...")
fetcher = StealthyFetcher()
page = fetcher.fetch("https://news.ycombinator.com/")
try:
results = []
# Each HN item is a <tr> with class athing
posts = page.css("tr.athing")
for post in posts[:15]:
title_elem = post.css_first(".titleline > a")
if not title_elem:
continue
title = title_elem.text
url = title_elem.attrib.get('href', '')
score = "Check HN for score"
results.append({
"title": title,
"score": score,
"url": url if url.startswith('http') else "https://news.ycombinator.com/" + url
})
print("\n=== 🚀 [Scrapling Crawler] Hacker News 抓取结果 ===")
print(json.dumps(results[:5], indent=2, ensure_ascii=False))
print("=========================================\n")
except Exception as e:
print("Failed to parse.")
print(traceback.format_exc())
if __name__ == "__main__":
fetch_hn()