from bs4 import BeautifulSoup
soup = BeautifulSoup
# 示例：抓取所有文章标题
titles = soup.select
data =
for title in titles:
link = title
text = title.get_text
data.append
print
4️⃣ 数据清洗 & 存储
import pandas as pd
df = pd.DataFrame
df.drop_duplicates
df.to_csv
print
5️⃣ 高级技巧：代理池 & 随机 User-Agent
防止 IP 被封或出现验证码。
import random
from itertools import cycle
proxies_list =
proxy_pool = cycle
def get_proxy:
return next
headers = random.choice()
response = requests.get(url,headers=headers,proxies={'http': get_proxy,'https': get_proxy})
Pandas 与 SQL 自动化存储方案
Pandas 可直接写入数据库。实现“一键”保存：
from sqlalchemy import create_engine
engine = create_engine
df.to_sql
print
MVP 示例：从简书文章列表批量抓取标题与链接并保存为 CSV 文件——完整代码块如下：
import requests
from bs4 import BeautifulSoup
import pandas as pd
from tqdm import tqdm
BASE_URL = 'https://www.jianshu.com'
LIST_URL_TEMPLATE = BASE_URL + '/tag/{tag}/posts?page={page}'
HEADERS_LIST =
def fetch_page:
resp = requests.get)
resp.raise_for_status
return resp.text
def parse_page:
soup = BeautifulSoup
posts_htmls = soup.select
items=
for post in posts_htmls:
title_tag=post.select_one
if not title_tag: continue
title=title_tag.get_text
link=BASE_URL+title_tag
items.append
return items
def main:
all_items=
for page in tqdm):
url=LIST_URL_TEMPLATE.format
html=fetch_page
items=parse_page
all_items.extend
from bs4 import BeautifulSoup
soup = BeautifulSoup
# 示例：抓取所有文章标题
titles = soup.select
data =
for title in titles:
link = title
text = title.get_text
data.append
print
4️⃣ 数据清洗 & 存储
import pandas as pd
df = pd.DataFrame
df.drop_duplicates
df.to_csv
print
5️⃣ 高级技巧：代理池 & 随机 User-Agent
防止 IP 被封或出现验证码。
import random
from itertools import cycle
proxies_list =
proxy_pool = cycle
def get_proxy:
return next
headers = random.choice()
response = requests.get(url,headers=headers,proxies={'http': get_proxy,'https': get_proxy})
Pandas 与 SQL 自动化存储方案
Pandas 可直接写入数据库。实现“一键”保存：
from sqlalchemy import create_engine
engine = create_engine
df.to_sql
print
MVP 示例：从简书文章列表批量抓取标题与链接并保存为 CSV 文件——完整代码块如下：
import requests
from bs4 import BeautifulSoup
import pandas as pd
from tqdm import tqdm
BASE_URL = 'https://www.jianshu.com'
LIST_URL_TEMPLATE = BASE_URL + '/tag/{tag}/posts?page={page}'
HEADERS_LIST =
def fetch_page:
resp = requests.get)
resp.raise_for_status
return resp.text
def parse_page:
soup = BeautifulSoup
posts_htmls = soup.select
items=
for post in posts_htmls:
title_tag=post.select_one
if not title_tag: continue
title=title_tag.get_text
link=BASE_URL+title_tag
items.append
return items
def main:
all_items=
for page in tqdm):
url=LIST_URL_TEMPLATE.format
html=fetch_page
items=parse_page
all_items.extend
