如何使用Python3爬虫技术分析Ajax，高效抓取今日头条最新街拍美图？

本文共计547个文字，预计阅读时间需要3分钟。

“假原创以下开头内容，不要啰嗦，不超过100个字，直接输出结果：’

# coding=utf-8
from urllib.parse import urlencode
import requests
from requests.exceptions import RequestException,Timeout
import json
from bs4 import BeautifulSoup
from pymongo import MongoClient
from multiprocessing import Pool
import os
import string
from hashlib import md5


def get_response(url):
try:
headers = {
"user-agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.62 Safari/537.36"
}
# proxies = {'www.toutiao.com/search_content/?" + urlencode(data)
print(url)
try:
response = get_response(url)
print(response.status_code)
if response.status_code == 200:
return response.text
return None
except RequestException:
print('request error')
return None

def conn_mongodb():
client = MongoClient('localhost', 27017)
db = client['jiepai']
jiepai = db['jiepai']
return jiepai

def save_image_url(data):
jiepai = conn_mongodb()
jiepai.update({'title':data.get('title')}, {'$set':data}, upsert=True)

def get_image_url():
jiepai = conn_mongodb()
data = jiepai.find({}, {'title': 1, 'images_list': 1, '_id': 0})
return data


def download_image(data):

base_dir = os.path.abspath(os.path.dirname(__file__))
if not os.path.exists(base_dir + '\jiepai'):
os.mkdir(base_dir + '\jiepai')
for item in data:
print(item.get('title'))
title = item.get('title')
images_list = item.get('images_list')
print('images_lsit',images_list)
# every file name
file_name = title.strip(string.punctuation)
file_name = str(file_name).replace('?','')
if not os.path.exists(base_dir + '\jiepai/' + file_name):
os.mkdir(base_dir + '\jiepai\\' + file_name)
# save images path
file_path = base_dir + '\jiepai\\' + file_name
for image_url in images_list:
print(image_url)
response = get_response(image_url)
html = response.content
image_name = md5(html).hexdigest() + '.jpg'

with open(file_path + '\\' + image_name, 'wb') as f:
f.write(html)
print('download success')


def parse_page_index(html):
data = json.loads(html)
if data and 'data' in data.keys():
for item in data.get('data'):
a_gourp_image_detail = {}
images_list = []
title = item.get('title')
# print(title)
if title is not None:
a_gourp_image_detail['title'] = title
images = item.get('image_detail')
# print(images)
if images:
for image in images:
# print(image.get('url'))
images_list.append(image.get('url'))
# if images_list:
a_gourp_image_detail['images_list'] = list(set(images_list))
print(a_gourp_image_detail)
save_image_url(a_gourp_image_detail)


def main(offset):

html = get_page_index(offset, '街拍')
# print(html)
parse_page_index(html)


if __name__ == "__main__":
# 多进程爬取图片链接，并保存到 Mongodb
# groups = [x*20 for x in range(0,5)]
# pool = Pool()
# pool.map(main, groups)

# 从 mongodb 中获取链接，多进程下载图片，并保存
data = get_image_url()
datas = [item for item in data]

pool = Pool()
pool.map(download_image, data)
# download_image()