Python 百度贴吧爬虫

	# coding:utf-8
	import requests
	from lxml import etree
	import json
	class Tieba(object):
	def __init__(self, name):
	self.url = 'https://tieba.baidu.com/f?kw={}'.format(name)
	self.heardes = {
	"User-Agent": "Mozilla/4.0 (compatible; MSIE 5.00; Windows 98)"
	}
	def get_data(self, url):
	response = requests.get(url, headers=self.heardes)
	with open("temp.html", "wb") as f:
	f.write(response.content)
	return response.content
	def parse_data(self, data):
	# 创建element对象
	# data = data.decode().replace("<!--", "")
	html = etree.HTML(data)
	el_list = html.xpath('//li[@class=" j_thread_list clearfix thread_item_box"]/div/div[2]/div[1]/div[1]/a')
	print(len(el_list))
	data_list = []
	for el in el_list:
	temp = {}
	temp['title'] = el.xpath("./text()")[0]
	temp['link'] = 'https://tieba.baidu.com' + el.xpath("./@href")[0]
	data_list.append(temp)
	# 获取下一页url
	try:
	next_url = 'https:' + html.xpath('//a[contains(text(),"下一页>")]/@href')[0]
	except:
	next_url = None
	return data_list, next_url
	def save_data(self, data_list):
	for data in data_list:
	with open("temp.json", "a+", encoding='utf-8') as f:
	data_temp = json.dumps(data, ensure_ascii=False)
	print(data_temp)
	f.write(data_temp)
	# f.write(data)
	def run(self):
	# url
	# hearders
	next_url = self.url
	while True:
	# 发送请求,获取响应
	data = self.get_data(self.url)
	# 从相应中提取数据(数据和翻页用的url)
	data_list, next_url = self.parse_data(data)
	self.save_data(data_list)
	print(next_url)
	# 判断是否终结
	if next_url == None:
	break
	if __name__ == '__main__':
	tieba = Tieba("华东理工大学吧")
	tieba.run()

hosted with ❤ by GitHub