selenium模块唯一的缺点就是占内存,运行慢,好处就是入门爬取js数据简单
from selenium import webdriver
from bs4 import BeautifulSoup
# 爬取百度今日热点
url = 'http://top.baidu.com/buzz?b=341&c=513&fr=topbuzz_b1'
browser = webdriver.PhantomJS()
browser.get(url)
soup = BeautifulSoup(browser.page_source, 'lxml')
# 标题
title = [i.get_text().strip() for i in soup.select('tbody tr .keyword .list-title')]
# print(title)
# 搜索指数
inde = [i.get_text().strip() for i in soup.select('tbody tr .last')][1:]
# print(inde[1:])
# 传送url
url_texi = [i.get('href') for i in soup.select('tbody tr .keyword .list-title')]
# print(url_texi)
for t, ie, u_t in zip(title, inde, url_texi):
print('标题:{},热度:{},传送:{}'.format(t, ie, u_t))
input('按回车键退出。。。')