通过CrawlSpider爬取网易社会招聘信息

通过CrawlSpider爬取网易社会招聘信息

1.创建工程

scrapy startproject 项目名称

2.创建crawlspider爬虫

scrapy genspider -t crawl 爬虫名 爬虫的范围.com

3.爬虫代码如下

# -*- coding: utf-8 -*-from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Ruleclass WangyishezhaoSpider(CrawlSpider):name = 'wangyishezhao'allowed_domains = ['163.com']start_urls = ['https://hr.163.com/position/list.do?postType=01']rules = (Rule(LinkExtractor(restrict_xpaths='//div[@class="m-page"]/a'), follow=True),Rule(LinkExtractor(restrict_xpaths='//tbody/tr/td[1]/a'), callback='parse_item', follow=False),)def parse_item(self, response):data_dict=dict()data_dict['job_title'] = response.xpath('//div[1]//div[1]/h2/text()').extract_first()data_dict['job_address'] = response.xpath('//table[@class="job-params"]//td[4]/text()').extract_first()# print(data_dict)return data_dict# print('#'*100)# print(response.url)

4.通过pipeline管道进行保存数据

# -*- coding: utf-8 -*-
from scrapy.exporters import JsonItemExporterclass WangyishezhaoPipeline(object):def open_spider(self,spider):self.file = open('wangyi.json','wb')# 写入器self.write = JsonItemExporter(self.file)self.write.start_exporting()def process_item(self, item, spider):self.write.export_item(item)return itemdef close_spider(self,spider):self.write.finish_exporting()self.file.close()

5.定义中间件，设置用户代理和ip代理

class UserAgentMiddleware(object):def process_request(self,request,spider):# 1.随机的user-agentrandom_user = random.choice(USER_AGENT_LIST)# 2.request.header['User-Agent']request.headers['User-Agent']=random_userdef process_response(self, request, response, spider):print('#' * 100)print(request.headers['User-Agent'])return responseclass ProxyMiddleware(object):def process_request(self,request,spider):# 1.代理ipproxy = random.choice(proxies)# 2.设置 request.meta['proxy']request.meta['proxy']=proxyreturn None

6.最后在settings配置文件中进行配置

BOT_NAME = 'WangYiSheZhao'SPIDER_MODULES = ['WangYiSheZhao.spiders']
NEWSPIDER_MODULE = 'WangYiSheZhao.spiders'# Crawl responsibly by identifying yourself (and your website) on the user-agent
USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.84 Safari/537.36'ROBOTSTXT_OBEY = FalseDOWNLOAD_DELAY = 0.5DOWNLOADER_MIDDLEWARES = {'WangYiSheZhao.middlewares.UserAgentMiddleware': 543,'WangYiSheZhao.middlewares.ProxyMiddleware': 555,
}ITEM_PIPELINES = {'WangYiSheZhao.pipelines.WangyishezhaoPipeline': 300,
}USER_AGENT_LIST = ["Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36","Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko","Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 SE 2.X MetaSr 1.0","Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; SE 2.X MetaSr 1.0)","Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36 OPR/26.0.1656.60",
"Opera/8.0 (Windows NT 5.1; U; en)","Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/2.0.0 Opera 9.50","Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 9.50",
]proxies = ['http://58.218.200.223:30054','http://58.218.200.223:30153','http://58.218.200.223:30370','http://58.218.200.223:30288','http://58.218.200.223:30044',
]

运行结果如下图所示

通过json.cn渲染后的结果