使用scrapy框架采集企业黄页数据

本文主要是介绍使用scrapy框架采集企业黄页数据，希望对大家解决编程问题提供一定的参考价值，需要的开发者们随着小编来一起学习吧！

今天闲着无聊，好就没用scrapy框架写爬虫了，于是就写了一个企业黄页的爬虫脚本，采集顺企网黄页数据，保存到mongodb，话不多说，简单看下代码吧。

下面部分是spider代码（习惯使用beautifulsoup了QAQ）

import scrapy
from bs4 import BeautifulSoup
from shunqi.items import ShunqiItem
import reclass EpregSpider(scrapy.Spider):name = 'epreg'allowed_domains = ['11467.com']start_url = "http://b2b.11467.com/"def start_requests(self):yield scrapy.Request(url=self.start_url,callback=self.get_city)def get_city(self,response):soup = BeautifulSoup(response.text,"lxml")dl_listtxt = soup.find(text="按城市浏览全国公司黄页").findNext("div").find_all("dl",attrs={"class":"listtxt"})for dl in dl_listtxt:province = dl.find("dt").text.strip()dd_list = dl.find_all("dd")for dd in dd_list:city = dd.text.strip()city_link = "https:" + dd.find("a")["href"]yield scrapy.Request(url=city_link,callback=self.get_class,meta={"city":city,"province":province})def get_class(self,response):city = response.meta["city"]province = response.meta["province"]soup = BeautifulSoup(response.text,"lxml")class_all = soup.find(text=re.compile("公司分类")).findNext("ul").find_all("li")for li in class_all:dd_all = li.find("dl").find_all("dd")for dd in dd_all:cor_link = "https:" + dd.find("a")["href"]yield scrapy.Request(url=cor_link,callback=self.get_cor,meta={"city":city,"province":province})def get_cor(self,response):city = response.meta["city"]province = response.meta["province"]soup = BeautifulSoup(response.text,"lxml")company_list = soup.find("ul",attrs={"class":"companylist"})if company_list:company_list1 = company_list.find_all("div",attrs={"class","f_l"})for li in company_list1:whole_url = "https:" + li.find("h4").findNext("a")["href"]if "qiye" in whole_url:whole_url = whole_url + "#contact"else:whole_url = "https:" + li.find("h4").findNext("a")["href"]yield scrapy.Request(url=whole_url,callback=self.get_info,meta={"url":whole_url,"city":city,"province":province})is_pages = soup.find("div",attrs={"class","pages"})if is_pages:max_page = is_pages.find(text="尾页").findParent("a")["href"].split("-")[1].replace(".htm")page_href = is_pages.find(text="尾页").findParent("a")["href"]for page in range(2,int(max_page)+1):page_url = page_href.split("-")[0] + "-" + str(page) + ".htm"yield scrapy.Request(url=page_url,callback=self.get_cor,meta={"city":city,"province":province})def deal_item(self,soup,text,label):try:data = soup.find(text=re.compile("%s" % text)).findNext("%s" % label).text.strip().replace("\r\n","")except:data = Nonereturn datadef get_info(self,response):item = ShunqiItem()whole_url = response.meta["url"]if "qiye" in whole_url:id = re.search("qiye\/(\d+)\.htm",whole_url,re.S).group(1)else:id = re.search("co\/(\d+)\.htm",whole_url,re.S).group(1)city= response.meta["city"]province = response.meta["province"]soup = BeautifulSoup(response.text,"lxml")adress = self.deal_item(soup,"公司地址：","dd")phone = self.deal_item(soup,"固定电话：","dd")manager = self.deal_item(soup,"经理：","dd")if self.deal_item(soup,"手机号码：","dd") == None:link_phone = self.deal_item(soup,"经理手机：","dd")else:link_phone= self.deal_item(soup,"手机号码：","dd")email = self.deal_item(soup,"电子邮件：","dd")cor_name = self.deal_item(soup,"法人名称：","td")business = self.deal_item(soup,"经营范围：","td")main_product = self.deal_item(soup,"主要经营产品：","td")for field in item.fields:item[field] = eval(field)yield item