Python爬取58足浴上网站信息

以58足浴（http://bj.58.com/zuyu/pn1/?PGTID=0d306b61-0000-186a-d0e6-09e79d939b21&ClickID=1）的该网页为例来实战下Xpath。想要爬取的信息包括：标题、类型、临近、转让费、租金、面积。

1. 使用基础库完成

先不使用框架，自己手写爬取该页的代码：

# -*- coding: utf-8 -*-
import codecs
import reimport pandas as pd
import sys
from lxml import etree
import requestsreload(sys)
sys.setdefaultencoding("utf-8")
# print res
# print type(res)
# with codecs.open("bj.58.html", mode="wb") as f:
#     f.write(res)
from lxml.etree import _Elementblank = ""  # 空字符串
colon_en = ":"  # 英文冒号
colon_zh = u"："  # 中文冒号
forward_slash = "/" # 正斜杠
br_label = "<br>"  # 换行标签pattern_space = re.compile("\s+")  # 空格
pattern_line = re.compile("<br\s*?/?>")  # 换行
pattern_label = re.compile("</?\w+[^>]*>")  # HTML标签def crawl_data(url):data = {"title": [],"kind": [],"approach": [],"trans_fee": [],"rent": [],"area": []}response = requests.get(url)res = response.contenttree = etree.HTML(res)frame = tree.xpath("//*[@id='infolist']/table/tr")# one = frame[0]# print one.xpath(".//text()")# print one.xpath("string()")for one in frame:# 标题提取 method 1raw_title = blank.join(one.xpath("./td[@class='t']/a/text()"))title = re.sub(pattern_space, blank, raw_title)# print("title: %s" % title)# method 2# title = one.xpath("string(./td[@class='t']/a)")data["title"].append(title)print("title: %s" % title)# 类型和临近位置提取raw_kind_and_approach = blank.join(one.xpath("./td[@class='t']/text()"))kind_and_approach = re.sub(pattern_space, blank, raw_kind_and_approach)k_and_a_list = kind_and_approach.split(forward_slash)kind = ""approach = ""for thing in k_and_a_list:if u"类型" in thing:kind = thing.split(colon_en)[1]elif u"临近" in thing:approach = thing.split(colon_en)[1]data["kind"].append(kind)data["approach"].append(approach)print("kind: %s, approach: %s" % (kind, approach))# 转让费和租金提取transfer_fee_and_rent = etree.tostring(one.xpath("./td[3]")[0], encoding="utf-8")# print("transfer_fee_and_rent: %s" % transfer_fee_and_rent)t_and_r_list = re.sub(pattern_space, blank, transfer_fee_and_rent).split(br_label)# 针对转让费为面议或租金为面议或都为面议的情况进行处理t_and_r_list = t_and_r_list if len(t_and_r_list) == 2 else t_and_r_list * 2transfer_fee = re.sub(pattern_label, blank, t_and_r_list[0]).split(colon_zh)[-1]rent = re.sub(pattern_label, blank, t_and_r_list[1]).split(colon_zh)[-1]data["trans_fee"].append(transfer_fee)data["rent"].append(rent)print("transfer_fee: %s, rent: %s" % (transfer_fee, rent))# 面积提取raw_area = etree.tostring(one.xpath("./td[position()=4]")[0], encoding="utf-8")area = re.sub(pattern_label, blank, raw_area)area = re.sub(pattern_space, blank, area)data["area"].append(area)print("area: %s" % area)print("-" * 50)# data.append(item)return datadef write_csv(data, file):df = pd.DataFrame(data)df.to_csv(file, index=False, encoding="gbk")if __name__ == "__main__":url = "http://bj.58.com/zuyu/pn1/?PGTID=0d306b61-0000-186a-d0e6-09e79d939b21&ClickID=1"data = crawl_data(url)out_file = "bj_58.csv"write_csv(data, out_file)# print("data: %s" % data)

运行后结果：

2. 使用Scrapy框架完成

命令行中输入 scrapy startproject tutorial来创建一个tutorial工程。

在items.py中添加一个新的Item：

class ZuYuItem(scrapy.Item):title = scrapy.Field()  # 标题kind = scrapy.Field()  # 类型approach = scrapy.Field()  # 临近transfer_fee = scrapy.Field()  # 转让费rent = scrapy.Field()  # 租金area = scrapy.Field()  # 面积

在spiders目录下创建一个名为 bj_58.py 的新的Python文件。内容如下：

# -*- coding: utf-8 -*-
import reimport scrapyfrom tutorial.items import ZuYuItemclass BJ58Spider(scrapy.Spider):"""scrapy crawl bj_58 -o res.csv"""name = "bj_58"start_urls = ["http://bj.58.com/zuyu/pn1/?PGTID=0d306b61-0000-186a-d0e6-09e79d939b21&ClickID=1"]def parse(self, response):blank = ""  # 空字符串colon_en = ":"  # 英文冒号colon_zh = u"："  # 中文冒号forward_slash = "/" # 正斜杠br_label = "<br>"  # 换行标签pattern_space = re.compile("\s+")  # 空格pattern_line = re.compile("<br\s*?/?>")  # 换行pattern_label = re.compile("</?\w+[^>]*>")  # HTML标签item = ZuYuItem()frame = response.xpath("//*[@id='infolist']/table/tr")# one = frame[0]# print one.xpath(".//text()").extract()  # 提取每个选择器所对应# print one.xpath("string()").extract_first()for one in frame:# 标题提取 method 1raw_title = blank.join(one.xpath("./td[@class='t']/a/text()").extract())title = re.sub(pattern_space, blank, raw_title)# method 2# title = one.xpath("string(./td[@class='t']/a)").extract_first()item["title"] = title# 类型和临近位置提取raw_kind_and_approach = blank.join(one.xpath("./td[@class='t']/text()").extract())kind_and_approach = re.sub(pattern_space, blank, raw_kind_and_approach)k_and_a_list = kind_and_approach.split(forward_slash)kind = ""approach = ""for thing in k_and_a_list:if u"类型" in thing:kind = thing.split(colon_en)[1]elif u"临近" in thing:approach = thing.split(colon_en)[1]item["kind"] = kinditem["approach"] = approach# 转让费和租金提取transfer_fee_and_rent = one.xpath("./td[position()=3]").extract_first()t_and_r_list = re.sub(pattern_space, blank, transfer_fee_and_rent).split(br_label)self.log("title: %s" % title)self.log("t_and_r_list: %s" % t_and_r_list)t_and_r_list = t_and_r_list if len(t_and_r_list) == 2 else t_and_r_list * 2self.log("t_and_r_list: %s" % t_and_r_list)transfer_fee = re.sub(pattern_label, blank, t_and_r_list[0]).split(colon_zh)[-1]rent = re.sub(pattern_label, blank, t_and_r_list[1]).split(colon_zh)[-1]item["transfer_fee"] = transfer_feeitem["rent"] = rent# 面积提取raw_area = one.xpath("./td[position()=4]").extract_first()area = re.sub(pattern_label, blank, raw_area)item["area"] = areayield item

在命令行中输入  scrapy crawl bj_58 -o res.csv 将结果存入res.csv文件中