基础篇(7)多线线程+对象实现爬虫

本文主要是介绍基础篇(7)多线线程+对象实现爬虫，希望对大家解决编程问题提供一定的参考价值，需要的开发者们随着小编来一起学习吧！

 

 

import time
import random
import re
import os
from urllib import request
import requests
import threading
from lxml import etree
from queue import Queue  # 这个队列是线程队列
"""
多线程的Queue就是线程安全的，所有我们不用考虑锁的问题
"""class Procuder(threading.Thread):"""生成者继承threading.Thread需要实现__init__方法和run()"""headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36'}def __init__(self,url_queue,img_queue,*args,**kwargs):""":param url_queue: url地址队列:param img_queue: 图片地址队列:param args: threading.Thread类的元组参数:param kwargs: threading.Thread的字典参数"""super(Procuder,self).__init__(*args,**kwargs)self.url_queue =url_queueself.img_queue =img_queuedef run(self):while True:if self.url_queue.empty():breakurl =self.url_queue.get() # 从队列中获取一个urlself.parse_page(url)def parse_page(self,url):response = requests.get(url,headers=self.headers)if response.status_code==200:text = response.texthtml = etree.HTML(text)imgs = html.xpath("//div[@class='page-content text-center']//img[@class!='gif']")for img in imgs:img_url = img.get("data-original")alt = img.get("alt")alt = re.sub(r'[\.\*\?。？！!，,]',"",alt)suffix = os.path.splitext(img_url)[1]file_name = alt +suffixself.img_queue.put((img_url,file_name))class Consumer(threading.Thread):def __init__(self,url_queue,img_queue,*args,**kwargs):super(Consumer,self).__init__(*args,**kwargs)self.url_queue = url_queueself.img_queue = img_queuedef run(self):while True:if self.url_queue.empty() and self.img_queue.empty():breakimg_url, filename = self.img_queue.get()request.urlretrieve(img_url, 'images/' + filename)print(filename + '  下载完成！')
def main():url_queue = Queue(100)img_queue = Queue(1000)for page in range(1,101):url = "http://www.doutula.com/photo/list/?page={}".format(str(page))url_queue.put(url)for i in range(5):p_thrd = Procuder(url_queue,img_queue)p_thrd.start()for j in range(10):c_thrd = Consumer(url_queue,img_queue)c_thrd.start()
if __name__ == '__main__':main()

 

这篇关于基础篇(7)多线线程+对象实现爬虫的文章就介绍到这儿，希望我们推荐的文章对编程师们有所帮助！

---