webmagic-爬取51招聘信息

本文主要是介绍webmagic-爬取51招聘信息，希望对大家解决编程问题提供一定的参考价值，需要的开发者们随着小编来一起学习吧！

点击资料或前往github查看源码WebMagic

如果是采用@Autowired注入变量，则当前类的实例，必须也是spring 容器注入才能成功注入，应该也采用@Autowired注入，要不就不要new需要注入其他对象的当前类

使用springboot开启定时任务，使用自定义pipeline将数据存储到数据库，根据传入的url获取页面，和jquery相似的选择器方法解析页面存入自己想得到的信息，有些数据有几个标签，获取文本内容不要用css()方法，要加上jsoup解析，text（）方法可以获取到所有文本内容，而css要选中标签，用了两种标签数据就会出现问题。
ps：爬取前查看得到的html，会与网页的不一样

package com.example.demo.task;import com.example.demo.pojo.JobInfo;
import com.example.demo.utils.MathSalary;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;import com.fasterxml.jackson.databind.ObjectMapper;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import org.springframework.util.StringUtils;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.BloomFilterDuplicateRemover;
import us.codecraft.webmagic.scheduler.QueueScheduler;
import us.codecraft.webmagic.selector.Selectable;import java.util.Date;@Component
public class JobProcessor implements PageProcessor {private String url = "https://search.51job.com/list/030200,000000,0000,00,9,99,java,2,1.html?lang=c&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&ord_field=0&dibiaoid=0&line=&welfare=";@Autowiredprivate ObjectMapper objectMapper;private int count = 1;@Overridepublic void process(Page page) {//爬取解析的与页面不同，此次爬取得到数据中是以json形式存在script中String jsonString = page.getHtml().css("script").regex(".*SEARCH_RESULT.*").toString();//不为空即为招聘页if (!StringUtils.isEmpty(jsonString)) {//截取json类型字符串String substring = jsonString.substring(jsonString.indexOf("{"), jsonString.lastIndexOf("}") + 1);try {JsonNode jsonNode = objectMapper.readTree(substring);JsonNode array = jsonNode.get("engine_search_result");if (array.isArray()) {for (JsonNode node : array) {//获取详情链接String detailUrl = node.get("job_href").asText();page.addTargetRequest(detailUrl);page.addTargetRequest(detailUrl);}// 获取下一页的urlString bkUrl = "https://search.51job.com/list/030200,000000,0000,00,9,99,java,2," + (++count) + ".html?lang=c&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&ord_field=0&dibiaoid=0&line=&welfare=";//                    // 把url放到任务队列中page.addTargetRequest(bkUrl);} else {count = 0;return;}//} catch (JsonProcessingException e) {e.printStackTrace();}} else {//进入详情页,保存数据到实体类，再通过page.putfiled（即resultItems）给Pipeline传入数据库saveJobInfo(page);}}//解析详情页private void saveJobInfo(Page page) {JobInfo jobInfo = new JobInfo();Selectable detailInfo = page.getHtml().css("div.tCompany_center");if (detailInfo == null) {return;}String addr = detailInfo.css(".ltype[title]").toString();jobInfo.setCompanyName(detailInfo.css("p.cname a", "text").toString());jobInfo.setCompanyAddr(addr.substring(addr.lastIndexOf("=") + 1, addr.indexOf("&")).trim());jobInfo.setCompanyInfo(detailInfo.css("div.tBorderTop_box div.tmsg", "text").toString().trim());jobInfo.setJobAddr(detailInfo.css("div.tBorderTop_box div.bmsg>p.fp", "text").toString());jobInfo.setJobInfo(Jsoup.parse(detailInfo.css("div.job_msg").toString()).text());jobInfo.setJobName(detailInfo.css("div.cn h1", "text").toString());// 设置当前链接jobInfo.setUrl(page.getUrl().toString());// 获取薪资// 有的没有写薪资, 先获取薪资的字符串String salaryText = detailInfo.css("div.cn strong", "text").toString();// 看看是否没有薪资这个字段if (!StringUtils.isEmpty(salaryText)) {// 使用工具类转换薪资字符串Integer[] salary = MathSalary.getSalary(salaryText);jobInfo.setSalaryMin(salary[0]);jobInfo.setSalaryMax(salary[1]);} else {// 没有则设为零jobInfo.setSalaryMax(0);jobInfo.setSalaryMin(0);}jobInfo.setTime((new Date().getYear() + 1900) + "-" + addr.substring(addr.lastIndexOf("发") - 5, addr.lastIndexOf("发")));page.putField("jobInfo", jobInfo);}//网页编码格式private Site site = Site.me().setCharset("gbk")//超时时间10s.setTimeOut(10000)//重试次数3次.setRetryTimes(3)//重试间隔时间3s.setSleepTime(3000);@Overridepublic Site getSite() {return site;}@Autowiredprivate JobInfoPipeline jobInfoPipeline;@Scheduled(initialDelay = 1000, fixedDelay = 100 * 1000)public void process() {Spider.create(this)//自定义PipeLine存入数据库中.addPipeline(jobInfoPipeline)//设置爬取url.addUrl(url)//线程数10.thread(10)//使用内存任务设置Bloom过滤去重器，初始化10万位空间.setScheduler(new QueueScheduler().setDuplicateRemover(new BloomFilterDuplicateRemover(100000))).run();//运行Spider}
}