3.3 【实战】爬取BOSS直聘招聘信息

本文主要是介绍3.3 【实战】爬取BOSS直聘招聘信息，希望对大家解决编程问题提供一定的参考价值，需要的开发者们随着小编来一起学习吧！

第四节：爬取招聘网站信息

课程目标

对boss直聘招聘信息爬取并进行简单的分析

课程内容

编码实现

解析网站：https://www.spidertools.cn/#/curl2Request

1. 爬虫部分

import requests
import pandas as pd
from tqdm import tqdm
import time
def get_job_list(page=1):headers = {"accept": "application/json, text/plain, */*","accept-language": "zh-CN,zh;q=0.9","priority": "u=1, i","referer": "https://www.zhipin.com/web/geek/job?query=python%E5%BC%80%E5%8F%91&city=101270100","sec-ch-ua": "\"Not)A;Brand\";v=\"99\", \"Google Chrome\";v=\"127\", \"Chromium\";v=\"127\"","sec-ch-ua-mobile": "?0","sec-ch-ua-platform": "\"Windows\"","sec-fetch-dest": "empty","sec-fetch-mode": "cors","sec-fetch-site": "same-origin","traceid": "F-a05d1b8WJPFaqSiJ","user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36","x-requested-with": "XMLHttpRequest"}cookies = {"Hm_lvt_194df3105ad7148dcf2b98a91b5e727a": "1724658599","HMACCOUNT": "E109D3011F5CA4E9","__g": "-","Hm_lpvt_194df3105ad7148dcf2b98a91b5e727a": "1724674759","__fid": "ef51d0c8944e0fc0a3e5db1854b55699","__zp_stoken__": "ad34fw47CtMKKH0VeaxofG1LCiH9qecONUcKKw4zDgXFQVMK4UMK8w4dkcsOMw4HCrsKzw4HCscKeasK3wrjCtcK1wpdZwqzCssKjUcKRUMKgUMSMwqTCpMSnxInFgcOmf8O0w4XCqEI2FBAUEg4WGhYYHBYaERMXFBAUEg4UEBQSDkI5xIFxMkRDSEM1VlpWEVlpaFFqVRNeTk5HRWEXGV5FP0FDR0HDj0HDhB%2FDi0XDghvDjUHCvmFDT0FFw4caME3DgCAXwr5KEcKAF0AVZWLDk23EjVYnw4TCvj1DR8ODxYRIQCNJPkRKREJPREAzQivDkW%2FEjVwhw4LCszRDHkxARE1AREBETz5GNERLYi5AQTNKEg4XFR03T8OFw4DDh8OrQEQ%3D","__c": "1724658600","__a": "39645613.1724658600..1724658600.19.1.19.19"
}url = "https://www.zhipin.com/wapi/zpgeek/search/joblist.json"params = {"scene": "1","query": "python开发","city": "101270100","experience": "","payType": "","partTime": "","degree": "","industry": "","scale": "","stage": "","position": "","jobType": "","salary": "","multiBusinessDistrict": "","multiSubway": "","page": f"{page}","pageSize": "30"}response = requests.get(url, headers=headers, cookies=cookies, params=params)rj  = response.json()zpData = rj['zpData']jobList = zpData['jobList']return jobList
jobs = []
for i in tqdm(range(1, 3)):jobList = get_job_list(i)jobs.extend(jobList)time.sleep(3)
infos = []
for job in jobs:info = {"职位名称":job[ 'jobName'],"薪资":job[ 'salaryDesc'],"学历要求":job[ 'jobDegree'],"要求技能":job[ 'skills'],"公示名称":job[ 'brandName'],"人员规模":job[ 'brandScaleName'],"福利":job[ 'welfareList'],"行业":job[ 'brandIndustry'],"老板名字":job[ 'bossName'],"职位":job[ 'bossTitle'],"所在城市":job[ 'cityName'],"所在区":job[ 'areaDistrict'],"区域":job[ 'businessDistrict'],"在线状态":"在线" if job["bossOnline"] else "离线",
}   infos.append(info)
df = pd.DataFrame(infos)
df.to_csv("boss直聘python开发岗位信息.csv", index=False)

2. 数据分析部分

import pandas as pd
df = pd.read_csv("boss直聘python开发岗位信息.csv")
df.head()
def print_skill_as(skills):splites_skills = []for sk in skills:splites_skills.extend(sk)se = pd.DataFrame({'技能': splites_skills,})gp = se.groupby("技能")dt = gp.size().sort_values(ascending=False)print(dt)
gp = df.groupby("学历要求")
gp.size().sort_values(ascending=False)
a_df =df[df["人员规模"] == "20-99人"]
b_df =df[df["人员规模"] == "100-499人"]
gp = b_df.groupby("薪资")
gp.size().sort_values(ascending=False)
print_skill_as(b_df["福利"])