python selenium爬取街舞网站视频数据

前言

想要做一个街舞网站视频的app，利用爬取的街舞视频存入本地，展现到app上

爬取网站：https://www.vhiphop.com/

Vhiphop网站也是我很喜欢的街舞网站之一，这里聚集了很多我喜欢的舞者视频和教学

爬取代码：

# !/usr/bin/env python
# -*- coding: utf-8 -*-# 爬取Vhiphop网站import requests
import importlib
import lxml.html
import os
import sys
importlib.reload(sys)
import jsonfrom selenium import webdrivervideo_list = []def selenium_get_vhiphop_html(http_url):# 更改代理ipoptions = webdriver.ChromeOptions()options.add_argument("--proxy-server=http://110.73.2.248:8123")#获取chromedriver的绝对路径driver_path = r'E:\fiddler\chromedriver.exe'# 初始化一个driver，并且指定chromedriver的路径driver = webdriver.Chrome(executable_path=driver_path)# driver = webdriver.Chrome(executable_path=driver_path,, chrome_options=options)# 请求网页driver.get(http_url)# 通过page_source获取网页源代码 热键 ctrl+alt+Lreturn driver.page_source# request请求，但response内容隐藏
def parse_vhiphop_url(http_url):#拿到请求头 模拟用户搜索headers = {"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36"}response = requests.get(http_url,headers=headers,verify=False)# 响应首页内容return response.text# 处理html数据，得到所有的数据
def get_vhiphop_video(html_data):"""分析全网数据，并提取视频信息"""# 获得etree对象metree = lxml.html.etree# 获得解析对象parser = metree.HTML(html_data)# 解析并获得div列表li_list = parser.xpath("//ul[@class='video-lists-box']/li")# 遍历div列表for li_element in li_list:# 拿到一个个li标签video_title = li_element.xpath("./a[@class='video-lists-link']/@title")# 40个popping信息item = {}# 获取标题item["video_title"] = video_title[0]# 获取ul地址，以用查找视频地址video_data_html_url = li_element.xpath("./a[@class='video-lists-link']/@href")[0]if not video_data_html_url.startswith("http"):video_data_html_url = "https://www.vhiphop.com"+video_data_html_url# 获取街舞视频播放页面html数据# video_data = selenium_get_vhiphop_html(video_data_html_url)# 解析html数据video_url = video_real_url(video_data_html_url)item["video_url"] = video_url# video封面图片地址video_img_url = li_element.xpath("./a[@class='video-lists-link']/div/img/@data-src")[0]item["video_img_url"] = video_img_url# 获取街舞类型：popping lacking breakingvideo_type = li_element.xpath("./a[@class='video-lists-link']/div[@class='video-meta-box']/div[@class='video-title-box']/span/text()")[0]item["video_type"] = video_typevideo_list.append(item)def selenium_get_video_html(html_url):# 获取chromedriver的绝对路径driver_path = r'E:\fiddler\chromedriver.exe'# 初始化一个driver，并且指定chromedriver的路径driver = webdriver.Chrome(executable_path=driver_path)# driver = webdriver.Chrome(executable_path=driver_path,, chrome_options=options)# 请求网页driver.get(html_url)# 通过page_source获取网页源代码 热键 ctrl+alt+Lreturn driver.page_sourcedef video_real_url(video_html_url):html_data = selenium_get_video_html(video_html_url)# 获得etree对象metree = lxml.html.etree# 获得解析对象parser = metree.HTML(html_data)# 解析并获得divvideo_url = parser.xpath("//div[@class='video-container cp']/div/video[@class='vjs-tech']/@src")return video_urldef result(http_url):item = {}html_data = selenium_get_vhiphop_html(http_url)get_vhiphop_video(html_data)def save_video_datas_json_file(video_list):"""以json格式保存到本地"""# 当目录不存在时则创建新目录video_dir_name = "./videos"if not os.path.exists(video_dir_name):# 创建目录os.makedirs(video_dir_name)# 写入数据 python类型的数据转换为字符串video_json_strs = json.dumps(video_list,ensure_ascii=False,indent=2)# 写入文件video_file = open(video_dir_name+"/video.json","w",encoding="utf-8")video_file.write(video_json_strs)video_file.close()print("总共个数："+len(video_list))print("视频数据已经保存成功！谢谢！")def main():# 推荐页http_url_index = "https://www.vhiphop.com/"# choreograph = 1# 编舞http_url_choreograph = "https://www.vhiphop.com/videos?sort=1"result(http_url_choreograph)# poppinghttp_url_popping = "https://www.vhiphop.com/videos?sort=2"result(http_url_popping)# hiphophttp_url_hiphop = "https://www.vhiphop.com/videos?sort=3"result(http_url_hiphop)# jazzhttp_url_jazz = "https://www.vhiphop.com/videos?sort=4"result(http_url_jazz)# lockinghttp_url_locking = "https://www.vhiphop.com/videos?sort=5"result(http_url_locking)# wacckinghttp_url_waccking = "https://www.vhiphop.com/videos?sort=6"result(http_url_waccking)# breakinghttp_url_breaking = "https://www.vhiphop.com/videos?sort=7"result(http_url_breaking)# househttp_url_house = "https://www.vhiphop.com/videos?sort=8"result(http_url_house)# contemporaryhttp_url_contemporary = "https://www.vhiphop.com/videos?sort=9"result(http_url_contemporary)# reggaehttp_url_reggae = "https://www.vhiphop.com/videos?sort=10"result(http_url_reggae)# freestlyhttp_url_freestly = "https://www.vhiphop.com/videos?sort=11"result(http_url_freestly)# krumpinghttp_url_krumping = "https://www.vhiphop.com/videos?sort=12"result(http_url_krumping)# othershttp_url_others = "https://www.vhiphop.com/videos?sort=13"result(http_url_others)# 解析网址，请求数据# [{视频1},{视频2},{视频3}]# 保存文件save_video_datas_json_file(video_list)# item_popping = get_vhiphop_video(html_data_index)if __name__ == '__main__':main()

爬取的数据以json存入文本

response内容隐藏

request请求相应的response内容不全

检查网页元素可以看到 response内容被隐藏，可以通过selenium直接获取浏览器的Element

缺点

因为需要安装浏览器驱动，所以每次请求网页时都需要打开浏览器，加载时间实在太长。建议采用多线程爬取，等待时间缩短一些

后续优化

等我写好app了在搞