网络爬虫--HtmlUnit

本文主要是介绍网络爬虫--HtmlUnit，希望对大家解决编程问题提供一定的参考价值，需要的开发者们随着小编来一起学习吧！

1、简介
htmlunit 是一款开源的java 页面分析工具，读取页面后，可以有效的使用htmlunit分析页面上的内容。项目可以模拟浏览器运行，被誉为java浏览器的开源实现。是一个没有界面的浏览器，运行速度迅速。是junit的扩展之一
2、官方API文档
HtmlUnit官网API文档
3、pom文件

<dependency><groupId>net.sourceforge.htmlunit</groupId><artifactId>htmlunit</artifactId><version>2.26</version></dependency><dependency><groupId>junit</groupId><artifactId>junit</artifactId><version>4.12</version></dependency><dependency><groupId>org.slf4j</groupId><artifactId>slf4j-log4j12</artifactId><version>1.7.21</version></dependency>

4、测试代码

package com.sun.htmlunit;import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.DomElement;
import com.gargoylesoftware.htmlunit.html.DomNodeList;
import com.gargoylesoftware.htmlunit.html.HtmlDivision;
import com.gargoylesoftware.htmlunit.html.HtmlForm;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import com.gargoylesoftware.htmlunit.html.HtmlSubmitInput;
import com.gargoylesoftware.htmlunit.html.HtmlTextInput;/*** htmlunit测试* @author sunt* @dade 2017年4月17日上午11:04:22* @version v1.0*/
public class HtmlUnitTestDemo {/*** 日志*/private static Logger logger = LoggerFactory.getLogger(HtmlUnitTestDemo.class);/*** htmlunit入门测试*/@Testpublic void test1() {//实例化web客户端WebClient client = new WebClient();try {//解析请求url页面HtmlPage page = client.getPage("http://blog.csdn.net/u010427935");logger.info("===========>获取请求页面的html:" + page.asXml());logger.info("=============>获取请求页面内容:" + page.asText());} catch (Exception e) {logger.error("=====HtmlUnitTestDemo===test1===>" + e.getMessage());}finally{//关闭客户端、释放内存client.close();}}/*** 模拟浏览器的请求,解决部分网址拒绝访问*/@Testpublic void test2() {//初始化web客户端WebClient client = new WebClient(BrowserVersion.FIREFOX_52);try {HtmlPage page = client.getPage("http://mvnrepository.com");logger.debug("=====================>获取请求页面的html内容:" + page.asXml());} catch (Exception e) {logger.error("=======test2==========>" + e.getMessage());}finally{//关闭客户端释放内存client.close();}}/*** 获取指定元素*/@Testpublic void test3() {//初始化web客户端WebClient client = new WebClient(BrowserVersion.FIREFOX_52);try {HtmlPage page = client.getPage("http://blog.csdn.net/u010427935");//获取指定id的html内容HtmlDivision division = page.getHtmlElementById("navMenu");logger.debug("================>指定html的内容:" + division.asXml());//通过name获取html内容DomNodeList<DomElement> tagList = page.getElementsByTagName("a");for (DomElement domElement : tagList) {logger.debug("===========>byTagName:" + domElement.asXml());}} catch (Exception e) {logger.error("=====test3===========>" + e.getMessage());}finally{//关闭客户端释放内存client.close();}}/*** 模拟点击事件实现搜索功能*/@Testpublic void test4() {//实例化web客户端WebClient client = new WebClient(BrowserVersion.FIREFOX_52);try {//获取解析的页面HtmlPage page = client.getPage("http://blog.java1234.com/index.html");//获取提交的表单HtmlForm form = page.getFormByName("myform");//获取输入框HtmlTextInput input = form.getInputByName("q");//获取提交的按钮HtmlSubmitInput submitInput = form.getInputByName("submitButton");for (int i = 0; i < 1000; i++) {//设置输入框的值input.setValueAttribute("java" + i);//模拟点击、提交表单HtmlPage result = submitInput.click();logger.info("===========>搜索的结果:" + result.asXml());}} catch (Exception e) {logger.error("===========test4========>" + e.getMessage());}finally{//关闭客户端释放内存client.close();}}/*** 代理IP*/@Testpublic void test5() {//初始化客户端WebClient client = new WebClient(BrowserVersion.FIREFOX_52, "58.118.185.100", 8998);try {HtmlPage page = client.getPage("http://blog.csdn.net/u010427935");logger.debug("=======html内容：====>" + page.asXml());} catch (Exception e) {logger.error("===========test5=========>" + e.getMessage());}finally{client.close();}}/*** 对于非js加载的页面取消js和css的解析*/@Testpublic void test6() {//实例化客户端WebClient client = new WebClient(BrowserVersion.FIREFOX_52);//取消客户端对js和css的解析client.getOptions().setCssEnabled(false);client.getOptions().setJavaScriptEnabled(false);try {HtmlPage page = client.getPage("http://blog.csdn.net/u010427935");logger.debug("==============>获取的html内容:" + page.asXml());} catch (Exception e) {logger.error("========error========test6====>" + e.getMessage());}finally{client.close();}}/*** 爬取ajax加载的页面httpclient无法抓去到数据*/@Testpublic void test7() {WebClient client = new WebClient(BrowserVersion.FIREFOX_52);HtmlPage page = null;try {page = client.getPage("https://pan.baidu.com/share/home?uk=305605848#category/type=0");//线程休眠等待js加载Thread.sleep(10000);logger.info("==========>抓去到的html内容:" + page.asXml());} catch (Exception e) {logger.error("=====error===test7=====>" + e.getMessage());}finally{client.close();}}
}