推荐 - Jsoup（附网页批量抓取例子）

本文主要是介绍推荐 - Jsoup（附网页批量抓取例子），希望对大家解决编程问题提供一定的参考价值，需要的开发者们随着小编来一起学习吧！

目标：做一个简单的网站爬虫（怎么听怎么象virus。。。），访问父网站下的超链接，提取里面的文本内容。

开始时，手工写HTML的标签解析，部分代码如下：

	/** * 获取Href List分析结果 * * @return List<String>* @throws IOException*/ public List<String> getHrefList() throws IOException { Set<String> rawHrefList = parseHref();hrefList = rewriteHref(rawHrefList);return hrefList; }/** * 解析网页链接 * * @return List<String>* @throws IOException */ private List<String> parserHref() throws IOException {List<String> rawHrefList = new ArrayList<String>();URL url = new URL(parentUrl); HttpURLConnection connection = (HttpURLConnection) url.openConnection(); connection.setDoOutput(true);InputStreamReader isr = new InputStreamReader( connection.getInputStream(), getCharset(connection.getContentType())); BufferedReader br = new BufferedReader(isr); String str = null;List<String> rs = null; while ((str = br.readLine()) != null) { rs = getHref(str); if (rs != null) rawHrefList.addAll(rs); }br.close();isr.close();connection.disconnect();return rawHrefList;} /** * 获取网页编码方式 * * @param str* @return String*/ private String getCharset(String str) { Pattern pattern = Pattern.compile("charset=.*"); Matcher matcher = pattern.matcher(str); if (matcher.find()) return matcher.group(0).split("charset=")[1]; return null; }/** * 从一行字符串中读取多条链接* * @param str* @return List<String>*/ private List<String> getHref(String str) { List<String> hrefSet = new ArrayList<String>();Pattern pattern = Pattern.compile("href=\".*?\"");final int URL_START_POSITION = 6;while(str.length() > 0) {Matcher matcher = pattern.matcher(str); if (matcher.find()) {// 从href中截取URLhrefSet.add(matcher.group(0).substring(URL_START_POSITION, matcher.group(0).length()-1));// 去掉已读取字符串str = str.substring(matcher.end());} else {str = "";}}if (hrefSet.size() > 0)return hrefSet;return null; }/** * 改写* * @param hrefList* @return List<String>*/ private List<String> rewriteHref(List<String> hrefList) {List<String> distinctHrefList = distinct(hrefList);return distinctByParent(distinctHrefList);}/** * 去重：去掉重复链接* * @param hrefList* @return List<String>*/ private List<String> distinct(List<String> hrefList) {List<String> distinctHrefList = new ArrayList<String>();Set<String> hrefSet = new HashSet<String>();Iterator<String> hrefItr = hrefList.iterator();while (hrefItr.hasNext()) {String hrefStr = (String)hrefItr.next();hrefSet.add(hrefStr);}distinctHrefList.addAll(hrefSet);return distinctHrefList;}

做着做着就觉得好麻烦呀，做语义分析，各种情况都要考虑，我的目标不是做一个分词工具出来，需要找个好工具才行，网上搜了一下，开始时看到html parser，不过发现代码都是2006年的，忽略，然后又看到Jsoup 和 Jericho HTML Parser，两者看起来半斤八两，不过Jericho 名字好难记，Jsoup 网站和文档好像丰富一些，OK，决定使用Jsoup 。用了才发现，太好用啦，代码清楚多了，开发效率提高，推荐！使用Jsoup后的全部代码如下（注意：现在的服务器有保护机制，同ip瞬间大量访问会被block）：

package j2seTest2;import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.io.IOException;
import java.util.ArrayList;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicInteger;import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;public class HtmlParser {final int THREAD_COUNT = 10;private CountDownLatch threadCompletedCounter = new CountDownLatch(THREAD_COUNT);AtomicInteger paragraphCounter = new AtomicInteger(0);public static void main(String[] args) {
//		testParseHref();
//		testGetHrefList();
//		testParseParagraph();
//		testCrawlParagraph();testCrawlParagraphInMultiThreads();}/** * 获取Href List分析结果 * * @param urlStr* @return List<String>*/ public List<String> getHrefList(String urlStr) { Set<String> rawHrefList = parseHref(urlStr);return rewriteHref(rawHrefList, urlStr); }/** * 解析网页链接, 去掉重复链接* * @param urlStr* @return Set<String>*/ private Set<String> parseHref(String urlStr) {Set<String> rawHrefSet = new HashSet<String>();Document doc;try {doc = Jsoup.connect(urlStr).timeout(5000).get();Elements elements = doc.getElementsByAttribute("href");Iterator<Element> eleItr = elements.iterator();while(eleItr.hasNext()) {Element element = eleItr.next();rawHrefSet.add(element.attr("href"));}} catch (IOException e) {e.printStackTrace();}return rawHrefSet;} /** * 改写* * @param hrefList, parentUrl* @return List<String>*/ private List<String> rewriteHref(Set<String> hrefList, String parentUrl) {return distinctByParent(hrefList, parentUrl);}/** * 去重：只保留父链接开头的URL* * @param hrefList, parentUrl* @return List<String>*/ private List<String> distinctByParent(Set<String> hrefList, String parentUrl) {ArrayList<String> distinctHrefList = new ArrayList<String>();Iterator<String> hrefItr = hrefList.iterator();while (hrefItr.hasNext()) {String hrefStr = (String)hrefItr.next();if (hrefStr.indexOf(parentUrl) >= 0) {distinctHrefList.add(hrefStr);}}return distinctHrefList;}/** * 获取网页段落内容* * @param urlStr* @return String*/ public String getParagraph(String urlStr) { return parseParagraph(urlStr);}/** * 解析网页段落内容* * @param urlStr* @return String*/ private String parseParagraph(String urlStr) {String paragraph = "";final String HTML_TAG_P = "p";final String HTML_TAG_FONT = "font";Document doc;try {doc = Jsoup.connect(urlStr).timeout(5000).get();paragraph = paragraph.concat(getTextByTag(doc, HTML_TAG_P));paragraph = paragraph.concat(getTextByTag(doc, HTML_TAG_FONT));} catch (IOException e) {e.printStackTrace();}return paragraph;}/** * get element's text by tag* * @param doc, tag* @return String*/ private String getTextByTag(Document doc, String tag) {String text = "";Elements elements = doc.getElementsByTag(tag);Iterator<Element> eleItr = elements.iterator();while(eleItr.hasNext()) {Element element = eleItr.next();text = text.concat(element.text());}return text;}/** * crawl element's text in multi-thread* * @param htmlParser, hrefList*/ private void crawlInMultiThreads(final HtmlParser htmlParser, final List<String> hrefList) {ExecutorService executor = Executors.newFixedThreadPool(THREAD_COUNT);final Map<Integer, List<String>> hrefMap = new HashMap<Integer, List<String>>();int sizeOfHrefSubList = hrefList.size() / THREAD_COUNT + 1;int startIndexOfHrefSubList = 0;int endIndexOfHrefSubList = sizeOfHrefSubList;Integer mapIndex = 0;// 把hrefList 按线程数分为若干个subList，再subList 存储到map中for(;;) {int hrefSubListIndex = 0;List<String> hrefSubList = new ArrayList<String>();for (int index = startIndexOfHrefSubList; index < endIndexOfHrefSubList; index++) {hrefSubList.add(hrefSubListIndex, hrefList.get(index));hrefSubListIndex++;}hrefMap.put(mapIndex, hrefSubList);mapIndex++;if (endIndexOfHrefSubList == hrefList.size()) {break;}startIndexOfHrefSubList = endIndexOfHrefSubList;if (hrefList.size() - endIndexOfHrefSubList > sizeOfHrefSubList) {endIndexOfHrefSubList += sizeOfHrefSubList;} else {endIndexOfHrefSubList = hrefList.size();}}// 一个线程处理map中的一个entryfor (int threadIndex = 0; threadIndex < THREAD_COUNT; threadIndex++) {final Iterator<String> hrefItr = hrefMap.get(Integer.valueOf(threadIndex)).iterator();executor.submit(new Runnable() {public void run() {// 每个entry有若干个href url去处理while (hrefItr.hasNext()) {synchronized (hrefItr) {String paragraph = htmlParser.getParagraph(hrefItr.next());paragraphCounter.incrementAndGet();System.out.println("paragraphCounter:" + paragraphCounter);System.out.println(paragraph);System.out.println("");}}threadCompletedCounter.countDown();}});}closeThreadPool(executor);}private void closeThreadPool(final ExecutorService executor) {try {threadCompletedCounter.await();executor.shutdown();} catch (InterruptedException e) {e.printStackTrace();}}private static void testParseHref() {HtmlParser a = new HtmlParser();Set<String> hrefStr;String parentUrl = "http://www.defense.gov/bios/";hrefStr = a.parseHref(parentUrl);Iterator<String> hrefItr = hrefStr.iterator();while (hrefItr.hasNext()) {System.out.println(hrefItr.next());}}private static void testGetHrefList() {HtmlParser a = new HtmlParser(); List<String> hrefList;hrefList = a.getHrefList("http://www.defense.gov/bios/");System.out.println("url count:" + hrefList.size());for (int i = 0; i < hrefList.size(); i++) System.out.println(hrefList.get(i));}private static void testParseParagraph() {HtmlParser a = new HtmlParser(); String paragraph;
//		paragraph = a.parseParagraph("http://www.defense.gov/bios/biographydetail.aspx?biographyid=430");  //<p>paragraph = a.parseParagraph("http://www.defense.gov/bios/biographydetail.aspx?biographyid=185");  //<font>System.out.println("paragraph:");System.out.println(paragraph);}private static void testCrawlParagraph() {HtmlParser a = new HtmlParser(); List<String> hrefList;hrefList = a.getHrefList("http://www.defense.gov/bios/");Iterator<String> hrefItr = hrefList.iterator();int hrefCounter = 1;while (hrefItr.hasNext()) {String hrefUrl = (String) hrefItr.next();String paragraph = a.getParagraph(hrefUrl);System.out.println("paragraph " + hrefCounter + " from " + hrefUrl + ":");System.out.println(paragraph);System.out.println("");hrefCounter++;}}private static void testCrawlParagraphInMultiThreads() {HtmlParser a = new HtmlParser(); List<String> hrefList = a.getHrefList("http://www.defense.gov/bios/");a.crawlInMultiThreads(a, hrefList);}}

其它关于爬虫的介绍参考：
Java爬虫：http://blog.csdn.net/u012730840/article/details/19985897

这篇关于推荐 - Jsoup（附网页批量抓取例子）的文章就介绍到这儿，希望我们推荐的文章对编程师们有所帮助！

---