本文主要是介绍spider-java (Jsoup) (媒体信息的爬取)，希望对大家解决编程问题提供一定的参考价值，需要的开发者们随着小编来一起学习吧！
媒体基础信息爬取实例
GetAppname.java (代码为hive的udf，静态页面的获取)
package com.hb.hive.utils;import java.util.Random;import org.apache.hadoop.hive.ql.exec.UDF;
import org.apache.hadoop.io.Text;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;public class GetAppname extends UDF {/*** 各大应用市场获取app名称* * @param Str* @return*/public Text evaluate(Object... args) {if (args.length == 1) {return new Text(getAppName(args[0].toString()));} else {return new Text(getAppName(args[0].toString()));}}public static String getAppName(String app_id) {try {int t =new Random().nextInt(10)*3000;System.out.println(t);Thread.sleep( t);} catch (InterruptedException e) {// TODO Auto-generated catch blocke.printStackTrace();}String app_name = "";app_name = getAppNameByYingyongbao(app_id);if (app_name.equals("")) {app_name = getAppNameByXiaomi(app_id);if (app_name.equals("")) {app_name = getAppByWandoujia(app_id);if (app_name.equals("")&&app_id.matches("[0-9]+")) {app_name = getAppNameByAso100(app_id);}}}return app_name;}/*** from Yingyonggao* * @param app_id* @return*/public static String getAppNameByYingyongbao(String app_id) {try {Document doc = Jsoup.connect("http://android.myapp.com/myapp/detail.htm?apkName=" + app_id).get();Elements elementsByClass = doc.getElementsByClass("det-name-int");if (!elementsByClass.text().toString().equals("")) {return elementsByClass.text().toString();}} catch (Exception e) {System.out.println(e);return "";}return "";}/*** from Xiaomi* * @param app_id* @return*/public static String getAppNameByXiaomi(String app_id) {try {Document doc = Jsoup.connect("http://app.mi.com/details?id=" + app_id).get();Elements elementsByClass = doc.getElementsByClass("yellow-flower");if (!elementsByClass.attr("alt").equals("")) {return elementsByClass.attr("alt");}} catch (Exception e) {System.out.println(e);return "";}return "";}/*** from Aso100* * @param app_id* @return*/public static String getAppNameByAso100(String app_id) {try {Document doc = Jsoup.connect("https://aso100.com/app/rank/appid/" + app_id ).get();Elements elementsByClass = doc.getElementsByClass("name-str");if (!elementsByClass.text().toString().equals("")) {return elementsByClass.text().toString();}} catch (Exception e) {System.out.println(e);return "";}return "";}/*** from Wandoujia* * @param app_id* @return*/public static String getAppByWandoujia(String app_id) {try {Document doc = Jsoup.connect("http://www.wandoujia.com/apps/" + app_id).get();Elements elementsByClass = doc.getElementsByClass("app-name");if (!elementsByClass.text().toString().equals("")) {return elementsByClass.text().toString();}} catch (Exception e) {System.out.println(e);return "";}return "";}/*** from Itunes* * @param app_id* @return*/public static String getAppByItunes(String app_id) {try {Document doc = Jsoup.connect("https://itunes.apple.com/cn/app/id" + app_id).get();Elements elementsByClass = doc.getElementsByClass("artwork");if (!elementsByClass.attr("alt").equals("")) {return elementsByClass.attr("alt");}} catch (Exception e) {System.out.println(e);return "";}return "";}/*** from Googleplay 需要翻墙* * @param app_id* @return*/public static String getAppByGoogleplay(String app_id) {try {Document doc = Jsoup.connect("https://play.google.com/store/apps/details?id=" + app_id + "&hl=zh_CN").get();Elements elementsByClass = doc.getElementsByClass("id-app-title");if (!elementsByClass.text().toString().equals("")) {return elementsByClass.text().toString();}} catch (Exception e) {System.out.println(e);return "";}return "";}@SuppressWarnings("static-access")public static void main(String[] args) {
// System.out.println(getAppByItunes("1000114190"));System.out.println(new GetAppname().getAppName("com.yr.mmpic"));
// System.out.println(new
// GetAppname().getAppByGoogleplay("com.mandongkeji.comiclover"));
// System.out.println(new
// GetAppname().getAppByWandoujia("com.mandongkeji.comiclover"));
// System.out.println(new
// GetAppname().getAppNameByAso100("com.mandongkeji.comiclover")); }
}结合phantomjs实现Jsoup动态页面的获取
phantomjs-2.1.1-macosx+java
具体的搭建方式请百度
在安装目录（/usr/local/share/phantomjs-2.1.1-macosx/code.js）下添加code.js用于获取动态页面
code.js
system = require('system')
address = system.args[1];
var page = require('webpage').create();
var url = address;
page.open(url, function (status) {//Page is loaded!if (status !== 'success') {console.log('Unable to post!');} else {window.setTimeout(function () {page.render("test1.png"); //截图console.log(page.content);phantom.exit();}, 5000);}}); https://www.jianshu.com/p/96220e239c35
这篇关于spider-java (Jsoup) (媒体信息的爬取)的文章就介绍到这儿，希望我们推荐的文章对编程师们有所帮助！
