pig简单介绍

2024-05-09 06:48

文章标签 简单介绍 pig

本文主要是介绍pig简单介绍，希望对大家解决编程问题提供一定的参考价值，需要的开发者们随着小编来一起学习吧！

pig简单介绍

1.load:从文件中读取数据（）
用法：athletes = LOAD 'hdfs://master:9000/OlympicAthletes.csv' USING org.apache.pig.piggybank.storage.CSVExcelStorage(',', 'YES_MULTILINE', 'NOCHANGE', 'SKIP_INPUT_HEADER') AS (athlete:chararray, country:chararray, year:int, sport:chararray, gold:int, silver:int, bronze:int, total:int);
注意：load和=之间有空格
2. dump :把操作结果读取到屏幕
用法: DUMP athletes
3.limit：限制前面多少条
用法：athletes_lim = LIMIT athletes 10;
DUMP athletes_lim;
4.GROUP BY：按照字段分组（根据国家分组）
用法： athletes_grp_country = GROUP athletes BY country;
5.DESCRIBE(描述结构)
用法：describe athletes_grp_country
6.FOREACH … GENERATE（统计出每个国家（country）赢了多少金牌(sum(athletes.total))）
medal_sum = FOREACH athletes_grp_country GENERATE group AS country, SUM(athletes.total) as medal_count;
DUMP medal_sum;
注意：athletes_grp_country是按照国家分组后的结果
7.distinct（把重复国家去掉)
distinct_countries = DISTINCT (FOREACH athletes GENERATE country);
DUMP distinct_countries;
这里或者可以写成这样：
a = FOREACH athletes GENERATE country;
distinct_countries = DISTINCT a;
DUMP distinct_countries;
8. GROUP ALL
data_range = FOREACH (GROUP athletes ALL) GENERATE MIN(athletes.year) as min_year, MAX(athletes.year) as max_year;
DUMP data_range;
注意： MIN MAX区分大小写
9. order .... by ....排序（按照金牌数降序排序）
ordered_medals = ORDER medal_sum BY medal_count DESC;
ordered_medals_lim = LIMIT ordered_medals 1;
DUMP ordered_medals_lim;
10.filter(找出不是游泳，根据国家分组，最高金牌的国家的金牌数)
athletes_filter = FILTER athletes by sport != 'Swimming';
medal_sum = FOREACH (GROUP athletes_filter BY country) GENERATE group as country, SUM(athletes_filter.total_medals) as medal_count;
ordered_medals = ORDER medal_sum BY medal_count DESC;
ordered_medals_lim = LIMIT ordered_medals 1;
DUMP ordered_medals_lim;
注意：问题：多少国家只赢一个金牌？
deco = filter medial_sum by count_medial == 1;
suna= foreach (group deco all) generate COUNT(deco.count_medial);
11. join(他们已经获得了在连续的奥运会奖牌数相同的运动员)
copy复制 athletes_copy = FOREACH athletes GENERATE athlete, year as year2, total as total2;
athletes_join = JOIN athletes BY athlete, athletes_copy BY athlete;
athletes_join_filtered = FILTER athletes_join BY total == total2 AND year2 == year+4;
athletes_output = FOREACH athletes_join_filtered GENERATE athletes::athlete as athlete, total2, year, year2;
这里有点问题！！！！！！！！
12.split()
SPLIT athletes INTO
summer_olympics IF year % 4 == 0,
winter_olympics IF year % 4 != 0;

from pig_util import outputSchema

@outputSchema('score:int')
def calculate_score(gold, silver, bronze):
return 3 * gold + 2 * silver + bronze

REGISTER 'olympic_udfs.py' USING streaming_python AS udf

athlete_score = FOREACH athletes GENERATE athlete, udf.calculate_score(gold_medal, silver_medal, bronze_medal) as score;


问题：
找出哪个国家分数最高的冬季奥运会上我们的新指标

解决方法：
新建olympic_udfs.py内容如下4行：
from pig_util import outputSchema
@outputSchema('score:int')
def calculate_score(gold, silver, bronze):
return 3 * gold + 2 * silver + bronze

register '/usr/local/pig/contrib/piggybank/java/piggybank.jar'
REGISTER '/root/olympic_udfs.py' USING streaming_python AS udf

athletes = LOAD 'hdfs://master:9000/OlympicAthletes.csv' USING org.apache.pig.piggybank.storage.CSVExcelStorage(',', 'YES_MULTILINE', 'NOCHANGE', 'SKIP_INPUT_HEADER') AS (athlete:chararray, country:chararray, year:int, sport:chararray, gold:int, silver:int, bronze:int, total:int);
SPLIT athletes INTO summer_olympics IF year % 4 == 0, winter_olympics IF year % 4 != 0;

1.winter_medal_sum = FOREACH (GROUP winter_olympics BY country) GENERATE group AS country, SUM(winter_olympics.gold) as gold_sum, SUM(winter_olympics.silver) as silver_sum, SUM(winter_olympics.bronze) as bronze_sum;
3.country_scores = FOREACH winter_medal_sum GENERATE country, udf.calculate_score(gold_sum, silver_sum, bronze_sum) as score;
4.ordered_winter_medals = ORDER country_scores BY score DESC;
5.ordered_winter_medals_lim = LIMIT ordered_winter_medals 1;
DUMP ordered_winter_medals_lim;