執(zhí)行腳本：
PIGGYBANK_PATH=$PIG_HOME/contrib/piggybank/java/piggybank.jar
INPUT=pig/input/test-pig-full.txt
OUTPUT=pig/output/test-pig-output-$(date +%Y%m%d%H%M%S)
PIGSCRIPT=analyst_status_logs.pig
#analyst_500_404_month.pig
#analyst_500_404_day.pig
#analyst_404_percentage.pig
#analyst_500_percentage.pig
#analyst_unique_path.pig
#analyst_user_logs.pig
#analyst_status_logs.pig
pig -p PIGGYBANK_PATH=$PIGGYBANK_PATH -p INPUT=$INPUT -p OUTPUT=$OUTPUT $PIGSCRIPT
要分析的數(shù)據(jù)源，LOG 文件
46.20.45.18 - - [25/Dec/2012:23:00:25 +0100] "GET / HTTP/1.0" 302 - "-" "Pingdom.com_bot_version_1.4_(http://www.pingdom.com/)" "-" "-" 46.20.45.18 "" 11011AEC9542DB0983093A100E8733F8 0
46.20.45.18 - - [25/Dec/2012:23:00:25 +0100] "GET /sign-in.jspx HTTP/1.0" 200 3926 "-" "Pingdom.com_bot_version_1.4_(http://www.pingdom.com/)" "-" "-" 46.20.45.18 "" 11011AEC9542DB0983093A100E8733F8 0
69.59.28.19 - - [25/Dec/2012:23:01:25 +0100] "GET / HTTP/1.0" 302 - "-" "Pingdom.com_bot_version_1.4_(http://www.pingdom.com/)" "-" "-" 69.59.28.19 "" 36D80DE7FE52A2D89A8F53A012307B0A 15
PIG腳本：
--注冊JAR包，因?yàn)橐玫紻ateExtractor
register '$PIGGYBANK_PATH';
--聲明一個短函數(shù)名
DEFINE DATE_EXTRACT_MM
org.apache.pig.piggybank.evaluation.util.apachelogparser.DateExtractor('yyyy-MM');
DEFINE DATE_EXTRACT_DD
org.apache.pig.piggybank.evaluation.util.apachelogparser.DateExtractor('yyyy-MM-dd');
-- pig/input/test-pig-full.txt
--把數(shù)據(jù)從變量所指的文件加載到PIG中，并定義數(shù)據(jù)列名，此時的數(shù)據(jù)集為數(shù)組(a,b,c)
raw_logs = load '$INPUT' USING org.apache.pig.piggybank.storage.MyRegExLoader('^(\\S+) (\\S+) (\\S+) \\[([\\w:/]+\\s[+\\-]\\d{4})\\] "(\\S+) (\\S+) (HTTP[^"]+)" (\\S+) (\\S+) "([^"]*)" "([^"]*)" "(\\S+)" "(\\S+)" (\\S+) "(.*)" (\\S+) (\\S+)')
as (remoteAddr: chararray,
n2: chararray,
n3: chararray,
time: chararray,
method: chararray,
path:chararray,
protocol:chararray,
status: int,
bytes_string: chararray,
referrer: chararray,
browser: chararray,
n10:chararray,
remoteLogname: chararray,
remoteAddr12: chararray,
path2: chararray,
sessionid: chararray,
n15: chararray
);
--過濾數(shù)據(jù)
filter_logs = FILTER raw_logs BY not (browser matches '.*pingdom.*');
--item_logs = FOREACH raw_logs GENERATE browser;
--percent 500 logs
--重定義數(shù)據(jù)項(xiàng)，數(shù)據(jù)集只取2項(xiàng)status,month
reitem_percent_500_logs = FOREACH filter_logs GENERATE status,DATE_EXTRACT_MM(time) as month;
--分組數(shù)據(jù)集，此時的數(shù)據(jù)結(jié)構(gòu)為MAP(a{(aa,bb,cc),(dd,ee,ff)},b{(bb,cc,dd),(ff,gg,hh)})
group_month_percent_500_logs = GROUP reitem_percent_500_logs BY (month);
--重定義分組數(shù)據(jù)集數(shù)據(jù)項(xiàng)，進(jìn)行分組統(tǒng)計(jì)，此時要聯(lián)合分組數(shù)據(jù)集和原數(shù)據(jù)集統(tǒng)計(jì)
final_month_500_logs = FOREACH group_month_percent_500_logs
{
--對原數(shù)據(jù)集做count，因?yàn)槭窃趂oreachj里做count的，即使是對原數(shù)據(jù)集，也會自動會加month==group的條件
--從這里可以看出對于group里的數(shù)據(jù)集，完全沒用到
--這時是以每一行為單位的，統(tǒng)計(jì)MAP中的KEY-a對應(yīng)的數(shù)組在原數(shù)據(jù)集中的個數(shù)
total = COUNT(reitem_percent_500_logs);
--對原數(shù)據(jù)集做filter，因?yàn)槭窃趂oreachj里做count的，即使是對原數(shù)據(jù)集，也會自動會加month==group的條件
--重新過濾一下原數(shù)據(jù)集，得到status==500,month==group的數(shù)據(jù)集
t = filter reitem_percent_500_logs by status== 500; --create a bag which contains only T values
--重定義數(shù)據(jù)項(xiàng)，取group，統(tǒng)計(jì)結(jié)果
generate flatten(group) as col1, 100*(double)COUNT(t)/(double)total;
}
STORE final_month_500_logs into '$OUTPUT' using PigStorage(',');