Cloudera Academic Partnership 10

8/19/2019 Cloudera Academic Partnership 10

1/102

!"#!! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


2/102

!"#%! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


3/102

!"#&! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


4/102

!"#C! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


5/102

!"#H! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


6/102

!"#M! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


7/102

!"#Q! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


8/102

!"#R! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


9/102

!"#S! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32 grouped = COGROUP stores BY store_id,salespeople BY store_id;

grunt> DUMP grouped;(A,{(A,Anchorage)},{(4,Dieter,A)})(B,{(B,Boston)},{(1,Alice,B),(8,Hannah,B)})

(C,{(C,Chicago)},{(6,Fredo,C),(9,Irina,C)})(D,{(D,Dallas)},{(2,Bob,D),(7,George,D)})(E,{(E,Edmonton)},{})(F,{(F,Fargo)},{(3,Carlos,F),(5,Étienne,F)})(,{},{(10,Jack,)})


10/102

!"#!"! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


11/102

!"#!!! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


12/102

!"#!%! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


13/102

!"#!&! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


14/102

!"#!C! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


15/102

!"#!H! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


16/102

!"#!M! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32 cleaned = FOREACH joined GENERATE stores::store_id,stores::name, person_id, salespeople::name;

grunt> DUMP cleaned;(A,Anchorage,4,Dieter)(B,Boston,1,Alice)(B,Boston,8,Hannah)... (additional records omitted for brevity) ...


17/102

!"#!Q! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


18/102

!"#!R! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


19/102

!"#!S! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


20/102

!"#%"! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


21/102

!"#%!! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


22/102

!"#%%! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


23/102

!"#%&! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32 crossed = CROSS stores, salespeople;

grunt> DUMP crossed;(A,Anchorage,1,Alice,B)(A,Anchorage,2,Bob,D)(A,Anchorage,8,Hannah,B)(A,Anchorage,10,Jack,)(B,Boston,1,Alice,B)(B,Boston,2,Bob,D)(B,Boston,8,Hannah,B)

(B,Boston,10,Jack,)(D,Dallas,1,Alice,B)(D,Dallas,2,Bob,D)(D,Dallas,8,Hannah,B)(D,Dallas,10,Jack,)

! ^-2-/7*-, -J-/< 0(,,159- 3(45127:(2 (8 /-3(/6,

12 *?- stores 726 salespeople /-97:(2,


24/102

!"#%C! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


25/102

!"#%H! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


26/102

!"#%M! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


27/102

!"#%Q! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$321. U7:2 ,+00(/*, *?1, )1*? *?- SPLIT (0-/7*(/

– WX%'488($?8 ?443 ?$+ ;4 P2+2511& 4X


28/102

!"#%R! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32 SPLIT customers INTO

gold_program IF ltv >= 25000,silver_program IF ltv >= 10000

AND ltv < 25000;

grunt> DUMP gold_program;(Glynn,27800)(Ian,43800)(Jeff,29100)(Kai,34000)

grunt> DUMP silver_program;(Bruce,23500)(Charles,17800)(Dustin,21250)(Mirko,24200)

! A091* 3+,*(4-/, 12*( ./(+0, 8(/ /-)7/6, 0/(./74I 57,-6 (2 918-:4- J79+-


29/102

!"#%S! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$321. ,+00(/*, 3(44(2 AbU =(12 *


30/102

!"#&"! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


31/102

!"#&!! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


32/102

!"#&%! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


33/102

!"#&&! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$327/74-*-/,! !D5


34/102

!"#&C! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32 999;

bigsales_alice = FILTER bigsales BY name == 'Alice';STORE bigsales_alice INTO 'Alice';


35/102

!"#&H! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


36/102

!"#&M! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


37/102

!"#&Q! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


38/102

!"#&R! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


39/102

!"#&S! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


40/102

!"#C"! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


41/102


42/102

!"#C%! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


43/102

!"#C&! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


44/102

!"#CC! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


45/102

!"#CH! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


46/102

!"#CM! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


47/102

!"#CQ! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


48/102

!"#CR! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


49/102

!"#CS! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


50/102


51/102

!"#H!! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


52/102

!"#H%! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


53/102

!"#H&! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


54/102

!"#HC! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


55/102

!"#HH! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


56/102

!"#HM! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


57/102

!"#HQ! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


58/102

!"#HR! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


59/102

!"#HS! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$321..< j72X 726 E7*7c+ 7/- *)( -N7409-, (8 (0-2 ,(+/3- _Ec,– f$2 1. )1*?12 72(*?-/ 972.+7.-

W884?E51 H$(?+8


60/102


61/102

!"#M!! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


62/102

!"#M%! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


63/102

!"#M&! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$321. 726 '76((0 )/1*- *( 9(. O9-,

! '() '76((0P, L-5 _a 372 ?-90 1. 3/-7*-, D70Z-6+3- =(5, 8/(4 1. U7:2 3(6-

!

'() ,-J-/79 ,1409- 3?72.-, *( 1. U7:2 3(6- 372 47X- 1* /+2 87,*-/! L?13? /-,(+/3-, 7/- -,0-31799< ?-908+9 8(/ */(+59-,?((:2. >1. -//(/,

H() J'$2;148*$$E?) 5?3 G%EP(e5E$?


64/102

!"#MC! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$321.! !C$))(?)! !Q8(?) @53$$%m8 c4; Q[! !F5+5 L5P%1(?) 5?3 F4;2))(?)! !H4'B$'P5?


65/102

!"#MH! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


66/102

!"#MM! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


67/102

!"#MQ! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


68/102

!"#MR! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


69/102

!"#MS! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


70/102

!"#Q"! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$321.

#28+$P(e(?) C$) D4885)48 $? 5 H4'/K$; b58(8

log4j.logger.org.apache.pig=DEBUG

$ pig -log4jconf customlog.properties

#$?+'$11(?) #1(4?+/L(34 C$) (148


71/102

!"#Q!! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


72/102

!"#Q%! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


73/102

!"#Q&! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


74/102

!"#QC! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


75/102

J*4 K$;J'5


76/102

!"#QM! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


77/102

!"#QQ! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


78/102

!"#QR! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


79/102

!"#QS! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


80/102

!"#R"! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


81/102

!"#R!! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


82/102

!"#R%! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


83/102

!"#R&! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


84/102

!"#RC! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32-/8(/4723- FJ-/J1-) ! !Q?34'8+5?3(?) +*4 WX4


85/102

!"#RH! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


86/102

!"#RM! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$321. ;/(+59-,?((:2. W26 F0:41l7:(2

@$= H() C5E? b4


87/102

!"#RQ! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


88/102

!"#RR! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32 stores = LOAD 'stores' AS (store_id:chararray, name:chararray);

grunt> sales = LOAD 'sales' AS (store_id:chararray, price:int);

grunt> groups = GROUP sales BY store_id;

grunt> totals = FOREACH groups GENERATE group,SUM(sales.price) AS amount;grunt> joined = JOIN totals BY group,

stores BY store_id;grunt> result = FOREACH joined

GENERATE name, amount;

grunt> DUMP result;(Anchorage,9067)(Boston,9838)(Dallas,8198)(Edmonton,2479)

Q8(?) +*4 EXPLAIN `4&=$'3


89/102

!"#RS! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


90/102

!"#S"! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


91/102

!"#S!! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


92/102

!"#S%! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


93/102

!"#S&! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


94/102

!"#SC! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$3243 (? joined

( (

stores = LOAD 'stores' AS ( store_id:chararray, name:chararray );

sales = LOAD 'sales' AS ( store_id:chararray, price:int );joined = JOIN sales BY store_id, stores BY store_id;groups = GROUP joined BY sales::store_id;totals = FOREACH groups GENERATE

FLATTEN(joined.stores::name) AS name,SUM(joined.sales::price) AS amount;

unique = DISTINCT totals;region = FILTER unique BY name == 'Anchorage' OR name == 'Edmonton';sorted = ORDER region BY amount DESC;topone = LIMIT sorted 1;STORE topone INTO 'topstore';

a(1+4' Q?=5?+43 F5+5 78 W5'1& 78 H$88(;14


95/102

!"#SH! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


96/102

!"#SM! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


97/102

!"#SQ! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


98/102

!"#SR! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


99/102

!"#SS! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


100/102

!"#!"

"! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$32


101/102

!"#!"

!! #$%&'()*+ ,-.-/,-.0 #1$234'56 711 '()*+8 '484'9436 :$+ +$ ;4 '4%'$321. ;-,:2. 726 E17.2(,:3,– http://tiny.cloudera.com/dac08b

! D71912. U1,* 8(/ >1. _,-/,– http://tiny.cloudera.com/dac08c

! b+-,:(2, ;7..-6 )1*? o>1.p (2 A*73XFJ-/V()– http://tiny.cloudera.com/dac08d

! b+-,:(2, ;7..-6 )1*? o>1.U7:2p (2 A*73XFJ-/V()– http://tiny.cloudera.com/dac08e

b(;1($)'5%*&


102/102

;?- 8(99()12. (K-/ 4(/- 128(/47:(2 (2 *(013, 61,3+,,-6 12 *?1, 3?70*-/ ! >-/8(/4723- :0, 8(/ >1.

– http://pig.apache.org/docs/r0.10.0/perf.html#performance-enhancers

Documents

Cloudera Academic Partnership 10