From 57a66164d834208c3628caab8895ef8bdab3976b Mon Sep 17 00:00:00 2001 From: Vitaly Gridnev Date: Fri, 5 Aug 2016 23:56:53 +0300 Subject: [PATCH] embedded jars: replace spark-example.jar we can use python implementation of spark pi job. Change-Id: I5428b814eba17153d045c34fe3852704be0cc104 --- .../edp-examples/edp-spark/spark-example.jar | Bin 4268 -> 0 bytes .../edp-examples/edp-spark/spark-pi.py | 39 ++++++++++++++++++ sahara_tests/scenario/defaults/edp.yaml.mako | 4 +- 3 files changed, 41 insertions(+), 2 deletions(-) delete mode 100644 sahara_tests/scenario/defaults/edp-examples/edp-spark/spark-example.jar create mode 100644 sahara_tests/scenario/defaults/edp-examples/edp-spark/spark-pi.py diff --git a/sahara_tests/scenario/defaults/edp-examples/edp-spark/spark-example.jar b/sahara_tests/scenario/defaults/edp-examples/edp-spark/spark-example.jar deleted file mode 100644 index a85d79cb23542c8bc48670e69b6650305db737d6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4268 zcmb_f2T+sSwhp}oP>K`xe`f7}_WJ%cv)9^deJ1*3a5D)*?M=hp z9s_-G2?D*oIJm!e;HjzPFmz(ZPXcBrre{dFfhVWAx5Uwg;ShG+V-paZD*`Lbv9S`_ z1hN2`9vgn^ekA`B#Q3=`bB*I)Xdb`6f{<0JJNfVvLdTZ3p9=)xgK%_pg82I&{5&8| z!3a+uq?11cezG)ilW;^L{QWac4UNYxoyxQ7^p%Ta?$vRPu<%nULXl=gouX)K)+$P#eOkJ6n%4aP%~!^`Gunq&NnunXSKI}K^hfwzAg{2&8xR-vv@ zP7XTABPFLS0$N)b&m6UN$g8A??W$Z{m$`&0^VmtVQeoC8A8VS8vLvHW3om-4MK%jX zvE@;9((Y$M6tixgfBD&@Dwt`5i~LzQ+a|SCcH6sUY#i#Yd1%k<^MhOBmlZ5@Bk@MY z;ofv3mLI=@L1pDPE`%d=yt9*Rkhl?dXZh{T?L^auayFyfZm|ML55?$FWg*_BWzO&= znoOcWr({N0lG)O*#0A)Qr4Yp?d}?IIS*q2*@z&_Tlw}VvB4e^kdji|BT%BITG2jmr z(wWx|{1Gfthz#qv`!d_OJV=`c0}-ekZzUV&bsv>V7%N0Pb*-5u~SZ4W^R6#5i+trsop6ArO;9Zzkk$zrBJcXJS*KG@xIV0+Gm^nO?vgd^Kp|# zhd|sGQQlnU4UenN_ztGujsiCnVqKk1!R5zNojO1o9bg?w+0y;Co0j=bZBA_b~THFk^QSol>R%J2m&FzyuF-nc!8w!7f?)c1y~`^0=1K!5zp3#EVA8IEp5Z}$eAPN0hA3xXRQ>RvXN=2UxE?EJADY$f z#Pg+d2)}p@C@|xfKZX$FuN41UV#pf(De1Yxg0>w`xy9IxbDgGKTpGZ3Y|vE{_ilsn zs|7l)5Huk%Okmu_a_E<=kGa6I(!7y)!*UgN0&jTGPszp z`1ZYK?Zw=ipLzcJ+7CrRYd!<_#oWaFNm=GIULY55X(EG2*_0KZe@q5(&CxiNz})>Z zW6Oc2hf?NKu~#LPsg|ua{YqyfzK;8A97?2Mm@_v=Yu7=ZgT3R0vPj@mARNWvFdU^2 z(4Lj;B z$ccx@z>8sjz&0rQ)L-Y`L9Wu{A%!ndQb`F5?F8SrQcL%~)XYMAYB#{Ow2m&78O~(^ zmuDYe21VVn&Np7#bSv4Xt<;@DcDzW}6En+7_6@t=$S=k{J(5kbqMlzL!PG4RCt8Rdn_U$v%H_GC3I5%bgdSDL zjrblXv@7|4F`=b?-m-9JwNqi~NT#@iTQq#75t_QB&NR2xPwiJ&i^g(JOu1DjcHQkm zx(8|C!XwU4v_}O&xQEG1xxbVA5E8#BPZzD8Ib*TBwS~d1>=u9Ptn~guV&HfmF%$^q z^!6zP%e-jL%MsiOX+Y3i5#3*bLNlj^vMO_=GmRsp66+~n)XvI0bQy_$ZW*W?bM~_} zoD=KTa|`b^AKL8+I^{I)MUjni^y|6R+LNWr(=o^`I6yH6ls_`$ze5LvFg~jyd7uhmt+O>mpJlj$ycO2 zS(WFF_#%5)Cd>`9d9QrW*8!Wzrr+-V^w!jV&d$7WOxEr+$T5QVN#m1hb6Z2Htuq60 zBBizHB~SJ?dUS2;O!8Hi*kY(C7%g?TC<5T6kjo7Gs8YS?eT!`T$9GZ!U3;Q zOL6M%su5IkXG4Z(E-i83{?OJeS=aFJx8PJ%WXv$$$)=7w`?K7OoJh_Z=jZbvG*|#q2Cx2 z9}p-;_H$9mG30FOIQPIQ>KBYSuF3wACL&&_-TYH~1Jtjq6Cs!MD5ay?UTSk6lr&1( zDpLXx4L+wS5KH~Wqws^)sKCNSrVM23UGLjuCxuk6Rg$o9Tu6KWvXF%TR%iHU`Q%wF z!Oxnr99`9|zJAu4d`NC9+Vl0L^KNBY;+8gca!lMzu=ayGqy(bJIjEkoX!~-$*3#Gk zfl+BXF+Tx{b96qS+EZN!UJ*1)D>8?s=Qhk?J6ChR)w^ttop1U~H5jwX&R}E2+viy! z9#`Yx@1iJf5kindVfwpfYOrRn*{v%!Sg~@oR=URGwyZfN1!=gK?*1+BYMVJS;>yzW zKN3J1(%MqX(qaSv|ige7{xJW?{?y;vv)j0V-2(GvPjRn9OK7?&x; za-*wUoD3ie-3CA%UxURI`N;8uJN=cU{p*Itkoc{{3h^{4Ue&HDd7GP1VH_kLu3#8b zjQQ*oGxppkmZdDQ*O&5;X&i1=;37~q-4amdAv9CwEUo-8rp6!LeX1+5jH9uvoXcckgT8Scqo5u&UEjjg^;~>wmKlv$7TrzP~qzp&MC5Mo-yDXvp3J5k-T|+gV>} z%Ol4)yc*tUG3T!M;hyCl2C64`@H^J_HwrJiZks(secoV!c&=x&S6&{d{SBB`$BQrm9u!gQApM^#<0O<@xPd4;!N ziHw!Dt;T!uhmITvOe#P`J&@JG5R<&GQPEomwrh znjRy31Bxs2LxEXTF*Ty)7>6N<1a%4Ctv|%p20IaWe7q)`0dn$6< zJzZ%AY0((bh<57N-j*+2)<0;lBuPAbC&So#a=D=NLNL{;sZQXuoY$0cwW>yL`1$iD zDU@lQFRq z4|D}yEER(}R`IRst_z*qqr1I6xf{oK=n~DzKte*w0{F9D{a-P9qF4Wnp91tmz5dnp zcL{l-R)0bH*!=1Gn|A%3;Y4cwKKh?X&0lCaHvgvZ{4442CgOJl>ZU`A0ypg`F}sHliv0V&d2BX@8kbxr!&!~IF1GYpgex&NdSNfhLhDl E0HQl7aR2}S diff --git a/sahara_tests/scenario/defaults/edp-examples/edp-spark/spark-pi.py b/sahara_tests/scenario/defaults/edp-examples/edp-spark/spark-pi.py new file mode 100644 index 00000000..9c7627e0 --- /dev/null +++ b/sahara_tests/scenario/defaults/edp-examples/edp-spark/spark-pi.py @@ -0,0 +1,39 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import sys +from random import random +from operator import add + +from pyspark import SparkContext + + +if __name__ == "__main__": + """ + Usage: pi [partitions] + """ + sc = SparkContext(appName="PythonPi") + partitions = int(sys.argv[1]) if len(sys.argv) > 1 else 2 + n = 100000 * partitions + + def f(_): + x = random() * 2 - 1 + y = random() * 2 - 1 + return 1 if x ** 2 + y ** 2 < 1 else 0 + + count = sc.parallelize(range(1, n + 1), partitions).map(f).reduce(add) + print("Pi is roughly %f" % (4.0 * count / n)) + + sc.stop() diff --git a/sahara_tests/scenario/defaults/edp.yaml.mako b/sahara_tests/scenario/defaults/edp.yaml.mako index 65c88653..f0a9464a 100644 --- a/sahara_tests/scenario/defaults/edp.yaml.mako +++ b/sahara_tests/scenario/defaults/edp.yaml.mako @@ -65,9 +65,9 @@ edp_jobs_flow: - type: Spark main_lib: type: database - source: edp-examples/edp-spark/spark-example.jar + source: edp-examples/edp-spark/spark-pi.py configs: - edp.java.main_class: org.apache.spark.examples.SparkPi + edp.java.main_class: main args: - 4 spark_wordcount: