From 43cbcda7efbbfacf0445c24b77ef0ecfa1b6a567 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Thu, 26 Mar 2020 18:26:40 +0100 Subject: [PATCH] unit test for SparkGraphImporterJob --- dhp-workflows/dhp-graph-mapper/pom.xml | 5 ++ .../dhp/graph/SparkGraphImporterJob.java | 33 ++++++---- .../dhp/graph/input_graph_parameters.json | 8 +-- .../dnetlib/dhp/graph/oozie_app/workflow.xml | 8 +-- .../dhp/graph/SparkGraphImporterJobTest.java | 62 +++++++++--------- .../graph/sample/dataset/dataset_10.json.gz | Bin 2668 -> 6736 bytes pom.xml | 6 ++ 7 files changed, 72 insertions(+), 50 deletions(-) diff --git a/dhp-workflows/dhp-graph-mapper/pom.xml b/dhp-workflows/dhp-graph-mapper/pom.xml index 802c3ff21..0b86dcdf1 100644 --- a/dhp-workflows/dhp-graph-mapper/pom.xml +++ b/dhp-workflows/dhp-graph-mapper/pom.xml @@ -19,6 +19,11 @@ org.apache.spark spark-sql_2.11 + + org.apache.spark + spark-hive_2.11 + test + eu.dnetlib.dhp diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/graph/SparkGraphImporterJob.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/graph/SparkGraphImporterJob.java index 95c3cd480..dbbb88b88 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/graph/SparkGraphImporterJob.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/graph/SparkGraphImporterJob.java @@ -18,29 +18,38 @@ public class SparkGraphImporterJob { "/eu/dnetlib/dhp/graph/input_graph_parameters.json"))); parser.parseArgument(args); + new SparkGraphImporterJob().run(parser); + } + + private void run(ArgumentApplicationParser parser) { try(SparkSession spark = getSparkSession(parser)) { - final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext()); final String inputPath = parser.get("sourcePath"); final String hiveDbName = parser.get("hive_db_name"); - spark.sql(String.format("DROP DATABASE IF EXISTS %s CASCADE", hiveDbName)); - spark.sql(String.format("CREATE DATABASE IF NOT EXISTS %s", hiveDbName)); - - // Read the input file and convert it into RDD of serializable object - GraphMappingUtils.types.forEach((name, clazz) -> spark.createDataset(sc.textFile(inputPath + "/" + name) - .map(s -> new ObjectMapper().readValue(s, clazz)) - .rdd(), Encoders.bean(clazz)) - .write() - .mode(SaveMode.Overwrite) - .saveAsTable(hiveDbName + "." + name)); + runWith(spark, inputPath, hiveDbName); } } + // public for testing + public void runWith(SparkSession spark, String inputPath, String hiveDbName) { + + spark.sql(String.format("DROP DATABASE IF EXISTS %s CASCADE", hiveDbName)); + spark.sql(String.format("CREATE DATABASE IF NOT EXISTS %s", hiveDbName)); + + final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext()); + // Read the input file and convert it into RDD of serializable object + GraphMappingUtils.types.forEach((name, clazz) -> spark.createDataset(sc.textFile(inputPath + "/" + name) + .map(s -> new ObjectMapper().readValue(s, clazz)) + .rdd(), Encoders.bean(clazz)) + .write() + .mode(SaveMode.Overwrite) + .saveAsTable(hiveDbName + "." + name)); + } + private static SparkSession getSparkSession(ArgumentApplicationParser parser) { SparkConf conf = new SparkConf(); conf.set("hive.metastore.uris", parser.get("hive_metastore_uris")); - return SparkSession .builder() .appName(SparkGraphImporterJob.class.getSimpleName()) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/graph/input_graph_parameters.json b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/graph/input_graph_parameters.json index 86fca71f3..13c7abd51 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/graph/input_graph_parameters.json +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/graph/input_graph_parameters.json @@ -1,6 +1,6 @@ [ - {"paramName":"mt", "paramLongName":"master", "paramDescription": "should be local or yarn", "paramRequired": true}, - {"paramName":"s", "paramLongName":"sourcePath", "paramDescription": "the path of the sequencial file to read", "paramRequired": true}, - {"paramName":"h", "paramLongName":"hive_metastore_uris","paramDescription": "the hive metastore uris", "paramRequired": true}, - {"paramName":"db", "paramLongName":"hive_db_name", "paramDescription": "the target hive database name", "paramRequired": true} + {"paramName":"mt", "paramLongName":"master", "paramDescription": "should be local or yarn", "paramRequired": true}, + {"paramName":"s", "paramLongName":"sourcePath", "paramDescription": "the path of the sequencial file to read", "paramRequired": true}, + {"paramName":"h", "paramLongName":"hive_metastore_uris","paramDescription": "the hive metastore uris", "paramRequired": true}, + {"paramName":"db", "paramLongName":"hive_db_name", "paramDescription": "the target hive database name", "paramRequired": true} ] \ No newline at end of file diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/graph/oozie_app/workflow.xml b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/graph/oozie_app/workflow.xml index bbee2f01c..e63bbbbfb 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/graph/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/graph/oozie_app/workflow.xml @@ -59,10 +59,10 @@ --conf spark.sql.queryExecutionListeners="com.cloudera.spark.lineage.NavigatorQueryListener" --conf spark.sql.warehouse.dir="/user/hive/warehouse" - -mt yarn-cluster - --sourcePath${sourcePath} - --hive_db_name${hive_db_name} - --hive_metastore_uris${hive_metastore_uris} + -mt yarn + -s${sourcePath} + -db${hive_db_name} + -h${hive_metastore_uris} diff --git a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/graph/SparkGraphImporterJobTest.java b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/graph/SparkGraphImporterJobTest.java index c7743d684..511adf3f1 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/graph/SparkGraphImporterJobTest.java +++ b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/graph/SparkGraphImporterJobTest.java @@ -1,52 +1,54 @@ package eu.dnetlib.dhp.graph; -import org.apache.spark.api.java.JavaSparkContext; -import org.apache.spark.sql.Encoders; +import org.apache.spark.SparkConf; import org.apache.spark.sql.SparkSession; import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -import scala.Tuple2; import java.nio.file.Path; -import java.util.List; -import java.util.stream.Collectors; public class SparkGraphImporterJobTest { - private static final long MAX = 1000L; + private final static String TEST_DB_NAME = "test"; - @Disabled("must be parametrized to run locally") - public void testImport(@TempDir Path outPath) throws Exception { - SparkGraphImporterJob.main(new String[] { - "-mt", "local[*]", - "-s", getClass().getResource("/eu/dnetlib/dhp/graph/sample").getPath(), - "-h", "", - "-db", "test" - }); + @Test + public void testImport(@TempDir Path outPath) { + try(SparkSession spark = testSparkSession(outPath.toString())) { - countEntities(outPath.toString()).forEach(t -> { - System.out.println(t); - Assertions.assertEquals(MAX, t._2().longValue(), String.format("mapped %s must be %s", t._1(), MAX)); - }); + new SparkGraphImporterJob().runWith( + spark, + getClass().getResource("/eu/dnetlib/dhp/graph/sample").getPath(), + TEST_DB_NAME); + + GraphMappingUtils.types.forEach((name, clazz) -> { + final long count = spark.read().table(TEST_DB_NAME + "." + name).count(); + if (name.equals("relation")) { + Assertions.assertEquals(100, count, String.format("%s should be 100", name)); + } else { + Assertions.assertEquals(10, count, String.format("%s should be 10", name)); + } + }); + } } - public static List> countEntities(final String inputPath) { + private SparkSession testSparkSession(final String inputPath) { + SparkConf conf = new SparkConf(); - final SparkSession spark = SparkSession + conf.set("spark.driver.host", "localhost"); + conf.set("hive.metastore.local", "true"); + conf.set("hive.metastore.warehouse.dir", inputPath + "/warehouse"); + conf.set("spark.sql.warehouse.dir", inputPath); + conf.set("javax.jdo.option.ConnectionURL", String.format("jdbc:derby:;databaseName=%s/junit_metastore_db;create=true", inputPath)); + conf.set("spark.ui.enabled", "false"); + + return SparkSession .builder() .appName(SparkGraphImporterJobTest.class.getSimpleName()) .master("local[*]") + .config(conf) + .enableHiveSupport() .getOrCreate(); - //final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext()); - - return GraphMappingUtils.types.entrySet() - .stream() - .map(entry -> { - final Long count = spark.read().load(inputPath + "/" + entry.getKey()).as(Encoders.bean(entry.getValue())).count(); - return new Tuple2(entry.getKey(), count); - }) - .collect(Collectors.toList()); } + } diff --git a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/graph/sample/dataset/dataset_10.json.gz b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/graph/sample/dataset/dataset_10.json.gz index ce0b9709be06a9fe6749958303807a15b481a8d8..0da3c4071ab705e39e22e33967e39de56f788855 100644 GIT binary patch literal 6736 zcmV-W8n5LaiwFpv<9uEK17u-zVRL14UokK)YIARH0PS7flG`?xzMrQ+scNcbs?=(d z67_f2X*-!t*@??`rYbw0N`WLKp+f=+NOq!RUA1|JeF9(l!WUDy%dKwn6#FE54giXj z`ajbiGB$Qy8MOou2j@FD0C7GX{M1WmLC?fO_U1qJxVYdsk5bl~FKC*xK76xaO0i`3 zMZ!{6uq4{(UzmvNvV|)1qBrk(gIUkk#NXFSUNAv*$sYVr|4rk9OMwl?Da~^Zt@QFu z&a<^t#XMFr&%dYXw{ykTGUtU<8>p^}3YxLI6wc!n%b0#}B3Lo6+dk24=esiYiPw~R z{kq?Sc8hWi>S6`YSYFU<-JAR2H1xu7I-btF8FU(33cdf9$v67{hO_BxKKFto36mfW zg212okr&OPWV9F#<0+es!vM7<6<=~e(=&L$KYI79{!h?~Oj8yEFbgF!c*IY=Z`lT9 zk!vOrNfkbAKF8-lV;3|nv9piZdix9NiadhW3z`yk^wsI-r%ub8rMeYW7ch_>L2Eb5L(_t3BmpNtE2263>iH9> zvnq=9pNA)Z4nBo{JN0onDC6^Zz>*Ta23hov-<8l7qMI;LSOY~t=U3I62i_p`yn&BW z9f#wqeqbV@TCuVkNm4i8|^rY%tY znV0ICD)1v9`YkvuJDPE5Lwkz_Pr1cdkM{mv%P&=V&eK$qPniIwm>r(E4|{p)L-LP}A9>(K)^FQaqt6nh^3l!H&?G$#ooDJ7>eD+P~f%#96P zG8?|43C}a>U}zJD3$8%-2~8Xft-`Q;mjM7Gk^hJ`?v4l z&H9;4x9iRZ2~hM`rg1Fl9z7z)atf%!3$JPV-xCeO1MZkI4?v4fFZGj|FFDJ zv|vkq!}~@?xgh3=`<>eR;jjaU)+sO8H~R&e(jOFY5dWSbch*FefG?zifE-YZ=PV%> z-y7Bp4&Khet9Q~x6Jz60o1*JYG%!hi;j7>X(#l|!{h7!QU)k>`pp zS4CY&)G)yZjCiKc*-nA@H5a)ZNdQ;MWAJEuqsm5&W5s9zZVWtP29oKZu`Qh+N1G!N zABp66H1Licw)Yr(d!RLC{_@%5FAgO8at+hUiLP*k80JaeWHPxX#=2MFbq)OG;Iiih zql30T0m`glClBF zb9{+afpd$U+EK_<#ueG5cZt;W;G()&?crxNCDGiuSjH=u%^1y)RZuy`!(Zc4VfQ!H z?@IefmefRy9E1dS+nf;-cERI%e&j#yKI_$9MdL7py^4@yhpE{`vY(KRNs#iX$10uH{IE$uypX{%9O5W>GXvg2)@s!f~>Q;xL{>y<0evw^jcS zawP9Tx0}rFNw*7zv-e52GxD@ew+n(v?-Jc^a^(9*e)!TK&x6@K4Bs!^?m50rKHI)k zNOpTlRnZ`sRe#nrGgF%V_Ooh`+HPanIhxs>XlCeMF7!M6r|_B&iy5+kUs3`C6Py}z z`?76r+C7F}3yDx++x$YhxJ9dYJZFW2pj`-@U+Pr6=!~37vBY;@@|uLkfde!u;uGu=g|CBuMft_1T$fk1-?S*4#ZNx>$t*x-BXAGdBKQc7_8a;LP~)` zlnKW^TGd}jvO+GyE-nL_fHwk?H%BB+y>4AhI+yyZ+sMZPN&@n^|&y_hio`5OwoZb z1@1<5IZWX^vI|qVFvSCgDQ4bma&4GmF`bRZqj*H)Q4C)#m={m{(Qp{~laT6T;x~pV zZmZtSFva-3VTy3{KBE-lPDClD-jzuKFiJ7PqyR&2K6=kd0d9O9?UQ@&q}#{t>OI-3 z#DDu)*ZO(@Rhrf6K*-esVNL+cxpScr7aI9+XO`wZ@D3bSZ=$$k9W8UrGjP8!M=tbm zac4SO1>>3B7|k#5j5=W21SS~yJ}k0e!iTpGb23d&E052aih0c4xamr4s~$8`Fug&4 zzZC*GJw5sKQB4kafv-Q1J={I;^$Dyp0s7}MV@mdGUz zOeliqOZ(iMq}z9b(JMTE6(m|wj6~&BDR?=x@i&Iypf!@ObjZjWFFXRRcaA^v{rSCW zE#Vq`cu^zt%2gS33&Tg*_H7}plo)YZGi9a9wr^l~Eil^^RK%zfHpeQ5ST#6!C9JMN z#)xm<>P{3Cz5qQL3}cT| ze>RI|Vd5v@csvb#x|k%xI1FzIVcuH3gCWdtFb;?J3}H^j?=ObAlc)U{rhj7$)1S?U zQy0T@F-#Z3{8*-z{SXb(z+FhDbs+x669rmxNVJZG^Glrp0p4aEp`zvUzqsmFGUe{r zX(xsRv$3lj#)e*IIZzyv!ohPZlj6yKbk@9@tQw0h43ow(TeE^&vPEN%^$AoZpO-pj zosv_T-vF#5Q*_ZJ?oOuIgL$7!0k5v``~}P5qY3bTR-GNbXHXvZy}+u##_$L)x%n6| zQYKc^p2yYg-At5#_Sf}`9exjeW#5%N<&ols{y^c49%N#T`MsiMBve#Krjx5D9Rh7r zUA!)&7i&hw7;5KbO~t_4rJy9Ef1L^cX#De&DJeEV#kschjj`YwK(u40v4o2lpa9j0S-} zeP5)9TAubv4>noGm86H^d^DT~eygO1zk>&YAs?f6(rLuVnGo_~7yq+QIVe_@xpeos zV>CEM!$&-CxbSVqKscofwtXjDaH`$d?Lwcvb|AExB_f5TAM`EZI$Ng;2)65(nh`z+ zMVnB3!UUeT<<3Dl3WI~^4itv7{h;s>vGY68GUx=(8zR_}n$L(~nO8^8W4j`&kJ;b6 zN_HfME*iz%Nes_!W)tKLQxI(5G;=VByCp#Hz=rIHgBX6d&a}zW+*SHpZhJ8zbRnJ4BoJ6B^I8vP$G|On zGC~g;o5~{I{$ehj-o7C@$NfE>;Nh|+YyDxm^*t^GvMdf(f0}WD1@kh~VpYmBspWfL zl=`0Q!-fzpkx><6$3&q_;`x|@>Dj(54*FhauFx&Y=$z@C?I>~qGjFDoP&87to3s=b zJ6%d=ls|AC2Xqf)uy7IcHBB>}VI-#!Q=l+>HZnfwrc9usT7qP61#XoX3(M#RFhm9m zt5UmHRFghthL?4IjVg)#cO@VNKb@C)(LmS_nvG(pC=?`454k9spF&Ala5O(#62w(F z+6w40D5iyjk~e`U0> zn+i~{0p3?&Ohj*-%7%St=aAcBfyly8H(a{DwJH1X6-BUg&lJhnxm0~y%luB!#6b6U zqN2ubtg|SX$~Zh68))|TRZpw`85#zPb~ru+_?hi5Y1M|_-3BSZbmFwk4V{k*+?39Z zN;hQtD<}*;X*C>Ljv;wLJ#$;ZtKPmIcJt?zNUmj~hu7}*1^6887Q09xt3f%KoV6w^ z!?&7(5e=W4I1#ioRw5hCXuA|UfnY!^5niZJ)2w8rza8Y(XKC*N9b-4K&+J4I%T&8t ztAU5~y_lVcl9OeFTLyo_6jrdfzO#ao7dn5AS;5rla6ZIdxtQ?-Qqd+yzIWt}old(( zMf*2{WDZS7Mf=#XbyT#DP|>_jQ_y z)LSN}RLDNO;FhX!(|Pr0O(koT?m6DrgYw2sWme)Is7vzEdeHMJRo{YN7fX^DSLonq z6%MA8VdoM?OX`Rjt&fe^rCn*6x|iY6aEC$%P!I`Aq;gKFT`11Z$P>QQi8~&#h~mvA zXQwBRZ@t;XVe|g68TC=>ruYTL3yCncnK0bF9-jacSrbr=Wan=5bQSnm0rKqGkOn79>(Fsi@dPNcj-Rx=Q%2L9OVe!UDF# ziRmx}eQw`oc(oCEoD#EVr>Q=;J(w5RC%`$~zRmjp0rZh7!$!fTzWf0yB&Y`o@Fh}c zJ`6MuJv!h10;X%oNs5)i@K{W{Tv?C`mhHAjG+!mABGHT>7?Gvz^M|!Ex*LlL*nM1km26}xC8P`U*u(#Bop|o z1w#)5K!ajtgdPz0!0vY0^n$yWCS?wHX&^0EQQ4_mEK|CsyJ* z1((jz7V`+6R%eg0dE9EH_L~~FR=;kN4~@ZAG(_+?Och2(TN?LcxBrp|qP9_J6scW2 zF{Pl(;CsPK8;Y6Gm-W$9TiYXA>{woBzuVoou-yZ)fu=IKsg3%obM$hn0a&tb( z#a!CW@)FCHbS$E_Q-9&+rMnfSZ6K=T1Nr51mkyW?xGA1zE~`tcKs;wQwspZ3#LRV-C?m@%6oK%VYZ7Z`U@{{m}Y|slV_$esOl6oNxd@A0ZDq>a?P<` zx@a3+N`Hib^Mb{q@*7kWRlpFz?vSUDNKs{pxW@XB1@m0ylzd*|OY|N&XYbZ2THWk) zb>gHtSB=TD0-Xcl&da5y&Cq(vuc#A;;{@}*JmrUa!X!nQVaEi*D=jI_LnyXyW6g}v z>TQljOod`77ezzgCYO+AKtLy}eWx9LL8m}jLwU<<;9|uiOJ1pOnxZhq;D2Vg}ujm6SiVw2+gc`!*rwfx)<{*WWZP4aesaQ?> z+Uyh1I73E?<{sRhjLMoPk`y@RR}yF-(C(RlhZbdFw`TSb`-IDE`!+GSq~vchU+aB6 zKfuCY!lZ`oaC6xDGraw)0q9ejD4z7Io}RKt05RGKbWP~|R4`D!EBX4W(Pd~ThR@L_ zG+~-Ck(l%>Oj;mNb^ToiC!OtwTD}Ol7h~{y7Gc4&mSOU4OTg2nRkoE5rf@CW7r(j*u(J)t>SnoC1dn<5JaIy{WU zOeceq=y~v0Fhrzs?NWX%aBI0Df#t+B$8`FyS3RAf>GBXnO(&S-yp%>@O?N`UkkPlI zBx5mdaAnmyUOiyz1ysKtyvW}(eM629w)CSRp~YO;{@L;KHFg!Ci(nA0w(soCcrb>g zW+W{r^t$R%N(#Kc(XyU28EYX;jxpOP_DJjSRd26hP2mAcF1dCGn9coFk5GKx#rCZs zcqzG5)Y8*5X-d#tqOc~dY)3)_d(WA&x#9aDYKGgGn8*&A|GWfOr*m9Mq@|%Z+r9y~ z@=HFnk6_RU29+>F0y>t#tf$?OL0S(g_yY64ZS=FCLv;HNpPT?HT$CGzS+iV-@4@p3 zg-)lAIrLGY>6k+wF1C(2)Dh;;wA0KXHw8N8kYf%#By(sIde<_C7IDOe)Efp^_rr4p< zuCv1ZEcnN8{pzsj%EWJzx5RA~hH&P4123qu_vqX- z&5R_P58@ILc7_c3xl~I=KBGv*O5`<3y{-q#@fy1b^WAxk`|#J4L1O`mAvxskB*s2@ zB=u#AP-TDEpOoMZ0JGAh_WI-mi&RNTo4;Z*uNP(1xRy~T3pcS9=@?Bn_OjZfQ?3)aH zVI=zfIeI#s60Be<*6MI6!CXo(mlDjS1pCM)SeFmwk3{TqJhKnka2(I96FjqVr+H@1 zBRig%c%5W?%gMvd7~pgccjfr+)&}UrT6(4QmaLP+q8&N!;!1P=p-66`BCyPwR3e&qg zAX)|DF}xy~H;}{}S#)r;J1#T`Ia!q%x+{lHhXZ&b`u zJeaQ|){R$1Lh<;bKKoeISHs-#1;()_x@Zn}XHT3;(PY8yTTN=MrcW?TfQRh4Iq{4q zWuH-%(oEB3jUnO9DduJtnypLiB2LnbE|t6wQ6i$}5+72)I~yJ-f7Sav28(yC)MxBh zc>57kdUiMGl+ArSsJN@^o66XS-P(+7Y+ZA>9}iVysWnSEb}`jX(au=*fNp$wsXZVs z@&1(UTfEVyzD6gJrax7#0!s_&lNU5A;g_dSaQjXieL_{llyr2Fdku%X9P09Yr&C84 mX+%97UBuBvK2*90Mg_Y~7jYih(M4YW>Hh(z`8FgaPyqlr-ru7D literal 2668 zcmV-y3X}C8iwFpvV|-o!17u-zVRL14UokK)YIARH0PS7tZreB({=ZK__}4DrD3UDM zl8nV7ZD+QfNz!iJL3dgV0!pGRP9)JoQc2xT-e%uqUu+L`iDK86q;0aX%b-Cml7~Fx zxsc~OwDPGJV~JlB3F`%)dbGHpf`%FC1qsds=|f3EI4ALSK}<3tNgU3r8d5}ETggiy zdqK}M{9Z$;>Svr?kOEgNdr;9m<496gsDYyl3qfI&o|p@o&ls0M#F-G6I6DtGnK3~n z<8vUdnie=Gw`nY*l;ou9I4+0`y0MR{vBhU^vCmg!i^g;`mV4O~A zfwLFT!Ta7F$$x`YVOd5Z5KO{Z4qbfeos&86!e*q18Rn|9Ab{>*tP7l#YOW_FrufY- zCs5QWK~ZMe6?8aLQ-N;U#+GSVj^p}=wrP3!%z}^B$Oh6sN}SPz622(Mn9(I+)g_-v zgvMubJ|k-+ab0`W*P41eO;yrUTFR0}DJsr;AUFTO93EKS8`BJIKX8>q-}TzYlH)=o zjOTbI!N3e$+ptESq97ljbuEhbFdD=3=F)v?wD}nk5vQ|uti5ErJMzahq+Q)S?0D;h6em^#N-e-P`^r;F)3uDkBYINZQivG3I-&d z2CMlu=G?L6ys_NC8o|63PdSlTsoyEAy_n%jK^k^0FE0&cWtc}Pg$WvrPY+)ny*RWD zQ}J{bM~9XFz@r#*QLw)S@)xilVxm5*Vymp zttv-aOiMgnVt^FWrNMtFrWqCKiox3+Ru9_MUMa^|77{G1QdCifzk@+6IJP*B0@E}% zc(OQJUX`k3nW?7pvzjuYQMK~hR%knw_I$VYeEmhSYMJhrOn=Nu8lwr#%dA?_F?vSG z3>_293tCLa=oDXos1GKRomZk+BYij;T{0doO{yzhvJuRVA`@hY_>Yn9o>3**>CM#d$TfB zr>xpU+O=Gc3iTpxUPO|Im`@qR+O_i52XMhk9yO}0j{Fkhrsk=Nv+BRLa~O zx7uY~Eyq<2fsd4lN`(~TCR;suBJb4>v?6RByJ_7ZS&!5GyY)2 zj8?eK8kAF;rZ&yE!_WjUsX~t%TYXF}Xw+@|^yBqs*tf+s0#|1dS~RiFCbTteX+uzy zd8iI$b>)<>PV3(PcJ-G}TK?Z8aWiyA18ZI4#!ZsMat28h#(tQLoY1o)cW4h?&-8s0 z`&$w>TgmS)adVd(z~J^t04s1D!yDae4q))0NdRwP`onEW0PCKU0BQ+b8mRp8Btjj?YNT?fU4TX^zhn7t zLn=Mf_SYemL*KLsw!-1aa_rFcJdcpXjiS&UTEwxUElA~7@&~_Se)7t4TnG5t5V`9s z=KFQU+yd<}ycz)BlPl(p25&}}{rg3i-=$?=T_0y3)D?0xM$h5-C#oRT`uSfJ3ZABA zF{9P!VEs0FzDu(!rr8w-lbKm zf0j@{>S6!;AJ``{(nlv)%9LWE3USd_1I{a==Ou=bOHh-RrmT8-&6+3RMZh2#o5zaL z5{Q4TC?`>xQz@Zu#Q@31=xs%*I-A392cmZ?@qF<%J*<<4&%Y8rAKDr|-vFeD}?qNr>%IeiwP|Y8)VZwPl>e4omT!)R z=DJ)Kwxci}x+7@XB6R!VdRcn+l~_7_RPVy z+|D-gNAX6b!SF6=0>?B)BmX|r1T{Yol_nUB%%QpU?<$!0oF+JmaZdYaqQ0#{1>t@4 zlQC9>qK{6EBV&x75T29Y*}(||+hl$8!srqR6Qdzp$R1RE>fC|;iiq9?bnaj;Wx5hu zJ}q;?tFLOP047raGfR+2S#*90XDfkdfkMjEh7i$$Xpnff1g=414HEC@B5IIW15sZL z5;t**Mxu7w1!yGdJC^@uBx+<@!{OR@dxvpi*%;%*OMGJC1e*k>z`#vjvIT zO8(HdjC}>J`;P7CTgH!2Q@7kQHe0yPu?@%Z?*bpXreXW;H--;a@bh4|j9VRE1s|HO zf)8yc7&!L}A5LI@o@O9>g&dyX4>VWkUZFrl^a`sZo;tBW-H#b8D^&b!H1D8^WgksS zfs2BS(Gwb_nQHbL&(vFKD*Cs0TBx#DgmCmS1MYZJR&{Ynx%}8iM}?ST-a(QJOx)yQ z0iwLYw+1S|5~v()eEFCLwRXkzYM^reWV#ZlJgQ+xIO-P_T#08ui=(u<2P-l{s`r~! z45@ws1dfQP4WfE^&4C6kcS`}j5pZd`I^fp8r3NlFaJeJEWfLJiHsDhGQGL7hyRgkQ aw_9%jaN0DrX+E-?!v6sLz=ksEjQ{|q-3)C2 diff --git a/pom.xml b/pom.xml index 6594943a8..861b7a2ea 100644 --- a/pom.xml +++ b/pom.xml @@ -143,6 +143,12 @@ ${dhp.spark.version} provided + + org.apache.spark + spark-hive_2.11 + ${dhp.spark.version} + test + org.slf4j