From 32fd75691f17af421c8f5e7c4a8b4a0796c4cb9c Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Fri, 13 Aug 2021 10:15:42 +0200 Subject: [PATCH] refactoring --- .../project/utils/CSVParser.java | 47 ---- .../project/utils/CSVProject.java | 200 ------------------ .../actionmanager/project/CSVParserTest.java | 31 --- .../project/preparedProgramme_whole.json.gz | Bin 15986 -> 0 bytes .../dhp/oa/graph/hostedbymap/GetCSV.java | 74 ++----- .../dhp/oa/graph/hostedbymap/TestReadCSV.java | 112 ---------- 6 files changed, 21 insertions(+), 443 deletions(-) delete mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/utils/CSVParser.java delete mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/utils/CSVProject.java delete mode 100644 dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/CSVParserTest.java delete mode 100644 dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/project/preparedProgramme_whole.json.gz delete mode 100644 dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/hostedbymap/TestReadCSV.java diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/utils/CSVParser.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/utils/CSVParser.java deleted file mode 100644 index c53cd2127..000000000 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/utils/CSVParser.java +++ /dev/null @@ -1,47 +0,0 @@ - -package eu.dnetlib.dhp.actionmanager.project.utils; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Set; - -import org.apache.commons.csv.CSVFormat; -import org.apache.commons.csv.CSVRecord; -import org.apache.commons.lang.reflect.FieldUtils; - -/** - * Reads a generic csv and maps it into classes that mirror its schema - */ -public class CSVParser { - - public List parse(String csvFile, String classForName) - throws ClassNotFoundException, IOException, IllegalAccessException, InstantiationException { - return parse(csvFile, classForName, ';'); - } - - public List parse(String csvFile, String classForName, char delimiter) - throws ClassNotFoundException, IOException, IllegalAccessException, InstantiationException { - final CSVFormat format = CSVFormat.EXCEL - .withHeader() - .withDelimiter(delimiter) - .withQuote('"') - .withTrim(); - List ret = new ArrayList<>(); - final org.apache.commons.csv.CSVParser parser = org.apache.commons.csv.CSVParser.parse(csvFile, format); - final Set headers = parser.getHeaderMap().keySet(); - Class clazz = Class.forName(classForName); - for (CSVRecord csvRecord : parser.getRecords()) { - - @SuppressWarnings("unchecked") - final R cc = (R) clazz.newInstance(); - for (String header : headers) { - FieldUtils.writeField(cc, header, csvRecord.get(header), true); - - } - ret.add(cc); - } - - return ret; - } -} diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/utils/CSVProject.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/utils/CSVProject.java deleted file mode 100644 index 268d5f28c..000000000 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/utils/CSVProject.java +++ /dev/null @@ -1,200 +0,0 @@ - -package eu.dnetlib.dhp.actionmanager.project.utils; - -import java.io.Serializable; - -/** - * the mmodel for the projects csv file - */ -public class CSVProject implements Serializable { - private String rcn; - private String id; - private String acronym; - private String status; - private String programme; - private String topics; - private String frameworkProgramme; - private String title; - private String startDate; - private String endDate; - private String projectUrl; - private String objective; - private String totalCost; - private String ecMaxContribution; - private String call; - private String fundingScheme; - private String coordinator; - private String coordinatorCountry; - private String participants; - private String participantCountries; - private String subjects; - - public String getRcn() { - return rcn; - } - - public void setRcn(String rcn) { - this.rcn = rcn; - } - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public String getAcronym() { - return acronym; - } - - public void setAcronym(String acronym) { - this.acronym = acronym; - } - - public String getStatus() { - return status; - } - - public void setStatus(String status) { - this.status = status; - } - - public String getProgramme() { - return programme; - } - - public void setProgramme(String programme) { - this.programme = programme; - } - - public String getTopics() { - return topics; - } - - public void setTopics(String topics) { - this.topics = topics; - } - - public String getFrameworkProgramme() { - return frameworkProgramme; - } - - public void setFrameworkProgramme(String frameworkProgramme) { - this.frameworkProgramme = frameworkProgramme; - } - - public String getTitle() { - return title; - } - - public void setTitle(String title) { - this.title = title; - } - - public String getStartDate() { - return startDate; - } - - public void setStartDate(String startDate) { - this.startDate = startDate; - } - - public String getEndDate() { - return endDate; - } - - public void setEndDate(String endDate) { - this.endDate = endDate; - } - - public String getProjectUrl() { - return projectUrl; - } - - public void setProjectUrl(String projectUrl) { - this.projectUrl = projectUrl; - } - - public String getObjective() { - return objective; - } - - public void setObjective(String objective) { - this.objective = objective; - } - - public String getTotalCost() { - return totalCost; - } - - public void setTotalCost(String totalCost) { - this.totalCost = totalCost; - } - - public String getEcMaxContribution() { - return ecMaxContribution; - } - - public void setEcMaxContribution(String ecMaxContribution) { - this.ecMaxContribution = ecMaxContribution; - } - - public String getCall() { - return call; - } - - public void setCall(String call) { - this.call = call; - } - - public String getFundingScheme() { - return fundingScheme; - } - - public void setFundingScheme(String fundingScheme) { - this.fundingScheme = fundingScheme; - } - - public String getCoordinator() { - return coordinator; - } - - public void setCoordinator(String coordinator) { - this.coordinator = coordinator; - } - - public String getCoordinatorCountry() { - return coordinatorCountry; - } - - public void setCoordinatorCountry(String coordinatorCountry) { - this.coordinatorCountry = coordinatorCountry; - } - - public String getParticipants() { - return participants; - } - - public void setParticipants(String participants) { - this.participants = participants; - } - - public String getParticipantCountries() { - return participantCountries; - } - - public void setParticipantCountries(String participantCountries) { - this.participantCountries = participantCountries; - } - - public String getSubjects() { - return subjects; - } - - public void setSubjects(String subjects) { - this.subjects = subjects; - } - -} diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/CSVParserTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/CSVParserTest.java deleted file mode 100644 index dd7e1910f..000000000 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/CSVParserTest.java +++ /dev/null @@ -1,31 +0,0 @@ - -package eu.dnetlib.dhp.actionmanager.project; - -import java.util.List; - -import org.apache.commons.io.IOUtils; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import eu.dnetlib.dhp.actionmanager.project.utils.CSVParser; - -class CSVParserTest { - - @Test - void readProgrammeTest() throws Exception { - - String programmecsv = IOUtils - .toString( - getClass() - .getClassLoader() - .getResourceAsStream("eu/dnetlib/dhp/actionmanager/project/programme.csv")); - - CSVParser csvParser = new CSVParser(); - - List pl = csvParser.parse(programmecsv, "eu.dnetlib.dhp.actionmanager.project.utils.CSVProgramme"); - - Assertions.assertEquals(24, pl.size()); - - } - -} diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/project/preparedProgramme_whole.json.gz b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/project/preparedProgramme_whole.json.gz deleted file mode 100644 index 620e1abfbf28cf33bc6307fc9ada17ce4a048a71..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 15986 zcmV-&K8?X2iwFpXn5JL=18{O>aA9&~WKeQ%XL4a}ZDn6~Xm4y~E^2dcZUF6l-E!MV zlJ5OJ1!69?JrShGq9n`hJ2p*?*<;HJQQnz}*oX}fERq<302@D2Yj!tcVqf80o%25X zBJ(6Cvwnc8ViiSz_#xfC=xB>1P+3`3S^4|rfBr7^!r#6B-P^a97iVvN*ZZ9p%?bSd zaqwpFra%7r`uz39>r40#zDoTx!0#saADp2dFJgC1;NxT!#pyl&4*m|@aGAMF{ELM6 z#=uPyf8l#>>PI-%H1d3sx`E@Z+#n#~k|fT5In%Wpr#e7TFbw=|^l3c2nf~do1NgT_i8|7MP{QQ#M)G)jz4O2S)9WVCRr04iB;1Y&TV|M|BJ^10$aba>PiQ_2tlQsM#^nz@TUu5BPR@=Rb43 zhXDT&zfT>1PTYikIU|8j7DIMQbm#8H<5 zv~b2*9Bl~w*iRG3^HcxtB*gc1z=={@U~9e|kgV*m^S5V~BY39nvCl3e#*aOq4UWwt zS@~W-*t;J|l8^wFA|P%^V*1%H8O#y>{RES8{PiY|9*IN3M?a3jH3^Y78xk*~c=FMFj$s)23H$2NpT%r1wSNBZND^g0kwb7zd>|dyhku=}+!W|eHn3+C z_)D20JbX-it1PX)5e_Ij|LpwAu=CGg=by{_{*L$|jO5`InA~wgoGo!?jK**`*{HLw>GtnxU9Wn0 zt1##C{NmhlBfM2_ggbWR@W8_fg?}Rfz;w8D;ErBo1XciOaFXXFC2Noj5nMq|mJnPg zNq)GSO~A=ddd}0z4+vgzG2C19Dq9ghb^?C^7v`GaTLSWk5(9n-bMagpbG-q90rDb+ z52El9=FPSMXcDV13ZkXIy-3Yp9M_|d1LsvS(`fcL)NDZH{hltz$?>o91@et?LOn3& zCTTLD3e=T)nS9QI)bG!IIN3mH=*G`Ta~!$e3RnhW*GX_|iHY`{B-`N3V4|~UXO7fH z8@vG%ocIa^Uy}MBe3AIekV-&z{)l(t+<~hxTex_@>C_{T;ckFeBHXr%ehj*x3&Mdt z;%-uR2E0-RcUJBrKAJ?E6&;3dZz^Hoz5Mo$77IcLK*j~+S6HmjV;u9Temiq8e*2dQ^c`cL2y49Pwwpe8Jz@uNpS!Do<= zkfcvhjEY8;V?GJz8OTE)))~|g)T&qh1~%rW;t$|A34u#O)%J@=+#}(N+A4HL{JQKp%-E;jr*H?s zzg3$arU@6SSTM{8WF>xmmkErwWU- z)ju#PLMdBhFxmz-04PI142<~XQ{wT2)xGU{%oV_Tkg>&B~&>|s)?U6V<-!8d1jLTid?z>ICjQg z@4noQhd0jTW_mxl|9U@mzFfl}H(&m3ct81a;|y;;IQQex$D1#ozWnj8&MVw3`jX`C z!$}B=VVZ$m4Vz^WhS6i85a%lJbN-SS7v~TPZT_-Kv?_jpi#PAA(M)BRPlg|EoohEq zox3QFm{+{xsHT~sdZL@AcPo2xaB05p&gg&g&IE;rs-R#Rf$BqK?TTAH_|KRu@d&Wf zu_kjkpka++9YVD@YFsWS3a=x-L7(Nj(lU+82dWu@ zR1^5pqdxe{` z6WA!YE+IW6e~#i{4(9L#4@3fk_VPOdh{PRrM-~BdZ97xRM3g?3&1x?TfFUsS3KXc+ zxA$?Y(Uq{LHQd?VMpI@+#kz?SKg?-1)6!mx+!_DV2|8s zikwnv&^X{fm(Br3vwUP(2ok?XiJy}V3Fky4PDqmlg;%*zA1=Q&n(cRBqlC zr=~hC-kOg0fNrpJgI|ruBXg%8h>bNmwAnI*ZW!fL+`7}I`#sa{F1(S4XCGCYYK!~D zK=WR_RRz1nwXr2!?qi?^qa5+l4gExfsW-6;LJc&s9_+06(HAijn%06u(flemCQe9D zZ^O$51K^n-piE5s0NoE!?Oe#-Y?y>R+&_ZW^M8I&5*0?jLdkmxfOqzNr-s1#&@*usfi4b(;X4HF?1W_He3UL&zx72+q2)l2Ma4n`-MajJ`*}{8UqO(IBlmZ ze~}LETt9Gr{z_GlgJ)1Zx?MLdT9n*oY-;z}=R7bgC55r0fW+phY1^^#y`MXZi22^s zb{{w2lT_8jG}bGe(<;e-389 zp)ud%JW1-#eUdmLd9YAWQ-=8T$fVJe8_!`1&XNiNAs34&fjzK_qingt@A`^wrvgt( z9s7LdG@C~*d>Xbwt{Q_rv&W#{DVdC;FHB&evzj>05bUCEW6&0gYKcJ`->o`{-um$?XM!5yc=qP=ysWY`Cv~$uQ~lS?Ki4c1AZP zI+|$SEIUI<;e@2uf}f;dAG45aK;x}$uGDNYI5%FU_2acof1ccbq$v~EU+x^H5C0YK zt)ri@ZGUhIX(0#KlDHe_`2^;rB`2Gs=vS~m)zDE)Blygl@k5fkMm%aqd9j~xAu{(9 zHv#&NxRSf#Yz|H3tu(AwS88%LoYAXVy{yw12!TZp#g(_p))=*MH-5@eL@jlfm6ok! z9q4t0a}$(FSCo>&Y9mnWXFukXib-MNA2#tui%FX!T=PnpLz3tE9vRd8aKkFVkgmrM z(e(3pYe$rMu(Ap-Sv8Z8zKsUqh=&i0q7zrB-pbgfyJc`@p)l2l!6Sjq)`NB8F8#uP|9axW8a+j^_maBg+{pPSc?*Ytj93 z+sc$Q5n;OY2Yl~s+kAY~2a@*ZG3p`Gcv}JFy&h>?EbK={Ljqx+!PPQfV}$>l7Ww|D z zwwmE+WMEuI>;9Q$<+8#o_tEtOcjo8XdX|u#+6#QQal`*01^z`qs5hsQ;>BpAyqNnj zXvB`N3-Z#jS}C-rxVK4%24rP3VPr{GrwOqo^R$;AiKTO>$)edF#z3c5**CfQ@O65B zHyM6%K8=ST#&^??lUtZXd1t52*#QP2&aI!YdjaAbqP(Y=TKlUI37Xa^majSnde9Wc$=8&*RAiYT{o?X z`Z_dK`=r$!J=mcFnal%fndvd7+&pCWy|foN4o*lju~IPbkV=Y{)7ZwZk-$UFM{Q)N z#>1sKf_J7vc@DwMT>D-e&2YbQS6W2Pz-vMpcM$qI4LeZr z&NgdLf?A=nA3s|8D8?jJzo)9JNbn|(7C0JxkY-N8IRQx*JLxm(u?o}{{~k#p^E`*M z%AP$IB&P2>pr##g>{v_R7rK^(R;~RrTfWNqmYiLK<;==MBwsa0P>~%F?o*LSBNrrt zoSP0B?1u_nJ-B?E%G)ESN=p-2{=y;3mJOu}D~vp`&8b&#>$nRD@?Q-+%BI zzNJQQeN9O-NtVay3`|6AB?|jI&sLN&6`pY9oy=TzCQfo$YRcQ0{>B)m8hmffW#DIU zdQ~@4o6X7lu>0rGvUr6_dtb`1X6wm)pXigiIDSp z&wqSpE^&CPd;f|UHiPPZ^jS?LBJgozhZ#ZwU_dfcnR$ah8s81;TgJNUyYgSji5+#$ zyXn%Yh9{uqmYO~DmpTyoXW}LzcI=|Ul%R}RqMl+)&^)@EvVru{YHaB;!?X&;+LkfP z8r(8U$q36K%CEA#gvl$YoBK$M{=`{J*gtWte(@M=}1FT7Tbi@!1FSleqk9tM_*~x~Wgcv_`o07m6$|J09zi|4MvT|3 z;?SEc=CJ>SOA)jtpPe;~Koh$>dK=DI?F;q$(hoQw6s_MkAntvhj)u${M@-tka zVAenU@Bc*e=qY({`=bm$8RqfY`lv4KL4z$KoN+Gb~Mtlwt zhtB5!3QFq+j#4D~6bme7_9C-|HLd5Jor6s^&b5wlsp^=7>gcuLR-NgcAHOQS2Uvat zjI;*!;sCiEX^P$f0nRZgu4Gz-|lC*4X zbAWTJpX&(3zYlvXVSu40Q|D=?yYXCY+tBK5Y4l(*$aZA`@&GN(gYwf{95ASEQ@YW9 zk8}*-zLT~N5Y_h~Ohd)CwL4I>9aFB0zF?N$9Br~uCkaA2_ifo=8mQ+zYq`yBd|Xe> zGRYy8WgDf4Xi5abPM}5*O>&H9pfzt`iNgHCrb*Msn>1#Gt8Sk=ouG2JJx$GNWJX?_ znG(g9*uo)ph=ATw0lR0;b!?q_WtX=RU3@QxnZEe) z7fm|qD!bG4kf%+JH#eKcB@D^0sUsQ-!YS4wTlnXk6fGj~Z;LbY=8!ja9?z_cCEbQC z4_E_rCmEX)vfACZN|rj70r&Hlu|$a|^gUkBw~A8w*L&wr#6nH>C!)BW&#wmcdI14EhVq#1JH%dVBi%5%LxnR zPV`mjujWgrkp%T7ut<)}EieoUB3X3eC34N8ykS(&MCfuff&wyt9^4*o-@F|)^K*DM zYkBAB)we(`WtgpVrlxAk-8@x~v_)GO=-IfuK>_Z^O&rZL_IfP`!($tkszn|rj~zT< zz>2lqT0sLO>3K{$JaQnT@Gj&H#R4s=1c#{kNQ%%63Y#eOmEL@d1jQURd8#Jdag_Spaxsu1l< zQNDIivluK0E9bg4_!C2Z!zRf|B042+CA;QL)wxkh)77cEs#mpUNsT(+URAVtyJ!9h z!yeoezB)Ia!9T$ojzRlKX(m8UzpO@PR9ZE|kZR?-`%f+1#6Gncnjl|=pno&4cqK38 zyjFp^-oDFG9{O<1czyDA{C)LS8h*HiMJ8SQ%T+3_2Y2pnQog5En?dL1{*QI(irAmZntTZR4yfg&m_1G!c*mWN=#)xc z@O0&-XtZsJM|1sjhmf*(2feA`Lv#>rl@HeeqsmM9W?j8L+lnGFMNBj|Y+qna z$J5~*Yk6)VsCrBF7hbI%086>u#g5WkTn0VW2Hkc!4GdbsWefJ?K2yb*_2c0;b+U5f zJQMyrFe6|G+BXk@Smb0Qe~NKU5|YA5N9S3~U>U2~G!^4~P& zOUXgPBXaFxI1>NjA&6_ESSs<{5mpqUu-JbB$5wn;hI*~lv`Hb~a-28S%Gsa+F9_b*$FFtrt27&InMe&=!HjUCI>0UX zCPU9-Z33LwPad2FLdb2W30Z$t*>teJv7|0BPVZ16YUeDDe-&+<66@4Q?VK>5| zT7Krdx}9F{)%SRMdj~5px%s0b;-sT7trPz9%MDE~wbN!3 zM;nmta|ZB`qp4Dfh)6s%Er=H5`WvQ9|CenDJ}<3W)~vB=m^ylZ`Z&V)wZ+@DO?GAG z$(7m&dK|83wgm|HIRd97MG=GE{*4yyiw1Hs^|fW>MR!)k2R}E@%UX9#}5%gKEll zT-ArZ_jFx%ht2y0D+~1`2pc+$qDNK=t?T=(>PwC>7ozn%1wE=~Zr^gDqvnhm7Gu>| zS0&fmaRlQ;4$wi9?<$;E1M%uHKeq)0&DFrFbP2c6=b30p^@#N7%fRv64U=v89tOb< zGRa*ESciW58RVbP{D6(tlMNC6EpNPCH0x_&?$rW?$2AMpS7kvtd#fkYP19`tTnIYs zT#&k09k>QF3NJ!*CZ<#!5UApL98|Par=Aq%DQ8Cr=?pl|N8w39uTe}kctXOId(gq& z%wIztPDN3R7X<;U#7*Fqh9d=><3%no-3+}nX(S4~SqpCZZ2=9!;<0QzbjHI)zoNf| zC*Wg21JBmjz-g`T7}SeFb7#U{=@+aM%SIWHFQ!;JJeA*CEu390h++Zw#T#$!LU@~+-f4KKbG*SOIMH#-w7pob zm95&UjTIl*-pw+)aY&XSuixf;t0p zaABag^N{^uGd}_z`{6gy1P`Gw82-g14}3X4N0{&_oqj2J^XjLcCjFo9C%@OcgF07Q zbW%1n2QBx;(f)@A&6aneQd%v!D zRRftkb)ptaV@N2)+}(NQoFvZ7z{K_(4G7h-3GuNZqZEf3+YRs15+Re>&Sv zXyQjX z6zV&(%n#@#4S%5`9|Zmq=g7kcOx)=k>g6{#9GR(gYSqFjd+37nKq4s%bwasQ>bFNpZvc!@i`ZZ-8*jaid4$nKYRBp<5GzH8h zXWCkoaw)phQba=mT3(7M-?CcWT%~*^YZcy(1R(n}nyGExtXhxgX$p$t63$9kSg4|N z<}8Y_>0tp{e!O;b*=ZbV1eY66T^I{O!lhboCP4#XK21zkyhcGfh-B_8Sw^3dV&gr%9nykF?j8(!k=SGd6g=vhvs+An^!TX|=CXHQWsoaBr zCG&=990d*FY1El~Uuh!E=HFK!Nxf){m2{V{i76=(l^K^-2m2CeJ$@=_J26SJAf<^X zn`{>NUhv#sVA{$&cZFA4-rX&L8z8Y-uin}T(ehZ$ro_2=3|~%^)R$ndC%Vm0t*p82 z@oMdEwDqD=W9QY+1_rWFbwL@B4mv0ODRl^k86M);*|HN}hFZ4C=d4ULk98Na zqB^fyOxtL6W*X+u_5sfgedSIcp&IAUG8j3H=;Fx9rii7W6k*~I)J9*x7e=@^6HhKs zS+B9uy%5}dVQv%iMuiR!I=Gc%B zG!*xt$LmCK_UPW7lMM;yd73s%Bterf@UhiPH!Gsh$h}U%qOWepmEdFP8C6@ZMv2m* zil##}(5n$JHZ%SuLxXA|@RVqJHOtT9-!RN7dZh3Eb@JZ%;cJL?GyfVhvEmG=XQy7d z7>cOp4_oS`I_@E1Ni)b6k3F?Gzgn=tP^;Bgqsm8PwJut%N|x{S=p&qdZkR>0zp;o* zQYT8rK9=YXGOF31))%T)rv=^OQjZee(%nFuhcJ2y$efE3){=^XCoL8{Ry8Lct2&T+ zQ87OQ%ywD3JN*Rvn6d$DW@k+{^-fMSK)8m0XqLujG-@qd?Rg0jb3@Xqw**vTsknB_ z^4a&Hb}PQ7lmI-q`%%|o=~g+?G!L&S(sTKGsobgK5cWf0fpMl@_I(g zRU#wz%l_!(Ba(!pXPn(nfeBnwtZi-MFV>a`p+`iYh2t|R8*?|}>g<*1;j2VccR zutJF=v-b97=2kh5^f@Q54nj1=&X&IU&e)NgB2k&9O-J zz^FmnxiL0v(aAedpDGvY)t*nCA8OU_nYHDlnvgNM{{XC6qSvH-d8af(R0yf3zoPU? zt>oNRe(_cnGq*HNlfp5aRFOH;=B=bRT!6lFbPX-dmIq>nQNkC7Pwq2wKjLM|F8*IKR9#cv z2TZL}0&7RQ!+AE8NZVyA7d?^~E^>VN%#C9oOmv5KFZ8JpNU<9tzoRjy|N9d0RX) zoQ1qZoLh8{zANWMC#dm-ej4R)|5y?p1hqzno5|=yE2_S!HkBM$CzYDMY6BG4@SJ%| z9BYgcwmkMMKqX`Q zHhK0|zKhQ4(`n0NA-eG-K;yWou(XudZvojdz!C;!<1WJ}@pB=}{_18Cwjp*L0j{!} z*18aV_8Lyz>KA5b(3uL`vhOk<_vUYWmuougkG4E8Eg^hJ@~F+K(DUGI&Bqj>SUo_k zb8=_g_?RQ@jRni~LM~fxgR9qPiX8d^VzWlL{f3M-L1p7s8XHIz=th`oCeYQM=D8w= zjf`jl2*9vG4z&cDE-PH1 z>F+!(S_D%mmQBDX#{Uq%!GNb*ge?WEpU4Ny67K)wzvnL7A~Cw>gzlOI&+@n)tvMIX z>=+FZo(Jsy5=R^05!c8=QW4R1cQ*Ss$DnHY@81|3wBj&Z_KntFZX#h}7llDc9#*JR zP?|^A52Re!-zeU4bp{;`@N(5DXmFA?M4=bW@|V-ux6%c~r;R1hmt7xDkyeRpj=r7i zDjj zAu}hNAbQSW(4q~;Q^f-@8QpW4P5SM({FTykl|ZAl$uDvWr_DSl<*zw(whj0A zZIE`&-mvvE)Ovt@CJ2WYEx{lJ;exFlK`tizwx_KX-FVJFp!EEbxm1vHAiXWqvW1Hm z0>x|wLf5_*M}4MByJ5uY0;3RT#0v3JH(nB!tmTJ3!bap$g5yMpWo1kfg@Qi+(TV^8 z5dySFSt65J4socIX701ZG{51j754eMS+k0A#;a=&(qOx%gC&1vwe1B9}Lik|NWn69z7)wZhw^F zC&L_1a6bx5m{fGg{`4WRjcA^rmRezsBM!D3)a5{81q6~>H^g9pmQghSwo=?J!#Dy* zsRUD6Ohs>I3aV@QM>9TMy zEwK}9vgH}p1vsfPM_pb^2JO}YGnJi)dREwvu-)oaDv#ji3)SK-9JasUWTmiJxL_{L zjiJ9~rJ~QE@sl(+Ys@_A>LP4vMw-fMx);y3`r0xKBB~#AO&Dg*LBVI}TcjO5s2Q1u zhmA_?DF0=1>T6YMYgcHc$!cDF8}6jWcQyOOl}}bL#{lgY77uO&Z2v*lev+^d6}vii zXA-$(-*pVG{2E!)xljL))lhm^4bIJ2HrT^TO))n0Svr3YTQ*7agatP29SM9Yl0N#& zl{0eT7~v(VM#tLsQ0&z?>aZC#+tun2+Ej63?96*c%fVG(enNAAJf&`w+x1$Uy6tGO zku(cExhsOZ4&#sxiW?~^!m5^c?l;{=!^Mr$ubRG=z71FA*7e@=3{4hGZuOyAE%mjb zw9rozer}JdD23XP=h>>#nUYuXpF5k=7XKys48-pmG zb02LwfdiBgDYuEK1?W;IP*xg`I@HItXf|Sks1KG zIWPWPH|jpXIl!(sc)ejr+X*oK4GOp$ute$wx5TeFR?l-9=4+ zNS-M5n(%<%MLeF$K4)PWxhO{5oT$HHS6CM>d0D!MN7IPbHLT&tWNX&kgQHz@pDp6} z&4ERrC)x;VS^V#S?f$+Nm2a=_T;0W04I9vW`CjTBY>MD1b1}_Wl`6Jjj%n5?g#Zej zRfI^&G^vN$Ltu`2MCbr4Rc~&@f;FH-tY{?%z?bukTzo1}WG*hZ;m8 z?0Rylzv|MSKdI<9xwDjdNfxe%^T*LFN|X1_9sVD)z3BKfafiZ~RoZS{=O`XB*<;NB ztUf1I%2}+%w|Z#|Q>K1MV1R(j;bjobumBQcW(X;l!epOeD=#rk9`A;1jmj5lU5pI= zz~3^2im%aF;jXL+sKI!Wu?57^`EwKpbEl6Xt02R|pq^|!45>eO5paN-sqCc^4=BE? zM&@5iEJ+iQkan2#$S0C?F%b<;&4z)X?5BHL?=9e15&TC&6zj&0eTk%MChP9Es@90L zk+@QTRN#wF@>+R^H8RhI#ItM_zxwH?N&n~jNsT~h-Q*=4H1kdnU`x}fMYC2Is!sBbB~fo^#JVlBg+334|RbEU@0eyvJ{LQ zM(87mNi6PI_#~Lqz7TvO&&&?z!%0sAkuH)51LCU_l&VEv(!3hQ-Waxq+}!W0)#Peb zOCA2cCe5vh7Jauly);CRYah8<#KHuK6R^?KAxDnPL)8E%Oi6BZjsR1NUR7f;f|PF+ z3e^a+`Okm+ALmBivWEOu!}sl~tp);G)JC^P*P5ExfO9o9^E$$s3UM;z4i#z#iUOhS z)a}U@o0wE$?ol@_sW2lAlgxJ446LQSQg=}bdggj7LaRI6`8>u#54!AqxRF^0(t5kpW)1zjAmgiv3s>D18_2O1qVl$OOYKIMjO0E2`F`S71u7T8m ztEG%Cju{XK8U3bwmO)tc3sBViz?XCThzJY+1ECMQN~ zH^?3d={|;!Smc_9v_D*o~M=i{?RPtA+svQ}%cj37B6iH>4b-nz;W9bwaRK78E6gq^pQSCFKwp zx@Z_3`nGPZ9`!zKienubR;Vf*M!p&Qx&a&O5$N6DD4;^5+*qWP~Mw~ z9wge(MQDDagA?c)=&|DIUEPFVjg6(+pLva4vDzkHHT$iptpl}`MqtV<2}P4$yFrgh z@!V=eav0L|R_a`|sg99RUB~$pJ@xTQ>e0eY&y8nMm?!fwuULVVci_CLUGMEts1==? zG6>GL@R(D7odxWgSFr#=eudQ0?8Q-%^cMsy|3H%7IbO8B;brzC7=C4xg@MQWr?jXGJ5)bRxT2_q>-Ri9l$K9|uuy1J&c4}_Z z>gg?Q_+0z#w<0?oo~YRi2DH@-tiNd8*V2VQ(|!T8kdz7#Rw%#)t5z7myIauA!cFvq zmA=e<2{M}ex(OmbW&N)FB)2mc8QrJ+Q4Y&fF~Gp5Kt;*ik3sDc$a*NOyPJsZ&N)hQ zRaNqxt$T9CSj1ExK@1 zS8CJgfn^GJtiNSnG=NCZctbauwN>A|n`sVLE0J20Ut8?#%-+vOD@j?HN=@*0bZF6` zC>Ee8;>Ymq--sDYwd45L{#|XG=1S{rdYhe)7Pd8~6I|*}tLrSLmMg5?HceZBMdk_) z>4^L{MXa%hU;D8P@t82*FwcMzF-iU=_$jS!z$`JQS_v&l2B2VN4Ta%zWZSv*aPBEC zSW98v(_@X6XKr>2UFg?^BuB-N^Ef4+U?BIFd$4nc7}fJ#qOd5I3XE`>?P7_ zK|9+$h=NC0?bilD^(b&LkpI_)9NwH%Z zjbq-34@Xg$BEa@7h7+oJU2gASqfc)B=#2j|8l!u5G^Vw0f1Z38-{9xB!@K**XmUHe zpM1HYan_dSWLu$444acVu9@=1OTM1aoksH}KyHB@y%B0t%ap5Xx6wJG6~~ArxOYS; zVjonM{1zXtNlY;UR4kM0@%>i){@zbjQquQ+dkJlM&v$B>EM{?XZYdNfGDdoZUyI%G zN!ph$`Xmb6Sc9KB=2VtLC)PqP%V8v{ve$@yaG@4gj}t^I@!{@*o%GLt{J-%X#enhS zE3!`ANRc;m;&;Vi7V+PiMRzWi4* z(B0v)Z5p1h5wf*Tg(lO9ImFEUG)y6v9N8BRiyD8Rkdjn84VnLFYm?kneoY6SGe_#8 zQ3E5|_QI(lr&7eKyT*_gin~S&9U8hO*9HwM;lG2Fn0_9cnkScMvGiKs{#xz2w}CRA zTVV?H4i8c?0Z5GJ{sK;FI0w5q-(>DDnaeMZQ&-BrhC$>#h`V+JvJAVv2lS?eSz_Ok zz^A!wMdi8@@jjG$E#BL=ZO=|acCl%qm#VjYL@Q#vEKL~&Ay$p?3=>(LM3FwPbfR^Z=Z)lGP0|Nc9AJ{)Z1O?x?u9uYzG zCT1D=MGBA$tf_w3Qh(WZX_10iOrOOgp3I4DJgxOM>wx&9gEjkdpSZPIePhInQL* zYgL{ZGiX_34F!pNJerO_)oBT})g(g@eyj~I7=2ojmvpBxSK;|~56@2uWUk_<+n3-L57EKY zYLOq4{&q&a9a`k)I&hapm!JM+^D!WKw59?<#Tc6N-RvHEt1|4){ zvOL3sQZ*y#6Y4+tGuV3vWjb?bxXz5@zbmk78<5eFv5<1uVTEH;*@#z2_7(#oy%kW?@aO zqH*dj2<)49&X$y2|kDA7-w5Q6~lN~h! z>MK4w;%TaP?>lqS=wJsXro~>;H_)EXoUTD^6$D=J4iw_DsK&xL9KDkKX3bai2FAeUZrz*8n0XU#dPS??}`dM(SakaWhM&`|(}{ZjS;G3CyQzy=ko8TdnmXSkO&aF7x#n3* z0xL?A5YZ@k@Pi;xV7rT}(5FRB0V-p@0ML@IM}3VykETOj2vs~&HSY8>0(q?ysTGLa z$OjEK(w3g3T>)H=an|s*C6gPXp(;ADy&$7%QIf*{Nx~HDMdW1(I*MT{VSzmJA5&ih z^ZlTO$X{}7iBo$cEw;dz*3lyypOc0TV)Oba^5Uxkv$x{X7{xA=x>T+)N!5+eFGpZC zAzo$-Eyv63fVfoiVs_`h9or#&iP%<>0(G6*)=EkP{kS2&rp^-Uzu5sppKOLF@_mtq z%k(HLYA?Q4;UFcgOFo%i5M}e?nNw7_TjtNPYU8c4L2FfF6U$ex&)z8M{76RJjnFO8 z9V92S@jfJcBn>m)tZ7|nRpLN|2!+D5sg+Mw?xPz#L3@Q+rh4 zIjZE^eK3fCJ;!O@pWNVtEBlW3WsZi=Z_A?AP>H753~z9%l`|uYC1s1GjoitMjU7p} z;c>Y|Oi+Pq-{XG}DF4BeZ48P3K*}@>bekEa7*XSp_X5QC%Z^HpnuJ*0QP}68FgP0i zSvd3bnu2XdYt`3usm3+Hm6{a4s{GV2C~X?yr54mSR@|{s`$ZH>zSy_EoMnE1S35@# zcWHHrQc=q7Gkxn+7(5@Vb#+oIR133(i#|y45S(L-vhXN-%N*^dRXv0Edd845`62)>!mh2BZdOMuIA`2AV(s1uoxdyvInQx07o#4{9GdcCH zy)xS+S_4Jl!@{Ne8p5zY!2bW0#zBKZ4GZ)->CrR7*0lrHpER$02p= zOC~sW&;gS<3aPe87lcXA_n^uw!7OCK08G&pLYk~+WIpGnx}-=U5nb ztgfBo;E#$ye`d#^R}s_iKRDCc@o}pl&MSn}gD)?=QfrU7qld=Dc5D-@0&GM}!zXz^ zp-PI&t|HIfV^t{|-StKd3d!v9(24^q008M++6!0b)N+Wb@#WaijI=Fb!5%zU<;Pt# z10Fkb;<1i%%xA|g?FwV z37V{~C)D}U$VDdBdk1(xjq|y`^yvX)FX$EZwD`0~i#ZF0eZEfB(z>2hC^44Ad clazz = Class.forName(classForName); - - ObjectMapper mapper = new ObjectMapper(); - - new CsvToBeanBuilder(in) - .withType(clazz) - .withMultilineLimit(1) - .build() - .parse() - .forEach(line -> { - try { - writer.write(mapper.writeValueAsString(line)); - writer.newLine(); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); - - writer.close(); in.close(); - if (shouldReplace) { + if (Boolean.TRUE.equals(shouldReplace)) { File f = new File("/tmp/DOAJ.csv"); f.delete(); } diff --git a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/hostedbymap/TestReadCSV.java b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/hostedbymap/TestReadCSV.java deleted file mode 100644 index 89259d814..000000000 --- a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/hostedbymap/TestReadCSV.java +++ /dev/null @@ -1,112 +0,0 @@ - -package eu.dnetlib.dhp.oa.graph.hostedbymap; - -import java.io.*; -import java.net.URL; -import java.net.URLConnection; -import java.nio.charset.Charset; -import java.util.List; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.opencsv.bean.CsvToBeanBuilder; - -import eu.dnetlib.dhp.oa.graph.hostedbymap.model.UnibiGoldModel; - -public class TestReadCSV { - - @Test - public void testCSVUnibi() throws FileNotFoundException { - - final String sourcePath = getClass() - .getResource("/eu/dnetlib/dhp/oa/graph/hostedbymap/unibiGold.csv") - .getPath(); - - List beans = new CsvToBeanBuilder(new FileReader(sourcePath)) - .withType(UnibiGoldModel.class) - .build() - .parse(); - - Assertions.assertEquals(36, beans.size()); - Assertions.assertEquals(1, beans.stream().filter(e -> e.getIssn().equals("0001-625X")).count()); - Assertions - .assertTrue( - beans - .stream() - .anyMatch(e -> e.getIssn().equals("0001-625X") && e.getTitle().equals("Acta Mycologica"))); - Assertions.assertTrue(beans.stream().allMatch(e -> e.getIssn().equals(e.getIssn_l()))); - - } - - @Disabled - @Test - public void testCSVUrlUnibi() throws IOException { - - URL csv = new URL("https://pub.uni-bielefeld.de/download/2944717/2944718/issn_gold_oa_version_4.csv"); - - BufferedReader in = new BufferedReader(new InputStreamReader(csv.openStream())); - ObjectMapper mapper = new ObjectMapper(); - - new CsvToBeanBuilder(in) - .withType(eu.dnetlib.dhp.oa.graph.hostedbymap.model.UnibiGoldModel.class) - .build() - .parse() - .forEach(line -> - - { - try { - System.out.println(mapper.writeValueAsString(line)); - } catch (JsonProcessingException e) { - e.printStackTrace(); - } - } - - ); - } - - @Disabled - @Test - public void testCSVUrlDOAJ() throws IOException { - - URLConnection connection = new URL("https://doaj.org/csv").openConnection(); - connection - .setRequestProperty( - "User-Agent", - "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.95 Safari/537.11"); - connection.connect(); - - BufferedReader in = new BufferedReader( - new InputStreamReader(connection.getInputStream(), Charset.forName("UTF-8"))); - // BufferedReader in = new BufferedReader(new FileReader("/tmp/DOAJ.csv")); - PrintWriter writer = new PrintWriter(new BufferedWriter(new FileWriter("/tmp/DOAJ_1.csv"))); - String line = null; - while ((line = in.readLine()) != null) { - writer.println(line.replace("\\\"", "\"")); - } - writer.close(); - in.close(); - in = new BufferedReader(new FileReader("/tmp/DOAJ_1.csv")); - ObjectMapper mapper = new ObjectMapper(); - - new CsvToBeanBuilder(in) - .withType(eu.dnetlib.dhp.oa.graph.hostedbymap.model.DOAJModel.class) - .withMultilineLimit(1) - .build() - .parse() - .forEach(lline -> - - { - try { - System.out.println(mapper.writeValueAsString(lline)); - } catch (JsonProcessingException e) { - e.printStackTrace(); - } - } - - ); - } -}