forked from D-Net/dnet-hadoop
refactoring
This commit is contained in:
parent
0e447add66
commit
adcbf0e29a
|
|
@ -0,0 +1,7 @@
|
|||
#sandboxName when not provided explicitly will be generated
|
||||
sandboxName=${sandboxName}
|
||||
sandboxDir=/user/${dhp.hadoop.frontend.user.name}/${sandboxName}
|
||||
workingDir=${sandboxDir}/working_dir
|
||||
oozie.wf.application.path = ${nameNode}${sandboxDir}/${oozieAppDir}
|
||||
oozieTopWfApplicationPath = ${oozie.wf.application.path}
|
||||
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package eu.dnetlib.dhp;
|
||||
package eu.dnetlib.dhp.bulktag;
|
||||
|
||||
import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
|
||||
|
||||
|
|
@ -28,7 +28,7 @@ public class SparkBulkTagJob2 {
|
|||
String jsonConfiguration =
|
||||
IOUtils.toString(
|
||||
SparkBulkTagJob2.class.getResourceAsStream(
|
||||
"/eu/dnetlib/dhp/bulktag/input_bulktag_parameters.json"));
|
||||
"/eu/dnetlib/dhp/bulktag/input_bulkTag_parameters.json"));
|
||||
|
||||
final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration);
|
||||
|
||||
|
|
|
|||
|
|
@ -65,13 +65,16 @@ public class ResultTagger implements Serializable {
|
|||
|
||||
// tagging for Subject
|
||||
final Set<String> subjects = new HashSet<>();
|
||||
result.getSubject().stream()
|
||||
Optional<List<StructuredProperty>> oresultsubj = Optional.ofNullable(result.getSubject());
|
||||
if (oresultsubj.isPresent()) {
|
||||
oresultsubj.get().stream()
|
||||
.map(subject -> subject.getValue())
|
||||
.filter(StringUtils::isNotBlank)
|
||||
.map(String::toLowerCase)
|
||||
.map(String::trim)
|
||||
.collect(Collectors.toCollection(HashSet::new))
|
||||
.forEach(s -> subjects.addAll(conf.getCommunityForSubjectValue(s)));
|
||||
}
|
||||
|
||||
communities.addAll(subjects);
|
||||
|
||||
|
|
@ -79,23 +82,32 @@ public class ResultTagger implements Serializable {
|
|||
final Set<String> datasources = new HashSet<>();
|
||||
final Set<String> tmp = new HashSet<>();
|
||||
|
||||
for (Instance i : result.getInstance()) {
|
||||
Optional<List<Instance>> oresultinstance = Optional.ofNullable(result.getInstance());
|
||||
if (oresultinstance.isPresent()) {
|
||||
for (Instance i : oresultinstance.get()) {
|
||||
tmp.add(StringUtils.substringAfter(i.getCollectedfrom().getKey(), "|"));
|
||||
tmp.add(StringUtils.substringAfter(i.getHostedby().getKey(), "|"));
|
||||
}
|
||||
|
||||
result.getInstance().stream()
|
||||
oresultinstance.get().stream()
|
||||
.map(i -> new Pair<>(i.getCollectedfrom().getKey(), i.getHostedby().getKey()))
|
||||
.flatMap(p -> Stream.of(p.getFst(), p.getSnd()))
|
||||
.map(s -> StringUtils.substringAfter(s, "|"))
|
||||
.collect(Collectors.toCollection(HashSet::new))
|
||||
.forEach(dsId -> datasources.addAll(conf.getCommunityForDatasource(dsId, param)));
|
||||
.forEach(
|
||||
dsId ->
|
||||
datasources.addAll(
|
||||
conf.getCommunityForDatasource(dsId, param)));
|
||||
}
|
||||
|
||||
communities.addAll(datasources);
|
||||
|
||||
/*Tagging for Zenodo Communities*/
|
||||
final Set<String> czenodo = new HashSet<>();
|
||||
result.getContext().stream()
|
||||
|
||||
Optional<List<Context>> oresultcontext = Optional.ofNullable(result.getContext());
|
||||
if (oresultcontext.isPresent()) {
|
||||
oresultcontext.get().stream()
|
||||
.filter(c -> c.getId().contains(ZENODO_COMMUNITY_INDICATOR))
|
||||
.collect(Collectors.toList())
|
||||
.forEach(
|
||||
|
|
@ -103,8 +115,10 @@ public class ResultTagger implements Serializable {
|
|||
czenodo.addAll(
|
||||
conf.getCommunityForZenodoCommunityValue(
|
||||
c.getId()
|
||||
.substring(c.getId().lastIndexOf("/") + 1)
|
||||
.substring(
|
||||
c.getId().lastIndexOf("/") + 1)
|
||||
.trim())));
|
||||
}
|
||||
|
||||
communities.addAll(czenodo);
|
||||
|
||||
|
|
|
|||
|
|
@ -67,8 +67,8 @@
|
|||
<name-node>${nameNode}</name-node>
|
||||
<master>yarn-cluster</master>
|
||||
<mode>cluster</mode>
|
||||
<name>bulkTagging</name>
|
||||
<class>eu.dnetlib.dhp.SparkBulkTagJob2</class>
|
||||
<name>bulkTagging-publication</name>
|
||||
<class>eu.dnetlib.dhp.bulktag.SparkBulkTagJob2</class>
|
||||
<jar>dhp-bulktag-${projectVersion}.jar</jar>
|
||||
<spark-opts>
|
||||
--num-executors=${sparkExecutorNumber}
|
||||
|
|
@ -96,8 +96,8 @@
|
|||
<name-node>${nameNode}</name-node>
|
||||
<master>yarn-cluster</master>
|
||||
<mode>cluster</mode>
|
||||
<name>bulkTagging</name>
|
||||
<class>eu.dnetlib.dhp.SparkBulkTagJob2</class>
|
||||
<name>bulkTagging-dataset</name>
|
||||
<class>eu.dnetlib.dhp.bulktag.SparkBulkTagJob2</class>
|
||||
<jar>dhp-bulktag-${projectVersion}.jar</jar>
|
||||
<spark-opts>
|
||||
--num-executors=${sparkExecutorNumber}
|
||||
|
|
@ -125,8 +125,8 @@
|
|||
<name-node>${nameNode}</name-node>
|
||||
<master>yarn-cluster</master>
|
||||
<mode>cluster</mode>
|
||||
<name>bulkTagging</name>
|
||||
<class>eu.dnetlib.dhp.SparkBulkTagJob2</class>
|
||||
<name>bulkTagging-orp</name>
|
||||
<class>eu.dnetlib.dhp.bulktag.SparkBulkTagJob2</class>
|
||||
<jar>dhp-bulktag-${projectVersion}.jar</jar>
|
||||
<spark-opts>
|
||||
--num-executors=${sparkExecutorNumber}
|
||||
|
|
@ -154,8 +154,8 @@
|
|||
<name-node>${nameNode}</name-node>
|
||||
<master>yarn-cluster</master>
|
||||
<mode>cluster</mode>
|
||||
<name>bulkTagging</name>
|
||||
<class>eu.dnetlib.dhp.SparkBulkTagJob2</class>
|
||||
<name>bulkTagging-software</name>
|
||||
<class>eu.dnetlib.dhp.bulktag.SparkBulkTagJob2</class>
|
||||
<jar>dhp-bulktag-${projectVersion}.jar</jar>
|
||||
<spark-opts>
|
||||
--num-executors=${sparkExecutorNumber}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ package eu.dnetlib.dhp;
|
|||
import static eu.dnetlib.dhp.community.TagginConstants.ZENODO_COMMUNITY_INDICATOR;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import eu.dnetlib.dhp.bulktag.SparkBulkTagJob2;
|
||||
import eu.dnetlib.dhp.schema.oaf.Dataset;
|
||||
import eu.dnetlib.dhp.schema.oaf.OtherResearchProduct;
|
||||
import eu.dnetlib.dhp.schema.oaf.Publication;
|
||||
|
|
|
|||
|
|
@ -141,4 +141,15 @@ public class CommunityConfigurationFactoryTest {
|
|||
|
||||
System.out.println(cc.toJson());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void temporaneo() throws Exception {
|
||||
String xml =
|
||||
IOUtils.toString(
|
||||
getClass()
|
||||
.getResourceAsStream(
|
||||
"/eu/dnetlib/dhp/communityconfiguration/tagging_conf.xml"));
|
||||
final CommunityConfiguration cc = CommunityConfigurationFactory.newInstance(xml);
|
||||
System.out.println(cc.toJson());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -9,8 +9,17 @@
|
|||
</zenodocommunity>
|
||||
</zenodocommunities>
|
||||
<organizations/>
|
||||
</community>
|
||||
<community id="clarin">
|
||||
</community> Result: 2 <community id="fet-fp7">
|
||||
<subjects/>
|
||||
<datasources/>
|
||||
<zenodocommunities/>
|
||||
<organizations/>
|
||||
</community> Result: 3 <community id="fet-h2020">
|
||||
<subjects/>
|
||||
<datasources/>
|
||||
<zenodocommunities/>
|
||||
<organizations/>
|
||||
</community> Result: 4 <community id="clarin">
|
||||
<subjects/>
|
||||
<datasources>
|
||||
<datasource>
|
||||
|
|
@ -20,39 +29,40 @@
|
|||
</datasources>
|
||||
<zenodocommunities/>
|
||||
<organizations/>
|
||||
</community>
|
||||
<community id="ee">
|
||||
</community> Result: 5 <community id="rda">
|
||||
<subjects/>
|
||||
<datasources/>
|
||||
<zenodocommunities>
|
||||
<zenodocommunity>
|
||||
<zenodoid>rda</zenodoid>
|
||||
<selcriteria/>
|
||||
</zenodocommunity>
|
||||
</zenodocommunities>
|
||||
<organizations/>
|
||||
</community> Result: 6 <community id="ee">
|
||||
<subjects>
|
||||
<subject>SDG13 - Climate action</subject>
|
||||
<subject>SDG8 - Decent work and economic
|
||||
growth</subject>
|
||||
<subject>SDG8 - Decent work and economic growth</subject>
|
||||
<subject>SDG15 - Life on land</subject>
|
||||
<subject>SDG2 - Zero hunger</subject>
|
||||
<subject>SDG17 - Partnerships for the
|
||||
goals</subject>
|
||||
<subject>SDG17 - Partnerships for the goals</subject>
|
||||
<subject>SDG10 - Reduced inequalities</subject>
|
||||
<subject>SDG5 - Gender equality</subject>
|
||||
<subject>SDG12 - Responsible
|
||||
consumption and production</subject>
|
||||
<subject>SDG12 - Responsible consumption and production</subject>
|
||||
<subject>SDG14 - Life below water</subject>
|
||||
<subject>SDG6 - Clean water and
|
||||
sanitation</subject>
|
||||
<subject>SDG6 - Clean water and sanitation</subject>
|
||||
<subject>SDG11 - Sustainable cities and communities</subject>
|
||||
<subject>SDG1 - No poverty</subject>
|
||||
<subject>SDG3 -
|
||||
Good health and well being</subject>
|
||||
<subject>SDG3 - Good health and well being</subject>
|
||||
<subject>SDG7 - Affordable and clean energy</subject>
|
||||
<subject>SDG4 - Quality
|
||||
education</subject>
|
||||
<subject>SDG4 - Quality education</subject>
|
||||
<subject>SDG9 - Industry innovation and infrastructure</subject>
|
||||
<subject>SDG16 - Peace justice
|
||||
and strong institutions</subject>
|
||||
<subject>SDG16 - Peace justice and strong institutions</subject>
|
||||
</subjects>
|
||||
<datasources/>
|
||||
<zenodocommunities/>
|
||||
<organizations/>
|
||||
</community>
|
||||
<community id="dh-ch">
|
||||
</community> Result: 7 <community id="dh-ch">
|
||||
<subjects>
|
||||
<subject>modern art</subject>
|
||||
<subject>monuments</subject>
|
||||
|
|
@ -243,8 +253,7 @@
|
|||
</zenodocommunity>
|
||||
</zenodocommunities>
|
||||
<organizations/>
|
||||
</community>
|
||||
<community id="fam">
|
||||
</community> Result: 8 <community id="fam">
|
||||
<subjects>
|
||||
<subject>Stock Assessment</subject>
|
||||
<subject>pelagic</subject>
|
||||
|
|
@ -363,8 +372,7 @@
|
|||
</zenodocommunity>
|
||||
</zenodocommunities>
|
||||
<organizations/>
|
||||
</community>
|
||||
<community id="ni">
|
||||
</community> Result: 9 <community id="ni">
|
||||
<subjects>
|
||||
<subject>brain mapping</subject>
|
||||
<subject>brain imaging</subject>
|
||||
|
|
@ -478,8 +486,7 @@
|
|||
</zenodocommunity>
|
||||
</zenodocommunities>
|
||||
<organizations/>
|
||||
</community>
|
||||
<community id="mes">
|
||||
</community> Result: 10 <community id="mes">
|
||||
<subjects>
|
||||
<subject>marine</subject>
|
||||
<subject>ocean</subject>
|
||||
|
|
@ -679,8 +686,7 @@
|
|||
</zenodocommunity>
|
||||
</zenodocommunities>
|
||||
<organizations/>
|
||||
</community>
|
||||
<community id="instruct">
|
||||
</community> Result: 11 <community id="instruct">
|
||||
<subjects/>
|
||||
<datasources/>
|
||||
<zenodocommunities>
|
||||
|
|
@ -694,8 +700,12 @@
|
|||
</zenodocommunity>
|
||||
</zenodocommunities>
|
||||
<organizations/>
|
||||
</community>
|
||||
<community id="aginfra">
|
||||
</community> Result: 12 <community id="elixir-gr">
|
||||
<subjects/>
|
||||
<datasources/>
|
||||
<zenodocommunities/>
|
||||
<organizations/>
|
||||
</community> Result: 13 <community id="aginfra">
|
||||
<subjects>
|
||||
<subject>animal production and health</subject>
|
||||
<subject>fisheries and aquaculture</subject>
|
||||
|
|
@ -817,8 +827,7 @@
|
|||
</zenodocommunity>
|
||||
</zenodocommunities>
|
||||
<organizations/>
|
||||
</community>
|
||||
<community id="dariah">
|
||||
</community> Result: 14 <community id="dariah">
|
||||
<subjects/>
|
||||
<datasources>
|
||||
<datasource>
|
||||
|
|
@ -837,8 +846,17 @@
|
|||
</zenodocommunity>
|
||||
</zenodocommunities>
|
||||
<organizations/>
|
||||
</community>
|
||||
<community id="beopen">
|
||||
</community> Result: 15 <community id="risis">
|
||||
<subjects/>
|
||||
<datasources/>
|
||||
<zenodocommunities/>
|
||||
<organizations/>
|
||||
</community> Result: 16 <community id="epos">
|
||||
<subjects/>
|
||||
<datasources/>
|
||||
<zenodocommunities/>
|
||||
<organizations/>
|
||||
</community> Result: 17 <community id="beopen">
|
||||
<subjects>
|
||||
<subject>Green Transport</subject>
|
||||
<subject>City mobility systems</subject>
|
||||
|
|
@ -1136,8 +1154,125 @@
|
|||
</zenodocommunity>
|
||||
</zenodocommunities>
|
||||
<organizations/>
|
||||
</community>
|
||||
<community id="covid-19">
|
||||
</community> Result: 18 <community id="euromarine">
|
||||
<subjects/>
|
||||
<datasources/>
|
||||
<zenodocommunities/>
|
||||
<organizations/>
|
||||
</community> Result: 19 <community id="ifremer">
|
||||
<subjects/>
|
||||
<datasources/>
|
||||
<zenodocommunities/>
|
||||
<organizations/>
|
||||
</community> Result: 20 <community id="oa-pg">
|
||||
<subjects/>
|
||||
<datasources/>
|
||||
<zenodocommunities/>
|
||||
<organizations/>
|
||||
</community> Result: 21 <community id="science-innovation-policy">
|
||||
<subjects>
|
||||
<subject>Sustainability-oriented science policy</subject>
|
||||
<subject> STI policies</subject>
|
||||
<subject>science—society relations</subject>
|
||||
<subject>Science & Technology Policy</subject>
|
||||
<subject>Innovation policy</subject>
|
||||
<subject>science policy</subject>
|
||||
<subject>Policy and Law</subject>
|
||||
</subjects>
|
||||
<datasources>
|
||||
<datasource>
|
||||
<openaireId>doajarticles::c6f0ed5fa41e98863e7c73501fe4bd6d</openaireId>
|
||||
<selcriteria/>
|
||||
</datasource>
|
||||
<datasource>
|
||||
<openaireId>doajarticles::ae4c7286c79590f19fdca670156ce816</openaireId>
|
||||
<selcriteria/>
|
||||
</datasource>
|
||||
<datasource>
|
||||
<openaireId>doajarticles::0f664bce92ce953e0c7a92068c46bfb3</openaireId>
|
||||
<selcriteria/>
|
||||
</datasource>
|
||||
<datasource>
|
||||
<openaireId>doajarticles::00017183dc4c858fb77541985323a4ef</openaireId>
|
||||
<selcriteria/>
|
||||
</datasource>
|
||||
<datasource>
|
||||
<openaireId>doajarticles::93b306f458cce3d7aaaf58c0a725f4f9</openaireId>
|
||||
<selcriteria/>
|
||||
</datasource>
|
||||
<datasource>
|
||||
<openaireId>doajarticles::9dbf8fbf3e9fe0fe1fc01e55fbd90bfc</openaireId>
|
||||
<selcriteria/>
|
||||
</datasource>
|
||||
<datasource>
|
||||
<openaireId>doajarticles::a2bda8785c863279bba4b8f34827b4c9</openaireId>
|
||||
<selcriteria/>
|
||||
</datasource>
|
||||
<datasource>
|
||||
<openaireId>doajarticles::019a1fcb42c3fea1c1b689df76330b58</openaireId>
|
||||
<selcriteria/>
|
||||
</datasource>
|
||||
<datasource>
|
||||
<openaireId>doajarticles::0daa8281938831e9c82bfed8b55a2975</openaireId>
|
||||
<selcriteria/>
|
||||
</datasource>
|
||||
<datasource>
|
||||
<openaireId>doajarticles::f67ad6d268162079b3abd51a24468744</openaireId>
|
||||
<selcriteria/>
|
||||
</datasource>
|
||||
<datasource>
|
||||
<openaireId>doajarticles::c6f0ed5fa41e98863e7c73501fe4bd6d</openaireId>
|
||||
<selcriteria/>
|
||||
</datasource>
|
||||
<datasource>
|
||||
<openaireId>doajarticles::ad114356e196a4a3d84dda59c720dacd</openaireId>
|
||||
<selcriteria/>
|
||||
</datasource>
|
||||
<datasource>
|
||||
<openaireId>doajarticles::01e8a54fdecaaf354c67a2dd74ae7d4f</openaireId>
|
||||
<selcriteria/>
|
||||
</datasource>
|
||||
<datasource>
|
||||
<openaireId>doajarticles::449305f096b10a9464449ff2d0e10e06</openaireId>
|
||||
<selcriteria/>
|
||||
</datasource>
|
||||
<datasource>
|
||||
<openaireId>doajarticles::982c0c0ac378256254cce2fa6572bb6c</openaireId>
|
||||
<selcriteria/>
|
||||
</datasource>
|
||||
<datasource>
|
||||
<openaireId>doajarticles::49d6ed47138884566ce93cf0ccb12c02</openaireId>
|
||||
<selcriteria/>
|
||||
</datasource>
|
||||
<datasource>
|
||||
<openaireId>doajarticles::a98e820dbc2e8ee0fc84ab66f263267c</openaireId>
|
||||
<selcriteria/>
|
||||
</datasource>
|
||||
<datasource>
|
||||
<openaireId>doajarticles::50b1ce37427b36368f8f0f1317e47f83</openaireId>
|
||||
<selcriteria/>
|
||||
</datasource>
|
||||
<datasource>
|
||||
<openaireId>doajarticles::f0ec29b7450b2ac5d0ad45327eeb531a</openaireId>
|
||||
<selcriteria/>
|
||||
</datasource>
|
||||
<datasource>
|
||||
<openaireId>doajarticles::d8d421d3b0349a7aaa93758b27a54e84</openaireId>
|
||||
<selcriteria/>
|
||||
</datasource>
|
||||
<datasource>
|
||||
<openaireId>doajarticles::7ffc35ac5133da01d421ccf8af5b70bc</openaireId>
|
||||
<selcriteria/>
|
||||
</datasource>
|
||||
</datasources>
|
||||
<zenodocommunities>
|
||||
<zenodocommunity>
|
||||
<zenodoid>risis</zenodoid>
|
||||
<selcriteria/>
|
||||
</zenodocommunity>
|
||||
</zenodocommunities>
|
||||
<organizations/>
|
||||
</community> Result: 22 <community id="covid-19">
|
||||
<subjects>
|
||||
<subject>COVID-19</subject>
|
||||
<subject>Severe acute respiratory syndrome coronavirus 2</subject>
|
||||
|
|
@ -1157,7 +1292,6 @@
|
|||
<openaireId>opendoar____::358aee4cc897452c00244351e4d91f69</openaireId>
|
||||
<selcriteria>{"criteria":[{"constraint":[{"verb":"contains","field":"title","value":"COVID-19"}]},
|
||||
{"constraint":[{"verb":"contains","field":"title","value":"SARS-CoV-2"}]},
|
||||
{"constraint":[{"verb":"contains","field":"title","value":"sars-cov-2"}]},
|
||||
{"constraint":[{"verb":"contains","field":"title","value":"2019-nCoV"}]}]}
|
||||
</selcriteria>
|
||||
</datasource>
|
||||
|
|
|
|||
Loading…
Reference in New Issue