refactoring

2020-04-24 10:47:43 +02:00 · 2020-04-24 10:47:43 +02:00 · adcbf0e29a
parent 0e447add66
commit adcbf0e29a
8 changed files with 1474 additions and 1283 deletions
--- a/dhp-workflows/dhp-bulktag/project-default.properties
+++ b/dhp-workflows/dhp-bulktag/project-default.properties
@ -0,0 +1,7 @@
+#sandboxName when not provided explicitly will be generated
+sandboxName=${sandboxName}
+sandboxDir=/user/${dhp.hadoop.frontend.user.name}/${sandboxName}
+workingDir=${sandboxDir}/working_dir
+oozie.wf.application.path = ${nameNode}${sandboxDir}/${oozieAppDir}
+oozieTopWfApplicationPath = ${oozie.wf.application.path}
+
--- a/dhp-workflows/dhp-bulktag/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob2.java
+++ b/dhp-workflows/dhp-bulktag/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob2.java
@ -1,4 +1,4 @@
-package eu.dnetlib.dhp;
+package eu.dnetlib.dhp.bulktag;

 import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;

@ -28,7 +28,7 @@ public class SparkBulkTagJob2 {
        String jsonConfiguration =
                IOUtils.toString(
                        SparkBulkTagJob2.class.getResourceAsStream(
-                                "/eu/dnetlib/dhp/bulktag/input_bulktag_parameters.json"));
+                                "/eu/dnetlib/dhp/bulktag/input_bulkTag_parameters.json"));

        final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration);

--- a/dhp-workflows/dhp-bulktag/src/main/java/eu/dnetlib/dhp/community/ResultTagger.java
+++ b/dhp-workflows/dhp-bulktag/src/main/java/eu/dnetlib/dhp/community/ResultTagger.java
@ -65,13 +65,16 @@ public class ResultTagger implements Serializable {

        // tagging for Subject
        final Set<String> subjects = new HashSet<>();
-        result.getSubject().stream()
+        Optional<List<StructuredProperty>> oresultsubj = Optional.ofNullable(result.getSubject());
+        if (oresultsubj.isPresent()) {
+            oresultsubj.get().stream()
                    .map(subject -> subject.getValue())
                    .filter(StringUtils::isNotBlank)
                    .map(String::toLowerCase)
                    .map(String::trim)
                    .collect(Collectors.toCollection(HashSet::new))
                    .forEach(s -> subjects.addAll(conf.getCommunityForSubjectValue(s)));
+        }

        communities.addAll(subjects);

@ -79,23 +82,32 @@ public class ResultTagger implements Serializable {
        final Set<String> datasources = new HashSet<>();
        final Set<String> tmp = new HashSet<>();

-        for (Instance i : result.getInstance()) {
+        Optional<List<Instance>> oresultinstance = Optional.ofNullable(result.getInstance());
+        if (oresultinstance.isPresent()) {
+            for (Instance i : oresultinstance.get()) {
                tmp.add(StringUtils.substringAfter(i.getCollectedfrom().getKey(), "|"));
                tmp.add(StringUtils.substringAfter(i.getHostedby().getKey(), "|"));
            }

-        result.getInstance().stream()
+            oresultinstance.get().stream()
                    .map(i -> new Pair<>(i.getCollectedfrom().getKey(), i.getHostedby().getKey()))
                    .flatMap(p -> Stream.of(p.getFst(), p.getSnd()))
                    .map(s -> StringUtils.substringAfter(s, "|"))
                    .collect(Collectors.toCollection(HashSet::new))
-                .forEach(dsId -> datasources.addAll(conf.getCommunityForDatasource(dsId, param)));
+                    .forEach(
+                            dsId ->
+                                    datasources.addAll(
+                                            conf.getCommunityForDatasource(dsId, param)));
+        }

        communities.addAll(datasources);

        /*Tagging for Zenodo Communities*/
        final Set<String> czenodo = new HashSet<>();
-        result.getContext().stream()
+
+        Optional<List<Context>> oresultcontext = Optional.ofNullable(result.getContext());
+        if (oresultcontext.isPresent()) {
+            oresultcontext.get().stream()
                    .filter(c -> c.getId().contains(ZENODO_COMMUNITY_INDICATOR))
                    .collect(Collectors.toList())
                    .forEach(
@ -103,8 +115,10 @@ public class ResultTagger implements Serializable {
                                    czenodo.addAll(
                                            conf.getCommunityForZenodoCommunityValue(
                                                    c.getId()
-                                                        .substring(c.getId().lastIndexOf("/") + 1)
+                                                            .substring(
+                                                                    c.getId().lastIndexOf("/") + 1)
                                                            .trim())));
+        }

        communities.addAll(czenodo);

--- a/dhp-workflows/dhp-bulktag/src/main/resources/eu/dnetlib/dhp/bulktag/oozie_app/workflow.xml
+++ b/dhp-workflows/dhp-bulktag/src/main/resources/eu/dnetlib/dhp/bulktag/oozie_app/workflow.xml
@ -67,8 +67,8 @@
            <name-node>${nameNode}</name-node>
            <master>yarn-cluster</master>
            <mode>cluster</mode>
-            <name>bulkTagging</name>
-            <class>eu.dnetlib.dhp.SparkBulkTagJob2</class>
+            <name>bulkTagging-publication</name>
+            <class>eu.dnetlib.dhp.bulktag.SparkBulkTagJob2</class>
            <jar>dhp-bulktag-${projectVersion}.jar</jar>
            <spark-opts>
                --num-executors=${sparkExecutorNumber}
@ -96,8 +96,8 @@
            <name-node>${nameNode}</name-node>
            <master>yarn-cluster</master>
            <mode>cluster</mode>
-            <name>bulkTagging</name>
-            <class>eu.dnetlib.dhp.SparkBulkTagJob2</class>
+            <name>bulkTagging-dataset</name>
+            <class>eu.dnetlib.dhp.bulktag.SparkBulkTagJob2</class>
            <jar>dhp-bulktag-${projectVersion}.jar</jar>
            <spark-opts>
                --num-executors=${sparkExecutorNumber}
@ -125,8 +125,8 @@
            <name-node>${nameNode}</name-node>
            <master>yarn-cluster</master>
            <mode>cluster</mode>
-            <name>bulkTagging</name>
-            <class>eu.dnetlib.dhp.SparkBulkTagJob2</class>
+            <name>bulkTagging-orp</name>
+            <class>eu.dnetlib.dhp.bulktag.SparkBulkTagJob2</class>
            <jar>dhp-bulktag-${projectVersion}.jar</jar>
            <spark-opts>
                --num-executors=${sparkExecutorNumber}
@ -154,8 +154,8 @@
            <name-node>${nameNode}</name-node>
            <master>yarn-cluster</master>
            <mode>cluster</mode>
-            <name>bulkTagging</name>
-            <class>eu.dnetlib.dhp.SparkBulkTagJob2</class>
+            <name>bulkTagging-software</name>
+            <class>eu.dnetlib.dhp.bulktag.SparkBulkTagJob2</class>
            <jar>dhp-bulktag-${projectVersion}.jar</jar>
            <spark-opts>
                --num-executors=${sparkExecutorNumber}
--- a/dhp-workflows/dhp-bulktag/src/test/java/eu/dnetlib/dhp/BulkTagJobTest.java
+++ b/dhp-workflows/dhp-bulktag/src/test/java/eu/dnetlib/dhp/BulkTagJobTest.java
@ -3,6 +3,7 @@ package eu.dnetlib.dhp;
 import static eu.dnetlib.dhp.community.TagginConstants.ZENODO_COMMUNITY_INDICATOR;

 import com.fasterxml.jackson.databind.ObjectMapper;
+import eu.dnetlib.dhp.bulktag.SparkBulkTagJob2;
 import eu.dnetlib.dhp.schema.oaf.Dataset;
 import eu.dnetlib.dhp.schema.oaf.OtherResearchProduct;
 import eu.dnetlib.dhp.schema.oaf.Publication;
--- a/dhp-workflows/dhp-bulktag/src/test/java/eu/dnetlib/dhp/CommunityConfigurationFactoryTest.java
+++ b/dhp-workflows/dhp-bulktag/src/test/java/eu/dnetlib/dhp/CommunityConfigurationFactoryTest.java
@ -141,4 +141,15 @@ public class CommunityConfigurationFactoryTest {

        System.out.println(cc.toJson());
    }
+
+    @Test
+    public void temporaneo() throws Exception {
+        String xml =
+                IOUtils.toString(
+                        getClass()
+                                .getResourceAsStream(
+                                        "/eu/dnetlib/dhp/communityconfiguration/tagging_conf.xml"));
+        final CommunityConfiguration cc = CommunityConfigurationFactory.newInstance(xml);
+        System.out.println(cc.toJson());
+    }
 }
--- a/dhp-workflows/dhp-bulktag/src/test/resources/eu/dnetlib/dhp/communityconfiguration/tagging_conf.json
+++ b/dhp-workflows/dhp-bulktag/src/test/resources/eu/dnetlib/dhp/communityconfiguration/tagging_conf.json
--- a/dhp-workflows/dhp-bulktag/src/test/resources/eu/dnetlib/dhp/communityconfiguration/tagging_conf.xml
+++ b/dhp-workflows/dhp-bulktag/src/test/resources/eu/dnetlib/dhp/communityconfiguration/tagging_conf.xml
@ -9,8 +9,17 @@
            </zenodocommunity>
        </zenodocommunities>
        <organizations/>
-    </community>
-    <community id="clarin">
+    </community> Result: 2 <community id="fet-fp7">
+    <subjects/>
+    <datasources/>
+    <zenodocommunities/>
+    <organizations/>
+</community> Result: 3 <community id="fet-h2020">
+    <subjects/>
+    <datasources/>
+    <zenodocommunities/>
+    <organizations/>
+</community> Result: 4 <community id="clarin">
    <subjects/>
    <datasources>
        <datasource>
@ -20,39 +29,40 @@
    </datasources>
    <zenodocommunities/>
    <organizations/>
-    </community>
-    <community id="ee">
+</community> Result: 5 <community id="rda">
+    <subjects/>
+    <datasources/>
+    <zenodocommunities>
+        <zenodocommunity>
+            <zenodoid>rda</zenodoid>
+            <selcriteria/>
+        </zenodocommunity>
+    </zenodocommunities>
+    <organizations/>
+</community> Result: 6 <community id="ee">
    <subjects>
        <subject>SDG13 - Climate action</subject>
-            <subject>SDG8 - Decent work and economic
-                growth</subject>
+        <subject>SDG8 - Decent work and economic growth</subject>
        <subject>SDG15 - Life on land</subject>
        <subject>SDG2 - Zero hunger</subject>
-            <subject>SDG17 - Partnerships for the
-                goals</subject>
+        <subject>SDG17 - Partnerships for the goals</subject>
        <subject>SDG10 - Reduced inequalities</subject>
        <subject>SDG5 - Gender equality</subject>
-            <subject>SDG12 - Responsible
-                consumption and production</subject>
+        <subject>SDG12 - Responsible consumption and production</subject>
        <subject>SDG14 - Life below water</subject>
-            <subject>SDG6 - Clean water and
-                sanitation</subject>
+        <subject>SDG6 - Clean water and sanitation</subject>
        <subject>SDG11 - Sustainable cities and communities</subject>
        <subject>SDG1 - No poverty</subject>
-            <subject>SDG3 -
-                Good health and well being</subject>
+        <subject>SDG3 - Good health and well being</subject>
        <subject>SDG7 - Affordable and clean energy</subject>
-            <subject>SDG4 - Quality
-                education</subject>
+        <subject>SDG4 - Quality education</subject>
        <subject>SDG9 - Industry innovation and infrastructure</subject>
-            <subject>SDG16 - Peace justice
-                and strong institutions</subject>
+        <subject>SDG16 - Peace justice and strong institutions</subject>
    </subjects>
    <datasources/>
    <zenodocommunities/>
    <organizations/>
-    </community>
-    <community id="dh-ch">
+</community> Result: 7 <community id="dh-ch">
    <subjects>
        <subject>modern art</subject>
        <subject>monuments</subject>
@ -243,8 +253,7 @@
        </zenodocommunity>
    </zenodocommunities>
    <organizations/>
-    </community>
-    <community id="fam">
+</community> Result: 8 <community id="fam">
    <subjects>
        <subject>Stock Assessment</subject>
        <subject>pelagic</subject>
@ -363,8 +372,7 @@
        </zenodocommunity>
    </zenodocommunities>
    <organizations/>
-    </community>
-    <community id="ni">
+</community> Result: 9 <community id="ni">
    <subjects>
        <subject>brain mapping</subject>
        <subject>brain imaging</subject>
@ -478,8 +486,7 @@
        </zenodocommunity>
    </zenodocommunities>
    <organizations/>
-    </community>
-    <community id="mes">
+</community> Result: 10 <community id="mes">
    <subjects>
        <subject>marine</subject>
        <subject>ocean</subject>
@ -679,8 +686,7 @@
        </zenodocommunity>
    </zenodocommunities>
    <organizations/>
-    </community>
-    <community id="instruct">
+</community> Result: 11 <community id="instruct">
    <subjects/>
    <datasources/>
    <zenodocommunities>
@ -694,8 +700,12 @@
        </zenodocommunity>
    </zenodocommunities>
    <organizations/>
-    </community>
-    <community id="aginfra">
+</community> Result: 12 <community id="elixir-gr">
+    <subjects/>
+    <datasources/>
+    <zenodocommunities/>
+    <organizations/>
+</community> Result: 13 <community id="aginfra">
    <subjects>
        <subject>animal production and health</subject>
        <subject>fisheries and aquaculture</subject>
@ -817,8 +827,7 @@
        </zenodocommunity>
    </zenodocommunities>
    <organizations/>
-    </community>
-    <community id="dariah">
+</community> Result: 14 <community id="dariah">
    <subjects/>
    <datasources>
        <datasource>
@ -837,8 +846,17 @@
        </zenodocommunity>
    </zenodocommunities>
    <organizations/>
-    </community>
-    <community id="beopen">
+</community> Result: 15 <community id="risis">
+    <subjects/>
+    <datasources/>
+    <zenodocommunities/>
+    <organizations/>
+</community> Result: 16 <community id="epos">
+    <subjects/>
+    <datasources/>
+    <zenodocommunities/>
+    <organizations/>
+</community> Result: 17 <community id="beopen">
    <subjects>
        <subject>Green Transport</subject>
        <subject>City mobility systems</subject>
@ -1136,8 +1154,125 @@
        </zenodocommunity>
    </zenodocommunities>
    <organizations/>
-    </community>
-    <community id="covid-19">
+</community> Result: 18 <community id="euromarine">
+    <subjects/>
+    <datasources/>
+    <zenodocommunities/>
+    <organizations/>
+</community> Result: 19 <community id="ifremer">
+    <subjects/>
+    <datasources/>
+    <zenodocommunities/>
+    <organizations/>
+</community> Result: 20 <community id="oa-pg">
+    <subjects/>
+    <datasources/>
+    <zenodocommunities/>
+    <organizations/>
+</community> Result: 21 <community id="science-innovation-policy">
+    <subjects>
+        <subject>Sustainability-oriented science policy</subject>
+        <subject> STI policies</subject>
+        <subject>science—society relations</subject>
+        <subject>Science &amp; Technology Policy</subject>
+        <subject>Innovation policy</subject>
+        <subject>science policy</subject>
+        <subject>Policy and Law</subject>
+    </subjects>
+    <datasources>
+        <datasource>
+            <openaireId>doajarticles::c6f0ed5fa41e98863e7c73501fe4bd6d</openaireId>
+            <selcriteria/>
+        </datasource>
+        <datasource>
+            <openaireId>doajarticles::ae4c7286c79590f19fdca670156ce816</openaireId>
+            <selcriteria/>
+        </datasource>
+        <datasource>
+            <openaireId>doajarticles::0f664bce92ce953e0c7a92068c46bfb3</openaireId>
+            <selcriteria/>
+        </datasource>
+        <datasource>
+            <openaireId>doajarticles::00017183dc4c858fb77541985323a4ef</openaireId>
+            <selcriteria/>
+        </datasource>
+        <datasource>
+            <openaireId>doajarticles::93b306f458cce3d7aaaf58c0a725f4f9</openaireId>
+            <selcriteria/>
+        </datasource>
+        <datasource>
+            <openaireId>doajarticles::9dbf8fbf3e9fe0fe1fc01e55fbd90bfc</openaireId>
+            <selcriteria/>
+        </datasource>
+        <datasource>
+            <openaireId>doajarticles::a2bda8785c863279bba4b8f34827b4c9</openaireId>
+            <selcriteria/>
+        </datasource>
+        <datasource>
+            <openaireId>doajarticles::019a1fcb42c3fea1c1b689df76330b58</openaireId>
+            <selcriteria/>
+        </datasource>
+        <datasource>
+            <openaireId>doajarticles::0daa8281938831e9c82bfed8b55a2975</openaireId>
+            <selcriteria/>
+        </datasource>
+        <datasource>
+            <openaireId>doajarticles::f67ad6d268162079b3abd51a24468744</openaireId>
+            <selcriteria/>
+        </datasource>
+        <datasource>
+            <openaireId>doajarticles::c6f0ed5fa41e98863e7c73501fe4bd6d</openaireId>
+            <selcriteria/>
+        </datasource>
+        <datasource>
+            <openaireId>doajarticles::ad114356e196a4a3d84dda59c720dacd</openaireId>
+            <selcriteria/>
+        </datasource>
+        <datasource>
+            <openaireId>doajarticles::01e8a54fdecaaf354c67a2dd74ae7d4f</openaireId>
+            <selcriteria/>
+        </datasource>
+        <datasource>
+            <openaireId>doajarticles::449305f096b10a9464449ff2d0e10e06</openaireId>
+            <selcriteria/>
+        </datasource>
+        <datasource>
+            <openaireId>doajarticles::982c0c0ac378256254cce2fa6572bb6c</openaireId>
+            <selcriteria/>
+        </datasource>
+        <datasource>
+            <openaireId>doajarticles::49d6ed47138884566ce93cf0ccb12c02</openaireId>
+            <selcriteria/>
+        </datasource>
+        <datasource>
+            <openaireId>doajarticles::a98e820dbc2e8ee0fc84ab66f263267c</openaireId>
+            <selcriteria/>
+        </datasource>
+        <datasource>
+            <openaireId>doajarticles::50b1ce37427b36368f8f0f1317e47f83</openaireId>
+            <selcriteria/>
+        </datasource>
+        <datasource>
+            <openaireId>doajarticles::f0ec29b7450b2ac5d0ad45327eeb531a</openaireId>
+            <selcriteria/>
+        </datasource>
+        <datasource>
+            <openaireId>doajarticles::d8d421d3b0349a7aaa93758b27a54e84</openaireId>
+            <selcriteria/>
+        </datasource>
+        <datasource>
+            <openaireId>doajarticles::7ffc35ac5133da01d421ccf8af5b70bc</openaireId>
+            <selcriteria/>
+        </datasource>
+    </datasources>
+    <zenodocommunities>
+        <zenodocommunity>
+            <zenodoid>risis</zenodoid>
+            <selcriteria/>
+        </zenodocommunity>
+    </zenodocommunities>
+    <organizations/>
+</community> Result: 22 <community id="covid-19">
    <subjects>
        <subject>COVID-19</subject>
        <subject>Severe acute respiratory syndrome coronavirus 2</subject>
@ -1157,7 +1292,6 @@
            <openaireId>opendoar____::358aee4cc897452c00244351e4d91f69</openaireId>
            <selcriteria>{"criteria":[{"constraint":[{"verb":"contains","field":"title","value":"COVID-19"}]},
                {"constraint":[{"verb":"contains","field":"title","value":"SARS-CoV-2"}]},
-                    {"constraint":[{"verb":"contains","field":"title","value":"sars-cov-2"}]},
                {"constraint":[{"verb":"contains","field":"title","value":"2019-nCoV"}]}]}
            </selcriteria>
        </datasource>
@ -1252,5 +1386,5 @@
        </zenodocommunity>
    </zenodocommunities>
    <organizations/>
-    </community>
+</community>
 </communities>