dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/ModelSupport.java

@@ -50,6 +50,172 @@ public class ModelSupport {
     oafTypes.put("relation", Relation.class);
   }
 
+  public static final Map<String, String> entityIdPrefix = Maps.newHashMap();
+
+  static {
+    entityIdPrefix.put("datasource", "10");
+    entityIdPrefix.put("organization", "20");
+    entityIdPrefix.put("project", "40");
+    entityIdPrefix.put("result","50");
+  }
+
+  public static final Map<String, RelationInverse> relationInverseMap = Maps.newHashMap();
+
+  static {
+    relationInverseMap.put("personResult_authorship_isAuthorOf", new RelationInverse()
+            .setRelation("isAuthorOf")
+            .setInverse("hasAuthor")
+            .setRelType("personResult")
+            .setSubReltype("authorship"));
+    relationInverseMap.put("personResult_authorship_hasAuthor",  new RelationInverse()
+            .setInverse("isAuthorOf")
+            .setRelation("hasAuthor")
+            .setRelType("personResult")
+            .setSubReltype("authorship"));
+    relationInverseMap.put("projectOrganization_participation_isParticipant", new RelationInverse()
+            .setRelation("isParticipant")
+            .setInverse("hasParticipant")
+            .setRelType("projectOrganization")
+            .setSubReltype("participation"));
+    relationInverseMap.put("projectOrganization_participation_hasParticipant",  new RelationInverse()
+            .setInverse("isParticipant")
+            .setRelation("hasParticipant")
+            .setRelType("projectOrganization")
+            .setSubReltype("participation"));
+    relationInverseMap.put("resultOrganization_affiliation_hasAuthorInstitution", new RelationInverse()
+            .setRelation("hasAuthorInstitution")
+            .setInverse("isAuthorInstitutionOf")
+            .setRelType("resultOrganization")
+            .setSubReltype("affiliation"));
+    relationInverseMap.put("resultOrganization_affiliation_isAuthorInstitutionOf", new RelationInverse()
+            .setInverse("hasAuthorInstitution")
+            .setRelation("isAuthorInstitutionOf")
+            .setRelType("resultOrganization")
+            .setSubReltype("affiliation"));
+    relationInverseMap.put("organizationOrganization_dedup_merges", new RelationInverse()
+            .setRelation("merges")
+            .setInverse("isMergedIn")
+    .setRelType("organizationOrganization")
+    .setSubReltype("dedup"));
+    relationInverseMap.put("organizationOrganization_dedup_isMergedIn",  new RelationInverse()
+            .setInverse("merges")
+            .setRelation("isMergedIn")
+            .setRelType("organizationOrganization")
+            .setSubReltype("dedup"));
+    relationInverseMap.put("organizationOrganization_dedupSimilarity_isSimilarTo",  new RelationInverse()
+            .setInverse("isSimilarTo")
+            .setRelation("isSimilarTo")
+            .setRelType("organizationOrganization")
+            .setSubReltype("dedupSimilarity"));
+
+    relationInverseMap.put("resultProject_outcome_isProducedBy", new RelationInverse()
+            .setRelation("isProducedBy")
+            .setInverse("produces")
+    .setRelType("resultProject")
+    .setSubReltype("outcome"));
+    relationInverseMap.put("resultProject_outcome_produces",  new RelationInverse()
+            .setInverse("isProducedBy")
+            .setRelation("produces")
+            .setRelType("resultProject")
+            .setSubReltype("outcome"));
+    relationInverseMap.put("projectPerson_contactPerson_isContact", new RelationInverse()
+            .setRelation("isContact")
+            .setInverse("hasContact")
+    .setRelType("projectPerson")
+    .setSubReltype("contactPerson"));
+    relationInverseMap.put("projectPerson_contactPerson_hasContact", new RelationInverse()
+            .setInverse("isContact")
+            .setRelation("hasContact")
+    .setRelType("personPerson")
+    .setSubReltype("coAuthorship"));
+    relationInverseMap.put("personPerson_coAuthorship_isCoauthorOf", new RelationInverse()
+            .setInverse("isCoAuthorOf")
+            .setRelation("isCoAuthorOf")
+            .setRelType("personPerson")
+            .setSubReltype("coAuthorship"));
+    relationInverseMap.put("personPerson_dedup_merges", new RelationInverse()
+            .setInverse("isMergedIn")
+            .setRelation("merges")
+    .setRelType("personPerson")
+    .setSubReltype("dedup"));
+    relationInverseMap.put("personPerson_dedup_isMergedIn", new RelationInverse()
+            .setInverse("merges")
+            .setRelation("isMergedIn")
+            .setRelType("personPerson")
+            .setSubReltype("dedup"));
+    relationInverseMap.put("personPerson_dedupSimilarity_isSimilarTo", new RelationInverse()
+            .setInverse("isSimilarTo")
+            .setRelation("isSimilarTo")
+            .setRelType("personPerson")
+            .setSubReltype("dedupSimilarity"));
+    relationInverseMap.put("datasourceOrganization_provision_isProvidedBy", new RelationInverse()
+            .setInverse("provides")
+            .setRelation("isProvidedBy")
+    .setRelType("datasourceOrganization")
+    .setSubReltype("provision"));
+    relationInverseMap.put("datasourceOrganization_provision_provides", new RelationInverse()
+            .setInverse("isProvidedBy")
+            .setRelation("provides")
+            .setRelType("datasourceOrganization")
+            .setSubReltype("provision"));
+    relationInverseMap.put("resultResult_similarity_hasAmongTopNSimilarDocuments", new RelationInverse()
+            .setInverse("isAmongTopNSimilarDocuments")
+            .setRelation("hasAmongTopNSimilarDocuments")
+    .setRelType("resultResult")
+    .setSubReltype("similarity"));
+    relationInverseMap.put("resultResult_similarity_isAmongTopNSimilarDocuments", new RelationInverse()
+            .setInverse("hasAmongTopNSimilarDocuments")
+            .setRelation("isAmongTopNSimilarDocuments")
+            .setRelType("resultResult")
+            .setSubReltype("similarity"));
+    relationInverseMap.put("resultResult_relationship_isRelatedTo", new RelationInverse()
+            .setInverse("isRelatedTo")
+            .setRelation("isRelatedTo")
+            .setRelType("resultResult")
+    .setSubReltype("relationship"));
+    relationInverseMap.put("resultResult_similarity_isAmongTopNSimilarDocuments", new RelationInverse()
+            .setInverse("hasAmongTopNSimilarDocuments")
+            .setRelation("isAmongTopNSimilarDocuments")
+            .setRelType("resultResult")
+            .setSubReltype("similarity"));
+    relationInverseMap.put("resultResult_supplement_isSupplementTo", new RelationInverse()
+            .setInverse("isSupplementedBy")
+            .setRelation("isSupplementTo")
+            .setRelType("resultResult")
+    .setSubReltype("supplement"));
+    relationInverseMap.put("resultResult_supplement_isSupplementedBy", new RelationInverse()
+            .setInverse("isSupplementTo")
+            .setRelation("isSupplementedBy")
+            .setRelType("resultResult")
+            .setSubReltype("supplement"));
+    relationInverseMap.put("resultResult_part_isPartOf", new RelationInverse()
+            .setInverse("hasPart")
+            .setRelation("isPartOf")
+            .setRelType("resultResult")
+    .setSubReltype("part"));
+    relationInverseMap.put("resultResult_part_hasPart", new RelationInverse()
+            .setInverse("isPartOf")
+            .setRelation("hasPart")
+            .setRelType("resultResult")
+    .setSubReltype("part"));
+    relationInverseMap.put("resultResult_dedup_merges", new RelationInverse()
+            .setInverse("isMergedIn")
+            .setRelation("merges")
+            .setRelType("resultResult")
+    .setSubReltype("dedup"));
+    relationInverseMap.put("resultResult_dedup_isMergedIn", new RelationInverse()
+            .setInverse("merges")
+            .setRelation("isMergedIn")
+            .setRelType("resultResult")
+    .setSubReltype("dedup"));
+    relationInverseMap.put("resultResult_dedupSimilarity_isSimilarTo", new RelationInverse()
+            .setInverse("isSimilarTo")
+            .setRelation("isSimilarTo")
+            .setRelType("resultResult")
+    .setSubReltype("dedupSimilarity"));
+
+  }
+
   private static final String schemeTemplate = "dnet:%s_%s_relations";
 
   private ModelSupport() {}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/RelationInverse.java

@@ -1,4 +1,46 @@
 package eu.dnetlib.dhp.schema.common;
 
-public class RelationInverse {
+public class RelationInverse  {
+    private String relation;
+    private String inverse;
+    private String relType;
+    private String subReltype;
+
+    public String getRelType() {
+        return relType;
+    }
+
+    public RelationInverse setRelType(String relType) {
+        this.relType = relType;
+        return this;
+    }
+
+    public String getSubReltype() {
+        return subReltype;
+    }
+
+    public RelationInverse setSubReltype(String subReltype) {
+        this.subReltype = subReltype;
+        return this;
+    }
+
+    public String getRelation() {
+        return relation;
+    }
+
+    public RelationInverse setRelation(String relation) {
+        this.relation = relation;
+        return this;
+    }
+
+    public String getInverse() {
+        return inverse;
+    }
+
+    public RelationInverse setInverse(String inverse) {
+        this.inverse = inverse;
+        return this;
+    }
+
+
 }

dhp-workflows/dhp-blacklist/src/main/resources/eu/dnetlib/dhp/blacklist/oozie_app/config-default.xml

@@ -0,0 +1,54 @@
+<configuration>
+    <property>
+        <name>jobTracker</name>
+        <value>yarnRM</value>
+    </property>
+    <property>
+        <name>nameNode</name>
+        <value>hdfs://nameservice1</value>
+    </property>
+    <property>
+        <name>oozie.use.system.libpath</name>
+        <value>true</value>
+    </property>
+    <property>
+        <name>oozie.action.sharelib.for.spark</name>
+        <value>spark2</value>
+    </property>
+    <property>
+        <name>hive_metastore_uris</name>
+        <value>thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083</value>
+    </property>
+    <property>
+        <name>spark2YarnHistoryServerAddress</name>
+        <value>http://iis-cdh5-test-gw.ocean.icm.edu.pl:18089</value>
+    </property>
+    <property>
+        <name>spark2ExtraListeners</name>
+        <value>com.cloudera.spark.lineage.NavigatorAppListener</value>
+    </property>
+    <property>
+        <name>spark2SqlQueryExecutionListeners</name>
+        <value>com.cloudera.spark.lineage.NavigatorQueryListener</value>
+    </property>
+    <property>
+        <name>sparkExecutorNumber</name>
+        <value>4</value>
+    </property>
+    <property>
+        <name>spark2EventLogDir</name>
+        <value>/user/spark/spark2ApplicationHistory</value>
+    </property>
+    <property>
+        <name>sparkDriverMemory</name>
+        <value>15G</value>
+    </property>
+    <property>
+        <name>sparkExecutorMemory</name>
+        <value>6G</value>
+    </property>
+    <property>
+        <name>sparkExecutorCores</name>
+        <value>1</value>
+    </property>
+</configuration>

dhp-workflows/dhp-blacklist/src/main/resources/eu/dnetlib/dhp/blacklist/oozie_app/workflow.xml

@@ -0,0 +1,227 @@
+<workflow-app name="bulk_tagging" xmlns="uri:oozie:workflow:0.5">
+<parameters>
+    <property>
+        <name>sourcePath</name>
+        <description>the source path</description>
+    </property>
+    <property>
+        <name>sparkDriverMemory</name>
+        <description>memory for driver process</description>
+    </property>
+    <property>
+        <name>sparkExecutorMemory</name>
+        <description>memory for individual executor</description>
+    </property>
+    <property>
+        <name>sparkExecutorCores</name>
+        <description>number of cores used by single executor</description>
+    </property>
+    <property>
+        <name>isLookUpUrl</name>
+        <description>the isLookup service endpoint</description>
+    </property>
+    <property>
+        <name>pathMap</name>
+        <description>the json path associated to each selection field</description>
+    </property>
+    <property>
+        <name>outputPath</name>
+        <description>the output path</description>
+    </property>
+</parameters>
+
+<start to="reset-outputpath"/>
+
+<kill name="Kill">
+    <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
+</kill>
+
+    <action name="reset-outputpath">
+        <fs>
+            <delete path='${outputPath}/relation'/>
+            <delete path='${outputPath}/publication'/>
+            <delete path='${outputPath}/dataset'/>
+            <delete path='${outputPath}/otherresearchproduct'/>
+            <delete path='${outputPath}/software'/>
+            <delete path='${outputPath}/organization'/>
+            <delete path='${outputPath}/project'/>
+            <delete path='${outputPath}/datasource'/>
+        </fs>
+        <ok to="copy_entities"/>
+        <error to="Kill"/>
+    </action>
+
+    <fork name="copy_entities">
+        <path start="copy_relation"/>
+        <path start="copy_organization"/>
+        <path start="copy_projects"/>
+        <path start="copy_datasources"/>
+    </fork>
+
+    <action name="copy_relation">
+        <distcp xmlns="uri:oozie:distcp-action:0.2">
+            <job-tracker>${jobTracker}</job-tracker>
+            <name-node>${nameNode}</name-node>
+            <arg>${nameNode}/${sourcePath}/relation</arg>
+            <arg>${nameNode}/${outputPath}/relation</arg>
+        </distcp>
+        <ok to="copy_wait"/>
+        <error to="Kill"/>
+    </action>
+
+    <action name="copy_organization">
+        <distcp xmlns="uri:oozie:distcp-action:0.2">
+            <job-tracker>${jobTracker}</job-tracker>
+            <name-node>${nameNode}</name-node>
+            <arg>${nameNode}/${sourcePath}/organization</arg>
+            <arg>${nameNode}/${outputPath}/organization</arg>
+        </distcp>
+        <ok to="copy_wait"/>
+        <error to="Kill"/>
+    </action>
+
+    <action name="copy_projects">
+        <distcp xmlns="uri:oozie:distcp-action:0.2">
+            <job-tracker>${jobTracker}</job-tracker>
+            <name-node>${nameNode}</name-node>
+            <arg>${nameNode}/${sourcePath}/project</arg>
+            <arg>${nameNode}/${outputPath}/project</arg>
+        </distcp>
+        <ok to="copy_wait"/>
+        <error to="Kill"/>
+    </action>
+
+    <action name="copy_datasources">
+        <distcp xmlns="uri:oozie:distcp-action:0.2">
+            <job-tracker>${jobTracker}</job-tracker>
+            <name-node>${nameNode}</name-node>
+            <arg>${nameNode}/${sourcePath}/datasource</arg>
+            <arg>${nameNode}/${outputPath}/datasource</arg>
+        </distcp>
+        <ok to="copy_wait"/>
+        <error to="Kill"/>
+    </action>
+
+    <join name="copy_wait" to="fork_exec_bulktag"/>
+
+    <fork name="fork_exec_bulktag">
+    <path start="join_bulktag_publication"/>
+    <path start="join_bulktag_dataset"/>
+    <path start="join_bulktag_otherresearchproduct"/>
+    <path start="join_bulktag_software"/>
+</fork>
+    <action name="join_bulktag_publication">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <job-tracker>${jobTracker}</job-tracker>
+            <name-node>${nameNode}</name-node>
+            <master>yarn-cluster</master>
+            <mode>cluster</mode>
+            <name>bulkTagging-publication</name>
+            <class>eu.dnetlib.dhp.bulktag.SparkBulkTagJob2</class>
+            <jar>dhp-bulktag-${projectVersion}.jar</jar>
+            <spark-opts>
+                --num-executors=${sparkExecutorNumber}
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${sourcePath}/publication</arg>
+            <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Publication</arg>
+            <arg>--outputPath</arg><arg>${outputPath}/publication</arg>
+            <arg>--pathMap</arg><arg>${pathMap}</arg>
+            <arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg>
+        </spark>
+        <ok to="wait"/>
+        <error to="Kill"/>
+    </action>
+    <action name="join_bulktag_dataset">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <job-tracker>${jobTracker}</job-tracker>
+            <name-node>${nameNode}</name-node>
+            <master>yarn-cluster</master>
+            <mode>cluster</mode>
+            <name>bulkTagging-dataset</name>
+            <class>eu.dnetlib.dhp.bulktag.SparkBulkTagJob2</class>
+            <jar>dhp-bulktag-${projectVersion}.jar</jar>
+            <spark-opts>
+                --num-executors=${sparkExecutorNumber}
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${sourcePath}/dataset</arg>
+            <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Dataset</arg>
+            <arg>--outputPath</arg><arg>${outputPath}/dataset</arg>
+            <arg>--pathMap</arg><arg>${pathMap}</arg>
+            <arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg>
+        </spark>
+        <ok to="wait"/>
+        <error to="Kill"/>
+    </action>
+    <action name="join_bulktag_otherresearchproduct">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <job-tracker>${jobTracker}</job-tracker>
+            <name-node>${nameNode}</name-node>
+            <master>yarn-cluster</master>
+            <mode>cluster</mode>
+            <name>bulkTagging-orp</name>
+            <class>eu.dnetlib.dhp.bulktag.SparkBulkTagJob2</class>
+            <jar>dhp-bulktag-${projectVersion}.jar</jar>
+            <spark-opts>
+                --num-executors=${sparkExecutorNumber}
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${sourcePath}/otherresearchproduct</arg>
+            <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.OtherResearchProduct</arg>
+            <arg>--outputPath</arg><arg>${outputPath}/otherresearchproduct</arg>
+            <arg>--pathMap</arg><arg>${pathMap}</arg>
+            <arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg>
+        </spark>
+        <ok to="wait"/>
+        <error to="Kill"/>
+    </action>
+    <action name="join_bulktag_software">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <job-tracker>${jobTracker}</job-tracker>
+            <name-node>${nameNode}</name-node>
+            <master>yarn-cluster</master>
+            <mode>cluster</mode>
+            <name>bulkTagging-software</name>
+            <class>eu.dnetlib.dhp.bulktag.SparkBulkTagJob2</class>
+            <jar>dhp-bulktag-${projectVersion}.jar</jar>
+            <spark-opts>
+                --num-executors=${sparkExecutorNumber}
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${sourcePath}/software</arg>
+            <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Software</arg>
+            <arg>--outputPath</arg><arg>${outputPath}/software</arg>
+            <arg>--pathMap</arg><arg>${pathMap}</arg>
+            <arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg>
+        </spark>
+        <ok to="wait"/>
+        <error to="Kill"/>
+    </action>
+<join name="wait" to="End"/>
+<end name="End"/>
+</workflow-app>