-

:

@@ -0,0 +1,7 @@
+Merge remote-tracking branch 'upstream/master'
+
+# Please enter a commit message to explain why this merge is necessary,
+# especially if it merges an updated upstream into a topic branch.
+#
+# Lines starting with '#' will be ignored, and an empty message aborts
+# the commit.

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/ModelSupport.java

@@ -1,11 +1,14 @@
+
 package eu.dnetlib.dhp.schema.common;
 
-import com.google.common.collect.Maps;
-import eu.dnetlib.dhp.schema.oaf.*;
 import java.util.Map;
 import java.util.Optional;
 import java.util.function.Function;
 
+import com.google.common.collect.Maps;
+
+import eu.dnetlib.dhp.schema.oaf.*;
+
 /** Oaf model utility methods. */
 public class ModelSupport {
 

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateDbEntitiesApplication.java

@@ -51,7 +51,7 @@ import eu.dnetlib.dhp.schema.oaf.Software;
 import eu.dnetlib.dhp.schema.oaf.StructuredProperty;
 
 public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
-		implements Closeable {
+	implements Closeable {
 
 	private static final Log log = LogFactory.getLog(MigrateDbEntitiesApplication.class);
 
@@ -61,9 +61,10 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
 
 	public static void main(final String[] args) throws Exception {
 		final ArgumentApplicationParser parser = new ArgumentApplicationParser(
-				IOUtils
+			IOUtils
-						.toString(MigrateDbEntitiesApplication.class
+				.toString(
-								.getResourceAsStream("/eu/dnetlib/dhp/oa/graph/migrate_db_entities_parameters.json")));
+					MigrateDbEntitiesApplication.class
+						.getResourceAsStream("/eu/dnetlib/dhp/oa/graph/migrate_db_entities_parameters.json")));
 
 		parser.parseArgument(args);
 
@@ -76,7 +77,7 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
 		final boolean processClaims = parser.get("action") != null && parser.get("action").equalsIgnoreCase("claims");
 
 		try (final MigrateDbEntitiesApplication smdbe = new MigrateDbEntitiesApplication(hdfsPath, dbUrl, dbUser,
-				dbPassword)) {
+			dbPassword)) {
 			if (processClaims) {
 				log.info("Processing claims...");
 				smdbe.execute("queryClaims.sql", smdbe::processClaims);
@@ -107,15 +108,15 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
 	}
 
 	public MigrateDbEntitiesApplication(
-			final String hdfsPath, final String dbUrl, final String dbUser, final String dbPassword)
+		final String hdfsPath, final String dbUrl, final String dbUser, final String dbPassword)
-			throws Exception {
+		throws Exception {
 		super(hdfsPath);
 		this.dbClient = new DbClient(dbUrl, dbUser, dbPassword);
 		this.lastUpdateTimestamp = new Date().getTime();
 	}
 
 	public void execute(final String sqlFile, final Function<ResultSet, List<Oaf>> producer)
-			throws Exception {
+		throws Exception {
 		final String sql = IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/dhp/oa/graph/sql/" + sqlFile));
 
 		final Consumer<ResultSet> consumer = rs -> producer.apply(rs).forEach(oaf -> emitOaf(oaf));
@@ -134,7 +135,10 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
 			ds.setId(createOpenaireId(10, rs.getString("datasourceid"), true));
 			ds.setOriginalId(Arrays.asList(rs.getString("datasourceid")));
 			ds
-					.setCollectedfrom(listKeyValues(createOpenaireId(10, rs.getString("collectedfromid"), true), rs.getString("collectedfromname")));
+				.setCollectedfrom(
+					listKeyValues(
+						createOpenaireId(10, rs.getString("collectedfromid"), true),
+						rs.getString("collectedfromname")));
 			ds.setPid(new ArrayList<>());
 			ds.setDateofcollection(asString(rs.getDate("dateofcollection")));
 			ds.setDateoftransformation(null); // Value not returned by the SQL query
@@ -175,7 +179,7 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
 			ds.setCertificates(field(rs.getString("certificates"), info));
 			ds.setPolicies(new ArrayList<>()); // The sql query returns an empty array
 			ds
-					.setJournal(prepareJournal(rs.getString("officialname"), rs.getString("journal"), info)); // Journal
+				.setJournal(prepareJournal(rs.getString("officialname"), rs.getString("journal"), info)); // Journal
 			ds.setDataInfo(info);
 			ds.setLastupdatetimestamp(lastUpdateTimestamp);
 
@@ -195,7 +199,10 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
 			p.setId(createOpenaireId(40, rs.getString("projectid"), true));
 			p.setOriginalId(Arrays.asList(rs.getString("projectid")));
 			p
-					.setCollectedfrom(listKeyValues(createOpenaireId(10, rs.getString("collectedfromid"), true), rs.getString("collectedfromname")));
+				.setCollectedfrom(
+					listKeyValues(
+						createOpenaireId(10, rs.getString("collectedfromid"), true),
+						rs.getString("collectedfromname")));
 			p.setPid(new ArrayList<>());
 			p.setDateofcollection(asString(rs.getDate("dateofcollection")));
 			p.setDateoftransformation(asString(rs.getDate("dateoftransformation")));
@@ -212,7 +219,7 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
 			p.setDuration(field(Integer.toString(rs.getInt("duration")), info));
 			p.setEcsc39(field(Boolean.toString(rs.getBoolean("ecsc39")), info));
 			p
-					.setOamandatepublications(field(Boolean.toString(rs.getBoolean("oamandatepublications")), info));
+				.setOamandatepublications(field(Boolean.toString(rs.getBoolean("oamandatepublications")), info));
 			p.setEcarticle29_3(field(Boolean.toString(rs.getBoolean("ecarticle29_3")), info));
 			p.setSubjects(prepareListOfStructProps(rs.getArray("subjects"), info));
 			p.setFundingtree(prepareListFields(rs.getArray("fundingtree"), info));
@@ -249,7 +256,10 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
 			o.setId(createOpenaireId(20, rs.getString("organizationid"), true));
 			o.setOriginalId(Arrays.asList(rs.getString("organizationid")));
 			o
-					.setCollectedfrom(listKeyValues(createOpenaireId(10, rs.getString("collectedfromid"), true), rs.getString("collectedfromname")));
+				.setCollectedfrom(
+					listKeyValues(
+						createOpenaireId(10, rs.getString("collectedfromid"), true),
+						rs.getString("collectedfromname")));
 			o.setPid(new ArrayList<>());
 			o.setDateofcollection(asString(rs.getDate("dateofcollection")));
 			o.setDateoftransformation(asString(rs.getDate("dateoftransformation")));
@@ -264,12 +274,14 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
 			o.setEclegalperson(field(Boolean.toString(rs.getBoolean("eclegalperson")), info));
 			o.setEcnonprofit(field(Boolean.toString(rs.getBoolean("ecnonprofit")), info));
 			o
-					.setEcresearchorganization(field(Boolean.toString(rs.getBoolean("ecresearchorganization")), info));
+				.setEcresearchorganization(field(Boolean.toString(rs.getBoolean("ecresearchorganization")), info));
 			o.setEchighereducation(field(Boolean.toString(rs.getBoolean("echighereducation")), info));
 			o
-					.setEcinternationalorganizationeurinterests(field(Boolean.toString(rs.getBoolean("ecinternationalorganizationeurinterests")), info));
+				.setEcinternationalorganizationeurinterests(
+					field(Boolean.toString(rs.getBoolean("ecinternationalorganizationeurinterests")), info));
 			o
-					.setEcinternationalorganization(field(Boolean.toString(rs.getBoolean("ecinternationalorganization")), info));
+				.setEcinternationalorganization(
+					field(Boolean.toString(rs.getBoolean("ecinternationalorganization")), info));
 			o.setEcenterprise(field(Boolean.toString(rs.getBoolean("ecenterprise")), info));
 			o.setEcsmevalidated(field(Boolean.toString(rs.getBoolean("ecsmevalidated")), info));
 			o.setEcnutscode(field(Boolean.toString(rs.getBoolean("ecnutscode")), info));
@@ -288,7 +300,8 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
 			final DataInfo info = prepareDataInfo(rs);
 			final String orgId = createOpenaireId(20, rs.getString("organization"), true);
 			final String dsId = createOpenaireId(10, rs.getString("datasource"), true);
-			final List<KeyValue> collectedFrom = listKeyValues(createOpenaireId(10, rs.getString("collectedfromid"), true), rs.getString("collectedfromname"));
+			final List<KeyValue> collectedFrom = listKeyValues(
+				createOpenaireId(10, rs.getString("collectedfromid"), true), rs.getString("collectedfromname"));
 
 			final Relation r1 = new Relation();
 			r1.setRelType("datasourceOrganization");
@@ -321,7 +334,8 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
 			final DataInfo info = prepareDataInfo(rs);
 			final String orgId = createOpenaireId(20, rs.getString("resporganization"), true);
 			final String projectId = createOpenaireId(40, rs.getString("project"), true);
-			final List<KeyValue> collectedFrom = listKeyValues(createOpenaireId(10, rs.getString("collectedfromid"), true), rs.getString("collectedfromname"));
+			final List<KeyValue> collectedFrom = listKeyValues(
+				createOpenaireId(10, rs.getString("collectedfromid"), true), rs.getString("collectedfromname"));
 
 			final Relation r1 = new Relation();
 			r1.setRelType("projectOrganization");
@@ -351,10 +365,12 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
 
 	public List<Oaf> processClaims(final ResultSet rs) {
 
-		final DataInfo info =
+		final DataInfo info = dataInfo(
-				dataInfo(false, null, false, false, qualifier("user:claim", "user:claim", "dnet:provenanceActions", "dnet:provenanceActions"), "0.9");
+			false, null, false, false,
+			qualifier("user:claim", "user:claim", "dnet:provenanceActions", "dnet:provenanceActions"), "0.9");
 
-		final List<KeyValue> collectedFrom = listKeyValues(createOpenaireId(10, "infrastruct_::openaire", true), "OpenAIRE");
+		final List<KeyValue> collectedFrom = listKeyValues(
+			createOpenaireId(10, "infrastruct_::openaire", true), "OpenAIRE");
 
 		try {
 
@@ -440,11 +456,15 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
 		final String inferenceprovenance = rs.getString("inferenceprovenance");
 		final Boolean inferred = rs.getBoolean("inferred");
 		final String trust = rs.getString("trust");
-		return dataInfo(deletedbyinference, inferenceprovenance, inferred, false, MigrationConstants.ENTITYREGISTRY_PROVENANCE_ACTION, trust);
+		return dataInfo(
+			deletedbyinference, inferenceprovenance, inferred, false,
+			MigrationConstants.ENTITYREGISTRY_PROVENANCE_ACTION, trust);
 	}
 
 	private Qualifier prepareQualifierSplitting(final String s) {
-		if (StringUtils.isBlank(s)) { return null; }
+		if (StringUtils.isBlank(s)) {
+			return null;
+		}
 		final String[] arr = s.split("@@@");
 		return arr.length == 4 ? qualifier(arr[0], arr[1], arr[2], arr[3]) : null;
 	}
@@ -458,19 +478,23 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
 	}
 
 	private StructuredProperty prepareStructProp(final String s, final DataInfo dataInfo) {
-		if (StringUtils.isBlank(s)) { return null; }
+		if (StringUtils.isBlank(s)) {
+			return null;
+		}
 		final String[] parts = s.split("###");
 		if (parts.length == 2) {
 			final String value = parts[0];
 			final String[] arr = parts[1].split("@@@");
-			if (arr.length == 4) { return structuredProperty(value, arr[0], arr[1], arr[2], arr[3], dataInfo); }
+			if (arr.length == 4) {
+				return structuredProperty(value, arr[0], arr[1], arr[2], arr[3], dataInfo);
+			}
 		}
 		return null;
 	}
 
 	private List<StructuredProperty> prepareListOfStructProps(
-			final Array array,
+		final Array array,
-			final DataInfo dataInfo) throws SQLException {
+		final DataInfo dataInfo) throws SQLException {
 		final List<StructuredProperty> res = new ArrayList<>();
 		if (array != null) {
 			for (final String s : (String[]) array.getArray()) {
@@ -489,8 +513,10 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
 			final String[] arr = sj.split("@@@");
 			if (arr.length == 3) {
 				final String issn = StringUtils.isNotBlank(arr[0]) ? arr[0].trim() : null;
-				final String eissn = StringUtils.isNotBlank(arr[1]) ? arr[1].trim() : null;;
+				final String eissn = StringUtils.isNotBlank(arr[1]) ? arr[1].trim() : null;
-				final String lissn = StringUtils.isNotBlank(arr[2]) ? arr[2].trim() : null;;
+				;
+				final String lissn = StringUtils.isNotBlank(arr[2]) ? arr[2].trim() : null;
+				;
 				if (issn != null || eissn != null || lissn != null) {
 					return journal(name, issn, eissn, eissn, null, null, null, null, null, null, null, info);
 				}
@@ -504,5 +530,4 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
 		super.close();
 		dbClient.close();
 	}
-
 }

dhp-workflows/dhp-propagation/src/main/java/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/SparkResultToOrganizationFromIstRepoJob2.java

@@ -99,6 +99,7 @@ public class SparkResultToOrganizationFromIstRepoJob2 {
 		String outputPath,
 		Class<? extends Result> resultClazz,
 		String resultType,
+
 		Boolean saveGraph) {
 		final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext());
 

dhp-workflows/dhp-propagation/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromsemrel/input_communitytoresult_parameters.json

@@ -6,6 +6,12 @@
     "paramDescription": "the path of the sequencial file to read",
     "paramRequired": true
   },
+  {
+    "paramName":"sg",
+    "paramLongName":"saveGraph",
+    "paramDescription": "true if the new version of the graph must be saved",
+    "paramRequired": false
+  },
   {
     "paramName":"h",
     "paramLongName":"hive_metastore_uris",

dhp-workflows/dhp-propagation/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromsemrel/oozie_app/workflow.xml

@@ -12,6 +12,10 @@
         <name>isLookupUrl</name>
         <description>the isLookup service endpoint</description>
     </property>
+    <property>
+        <name>outputPath</name>
+        <description>the output path</description>
+    </property>
 </parameters>
 
     <start to="reset-outputpath"/>
@@ -21,22 +25,72 @@
     </kill>
     <action name="reset-outputpath">
         <fs>
-            <delete path='${workingDir}/resulttocommunitysemrel_propagation'/>
+            <delete path='${outputPath}/relation'/>
+            <delete path='${outputPath}/datasource'/>
+            <delete path='${outputPath}/organization'/>
+            <delete path='${outputPath}/project'/>
+            <delete path='${outputPath}/publication'/>
+            <delete path='${outputPath}/dataset'/>
+            <delete path='${outputPath}/software'/>
+            <delete path='${outputPath}/otherresearchproduct'/>
+            <delete path='${workingDir}/preparedInfo'/>
         </fs>
-        <ok to="copy_relation"/>
+        <ok to="copy_entities"/>
         <error to="Kill"/>
     </action>
+    <fork name="copy_entities">
+        <path start="copy_relation"/>
+        <path start="copy_organization"/>
+        <path start="copy_projects"/>
+        <path start="copy_datasources"/>
+    </fork>
+
     <action name="copy_relation">
         <distcp xmlns="uri:oozie:distcp-action:0.2">
             <job-tracker>${jobTracker}</job-tracker>
             <name-node>${nameNode}</name-node>
             <arg>${nameNode}/${sourcePath}/relation</arg>
-            <arg>${nameNode}/${workingDir}/projecttoresult_propagation/relation</arg>
+            <arg>${nameNode}/${outputPath}/relation</arg>
         </distcp>
-        <ok to="fork_prepare_assoc_step1"/>
+        <ok to="copy_wait"/>
         <error to="Kill"/>
     </action>
 
+    <action name="copy_organization">
+        <distcp xmlns="uri:oozie:distcp-action:0.2">
+            <job-tracker>${jobTracker}</job-tracker>
+            <name-node>${nameNode}</name-node>
+            <arg>${nameNode}/${sourcePath}/organization</arg>
+            <arg>${nameNode}/${outputPath}/organization</arg>
+        </distcp>
+        <ok to="copy_wait"/>
+        <error to="Kill"/>
+    </action>
+
+    <action name="copy_projects">
+        <distcp xmlns="uri:oozie:distcp-action:0.2">
+            <job-tracker>${jobTracker}</job-tracker>
+            <name-node>${nameNode}</name-node>
+            <arg>${nameNode}/${sourcePath}/project</arg>
+            <arg>${nameNode}/${outputPath}/project</arg>
+        </distcp>
+        <ok to="copy_wait"/>
+        <error to="Kill"/>
+    </action>
+
+    <action name="copy_datasources">
+        <distcp xmlns="uri:oozie:distcp-action:0.2">
+            <job-tracker>${jobTracker}</job-tracker>
+            <name-node>${nameNode}</name-node>
+            <arg>${nameNode}/${sourcePath}/datasource</arg>
+            <arg>${nameNode}/${outputPath}/datasource</arg>
+        </distcp>
+        <ok to="copy_wait"/>
+        <error to="Kill"/>
+    </action>
+
+    <join name="copy_wait" to="fork_prepare_assoc_step1"/>
+
     <fork name="fork_prepare_assoc_step1">
         <path start="join_prepare_publication"/>
         <path start="join_prepare_dataset"/>
@@ -67,7 +121,7 @@
             <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Publication</arg>
             <arg>--outputPath</arg><arg>${workingDir}/preparedInfo/targetCommunityAssoc</arg>
             <arg>--allowedsemrels</arg><arg>${allowedsemrels}</arg>
-            <arg>--isLookupUrl</arg><arg>${isLookupUrl}</arg>
+            <arg>--isLookupUrl</arg><arg>${isLookUpUrl}</arg>
         </spark>
         <ok to="wait"/>
         <error to="Kill"/>
@@ -215,7 +269,8 @@
             <arg>--sourcePath</arg><arg>${sourcePath}/publication</arg>
             <arg>--hive_metastore_uris</arg><arg>${hive_metastore_uris}</arg>
             <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Publication</arg>
-            <arg>--outputPath</arg><arg>${workingDir}/publication</arg>
+            <arg>--outputPath</arg><arg>${outputPath}/publication</arg>
+            <arg>--saveGrap</arg><arg>${saveGraph</arg>
         </spark>
         <ok to="wait2"/>
         <error to="Kill"/>
@@ -242,7 +297,8 @@
             <arg>--sourcePath</arg><arg>${sourcePath}/dataset</arg>
             <arg>--hive_metastore_uris</arg><arg>${hive_metastore_uris}</arg>
             <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Dataset</arg>
-            <arg>--outputPath</arg><arg>${workingDir}/dataset</arg>
+            <arg>--outputPath</arg><arg>${outputPath}/dataset</arg>
+            <arg>--saveGrap</arg><arg>${saveGraph</arg>
         </spark>
         <ok to="wait2"/>
         <error to="Kill"/>
@@ -269,7 +325,8 @@
             <arg>--sourcePath</arg><arg>${sourcePath}/otherresearchproduct</arg>
             <arg>--hive_metastore_uris</arg><arg>${hive_metastore_uris}</arg>
             <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.OtherResearchProduct</arg>
-            <arg>--outputPath</arg><arg>${workingDir}/otherresearchproduct</arg>
+            <arg>--outputPath</arg><arg>${outputPath}/otherresearchproduct</arg>
+            <arg>--saveGrap</arg><arg>${saveGraph</arg>
         </spark>
         <ok to="wait2"/>
         <error to="Kill"/>
@@ -296,7 +353,8 @@
             <arg>--sourcePath</arg><arg>${sourcePath}/software</arg>
             <arg>--hive_metastore_uris</arg><arg>${hive_metastore_uris}</arg>
             <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Software</arg>
-            <arg>--outputPath</arg><arg>${workingDir}/software</arg>
+            <arg>--outputPath</arg><arg>${outputPath}/software</arg>
+            <arg>--saveGrap</arg><arg>${saveGraph</arg>
         </spark>
         <ok to="wait2"/>
         <error to="Kill"/>

dhp-workflows/dhp-propagation/src/test/java/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/Result2OrganizationJobTest.java

@@ -88,6 +88,7 @@ public class Result2OrganizationJobTest {
 					"",
 					"-resultTableName",
 					"eu.dnetlib.dhp.schema.oaf.Software",
+
 					"-saveGraph",
 					"true",
 					"-outputPath",
@@ -138,6 +139,7 @@ public class Result2OrganizationJobTest {
 					"",
 					"-resultTableName",
 					"eu.dnetlib.dhp.schema.oaf.Software",
+
 					"-saveGraph",
 					"true",
 					"-outputPath",
@@ -211,6 +213,7 @@ public class Result2OrganizationJobTest {
 					"",
 					"-resultTableName",
 					"eu.dnetlib.dhp.schema.oaf.Software",
+
 					"-saveGraph",
 					"true",
 					"-outputPath",