added datasource blacklist for the organization to result propagation through institutional repositories

dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/PrepareResultInstRepoAssociation.java

@@ -22,6 +22,11 @@ import eu.dnetlib.dhp.schema.oaf.Datasource;
 import eu.dnetlib.dhp.schema.oaf.Organization;
 import eu.dnetlib.dhp.schema.oaf.Relation;
 
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+
 public class PrepareResultInstRepoAssociation {
 
 	private static final Logger log = LoggerFactory.getLogger(PrepareResultInstRepoAssociation.class);
@@ -51,6 +56,10 @@ public class PrepareResultInstRepoAssociation {
 		final String alreadyLinkedPath = parser.get("alreadyLinkedPath");
 		log.info("alreadyLinkedPath {}: ", alreadyLinkedPath);
 
+		List<String> blacklist = Optional.ofNullable(parser.get("blacklist"))
+				.map(v -> Arrays.asList(v.split(";")))
+				.orElse(new ArrayList<>());
+
 		SparkConf conf = new SparkConf();
 		conf.set("hive.metastore.uris", parser.get("hive_metastore_uris"));
 
@@ -61,7 +70,7 @@ public class PrepareResultInstRepoAssociation {
 				readNeededResources(spark, inputPath);
 
 				removeOutputDir(spark, datasourceOrganizationPath);
-				prepareDatasourceOrganization(spark, datasourceOrganizationPath);
+				prepareDatasourceOrganization(spark, datasourceOrganizationPath, blacklist);
 
 				removeOutputDir(spark, alreadyLinkedPath);
 				prepareAlreadyLinkedAssociation(spark, alreadyLinkedPath);
@@ -80,7 +89,15 @@ public class PrepareResultInstRepoAssociation {
 	}
 
 	private static void prepareDatasourceOrganization(
-		SparkSession spark, String datasourceOrganizationPath) {
+		SparkSession spark, String datasourceOrganizationPath, List<String> blacklist) {
+		String blacklisted = "";
+		if(blacklist.size() > 0 ){
+			blacklisted = " AND  d.id != '" + blacklist.get(0) + "'";
+			for (int i = 1; i < blacklist.size(); i++) {
+				blacklisted += " AND d.id != '" + blacklist.get(i) + "'";
+			}
+		}
+
 
 		String query = "SELECT source datasourceId, target organizationId "
 			+ "FROM ( SELECT id "
@@ -88,7 +105,7 @@ public class PrepareResultInstRepoAssociation {
 			+ "WHERE datasourcetype.classid = '"
 			+ INSTITUTIONAL_REPO_TYPE
 			+ "' "
-			+ "AND datainfo.deletedbyinference = false  ) d "
+			+ "AND datainfo.deletedbyinference = false  " + blacklisted + " ) d "
 			+ "JOIN ( SELECT source, target "
 			+ "FROM relation "
 			+ "WHERE lower(relclass) = '"

dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/input_prepareresultorg_parameters.json

@@ -28,5 +28,10 @@
     "paramLongName": "isSparkSessionManaged",
     "paramDescription": "the path where prepared info have been stored",
     "paramRequired": false
-  }
+  },{
+  "paramName": "bl",
+  "paramLongName": "blacklist",
+  "paramDescription": "institutional repositories that should not be considered for the propagation",
+  "paramRequired": false
+}
 ]

dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/oozie_app/workflow.xml

@@ -141,6 +141,7 @@
             <arg>--hive_metastore_uris</arg><arg>${hive_metastore_uris}</arg>
             <arg>--datasourceOrganizationPath</arg><arg>${workingDir}/preparedInfo/datasourceOrganization</arg>
             <arg>--alreadyLinkedPath</arg><arg>${workingDir}/preparedInfo/alreadyLinked</arg>
+            <arg>--blacklist</arg><arg>${blacklist}</arg>
         </spark>
         <ok to="fork_join_apply_resulttoorganization_propagation"/>
         <error to="Kill"/>