mergin with branch beta

dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/ConversionUtils.java

@@ -2,7 +2,10 @@
 package eu.dnetlib.dhp.broker.oa.util;
 
 import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import java.util.function.Function;
 import java.util.stream.Collectors;
@@ -71,7 +74,7 @@ public class ConversionUtils {
 		res.setOpenaireId(cleanOpenaireId(d.getId()));
 		res.setOriginalId(first(d.getOriginalId()));
 		res.setTitle(structPropValue(d.getTitle()));
-		res.setPids(mappedList(d.getPid(), ConversionUtils::oafPidToBrokerPid));
+		res.setPids(allResultPids(d));
 		res.setInstances(flatMappedList(d.getInstance(), ConversionUtils::oafInstanceToBrokerInstances));
 		res.setCollectedFrom(mappedFirst(d.getCollectedfrom(), KeyValue::getValue));
 		return res;
@@ -86,7 +89,7 @@ public class ConversionUtils {
 		res.setOpenaireId(cleanOpenaireId(p.getId()));
 		res.setOriginalId(first(p.getOriginalId()));
 		res.setTitle(structPropValue(p.getTitle()));
-		res.setPids(mappedList(p.getPid(), ConversionUtils::oafPidToBrokerPid));
+		res.setPids(allResultPids(p));
 		res.setInstances(flatMappedList(p.getInstance(), ConversionUtils::oafInstanceToBrokerInstances));
 		res.setCollectedFrom(mappedFirst(p.getCollectedfrom(), KeyValue::getValue));
 
@@ -115,7 +118,7 @@ public class ConversionUtils {
 		res
 			.setJournal(
 				result instanceof Publication ? oafJournalToBrokerJournal(((Publication) result).getJournal()) : null);
-		res.setPids(mappedList(result.getPid(), ConversionUtils::oafPidToBrokerPid));
+		res.setPids(allResultPids(result));
 		res.setInstances(flatMappedList(result.getInstance(), ConversionUtils::oafInstanceToBrokerInstances));
 		res
 			.setExternalReferences(mappedList(result.getExternalReference(), ConversionUtils::oafExtRefToBrokerExtRef));
@@ -123,6 +126,26 @@ public class ConversionUtils {
 		return res;
 	}
 
+	protected static List<OaBrokerTypedValue> allResultPids(final Result result) {
+		final Map<String, StructuredProperty> map = new HashMap<>();
+
+		if (result.getPid() != null) {
+			result.getPid().forEach(sp -> map.put(sp.getValue(), sp));
+		}
+
+		if (result.getInstance() != null) {
+			result.getInstance().forEach(i -> {
+				if (i.getPid() != null) {
+					i.getPid().forEach(sp -> map.put(sp.getValue(), sp));
+				}
+				if (i.getAlternateIdentifier() != null) {
+					i.getAlternateIdentifier().forEach(sp -> map.put(sp.getValue(), sp));
+				}
+			});
+		}
+		return mappedList(map.values(), ConversionUtils::oafPidToBrokerPid);
+	}
+
 	public static String cleanOpenaireId(final String id) {
 		return id.contains("|") ? StringUtils.substringAfter(id, "|") : id;
 	}
@@ -283,18 +306,6 @@ public class ConversionUtils {
 			: new ArrayList<>();
 	}
 
-	private static List<OaBrokerTypedValue> structPropTypedList(final List<StructuredProperty> list) {
-		if (list == null) {
-			return new ArrayList<>();
-		}
-
-		return list
-			.stream()
-			.map(ConversionUtils::oafStructPropToBrokerTypedValue)
-			.filter(Objects::nonNull)
-			.collect(Collectors.toList());
-	}
-
 	private static List<OaBrokerTypedValue> subjectList(final List<Subject> list) {
 		if (list == null) {
 			return new ArrayList<>();
@@ -307,7 +318,19 @@ public class ConversionUtils {
 			.collect(Collectors.toList());
 	}
 
-	private static <F, T> List<T> mappedList(final List<F> list, final Function<F, T> func) {
+	private static List<OaBrokerTypedValue> structPropTypedList(final List<StructuredProperty> list) {
+		if (list == null) {
+			return new ArrayList<>();
+		}
+
+		return list
+			.stream()
+			.map(ConversionUtils::oafStructPropToBrokerTypedValue)
+			.filter(Objects::nonNull)
+			.collect(Collectors.toList());
+	}
+
+	private static <F, T> List<T> mappedList(final Collection<F> list, final Function<F, T> func) {
 		if (list == null) {
 			return new ArrayList<>();
 		}

dhp-workflows/dhp-broker-events/src/test/java/eu/dnetlib/dhp/broker/oa/util/ConversionUtilsTest.java

@@ -0,0 +1,94 @@
+
+package eu.dnetlib.dhp.broker.oa.util;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import eu.dnetlib.broker.objects.OaBrokerTypedValue;
+import eu.dnetlib.dhp.schema.oaf.Instance;
+import eu.dnetlib.dhp.schema.oaf.Qualifier;
+import eu.dnetlib.dhp.schema.oaf.Result;
+import eu.dnetlib.dhp.schema.oaf.StructuredProperty;
+
+class ConversionUtilsTest {
+
+	@BeforeEach
+	void setUp() throws Exception {
+	}
+
+	@Test
+	void testAllResultPids() {
+		final Qualifier qf = new Qualifier();
+		qf.setClassid("test");
+		qf.setClassname("test");
+		qf.setSchemeid("test");
+		qf.setSchemename("test");
+
+		final StructuredProperty sp1 = new StructuredProperty();
+		sp1.setValue("1");
+		sp1.setQualifier(qf);
+
+		final StructuredProperty sp2 = new StructuredProperty();
+		sp2.setValue("2");
+		sp2.setQualifier(qf);
+
+		final StructuredProperty sp3 = new StructuredProperty();
+		sp3.setValue("3");
+		sp3.setQualifier(qf);
+
+		final StructuredProperty sp4a = new StructuredProperty();
+		sp4a.setValue("4");
+		sp4a.setQualifier(qf);
+
+		final StructuredProperty sp4b = new StructuredProperty();
+		sp4b.setValue("4");
+		sp4b.setQualifier(qf);
+
+		final StructuredProperty sp5 = new StructuredProperty();
+		sp5.setValue("5");
+		sp5.setQualifier(qf);
+
+		final StructuredProperty sp6a = new StructuredProperty();
+		sp6a.setValue("6");
+		sp6a.setQualifier(qf);
+
+		final StructuredProperty sp6b = new StructuredProperty();
+		sp6b.setValue("6");
+		sp6b.setQualifier(qf);
+
+		final Result oaf = new Result();
+		oaf.setPid(new ArrayList<>());
+		oaf.getPid().add(sp1);
+		oaf.getPid().add(sp2);
+		oaf.getPid().add(sp4a);
+
+		final Instance instance1 = new Instance();
+		instance1.setPid(new ArrayList<>());
+		instance1.setAlternateIdentifier(new ArrayList<>());
+		instance1.getPid().add(sp3);
+		instance1.getPid().add(sp4b);
+		instance1.getAlternateIdentifier().add(sp5);
+		instance1.getAlternateIdentifier().add(sp6a);
+
+		final Instance instance2 = new Instance();
+		instance2.setPid(new ArrayList<>());
+		instance2.setAlternateIdentifier(new ArrayList<>());
+		instance2.getPid().add(sp6b);
+
+		oaf.setInstance(new ArrayList<>());
+		oaf.getInstance().add(instance1);
+		oaf.getInstance().add(instance2);
+
+		final List<OaBrokerTypedValue> list = ConversionUtils.allResultPids(oaf);
+
+		// list.forEach(x -> System.out.println(x.getValue()));
+
+		assertEquals(6, list.size());
+	}
+
+}

dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/PropagationConstant.java

@@ -43,7 +43,7 @@ public class PropagationConstant {
 
 	public final static String NULL = "NULL";
 
-	public static final String INSTITUTIONAL_REPO_TYPE = "pubsrepository::institutional";
+	public static final String INSTITUTIONAL_REPO_TYPE = "institutional";
 
 	public static final String PROPAGATION_DATA_INFO_TYPE = "propagation";
 
@@ -233,9 +233,9 @@ public class PropagationConstant {
 
 		if (HdfsSupport.exists(inputPath, spark.sparkContext().hadoopConfiguration())) {
 			return spark
-					.read()
+				.read()
-					.textFile(inputPath)
+				.textFile(inputPath)
-					.map((MapFunction<String, R>) value -> OBJECT_MAPPER.readValue(value, clazz), Encoders.bean(clazz));
+				.map((MapFunction<String, R>) value -> OBJECT_MAPPER.readValue(value, clazz), Encoders.bean(clazz));
 		} else {
 			return spark.emptyDataset(Encoders.bean(clazz));
 		}

dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/PrepareResultInstRepoAssociation.java

@@ -102,7 +102,7 @@ public class PrepareResultInstRepoAssociation {
 		String query = "SELECT source datasourceId, target organizationId "
 			+ "FROM ( SELECT id "
 			+ "FROM datasource "
-			+ "WHERE datasourcetype.classid = '"
+			+ "WHERE lower(jurisdiction.classid) = '"
 			+ INSTITUTIONAL_REPO_TYPE
 			+ "' "
 			+ "AND datainfo.deletedbyinference = false  " + blacklisted + " ) d "

dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/oozie_app/workflow.xml

@@ -219,7 +219,7 @@
         <error to="Kill"/>
     </action>
 
-    <join name="wait" to="eosc_tag"/>
+    <join name="wait" to="End"/>
 
     <action name="eosc_tag">
         <spark xmlns="uri:oozie:spark-action:0.2">

dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/query.xq

@@ -55,4 +55,5 @@ return
 {$zc/param[./@name='selcriteria']/text()}
 </selcriteria>
 </zenodocommunity>}
-</zenodocommunities>
+</zenodocommunities>
+</community>