diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/ConversionUtils.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/ConversionUtils.java index 5e7adec79..4bf4d3341 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/ConversionUtils.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/ConversionUtils.java @@ -2,7 +2,10 @@ package eu.dnetlib.dhp.broker.oa.util; import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.function.Function; import java.util.stream.Collectors; @@ -71,7 +74,7 @@ public class ConversionUtils { res.setOpenaireId(cleanOpenaireId(d.getId())); res.setOriginalId(first(d.getOriginalId())); res.setTitle(structPropValue(d.getTitle())); - res.setPids(mappedList(d.getPid(), ConversionUtils::oafPidToBrokerPid)); + res.setPids(allResultPids(d)); res.setInstances(flatMappedList(d.getInstance(), ConversionUtils::oafInstanceToBrokerInstances)); res.setCollectedFrom(mappedFirst(d.getCollectedfrom(), KeyValue::getValue)); return res; @@ -86,7 +89,7 @@ public class ConversionUtils { res.setOpenaireId(cleanOpenaireId(p.getId())); res.setOriginalId(first(p.getOriginalId())); res.setTitle(structPropValue(p.getTitle())); - res.setPids(mappedList(p.getPid(), ConversionUtils::oafPidToBrokerPid)); + res.setPids(allResultPids(p)); res.setInstances(flatMappedList(p.getInstance(), ConversionUtils::oafInstanceToBrokerInstances)); res.setCollectedFrom(mappedFirst(p.getCollectedfrom(), KeyValue::getValue)); @@ -115,7 +118,7 @@ public class ConversionUtils { res .setJournal( result instanceof Publication ? oafJournalToBrokerJournal(((Publication) result).getJournal()) : null); - res.setPids(mappedList(result.getPid(), ConversionUtils::oafPidToBrokerPid)); + res.setPids(allResultPids(result)); res.setInstances(flatMappedList(result.getInstance(), ConversionUtils::oafInstanceToBrokerInstances)); res .setExternalReferences(mappedList(result.getExternalReference(), ConversionUtils::oafExtRefToBrokerExtRef)); @@ -123,6 +126,26 @@ public class ConversionUtils { return res; } + protected static List allResultPids(final Result result) { + final Map map = new HashMap<>(); + + if (result.getPid() != null) { + result.getPid().forEach(sp -> map.put(sp.getValue(), sp)); + } + + if (result.getInstance() != null) { + result.getInstance().forEach(i -> { + if (i.getPid() != null) { + i.getPid().forEach(sp -> map.put(sp.getValue(), sp)); + } + if (i.getAlternateIdentifier() != null) { + i.getAlternateIdentifier().forEach(sp -> map.put(sp.getValue(), sp)); + } + }); + } + return mappedList(map.values(), ConversionUtils::oafPidToBrokerPid); + } + public static String cleanOpenaireId(final String id) { return id.contains("|") ? StringUtils.substringAfter(id, "|") : id; } @@ -283,18 +306,6 @@ public class ConversionUtils { : new ArrayList<>(); } - private static List structPropTypedList(final List list) { - if (list == null) { - return new ArrayList<>(); - } - - return list - .stream() - .map(ConversionUtils::oafStructPropToBrokerTypedValue) - .filter(Objects::nonNull) - .collect(Collectors.toList()); - } - private static List subjectList(final List list) { if (list == null) { return new ArrayList<>(); @@ -307,7 +318,19 @@ public class ConversionUtils { .collect(Collectors.toList()); } - private static List mappedList(final List list, final Function func) { + private static List structPropTypedList(final List list) { + if (list == null) { + return new ArrayList<>(); + } + + return list + .stream() + .map(ConversionUtils::oafStructPropToBrokerTypedValue) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + + private static List mappedList(final Collection list, final Function func) { if (list == null) { return new ArrayList<>(); } diff --git a/dhp-workflows/dhp-broker-events/src/test/java/eu/dnetlib/dhp/broker/oa/util/ConversionUtilsTest.java b/dhp-workflows/dhp-broker-events/src/test/java/eu/dnetlib/dhp/broker/oa/util/ConversionUtilsTest.java new file mode 100644 index 000000000..fc630df05 --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/test/java/eu/dnetlib/dhp/broker/oa/util/ConversionUtilsTest.java @@ -0,0 +1,94 @@ + +package eu.dnetlib.dhp.broker.oa.util; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import eu.dnetlib.broker.objects.OaBrokerTypedValue; +import eu.dnetlib.dhp.schema.oaf.Instance; +import eu.dnetlib.dhp.schema.oaf.Qualifier; +import eu.dnetlib.dhp.schema.oaf.Result; +import eu.dnetlib.dhp.schema.oaf.StructuredProperty; + +class ConversionUtilsTest { + + @BeforeEach + void setUp() throws Exception { + } + + @Test + void testAllResultPids() { + final Qualifier qf = new Qualifier(); + qf.setClassid("test"); + qf.setClassname("test"); + qf.setSchemeid("test"); + qf.setSchemename("test"); + + final StructuredProperty sp1 = new StructuredProperty(); + sp1.setValue("1"); + sp1.setQualifier(qf); + + final StructuredProperty sp2 = new StructuredProperty(); + sp2.setValue("2"); + sp2.setQualifier(qf); + + final StructuredProperty sp3 = new StructuredProperty(); + sp3.setValue("3"); + sp3.setQualifier(qf); + + final StructuredProperty sp4a = new StructuredProperty(); + sp4a.setValue("4"); + sp4a.setQualifier(qf); + + final StructuredProperty sp4b = new StructuredProperty(); + sp4b.setValue("4"); + sp4b.setQualifier(qf); + + final StructuredProperty sp5 = new StructuredProperty(); + sp5.setValue("5"); + sp5.setQualifier(qf); + + final StructuredProperty sp6a = new StructuredProperty(); + sp6a.setValue("6"); + sp6a.setQualifier(qf); + + final StructuredProperty sp6b = new StructuredProperty(); + sp6b.setValue("6"); + sp6b.setQualifier(qf); + + final Result oaf = new Result(); + oaf.setPid(new ArrayList<>()); + oaf.getPid().add(sp1); + oaf.getPid().add(sp2); + oaf.getPid().add(sp4a); + + final Instance instance1 = new Instance(); + instance1.setPid(new ArrayList<>()); + instance1.setAlternateIdentifier(new ArrayList<>()); + instance1.getPid().add(sp3); + instance1.getPid().add(sp4b); + instance1.getAlternateIdentifier().add(sp5); + instance1.getAlternateIdentifier().add(sp6a); + + final Instance instance2 = new Instance(); + instance2.setPid(new ArrayList<>()); + instance2.setAlternateIdentifier(new ArrayList<>()); + instance2.getPid().add(sp6b); + + oaf.setInstance(new ArrayList<>()); + oaf.getInstance().add(instance1); + oaf.getInstance().add(instance2); + + final List list = ConversionUtils.allResultPids(oaf); + + // list.forEach(x -> System.out.println(x.getValue())); + + assertEquals(6, list.size()); + } + +} diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/PropagationConstant.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/PropagationConstant.java index 88ec9a142..673dbf7af 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/PropagationConstant.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/PropagationConstant.java @@ -43,7 +43,7 @@ public class PropagationConstant { public final static String NULL = "NULL"; - public static final String INSTITUTIONAL_REPO_TYPE = "pubsrepository::institutional"; + public static final String INSTITUTIONAL_REPO_TYPE = "institutional"; public static final String PROPAGATION_DATA_INFO_TYPE = "propagation"; @@ -233,9 +233,9 @@ public class PropagationConstant { if (HdfsSupport.exists(inputPath, spark.sparkContext().hadoopConfiguration())) { return spark - .read() - .textFile(inputPath) - .map((MapFunction) value -> OBJECT_MAPPER.readValue(value, clazz), Encoders.bean(clazz)); + .read() + .textFile(inputPath) + .map((MapFunction) value -> OBJECT_MAPPER.readValue(value, clazz), Encoders.bean(clazz)); } else { return spark.emptyDataset(Encoders.bean(clazz)); } diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/PrepareResultInstRepoAssociation.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/PrepareResultInstRepoAssociation.java index 50ab997b6..1663afb32 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/PrepareResultInstRepoAssociation.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/PrepareResultInstRepoAssociation.java @@ -102,7 +102,7 @@ public class PrepareResultInstRepoAssociation { String query = "SELECT source datasourceId, target organizationId " + "FROM ( SELECT id " + "FROM datasource " - + "WHERE datasourcetype.classid = '" + + "WHERE lower(jurisdiction.classid) = '" + INSTITUTIONAL_REPO_TYPE + "' " + "AND datainfo.deletedbyinference = false " + blacklisted + " ) d " diff --git a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/oozie_app/workflow.xml b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/oozie_app/workflow.xml index 9c1bbdf72..d8f1946bb 100644 --- a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/oozie_app/workflow.xml @@ -219,7 +219,7 @@ - + diff --git a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/query.xq b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/query.xq index 6fbd74c8f..a9c0d9e3f 100644 --- a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/query.xq +++ b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/query.xq @@ -55,4 +55,5 @@ return {$zc/param[./@name='selcriteria']/text()} } - \ No newline at end of file + + \ No newline at end of file