mergin with branch beta
This commit is contained in:
commit
50cdc76987
|
@ -2,7 +2,10 @@
|
||||||
package eu.dnetlib.dhp.broker.oa.util;
|
package eu.dnetlib.dhp.broker.oa.util;
|
||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
|
import java.util.Collection;
|
||||||
|
import java.util.HashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
import java.util.function.Function;
|
import java.util.function.Function;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
@ -71,7 +74,7 @@ public class ConversionUtils {
|
||||||
res.setOpenaireId(cleanOpenaireId(d.getId()));
|
res.setOpenaireId(cleanOpenaireId(d.getId()));
|
||||||
res.setOriginalId(first(d.getOriginalId()));
|
res.setOriginalId(first(d.getOriginalId()));
|
||||||
res.setTitle(structPropValue(d.getTitle()));
|
res.setTitle(structPropValue(d.getTitle()));
|
||||||
res.setPids(mappedList(d.getPid(), ConversionUtils::oafPidToBrokerPid));
|
res.setPids(allResultPids(d));
|
||||||
res.setInstances(flatMappedList(d.getInstance(), ConversionUtils::oafInstanceToBrokerInstances));
|
res.setInstances(flatMappedList(d.getInstance(), ConversionUtils::oafInstanceToBrokerInstances));
|
||||||
res.setCollectedFrom(mappedFirst(d.getCollectedfrom(), KeyValue::getValue));
|
res.setCollectedFrom(mappedFirst(d.getCollectedfrom(), KeyValue::getValue));
|
||||||
return res;
|
return res;
|
||||||
|
@ -86,7 +89,7 @@ public class ConversionUtils {
|
||||||
res.setOpenaireId(cleanOpenaireId(p.getId()));
|
res.setOpenaireId(cleanOpenaireId(p.getId()));
|
||||||
res.setOriginalId(first(p.getOriginalId()));
|
res.setOriginalId(first(p.getOriginalId()));
|
||||||
res.setTitle(structPropValue(p.getTitle()));
|
res.setTitle(structPropValue(p.getTitle()));
|
||||||
res.setPids(mappedList(p.getPid(), ConversionUtils::oafPidToBrokerPid));
|
res.setPids(allResultPids(p));
|
||||||
res.setInstances(flatMappedList(p.getInstance(), ConversionUtils::oafInstanceToBrokerInstances));
|
res.setInstances(flatMappedList(p.getInstance(), ConversionUtils::oafInstanceToBrokerInstances));
|
||||||
res.setCollectedFrom(mappedFirst(p.getCollectedfrom(), KeyValue::getValue));
|
res.setCollectedFrom(mappedFirst(p.getCollectedfrom(), KeyValue::getValue));
|
||||||
|
|
||||||
|
@ -115,7 +118,7 @@ public class ConversionUtils {
|
||||||
res
|
res
|
||||||
.setJournal(
|
.setJournal(
|
||||||
result instanceof Publication ? oafJournalToBrokerJournal(((Publication) result).getJournal()) : null);
|
result instanceof Publication ? oafJournalToBrokerJournal(((Publication) result).getJournal()) : null);
|
||||||
res.setPids(mappedList(result.getPid(), ConversionUtils::oafPidToBrokerPid));
|
res.setPids(allResultPids(result));
|
||||||
res.setInstances(flatMappedList(result.getInstance(), ConversionUtils::oafInstanceToBrokerInstances));
|
res.setInstances(flatMappedList(result.getInstance(), ConversionUtils::oafInstanceToBrokerInstances));
|
||||||
res
|
res
|
||||||
.setExternalReferences(mappedList(result.getExternalReference(), ConversionUtils::oafExtRefToBrokerExtRef));
|
.setExternalReferences(mappedList(result.getExternalReference(), ConversionUtils::oafExtRefToBrokerExtRef));
|
||||||
|
@ -123,6 +126,26 @@ public class ConversionUtils {
|
||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
protected static List<OaBrokerTypedValue> allResultPids(final Result result) {
|
||||||
|
final Map<String, StructuredProperty> map = new HashMap<>();
|
||||||
|
|
||||||
|
if (result.getPid() != null) {
|
||||||
|
result.getPid().forEach(sp -> map.put(sp.getValue(), sp));
|
||||||
|
}
|
||||||
|
|
||||||
|
if (result.getInstance() != null) {
|
||||||
|
result.getInstance().forEach(i -> {
|
||||||
|
if (i.getPid() != null) {
|
||||||
|
i.getPid().forEach(sp -> map.put(sp.getValue(), sp));
|
||||||
|
}
|
||||||
|
if (i.getAlternateIdentifier() != null) {
|
||||||
|
i.getAlternateIdentifier().forEach(sp -> map.put(sp.getValue(), sp));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return mappedList(map.values(), ConversionUtils::oafPidToBrokerPid);
|
||||||
|
}
|
||||||
|
|
||||||
public static String cleanOpenaireId(final String id) {
|
public static String cleanOpenaireId(final String id) {
|
||||||
return id.contains("|") ? StringUtils.substringAfter(id, "|") : id;
|
return id.contains("|") ? StringUtils.substringAfter(id, "|") : id;
|
||||||
}
|
}
|
||||||
|
@ -283,18 +306,6 @@ public class ConversionUtils {
|
||||||
: new ArrayList<>();
|
: new ArrayList<>();
|
||||||
}
|
}
|
||||||
|
|
||||||
private static List<OaBrokerTypedValue> structPropTypedList(final List<StructuredProperty> list) {
|
|
||||||
if (list == null) {
|
|
||||||
return new ArrayList<>();
|
|
||||||
}
|
|
||||||
|
|
||||||
return list
|
|
||||||
.stream()
|
|
||||||
.map(ConversionUtils::oafStructPropToBrokerTypedValue)
|
|
||||||
.filter(Objects::nonNull)
|
|
||||||
.collect(Collectors.toList());
|
|
||||||
}
|
|
||||||
|
|
||||||
private static List<OaBrokerTypedValue> subjectList(final List<Subject> list) {
|
private static List<OaBrokerTypedValue> subjectList(final List<Subject> list) {
|
||||||
if (list == null) {
|
if (list == null) {
|
||||||
return new ArrayList<>();
|
return new ArrayList<>();
|
||||||
|
@ -307,7 +318,19 @@ public class ConversionUtils {
|
||||||
.collect(Collectors.toList());
|
.collect(Collectors.toList());
|
||||||
}
|
}
|
||||||
|
|
||||||
private static <F, T> List<T> mappedList(final List<F> list, final Function<F, T> func) {
|
private static List<OaBrokerTypedValue> structPropTypedList(final List<StructuredProperty> list) {
|
||||||
|
if (list == null) {
|
||||||
|
return new ArrayList<>();
|
||||||
|
}
|
||||||
|
|
||||||
|
return list
|
||||||
|
.stream()
|
||||||
|
.map(ConversionUtils::oafStructPropToBrokerTypedValue)
|
||||||
|
.filter(Objects::nonNull)
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
}
|
||||||
|
|
||||||
|
private static <F, T> List<T> mappedList(final Collection<F> list, final Function<F, T> func) {
|
||||||
if (list == null) {
|
if (list == null) {
|
||||||
return new ArrayList<>();
|
return new ArrayList<>();
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,94 @@
|
||||||
|
|
||||||
|
package eu.dnetlib.dhp.broker.oa.util;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.BeforeEach;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
|
import eu.dnetlib.broker.objects.OaBrokerTypedValue;
|
||||||
|
import eu.dnetlib.dhp.schema.oaf.Instance;
|
||||||
|
import eu.dnetlib.dhp.schema.oaf.Qualifier;
|
||||||
|
import eu.dnetlib.dhp.schema.oaf.Result;
|
||||||
|
import eu.dnetlib.dhp.schema.oaf.StructuredProperty;
|
||||||
|
|
||||||
|
class ConversionUtilsTest {
|
||||||
|
|
||||||
|
@BeforeEach
|
||||||
|
void setUp() throws Exception {
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testAllResultPids() {
|
||||||
|
final Qualifier qf = new Qualifier();
|
||||||
|
qf.setClassid("test");
|
||||||
|
qf.setClassname("test");
|
||||||
|
qf.setSchemeid("test");
|
||||||
|
qf.setSchemename("test");
|
||||||
|
|
||||||
|
final StructuredProperty sp1 = new StructuredProperty();
|
||||||
|
sp1.setValue("1");
|
||||||
|
sp1.setQualifier(qf);
|
||||||
|
|
||||||
|
final StructuredProperty sp2 = new StructuredProperty();
|
||||||
|
sp2.setValue("2");
|
||||||
|
sp2.setQualifier(qf);
|
||||||
|
|
||||||
|
final StructuredProperty sp3 = new StructuredProperty();
|
||||||
|
sp3.setValue("3");
|
||||||
|
sp3.setQualifier(qf);
|
||||||
|
|
||||||
|
final StructuredProperty sp4a = new StructuredProperty();
|
||||||
|
sp4a.setValue("4");
|
||||||
|
sp4a.setQualifier(qf);
|
||||||
|
|
||||||
|
final StructuredProperty sp4b = new StructuredProperty();
|
||||||
|
sp4b.setValue("4");
|
||||||
|
sp4b.setQualifier(qf);
|
||||||
|
|
||||||
|
final StructuredProperty sp5 = new StructuredProperty();
|
||||||
|
sp5.setValue("5");
|
||||||
|
sp5.setQualifier(qf);
|
||||||
|
|
||||||
|
final StructuredProperty sp6a = new StructuredProperty();
|
||||||
|
sp6a.setValue("6");
|
||||||
|
sp6a.setQualifier(qf);
|
||||||
|
|
||||||
|
final StructuredProperty sp6b = new StructuredProperty();
|
||||||
|
sp6b.setValue("6");
|
||||||
|
sp6b.setQualifier(qf);
|
||||||
|
|
||||||
|
final Result oaf = new Result();
|
||||||
|
oaf.setPid(new ArrayList<>());
|
||||||
|
oaf.getPid().add(sp1);
|
||||||
|
oaf.getPid().add(sp2);
|
||||||
|
oaf.getPid().add(sp4a);
|
||||||
|
|
||||||
|
final Instance instance1 = new Instance();
|
||||||
|
instance1.setPid(new ArrayList<>());
|
||||||
|
instance1.setAlternateIdentifier(new ArrayList<>());
|
||||||
|
instance1.getPid().add(sp3);
|
||||||
|
instance1.getPid().add(sp4b);
|
||||||
|
instance1.getAlternateIdentifier().add(sp5);
|
||||||
|
instance1.getAlternateIdentifier().add(sp6a);
|
||||||
|
|
||||||
|
final Instance instance2 = new Instance();
|
||||||
|
instance2.setPid(new ArrayList<>());
|
||||||
|
instance2.setAlternateIdentifier(new ArrayList<>());
|
||||||
|
instance2.getPid().add(sp6b);
|
||||||
|
|
||||||
|
oaf.setInstance(new ArrayList<>());
|
||||||
|
oaf.getInstance().add(instance1);
|
||||||
|
oaf.getInstance().add(instance2);
|
||||||
|
|
||||||
|
final List<OaBrokerTypedValue> list = ConversionUtils.allResultPids(oaf);
|
||||||
|
|
||||||
|
// list.forEach(x -> System.out.println(x.getValue()));
|
||||||
|
|
||||||
|
assertEquals(6, list.size());
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -43,7 +43,7 @@ public class PropagationConstant {
|
||||||
|
|
||||||
public final static String NULL = "NULL";
|
public final static String NULL = "NULL";
|
||||||
|
|
||||||
public static final String INSTITUTIONAL_REPO_TYPE = "pubsrepository::institutional";
|
public static final String INSTITUTIONAL_REPO_TYPE = "institutional";
|
||||||
|
|
||||||
public static final String PROPAGATION_DATA_INFO_TYPE = "propagation";
|
public static final String PROPAGATION_DATA_INFO_TYPE = "propagation";
|
||||||
|
|
||||||
|
@ -233,9 +233,9 @@ public class PropagationConstant {
|
||||||
|
|
||||||
if (HdfsSupport.exists(inputPath, spark.sparkContext().hadoopConfiguration())) {
|
if (HdfsSupport.exists(inputPath, spark.sparkContext().hadoopConfiguration())) {
|
||||||
return spark
|
return spark
|
||||||
.read()
|
.read()
|
||||||
.textFile(inputPath)
|
.textFile(inputPath)
|
||||||
.map((MapFunction<String, R>) value -> OBJECT_MAPPER.readValue(value, clazz), Encoders.bean(clazz));
|
.map((MapFunction<String, R>) value -> OBJECT_MAPPER.readValue(value, clazz), Encoders.bean(clazz));
|
||||||
} else {
|
} else {
|
||||||
return spark.emptyDataset(Encoders.bean(clazz));
|
return spark.emptyDataset(Encoders.bean(clazz));
|
||||||
}
|
}
|
||||||
|
|
|
@ -102,7 +102,7 @@ public class PrepareResultInstRepoAssociation {
|
||||||
String query = "SELECT source datasourceId, target organizationId "
|
String query = "SELECT source datasourceId, target organizationId "
|
||||||
+ "FROM ( SELECT id "
|
+ "FROM ( SELECT id "
|
||||||
+ "FROM datasource "
|
+ "FROM datasource "
|
||||||
+ "WHERE datasourcetype.classid = '"
|
+ "WHERE lower(jurisdiction.classid) = '"
|
||||||
+ INSTITUTIONAL_REPO_TYPE
|
+ INSTITUTIONAL_REPO_TYPE
|
||||||
+ "' "
|
+ "' "
|
||||||
+ "AND datainfo.deletedbyinference = false " + blacklisted + " ) d "
|
+ "AND datainfo.deletedbyinference = false " + blacklisted + " ) d "
|
||||||
|
|
|
@ -219,7 +219,7 @@
|
||||||
<error to="Kill"/>
|
<error to="Kill"/>
|
||||||
</action>
|
</action>
|
||||||
|
|
||||||
<join name="wait" to="eosc_tag"/>
|
<join name="wait" to="End"/>
|
||||||
|
|
||||||
<action name="eosc_tag">
|
<action name="eosc_tag">
|
||||||
<spark xmlns="uri:oozie:spark-action:0.2">
|
<spark xmlns="uri:oozie:spark-action:0.2">
|
||||||
|
|
|
@ -56,3 +56,4 @@ return
|
||||||
</selcriteria>
|
</selcriteria>
|
||||||
</zenodocommunity>}
|
</zenodocommunity>}
|
||||||
</zenodocommunities>
|
</zenodocommunities>
|
||||||
|
</community>
|
Loading…
Reference in New Issue