sabeel
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Pull Requests Activity

Considering instance pids and alteternative identifiers

This commit is contained in:
Michele Artini 2023-01-11 09:37:34 +01:00 committed by Claudio Atzori
parent 7b7520850b
commit 91b845f611
2 changed files with 141 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

117
dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/ConversionUtils.java
View File

@ -2,7 +2,10 @@
package eu.dnetlib.dhp.broker.oa.util;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.function.Function;
import java.util.stream.Collectors;
@ -26,19 +29,29 @@ import eu.dnetlib.broker.objects.OaBrokerRelatedPublication;
import eu.dnetlib.broker.objects.OaBrokerRelatedSoftware;
import eu.dnetlib.broker.objects.OaBrokerTypedValue;
import eu.dnetlib.dhp.schema.common.ModelConstants;
import eu.dnetlib.dhp.schema.oaf.*;
import eu.dnetlib.dhp.schema.oaf.Author;
import eu.dnetlib.dhp.schema.oaf.Dataset;
import eu.dnetlib.dhp.schema.oaf.Datasource;
import eu.dnetlib.dhp.schema.oaf.ExternalReference;
import eu.dnetlib.dhp.schema.oaf.Field;
import eu.dnetlib.dhp.schema.oaf.Instance;
import eu.dnetlib.dhp.schema.oaf.Journal;
import eu.dnetlib.dhp.schema.oaf.KeyValue;
import eu.dnetlib.dhp.schema.oaf.Project;
import eu.dnetlib.dhp.schema.oaf.Publication;
import eu.dnetlib.dhp.schema.oaf.Qualifier;
import eu.dnetlib.dhp.schema.oaf.Result;
import eu.dnetlib.dhp.schema.oaf.Software;
import eu.dnetlib.dhp.schema.oaf.StructuredProperty;
public class ConversionUtils {
private static final Logger log = LoggerFactory.getLogger(ConversionUtils.class);
private ConversionUtils() {
}
private ConversionUtils() {}
public static List<OaBrokerInstance> oafInstanceToBrokerInstances(final Instance i) {
if (i == null) {
return new ArrayList<>();
}
if (i == null) { return new ArrayList<>(); }
return mappedList(i.getUrl(), url -> {
final OaBrokerInstance res = new OaBrokerInstance();
@ -58,35 +71,27 @@ public class ConversionUtils {
return sp != null ? new OaBrokerTypedValue(classId(sp.getQualifier()), sp.getValue()) : null;
}
public static OaBrokerTypedValue oafSubjectToBrokerTypedValue(final Subject sp) {
return sp != null ? new OaBrokerTypedValue(classId(sp.getQualifier()), sp.getValue()) : null;
}
public static OaBrokerRelatedDataset oafDatasetToBrokerDataset(final Dataset d) {
if (d == null) {
return null;
}
if (d == null) { return null; }
final OaBrokerRelatedDataset res = new OaBrokerRelatedDataset();
res.setOpenaireId(cleanOpenaireId(d.getId()));
res.setOriginalId(first(d.getOriginalId()));
res.setTitle(structPropValue(d.getTitle()));
res.setPids(mappedList(d.getPid(), ConversionUtils::oafPidToBrokerPid));
res.setPids(allResultPids(d));
res.setInstances(flatMappedList(d.getInstance(), ConversionUtils::oafInstanceToBrokerInstances));
res.setCollectedFrom(mappedFirst(d.getCollectedfrom(), KeyValue::getValue));
return res;
}
public static OaBrokerRelatedPublication oafPublicationToBrokerPublication(final Publication p) {
if (p == null) {
return null;
}
if (p == null) { return null; }
final OaBrokerRelatedPublication res = new OaBrokerRelatedPublication();
res.setOpenaireId(cleanOpenaireId(p.getId()));
res.setOriginalId(first(p.getOriginalId()));
res.setTitle(structPropValue(p.getTitle()));
res.setPids(mappedList(p.getPid(), ConversionUtils::oafPidToBrokerPid));
res.setPids(allResultPids(p));
res.setInstances(flatMappedList(p.getInstance(), ConversionUtils::oafInstanceToBrokerInstances));
res.setCollectedFrom(mappedFirst(p.getCollectedfrom(), KeyValue::getValue));
@ -94,9 +99,7 @@ public class ConversionUtils {
}
public static OaBrokerMainEntity oafResultToBrokerResult(final Result result) {
if (result == null) {
return null;
}
if (result == null) { return null; }
final OaBrokerMainEntity res = new OaBrokerMainEntity();
@ -106,16 +109,15 @@ public class ConversionUtils {
res.setTitles(structPropList(result.getTitle()));
res.setAbstracts(fieldList(result.getDescription()));
res.setLanguage(classId(result.getLanguage()));
res.setSubjects(subjectList(result.getSubject()));
res.setSubjects(structPropTypedList(result.getSubject()));
res.setCreators(mappedList(result.getAuthor(), ConversionUtils::oafAuthorToBrokerAuthor));
res.setPublicationdate(fieldValue(result.getDateofacceptance()));
res.setPublisher(fieldValue(result.getPublisher()));
res.setEmbargoenddate(fieldValue(result.getEmbargoenddate()));
res.setContributor(fieldList(result.getContributor()));
res
.setJournal(
result instanceof Publication ? oafJournalToBrokerJournal(((Publication) result).getJournal()) : null);
res.setPids(mappedList(result.getPid(), ConversionUtils::oafPidToBrokerPid));
.setJournal(result instanceof Publication ? oafJournalToBrokerJournal(((Publication) result).getJournal()) : null);
res.setPids(allResultPids(result));
res.setInstances(flatMappedList(result.getInstance(), ConversionUtils::oafInstanceToBrokerInstances));
res
.setExternalReferences(mappedList(result.getExternalReference(), ConversionUtils::oafExtRefToBrokerExtRef));
@ -123,14 +125,23 @@ public class ConversionUtils {
return res;
}
protected static List<OaBrokerTypedValue> allResultPids(final Result result) {
final Map<String, StructuredProperty> map = new HashMap<>();
result.getPid().forEach(sp -> map.put(sp.getValue(), sp));
result.getInstance().forEach(i -> {
i.getPid().forEach(sp -> map.put(sp.getValue(), sp));
i.getAlternateIdentifier().forEach(sp -> map.put(sp.getValue(), sp));
});
final List<OaBrokerTypedValue> pids = mappedList(map.values(), ConversionUtils::oafPidToBrokerPid);
return pids;
}
public static String cleanOpenaireId(final String id) {
return id.contains("|") ? StringUtils.substringAfter(id, "|") : id;
}
private static OaBrokerAuthor oafAuthorToBrokerAuthor(final Author author) {
if (author == null) {
return null;
}
if (author == null) { return null; }
final String pids = author.getPid() != null ? author
.getPid()
@ -154,9 +165,7 @@ public class ConversionUtils {
}
private static OaBrokerJournal oafJournalToBrokerJournal(final Journal journal) {
if (journal == null) {
return null;
}
if (journal == null) { return null; }
final OaBrokerJournal res = new OaBrokerJournal();
res.setName(journal.getName());
@ -168,9 +177,7 @@ public class ConversionUtils {
}
private static OaBrokerExternalReference oafExtRefToBrokerExtRef(final ExternalReference ref) {
if (ref == null) {
return null;
}
if (ref == null) { return null; }
final OaBrokerExternalReference res = new OaBrokerExternalReference();
res.setRefidentifier(ref.getRefidentifier());
@ -181,9 +188,7 @@ public class ConversionUtils {
}
public static OaBrokerProject oafProjectToBrokerProject(final Project p) {
if (p == null) {
return null;
}
if (p == null) { return null; }
final OaBrokerProject res = new OaBrokerProject();
res.setOpenaireId(cleanOpenaireId(p.getId()));
@ -207,9 +212,7 @@ public class ConversionUtils {
}
public static OaBrokerRelatedSoftware oafSoftwareToBrokerSoftware(final Software sw) {
if (sw == null) {
return null;
}
if (sw == null) { return null; }
final OaBrokerRelatedSoftware res = new OaBrokerRelatedSoftware();
res.setOpenaireId(cleanOpenaireId(sw.getId()));
@ -222,9 +225,7 @@ public class ConversionUtils {
}
public static OaBrokerRelatedDatasource oafDatasourceToBrokerDatasource(final Datasource ds) {
if (ds == null) {
return null;
}
if (ds == null) { return null; }
final OaBrokerRelatedDatasource res = new OaBrokerRelatedDatasource();
res.setName(StringUtils.defaultIfBlank(fieldValue(ds.getOfficialname()), fieldValue(ds.getEnglishname())));
@ -284,9 +285,7 @@ public class ConversionUtils {
}
private static List<OaBrokerTypedValue> structPropTypedList(final List<StructuredProperty> list) {
if (list == null) {
return new ArrayList<>();
}
if (list == null) { return new ArrayList<>(); }
return list
.stream()
@ -295,22 +294,8 @@ public class ConversionUtils {
.collect(Collectors.toList());
}
private static List<OaBrokerTypedValue> subjectList(final List<Subject> list) {
if (list == null) {
return new ArrayList<>();
}
return list
.stream()
.map(ConversionUtils::oafSubjectToBrokerTypedValue)
.filter(Objects::nonNull)
.collect(Collectors.toList());
}
private static <F, T> List<T> mappedList(final List<F> list, final Function<F, T> func) {
if (list == null) {
return new ArrayList<>();
}
private static <F, T> List<T> mappedList(final Collection<F> list, final Function<F, T> func) {
if (list == null) { return new ArrayList<>(); }
return list
.stream()
@ -321,9 +306,7 @@ public class ConversionUtils {
}
private static <F, T> List<T> flatMappedList(final List<F> list, final Function<F, List<T>> func) {
if (list == null) {
return new ArrayList<>();
}
if (list == null) { return new ArrayList<>(); }
return list
.stream()
@ -335,9 +318,7 @@ public class ConversionUtils {
}
private static <F, T> T mappedFirst(final List<F> list, final Function<F, T> func) {
if (list == null) {
return null;
}
if (list == null) { return null; }
return list
.stream()

92
dhp-workflows/dhp-broker-events/src/test/java/eu/dnetlib/dhp/broker/oa/util/ConversionUtilsTest.java Normal file
View File

@ -0,0 +1,92 @@
package eu.dnetlib.dhp.broker.oa.util;
import static org.junit.jupiter.api.Assertions.assertEquals;
import java.util.ArrayList;
import java.util.List;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import eu.dnetlib.broker.objects.OaBrokerTypedValue;
import eu.dnetlib.dhp.schema.oaf.Instance;
import eu.dnetlib.dhp.schema.oaf.Qualifier;
import eu.dnetlib.dhp.schema.oaf.Result;
import eu.dnetlib.dhp.schema.oaf.StructuredProperty;
class ConversionUtilsTest {
@BeforeEach
void setUp() throws Exception {}
@Test
void testAllResultPids() {
final Qualifier qf = new Qualifier();
qf.setClassid("test");
qf.setClassname("test");
qf.setSchemeid("test");
qf.setSchemename("test");
final StructuredProperty sp1 = new StructuredProperty();
sp1.setValue("1");
sp1.setQualifier(qf);
final StructuredProperty sp2 = new StructuredProperty();
sp2.setValue("2");
sp2.setQualifier(qf);
final StructuredProperty sp3 = new StructuredProperty();
sp3.setValue("3");
sp3.setQualifier(qf);
final StructuredProperty sp4a = new StructuredProperty();
sp4a.setValue("4");
sp4a.setQualifier(qf);
final StructuredProperty sp4b = new StructuredProperty();
sp4b.setValue("4");
sp4b.setQualifier(qf);
final StructuredProperty sp5 = new StructuredProperty();
sp5.setValue("5");
sp5.setQualifier(qf);
final StructuredProperty sp6a = new StructuredProperty();
sp6a.setValue("6");
sp6a.setQualifier(qf);
final StructuredProperty sp6b = new StructuredProperty();
sp6b.setValue("6");
sp6b.setQualifier(qf);
final Result oaf = new Result();
oaf.setPid(new ArrayList<>());
oaf.getPid().add(sp1);
oaf.getPid().add(sp2);
oaf.getPid().add(sp4a);
final Instance instance1 = new Instance();
instance1.setPid(new ArrayList<>());
instance1.setAlternateIdentifier(new ArrayList<>());
instance1.getPid().add(sp3);
instance1.getPid().add(sp4b);
instance1.getAlternateIdentifier().add(sp5);
instance1.getAlternateIdentifier().add(sp6a);
final Instance instance2 = new Instance();
instance2.setPid(new ArrayList<>());
instance2.setAlternateIdentifier(new ArrayList<>());
instance2.getPid().add(sp6b);
oaf.setInstance(new ArrayList<>());
oaf.getInstance().add(instance1);
oaf.getInstance().add(instance2);
final List<OaBrokerTypedValue> list = ConversionUtils.allResultPids(oaf);
// list.forEach(x -> System.out.println(x.getValue()));
assertEquals(6, list.size());
}
}