D-Net
/
dnet-hadoop
13
4
Fork 10
Code Issues 14 Pull Requests 8 Projects 2 Releases 1 Wiki Activity

mapping of oaf:country in results

This commit is contained in:
Michele Artini 2024-03-14 10:16:39 +01:00
parent a99942f7cf
commit 28576d104e
8 changed files with 266 additions and 151 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
dhp-workflows/dhp-doiboost/src/test/scala/eu/dnetlib/dhp/doiboost/crossref/CrossrefMappingTest.scala
View File

@ -23,10 +23,15 @@ class CrossrefMappingTest {
val mapper = new ObjectMapper() val mapper = new ObjectMapper()
@Test @Test
def testMissingAuthorParser():Unit = { def testMissingAuthorParser(): Unit = {
val json: String = Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/doiboost/crossref/s41567-022-01757-y.json")).mkString val json: String = Source
.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/doiboost/crossref/s41567-022-01757-y.json"))
.mkString
val result = Crossref2Oaf.convert(json) val result = Crossref2Oaf.convert(json)
result.filter(o => o.isInstanceOf[Publication]).map(p=> p.asInstanceOf[Publication]).foreach(p =>assertTrue(p.getAuthor.size()>0)) result
.filter(o => o.isInstanceOf[Publication])
.map(p => p.asInstanceOf[Publication])
.foreach(p => assertTrue(p.getAuthor.size() > 0))
} }
@Test @Test

2
dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/Constraints.java
View File

@ -53,7 +53,7 @@ public class Constraints implements Serializable {
for (Constraint sc : constraint) { for (Constraint sc : constraint) {
boolean verified = false; boolean verified = false;
if(!param.containsKey(sc.getField())) if (!param.containsKey(sc.getField()))
return false; return false;
for (String value : param.get(sc.getField())) { for (String value : param.get(sc.getField())) {
if (sc.verifyCriteria(value.trim())) { if (sc.verifyCriteria(value.trim())) {

187
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/AbstractMdRecordToOafMapper.java
View File

@ -1,16 +1,50 @@
package eu.dnetlib.dhp.oa.graph.raw; package eu.dnetlib.dhp.oa.graph.raw;
import static eu.dnetlib.dhp.schema.common.ModelConstants.*; import static eu.dnetlib.dhp.schema.common.ModelConstants.AFFILIATION;
import static eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils.*; import static eu.dnetlib.dhp.schema.common.ModelConstants.DNET_PID_TYPES;
import static eu.dnetlib.dhp.schema.common.ModelConstants.HAS_AUTHOR_INSTITUTION;
import static eu.dnetlib.dhp.schema.common.ModelConstants.IS_AUTHOR_INSTITUTION_OF;
import static eu.dnetlib.dhp.schema.common.ModelConstants.IS_PRODUCED_BY;
import static eu.dnetlib.dhp.schema.common.ModelConstants.OPENAIRE_COAR_RESOURCE_TYPES_3_1;
import static eu.dnetlib.dhp.schema.common.ModelConstants.OUTCOME;
import static eu.dnetlib.dhp.schema.common.ModelConstants.PRODUCES;
import static eu.dnetlib.dhp.schema.common.ModelConstants.REPOSITORY_PROVENANCE_ACTIONS;
import static eu.dnetlib.dhp.schema.common.ModelConstants.RESULT_ORGANIZATION;
import static eu.dnetlib.dhp.schema.common.ModelConstants.RESULT_PROJECT;
import static eu.dnetlib.dhp.schema.common.ModelConstants.UNKNOWN;
import static eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils.createOpenaireId; import static eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils.createOpenaireId;
import static eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils.dataInfo;
import static eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils.field;
import static eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils.journal;
import static eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils.keyValue;
import static eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils.listFields;
import static eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils.oaiIProvenance;
import static eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils.qualifier;
import static eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils.structuredProperty;
import static eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils.subject;
import java.util.*; import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.commons.validator.routines.UrlValidator; import org.apache.commons.validator.routines.UrlValidator;
import org.dom4j.*; import org.dom4j.Document;
import org.dom4j.DocumentException;
import org.dom4j.DocumentFactory;
import org.dom4j.DocumentHelper;
import org.dom4j.Element;
import org.dom4j.Node;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -21,7 +55,29 @@ import eu.dnetlib.dhp.common.Constants;
import eu.dnetlib.dhp.common.vocabulary.VocabularyGroup; import eu.dnetlib.dhp.common.vocabulary.VocabularyGroup;
import eu.dnetlib.dhp.schema.common.ModelConstants; import eu.dnetlib.dhp.schema.common.ModelConstants;
import eu.dnetlib.dhp.schema.common.ModelSupport; import eu.dnetlib.dhp.schema.common.ModelSupport;
import eu.dnetlib.dhp.schema.oaf.*; import eu.dnetlib.dhp.schema.oaf.AccessRight;
import eu.dnetlib.dhp.schema.oaf.Author;
import eu.dnetlib.dhp.schema.oaf.Context;
import eu.dnetlib.dhp.schema.oaf.Country;
import eu.dnetlib.dhp.schema.oaf.DataInfo;
import eu.dnetlib.dhp.schema.oaf.Dataset;
import eu.dnetlib.dhp.schema.oaf.EoscIfGuidelines;
import eu.dnetlib.dhp.schema.oaf.Field;
import eu.dnetlib.dhp.schema.oaf.GeoLocation;
import eu.dnetlib.dhp.schema.oaf.Instance;
import eu.dnetlib.dhp.schema.oaf.InstanceTypeMapping;
import eu.dnetlib.dhp.schema.oaf.Journal;
import eu.dnetlib.dhp.schema.oaf.KeyValue;
import eu.dnetlib.dhp.schema.oaf.OAIProvenance;
import eu.dnetlib.dhp.schema.oaf.Oaf;
import eu.dnetlib.dhp.schema.oaf.OafEntity;
import eu.dnetlib.dhp.schema.oaf.OtherResearchProduct;
import eu.dnetlib.dhp.schema.oaf.Publication;
import eu.dnetlib.dhp.schema.oaf.Qualifier;
import eu.dnetlib.dhp.schema.oaf.Result;
import eu.dnetlib.dhp.schema.oaf.Software;
import eu.dnetlib.dhp.schema.oaf.StructuredProperty;
import eu.dnetlib.dhp.schema.oaf.Subject;
import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory; import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory;
import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils; import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils;
@ -43,9 +99,7 @@ public abstract class AbstractMdRecordToOafMapper {
protected static final String DATACITE_SCHEMA_KERNEL_3_SLASH = "http://datacite.org/schema/kernel-3/"; protected static final String DATACITE_SCHEMA_KERNEL_3_SLASH = "http://datacite.org/schema/kernel-3/";
protected static final Qualifier ORCID_PID_TYPE = qualifier( protected static final Qualifier ORCID_PID_TYPE = qualifier(
ModelConstants.ORCID_PENDING, ModelConstants.ORCID_PENDING, ModelConstants.ORCID_CLASSNAME, DNET_PID_TYPES, DNET_PID_TYPES);
ModelConstants.ORCID_CLASSNAME,
DNET_PID_TYPES, DNET_PID_TYPES);
protected static final Qualifier MAG_PID_TYPE = qualifier( protected static final Qualifier MAG_PID_TYPE = qualifier(
"MAGIdentifier", "Microsoft Academic Graph Identifier", DNET_PID_TYPES, DNET_PID_TYPES); "MAGIdentifier", "Microsoft Academic Graph Identifier", DNET_PID_TYPES, DNET_PID_TYPES);
@ -118,7 +172,7 @@ public abstract class AbstractMdRecordToOafMapper {
return Lists.newArrayList(); return Lists.newArrayList();
} }
final DataInfo entityInfo = prepareDataInfo(doc, invisible); final DataInfo entityInfo = prepareDataInfo(doc, this.invisible);
final long lastUpdateTimestamp = new Date().getTime(); final long lastUpdateTimestamp = new Date().getTime();
final List<Instance> instances = prepareInstances(doc, entityInfo, collectedFrom, hostedBy); final List<Instance> instances = prepareInstances(doc, entityInfo, collectedFrom, hostedBy);
@ -126,7 +180,7 @@ public abstract class AbstractMdRecordToOafMapper {
final String type = getResultType(doc, instances); final String type = getResultType(doc, instances);
return createOafs(doc, type, instances, collectedFrom, entityInfo, lastUpdateTimestamp); return createOafs(doc, type, instances, collectedFrom, entityInfo, lastUpdateTimestamp);
} catch (DocumentException e) { } catch (final DocumentException e) {
log.error("Error with record:\n" + xml); log.error("Error with record:\n" + xml);
return Lists.newArrayList(); return Lists.newArrayList();
} }
@ -135,7 +189,7 @@ public abstract class AbstractMdRecordToOafMapper {
protected String getResultType(final Document doc, final List<Instance> instances) { protected String getResultType(final Document doc, final List<Instance> instances) {
final String type = doc.valueOf("//dr:CobjCategory/@type"); final String type = doc.valueOf("//dr:CobjCategory/@type");
if (StringUtils.isBlank(type) && vocs.vocabularyExists(ModelConstants.DNET_RESULT_TYPOLOGIES)) { if (StringUtils.isBlank(type) && this.vocs.vocabularyExists(ModelConstants.DNET_RESULT_TYPOLOGIES)) {
final String instanceType = instances final String instanceType = instances
.stream() .stream()
.map(i -> i.getInstancetype().getClassid()) .map(i -> i.getInstancetype().getClassid())
@ -143,7 +197,7 @@ public abstract class AbstractMdRecordToOafMapper {
.filter(s -> !UNKNOWN.equalsIgnoreCase(s)) .filter(s -> !UNKNOWN.equalsIgnoreCase(s))
.orElse("0000"); // Unknown .orElse("0000"); // Unknown
return Optional return Optional
.ofNullable(vocs.getSynonymAsQualifier(ModelConstants.DNET_RESULT_TYPOLOGIES, instanceType)) .ofNullable(this.vocs.getSynonymAsQualifier(ModelConstants.DNET_RESULT_TYPOLOGIES, instanceType))
.map(Qualifier::getClassid) .map(Qualifier::getClassid)
.orElse("0000"); .orElse("0000");
} }
@ -170,15 +224,14 @@ public abstract class AbstractMdRecordToOafMapper {
final DataInfo info, final DataInfo info,
final long lastUpdateTimestamp) { final long lastUpdateTimestamp) {
final OafEntity entity = createEntity( final OafEntity entity = createEntity(doc, type, instances, collectedFrom, info, lastUpdateTimestamp);
doc, type, instances, collectedFrom, info, lastUpdateTimestamp);
final Set<String> originalId = Sets.newHashSet(entity.getOriginalId()); final Set<String> originalId = Sets.newHashSet(entity.getOriginalId());
originalId.add(entity.getId()); originalId.add(entity.getId());
entity.setOriginalId(Lists.newArrayList(originalId)); entity.setOriginalId(Lists.newArrayList(originalId));
if (!forceOriginalId) { if (!this.forceOriginalId) {
final String id = IdentifierFactory.createIdentifier(entity, shouldHashId); final String id = IdentifierFactory.createIdentifier(entity, this.shouldHashId);
if (!id.equals(entity.getId())) { if (!id.equals(entity.getId())) {
entity.setId(id); entity.setId(id);
} }
@ -189,7 +242,7 @@ public abstract class AbstractMdRecordToOafMapper {
final DataInfo relationInfo = prepareDataInfo(doc, false); final DataInfo relationInfo = prepareDataInfo(doc, false);
if (!oafs.isEmpty()) { if (!oafs.isEmpty()) {
Set<Oaf> rels = Sets.newHashSet(); final Set<Oaf> rels = Sets.newHashSet();
rels.addAll(addProjectRels(doc, entity, relationInfo)); rels.addAll(addProjectRels(doc, entity, relationInfo));
rels.addAll(addOtherResultRels(doc, entity, relationInfo)); rels.addAll(addOtherResultRels(doc, entity, relationInfo));
@ -247,7 +300,8 @@ public abstract class AbstractMdRecordToOafMapper {
private List<Oaf> addProjectRels( private List<Oaf> addProjectRels(
final Document doc, final Document doc,
final OafEntity entity, DataInfo info) { final OafEntity entity,
final DataInfo info) {
final List<Oaf> res = new ArrayList<>(); final List<Oaf> res = new ArrayList<>();
@ -267,25 +321,29 @@ public abstract class AbstractMdRecordToOafMapper {
OafMapperUtils OafMapperUtils
.getRelation( .getRelation(
docId, projectId, RESULT_PROJECT, OUTCOME, IS_PRODUCED_BY, entity.getCollectedfrom(), docId, projectId, RESULT_PROJECT, OUTCOME, IS_PRODUCED_BY, entity.getCollectedfrom(),
info, entity.getLastupdatetimestamp(), validationdDate, null)); info, entity
.getLastupdatetimestamp(),
validationdDate, null));
res res
.add( .add(
OafMapperUtils OafMapperUtils
.getRelation( .getRelation(
projectId, docId, RESULT_PROJECT, OUTCOME, PRODUCES, entity.getCollectedfrom(), info, projectId, docId, RESULT_PROJECT, OUTCOME, PRODUCES, entity.getCollectedfrom(), info,
entity.getLastupdatetimestamp(), validationdDate, null)); entity
.getLastupdatetimestamp(),
validationdDate, null));
} }
} }
return res; return res;
} }
private List<Oaf> addRelations(Document doc, OafEntity entity, DataInfo info) { private List<Oaf> addRelations(final Document doc, final OafEntity entity, final DataInfo info) {
final List<Oaf> rels = Lists.newArrayList(); final List<Oaf> rels = Lists.newArrayList();
for (Object o : doc.selectNodes("//oaf:relation")) { for (final Object o : doc.selectNodes("//oaf:relation")) {
Element element = (Element) o; final Element element = (Element) o;
final String target = StringUtils.trim(element.getText()); final String target = StringUtils.trim(element.getText());
final String relType = element.attributeValue("relType"); final String relType = element.attributeValue("relType");
@ -309,15 +367,17 @@ public abstract class AbstractMdRecordToOafMapper {
OafMapperUtils OafMapperUtils
.getRelation( .getRelation(
entity.getId(), targetId, relType, subRelType, relClass, entity.getId(), targetId, relType, subRelType, relClass,
entity.getCollectedfrom(), info, entity.getCollectedfrom(), info, entity
entity.getLastupdatetimestamp(), validationDate, null)); .getLastupdatetimestamp(),
validationDate, null));
rels rels
.add( .add(
OafMapperUtils OafMapperUtils
.getRelation( .getRelation(
targetId, entity.getId(), relType, subRelType, relClassInverse, targetId, entity.getId(), relType, subRelType, relClassInverse,
entity.getCollectedfrom(), info, entity.getCollectedfrom(), info, entity
entity.getLastupdatetimestamp(), validationDate, null)); .getLastupdatetimestamp(),
validationDate, null));
} }
} }
} }
@ -325,24 +385,24 @@ public abstract class AbstractMdRecordToOafMapper {
return rels; return rels;
} }
private List<Oaf> addAffiliations(Document doc, OafEntity entity, DataInfo info) { private List<Oaf> addAffiliations(final Document doc, final OafEntity entity, final DataInfo info) {
final List<Oaf> rels = Lists.newArrayList(); final List<Oaf> rels = Lists.newArrayList();
for (Object o : doc.selectNodes("//datacite:affiliation[@affiliationIdentifierScheme='ROR']")) { for (final Object o : doc.selectNodes("//datacite:affiliation[@affiliationIdentifierScheme='ROR']")) {
Element element = (Element) o; final Element element = (Element) o;
String rorId = element.attributeValue("affiliationIdentifier"); final String rorId = element.attributeValue("affiliationIdentifier");
if (StringUtils.isNotBlank(rorId)) { if (StringUtils.isNotBlank(rorId)) {
String fullRorId = Constants.ROR_NS_PREFIX + "::" + rorId; final String fullRorId = Constants.ROR_NS_PREFIX + "::" + rorId;
String resultId = entity.getId(); final String resultId = entity.getId();
String orgId = createOpenaireId("organization", fullRorId, true); final String orgId = createOpenaireId("organization", fullRorId, true);
List<KeyValue> properties = Lists.newArrayList(); final List<KeyValue> properties = Lists.newArrayList();
String apcAmount = doc.valueOf("//oaf:processingchargeamount"); final String apcAmount = doc.valueOf("//oaf:processingchargeamount");
String apcCurrency = doc.valueOf("//oaf:processingchargeamount/@currency"); final String apcCurrency = doc.valueOf("//oaf:processingchargeamount/@currency");
if (StringUtils.isNotBlank(apcAmount) && StringUtils.isNotBlank(apcCurrency)) { if (StringUtils.isNotBlank(apcAmount) && StringUtils.isNotBlank(apcCurrency)) {
properties.add(OafMapperUtils.keyValue("apc_amount", apcAmount)); properties.add(OafMapperUtils.keyValue("apc_amount", apcAmount));
@ -354,15 +414,16 @@ public abstract class AbstractMdRecordToOafMapper {
OafMapperUtils OafMapperUtils
.getRelation( .getRelation(
resultId, orgId, RESULT_ORGANIZATION, AFFILIATION, HAS_AUTHOR_INSTITUTION, resultId, orgId, RESULT_ORGANIZATION, AFFILIATION, HAS_AUTHOR_INSTITUTION,
entity.getCollectedfrom(), info, entity.getLastupdatetimestamp(), null, entity.getCollectedfrom(), info, entity
properties)); .getLastupdatetimestamp(),
null, properties));
rels rels
.add( .add(
OafMapperUtils OafMapperUtils
.getRelation( .getRelation(
orgId, resultId, RESULT_ORGANIZATION, AFFILIATION, IS_AUTHOR_INSTITUTION_OF, orgId, resultId, RESULT_ORGANIZATION, AFFILIATION, IS_AUTHOR_INSTITUTION_OF, entity
entity.getCollectedfrom(), info, entity.getLastupdatetimestamp(), null, .getCollectedfrom(),
properties)); info, entity.getLastupdatetimestamp(), null, properties));
} }
} }
return rels; return rels;
@ -370,7 +431,8 @@ public abstract class AbstractMdRecordToOafMapper {
protected abstract List<Oaf> addOtherResultRels( protected abstract List<Oaf> addOtherResultRels(
final Document doc, final Document doc,
final OafEntity entity, DataInfo info); final OafEntity entity,
DataInfo info);
private void populateResultFields( private void populateResultFields(
final Result r, final Result r,
@ -391,7 +453,7 @@ public abstract class AbstractMdRecordToOafMapper {
r.setOaiprovenance(prepareOAIprovenance(doc)); r.setOaiprovenance(prepareOAIprovenance(doc));
r.setAuthor(prepareAuthors(doc, info)); r.setAuthor(prepareAuthors(doc, info));
r.setLanguage(prepareLanguages(doc)); r.setLanguage(prepareLanguages(doc));
r.setCountry(new ArrayList<>()); // NOT PRESENT IN MDSTORES r.setCountry(prepareCountries(doc, info));
r.setSubject(prepareSubjects(doc, info)); r.setSubject(prepareSubjects(doc, info));
r.setTitle(prepareTitles(doc, info)); r.setTitle(prepareTitles(doc, info));
r.setRelevantdate(prepareRelevantDates(doc, info)); r.setRelevantdate(prepareRelevantDates(doc, info));
@ -433,7 +495,7 @@ public abstract class AbstractMdRecordToOafMapper {
return list; return list;
} }
private List<EoscIfGuidelines> prepareEOSCIfGuidelines(Document doc, DataInfo info) { private List<EoscIfGuidelines> prepareEOSCIfGuidelines(final Document doc, final DataInfo info) {
final Set<EoscIfGuidelines> set = Sets.newHashSet(); final Set<EoscIfGuidelines> set = Sets.newHashSet();
for (final Object o : doc.selectNodes("//oaf:eoscifguidelines")) { for (final Object o : doc.selectNodes("//oaf:eoscifguidelines")) {
final String code = ((Node) o).valueOf("@code"); final String code = ((Node) o).valueOf("@code");
@ -520,7 +582,7 @@ public abstract class AbstractMdRecordToOafMapper {
protected abstract String findOriginalType(Document doc); protected abstract String findOriginalType(Document doc);
protected List<InstanceTypeMapping> prepareInstanceTypeMapping(Document doc) { protected List<InstanceTypeMapping> prepareInstanceTypeMapping(final Document doc) {
return Optional return Optional
.ofNullable(findOriginalType(doc)) .ofNullable(findOriginalType(doc))
.map(originalType -> { .map(originalType -> {
@ -531,6 +593,24 @@ public abstract class AbstractMdRecordToOafMapper {
.orElse(new ArrayList<>()); .orElse(new ArrayList<>());
} }
private List<Country> prepareCountries(final Document doc, final DataInfo info) {
final List<Country> list = new ArrayList<>();
for (final Object n : doc.selectNodes("//oaf:country")) {
final String code = ((Node) n).valueOf("@code").trim();
final String name = ((Node) n).getText().trim();
if (StringUtils.isNotBlank(code) || StringUtils.isNotBlank(name)) {
final Country country = new Country();
country.setClassid(StringUtils.isNotBlank(code) ? code : name);
country.setClassname(StringUtils.isNotBlank(name) ? name : code);
country.setSchemeid(ModelConstants.DNET_COUNTRY_TYPE);
country.setSchemename(ModelConstants.DNET_COUNTRY_TYPE);
country.setDataInfo(info);
list.add(country);
}
}
return list;
}
private Journal prepareJournal(final Document doc, final DataInfo info) { private Journal prepareJournal(final Document doc, final DataInfo info) {
final Node n = doc.selectSingleNode("//oaf:journal"); final Node n = doc.selectSingleNode("//oaf:journal");
if (n != null) { if (n != null) {
@ -587,7 +667,7 @@ public abstract class AbstractMdRecordToOafMapper {
} }
protected Qualifier prepareQualifier(final String classId, final String schemeId) { protected Qualifier prepareQualifier(final String classId, final String schemeId) {
return vocs.getTermAsQualifier(schemeId, classId); return this.vocs.getTermAsQualifier(schemeId, classId);
} }
protected List<StructuredProperty> prepareListStructPropsWithValidQualifier( protected List<StructuredProperty> prepareListStructPropsWithValidQualifier(
@ -601,8 +681,8 @@ public abstract class AbstractMdRecordToOafMapper {
for (final Object o : node.selectNodes(xpath)) { for (final Object o : node.selectNodes(xpath)) {
final Node n = (Node) o; final Node n = (Node) o;
final String classId = n.valueOf(xpathClassId).trim(); final String classId = n.valueOf(xpathClassId).trim();
if (vocs.termExists(schemeId, classId)) { if (this.vocs.termExists(schemeId, classId)) {
res.add(structuredProperty(n.getText(), vocs.getTermAsQualifier(schemeId, classId), info)); res.add(structuredProperty(n.getText(), this.vocs.getTermAsQualifier(schemeId, classId), info));
} }
} }
return res; return res;
@ -631,8 +711,9 @@ public abstract class AbstractMdRecordToOafMapper {
res res
.add( .add(
structuredProperty( structuredProperty(
n.getText(), n.valueOf("@classid"), n.valueOf("@classname"), n.valueOf("@schemeid"), n.getText(), n.valueOf("@classid"), n.valueOf("@classname"), n.valueOf("@schemeid"), n
n.valueOf("@schemename"), info)); .valueOf("@schemename"),
info));
} }
return res; return res;
} }
@ -663,7 +744,7 @@ public abstract class AbstractMdRecordToOafMapper {
final String identifier = n.valueOf("./*[local-name()='identifier']"); final String identifier = n.valueOf("./*[local-name()='identifier']");
final String baseURL = n.valueOf("./*[local-name()='baseURL']"); final String baseURL = n.valueOf("./*[local-name()='baseURL']");
final String metadataNamespace = n.valueOf("./*[local-name()='metadataNamespace']"); final String metadataNamespace = n.valueOf("./*[local-name()='metadataNamespace']");
final boolean altered = n.valueOf("@altered").equalsIgnoreCase("true"); final boolean altered = "true".equalsIgnoreCase(n.valueOf("@altered"));
final String datestamp = n.valueOf("./*[local-name()='datestamp']"); final String datestamp = n.valueOf("./*[local-name()='datestamp']");
final String harvestDate = n.valueOf("@harvestDate"); final String harvestDate = n.valueOf("@harvestDate");
@ -722,7 +803,7 @@ public abstract class AbstractMdRecordToOafMapper {
return res; return res;
} }
protected Set<String> validateUrl(Collection<String> url) { protected Set<String> validateUrl(final Collection<String> url) {
if (Objects.isNull(url)) { if (Objects.isNull(url)) {
return new HashSet<>(); return new HashSet<>();

16
dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/CopyHdfsOafSparkApplicationTest.java
View File

@ -64,14 +64,14 @@ public class CopyHdfsOafSparkApplicationTest {
@Test @Test
void isOafType_Datacite_ORP() throws IOException { void isOafType_Datacite_ORP() throws IOException {
assertTrue( assertTrue(
CopyHdfsOafSparkApplication CopyHdfsOafSparkApplication
.isOafType( .isOafType(
IOUtils IOUtils
.toString( .toString(
getClass() getClass()
.getResourceAsStream( .getResourceAsStream(
"/eu/dnetlib/dhp/oa/graph/raw/datacite_orp.json")), "/eu/dnetlib/dhp/oa/graph/raw/datacite_orp.json")),
"otherresearchproduct")); "otherresearchproduct"));
} }
} }

193
dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MappersTest.java
View File

@ -3,7 +3,13 @@ package eu.dnetlib.dhp.oa.graph.raw;
import static eu.dnetlib.dhp.schema.oaf.utils.GraphCleaningFunctions.cleanup; import static eu.dnetlib.dhp.schema.oaf.utils.GraphCleaningFunctions.cleanup;
import static eu.dnetlib.dhp.schema.oaf.utils.GraphCleaningFunctions.fixVocabularyNames; import static eu.dnetlib.dhp.schema.oaf.utils.GraphCleaningFunctions.fixVocabularyNames;
import static org.junit.jupiter.api.Assertions.*; import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.lenient;
import java.io.IOException; import java.io.IOException;
@ -14,8 +20,6 @@ import java.util.stream.Collectors;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.spark.api.java.function.MapFunction;
import org.apache.spark.sql.Encoders;
import org.dom4j.DocumentException; import org.dom4j.DocumentException;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
@ -30,8 +34,18 @@ import eu.dnetlib.dhp.common.vocabulary.VocabularyGroup;
import eu.dnetlib.dhp.oa.graph.clean.CleaningRuleMap; import eu.dnetlib.dhp.oa.graph.clean.CleaningRuleMap;
import eu.dnetlib.dhp.oa.graph.clean.OafCleaner; import eu.dnetlib.dhp.oa.graph.clean.OafCleaner;
import eu.dnetlib.dhp.schema.common.ModelConstants; import eu.dnetlib.dhp.schema.common.ModelConstants;
import eu.dnetlib.dhp.schema.oaf.*; import eu.dnetlib.dhp.schema.oaf.Author;
import eu.dnetlib.dhp.schema.oaf.utils.GraphCleaningFunctions; import eu.dnetlib.dhp.schema.oaf.Dataset;
import eu.dnetlib.dhp.schema.oaf.Field;
import eu.dnetlib.dhp.schema.oaf.Instance;
import eu.dnetlib.dhp.schema.oaf.InstanceTypeMapping;
import eu.dnetlib.dhp.schema.oaf.KeyValue;
import eu.dnetlib.dhp.schema.oaf.Oaf;
import eu.dnetlib.dhp.schema.oaf.OtherResearchProduct;
import eu.dnetlib.dhp.schema.oaf.Publication;
import eu.dnetlib.dhp.schema.oaf.Relation;
import eu.dnetlib.dhp.schema.oaf.Software;
import eu.dnetlib.dhp.schema.oaf.StructuredProperty;
import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory; import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory;
import eu.dnetlib.dhp.schema.oaf.utils.PidType; import eu.dnetlib.dhp.schema.oaf.utils.PidType;
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
@ -47,12 +61,12 @@ class MappersTest {
@BeforeEach @BeforeEach
public void setUp() throws Exception { public void setUp() throws Exception {
lenient().when(isLookUpService.quickSearchProfile(VocabularyGroup.VOCABULARIES_XQUERY)).thenReturn(vocs()); lenient().when(this.isLookUpService.quickSearchProfile(VocabularyGroup.VOCABULARIES_XQUERY)).thenReturn(vocs());
lenient() lenient()
.when(isLookUpService.quickSearchProfile(VocabularyGroup.VOCABULARY_SYNONYMS_XQUERY)) .when(this.isLookUpService.quickSearchProfile(VocabularyGroup.VOCABULARY_SYNONYMS_XQUERY))
.thenReturn(synonyms()); .thenReturn(synonyms());
vocs = VocabularyGroup.loadVocsFromIS(isLookUpService); this.vocs = VocabularyGroup.loadVocsFromIS(this.isLookUpService);
} }
@Test @Test
@ -60,12 +74,12 @@ class MappersTest {
final String xml = IOUtils.toString(Objects.requireNonNull(getClass().getResourceAsStream("oaf_record.xml"))); final String xml = IOUtils.toString(Objects.requireNonNull(getClass().getResourceAsStream("oaf_record.xml")));
final List<Oaf> list = new OafToOafMapper(vocs, false, true).processMdRecord(xml); final List<Oaf> list = new OafToOafMapper(this.vocs, false, true).processMdRecord(xml);
assertEquals(1, list.stream().filter(o -> o instanceof Publication).count()); assertEquals(1, list.stream().filter(Publication.class::isInstance).count());
assertEquals(4, list.stream().filter(o -> o instanceof Relation).count()); assertEquals(4, list.stream().filter(Relation.class::isInstance).count());
Publication p = (Publication) list.stream().filter(o -> o instanceof Publication).findFirst().get(); final Publication p = (Publication) list.stream().filter(Publication.class::isInstance).findFirst().get();
assertValidId(p.getId()); assertValidId(p.getId());
@ -83,7 +97,7 @@ class MappersTest {
final Optional<Author> author = p final Optional<Author> author = p
.getAuthor() .getAuthor()
.stream() .stream()
.filter(a -> a.getPid() != null && !a.getPid().isEmpty()) .filter(a -> (a.getPid() != null) && !a.getPid().isEmpty())
.findFirst(); .findFirst();
assertTrue(author.isPresent()); assertTrue(author.isPresent());
@ -124,7 +138,7 @@ class MappersTest {
assertNotNull(instance.getInstanceTypeMapping()); assertNotNull(instance.getInstanceTypeMapping());
assertEquals(1, instance.getInstanceTypeMapping().size()); assertEquals(1, instance.getInstanceTypeMapping().size());
Optional<InstanceTypeMapping> coarType = instance final Optional<InstanceTypeMapping> coarType = instance
.getInstanceTypeMapping() .getInstanceTypeMapping()
.stream() .stream()
.filter(itm -> ModelConstants.OPENAIRE_COAR_RESOURCE_TYPES_3_1.equals(itm.getVocabularyName())) .filter(itm -> ModelConstants.OPENAIRE_COAR_RESOURCE_TYPES_3_1.equals(itm.getVocabularyName()))
@ -134,7 +148,7 @@ class MappersTest {
assertNull(coarType.get().getTypeCode()); assertNull(coarType.get().getTypeCode());
assertNull(coarType.get().getTypeLabel()); assertNull(coarType.get().getTypeLabel());
Optional<InstanceTypeMapping> userType = instance final Optional<InstanceTypeMapping> userType = instance
.getInstanceTypeMapping() .getInstanceTypeMapping()
.stream() .stream()
.filter(itm -> ModelConstants.OPENAIRE_USER_RESOURCE_TYPES.equals(itm.getVocabularyName())) .filter(itm -> ModelConstants.OPENAIRE_USER_RESOURCE_TYPES.equals(itm.getVocabularyName()))
@ -157,9 +171,9 @@ class MappersTest {
assertEquals("https://oneecosystem.pensoft.net/article/13718/", p.getFulltext().get(0).getValue()); assertEquals("https://oneecosystem.pensoft.net/article/13718/", p.getFulltext().get(0).getValue());
// RESULT PROJECT // RESULT PROJECT
List<Relation> resultProject = list final List<Relation> resultProject = list
.stream() .stream()
.filter(o -> o instanceof Relation) .filter(Relation.class::isInstance)
.map(o -> (Relation) o) .map(o -> (Relation) o)
.filter(r -> ModelConstants.RESULT_PROJECT.equals(r.getRelType())) .filter(r -> ModelConstants.RESULT_PROJECT.equals(r.getRelType()))
.collect(Collectors.toList()); .collect(Collectors.toList());
@ -180,9 +194,9 @@ class MappersTest {
assertEquals(rp2.getSource(), rp1.getTarget()); assertEquals(rp2.getSource(), rp1.getTarget());
// AFFILIATIONS // AFFILIATIONS
List<Relation> affiliation = list final List<Relation> affiliation = list
.stream() .stream()
.filter(o -> o instanceof Relation) .filter(Relation.class::isInstance)
.map(o -> (Relation) o) .map(o -> (Relation) o)
.filter(r -> ModelConstants.RESULT_ORGANIZATION.equals(r.getRelType())) .filter(r -> ModelConstants.RESULT_ORGANIZATION.equals(r.getRelType()))
.collect(Collectors.toList()); .collect(Collectors.toList());
@ -196,9 +210,15 @@ class MappersTest {
assertEquals(aff1.getSource(), aff2.getTarget()); assertEquals(aff1.getSource(), aff2.getTarget());
assertEquals(aff2.getSource(), aff1.getTarget()); assertEquals(aff2.getSource(), aff1.getTarget());
assertEquals(3, p.getCountry().size());
assertEquals("IT", p.getCountry().get(0).getClassid());
assertEquals("FR", p.getCountry().get(1).getClassid());
assertEquals("DE", p.getCountry().get(2).getClassid());
} }
private void verifyRelation(Relation r) { private void verifyRelation(final Relation r) {
assertValidId(r.getSource()); assertValidId(r.getSource());
assertValidId(r.getTarget()); assertValidId(r.getTarget());
assertValidId(r.getCollectedfrom().get(0).getKey()); assertValidId(r.getCollectedfrom().get(0).getKey());
@ -215,7 +235,7 @@ class MappersTest {
final String xml = IOUtils final String xml = IOUtils
.toString(Objects.requireNonNull(getClass().getResourceAsStream("oaf_record_pubmed.xml"))); .toString(Objects.requireNonNull(getClass().getResourceAsStream("oaf_record_pubmed.xml")));
final List<Oaf> list = new OafToOafMapper(vocs, false, true).processMdRecord(xml); final List<Oaf> list = new OafToOafMapper(this.vocs, false, true).processMdRecord(xml);
assertEquals(1, list.size()); assertEquals(1, list.size());
assertTrue(list.get(0) instanceof Publication); assertTrue(list.get(0) instanceof Publication);
@ -237,7 +257,7 @@ class MappersTest {
final Optional<Author> author = p final Optional<Author> author = p
.getAuthor() .getAuthor()
.stream() .stream()
.filter(a -> a.getPid() != null && !a.getPid().isEmpty()) .filter(a -> (a.getPid() != null) && !a.getPid().isEmpty())
.findFirst(); .findFirst();
assertTrue(author.isPresent()); assertTrue(author.isPresent());
@ -270,7 +290,7 @@ class MappersTest {
assertEquals("OPEN", i.getAccessright().getClassid()); assertEquals("OPEN", i.getAccessright().getClassid());
}); });
Publication p_cleaned = cleanup(p, vocs); final Publication p_cleaned = cleanup(p, this.vocs);
assertEquals("0002", p_cleaned.getInstance().get(0).getRefereed().getClassid()); assertEquals("0002", p_cleaned.getInstance().get(0).getRefereed().getClassid());
assertEquals("nonPeerReviewed", p_cleaned.getInstance().get(0).getRefereed().getClassname()); assertEquals("nonPeerReviewed", p_cleaned.getInstance().get(0).getRefereed().getClassname());
@ -290,7 +310,7 @@ class MappersTest {
final String xml = IOUtils.toString(Objects.requireNonNull(getClass().getResourceAsStream("oaf_record.xml"))); final String xml = IOUtils.toString(Objects.requireNonNull(getClass().getResourceAsStream("oaf_record.xml")));
final List<Oaf> list = new OafToOafMapper(vocs, true, true).processMdRecord(xml); final List<Oaf> list = new OafToOafMapper(this.vocs, true, true).processMdRecord(xml);
assertFalse(list.isEmpty()); assertFalse(list.isEmpty());
assertTrue(list.get(0) instanceof Publication); assertTrue(list.get(0) instanceof Publication);
@ -307,15 +327,14 @@ class MappersTest {
.toString(Objects.requireNonNull(getClass().getResourceAsStream("odf_fwfebooklibrary.xml"))); .toString(Objects.requireNonNull(getClass().getResourceAsStream("odf_fwfebooklibrary.xml")));
assertThrows( assertThrows(
IllegalArgumentException.class, IllegalArgumentException.class, () -> new OdfToOafMapper(this.vocs, false, true).processMdRecord(xml));
() -> new OdfToOafMapper(vocs, false, true).processMdRecord(xml));
} }
@Test @Test
void testDataset() throws IOException, DocumentException { void testDataset() throws IOException, DocumentException {
final String xml = IOUtils.toString(Objects.requireNonNull(getClass().getResourceAsStream("odf_dataset.xml"))); final String xml = IOUtils.toString(Objects.requireNonNull(getClass().getResourceAsStream("odf_dataset.xml")));
final List<Oaf> list = new OdfToOafMapper(vocs, false, true).processMdRecord(xml); final List<Oaf> list = new OdfToOafMapper(this.vocs, false, true).processMdRecord(xml);
assertEquals(3, list.size()); assertEquals(3, list.size());
assertTrue(list.get(0) instanceof Dataset); assertTrue(list.get(0) instanceof Dataset);
@ -345,7 +364,7 @@ class MappersTest {
assertValidId(d.getId()); assertValidId(d.getId());
assertEquals("50|doi_________::000374d100a9db469bd42b69dbb40b36", d.getId()); assertEquals("50|doi_________::000374d100a9db469bd42b69dbb40b36", d.getId());
assertEquals(2, d.getOriginalId().size()); assertEquals(2, d.getOriginalId().size());
assertTrue(d.getOriginalId().stream().anyMatch(oid -> oid.equals("oai:zenodo.org:3234526"))); assertTrue(d.getOriginalId().stream().anyMatch("oai:zenodo.org:3234526"::equals));
assertValidId(d.getCollectedfrom().get(0).getKey()); assertValidId(d.getCollectedfrom().get(0).getKey());
assertTrue(StringUtils.isNotBlank(d.getTitle().get(0).getValue())); assertTrue(StringUtils.isNotBlank(d.getTitle().get(0).getValue()));
assertFalse(d.getAuthor().isEmpty()); assertFalse(d.getAuthor().isEmpty());
@ -353,7 +372,7 @@ class MappersTest {
final Optional<Author> author = d final Optional<Author> author = d
.getAuthor() .getAuthor()
.stream() .stream()
.filter(a -> a.getPid() != null && !a.getPid().isEmpty()) .filter(a -> (a.getPid() != null) && !a.getPid().isEmpty())
.findFirst(); .findFirst();
assertTrue(author.isPresent()); assertTrue(author.isPresent());
final Optional<StructuredProperty> oPid = author final Optional<StructuredProperty> oPid = author
@ -429,7 +448,7 @@ class MappersTest {
verifyTitle(d, "Subtitle", "survey"); verifyTitle(d, "Subtitle", "survey");
} }
private void verifyTitle(Dataset d, String titleType, String title) { private void verifyTitle(final Dataset d, final String titleType, final String title) {
Optional Optional
.of( .of(
d d
@ -448,7 +467,7 @@ class MappersTest {
final String xml = IOUtils final String xml = IOUtils
.toString(Objects.requireNonNull(getClass().getResourceAsStream("odf_bielefeld.xml"))); .toString(Objects.requireNonNull(getClass().getResourceAsStream("odf_bielefeld.xml")));
final List<Oaf> list = new OdfToOafMapper(vocs, false, true).processMdRecord(xml); final List<Oaf> list = new OdfToOafMapper(this.vocs, false, true).processMdRecord(xml);
assertEquals(1, list.size()); assertEquals(1, list.size());
assertTrue(list.get(0) instanceof Publication); assertTrue(list.get(0) instanceof Publication);
@ -458,7 +477,7 @@ class MappersTest {
assertValidId(p.getId()); assertValidId(p.getId());
assertEquals(2, p.getOriginalId().size()); assertEquals(2, p.getOriginalId().size());
assertTrue(p.getOriginalId().stream().anyMatch(oid -> oid.equals("oai:pub.uni-bielefeld.de:2949739"))); assertTrue(p.getOriginalId().stream().anyMatch("oai:pub.uni-bielefeld.de:2949739"::equals));
// assertEquals("oai:pub.uni-bielefeld.de:2949739", p.getOriginalId().get(0)); // assertEquals("oai:pub.uni-bielefeld.de:2949739", p.getOriginalId().get(0));
assertValidId(p.getCollectedfrom().get(0).getKey()); assertValidId(p.getCollectedfrom().get(0).getKey());
@ -489,7 +508,7 @@ class MappersTest {
assertEquals("OPEN", i.getAccessright().getClassid()); assertEquals("OPEN", i.getAccessright().getClassid());
}); });
Publication p_cleaned = cleanup(p, vocs); final Publication p_cleaned = cleanup(p, this.vocs);
assertEquals("0002", p_cleaned.getInstance().get(0).getRefereed().getClassid()); assertEquals("0002", p_cleaned.getInstance().get(0).getRefereed().getClassid());
assertEquals("nonPeerReviewed", p_cleaned.getInstance().get(0).getRefereed().getClassname()); assertEquals("nonPeerReviewed", p_cleaned.getInstance().get(0).getRefereed().getClassname());
} }
@ -499,7 +518,7 @@ class MappersTest {
final String xml = IOUtils final String xml = IOUtils
.toString(Objects.requireNonNull(getClass().getResourceAsStream("odf_opentrial.xml"))); .toString(Objects.requireNonNull(getClass().getResourceAsStream("odf_opentrial.xml")));
final List<Oaf> list = new OdfToOafMapper(vocs, false, true).processMdRecord(xml); final List<Oaf> list = new OdfToOafMapper(this.vocs, false, true).processMdRecord(xml);
assertEquals(1, list.size()); assertEquals(1, list.size());
assertTrue(list.get(0) instanceof Dataset); assertTrue(list.get(0) instanceof Dataset);
@ -608,7 +627,7 @@ class MappersTest {
assertTrue(i.getUrl().contains("http://apps.who.int/trialsearch/Trial3.aspx?trialid=NCT02321059")); assertTrue(i.getUrl().contains("http://apps.who.int/trialsearch/Trial3.aspx?trialid=NCT02321059"));
assertTrue(i.getUrl().contains("https://clinicaltrials.gov/ct2/show/NCT02321059")); assertTrue(i.getUrl().contains("https://clinicaltrials.gov/ct2/show/NCT02321059"));
Dataset d_cleaned = cleanup(d, vocs); final Dataset d_cleaned = cleanup(d, this.vocs);
assertEquals("0002", d_cleaned.getInstance().get(0).getRefereed().getClassid()); assertEquals("0002", d_cleaned.getInstance().get(0).getRefereed().getClassid());
assertEquals("nonPeerReviewed", d_cleaned.getInstance().get(0).getRefereed().getClassname()); assertEquals("nonPeerReviewed", d_cleaned.getInstance().get(0).getRefereed().getClassname());
} }
@ -616,11 +635,11 @@ class MappersTest {
@Test @Test
void test_record_from_Crossref() throws IOException { void test_record_from_Crossref() throws IOException {
final CleaningRuleMap mapping = CleaningRuleMap.create(vocs); final CleaningRuleMap mapping = CleaningRuleMap.create(this.vocs);
final String xml = IOUtils final String xml = IOUtils
.toString(Objects.requireNonNull(getClass().getResourceAsStream("oaf_crossref.xml"))); .toString(Objects.requireNonNull(getClass().getResourceAsStream("oaf_crossref.xml")));
final List<Oaf> list = new OafToOafMapper(vocs, false, true).processMdRecord(xml); final List<Oaf> list = new OafToOafMapper(this.vocs, false, true).processMdRecord(xml);
assertEquals(1, list.size()); assertEquals(1, list.size());
assertTrue(list.get(0) instanceof Publication); assertTrue(list.get(0) instanceof Publication);
@ -703,7 +722,7 @@ class MappersTest {
assertEquals("", p.getInstance().get(0).getRefereed().getClassid()); assertEquals("", p.getInstance().get(0).getRefereed().getClassid());
assertEquals("", p.getInstance().get(0).getRefereed().getClassname()); assertEquals("", p.getInstance().get(0).getRefereed().getClassname());
Publication p_cleaned = cleanup(p, vocs); final Publication p_cleaned = cleanup(p, this.vocs);
assertEquals("0001", p_cleaned.getInstance().get(0).getRefereed().getClassid()); assertEquals("0001", p_cleaned.getInstance().get(0).getRefereed().getClassid());
assertEquals("peerReviewed", p_cleaned.getInstance().get(0).getRefereed().getClassname()); assertEquals("peerReviewed", p_cleaned.getInstance().get(0).getRefereed().getClassname());
@ -730,15 +749,14 @@ class MappersTest {
.get(0) .get(0)
.getInstanceTypeMapping() .getInstanceTypeMapping()
.stream() .stream()
.noneMatch( .noneMatch(t -> ModelConstants.OPENAIRE_USER_RESOURCE_TYPES.equals(t.getVocabularyName())));
t -> ModelConstants.OPENAIRE_USER_RESOURCE_TYPES.equals(t.getVocabularyName())));
} }
@Test @Test
void testSoftware() throws IOException, DocumentException { void testSoftware() throws IOException, DocumentException {
final String xml = IOUtils.toString(Objects.requireNonNull(getClass().getResourceAsStream("odf_software.xml"))); final String xml = IOUtils.toString(Objects.requireNonNull(getClass().getResourceAsStream("odf_software.xml")));
final List<Oaf> list = new OdfToOafMapper(vocs, false, true).processMdRecord(xml); final List<Oaf> list = new OdfToOafMapper(this.vocs, false, true).processMdRecord(xml);
assertEquals(3, list.size()); assertEquals(3, list.size());
assertTrue(list.get(0) instanceof Software); assertTrue(list.get(0) instanceof Software);
@ -775,7 +793,7 @@ class MappersTest {
void testClaimDedup() throws IOException, DocumentException { void testClaimDedup() throws IOException, DocumentException {
final String xml = IOUtils final String xml = IOUtils
.toString(Objects.requireNonNull(getClass().getResourceAsStream("oaf_claim_dedup.xml"))); .toString(Objects.requireNonNull(getClass().getResourceAsStream("oaf_claim_dedup.xml")));
final List<Oaf> list = new OafToOafMapper(vocs, false, true).processMdRecord(xml); final List<Oaf> list = new OafToOafMapper(this.vocs, false, true).processMdRecord(xml);
assertNotNull(list); assertNotNull(list);
assertFalse(list.isEmpty()); assertFalse(list.isEmpty());
@ -788,7 +806,7 @@ class MappersTest {
@Test @Test
void testNakala() throws IOException, DocumentException { void testNakala() throws IOException, DocumentException {
final String xml = IOUtils.toString(Objects.requireNonNull(getClass().getResourceAsStream("odf_nakala.xml"))); final String xml = IOUtils.toString(Objects.requireNonNull(getClass().getResourceAsStream("odf_nakala.xml")));
final List<Oaf> list = new OdfToOafMapper(vocs, false, true).processMdRecord(xml); final List<Oaf> list = new OdfToOafMapper(this.vocs, false, true).processMdRecord(xml);
System.out.println("***************"); System.out.println("***************");
System.out.println(new ObjectMapper().writeValueAsString(list)); System.out.println(new ObjectMapper().writeValueAsString(list));
@ -816,7 +834,7 @@ class MappersTest {
@Test @Test
void testEnermaps() throws IOException, DocumentException { void testEnermaps() throws IOException, DocumentException {
final String xml = IOUtils.toString(Objects.requireNonNull(getClass().getResourceAsStream("enermaps.xml"))); final String xml = IOUtils.toString(Objects.requireNonNull(getClass().getResourceAsStream("enermaps.xml")));
final List<Oaf> list = new OdfToOafMapper(vocs, false, true).processMdRecord(xml); final List<Oaf> list = new OdfToOafMapper(this.vocs, false, true).processMdRecord(xml);
System.out.println("***************"); System.out.println("***************");
System.out.println(new ObjectMapper().writeValueAsString(list)); System.out.println(new ObjectMapper().writeValueAsString(list));
@ -842,7 +860,7 @@ class MappersTest {
void testClaimFromCrossref() throws IOException, DocumentException { void testClaimFromCrossref() throws IOException, DocumentException {
final String xml = IOUtils final String xml = IOUtils
.toString(Objects.requireNonNull(getClass().getResourceAsStream("oaf_claim_crossref.xml"))); .toString(Objects.requireNonNull(getClass().getResourceAsStream("oaf_claim_crossref.xml")));
final List<Oaf> list = new OafToOafMapper(vocs, false, true).processMdRecord(xml); final List<Oaf> list = new OafToOafMapper(this.vocs, false, true).processMdRecord(xml);
System.out.println("***************"); System.out.println("***************");
System.out.println(new ObjectMapper().writeValueAsString(list)); System.out.println(new ObjectMapper().writeValueAsString(list));
@ -858,7 +876,7 @@ class MappersTest {
@Test @Test
void testODFRecord() throws IOException, DocumentException { void testODFRecord() throws IOException, DocumentException {
final String xml = IOUtils.toString(Objects.requireNonNull(getClass().getResourceAsStream("odf_record.xml"))); final String xml = IOUtils.toString(Objects.requireNonNull(getClass().getResourceAsStream("odf_record.xml")));
final List<Oaf> list = new OdfToOafMapper(vocs, false, true).processMdRecord(xml); final List<Oaf> list = new OdfToOafMapper(this.vocs, false, true).processMdRecord(xml);
System.out.println("***************"); System.out.println("***************");
System.out.println(new ObjectMapper().writeValueAsString(list)); System.out.println(new ObjectMapper().writeValueAsString(list));
System.out.println("***************"); System.out.println("***************");
@ -867,12 +885,17 @@ class MappersTest {
assertValidId(p.getCollectedfrom().get(0).getKey()); assertValidId(p.getCollectedfrom().get(0).getKey());
System.out.println(p.getTitle().get(0).getValue()); System.out.println(p.getTitle().get(0).getValue());
assertTrue(StringUtils.isNotBlank(p.getTitle().get(0).getValue())); assertTrue(StringUtils.isNotBlank(p.getTitle().get(0).getValue()));
assertEquals(3, p.getCountry().size());
assertEquals("IT", p.getCountry().get(0).getClassid());
assertEquals("FR", p.getCountry().get(1).getClassid());
assertEquals("DE", p.getCountry().get(2).getClassid());
} }
@Test @Test
void testTextGrid() throws IOException, DocumentException { void testTextGrid() throws IOException, DocumentException {
final String xml = IOUtils.toString(Objects.requireNonNull(getClass().getResourceAsStream("textgrid.xml"))); final String xml = IOUtils.toString(Objects.requireNonNull(getClass().getResourceAsStream("textgrid.xml")));
final List<Oaf> list = new OdfToOafMapper(vocs, false, true).processMdRecord(xml); final List<Oaf> list = new OdfToOafMapper(this.vocs, false, true).processMdRecord(xml);
System.out.println("***************"); System.out.println("***************");
System.out.println(new ObjectMapper().writeValueAsString(list)); System.out.println(new ObjectMapper().writeValueAsString(list));
@ -906,7 +929,7 @@ class MappersTest {
@Test @Test
void testBologna() throws IOException, DocumentException { void testBologna() throws IOException, DocumentException {
final String xml = IOUtils.toString(Objects.requireNonNull(getClass().getResourceAsStream("oaf-bologna.xml"))); final String xml = IOUtils.toString(Objects.requireNonNull(getClass().getResourceAsStream("oaf-bologna.xml")));
final List<Oaf> list = new OafToOafMapper(vocs, false, true).processMdRecord(xml); final List<Oaf> list = new OafToOafMapper(this.vocs, false, true).processMdRecord(xml);
System.out.println("***************"); System.out.println("***************");
System.out.println(new ObjectMapper().writeValueAsString(list)); System.out.println(new ObjectMapper().writeValueAsString(list));
@ -923,7 +946,7 @@ class MappersTest {
@Test @Test
void testJairo() throws IOException, DocumentException { void testJairo() throws IOException, DocumentException {
final String xml = IOUtils.toString(Objects.requireNonNull(getClass().getResourceAsStream("oaf_jairo.xml"))); final String xml = IOUtils.toString(Objects.requireNonNull(getClass().getResourceAsStream("oaf_jairo.xml")));
final List<Oaf> list = new OafToOafMapper(vocs, false, true).processMdRecord(xml); final List<Oaf> list = new OafToOafMapper(this.vocs, false, true).processMdRecord(xml);
System.out.println("***************"); System.out.println("***************");
System.out.println(new ObjectMapper().writeValueAsString(list)); System.out.println(new ObjectMapper().writeValueAsString(list));
@ -938,7 +961,7 @@ class MappersTest {
assertEquals(1, p.getTitle().size()); assertEquals(1, p.getTitle().size());
assertTrue(StringUtils.isNotBlank(p.getTitle().get(0).getValue())); assertTrue(StringUtils.isNotBlank(p.getTitle().get(0).getValue()));
final Publication p_cleaned = cleanup(fixVocabularyNames(p), vocs); final Publication p_cleaned = cleanup(fixVocabularyNames(p), this.vocs);
assertNotNull(p_cleaned.getTitle()); assertNotNull(p_cleaned.getTitle());
assertFalse(p_cleaned.getTitle().isEmpty()); assertFalse(p_cleaned.getTitle().isEmpty());
@ -947,7 +970,7 @@ class MappersTest {
@Test @Test
void test_instance_url_validation() throws IOException { void test_instance_url_validation() throws IOException {
final String xml = IOUtils.toString(Objects.requireNonNull(getClass().getResourceAsStream("idus_sevilla.xml"))); final String xml = IOUtils.toString(Objects.requireNonNull(getClass().getResourceAsStream("idus_sevilla.xml")));
final List<Oaf> list = new OafToOafMapper(vocs, false, true).processMdRecord(xml); final List<Oaf> list = new OafToOafMapper(this.vocs, false, true).processMdRecord(xml);
final Publication p = (Publication) list.get(0); final Publication p = (Publication) list.get(0);
@ -961,7 +984,7 @@ class MappersTest {
@Test @Test
void testZenodo() throws IOException, DocumentException { void testZenodo() throws IOException, DocumentException {
final String xml = IOUtils.toString(Objects.requireNonNull(getClass().getResourceAsStream("odf_zenodo.xml"))); final String xml = IOUtils.toString(Objects.requireNonNull(getClass().getResourceAsStream("odf_zenodo.xml")));
final List<Oaf> list = new OdfToOafMapper(vocs, false, true).processMdRecord(xml); final List<Oaf> list = new OdfToOafMapper(this.vocs, false, true).processMdRecord(xml);
System.out.println("***************"); System.out.println("***************");
System.out.println(new ObjectMapper().writeValueAsString(list)); System.out.println(new ObjectMapper().writeValueAsString(list));
@ -982,7 +1005,7 @@ class MappersTest {
Author author = p Author author = p
.getAuthor() .getAuthor()
.stream() .stream()
.filter(a -> a.getPid().stream().anyMatch(pi -> pi.getValue().equals("0000-0003-3272-8007"))) .filter(a -> a.getPid().stream().anyMatch(pi -> "0000-0003-3272-8007".equals(pi.getValue())))
.findFirst() .findFirst()
.get(); .get();
assertNotNull(author); assertNotNull(author);
@ -993,7 +1016,7 @@ class MappersTest {
author = p author = p
.getAuthor() .getAuthor()
.stream() .stream()
.filter(a -> a.getPid().stream().anyMatch(pi -> pi.getValue().equals("0000-0003-3272-8008"))) .filter(a -> a.getPid().stream().anyMatch(pi -> "0000-0003-3272-8008".equals(pi.getValue())))
.findFirst() .findFirst()
.get(); .get();
assertNotNull(author); assertNotNull(author);
@ -1008,7 +1031,7 @@ class MappersTest {
final String xml = IOUtils final String xml = IOUtils
.toString(Objects.requireNonNull(getClass().getResourceAsStream("odf_from_hdfs.xml"))); .toString(Objects.requireNonNull(getClass().getResourceAsStream("odf_from_hdfs.xml")));
final List<Oaf> list = new OdfToOafMapper(vocs, false, true).processMdRecord(xml); final List<Oaf> list = new OdfToOafMapper(this.vocs, false, true).processMdRecord(xml);
assertEquals(1, list.size()); assertEquals(1, list.size());
@ -1020,7 +1043,7 @@ class MappersTest {
assertValidId(p.getId()); assertValidId(p.getId());
assertEquals(2, p.getOriginalId().size()); assertEquals(2, p.getOriginalId().size());
assertTrue(p.getOriginalId().stream().anyMatch(oid -> oid.equals("df76e73f-0483-49a4-a9bb-63f2f985574a"))); assertTrue(p.getOriginalId().stream().anyMatch("df76e73f-0483-49a4-a9bb-63f2f985574a"::equals));
assertValidId(p.getCollectedfrom().get(0).getKey()); assertValidId(p.getCollectedfrom().get(0).getKey());
assertFalse(p.getAuthor().isEmpty()); assertFalse(p.getAuthor().isEmpty());
@ -1047,7 +1070,7 @@ class MappersTest {
assertEquals("UNKNOWN", i.getAccessright().getClassid()); assertEquals("UNKNOWN", i.getAccessright().getClassid());
}); });
Dataset p_cleaned = cleanup(p, vocs); final Dataset p_cleaned = cleanup(p, this.vocs);
assertEquals("0002", p_cleaned.getInstance().get(0).getRefereed().getClassid()); assertEquals("0002", p_cleaned.getInstance().get(0).getRefereed().getClassid());
assertEquals("nonPeerReviewed", p_cleaned.getInstance().get(0).getRefereed().getClassname()); assertEquals("nonPeerReviewed", p_cleaned.getInstance().get(0).getRefereed().getClassname());
} }
@ -1055,7 +1078,7 @@ class MappersTest {
@Test @Test
void testXMLEncodedURL() throws IOException, DocumentException { void testXMLEncodedURL() throws IOException, DocumentException {
final String xml = IOUtils.toString(Objects.requireNonNull(getClass().getResourceAsStream("encoded-url.xml"))); final String xml = IOUtils.toString(Objects.requireNonNull(getClass().getResourceAsStream("encoded-url.xml")));
final List<Oaf> list = new OafToOafMapper(vocs, false, true).processMdRecord(xml); final List<Oaf> list = new OafToOafMapper(this.vocs, false, true).processMdRecord(xml);
System.out.println("***************"); System.out.println("***************");
System.out.println(new ObjectMapper().writeValueAsString(list)); System.out.println(new ObjectMapper().writeValueAsString(list));
@ -1064,7 +1087,7 @@ class MappersTest {
final Publication p = (Publication) list.get(0); final Publication p = (Publication) list.get(0);
assertTrue(p.getInstance().size() > 0); assertTrue(p.getInstance().size() > 0);
String decoded = "https://www.ec.europa.eu/research/participants/documents/downloadPublic?documentIds=080166e5af388993&appId=PPGMS"; final String decoded = "https://www.ec.europa.eu/research/participants/documents/downloadPublic?documentIds=080166e5af388993&appId=PPGMS";
assertEquals(decoded, p.getInstance().get(0).getUrl().get(0)); assertEquals(decoded, p.getInstance().get(0).getUrl().get(0));
} }
@ -1072,7 +1095,7 @@ class MappersTest {
void testXMLEncodedURL_ODF() throws IOException, DocumentException { void testXMLEncodedURL_ODF() throws IOException, DocumentException {
final String xml = IOUtils final String xml = IOUtils
.toString(Objects.requireNonNull(getClass().getResourceAsStream("encoded-url_odf.xml"))); .toString(Objects.requireNonNull(getClass().getResourceAsStream("encoded-url_odf.xml")));
final List<Oaf> list = new OdfToOafMapper(vocs, false, true).processMdRecord(xml); final List<Oaf> list = new OdfToOafMapper(this.vocs, false, true).processMdRecord(xml);
System.out.println("***************"); System.out.println("***************");
System.out.println(new ObjectMapper().writeValueAsString(list)); System.out.println(new ObjectMapper().writeValueAsString(list));
@ -1080,7 +1103,7 @@ class MappersTest {
final Dataset p = (Dataset) list.get(0); final Dataset p = (Dataset) list.get(0);
assertFalse(p.getInstance().isEmpty()); assertFalse(p.getInstance().isEmpty());
for (String url : p.getInstance().get(0).getUrl()) { for (final String url : p.getInstance().get(0).getUrl()) {
System.out.println(url); System.out.println(url);
assertFalse(url.contains("&amp;")); assertFalse(url.contains("&amp;"));
} }
@ -1089,16 +1112,16 @@ class MappersTest {
@Test @Test
void testOpenAPC() throws IOException { void testOpenAPC() throws IOException {
final String xml = IOUtils.toString(Objects.requireNonNull(getClass().getResourceAsStream("oaf_openapc.xml"))); final String xml = IOUtils.toString(Objects.requireNonNull(getClass().getResourceAsStream("oaf_openapc.xml")));
final List<Oaf> list = new OafToOafMapper(vocs, true, true).processMdRecord(xml); final List<Oaf> list = new OafToOafMapper(this.vocs, true, true).processMdRecord(xml);
System.out.println("***************"); System.out.println("***************");
System.out.println(new ObjectMapper().writeValueAsString(list)); System.out.println(new ObjectMapper().writeValueAsString(list));
System.out.println("***************"); System.out.println("***************");
final Optional<Oaf> o = list.stream().filter(r -> r instanceof Publication).findFirst(); final Optional<Oaf> o = list.stream().filter(Publication.class::isInstance).findFirst();
assertTrue(o.isPresent()); assertTrue(o.isPresent());
Publication p = (Publication) o.get(); final Publication p = (Publication) o.get();
assertFalse(p.getInstance().isEmpty()); assertFalse(p.getInstance().isEmpty());
assertEquals("https://doi.org/10.1155/2015/439379", p.getInstance().get(0).getUrl().get(0)); assertEquals("https://doi.org/10.1155/2015/439379", p.getInstance().get(0).getUrl().get(0));
@ -1109,14 +1132,14 @@ class MappersTest {
assertEquals("1721.47", p.getProcessingchargeamount().getValue()); assertEquals("1721.47", p.getProcessingchargeamount().getValue());
assertEquals("EUR", p.getProcessingchargecurrency().getValue()); assertEquals("EUR", p.getProcessingchargecurrency().getValue());
List<Oaf> affiliations = list.stream().filter(r -> r instanceof Relation).collect(Collectors.toList()); final List<Oaf> affiliations = list.stream().filter(Relation.class::isInstance).collect(Collectors.toList());
assertEquals(2, affiliations.size()); assertEquals(2, affiliations.size());
for (Oaf aff : affiliations) { for (final Oaf aff : affiliations) {
Relation r = (Relation) aff; final Relation r = (Relation) aff;
assertEquals(ModelConstants.AFFILIATION, r.getSubRelType()); assertEquals(ModelConstants.AFFILIATION, r.getSubRelType());
assertEquals(ModelConstants.RESULT_ORGANIZATION, r.getRelType()); assertEquals(ModelConstants.RESULT_ORGANIZATION, r.getRelType());
String source = r.getSource(); final String source = r.getSource();
if (StringUtils.startsWith(source, "50")) { if (StringUtils.startsWith(source, "50")) {
assertEquals(ModelConstants.HAS_AUTHOR_INSTITUTION, r.getRelClass()); assertEquals(ModelConstants.HAS_AUTHOR_INSTITUTION, r.getRelClass());
} else if (StringUtils.startsWith(source, "20")) { } else if (StringUtils.startsWith(source, "20")) {
@ -1127,7 +1150,7 @@ class MappersTest {
throw new IllegalArgumentException("invalid source / target prefixes for affiliation relations"); throw new IllegalArgumentException("invalid source / target prefixes for affiliation relations");
} }
List<KeyValue> apcInfo = r.getProperties(); final List<KeyValue> apcInfo = r.getProperties();
assertEquals( assertEquals(
"EUR", apcInfo "EUR", apcInfo
.stream() .stream()
@ -1148,7 +1171,7 @@ class MappersTest {
@Test @Test
void testROHub() throws IOException { void testROHub() throws IOException {
final String xml = IOUtils.toString(Objects.requireNonNull(getClass().getResourceAsStream("rohub.xml"))); final String xml = IOUtils.toString(Objects.requireNonNull(getClass().getResourceAsStream("rohub.xml")));
final List<Oaf> list = new OdfToOafMapper(vocs, false, true).processMdRecord(xml); final List<Oaf> list = new OdfToOafMapper(this.vocs, false, true).processMdRecord(xml);
System.out.println("***************"); System.out.println("***************");
System.out.println(new ObjectMapper().writeValueAsString(list)); System.out.println(new ObjectMapper().writeValueAsString(list));
System.out.println("***************"); System.out.println("***************");
@ -1160,7 +1183,7 @@ class MappersTest {
assertTrue(StringUtils.isNotBlank(p.getTitle().get(0).getValue())); assertTrue(StringUtils.isNotBlank(p.getTitle().get(0).getValue()));
assertEquals(1, p.getInstance().size()); assertEquals(1, p.getInstance().size());
assertEquals("https://w3id.org/ro-id/0ab171a7-45c5-4194-82d4-850955504bca", p.getPid().get(0).getValue()); assertEquals("https://w3id.org/ro-id/0ab171a7-45c5-4194-82d4-850955504bca", p.getPid().get(0).getValue());
Instance inst = p.getInstance().get(0); final Instance inst = p.getInstance().get(0);
assertEquals("https://w3id.org/ro-id/0ab171a7-45c5-4194-82d4-850955504bca", inst.getPid().get(0).getValue()); assertEquals("https://w3id.org/ro-id/0ab171a7-45c5-4194-82d4-850955504bca", inst.getPid().get(0).getValue());
assertEquals("https://w3id.org/ro-id/0ab171a7-45c5-4194-82d4-850955504bca", inst.getUrl().get(0)); assertEquals("https://w3id.org/ro-id/0ab171a7-45c5-4194-82d4-850955504bca", inst.getUrl().get(0));
assertEquals(1, p.getEoscifguidelines().size()); assertEquals(1, p.getEoscifguidelines().size());
@ -1174,10 +1197,10 @@ class MappersTest {
@Test @Test
void test_Zenodo2() throws IOException { void test_Zenodo2() throws IOException {
final String xml = IOUtils.toString(Objects.requireNonNull(getClass().getResourceAsStream("odf_zenodo2.xml"))); final String xml = IOUtils.toString(Objects.requireNonNull(getClass().getResourceAsStream("odf_zenodo2.xml")));
final List<Oaf> list = new OdfToOafMapper(vocs, false, true).processMdRecord(xml); final List<Oaf> list = new OdfToOafMapper(this.vocs, false, true).processMdRecord(xml);
assertEquals(3, list.size()); assertEquals(3, list.size());
Publication p = cleanup((Publication) list.get(0), vocs); final Publication p = cleanup((Publication) list.get(0), this.vocs);
assertNotNull(p.getInstance()); assertNotNull(p.getInstance());
assertEquals(1, p.getInstance().size()); assertEquals(1, p.getInstance().size());
@ -1187,7 +1210,7 @@ class MappersTest {
assertNotNull(instance.getInstanceTypeMapping()); assertNotNull(instance.getInstanceTypeMapping());
assertEquals(1, instance.getInstanceTypeMapping().size()); assertEquals(1, instance.getInstanceTypeMapping().size());
Optional<InstanceTypeMapping> coarType = instance final Optional<InstanceTypeMapping> coarType = instance
.getInstanceTypeMapping() .getInstanceTypeMapping()
.stream() .stream()
.filter(itm -> ModelConstants.OPENAIRE_COAR_RESOURCE_TYPES_3_1.equals(itm.getVocabularyName())) .filter(itm -> ModelConstants.OPENAIRE_COAR_RESOURCE_TYPES_3_1.equals(itm.getVocabularyName()))
@ -1203,7 +1226,7 @@ class MappersTest {
void testROHub2() throws IOException { void testROHub2() throws IOException {
final String xml = IOUtils final String xml = IOUtils
.toString(Objects.requireNonNull(getClass().getResourceAsStream("rohub-modified.xml"))); .toString(Objects.requireNonNull(getClass().getResourceAsStream("rohub-modified.xml")));
final List<Oaf> list = new OdfToOafMapper(vocs, false, true).processMdRecord(xml); final List<Oaf> list = new OdfToOafMapper(this.vocs, false, true).processMdRecord(xml);
System.out.println("***************"); System.out.println("***************");
System.out.println(new ObjectMapper().writeValueAsString(list)); System.out.println(new ObjectMapper().writeValueAsString(list));
System.out.println("***************"); System.out.println("***************");
@ -1216,13 +1239,13 @@ class MappersTest {
assertEquals("w3id", (p.getPid().get(0).getQualifier().getClassid())); assertEquals("w3id", (p.getPid().get(0).getQualifier().getClassid()));
assertEquals("https://w3id.org/ro-id/0ab171a7-45c5-4194-82d4-850955504bca", (p.getPid().get(0).getValue())); assertEquals("https://w3id.org/ro-id/0ab171a7-45c5-4194-82d4-850955504bca", (p.getPid().get(0).getValue()));
assertEquals(1, list.stream().filter(o -> o instanceof OtherResearchProduct).count()); assertEquals(1, list.stream().filter(OtherResearchProduct.class::isInstance).count());
assertEquals(6, list.stream().filter(o -> o instanceof Relation).count()); assertEquals(6, list.stream().filter(Relation.class::isInstance).count());
for (Oaf oaf : list) { for (final Oaf oaf : list) {
if (oaf instanceof Relation) { if (oaf instanceof Relation) {
String source = ((Relation) oaf).getSource(); final String source = ((Relation) oaf).getSource();
String target = ((Relation) oaf).getTarget(); final String target = ((Relation) oaf).getTarget();
assertNotEquals(source, target); assertNotEquals(source, target);
assertTrue(source.equals(p.getId()) || target.equals(p.getId())); assertTrue(source.equals(p.getId()) || target.equals(p.getId()));
assertNotNull(((Relation) oaf).getSubRelType()); assertNotNull(((Relation) oaf).getSubRelType());
@ -1235,7 +1258,7 @@ class MappersTest {
@Test @Test
void testRiunet() throws IOException, DocumentException { void testRiunet() throws IOException, DocumentException {
final String xml = IOUtils.toString(Objects.requireNonNull(getClass().getResourceAsStream("riunet.xml"))); final String xml = IOUtils.toString(Objects.requireNonNull(getClass().getResourceAsStream("riunet.xml")));
final List<Oaf> list = new OdfToOafMapper(vocs, false, true).processMdRecord(xml); final List<Oaf> list = new OdfToOafMapper(this.vocs, false, true).processMdRecord(xml);
System.out.println("***************"); System.out.println("***************");
System.out.println(new ObjectMapper().writeValueAsString(list)); System.out.println(new ObjectMapper().writeValueAsString(list));
System.out.println("***************"); System.out.println("***************");
@ -1248,7 +1271,7 @@ class MappersTest {
void testEOSCFuture_ROHub() throws IOException { void testEOSCFuture_ROHub() throws IOException {
final String xml = IOUtils final String xml = IOUtils
.toString(Objects.requireNonNull(getClass().getResourceAsStream("photic-zone-transformed.xml"))); .toString(Objects.requireNonNull(getClass().getResourceAsStream("photic-zone-transformed.xml")));
final List<Oaf> list = new OdfToOafMapper(vocs, false, true).processMdRecord(xml); final List<Oaf> list = new OdfToOafMapper(this.vocs, false, true).processMdRecord(xml);
final OtherResearchProduct rocrate = (OtherResearchProduct) list.get(0); final OtherResearchProduct rocrate = (OtherResearchProduct) list.get(0);
assertNotNull(rocrate.getEoscifguidelines()); assertNotNull(rocrate.getEoscifguidelines());
System.out.println("***************"); System.out.println("***************");
@ -1260,7 +1283,7 @@ class MappersTest {
void testD4ScienceTraining() throws IOException { void testD4ScienceTraining() throws IOException {
final String xml = IOUtils final String xml = IOUtils
.toString(Objects.requireNonNull(getClass().getResourceAsStream("d4science-1-training.xml"))); .toString(Objects.requireNonNull(getClass().getResourceAsStream("d4science-1-training.xml")));
final List<Oaf> list = new OdfToOafMapper(vocs, false, true).processMdRecord(xml); final List<Oaf> list = new OdfToOafMapper(this.vocs, false, true).processMdRecord(xml);
final OtherResearchProduct trainingMaterial = (OtherResearchProduct) list.get(0); final OtherResearchProduct trainingMaterial = (OtherResearchProduct) list.get(0);
System.out.println("***************"); System.out.println("***************");
System.out.println(new ObjectMapper().writeValueAsString(trainingMaterial)); System.out.println(new ObjectMapper().writeValueAsString(trainingMaterial));
@ -1271,7 +1294,7 @@ class MappersTest {
void testD4ScienceDataset() throws IOException { void testD4ScienceDataset() throws IOException {
final String xml = IOUtils final String xml = IOUtils
.toString(Objects.requireNonNull(getClass().getResourceAsStream("d4science-2-dataset.xml"))); .toString(Objects.requireNonNull(getClass().getResourceAsStream("d4science-2-dataset.xml")));
final List<Oaf> list = new OdfToOafMapper(vocs, false, true).processMdRecord(xml); final List<Oaf> list = new OdfToOafMapper(this.vocs, false, true).processMdRecord(xml);
final Dataset trainingMaterial = (Dataset) list.get(0); final Dataset trainingMaterial = (Dataset) list.get(0);
System.out.println("***************"); System.out.println("***************");
System.out.println(new ObjectMapper().writeValueAsString(trainingMaterial)); System.out.println(new ObjectMapper().writeValueAsString(trainingMaterial));
@ -1282,7 +1305,7 @@ class MappersTest {
void testNotWellFormed() throws IOException { void testNotWellFormed() throws IOException {
final String xml = IOUtils final String xml = IOUtils
.toString(Objects.requireNonNull(getClass().getResourceAsStream("oaf_notwellformed.xml"))); .toString(Objects.requireNonNull(getClass().getResourceAsStream("oaf_notwellformed.xml")));
final List<Oaf> actual = new OafToOafMapper(vocs, false, true).processMdRecord(xml); final List<Oaf> actual = new OafToOafMapper(this.vocs, false, true).processMdRecord(xml);
assertNotNull(actual); assertNotNull(actual);
assertTrue(actual.isEmpty()); assertTrue(actual.isEmpty());
} }

3
dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/oaf_record.xml
View File

@ -71,6 +71,9 @@
subRelType="outcome" subRelType="outcome"
targetType="project" targetType="project"
validationDate="2020-01-01">corda_______::226852</oaf:relation> validationDate="2020-01-01">corda_______::226852</oaf:relation>
<oaf:country code="IT">Italy</oaf:country>
<oaf:country code="FR" />
<oaf:country>DE</oaf:country>
</metadata> </metadata>
<about xmlns:oai="http://www.openarchives.org/OAI/2.0/"> <about xmlns:oai="http://www.openarchives.org/OAI/2.0/">
<provenance xmlns="http://www.openarchives.org/OAI/2.0/provenance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/provenance http://www.openarchives.org/OAI/2.0/provenance.xsd"> <provenance xmlns="http://www.openarchives.org/OAI/2.0/provenance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/provenance http://www.openarchives.org/OAI/2.0/provenance.xsd">

2
dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/odf_dataset.xml
View File

@ -91,7 +91,7 @@
<oaf:hostedBy id="re3data_____::r3d100010468" name="Zenodo"/> <oaf:hostedBy id="re3data_____::r3d100010468" name="Zenodo"/>
<oaf:projectid validationDate="2020-01-01">corda_______::226852</oaf:projectid> <oaf:projectid validationDate="2020-01-01">corda_______::226852</oaf:projectid>
<oaf:collectedFrom id="re3data_____::r3d100010468" name="Zenodo"/> <oaf:collectedFrom id="re3data_____::r3d100010468" name="Zenodo"/>
<oaf:refereed>0001</oaf:refereed>s <oaf:refereed>0001</oaf:refereed>
</metadata> </metadata>
<about xmlns:dc="http://purl.org/dc/elements/1.1/" <about xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:dri="http://www.driver-repository.eu/namespace/dri" xmlns:dri="http://www.driver-repository.eu/namespace/dri"

3
dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/odf_record.xml
View File

@ -78,6 +78,9 @@
<oaf:projectid>corda_______::630786</oaf:projectid> <oaf:projectid>corda_______::630786</oaf:projectid>
<oaf:hostedBy id="re3data_____::r3d100010386" name="LINDAT/CLARIN repository"/> <oaf:hostedBy id="re3data_____::r3d100010386" name="LINDAT/CLARIN repository"/>
<oaf:collectedFrom id="re3data_____::r3d100010386" name="LINDAT/CLARIN repository"/> <oaf:collectedFrom id="re3data_____::r3d100010386" name="LINDAT/CLARIN repository"/>
<oaf:country code="IT">Italy</oaf:country>
<oaf:country code="FR" />
<oaf:country>DE</oaf:country>
</metadata> </metadata>
<about xmlns:dc="http://purl.org/dc/elements/1.1/" <about xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:dri="http://www.driver-repository.eu/namespace/dri" xmlns:prov="http://www.openarchives.org/OAI/2.0/provenance"> xmlns:dri="http://www.driver-repository.eu/namespace/dri" xmlns:prov="http://www.openarchives.org/OAI/2.0/provenance">