2020-04-28 11:23:29 +02:00
|
|
|
|
2020-04-10 17:53:07 +02:00
|
|
|
package eu.dnetlib.dhp.oa.graph.raw;
|
2020-03-02 16:12:14 +01:00
|
|
|
|
2020-06-12 10:58:02 +02:00
|
|
|
import static eu.dnetlib.dhp.schema.common.ModelConstants.*;
|
2021-04-27 15:44:01 +02:00
|
|
|
import static eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils.*;
|
2020-05-22 12:25:01 +02:00
|
|
|
|
2021-08-20 17:03:30 +02:00
|
|
|
import java.io.UnsupportedEncodingException;
|
|
|
|
import java.net.URLDecoder;
|
2020-05-22 12:25:01 +02:00
|
|
|
import java.util.ArrayList;
|
2021-03-16 14:19:32 +01:00
|
|
|
import java.util.HashSet;
|
2020-05-22 12:25:01 +02:00
|
|
|
import java.util.List;
|
2021-03-16 14:19:32 +01:00
|
|
|
import java.util.Set;
|
2020-05-08 09:43:26 +02:00
|
|
|
import java.util.stream.Collectors;
|
2020-04-28 11:23:29 +02:00
|
|
|
|
|
|
|
import org.apache.commons.lang3.StringUtils;
|
|
|
|
import org.dom4j.Document;
|
2020-05-15 12:26:16 +02:00
|
|
|
import org.dom4j.Element;
|
2020-04-28 11:23:29 +02:00
|
|
|
import org.dom4j.Node;
|
|
|
|
|
2020-05-06 13:20:02 +02:00
|
|
|
import com.google.common.collect.Lists;
|
|
|
|
|
2020-05-25 10:35:39 +02:00
|
|
|
import eu.dnetlib.dhp.common.PacePerson;
|
2021-01-25 15:43:04 +01:00
|
|
|
import eu.dnetlib.dhp.common.vocabulary.VocabularyGroup;
|
2020-10-16 17:02:10 +02:00
|
|
|
import eu.dnetlib.dhp.schema.oaf.*;
|
2021-04-27 15:44:01 +02:00
|
|
|
import eu.dnetlib.dhp.schema.oaf.utils.CleaningFunctions;
|
2021-03-16 14:19:32 +01:00
|
|
|
import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory;
|
2021-04-27 15:44:01 +02:00
|
|
|
import eu.dnetlib.dhp.schema.oaf.utils.ModelHardLimits;
|
2020-02-04 15:25:47 +01:00
|
|
|
|
2020-03-02 16:12:14 +01:00
|
|
|
public class OafToOafMapper extends AbstractMdRecordToOafMapper {
|
2020-02-05 15:35:40 +01:00
|
|
|
|
2021-07-19 17:43:52 +02:00
|
|
|
public OafToOafMapper(final VocabularyGroup vocs, final boolean invisible, final boolean shouldHashId,
|
|
|
|
final boolean forceOrginalId) {
|
|
|
|
super(vocs, invisible, shouldHashId, forceOrginalId);
|
|
|
|
}
|
|
|
|
|
2020-11-30 12:00:38 +01:00
|
|
|
public OafToOafMapper(final VocabularyGroup vocs, final boolean invisible, final boolean shouldHashId) {
|
|
|
|
super(vocs, invisible, shouldHashId);
|
2020-04-28 11:23:29 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
protected List<Author> prepareAuthors(final Document doc, final DataInfo info) {
|
|
|
|
final List<Author> res = new ArrayList<>();
|
|
|
|
int pos = 1;
|
|
|
|
for (final Object o : doc.selectNodes("//dc:creator")) {
|
2020-05-15 12:26:16 +02:00
|
|
|
final Element e = (Element) o;
|
2020-04-28 11:23:29 +02:00
|
|
|
final Author author = new Author();
|
2020-05-15 12:26:16 +02:00
|
|
|
author.setFullname(e.getText());
|
2020-04-28 11:23:29 +02:00
|
|
|
author.setRank(pos++);
|
2020-05-15 12:26:16 +02:00
|
|
|
final PacePerson p = new PacePerson(e.getText(), false);
|
2020-04-28 11:23:29 +02:00
|
|
|
if (p.isAccurate()) {
|
|
|
|
author.setName(p.getNormalisedFirstName());
|
|
|
|
author.setSurname(p.getNormalisedSurname());
|
|
|
|
}
|
2020-05-15 12:26:16 +02:00
|
|
|
|
2020-05-22 12:25:01 +02:00
|
|
|
final String pid = e.valueOf("./@nameIdentifier");
|
2020-05-22 12:34:00 +02:00
|
|
|
final String type = e
|
|
|
|
.valueOf("./@nameIdentifierScheme")
|
2020-05-22 12:25:01 +02:00
|
|
|
.trim()
|
|
|
|
.toUpperCase()
|
2021-08-11 12:13:22 +02:00
|
|
|
.replace(" ", "")
|
|
|
|
.replace("_", "");
|
2020-05-15 12:26:16 +02:00
|
|
|
|
2020-05-15 17:06:01 +02:00
|
|
|
author.setPid(new ArrayList<>());
|
2020-05-22 12:25:01 +02:00
|
|
|
|
|
|
|
if (StringUtils.isNotBlank(pid)) {
|
2021-05-26 18:20:23 +02:00
|
|
|
if (type.toLowerCase().startsWith(ORCID)) {
|
2020-05-22 12:34:00 +02:00
|
|
|
final String cleanedId = pid
|
|
|
|
.replaceAll("http://orcid.org/", "")
|
|
|
|
.replaceAll("https://orcid.org/", "");
|
2020-05-22 12:25:01 +02:00
|
|
|
author.getPid().add(structuredProperty(cleanedId, ORCID_PID_TYPE, info));
|
|
|
|
} else if (type.startsWith("MAGID")) {
|
|
|
|
author.getPid().add(structuredProperty(pid, MAG_PID_TYPE, info));
|
|
|
|
}
|
2020-05-15 12:26:16 +02:00
|
|
|
}
|
|
|
|
|
2020-04-28 11:23:29 +02:00
|
|
|
res.add(author);
|
|
|
|
}
|
|
|
|
return res;
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
protected Qualifier prepareLanguages(final Document doc) {
|
2020-05-27 11:34:13 +02:00
|
|
|
return prepareQualifier(doc, "//dc:language", DNET_LANGUAGES);
|
2020-04-28 11:23:29 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2022-08-04 11:39:39 +02:00
|
|
|
protected List<Subject> prepareSubjects(final Document doc, final DataInfo info) {
|
|
|
|
return prepareSubjectList(doc, "//dc:subject", info);
|
2020-04-28 11:23:29 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
protected List<StructuredProperty> prepareTitles(final Document doc, final DataInfo info) {
|
|
|
|
return prepareListStructProps(doc, "//dc:title", MAIN_TITLE_QUALIFIER, info);
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
protected List<Field<String>> prepareDescriptions(final Document doc, final DataInfo info) {
|
2020-10-30 10:56:42 +01:00
|
|
|
return prepareListFields(doc, "//dc:description", info)
|
|
|
|
.stream()
|
|
|
|
.map(d -> {
|
|
|
|
d.setValue(StringUtils.left(d.getValue(), ModelHardLimits.MAX_ABSTRACT_LENGTH));
|
|
|
|
return d;
|
|
|
|
})
|
|
|
|
.collect(Collectors.toList());
|
2020-04-28 11:23:29 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
protected Field<String> preparePublisher(final Document doc, final DataInfo info) {
|
|
|
|
return prepareField(doc, "//dc:publisher", info);
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
protected List<Field<String>> prepareFormats(final Document doc, final DataInfo info) {
|
|
|
|
return prepareListFields(doc, "//dc:format", info);
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
protected List<Field<String>> prepareContributors(final Document doc, final DataInfo info) {
|
|
|
|
return prepareListFields(doc, "//dc:contributor", info);
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
protected List<Field<String>> prepareCoverages(final Document doc, final DataInfo info) {
|
|
|
|
return prepareListFields(doc, "//dc:coverage", info);
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
protected List<Instance> prepareInstances(
|
|
|
|
final Document doc,
|
|
|
|
final DataInfo info,
|
|
|
|
final KeyValue collectedfrom,
|
|
|
|
final KeyValue hostedby) {
|
2020-05-06 13:20:02 +02:00
|
|
|
|
|
|
|
final Instance instance = new Instance();
|
|
|
|
instance
|
2020-05-27 11:34:13 +02:00
|
|
|
.setInstancetype(prepareQualifier(doc, "//dr:CobjCategory", DNET_PUBLICATION_RESOURCE));
|
2020-05-06 13:20:02 +02:00
|
|
|
instance.setCollectedfrom(collectedfrom);
|
|
|
|
instance.setHostedby(hostedby);
|
2021-03-16 14:19:32 +01:00
|
|
|
|
|
|
|
final List<StructuredProperty> alternateIdentifier = prepareResultPids(doc, info);
|
|
|
|
final List<StructuredProperty> pid = IdentifierFactory.getPids(alternateIdentifier, collectedfrom);
|
|
|
|
|
|
|
|
final Set<StructuredProperty> pids = pid.stream().collect(Collectors.toCollection(HashSet::new));
|
|
|
|
|
|
|
|
instance
|
|
|
|
.setAlternateIdentifier(
|
|
|
|
alternateIdentifier.stream().filter(i -> !pids.contains(i)).collect(Collectors.toList()));
|
|
|
|
instance.setPid(pid);
|
|
|
|
|
2020-05-06 13:20:02 +02:00
|
|
|
instance.setDateofacceptance(field(doc.valueOf("//oaf:dateAccepted"), info));
|
|
|
|
instance.setDistributionlocation(doc.valueOf("//oaf:distributionlocation"));
|
|
|
|
instance
|
2021-01-12 15:36:38 +01:00
|
|
|
.setAccessright(prepareAccessRight(doc, "//oaf:accessrights", DNET_ACCESS_MODES));
|
2020-05-06 13:20:02 +02:00
|
|
|
instance.setLicense(field(doc.valueOf("//oaf:license"), info));
|
2020-06-09 19:52:53 +02:00
|
|
|
instance.setRefereed(prepareQualifier(doc, "//oaf:refereed", DNET_REVIEW_LEVELS));
|
2020-05-06 13:20:02 +02:00
|
|
|
instance
|
2020-05-22 12:25:01 +02:00
|
|
|
.setProcessingchargeamount(field(doc.valueOf("//oaf:processingchargeamount"), info));
|
2020-05-06 13:20:02 +02:00
|
|
|
instance
|
2020-05-22 12:25:01 +02:00
|
|
|
.setProcessingchargecurrency(field(doc.valueOf("//oaf:processingchargeamount/@currency"), info));
|
2020-05-06 13:20:02 +02:00
|
|
|
|
2020-05-22 12:25:01 +02:00
|
|
|
final List<Node> nodes = Lists.newArrayList(doc.selectNodes("//dc:identifier"));
|
2022-09-19 11:19:10 +02:00
|
|
|
final List<String> url = nodes
|
|
|
|
.stream()
|
|
|
|
.filter(n -> StringUtils.isNotBlank(n.getText()))
|
|
|
|
.map(n -> n.getText().trim())
|
|
|
|
.filter(u -> u.startsWith("http"))
|
|
|
|
.map(s -> {
|
|
|
|
try {
|
|
|
|
return URLDecoder.decode(s, "UTF-8");
|
|
|
|
} catch (Throwable t) {
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
})
|
|
|
|
.distinct()
|
|
|
|
.collect(Collectors.toCollection(ArrayList::new));
|
|
|
|
final Set<String> validUrl = validateUrl(url);
|
|
|
|
if (!validUrl.isEmpty()) {
|
|
|
|
instance.setUrl(new ArrayList<>());
|
|
|
|
instance.getUrl().addAll(validUrl);
|
|
|
|
}
|
2020-05-08 09:43:26 +02:00
|
|
|
|
2020-05-06 13:20:02 +02:00
|
|
|
return Lists.newArrayList(instance);
|
2020-04-28 11:23:29 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
protected List<Field<String>> prepareSources(final Document doc, final DataInfo info) {
|
|
|
|
return prepareListFields(doc, "//dc:source", info);
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
protected List<StructuredProperty> prepareRelevantDates(final Document doc, final DataInfo info) {
|
|
|
|
return new ArrayList<>(); // NOT PRESENT IN OAF
|
|
|
|
}
|
|
|
|
|
|
|
|
// SOFTWARES
|
|
|
|
|
|
|
|
@Override
|
|
|
|
protected Qualifier prepareSoftwareProgrammingLanguage(final Document doc, final DataInfo info) {
|
|
|
|
return null; // NOT PRESENT IN OAF
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
protected Field<String> prepareSoftwareCodeRepositoryUrl(
|
2020-05-22 12:25:01 +02:00
|
|
|
final Document doc,
|
|
|
|
final DataInfo info) {
|
2020-04-28 11:23:29 +02:00
|
|
|
return null; // NOT PRESENT IN OAF
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
protected List<StructuredProperty> prepareSoftwareLicenses(
|
2020-05-22 12:25:01 +02:00
|
|
|
final Document doc,
|
|
|
|
final DataInfo info) {
|
2020-04-28 11:23:29 +02:00
|
|
|
return new ArrayList<>(); // NOT PRESENT IN OAF
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
protected List<Field<String>> prepareSoftwareDocumentationUrls(
|
2020-05-22 12:25:01 +02:00
|
|
|
final Document doc,
|
|
|
|
final DataInfo info) {
|
2020-04-28 11:23:29 +02:00
|
|
|
return new ArrayList<>(); // NOT PRESENT IN OAF
|
|
|
|
}
|
|
|
|
|
|
|
|
// DATASETS
|
|
|
|
@Override
|
|
|
|
protected List<GeoLocation> prepareDatasetGeoLocations(final Document doc, final DataInfo info) {
|
|
|
|
return new ArrayList<>(); // NOT PRESENT IN OAF
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
protected Field<String> prepareDatasetMetadataVersionNumber(
|
2020-05-22 12:25:01 +02:00
|
|
|
final Document doc,
|
|
|
|
final DataInfo info) {
|
2020-04-28 11:23:29 +02:00
|
|
|
return null; // NOT PRESENT IN OAF
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
protected Field<String> prepareDatasetLastMetadataUpdate(
|
2020-05-22 12:25:01 +02:00
|
|
|
final Document doc,
|
|
|
|
final DataInfo info) {
|
2020-04-28 11:23:29 +02:00
|
|
|
return null; // NOT PRESENT IN OAF
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
protected Field<String> prepareDatasetVersion(final Document doc, final DataInfo info) {
|
|
|
|
return null; // NOT PRESENT IN OAF
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
protected Field<String> prepareDatasetSize(final Document doc, final DataInfo info) {
|
|
|
|
return null; // NOT PRESENT IN OAF
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
protected Field<String> prepareDatasetDevice(final Document doc, final DataInfo info) {
|
|
|
|
return null; // NOT PRESENT IN OAF
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
protected Field<String> prepareDatasetStorageDate(final Document doc, final DataInfo info) {
|
|
|
|
return null; // NOT PRESENT IN OAF
|
|
|
|
}
|
|
|
|
|
|
|
|
// OTHER PRODUCTS
|
|
|
|
|
|
|
|
@Override
|
|
|
|
protected List<Field<String>> prepareOtherResearchProductTools(
|
2020-05-22 12:25:01 +02:00
|
|
|
final Document doc,
|
|
|
|
final DataInfo info) {
|
2020-04-28 11:23:29 +02:00
|
|
|
return new ArrayList<>(); // NOT PRESENT IN OAF
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
protected List<Field<String>> prepareOtherResearchProductContactGroups(
|
2020-05-22 12:25:01 +02:00
|
|
|
final Document doc,
|
|
|
|
final DataInfo info) {
|
2020-04-28 11:23:29 +02:00
|
|
|
return new ArrayList<>(); // NOT PRESENT IN OAF
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
protected List<Field<String>> prepareOtherResearchProductContactPersons(
|
2020-05-22 12:25:01 +02:00
|
|
|
final Document doc,
|
|
|
|
final DataInfo info) {
|
2020-04-28 11:23:29 +02:00
|
|
|
return new ArrayList<>(); // NOT PRESENT IN OAF
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
protected List<Oaf> addOtherResultRels(
|
2020-10-16 17:02:10 +02:00
|
|
|
final Document doc,
|
|
|
|
final OafEntity entity) {
|
2020-04-28 11:23:29 +02:00
|
|
|
|
2020-10-16 16:00:19 +02:00
|
|
|
final String docId = entity.getId();
|
2020-04-28 11:23:29 +02:00
|
|
|
final List<Oaf> res = new ArrayList<>();
|
|
|
|
|
|
|
|
for (final Object o : doc.selectNodes("//*[local-name()='relatedDataset']")) {
|
|
|
|
|
|
|
|
final String originalId = ((Node) o).getText();
|
|
|
|
|
|
|
|
if (StringUtils.isNotBlank(originalId)) {
|
|
|
|
|
|
|
|
final String otherId = createOpenaireId(50, originalId, false);
|
|
|
|
|
2020-05-06 13:20:02 +02:00
|
|
|
res
|
2020-05-26 13:11:09 +02:00
|
|
|
.add(
|
|
|
|
getRelation(
|
2020-10-16 16:00:19 +02:00
|
|
|
docId, otherId, RESULT_RESULT, RELATIONSHIP, IS_RELATED_TO, entity));
|
2020-05-06 13:20:02 +02:00
|
|
|
res
|
2020-05-26 13:11:09 +02:00
|
|
|
.add(
|
|
|
|
getRelation(
|
2020-10-16 16:00:19 +02:00
|
|
|
otherId, docId, RESULT_RESULT, RELATIONSHIP, IS_RELATED_TO, entity));
|
2020-04-28 11:23:29 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
return res;
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
protected Qualifier prepareResourceType(final Document doc, final DataInfo info) {
|
|
|
|
return null; // NOT PRESENT IN OAF
|
|
|
|
}
|
2020-05-26 13:06:55 +02:00
|
|
|
|
|
|
|
@Override
|
|
|
|
protected List<StructuredProperty> prepareResultPids(final Document doc, final DataInfo info) {
|
2020-05-27 11:34:13 +02:00
|
|
|
return prepareListStructPropsWithValidQualifier(
|
2020-10-30 10:56:42 +01:00
|
|
|
doc, "//oaf:identifier", "@identifierType", DNET_PID_TYPES, info)
|
|
|
|
.stream()
|
2020-11-03 12:19:46 +01:00
|
|
|
.map(CleaningFunctions::normalizePidValue)
|
2020-10-30 10:56:42 +01:00
|
|
|
.collect(Collectors.toList());
|
2020-05-26 13:06:55 +02:00
|
|
|
}
|
2020-10-30 10:56:42 +01:00
|
|
|
|
2020-02-04 15:25:47 +01:00
|
|
|
}
|