2020-02-05 15:35:40 +01:00
|
|
|
package eu.dnetlib.dhp.migration;
|
|
|
|
|
2020-02-07 16:02:49 +01:00
|
|
|
import java.util.ArrayList;
|
2020-02-10 16:04:06 +01:00
|
|
|
import java.util.Arrays;
|
2020-02-05 15:35:40 +01:00
|
|
|
import java.util.List;
|
|
|
|
import java.util.Map;
|
|
|
|
|
2020-02-07 16:02:49 +01:00
|
|
|
import org.apache.commons.lang3.StringUtils;
|
2020-02-05 15:35:40 +01:00
|
|
|
import org.apache.commons.logging.Log;
|
|
|
|
import org.apache.commons.logging.LogFactory;
|
|
|
|
import org.dom4j.Document;
|
2020-02-07 16:02:49 +01:00
|
|
|
import org.dom4j.Node;
|
2020-02-05 15:35:40 +01:00
|
|
|
|
|
|
|
import eu.dnetlib.dhp.schema.oaf.Author;
|
|
|
|
import eu.dnetlib.dhp.schema.oaf.DataInfo;
|
|
|
|
import eu.dnetlib.dhp.schema.oaf.Field;
|
|
|
|
import eu.dnetlib.dhp.schema.oaf.GeoLocation;
|
|
|
|
import eu.dnetlib.dhp.schema.oaf.Instance;
|
|
|
|
import eu.dnetlib.dhp.schema.oaf.KeyValue;
|
|
|
|
import eu.dnetlib.dhp.schema.oaf.Oaf;
|
|
|
|
import eu.dnetlib.dhp.schema.oaf.Qualifier;
|
2020-02-10 16:04:06 +01:00
|
|
|
import eu.dnetlib.dhp.schema.oaf.Relation;
|
2020-02-05 15:35:40 +01:00
|
|
|
import eu.dnetlib.dhp.schema.oaf.StructuredProperty;
|
|
|
|
|
|
|
|
public class OdfMigrationExecutor extends AbstractMongoExecutor {
|
|
|
|
|
|
|
|
private static final Log log = LogFactory.getLog(OdfMigrationExecutor.class);
|
|
|
|
|
|
|
|
public OdfMigrationExecutor(final String hdfsPath, final String hdfsNameNode, final String hdfsUser, final String mongoBaseUrl, final String mongoDb,
|
|
|
|
final String dbUrl, final String dbUser,
|
|
|
|
final String dbPassword) throws Exception {
|
|
|
|
super(hdfsPath, hdfsNameNode, hdfsUser, mongoBaseUrl, mongoDb, dbUrl, dbUser, dbPassword);
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
protected void registerNamespaces(final Map<String, String> nsContext) {
|
2020-02-13 15:34:13 +01:00
|
|
|
super.registerNamespaces(nsContext);
|
2020-02-07 16:02:49 +01:00
|
|
|
nsContext.put("dc", "http://datacite.org/schema/kernel-3");
|
|
|
|
}
|
2020-02-05 15:35:40 +01:00
|
|
|
|
2020-02-07 16:02:49 +01:00
|
|
|
@Override
|
|
|
|
protected List<StructuredProperty> prepareTitles(final Document doc, final DataInfo info) {
|
|
|
|
return prepareListStructProps(doc, "//dc:title", MAIN_TITLE_QUALIFIER, info);
|
2020-02-05 15:35:40 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2020-02-07 16:02:49 +01:00
|
|
|
protected List<Author> prepareAuthors(final Document doc, final DataInfo info) {
|
|
|
|
final List<Author> res = new ArrayList<>();
|
|
|
|
int pos = 1;
|
|
|
|
for (final Object o : doc.selectNodes("//dc:creator")) {
|
|
|
|
final Node n = (Node) o;
|
|
|
|
final Author author = new Author();
|
|
|
|
author.setFullname(n.valueOf("./dc:creatorName"));
|
|
|
|
author.setName(n.valueOf("./dc:givenName"));
|
|
|
|
author.setSurname(n.valueOf("./dc:familyName"));
|
|
|
|
author.setAffiliation(prepareListFields(doc, "./dc:affiliation", info));
|
|
|
|
author.setPid(preparePids(doc, info));
|
|
|
|
author.setRank(pos++);
|
|
|
|
res.add(author);
|
|
|
|
}
|
|
|
|
return res;
|
|
|
|
}
|
|
|
|
|
|
|
|
private List<StructuredProperty> preparePids(final Document doc, final DataInfo info) {
|
|
|
|
final List<StructuredProperty> res = new ArrayList<>();
|
|
|
|
for (final Object o : doc.selectNodes("./dc:nameIdentifier")) {
|
|
|
|
res.add(structuredProperty(((Node) o).getText(), prepareQualifier((Node) o, "./@nameIdentifierScheme", "dnet:pid_types", "dnet:pid_types"), info));
|
|
|
|
}
|
|
|
|
return res;
|
2020-02-05 15:35:40 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2020-02-07 16:02:49 +01:00
|
|
|
protected List<Instance> prepareInstances(final Document doc, final DataInfo info, final KeyValue collectedfrom, final KeyValue hostedby) {
|
|
|
|
final List<Instance> res = new ArrayList<>();
|
|
|
|
for (final Object o : doc.selectNodes("//dc:alternateIdentifier[@alternateIdentifierType='URL']")) {
|
|
|
|
final Instance instance = new Instance();
|
2020-02-17 10:40:53 +01:00
|
|
|
instance.setUrl(Arrays.asList(((Node) o).getText().trim()));
|
2020-02-07 16:02:49 +01:00
|
|
|
instance.setInstancetype(prepareQualifier(doc, "//dr:CobjCategory", "dnet:publication_resource", "dnet:publication_resource"));
|
|
|
|
instance.setCollectedfrom(collectedfrom);
|
|
|
|
instance.setHostedby(hostedby);
|
|
|
|
instance.setDateofacceptance(field(doc.valueOf("//oaf:dateAccepted"), info));
|
|
|
|
instance.setDistributionlocation(doc.valueOf("//oaf:distributionlocation"));
|
|
|
|
instance.setAccessright(prepareQualifier(doc, "//oaf:accessrights", "dnet:access_modes", "dnet:access_modes"));
|
|
|
|
instance.setLicense(field(doc.valueOf("//oaf:license"), info));
|
2020-02-17 10:40:53 +01:00
|
|
|
instance.setRefereed(field(doc.valueOf("//oaf:refereed"), info));
|
|
|
|
instance.setProcessingchargeamount(field(doc.valueOf("//oaf:processingchargeamount"), info));
|
|
|
|
instance.setProcessingchargecurrency(field(doc.valueOf("//oaf:processingchargeamount/@currency"), info));
|
2020-02-07 16:02:49 +01:00
|
|
|
res.add(instance);
|
|
|
|
}
|
|
|
|
return res;
|
2020-02-05 15:35:40 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2020-02-07 16:02:49 +01:00
|
|
|
protected List<Field<String>> prepareSources(final Document doc, final DataInfo info) {
|
|
|
|
return new ArrayList<>(); // Not present in ODF ???
|
2020-02-05 15:35:40 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
protected List<StructuredProperty> prepareRelevantDates(final Document doc, final DataInfo info) {
|
2020-02-07 16:02:49 +01:00
|
|
|
final List<StructuredProperty> res = new ArrayList<>();
|
|
|
|
for (final Object o : doc.selectNodes("//dc:date")) {
|
|
|
|
final String dateType = ((Node) o).valueOf("@dateType");
|
|
|
|
if (StringUtils.isBlank(dateType) && !dateType.equalsIgnoreCase("Accepted") && !dateType.equalsIgnoreCase("Issued")
|
|
|
|
&& !dateType.equalsIgnoreCase("Updated") && !dateType.equalsIgnoreCase("Available")) {
|
|
|
|
res.add(structuredProperty(((Node) o).getText(), "UNKNOWN", "UNKNOWN", "dnet:dataCite_date", "dnet:dataCite_date", info));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return res;
|
2020-02-05 15:35:40 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
protected List<Field<String>> prepareCoverages(final Document doc, final DataInfo info) {
|
2020-02-07 16:02:49 +01:00
|
|
|
return new ArrayList<>(); // Not present in ODF ???
|
2020-02-05 15:35:40 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
protected List<Field<String>> prepareContributors(final Document doc, final DataInfo info) {
|
2020-02-07 16:02:49 +01:00
|
|
|
return prepareListFields(doc, "//dc:contributorName", info);
|
2020-02-05 15:35:40 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
protected List<Field<String>> prepareFormats(final Document doc, final DataInfo info) {
|
2020-02-07 16:02:49 +01:00
|
|
|
return prepareListFields(doc, "//dc:format", info);
|
2020-02-05 15:35:40 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
protected Field<String> preparePublisher(final Document doc, final DataInfo info) {
|
2020-02-07 16:02:49 +01:00
|
|
|
return prepareField(doc, "//dc:publisher", info);
|
2020-02-05 15:35:40 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
protected List<Field<String>> prepareDescriptions(final Document doc, final DataInfo info) {
|
2020-02-07 16:02:49 +01:00
|
|
|
return prepareListFields(doc, "//dc:description[@descriptionType='Abstract']", info);
|
2020-02-05 15:35:40 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
protected List<StructuredProperty> prepareSubjects(final Document doc, final DataInfo info) {
|
2020-02-07 16:02:49 +01:00
|
|
|
return prepareListStructProps(doc, "//dc:subject", info);
|
2020-02-05 15:35:40 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
protected Qualifier prepareLanguages(final Document doc) {
|
2020-02-07 16:02:49 +01:00
|
|
|
return prepareQualifier(doc, "//dc:language", "dnet:languages", "dnet:languages");
|
2020-02-05 15:35:40 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
protected List<Field<String>> prepareOtherResearchProductTools(final Document doc, final DataInfo info) {
|
2020-02-07 16:02:49 +01:00
|
|
|
return new ArrayList<>(); // Not present in ODF ???
|
2020-02-05 15:35:40 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
protected List<Field<String>> prepareOtherResearchProductContactGroups(final Document doc, final DataInfo info) {
|
2020-02-07 16:02:49 +01:00
|
|
|
return prepareListFields(doc, "//dc:contributor[@contributorType='ContactGroup']/dc:contributorName", info);
|
2020-02-05 15:35:40 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
protected List<Field<String>> prepareOtherResearchProductContactPersons(final Document doc, final DataInfo info) {
|
2020-02-07 16:02:49 +01:00
|
|
|
return prepareListFields(doc, "//dc:contributor[@contributorType='ContactPerson']/dc:contributorName", info);
|
2020-02-05 15:35:40 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
protected Qualifier prepareSoftwareProgrammingLanguage(final Document doc, final DataInfo info) {
|
2020-02-07 16:02:49 +01:00
|
|
|
return prepareQualifier(doc, "//dc:format", "dnet:programming_languages", "dnet:programming_languages");
|
2020-02-05 15:35:40 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
protected Field<String> prepareSoftwareCodeRepositoryUrl(final Document doc, final DataInfo info) {
|
2020-02-07 16:02:49 +01:00
|
|
|
return null; // Not present in ODF ???
|
2020-02-05 15:35:40 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
protected List<StructuredProperty> prepareSoftwareLicenses(final Document doc, final DataInfo info) {
|
2020-02-07 16:02:49 +01:00
|
|
|
return new ArrayList<>(); // Not present in ODF ???
|
2020-02-05 15:35:40 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
protected List<Field<String>> prepareSoftwareDocumentationUrls(final Document doc, final DataInfo info) {
|
2020-02-07 16:02:49 +01:00
|
|
|
return prepareListFields(doc, "//dc:relatedIdentifier[@relatedIdentifierType='URL' and @relationType='IsDocumentedBy']", info);
|
2020-02-05 15:35:40 +01:00
|
|
|
}
|
|
|
|
|
2020-02-07 16:02:49 +01:00
|
|
|
// DATASETS
|
|
|
|
|
2020-02-05 15:35:40 +01:00
|
|
|
@Override
|
|
|
|
protected List<GeoLocation> prepareDatasetGeoLocations(final Document doc, final DataInfo info) {
|
2020-02-07 16:02:49 +01:00
|
|
|
final List<GeoLocation> res = new ArrayList<>();
|
|
|
|
|
|
|
|
for (final Object o : doc.selectNodes("//dc:geoLocation")) {
|
|
|
|
final GeoLocation loc = new GeoLocation();
|
|
|
|
loc.setBox(((Node) o).valueOf("./dc:geoLocationBox"));
|
|
|
|
loc.setPlace(((Node) o).valueOf("./dc:geoLocationPlace"));
|
|
|
|
loc.setPoint(((Node) o).valueOf("./dc:geoLocationPoint"));
|
|
|
|
res.add(loc);
|
|
|
|
}
|
|
|
|
return res;
|
2020-02-05 15:35:40 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
protected Field<String> prepareDatasetMetadataVersionNumber(final Document doc, final DataInfo info) {
|
2020-02-10 16:04:06 +01:00
|
|
|
return null; // Not present in ODF ???
|
2020-02-05 15:35:40 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
protected Field<String> prepareDatasetLastMetadataUpdate(final Document doc, final DataInfo info) {
|
2020-02-07 16:02:49 +01:00
|
|
|
return prepareField(doc, "//dc:date[@dateType='Updated']", info);
|
2020-02-05 15:35:40 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
protected Field<String> prepareDatasetVersion(final Document doc, final DataInfo info) {
|
2020-02-07 16:02:49 +01:00
|
|
|
return prepareField(doc, "//dc:version", info);
|
2020-02-05 15:35:40 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
protected Field<String> prepareDatasetSize(final Document doc, final DataInfo info) {
|
2020-02-07 16:02:49 +01:00
|
|
|
return prepareField(doc, "//dc:size", info);
|
2020-02-05 15:35:40 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
protected Field<String> prepareDatasetDevice(final Document doc, final DataInfo info) {
|
2020-02-07 16:02:49 +01:00
|
|
|
return null; // Not present in ODF ???
|
2020-02-05 15:35:40 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
protected Field<String> prepareDatasetStorageDate(final Document doc, final DataInfo info) {
|
2020-02-07 16:02:49 +01:00
|
|
|
return prepareField(doc, "//dc:date[@dateType='Issued']", info);
|
2020-02-05 15:35:40 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2020-02-10 16:04:06 +01:00
|
|
|
protected List<Oaf> addOtherResultRels(final Document doc, final KeyValue collectedFrom, final DataInfo info, final long lastUpdateTimestamp) {
|
|
|
|
|
|
|
|
final String docId = createOpenaireId(50, doc.valueOf("//dri:objIdentifier"));
|
|
|
|
|
|
|
|
final List<Oaf> res = new ArrayList<>();
|
|
|
|
|
|
|
|
for (final Object o : doc.selectNodes("//*[local-name() = 'resource']//*[local-name()='relatedIdentifier' and ./@relatedIdentifierType='OPENAIRE']")) {
|
|
|
|
final String otherId = createOpenaireId(50, ((Node) o).getText());
|
|
|
|
final String type = ((Node) o).valueOf("@relationType");
|
|
|
|
|
|
|
|
if (type.equals("IsSupplementTo")) {
|
|
|
|
res.add(prepareOtherResultRel(collectedFrom, info, lastUpdateTimestamp, docId, otherId, "supplement", "isSupplementTo"));
|
|
|
|
res.add(prepareOtherResultRel(collectedFrom, info, lastUpdateTimestamp, otherId, docId, "supplement", "isSupplementedBy"));
|
|
|
|
} else if (type.equals("IsPartOf")) {
|
|
|
|
res.add(prepareOtherResultRel(collectedFrom, info, lastUpdateTimestamp, docId, otherId, "part", "IsPartOf"));
|
|
|
|
res.add(prepareOtherResultRel(collectedFrom, info, lastUpdateTimestamp, otherId, docId, "part", "HasParts"));
|
|
|
|
} else {}
|
|
|
|
}
|
|
|
|
return res;
|
|
|
|
}
|
|
|
|
|
|
|
|
private Relation prepareOtherResultRel(final KeyValue collectedFrom,
|
2020-02-05 15:35:40 +01:00
|
|
|
final DataInfo info,
|
2020-02-10 16:04:06 +01:00
|
|
|
final long lastUpdateTimestamp,
|
|
|
|
final String source,
|
|
|
|
final String target,
|
|
|
|
final String subRelType,
|
|
|
|
final String relClass) {
|
|
|
|
final Relation r = new Relation();
|
|
|
|
r.setRelType("resultResult");
|
|
|
|
r.setSubRelType(subRelType);
|
|
|
|
r.setRelClass(relClass);
|
|
|
|
r.setSource(source);
|
|
|
|
r.setTarget(target);
|
|
|
|
r.setCollectedFrom(Arrays.asList(collectedFrom));
|
|
|
|
r.setDataInfo(info);
|
|
|
|
r.setLastupdatetimestamp(lastUpdateTimestamp);
|
|
|
|
return r;
|
|
|
|
}
|
2020-02-05 15:35:40 +01:00
|
|
|
|
2020-02-10 16:04:06 +01:00
|
|
|
@Override
|
|
|
|
protected Qualifier prepareResourceType(final Document doc, final DataInfo info) {
|
|
|
|
return prepareQualifier(doc, "//*[local-name() = 'resource']//*[local-name() = 'resourceType']", "dnet:dataCite_resource", "dnet:dataCite_resource");
|
2020-02-05 15:35:40 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
}
|