przemyslaw.jacewicz
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request '[graph raw] datainfo.invisible set as true only for entities' (#336) from invisible_relations into beta 

Reviewed-on: D-Net/dnet-hadoop#336
This commit is contained in:
Claudio Atzori 2023-09-04 16:14:47 +02:00
parent 15666e86a8 adec6692ca
commit 9f5d16624c
3 changed files with 38 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/AbstractMdRecordToOafMapper.java
View File

@ -117,14 +117,14 @@ public abstract class AbstractMdRecordToOafMapper {
return Lists.newArrayList();
}
final DataInfo info = prepareDataInfo(doc, invisible);
final DataInfo entityInfo = prepareDataInfo(doc, invisible);
final long lastUpdateTimestamp = new Date().getTime();
final List<Instance> instances = prepareInstances(doc, info, collectedFrom, hostedBy);
final List<Instance> instances = prepareInstances(doc, entityInfo, collectedFrom, hostedBy);
final String type = getResultType(doc, instances);
return createOafs(doc, type, instances, collectedFrom, info, lastUpdateTimestamp);
return createOafs(doc, type, instances, collectedFrom, entityInfo, lastUpdateTimestamp);
} catch (DocumentException e) {
log.error("Error with record:\n" + xml);
return Lists.newArrayList();
@ -184,13 +184,15 @@ public abstract class AbstractMdRecordToOafMapper {
final List<Oaf> oafs = Lists.newArrayList(entity);
final DataInfo relationInfo = prepareDataInfo(doc, false);
if (!oafs.isEmpty()) {
Set<Oaf> rels = Sets.newHashSet();
rels.addAll(addProjectRels(doc, entity));
rels.addAll(addOtherResultRels(doc, entity));
rels.addAll(addRelations(doc, entity));
rels.addAll(addAffiliations(doc, entity));
rels.addAll(addProjectRels(doc, entity, relationInfo));
rels.addAll(addOtherResultRels(doc, entity, relationInfo));
rels.addAll(addRelations(doc, entity, relationInfo));
rels.addAll(addAffiliations(doc, entity, relationInfo));
oafs.addAll(rels);
}
@ -243,7 +245,7 @@ public abstract class AbstractMdRecordToOafMapper {
private List<Oaf> addProjectRels(
final Document doc,
final OafEntity entity) {
final OafEntity entity, DataInfo info) {
final List<Oaf> res = new ArrayList<>();
@ -262,18 +264,21 @@ public abstract class AbstractMdRecordToOafMapper {
.add(
OafMapperUtils
.getRelation(
docId, projectId, RESULT_PROJECT, OUTCOME, IS_PRODUCED_BY, entity, validationdDate));
docId, projectId, RESULT_PROJECT, OUTCOME, IS_PRODUCED_BY, entity.getCollectedfrom(),
info, entity.getLastupdatetimestamp(), validationdDate, null));
res
.add(
OafMapperUtils
.getRelation(projectId, docId, RESULT_PROJECT, OUTCOME, PRODUCES, entity, validationdDate));
.getRelation(
projectId, docId, RESULT_PROJECT, OUTCOME, PRODUCES, entity.getCollectedfrom(), info,
entity.getLastupdatetimestamp(), validationdDate, null));
}
}
return res;
}
private List<Oaf> addRelations(Document doc, OafEntity entity) {
private List<Oaf> addRelations(Document doc, OafEntity entity, DataInfo info) {
final List<Oaf> rels = Lists.newArrayList();
@ -301,14 +306,16 @@ public abstract class AbstractMdRecordToOafMapper {
.add(
OafMapperUtils
.getRelation(
entity.getId(), targetId, relType, subRelType, relClass, entity,
validationDate));
entity.getId(), targetId, relType, subRelType, relClass,
entity.getCollectedfrom(), info,
entity.getLastupdatetimestamp(), validationDate, null));
rels
.add(
OafMapperUtils
.getRelation(
targetId, entity.getId(), relType, subRelType, relClassInverse, entity,
validationDate));
targetId, entity.getId(), relType, subRelType, relClassInverse,
entity.getCollectedfrom(), info,
entity.getLastupdatetimestamp(), validationDate, null));
}
}
}
@ -316,7 +323,7 @@ public abstract class AbstractMdRecordToOafMapper {
return rels;
}
private List<Oaf> addAffiliations(Document doc, OafEntity entity) {
private List<Oaf> addAffiliations(Document doc, OafEntity entity, DataInfo info) {
final List<Oaf> rels = Lists.newArrayList();
for (Object o : doc.selectNodes("//datacite:affiliation[@affiliationIdentifierScheme='ROR']")) {
@ -345,14 +352,14 @@ public abstract class AbstractMdRecordToOafMapper {
OafMapperUtils
.getRelation(
resultId, orgId, RESULT_ORGANIZATION, AFFILIATION, HAS_AUTHOR_INSTITUTION,
entity.getCollectedfrom(), entity.getDataInfo(), entity.getLastupdatetimestamp(), null,
entity.getCollectedfrom(), info, entity.getLastupdatetimestamp(), null,
properties));
rels
.add(
OafMapperUtils
.getRelation(
orgId, resultId, RESULT_ORGANIZATION, AFFILIATION, IS_AUTHOR_INSTITUTION_OF,
entity.getCollectedfrom(), entity.getDataInfo(), entity.getLastupdatetimestamp(), null,
entity.getCollectedfrom(), info, entity.getLastupdatetimestamp(), null,
properties));
}
}
@ -361,7 +368,7 @@ public abstract class AbstractMdRecordToOafMapper {
protected abstract List<Oaf> addOtherResultRels(
final Document doc,
final OafEntity entity);
final OafEntity entity, DataInfo info);
private void populateResultFields(
final Result r,

9
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OafToOafMapper.java
View File

@ -4,7 +4,6 @@ package eu.dnetlib.dhp.oa.graph.raw;
import static eu.dnetlib.dhp.schema.common.ModelConstants.*;
import static eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils.*;
import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.util.ArrayList;
import java.util.HashSet;
@ -292,7 +291,7 @@ public class OafToOafMapper extends AbstractMdRecordToOafMapper {
@Override
protected List<Oaf> addOtherResultRels(
final Document doc,
final OafEntity entity) {
final OafEntity entity, DataInfo info) {
final String docId = entity.getId();
final List<Oaf> res = new ArrayList<>();
@ -308,11 +307,13 @@ public class OafToOafMapper extends AbstractMdRecordToOafMapper {
res
.add(
getRelation(
docId, otherId, RESULT_RESULT, RELATIONSHIP, IS_RELATED_TO, entity));
docId, otherId, RESULT_RESULT, RELATIONSHIP, IS_RELATED_TO, entity.getCollectedfrom(), info,
entity.getLastupdatetimestamp(), null, null));
res
.add(
getRelation(
otherId, docId, RESULT_RESULT, RELATIONSHIP, IS_RELATED_TO, entity));
otherId, docId, RESULT_RESULT, RELATIONSHIP, IS_RELATED_TO, entity.getCollectedfrom(), info,
entity.getLastupdatetimestamp(), null, null));
}
}
return res;

17
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OdfToOafMapper.java
View File

@ -5,15 +5,11 @@ import static eu.dnetlib.dhp.schema.common.ModelConstants.*;
import static eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils.*;
import static eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils.structuredProperty;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLDecoder;
import java.util.*;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.validator.routines.UrlValidator;
import org.dom4j.Document;
import org.dom4j.Element;
import org.dom4j.Node;
@ -27,7 +23,6 @@ import eu.dnetlib.dhp.schema.common.RelationInverse;
import eu.dnetlib.dhp.schema.oaf.*;
import eu.dnetlib.dhp.schema.oaf.utils.CleaningFunctions;
import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory;
import eu.dnetlib.dhp.schema.oaf.utils.PidType;
public class OdfToOafMapper extends AbstractMdRecordToOafMapper {
@ -397,7 +392,7 @@ public class OdfToOafMapper extends AbstractMdRecordToOafMapper {
@Override
protected List<Oaf> addOtherResultRels(
final Document doc,
final OafEntity entity) {
final OafEntity entity, DataInfo info) {
final String docId = entity.getId();
@ -413,7 +408,7 @@ public class OdfToOafMapper extends AbstractMdRecordToOafMapper {
final String relType = ((Node) o).valueOf("@relationType");
String otherId = guessRelatedIdentifier(idType, originalId);
if (StringUtils.isNotBlank(otherId)) {
res.addAll(getRelations(relType, docId, otherId, entity));
res.addAll(getRelations(relType, docId, otherId, entity, info));
}
}
@ -434,18 +429,20 @@ public class OdfToOafMapper extends AbstractMdRecordToOafMapper {
}
protected List<Oaf> getRelations(final String reltype, final String entityId, final String otherId,
final OafEntity entity) {
final OafEntity entity, DataInfo info) {
final List<Oaf> res = new ArrayList<>();
RelationInverse rel = ModelSupport.findRelation(reltype);
if (rel != null) {
res
.add(
getRelation(
entityId, otherId, rel.getRelType(), rel.getSubReltype(), rel.getRelClass(), entity));
entityId, otherId, rel.getRelType(), rel.getSubReltype(), rel.getRelClass(),
entity.getCollectedfrom(), info, entity.getLastupdatetimestamp(), null, null));
res
.add(
getRelation(
otherId, entityId, rel.getRelType(), rel.getSubReltype(), rel.getInverseRelClass(), entity));
otherId, entityId, rel.getRelType(), rel.getSubReltype(), rel.getInverseRelClass(),
entity.getCollectedfrom(), info, entity.getLastupdatetimestamp(), null, null));
}
return res;