D-Net
/
dnet-hadoop
13
4
Fork 10
Code Issues 14 Pull Requests 9 Projects 2 Releases 1 Wiki Activity

-Renamed RelationInverse into RelationLabel. 

-Removed findRelation from ModelSupport
-code formatted
This commit is contained in:
Sandro La Bruzzo 2023-04-28 11:44:28 +02:00
parent 8c9a77d7eb
commit f8f4b9a018
17 changed files with 502 additions and 660 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
dhp-common/src/main/java/eu/dnetlib/dhp/common/api/ZenodoAPIClient.java
View File

@ -9,13 +9,13 @@ import java.util.concurrent.TimeUnit;
import org.apache.http.HttpHeaders;
import org.apache.http.entity.ContentType;
import org.jetbrains.annotations.NotNull;
import com.google.gson.Gson;
import eu.dnetlib.dhp.common.api.zenodo.ZenodoModel;
import eu.dnetlib.dhp.common.api.zenodo.ZenodoModelList;
import okhttp3.*;
import org.jetbrains.annotations.NotNull;
public class ZenodoAPIClient implements Serializable {
@ -155,7 +155,6 @@ public class ZenodoAPIClient implements Serializable {
conn.setDoOutput(true);
conn.setRequestMethod("PUT");
try (OutputStream os = conn.getOutputStream()) {
byte[] input = metadata.getBytes("utf-8");
os.write(input, 0, input.length);
@ -169,7 +168,6 @@ public class ZenodoAPIClient implements Serializable {
return responseCode;
}
private boolean checkOKStatus(int responseCode) {
@ -233,7 +231,6 @@ public class ZenodoAPIClient implements Serializable {
conn.setDoOutput(true);
conn.setRequestMethod("POST");
try (OutputStream os = conn.getOutputStream()) {
byte[] input = json.getBytes("utf-8");
os.write(input, 0, input.length);
@ -296,7 +293,6 @@ public class ZenodoAPIClient implements Serializable {
ZenodoModel zenodoModel = new Gson().fromJson(body, ZenodoModel.class);
bucket = zenodoModel.getLinks().getBucket();
return responseCode;
}
@ -331,8 +327,6 @@ public class ZenodoAPIClient implements Serializable {
conn.setDoOutput(true);
conn.setRequestMethod("GET");
String body = getBody(conn);
int responseCode = conn.getResponseCode();
@ -341,12 +335,8 @@ public class ZenodoAPIClient implements Serializable {
if (!checkOKStatus(responseCode))
throw new IOException("Unexpected code " + responseCode + body);
return body;
}
private String getBucket(String inputUurl) throws IOException {
@ -370,8 +360,6 @@ public class ZenodoAPIClient implements Serializable {
return zenodoModel.getLinks().getBucket();
}
}

1
dhp-common/src/main/java/eu/dnetlib/dhp/schema/common/ModelConstants.java
View File

@ -86,7 +86,6 @@ public class ModelConstants {
public static final Qualifier PROVENANCE_ACTION_SET_QUALIFIER = qualifier(
SYSIMPORT_ACTIONSET, SYSIMPORT_ACTIONSET, DNET_PROVENANCE_ACTIONS);
public static final String UNKNOWN = "UNKNOWN";
public static final String NOT_AVAILABLE = "not available";

44
dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/common/ModelSupport.java
View File

@ -2,16 +2,12 @@
package eu.dnetlib.dhp.schema.oaf.common;
import static com.google.common.base.Preconditions.checkArgument;
import static eu.dnetlib.dhp.schema.common.ModelConstants.*;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.text.ParseException;
import java.util.Date;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.*;
import java.util.function.Function;
import org.apache.commons.codec.binary.Hex;
@ -96,26 +92,6 @@ public class ModelSupport {
idPrefixEntity.put("50", "result");
}
private static void set(Map<String, RelationInverse> relationInverseMap, String relType, String subRelType,
String relClass, String inverseRelClass) {
relationInverseMap
.put(
rel(relType, subRelType, relClass), new RelationInverse()
.setInverseRelClass(inverseRelClass)
.setRelClass(relClass)
.setRelType(relType)
.setSubReltype(subRelType));
if (!relClass.equals(inverseRelClass)) {
relationInverseMap
.put(
rel(relType, subRelType, inverseRelClass), new RelationInverse()
.setInverseRelClass(relClass)
.setRelClass(inverseRelClass)
.setRelType(relType)
.setSubReltype(subRelType));
}
}
/**
* Helper method: combines the relation attributes
* @param relType
@ -127,6 +103,24 @@ public class ModelSupport {
return String.format("%s_%s_%s", relType, subRelType, relClass);
}
/**
* Helper method: deserialize the relation attributes serialized with rel
* @param relType
* @param subRelType
* @param relClass
* @return
*/
public static RelationLabel unRel(String deserialization) {
final String[] s = deserialization.split("_");
if (s!= null && s.length==3) {
final Relation.RELTYPE currentRelType = Relation.RELTYPE.valueOf(s[0]);
final Relation.SUBRELTYPE currentSubRelType = Relation.SUBRELTYPE.valueOf(s[1]);
final Relation.RELCLASS currentRelClass = Relation.RELCLASS.valueOf(s[2]);
return new RelationLabel(currentRelClass, currentRelType, currentSubRelType);
}
throw new IllegalArgumentException("Invalid relationship format for "+ deserialization);
}
private static final String schemeTemplate = "dnet:%s_%s_relations";
public static final String DATE_FORMAT = "yyyy-MM-dd";

46
dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/common/RelationInverse.java
View File

@ -1,46 +0,0 @@
package eu.dnetlib.dhp.schema.oaf.common;
public class RelationInverse {
private String relClass;
private String inverseRelClass;
private String relType;
private String subReltype;
public String getRelType() {
return relType;
}
public RelationInverse setRelType(String relType) {
this.relType = relType;
return this;
}
public String getSubReltype() {
return subReltype;
}
public RelationInverse setSubReltype(String subReltype) {
this.subReltype = subReltype;
return this;
}
public String getRelClass() {
return relClass;
}
public RelationInverse setRelClass(String relClass) {
this.relClass = relClass;
return this;
}
public String getInverseRelClass() {
return inverseRelClass;
}
public RelationInverse setInverseRelClass(String inverseRelClass) {
this.inverseRelClass = inverseRelClass;
return this;
}
}

33
dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/common/RelationLabel.java Normal file
View File

@ -0,0 +1,33 @@
package eu.dnetlib.dhp.schema.oaf.common;
import eu.dnetlib.dhp.schema.oaf.Relation;
public class RelationLabel {
private final Relation.RELCLASS relClass;
private final Relation.RELTYPE relType;
private final Relation.SUBRELTYPE subReltype;
public RelationLabel(Relation.RELCLASS relClass, Relation.RELTYPE relType, Relation.SUBRELTYPE subReltype) {
this.relClass = relClass;
this.relType = relType;
this.subReltype = subReltype;
}
public RelationLabel inverse() {
return new RelationLabel(relClass.getInverse(), relType, subReltype);
}
public Relation.RELTYPE getRelType() {
return relType;
}
public Relation.SUBRELTYPE getSubReltype() {
return subReltype;
}
public Relation.RELCLASS getRelClass() {
return relClass;
}
}

59
dhp-common/src/main/java/eu/dnetlib/dhp/schema/sx/OafUtils.scala
View File

@ -1,59 +0,0 @@
package eu.dnetlib.dhp.schema.sx
import eu.dnetlib.dhp.schema.common.ModelConstants
import eu.dnetlib.dhp.schema.oaf._
object OafUtils {
def generateKeyValue(key: String, value: String): KeyValue = {
val kv: KeyValue = new KeyValue()
kv.setKey(key)
kv.setValue(value)
kv
}
def generateDataInfo(trust: Float = 0.9f, invisible: Boolean = false): DataInfo = {
val di = new DataInfo
di.setInferred(false)
di.setTrust(trust)
di.setProvenanceaction(createQualifier(ModelConstants.SYSIMPORT_ACTIONSET, ModelConstants.DNET_PROVENANCE_ACTIONS))
di
}
def createQualifier(cls: String, sch: String): Qualifier = {
createQualifier(cls, cls, sch)
}
def createQualifier(classId: String, className: String, schemeId: String): Qualifier = {
val q: Qualifier = new Qualifier
q.setClassid(classId)
q.setClassname(className)
q.setSchemeid(schemeId)
q
}
def createAccessRight(classId: String, className: String, schemeId: String): AccessRight = {
val accessRight: AccessRight = new AccessRight
accessRight.setClassid(classId)
accessRight.setClassname(className)
accessRight.setSchemeid(schemeId)
accessRight
}
def createSP(value: String, classId: String,className:String, schemeId: String): StructuredProperty = {
val sp = new StructuredProperty
sp.setQualifier(createQualifier(classId,className, schemeId))
sp.setValue(value)
sp
}
def createSP(value: String, classId: String, schemeId: String): StructuredProperty = {
val sp = new StructuredProperty
sp.setQualifier(createQualifier(classId, schemeId))
sp.setValue(value)
sp
}
}

25
dhp-common/src/main/java/eu/dnetlib/scholexplorer/relation/RelInfo.java
View File

@ -1,25 +0,0 @@
package eu.dnetlib.scholexplorer.relation;
import java.io.Serializable;
public class RelInfo implements Serializable {
private String original;
private String inverse;
public String getOriginal() {
return original;
}
public void setOriginal(String original) {
this.original = original;
}
public String getInverse() {
return inverse;
}
public void setInverse(String inverse) {
this.inverse = inverse;
}
}

20
dhp-common/src/main/java/eu/dnetlib/scholexplorer/relation/RelationMapper.java
View File

@ -1,20 +0,0 @@
package eu.dnetlib.scholexplorer.relation;
import java.io.Serializable;
import java.util.HashMap;
import org.apache.commons.io.IOUtils;
import com.fasterxml.jackson.databind.ObjectMapper;
public class RelationMapper extends HashMap<String, RelInfo> implements Serializable {
public static RelationMapper load() throws Exception {
final String json = IOUtils.toString(RelationMapper.class.getResourceAsStream("relations.json"));
ObjectMapper mapper = new ObjectMapper();
return mapper.readValue(json, RelationMapper.class);
}
}

11
dhp-common/src/test/java/eu/dnetlib/dhp/schema/oaf/common/ModelSupportTest.java
View File

@ -36,15 +36,4 @@ public class ModelSupportTest {
}
}
@Nested
class InverseRelation {
@Test
void findRelations() {
assertNotNull(ModelSupport.findRelation("isMetadataFor"));
assertNotNull(ModelSupport.findRelation("ismetadatafor"));
assertNotNull(ModelSupport.findRelation("ISMETADATAFOR"));
assertNotNull(ModelSupport.findRelation("isRelatedTo"));
}
}
}

16
dhp-common/src/test/java/eu/dnetlib/scholexplorer/relation/RelationMapperTest.java
View File

@ -1,16 +0,0 @@
package eu.dnetlib.scholexplorer.relation;
import static org.junit.jupiter.api.Assertions.assertFalse;
import org.junit.jupiter.api.Test;
class RelationMapperTest {
@Test
void testLoadRels() throws Exception {
RelationMapper relationMapper = RelationMapper.load();
assertFalse(relationMapper.isEmpty());
}
}

11
dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/datacite/DataciteModelConstants.scala
View File

@ -2,7 +2,7 @@ package eu.dnetlib.dhp.datacite
import eu.dnetlib.dhp.schema.common.ModelConstants
import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils
import eu.dnetlib.dhp.schema.oaf.{DataInfo, EntityDataInfo, KeyValue}
import eu.dnetlib.dhp.schema.oaf.{DataInfo, EntityDataInfo, KeyValue, Relation}
import java.io.InputStream
import java.time.format.DateTimeFormatter
@ -66,12 +66,13 @@ class DataciteModelConstants extends Serializable {}
object DataciteModelConstants {
val REL_TYPE_VALUE: String = "resultResult"
val REL_TYPE_VALUE =Relation.RELTYPE.resultResult
val DATE_RELATION_KEY = "RelationDate"
val DATACITE_FILTER_PATH = "/eu/dnetlib/dhp/datacite/datacite_filter"
val DOI_CLASS = "doi"
val SUBJ_CLASS = "keywords"
val DATACITE_NAME = "Datacite"
val PMID = "pmid"
val ARXIV = "arxiv"
val dataInfo: EntityDataInfo = dataciteDataInfo(0.9f)
val relDataInfo = OafMapperUtils.fromEntityDataInfo(dataInfo);
@ -138,4 +139,8 @@ object DataciteModelConstants {
Pattern.compile("(19|20)\\d\\d", Pattern.MULTILINE)
)
def validIdentifiersInRelation(relatedIdentifierType:String):Boolean = {
relatedIdentifierType.equalsIgnoreCase(ModelConstants.DOI) || relatedIdentifierType.equalsIgnoreCase(PMID) ||
relatedIdentifierType.equalsIgnoreCase(ARXIV)
}
}

44
dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/datacite/DataciteToOAFTransformation.scala
View File

@ -100,7 +100,7 @@ object DataciteToOAFTransformation {
}
/** This utility method indicates whether the embargo date has been reached
* @param embargo_end_date
* @param embargo_end_date the end date of embargo
* @return True if the embargo date has been reached, false otherwise
*/
def embargo_end(embargo_end_date: String): Boolean = {
@ -345,8 +345,8 @@ object DataciteToOAFTransformation {
// DOI is mapped on a PID inside a Instance object
val doi_q = OafMapperUtils.qualifier(
"doi",
"doi",
ModelConstants.DOI,
ModelConstants.DOI,
ModelConstants.DNET_PID_TYPES
)
val pid = OafMapperUtils.structuredProperty(doi, doi_q)
@ -615,44 +615,52 @@ object DataciteToOAFTransformation {
List(result)
}
//TODO @CLAUDIO we need to define relation in which verse
/**
* This function generate unresolved relation from the original Datacite document
* @param rels the related identifier section on the document
* @param id the source record Identifier
* @param date the date of collection
* @return a List of OAF relation
*/
private def generateRelations(
rels: List[RelatedIdentifierType],
id: String,
date: String
): List[Relation] = {
// TODO We need to check how to generate realtions
// in the previous implementation we create all Bidirection Relations
// related to a DOI pid or arxiv,
val bidirectionalRels: List[Relation] = rels
.filter(r =>
Relation.RELCLASS.exists(r.relationType) && (r.relatedIdentifierType.equalsIgnoreCase("doi") ||
r.relatedIdentifierType.equalsIgnoreCase("pmid") ||
r.relatedIdentifierType.equalsIgnoreCase("arxiv"))
Relation.RELCLASS.exists(r.relationType) && validIdentifiersInRelation(r.relatedIdentifierType)
)
.map(r => {
val subRelType = subRelTypeMapping(r.relationType).relType
val subRelType = Relation.SUBRELTYPE.valueOf(r.relationType)
val target = DHPUtils.generateUnresolvedIdentifier(r.relatedIdentifier, r.relatedIdentifierType)
relation(id, target, subRelType, r.relationType, date)
relation(id, target, subRelType, Relation.RELCLASS.valueOf(r.relationType), date)
})
val citationRels: List[Relation] = rels
.filter(r =>
(r.relatedIdentifierType.equalsIgnoreCase("doi") ||
r.relatedIdentifierType.equalsIgnoreCase("pmid") ||
r.relatedIdentifierType.equalsIgnoreCase("arxiv")) &&
.filter(r =>validIdentifiersInRelation(r.relatedIdentifierType) &&
(r.relationType.toLowerCase.contains("cite") || r.relationType.toLowerCase.contains("reference"))
)
.map(r => {
r.relationType match {
case ModelConstants.CITES | ModelConstants.REFERENCES =>
case Relation.RELCLASS.Cites.toString | Relation.RELCLASS.References.toString =>
val target = DHPUtils.generateUnresolvedIdentifier(r.relatedIdentifier, r.relatedIdentifierType)
relation(id, target, ModelConstants.CITATION, ModelConstants.CITES, date)
case ModelConstants.IS_CITED_BY | ModelConstants.IS_REFERENCED_BY =>
relation(id, target, Relation.SUBRELTYPE.citation, Relation.RELCLASS.Cites, date)
case Relation.RELCLASS.IsCitedBy.toString | Relation.RELCLASS.IsReferencedBy.toString =>
val source = DHPUtils.generateUnresolvedIdentifier(r.relatedIdentifier, r.relatedIdentifierType)
relation(source, id, ModelConstants.CITATION, ModelConstants.CITES, date)
relation(source, id, Relation.SUBRELTYPE.citation, Relation.RELCLASS.Cites, date)
}
})
citationRels ::: bidirectionalRels
}
def relation(source: String, target: String, subRelType: String, relClass: String, date: String): Relation = {
def relation(source: String, target: String, subRelType: Relation.SUBRELTYPE, relClass: Relation.RELCLASS, date: String): Relation = {
val rel = new Relation
rel.setProvenance(Lists.newArrayList(OafMapperUtils.getProvenance(DATACITE_COLLECTED_FROM, relDataInfo)))

22
dhp-workflows/dhp-blacklist/src/main/java/eu/dnetlib/dhp/blacklist/ReadBlacklistFromDB.java
View File

@ -14,7 +14,7 @@ import java.util.function.Consumer;
import java.util.function.Function;
import eu.dnetlib.dhp.schema.oaf.common.ModelSupport;
import eu.dnetlib.dhp.schema.oaf.common.RelationInverse;
import eu.dnetlib.dhp.schema.oaf.common.RelationLabel;
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@ -76,30 +76,22 @@ public class ReadBlacklistFromDB implements Closeable {
public List<Relation> processBlacklistEntry(ResultSet rs) {
try {
Relation direct = new Relation();
Relation inverse = new Relation();
String source_prefix = ModelSupport.entityIdPrefix.get(rs.getString("source_type"));
String target_prefix = ModelSupport.entityIdPrefix.get(rs.getString("target_type"));
String source_direct = source_prefix + "|" + rs.getString("source");
direct.setSource(source_direct);
inverse.setTarget(source_direct);
String target_direct = target_prefix + "|" + rs.getString("target");
direct.setTarget(target_direct);
inverse.setSource(target_direct);
String encoding = rs.getString("relationship");
RelationInverse ri = ModelSupport.findInverse(encoding);
direct.setRelClass(ri.getRelClass());
inverse.setRelClass(ri.getInverseRelClass());
direct.setRelType(ri.getRelType());
inverse.setRelType(ri.getRelType());
direct.setSubRelType(ri.getSubReltype());
inverse.setSubRelType(ri.getSubReltype());
return Arrays.asList(direct, inverse);
final RelationLabel directLabel = ModelSupport.unRel(encoding);
direct.setRelClass(directLabel.getRelClass());
direct.setRelType(directLabel.getRelType());
direct.setSubRelType(directLabel.getSubReltype());
return Arrays.asList(direct, direct.inverse());
} catch (final SQLException e) {
throw new RuntimeException(e);
}

4
dhp-workflows/dhp-blacklist/src/test/java/eu/dnetlib/dhp/blacklist/BlacklistRelationTest.java
View File

@ -5,7 +5,7 @@ import java.util.Arrays;
import java.util.List;
import eu.dnetlib.dhp.schema.oaf.common.ModelSupport;
import eu.dnetlib.dhp.schema.oaf.common.RelationInverse;
import eu.dnetlib.dhp.schema.oaf.common.RelationLabel;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@ -25,7 +25,7 @@ public class BlacklistRelationTest {
"resultProject_outcome_isProducedBy");
rels.forEach(r -> {
RelationInverse inverse = ModelSupport.relationInverseMap.get(r);
RelationLabel inverse = ModelSupport.relationInverseMap.get(r);
Assertions.assertNotNull(inverse);
Assertions.assertNotNull(inverse.getRelType());
Assertions.assertNotNull(inverse.getSubReltype());

6
dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/SparkGenerateDoiBoost.scala
View File

@ -203,9 +203,9 @@ object SparkGenerateDoiBoost {
val r: Relation = new Relation
r.setSource(pub.getId)
r.setTarget(affId)
r.setRelType(ModelConstants.RESULT_ORGANIZATION)
r.setRelClass(ModelConstants.HAS_AUTHOR_INSTITUTION)
r.setSubRelType(ModelConstants.AFFILIATION)
r.setRelType(Relation.RELTYPE.resultOrganization)
r.setRelClass(Relation.RELCLASS.hasAuthorInstitution)
r.setSubRelType(Relation.SUBRELTYPE.affiliation)
r.setProvenance(OafMapperUtils.getProvenance(pub.getCollectedfrom, dataInfo))
List(r)

4
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OdfToOafMapper.java
View File

@ -11,7 +11,7 @@ import java.util.*;
import java.util.stream.Collectors;
import eu.dnetlib.dhp.schema.oaf.common.ModelSupport;
import eu.dnetlib.dhp.schema.oaf.common.RelationInverse;
import eu.dnetlib.dhp.schema.oaf.common.RelationLabel;
import org.apache.commons.lang3.StringUtils;
import org.dom4j.Document;
import org.dom4j.Element;
@ -401,7 +401,7 @@ public class OdfToOafMapper extends AbstractMdRecordToOafMapper {
protected List<Oaf> getRelations(final String reltype, final String entityId, final String otherId,
final Entity entity) {
final List<Oaf> res = new ArrayList<>();
RelationInverse rel = ModelSupport.findRelation(reltype);
RelationLabel rel = ModelSupport.findRelation(reltype);
if (rel != null) {
res
.add(