D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 13 Projects 2 Releases 2 Wiki Activity

Merge branch 'master' of https://code-repo.d2science.org/D-Net/dnet-hadoop

This commit is contained in:
Claudio Atzori 2019-10-24 17:43:55 +02:00
parent 0d88f9a6a4 2dd9572f41
commit d46371ceab
6 changed files with 151 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/ExtraInfo.java
View File

@ -18,39 +18,44 @@ public class ExtraInfo implements Serializable {
return name;
}
public void setName(String name) {
public ExtraInfo setName(String name) {
this.name = name;
return this;
}
public String getTypology() {
return typology;
}
public void setTypology(String typology) {
public ExtraInfo setTypology(String typology) {
this.typology = typology;
return this;
}
public String getProvenance() {
return provenance;
}
public void setProvenance(String provenance) {
public ExtraInfo setProvenance(String provenance) {
this.provenance = provenance;
return this;
}
public String getTrust() {
return trust;
}
public void setTrust(String trust) {
public ExtraInfo setTrust(String trust) {
this.trust = trust;
return this;
}
public String getValue() {
return value;
}
public void setValue(String value) {
public ExtraInfo setValue(String value) {
this.value = value;
return this;
}
}

3
dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/OAIProvenance.java
View File

@ -10,7 +10,8 @@ public class OAIProvenance implements Serializable {
return originDescription;
}
public void setOriginDescription(OriginDescription originDescription) {
public OAIProvenance setOriginDescription(OriginDescription originDescription) {
this.originDescription = originDescription;
return this;
}
}

21
dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/OriginDescription.java
View File

@ -22,55 +22,62 @@ public class OriginDescription implements Serializable {
return harvestDate;
}
public void setHarvestDate(String harvestDate) {
public OriginDescription setHarvestDate(String harvestDate) {
this.harvestDate = harvestDate;
return this;
}
public Boolean getAltered() {
return altered;
}
public void setAltered(Boolean altered) {
public OriginDescription setAltered(Boolean altered) {
this.altered = altered;
return this;
}
public String getBaseURL() {
return baseURL;
}
public void setBaseURL(String baseURL) {
public OriginDescription setBaseURL(String baseURL) {
this.baseURL = baseURL;
return this;
}
public String getIdentifier() {
return identifier;
}
public void setIdentifier(String identifier) {
public OriginDescription setIdentifier(String identifier) {
this.identifier = identifier;
return this;
}
public String getDatestamp() {
return datestamp;
}
public void setDatestamp(String datestamp) {
public OriginDescription setDatestamp(String datestamp) {
this.datestamp = datestamp;
return this;
}
public String getMetadataNamespace() {
return metadataNamespace;
}
public void setMetadataNamespace(String metadataNamespace) {
public OriginDescription setMetadataNamespace(String metadataNamespace) {
this.metadataNamespace = metadataNamespace;
return this;
}
public OriginDescription getOriginDescription() {
return originDescription;
}
public void setOriginDescription(OriginDescription originDescription) {
public OriginDescription setOriginDescription(OriginDescription originDescription) {
this.originDescription = originDescription;
return this;
}
}

31
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/graph/ProtoConverter.java
View File

@ -67,22 +67,37 @@ public class ProtoConverter implements Serializable {
private static Datasource convertDataSource(OafProtos.Oaf oaf) {
final Datasource result = new Datasource();
//setting oaf field
//TODO waiting claudio for this method
//result.setDataInfo(DataInfo.fromOaf(oaf.getDataInfo()));
//Set Oaf Fields
result.setDataInfo(ProtoUtils.mapDataInfo(oaf.getDataInfo()));
result.setLastupdatetimestamp(oaf.getLastupdatetimestamp());
//setting Entity fields
result.setId(oaf.getEntity().getId());
result.setOriginalId(oaf.getEntity().getOriginalIdList());
final OafProtos.OafEntity entity = oaf.getEntity();
//TODO waiting claudio for this method
result.setCollectedfrom(oaf.getEntity().getCollectedfromList()
result.setId(entity.getId());
result.setOriginalId(entity.getOriginalIdList());
result.setCollectedfrom(entity.getCollectedfromList()
.stream()
.map(s->new KeyValue())
.map(ProtoUtils::mapKV)
.collect(Collectors.toList()));
result.setPid(entity.getPidList()
.stream()
.map(ProtoUtils::mapStructuredProperty)
.collect(Collectors.toList()));
result.setDateofcollection(entity.getDateofcollection());
result.setDateoftransformation(entity.getDateoftransformation());
result.setExtraInfo(entity.getExtraInfoList()
.stream()
.map(ProtoUtils::mapExtraInfo)
.collect(Collectors.toList()));
return result;
}

29
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/graph/ProtoUtils.java
View File

@ -45,6 +45,35 @@ public class ProtoUtils {
.setDataInfo(sp.hasDataInfo() ? mapDataInfo(sp.getDataInfo()) : null);
}
public static ExtraInfo mapExtraInfo(FieldTypeProtos.ExtraInfo extraInfo) {
return new ExtraInfo()
.setName(extraInfo.getName())
.setTypology(extraInfo.getTypology())
.setProvenance(extraInfo.getProvenance())
.setTrust(extraInfo.getTrust())
.setValue(extraInfo.getValue());
}
public static OAIProvenance mapOAIProvenance(FieldTypeProtos.OAIProvenance oaiProvenance) {
return new OAIProvenance().setOriginDescription(mapOriginalDescription(oaiProvenance.getOriginDescription()));
}
public static OriginDescription mapOriginalDescription(FieldTypeProtos.OAIProvenance.OriginDescription originDescription) {
final OriginDescription originDescriptionResult = new OriginDescription()
.setHarvestDate(originDescription.getHarvestDate())
.setAltered(originDescription.getAltered())
.setBaseURL(originDescription.getBaseURL())
.setIdentifier(originDescription.getIdentifier())
.setDatestamp(originDescription.getDatestamp())
.setMetadataNamespace(originDescription.getMetadataNamespace());
if (originDescription.hasOriginDescription())
originDescriptionResult.setOriginDescription(mapOriginalDescription(originDescription.getOriginDescription()));
return originDescriptionResult;
}
public static Field<String> mapStringField(FieldTypeProtos.StringField s) {
return new Field<String>()
.setValue(s.getValue())

74
dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/graph/organization.json
View File

@ -1 +1,73 @@
{"kind": "entity","entity": {"type": "datasource","datasource": {"metadata": {"officialname": {"value": "CRIS UNS (Current Research Information System University of Novi Sad)"},"englishname": {"value": "CRIS UNS (Current Research Information System University of Novi Sad)"},"websiteurl": {"value": "https://cris.uns.ac.rs/"},"accessinfopackage": [{"value": "https://cris.uns.ac.rs/OAIHandlerOpenAIRECRIS"}],"namespaceprefix": {"value": "CrisUnsNoviS"},"datasourcetype": {"classid": "crissystem","classname": "CRIS System","schemeid": "dnet:datasource_typologies","schemename": "dnet:datasource_typologies"},"openairecompatibility": {"classid": "openaire-cris_1.1","classname": "OpenAIRE CRIS v1.1","schemeid": "dnet:datasourceCompatibilityLevel","schemename": "dnet:datasourceCompatibilityLevel"},"latitude": {"value": "0.0"},"longitude": {"value": "0.0"},"journal": {"issnPrinted": "","issnOnline": "","issnLinking": ""}}},"originalId": ["CRIS_UNS____::openaire"],"collectedfrom": [{"key": "","value": ""}],"dateofcollection": "2019-04-04","id": "10|CRIS_UNS____::f66f1bd369679b5b077dcdf006089556","dateoftransformation": ""},"dataInfo": {"inferred": false,"deletedbyinference": false,"trust": "0.9","provenanceaction": {"classid": "sysimport:crosswalk:entityregistry","classname": "sysimport:crosswalk:entityregistry","schemeid": "dnet:provenance_actions","schemename": "dnet:provenance_actions"}}}
{
"kind": "entity",
"entity": {
"type": "datasource",
"datasource": {
"metadata": {
"officialname": {
"value": "CRIS UNS (Current Research Information System University of Novi Sad)"
},
"englishname": {
"value": "CRIS UNS (Current Research Information System University of Novi Sad)"
},
"websiteurl": {
"value": "https://cris.uns.ac.rs/"
},
"accessinfopackage": [
{
"value": "https://cris.uns.ac.rs/OAIHandlerOpenAIRECRIS"
}
],
"namespaceprefix": {
"value": "CrisUnsNoviS"
},
"datasourcetype": {
"classid": "crissystem",
"classname": "CRIS System",
"schemeid": "dnet:datasource_typologies",
"schemename": "dnet:datasource_typologies"
},
"openairecompatibility": {
"classid": "openaire-cris_1.1",
"classname": "OpenAIRE CRIS v1.1",
"schemeid": "dnet:datasourceCompatibilityLevel",
"schemename": "dnet:datasourceCompatibilityLevel"
},
"latitude": {
"value": "0.0"
},
"longitude": {
"value": "0.0"
},
"journal": {
"issnPrinted": "",
"issnOnline": "",
"issnLinking": ""
}
}
},
"originalId": [
"CRIS_UNS____::openaire"
],
"collectedfrom": [
{
"key": "",
"value": ""
}
],
"dateofcollection": "2019-04-04",
"id": "10|CRIS_UNS____::f66f1bd369679b5b077dcdf006089556",
"dateoftransformation": ""
},
"dataInfo": {
"inferred": false,
"deletedbyinference": false,
"trust": "0.9",
"provenanceaction": {
"classid": "sysimport:crosswalk:entityregistry",
"classname": "sysimport:crosswalk:entityregistry",
"schemeid": "dnet:provenance_actions",
"schemename": "dnet:provenance_actions"
}
}
}