enrico.ottonello
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

dataset, orp and software mapping implemented. addition of test resources for results. implementation of tests to check the result of the mapping

This commit is contained in:
miconis 2019-10-25 12:48:25 +02:00
parent 4908165e05
commit 551eda1600
6 changed files with 105 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/graph/ProtoConverter.java
View File

@ -80,7 +80,7 @@ public class ProtoConverter implements Serializable {
.setEcsmevalidated(mapStringField(m.getEcsmevalidated()))
.setEcnutscode(mapStringField(m.getEcnutscode()))
.setCountry(mapQualifier(m.getCountry()));
}
}
private static Datasource convertDataSource(OafProtos.Oaf oaf) {
final DatasourceProtos.Datasource.Metadata m = oaf.getEntity().getDatasource().getMetadata();
@ -184,19 +184,47 @@ public class ProtoConverter implements Serializable {
return createPublication(oaf);
case "software":
return createSoftware(oaf);
case "orp":
case "other":
return createORP(oaf);
default:
throw new RuntimeException("received unknown type :" + oaf.getEntity().getResult().getMetadata().getResulttype().getClassid());
throw new RuntimeException("received unknown type: " + oaf.getEntity().getResult().getMetadata().getResulttype().getClassid());
}
}
private static Software createSoftware(OafProtos.Oaf oaf) {
return new Software();
ResultProtos.Result.Metadata m = oaf.getEntity().getResult().getMetadata();
Software software = setOaf(new Software(), oaf);
setEntity(software, oaf);
return setResult(software, oaf)
.setDocumentationUrl(m.getDocumentationUrlList()
.stream()
.map(ProtoUtils::mapStringField)
.collect(Collectors.toList()))
.setLicense(m.getLicenseList()
.stream()
.map(ProtoUtils::mapStructuredProperty)
.collect(Collectors.toList()))
.setCodeRepositoryUrl(ProtoUtils.mapStringField(m.getCodeRepositoryUrl()))
.setProgrammingLanguage(ProtoUtils.mapQualifier(m.getProgrammingLanguage()));
}
private static OtherResearchProducts createORP(OafProtos.Oaf oaf) {
return new OtherResearchProducts();
ResultProtos.Result.Metadata m = oaf.getEntity().getResult().getMetadata();
OtherResearchProducts otherResearchProducts = setOaf(new OtherResearchProducts(), oaf);
setEntity(otherResearchProducts, oaf);
return setResult(otherResearchProducts, oaf)
.setContactperson(m.getContactpersonList()
.stream()
.map(ProtoUtils::mapStringField)
.collect(Collectors.toList()))
.setContactgroup(m.getContactgroupList()
.stream()
.map(ProtoUtils::mapStringField)
.collect(Collectors.toList()))
.setTool(m.getToolList()
.stream()
.map(ProtoUtils::mapStringField)
.collect(Collectors.toList()));
}
private static Publication createPublication(OafProtos.Oaf oaf) {
@ -206,11 +234,23 @@ public class ProtoConverter implements Serializable {
setEntity(publication, oaf);
return setResult(publication, oaf)
.setJournal(mapJournal(m.getJournal()));
}
private static Dataset createDataset(OafProtos.Oaf oaf) {
return new Dataset();
ResultProtos.Result.Metadata m = oaf.getEntity().getResult().getMetadata();
Dataset dataset = setOaf(new Dataset(), oaf);
setEntity(dataset, oaf);
return setResult(dataset, oaf)
.setStoragedate(ProtoUtils.mapStringField(m.getStoragedate()))
.setDevice(ProtoUtils.mapStringField(m.getDevice()))
.setSize(ProtoUtils.mapStringField(m.getSize()))
.setVersion(ProtoUtils.mapStringField(m.getVersion()))
.setLastmetadataupdate(ProtoUtils.mapStringField(m.getLastmetadataupdate()))
.setMetadataversionnumber(ProtoUtils.mapStringField(m.getMetadataversionnumber()))
.setGeolocation(m.getGeolocationList()
.stream()
.map(ProtoUtils::mapGeolocation)
.collect(Collectors.toList()));
}
}

7
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/graph/ProtoUtils.java
View File

@ -226,4 +226,11 @@ public class ProtoUtils {
}
public static GeoLocation mapGeolocation(ResultProtos.Result.GeoLocation geoLocation) {
return new GeoLocation()
.setPoint(geoLocation.getPoint())
.setBox(geoLocation.getBox())
.setPlace(geoLocation.getPlace());
}
}

53
dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/graph/ProtoConverterTest.java
View File

@ -1,9 +1,7 @@
package eu.dnetlib.dhp.graph;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dnetlib.dhp.schema.oaf.Datasource;
import eu.dnetlib.dhp.schema.oaf.Oaf;
import eu.dnetlib.dhp.schema.oaf.Publication;
import eu.dnetlib.dhp.schema.oaf.*;
import org.apache.commons.io.IOUtils;
import static org.junit.Assert.*;
import org.junit.Test;
@ -29,8 +27,6 @@ public class ProtoConverterTest {
ObjectMapper mapper = new ObjectMapper();
System.out.println(mapper.writeValueAsString(result));
}
@Test
@ -44,7 +40,52 @@ public class ProtoConverterTest {
Publication p = (Publication) result;
ObjectMapper mapper = new ObjectMapper();
System.out.println(mapper.writeValueAsString(result));
System.out.println(mapper.writeValueAsString(p));
}
@Test
public void convertDatasetTest() throws Exception {
final String json = IOUtils.toString(this.getClass().getResourceAsStream("/eu/dnetlib/dhp/graph/dataset.json"));
Oaf result = ProtoConverter.convert(json);
assertNotNull(result);
assertTrue(result instanceof Dataset);
Dataset d = (Dataset) result;
ObjectMapper mapper = new ObjectMapper();
System.out.println(mapper.writeValueAsString(d));
}
@Test
public void convertORPTest() throws Exception {
final String json = IOUtils.toString(this.getClass().getResourceAsStream("/eu/dnetlib/dhp/graph/orp.json"));
Oaf result = ProtoConverter.convert(json);
assertNotNull(result);
assertTrue(result instanceof OtherResearchProducts);
OtherResearchProducts orp = (OtherResearchProducts) result;
ObjectMapper mapper = new ObjectMapper();
System.out.println(mapper.writeValueAsString(orp));
}
@Test
public void convertSoftware() throws Exception {
final String json = IOUtils.toString(this.getClass().getResourceAsStream("/eu/dnetlib/dhp/graph/software.json"));
Oaf result = ProtoConverter.convert(json);
assertNotNull(result);
assertTrue(result instanceof Software);
Software s = (Software) result;
ObjectMapper mapper = new ObjectMapper();
System.out.println(mapper.writeValueAsString(s));
}

1
dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/graph/dataset.json Normal file
View File

@ -0,0 +1 @@
{"kind":1,"dataInfo":{"deletedbyinference":true,"provenanceaction":{"classid":"sysimport:actionset","classname":"sysimport:actionset","schemename":"dnet:provenanceActions","schemeid":"dnet:provenanceActions"},"inferred":true,"inferenceprovenance":"dedup-similarity-result-levenstein","invisible":false,"trust":"0.9"},"entity":{"pid":[{"qualifier":{"classid":"doi","classname":"doi","schemename":"dnet:pid_types","schemeid":"dnet:pid_types"},"value":"10.5517/cc11xr4v"}],"result":{"instance":[{"url":["http://dx.doi.org/10.5517/cc11xr4v"],"collectedfrom":{"value":"scholExplorer","key":"10|openaire____::e034d6a11054f5ade9221ebac484e864"},"hostedby":{"value":"Unknown Repository","key":"10|openaire____::55045bd2a65019fd8e6741a755395c8c"},"accessright":{"classid":"UNKNOWN","classname":"not available","schemename":"dnet:access_modes","schemeid":"dnet:access_modes"},"instancetype":{"classid":"0000","classname":"Unknown","schemename":"dnet:publication_resource","schemeid":"dnet:publication_resource"}}],"metadata":{"publisher":{"value":"Cambridge Crystallographic Data Centre"},"description":[{"value":"An entry from the Cambridge Structural Database, the world’s repository for small molecule crystal structures. The entry contains experimental data from a crystal diffraction study. The deposited dataset for this entry is freely available from the CCDC and typically includes 3D coordinates, cell parameters, space group, experimental conditions and quality measures."}],"language":{"classid":"und","classname":"Undetermined","schemename":"dent:languages","schemeid":"dent:languages"},"title":[{"qualifier":{"classid":"main title","classname":"main title","schemename":"dnet:dataCite_title","schemeid":"dnet:dataCite_title"},"value":"CCDC 980937: Experimental Crystal Structure Determination"}],"author":[{"fullname":"Yuan, Xian-You","rank":1},{"fullname":"Ou, Guang-Chuan","rank":2},{"fullname":"Yuan, Lin","rank":3},{"fullname":"Zhang, Xin-Yu","rank":4},{"fullname":"Zhang, Min","rank":5}],"resulttype":{"classid":"dataset","classname":"dataset","schemename":"dnet:result_typologies","schemeid":"dnet:result_typologies"},"relevantdate":[{"qualifier":{"classid":"dnet:date","classname":"dnet:date","schemename":"dnet:date","schemeid":"dnet:date"},"value":"2016-01-01"}],"subject":[{"qualifier":{"classid":"keyword","classname":"keyword","schemename":"dnet:subject","schemeid":"dnet:subject"},"value":"Experimental 3D Coordinates"},{"qualifier":{"classid":"keyword","classname":"keyword","schemename":"dnet:subject","schemeid":"dnet:subject"},"value":"Crystal Structure"},{"qualifier":{"classid":"keyword","classname":"keyword","schemename":"dnet:subject","schemeid":"dnet:subject"},"value":"(5-ethyl-2-methyl-2-phenyl-1,3-dioxan-5-yl)methanol"},{"qualifier":{"classid":"keyword","classname":"keyword","schemename":"dnet:subject","schemeid":"dnet:subject"},"value":"Crystal System"},{"qualifier":{"classid":"keyword","classname":"keyword","schemename":"dnet:subject","schemeid":"dnet:subject"},"value":"Space Group"},{"qualifier":{"classid":"keyword","classname":"keyword","schemename":"dnet:subject","schemeid":"dnet:subject"},"value":"Cell Parameters"},{"qualifier":{"classid":"keyword","classname":"keyword","schemename":"dnet:subject","schemeid":"dnet:subject"},"value":"Crystallography"}]}},"collectedfrom":[{"value":"scholExplorer","key":"10|openaire____::e034d6a11054f5ade9221ebac484e864"}],"dateofcollection":"2019-10-22T14:29:26+00:00","type":50,"id":"50|scholexplore::000023d184acb169596e3e6004abb421"}}

1
dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/graph/orp.json Normal file
View File

@ -0,0 +1 @@
{"kind":1,"dataInfo":{"trust":"0.9","invisible":false,"deletedbyinference":false,"inferred":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemename":"dnet:provenanceActions","schemeid":"dnet:provenanceActions"}},"entity":{"dateoftransformation":"","pid":[{"qualifier":{"classid":"doi","classname":"doi","schemename":"dnet:pid_types","schemeid":"dnet:pid_types"},"value":"10.3203/iwf/c-13106eng"},{"qualifier":{"classid":"doi","classname":"doi","schemename":"dnet:pid_types","schemeid":"dnet:pid_types"},"value":"https://doi.org/10.3203/iwf/c-13106eng"}],"originalId":["https://doi.org/10.3203/iwf/c-13106eng","http://dx.doi.org/10.3203/iwf/c-13106eng","10.3203/iwf/c-13106eng"],"oaiprovenance":{"originDescription":{"metadataNamespace":"","altered":true,"baseURL":"https%3A%2F%2Foai.datacite.org%2Foai","datestamp":"","harvestDate":"2019-04-03T17:58:12.853Z","identifier":"10.3203/iwf/c-13106eng"}},"result":{"instance":[{"hostedby":{"value":"Unknown Repository","key":"10|openaire____::55045bd2a65019fd8e6741a755395c8c"},"license":{"value":""},"url":["http://dx.doi.org/10.3203/iwf/c-13106eng"],"distributionlocation":"","dateofacceptance":{"value":"2007-01-01"},"collectedfrom":{"value":"Datacite","key":"10|openaire____::9e3be59865b2c1c335d32dae2fe7b254"},"accessright":{"classid":"UNKNOWN","classname":"UNKNOWN","schemename":"dnet:access_modes","schemeid":"dnet:access_modes"},"instancetype":{"classid":"0000","classname":"Unknown","schemename":"dnet:dataCite_resource","schemeid":"dnet:dataCite_resource"}}],"metadata":{"publisher":{"value":"IWF (Göttingen)"},"license":[{"value":""}],"description":[{"value":"A 2D animation explains the molecular structure of histone octamers. From the CD-ROM: BEREITER-HAHN, JÜRGEN / PETERS, WINFRIED S. (Frankfurt a. M.). The Cell IV - Nucleus of Life - From Gene to Proteins (C 7103)"}],"language":{"classid":"eng","classname":"English","schemename":"dnet:languages","schemeid":"dnet:languages"},"title":[{"qualifier":{"classid":"main title","classname":"main title","schemename":"dnet:dataCite_title","schemeid":"dnet:dataCite_title"},"value":"Histone Octamer"}],"author":[{"fullname":"IWF","rank":1}],"resulttype":{"classid":"other","classname":"other","schemename":"dnet:result_typologies","schemeid":"dnet:result_typologies"},"version":{"value":"None"},"storagedate":{"value":"2007"},"dateofacceptance":{"value":"2007-01-01"},"size":{"value":""},"subject":[{"qualifier":{"classid":"keyword","classname":"keyword","schemename":"dnet:subject_classification_typologies","schemeid":"dnet:subject_classification_typologies"},"value":"Life Sciences"},{"qualifier":{"classid":"keyword","classname":"keyword","schemename":"dnet:subject_classification_typologies","schemeid":"dnet:subject_classification_typologies"},"value":"histone"},{"qualifier":{"classid":"keyword","classname":"keyword","schemename":"dnet:subject_classification_typologies","schemeid":"dnet:subject_classification_typologies"},"value":"nucleosome"}]}},"collectedfrom":[{"value":"Datacite","key":"10|openaire____::9e3be59865b2c1c335d32dae2fe7b254"}],"dateofcollection":"2018-10-28T00:39:04.337Z","type":50,"id":"50|datacite____::0000228dcefe42612ec4bd83810fe348"}}

1
dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/graph/software.json Normal file
View File

File diff suppressed because one or more lines are too long