D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 13 Projects 2 Releases 2 Wiki Activity

dataset, orp and software mapping implemented. addition of test resources for results. implementation of tests to check the result of the mapping

This commit is contained in:
miconis 2019-10-25 12:48:25 +02:00
parent 4908165e05
commit 551eda1600
6 changed files with 105 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/graph/ProtoConverter.java
View File

@ -184,7 +184,7 @@ public class ProtoConverter implements Serializable {
return createPublication(oaf);
case "software":
return createSoftware(oaf);
case "orp":
case "other":
return createORP(oaf);
default:
throw new RuntimeException("received unknown type: " + oaf.getEntity().getResult().getMetadata().getResulttype().getClassid());
@ -192,11 +192,39 @@ public class ProtoConverter implements Serializable {
}
private static Software createSoftware(OafProtos.Oaf oaf) {
return new Software();
ResultProtos.Result.Metadata m = oaf.getEntity().getResult().getMetadata();
Software software = setOaf(new Software(), oaf);
setEntity(software, oaf);
return setResult(software, oaf)
.setDocumentationUrl(m.getDocumentationUrlList()
.stream()
.map(ProtoUtils::mapStringField)
.collect(Collectors.toList()))
.setLicense(m.getLicenseList()
.stream()
.map(ProtoUtils::mapStructuredProperty)
.collect(Collectors.toList()))
.setCodeRepositoryUrl(ProtoUtils.mapStringField(m.getCodeRepositoryUrl()))
.setProgrammingLanguage(ProtoUtils.mapQualifier(m.getProgrammingLanguage()));
}
private static OtherResearchProducts createORP(OafProtos.Oaf oaf) {
return new OtherResearchProducts();
ResultProtos.Result.Metadata m = oaf.getEntity().getResult().getMetadata();
OtherResearchProducts otherResearchProducts = setOaf(new OtherResearchProducts(), oaf);
setEntity(otherResearchProducts, oaf);
return setResult(otherResearchProducts, oaf)
.setContactperson(m.getContactpersonList()
.stream()
.map(ProtoUtils::mapStringField)
.collect(Collectors.toList()))
.setContactgroup(m.getContactgroupList()
.stream()
.map(ProtoUtils::mapStringField)
.collect(Collectors.toList()))
.setTool(m.getToolList()
.stream()
.map(ProtoUtils::mapStringField)
.collect(Collectors.toList()));
}
private static Publication createPublication(OafProtos.Oaf oaf) {
@ -206,11 +234,23 @@ public class ProtoConverter implements Serializable {
setEntity(publication, oaf);
return setResult(publication, oaf)
.setJournal(mapJournal(m.getJournal()));
}
private static Dataset createDataset(OafProtos.Oaf oaf) {
return new Dataset();
ResultProtos.Result.Metadata m = oaf.getEntity().getResult().getMetadata();
Dataset dataset = setOaf(new Dataset(), oaf);
setEntity(dataset, oaf);
return setResult(dataset, oaf)
.setStoragedate(ProtoUtils.mapStringField(m.getStoragedate()))
.setDevice(ProtoUtils.mapStringField(m.getDevice()))
.setSize(ProtoUtils.mapStringField(m.getSize()))
.setVersion(ProtoUtils.mapStringField(m.getVersion()))
.setLastmetadataupdate(ProtoUtils.mapStringField(m.getLastmetadataupdate()))
.setMetadataversionnumber(ProtoUtils.mapStringField(m.getMetadataversionnumber()))
.setGeolocation(m.getGeolocationList()
.stream()
.map(ProtoUtils::mapGeolocation)
.collect(Collectors.toList()));
}
}

7
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/graph/ProtoUtils.java
View File

@ -226,4 +226,11 @@ public class ProtoUtils {
}
public static GeoLocation mapGeolocation(ResultProtos.Result.GeoLocation geoLocation) {
return new GeoLocation()
.setPoint(geoLocation.getPoint())
.setBox(geoLocation.getBox())
.setPlace(geoLocation.getPlace());
}
}

53
dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/graph/ProtoConverterTest.java
View File

@ -1,9 +1,7 @@
package eu.dnetlib.dhp.graph;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dnetlib.dhp.schema.oaf.Datasource;
import eu.dnetlib.dhp.schema.oaf.Oaf;
import eu.dnetlib.dhp.schema.oaf.Publication;
import eu.dnetlib.dhp.schema.oaf.*;
import org.apache.commons.io.IOUtils;
import static org.junit.Assert.*;
import org.junit.Test;
@ -29,8 +27,6 @@ public class ProtoConverterTest {
ObjectMapper mapper = new ObjectMapper();
System.out.println(mapper.writeValueAsString(result));
}
@Test
@ -44,7 +40,52 @@ public class ProtoConverterTest {
Publication p = (Publication) result;
ObjectMapper mapper = new ObjectMapper();
System.out.println(mapper.writeValueAsString(result));
System.out.println(mapper.writeValueAsString(p));
}
@Test
public void convertDatasetTest() throws Exception {
final String json = IOUtils.toString(this.getClass().getResourceAsStream("/eu/dnetlib/dhp/graph/dataset.json"));
Oaf result = ProtoConverter.convert(json);
assertNotNull(result);
assertTrue(result instanceof Dataset);
Dataset d = (Dataset) result;
ObjectMapper mapper = new ObjectMapper();
System.out.println(mapper.writeValueAsString(d));
}
@Test
public void convertORPTest() throws Exception {
final String json = IOUtils.toString(this.getClass().getResourceAsStream("/eu/dnetlib/dhp/graph/orp.json"));
Oaf result = ProtoConverter.convert(json);
assertNotNull(result);
assertTrue(result instanceof OtherResearchProducts);
OtherResearchProducts orp = (OtherResearchProducts) result;
ObjectMapper mapper = new ObjectMapper();
System.out.println(mapper.writeValueAsString(orp));
}
@Test
public void convertSoftware() throws Exception {
final String json = IOUtils.toString(this.getClass().getResourceAsStream("/eu/dnetlib/dhp/graph/software.json"));
Oaf result = ProtoConverter.convert(json);
assertNotNull(result);
assertTrue(result instanceof Software);
Software s = (Software) result;
ObjectMapper mapper = new ObjectMapper();
System.out.println(mapper.writeValueAsString(s));
}

1
dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/graph/dataset.json Normal file
View File

@ -0,0 +1 @@
{"kind":1,"dataInfo":{"deletedbyinference":true,"provenanceaction":{"classid":"sysimport:actionset","classname":"sysimport:actionset","schemename":"dnet:provenanceActions","schemeid":"dnet:provenanceActions"},"inferred":true,"inferenceprovenance":"dedup-similarity-result-levenstein","invisible":false,"trust":"0.9"},"entity":{"pid":[{"qualifier":{"classid":"doi","classname":"doi","schemename":"dnet:pid_types","schemeid":"dnet:pid_types"},"value":"10.5517/cc11xr4v"}],"result":{"instance":[{"url":["http://dx.doi.org/10.5517/cc11xr4v"],"collectedfrom":{"value":"scholExplorer","key":"10|openaire____::e034d6a11054f5ade9221ebac484e864"},"hostedby":{"value":"Unknown Repository","key":"10|openaire____::55045bd2a65019fd8e6741a755395c8c"},"accessright":{"classid":"UNKNOWN","classname":"not available","schemename":"dnet:access_modes","schemeid":"dnet:access_modes"},"instancetype":{"classid":"0000","classname":"Unknown","schemename":"dnet:publication_resource","schemeid":"dnet:publication_resource"}}],"metadata":{"publisher":{"value":"Cambridge Crystallographic Data Centre"},"description":[{"value":"An entry from the Cambridge Structural Database, the world’s repository for small molecule crystal structures. The entry contains experimental data from a crystal diffraction study. The deposited dataset for this entry is freely available from the CCDC and typically includes 3D coordinates, cell parameters, space group, experimental conditions and quality measures."}],"language":{"classid":"und","classname":"Undetermined","schemename":"dent:languages","schemeid":"dent:languages"},"title":[{"qualifier":{"classid":"main title","classname":"main title","schemename":"dnet:dataCite_title","schemeid":"dnet:dataCite_title"},"value":"CCDC 980937: Experimental Crystal Structure Determination"}],"author":[{"fullname":"Yuan, Xian-You","rank":1},{"fullname":"Ou, Guang-Chuan","rank":2},{"fullname":"Yuan, Lin","rank":3},{"fullname":"Zhang, Xin-Yu","rank":4},{"fullname":"Zhang, Min","rank":5}],"resulttype":{"classid":"dataset","classname":"dataset","schemename":"dnet:result_typologies","schemeid":"dnet:result_typologies"},"relevantdate":[{"qualifier":{"classid":"dnet:date","classname":"dnet:date","schemename":"dnet:date","schemeid":"dnet:date"},"value":"2016-01-01"}],"subject":[{"qualifier":{"classid":"keyword","classname":"keyword","schemename":"dnet:subject","schemeid":"dnet:subject"},"value":"Experimental 3D Coordinates"},{"qualifier":{"classid":"keyword","classname":"keyword","schemename":"dnet:subject","schemeid":"dnet:subject"},"value":"Crystal Structure"},{"qualifier":{"classid":"keyword","classname":"keyword","schemename":"dnet:subject","schemeid":"dnet:subject"},"value":"(5-ethyl-2-methyl-2-phenyl-1,3-dioxan-5-yl)methanol"},{"qualifier":{"classid":"keyword","classname":"keyword","schemename":"dnet:subject","schemeid":"dnet:subject"},"value":"Crystal System"},{"qualifier":{"classid":"keyword","classname":"keyword","schemename":"dnet:subject","schemeid":"dnet:subject"},"value":"Space Group"},{"qualifier":{"classid":"keyword","classname":"keyword","schemename":"dnet:subject","schemeid":"dnet:subject"},"value":"Cell Parameters"},{"qualifier":{"classid":"keyword","classname":"keyword","schemename":"dnet:subject","schemeid":"dnet:subject"},"value":"Crystallography"}]}},"collectedfrom":[{"value":"scholExplorer","key":"10|openaire____::e034d6a11054f5ade9221ebac484e864"}],"dateofcollection":"2019-10-22T14:29:26+00:00","type":50,"id":"50|scholexplore::000023d184acb169596e3e6004abb421"}}

1
dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/graph/orp.json Normal file
View File

@ -0,0 +1 @@
{"kind":1,"dataInfo":{"trust":"0.9","invisible":false,"deletedbyinference":false,"inferred":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemename":"dnet:provenanceActions","schemeid":"dnet:provenanceActions"}},"entity":{"dateoftransformation":"","pid":[{"qualifier":{"classid":"doi","classname":"doi","schemename":"dnet:pid_types","schemeid":"dnet:pid_types"},"value":"10.3203/iwf/c-13106eng"},{"qualifier":{"classid":"doi","classname":"doi","schemename":"dnet:pid_types","schemeid":"dnet:pid_types"},"value":"https://doi.org/10.3203/iwf/c-13106eng"}],"originalId":["https://doi.org/10.3203/iwf/c-13106eng","http://dx.doi.org/10.3203/iwf/c-13106eng","10.3203/iwf/c-13106eng"],"oaiprovenance":{"originDescription":{"metadataNamespace":"","altered":true,"baseURL":"https%3A%2F%2Foai.datacite.org%2Foai","datestamp":"","harvestDate":"2019-04-03T17:58:12.853Z","identifier":"10.3203/iwf/c-13106eng"}},"result":{"instance":[{"hostedby":{"value":"Unknown Repository","key":"10|openaire____::55045bd2a65019fd8e6741a755395c8c"},"license":{"value":""},"url":["http://dx.doi.org/10.3203/iwf/c-13106eng"],"distributionlocation":"","dateofacceptance":{"value":"2007-01-01"},"collectedfrom":{"value":"Datacite","key":"10|openaire____::9e3be59865b2c1c335d32dae2fe7b254"},"accessright":{"classid":"UNKNOWN","classname":"UNKNOWN","schemename":"dnet:access_modes","schemeid":"dnet:access_modes"},"instancetype":{"classid":"0000","classname":"Unknown","schemename":"dnet:dataCite_resource","schemeid":"dnet:dataCite_resource"}}],"metadata":{"publisher":{"value":"IWF (Göttingen)"},"license":[{"value":""}],"description":[{"value":"A 2D animation explains the molecular structure of histone octamers. From the CD-ROM: BEREITER-HAHN, JÜRGEN / PETERS, WINFRIED S. (Frankfurt a. M.). The Cell IV - Nucleus of Life - From Gene to Proteins (C 7103)"}],"language":{"classid":"eng","classname":"English","schemename":"dnet:languages","schemeid":"dnet:languages"},"title":[{"qualifier":{"classid":"main title","classname":"main title","schemename":"dnet:dataCite_title","schemeid":"dnet:dataCite_title"},"value":"Histone Octamer"}],"author":[{"fullname":"IWF","rank":1}],"resulttype":{"classid":"other","classname":"other","schemename":"dnet:result_typologies","schemeid":"dnet:result_typologies"},"version":{"value":"None"},"storagedate":{"value":"2007"},"dateofacceptance":{"value":"2007-01-01"},"size":{"value":""},"subject":[{"qualifier":{"classid":"keyword","classname":"keyword","schemename":"dnet:subject_classification_typologies","schemeid":"dnet:subject_classification_typologies"},"value":"Life Sciences"},{"qualifier":{"classid":"keyword","classname":"keyword","schemename":"dnet:subject_classification_typologies","schemeid":"dnet:subject_classification_typologies"},"value":"histone"},{"qualifier":{"classid":"keyword","classname":"keyword","schemename":"dnet:subject_classification_typologies","schemeid":"dnet:subject_classification_typologies"},"value":"nucleosome"}]}},"collectedfrom":[{"value":"Datacite","key":"10|openaire____::9e3be59865b2c1c335d32dae2fe7b254"}],"dateofcollection":"2018-10-28T00:39:04.337Z","type":50,"id":"50|datacite____::0000228dcefe42612ec4bd83810fe348"}}

1
dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/graph/software.json Normal file
View File

File diff suppressed because one or more lines are too long