miriam.baglioni
/
dnet-hadoop
forked from D-Net/dnet-hadoop
2
0
Fork 0
Code Issues 2 Pull Requests Releases Wiki Activity

merge branch with master

This commit is contained in:
Miriam Baglioni 2020-12-14 13:35:27 +01:00
parent 9d5e6eaf3f 1506f49052
commit 24bbd30e80
6 changed files with 912 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
dhp-workflows/dhp-graph-provision/pom.xml
View File

@ -59,6 +59,10 @@
<artifactId>solr-test-framework</artifactId>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
</exclusion>
<exclusion>
<groupId>com.carrotsearch</groupId>
<artifactId>*</artifactId>
@ -140,6 +144,12 @@
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
<exclusions>
<exclusion>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
</exclusion>
</exclusions>
</dependency>

52
dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/AuthorPidTypeComparator.java Normal file
View File

@ -0,0 +1,52 @@
package eu.dnetlib.dhp.oa.provision.utils;
import java.util.Comparator;
import java.util.Optional;
import eu.dnetlib.dhp.schema.common.ModelConstants;
import eu.dnetlib.dhp.schema.common.ModelSupport;
import eu.dnetlib.dhp.schema.oaf.Qualifier;
import eu.dnetlib.dhp.schema.oaf.StructuredProperty;
public class AuthorPidTypeComparator implements Comparator<StructuredProperty> {
@Override
public int compare(StructuredProperty left, StructuredProperty right) {
String lClass = Optional
.ofNullable(left)
.map(StructuredProperty::getQualifier)
.map(Qualifier::getClassid)
.orElse(null);
String rClass = Optional
.ofNullable(right)
.map(StructuredProperty::getQualifier)
.map(Qualifier::getClassid)
.orElse(null);
if (lClass == null && rClass == null)
return 0;
if (lClass == null)
return 1;
if (rClass == null)
return -1;
if (lClass.equals(rClass))
return 0;
if (lClass.equals(ModelConstants.ORCID))
return -1;
if (rClass.equals(ModelConstants.ORCID))
return 1;
if (lClass.equals(ModelConstants.ORCID_PENDING))
return -1;
if (rClass.equals(ModelConstants.ORCID_PENDING))
return 1;
return 0;
}
}

12
dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlRecordFactory.java
View File

@ -254,6 +254,18 @@ public class XmlRecordFactory implements Serializable {
p -> p,
(p1, p2) -> p1))
.values()
.stream()
.collect(
Collectors
.groupingBy(
p -> p.getValue(),
Collectors
.mapping(
p -> p,
Collectors.minBy(new AuthorPidTypeComparator()))))
.values()
.stream()
.map(op -> op.get())
.forEach(
sp -> {
String pidType = getAuthorPidType(sp.getQualifier().getClassid());

4
dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/SolrAdminApplicationTest.java
View File

@ -6,15 +6,13 @@ import org.apache.solr.client.solrj.response.UpdateResponse;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import junit.framework.Assert;
public class SolrAdminApplicationTest extends SolrTest {
@Test
public void testPing() throws Exception {
SolrPingResponse pingResponse = miniCluster.getSolrClient().ping();
log.info("pingResponse: '{}'", pingResponse.getStatus());
Assert.assertTrue(pingResponse.getStatus() == 0);
Assertions.assertTrue(pingResponse.getStatus() == 0);
}
@Test

26
dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/XmlRecordFactoryTest.java
View File

@ -14,33 +14,33 @@ import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dnetlib.dhp.oa.provision.model.JoinedEntity;
import eu.dnetlib.dhp.oa.provision.utils.ContextMapper;
import eu.dnetlib.dhp.oa.provision.utils.XmlRecordFactory;
import eu.dnetlib.dhp.schema.oaf.Publication;
//TODO to enable it we need to update the joined_entity.json test file
@Disabled
public class XmlRecordFactoryTest {
private static final String otherDsTypeId = "scholarcomminfra,infospace,pubsrepository::mock,entityregistry,entityregistry::projects,entityregistry::repositories,websource";
private static ObjectMapper OBJECT_MAPPER = new ObjectMapper()
.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
@Test
public void testXMLRecordFactory() throws IOException, DocumentException {
String json = IOUtils.toString(getClass().getResourceAsStream("joined_entity.json"));
assertNotNull(json);
JoinedEntity je = new ObjectMapper().readValue(json, JoinedEntity.class);
assertNotNull(je);
ContextMapper contextMapper = new ContextMapper();
XmlRecordFactory xmlRecordFactory = new XmlRecordFactory(contextMapper, false, XmlConverterJob.schemaLocation,
otherDsTypeId);
String xml = xmlRecordFactory.build(je);
Publication p = OBJECT_MAPPER
.readValue(IOUtils.toString(getClass().getResourceAsStream("publication.json")), Publication.class);
String xml = xmlRecordFactory.build(new JoinedEntity<>(p));
assertNotNull(xml);
@ -48,6 +48,14 @@ public class XmlRecordFactoryTest {
assertNotNull(doc);
System.out.println(doc.asXML());
Assertions.assertEquals("0000-0001-9613-6638", doc.valueOf("//creator[@rank = '1']/@orcid"));
Assertions.assertEquals("0000-0001-9613-6639", doc.valueOf("//creator[@rank = '1']/@orcid_pending"));
Assertions.assertEquals("0000-0001-9613-9956", doc.valueOf("//creator[@rank = '2']/@orcid"));
Assertions.assertEquals("", doc.valueOf("//creator[@rank = '2']/@orcid_pending"));
// TODO add assertions based of values extracted from the XML record
}
}

820
dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/publication.json Normal file
View File

@ -0,0 +1,820 @@
{
"author": [
{
"affiliation": [],
"fullname": "Lee, Jaehyun",
"name": "Jaehyun",
"pid": [
{
"qualifier": {
"classid": "orcid",
"classname": "Open Researcher and Contributor ID",
"schemeid": "dnet:pid_types",
"schemename": "dnet:pid_types"
},
"value": "0000-0001-9613-6638"
},
{
"qualifier": {
"classid": "orcid_pending",
"classname": "Open Researcher and Contributor ID",
"schemeid": "dnet:pid_types",
"schemename": "dnet:pid_types"
},
"value": "0000-0001-9613-6639"
}
],
"rank": 1,
"surname": "Lee"
},
{
"affiliation": [],
"fullname": "Berrada, Salim",
"name": "Salim",
"pid": [
{
"qualifier": {
"classid": "orcid",
"classname": "Open Researcher and Contributor ID",
"schemeid": "dnet:pid_types",
"schemename": "dnet:pid_types"
},
"value": "0000-0001-9613-9956"
},
{
"qualifier": {
"classid": "orcid_pending",
"classname": "Open Researcher and Contributor ID",
"schemeid": "dnet:pid_types",
"schemename": "dnet:pid_types"
},
"value": "0000-0001-9613-9956"
}
],
"rank": 2,
"surname": "Berrada"
},
{
"affiliation": [],
"fullname": "Adamu-Lema, Fikru",
"name": "Fikru",
"pid": [],
"rank": 3,
"surname": "Adamu-Lema"
},
{
"affiliation": [],
"fullname": "Nagy, Nicole",
"name": "Nicole",
"pid": [],
"rank": 4,
"surname": "Nagy"
},
{
"affiliation": [],
"fullname": "Georgiev, Vihar P.",
"name": "Vihar P.",
"pid": [],
"rank": 5,
"surname": "Georgiev"
},
{
"affiliation": [],
"fullname": "Sadi, Toufik",
"name": "Toufik",
"pid": [],
"rank": 6,
"surname": "Sadi"
},
{
"affiliation": [],
"fullname": "Liang, Jie",
"name": "Jie",
"pid": [],
"rank": 7,
"surname": "Liang"
},
{
"affiliation": [],
"fullname": "Ramos, Raphael",
"name": "Raphael",
"pid": [],
"rank": 8,
"surname": "Ramos"
},
{
"affiliation": [],
"fullname": "Carrillo-Nunez, Hamilton",
"name": "Hamilton",
"pid": [],
"rank": 9,
"surname": "Carrillo-Nunez"
},
{
"affiliation": [],
"fullname": "Kalita, Dipankar",
"name": "Dipankar",
"pid": [],
"rank": 10,
"surname": "Kalita"
},
{
"affiliation": [],
"fullname": "Lilienthal, Katharina",
"name": "Katharina",
"pid": [],
"rank": 11,
"surname": "Lilienthal"
},
{
"affiliation": [],
"fullname": "Wislicenus, Marcus",
"name": "Marcus",
"pid": [],
"rank": 12,
"surname": "Wislicenus"
},
{
"affiliation": [],
"fullname": "Pandey, Reeturaj",
"name": "Reeturaj",
"pid": [],
"rank": 13,
"surname": "Pandey"
},
{
"affiliation": [],
"fullname": "Chen, Bingan",
"name": "Bingan",
"pid": [],
"rank": 14,
"surname": "Chen"
},
{
"affiliation": [],
"fullname": "Teo, Kenneth B.K.",
"name": "Kenneth B. K.",
"pid": [],
"rank": 15,
"surname": "Teo"
},
{
"affiliation": [],
"fullname": "Goncalves, Goncalo",
"name": "Goncalo",
"pid": [],
"rank": 16,
"surname": "Goncalves"
},
{
"affiliation": [],
"fullname": "Okuno, Hanako",
"name": "Hanako",
"pid": [],
"rank": 17,
"surname": "Okuno"
},
{
"affiliation": [],
"fullname": "Uhlig, Benjamin",
"name": "Benjamin",
"pid": [],
"rank": 18,
"surname": "Uhlig"
},
{
"affiliation": [],
"fullname": "Todri-Sanial, Aida",
"name": "Aida",
"pid": [],
"rank": 19,
"surname": "Todri-Sanial"
},
{
"affiliation": [],
"fullname": "Dijon",
"name": "",
"pid": [],
"rank": 20,
"surname": ""
},
{
"affiliation": [],
"fullname": "Jean",
"name": "",
"pid": [],
"rank": 21,
"surname": ""
}
],
"collectedfrom": [
{
"dataInfo": {
"deletedbyinference": false,
"inferenceprovenance": "",
"inferred": false,
"invisible": false,
"provenanceaction": {
"classid": "",
"classname": "",
"schemeid": "",
"schemename": ""
},
"trust": ""
},
"key": "10|CSC_________::a2b9ce8435390bcbfc05f3cae3948747",
"value": "VIRTA"
}
],
"context": [],
"contributor": [],
"country": [],
"coverage": [],
"dataInfo": {
"deletedbyinference": false,
"inferenceprovenance": "",
"inferred": false,
"invisible": false,
"provenanceaction": {
"classid": "sysimport:crosswalk:datasetarchive",
"classname": "sysimport:crosswalk:datasetarchive",
"schemeid": "dnet:provenanceActions",
"schemename": "dnet:provenanceActions"
},
"trust": "0.9"
},
"dateofacceptance": {
"dataInfo": {
"deletedbyinference": false,
"inferenceprovenance": "",
"inferred": false,
"invisible": false,
"provenanceaction": {
"classid": "",
"classname": "",
"schemeid": "",
"schemename": ""
},
"trust": ""
},
"value": "2018-01-01"
},
"dateofcollection": "2020-01-27T11:32:33.729Z",
"dateoftransformation": "2020-01-27T12:03:59.662Z",
"description": [],
"embargoenddate": {
"dataInfo": {
"deletedbyinference": false,
"inferenceprovenance": "",
"inferred": false,
"invisible": false,
"provenanceaction": {
"classid": "",
"classname": "",
"schemeid": "",
"schemename": ""
},
"trust": ""
},
"value": ""
},
"extraInfo": [],
"format": [],
"fulltext": [],
"id": "50|CSC_________::0000ec4dd9df012feaafa77e71a0fb4c",
"instance": [
{
"accessright": {
"classid": "OPEN",
"classname": "Open Access",
"schemeid": "dnet:access_modes",
"schemename": "dnet:access_modes"
},
"collectedfrom": {
"dataInfo": {
"deletedbyinference": false,
"inferenceprovenance": "",
"inferred": false,
"invisible": false,
"provenanceaction": {
"classid": "",
"classname": "",
"schemeid": "",
"schemename": ""
},
"trust": ""
},
"key": "10|CSC_________::a2b9ce8435390bcbfc05f3cae3948747",
"value": "VIRTA"
},
"dateofacceptance": {
"dataInfo": {
"deletedbyinference": false,
"inferenceprovenance": "",
"inferred": false,
"invisible": false,
"provenanceaction": {
"classid": "",
"classname": "",
"schemeid": "",
"schemename": ""
},
"trust": ""
},
"value": "2018-01-01"
},
"distributionlocation": "",
"hostedby": {
"dataInfo": {
"deletedbyinference": false,
"inferenceprovenance": "",
"inferred": false,
"invisible": false,
"provenanceaction": {
"classid": "",
"classname": "",
"schemeid": "",
"schemename": ""
},
"trust": ""
},
"key": "10|CSC_________::a2b9ce8435390bcbfc05f3cae3948747",
"value": "VIRTA"
},
"instancetype": {
"classid": "0001",
"classname": "Article",
"schemeid": "dnet:dataCite_resource",
"schemename": "dnet:dataCite_resource"
},
"license": {
"dataInfo": {
"deletedbyinference": false,
"inferenceprovenance": "",
"inferred": false,
"invisible": false,
"provenanceaction": {
"classid": "",
"classname": "",
"schemeid": "",
"schemename": ""
},
"trust": ""
},
"value": ""
},
"url": [
"http://juuli.fi/Record/0331473718",
"http://dx.doi.org/10.1109/TED.2018.2853550"
]
}
],
"journal": {
"conferencedate": "",
"conferenceplace": "",
"dataInfo": {
"deletedbyinference": false,
"inferenceprovenance": "",
"inferred": false,
"invisible": false,
"provenanceaction": {
"classid": "",
"classname": "",
"schemeid": "",
"schemename": ""
},
"trust": ""
},
"edition": "",
"ep": "3892",
"iss": "9",
"issnLinking": "",
"issnOnline": "",
"issnPrinted": "0018-9383",
"name": "IEEE Transactions on Electron Devices",
"sp": "3884",
"vol": "65"
},
"language": {
"classid": "en",
"classname": "en",
"schemeid": "dnet:languages",
"schemename": "dnet:languages"
},
"lastupdatetimestamp": 0,
"originalId": [
"0331473718",
"10.1109/TED.2018.2853550",
"http://juuli.fi/Record/0331473718"
],
"pid": [
{
"dataInfo": {
"deletedbyinference": false,
"inferenceprovenance": "",
"inferred": false,
"invisible": false,
"provenanceaction": {
"classid": "",
"classname": "",
"schemeid": "",
"schemename": ""
},
"trust": ""
},
"qualifier": {
"classid": "doi",
"classname": "doi",
"schemeid": "dnet:pid_types",
"schemename": "dnet:pid_types"
},
"value": "10.1109/TED.2018.2853550"
}
],
"publisher": {
"dataInfo": {
"deletedbyinference": false,
"inferenceprovenance": "",
"inferred": false,
"invisible": false,
"provenanceaction": {
"classid": "",
"classname": "",
"schemeid": "",
"schemename": ""
},
"trust": ""
},
"value": ""
},
"relevantdate": [],
"resourcetype": {
"classid": "0001",
"classname": "Article",
"schemeid": "dnet:dataCite_resource",
"schemename": "dnet:dataCite_resource"
},
"resulttype": {
"classid": "publication",
"classname": "publication",
"schemeid": "dnet:result_typologies",
"schemename": "dnet:result_typologies"
},
"source": [],
"subject": [
{
"dataInfo": {
"deletedbyinference": false,
"inferenceprovenance": "",
"inferred": false,
"invisible": false,
"provenanceaction": {
"classid": "",
"classname": "",
"schemeid": "",
"schemename": ""
},
"trust": ""
},
"qualifier": {
"classid": "http://finto.fi/okm-tieteenala/en/",
"classname": "finto",
"schemeid": "dnet:subject_classification_typologies",
"schemename": "dnet:subject_classification_typologies"
},
"value": "ta114"
},
{
"dataInfo": {
"deletedbyinference": false,
"inferenceprovenance": "",
"inferred": false,
"invisible": false,
"provenanceaction": {
"classid": "",
"classname": "",
"schemeid": "",
"schemename": ""
},
"trust": ""
},
"qualifier": {
"classid": "keyword",
"classname": "keyword",
"schemeid": "dnet:subject_classification_typologies",
"schemename": "dnet:subject_classification_typologies"
},
"value": "Conductivity"
},
{
"dataInfo": {
"deletedbyinference": false,
"inferenceprovenance": "",
"inferred": false,
"invisible": false,
"provenanceaction": {
"classid": "",
"classname": "",
"schemeid": "",
"schemename": ""
},
"trust": ""
},
"qualifier": {
"classid": "keyword",
"classname": "keyword",
"schemeid": "dnet:subject_classification_typologies",
"schemename": "dnet:subject_classification_typologies"
},
"value": "Contacts"
},
{
"dataInfo": {
"deletedbyinference": false,
"inferenceprovenance": "",
"inferred": false,
"invisible": false,
"provenanceaction": {
"classid": "",
"classname": "",
"schemeid": "",
"schemename": ""
},
"trust": ""
},
"qualifier": {
"classid": "keyword",
"classname": "keyword",
"schemeid": "dnet:subject_classification_typologies",
"schemename": "dnet:subject_classification_typologies"
},
"value": "Cu-carbon nanotubes (CNT) composites"
},
{
"dataInfo": {
"deletedbyinference": false,
"inferenceprovenance": "",
"inferred": false,
"invisible": false,
"provenanceaction": {
"classid": "",
"classname": "",
"schemeid": "",
"schemename": ""
},
"trust": ""
},
"qualifier": {
"classid": "keyword",
"classname": "keyword",
"schemeid": "dnet:subject_classification_typologies",
"schemename": "dnet:subject_classification_typologies"
},
"value": "density functional theory (DFT)"
},
{
"dataInfo": {
"deletedbyinference": false,
"inferenceprovenance": "",
"inferred": false,
"invisible": false,
"provenanceaction": {
"classid": "",
"classname": "",
"schemeid": "",
"schemename": ""
},
"trust": ""
},
"qualifier": {
"classid": "keyword",
"classname": "keyword",
"schemeid": "dnet:subject_classification_typologies",
"schemename": "dnet:subject_classification_typologies"
},
"value": "Discrete Fourier transforms"
},
{
"dataInfo": {
"deletedbyinference": false,
"inferenceprovenance": "",
"inferred": false,
"invisible": false,
"provenanceaction": {
"classid": "",
"classname": "",
"schemeid": "",
"schemename": ""
},
"trust": ""
},
"qualifier": {
"classid": "keyword",
"classname": "keyword",
"schemeid": "dnet:subject_classification_typologies",
"schemename": "dnet:subject_classification_typologies"
},
"value": "Electromigration"
},
{
"dataInfo": {
"deletedbyinference": false,
"inferenceprovenance": "",
"inferred": false,
"invisible": false,
"provenanceaction": {
"classid": "",
"classname": "",
"schemeid": "",
"schemename": ""
},
"trust": ""
},
"qualifier": {
"classid": "keyword",
"classname": "keyword",
"schemeid": "dnet:subject_classification_typologies",
"schemename": "dnet:subject_classification_typologies"
},
"value": "electromigration (EM)"
},
{
"dataInfo": {
"deletedbyinference": false,
"inferenceprovenance": "",
"inferred": false,
"invisible": false,
"provenanceaction": {
"classid": "",
"classname": "",
"schemeid": "",
"schemename": ""
},
"trust": ""
},
"qualifier": {
"classid": "keyword",
"classname": "keyword",
"schemeid": "dnet:subject_classification_typologies",
"schemename": "dnet:subject_classification_typologies"
},
"value": "electrothermal"
},
{
"dataInfo": {
"deletedbyinference": false,
"inferenceprovenance": "",
"inferred": false,
"invisible": false,
"provenanceaction": {
"classid": "",
"classname": "",
"schemeid": "",
"schemename": ""
},
"trust": ""
},
"qualifier": {
"classid": "keyword",
"classname": "keyword",
"schemeid": "dnet:subject_classification_typologies",
"schemename": "dnet:subject_classification_typologies"
},
"value": "interconnects"
},
{
"dataInfo": {
"deletedbyinference": false,
"inferenceprovenance": "",
"inferred": false,
"invisible": false,
"provenanceaction": {
"classid": "",
"classname": "",
"schemeid": "",
"schemename": ""
},
"trust": ""
},
"qualifier": {
"classid": "keyword",
"classname": "keyword",
"schemeid": "dnet:subject_classification_typologies",
"schemename": "dnet:subject_classification_typologies"
},
"value": "Lattices"
},
{
"dataInfo": {
"deletedbyinference": false,
"inferenceprovenance": "",
"inferred": false,
"invisible": false,
"provenanceaction": {
"classid": "",
"classname": "",
"schemeid": "",
"schemename": ""
},
"trust": ""
},
"qualifier": {
"classid": "keyword",
"classname": "keyword",
"schemeid": "dnet:subject_classification_typologies",
"schemename": "dnet:subject_classification_typologies"
},
"value": "multiscale simulation"
},
{
"dataInfo": {
"deletedbyinference": false,
"inferenceprovenance": "",
"inferred": false,
"invisible": false,
"provenanceaction": {
"classid": "",
"classname": "",
"schemeid": "",
"schemename": ""
},
"trust": ""
},
"qualifier": {
"classid": "keyword",
"classname": "keyword",
"schemeid": "dnet:subject_classification_typologies",
"schemename": "dnet:subject_classification_typologies"
},
"value": "Resistance"
},
{
"dataInfo": {
"deletedbyinference": false,
"inferenceprovenance": "",
"inferred": false,
"invisible": false,
"provenanceaction": {
"classid": "",
"classname": "",
"schemeid": "",
"schemename": ""
},
"trust": ""
},
"qualifier": {
"classid": "keyword",
"classname": "keyword",
"schemeid": "dnet:subject_classification_typologies",
"schemename": "dnet:subject_classification_typologies"
},
"value": "self-heating."
},
{
"dataInfo": {
"deletedbyinference": false,
"inferenceprovenance": "",
"inferred": false,
"invisible": false,
"provenanceaction": {
"classid": "",
"classname": "",
"schemeid": "",
"schemename": ""
},
"trust": ""
},
"qualifier": {
"classid": "keyword",
"classname": "keyword",
"schemeid": "dnet:subject_classification_typologies",
"schemename": "dnet:subject_classification_typologies"
},
"value": "Thermal conductivity"
}
],
"title": [
{
"dataInfo": {
"deletedbyinference": false,
"inferenceprovenance": "",
"inferred": false,
"invisible": false,
"provenanceaction": {
"classid": "",
"classname": "",
"schemeid": "",
"schemename": ""
},
"trust": ""
},
"qualifier": {
"classid": "main title",
"classname": "main title",
"schemeid": "dnet:dataCite_title",
"schemename": "dnet:dataCite_title"
},
"value": "Understanding Electromigration in Cu-CNT Composite Interconnects A Multiscale Electrothermal Simulation Study"
}
]
}