Compare commits
2 Commits
07ca5edbc8
...
c001b49645
Author | SHA1 | Date |
---|---|---|
Miriam Baglioni | c001b49645 | |
Miriam Baglioni | 4342779a01 |
|
@ -79,6 +79,11 @@ public class CreatePersonAS {
|
||||||
"/eu/dnetlib/dhp/actionmanager/person/")
|
"/eu/dnetlib/dhp/actionmanager/person/")
|
||||||
.getPath();
|
.getPath();
|
||||||
|
|
||||||
|
// spark.read()
|
||||||
|
// .parquet("/Users/miriam/Downloads/part-00000-761dbe11-9f51-4275-a8fd-592649f334ef-c000.snappy.parquet")
|
||||||
|
// .write()
|
||||||
|
// .json("/tmp/part-00000.json");
|
||||||
|
|
||||||
//
|
//
|
||||||
// spark
|
// spark
|
||||||
// .read()
|
// .read()
|
||||||
|
@ -123,30 +128,30 @@ public class CreatePersonAS {
|
||||||
.filter(v -> "eu.dnetlib.dhp.schema.oaf.Person".equalsIgnoreCase(v._1().toString()))
|
.filter(v -> "eu.dnetlib.dhp.schema.oaf.Person".equalsIgnoreCase(v._1().toString()))
|
||||||
.map(value -> OBJECT_MAPPER.readValue(value._2().toString(), AtomicAction.class))
|
.map(value -> OBJECT_MAPPER.readValue(value._2().toString(), AtomicAction.class))
|
||||||
.map(aa -> ((Person) aa.getPayload()));
|
.map(aa -> ((Person) aa.getPayload()));
|
||||||
//
|
|
||||||
Assertions.assertEquals(8, people.count());
|
Assertions.assertEquals(8, people.count());
|
||||||
Assertions
|
Assertions
|
||||||
.assertEquals(
|
.assertEquals(
|
||||||
"Manuel Edelberto",
|
"Seda",
|
||||||
people
|
people
|
||||||
.filter(
|
.filter(
|
||||||
p -> p.getPid().stream().anyMatch(id -> id.getValue().equalsIgnoreCase("0000-0003-0046-4895")))
|
p -> p.getPid().stream().anyMatch(id -> id.getValue().equalsIgnoreCase("0000-0001-6544-2588")))
|
||||||
.first()
|
.first()
|
||||||
.getGivenName());
|
.getGivenName());
|
||||||
Assertions
|
Assertions
|
||||||
.assertEquals(
|
.assertEquals(
|
||||||
"Ortega Coello",
|
"Ardahan Sevgili",
|
||||||
people
|
people
|
||||||
.filter(
|
.filter(
|
||||||
p -> p.getPid().stream().anyMatch(id -> id.getValue().equalsIgnoreCase("0000-0003-0046-4895")))
|
p -> p.getPid().stream().anyMatch(id -> id.getValue().equalsIgnoreCase("0000-0001-6544-2588")))
|
||||||
.first()
|
.first()
|
||||||
.getFamilyName());
|
.getFamilyName());
|
||||||
Assertions
|
Assertions
|
||||||
.assertEquals(
|
.assertEquals(
|
||||||
1,
|
0,
|
||||||
people
|
people
|
||||||
.filter(
|
.filter(
|
||||||
p -> p.getPid().stream().anyMatch(id -> id.getValue().equalsIgnoreCase("0000-0003-0046-4895")))
|
p -> p.getPid().stream().anyMatch(id -> id.getValue().equalsIgnoreCase("0000-0001-6544-2588")))
|
||||||
.first()
|
.first()
|
||||||
.getAlternativeNames()
|
.getAlternativeNames()
|
||||||
.size());
|
.size());
|
||||||
|
@ -155,7 +160,7 @@ public class CreatePersonAS {
|
||||||
2,
|
2,
|
||||||
people
|
people
|
||||||
.filter(
|
.filter(
|
||||||
p -> p.getPid().stream().anyMatch(id -> id.getValue().equalsIgnoreCase("0000-0003-0046-4895")))
|
p -> p.getPid().stream().anyMatch(id -> id.getValue().equalsIgnoreCase("0000-0001-6544-2588")))
|
||||||
.first()
|
.first()
|
||||||
.getPid()
|
.getPid()
|
||||||
.size());
|
.size());
|
||||||
|
@ -164,18 +169,14 @@ public class CreatePersonAS {
|
||||||
.assertTrue(
|
.assertTrue(
|
||||||
people
|
people
|
||||||
.filter(
|
.filter(
|
||||||
p -> p.getPid().stream().anyMatch(id -> id.getValue().equalsIgnoreCase("0000-0003-0046-4895")))
|
p -> p.getPid().stream().anyMatch(id -> id.getValue().equalsIgnoreCase("0000-0001-6544-2588")))
|
||||||
.first()
|
.first()
|
||||||
.getPid()
|
.getPid()
|
||||||
.stream()
|
.stream()
|
||||||
.anyMatch(
|
.anyMatch(
|
||||||
p -> p.getQualifier().getClassname().equalsIgnoreCase("Scopus Author ID")
|
p -> p.getQualifier().getClassname().equalsIgnoreCase("Scopus Author ID")
|
||||||
&& p.getValue().equalsIgnoreCase("6603539671")));
|
&& p.getValue().equalsIgnoreCase("57203318816")));
|
||||||
relations
|
|
||||||
.filter(
|
|
||||||
r -> r.getSource().equalsIgnoreCase("30|orcid_______::" + DHPUtils.md5("0000-0001-6291-9619"))
|
|
||||||
&& r.getRelClass().equalsIgnoreCase(ModelConstants.RESULT_PERSON_HASAUTHORED))
|
|
||||||
.foreach(r -> System.out.println(new ObjectMapper().writeValueAsString(r)));
|
|
||||||
|
|
||||||
Assertions
|
Assertions
|
||||||
.assertEquals(
|
.assertEquals(
|
||||||
|
@ -227,8 +228,7 @@ public class CreatePersonAS {
|
||||||
r -> r.getSource().equalsIgnoreCase("30|orcid_______::" + DHPUtils.md5("0000-0001-6291-9619"))
|
r -> r.getSource().equalsIgnoreCase("30|orcid_______::" + DHPUtils.md5("0000-0001-6291-9619"))
|
||||||
&& r.getRelClass().equalsIgnoreCase(ModelConstants.PERSON_PERSON_HASCOAUTHORED))
|
&& r.getRelClass().equalsIgnoreCase(ModelConstants.PERSON_PERSON_HASCOAUTHORED))
|
||||||
.count());
|
.count());
|
||||||
Assertions.assertEquals(37, relations.count());
|
Assertions.assertEquals(38, relations.count());
|
||||||
relations.foreach(r -> System.out.println(new ObjectMapper().writeValueAsString(r)));
|
|
||||||
|
|
||||||
//check contribution from publisher papers
|
//check contribution from publisher papers
|
||||||
//the relation was merged with the other one already extracted from orcid
|
//the relation was merged with the other one already extracted from orcid
|
||||||
|
@ -242,13 +242,7 @@ public class CreatePersonAS {
|
||||||
List<KeyValue> properties = filterRelations.first().getProperties();
|
List<KeyValue> properties = filterRelations.first().getProperties();
|
||||||
Assertions.assertFalse(properties.isEmpty());
|
Assertions.assertFalse(properties.isEmpty());
|
||||||
Assertions.assertEquals(4, properties.size());
|
Assertions.assertEquals(4, properties.size());
|
||||||
properties.forEach(p-> {
|
|
||||||
try {
|
|
||||||
System.out.println(new ObjectMapper().writeValueAsString(p));
|
|
||||||
} catch (JsonProcessingException e) {
|
|
||||||
throw new RuntimeException(e);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
Assertions.assertEquals(1, properties.stream().filter(p->p.getKey().equalsIgnoreCase("corresponding")).count());
|
Assertions.assertEquals(1, properties.stream().filter(p->p.getKey().equalsIgnoreCase("corresponding")).count());
|
||||||
Assertions.assertEquals(1, properties.stream().filter(p->p.getKey().equalsIgnoreCase("corresponding") &&
|
Assertions.assertEquals(1, properties.stream().filter(p->p.getKey().equalsIgnoreCase("corresponding") &&
|
||||||
p.getValue().equalsIgnoreCase("true")).count());
|
p.getValue().equalsIgnoreCase("true")).count());
|
||||||
|
@ -259,7 +253,12 @@ public class CreatePersonAS {
|
||||||
).count());
|
).count());
|
||||||
Assertions.assertEquals(2, properties.stream().filter(p->p.getKey().equalsIgnoreCase("role")).count());
|
Assertions.assertEquals(2, properties.stream().filter(p->p.getKey().equalsIgnoreCase("role")).count());
|
||||||
|
|
||||||
|
JavaRDD<Relation> filterAffiliation = relations.filter(r -> r.getRelClass().equalsIgnoreCase(ModelConstants.ORG_PERSON_PARTICIPATES));
|
||||||
|
JavaRDD<Relation> rels = filterAffiliation;
|
||||||
|
rels.foreach(r-> System.out.println(new ObjectMapper().writeValueAsString(r)));
|
||||||
|
Assertions.assertEquals(4, filterAffiliation.count());
|
||||||
|
Assertions.assertEquals(3, filterAffiliation.filter(r -> r.getCollectedfrom().get(0).getValue().equalsIgnoreCase("OpenAIRE")).count());
|
||||||
|
Assertions.assertEquals(1, filterAffiliation.filter(r -> r.getCollectedfrom().get(0).getValue().equalsIgnoreCase("ORCID")).count());
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Binary file not shown.
|
@ -13,4 +13,4 @@
|
||||||
{"affiliationId":{"schema":"ROR","value":"https://ror.org/05qderh61"},"departmentName":"University of Calabar","endDate":"","orcid":"0000-0002-4029-3878","startDate":""}
|
{"affiliationId":{"schema":"ROR","value":"https://ror.org/05qderh61"},"departmentName":"University of Calabar","endDate":"","orcid":"0000-0002-4029-3878","startDate":""}
|
||||||
{"affiliationId":{"schema":"ROR","value":"https://ror.org/02s6k3f65"},"departmentName":"Department of Pharmaceutical Sciences","orcid":"0009-0000-8536-4302","roleTitle":""}
|
{"affiliationId":{"schema":"ROR","value":"https://ror.org/02s6k3f65"},"departmentName":"Department of Pharmaceutical Sciences","orcid":"0009-0000-8536-4302","roleTitle":""}
|
||||||
{"departmentName":"Jurídico","endDate":"","orcid":"0000-0002-9980-4564","roleTitle":"Advogada","startDate":"2018-02-08"}
|
{"departmentName":"Jurídico","endDate":"","orcid":"0000-0002-9980-4564","roleTitle":"Advogada","startDate":"2018-02-08"}
|
||||||
{"affiliationId":{"schema":"ROR","value":"https://ror.org/01dw0ab98"},"departmentName":"眼科","endDate":"2010-07-01","orcid":"0009-0002-1415-0125","roleTitle":""}
|
{"affiliationId":{"schema":"ROR","value":"https://ror.org/01dw0ab98"},"departmentName":"眼科","endDate":"2010-07-01","orcid":"0000-0001-6544-2588","roleTitle":""}
|
Loading…
Reference in New Issue