adding test classes
This commit is contained in:
parent
2fe2f0aa9e
commit
33eaacdd58
|
|
@ -63,7 +63,7 @@ public class SparkDumpOrganizationProject implements Serializable {
|
|||
conf,
|
||||
isSparkSessionManaged,
|
||||
spark -> {
|
||||
Utils.removeOutputDir(spark, outputPath);
|
||||
Utils.removeOutputDir(spark, outputPath + "/organizationProject");
|
||||
dumpRelation(spark, inputPath, outputPath);
|
||||
|
||||
});
|
||||
|
|
@ -72,6 +72,7 @@ public class SparkDumpOrganizationProject implements Serializable {
|
|||
|
||||
private static void dumpRelation(SparkSession spark, String inputPath, String outputPath) {
|
||||
Dataset<Organization> organization = Utils.readPath(spark, outputPath + "organization", Organization.class);
|
||||
|
||||
Dataset<Project> project = Utils.readPath(spark, outputPath + "project", Project.class);
|
||||
|
||||
Dataset<Relation> relation = Utils
|
||||
|
|
|
|||
|
|
@ -87,7 +87,7 @@ public class ExtendAffiliationTest {
|
|||
|
||||
spark
|
||||
.read()
|
||||
.textFile("/eu/dnetlib/dhp/oa/graph/dump/eosc/working/publication")
|
||||
.textFile(workingPath + "publication")
|
||||
.write()
|
||||
.text(workingDir.toString() + "/working/publication");
|
||||
|
||||
|
|
@ -96,25 +96,33 @@ public class ExtendAffiliationTest {
|
|||
"-outputPath", workingDir.toString() + "/",
|
||||
"-sourcePath", sourcePath,
|
||||
"-resultType", "publication",
|
||||
"-workingPath", workingDir.toString() + "/working"
|
||||
"-workingPath", workingDir.toString() + "/working/"
|
||||
});
|
||||
|
||||
/*
|
||||
affiliation relationships
|
||||
20|13811704aa70::51a6ade52065e3b371d1ae822e07f1ff -> 50|06cdd3ff4700::93859bd27121c3ee7c6ee4bfb1790cba
|
||||
|
||||
*/
|
||||
|
||||
JavaRDD<Result> tmp = sc
|
||||
.textFile(workingDir.toString() + "/working/publicationextendedaffiliation")
|
||||
.map(item -> OBJECT_MAPPER.readValue(item, Result.class));
|
||||
|
||||
Assertions.assertEquals(3, tmp.count());
|
||||
|
||||
tmp.foreach(p->System.out.println(OBJECT_MAPPER.writeValueAsString(p)));
|
||||
|
||||
Assertions
|
||||
.assertEquals(
|
||||
2,
|
||||
1,
|
||||
tmp
|
||||
.filter(r -> Optional.ofNullable(r.getAffiliation()).isPresent() && r.getAffiliation().size() > 0)
|
||||
.count());
|
||||
|
||||
Assertions
|
||||
.assertEquals(
|
||||
2,
|
||||
1,
|
||||
tmp
|
||||
.filter(r -> r.getId().equalsIgnoreCase("50|06cdd3ff4700::93859bd27121c3ee7c6ee4bfb1790cba"))
|
||||
.first()
|
||||
|
|
@ -129,7 +137,6 @@ public class ExtendAffiliationTest {
|
|||
Assertions
|
||||
.assertTrue(
|
||||
affiliations.stream().anyMatch(a -> a.getName().equalsIgnoreCase("Doris Engineering (France)")));
|
||||
Assertions.assertTrue(affiliations.stream().anyMatch(a -> a.getName().equalsIgnoreCase("RENNES METROPOLE")));
|
||||
|
||||
Affiliation organization = affiliations
|
||||
.stream()
|
||||
|
|
@ -154,56 +161,7 @@ public class ExtendAffiliationTest {
|
|||
&& p.getType().equalsIgnoreCase("ror")));
|
||||
Assertions.assertEquals(2, organization.getPid().size());
|
||||
|
||||
organization = affiliations
|
||||
.stream()
|
||||
.filter(a -> a.getId().equalsIgnoreCase("20|MetisRadboud::b58bdbe8ae5acead04fc76777d2f8017"))
|
||||
.findFirst()
|
||||
.get();
|
||||
Assertions.assertEquals("RENNES METROPOLE", organization.getName());
|
||||
Assertions.assertEquals(1, organization.getPid().size());
|
||||
Assertions
|
||||
.assertTrue(
|
||||
organization.getPid().get(0).getValue().equalsIgnoreCase("892062829")
|
||||
&& organization.getPid().get(0).getType().equalsIgnoreCase("pic"));
|
||||
|
||||
Assertions
|
||||
.assertEquals(
|
||||
1,
|
||||
tmp
|
||||
.filter(r -> r.getId().equalsIgnoreCase("50|06cdd3ff4700::cd7711c65d518859f1d87056e2c45d98"))
|
||||
.first()
|
||||
.getAffiliation()
|
||||
.size());
|
||||
Assertions
|
||||
.assertEquals(
|
||||
"MIKARE RESEARCH",
|
||||
tmp
|
||||
.filter(r -> r.getId().equalsIgnoreCase("50|06cdd3ff4700::cd7711c65d518859f1d87056e2c45d98"))
|
||||
.first()
|
||||
.getAffiliation()
|
||||
.get(0)
|
||||
.getName());
|
||||
Assertions
|
||||
.assertEquals(
|
||||
0,
|
||||
tmp
|
||||
.filter(r -> r.getId().equalsIgnoreCase("50|06cdd3ff4700::cd7711c65d518859f1d87056e2c45d98"))
|
||||
.first()
|
||||
.getAffiliation()
|
||||
.get(0)
|
||||
.getPid()
|
||||
.size());
|
||||
|
||||
Assertions
|
||||
.assertFalse(
|
||||
Optional
|
||||
.ofNullable(
|
||||
tmp
|
||||
.filter(
|
||||
r -> r.getId().equalsIgnoreCase("50|06cdd3ff4700::ff21e3c55d527fa7db171137c5fd1f1f"))
|
||||
.first()
|
||||
.getAffiliation())
|
||||
.isPresent());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -244,6 +202,7 @@ public class ExtendAffiliationTest {
|
|||
rels.foreach(r -> Assertions.assertTrue(r.getSource().startsWith("50|")));
|
||||
rels.foreach(r -> Assertions.assertTrue(r.getTarget().startsWith("20|")));
|
||||
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -160,8 +160,10 @@ public class ExtendProjectTest {
|
|||
rels.foreach(r -> Assertions.assertTrue(r.getSource().startsWith("50|")));
|
||||
rels.foreach(r -> Assertions.assertTrue(r.getTarget().startsWith("40|")));
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,118 @@
|
|||
|
||||
package eu.dnetlib.dhp.oa.graph.dump.eosc;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import eu.dnetlib.dhp.eosc.model.Relation;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.spark.SparkConf;
|
||||
import org.apache.spark.api.java.JavaRDD;
|
||||
import org.apache.spark.api.java.JavaSparkContext;
|
||||
import org.apache.spark.sql.SparkSession;
|
||||
import org.junit.jupiter.api.AfterAll;
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.HashMap;
|
||||
|
||||
/**
|
||||
* @author miriam.baglioni
|
||||
* @Date 25/10/23
|
||||
*/
|
||||
public class OrganizationProjectRelationTest {
|
||||
|
||||
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
|
||||
|
||||
private static SparkSession spark;
|
||||
|
||||
private static Path workingDir;
|
||||
|
||||
private static final Logger log = LoggerFactory
|
||||
.getLogger(OrganizationProjectRelationTest.class);
|
||||
|
||||
private static HashMap<String, String> map = new HashMap<>();
|
||||
|
||||
@BeforeAll
|
||||
public static void beforeAll() throws IOException {
|
||||
workingDir = Files
|
||||
.createTempDirectory(OrganizationProjectRelationTest.class.getSimpleName());
|
||||
log.info("using work dir {}", workingDir);
|
||||
|
||||
SparkConf conf = new SparkConf();
|
||||
conf.setAppName(OrganizationProjectRelationTest.class.getSimpleName());
|
||||
|
||||
conf.setMaster("local[*]");
|
||||
conf.set("spark.driver.host", "localhost");
|
||||
conf.set("hive.metastore.local", "true");
|
||||
conf.set("spark.ui.enabled", "false");
|
||||
conf.set("spark.sql.warehouse.dir", workingDir.toString());
|
||||
conf.set("hive.metastore.warehouse.dir", workingDir.resolve("warehouse").toString());
|
||||
|
||||
spark = SparkSession
|
||||
.builder()
|
||||
.appName(OrganizationProjectRelationTest.class.getSimpleName())
|
||||
.config(conf)
|
||||
.getOrCreate();
|
||||
}
|
||||
|
||||
@AfterAll
|
||||
public static void afterAll() throws IOException {
|
||||
FileUtils.deleteDirectory(workingDir.toFile());
|
||||
spark.stop();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void ExtendEoscResultWithProjectTest() throws Exception {
|
||||
final String sourcePath = getClass()
|
||||
.getResource("/eu/dnetlib/dhp/oa/graph/dump/eosc/input")
|
||||
.getPath();
|
||||
|
||||
final String workingPath = getClass()
|
||||
.getResource("/eu/dnetlib/dhp/oa/graph/dump/eosc/working/")
|
||||
.getPath();
|
||||
|
||||
final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
|
||||
|
||||
spark
|
||||
.read()
|
||||
.textFile(workingPath + "organization")
|
||||
.write()
|
||||
.text(workingDir.toString() + "/working/organization");
|
||||
|
||||
spark
|
||||
.read()
|
||||
.textFile(workingPath + "project")
|
||||
.write()
|
||||
.text(workingDir.toString() + "/working/project");
|
||||
|
||||
|
||||
|
||||
SparkDumpOrganizationProject.main(new String[] {
|
||||
"-isSparkSessionManaged", Boolean.FALSE.toString(),
|
||||
"-outputPath", workingDir.toString() + "/working/",
|
||||
"-sourcePath", sourcePath
|
||||
|
||||
});
|
||||
|
||||
JavaRDD<Relation> tmp = sc
|
||||
.textFile(workingDir.toString() + "/working/organizationProject")
|
||||
.map(item -> OBJECT_MAPPER.readValue(item, Relation.class));
|
||||
|
||||
Assertions.assertEquals(3, tmp.count());
|
||||
|
||||
Assertions.assertEquals(1,tmp.filter(r -> r.getSource().equalsIgnoreCase("20|chistera____::9146e9ef10640675f361d674e77bd254")).count());
|
||||
Assertions.assertEquals(2,tmp.filter(r -> r.getSource().equalsIgnoreCase("20|corda__h2020::dfe84ab5cad50d4dcfaf5bd0c86e1b64")).count());
|
||||
|
||||
Assertions.assertEquals(1,tmp.filter(r -> r.getSource().equalsIgnoreCase("20|chistera____::9146e9ef10640675f361d674e77bd254") &&
|
||||
r.getTarget().equalsIgnoreCase("40|nsf_________::d1c070f4252c32e23ccc3f4211c9c621")).count());
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -13,4 +13,8 @@
|
|||
{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"iis::document_affiliations","inferred":true,"invisible":false,"provenanceaction":{"classid":"iis","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.8847"},"lastupdatetimestamp":1658466741040,"properties":[],"relClass":"isAuthorInstitutionOf","relType":"resultOrganization","target":"20|chistera____::9146e9ef10640675f361d674e77bd254","subRelType":"affiliation","source":"50|355e65625b88::38d0ab3b2212878dee7072170f1561ee","validated":false}
|
||||
{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"iis::document_affiliations","inferred":true,"invisible":false,"provenanceaction":{"classid":"iis","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.8847"},"lastupdatetimestamp":1658466737372,"properties":[],"relClass":"isAuthorInstitutionOf","relType":"resultOrganization","target":"20|chistera____::9146e9ef10640675f361d674e77bd254","subRelType":"affiliation","source":"50|06cdd3ff4700::cd7711c65d518859f1d87056e2c45d98","validated":false}
|
||||
{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"iis::document_affiliations","inferred":true,"invisible":false,"provenanceaction":{"classid":"iis","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.8998"},"lastupdatetimestamp":1658466717565,"properties":[],"relClass":"isAuthorInstitutionOf","relType":"resultOrganization","target":"20|aka_________::04ab269cfcf6bd571b6285151ec554b5","subRelType":"affiliation","source":"50|355e65625b88::38d0ab3b2212878dee7072170f1561ee","validated":false}
|
||||
{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"iis::document_affiliations","inferred":true,"invisible":false,"provenanceaction":{"classid":"iis","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.8998"},"lastupdatetimestamp":1658466733174,"properties":[],"relClass":"isAuthorInstitutionOf","relType":"resultOrganization","target":"20|corda__h2020::dfe84ab5cad50d4dcfaf5bd0c86e1b64","subRelType":"affiliation","source":"50|06cdd3ff4700::cd7711c65d518859f1d87056e2c45d98","validated":false}
|
||||
{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"iis::document_affiliations","inferred":true,"invisible":false,"provenanceaction":{"classid":"iis","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.8998"},"lastupdatetimestamp":1658466733174,"properties":[],"relClass":"isAuthorInstitutionOf","relType":"resultOrganization","target":"20|corda__h2020::dfe84ab5cad50d4dcfaf5bd0c86e1b64","subRelType":"affiliation","source":"50|06cdd3ff4700::cd7711c65d518859f1d87056e2c45d98","validated":false}
|
||||
{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"iis::document_affiliations","inferred":true,"invisible":false,"provenanceaction":{"classid":"iis","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.8847"},"lastupdatetimestamp":1658466741040,"properties":[],"relClass":"isParticipant","relType":"resultOrganization","target":"40|nsf_________::d1c070f4252c32e23ccc3f4211c9c621","subRelType":"participation","source":"20|chistera____::9146e9ef10640675f361d674e77bd254","validated":false}
|
||||
{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"iis::document_affiliations","inferred":true,"invisible":false,"provenanceaction":{"classid":"iis","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.8847"},"lastupdatetimestamp":1658466737372,"properties":[],"relClass":"isParticipant","relType":"resultOrganization","target":"40|ukri________::081b09db1211a7b89eb3610d3160e9ba","subRelType":"participation","source":"20|corda__h2020::dfe84ab5cad50d4dcfaf5bd0c86e1b64","validated":false}
|
||||
{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"iis::document_affiliations","inferred":true,"invisible":false,"provenanceaction":{"classid":"iis","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.8998"},"lastupdatetimestamp":1658466717565,"properties":[],"relClass":"isParticipant","relType":"resultOrganization","target":"40|nsf_________::d1c070f4252c32e23ccc3f4211c9c621","subRelType":"participation","source":"20|nih_________::7523ba08be91b521952082f0c25daf5f","validated":false}
|
||||
{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"iis::document_affiliations","inferred":true,"invisible":false,"provenanceaction":{"classid":"iis","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.8998"},"lastupdatetimestamp":1658466733174,"properties":[],"relClass":"isParticipant","relType":"resultOrganization","target":"40|corda__h2020::5e49c0ee515f36e416a00cc292dfb310","subRelType":"participation","source":"20|corda__h2020::dfe84ab5cad50d4dcfaf5bd0c86e1b64","validated":false}
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
{"legalshortname":null,"legalname":"School of Computer Science, University of Birmingham","websiteurl":null,"alternativenames":[],"country":{"code":"GB","label":"United Kingdom"},"id":"20|chistera____::9146e9ef10640675f361d674e77bd254","pid":[]}
|
||||
{"legalshortname":"EVOTHINGS","legalname":"EVOTHINGS AB","websiteurl":"https://evothings.com","alternativenames":[],"country":{"code":"SE","label":"Sweden"},"id":"20|corda__h2020::dfe84ab5cad50d4dcfaf5bd0c86e1b64","pid":[{"type":"PIC","value":"922724335"}]}
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
{"id":"40|corda__h2020::5e49c0ee515f36e416a00cc292dfb310","websiteurl":null,"code":"287027","acronym":null,"title":"Advanced numerical computation methods for massive parabolic problems","startdate":"2015-09-01","enddate":"2019-08-31","callidentifier":"Academy Project LT","keywords":null,"openaccessmandateforpublications":false,"openaccessmandatefordataset":false,"subject":[],"funding":[{"shortName":"AKA","name":"Academy of Finland","jurisdiction":"FI","fundingStream":null,"funding_stream":null}],"summary":null,"granted":{"currency":"EUR","totalcost":0.0,"fundedamount":644270.0},"h2020programme":[]}
|
||||
{"id":"40|nsf_________::d1c070f4252c32e23ccc3f4211c9c621","websiteurl":null,"code":"328474","acronym":null,"title":"A novel family-based sequencing approach and dissection of regulatory networks underlying a colour polymorphism","startdate":"2020-01-01","enddate":"2022-01-31","callidentifier":"Molecular Regulatory Networks of Life (R’Life), call for Academy-funded researchers BTY","keywords":null,"openaccessmandateforpublications":false,"openaccessmandatefordataset":false,"subject":[],"funding":[{"shortName":"AKA","name":"Academy of Finland","jurisdiction":"FI","fundingStream":null,"funding_stream":null}],"summary":null,"granted":{"currency":"EUR","totalcost":0.0,"fundedamount":250000.0},"h2020programme":[]}
|
||||
{"id":"40|nserc_______::1e5e62235d094afd01cd56e65112fc63","websiteurl":null,"code":"107987","acronym":null,"title":"Structure of the upper mantle beneath Central Fennoscandian Shield from seismic anisotropy studies","startdate":"2005-02-15","enddate":"2005-12-31","callidentifier":"Apurahat tutkijainvaihtoon ja muuhun kahdenvälisiin sopimuksiin perustuvaan yhteistoimintaan LT","keywords":null,"openaccessmandateforpublications":false,"openaccessmandatefordataset":false,"subject":[],"funding":[{"shortName":"AKA","name":"Academy of Finland","jurisdiction":"FI","fundingStream":null,"funding_stream":null}],"summary":null,"granted":{"currency":"EUR","totalcost":0.0,"fundedamount":700.0},"h2020programme":[]}
|
||||
{"id":"40|corda_______::175629cbea2038ed02c85e7132fc4be2","websiteurl":null,"code":"201608","acronym":null,"title":"Symbiotic Legumes For Sustainable Food Production and Prevention of Land Deglaration in China","startdate":"2003-01-01","enddate":"2005-12-31","callidentifier":"Appropriations for development studies BY","keywords":null,"openaccessmandateforpublications":false,"openaccessmandatefordataset":false,"subject":[],"funding":[{"shortName":"AKA","name":"Academy of Finland","jurisdiction":"FI","fundingStream":null,"funding_stream":null}],"summary":null,"granted":{"currency":"EUR","totalcost":0.0,"fundedamount":300010.0},"h2020programme":[]}
|
||||
{"id":"40|ukri________::081b09db1211a7b89eb3610d3160e9ba","websiteurl":null,"code":"200618","acronym":null,"title":"Atomic Emission.","startdate":"2002-04-24","enddate":"2002-12-31","callidentifier":"Researcher exchange to Finland LT","keywords":null,"openaccessmandateforpublications":false,"openaccessmandatefordataset":false,"subject":[],"funding":[{"shortName":"AKA","name":"Academy of Finland","jurisdiction":"FI","fundingStream":null,"funding_stream":null}],"summary":null,"granted":{"currency":"EUR","totalcost":0.0,"fundedamount":2760.0},"h2020programme":[]}
|
||||
Loading…
Reference in New Issue