miriam.baglioni
/
dnet-hadoop
forked from D-Net/dnet-hadoop
2
0
Fork 0
Code Issues 2 Pull Requests Releases Wiki Activity

first part of the test

This commit is contained in:
Miriam Baglioni 2020-11-02 18:24:25 +01:00
parent 72fb425787
commit 43ddeedd6a
1 changed files with 31 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/clean/CleanOrcidTest.java
View File

@ -1,15 +1,16 @@
package eu.dnetlib.dhp.oa.graph.clean;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.gson.Gson;
import eu.dnetlib.dhp.oa.graph.clean.authorpids.CleanAuthorPidsSparkJob;
import eu.dnetlib.dhp.oa.graph.clean.authorpids.OrcidAuthotitative;
import eu.dnetlib.dhp.oa.graph.dump.DumpJobTest;
import eu.dnetlib.dhp.oa.graph.dump.community.CommunityMap;
import eu.dnetlib.dhp.oa.graph.clean.authorpids.PrepareResultsSparkJob;
import eu.dnetlib.dhp.oa.graph.clean.authorpids.ResultInfo;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.io.Text;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.SparkSession;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
@ -20,10 +21,8 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.List;
public class LoadOrcidSeqFileTest {
public class CleanOrcidTest {
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
@ -31,15 +30,15 @@ public class LoadOrcidSeqFileTest {
private static Path workingDir;
private static final Logger log = LoggerFactory.getLogger(LoadOrcidSeqFileTest.class);
private static final Logger log = LoggerFactory.getLogger(CleanOrcidTest.class);
@BeforeAll
public static void beforeAll() throws IOException {
workingDir = Files.createTempDirectory(LoadOrcidSeqFileTest.class.getSimpleName());
workingDir = Files.createTempDirectory(CleanOrcidTest.class.getSimpleName());
log.info("using work dir {}", workingDir);
SparkConf conf = new SparkConf();
conf.setAppName(LoadOrcidSeqFileTest.class.getSimpleName());
conf.setAppName(CleanOrcidTest.class.getSimpleName());
conf.setMaster("local[*]");
conf.set("spark.driver.host", "localhost");
@ -50,7 +49,7 @@ public class LoadOrcidSeqFileTest {
spark = SparkSession
.builder()
.appName(LoadOrcidSeqFileTest.class.getSimpleName())
.appName(CleanOrcidTest.class.getSimpleName())
.config(conf)
.getOrCreate();
}
@ -62,13 +61,26 @@ public class LoadOrcidSeqFileTest {
}
@Test
public void readSeqFile() {
public void prepareInfoTest() throws Exception {
final String sourcePath = getClass()
.getResource("/eu/dnetlib/dhp/oa/graph/clean/part-00000.gz" )
.getPath();
PrepareResultsSparkJob.main(new String[] {
"-isSparkSessionManaged", Boolean.FALSE.toString(),
"-outputPath", workingDir.toString() + "/preparedInfo",
"-inputPath", sourcePath,
"-graphTableClassName", "eu.dnetlib.dhp.schema.oaf.Publication"
});
final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
JavaRDD<ResultInfo> tmp = sc
.textFile(workingDir.toString() + "/preparedInfo")
.map(item -> OBJECT_MAPPER.readValue(item, ResultInfo.class));
JavaPairRDD<Text, Text> tmp = CleanAuthorPidsSparkJob.loadAuthoritativeOrcid(spark, getClass()
.getResource("/eu/dnetlib/dhp/oa/graph/clean/authors.seq")
.getPath());
tmp.foreach(pair -> System.out.println(pair._2().toString()));
}
}