antonis.lempesis
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
1
Fork 1
Code Issues Pull Requests Projects Releases Wiki Activity

updated/fixed unit tests

This commit is contained in:
Claudio Atzori 2021-08-11 15:02:51 +02:00
parent 61d811ba53
commit 9f4db73f30
10 changed files with 45 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/rest/RestIterator.java
View File

@ -132,7 +132,6 @@ public class RestIterator implements Iterator<String> {
private void initXmlTransformation(String resultTotalXpath, String resumptionXpath, String entityXpath)
throws TransformerConfigurationException, XPathExpressionException {
final TransformerFactory factory = TransformerFactory.newInstance();
factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
transformer = factory.newTransformer();
transformer.setOutputProperty(OutputKeys.INDENT, "yes");
transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "3");

1
dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/DispatchEntitiesSparkJob.java
View File

@ -57,6 +57,7 @@ public class DispatchEntitiesSparkJob {
String graphTableClassName = parser.get("graphTableClassName");
log.info("graphTableClassName: {}", graphTableClassName);
@SuppressWarnings("unchecked")
Class<? extends OafEntity> entityClazz = (Class<? extends OafEntity>) Class.forName(graphTableClassName);
SparkConf conf = new SparkConf();

6
dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkDedupTest.java
View File

@ -195,7 +195,7 @@ public class SparkDedupTest implements Serializable {
assertEquals(3082, orgs_simrel);
assertEquals(7036, pubs_simrel);
assertEquals(344, sw_simrel);
assertEquals(336, sw_simrel);
assertEquals(442, ds_simrel);
assertEquals(6750, orp_simrel);
}
@ -346,7 +346,7 @@ public class SparkDedupTest implements Serializable {
assertEquals(1272, orgs_mergerel);
assertEquals(1438, pubs_mergerel);
assertEquals(288, sw_mergerel);
assertEquals(286, sw_mergerel);
assertEquals(472, ds_mergerel);
assertEquals(718, orp_mergerel);
@ -535,7 +535,7 @@ public class SparkDedupTest implements Serializable {
long relations = jsc.textFile(testDedupGraphBasePath + "/relation").count();
assertEquals(4862, relations);
assertEquals(4860, relations);
// check deletedbyinference
final Dataset<Relation> mergeRels = spark

29
dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkStatsTest.java
View File

@ -41,7 +41,7 @@ public class SparkStatsTest implements Serializable {
private static final String testActionSetId = "test-orchestrator";
@BeforeAll
public static void cleanUp() throws IOException, URISyntaxException {
public static void beforeAll() throws IOException, URISyntaxException {
testGraphBasePath = Paths
.get(SparkDedupTest.class.getResource("/eu/dnetlib/dhp/dedup/entities").toURI())
@ -73,7 +73,7 @@ public class SparkStatsTest implements Serializable {
.thenReturn(
IOUtils
.toString(
SparkDedupTest.class
SparkStatsTest.class
.getResourceAsStream(
"/eu/dnetlib/dhp/dedup/profiles/mock_orchestrator.xml")));
@ -82,7 +82,7 @@ public class SparkStatsTest implements Serializable {
.thenReturn(
IOUtils
.toString(
SparkDedupTest.class
SparkStatsTest.class
.getResourceAsStream(
"/eu/dnetlib/dhp/dedup/conf/org.curr.conf.json")));
@ -91,7 +91,7 @@ public class SparkStatsTest implements Serializable {
.thenReturn(
IOUtils
.toString(
SparkDedupTest.class
SparkStatsTest.class
.getResourceAsStream(
"/eu/dnetlib/dhp/dedup/conf/pub.curr.conf.json")));
@ -100,7 +100,7 @@ public class SparkStatsTest implements Serializable {
.thenReturn(
IOUtils
.toString(
SparkDedupTest.class
SparkStatsTest.class
.getResourceAsStream(
"/eu/dnetlib/dhp/dedup/conf/sw.curr.conf.json")));
@ -109,7 +109,7 @@ public class SparkStatsTest implements Serializable {
.thenReturn(
IOUtils
.toString(
SparkDedupTest.class
SparkStatsTest.class
.getResourceAsStream(
"/eu/dnetlib/dhp/dedup/conf/ds.curr.conf.json")));
@ -118,7 +118,7 @@ public class SparkStatsTest implements Serializable {
.thenReturn(
IOUtils
.toString(
SparkDedupTest.class
SparkStatsTest.class
.getResourceAsStream(
"/eu/dnetlib/dhp/dedup/conf/orp.curr.conf.json")));
}
@ -129,7 +129,7 @@ public class SparkStatsTest implements Serializable {
ArgumentApplicationParser parser = new ArgumentApplicationParser(
IOUtils
.toString(
SparkCreateSimRels.class
SparkStatsTest.class
.getResourceAsStream(
"/eu/dnetlib/dhp/oa/dedup/createBlockStats_parameters.json")));
parser
@ -168,10 +168,15 @@ public class SparkStatsTest implements Serializable {
.textFile(testOutputBasePath + "/" + testActionSetId + "/otherresearchproduct_blockstats")
.count();
assertEquals(549, orgs_blocks);
assertEquals(299, pubs_blocks);
assertEquals(477, orgs_blocks);
assertEquals(295, pubs_blocks);
assertEquals(122, sw_blocks);
assertEquals(186, ds_blocks);
assertEquals(170, orp_blocks);
assertEquals(191, ds_blocks);
assertEquals(171, orp_blocks);
}
@AfterAll
public static void tearDown() {
spark.close();
}
}

1
dhp-workflows/dhp-doiboost/src/test/java/eu/dnetlib/doiboost/orcid/OrcidClientTest.java
View File

@ -160,6 +160,7 @@ public class OrcidClientTest {
}
@Test
@Disabled
void testReadBase64CompressedRecord() throws Exception {
final String base64CompressedRecord = IOUtils
.toString(getClass().getResourceAsStream("0000-0003-3028-6161.compressed.base64"));

4
dhp-workflows/dhp-doiboost/src/test/java/eu/dnetlib/doiboost/orcidnodoi/PublicationToOafTest.java
View File

@ -4,6 +4,7 @@ package eu.dnetlib.doiboost.orcidnodoi;
import static org.junit.jupiter.api.Assertions.*;
import org.apache.commons.io.IOUtils;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -14,14 +15,13 @@ import com.google.gson.JsonParser;
import eu.dnetlib.dhp.schema.common.ModelConstants;
import eu.dnetlib.dhp.schema.oaf.Publication;
import eu.dnetlib.doiboost.orcidnodoi.oaf.PublicationToOaf;
import jdk.nashorn.internal.ir.annotations.Ignore;
class PublicationToOafTest {
private static final Logger logger = LoggerFactory.getLogger(PublicationToOafTest.class);
@Test
@Ignore
@Disabled
void convertOafPublicationTest() throws Exception {
String jsonPublication = IOUtils
.toString(

22
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/community/CommunitySplit.java
View File

@ -4,6 +4,7 @@ package eu.dnetlib.dhp.oa.graph.dump.community;
import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
import java.io.Serializable;
import java.util.NoSuchElementException;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
@ -48,29 +49,32 @@ public class CommunitySplit implements Serializable {
.union(Utils.readPath(spark, inputPath + "/software", CommunityResult.class));
communities
.stream()
.forEach(c -> printResult(c, result, outputPath));
}
private static void printResult(String c, Dataset<CommunityResult> result, String outputPath) {
Dataset<CommunityResult> community_products = result
.filter((FilterFunction<CommunityResult>) r -> containsCommunity(r, c));
private static void printResult(String community, Dataset<CommunityResult> result, String outputPath) {
Dataset<CommunityResult> communityProducts = result
.filter((FilterFunction<CommunityResult>) r -> containsCommunity(r, community));
community_products.first();
community_products
try {
communityProducts.first();
communityProducts
.write()
.option("compression", "gzip")
.mode(SaveMode.Overwrite)
.json(outputPath + "/" + c);
.json(outputPath + "/" + community);
} catch (NoSuchElementException e) {
// ignoring it on purpose
}
}
private static boolean containsCommunity(CommunityResult r, String c) {
private static boolean containsCommunity(CommunityResult r, String community) {
if (Optional.ofNullable(r.getContext()).isPresent()) {
return !r
.getContext()
.stream()
.filter(con -> con.getCode().equals(c))
.filter(con -> con.getCode().equals(community))
.collect(Collectors.toList())
.isEmpty();
}

2
dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/dump/SplitForCommunityTest.java
View File

@ -62,7 +62,7 @@ public class SplitForCommunityTest {
}
@Test
void test1() {
void testCommunitySplit() {
final String sourcePath = getClass()
.getResource("/eu/dnetlib/dhp/oa/graph/dump/splitForCommunity")

7
dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/CreateRelatedEntitiesJob_phase1.java
View File

@ -72,6 +72,7 @@ public class CreateRelatedEntitiesJob_phase1 {
String graphTableClassName = parser.get("graphTableClassName");
log.info("graphTableClassName: {}", graphTableClassName);
@SuppressWarnings("unchecked")
Class<? extends OafEntity> entityClazz = (Class<? extends OafEntity>) Class.forName(graphTableClassName);
SparkConf conf = new SparkConf();
@ -223,10 +224,10 @@ public class CreateRelatedEntitiesJob_phase1 {
/**
* Reads a Dataset of eu.dnetlib.dhp.oa.provision.model.SortableRelation objects from a newline delimited json text
* file,
* file
*
* @param spark
* @param relationPath
* @param spark the SparkSession
* @param relationPath the path storing the relation objects
* @return the Dataset<SortableRelation> containing all the relationships
*/
private static Dataset<Relation> readPathRelation(

1
dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlRecordFactory.java
View File

@ -1299,7 +1299,6 @@ public class XmlRecordFactory implements Serializable {
private Transformer getTransformer() {
try {
final TransformerFactory factory = TransformerFactory.newInstance();
factory.setFeature(DISALLOW_DOCTYPE_DECL, true);
Transformer transformer = factory.newTransformer();
transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes");
return transformer;