forked from D-Net/dnet-hadoop
adjusted test assertions to reflect update ordering defined in SortableRelationKey
This commit is contained in:
parent
fd289d389c
commit
ee0b2191f8
|
@ -4,6 +4,7 @@ package eu.dnetlib.dhp.oa.provision;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.nio.file.Files;
|
import java.nio.file.Files;
|
||||||
import java.nio.file.Path;
|
import java.nio.file.Path;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
import org.apache.commons.io.FileUtils;
|
import org.apache.commons.io.FileUtils;
|
||||||
import org.apache.spark.SparkConf;
|
import org.apache.spark.SparkConf;
|
||||||
|
@ -20,8 +21,6 @@ import org.junit.jupiter.api.io.TempDir;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
||||||
|
|
||||||
import eu.dnetlib.dhp.oa.provision.model.ProvisionModelSupport;
|
import eu.dnetlib.dhp.oa.provision.model.ProvisionModelSupport;
|
||||||
import eu.dnetlib.dhp.schema.oaf.Relation;
|
import eu.dnetlib.dhp.schema.oaf.Relation;
|
||||||
|
|
||||||
|
@ -31,7 +30,8 @@ public class PrepareRelationsJobTest {
|
||||||
|
|
||||||
public static final String SUBRELTYPE = "subRelType";
|
public static final String SUBRELTYPE = "subRelType";
|
||||||
public static final String OUTCOME = "outcome";
|
public static final String OUTCOME = "outcome";
|
||||||
public static final String SUPPLEMENT = "supplement";
|
public static final String PARTICIPATION = "participation";
|
||||||
|
public static final String AFFILIATION = "affiliation";
|
||||||
|
|
||||||
private static SparkSession spark;
|
private static SparkSession spark;
|
||||||
|
|
||||||
|
@ -64,7 +64,7 @@ public class PrepareRelationsJobTest {
|
||||||
@Test
|
@Test
|
||||||
public void testRunPrepareRelationsJob(@TempDir Path testPath) throws Exception {
|
public void testRunPrepareRelationsJob(@TempDir Path testPath) throws Exception {
|
||||||
|
|
||||||
final int maxRelations = 10;
|
final int maxRelations = 20;
|
||||||
PrepareRelationsJob
|
PrepareRelationsJob
|
||||||
.main(
|
.main(
|
||||||
new String[] {
|
new String[] {
|
||||||
|
@ -73,7 +73,8 @@ public class PrepareRelationsJobTest {
|
||||||
"-outputPath", testPath.toString(),
|
"-outputPath", testPath.toString(),
|
||||||
"-relPartitions", "10",
|
"-relPartitions", "10",
|
||||||
"-relationFilter", "asd",
|
"-relationFilter", "asd",
|
||||||
"-maxRelations", String.valueOf(maxRelations)
|
"-sourceMaxRelations", String.valueOf(maxRelations),
|
||||||
|
"-targetMaxRelations", String.valueOf(maxRelations * 100)
|
||||||
});
|
});
|
||||||
|
|
||||||
Dataset<Relation> out = spark
|
Dataset<Relation> out = spark
|
||||||
|
@ -82,19 +83,31 @@ public class PrepareRelationsJobTest {
|
||||||
.as(Encoders.bean(Relation.class))
|
.as(Encoders.bean(Relation.class))
|
||||||
.cache();
|
.cache();
|
||||||
|
|
||||||
Assertions.assertEquals(10, out.count());
|
Assertions.assertEquals(maxRelations, out.count());
|
||||||
|
|
||||||
Dataset<Row> freq = out
|
Dataset<Row> freq = out
|
||||||
.toDF()
|
.toDF()
|
||||||
.cube(SUBRELTYPE)
|
.cube(SUBRELTYPE)
|
||||||
.count()
|
.count()
|
||||||
.filter((FilterFunction<Row>) value -> !value.isNullAt(0));
|
.filter((FilterFunction<Row>) value -> !value.isNullAt(0));
|
||||||
long outcome = freq.filter(freq.col(SUBRELTYPE).equalTo(OUTCOME)).collectAsList().get(0).getAs("count");
|
|
||||||
long supplement = freq.filter(freq.col(SUBRELTYPE).equalTo(SUPPLEMENT)).collectAsList().get(0).getAs("count");
|
|
||||||
|
|
||||||
Assertions.assertTrue(outcome > supplement);
|
log.info(freq.collectAsList().toString());
|
||||||
|
|
||||||
|
long outcome = getRows(freq, OUTCOME).get(0).getAs("count");
|
||||||
|
long participation = getRows(freq, PARTICIPATION).get(0).getAs("count");
|
||||||
|
long affiliation = getRows(freq, AFFILIATION).get(0).getAs("count");
|
||||||
|
|
||||||
|
Assertions.assertTrue(participation == outcome);
|
||||||
|
Assertions.assertTrue(outcome > affiliation);
|
||||||
|
Assertions.assertTrue(participation > affiliation);
|
||||||
|
|
||||||
Assertions.assertEquals(7, outcome);
|
Assertions.assertEquals(7, outcome);
|
||||||
Assertions.assertEquals(3, supplement);
|
Assertions.assertEquals(7, participation);
|
||||||
|
Assertions.assertEquals(6, affiliation);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected List<Row> getRows(Dataset<Row> freq, String col) {
|
||||||
|
return freq.filter(freq.col(SUBRELTYPE).equalTo(col)).collectAsList();
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue