diff --git a/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkDedupTest.java b/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkDedupTest.java index 549988767f..2b9559867e 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkDedupTest.java +++ b/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkDedupTest.java @@ -206,11 +206,16 @@ public class SparkDedupTest implements Serializable { .load(DedupUtility.createSimRelPath(testOutputBasePath, testActionSetId, "otherresearchproduct")) .count(); - assertEquals(3082, orgs_simrel); - assertEquals(7036, pubs_simrel); + assertEquals(3076, orgs_simrel); + assertEquals(7040, pubs_simrel); assertEquals(336, sw_simrel); assertEquals(442, ds_simrel); - assertEquals(6750, orp_simrel); + assertEquals(6784, orp_simrel); +// System.out.println("org_simrel = " + orgs_simrel); +// System.out.println("pub_simrel = " + pubs_simrel); +// System.out.println("sw_simrel = " + sw_simrel); +// System.out.println("ds_simrel = " + ds_simrel); +// System.out.println("orp_simrel = " + orp_simrel); } @Test @@ -258,10 +263,14 @@ public class SparkDedupTest implements Serializable { .count(); // entities simrels supposed to be equal to the number of previous step (no rels in whitelist) - assertEquals(3082, orgs_simrel); - assertEquals(7036, pubs_simrel); + assertEquals(3076, orgs_simrel); + assertEquals(7040, pubs_simrel); assertEquals(442, ds_simrel); - assertEquals(6750, orp_simrel); + assertEquals(6784, orp_simrel); +// System.out.println("org_simrel = " + orgs_simrel); +// System.out.println("pub_simrel = " + pubs_simrel); +// System.out.println("ds_simrel = " + ds_simrel); +// System.out.println("orp_simrel = " + orp_simrel); // entities simrels to be different from the number of previous step (new simrels in the whitelist) Dataset sw_simrel = spark @@ -288,6 +297,7 @@ public class SparkDedupTest implements Serializable { .count() > 0); assertEquals(338, sw_simrel.count()); +// System.out.println("sw_simrel_count = " + sw_simrel.count()); } @@ -435,11 +445,16 @@ public class SparkDedupTest implements Serializable { .load(testOutputBasePath + "/" + testActionSetId + "/otherresearchproduct_mergerel") .count(); - assertEquals(1272, orgs_mergerel); - assertEquals(1438, pubs_mergerel); + assertEquals(1268, orgs_mergerel); + assertEquals(1444, pubs_mergerel); assertEquals(286, sw_mergerel); assertEquals(472, ds_mergerel); - assertEquals(718, orp_mergerel); + assertEquals(738, orp_mergerel); +// System.out.println("org_simrel = " + orgs_mergerel); +// System.out.println("pub_simrel = " + pubs_mergerel); +// System.out.println("sw_simrel = " + sw_mergerel); +// System.out.println("ds_simrel = " + ds_mergerel); +// System.out.println("orp_simrel = " + orp_mergerel); } @@ -483,11 +498,16 @@ public class SparkDedupTest implements Serializable { testOutputBasePath + "/" + testActionSetId + "/otherresearchproduct_deduprecord") .count(); - assertEquals(85, orgs_deduprecord); - assertEquals(65, pubs_deduprecord); + assertEquals(86, orgs_deduprecord); + assertEquals(67, pubs_deduprecord); assertEquals(49, sw_deduprecord); assertEquals(97, ds_deduprecord); - assertEquals(89, orp_deduprecord); + assertEquals(92, orp_deduprecord); +// System.out.println("org_simrel = " + orgs_deduprecord); +// System.out.println("pub_simrel = " + pubs_deduprecord); +// System.out.println("sw_simrel = " + sw_deduprecord); +// System.out.println("ds_simrel = " + ds_deduprecord); +// System.out.println("orp_simrel = " + orp_deduprecord); } @Test @@ -566,13 +586,18 @@ public class SparkDedupTest implements Serializable { .distinct() .count(); - assertEquals(896, publications); - assertEquals(838, organizations); + assertEquals(898, publications); + assertEquals(839, organizations); assertEquals(100, projects); assertEquals(100, datasource); assertEquals(198, softwares); assertEquals(389, dataset); - assertEquals(517, otherresearchproduct); + assertEquals(520, otherresearchproduct); +// System.out.println("publications = " + publications); +// System.out.println("organizations = " + organizations); +// System.out.println("softwares = " + softwares); +// System.out.println("dataset = " + dataset); +// System.out.println("orp = " + otherresearchproduct); long deletedOrgs = jsc .textFile(testDedupGraphBasePath + "/organization") @@ -627,6 +652,8 @@ public class SparkDedupTest implements Serializable { long relations = jsc.textFile(testDedupGraphBasePath + "/relation").count(); assertEquals(4860, relations); +// System.out.println("relations = " + relations); + // check deletedbyinference final Dataset mergeRels = spark diff --git a/pom.xml b/pom.xml index 603a4cf1e1..86adee9022 100644 --- a/pom.xml +++ b/pom.xml @@ -801,7 +801,7 @@ [4.0.3] [6.0.5] [3.1.6] - [4.1.7] + [4.1.12] [2.6.1] 7.5.0 4.7.2