From 7e8eff40c1660d5d277fdc36ff87d2e264e6549d Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Tue, 12 Dec 2023 08:54:15 +0100 Subject: [PATCH 1/2] [graph provision] added tests for the new model fields --- .../dhp/oa/provision/XmlIndexingJobTest.java | 34 ++- .../oa/provision/XmlRecordFactoryTest.java | 8 +- .../eu/dnetlib/dhp/oa/provision/fields.xml | 242 ++++++++++-------- .../dnetlib/dhp/oa/provision/publication.json | 10 +- .../conf/exploreTestConfig/managed-schema | 18 +- .../solr/conf/testConfig/managed-schema | 6 + 6 files changed, 183 insertions(+), 135 deletions(-) diff --git a/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/XmlIndexingJobTest.java b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/XmlIndexingJobTest.java index 6f19565784..47f0ec8c39 100644 --- a/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/XmlIndexingJobTest.java +++ b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/XmlIndexingJobTest.java @@ -4,6 +4,7 @@ package eu.dnetlib.dhp.oa.provision; import java.io.IOException; import java.io.StringReader; import java.net.URI; +import java.util.Map; import org.apache.commons.io.IOUtils; import org.apache.hadoop.io.Text; @@ -11,6 +12,7 @@ import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.SolrInputField; import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.JavaSparkContext; @@ -30,6 +32,8 @@ import eu.dnetlib.dhp.oa.provision.utils.ISLookupClient; import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException; import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; +import static org.junit.jupiter.api.Assertions.assertEquals; + @ExtendWith(MockitoExtension.class) public class XmlIndexingJobTest extends SolrTest { @@ -101,14 +105,34 @@ public class XmlIndexingJobTest extends SolrTest { new XmlIndexingJob(spark, inputPath, FORMAT, batchSize, XmlIndexingJob.OutputFormat.SOLR, null) .run(isLookupClient); - Assertions.assertEquals(0, miniCluster.getSolrClient().commit().getStatus()); + assertEquals(0, miniCluster.getSolrClient().commit().getStatus()); QueryResponse rsp = miniCluster.getSolrClient().query(new SolrQuery().add(CommonParams.Q, "*:*")); - Assertions - .assertEquals( + assertEquals( nRecord, rsp.getResults().getNumFound(), "the number of indexed records should be equal to the number of input records"); + + + rsp = miniCluster.getSolrClient().query(new SolrQuery().add(CommonParams.Q, "isgreen:true")); + assertEquals( + 0, rsp.getResults().getNumFound(), + "the number of indexed records having isgreen = true"); + + rsp = miniCluster.getSolrClient().query(new SolrQuery().add(CommonParams.Q, "openaccesscolor:bronze")); + assertEquals( + 0, rsp.getResults().getNumFound(), + "the number of indexed records having openaccesscolor = bronze"); + + rsp = miniCluster.getSolrClient().query(new SolrQuery().add(CommonParams.Q, "isindiamondjournal:true")); + assertEquals( + 0, rsp.getResults().getNumFound(), + "the number of indexed records having isindiamondjournal = true"); + + rsp = miniCluster.getSolrClient().query(new SolrQuery().add(CommonParams.Q, "publiclyfunded:true")); + assertEquals( + 0, rsp.getResults().getNumFound(), + "the number of indexed records having publiclyfunded = publiclyfunded"); } @Test @@ -126,7 +150,7 @@ public class XmlIndexingJobTest extends SolrTest { .map(s -> new SAXReader().read(new StringReader(s)).valueOf(ID_XPATH)) .distinct() .count(); - Assertions.assertEquals(nRecord, xmlIdUnique, "IDs should be unique among input records"); + assertEquals(nRecord, xmlIdUnique, "IDs should be unique among input records"); final String outputPath = workingDir.resolve("outputPath").toAbsolutePath().toString(); new XmlIndexingJob(spark, inputPath, FORMAT, batchSize, XmlIndexingJob.OutputFormat.HDFS, outputPath) @@ -142,7 +166,7 @@ public class XmlIndexingJobTest extends SolrTest { }, Encoders.STRING()) .distinct() .count(); - Assertions.assertEquals(xmlIdUnique, docIdUnique, "IDs should be unique among the output records"); + assertEquals(xmlIdUnique, docIdUnique, "IDs should be unique among the output records"); } diff --git a/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/XmlRecordFactoryTest.java b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/XmlRecordFactoryTest.java index 88bffd0e79..097d36d054 100644 --- a/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/XmlRecordFactoryTest.java +++ b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/XmlRecordFactoryTest.java @@ -32,7 +32,7 @@ public class XmlRecordFactoryTest { .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); @Test - public void testXMLRecordFactory() throws IOException, DocumentException { + void testXMLRecordFactory() throws IOException, DocumentException { final ContextMapper contextMapper = new ContextMapper(); @@ -93,6 +93,12 @@ public class XmlRecordFactoryTest { "https://osf.io/preprints/socarxiv/7vgtu/download", doc.valueOf("//*[local-name() = 'result']/fulltext[1]")); + assertEquals("true", doc.valueOf("//*[local-name() = 'result']/isgreen/text()")); + assertEquals("bronze", doc.valueOf("//*[local-name() = 'result']/openaccesscolor/text()")); + assertEquals("true", doc.valueOf("//*[local-name() = 'result']/isindiamondjournal/text()")); + assertEquals("true", doc.valueOf("//*[local-name() = 'result']/publiclyfunded/text()")); + + System.out.println(doc.asXML()); } @Test diff --git a/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/fields.xml b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/fields.xml index 0bf588a57b..513ead1dd0 100644 --- a/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/fields.xml +++ b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/fields.xml @@ -1,116 +1,134 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/publication.json b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/publication.json index 4ba20292c6..a89ec62d55 100644 --- a/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/publication.json +++ b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/publication.json @@ -1048,8 +1048,8 @@ "schemename": "dnet:dataCite_resource" }, "refereed": { - "classid": "0001", - "classname": "peerReviewed", + "classid": "0002", + "classname": "nonPeerReviewed", "schemeid": "dnet:review_levels", "schemename": "dnet:review_levels" }, @@ -1949,5 +1949,9 @@ "schemename": "dnet:provenanceActions" } } - } + }, + "isGreen": true, + "openAccessColor": "bronze", + "isInDiamondJournal": true, + "publiclyFunded": true } \ No newline at end of file diff --git a/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/solr/conf/exploreTestConfig/managed-schema b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/solr/conf/exploreTestConfig/managed-schema index 39c811f83f..28a4396190 100644 --- a/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/solr/conf/exploreTestConfig/managed-schema +++ b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/solr/conf/exploreTestConfig/managed-schema @@ -219,20 +219,6 @@ - - - - - - - - - - - - - - @@ -368,6 +354,10 @@ + + + + diff --git a/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/solr/conf/testConfig/managed-schema b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/solr/conf/testConfig/managed-schema index 977e0b2d72..962e0cc6fd 100644 --- a/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/solr/conf/testConfig/managed-schema +++ b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/solr/conf/testConfig/managed-schema @@ -365,7 +365,13 @@ + + + + + + From ff924215b8e457b3e05e98792db0c6b3bdd0fa0d Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Tue, 12 Dec 2023 11:21:30 +0100 Subject: [PATCH 2/2] [graph provision] added tests for new peerreviewed field --- .../provision/IndexRecordTransformerTest.java | 20 ++++++++++++++++++- .../dhp/oa/provision/XmlIndexingJobTest.java | 7 ++++++- .../oa/provision/XmlRecordFactoryTest.java | 2 -- .../eu/dnetlib/dhp/oa/provision/fields.xml | 3 +++ .../conf/exploreTestConfig/managed-schema | 1 + .../solr/conf/testConfig/managed-schema | 1 + 6 files changed, 30 insertions(+), 4 deletions(-) diff --git a/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/IndexRecordTransformerTest.java b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/IndexRecordTransformerTest.java index ce593cf076..e07ba1b4ea 100644 --- a/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/IndexRecordTransformerTest.java +++ b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/IndexRecordTransformerTest.java @@ -78,6 +78,22 @@ public class IndexRecordTransformerTest { testRecordTransformation(record); } + @Test + void testPeerReviewed() throws IOException, TransformerException { + + final XmlRecordFactory xmlRecordFactory = new XmlRecordFactory(contextMapper, false, + XmlConverterJob.schemaLocation); + + final Publication p = load("publication.json", Publication.class); + + final JoinedEntity je = new JoinedEntity<>(p); + final String record = xmlRecordFactory.build(je); + assertNotNull(record); + SolrInputDocument solrDoc = testRecordTransformation(record); + + assertEquals("true", solrDoc.get("peerreviewed").getValue()); + } + @Test public void testRiunet() throws IOException, TransformerException { @@ -184,7 +200,7 @@ public class IndexRecordTransformerTest { } } - private void testRecordTransformation(final String record) throws IOException, TransformerException { + private SolrInputDocument testRecordTransformation(final String record) throws IOException, TransformerException { final String fields = IOUtils.toString(getClass().getResourceAsStream("fields.xml")); final String xslt = IOUtils.toString(getClass().getResourceAsStream("layoutToRecordTransformer.xsl")); @@ -200,6 +216,8 @@ public class IndexRecordTransformerTest { Assertions.assertNotNull(xmlDoc); System.out.println(xmlDoc); + + return solrDoc; } private T load(final String fileName, final Class clazz) throws IOException { diff --git a/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/XmlIndexingJobTest.java b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/XmlIndexingJobTest.java index 47f0ec8c39..b62acbac34 100644 --- a/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/XmlIndexingJobTest.java +++ b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/XmlIndexingJobTest.java @@ -132,7 +132,12 @@ public class XmlIndexingJobTest extends SolrTest { rsp = miniCluster.getSolrClient().query(new SolrQuery().add(CommonParams.Q, "publiclyfunded:true")); assertEquals( 0, rsp.getResults().getNumFound(), - "the number of indexed records having publiclyfunded = publiclyfunded"); + "the number of indexed records having publiclyfunded = true"); + + rsp = miniCluster.getSolrClient().query(new SolrQuery().add(CommonParams.Q, "peerreviewed:true")); + assertEquals( + 0, rsp.getResults().getNumFound(), + "the number of indexed records having peerreviewed = true"); } @Test diff --git a/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/XmlRecordFactoryTest.java b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/XmlRecordFactoryTest.java index 097d36d054..de69795f88 100644 --- a/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/XmlRecordFactoryTest.java +++ b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/XmlRecordFactoryTest.java @@ -97,8 +97,6 @@ public class XmlRecordFactoryTest { assertEquals("bronze", doc.valueOf("//*[local-name() = 'result']/openaccesscolor/text()")); assertEquals("true", doc.valueOf("//*[local-name() = 'result']/isindiamondjournal/text()")); assertEquals("true", doc.valueOf("//*[local-name() = 'result']/publiclyfunded/text()")); - - System.out.println(doc.asXML()); } @Test diff --git a/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/fields.xml b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/fields.xml index 513ead1dd0..ba3fa83df1 100644 --- a/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/fields.xml +++ b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/fields.xml @@ -74,6 +74,9 @@ + + + diff --git a/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/solr/conf/exploreTestConfig/managed-schema b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/solr/conf/exploreTestConfig/managed-schema index 28a4396190..4e85ca3be1 100644 --- a/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/solr/conf/exploreTestConfig/managed-schema +++ b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/solr/conf/exploreTestConfig/managed-schema @@ -358,6 +358,7 @@ + diff --git a/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/solr/conf/testConfig/managed-schema b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/solr/conf/testConfig/managed-schema index 962e0cc6fd..e191c6223b 100644 --- a/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/solr/conf/testConfig/managed-schema +++ b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/solr/conf/testConfig/managed-schema @@ -371,6 +371,7 @@ +