dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/SolrConfigExploreTest.java

@@ -0,0 +1,126 @@
+
+package eu.dnetlib.dhp.oa.provision;
+
+import java.io.IOException;
+import java.net.URI;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.spark.SparkConf;
+import org.apache.spark.sql.SparkSession;
+import org.junit.jupiter.api.*;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+import eu.dnetlib.dhp.oa.provision.model.SerializableSolrInputDocument;
+import eu.dnetlib.dhp.oa.provision.utils.ISLookupClient;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+
+@ExtendWith(MockitoExtension.class)
+public class SolrConfigExploreTest extends SolrExploreTest {
+
+    protected static SparkSession spark;
+
+    private static final Integer batchSize = 100;
+
+    @Mock
+    private ISLookUpService isLookUpService;
+
+    @Mock
+    private ISLookupClient isLookupClient;
+
+    @BeforeEach
+    public void prepareMocks() throws ISLookUpException, IOException {
+        isLookupClient.setIsLookup(isLookUpService);
+
+        int solrPort = URI.create("http://" + miniCluster.getZkClient().getZkServerAddress()).getPort();
+
+        Mockito
+                .when(isLookupClient.getDsId(Mockito.anyString()))
+                .thenReturn("313f0381-23b6-466f-a0b8-c72a9679ac4b_SW5kZXhEU1Jlc291cmNlcy9JbmRleERTUmVzb3VyY2VUeXBl");
+        Mockito.when(isLookupClient.getZkHost()).thenReturn(String.format("127.0.0.1:%s/solr", solrPort));
+        Mockito
+                .when(isLookupClient.getLayoutSource(Mockito.anyString()))
+                .thenReturn(IOUtils.toString(getClass().getResourceAsStream("fields.xml")));
+        Mockito
+                .when(isLookupClient.getLayoutTransformer())
+                .thenReturn(IOUtils.toString(getClass().getResourceAsStream("layoutToRecordTransformer.xsl")));
+    }
+
+    @BeforeAll
+    public static void before() {
+
+        SparkConf conf = new SparkConf();
+        conf.setAppName(XmlIndexingJobTest.class.getSimpleName());
+        conf.registerKryoClasses(new Class[] {
+                SerializableSolrInputDocument.class
+        });
+
+        conf.setMaster("local[1]");
+        conf.set("spark.driver.host", "localhost");
+        conf.set("hive.metastore.local", "true");
+        conf.set("spark.ui.enabled", "false");
+        conf.set("spark.sql.warehouse.dir", workingDir.resolve("spark").toString());
+
+        spark = SparkSession
+                .builder()
+                .appName(XmlIndexingJobTest.class.getSimpleName())
+                .config(conf)
+                .getOrCreate();
+
+
+    }
+
+    @AfterAll
+    public static void tearDown() {
+        spark.stop();
+    }
+
+    @Test
+    public void testSolrConfig() throws Exception {
+
+        String inputPath = "src/test/resources/eu/dnetlib/dhp/oa/provision/xml";
+
+        new XmlIndexingJob(spark, inputPath, FORMAT, batchSize, XmlIndexingJob.OutputFormat.SOLR, null).run(isLookupClient);
+        Assertions.assertEquals(0, miniCluster.getSolrClient().commit().getStatus());
+
+        String[] queryStrings = {
+                "cancer",
+                "graph",
+                "graphs"
+        };
+
+        for (String q : queryStrings) {
+            SolrQuery query = new SolrQuery();
+            query.setRequestHandler("/exploreSearch");
+            query.add(CommonParams.Q, q);
+            query.set("debugQuery", "on");
+
+            log.info("Submit query to Solr with params: {}", query.toString());
+            QueryResponse rsp = miniCluster.getSolrClient().query(query);
+//            System.out.println(rsp.getHighlighting());
+//            System.out.println(rsp.getExplainMap());
+
+            for (SolrDocument doc : rsp.getResults()) {
+                System.out.println(
+                        doc.get("score") + "\t" +
+                        doc.get("__indexrecordidentifier") + "\t" +
+                        doc.get("resultidentifier") + "\t" +
+                        doc.get("resultauthor") + "\t" +
+                        doc.get("resultacceptanceyear") + "\t" +
+                        doc.get("resultsubject") + "\t" +
+                        doc.get("resulttitle") + "\t" +
+                        doc.get("relprojectname") + "\t" +
+                        doc.get("resultdescription") + "\t" +
+                        doc.get("__all") + "\t"
+                );
+            }
+        }
+    }
+}

dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/SolrConfigTest.java

@@ -0,0 +1,131 @@
+
+package eu.dnetlib.dhp.oa.provision;
+
+import java.io.IOException;
+import java.io.StringReader;
+import java.net.URI;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.io.Text;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrInputField;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.spark.SparkConf;
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.api.java.function.MapFunction;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Encoders;
+import org.apache.spark.sql.SparkSession;
+import org.dom4j.io.SAXReader;
+import org.junit.jupiter.api.*;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+import eu.dnetlib.dhp.oa.provision.model.SerializableSolrInputDocument;
+import eu.dnetlib.dhp.oa.provision.utils.ISLookupClient;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+
+@ExtendWith(MockitoExtension.class)
+public class SolrConfigTest extends SolrTest {
+
+    protected static SparkSession spark;
+
+    private static final Integer batchSize = 100;
+
+    @Mock
+    private ISLookUpService isLookUpService;
+
+    @Mock
+    private ISLookupClient isLookupClient;
+
+    @BeforeEach
+    public void prepareMocks() throws ISLookUpException, IOException {
+        isLookupClient.setIsLookup(isLookUpService);
+
+        int solrPort = URI.create("http://" + miniCluster.getZkClient().getZkServerAddress()).getPort();
+
+        Mockito
+                .when(isLookupClient.getDsId(Mockito.anyString()))
+                .thenReturn("313f0381-23b6-466f-a0b8-c72a9679ac4b_SW5kZXhEU1Jlc291cmNlcy9JbmRleERTUmVzb3VyY2VUeXBl");
+        Mockito.when(isLookupClient.getZkHost()).thenReturn(String.format("127.0.0.1:%s/solr", solrPort));
+        Mockito
+                .when(isLookupClient.getLayoutSource(Mockito.anyString()))
+                .thenReturn(IOUtils.toString(getClass().getResourceAsStream("fields.xml")));
+        Mockito
+                .when(isLookupClient.getLayoutTransformer())
+                .thenReturn(IOUtils.toString(getClass().getResourceAsStream("layoutToRecordTransformer.xsl")));
+    }
+
+    @BeforeAll
+    public static void before() {
+
+        SparkConf conf = new SparkConf();
+        conf.setAppName(XmlIndexingJobTest.class.getSimpleName());
+        conf.registerKryoClasses(new Class[] {
+                SerializableSolrInputDocument.class
+        });
+
+        conf.setMaster("local[1]");
+        conf.set("spark.driver.host", "localhost");
+        conf.set("hive.metastore.local", "true");
+        conf.set("spark.ui.enabled", "false");
+        conf.set("spark.sql.warehouse.dir", workingDir.resolve("spark").toString());
+
+        spark = SparkSession
+                .builder()
+                .appName(XmlIndexingJobTest.class.getSimpleName())
+                .config(conf)
+                .getOrCreate();
+
+
+    }
+
+    @AfterAll
+    public static void tearDown() {
+        spark.stop();
+    }
+
+    @Test
+    public void testSolrConfig() throws Exception {
+
+        String inputPath = "src/test/resources/eu/dnetlib/dhp/oa/provision/xml";
+
+        new XmlIndexingJob(spark, inputPath, FORMAT, batchSize, XmlIndexingJob.OutputFormat.SOLR, null).run(isLookupClient);
+        Assertions.assertEquals(0, miniCluster.getSolrClient().commit().getStatus());
+
+        String[] queryStrings = {
+            "cancer",
+            "graph",
+            "graphs"
+        };
+
+        for (String q : queryStrings) {
+            SolrQuery query = new SolrQuery();
+            query.add(CommonParams.Q, q);
+
+            log.info("Submit query to Solr with params: {}", query.toString());
+            QueryResponse rsp = miniCluster.getSolrClient().query(query);
+
+            for (SolrDocument doc : rsp.getResults()) {
+                System.out.println(
+                        doc.get("score") + "\t" +
+                        doc.get("__indexrecordidentifier") + "\t" +
+                        doc.get("resultidentifier") + "\t" +
+                        doc.get("resultauthor") + "\t" +
+                        doc.get("resultacceptanceyear") + "\t" +
+                        doc.get("resultsubject") + "\t" +
+                        doc.get("resulttitle") + "\t" +
+                        doc.get("relprojectname") + "\t" +
+                        doc.get("resultdescription") + "\t" +
+                        doc.get("__all") + "\t"
+                );
+            }
+        }
+    }
+}

dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/SolrExploreTest.java

@@ -0,0 +1,109 @@
+
+package eu.dnetlib.dhp.oa.provision;
+
+import java.io.File;
+import java.nio.file.Path;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.solr.client.solrj.embedded.JettyConfig;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.ConfigSetAdminRequest;
+import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.cloud.MiniSolrCloudCluster;
+import org.apache.solr.common.params.CollectionParams;
+import org.apache.solr.common.params.CoreAdminParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.io.TempDir;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public abstract class SolrExploreTest {
+
+    protected static final Logger log = LoggerFactory.getLogger(SolrTest.class);
+
+    protected static final String FORMAT = "test";
+    protected static final String DEFAULT_COLLECTION = FORMAT + "-index-openaire";
+    protected static final String CONFIG_NAME = "testConfig";
+
+    protected static MiniSolrCloudCluster miniCluster;
+
+    @TempDir
+    public static Path workingDir;
+
+    @BeforeAll
+    public static void setup() throws Exception {
+
+        // random unassigned HTTP port
+        final int jettyPort = 0;
+        final JettyConfig jettyConfig = JettyConfig.builder().setPort(jettyPort).build();
+
+        log.info(String.format("working directory: %s", workingDir.toString()));
+        System.setProperty("solr.log.dir", workingDir.resolve("logs").toString());
+
+        // create a MiniSolrCloudCluster instance
+        miniCluster = new MiniSolrCloudCluster(2, workingDir.resolve("solr"), jettyConfig);
+
+        // Upload Solr configuration directory to ZooKeeper
+        String solrZKConfigDir = "src/test/resources/eu/dnetlib/dhp/oa/provision/solr/conf/exploreTestConfig";
+        File configDir = new File(solrZKConfigDir);
+
+        miniCluster.uploadConfigSet(configDir.toPath(), CONFIG_NAME);
+
+        // override settings in the solrconfig include
+        System.setProperty("solr.tests.maxBufferedDocs", "100000");
+        System.setProperty("solr.tests.maxIndexingThreads", "-1");
+        System.setProperty("solr.tests.ramBufferSizeMB", "100");
+
+        // use non-test classes so RandomizedRunner isn't necessary
+        System.setProperty("solr.tests.mergeScheduler", "org.apache.lucene.index.ConcurrentMergeScheduler");
+        System.setProperty("solr.directoryFactory", "solr.RAMDirectoryFactory");
+        System.setProperty("solr.lock.type", "single");
+
+        log.info(new ConfigSetAdminRequest.List().process(miniCluster.getSolrClient()).toString());
+        log
+                .info(
+                        CollectionAdminRequest.ClusterStatus
+                                .getClusterStatus()
+                                .process(miniCluster.getSolrClient())
+                                .toString());
+
+        NamedList<Object> res = createCollection(
+                miniCluster.getSolrClient(), DEFAULT_COLLECTION, 4, 2, 20, CONFIG_NAME);
+        res.forEach(o -> log.info(o.toString()));
+
+        miniCluster.getSolrClient().setDefaultCollection(DEFAULT_COLLECTION);
+
+        log
+                .info(
+                        CollectionAdminRequest.ClusterStatus
+                                .getClusterStatus()
+                                .process(miniCluster.getSolrClient())
+                                .toString());
+
+    }
+
+    @AfterAll
+    public static void shutDown() throws Exception {
+        miniCluster.shutdown();
+        FileUtils.deleteDirectory(workingDir.toFile());
+    }
+
+    protected static NamedList<Object> createCollection(CloudSolrClient client, String name, int numShards,
+                                                        int replicationFactor, int maxShardsPerNode, String configName) throws Exception {
+        ModifiableSolrParams modParams = new ModifiableSolrParams();
+        modParams.set(CoreAdminParams.ACTION, CollectionParams.CollectionAction.CREATE.name());
+        modParams.set("name", name);
+        modParams.set("numShards", numShards);
+        modParams.set("replicationFactor", replicationFactor);
+        modParams.set("collection.configName", configName);
+        modParams.set("maxShardsPerNode", maxShardsPerNode);
+        QueryRequest request = new QueryRequest(modParams);
+        request.setPath("/admin/collections");
+        return client.request(request);
+    }
+
+}