WIP: initial commit

dhp-workflows/dhp-graph-provision/pom.xml

@@ -80,10 +80,7 @@
             <groupId>org.apache.solr</groupId>
             <artifactId>solr-solrj</artifactId>
         </dependency>
-        <dependency>
-            <groupId>com.lucidworks.spark</groupId>
-            <artifactId>spark-solr</artifactId>
-        </dependency>
+
 
         <!-- the solr-test-framework requires the old junit:junit test framework -->
         <dependency>
@@ -218,6 +215,11 @@
             <version>${project.version}</version>
         </dependency>
 
+        <dependency>
+            <groupId>eu.dnetlib.dhp</groupId>
+            <artifactId>openaire-solr-importer</artifactId>
+        </dependency>
+
     </dependencies>
 
     <profiles>

dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/oozie_app/workflow.xml

@@ -715,7 +715,7 @@
             <master>yarn</master>
             <mode>cluster</mode>
             <name>to_solr_index</name>
-            <class>eu.dnetlib.dhp.oa.provision.XmlIndexingJob</class>
+            <class>eu.dnetlib.dhp.RecordImporterApplication</class>
             <jar>dhp-graph-provision-${projectVersion}.jar</jar>
             <spark-opts>
                 --executor-memory=${sparkExecutorMemoryForIndexing}
@@ -730,7 +730,7 @@
                 --conf spark.hadoop.mapreduce.map.speculative=false
                 --conf spark.hadoop.mapreduce.reduce.speculative=false
             </spark-opts>
-            <arg>--inputPath</arg><arg>${workingDir}/xml_json</arg>
+            <arg>--path</arg><arg>${workingDir}/xml_json</arg>
             <arg>--isLookupUrl</arg><arg>${isLookupUrl}</arg>
             <arg>--shadowFormat</arg><arg>${shadowFormat}</arg>
             <arg>--batchSize</arg><arg>${batchSize}</arg>

pom.xml

@@ -177,6 +177,11 @@
                 <artifactId>dhp-schemas</artifactId>
                 <version>${dhp-schemas.version}</version>
             </dependency>
+            <dependency>
+                <groupId>eu.dnetlib.dhp</groupId>
+                <artifactId>openaire-solr-importer</artifactId>
+                <version>[1.0.0]</version>
+            </dependency>
             <dependency>
                 <groupId>org.apache.hadoop</groupId>
                 <artifactId>hadoop-hdfs</artifactId>
@@ -960,8 +965,8 @@
         <org.slf4j.version>1.7.25</org.slf4j.version>
         <reflections.version>0.9.10</reflections.version>
         <scala-xml.version>1.3.0</scala-xml.version>
-        <solr.version>7.5.0</solr.version>
-        <sparksolr.version>3.6.0</sparksolr.version>
+        <solr.version>9.7.0</solr.version>
+
         <unidecode.version>0.0.7</unidecode.version>
         <vtd.version>[2.12,3.0)</vtd.version>
         <zookeeper.version>3.4.6</zookeeper.version>