Merge branch 'master' of code-repo.d4science.org:D-Net/dnet-hadoop

dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/PartitionEventsByDsIdJob.java

@@ -4,8 +4,13 @@ package eu.dnetlib.dhp.broker.oa;
 import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
 
 import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashSet;
 import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
 
+import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configuration;
@@ -29,15 +34,14 @@ import eu.dnetlib.dhp.broker.oa.util.ClusterUtils;
 public class PartitionEventsByDsIdJob {
 
 	private static final Logger log = LoggerFactory.getLogger(PartitionEventsByDsIdJob.class);
-	private static final String OPENDOAR_NSPREFIX = "opendoar____::";
+	private static final String OPENDOAR_NSPREFIX = "10|opendoar____::";
 
 	public static void main(final String[] args) throws Exception {
 
 		final ArgumentApplicationParser parser = new ArgumentApplicationParser(
 			IOUtils
-				.toString(
+				.toString(PartitionEventsByDsIdJob.class
-					PartitionEventsByDsIdJob.class
+					.getResourceAsStream("/eu/dnetlib/dhp/broker/oa/od_partitions_params.json")));
-						.getResourceAsStream("/eu/dnetlib/dhp/broker/oa/common_params.json")));
 		parser.parseArgument(args);
 
 		final Boolean isSparkSessionManaged = Optional
@@ -54,13 +58,25 @@ public class PartitionEventsByDsIdJob {
 		final String partitionPath = parser.get("workingPath") + "/eventsByOpendoarId";
 		log.info("partitionPath: {}", partitionPath);
 
+		final String opendoarIds = parser.get("opendoarIds");
+		log.info("opendoarIds: {}", opendoarIds);
+
+		final Set<String> validOpendoarIds = new HashSet<>();
+		if (!opendoarIds.trim().equals("-")) {
+			validOpendoarIds.addAll(Arrays.stream(opendoarIds.split(","))
+				.map(String::trim)
+				.filter(StringUtils::isNotBlank)
+				.map(s -> OPENDOAR_NSPREFIX + DigestUtils.md5Hex(s))
+				.collect(Collectors.toSet()));
+		}
+
 		runWithSparkSession(conf, isSparkSessionManaged, spark -> {
 
 			ClusterUtils
 				.readPath(spark, eventsPath, Event.class)
 				.filter(e -> StringUtils.isNotBlank(e.getMap().getTargetDatasourceId()))
-				.filter(e -> e.getMap().getTargetDatasourceId().contains(OPENDOAR_NSPREFIX))
+				.filter(e -> e.getMap().getTargetDatasourceId().startsWith(OPENDOAR_NSPREFIX))
-				.limit(10000)
+				.filter(e -> validOpendoarIds.contains(e.getMap().getTargetDatasourceId()))
 				.map(e -> messageFromNotification(e), Encoders.bean(ShortEventMessageWithGroupId.class))
 				.coalesce(1)
 				.write()

dhp-workflows/dhp-broker-events/src/main/resources/eu/dnetlib/dhp/broker/oa/od_partitions_params.json

@@ -0,0 +1,14 @@
+[
+	{
+		"paramName": "o",
+		"paramLongName": "workingPath",
+		"paramDescription": "the path where the temporary data will be stored",
+		"paramRequired": true
+	},
+	{
+		"paramName": "list",
+		"paramLongName": "opendoarIds",
+		"paramDescription": "the opendoar IDs whitelist (comma separated)",
+		"paramRequired": true
+	}
+]

dhp-workflows/dhp-broker-events/src/main/resources/eu/dnetlib/dhp/broker/oa/opendoarPartition/oozie_app/workflow.xml

@@ -1,60 +1,13 @@
-<workflow-app name="create broker events - partial" xmlns="uri:oozie:workflow:0.5">
+<workflow-app name="partitionEventsByOpendoarIds" xmlns="uri:oozie:workflow:0.5">
 
     <parameters>
         <property>
-            <name>graphInputPath</name>
+            <name>opendoarIds</name>
-            <description>the path where the graph is stored</description>
+            <description>the opendoar IDs whitelist (comma separated)</description>
         </property>
         <property>
             <name>workingPath</name>
             <description>the path where the the generated data will be stored</description>
-        </property>
-		<property>
-            <name>datasourceIdWhitelist</name>
-            <value>-</value>
-            <description>a white list (comma separeted, - for empty list) of datasource ids</description>
-        </property>
-		<property>
-            <name>datasourceTypeWhitelist</name>
-            <value>-</value>
-            <description>a white list (comma separeted, - for empty list) of datasource types</description>
-        </property>
-		<property>
-            <name>datasourceIdBlacklist</name>
-            <value>-</value>
-            <description>a black list (comma separeted, - for empty list) of datasource ids</description>
-        </property>
-        <property>
-            <name>esEventIndexName</name>
-            <description>the elasticsearch index name for events</description>
-        </property>
-        <property>
-            <name>esNotificationsIndexName</name>
-            <description>the elasticsearch index name for notifications</description>
-        </property>
-        <property>
-            <name>esIndexHost</name>
-            <description>the elasticsearch host</description>
-        </property>
-        <property>
-        	<name>maxIndexedEventsForDsAndTopic</name>
-        	<description>the max number of events for each couple (ds/topic)</description>
-        </property>
-        <property>
-        	<name>brokerApiBaseUrl</name>
-        	<description>the url of the broker service api</description>
-        </property>
-        <property>
-        	<name>brokerDbUrl</name>
-        	<description>the url of the broker database</description>
-        </property>
-        <property>
-        	<name>brokerDbUser</name>
-        	<description>the user of the broker database</description>
-        </property>
-        <property>
-        	<name>brokerDbPassword</name>
-        	<description>the password of the broker database</description>
         </property>
         <property>
             <name>sparkDriverMemory</name>
@@ -111,13 +64,13 @@
         </configuration>
     </global>
 
-    <start to="partition"/>
+    <start to="opendoarPartition"/>
 
     <kill name="Kill">
         <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
     </kill>
     
-   <action name="partition">
+   <action name="opendoarPartition">
         <spark xmlns="uri:oozie:spark-action:0.2">
             <master>yarn</master>
             <mode>cluster</mode>
@@ -134,8 +87,8 @@
                 --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
                 --conf spark.sql.shuffle.partitions=3840
             </spark-opts>
-            <arg>--graphPath</arg><arg>${graphInputPath}</arg>
             <arg>--workingPath</arg><arg>${workingPath}</arg>
+            <arg>--opendoarIds</arg><arg>${opendoarIds}</arg>
         </spark>
         <ok to="End"/>
         <error to="Kill"/>

dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/XmlRecordFactoryTest.java

@@ -5,31 +5,23 @@ import static org.junit.jupiter.api.Assertions.*;
 
 import java.io.IOException;
 import java.io.StringReader;
-import java.util.List;
 
 import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.StringUtils;
 import org.dom4j.Document;
 import org.dom4j.DocumentException;
 import org.dom4j.io.SAXReader;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
-import org.mockito.Mock;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 
 import eu.dnetlib.dhp.oa.provision.model.JoinedEntity;
 import eu.dnetlib.dhp.oa.provision.utils.ContextMapper;
 import eu.dnetlib.dhp.oa.provision.utils.XmlRecordFactory;
-import eu.dnetlib.dhp.schema.oaf.Oaf;
-import eu.dnetlib.dhp.schema.oaf.OafEntity;
-import eu.dnetlib.dhp.schema.oaf.OafMapperUtils;
-import eu.dnetlib.dhp.schema.oaf.Publication;
-import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
 
 //TODO to enable it we need to update the joined_entity.json test file
-//@Disabled
+@Disabled
 public class XmlRecordFactoryTest {
 
 	private static final String otherDsTypeId = "scholarcomminfra,infospace,pubsrepository::mock,entityregistry,entityregistry::projects,entityregistry::repositories,websource";
@@ -43,27 +35,6 @@ public class XmlRecordFactoryTest {
 		JoinedEntity je = new ObjectMapper().readValue(json, JoinedEntity.class);
 		assertNotNull(je);
 
-		Document doc = buildXml(je);
-		//// TODO specific test assertion on doc
-	}
-
-	@Test
-	void testBologna() throws IOException, DocumentException {
-		final String json = IOUtils.toString(getClass().getResourceAsStream("oaf-bologna.json"));
-		Publication oaf = new ObjectMapper().readValue(json, Publication.class);
-		assertNotNull(oaf);
-		JoinedEntity je = new JoinedEntity();
-		je.setEntity(oaf);
-		assertNotNull(je);
-
-		Document doc = buildXml(je);
-		// TODO specific test assertion on doc
-
-		System.out.println(doc.asXML());
-
-	}
-
-	private Document buildXml(JoinedEntity je) throws DocumentException {
 		ContextMapper contextMapper = new ContextMapper();
 
 		XmlRecordFactory xmlRecordFactory = new XmlRecordFactory(contextMapper, false, XmlConverterJob.schemaLocation,
@@ -78,7 +49,5 @@ public class XmlRecordFactoryTest {
 		assertNotNull(doc);
 
 		// TODO add assertions based of values extracted from the XML record
-
-		return doc;
 	}
 }