compilation after merging

This commit is contained in:
Miriam Baglioni 2024-01-23 11:43:08 +01:00
parent 6e58d79623
commit f7d06dc661
9 changed files with 194 additions and 153 deletions

View File

@ -95,7 +95,7 @@ public class SparkAtomicActionScoreJob implements Serializable {
return projectScores.map((MapFunction<BipProjectModel, Project>) bipProjectScores -> { return projectScores.map((MapFunction<BipProjectModel, Project>) bipProjectScores -> {
Project project = new Project(); Project project = new Project();
//project.setId(bipProjectScores.getProjectId()); project.setId(bipProjectScores.getProjectId());
project.setMeasures(bipProjectScores.toMeasures()); project.setMeasures(bipProjectScores.toMeasures());
return project; return project;
}, Encoders.bean(Project.class)) }, Encoders.bean(Project.class))

View File

@ -1,3 +1,4 @@
package eu.dnetlib.dhp.bulktag.actions; package eu.dnetlib.dhp.bulktag.actions;
import java.io.Serializable; import java.io.Serializable;

View File

@ -1,3 +1,4 @@
package eu.dnetlib.dhp.bulktag.actions; package eu.dnetlib.dhp.bulktag.actions;
import java.io.Serializable; import java.io.Serializable;
@ -36,14 +37,14 @@ public class ExecSubstringAction implements Serializable {
this.to = to; this.to = to;
} }
public String execSubstring(){ public String execSubstring() {
int to = Integer.valueOf(this.to); int to = Integer.valueOf(this.to);
int from = Integer.valueOf(this.from); int from = Integer.valueOf(this.from);
if(to < from || from > this.value.length()) if (to < from || from > this.value.length())
return ""; return "";
if(from < 0) if (from < 0)
from = 0; from = 0;
if (to > this.value.length()) if (to > this.value.length())
to = this.value.length(); to = this.value.length();

View File

@ -1,3 +1,4 @@
package eu.dnetlib.dhp.bulktag.actions; package eu.dnetlib.dhp.bulktag.actions;
import java.io.Serializable; import java.io.Serializable;
@ -11,7 +12,6 @@ public class MapModel implements Serializable {
private String path; private String path;
private Action action; private Action action;
public String getPath() { public String getPath() {
return path; return path;
} }

View File

@ -1,3 +1,4 @@
package eu.dnetlib.dhp.bulktag.actions; package eu.dnetlib.dhp.bulktag.actions;
import java.io.Serializable; import java.io.Serializable;

View File

@ -1,11 +1,11 @@
package eu.dnetlib.dhp.bulktag.community; package eu.dnetlib.dhp.bulktag.community;
import eu.dnetlib.dhp.bulktag.actions.MapModel;
import java.io.Serializable; import java.io.Serializable;
import java.util.HashMap; import java.util.HashMap;
import eu.dnetlib.dhp.bulktag.actions.MapModel;
public class ProtoMap extends HashMap<String, MapModel> implements Serializable { public class ProtoMap extends HashMap<String, MapModel> implements Serializable {
public ProtoMap() { public ProtoMap() {

View File

@ -10,9 +10,6 @@ import java.lang.reflect.Method;
import java.util.*; import java.util.*;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import com.jayway.jsonpath.PathNotFoundException;
import eu.dnetlib.dhp.bulktag.actions.MapModel;
import eu.dnetlib.dhp.bulktag.actions.Parameters;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -20,7 +17,10 @@ import org.slf4j.LoggerFactory;
import com.google.gson.Gson; import com.google.gson.Gson;
import com.jayway.jsonpath.DocumentContext; import com.jayway.jsonpath.DocumentContext;
import com.jayway.jsonpath.JsonPath; import com.jayway.jsonpath.JsonPath;
import com.jayway.jsonpath.PathNotFoundException;
import eu.dnetlib.dhp.bulktag.actions.MapModel;
import eu.dnetlib.dhp.bulktag.actions.Parameters;
import eu.dnetlib.dhp.bulktag.eosc.EoscIFTag; import eu.dnetlib.dhp.bulktag.eosc.EoscIFTag;
import eu.dnetlib.dhp.schema.oaf.*; import eu.dnetlib.dhp.schema.oaf.*;
import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils; import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils;
@ -40,7 +40,8 @@ public class ResultTagger implements Serializable {
return (tmp != clist.size()); return (tmp != clist.size());
} }
private Map<String, List<String>> getParamMap(final Result result, Map<String, MapModel> params) throws NoSuchMethodException, InvocationTargetException { private Map<String, List<String>> getParamMap(final Result result, Map<String, MapModel> params)
throws NoSuchMethodException, InvocationTargetException {
Map<String, List<String>> param = new HashMap<>(); Map<String, List<String>> param = new HashMap<>();
String json = new Gson().toJson(result, Result.class); String json = new Gson().toJson(result, Result.class);
DocumentContext jsonContext = JsonPath.parse(json); DocumentContext jsonContext = JsonPath.parse(json);
@ -54,30 +55,34 @@ public class ResultTagger implements Serializable {
try { try {
String path = mapModel.getPath(); String path = mapModel.getPath();
Object obj = jsonContext.read(path); Object obj = jsonContext.read(path);
List<String> pathValue ; List<String> pathValue;
if(obj instanceof java.lang.String) if (obj instanceof java.lang.String)
pathValue = Arrays.asList((String)obj); pathValue = Arrays.asList((String) obj);
else else
pathValue = (List<String>)obj; pathValue = (List<String>) obj;
if(Optional.ofNullable(mapModel.getAction()).isPresent()){ if (Optional.ofNullable(mapModel.getAction()).isPresent()) {
Class<?> c = Class.forName(mapModel.getAction().getClazz()); Class<?> c = Class.forName(mapModel.getAction().getClazz());
Object class_instance = c.newInstance(); Object class_instance = c.newInstance();
Method setField = c.getMethod("setValue", String.class); Method setField = c.getMethod("setValue", String.class);
setField.invoke(class_instance, pathValue.get(0)); setField.invoke(class_instance, pathValue.get(0));
for(Parameters p : mapModel.getAction().getParams()){ for (Parameters p : mapModel.getAction().getParams()) {
setField = c.getMethod("set" + p.getParamName(), String.class); setField = c.getMethod("set" + p.getParamName(), String.class);
setField.invoke(class_instance, p.getParamValue()); setField.invoke(class_instance, p.getParamValue());
} }
param.put(key,Arrays.asList((String)c.getMethod(mapModel.getAction().getMethod()).invoke(class_instance))); param
.put(
key, Arrays
.asList((String) c.getMethod(mapModel.getAction().getMethod()).invoke(class_instance)));
} }
else{ else {
param.put(key, pathValue); param.put(key, pathValue);
} }
} catch (PathNotFoundException | ClassNotFoundException | InstantiationException | IllegalAccessException e) { } catch (PathNotFoundException | ClassNotFoundException | InstantiationException
| IllegalAccessException e) {
param.put(key, new ArrayList<>()); param.put(key, new ArrayList<>());
} }
} }
@ -86,9 +91,8 @@ public class ResultTagger implements Serializable {
} }
public <R extends Result> R enrichContextCriteria( public <R extends Result> R enrichContextCriteria(
final R result, final CommunityConfiguration conf, final Map<String, MapModel> criteria) throws InvocationTargetException, NoSuchMethodException { final R result, final CommunityConfiguration conf, final Map<String, MapModel> criteria)
throws InvocationTargetException, NoSuchMethodException {
// Verify if the entity is deletedbyinference. In case verify if to clean the context list // Verify if the entity is deletedbyinference. In case verify if to clean the context list
// from all the zenodo communities // from all the zenodo communities

View File

@ -33,13 +33,14 @@ public class BulkTagJobTest {
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
public static final String pathMap = "{\"author\":{\"path\":\"$['author'][*]['fullname']\"}," + public static final String pathMap = "{\"author\":{\"path\":\"$['author'][*]['fullname']\"}," +
" \"title\":{\"path\":\"$['title'][*]['value']\"}, "+ " \"title\":{\"path\":\"$['title'][*]['value']\"}, " +
" \"orcid\":{\"path\":\"$['author'][*]['pid'][*][?(@['qualifier']['classid']=='orcid')]['value']\"} , " + " \"orcid\":{\"path\":\"$['author'][*]['pid'][*][?(@['qualifier']['classid']=='orcid')]['value']\"} , " +
" \"orcid_pending\":{\"path\":\"$['author'][*]['pid'][*][?(@['qualifier']['classid']=='orcid_pending')]['value']\"} ,"+ " \"orcid_pending\":{\"path\":\"$['author'][*]['pid'][*][?(@['qualifier']['classid']=='orcid_pending')]['value']\"} ,"
"\"contributor\" : {\"path\":\"$['contributor'][*]['value']\"},"+ +
" \"description\" : {\"path\":\"$['description'][*]['value']\"},"+ "\"contributor\" : {\"path\":\"$['contributor'][*]['value']\"}," +
" \"description\" : {\"path\":\"$['description'][*]['value']\"}," +
" \"subject\" :{\"path\":\"$['subject'][*]['value']\"}, " + " \"subject\" :{\"path\":\"$['subject'][*]['value']\"}, " +
" \"fos\" : {\"path\":\"$['subject'][?(@['qualifier']['classid']=='FOS')].value\"} , "+ " \"fos\" : {\"path\":\"$['subject'][?(@['qualifier']['classid']=='FOS')].value\"} , " +
"\"sdg\" : {\"path\":\"$['subject'][?(@['qualifier']['classid']=='SDG')].value\"}," + "\"sdg\" : {\"path\":\"$['subject'][?(@['qualifier']['classid']=='SDG')].value\"}," +
"\"journal\":{\"path\":\"$['journal'].name\"}," + "\"journal\":{\"path\":\"$['journal'].name\"}," +
"\"hostedby\":{\"path\":\"$['instance'][*]['hostedby']['key']\"}," + "\"hostedby\":{\"path\":\"$['instance'][*]['hostedby']['key']\"}," +
@ -47,12 +48,11 @@ public class BulkTagJobTest {
"\"publisher\":{\"path\":\"$['publisher'].value\"}," + "\"publisher\":{\"path\":\"$['publisher'].value\"}," +
"\"publicationyear\":{\"path\":\"$['dateofacceptance'].value\", " + "\"publicationyear\":{\"path\":\"$['dateofacceptance'].value\", " +
" \"action\":{\"clazz\":\"eu.dnetlib.dhp.bulktag.actions.ExecSubstringAction\"," + " \"action\":{\"clazz\":\"eu.dnetlib.dhp.bulktag.actions.ExecSubstringAction\"," +
"\"method\":\"execSubstring\","+ "\"method\":\"execSubstring\"," +
"\"params\":[" + "\"params\":[" +
"{\"paramName\":\"From\", \"paramValue\":0}, " + "{\"paramName\":\"From\", \"paramValue\":0}, " +
"{\"paramName\":\"To\",\"paramValue\":4}]}}}"; "{\"paramName\":\"To\",\"paramValue\":4}]}}}";
private static SparkSession spark; private static SparkSession spark;
private static Path workingDir; private static Path workingDir;
@ -1609,11 +1609,9 @@ public class BulkTagJobTest {
Assertions.assertEquals(0, spark.sql(query).count()); Assertions.assertEquals(0, spark.sql(query).count());
} }
@Test @Test
void pubdateTest() throws Exception { void pubdateTest() throws Exception {
final String pathMap = BulkTagJobTest.pathMap; final String pathMap = BulkTagJobTest.pathMap;
SparkBulkTagJob SparkBulkTagJob
.main( .main(
@ -1643,7 +1641,6 @@ public class BulkTagJobTest {
verificationDataset.createOrReplaceTempView("dataset"); verificationDataset.createOrReplaceTempView("dataset");
String query = "select id, MyT.id community, MyD.provenanceaction.classid " String query = "select id, MyT.id community, MyD.provenanceaction.classid "
+ "from dataset " + "from dataset "
+ "lateral view explode(context) c as MyT " + "lateral view explode(context) c as MyT "
@ -1654,15 +1651,52 @@ public class BulkTagJobTest {
queryResult.show(false); queryResult.show(false);
Assertions.assertEquals(5, queryResult.count()); Assertions.assertEquals(5, queryResult.count());
Assertions.assertEquals(1, queryResult.filter((FilterFunction<Row>) r -> r.getAs("id").equals("50|od______3989::02dd5d2c222191b0b9bd4f33c8e96529")).count()); Assertions
Assertions.assertEquals(1, queryResult.filter((FilterFunction<Row>) r -> r.getAs("id").equals("50|od______3989::2f4f3c820c450bd08dac08d07cc82dcf")).count()); .assertEquals(
Assertions.assertEquals(1, queryResult.filter((FilterFunction<Row>) r -> r.getAs("id").equals("50|od______3989::7fcbe3a03280663cddebfd3cb9203177")).count()); 1,
Assertions.assertEquals(1, queryResult.filter((FilterFunction<Row>) r -> r.getAs("id").equals("50|od______3989::d791339867bec6d3eb2104deeb4e4961")).count()); queryResult
Assertions.assertEquals(1, queryResult.filter((FilterFunction<Row>) r -> r.getAs("id").equals("50|od______3989::d90d3a1f64ad264b5ebed8a35b280343")).count()); .filter(
(FilterFunction<Row>) r -> r
.getAs("id")
.equals("50|od______3989::02dd5d2c222191b0b9bd4f33c8e96529"))
.count());
Assertions
.assertEquals(
1,
queryResult
.filter(
(FilterFunction<Row>) r -> r
.getAs("id")
.equals("50|od______3989::2f4f3c820c450bd08dac08d07cc82dcf"))
.count());
Assertions
.assertEquals(
1,
queryResult
.filter(
(FilterFunction<Row>) r -> r
.getAs("id")
.equals("50|od______3989::7fcbe3a03280663cddebfd3cb9203177"))
.count());
Assertions
.assertEquals(
1,
queryResult
.filter(
(FilterFunction<Row>) r -> r
.getAs("id")
.equals("50|od______3989::d791339867bec6d3eb2104deeb4e4961"))
.count());
Assertions
.assertEquals(
1,
queryResult
.filter(
(FilterFunction<Row>) r -> r
.getAs("id")
.equals("50|od______3989::d90d3a1f64ad264b5ebed8a35b280343"))
.count());
} }
} }