forked from D-Net/dnet-hadoop
compilation after merging
This commit is contained in:
parent
6e58d79623
commit
f7d06dc661
|
@ -95,7 +95,7 @@ public class SparkAtomicActionScoreJob implements Serializable {
|
||||||
|
|
||||||
return projectScores.map((MapFunction<BipProjectModel, Project>) bipProjectScores -> {
|
return projectScores.map((MapFunction<BipProjectModel, Project>) bipProjectScores -> {
|
||||||
Project project = new Project();
|
Project project = new Project();
|
||||||
//project.setId(bipProjectScores.getProjectId());
|
project.setId(bipProjectScores.getProjectId());
|
||||||
project.setMeasures(bipProjectScores.toMeasures());
|
project.setMeasures(bipProjectScores.toMeasures());
|
||||||
return project;
|
return project;
|
||||||
}, Encoders.bean(Project.class))
|
}, Encoders.bean(Project.class))
|
||||||
|
|
|
@ -1,3 +1,4 @@
|
||||||
|
|
||||||
package eu.dnetlib.dhp.bulktag.actions;
|
package eu.dnetlib.dhp.bulktag.actions;
|
||||||
|
|
||||||
import java.io.Serializable;
|
import java.io.Serializable;
|
||||||
|
|
|
@ -1,3 +1,4 @@
|
||||||
|
|
||||||
package eu.dnetlib.dhp.bulktag.actions;
|
package eu.dnetlib.dhp.bulktag.actions;
|
||||||
|
|
||||||
import java.io.Serializable;
|
import java.io.Serializable;
|
||||||
|
@ -36,14 +37,14 @@ public class ExecSubstringAction implements Serializable {
|
||||||
this.to = to;
|
this.to = to;
|
||||||
}
|
}
|
||||||
|
|
||||||
public String execSubstring(){
|
public String execSubstring() {
|
||||||
int to = Integer.valueOf(this.to);
|
int to = Integer.valueOf(this.to);
|
||||||
int from = Integer.valueOf(this.from);
|
int from = Integer.valueOf(this.from);
|
||||||
|
|
||||||
if(to < from || from > this.value.length())
|
if (to < from || from > this.value.length())
|
||||||
return "";
|
return "";
|
||||||
|
|
||||||
if(from < 0)
|
if (from < 0)
|
||||||
from = 0;
|
from = 0;
|
||||||
if (to > this.value.length())
|
if (to > this.value.length())
|
||||||
to = this.value.length();
|
to = this.value.length();
|
||||||
|
|
|
@ -1,3 +1,4 @@
|
||||||
|
|
||||||
package eu.dnetlib.dhp.bulktag.actions;
|
package eu.dnetlib.dhp.bulktag.actions;
|
||||||
|
|
||||||
import java.io.Serializable;
|
import java.io.Serializable;
|
||||||
|
@ -11,7 +12,6 @@ public class MapModel implements Serializable {
|
||||||
private String path;
|
private String path;
|
||||||
private Action action;
|
private Action action;
|
||||||
|
|
||||||
|
|
||||||
public String getPath() {
|
public String getPath() {
|
||||||
return path;
|
return path;
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,3 +1,4 @@
|
||||||
|
|
||||||
package eu.dnetlib.dhp.bulktag.actions;
|
package eu.dnetlib.dhp.bulktag.actions;
|
||||||
|
|
||||||
import java.io.Serializable;
|
import java.io.Serializable;
|
||||||
|
|
|
@ -1,11 +1,11 @@
|
||||||
|
|
||||||
package eu.dnetlib.dhp.bulktag.community;
|
package eu.dnetlib.dhp.bulktag.community;
|
||||||
|
|
||||||
import eu.dnetlib.dhp.bulktag.actions.MapModel;
|
|
||||||
|
|
||||||
import java.io.Serializable;
|
import java.io.Serializable;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
|
|
||||||
|
import eu.dnetlib.dhp.bulktag.actions.MapModel;
|
||||||
|
|
||||||
public class ProtoMap extends HashMap<String, MapModel> implements Serializable {
|
public class ProtoMap extends HashMap<String, MapModel> implements Serializable {
|
||||||
|
|
||||||
public ProtoMap() {
|
public ProtoMap() {
|
||||||
|
|
|
@ -10,9 +10,6 @@ import java.lang.reflect.Method;
|
||||||
import java.util.*;
|
import java.util.*;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
import com.jayway.jsonpath.PathNotFoundException;
|
|
||||||
import eu.dnetlib.dhp.bulktag.actions.MapModel;
|
|
||||||
import eu.dnetlib.dhp.bulktag.actions.Parameters;
|
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
@ -20,7 +17,10 @@ import org.slf4j.LoggerFactory;
|
||||||
import com.google.gson.Gson;
|
import com.google.gson.Gson;
|
||||||
import com.jayway.jsonpath.DocumentContext;
|
import com.jayway.jsonpath.DocumentContext;
|
||||||
import com.jayway.jsonpath.JsonPath;
|
import com.jayway.jsonpath.JsonPath;
|
||||||
|
import com.jayway.jsonpath.PathNotFoundException;
|
||||||
|
|
||||||
|
import eu.dnetlib.dhp.bulktag.actions.MapModel;
|
||||||
|
import eu.dnetlib.dhp.bulktag.actions.Parameters;
|
||||||
import eu.dnetlib.dhp.bulktag.eosc.EoscIFTag;
|
import eu.dnetlib.dhp.bulktag.eosc.EoscIFTag;
|
||||||
import eu.dnetlib.dhp.schema.oaf.*;
|
import eu.dnetlib.dhp.schema.oaf.*;
|
||||||
import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils;
|
import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils;
|
||||||
|
@ -40,7 +40,8 @@ public class ResultTagger implements Serializable {
|
||||||
return (tmp != clist.size());
|
return (tmp != clist.size());
|
||||||
}
|
}
|
||||||
|
|
||||||
private Map<String, List<String>> getParamMap(final Result result, Map<String, MapModel> params) throws NoSuchMethodException, InvocationTargetException {
|
private Map<String, List<String>> getParamMap(final Result result, Map<String, MapModel> params)
|
||||||
|
throws NoSuchMethodException, InvocationTargetException {
|
||||||
Map<String, List<String>> param = new HashMap<>();
|
Map<String, List<String>> param = new HashMap<>();
|
||||||
String json = new Gson().toJson(result, Result.class);
|
String json = new Gson().toJson(result, Result.class);
|
||||||
DocumentContext jsonContext = JsonPath.parse(json);
|
DocumentContext jsonContext = JsonPath.parse(json);
|
||||||
|
@ -54,30 +55,34 @@ public class ResultTagger implements Serializable {
|
||||||
try {
|
try {
|
||||||
String path = mapModel.getPath();
|
String path = mapModel.getPath();
|
||||||
Object obj = jsonContext.read(path);
|
Object obj = jsonContext.read(path);
|
||||||
List<String> pathValue ;
|
List<String> pathValue;
|
||||||
if(obj instanceof java.lang.String)
|
if (obj instanceof java.lang.String)
|
||||||
pathValue = Arrays.asList((String)obj);
|
pathValue = Arrays.asList((String) obj);
|
||||||
else
|
else
|
||||||
pathValue = (List<String>)obj;
|
pathValue = (List<String>) obj;
|
||||||
if(Optional.ofNullable(mapModel.getAction()).isPresent()){
|
if (Optional.ofNullable(mapModel.getAction()).isPresent()) {
|
||||||
Class<?> c = Class.forName(mapModel.getAction().getClazz());
|
Class<?> c = Class.forName(mapModel.getAction().getClazz());
|
||||||
Object class_instance = c.newInstance();
|
Object class_instance = c.newInstance();
|
||||||
Method setField = c.getMethod("setValue", String.class);
|
Method setField = c.getMethod("setValue", String.class);
|
||||||
setField.invoke(class_instance, pathValue.get(0));
|
setField.invoke(class_instance, pathValue.get(0));
|
||||||
for(Parameters p : mapModel.getAction().getParams()){
|
for (Parameters p : mapModel.getAction().getParams()) {
|
||||||
setField = c.getMethod("set" + p.getParamName(), String.class);
|
setField = c.getMethod("set" + p.getParamName(), String.class);
|
||||||
setField.invoke(class_instance, p.getParamValue());
|
setField.invoke(class_instance, p.getParamValue());
|
||||||
}
|
}
|
||||||
|
|
||||||
param.put(key,Arrays.asList((String)c.getMethod(mapModel.getAction().getMethod()).invoke(class_instance)));
|
param
|
||||||
|
.put(
|
||||||
|
key, Arrays
|
||||||
|
.asList((String) c.getMethod(mapModel.getAction().getMethod()).invoke(class_instance)));
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
else{
|
else {
|
||||||
param.put(key, pathValue);
|
param.put(key, pathValue);
|
||||||
}
|
}
|
||||||
|
|
||||||
} catch (PathNotFoundException | ClassNotFoundException | InstantiationException | IllegalAccessException e) {
|
} catch (PathNotFoundException | ClassNotFoundException | InstantiationException
|
||||||
|
| IllegalAccessException e) {
|
||||||
param.put(key, new ArrayList<>());
|
param.put(key, new ArrayList<>());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -86,9 +91,8 @@ public class ResultTagger implements Serializable {
|
||||||
}
|
}
|
||||||
|
|
||||||
public <R extends Result> R enrichContextCriteria(
|
public <R extends Result> R enrichContextCriteria(
|
||||||
final R result, final CommunityConfiguration conf, final Map<String, MapModel> criteria) throws InvocationTargetException, NoSuchMethodException {
|
final R result, final CommunityConfiguration conf, final Map<String, MapModel> criteria)
|
||||||
|
throws InvocationTargetException, NoSuchMethodException {
|
||||||
|
|
||||||
|
|
||||||
// Verify if the entity is deletedbyinference. In case verify if to clean the context list
|
// Verify if the entity is deletedbyinference. In case verify if to clean the context list
|
||||||
// from all the zenodo communities
|
// from all the zenodo communities
|
||||||
|
|
|
@ -33,13 +33,14 @@ public class BulkTagJobTest {
|
||||||
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
|
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
|
||||||
|
|
||||||
public static final String pathMap = "{\"author\":{\"path\":\"$['author'][*]['fullname']\"}," +
|
public static final String pathMap = "{\"author\":{\"path\":\"$['author'][*]['fullname']\"}," +
|
||||||
" \"title\":{\"path\":\"$['title'][*]['value']\"}, "+
|
" \"title\":{\"path\":\"$['title'][*]['value']\"}, " +
|
||||||
" \"orcid\":{\"path\":\"$['author'][*]['pid'][*][?(@['qualifier']['classid']=='orcid')]['value']\"} , " +
|
" \"orcid\":{\"path\":\"$['author'][*]['pid'][*][?(@['qualifier']['classid']=='orcid')]['value']\"} , " +
|
||||||
" \"orcid_pending\":{\"path\":\"$['author'][*]['pid'][*][?(@['qualifier']['classid']=='orcid_pending')]['value']\"} ,"+
|
" \"orcid_pending\":{\"path\":\"$['author'][*]['pid'][*][?(@['qualifier']['classid']=='orcid_pending')]['value']\"} ,"
|
||||||
"\"contributor\" : {\"path\":\"$['contributor'][*]['value']\"},"+
|
+
|
||||||
" \"description\" : {\"path\":\"$['description'][*]['value']\"},"+
|
"\"contributor\" : {\"path\":\"$['contributor'][*]['value']\"}," +
|
||||||
|
" \"description\" : {\"path\":\"$['description'][*]['value']\"}," +
|
||||||
" \"subject\" :{\"path\":\"$['subject'][*]['value']\"}, " +
|
" \"subject\" :{\"path\":\"$['subject'][*]['value']\"}, " +
|
||||||
" \"fos\" : {\"path\":\"$['subject'][?(@['qualifier']['classid']=='FOS')].value\"} , "+
|
" \"fos\" : {\"path\":\"$['subject'][?(@['qualifier']['classid']=='FOS')].value\"} , " +
|
||||||
"\"sdg\" : {\"path\":\"$['subject'][?(@['qualifier']['classid']=='SDG')].value\"}," +
|
"\"sdg\" : {\"path\":\"$['subject'][?(@['qualifier']['classid']=='SDG')].value\"}," +
|
||||||
"\"journal\":{\"path\":\"$['journal'].name\"}," +
|
"\"journal\":{\"path\":\"$['journal'].name\"}," +
|
||||||
"\"hostedby\":{\"path\":\"$['instance'][*]['hostedby']['key']\"}," +
|
"\"hostedby\":{\"path\":\"$['instance'][*]['hostedby']['key']\"}," +
|
||||||
|
@ -47,12 +48,11 @@ public class BulkTagJobTest {
|
||||||
"\"publisher\":{\"path\":\"$['publisher'].value\"}," +
|
"\"publisher\":{\"path\":\"$['publisher'].value\"}," +
|
||||||
"\"publicationyear\":{\"path\":\"$['dateofacceptance'].value\", " +
|
"\"publicationyear\":{\"path\":\"$['dateofacceptance'].value\", " +
|
||||||
" \"action\":{\"clazz\":\"eu.dnetlib.dhp.bulktag.actions.ExecSubstringAction\"," +
|
" \"action\":{\"clazz\":\"eu.dnetlib.dhp.bulktag.actions.ExecSubstringAction\"," +
|
||||||
"\"method\":\"execSubstring\","+
|
"\"method\":\"execSubstring\"," +
|
||||||
"\"params\":[" +
|
"\"params\":[" +
|
||||||
"{\"paramName\":\"From\", \"paramValue\":0}, " +
|
"{\"paramName\":\"From\", \"paramValue\":0}, " +
|
||||||
"{\"paramName\":\"To\",\"paramValue\":4}]}}}";
|
"{\"paramName\":\"To\",\"paramValue\":4}]}}}";
|
||||||
|
|
||||||
|
|
||||||
private static SparkSession spark;
|
private static SparkSession spark;
|
||||||
|
|
||||||
private static Path workingDir;
|
private static Path workingDir;
|
||||||
|
@ -1609,11 +1609,9 @@ public class BulkTagJobTest {
|
||||||
Assertions.assertEquals(0, spark.sql(query).count());
|
Assertions.assertEquals(0, spark.sql(query).count());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void pubdateTest() throws Exception {
|
void pubdateTest() throws Exception {
|
||||||
|
|
||||||
|
|
||||||
final String pathMap = BulkTagJobTest.pathMap;
|
final String pathMap = BulkTagJobTest.pathMap;
|
||||||
SparkBulkTagJob
|
SparkBulkTagJob
|
||||||
.main(
|
.main(
|
||||||
|
@ -1643,7 +1641,6 @@ public class BulkTagJobTest {
|
||||||
|
|
||||||
verificationDataset.createOrReplaceTempView("dataset");
|
verificationDataset.createOrReplaceTempView("dataset");
|
||||||
|
|
||||||
|
|
||||||
String query = "select id, MyT.id community, MyD.provenanceaction.classid "
|
String query = "select id, MyT.id community, MyD.provenanceaction.classid "
|
||||||
+ "from dataset "
|
+ "from dataset "
|
||||||
+ "lateral view explode(context) c as MyT "
|
+ "lateral view explode(context) c as MyT "
|
||||||
|
@ -1654,15 +1651,52 @@ public class BulkTagJobTest {
|
||||||
queryResult.show(false);
|
queryResult.show(false);
|
||||||
Assertions.assertEquals(5, queryResult.count());
|
Assertions.assertEquals(5, queryResult.count());
|
||||||
|
|
||||||
Assertions.assertEquals(1, queryResult.filter((FilterFunction<Row>) r -> r.getAs("id").equals("50|od______3989::02dd5d2c222191b0b9bd4f33c8e96529")).count());
|
Assertions
|
||||||
Assertions.assertEquals(1, queryResult.filter((FilterFunction<Row>) r -> r.getAs("id").equals("50|od______3989::2f4f3c820c450bd08dac08d07cc82dcf")).count());
|
.assertEquals(
|
||||||
Assertions.assertEquals(1, queryResult.filter((FilterFunction<Row>) r -> r.getAs("id").equals("50|od______3989::7fcbe3a03280663cddebfd3cb9203177")).count());
|
1,
|
||||||
Assertions.assertEquals(1, queryResult.filter((FilterFunction<Row>) r -> r.getAs("id").equals("50|od______3989::d791339867bec6d3eb2104deeb4e4961")).count());
|
queryResult
|
||||||
Assertions.assertEquals(1, queryResult.filter((FilterFunction<Row>) r -> r.getAs("id").equals("50|od______3989::d90d3a1f64ad264b5ebed8a35b280343")).count());
|
.filter(
|
||||||
|
(FilterFunction<Row>) r -> r
|
||||||
|
.getAs("id")
|
||||||
|
.equals("50|od______3989::02dd5d2c222191b0b9bd4f33c8e96529"))
|
||||||
|
.count());
|
||||||
|
Assertions
|
||||||
|
.assertEquals(
|
||||||
|
1,
|
||||||
|
queryResult
|
||||||
|
.filter(
|
||||||
|
(FilterFunction<Row>) r -> r
|
||||||
|
.getAs("id")
|
||||||
|
.equals("50|od______3989::2f4f3c820c450bd08dac08d07cc82dcf"))
|
||||||
|
.count());
|
||||||
|
Assertions
|
||||||
|
.assertEquals(
|
||||||
|
1,
|
||||||
|
queryResult
|
||||||
|
.filter(
|
||||||
|
(FilterFunction<Row>) r -> r
|
||||||
|
.getAs("id")
|
||||||
|
.equals("50|od______3989::7fcbe3a03280663cddebfd3cb9203177"))
|
||||||
|
.count());
|
||||||
|
Assertions
|
||||||
|
.assertEquals(
|
||||||
|
1,
|
||||||
|
queryResult
|
||||||
|
.filter(
|
||||||
|
(FilterFunction<Row>) r -> r
|
||||||
|
.getAs("id")
|
||||||
|
.equals("50|od______3989::d791339867bec6d3eb2104deeb4e4961"))
|
||||||
|
.count());
|
||||||
|
Assertions
|
||||||
|
.assertEquals(
|
||||||
|
1,
|
||||||
|
queryResult
|
||||||
|
.filter(
|
||||||
|
(FilterFunction<Row>) r -> r
|
||||||
|
.getAs("id")
|
||||||
|
.equals("50|od______3989::d90d3a1f64ad264b5ebed8a35b280343"))
|
||||||
|
.count());
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue