D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 13 Projects 2 Releases 2 Wiki Activity

master #11

Manually merged
claudio.atzori merged 275 commits from :master into enrichment_wfs 2020-05-11 15:14:56 +02:00
Conversation 1 Commits 275 Files Changed 156 +25 -4
1 changed files with 25 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 8438702b3d - Show all commits

29
dhp-workflows/dhp-propagation/src/main/java/eu/dnetlib/dhp/PropagationConstant.java
View File

@ -4,7 +4,6 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dnetlib.dhp.schema.oaf.*;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.function.PairFunction;
@ -19,6 +18,8 @@ public class PropagationConstant {
public final static String PROPAGATION_DATA_INFO_TYPE = "propagation";
public static final String TRUE = "true";
public final static String DNET_COUNTRY_SCHEMA = "dnet:countries";
public final static String DNET_SCHEMA_NAME = "dnet:provenanceActions";
@ -84,6 +85,8 @@ public class PropagationConstant {
public static DataInfo getDataInfo(String inference_provenance, String inference_class_id, String inference_class_name){
DataInfo di = new DataInfo();
di.setInferred(true);
di.setDeletedbyinference(false);
di.setTrust("0.85");
di.setInferenceprovenance(inference_provenance);
di.setProvenanceaction(getQualifier(inference_class_id, inference_class_name));
return di;
@ -119,11 +122,25 @@ public class PropagationConstant {
return relations
.filter(r -> !r.getDataInfo().getDeletedbyinference())
.filter(r -> allowedsemrel.contains(r.getRelClass()) && RELATION_RESULTRESULT_REL_TYPE.equals(r.getRelType()))
.map(r -> new TypedRow().setSourceId(r.getSource()).setTargetId(r.getTarget()))
.map(r -> {
TypedRow tr = new TypedRow();
tr.setSourceId(r.getSource());
tr.setTargetId(r.getTarget());
return tr;
})
.mapToPair(toPair());
}
public static String getConstraintList(String text, List<String> constraints){
String ret = " and (" + text + constraints.get(0) + "'";
for (int i =1; i < constraints.size(); i++){
ret += " OR " + text + constraints.get(i) + "'";
}
ret += ")";
return ret;
}
public static List<TypedRow> getTypedRowsDatasourceResult(OafEntity oaf) {
List<TypedRow> lst = new ArrayList<>();
@ -155,7 +172,11 @@ public class PropagationConstant {
datasources_provenance.add(i.getHostedby().getKey());
}
for (String dsId : datasources_provenance) {
lst.add(new TypedRow().setSourceId(dsId).setTargetId(oaf.getId()).setType(type));
TypedRow tr = new TypedRow();
tr.setSourceId(dsId);
tr.setTargetId(oaf.getId());
tr.setType(type);
lst.add(tr);
}
return lst;
}
@ -194,8 +215,8 @@ public class PropagationConstant {
public static void createOutputDirs(String outputPath, FileSystem fs) throws IOException {
if (fs.exists(new Path(outputPath))) {
fs.delete(new Path(outputPath), true);
fs.mkdirs(new Path(outputPath));
}
fs.mkdirs(new Path(outputPath));
}
}