D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 10 Projects 2 Releases 2 Wiki Activity

[stats wf] indicators across stats dbs & updates in the org ids #248

Closed
dimitris.pierrakos wants to merge 1742 commits from beta into beta2master_sept_2022
pull from: beta
merge into: D-Net:beta2master_sept_2022
Conversation 3 Commits 1742 Files Changed 1525 +17 -19
4 changed files with 17 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 1cb79719a7 - Show all commits

11
dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/ResultTagger.java
View File

@ -152,7 +152,8 @@ public class ResultTagger implements Serializable {
} }
result.getContext().forEach(c -> { result.getContext().forEach(c -> {
if (communities.contains(c.getId())) { String cId = c.getId();
if (communities.contains(cId)) {
Optional<List<DataInfo>> opt_dataInfoList = Optional.ofNullable(c.getDataInfo()); Optional<List<DataInfo>> opt_dataInfoList = Optional.ofNullable(c.getDataInfo());
List<DataInfo> dataInfoList; List<DataInfo> dataInfoList;
if (opt_dataInfoList.isPresent()) if (opt_dataInfoList.isPresent())
@ -161,19 +162,19 @@ public class ResultTagger implements Serializable {
dataInfoList = new ArrayList<>(); dataInfoList = new ArrayList<>();
c.setDataInfo(dataInfoList); c.setDataInfo(dataInfoList);
} }
if (subjects.contains(c)) if (subjects.contains(cId))
dataInfoList dataInfoList
.add(OafMapperUtils.dataInfo(false, BULKTAG_DATA_INFO_TYPE, true, false, .add(OafMapperUtils.dataInfo(false, BULKTAG_DATA_INFO_TYPE, true, false,
OafMapperUtils.qualifier(CLASS_ID_SUBJECT, CLASS_NAME_BULKTAG_SUBJECT, DNET_PROVENANCE_ACTIONS, DNET_PROVENANCE_ACTIONS), TAGGING_TRUST)); OafMapperUtils.qualifier(CLASS_ID_SUBJECT, CLASS_NAME_BULKTAG_SUBJECT, DNET_PROVENANCE_ACTIONS, DNET_PROVENANCE_ACTIONS), TAGGING_TRUST));
if (datasources.contains(c)) if (datasources.contains(cId))
dataInfoList dataInfoList
.add(OafMapperUtils.dataInfo(false, BULKTAG_DATA_INFO_TYPE, true, false, .add(OafMapperUtils.dataInfo(false, BULKTAG_DATA_INFO_TYPE, true, false,
OafMapperUtils.qualifier(CLASS_ID_DATASOURCE, CLASS_NAME_BULKTAG_DATASOURCE, DNET_PROVENANCE_ACTIONS, DNET_PROVENANCE_ACTIONS), TAGGING_TRUST)); OafMapperUtils.qualifier(CLASS_ID_DATASOURCE, CLASS_NAME_BULKTAG_DATASOURCE, DNET_PROVENANCE_ACTIONS, DNET_PROVENANCE_ACTIONS), TAGGING_TRUST));
if (czenodo.contains(c)) if (czenodo.contains(cId))
dataInfoList dataInfoList
.add(OafMapperUtils.dataInfo(false, BULKTAG_DATA_INFO_TYPE, true, false, .add(OafMapperUtils.dataInfo(false, BULKTAG_DATA_INFO_TYPE, true, false,
OafMapperUtils.qualifier(CLASS_ID_CZENODO, CLASS_NAME_BULKTAG_ZENODO, DNET_PROVENANCE_ACTIONS, DNET_PROVENANCE_ACTIONS), TAGGING_TRUST)); OafMapperUtils.qualifier(CLASS_ID_CZENODO, CLASS_NAME_BULKTAG_ZENODO, DNET_PROVENANCE_ACTIONS, DNET_PROVENANCE_ACTIONS), TAGGING_TRUST));
if (aconstraints.contains(c)) if (aconstraints.contains(cId))
dataInfoList dataInfoList
.add( .add(
OafMapperUtils.dataInfo(false, BULKTAG_DATA_INFO_TYPE, true, false, OafMapperUtils.dataInfo(false, BULKTAG_DATA_INFO_TYPE, true, false,

15
dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java
View File

@ -21,6 +21,7 @@ import org.apache.commons.io.IOUtils;
import org.apache.spark.SparkConf; import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.ForeachFunction;
import org.apache.spark.sql.Encoders; import org.apache.spark.sql.Encoders;
import org.apache.spark.sql.Row; import org.apache.spark.sql.Row;
import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.SparkSession;
@ -778,18 +779,4 @@ public class BulkTagJobTest {
1, idExplodeCommunity.filter("provenance = 'community:advconstraint'").count()); 1, idExplodeCommunity.filter("provenance = 'community:advconstraint'").count());
} }
// @Test
// void test1(){
// ProtoMap params = new Gson().fromJson(pathMap, ProtoMap.class);
// HashMap<String, String> param = new HashMap<>();
// for (String key : params.keySet()) {
// try {
// param.put(key, jsonContext.read(params.get(key)));
// } catch (com.jayway.jsonpath.PathNotFoundException e) {
// param.put(key, new ArrayList<>());
// }
// }
// return param;
// }
// }
} }

10
dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_subject_datasource/dataset_10.json Normal file
View File

File diff suppressed because one or more lines are too long

BIN
dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_subject_datasource/dataset_10.json.gz
View File

Binary file not shown.