forked from D-Net/dnet-hadoop
code formatting
This commit is contained in:
parent
b60985cf68
commit
57dbeb08d2
|
@ -5,9 +5,10 @@ import java.io.Serializable;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
import com.google.gson.Gson;
|
|
||||||
import org.apache.avro.generic.GenericData;
|
import org.apache.avro.generic.GenericData;
|
||||||
|
|
||||||
|
import com.google.gson.Gson;
|
||||||
|
|
||||||
/** Created by miriam on 01/08/2018. */
|
/** Created by miriam on 01/08/2018. */
|
||||||
public class Community implements Serializable {
|
public class Community implements Serializable {
|
||||||
|
|
||||||
|
|
|
@ -77,7 +77,7 @@ public class CommunityConfiguration implements Serializable {
|
||||||
if (zenodocommunityMap == null) {
|
if (zenodocommunityMap == null) {
|
||||||
zenodocommunityMap = Maps.newHashMap();
|
zenodocommunityMap = Maps.newHashMap();
|
||||||
}
|
}
|
||||||
if(selectionConstraintsMap == null){
|
if (selectionConstraintsMap == null) {
|
||||||
selectionConstraintsMap = Maps.newHashMap();
|
selectionConstraintsMap = Maps.newHashMap();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -91,10 +91,11 @@ public class CommunityConfigurationFactory {
|
||||||
|
|
||||||
private static SelectionConstraints parseConstrains(Node node) {
|
private static SelectionConstraints parseConstrains(Node node) {
|
||||||
Node aconstraints = node.selectSingleNode("./advancedConstraints");
|
Node aconstraints = node.selectSingleNode("./advancedConstraints");
|
||||||
if(aconstraints == null){
|
if (aconstraints == null) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
SelectionConstraints selectionConstraints = new Gson().fromJson(aconstraints.getText(), SelectionConstraints.class);
|
SelectionConstraints selectionConstraints = new Gson()
|
||||||
|
.fromJson(aconstraints.getText(), SelectionConstraints.class);
|
||||||
|
|
||||||
selectionConstraints.setSelection(resolver);
|
selectionConstraints.setSelection(resolver);
|
||||||
return selectionConstraints;
|
return selectionConstraints;
|
||||||
|
|
|
@ -70,7 +70,7 @@ public class QueryInformationSystem {
|
||||||
+ " </zenodocommunity>} "
|
+ " </zenodocommunity>} "
|
||||||
+ " </zenodocommunities> "
|
+ " </zenodocommunities> "
|
||||||
+ "<advancedConstraint>"
|
+ "<advancedConstraint>"
|
||||||
+"{$x//CONFIGURATION/context/param[./@name='advancedConstraint']/text()} "
|
+ "{$x//CONFIGURATION/context/param[./@name='advancedConstraint']/text()} "
|
||||||
+ "</advancedConstraint>"
|
+ "</advancedConstraint>"
|
||||||
+ " </community>";
|
+ " </community>";
|
||||||
|
|
||||||
|
|
|
@ -9,16 +9,16 @@ import java.util.*;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
import java.util.stream.Stream;
|
import java.util.stream.Stream;
|
||||||
|
|
||||||
import eu.dnetlib.dhp.schema.common.ModelConstants;
|
|
||||||
import eu.dnetlib.dhp.schema.common.ModelSupport;
|
|
||||||
import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils;
|
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
|
||||||
import com.google.gson.Gson;
|
import com.google.gson.Gson;
|
||||||
import com.jayway.jsonpath.DocumentContext;
|
import com.jayway.jsonpath.DocumentContext;
|
||||||
import com.jayway.jsonpath.JsonPath;
|
import com.jayway.jsonpath.JsonPath;
|
||||||
|
|
||||||
|
import eu.dnetlib.dhp.schema.common.ModelConstants;
|
||||||
|
import eu.dnetlib.dhp.schema.common.ModelSupport;
|
||||||
import eu.dnetlib.dhp.schema.oaf.*;
|
import eu.dnetlib.dhp.schema.oaf.*;
|
||||||
|
import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils;
|
||||||
|
|
||||||
/** Created by miriam on 02/08/2018. */
|
/** Created by miriam on 02/08/2018. */
|
||||||
public class ResultTagger implements Serializable {
|
public class ResultTagger implements Serializable {
|
||||||
|
@ -134,11 +134,17 @@ public class ResultTagger implements Serializable {
|
||||||
/* Tagging for Advanced Constraints */
|
/* Tagging for Advanced Constraints */
|
||||||
final Set<String> aconstraints = new HashSet<>();
|
final Set<String> aconstraints = new HashSet<>();
|
||||||
|
|
||||||
conf.getSelectionConstraintsMap().keySet()
|
conf
|
||||||
|
.getSelectionConstraintsMap()
|
||||||
|
.keySet()
|
||||||
.forEach(communityId -> {
|
.forEach(communityId -> {
|
||||||
if(conf.getSelectionConstraintsMap().get(communityId) != null &&
|
if (conf.getSelectionConstraintsMap().get(communityId) != null &&
|
||||||
conf.getSelectionConstraintsMap().get(communityId)
|
conf
|
||||||
.getCriteria().stream().anyMatch(crit -> crit.verifyCriteria(param)))
|
.getSelectionConstraintsMap()
|
||||||
|
.get(communityId)
|
||||||
|
.getCriteria()
|
||||||
|
.stream()
|
||||||
|
.anyMatch(crit -> crit.verifyCriteria(param)))
|
||||||
aconstraints.add(communityId);
|
aconstraints.add(communityId);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
@ -163,21 +169,48 @@ public class ResultTagger implements Serializable {
|
||||||
}
|
}
|
||||||
if (subjects.contains(c))
|
if (subjects.contains(c))
|
||||||
dataInfoList
|
dataInfoList
|
||||||
.add(OafMapperUtils.dataInfo(false, BULKTAG_DATA_INFO_TYPE, true, false,
|
.add(
|
||||||
OafMapperUtils.qualifier(CLASS_ID_SUBJECT, CLASS_NAME_BULKTAG_SUBJECT, DNET_PROVENANCE_ACTIONS, DNET_PROVENANCE_ACTIONS), TAGGING_TRUST));
|
OafMapperUtils
|
||||||
|
.dataInfo(
|
||||||
|
false, BULKTAG_DATA_INFO_TYPE, true, false,
|
||||||
|
OafMapperUtils
|
||||||
|
.qualifier(
|
||||||
|
CLASS_ID_SUBJECT, CLASS_NAME_BULKTAG_SUBJECT, DNET_PROVENANCE_ACTIONS,
|
||||||
|
DNET_PROVENANCE_ACTIONS),
|
||||||
|
TAGGING_TRUST));
|
||||||
if (datasources.contains(c))
|
if (datasources.contains(c))
|
||||||
dataInfoList
|
dataInfoList
|
||||||
.add(OafMapperUtils.dataInfo(false, BULKTAG_DATA_INFO_TYPE, true, false,
|
.add(
|
||||||
OafMapperUtils.qualifier(CLASS_ID_DATASOURCE, CLASS_NAME_BULKTAG_DATASOURCE, DNET_PROVENANCE_ACTIONS, DNET_PROVENANCE_ACTIONS), TAGGING_TRUST));
|
OafMapperUtils
|
||||||
|
.dataInfo(
|
||||||
|
false, BULKTAG_DATA_INFO_TYPE, true, false,
|
||||||
|
OafMapperUtils
|
||||||
|
.qualifier(
|
||||||
|
CLASS_ID_DATASOURCE, CLASS_NAME_BULKTAG_DATASOURCE, DNET_PROVENANCE_ACTIONS,
|
||||||
|
DNET_PROVENANCE_ACTIONS),
|
||||||
|
TAGGING_TRUST));
|
||||||
if (czenodo.contains(c))
|
if (czenodo.contains(c))
|
||||||
dataInfoList
|
dataInfoList
|
||||||
.add(OafMapperUtils.dataInfo(false, BULKTAG_DATA_INFO_TYPE, true, false,
|
.add(
|
||||||
OafMapperUtils.qualifier(CLASS_ID_CZENODO, CLASS_NAME_BULKTAG_ZENODO, DNET_PROVENANCE_ACTIONS, DNET_PROVENANCE_ACTIONS), TAGGING_TRUST));
|
OafMapperUtils
|
||||||
|
.dataInfo(
|
||||||
|
false, BULKTAG_DATA_INFO_TYPE, true, false,
|
||||||
|
OafMapperUtils
|
||||||
|
.qualifier(
|
||||||
|
CLASS_ID_CZENODO, CLASS_NAME_BULKTAG_ZENODO, DNET_PROVENANCE_ACTIONS,
|
||||||
|
DNET_PROVENANCE_ACTIONS),
|
||||||
|
TAGGING_TRUST));
|
||||||
if (aconstraints.contains(c))
|
if (aconstraints.contains(c))
|
||||||
dataInfoList
|
dataInfoList
|
||||||
.add(
|
.add(
|
||||||
OafMapperUtils.dataInfo(false, BULKTAG_DATA_INFO_TYPE, true, false,
|
OafMapperUtils
|
||||||
OafMapperUtils.qualifier(CLASS_ID_ADVANCED_CONSTRAINT, CLASS_NAME_BULKTAG_ADVANCED_CONSTRAINT, DNET_PROVENANCE_ACTIONS, DNET_PROVENANCE_ACTIONS), TAGGING_TRUST));
|
.dataInfo(
|
||||||
|
false, BULKTAG_DATA_INFO_TYPE, true, false,
|
||||||
|
OafMapperUtils
|
||||||
|
.qualifier(
|
||||||
|
CLASS_ID_ADVANCED_CONSTRAINT, CLASS_NAME_BULKTAG_ADVANCED_CONSTRAINT,
|
||||||
|
DNET_PROVENANCE_ACTIONS, DNET_PROVENANCE_ACTIONS),
|
||||||
|
TAGGING_TRUST));
|
||||||
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
@ -198,21 +231,48 @@ public class ResultTagger implements Serializable {
|
||||||
List<DataInfo> dataInfoList = new ArrayList<>();
|
List<DataInfo> dataInfoList = new ArrayList<>();
|
||||||
if (subjects.contains(c))
|
if (subjects.contains(c))
|
||||||
dataInfoList
|
dataInfoList
|
||||||
.add(OafMapperUtils.dataInfo(false, BULKTAG_DATA_INFO_TYPE, true, false,
|
.add(
|
||||||
OafMapperUtils.qualifier(CLASS_ID_SUBJECT, CLASS_NAME_BULKTAG_SUBJECT, DNET_PROVENANCE_ACTIONS, DNET_PROVENANCE_ACTIONS), TAGGING_TRUST));
|
OafMapperUtils
|
||||||
|
.dataInfo(
|
||||||
|
false, BULKTAG_DATA_INFO_TYPE, true, false,
|
||||||
|
OafMapperUtils
|
||||||
|
.qualifier(
|
||||||
|
CLASS_ID_SUBJECT, CLASS_NAME_BULKTAG_SUBJECT, DNET_PROVENANCE_ACTIONS,
|
||||||
|
DNET_PROVENANCE_ACTIONS),
|
||||||
|
TAGGING_TRUST));
|
||||||
if (datasources.contains(c))
|
if (datasources.contains(c))
|
||||||
dataInfoList
|
dataInfoList
|
||||||
.add(OafMapperUtils.dataInfo(false, BULKTAG_DATA_INFO_TYPE, true, false,
|
.add(
|
||||||
OafMapperUtils.qualifier(CLASS_ID_DATASOURCE, CLASS_NAME_BULKTAG_DATASOURCE, DNET_PROVENANCE_ACTIONS, DNET_PROVENANCE_ACTIONS), TAGGING_TRUST));
|
OafMapperUtils
|
||||||
|
.dataInfo(
|
||||||
|
false, BULKTAG_DATA_INFO_TYPE, true, false,
|
||||||
|
OafMapperUtils
|
||||||
|
.qualifier(
|
||||||
|
CLASS_ID_DATASOURCE, CLASS_NAME_BULKTAG_DATASOURCE,
|
||||||
|
DNET_PROVENANCE_ACTIONS, DNET_PROVENANCE_ACTIONS),
|
||||||
|
TAGGING_TRUST));
|
||||||
if (czenodo.contains(c))
|
if (czenodo.contains(c))
|
||||||
dataInfoList
|
dataInfoList
|
||||||
.add(OafMapperUtils.dataInfo(false, BULKTAG_DATA_INFO_TYPE, true, false,
|
.add(
|
||||||
OafMapperUtils.qualifier(CLASS_ID_CZENODO, CLASS_NAME_BULKTAG_ZENODO, DNET_PROVENANCE_ACTIONS, DNET_PROVENANCE_ACTIONS), TAGGING_TRUST));
|
OafMapperUtils
|
||||||
|
.dataInfo(
|
||||||
|
false, BULKTAG_DATA_INFO_TYPE, true, false,
|
||||||
|
OafMapperUtils
|
||||||
|
.qualifier(
|
||||||
|
CLASS_ID_CZENODO, CLASS_NAME_BULKTAG_ZENODO, DNET_PROVENANCE_ACTIONS,
|
||||||
|
DNET_PROVENANCE_ACTIONS),
|
||||||
|
TAGGING_TRUST));
|
||||||
if (aconstraints.contains(c))
|
if (aconstraints.contains(c))
|
||||||
dataInfoList
|
dataInfoList
|
||||||
.add(
|
.add(
|
||||||
OafMapperUtils.dataInfo(false, BULKTAG_DATA_INFO_TYPE, true, false,
|
OafMapperUtils
|
||||||
OafMapperUtils.qualifier(CLASS_ID_ADVANCED_CONSTRAINT, CLASS_NAME_BULKTAG_ADVANCED_CONSTRAINT, DNET_PROVENANCE_ACTIONS, DNET_PROVENANCE_ACTIONS), TAGGING_TRUST));
|
.dataInfo(
|
||||||
|
false, BULKTAG_DATA_INFO_TYPE, true, false,
|
||||||
|
OafMapperUtils
|
||||||
|
.qualifier(
|
||||||
|
CLASS_ID_ADVANCED_CONSTRAINT, CLASS_NAME_BULKTAG_ADVANCED_CONSTRAINT,
|
||||||
|
DNET_PROVENANCE_ACTIONS, DNET_PROVENANCE_ACTIONS),
|
||||||
|
TAGGING_TRUST));
|
||||||
|
|
||||||
context.setDataInfo(dataInfoList);
|
context.setDataInfo(dataInfoList);
|
||||||
return context;
|
return context;
|
||||||
|
|
|
@ -11,11 +11,6 @@ import java.util.HashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
import com.google.gson.Gson;
|
|
||||||
import com.jayway.jsonpath.DocumentContext;
|
|
||||||
import com.jayway.jsonpath.JsonPath;
|
|
||||||
import eu.dnetlib.dhp.bulktag.community.ProtoMap;
|
|
||||||
import eu.dnetlib.dhp.schema.oaf.*;
|
|
||||||
import org.apache.commons.io.FileUtils;
|
import org.apache.commons.io.FileUtils;
|
||||||
import org.apache.commons.io.IOUtils;
|
import org.apache.commons.io.IOUtils;
|
||||||
import org.apache.spark.SparkConf;
|
import org.apache.spark.SparkConf;
|
||||||
|
@ -32,6 +27,12 @@ import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import com.google.gson.Gson;
|
||||||
|
import com.jayway.jsonpath.DocumentContext;
|
||||||
|
import com.jayway.jsonpath.JsonPath;
|
||||||
|
|
||||||
|
import eu.dnetlib.dhp.bulktag.community.ProtoMap;
|
||||||
|
import eu.dnetlib.dhp.schema.oaf.*;
|
||||||
|
|
||||||
public class BulkTagJobTest {
|
public class BulkTagJobTest {
|
||||||
|
|
||||||
|
@ -44,7 +45,7 @@ public class BulkTagJobTest {
|
||||||
+ " \"orcid\" : \"$['author'][*]['pid'][*][?(@['key']=='ORCID')]['value']\","
|
+ " \"orcid\" : \"$['author'][*]['pid'][*][?(@['key']=='ORCID')]['value']\","
|
||||||
+ " \"contributor\" : \"$['contributor'][*]['value']\","
|
+ " \"contributor\" : \"$['contributor'][*]['value']\","
|
||||||
+ " \"description\" : \"$['description'][*]['value']\", "
|
+ " \"description\" : \"$['description'][*]['value']\", "
|
||||||
+" \"subject\" :\"$['subject'][*]['value']\" }";
|
+ " \"subject\" :\"$['subject'][*]['value']\" }";
|
||||||
|
|
||||||
private static SparkSession spark;
|
private static SparkSession spark;
|
||||||
|
|
||||||
|
|
|
@ -10,7 +10,6 @@ import java.util.stream.Collectors;
|
||||||
|
|
||||||
import javax.swing.text.html.Option;
|
import javax.swing.text.html.Option;
|
||||||
|
|
||||||
import eu.dnetlib.dhp.schema.oaf.utils.PidType;
|
|
||||||
import org.apache.commons.io.IOUtils;
|
import org.apache.commons.io.IOUtils;
|
||||||
import org.apache.spark.SparkConf;
|
import org.apache.spark.SparkConf;
|
||||||
import org.apache.spark.api.java.function.FilterFunction;
|
import org.apache.spark.api.java.function.FilterFunction;
|
||||||
|
@ -33,6 +32,7 @@ import eu.dnetlib.dhp.oa.graph.clean.CleanContextSparkJob;
|
||||||
import eu.dnetlib.dhp.schema.oaf.Country;
|
import eu.dnetlib.dhp.schema.oaf.Country;
|
||||||
import eu.dnetlib.dhp.schema.oaf.Result;
|
import eu.dnetlib.dhp.schema.oaf.Result;
|
||||||
import eu.dnetlib.dhp.schema.oaf.StructuredProperty;
|
import eu.dnetlib.dhp.schema.oaf.StructuredProperty;
|
||||||
|
import eu.dnetlib.dhp.schema.oaf.utils.PidType;
|
||||||
|
|
||||||
public class CleanCountrySparkJob implements Serializable {
|
public class CleanCountrySparkJob implements Serializable {
|
||||||
private static final Logger log = LoggerFactory.getLogger(CleanCountrySparkJob.class);
|
private static final Logger log = LoggerFactory.getLogger(CleanCountrySparkJob.class);
|
||||||
|
@ -113,7 +113,10 @@ public class CleanCountrySparkJob implements Serializable {
|
||||||
if (r
|
if (r
|
||||||
.getPid()
|
.getPid()
|
||||||
.stream()
|
.stream()
|
||||||
.anyMatch(p -> p.getQualifier().getClassid()
|
.anyMatch(
|
||||||
|
p -> p
|
||||||
|
.getQualifier()
|
||||||
|
.getClassid()
|
||||||
.equals(PidType.doi) && pidInParam(p.getValue(), verifyParam))) {
|
.equals(PidType.doi) && pidInParam(p.getValue(), verifyParam))) {
|
||||||
r
|
r
|
||||||
.setCountry(
|
.setCountry(
|
||||||
|
|
|
@ -422,7 +422,8 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i
|
||||||
|
|
||||||
final Relation r2 = OafMapperUtils
|
final Relation r2 = OafMapperUtils
|
||||||
.getRelation(
|
.getRelation(
|
||||||
orgId, dsId, DATASOURCE_ORGANIZATION, PROVISION, PROVIDES, collectedFrom, info, lastUpdateTimestamp);
|
orgId, dsId, DATASOURCE_ORGANIZATION, PROVISION, PROVIDES, collectedFrom, info,
|
||||||
|
lastUpdateTimestamp);
|
||||||
|
|
||||||
return Arrays.asList(r1, r2);
|
return Arrays.asList(r1, r2);
|
||||||
} catch (final Exception e) {
|
} catch (final Exception e) {
|
||||||
|
|
Loading…
Reference in New Issue