forked from D-Net/dnet-hadoop
Merge branch 'beta' of https://code-repo.d4science.org/D-Net/dnet-hadoop into beta
This commit is contained in:
commit
0c74f5667e
|
@ -52,4 +52,10 @@ public class Constants {
|
||||||
public static final String CONTENT_INVALIDRECORDS = "InvalidRecords";
|
public static final String CONTENT_INVALIDRECORDS = "InvalidRecords";
|
||||||
public static final String CONTENT_TRANSFORMEDRECORDS = "transformedItems";
|
public static final String CONTENT_TRANSFORMEDRECORDS = "transformedItems";
|
||||||
|
|
||||||
|
// IETF Draft and used by Repositories like ZENODO , not included in APACHE HTTP java packages
|
||||||
|
// see https://ietf-wg-httpapi.github.io/ratelimit-headers/draft-ietf-httpapi-ratelimit-headers.html
|
||||||
|
public static final String HTTPHEADER_IETF_DRAFT_RATELIMIT_LIMIT = "X-RateLimit-Limit";
|
||||||
|
public static final String HTTPHEADER_IETF_DRAFT_RATELIMIT_REMAINING = "X-RateLimit-Remaining";
|
||||||
|
public static final String HTTPHEADER_IETF_DRAFT_RATELIMIT_RESET = "X-RateLimit-Reset";
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -15,12 +15,13 @@ import org.apache.http.HttpHeaders;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import eu.dnetlib.dhp.common.Constants;
|
||||||
import eu.dnetlib.dhp.common.aggregation.AggregatorReport;
|
import eu.dnetlib.dhp.common.aggregation.AggregatorReport;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Migrated from https://svn.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-modular-collector-service/trunk/src/main/java/eu/dnetlib/data/collector/plugins/HttpConnector.java
|
* Migrated from https://svn.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-modular-collector-service/trunk/src/main/java/eu/dnetlib/data/collector/plugins/HttpConnector.java
|
||||||
*
|
*
|
||||||
* @author jochen, michele, andrea, alessia, claudio
|
* @author jochen, michele, andrea, alessia, claudio, andreas
|
||||||
*/
|
*/
|
||||||
public class HttpConnector2 {
|
public class HttpConnector2 {
|
||||||
|
|
||||||
|
@ -112,6 +113,17 @@ public class HttpConnector2 {
|
||||||
}
|
}
|
||||||
|
|
||||||
int retryAfter = obtainRetryAfter(urlConn.getHeaderFields());
|
int retryAfter = obtainRetryAfter(urlConn.getHeaderFields());
|
||||||
|
String rateLimit = urlConn.getHeaderField(Constants.HTTPHEADER_IETF_DRAFT_RATELIMIT_LIMIT);
|
||||||
|
String rateRemaining = urlConn.getHeaderField(Constants.HTTPHEADER_IETF_DRAFT_RATELIMIT_REMAINING);
|
||||||
|
|
||||||
|
if ((rateLimit != null) && (rateRemaining != null) && (Integer.parseInt(rateRemaining) < 2)) {
|
||||||
|
if (retryAfter > 0) {
|
||||||
|
backoffAndSleep(retryAfter);
|
||||||
|
} else {
|
||||||
|
backoffAndSleep(1000);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (is2xx(urlConn.getResponseCode())) {
|
if (is2xx(urlConn.getResponseCode())) {
|
||||||
input = urlConn.getInputStream();
|
input = urlConn.getInputStream();
|
||||||
responseType = urlConn.getContentType();
|
responseType = urlConn.getContentType();
|
||||||
|
|
|
@ -66,11 +66,11 @@ public class OaiCollectorPlugin implements CollectorPlugin {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (fromDate != null && !fromDate.matches(DATE_REGEX) && !fromDate.matches(UTC_DATETIME_REGEX)) {
|
if (fromDate != null && !fromDate.matches(DATE_REGEX) && !fromDate.matches(UTC_DATETIME_REGEX)) {
|
||||||
throw new CollectorException("Invalid date (YYYY-MM-DD): " + fromDate);
|
throw new CollectorException("Invalid date (YYYY-MM-DD or YYYY-MM-DDT00:00:00Z): " + fromDate);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (untilDate != null && !untilDate.matches(DATE_REGEX) && !untilDate.matches(UTC_DATETIME_REGEX)) {
|
if (untilDate != null && !untilDate.matches(DATE_REGEX) && !untilDate.matches(UTC_DATETIME_REGEX)) {
|
||||||
throw new CollectorException("Invalid date (YYYY-MM-DD): " + untilDate);
|
throw new CollectorException("Invalid date (YYYY-MM-DD or YYYY-MM-DDT00:00:00Z): " + untilDate);
|
||||||
}
|
}
|
||||||
|
|
||||||
final Iterator<Iterator<String>> iters = sets
|
final Iterator<Iterator<String>> iters = sets
|
||||||
|
|
|
@ -68,27 +68,28 @@ public class PropagationConstant {
|
||||||
getDataInfo(
|
getDataInfo(
|
||||||
PROPAGATION_DATA_INFO_TYPE,
|
PROPAGATION_DATA_INFO_TYPE,
|
||||||
PROPAGATION_COUNTRY_INSTREPO_CLASS_ID,
|
PROPAGATION_COUNTRY_INSTREPO_CLASS_ID,
|
||||||
PROPAGATION_COUNTRY_INSTREPO_CLASS_NAME));
|
PROPAGATION_COUNTRY_INSTREPO_CLASS_NAME,
|
||||||
|
ModelConstants.DNET_PROVENANCE_ACTIONS));
|
||||||
return nc;
|
return nc;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static DataInfo getDataInfo(
|
public static DataInfo getDataInfo(
|
||||||
String inference_provenance, String inference_class_id, String inference_class_name) {
|
String inference_provenance, String inference_class_id, String inference_class_name, String qualifierSchema) {
|
||||||
DataInfo di = new DataInfo();
|
DataInfo di = new DataInfo();
|
||||||
di.setInferred(true);
|
di.setInferred(true);
|
||||||
di.setDeletedbyinference(false);
|
di.setDeletedbyinference(false);
|
||||||
di.setTrust("0.85");
|
di.setTrust("0.85");
|
||||||
di.setInferenceprovenance(inference_provenance);
|
di.setInferenceprovenance(inference_provenance);
|
||||||
di.setProvenanceaction(getQualifier(inference_class_id, inference_class_name));
|
di.setProvenanceaction(getQualifier(inference_class_id, inference_class_name, qualifierSchema));
|
||||||
return di;
|
return di;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static Qualifier getQualifier(String inference_class_id, String inference_class_name) {
|
public static Qualifier getQualifier(String inference_class_id, String inference_class_name, String qualifierSchema) {
|
||||||
Qualifier pa = new Qualifier();
|
Qualifier pa = new Qualifier();
|
||||||
pa.setClassid(inference_class_id);
|
pa.setClassid(inference_class_id);
|
||||||
pa.setClassname(inference_class_name);
|
pa.setClassname(inference_class_name);
|
||||||
pa.setSchemeid(ModelConstants.DNET_PID_TYPES);
|
pa.setSchemeid(qualifierSchema);
|
||||||
pa.setSchemename(ModelConstants.DNET_PID_TYPES);
|
pa.setSchemename(qualifierSchema);
|
||||||
return pa;
|
return pa;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -107,7 +108,7 @@ public class PropagationConstant {
|
||||||
r.setRelClass(rel_class);
|
r.setRelClass(rel_class);
|
||||||
r.setRelType(rel_type);
|
r.setRelType(rel_type);
|
||||||
r.setSubRelType(subrel_type);
|
r.setSubRelType(subrel_type);
|
||||||
r.setDataInfo(getDataInfo(inference_provenance, inference_class_id, inference_class_name));
|
r.setDataInfo(getDataInfo(inference_provenance, inference_class_id, inference_class_name, ModelConstants.DNET_PROVENANCE_ACTIONS));
|
||||||
return r;
|
return r;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -173,13 +173,14 @@ public class SparkOrcidToResultFromSemRelJob {
|
||||||
if (toaddpid) {
|
if (toaddpid) {
|
||||||
StructuredProperty p = new StructuredProperty();
|
StructuredProperty p = new StructuredProperty();
|
||||||
p.setValue(autoritative_author.getOrcid());
|
p.setValue(autoritative_author.getOrcid());
|
||||||
p.setQualifier(getQualifier(ModelConstants.ORCID_PENDING, ModelConstants.ORCID_CLASSNAME));
|
p.setQualifier(getQualifier(ModelConstants.ORCID_PENDING, ModelConstants.ORCID_CLASSNAME, ModelConstants.DNET_PID_TYPES));
|
||||||
p
|
p
|
||||||
.setDataInfo(
|
.setDataInfo(
|
||||||
getDataInfo(
|
getDataInfo(
|
||||||
PROPAGATION_DATA_INFO_TYPE,
|
PROPAGATION_DATA_INFO_TYPE,
|
||||||
PROPAGATION_ORCID_TO_RESULT_FROM_SEM_REL_CLASS_ID,
|
PROPAGATION_ORCID_TO_RESULT_FROM_SEM_REL_CLASS_ID,
|
||||||
PROPAGATION_ORCID_TO_RESULT_FROM_SEM_REL_CLASS_NAME));
|
PROPAGATION_ORCID_TO_RESULT_FROM_SEM_REL_CLASS_NAME,
|
||||||
|
ModelConstants.DNET_PROVENANCE_ACTIONS));
|
||||||
|
|
||||||
Optional<List<StructuredProperty>> authorPid = Optional.ofNullable(author.getPid());
|
Optional<List<StructuredProperty>> authorPid = Optional.ofNullable(author.getPid());
|
||||||
if (authorPid.isPresent()) {
|
if (authorPid.isPresent()) {
|
||||||
|
|
|
@ -10,6 +10,7 @@ import java.util.List;
|
||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
import eu.dnetlib.dhp.schema.common.ModelConstants;
|
||||||
import org.apache.commons.io.IOUtils;
|
import org.apache.commons.io.IOUtils;
|
||||||
import org.apache.spark.SparkConf;
|
import org.apache.spark.SparkConf;
|
||||||
import org.apache.spark.api.java.function.MapFunction;
|
import org.apache.spark.api.java.function.MapFunction;
|
||||||
|
@ -128,7 +129,8 @@ public class SparkResultToCommunityFromOrganizationJob {
|
||||||
getDataInfo(
|
getDataInfo(
|
||||||
PROPAGATION_DATA_INFO_TYPE,
|
PROPAGATION_DATA_INFO_TYPE,
|
||||||
PROPAGATION_RESULT_COMMUNITY_ORGANIZATION_CLASS_ID,
|
PROPAGATION_RESULT_COMMUNITY_ORGANIZATION_CLASS_ID,
|
||||||
PROPAGATION_RESULT_COMMUNITY_ORGANIZATION_CLASS_NAME)));
|
PROPAGATION_RESULT_COMMUNITY_ORGANIZATION_CLASS_NAME,
|
||||||
|
ModelConstants.DNET_PROVENANCE_ACTIONS)));
|
||||||
propagatedContexts.add(newContext);
|
propagatedContexts.add(newContext);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -7,6 +7,7 @@ import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkHiveSession;
|
||||||
import java.util.*;
|
import java.util.*;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
import eu.dnetlib.dhp.schema.common.ModelConstants;
|
||||||
import org.apache.commons.io.IOUtils;
|
import org.apache.commons.io.IOUtils;
|
||||||
import org.apache.spark.SparkConf;
|
import org.apache.spark.SparkConf;
|
||||||
import org.apache.spark.api.java.function.MapFunction;
|
import org.apache.spark.api.java.function.MapFunction;
|
||||||
|
@ -124,7 +125,8 @@ public class SparkResultToCommunityThroughSemRelJob {
|
||||||
getDataInfo(
|
getDataInfo(
|
||||||
PROPAGATION_DATA_INFO_TYPE,
|
PROPAGATION_DATA_INFO_TYPE,
|
||||||
PROPAGATION_RESULT_COMMUNITY_SEMREL_CLASS_ID,
|
PROPAGATION_RESULT_COMMUNITY_SEMREL_CLASS_ID,
|
||||||
PROPAGATION_RESULT_COMMUNITY_SEMREL_CLASS_NAME)));
|
PROPAGATION_RESULT_COMMUNITY_SEMREL_CLASS_NAME,
|
||||||
|
ModelConstants.DNET_PROVENANCE_ACTIONS)));
|
||||||
return newContext;
|
return newContext;
|
||||||
}
|
}
|
||||||
return null;
|
return null;
|
||||||
|
|
|
@ -12,6 +12,7 @@ import eu.dnetlib.dhp.schema.common.ModelConstants;
|
||||||
import eu.dnetlib.dhp.schema.oaf.AccessRight;
|
import eu.dnetlib.dhp.schema.oaf.AccessRight;
|
||||||
import eu.dnetlib.dhp.schema.oaf.Country;
|
import eu.dnetlib.dhp.schema.oaf.Country;
|
||||||
import eu.dnetlib.dhp.schema.oaf.Qualifier;
|
import eu.dnetlib.dhp.schema.oaf.Qualifier;
|
||||||
|
import eu.dnetlib.dhp.schema.oaf.Relation;
|
||||||
|
|
||||||
public class CleaningRuleMap extends HashMap<Class<?>, SerializableConsumer<Object>> implements Serializable {
|
public class CleaningRuleMap extends HashMap<Class<?>, SerializableConsumer<Object>> implements Serializable {
|
||||||
|
|
||||||
|
@ -24,15 +25,29 @@ public class CleaningRuleMap extends HashMap<Class<?>, SerializableConsumer<Obje
|
||||||
CleaningRuleMap mapping = new CleaningRuleMap();
|
CleaningRuleMap mapping = new CleaningRuleMap();
|
||||||
mapping.put(Qualifier.class, o -> cleanQualifier(vocabularies, (Qualifier) o));
|
mapping.put(Qualifier.class, o -> cleanQualifier(vocabularies, (Qualifier) o));
|
||||||
mapping.put(AccessRight.class, o -> cleanQualifier(vocabularies, (AccessRight) o));
|
mapping.put(AccessRight.class, o -> cleanQualifier(vocabularies, (AccessRight) o));
|
||||||
mapping.put(Country.class, o -> {
|
mapping.put(Country.class, o -> cleanCountry(vocabularies, (Country) o));
|
||||||
final Country c = (Country) o;
|
mapping.put(Relation.class, o -> cleanRelation(vocabularies, (Relation) o));
|
||||||
|
return mapping;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void cleanRelation(VocabularyGroup vocabularies, Relation r) {
|
||||||
|
if (vocabularies.vocabularyExists(ModelConstants.DNET_RELATION_SUBRELTYPE)) {
|
||||||
|
Qualifier newValue = vocabularies.lookup(ModelConstants.DNET_RELATION_SUBRELTYPE, r.getSubRelType());
|
||||||
|
r.setSubRelType(newValue.getClassid());
|
||||||
|
}
|
||||||
|
if (vocabularies.vocabularyExists(ModelConstants.DNET_RELATION_RELCLASS)) {
|
||||||
|
Qualifier newValue = vocabularies.lookup(ModelConstants.DNET_RELATION_RELCLASS, r.getRelClass());
|
||||||
|
r.setRelClass(newValue.getClassid());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void cleanCountry(VocabularyGroup vocabularies, Country o) {
|
||||||
|
final Country c = o;
|
||||||
if (StringUtils.isBlank(c.getSchemeid())) {
|
if (StringUtils.isBlank(c.getSchemeid())) {
|
||||||
c.setSchemeid(ModelConstants.DNET_COUNTRY_TYPE);
|
c.setSchemeid(ModelConstants.DNET_COUNTRY_TYPE);
|
||||||
c.setSchemename(ModelConstants.DNET_COUNTRY_TYPE);
|
c.setSchemename(ModelConstants.DNET_COUNTRY_TYPE);
|
||||||
}
|
}
|
||||||
cleanQualifier(vocabularies, c);
|
cleanQualifier(vocabularies, c);
|
||||||
});
|
|
||||||
return mapping;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private static <Q extends Qualifier> void cleanQualifier(VocabularyGroup vocabularies, Q q) {
|
private static <Q extends Qualifier> void cleanQualifier(VocabularyGroup vocabularies, Q q) {
|
||||||
|
|
|
@ -5,6 +5,7 @@ import static org.junit.jupiter.api.Assertions.*;
|
||||||
import static org.mockito.Mockito.lenient;
|
import static org.mockito.Mockito.lenient;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
import java.util.Collection;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.stream.Stream;
|
import java.util.stream.Stream;
|
||||||
|
@ -16,12 +17,12 @@ import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
import org.mockito.Mock;
|
import org.mockito.Mock;
|
||||||
import org.mockito.junit.jupiter.MockitoExtension;
|
import org.mockito.junit.jupiter.MockitoExtension;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.databind.DeserializationFeature;
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
|
||||||
import eu.dnetlib.dhp.common.vocabulary.VocabularyGroup;
|
import eu.dnetlib.dhp.common.vocabulary.VocabularyGroup;
|
||||||
import eu.dnetlib.dhp.schema.common.ModelConstants;
|
import eu.dnetlib.dhp.schema.common.ModelConstants;
|
||||||
import eu.dnetlib.dhp.schema.oaf.*;
|
import eu.dnetlib.dhp.schema.oaf.*;
|
||||||
import eu.dnetlib.dhp.schema.oaf.utils.CleaningFunctions;
|
|
||||||
import eu.dnetlib.dhp.schema.oaf.utils.GraphCleaningFunctions;
|
import eu.dnetlib.dhp.schema.oaf.utils.GraphCleaningFunctions;
|
||||||
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
|
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
|
||||||
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
|
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
|
||||||
|
@ -29,7 +30,8 @@ import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
|
||||||
@ExtendWith(MockitoExtension.class)
|
@ExtendWith(MockitoExtension.class)
|
||||||
public class GraphCleaningFunctionsTest {
|
public class GraphCleaningFunctionsTest {
|
||||||
|
|
||||||
public static final ObjectMapper MAPPER = new ObjectMapper();
|
public static final ObjectMapper MAPPER = new ObjectMapper()
|
||||||
|
.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
|
||||||
|
|
||||||
@Mock
|
@Mock
|
||||||
private ISLookUpService isLookUpService;
|
private ISLookUpService isLookUpService;
|
||||||
|
@ -49,6 +51,23 @@ public class GraphCleaningFunctionsTest {
|
||||||
mapping = CleaningRuleMap.create(vocabularies);
|
mapping = CleaningRuleMap.create(vocabularies);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testCleanRelations() throws Exception {
|
||||||
|
|
||||||
|
List<String> lines = IOUtils
|
||||||
|
.readLines(getClass().getResourceAsStream("/eu/dnetlib/dhp/oa/graph/clean/relation.json"));
|
||||||
|
for (String json : lines) {
|
||||||
|
Relation r_in = MAPPER.readValue(json, Relation.class);
|
||||||
|
assertNotNull(r_in);
|
||||||
|
|
||||||
|
assertFalse(vocabularies.getTerms(ModelConstants.DNET_RELATION_RELCLASS).contains(r_in.getRelClass()));
|
||||||
|
|
||||||
|
Relation r_out = OafCleaner.apply(r_in, mapping);
|
||||||
|
assertTrue(vocabularies.getTerms(ModelConstants.DNET_RELATION_RELCLASS).contains(r_out.getRelClass()));
|
||||||
|
assertTrue(vocabularies.getTerms(ModelConstants.DNET_RELATION_SUBRELTYPE).contains(r_out.getSubRelType()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void testCleaning() throws Exception {
|
void testCleaning() throws Exception {
|
||||||
|
|
||||||
|
@ -87,7 +106,7 @@ public class GraphCleaningFunctionsTest {
|
||||||
p_out
|
p_out
|
||||||
.getPid()
|
.getPid()
|
||||||
.stream()
|
.stream()
|
||||||
.map(p -> p.getQualifier())
|
.map(StructuredProperty::getQualifier)
|
||||||
.allMatch(q -> pidTerms.contains(q.getClassid())));
|
.allMatch(q -> pidTerms.contains(q.getClassid())));
|
||||||
|
|
||||||
List<Instance> poi = p_out.getInstance();
|
List<Instance> poi = p_out.getInstance();
|
||||||
|
@ -101,8 +120,8 @@ public class GraphCleaningFunctionsTest {
|
||||||
assertEquals(2, poii.getPid().size());
|
assertEquals(2, poii.getPid().size());
|
||||||
|
|
||||||
assertTrue(
|
assertTrue(
|
||||||
poii.getPid().stream().filter(s -> s.getValue().equals("10.1007/s109090161569x")).findFirst().isPresent());
|
poii.getPid().stream().anyMatch(s -> s.getValue().equals("10.1007/s109090161569x")));
|
||||||
assertTrue(poii.getPid().stream().filter(s -> s.getValue().equals("10.1008/abcd")).findFirst().isPresent());
|
assertTrue(poii.getPid().stream().anyMatch(s -> s.getValue().equals("10.1008/abcd")));
|
||||||
|
|
||||||
assertNotNull(poii.getAlternateIdentifier());
|
assertNotNull(poii.getAlternateIdentifier());
|
||||||
assertEquals(2, poii.getAlternateIdentifier().size());
|
assertEquals(2, poii.getAlternateIdentifier().size());
|
||||||
|
@ -111,16 +130,12 @@ public class GraphCleaningFunctionsTest {
|
||||||
poii
|
poii
|
||||||
.getAlternateIdentifier()
|
.getAlternateIdentifier()
|
||||||
.stream()
|
.stream()
|
||||||
.filter(s -> s.getValue().equals("10.1007/s109090161569x"))
|
.anyMatch(s -> s.getValue().equals("10.1007/s109090161569x")));
|
||||||
.findFirst()
|
|
||||||
.isPresent());
|
|
||||||
assertTrue(
|
assertTrue(
|
||||||
poii
|
poii
|
||||||
.getAlternateIdentifier()
|
.getAlternateIdentifier()
|
||||||
.stream()
|
.stream()
|
||||||
.filter(s -> s.getValue().equals("10.1009/qwerty"))
|
.anyMatch(s -> s.getValue().equals("10.1009/qwerty")));
|
||||||
.findFirst()
|
|
||||||
.isPresent());
|
|
||||||
|
|
||||||
Publication p_cleaned = GraphCleaningFunctions.cleanup(p_out);
|
Publication p_cleaned = GraphCleaningFunctions.cleanup(p_out);
|
||||||
|
|
||||||
|
@ -142,8 +157,8 @@ public class GraphCleaningFunctionsTest {
|
||||||
assertEquals(2, pcii.getPid().size());
|
assertEquals(2, pcii.getPid().size());
|
||||||
|
|
||||||
assertTrue(
|
assertTrue(
|
||||||
pcii.getPid().stream().filter(s -> s.getValue().equals("10.1007/s109090161569x")).findFirst().isPresent());
|
pcii.getPid().stream().anyMatch(s -> s.getValue().equals("10.1007/s109090161569x")));
|
||||||
assertTrue(pcii.getPid().stream().filter(s -> s.getValue().equals("10.1008/abcd")).findFirst().isPresent());
|
assertTrue(pcii.getPid().stream().anyMatch(s -> s.getValue().equals("10.1008/abcd")));
|
||||||
|
|
||||||
assertNotNull(pcii.getAlternateIdentifier());
|
assertNotNull(pcii.getAlternateIdentifier());
|
||||||
assertEquals(1, pcii.getAlternateIdentifier().size());
|
assertEquals(1, pcii.getAlternateIdentifier().size());
|
||||||
|
@ -151,9 +166,7 @@ public class GraphCleaningFunctionsTest {
|
||||||
pcii
|
pcii
|
||||||
.getAlternateIdentifier()
|
.getAlternateIdentifier()
|
||||||
.stream()
|
.stream()
|
||||||
.filter(s -> s.getValue().equals("10.1009/qwerty"))
|
.anyMatch(s -> s.getValue().equals("10.1009/qwerty")));
|
||||||
.findFirst()
|
|
||||||
.isPresent());
|
|
||||||
|
|
||||||
getAuthorPids(p_cleaned).forEach(pid -> {
|
getAuthorPids(p_cleaned).forEach(pid -> {
|
||||||
System.out
|
System.out
|
||||||
|
@ -172,17 +185,17 @@ public class GraphCleaningFunctionsTest {
|
||||||
return pub
|
return pub
|
||||||
.getAuthor()
|
.getAuthor()
|
||||||
.stream()
|
.stream()
|
||||||
.map(a -> a.getPid())
|
.map(Author::getPid)
|
||||||
.flatMap(p -> p.stream())
|
.flatMap(Collection::stream)
|
||||||
.map(s -> s.getQualifier());
|
.map(StructuredProperty::getQualifier);
|
||||||
}
|
}
|
||||||
|
|
||||||
private Stream<StructuredProperty> getAuthorPids(Result pub) {
|
private Stream<StructuredProperty> getAuthorPids(Result pub) {
|
||||||
return pub
|
return pub
|
||||||
.getAuthor()
|
.getAuthor()
|
||||||
.stream()
|
.stream()
|
||||||
.map(a -> a.getPid())
|
.map(Author::getPid)
|
||||||
.flatMap(p -> p.stream());
|
.flatMap(Collection::stream);
|
||||||
}
|
}
|
||||||
|
|
||||||
private List<String> vocs() throws IOException {
|
private List<String> vocs() throws IOException {
|
||||||
|
|
|
@ -0,0 +1,10 @@
|
||||||
|
{"relType":"resultResult","subRelType":"citation","relClass":"cites","source":"50|4ScienceCRIS::f66f1bd369679b5b077dcdf006089556","target":"50|openaire____::007a4870b31056f89b768cf508e1538e"}
|
||||||
|
{"relType":"resultResult","subRelType":"citation","relClass":"isCitedBy","source":"50|openaire____::007a4870b31056f89b768cf508e1538e","target":"50|4ScienceCRIS::f66f1bd369679b5b077dcdf006089556"}
|
||||||
|
{"relType":"resultResult","subRelType":"supplement","relClass":"isSupplementTo","source":"50|openaire____::007a4870b31056f89b768cf508e1538e","target":"50|4ScienceCRIS::f66f1bd369679b5b077dcdf006089556"}
|
||||||
|
{"relType":"resultResult","subRelType":"supplement","relClass":"isSupplementedBy","source":"50|openaire____::007a4870b31056f89b768cf508e1538e","target":"50|4ScienceCRIS::f66f1bd369679b5b077dcdf006089556"}
|
||||||
|
{"relType":"resultResult","subRelType":"part","relClass":"isPartOf","source":"50|openaire____::007a4870b31056f89b768cf508e1538e","target":"50|4ScienceCRIS::f66f1bd369679b5b077dcdf006089556"}
|
||||||
|
{"relType":"resultResult","subRelType":"part","relClass":"hasPart","source":"50|openaire____::007a4870b31056f89b768cf508e1538e","target":"50|4ScienceCRIS::f66f1bd369679b5b077dcdf006089556"}
|
||||||
|
{"relType":"resultResult","subRelType":"review","relClass":"isReviewedBy","source":"50|openaire____::007a4870b31056f89b768cf508e1538e","target":"50|4ScienceCRIS::f66f1bd369679b5b077dcdf006089556"}
|
||||||
|
{"relType":"resultResult","subRelType":"review","relClass":"reviews","source":"50|openaire____::007a4870b31056f89b768cf508e1538e","target":"50|4ScienceCRIS::f66f1bd369679b5b077dcdf006089556"}
|
||||||
|
{"relType":"resultResult","subRelType":"relationship","relClass":"isRelatedTo","source":"50|openaire____::007a4870b31056f89b768cf508e1538e","target":"50|4ScienceCRIS::f66f1bd369679b5b077dcdf006089556"}
|
||||||
|
{"relType":"resultResult","subRelType":"publicationDataset","relClass":"isRelatedTo","source":"50|openaire____::007a4870b31056f89b768cf508e1538e","target":"50|4ScienceCRIS::f66f1bd369679b5b077dcdf006089556"}
|
|
@ -1232,3 +1232,13 @@ dnet:review_levels @=@ 0001 @=@ 印刷物/電子媒体-紀要論文(査読有
|
||||||
dnet:review_levels @=@ 0001 @=@ 印刷物/電子媒体-雑誌記事(査読有り)
|
dnet:review_levels @=@ 0001 @=@ 印刷物/電子媒体-雑誌記事(査読有り)
|
||||||
dnet:review_levels @=@ 0001 @=@ 原著論文(査読有り)
|
dnet:review_levels @=@ 0001 @=@ 原著論文(査読有り)
|
||||||
dnet:review_levels @=@ 0001 @=@ 査読論文
|
dnet:review_levels @=@ 0001 @=@ 査読論文
|
||||||
|
dnet:relation_relClass @=@ Cites @=@ cites
|
||||||
|
dnet:relation_relClass @=@ IsCitedBy @=@ isCitedBy
|
||||||
|
dnet:relation_relClass @=@ HasPart @=@ hasPart
|
||||||
|
dnet:relation_relClass @=@ IsPartOf @=@ isPartOf
|
||||||
|
dnet:relation_relClass @=@ IsReviewedBy @=@ isReviewedBy
|
||||||
|
dnet:relation_relClass @=@ Reviews @=@ reviews
|
||||||
|
dnet:relation_relClass @=@ IsSupplementTo @=@ isSupplementTo
|
||||||
|
dnet:relation_relClass @=@ IsSupplementedBy @=@ isSupplementedBy
|
||||||
|
dnet:relation_relClass @=@ IsRelatedTo @=@ isRelatedTo
|
||||||
|
dnet:relation_subRelType @=@ relationship @=@ publicationDataset
|
|
@ -1080,3 +1080,40 @@ dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/AUTHOR/ORCID @=@ An Ope
|
||||||
dnet:review_levels @=@ dnet:review_levels @=@ 0000 @=@ Unknown
|
dnet:review_levels @=@ dnet:review_levels @=@ 0000 @=@ Unknown
|
||||||
dnet:review_levels @=@ dnet:review_levels @=@ 0002 @=@ nonPeerReviewed
|
dnet:review_levels @=@ dnet:review_levels @=@ 0002 @=@ nonPeerReviewed
|
||||||
dnet:review_levels @=@ dnet:review_levels @=@ 0001 @=@ peerReviewed
|
dnet:review_levels @=@ dnet:review_levels @=@ 0001 @=@ peerReviewed
|
||||||
|
dnet:relation_relClass @=@ dnet:relation_relClass @=@ Cites @=@ Cites
|
||||||
|
dnet:relation_relClass @=@ dnet:relation_relClass @=@ IsCitedBy @=@ IsCitedBy
|
||||||
|
dnet:relation_relClass @=@ dnet:relation_relClass @=@ HasPart @=@ HasPart
|
||||||
|
dnet:relation_relClass @=@ dnet:relation_relClass @=@ IsPartOf @=@ IsPartOf
|
||||||
|
dnet:relation_relClass @=@ dnet:relation_relClass @=@ IsReviewedBy @=@ IsReviewedBy
|
||||||
|
dnet:relation_relClass @=@ dnet:relation_relClass @=@ Reviews @=@ Reviews
|
||||||
|
dnet:relation_relClass @=@ dnet:relation_relClass @=@ IsSupplementTo @=@ IsSupplementTo
|
||||||
|
dnet:relation_relClass @=@ dnet:relation_relClass @=@ IsSupplementedBy @=@ IsSupplementedBy
|
||||||
|
dnet:relation_relClass @=@ dnet:relation_relClass @=@ IsRelatedTo @=@ IsRelatedTo
|
||||||
|
dnet:relation_relClass @=@ dnet:relation_relClass @=@ Compiles @=@ Compiles
|
||||||
|
dnet:relation_relClass @=@ dnet:relation_relClass @=@ Continues @=@ Continues
|
||||||
|
dnet:relation_relClass @=@ dnet:relation_relClass @=@ Documents @=@ Documents
|
||||||
|
dnet:relation_relClass @=@ dnet:relation_relClass @=@ HasAmongTopNSimilarDocuments @=@ HasAmongTopNSimilarDocuments
|
||||||
|
dnet:relation_relClass @=@ dnet:relation_relClass @=@ HasVersion @=@ HasVersion
|
||||||
|
dnet:relation_relClass @=@ dnet:relation_relClass @=@ IsAmongTopNSimilarDocuments @=@ IsAmongTopNSimilarDocuments
|
||||||
|
dnet:relation_relClass @=@ dnet:relation_relClass @=@ IsCompiledBy @=@ IsCompiledBy
|
||||||
|
dnet:relation_relClass @=@ dnet:relation_relClass @=@ IsContinuedBy @=@ IsContinuedBy
|
||||||
|
dnet:relation_relClass @=@ dnet:relation_relClass @=@ IsDerivedFrom @=@ IsDerivedFrom
|
||||||
|
dnet:relation_relClass @=@ dnet:relation_relClass @=@ IsDocumentedBy @=@ IsDocumentedBy
|
||||||
|
dnet:relation_relClass @=@ dnet:relation_relClass @=@ IsNewVersionOf @=@ IsNewVersionOf
|
||||||
|
dnet:relation_relClass @=@ dnet:relation_relClass @=@ IsObsoletedBy @=@ IsObsoletedBy
|
||||||
|
dnet:relation_relClass @=@ dnet:relation_relClass @=@ IsOriginalFormOf @=@ IsOriginalFormOf
|
||||||
|
dnet:relation_relClass @=@ dnet:relation_relClass @=@ IsPreviousVersionOf @=@ IsPreviousVersionOf
|
||||||
|
dnet:relation_relClass @=@ dnet:relation_relClass @=@ IsSourceOf @=@ IsSourceOf
|
||||||
|
dnet:relation_relClass @=@ dnet:relation_relClass @=@ IsVariantFormOf @=@ IsVariantFormOf
|
||||||
|
dnet:relation_subRelType @=@ dnet:relation_subRelType @=@ affiliation @=@ affiliation
|
||||||
|
dnet:relation_subRelType @=@ dnet:relation_subRelType @=@ citation @=@ citation
|
||||||
|
dnet:relation_subRelType @=@ dnet:relation_subRelType @=@ dedup @=@ dedup
|
||||||
|
dnet:relation_subRelType @=@ dnet:relation_subRelType @=@ outcome @=@ outcome
|
||||||
|
dnet:relation_subRelType @=@ dnet:relation_subRelType @=@ part @=@ part
|
||||||
|
dnet:relation_subRelType @=@ dnet:relation_subRelType @=@ participation @=@ participation
|
||||||
|
dnet:relation_subRelType @=@ dnet:relation_subRelType @=@ provision @=@ provision
|
||||||
|
dnet:relation_subRelType @=@ dnet:relation_subRelType @=@ relationship @=@ relationship
|
||||||
|
dnet:relation_subRelType @=@ dnet:relation_subRelType @=@ review @=@ review
|
||||||
|
dnet:relation_subRelType @=@ dnet:relation_subRelType @=@ similarity @=@ similarity
|
||||||
|
dnet:relation_subRelType @=@ dnet:relation_subRelType @=@ supplement @=@ supplement
|
||||||
|
dnet:relation_subRelType @=@ dnet:relation_subRelType @=@ version @=@ version
|
2
pom.xml
2
pom.xml
|
@ -753,7 +753,7 @@
|
||||||
<mockito-core.version>3.3.3</mockito-core.version>
|
<mockito-core.version>3.3.3</mockito-core.version>
|
||||||
<mongodb.driver.version>3.4.2</mongodb.driver.version>
|
<mongodb.driver.version>3.4.2</mongodb.driver.version>
|
||||||
<vtd.version>[2.12,3.0)</vtd.version>
|
<vtd.version>[2.12,3.0)</vtd.version>
|
||||||
<dhp-schemas.version>[2.7.17]</dhp-schemas.version>
|
<dhp-schemas.version>[2.7.18]</dhp-schemas.version>
|
||||||
<dnet-actionmanager-api.version>[4.0.3]</dnet-actionmanager-api.version>
|
<dnet-actionmanager-api.version>[4.0.3]</dnet-actionmanager-api.version>
|
||||||
<dnet-actionmanager-common.version>[6.0.5]</dnet-actionmanager-common.version>
|
<dnet-actionmanager-common.version>[6.0.5]</dnet-actionmanager-common.version>
|
||||||
<dnet-openaire-broker-common.version>[3.1.6]</dnet-openaire-broker-common.version>
|
<dnet-openaire-broker-common.version>[3.1.6]</dnet-openaire-broker-common.version>
|
||||||
|
|
Loading…
Reference in New Issue