diff --git a/dhp-build/dhp-build-assembly-resources/pom.xml b/dhp-build/dhp-build-assembly-resources/pom.xml index 8bae191d38..012ff89a36 100644 --- a/dhp-build/dhp-build-assembly-resources/pom.xml +++ b/dhp-build/dhp-build-assembly-resources/pom.xml @@ -6,7 +6,7 @@ eu.dnetlib.dhp dhp-build - 1.2.2-SNAPSHOT + 1.2.4-SNAPSHOT dhp-build-assembly-resources diff --git a/dhp-build/dhp-build-properties-maven-plugin/pom.xml b/dhp-build/dhp-build-properties-maven-plugin/pom.xml index ad8cd57b4a..256017e2c7 100644 --- a/dhp-build/dhp-build-properties-maven-plugin/pom.xml +++ b/dhp-build/dhp-build-properties-maven-plugin/pom.xml @@ -6,7 +6,7 @@ eu.dnetlib.dhp dhp-build - 1.2.2-SNAPSHOT + 1.2.4-SNAPSHOT dhp-build-properties-maven-plugin diff --git a/dhp-build/dhp-code-style/pom.xml b/dhp-build/dhp-code-style/pom.xml index 08f5de9ee8..e60e8076ef 100644 --- a/dhp-build/dhp-code-style/pom.xml +++ b/dhp-build/dhp-code-style/pom.xml @@ -5,7 +5,7 @@ eu.dnetlib.dhp dhp-code-style - 1.2.2-SNAPSHOT + 1.2.4-SNAPSHOT jar diff --git a/dhp-build/pom.xml b/dhp-build/pom.xml index 369e25b24b..12b999b9c8 100644 --- a/dhp-build/pom.xml +++ b/dhp-build/pom.xml @@ -4,7 +4,7 @@ eu.dnetlib.dhp dhp - 1.2.2-SNAPSHOT + 1.2.4-SNAPSHOT dhp-build pom diff --git a/dhp-common/pom.xml b/dhp-common/pom.xml index 60e66f45a2..0819a8bd20 100644 --- a/dhp-common/pom.xml +++ b/dhp-common/pom.xml @@ -5,7 +5,7 @@ eu.dnetlib.dhp dhp - 1.2.2-SNAPSHOT + 1.2.4-SNAPSHOT ../ diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/common/FunctionalInterfaceSupport.java b/dhp-common/src/main/java/eu/dnetlib/dhp/common/FunctionalInterfaceSupport.java index e793e3f295..c6c9d80447 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/common/FunctionalInterfaceSupport.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/common/FunctionalInterfaceSupport.java @@ -2,6 +2,7 @@ package eu.dnetlib.dhp.common; import java.io.Serializable; +import java.util.function.Consumer; import java.util.function.Supplier; /** Provides serializable and throwing extensions to standard functional interfaces. */ @@ -10,6 +11,16 @@ public class FunctionalInterfaceSupport { private FunctionalInterfaceSupport() { } + /** + * Serializable consumer of any kind of objects. To be used withing spark processing pipelines when supplying + * functions externally. + * + * @param + */ + @FunctionalInterface + public interface SerializableConsumer extends Consumer, Serializable { + } + /** * Serializable supplier of any kind of objects. To be used withing spark processing pipelines when supplying * functions externally. diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/common/PacePerson.java b/dhp-common/src/main/java/eu/dnetlib/dhp/common/PacePerson.java index 1909ddcca6..6e02ca614b 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/common/PacePerson.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/common/PacePerson.java @@ -16,6 +16,12 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.hash.Hashing; +/** + * PacePerson tries to derive information from the fullname string of an author. Such informations are Names, Surnames + * an Fullname split into terms. It provides also an additional field for the original data. The calculation of the + * names and the surnames is not always possible. When it is impossible to assert which are the names and the surnames, + * the lists are empty. + */ public class PacePerson { private static final String UTF8 = "UTF-8"; @@ -26,10 +32,19 @@ public class PacePerson { private static Set particles = null; + /** + * Capitalizes a string + * + * @param s the string to capitalize + * @return the input string with capital letter + */ public static final String capitalize(final String s) { return WordUtils.capitalize(s.toLowerCase(), ' ', '-'); } + /** + * Adds a dot to a string with length equals to 1 + */ public static final String dotAbbreviations(final String s) { return s.length() == 1 ? s + "." : s; } @@ -46,6 +61,12 @@ public class PacePerson { return h; } + /** + * The constructor of the class. It fills the fields of the class basing on the input fullname. + * + * @param s the input string (fullname of the author) + * @param aggressive set the string normalization type + */ public PacePerson(String s, final boolean aggressive) { original = s; s = Normalizer.normalize(s, Normalizer.Form.NFD); @@ -64,6 +85,7 @@ public class PacePerson { // s = s.replaceAll("[\\W&&[^,-]]", ""); } + // if the string contains a comma, it can derive surname and name by splitting on it if (s.contains(",")) { final String[] arr = s.split(","); if (arr.length == 1) { @@ -74,21 +96,23 @@ public class PacePerson { fullname.addAll(surname); fullname.addAll(name); } - } else { + } else { // otherwise, it should rely on CAPS terms and short terms fullname = splitTerms(s); int lastInitialPosition = fullname.size(); boolean hasSurnameInUpperCase = false; + // computes lastInitialPosition and hasSurnameInUpperCase for (int i = 0; i < fullname.size(); i++) { final String term = fullname.get(i); if (term.length() == 1) { - lastInitialPosition = i; + lastInitialPosition = i; // first word in the name longer than 1 (to avoid name with dots) } else if (term.equals(term.toUpperCase())) { - hasSurnameInUpperCase = true; + hasSurnameInUpperCase = true; // if one of the words is CAPS } } + // manages particular cases of fullnames if (lastInitialPosition < fullname.size() - 1) { // Case: Michele G. Artini name = fullname.subList(0, lastInitialPosition + 1); surname = fullname.subList(lastInitialPosition + 1, fullname.size()); diff --git a/dhp-common/src/test/java/eu/dnetlib/dhp/common/PacePersonTest.java b/dhp-common/src/test/java/eu/dnetlib/dhp/common/PacePersonTest.java new file mode 100644 index 0000000000..5ebd7213e8 --- /dev/null +++ b/dhp-common/src/test/java/eu/dnetlib/dhp/common/PacePersonTest.java @@ -0,0 +1,27 @@ + +package eu.dnetlib.dhp.common; + +import static org.junit.jupiter.api.Assertions.*; + +import org.junit.jupiter.api.Test; + +public class PacePersonTest { + + @Test + public void pacePersonTest1() { + + PacePerson p = new PacePerson("Artini, Michele", false); + assertEquals("Artini", p.getSurnameString()); + assertEquals("Michele", p.getNameString()); + assertEquals("Artini, Michele", p.getNormalisedFullname()); + } + + @Test + public void pacePersonTest2() { + PacePerson p = new PacePerson("Michele G. Artini", false); + assertEquals("Artini, Michele G.", p.getNormalisedFullname()); + assertEquals("Michele G", p.getNameString()); + assertEquals("Artini", p.getSurnameString()); + } + +} diff --git a/dhp-schemas/pom.xml b/dhp-schemas/pom.xml index 5e864cf940..2e5652b430 100644 --- a/dhp-schemas/pom.xml +++ b/dhp-schemas/pom.xml @@ -5,7 +5,7 @@ eu.dnetlib.dhp dhp - 1.2.2-SNAPSHOT + 1.2.4-SNAPSHOT ../ diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/ModelConstants.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/ModelConstants.java index e32dd10fa9..c5905e45b3 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/ModelConstants.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/ModelConstants.java @@ -1,6 +1,10 @@ package eu.dnetlib.dhp.schema.common; +import java.security.Key; + +import eu.dnetlib.dhp.schema.oaf.DataInfo; +import eu.dnetlib.dhp.schema.oaf.KeyValue; import eu.dnetlib.dhp.schema.oaf.Qualifier; public class ModelConstants { @@ -14,6 +18,7 @@ public class ModelConstants { public static final String DNET_DATA_CITE_RESOURCE = "dnet:dataCite_resource"; public static final String DNET_PROVENANCE_ACTIONS = "dnet:provenanceActions"; public static final String DNET_COUNTRY_TYPE = "dnet:countries"; + public static final String DNET_REVIEW_LEVELS = "dnet:review_levels"; public static final String SYSIMPORT_CROSSWALK_REPOSITORY = "sysimport:crosswalk:repository"; public static final String SYSIMPORT_CROSSWALK_ENTITYREGISTRY = "sysimport:crosswalk:entityregistry"; @@ -25,6 +30,10 @@ public class ModelConstants { public static final String ORP_RESULTTYPE_CLASSID = "other"; public static final String RESULT_RESULT = "resultResult"; + /** + * @deprecated Use {@link ModelConstants#RELATIONSHIP} instead. + */ + @Deprecated public static final String PUBLICATION_DATASET = "publicationDataset"; public static final String IS_RELATED_TO = "isRelatedTo"; public static final String SUPPLEMENT = "supplement"; @@ -34,6 +43,12 @@ public class ModelConstants { public static final String IS_PART_OF = "IsPartOf"; public static final String HAS_PARTS = "HasParts"; public static final String RELATIONSHIP = "relationship"; + public static final String CITATION = "citation"; + public static final String CITES = "cites"; + public static final String IS_CITED_BY = "IsCitedBy"; + public static final String REVIEW = "review"; + public static final String REVIEWS = "reviews"; + public static final String IS_REVIEWED_BY = "IsReviewedBy"; public static final String RESULT_PROJECT = "resultProject"; public static final String OUTCOME = "outcome"; @@ -84,6 +99,9 @@ public class ModelConstants { SYSIMPORT_CROSSWALK_ENTITYREGISTRY, SYSIMPORT_CROSSWALK_ENTITYREGISTRY, DNET_PROVENANCE_ACTIONS, DNET_PROVENANCE_ACTIONS); + public static final KeyValue UNKNOWN_REPOSITORY = keyValue( + "10|openaire____::55045bd2a65019fd8e6741a755395c8c", "Unknown Repository"); + private static Qualifier qualifier( final String classid, final String classname, @@ -96,4 +114,12 @@ public class ModelConstants { q.setSchemename(schemename); return q; } + + private static KeyValue keyValue(String key, String value) { + KeyValue kv = new KeyValue(); + kv.setKey(key); + kv.setValue(value); + kv.setDataInfo(new DataInfo()); + return kv; + } } diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/ModelSupport.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/ModelSupport.java index 9ee7c2debc..7d8be81ac9 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/ModelSupport.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/ModelSupport.java @@ -58,6 +58,18 @@ public class ModelSupport { oafTypes.put("relation", Relation.class); } + public static final Map idPrefixMap = Maps.newHashMap(); + + static { + idPrefixMap.put(Datasource.class, "10"); + idPrefixMap.put(Organization.class, "20"); + idPrefixMap.put(Project.class, "40"); + idPrefixMap.put(Dataset.class, "50"); + idPrefixMap.put(OtherResearchProduct.class, "50"); + idPrefixMap.put(Software.class, "50"); + idPrefixMap.put(Publication.class, "50"); + } + public static final Map entityIdPrefix = Maps.newHashMap(); static { @@ -289,6 +301,10 @@ public class ModelSupport { private ModelSupport() { } + public static String getIdPrefix(Class clazz) { + return idPrefixMap.get(clazz); + } + /** * Checks subclass-superclass relationship. * diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Dataset.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Dataset.java index 07ddbb00e9..b5587c6b7e 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Dataset.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Dataset.java @@ -10,6 +10,7 @@ public class Dataset extends Result implements Serializable { private Field storagedate; + // candidate for removal private Field device; private Field size; diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Instance.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Instance.java index 2b7d3846c0..29d4952619 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Instance.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Instance.java @@ -31,7 +31,7 @@ public class Instance implements Serializable { // typed results private Field processingchargecurrency; - private Field refereed; // peer-review status + private Qualifier refereed; // peer-review status public Field getLicense() { return license; @@ -113,11 +113,11 @@ public class Instance implements Serializable { this.processingchargecurrency = processingchargecurrency; } - public Field getRefereed() { + public Qualifier getRefereed() { return refereed; } - public void setRefereed(Field refereed) { + public void setRefereed(Qualifier refereed) { this.refereed = refereed; } diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Measure.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Measure.java new file mode 100644 index 0000000000..c0c14d10de --- /dev/null +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Measure.java @@ -0,0 +1,59 @@ + +package eu.dnetlib.dhp.schema.oaf; + +import java.util.List; + +import com.google.common.base.Objects; + +/** + * Represent a measure, must be further described by a system available resource providing name and descriptions. + */ +public class Measure { + + /** + * Unique measure identifier. + */ + private String id; + + /** + * List of units associated with this measure. KeyValue provides a pair to store the laber (key) and the value, plus + * common provenance information. + */ + private List unit; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public List getUnit() { + return unit; + } + + public void setUnit(List unit) { + this.unit = unit; + } + + public void mergeFrom(Measure m) { + // TODO + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + Measure measure = (Measure) o; + return Objects.equal(id, measure.id) && + Objects.equal(unit, measure.unit); + } + + @Override + public int hashCode() { + return Objects.hashCode(id, unit); + } +} diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Programme.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Programme.java new file mode 100644 index 0000000000..00dc32fbca --- /dev/null +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Programme.java @@ -0,0 +1,38 @@ + +package eu.dnetlib.dhp.schema.oaf; + +import java.io.Serializable; +import java.util.Objects; + +public class Programme implements Serializable { + private String code; + private String description; + + public String getCode() { + return code; + } + + public void setCode(String code) { + this.code = code; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + + Programme programme = (Programme) o; + return Objects.equals(code, programme.code); + } + +} diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Project.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Project.java index 924c08cc9c..1fcfb305e7 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Project.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Project.java @@ -58,6 +58,8 @@ public class Project extends OafEntity implements Serializable { private Float fundedamount; + private List programme; + public Field getWebsiteurl() { return websiteurl; } @@ -266,6 +268,14 @@ public class Project extends OafEntity implements Serializable { this.fundedamount = fundedamount; } + public List getProgramme() { + return programme; + } + + public void setProgramme(List programme) { + this.programme = programme; + } + @Override public void mergeFrom(OafEntity e) { super.mergeFrom(e); @@ -320,6 +330,9 @@ public class Project extends OafEntity implements Serializable { fundedamount = p.getFundedamount() != null && compareTrust(this, e) < 0 ? p.getFundedamount() : fundedamount; + + programme = mergeLists(programme, p.getProgramme()); + mergeOAFDataInfo(e); } } diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Relation.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Relation.java index c928992aa3..17a50d7aca 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Relation.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Relation.java @@ -41,6 +41,16 @@ public class Relation extends Oaf { */ private String target; + /** + * Was this relationship authoritatively validated? + */ + private Boolean validated; + + /** + * When was this relationship authoritatively validated. + */ + private String validationDate; + /** * List of relation specific properties. Values include 'similarityLevel', indicating the similarity score between a * pair of publications. @@ -95,6 +105,22 @@ public class Relation extends Oaf { this.properties = properties; } + public Boolean getValidated() { + return validated; + } + + public void setValidated(Boolean validated) { + this.validated = validated; + } + + public String getValidationDate() { + return validationDate; + } + + public void setValidationDate(String validationDate) { + this.validationDate = validationDate; + } + public void mergeFrom(final Relation r) { checkArgument(Objects.equals(getSource(), r.getSource()), "source ids must be equal"); @@ -137,4 +163,5 @@ public class Relation extends Oaf { public int hashCode() { return Objects.hash(relType, subRelType, relClass, source, target, collectedfrom); } + } diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Result.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Result.java index 213a585a8c..fdd42ab7d1 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Result.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Result.java @@ -2,11 +2,15 @@ package eu.dnetlib.dhp.schema.oaf; import java.io.Serializable; +import java.util.ArrayList; import java.util.Comparator; import java.util.List; +import java.util.stream.Collectors; public class Result extends OafEntity implements Serializable { + private List measures; + private List author; // resulttype allows subclassing results into publications | datasets | software @@ -51,6 +55,14 @@ public class Result extends OafEntity implements Serializable { private List instance; + public List getMeasures() { + return measures; + } + + public void setMeasures(List measures) { + this.measures = measures; + } + public List getAuthor() { return author; } @@ -229,6 +241,8 @@ public class Result extends OafEntity implements Serializable { Result r = (Result) e; + // TODO consider merging also Measures + instance = mergeLists(instance, r.getInstance()); if (r.getBestaccessright() != null && compareTrust(this, r) < 0) @@ -244,25 +258,33 @@ public class Result extends OafEntity implements Serializable { subject = mergeLists(subject, r.getSubject()); - //merge title lists: main title with higher trust and distinct between the others + // merge title lists: main title with higher trust and distinct between the others StructuredProperty baseMainTitle = null; - if(title != null) { + if (title != null) { baseMainTitle = getMainTitle(title); - title.remove(baseMainTitle); + if (baseMainTitle != null) { + final StructuredProperty p = baseMainTitle; + title = title.stream().filter(t -> t != p).collect(Collectors.toList()); + } } StructuredProperty newMainTitle = null; - if(r.getTitle() != null) { + if (r.getTitle() != null) { newMainTitle = getMainTitle(r.getTitle()); - r.getTitle().remove(newMainTitle); + if (newMainTitle != null) { + final StructuredProperty p = newMainTitle; + r.setTitle(r.getTitle().stream().filter(t -> t != p).collect(Collectors.toList())); + } } - if (newMainTitle != null && compareTrust(this, r) < 0 ) + if (newMainTitle != null && compareTrust(this, r) < 0) { baseMainTitle = newMainTitle; + } title = mergeLists(title, r.getTitle()); - if (title != null && baseMainTitle != null) + if (title != null && baseMainTitle != null) { title.add(baseMainTitle); + } relevantdate = mergeLists(relevantdate, r.getRelevantdate()); @@ -314,8 +336,9 @@ public class Result extends OafEntity implements Serializable { } private StructuredProperty getMainTitle(List titles) { - //need to check if the list of titles contains more than 1 main title? (in that case, we should chose which main title select in the list) - for (StructuredProperty title: titles) { + // need to check if the list of titles contains more than 1 main title? (in that case, we should chose which + // main title select in the list) + for (StructuredProperty title : titles) { if (title.getQualifier() != null && title.getQualifier().getClassid() != null) if (title.getQualifier().getClassid().equals("main title")) return title; diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Software.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Software.java index 40332bf53c..d25b5c9cec 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Software.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Software.java @@ -10,8 +10,10 @@ public class Software extends Result implements Serializable { private List> documentationUrl; + // candidate for removal private List license; + // candidate for removal private Field codeRepositoryUrl; private Qualifier programmingLanguage; diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/scholexplorer/DLIRelation.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/scholexplorer/DLIRelation.java index d2d2089c08..ca85fa14fc 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/scholexplorer/DLIRelation.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/scholexplorer/DLIRelation.java @@ -1,11 +1,25 @@ package eu.dnetlib.dhp.schema.scholexplorer; +import java.util.List; + +import eu.dnetlib.dhp.schema.oaf.KeyValue; import eu.dnetlib.dhp.schema.oaf.Relation; public class DLIRelation extends Relation { + private String dateOfCollection; + private List collectedFrom; + + public List getCollectedFrom() { + return collectedFrom; + } + + public void setCollectedFrom(List collectedFrom) { + this.collectedFrom = collectedFrom; + } + public String getDateOfCollection() { return dateOfCollection; } diff --git a/dhp-schemas/src/test/java/eu/dnetlib/dhp/schema/oaf/MeasureTest.java b/dhp-schemas/src/test/java/eu/dnetlib/dhp/schema/oaf/MeasureTest.java new file mode 100644 index 0000000000..26b4407c9b --- /dev/null +++ b/dhp-schemas/src/test/java/eu/dnetlib/dhp/schema/oaf/MeasureTest.java @@ -0,0 +1,57 @@ + +package eu.dnetlib.dhp.schema.oaf; + +import java.io.IOException; +import java.util.List; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Lists; + +public class MeasureTest { + + public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper() + .setSerializationInclusion(JsonInclude.Include.NON_NULL); + + @Test + public void testMeasureSerialization() throws IOException { + + Measure popularity = new Measure(); + popularity.setId("popularity"); + popularity + .setUnit( + Lists + .newArrayList( + unit("score", "0.5"))); + + Measure influence = new Measure(); + influence.setId("influence"); + influence + .setUnit( + Lists + .newArrayList( + unit("score", "0.3"))); + + List m = Lists.newArrayList(popularity, influence); + + String s = OBJECT_MAPPER.writeValueAsString(m); + System.out.println(s); + + List mm = OBJECT_MAPPER.readValue(s, new TypeReference>() { + }); + + Assertions.assertNotNull(mm); + } + + private KeyValue unit(String key, String value) { + KeyValue unit = new KeyValue(); + unit.setKey(key); + unit.setValue(value); + return unit; + } + +} diff --git a/dhp-workflows/dhp-actionmanager/pom.xml b/dhp-workflows/dhp-actionmanager/pom.xml index ec6247102e..0b4d25700b 100644 --- a/dhp-workflows/dhp-actionmanager/pom.xml +++ b/dhp-workflows/dhp-actionmanager/pom.xml @@ -4,7 +4,7 @@ eu.dnetlib.dhp dhp-workflows - 1.2.2-SNAPSHOT + 1.2.4-SNAPSHOT dhp-actionmanager diff --git a/dhp-workflows/dhp-actionmanager/src/main/java/eu/dnetlib/dhp/actionmanager/migration/ProtoConverter.java b/dhp-workflows/dhp-actionmanager/src/main/java/eu/dnetlib/dhp/actionmanager/migration/ProtoConverter.java index e55c0eb7b6..8ea877aec5 100644 --- a/dhp-workflows/dhp-actionmanager/src/main/java/eu/dnetlib/dhp/actionmanager/migration/ProtoConverter.java +++ b/dhp-workflows/dhp-actionmanager/src/main/java/eu/dnetlib/dhp/actionmanager/migration/ProtoConverter.java @@ -96,12 +96,21 @@ public class ProtoConverter implements Serializable { .stream() .distinct() .collect(Collectors.toCollection(ArrayList::new)) : null); - i.setRefereed(mapStringField(ri.getRefereed())); + i.setRefereed(mapRefereed(ri.getRefereed())); i.setProcessingchargeamount(mapStringField(ri.getProcessingchargeamount())); i.setProcessingchargecurrency(mapStringField(ri.getProcessingchargecurrency())); return i; } + private static Qualifier mapRefereed(FieldTypeProtos.StringField refereed) { + Qualifier q = new Qualifier(); + q.setClassid(refereed.getValue()); + q.setSchemename(refereed.getValue()); + q.setSchemeid("dnet:review_levels"); + q.setSchemename("dnet:review_levels"); + return q; + } + private static List convertExternalRefs(OafProtos.Oaf oaf) { ResultProtos.Result r = oaf.getEntity().getResult(); if (r.getExternalReferenceCount() > 0) { diff --git a/dhp-workflows/dhp-actionmanager/src/main/java/eu/dnetlib/dhp/actionmanager/promote/PromoteActionPayloadForGraphTableJob.java b/dhp-workflows/dhp-actionmanager/src/main/java/eu/dnetlib/dhp/actionmanager/promote/PromoteActionPayloadForGraphTableJob.java index 17bfc4af36..5fa9e67235 100644 --- a/dhp-workflows/dhp-actionmanager/src/main/java/eu/dnetlib/dhp/actionmanager/promote/PromoteActionPayloadForGraphTableJob.java +++ b/dhp-workflows/dhp-actionmanager/src/main/java/eu/dnetlib/dhp/actionmanager/promote/PromoteActionPayloadForGraphTableJob.java @@ -4,6 +4,7 @@ package eu.dnetlib.dhp.actionmanager.promote; import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; import static eu.dnetlib.dhp.schema.common.ModelSupport.isSubClass; +import java.io.IOException; import java.util.Objects; import java.util.Optional; import java.util.function.BiFunction; @@ -20,6 +21,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException; import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.common.FunctionalInterfaceSupport.SerializableSupplier; @@ -134,24 +136,39 @@ public class PromoteActionPayloadForGraphTableJob { .map( (MapFunction) value -> OBJECT_MAPPER.readValue(value, rowClazz), Encoders.bean(rowClazz)); - - /* - * return spark .read() .parquet(path) .as(Encoders.bean(rowClazz)); - */ } private static Dataset readActionPayload( SparkSession spark, String path, Class actionPayloadClazz) { logger.info("Reading action payload from path: {}", path); + return spark .read() .parquet(path) + .map((MapFunction) value -> extractPayload(value), Encoders.STRING()) .map( - (MapFunction) value -> OBJECT_MAPPER - .readValue(value. getAs("payload"), actionPayloadClazz), + (MapFunction) value -> decodePayload(actionPayloadClazz, value), Encoders.bean(actionPayloadClazz)); } + private static String extractPayload(Row value) { + try { + return value. getAs("payload"); + } catch (IllegalArgumentException | ClassCastException e) { + logger.error("cannot extract payload from action: {}", value.toString()); + throw e; + } + } + + private static A decodePayload(Class actionPayloadClazz, String payload) throws IOException { + try { + return OBJECT_MAPPER.readValue(payload, actionPayloadClazz); + } catch (UnrecognizedPropertyException e) { + logger.error("error decoding payload: {}", payload); + throw e; + } + } + private static Dataset promoteActionPayloadForGraphTable( Dataset rowDS, Dataset actionPayloadDS, diff --git a/dhp-workflows/dhp-aggregation/pom.xml b/dhp-workflows/dhp-aggregation/pom.xml index 9f082df70a..a1bc1c483d 100644 --- a/dhp-workflows/dhp-aggregation/pom.xml +++ b/dhp-workflows/dhp-aggregation/pom.xml @@ -4,7 +4,7 @@ eu.dnetlib.dhp dhp-workflows - 1.2.2-SNAPSHOT + 1.2.4-SNAPSHOT dhp-aggregation @@ -38,48 +38,6 @@ ${project.version} - - eu.dnetlib - dnet-actionmanager-common - - - eu.dnetlib - dnet-openaireplus-mapping-utils - - - saxonica - saxon - - - saxonica - saxon-dom - - - jgrapht - jgrapht - - - net.sf.ehcache - ehcache - - - org.springframework - spring-test - - - org.apache.* - * - - - apache - * - - - - - eu.dnetlib - dnet-openaire-data-protos - net.sf.saxon @@ -100,11 +58,15 @@ jaxen + - org.apache.hadoop - hadoop-distcp + org.apache.commons + commons-csv + 1.8 + + diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProgramme.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProgramme.java new file mode 100644 index 0000000000..c6dab13a06 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProgramme.java @@ -0,0 +1,123 @@ + +package eu.dnetlib.dhp.actionmanager.project; + +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; + +import java.util.HashMap; +import java.util.Optional; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.function.MapFunction; +import org.apache.spark.sql.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.dhp.actionmanager.project.csvutils.CSVProgramme; +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.common.HdfsSupport; +import scala.Tuple2; + +public class PrepareProgramme { + + private static final Logger log = LoggerFactory.getLogger(PrepareProgramme.class); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + public static void main(String[] args) throws Exception { + + String jsonConfiguration = IOUtils + .toString( + PrepareProgramme.class + .getResourceAsStream( + "/eu/dnetlib/dhp/actionmanager/project/prepare_programme_parameters.json")); + + final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); + + parser.parseArgument(args); + + Boolean isSparkSessionManaged = Optional + .ofNullable(parser.get("isSparkSessionManaged")) + .map(Boolean::valueOf) + .orElse(Boolean.TRUE); + + log.info("isSparkSessionManaged: {}", isSparkSessionManaged); + + final String programmePath = parser.get("programmePath"); + log.info("programmePath {}: ", programmePath); + + final String outputPath = parser.get("outputPath"); + log.info("outputPath {}: ", outputPath); + + SparkConf conf = new SparkConf(); + + runWithSparkSession( + conf, + isSparkSessionManaged, + spark -> { + removeOutputDir(spark, outputPath); + exec(spark, programmePath, outputPath); + }); + } + + private static void removeOutputDir(SparkSession spark, String path) { + HdfsSupport.remove(path, spark.sparkContext().hadoopConfiguration()); + } + + private static void exec(SparkSession spark, String programmePath, String outputPath) { + Dataset programme = readPath(spark, programmePath, CSVProgramme.class); + + programme + .toJavaRDD() + .filter(p -> !p.getCode().contains("FP7")) + .mapToPair(csvProgramme -> new Tuple2<>(csvProgramme.getCode(), csvProgramme)) + .reduceByKey((a, b) -> { + if (StringUtils.isEmpty(a.getShortTitle())) { + if (StringUtils.isEmpty(b.getShortTitle())) { + if (StringUtils.isEmpty(a.getTitle())) { + if (StringUtils.isNotEmpty(b.getTitle())) { + a.setShortTitle(b.getTitle()); + a.setLanguage(b.getLanguage()); + } + } else {// notIsEmpty a.getTitle + if (StringUtils.isEmpty(b.getTitle())) { + a.setShortTitle(a.getTitle()); + } else { + if (b.getLanguage().equalsIgnoreCase("en")) { + a.setShortTitle(b.getTitle()); + a.setLanguage(b.getLanguage()); + } else { + a.setShortTitle(a.getTitle()); + } + } + } + } else {// not isEmpty b.getShortTitle + a.setShortTitle(b.getShortTitle()); + // a.setLanguage(b.getLanguage()); + } + } + return a; + + }) + .map(p -> { + CSVProgramme csvProgramme = p._2(); + if (StringUtils.isEmpty(csvProgramme.getShortTitle())) { + csvProgramme.setShortTitle(csvProgramme.getTitle()); + } + return OBJECT_MAPPER.writeValueAsString(csvProgramme); + }) + .saveAsTextFile(outputPath); + + } + + public static Dataset readPath( + SparkSession spark, String inputPath, Class clazz) { + return spark + .read() + .textFile(inputPath) + .map((MapFunction) value -> OBJECT_MAPPER.readValue(value, clazz), Encoders.bean(clazz)); + } + +} diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProjects.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProjects.java new file mode 100644 index 0000000000..78aed1a697 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProjects.java @@ -0,0 +1,120 @@ + +package eu.dnetlib.dhp.actionmanager.project; + +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; + +import java.util.*; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.function.FlatMapFunction; +import org.apache.spark.api.java.function.MapFunction; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.SaveMode; +import org.apache.spark.sql.SparkSession; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.dhp.actionmanager.project.csvutils.CSVProgramme; +import eu.dnetlib.dhp.actionmanager.project.csvutils.CSVProject; +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.common.HdfsSupport; +import scala.Tuple2; + +public class PrepareProjects { + + private static final Logger log = LoggerFactory.getLogger(PrepareProgramme.class); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private static final HashMap programmeMap = new HashMap<>(); + + public static void main(String[] args) throws Exception { + + String jsonConfiguration = IOUtils + .toString( + PrepareProjects.class + .getResourceAsStream( + "/eu/dnetlib/dhp/actionmanager/project/prepare_project_parameters.json")); + + final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); + + parser.parseArgument(args); + + Boolean isSparkSessionManaged = Optional + .ofNullable(parser.get("isSparkSessionManaged")) + .map(Boolean::valueOf) + .orElse(Boolean.TRUE); + + log.info("isSparkSessionManaged: {}", isSparkSessionManaged); + + final String projectPath = parser.get("projectPath"); + log.info("projectPath {}: ", projectPath); + + final String outputPath = parser.get("outputPath"); + log.info("outputPath {}: ", outputPath); + + final String dbProjectPath = parser.get("dbProjectPath"); + log.info("dbProjectPath {}: ", dbProjectPath); + + SparkConf conf = new SparkConf(); + + runWithSparkSession( + conf, + isSparkSessionManaged, + spark -> { + removeOutputDir(spark, outputPath); + exec(spark, projectPath, dbProjectPath, outputPath); + }); + } + + private static void removeOutputDir(SparkSession spark, String path) { + HdfsSupport.remove(path, spark.sparkContext().hadoopConfiguration()); + } + + private static void exec(SparkSession spark, String projectPath, String dbProjectPath, String outputPath) { + Dataset project = readPath(spark, projectPath, CSVProject.class); + Dataset dbProjects = readPath(spark, dbProjectPath, ProjectSubset.class); + + dbProjects + .joinWith(project, dbProjects.col("code").equalTo(project.col("id")), "left") + .flatMap(getTuple2CSVProjectFlatMapFunction(), Encoders.bean(CSVProject.class)) + .filter(Objects::nonNull) + .write() + .mode(SaveMode.Overwrite) + .option("compression", "gzip") + .json(outputPath); + + } + + private static FlatMapFunction, CSVProject> getTuple2CSVProjectFlatMapFunction() { + return (FlatMapFunction, CSVProject>) value -> { + Optional csvProject = Optional.ofNullable(value._2()); + List csvProjectList = new ArrayList<>(); + if (csvProject.isPresent()) { + + String[] programme = csvProject.get().getProgramme().split(";"); + Arrays + .stream(programme) + .forEach(p -> { + CSVProject proj = new CSVProject(); + proj.setProgramme(p); + proj.setId(csvProject.get().getId()); + csvProjectList.add(proj); + }); + } + return csvProjectList.iterator(); + }; + } + + public static Dataset readPath( + SparkSession spark, String inputPath, Class clazz) { + return spark + .read() + .textFile(inputPath) + .map((MapFunction) value -> OBJECT_MAPPER.readValue(value, clazz), Encoders.bean(clazz)); + } +} diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/ProjectSubset.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/ProjectSubset.java new file mode 100644 index 0000000000..2fccbc5165 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/ProjectSubset.java @@ -0,0 +1,17 @@ + +package eu.dnetlib.dhp.actionmanager.project; + +import java.io.Serializable; + +public class ProjectSubset implements Serializable { + + private String code; + + public String getCode() { + return code; + } + + public void setCode(String code) { + this.code = code; + } +} diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/ReadProjectsFromDB.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/ReadProjectsFromDB.java new file mode 100644 index 0000000000..2d541d2f98 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/ReadProjectsFromDB.java @@ -0,0 +1,115 @@ + +package eu.dnetlib.dhp.actionmanager.project; + +import java.io.BufferedWriter; +import java.io.Closeable; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.nio.charset.StandardCharsets; +import java.sql.ResultSet; +import java.util.Arrays; +import java.util.List; +import java.util.function.Consumer; +import java.util.function.Function; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.common.DbClient; + +public class ReadProjectsFromDB implements Closeable { + + private final DbClient dbClient; + private static final Log log = LogFactory.getLog(ReadProjectsFromDB.class); + private final Configuration conf; + private final BufferedWriter writer; + private final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private final static String query = "SELECT code " + + "from projects where id like 'corda__h2020%' "; + + public static void main(final String[] args) throws Exception { + final ArgumentApplicationParser parser = new ArgumentApplicationParser( + IOUtils + .toString( + ReadProjectsFromDB.class + .getResourceAsStream( + "/eu/dnetlib/dhp/actionmanager/project/read_projects_db.json"))); + + parser.parseArgument(args); + + final String dbUrl = parser.get("postgresUrl"); + final String dbUser = parser.get("postgresUser"); + final String dbPassword = parser.get("postgresPassword"); + final String hdfsPath = parser.get("hdfsPath"); + final String hdfsNameNode = parser.get("hdfsNameNode"); + + try (final ReadProjectsFromDB rbl = new ReadProjectsFromDB(hdfsPath, hdfsNameNode, dbUrl, dbUser, + dbPassword)) { + + log.info("Processing projects..."); + rbl.execute(query, rbl::processProjectsEntry); + + } + } + + public void execute(final String sql, final Function> producer) throws Exception { + + final Consumer consumer = rs -> producer.apply(rs).forEach(r -> writeProject(r)); + + dbClient.processResults(sql, consumer); + } + + public List processProjectsEntry(ResultSet rs) { + try { + ProjectSubset p = new ProjectSubset(); + p.setCode(rs.getString("code")); + + return Arrays.asList(p); + + } catch (final Exception e) { + throw new RuntimeException(e); + } + } + + protected void writeProject(final ProjectSubset r) { + try { + writer.write(OBJECT_MAPPER.writeValueAsString(r)); + writer.newLine(); + } catch (final Exception e) { + throw new RuntimeException(e); + } + } + + public ReadProjectsFromDB( + final String hdfsPath, String hdfsNameNode, final String dbUrl, final String dbUser, final String dbPassword) + throws Exception { + + this.dbClient = new DbClient(dbUrl, dbUser, dbPassword); + this.conf = new Configuration(); + this.conf.set("fs.defaultFS", hdfsNameNode); + FileSystem fileSystem = FileSystem.get(this.conf); + Path hdfsWritePath = new Path(hdfsPath); + FSDataOutputStream fsDataOutputStream = null; + if (fileSystem.exists(hdfsWritePath)) { + fileSystem.delete(hdfsWritePath, false); + } + fsDataOutputStream = fileSystem.create(hdfsWritePath); + + this.writer = new BufferedWriter(new OutputStreamWriter(fsDataOutputStream, StandardCharsets.UTF_8)); + } + + @Override + public void close() throws IOException { + dbClient.close(); + writer.close(); + } +} diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/SparkAtomicActionJob.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/SparkAtomicActionJob.java new file mode 100644 index 0000000000..1023e2d19d --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/SparkAtomicActionJob.java @@ -0,0 +1,161 @@ + +package eu.dnetlib.dhp.actionmanager.project; + +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; + +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Consumer; + +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.SequenceFile; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapred.SequenceFileOutputFormat; +import org.apache.hadoop.mapred.TextOutputFormat; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.function.MapFunction; +import org.apache.spark.api.java.function.MapGroupsFunction; +import org.apache.spark.rdd.SequenceFileRDDFunctions; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.SaveMode; +import org.apache.spark.sql.SparkSession; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.dhp.actionmanager.project.csvutils.CSVProgramme; +import eu.dnetlib.dhp.actionmanager.project.csvutils.CSVProject; +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.common.HdfsSupport; +import eu.dnetlib.dhp.schema.action.AtomicAction; +import eu.dnetlib.dhp.schema.common.ModelSupport; +import eu.dnetlib.dhp.schema.oaf.Programme; +import eu.dnetlib.dhp.schema.oaf.Project; +import eu.dnetlib.dhp.utils.DHPUtils; +import scala.Function1; +import scala.Tuple2; +import scala.runtime.BoxedUnit; + +public class SparkAtomicActionJob { + private static final Logger log = LoggerFactory.getLogger(SparkAtomicActionJob.class); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private static final HashMap programmeMap = new HashMap<>(); + + public static void main(String[] args) throws Exception { + + String jsonConfiguration = IOUtils + .toString( + SparkAtomicActionJob.class + .getResourceAsStream( + "/eu/dnetlib/dhp/actionmanager/project/action_set_parameters.json")); + + final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); + + parser.parseArgument(args); + + Boolean isSparkSessionManaged = Optional + .ofNullable(parser.get("isSparkSessionManaged")) + .map(Boolean::valueOf) + .orElse(Boolean.TRUE); + + log.info("isSparkSessionManaged: {}", isSparkSessionManaged); + + String projectPath = parser.get("projectPath"); + log.info("projectPath: {}", projectPath); + + final String outputPath = parser.get("outputPath"); + log.info("outputPath {}: ", outputPath); + + final String programmePath = parser.get("programmePath"); + log.info("programmePath {}: ", programmePath); + + SparkConf conf = new SparkConf(); + + runWithSparkSession( + conf, + isSparkSessionManaged, + spark -> { + removeOutputDir(spark, outputPath); + getAtomicActions( + spark, + projectPath, + programmePath, + outputPath); + }); + } + + private static void removeOutputDir(SparkSession spark, String path) { + HdfsSupport.remove(path, spark.sparkContext().hadoopConfiguration()); + } + + private static void getAtomicActions(SparkSession spark, String projectPatH, + String programmePath, + String outputPath) { + + Dataset project = readPath(spark, projectPatH, CSVProject.class); + Dataset programme = readPath(spark, programmePath, CSVProgramme.class); + + project + .joinWith(programme, project.col("programme").equalTo(programme.col("code")), "left") + .map(c -> { + CSVProject csvProject = c._1(); + Optional csvProgramme = Optional.ofNullable(c._2()); + if (csvProgramme.isPresent()) { + Project p = new Project(); + p + .setId( + createOpenaireId( + ModelSupport.entityIdPrefix.get("project"), + "corda__h2020", csvProject.getId())); + Programme pm = new Programme(); + pm.setCode(csvProject.getProgramme()); + pm.setDescription(csvProgramme.get().getShortTitle()); + p.setProgramme(Arrays.asList(pm)); + return p; + } + + return null; + }, Encoders.bean(Project.class)) + .filter(Objects::nonNull) + .groupByKey( + (MapFunction) p -> p.getId(), + Encoders.STRING()) + .mapGroups((MapGroupsFunction) (s, it) -> { + Project first = it.next(); + it.forEachRemaining(p -> { + first.mergeFrom(p); + }); + return first; + }, Encoders.bean(Project.class)) + .toJavaRDD() + .map(p -> new AtomicAction(Project.class, p)) + .mapToPair( + aa -> new Tuple2<>(new Text(aa.getClazz().getCanonicalName()), + new Text(OBJECT_MAPPER.writeValueAsString(aa)))) + .saveAsHadoopFile(outputPath, Text.class, Text.class, SequenceFileOutputFormat.class); + + } + + public static Dataset readPath( + SparkSession spark, String inputPath, Class clazz) { + return spark + .read() + .textFile(inputPath) + .map((MapFunction) value -> OBJECT_MAPPER.readValue(value, clazz), Encoders.bean(clazz)); + } + + public static String createOpenaireId( + final String prefix, final String nsPrefix, final String id) { + + return String.format("%s|%s::%s", prefix, nsPrefix, DHPUtils.md5(id)); + + } +} diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/csvutils/CSVParser.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/csvutils/CSVParser.java new file mode 100644 index 0000000000..ef29a6b6a6 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/csvutils/CSVParser.java @@ -0,0 +1,37 @@ + +package eu.dnetlib.dhp.actionmanager.project.csvutils; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVRecord; +import org.apache.commons.lang.reflect.FieldUtils; + +public class CSVParser { + + public List parse(String csvFile, String classForName) + throws ClassNotFoundException, IOException, IllegalAccessException, InstantiationException { + final CSVFormat format = CSVFormat.EXCEL + .withHeader() + .withDelimiter(';') + .withQuote('"') + .withTrim(); + List ret = new ArrayList<>(); + final org.apache.commons.csv.CSVParser parser = org.apache.commons.csv.CSVParser.parse(csvFile, format); + final Set headers = parser.getHeaderMap().keySet(); + Class clazz = Class.forName(classForName); + for (CSVRecord csvRecord : parser.getRecords()) { + final Object cc = clazz.newInstance(); + for (String header : headers) { + FieldUtils.writeField(cc, header, csvRecord.get(header), true); + + } + ret.add((R) cc); + } + + return ret; + } +} diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/csvutils/CSVProgramme.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/csvutils/CSVProgramme.java new file mode 100644 index 0000000000..a9069e5104 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/csvutils/CSVProgramme.java @@ -0,0 +1,52 @@ + +package eu.dnetlib.dhp.actionmanager.project.csvutils; + +import java.io.Serializable; + +public class CSVProgramme implements Serializable { + private String rcn; + private String code; + private String title; + private String shortTitle; + private String language; + + public String getRcn() { + return rcn; + } + + public void setRcn(String rcn) { + this.rcn = rcn; + } + + public String getCode() { + return code; + } + + public void setCode(String code) { + this.code = code; + } + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } + + public String getShortTitle() { + return shortTitle; + } + + public void setShortTitle(String shortTitle) { + this.shortTitle = shortTitle; + } + + public String getLanguage() { + return language; + } + + public void setLanguage(String language) { + this.language = language; + } +} diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/csvutils/CSVProject.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/csvutils/CSVProject.java new file mode 100644 index 0000000000..ff18c6260e --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/csvutils/CSVProject.java @@ -0,0 +1,197 @@ + +package eu.dnetlib.dhp.actionmanager.project.csvutils; + +import java.io.Serializable; + +public class CSVProject implements Serializable { + private String rcn; + private String id; + private String acronym; + private String status; + private String programme; + private String topics; + private String frameworkProgramme; + private String title; + private String startDate; + private String endDate; + private String projectUrl; + private String objective; + private String totalCost; + private String ecMaxContribution; + private String call; + private String fundingScheme; + private String coordinator; + private String coordinatorCountry; + private String participants; + private String participantCountries; + private String subjects; + + public String getRcn() { + return rcn; + } + + public void setRcn(String rcn) { + this.rcn = rcn; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getAcronym() { + return acronym; + } + + public void setAcronym(String acronym) { + this.acronym = acronym; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public String getProgramme() { + return programme; + } + + public void setProgramme(String programme) { + this.programme = programme; + } + + public String getTopics() { + return topics; + } + + public void setTopics(String topics) { + this.topics = topics; + } + + public String getFrameworkProgramme() { + return frameworkProgramme; + } + + public void setFrameworkProgramme(String frameworkProgramme) { + this.frameworkProgramme = frameworkProgramme; + } + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } + + public String getStartDate() { + return startDate; + } + + public void setStartDate(String startDate) { + this.startDate = startDate; + } + + public String getEndDate() { + return endDate; + } + + public void setEndDate(String endDate) { + this.endDate = endDate; + } + + public String getProjectUrl() { + return projectUrl; + } + + public void setProjectUrl(String projectUrl) { + this.projectUrl = projectUrl; + } + + public String getObjective() { + return objective; + } + + public void setObjective(String objective) { + this.objective = objective; + } + + public String getTotalCost() { + return totalCost; + } + + public void setTotalCost(String totalCost) { + this.totalCost = totalCost; + } + + public String getEcMaxContribution() { + return ecMaxContribution; + } + + public void setEcMaxContribution(String ecMaxContribution) { + this.ecMaxContribution = ecMaxContribution; + } + + public String getCall() { + return call; + } + + public void setCall(String call) { + this.call = call; + } + + public String getFundingScheme() { + return fundingScheme; + } + + public void setFundingScheme(String fundingScheme) { + this.fundingScheme = fundingScheme; + } + + public String getCoordinator() { + return coordinator; + } + + public void setCoordinator(String coordinator) { + this.coordinator = coordinator; + } + + public String getCoordinatorCountry() { + return coordinatorCountry; + } + + public void setCoordinatorCountry(String coordinatorCountry) { + this.coordinatorCountry = coordinatorCountry; + } + + public String getParticipants() { + return participants; + } + + public void setParticipants(String participants) { + this.participants = participants; + } + + public String getParticipantCountries() { + return participantCountries; + } + + public void setParticipantCountries(String participantCountries) { + this.participantCountries = participantCountries; + } + + public String getSubjects() { + return subjects; + } + + public void setSubjects(String subjects) { + this.subjects = subjects; + } + +} diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/csvutils/ReadCSV.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/csvutils/ReadCSV.java new file mode 100644 index 0000000000..2b72b229ad --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/csvutils/ReadCSV.java @@ -0,0 +1,97 @@ + +package eu.dnetlib.dhp.actionmanager.project.csvutils; + +import java.io.BufferedWriter; +import java.io.Closeable; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.nio.charset.StandardCharsets; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.dhp.actionmanager.project.httpconnector.HttpConnector; +import eu.dnetlib.dhp.application.ArgumentApplicationParser; + +public class ReadCSV implements Closeable { + private static final Log log = LogFactory.getLog(ReadCSV.class); + private final Configuration conf; + private final BufferedWriter writer; + private final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private String csvFile; + + public static void main(final String[] args) throws Exception { + final ArgumentApplicationParser parser = new ArgumentApplicationParser( + IOUtils + .toString( + ReadCSV.class + .getResourceAsStream( + "/eu/dnetlib/dhp/actionmanager/project/parameters.json"))); + + parser.parseArgument(args); + + final String fileURL = parser.get("fileURL"); + final String hdfsPath = parser.get("hdfsPath"); + final String hdfsNameNode = parser.get("hdfsNameNode"); + final String classForName = parser.get("classForName"); + + try (final ReadCSV readCSV = new ReadCSV(hdfsPath, hdfsNameNode, fileURL)) { + + log.info("Getting CSV file..."); + readCSV.execute(classForName); + + } + } + + public void execute(final String classForName) throws Exception { + CSVParser csvParser = new CSVParser(); + csvParser + .parse(csvFile, classForName) + .stream() + .forEach(p -> write(p)); + + } + + @Override + public void close() throws IOException { + writer.close(); + } + + public ReadCSV( + final String hdfsPath, + final String hdfsNameNode, + final String fileURL) + throws Exception { + this.conf = new Configuration(); + this.conf.set("fs.defaultFS", hdfsNameNode); + HttpConnector httpConnector = new HttpConnector(); + FileSystem fileSystem = FileSystem.get(this.conf); + Path hdfsWritePath = new Path(hdfsPath); + FSDataOutputStream fsDataOutputStream = null; + if (fileSystem.exists(hdfsWritePath)) { + fileSystem.delete(hdfsWritePath, false); + } + fsDataOutputStream = fileSystem.create(hdfsWritePath); + + this.writer = new BufferedWriter(new OutputStreamWriter(fsDataOutputStream, StandardCharsets.UTF_8)); + this.csvFile = httpConnector.getInputSource(fileURL); + ; + } + + protected void write(final Object p) { + try { + writer.write(OBJECT_MAPPER.writeValueAsString(p)); + writer.newLine(); + } catch (final Exception e) { + throw new RuntimeException(e); + } + } + +} diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/httpconnector/CollectorPluginErrorLogList.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/httpconnector/CollectorPluginErrorLogList.java new file mode 100644 index 0000000000..9d3f882651 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/httpconnector/CollectorPluginErrorLogList.java @@ -0,0 +1,20 @@ + +package eu.dnetlib.dhp.actionmanager.project.httpconnector; + +import java.util.LinkedList; + +public class CollectorPluginErrorLogList extends LinkedList { + + private static final long serialVersionUID = -6925786561303289704L; + + @Override + public String toString() { + String log = new String(); + int index = 0; + for (String errorMessage : this) { + log += String.format("Retry #%s: %s / ", index++, errorMessage); + } + return log; + } + +} diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/httpconnector/CollectorServiceException.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/httpconnector/CollectorServiceException.java new file mode 100644 index 0000000000..9167d97b4c --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/httpconnector/CollectorServiceException.java @@ -0,0 +1,20 @@ + +package eu.dnetlib.dhp.actionmanager.project.httpconnector; + +public class CollectorServiceException extends Exception { + + private static final long serialVersionUID = 7523999812098059764L; + + public CollectorServiceException(String string) { + super(string); + } + + public CollectorServiceException(String string, Throwable exception) { + super(string, exception); + } + + public CollectorServiceException(Throwable exception) { + super(exception); + } + +} diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/httpconnector/HttpConnector.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/httpconnector/HttpConnector.java new file mode 100644 index 0000000000..e20518b559 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/httpconnector/HttpConnector.java @@ -0,0 +1,240 @@ + +package eu.dnetlib.dhp.actionmanager.project.httpconnector; + +import java.io.IOException; +import java.io.InputStream; +import java.net.*; +import java.security.GeneralSecurityException; +import java.security.cert.X509Certificate; +import java.util.List; +import java.util.Map; + +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.math.NumberUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** + * @author jochen, michele, andrea + */ +public class HttpConnector { + + private static final Log log = LogFactory.getLog(HttpConnector.class); + + private int maxNumberOfRetry = 6; + private int defaultDelay = 120; // seconds + private int readTimeOut = 120; // seconds + + private String responseType = null; + + private String userAgent = "Mozilla/5.0 (compatible; OAI; +http://www.openaire.eu)"; + + public HttpConnector() { + CookieHandler.setDefault(new CookieManager(null, CookiePolicy.ACCEPT_ALL)); + } + + /** + * Given the URL returns the content via HTTP GET + * + * @param requestUrl the URL + * @return the content of the downloaded resource + * @throws CollectorServiceException when retrying more than maxNumberOfRetry times + */ + public String getInputSource(final String requestUrl) throws CollectorServiceException { + return attemptDownlaodAsString(requestUrl, 1, new CollectorPluginErrorLogList()); + } + + /** + * Given the URL returns the content as a stream via HTTP GET + * + * @param requestUrl the URL + * @return the content of the downloaded resource as InputStream + * @throws CollectorServiceException when retrying more than maxNumberOfRetry times + */ + public InputStream getInputSourceAsStream(final String requestUrl) throws CollectorServiceException { + return attemptDownload(requestUrl, 1, new CollectorPluginErrorLogList()); + } + + private String attemptDownlaodAsString(final String requestUrl, final int retryNumber, + final CollectorPluginErrorLogList errorList) + throws CollectorServiceException { + try { + InputStream s = attemptDownload(requestUrl, 1, new CollectorPluginErrorLogList()); + try { + return IOUtils.toString(s); + } catch (IOException e) { + log.error("error while retrieving from http-connection occured: " + requestUrl, e); + Thread.sleep(defaultDelay * 1000); + errorList.add(e.getMessage()); + return attemptDownlaodAsString(requestUrl, retryNumber + 1, errorList); + } finally { + IOUtils.closeQuietly(s); + } + } catch (InterruptedException e) { + throw new CollectorServiceException(e); + } + } + + private InputStream attemptDownload(final String requestUrl, final int retryNumber, + final CollectorPluginErrorLogList errorList) + throws CollectorServiceException { + + if (retryNumber > maxNumberOfRetry) { + throw new CollectorServiceException("Max number of retries exceeded. Cause: \n " + errorList); + } + + log.debug("Downloading " + requestUrl + " - try: " + retryNumber); + try { + InputStream input = null; + + try { + final HttpURLConnection urlConn = (HttpURLConnection) new URL(requestUrl).openConnection(); + urlConn.setInstanceFollowRedirects(false); + urlConn.setReadTimeout(readTimeOut * 1000); + urlConn.addRequestProperty("User-Agent", userAgent); + + if (log.isDebugEnabled()) { + logHeaderFields(urlConn); + } + + int retryAfter = obtainRetryAfter(urlConn.getHeaderFields()); + if (retryAfter > 0 && urlConn.getResponseCode() == HttpURLConnection.HTTP_UNAVAILABLE) { + log.warn("waiting and repeating request after " + retryAfter + " sec."); + Thread.sleep(retryAfter * 1000); + errorList.add("503 Service Unavailable"); + urlConn.disconnect(); + return attemptDownload(requestUrl, retryNumber + 1, errorList); + } else if ((urlConn.getResponseCode() == HttpURLConnection.HTTP_MOVED_PERM) + || (urlConn.getResponseCode() == HttpURLConnection.HTTP_MOVED_TEMP)) { + final String newUrl = obtainNewLocation(urlConn.getHeaderFields()); + log.debug("The requested url has been moved to " + newUrl); + errorList + .add( + String + .format( + "%s %s. Moved to: %s", urlConn.getResponseCode(), urlConn.getResponseMessage(), + newUrl)); + urlConn.disconnect(); + return attemptDownload(newUrl, retryNumber + 1, errorList); + } else if (urlConn.getResponseCode() != HttpURLConnection.HTTP_OK) { + log + .error( + String + .format("HTTP error: %s %s", urlConn.getResponseCode(), urlConn.getResponseMessage())); + Thread.sleep(defaultDelay * 1000); + errorList.add(String.format("%s %s", urlConn.getResponseCode(), urlConn.getResponseMessage())); + urlConn.disconnect(); + return attemptDownload(requestUrl, retryNumber + 1, errorList); + } else { + input = urlConn.getInputStream(); + responseType = urlConn.getContentType(); + return input; + } + } catch (IOException e) { + log.error("error while retrieving from http-connection occured: " + requestUrl, e); + Thread.sleep(defaultDelay * 1000); + errorList.add(e.getMessage()); + return attemptDownload(requestUrl, retryNumber + 1, errorList); + } + } catch (InterruptedException e) { + throw new CollectorServiceException(e); + } + } + + private void logHeaderFields(final HttpURLConnection urlConn) throws IOException { + log.debug("StatusCode: " + urlConn.getResponseMessage()); + + for (Map.Entry> e : urlConn.getHeaderFields().entrySet()) { + if (e.getKey() != null) { + for (String v : e.getValue()) { + log.debug(" key: " + e.getKey() + " - value: " + v); + } + } + } + } + + private int obtainRetryAfter(final Map> headerMap) { + for (String key : headerMap.keySet()) { + if ((key != null) && key.toLowerCase().equals("retry-after") && (headerMap.get(key).size() > 0) + && NumberUtils.isCreatable(headerMap.get(key).get(0))) { + return Integer + .parseInt(headerMap.get(key).get(0)) + 10; + } + } + return -1; + } + + private String obtainNewLocation(final Map> headerMap) throws CollectorServiceException { + for (String key : headerMap.keySet()) { + if ((key != null) && key.toLowerCase().equals("location") && (headerMap.get(key).size() > 0)) { + return headerMap.get(key).get(0); + } + } + throw new CollectorServiceException("The requested url has been MOVED, but 'location' param is MISSING"); + } + + /** + * register for https scheme; this is a workaround and not intended for the use in trusted environments + */ + public void initTrustManager() { + final X509TrustManager tm = new X509TrustManager() { + + @Override + public void checkClientTrusted(final X509Certificate[] xcs, final String string) { + } + + @Override + public void checkServerTrusted(final X509Certificate[] xcs, final String string) { + } + + @Override + public X509Certificate[] getAcceptedIssuers() { + return null; + } + }; + try { + final SSLContext ctx = SSLContext.getInstance("TLS"); + ctx.init(null, new TrustManager[] { + tm + }, null); + HttpsURLConnection.setDefaultSSLSocketFactory(ctx.getSocketFactory()); + } catch (GeneralSecurityException e) { + log.fatal(e); + throw new IllegalStateException(e); + } + } + + public int getMaxNumberOfRetry() { + return maxNumberOfRetry; + } + + public void setMaxNumberOfRetry(final int maxNumberOfRetry) { + this.maxNumberOfRetry = maxNumberOfRetry; + } + + public int getDefaultDelay() { + return defaultDelay; + } + + public void setDefaultDelay(final int defaultDelay) { + this.defaultDelay = defaultDelay; + } + + public int getReadTimeOut() { + return readTimeOut; + } + + public void setReadTimeOut(final int readTimeOut) { + this.readTimeOut = readTimeOut; + } + + public String getResponseType() { + return responseType; + } + +} diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/action_set_parameters.json b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/action_set_parameters.json new file mode 100644 index 0000000000..a0856e10ef --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/action_set_parameters.json @@ -0,0 +1,26 @@ +[ + { + "paramName": "issm", + "paramLongName": "isSparkSessionManaged", + "paramDescription": "when true will stop SparkSession after job execution", + "paramRequired": false +}, +{ +"paramName": "pjp", +"paramLongName": "projectPath", +"paramDescription": "the URL from where to get the projects file", +"paramRequired": true +}, +{ +"paramName": "pp", +"paramLongName": "programmePath", +"paramDescription": "the URL from where to get the programme file", +"paramRequired": true +}, +{ +"paramName": "o", +"paramLongName": "outputPath", +"paramDescription": "the path of the new ActionSet", +"paramRequired": true +} +] \ No newline at end of file diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/config-default.xml b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/config-default.xml new file mode 100644 index 0000000000..fe82ae1940 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/config-default.xml @@ -0,0 +1,54 @@ + + + jobTracker + yarnRM + + + nameNode + hdfs://nameservice1 + + + oozie.use.system.libpath + true + + + oozie.action.sharelib.for.spark + spark2 + + + hive_metastore_uris + thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083 + + + spark2YarnHistoryServerAddress + http://iis-cdh5-test-gw.ocean.icm.edu.pl:18089 + + + spark2ExtraListeners + com.cloudera.spark.lineage.NavigatorAppListener + + + spark2SqlQueryExecutionListeners + com.cloudera.spark.lineage.NavigatorQueryListener + + + sparkExecutorNumber + 4 + + + spark2EventLogDir + /user/spark/spark2ApplicationHistory + + + sparkDriverMemory + 15G + + + sparkExecutorMemory + 6G + + + sparkExecutorCores + 1 + + \ No newline at end of file diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/workflow.xml b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/workflow.xml new file mode 100644 index 0000000000..1e34456752 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/workflow.xml @@ -0,0 +1,146 @@ + + + + projectFileURL + the url where to get the projects file + + + + programmeFileURL + the url where to get the programme file + + + + outputPath + path where to store the action set + + + + + + Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] + + + + + + + + + + + + + + + eu.dnetlib.dhp.actionmanager.project.csvutils.ReadCSV + --hdfsNameNode${nameNode} + --fileURL${projectFileURL} + --hdfsPath${workingDir}/projects + --classForNameeu.dnetlib.dhp.actionmanager.project.csvutils.CSVProject + + + + + + + + eu.dnetlib.dhp.actionmanager.project.csvutils.ReadCSV + --hdfsNameNode${nameNode} + --fileURL${programmeFileURL} + --hdfsPath${workingDir}/programme + --classForNameeu.dnetlib.dhp.actionmanager.project.csvutils.CSVProgramme + + + + + + + + eu.dnetlib.dhp.actionmanager.project.ReadProjectsFromDB + --hdfsPath${workingDir}/dbProjects + --hdfsNameNode${nameNode} + --postgresUrl${postgresURL} + --postgresUser${postgresUser} + --postgresPassword${postgresPassword} + + + + + + + + yarn + cluster + PrepareProgramme + eu.dnetlib.dhp.actionmanager.project.PrepareProgramme + dhp-aggregation-${projectVersion}.jar + + --executor-cores=${sparkExecutorCores} + --executor-memory=${sparkExecutorMemory} + --driver-memory=${sparkDriverMemory} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.sql.shuffle.partitions=3840 + + --programmePath${workingDir}/programme + --outputPath${workingDir}/preparedProgramme + + + + + + + + yarn + cluster + PrepareProjects + eu.dnetlib.dhp.actionmanager.project.PrepareProjects + dhp-aggregation-${projectVersion}.jar + + --executor-cores=${sparkExecutorCores} + --executor-memory=${sparkExecutorMemory} + --driver-memory=${sparkDriverMemory} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.sql.shuffle.partitions=3840 + + --projectPath${workingDir}/projects + --outputPath${workingDir}/preparedProjects + --dbProjectPath${workingDir}/dbProjects + + + + + + + + yarn + cluster + ProjectProgrammeAS + eu.dnetlib.dhp.actionmanager.project.SparkAtomicActionJob + dhp-aggregation-${projectVersion}.jar + + --executor-cores=${sparkExecutorCores} + --executor-memory=${sparkExecutorMemory} + --driver-memory=${sparkDriverMemory} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.sql.shuffle.partitions=3840 + + --projectPath${workingDir}/preparedProjects + --programmePath${workingDir}/preparedProgramme + --outputPath${outputPath} + + + + + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/parameters.json b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/parameters.json new file mode 100644 index 0000000000..dd3de70f6e --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/parameters.json @@ -0,0 +1,29 @@ +[ + + { + "paramName": "fu", + "paramLongName" : "fileURL", + "paramDescription" : "the url of the file to download", + "paramRequired" : true + }, + { + "paramName": "hp", + "paramLongName" : "hdfsPath", + "paramDescription" : "where to save the file", + "paramRequired" : true + }, + { + "paramName": "hnn", + "paramLongName" : "hdfsNameNode", + "paramDescription" : "the name node", + "paramRequired" : true + }, + { + "paramName": "cfn", + "paramLongName" : "classForName", + "paramDescription" : "the name of the class to deserialize the csv to", + "paramRequired" : true +} + + +] \ No newline at end of file diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/prepare_programme_parameters.json b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/prepare_programme_parameters.json new file mode 100644 index 0000000000..54083e1089 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/prepare_programme_parameters.json @@ -0,0 +1,20 @@ +[ + { + "paramName": "issm", + "paramLongName": "isSparkSessionManaged", + "paramDescription": "when true will stop SparkSession after job execution", + "paramRequired": false +}, +{ +"paramName": "pp", +"paramLongName": "programmePath", +"paramDescription": "the URL from where to get the programme file", +"paramRequired": true +}, +{ +"paramName": "o", +"paramLongName": "outputPath", +"paramDescription": "the path of the new ActionSet", +"paramRequired": true +} +] \ No newline at end of file diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/prepare_project_parameters.json b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/prepare_project_parameters.json new file mode 100644 index 0000000000..49f9c73062 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/prepare_project_parameters.json @@ -0,0 +1,26 @@ +[ + { + "paramName": "issm", + "paramLongName": "isSparkSessionManaged", + "paramDescription": "when true will stop SparkSession after job execution", + "paramRequired": false +}, +{ +"paramName": "pjp", +"paramLongName": "projectPath", +"paramDescription": "the URL from where to get the programme file", +"paramRequired": true +}, +{ +"paramName": "o", +"paramLongName": "outputPath", +"paramDescription": "the path of the new ActionSet", +"paramRequired": true +}, + { + "paramName": "dbp", + "paramLongName": "dbProjectPath", + "paramDescription": "the path of the project code read from db", + "paramRequired": true + } +] \ No newline at end of file diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/read_projects_db.json b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/read_projects_db.json new file mode 100644 index 0000000000..9a2eadaa7d --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/read_projects_db.json @@ -0,0 +1,32 @@ +[ + { + "paramName": "p", + "paramLongName": "hdfsPath", + "paramDescription": "the path where storing the sequential file", + "paramRequired": true + }, + { + "paramName": "nn", + "paramLongName": "hdfsNameNode", + "paramDescription": "the name node on hdfs", + "paramRequired": true + }, + { + "paramName": "pgurl", + "paramLongName": "postgresUrl", + "paramDescription": "postgres url, example: jdbc:postgresql://localhost:5432/testdb", + "paramRequired": true + }, + { + "paramName": "pguser", + "paramLongName": "postgresUser", + "paramDescription": "postgres user", + "paramRequired": false + }, + { + "paramName": "pgpasswd", + "paramLongName": "postgresPassword", + "paramDescription": "postgres password", + "paramRequired": false + } +] \ No newline at end of file diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/CSVParserTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/CSVParserTest.java new file mode 100644 index 0000000000..17fdd45111 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/CSVParserTest.java @@ -0,0 +1,41 @@ + +package eu.dnetlib.dhp.actionmanager.project; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import org.apache.commons.io.IOUtils; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import eu.dnetlib.dhp.actionmanager.project.csvutils.CSVParser; + +public class CSVParserTest { + + private static Path workingDir; + + @BeforeAll + public static void beforeAll() throws IOException { + workingDir = Files.createTempDirectory(CSVParserTest.class.getSimpleName()); + + } + + @Test + public void readProgrammeTest() throws Exception { + + String programmecsv = IOUtils + .toString( + getClass() + .getClassLoader() + .getResourceAsStream("eu/dnetlib/dhp/actionmanager/project/programme.csv")); + + CSVParser csvParser = new CSVParser(); + + List pl = csvParser.parse(programmecsv, "eu.dnetlib.dhp.actionmanager.project.csvutils.CSVProgramme"); + + System.out.println(pl.size()); + + } +} diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/PrepareProgrammeTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/PrepareProgrammeTest.java new file mode 100644 index 0000000000..7f890a8a35 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/PrepareProgrammeTest.java @@ -0,0 +1,94 @@ + +package eu.dnetlib.dhp.actionmanager.project; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.apache.commons.io.FileUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.SparkSession; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.dhp.actionmanager.project.csvutils.CSVProgramme; + +public class PrepareProgrammeTest { + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private static final ClassLoader cl = eu.dnetlib.dhp.actionmanager.project.PrepareProgrammeTest.class + .getClassLoader(); + + private static SparkSession spark; + + private static Path workingDir; + private static final Logger log = LoggerFactory + .getLogger(eu.dnetlib.dhp.actionmanager.project.PrepareProgrammeTest.class); + + @BeforeAll + public static void beforeAll() throws IOException { + workingDir = Files + .createTempDirectory(eu.dnetlib.dhp.actionmanager.project.PrepareProgrammeTest.class.getSimpleName()); + log.info("using work dir {}", workingDir); + + SparkConf conf = new SparkConf(); + conf.setAppName(eu.dnetlib.dhp.actionmanager.project.PrepareProgrammeTest.class.getSimpleName()); + + conf.setMaster("local[*]"); + conf.set("spark.driver.host", "localhost"); + conf.set("hive.metastore.local", "true"); + conf.set("spark.ui.enabled", "false"); + conf.set("spark.sql.warehouse.dir", workingDir.toString()); + conf.set("hive.metastore.warehouse.dir", workingDir.resolve("warehouse").toString()); + + spark = SparkSession + .builder() + .appName(PrepareProgrammeTest.class.getSimpleName()) + .config(conf) + .getOrCreate(); + } + + @AfterAll + public static void afterAll() throws IOException { + FileUtils.deleteDirectory(workingDir.toFile()); + spark.stop(); + } + + @Test + public void numberDistinctProgrammeTest() throws Exception { + PrepareProgramme + .main( + new String[] { + "-isSparkSessionManaged", + Boolean.FALSE.toString(), + "-programmePath", + getClass().getResource("/eu/dnetlib/dhp/actionmanager/project/whole_programme.json.gz").getPath(), + "-outputPath", + workingDir.toString() + "/preparedProgramme" + }); + + final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext()); + + JavaRDD tmp = sc + .textFile(workingDir.toString() + "/preparedProgramme") + .map(item -> OBJECT_MAPPER.readValue(item, CSVProgramme.class)); + + Assertions.assertEquals(277, tmp.count()); + + Dataset verificationDataset = spark.createDataset(tmp.rdd(), Encoders.bean(CSVProgramme.class)); + + Assertions.assertEquals(0, verificationDataset.filter("shortTitle =''").count()); + } + +} diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/PrepareProjectTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/PrepareProjectTest.java new file mode 100644 index 0000000000..5ff88e46f6 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/PrepareProjectTest.java @@ -0,0 +1,99 @@ + +package eu.dnetlib.dhp.actionmanager.project; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.apache.commons.io.FileUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.SparkSession; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.dhp.actionmanager.project.csvutils.CSVProgramme; +import eu.dnetlib.dhp.actionmanager.project.csvutils.CSVProject; + +public class PrepareProjectTest { + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private static final ClassLoader cl = PrepareProjectTest.class + .getClassLoader(); + + private static SparkSession spark; + + private static Path workingDir; + private static final Logger log = LoggerFactory + .getLogger(PrepareProjectTest.class); + + @BeforeAll + public static void beforeAll() throws IOException { + workingDir = Files + .createTempDirectory(PrepareProjectTest.class.getSimpleName()); + log.info("using work dir {}", workingDir); + + SparkConf conf = new SparkConf(); + conf.setAppName(PrepareProjectTest.class.getSimpleName()); + + conf.setMaster("local[*]"); + conf.set("spark.driver.host", "localhost"); + conf.set("hive.metastore.local", "true"); + conf.set("spark.ui.enabled", "false"); + conf.set("spark.sql.warehouse.dir", workingDir.toString()); + conf.set("hive.metastore.warehouse.dir", workingDir.resolve("warehouse").toString()); + + spark = SparkSession + .builder() + .appName(PrepareProjectTest.class.getSimpleName()) + .config(conf) + .getOrCreate(); + } + + @AfterAll + public static void afterAll() throws IOException { + FileUtils.deleteDirectory(workingDir.toFile()); + spark.stop(); + } + + @Test + public void numberDistinctProjectTest() throws Exception { + PrepareProjects + .main( + new String[] { + "-isSparkSessionManaged", + Boolean.FALSE.toString(), + "-projectPath", + getClass().getResource("/eu/dnetlib/dhp/actionmanager/project/projects_subset.json").getPath(), + "-outputPath", + workingDir.toString() + "/preparedProjects", + "-dbProjectPath", + getClass().getResource("/eu/dnetlib/dhp/actionmanager/project/dbProject").getPath(), + + }); + + final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext()); + + JavaRDD tmp = sc + .textFile(workingDir.toString() + "/preparedProjects") + .map(item -> OBJECT_MAPPER.readValue(item, CSVProject.class)); + + Assertions.assertEquals(8, tmp.count()); + + Dataset verificationDataset = spark.createDataset(tmp.rdd(), Encoders.bean(CSVProject.class)); + + Assertions.assertEquals(0, verificationDataset.filter("length(id) = 0").count()); + Assertions.assertEquals(0, verificationDataset.filter("length(programme) = 0").count()); + } + +} diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java new file mode 100644 index 0000000000..718cd8ebe0 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java @@ -0,0 +1,94 @@ + +package eu.dnetlib.dhp.actionmanager.project; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.io.Text; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.sql.SparkSession; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.dhp.schema.action.AtomicAction; +import eu.dnetlib.dhp.schema.oaf.Project; + +public class SparkUpdateProjectTest { + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private static final ClassLoader cl = eu.dnetlib.dhp.actionmanager.project.SparkUpdateProjectTest.class + .getClassLoader(); + + private static SparkSession spark; + + private static Path workingDir; + private static final Logger log = LoggerFactory + .getLogger(eu.dnetlib.dhp.actionmanager.project.SparkUpdateProjectTest.class); + + @BeforeAll + public static void beforeAll() throws IOException { + workingDir = Files + .createTempDirectory(eu.dnetlib.dhp.actionmanager.project.SparkUpdateProjectTest.class.getSimpleName()); + log.info("using work dir {}", workingDir); + + SparkConf conf = new SparkConf(); + conf.setAppName(eu.dnetlib.dhp.actionmanager.project.SparkUpdateProjectTest.class.getSimpleName()); + + conf.setMaster("local[*]"); + conf.set("spark.driver.host", "localhost"); + conf.set("hive.metastore.local", "true"); + conf.set("spark.ui.enabled", "false"); + conf.set("spark.sql.warehouse.dir", workingDir.toString()); + conf.set("hive.metastore.warehouse.dir", workingDir.resolve("warehouse").toString()); + + spark = SparkSession + .builder() + .appName(SparkUpdateProjectTest.class.getSimpleName()) + .config(conf) + .getOrCreate(); + } + + @AfterAll + public static void afterAll() throws IOException { + FileUtils.deleteDirectory(workingDir.toFile()); + spark.stop(); + } + + @Test + public void numberDistinctProgrammeTest() throws Exception { + SparkAtomicActionJob + .main( + new String[] { + "-isSparkSessionManaged", + Boolean.FALSE.toString(), + "-programmePath", + getClass() + .getResource("/eu/dnetlib/dhp/actionmanager/project/preparedProgramme_whole.json.gz") + .getPath(), + "-projectPath", + getClass().getResource("/eu/dnetlib/dhp/actionmanager/project/prepared_projects.json").getPath(), + "-outputPath", + workingDir.toString() + "/actionSet" + }); + + final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext()); + + JavaRDD tmp = sc + .sequenceFile(workingDir.toString() + "/actionSet", Text.class, Text.class) + .map(value -> OBJECT_MAPPER.readValue(value._2().toString(), AtomicAction.class)) + .map(aa -> ((Project) aa.getPayload())); + + Assertions.assertEquals(14, tmp.count()); + + } +} diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/httpconnector/HttpConnectorTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/httpconnector/HttpConnectorTest.java new file mode 100644 index 0000000000..51a7019cac --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/httpconnector/HttpConnectorTest.java @@ -0,0 +1,39 @@ + +package eu.dnetlib.dhp.actionmanager.project.httpconnector; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.ssl.SSLContextBuilder; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +public class HttpConnectorTest { + + private static final Log log = LogFactory.getLog(HttpConnectorTest.class); + private static HttpConnector connector; + + private static final String URL = "http://cordis.europa.eu/data/reference/cordisref-H2020programmes.csv"; + private static final String URL_MISCONFIGURED_SERVER = "https://www.alexandria.unisg.ch/cgi/oai2?verb=Identify"; + private static final String URL_GOODSNI_SERVER = "https://air.unimi.it/oai/openaire?verb=Identify"; + + private static final SSLContextBuilder sslContextBuilder = new SSLContextBuilder(); + private static SSLConnectionSocketFactory sslSocketFactory; + + @BeforeAll + public static void setUp() { + connector = new HttpConnector(); + } + + @Test + + public void testGetInputSource() throws CollectorServiceException { + System.out.println(connector.getInputSource(URL)); + } + + @Test + public void testGoodServers() throws CollectorServiceException { + System.out.println(connector.getInputSource(URL_GOODSNI_SERVER)); + } + +} diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collector/worker/DnetCollectorWorkerApplicationTests.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collector/worker/DnetCollectorWorkerApplicationTests.java index 87bd3be3d8..c745219fea 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collector/worker/DnetCollectorWorkerApplicationTests.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collector/worker/DnetCollectorWorkerApplicationTests.java @@ -8,6 +8,7 @@ import java.io.File; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import com.fasterxml.jackson.databind.ObjectMapper; @@ -19,6 +20,7 @@ import eu.dnetlib.dhp.collection.worker.utils.CollectorPluginFactory; import eu.dnetlib.message.Message; import eu.dnetlib.message.MessageManager; +@Disabled public class DnetCollectorWorkerApplicationTests { private final ArgumentApplicationParser argumentParser = mock(ArgumentApplicationParser.class); diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/project/dbProject b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/project/dbProject new file mode 100644 index 0000000000..f8e3c45898 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/project/dbProject @@ -0,0 +1,8 @@ +{"code":"894593"} +{"code":"897004"} +{"code":"896300"} +{"code":"892890"} +{"code":"886828"} +{"code":"8867767"} +{"code":"101003374"} +{"code":"886776"} \ No newline at end of file diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/project/preparedProgramme_whole.json.gz b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/project/preparedProgramme_whole.json.gz new file mode 100644 index 0000000000..1afa730616 Binary files /dev/null and b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/project/preparedProgramme_whole.json.gz differ diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/project/prepared_projects.json b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/project/prepared_projects.json new file mode 100644 index 0000000000..b8805b2db5 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/project/prepared_projects.json @@ -0,0 +1,16 @@ +{"rcn":"229267","id":"894593","acronym":"ICARUS","status":"SIGNED","programme":"H2020-EU.3.4.7.","topics":"SESAR-ER4-31-2019","frameworkProgramme":"H2020","title":"INTEGRATED COMMON ALTITUDE REFERENCE SYSTEM FOR U-SPACE","startDate":"2020-05-01","endDate":"2022-07-31","projectUrl":"","objective":"ICARUS project proposes an innovative solution to the challenge of the Common Altitude Reference inside VLL airspaces with the definition of a new U-space service and its validation in a real operational environment. In manned aviation, the methods of determining the altitude of an aircraft are based on pressure altitude difference measurements (e.g. QFE, QNH and FL) referred to a common datum. \nThe UA flights superimpose a new challenge, since a small drone may take off and land almost from everywhere, hence reducing the original significance of QFE settings, introduced on behalf of manned pilots to display on the altimeter the 0-height at touchdown on the local runway. In fact, the possibility for n drones to take off at n different places would generate a series of n different QFE corresponding to different heights of ground pressures referred to the take-off “Home points”. Therefore for a large number drones, new methodologies and procedures shall be put in place. The ICARUS defines a new U-space U3 service tightly coupled with the interface of the existing U-space services (e.g. Tracking, and Flight Planning services). The users of ICARUS service shall be remote pilots competent to fly in BVLOS in the specific category of UAS operations and ultralight GA pilots potentially sharing the same VLL airspace. \nThe ICARUS proposed approach foresees the realization of DTM service embedded in an Application Program Interface (API) that can be queried by UAS pilot/operator (or by drone itself) based on the actual positioning of the UA along its trajectory, computed by the (E)GNSS receiver. The output of the DTM service would provide information on distance from ground/obstacles in combination with the common altitude reference.\nAccuracy, continuity, integrity and availability requirements for GNSS-based altimetry together with accuracy and resolution requirements of the DTM to be provided by ICARUS service are key topics of the study.","totalCost":"1385286,25","ecMaxContribution":"1144587,5","call":"H2020-SESAR-2019-2","fundingScheme":"SESAR-RIA","coordinator":"E-GEOS SPA","coordinatorCountry":"IT","participants":"TOPVIEW SRL;TELESPAZIO SPA;DRONERADAR SP Z O.O.;EUROCONTROL - EUROPEAN ORGANISATION FOR THE SAFETY OF AIR NAVIGATION;EUROUSC ESPANA SL;POLITECNICO DI MILANO;UNIVERSITA DEGLI STUDI DI ROMA LA SAPIENZA","participantCountries":"IT;PL;BE;ES","subjects":""} +{"rcn":"229284","id":"897004","acronym":"ISLand","status":"SIGNED","programme":"H2020-EU.1.3.2.","topics":"MSCA-IF-2019","frameworkProgramme":"H2020","title":"Isolation and Segregation Landscape. Archaeology of quarantine in the Indian Ocean World","startDate":"2020-11-01","endDate":"2023-10-31","projectUrl":"","objective":"The proposed research presents an experimental and completely novel investigation within the historical archaeology,\napplied to isolated contexts. The main objective of ISLand is to provide a new way of thinking about human interactions\nwithin colonial empires and bringing colonial studies into dialogue with medical history and the emerging concept of\nhealthscaping. It seeks to do so by studying quarantine facilities in the Indian Ocean World during the long nineteenth\ncentury, a crucial period for the history of European empires in that region and a flashpoint for the conceptualization of\nmodern public health. Quarantine, traditionally viewed as merely a mechanism for the control of disease, will be analyzed as\nthe outward material response to important changes taking place socially, ecologically, and politically at the time.\nThe project is a part of an international, interdisciplinary effort, combining history, archaeology, and anthropology. The\nresearcher will tap numerous archival sources and archaeological data from selected sites, examine them through social and\nspatial analysis, and systematically analyze a test case in Mauritius through the most innovative methods that target\nlandscape and standing archaeology.\nThe broader impacts of ISLand have relevance for current European approaches to the migration crisis, where the threat of\ndisease has been ignited as a potentially debilitating consequence of immigration from extra-European countries. The\ntraining-through-research project at the Stanford University, the top institution where acquiring knowledge and skills in\nhistorical archaeology, will allow the applicant to develop into a position of professional maturity with a specific\ninterdisciplinary set of skills. With the support of the host institutions in EU, the researcher will promote historical archaeology\nin European academy, stimulating new approaches in usual archaeological research and an interdisciplinary approach with\ncultural anthropology.","totalCost":"253052,16","ecMaxContribution":"253052,16","call":"H2020-MSCA-IF-2019","fundingScheme":"MSCA-IF-GF","coordinator":"UNIVERSITEIT VAN AMSTERDAM","coordinatorCountry":"NL","participants":"","participantCountries":"","subjects":""} +{"rcn":"229281","id":"896300","acronym":"STRETCH","status":"SIGNED","programme":"H2020-EU.1.3.2.","topics":"MSCA-IF-2019","frameworkProgramme":"H2020","title":"Smart Textiles for RETrofitting and Monitoring of Cultural Heritage Buildings","startDate":"2020-09-01","endDate":"2022-08-31","projectUrl":"","objective":"This project aims to develop novel techniques using smart multifunctional materials for the combined seismic-plus-energy retrofitting, and Structural Health Monitoring (SHM) of the European cultural heritage buildings (CHB). The need for upgrading the existing old and CHB is becoming increasingly important for the EU countries, due to: (1) their poor structural performance during recent earthquakes (e.g. Italy, Greece) or other natural hazards (e.g. extreme weather conditions) that have resulted in significant economic losses, and loss of human lives; and (2) their low energy performance which increases significantly their energy consumption (buildings are responsible for 40% of EU energy consumption). Moreover, the SHM of the existing buildings is crucial for assessing continuously their structural integrity and thus to provide information for planning cost effective and sustainable maintenance decisions. Since replacing the old buildings with new is not financially feasible, and even it is not allowed for CHB, their lifetime extension requires considering simultaneously both structural and energy retrofitting. It is noted that the annual cost of repair and maintenance of existing European building stock is estimated to be about 50% of the total construction budget, currently standing at more than €300 billion. To achieve cost effectiveness, STRETCH explores a novel approach, which integrates technical textile reinforcement with thermal insulation systems and strain sensors to provide simultaneous structural-plus-energy retrofitting combined with SHM, tailored for masonry cultural heritage building envelopes. The effectiveness of the proposed retrofitting system will be validated experimentally and analytically. Moreover, draft guidelines and recommendations for determining future research on the use of smart composite materials for the concurrent retrofitting (structural-plus-energy) and SHM of the existing cultural heritage buildings envelopes will be proposed.","totalCost":"183473,28","ecMaxContribution":"183473,28","call":"H2020-MSCA-IF-2019","fundingScheme":"MSCA-IF-EF-ST","coordinator":"JRC -JOINT RESEARCH CENTRE- EUROPEAN COMMISSION","coordinatorCountry":"BE","participants":"","participantCountries":"","subjects":""} +{"rcn":"229265","id":"892890","acronym":"RhythmicPrediction","status":"SIGNED","programme":"H2020-EU.1.3.2.","topics":"MSCA-IF-2019","frameworkProgramme":"H2020","title":"Rhythmic prediction in speech perception: are our brain waves in sync with our native language?","startDate":"2021-01-01","endDate":"2022-12-31","projectUrl":"","objective":"Speech has rhythmic properties that widely differ across languages. When we listen to foreign languages, we may perceive them to be more musical, or rather more rap-like than our own. Even if we are unaware of it, the rhythm and melody of language, i.e. prosody, reflects its linguistic structure. On the one hand, prosody emphasizes content words and new information with stress and accents. On the other hand, it is aligned to phrase edges, marking them with boundary tones. Prosody hence helps the listener to focus on important words and to chunk sentences into phrases, and phrases into words. In fact, prosody is even used predictively, for instance to time the onset of the next word, the next piece of new information, or the total remaining length of the utterance, so the listener can seamlessly start their own speaking turn. \nSo, the listener, or rather their brain, is actively predicting when important speech events will happen, using prosody. How prosodic rhythms are exploited to predict speech timing, however, is unclear. No link between prosody and neural predictive processing has yet been empirically made. One hypothesis is that rhythm, such as the alternation of stressed and unstressed syllables, helps listeners time their attention. Similar behavior is best captured by the notion of an internal oscillator which can be set straight by attentional spikes. While neuroscientific evidence for the relation of neural oscillators to speech processing is starting to emerge, no link to the use of prosody nor predictive listening exists, yet. Furthermore, it is still unknown how native language knowledge affects cortical oscillations, and how oscillations are affected by cross-linguistic differences in rhythmic structure. The current project combines the standing knowledge of prosodic typology with the recent advances in neuroscience on cortical oscillations, to investigate the role of internal oscillators on native prosody perception, and active speech prediction.","totalCost":"191149,44","ecMaxContribution":"191149,44","call":"H2020-MSCA-IF-2019","fundingScheme":"MSCA-IF-EF-ST","coordinator":"UNIVERSITE DE GENEVE","coordinatorCountry":"CH","participants":"","participantCountries":"","subjects":""} +{"rcn":"229235","id":"886828","acronym":"ASAP","status":"SIGNED","programme":"H2020-EU.1.3.2.","topics":"MSCA-IF-2019","frameworkProgramme":"H2020","title":"Advanced Solutions for Asphalt Pavements","startDate":"2021-09-01","endDate":"2023-08-31","projectUrl":"","objective":"The Advanced Solutions for Asphalt Pavements (ASAP) project involves the development of a unique road paving technology which will use a bio-bitumen rejuvenator to rejuvenate aged asphalt bitumen. This technology will help to extend the lifespan of asphalt pavements (roads) and will reduce the environmental and economic impact of roads and road maintenance processes. Recycling and self-healing processes will replace fossil fuel dependent technology. Self-healing will involve rejuvenating aged asphalt bitumen using a bio-rejuvenator developed using microalgae oils (rejuvenating bio-oil). Microalgae has been selected because of its fast growth, versatility and ability to survive within hostile environments, such as wastewater. \n\nASAP will utilise microalgae, cultivated within the wastewater treatment process, as a source of the rejuvenating bio-oil. The solvent (Soxhlet) processes will be used to extract the oil from the microalgae. To ensure the efficiency of the oil extraction process, an ultrasonication process will be used to pre-treat the microalgae. The suitability of rejuvenating bio-oil as a replacement for the bitumen rejuvenator (fossil fuel based) will be ascertained via a series of standard bituminous and accelerated tests. A rejuvenator-binder diffusion numerical model will be developed, based on the Delft Lattice concrete diffusion model, to determine the conditions required for rejuvenation to occur and to ascertain the healing rate of the asphalt binder. These parameters will facilitate the selection and optimisation of the asphalt self-healing systems (specifically the amount of bio-oil rejuvenator and time required) to achieve full rejuvenation. \n\nThis novel approach will benchmark the effectiveness of this intervention against existing asphalt design and maintenance processes and assess feasibility. The ASAP project presents an opportunity to revolutionise road design and maintenance processes and reduce its environmental and financial costs.","totalCost":"187572,48","ecMaxContribution":"187572,48","call":"H2020-MSCA-IF-2019","fundingScheme":"MSCA-IF-EF-ST","coordinator":"NEDERLANDSE ORGANISATIE VOOR TOEGEPAST NATUURWETENSCHAPPELIJK ONDERZOEK TNO","coordinatorCountry":"NL","participants":"","participantCountries":"","subjects":""} +{"rcn":null,"id":"886776","acronym":null,"status":null,"programme":"H2020-EU.2.1.4.","topics":null,"frameworkProgramme":"H2020","title":"BIO-Based pESTicides production for sustainable agriculture management plan","startDate":"2020-05-01","endDate":"2023-04-30","projectUrl":"","objective":"The BIOBESTicide project will validate and demonstrate the production of an effective and cost-efficient biopesticide. The demonstration will be based on an innovative bio-based value chain starting from the valorisation of sustainable biomasses, i.e. beet pulp and sugar molasses and will exploit the properties of the oomycete Pythium oligandrum strain I-5180 to increase natural plant defenses, to produce an highly effective and eco-friendly biopesticide solution for vine plants protection. \nBIOVITIS, the project coordinator, has developed, at laboratory level (TRL4), an effective method to biocontrol one of the major causes of worldwide vineyards destruction, the Grapevine Trunk Diseases (GTDs). The protection system is based on the oomycete Pythium oligandrum strain I-5180 that, at applied at optimal time and concentration, colonises the root of vines and stimulates the natural plant defences against GTDs, providing a protection that ranges between 40% and 60%. \nBIOBESTicide project will respond to the increasing demands for innovative solutions for crop protection agents, transferring the technology to a DEMO Plant able to produce more than 10 T of a high-quality oomycete-based biopesticide product per year (TRL7). \nThe BIOBESTicide project will validate the efficiency of the formulated product on vineyards of different geographical areas.\nTo assure the safety of products under both health and environmental points of view, a full and complete approval dossier for Pythium oligandrum strain I-5180 will be submitted in all the European countries. \nA Life Cycle Sustainability Assessment (LCSA) will be conducted to assess the environmental, economic and social impacts of the developed products.\nThe adoption of the effective and cost-efficient biopesticide will have significant impacts with a potential ROI of 30 % in just 5 years and a total EBITDA of more than € 6,400,000.","totalCost":"4402772,5","ecMaxContribution":"3069653","call":"H2020-BBI-JTI-2019","fundingScheme":"BBI-IA-DEMO","coordinator":"BIOVITIS","coordinatorCountry":"FR","participants":"MERCIER FRERES SARL;FUNDACION TECNALIA RESEARCH & INNOVATION;LAMBERTI SPA;EURION CONSULTING;CIAOTECH Srl;STOWARZYSZENIE ZACHODNIOPOMORSKI KLASTER CHEMICZNY ZIELONA CHEMIA;NORDZUCKER AG;INSTITUT NATIONAL DE RECHERCHE POUR L'AGRICULTURE, L'ALIMENTATION ET L'ENVIRONNEMENT;INSTITUT FRANCAIS DE LA VIGNE ET DU VIN","participantCountries":"FR;ES;IT;PL;DE","subjects":""} +{"rcn":null,"id":"886776","acronym":null,"status":null,"programme":"H2020-EU.3.2.6.","topics":"BBI-2019-SO3-D4","frameworkProgramme":"H2020","title":"BIO-Based pESTicides production for sustainable agriculture management plan","startDate":"2020-05-01","endDate":"2023-04-30","projectUrl":"","objective":"The BIOBESTicide project will validate and demonstrate the production of an effective and cost-efficient biopesticide. The demonstration will be based on an innovative bio-based value chain starting from the valorisation of sustainable biomasses, i.e. beet pulp and sugar molasses and will exploit the properties of the oomycete Pythium oligandrum strain I-5180 to increase natural plant defenses, to produce an highly effective and eco-friendly biopesticide solution for vine plants protection. \nBIOVITIS, the project coordinator, has developed, at laboratory level (TRL4), an effective method to biocontrol one of the major causes of worldwide vineyards destruction, the Grapevine Trunk Diseases (GTDs). The protection system is based on the oomycete Pythium oligandrum strain I-5180 that, at applied at optimal time and concentration, colonises the root of vines and stimulates the natural plant defences against GTDs, providing a protection that ranges between 40% and 60%. \nBIOBESTicide project will respond to the increasing demands for innovative solutions for crop protection agents, transferring the technology to a DEMO Plant able to produce more than 10 T of a high-quality oomycete-based biopesticide product per year (TRL7). \nThe BIOBESTicide project will validate the efficiency of the formulated product on vineyards of different geographical areas.\nTo assure the safety of products under both health and environmental points of view, a full and complete approval dossier for Pythium oligandrum strain I-5180 will be submitted in all the European countries. \nA Life Cycle Sustainability Assessment (LCSA) will be conducted to assess the environmental, economic and social impacts of the developed products.\nThe adoption of the effective and cost-efficient biopesticide will have significant impacts with a potential ROI of 30 % in just 5 years and a total EBITDA of more than € 6,400,000.","totalCost":"4402772,5","ecMaxContribution":"3069653","call":"H2020-BBI-JTI-2019","fundingScheme":"BBI-IA-DEMO","coordinator":"BIOVITIS","coordinatorCountry":"FR","participants":"MERCIER FRERES SARL;FUNDACION TECNALIA RESEARCH & INNOVATION;LAMBERTI SPA;EURION CONSULTING;CIAOTECH Srl;STOWARZYSZENIE ZACHODNIOPOMORSKI KLASTER CHEMICZNY ZIELONA CHEMIA;NORDZUCKER AG;INSTITUT NATIONAL DE RECHERCHE POUR L'AGRICULTURE, L'ALIMENTATION ET L'ENVIRONNEMENT;INSTITUT FRANCAIS DE LA VIGNE ET DU VIN","participantCountries":"FR;ES;IT;PL;DE","subjects":""} +{"rcn":"229276","id":"895426","acronym":"DisMoBoH","status":"SIGNED","programme":"H2020-EU.1.3.2.","topics":"MSCA-IF-2019","frameworkProgramme":"H2020","title":"Dissecting the molecular building principles of locally formed transcriptional hubs","startDate":"2021-09-01","endDate":"2023-08-31","projectUrl":"","objective":"Numerous DNA variants have already been identified that modulate inter-individual molecular traits – most prominently gene expression. However, since finding mechanistic interpretations relating genotype to phenotype has proven challenging, the focus has shifted to higher-order regulatory features, i.e. chromatin accessibility, transcription factor (TF) binding and 3D chromatin interactions. This revealed at least two enhancer types: “lead” enhancers in which the presence of genetic variants modulates the activity of entire chromatin domains, and “dependent” ones in which variants induce subtle changes, affecting DNA accessibility, but not transcription. Although cell type-specific TFs are likely important, it remains unclear which sequence features are required to establish such enhancer hierarchies, and under which circumstances genetic variation results in altered enhancer-promoter contacts and differential gene expression. Here, we propose to investigate the molecular mechanisms that link DNA variation to TF binding, chromatin topology, and gene expression response. We will leverage data on enhancer hierarchy and sequence-specific TF binding to identify the sequence signatures that define “lead” enhancers. The results will guide the design of a synthetic locus that serves as an in vivo platform to systematically vary the building blocks of local transcriptional units: i) DNA sequence – including variations in TF binding site affinity and syntax, ii) molecular interactions between TFs, and iii) chromatin conformation. To validate our findings, we will perform optical reconstruction of chromatin architecture for a select number of DNA variants. By simultaneously perturbing co-dependent features, this proposal will provide novel mechanistic insights into the formation of local transcriptional hubs.","totalCost":"191149,44","ecMaxContribution":"191149,44","call":"H2020-MSCA-IF-2019","fundingScheme":"MSCA-IF-EF-RI","coordinator":"ECOLE POLYTECHNIQUE FEDERALE DE LAUSANNE","coordinatorCountry":"CH","participants":"","participantCountries":"","subjects":""} +{"rcn":"229288","id":"898218","acronym":"devUTRs","status":"SIGNED","programme":"H2020-EU.1.3.2.","topics":"MSCA-IF-2019","frameworkProgramme":"H2020","title":"Uncovering the roles of 5′UTRs in translational control during early zebrafish development","startDate":"2021-09-01","endDate":"2023-08-31","projectUrl":"","objective":"Following fertilisation, metazoan embryos are transcriptionally silent, and embryogenesis is controlled by maternally deposited factors. Developmental progression requires the synthesis of new mRNAs and proteins in a coordinated fashion. Many posttranscriptional mechanisms regulate the fate of maternal mRNAs, but it is less understood how translational control shapes early embryogenesis. In eukaryotes, translation starts at the mRNA 5′ end, consisting of the 5′ cap and 5′ untranslated region (UTR). Protein synthesis is primarily regulated at the translation initiation step by elements within the 5′UTR. However, the role of 5′UTRs in regulating the dynamics of mRNA translation during vertebrate embryogenesis remains unexplored. For example, all vertebrate ribosomal protein (RP) mRNAs harbor a conserved terminal oligopyrimidine tract (TOP) in their 5′UTR. RP levels must be tightly controlled to ensure proper organismal development, but if and how the TOP motif mediates RP mRNA translational regulation during embryogenesis is unclear. Overall, we lack a systematic understanding of the regulatory information contained in 5′UTRs. In this work, I aim to uncover the 5′UTR in vivo rules for mRNA translational regulation during zebrafish embryogenesis. I propose to apply imaging and biochemical approaches to characterise the role of the TOP motif in RP mRNA translational regulation during embryogenesis and identify the trans-acting factor(s) that bind(s) to it (Aim 1). To systematically assess the contribution of 5′UTRs to mRNA translational regulation during zebrafish embryogenesis, I will couple a massively parallel reporter assay of 5′UTRs to polysome profiling (Aim 2). By integrating the translational behaviour of 5′UTR reporters throughout embryogenesis with sequence-based regression models, I anticipate to uncover novel cis-regulatory elements in 5′UTRs with developmental roles.","totalCost":"191149,44","ecMaxContribution":"191149,44","call":"H2020-MSCA-IF-2019","fundingScheme":"MSCA-IF-EF-ST","coordinator":"UNIVERSITAT BASEL","coordinatorCountry":"CH","participants":"","participantCountries":"","subjects":""} +{"rcn":"229261","id":"893787","acronym":"HOLYHOST","status":"SIGNED","programme":"H2020-EU.1.3.2.","topics":"MSCA-IF-2019","frameworkProgramme":"H2020","title":"Welfare and Hosting buildings in the “Holy Land” between the 4th and the 7th c. AD","startDate":"2020-10-01","endDate":"2022-09-30","projectUrl":"","objective":"Between the 4th and the 7th century AD, many hospices dedicated to the poor, elderly, strangers and travelers were built in the countryside, along roads, around and inside cities. They were commissioned by the Church, rich pious men and women concerned by the redeem of their sins, as well as emperors who saw this as a guarantee of social stability. Welfare is thus an important phenomena of Late Antiquity, abundantly mentioned by ancient literary sources and inscriptions, particularly in the eastern part of the Empire. However, the buildings that provided shelter and care to the needy have not yet received sufficient attention from archaeologists. Except for buildings which were identified by their inventors as hostels dedicated to pilgrims, they are still invisible in the field. \nThe aim of the HOLYHOST research project is to bring this social history’s main topic on the field of archaeology. It will address the welfare issue through the archaeological and architectural survey and study of Ancient welfare and hosting establishments’ remains, in the Holy Land (Palestine and Jordan) and around. This work will contribute to a better understanding of the practices linked to hospitality, welfare, accommodation and care in Antiquity. Moreover, such establishments served as models for medieval and modern Islamic, Jewish and Christian waqf institutions (religious endowment), and welfare continues to be highly relevant nowadays, through issues still at the heart of contemporary challenges debated in Europe: poverty, social exclusion, migrant crisis, principle of reception and hospitality. This interdisciplinary and diachronic research project will thus offer many new research perspectives, in terms of history of architecture, evolution of care practices, social and political regulations.","totalCost":"196707,84","ecMaxContribution":"196707,84","call":"H2020-MSCA-IF-2019","fundingScheme":"MSCA-IF-EF-ST","coordinator":"UNIVERSITE PARIS I PANTHEON-SORBONNE","coordinatorCountry":"FR","participants":"","participantCountries":"","subjects":""} +{"rcn":"229282","id":"896189","acronym":"MICADO","status":"SIGNED","programme":"H2020-EU.1.3.2.","topics":"MSCA-IF-2019","frameworkProgramme":"H2020","title":"Microbial contribution to continental wetland carbon budget","startDate":"2021-01-04","endDate":"2023-01-03","projectUrl":"","objective":"Continental wetlands are major carbon dioxide sinks but the second largest source of methane. Monitoring of wetland methane emissions revealed large inter-site variability that is hard to explain in the framework of current biogeochemical theories. Methane production in wetlands is an anaerobic microbial driven process involving a complex set of microbial metabolisms depending on the availability of (i) energy (via the presence of specific redox couples), (ii) organic substrates and (iii) specific microbial communities. To understand the complexity of microbial drivers on wetland methane emissions and quantify their contribution, the MICADO project will set up a multidisciplinary approach linking isotope organic geochemistry and environmental microbiology to assess microbial functioning in situ. As an organic geochemist I have developed an innovative approach to trace in situ microbial activity via compound specific carbon isotope analysis of microbe macromolecules and organic metabolites. The host institution is a leader in France in environmental microbiology and biogeochemistry developing high-throughput metagenomics and microbial rate assessments, for which I will be trained during the MICADO project. These techniques are highly complementary and combined they will provide a comprehensive knowledge on microbial metabolisms involved in organic matter degradation encompassing their complexity and interactions. This will revisit the relationships between organic substrate availability and microbial communities and will contribute at estimating the impact of microbial activity on wetland methane emissions. This project will give me the opportunity to acquire fundamental knowledge and to develop original lines of research that will consolidate my position as an independent scientist in biogeochemistry.","totalCost":"196707,84","ecMaxContribution":"196707,84","call":"H2020-MSCA-IF-2019","fundingScheme":"MSCA-IF-EF-ST","coordinator":"CENTRE NATIONAL DE LA RECHERCHE SCIENTIFIQUE CNRS","coordinatorCountry":"FR","participants":"","participantCountries":"","subjects":""} +{"rcn":"229249","id":"891624","acronym":"CuTAN","status":"SIGNED","programme":"H2020-EU.1.3.2.","topics":"MSCA-IF-2019","frameworkProgramme":"H2020","title":"Copper-Catalyzed Multicomponent Reactions in Tandem Processes for Target Molecule Synthesis","startDate":"2021-02-01","endDate":"2023-01-31","projectUrl":"","objective":"The invention of processes that can form several bonds, stereocentres and rings in a single process is key to a sustainable future in synthetic chemistry. Multicomponent reactions and tandem procedures are two strategies that enable the rapid build-up of molecular complexity from simple reagents. By combining these two strategies into a single procedure, the diversity, complexity and value of products can be further enhanced along with the efficiency and economy of their construction. In this project, Dr Satpathi will develop novel copper-catalyzed multicomponent couplings of unsaturated hydrocarbons (e.g. allenes, enynes) with imines and boron reagents. These procedures will provide high-value amine products with universally high regio-, diastero- and enantiocontrol. The products will bear a variety of synthetic handles, for example, amino, alkynyl/alkenyl, and boryl groups, thus the products are primed for subsequent transformation. Dr Satpathi will exploit this functionality in tandem intramolecular couplings (e.g. intramolecular Suzuki/Buchwald-Hartwig reactions) to provide core cyclic structures of drug molecules and natural products. Thus, through a tandem procedure of; 1) copper-catalyzed borofunctionalization, and; 2) subsequent transition-metal catalyzed cyclization, he will gain efficient access to highly sought-after complex molecules. Overall, the process will provide high-value, chiral, cyclic motifs from abundant, achiral, linear substrates. Finally, Dr Satpathi has identified the phthalide-isoquinoline family of alkaloids as target molecules to display the power of his tandem methodology. Dr Satpathi has devised a novel route, which begins with our tandem multifunctionalization/cyclization reaction, to provide a range of these important alkaloids. The chosen alkaloids are of particular interest as they display a range of bioactivities – for example as natural products, receptor antagonists and on-market drugs.","totalCost":"212933,76","ecMaxContribution":"212933,76","call":"H2020-MSCA-IF-2019","fundingScheme":"MSCA-IF-EF-ST","coordinator":"THE UNIVERSITY OF MANCHESTER","coordinatorCountry":"UK","participants":"","participantCountries":"","subjects":""} +{"rcn":"229239","id":"887259","acronym":"ALEHOOP","status":"SIGNED","programme":"H2020-EU.2.1.4.","topics":"BBI-2019-SO3-D3","frameworkProgramme":"H2020","title":"Biorefineries for the valorisation of macroalgal residual biomass and legume processing by-products to obtain new protein value chains for high-value food and feed applications","startDate":"2020-06-01","endDate":"2024-05-31","projectUrl":"","objective":"ALEHOOP provides the demonstration at pilot scale of both sustainable macroalgae and legume-based biorefineries for the recovery of low-cost dietary proteins from alga-based and plant residual biomass and their validation to meet market requirements of consumers and industry in the food and feed sectors. In these sectors, consumers are demanding affordable functional natural proteins from alternative sources and industry is demanding low-cost bio-based protein formulations with better performance and higher sustainability. \nCurrent protein demand for the 7.3 billion inhabitants of the world is approximately 202 Mt. Due to the rise in meat consumption more proteins are therefore required for animal feeding. To satisfy the current protein demand, Europe imports over 30 Mt of soy from the Americas each year mainly for animal feeding, entailing 95% dependency of EU on imported soy. Current sources of proteins are becoming unsustainable from an economic and environmental perspective for Europe resulting in concerns for sustainability and food security and leading to search for new alternative proteins. \nALEHOOP addresses the obtaining of proteins from green macroalgal blooms, brown seaweed by-products from algae processors and legume processing by-products (peas, lupines, beans and lentils) as alternative protein sources for animal feeding (case of green seaweed) and food applications (case of brown seaweed and legume by-products), since they are low cost and under-exploited biomass that do not compete with traditional food crops for space and resources. This will reduce EU´s dependency on protein imports and contribute to our raw material security. The new proteins will be validated in foods for elderly, sporty and overweight people, vegetarians and healthy consumers as well as for animal feed creating cross-sectorial interconnection between these value chains and supporting the projected business plan.","totalCost":"6718370","ecMaxContribution":"5140274,41","call":"H2020-BBI-JTI-2019","fundingScheme":"BBI-IA-DEMO","coordinator":"CONTACTICA S.L.","coordinatorCountry":"ES","participants":"CENTIV GMBH;ALGINOR ASA;FUNDACION TECNALIA RESEARCH & INNOVATION;INDUKERN,S.A.;ASOCIACION NACIONAL DE FABRICANTES DE CONSERVAS DE PESCADOS Y MARISCOS-CENTRO TECNICO NACIONAL DE CONSERVACION DE PRODUCTOS DE LA PESCA;BIOZOON GMBH;EIGEN VERMOGEN VAN HET INSTITUUT VOOR LANDBOUW- EN VISSERIJONDERZOEK;BIOSURYA SL;VYZKUMNY USTAV VETERINARNIHO LEKARSTVI;NUTRITION SCIENCES;TECHNOLOGICAL UNIVERSITY DUBLIN;GARLAN, S.COOP.;ISANATUR SPAIN SL;UNIVERSIDAD DE VIGO;UNIVERSIDAD DE CADIZ","participantCountries":"DE;NO;ES;BE;CZ;IE","subjects":""} +{"rcn":"229239","id":"887259","acronym":"ALEHOOP","status":"SIGNED","programme":"H2020-EU.3.2.6.","topics":"BBI-2019-SO3-D3","frameworkProgramme":"H2020","title":"Biorefineries for the valorisation of macroalgal residual biomass and legume processing by-products to obtain new protein value chains for high-value food and feed applications","startDate":"2020-06-01","endDate":"2024-05-31","projectUrl":"","objective":"ALEHOOP provides the demonstration at pilot scale of both sustainable macroalgae and legume-based biorefineries for the recovery of low-cost dietary proteins from alga-based and plant residual biomass and their validation to meet market requirements of consumers and industry in the food and feed sectors. In these sectors, consumers are demanding affordable functional natural proteins from alternative sources and industry is demanding low-cost bio-based protein formulations with better performance and higher sustainability. \nCurrent protein demand for the 7.3 billion inhabitants of the world is approximately 202 Mt. Due to the rise in meat consumption more proteins are therefore required for animal feeding. To satisfy the current protein demand, Europe imports over 30 Mt of soy from the Americas each year mainly for animal feeding, entailing 95% dependency of EU on imported soy. Current sources of proteins are becoming unsustainable from an economic and environmental perspective for Europe resulting in concerns for sustainability and food security and leading to search for new alternative proteins. \nALEHOOP addresses the obtaining of proteins from green macroalgal blooms, brown seaweed by-products from algae processors and legume processing by-products (peas, lupines, beans and lentils) as alternative protein sources for animal feeding (case of green seaweed) and food applications (case of brown seaweed and legume by-products), since they are low cost and under-exploited biomass that do not compete with traditional food crops for space and resources. This will reduce EU´s dependency on protein imports and contribute to our raw material security. The new proteins will be validated in foods for elderly, sporty and overweight people, vegetarians and healthy consumers as well as for animal feed creating cross-sectorial interconnection between these value chains and supporting the projected business plan.","totalCost":"6718370","ecMaxContribution":"5140274,41","call":"H2020-BBI-JTI-2019","fundingScheme":"BBI-IA-DEMO","coordinator":"CONTACTICA S.L.","coordinatorCountry":"ES","participants":"CENTIV GMBH;ALGINOR ASA;FUNDACION TECNALIA RESEARCH & INNOVATION;INDUKERN,S.A.;ASOCIACION NACIONAL DE FABRICANTES DE CONSERVAS DE PESCADOS Y MARISCOS-CENTRO TECNICO NACIONAL DE CONSERVACION DE PRODUCTOS DE LA PESCA;BIOZOON GMBH;EIGEN VERMOGEN VAN HET INSTITUUT VOOR LANDBOUW- EN VISSERIJONDERZOEK;BIOSURYA SL;VYZKUMNY USTAV VETERINARNIHO LEKARSTVI;NUTRITION SCIENCES;TECHNOLOGICAL UNIVERSITY DUBLIN;GARLAN, S.COOP.;ISANATUR SPAIN SL;UNIVERSIDAD DE VIGO;UNIVERSIDAD DE CADIZ","participantCountries":"DE;NO;ES;BE;CZ;IE","subjects":""} +{"rcn":"229258","id":"892834","acronym":"DENVPOC","status":"SIGNED","programme":"H2020-EU.1.3.2.","topics":"MSCA-IF-2019","frameworkProgramme":"H2020","title":"qPCR Microfluidics point-of-care platform for dengue diagnosis","startDate":"2020-05-18","endDate":"2022-05-17","projectUrl":"","objective":"As a result of Global climate change and fast urbanization, global outbreaks of Dengue (DENV)/ Zika(ZIKV)/Chikungunya(CHIKV) virus have the potential to occur. The most common pathway of these infections in humans is through the female Aedes mosquito vector. DENV is an exanthematous febrile disease with varied clinical manifestations and progressions . Due to similarities in symptoms between DENV and ZIKV and CHIKV, it is difficult to make a differential diagnosis, impeding appropriate, timely medical intervention. Furthermore, cross-reactivity with ZIKV, which was recently related to microcephaly, is a serious issue. In 2016, in Brazil alone, there were 4180 microcephaly cases reported instead of 163 cases, more in line with yearly expected projections , , Thus, the sooner an accurate diagnostic which differentiates DENV from the other manifestations is critical; most especially at the early stages of the infection, to have a reliable diagnosis in pregnant women. In 2016, the OMS emergency committee declared that the outbreaks and the potentially resultant neurological disorders in Brazil were an important international state of emergency in public health, as a result of the associated secondary effects; these diseases became a Global concern. This project allows developing a highly and fast Multiplex qPCR POC platform by using FASTGENE technology with a minimal amount of patient serotype. It would reduce the time of analysis (30 to 90’ for a standard) and costs. Additionally, the sample preprocessing and thermalization will shorten real-time PCR amplification time and will be integrated within the microfluidic systems. This platform can result in a commercialized product whereupon a main market target would be pregnant women and people living or traveling through/from outbreak risk areas.","totalCost":"196707,84","ecMaxContribution":"196707,84","call":"H2020-MSCA-IF-2019","fundingScheme":"MSCA-IF-EF-SE","coordinator":"BFORCURE","coordinatorCountry":"FR","participants":"","participantCountries":"","subjects":""} +{"rcn":"229280","id":"895716","acronym":"DoMiCoP","status":"SIGNED","programme":"H2020-EU.1.3.2.","topics":"MSCA-IF-2019","frameworkProgramme":"H2020","title":"The Diffusion of Migration Control Practice. Actors, Processes and Effects.","startDate":"2021-03-01","endDate":"2023-02-28","projectUrl":"","objective":"DoMiCoP develops new understandings and perspectives to study migration control in practice in the European Union by asking one main question: how and why do communities of practice develop and diffuse the knowledge required to put migration control into action? Unlike the nexus between expert knowledge, epistemic communities and policy formulation, the nexus between everyday knowledge, communities of practice and policy implementation has not yet received systematic scholarly attention. My project bridges that gap by focusing on intermediate arenas in which communities of practice take shape most notably the meetings and trainings that gather state and non-state actors involved in putting asylum, detention and removal into practice. By building on field-based methodologies (interviews and participant observations), DoMiCoP sheds ethnographic light on the role that ‘learning from abroad’ plays in the implementation of migration control in the EU. My project’s aim is threefold: 1) Identifying arenas at intermediate levels in which communities of practice take shape; 2) Analysing the communities of practice by focusing on the configurations of actors and organizations involved, the motivations underlying their involvement, the process of knowledge development in interaction, the conflicts and negotiations; 3) Revealing the role of non-state organizations (private for profit and not-for-profit). From a theoretical point of view, this project goes beyond the classical view of the implementation as a test to assess the effectiveness of policy transfers towards an analysis of policy transfer at that level of policy-making. From an empirical point of view, the project expands knowledge about less-studied venues of policy-making and provides original thick descriptions. From a methodological point of view, the project engages with qualitative methods for the study of policy diffusion and aims at responding to their main challenges through participant observation.","totalCost":"163673,28","ecMaxContribution":"163673,28","call":"H2020-MSCA-IF-2019","fundingScheme":"MSCA-IF-EF-ST","coordinator":"EUROPEAN UNIVERSITY INSTITUTE","coordinatorCountry":"IT","participants":"","participantCountries":"","subjects":""} \ No newline at end of file diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/project/programme.csv b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/project/programme.csv new file mode 100644 index 0000000000..6a9c855a03 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/project/programme.csv @@ -0,0 +1,25 @@ +rcn;code;title;shortTitle;language +664331;H2020-EU.3.3.2.;Un approvisionnement en électricité à faible coût et à faibles émissions de carbone;Low-cost, low-carbon energy supply;fr +664355;H2020-EU.3.3.7.;Absorción por el mercado de la innovación energética - explotación del Programa Energía Inteligente - Europa Europe;Market uptake of energy innovation;es +664323;H2020-EU.3.3.1.;Ridurre il consumo di energia e le emissioni di carbonio grazie all'uso intelligente e sostenibile;Reducing energy consumption and carbon footprint;it +664233;H2020-EU.2.3.2.3.;Wsparcie innowacji rynkowych;Supporting market-driven innovation;pl +664199;H2020-EU.2.1.5.1.;Tecnologías para las fábricas del futuro;Technologies for Factories of the Future;es +664235;H2020-EU.3.;PRIORITÉ «Défis de société»;Societal Challenges;fr +664355;H2020-EU.3.3.7.;"Assorbimento di mercato dell'innovazione energetica - iniziative fondate sul programma ""Energia intelligente - Europa""";Market uptake of energy innovation;it +664355;H2020-EU.3.3.7.;"Markteinführung von Energieinnovationen – Aufbau auf ""Intelligente Energie – Europa";Market uptake of energy innovation;de +664235;H2020-EU.3.;"PRIORIDAD ""Retos de la sociedad""";Societal Challenges;es +664231;H2020-EU.2.3.2.2.;Mejorar la capacidad de innovación de las PYME;Enhancing the innovation capacity of SMEs;es +664223;H2020-EU.2.3.;LIDERAZGO INDUSTRIAL - Innovación en la pequeña y mediana empresa;Innovation in SMEs;es +664323;H2020-EU.3.3.1.;Réduire la consommation d'énergie et l'empreinte carbone en utilisant l'énergie de manière intelligente et durable;Reducing energy consumption and carbon footprint;fr +664323;H2020-EU.3.3.1.;Reducir el consumo de energía y la huella de carbono mediante un uso inteligente y sostenible;Reducing energy consumption and carbon footprint;es +664215;H2020-EU.2.1.6.4.;Beitrag der europäischen Forschung zu internationalen Weltraumpartnerschaften;Research in support of international space partnerships;de +664213;H2020-EU.2.1.6.3.;Permettere lo sfruttamento dei dati spaziali;;it +664213;H2020-EU.2.1.6.3.;Permettre l'exploitation des données spatiales;Enabling exploitation of space data;fr +664231;H2020-EU.2.3.2.2.;Zwiększenie zdolności MŚP pod względem innowacji;Enhancing the innovation capacity of SMEs;pl +664231;H2020-EU.2.3.2.2.;Rafforzare la capacità di innovazione delle PMI;Enhancing the innovation capacity of SMEs;it +664213;H2020-EU.2.1.6.3.;Grundlagen für die Nutzung von Weltraumdaten;Enabling exploitation of space data;de +664211;H2020-EU.2.1.6.2.;Favorecer los avances en las tecnologías espaciales;Enabling advances in space technology;es +664209;H2020-EU.2.1.6.1.;Assurer la compétitivité et l'indépendance de l'Europe et promouvoir l'innovation dans le secteur spatial européen;Competitiveness, non-dependence and innovation;fr +664231;H2020-EU.2.3.2.2.;Renforcement de la capacité d'innovation des PME;Enhancing the innovation capacity of SMEs;fr +664203;H2020-EU.2.1.5.3.;Tecnologías sostenibles, eficientes en su utilización de recursos y de baja emisión de carbono en las industrias de transformación de gran consumo energético;Sustainable, resource-efficient and low-carbon technologies in energy-intensive process industries;es +664103;H2020-EU.1.2.1.;FET Open;FET Open;es \ No newline at end of file diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/project/projects_subset.json b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/project/projects_subset.json new file mode 100644 index 0000000000..edf83fbc8c --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/project/projects_subset.json @@ -0,0 +1,16 @@ +{"rcn":"229267","id":"894593","acronym":"ICARUS","status":"SIGNED","programme":"H2020-EU.3.4.7.","topics":"SESAR-ER4-31-2019","frameworkProgramme":"H2020","title":"INTEGRATED COMMON ALTITUDE REFERENCE SYSTEM FOR U-SPACE","startDate":"2020-05-01","endDate":"2022-07-31","projectUrl":"","objective":"ICARUS project proposes an innovative solution to the challenge of the Common Altitude Reference inside VLL airspaces with the definition of a new U-space service and its validation in a real operational environment. In manned aviation, the methods of determining the altitude of an aircraft are based on pressure altitude difference measurements (e.g. QFE, QNH and FL) referred to a common datum. \nThe UA flights superimpose a new challenge, since a small drone may take off and land almost from everywhere, hence reducing the original significance of QFE settings, introduced on behalf of manned pilots to display on the altimeter the 0-height at touchdown on the local runway. In fact, the possibility for n drones to take off at n different places would generate a series of n different QFE corresponding to different heights of ground pressures referred to the take-off “Home points”. Therefore for a large number drones, new methodologies and procedures shall be put in place. The ICARUS defines a new U-space U3 service tightly coupled with the interface of the existing U-space services (e.g. Tracking, and Flight Planning services). The users of ICARUS service shall be remote pilots competent to fly in BVLOS in the specific category of UAS operations and ultralight GA pilots potentially sharing the same VLL airspace. \nThe ICARUS proposed approach foresees the realization of DTM service embedded in an Application Program Interface (API) that can be queried by UAS pilot/operator (or by drone itself) based on the actual positioning of the UA along its trajectory, computed by the (E)GNSS receiver. The output of the DTM service would provide information on distance from ground/obstacles in combination with the common altitude reference.\nAccuracy, continuity, integrity and availability requirements for GNSS-based altimetry together with accuracy and resolution requirements of the DTM to be provided by ICARUS service are key topics of the study.","totalCost":"1385286,25","ecMaxContribution":"1144587,5","call":"H2020-SESAR-2019-2","fundingScheme":"SESAR-RIA","coordinator":"E-GEOS SPA","coordinatorCountry":"IT","participants":"TOPVIEW SRL;TELESPAZIO SPA;DRONERADAR SP Z O.O.;EUROCONTROL - EUROPEAN ORGANISATION FOR THE SAFETY OF AIR NAVIGATION;EUROUSC ESPANA SL;POLITECNICO DI MILANO;UNIVERSITA DEGLI STUDI DI ROMA LA SAPIENZA","participantCountries":"IT;PL;BE;ES","subjects":""} +{"rcn":"229284","id":"897004","acronym":"ISLand","status":"SIGNED","programme":"H2020-EU.1.3.2.","topics":"MSCA-IF-2019","frameworkProgramme":"H2020","title":"Isolation and Segregation Landscape. Archaeology of quarantine in the Indian Ocean World","startDate":"2020-11-01","endDate":"2023-10-31","projectUrl":"","objective":"The proposed research presents an experimental and completely novel investigation within the historical archaeology,\napplied to isolated contexts. The main objective of ISLand is to provide a new way of thinking about human interactions\nwithin colonial empires and bringing colonial studies into dialogue with medical history and the emerging concept of\nhealthscaping. It seeks to do so by studying quarantine facilities in the Indian Ocean World during the long nineteenth\ncentury, a crucial period for the history of European empires in that region and a flashpoint for the conceptualization of\nmodern public health. Quarantine, traditionally viewed as merely a mechanism for the control of disease, will be analyzed as\nthe outward material response to important changes taking place socially, ecologically, and politically at the time.\nThe project is a part of an international, interdisciplinary effort, combining history, archaeology, and anthropology. The\nresearcher will tap numerous archival sources and archaeological data from selected sites, examine them through social and\nspatial analysis, and systematically analyze a test case in Mauritius through the most innovative methods that target\nlandscape and standing archaeology.\nThe broader impacts of ISLand have relevance for current European approaches to the migration crisis, where the threat of\ndisease has been ignited as a potentially debilitating consequence of immigration from extra-European countries. The\ntraining-through-research project at the Stanford University, the top institution where acquiring knowledge and skills in\nhistorical archaeology, will allow the applicant to develop into a position of professional maturity with a specific\ninterdisciplinary set of skills. With the support of the host institutions in EU, the researcher will promote historical archaeology\nin European academy, stimulating new approaches in usual archaeological research and an interdisciplinary approach with\ncultural anthropology.","totalCost":"253052,16","ecMaxContribution":"253052,16","call":"H2020-MSCA-IF-2019","fundingScheme":"MSCA-IF-GF","coordinator":"UNIVERSITEIT VAN AMSTERDAM","coordinatorCountry":"NL","participants":"","participantCountries":"","subjects":""} +{"rcn":"229281","id":"896300","acronym":"STRETCH","status":"SIGNED","programme":"H2020-EU.1.3.2.","topics":"MSCA-IF-2019","frameworkProgramme":"H2020","title":"Smart Textiles for RETrofitting and Monitoring of Cultural Heritage Buildings","startDate":"2020-09-01","endDate":"2022-08-31","projectUrl":"","objective":"This project aims to develop novel techniques using smart multifunctional materials for the combined seismic-plus-energy retrofitting, and Structural Health Monitoring (SHM) of the European cultural heritage buildings (CHB). The need for upgrading the existing old and CHB is becoming increasingly important for the EU countries, due to: (1) their poor structural performance during recent earthquakes (e.g. Italy, Greece) or other natural hazards (e.g. extreme weather conditions) that have resulted in significant economic losses, and loss of human lives; and (2) their low energy performance which increases significantly their energy consumption (buildings are responsible for 40% of EU energy consumption). Moreover, the SHM of the existing buildings is crucial for assessing continuously their structural integrity and thus to provide information for planning cost effective and sustainable maintenance decisions. Since replacing the old buildings with new is not financially feasible, and even it is not allowed for CHB, their lifetime extension requires considering simultaneously both structural and energy retrofitting. It is noted that the annual cost of repair and maintenance of existing European building stock is estimated to be about 50% of the total construction budget, currently standing at more than €300 billion. To achieve cost effectiveness, STRETCH explores a novel approach, which integrates technical textile reinforcement with thermal insulation systems and strain sensors to provide simultaneous structural-plus-energy retrofitting combined with SHM, tailored for masonry cultural heritage building envelopes. The effectiveness of the proposed retrofitting system will be validated experimentally and analytically. Moreover, draft guidelines and recommendations for determining future research on the use of smart composite materials for the concurrent retrofitting (structural-plus-energy) and SHM of the existing cultural heritage buildings envelopes will be proposed.","totalCost":"183473,28","ecMaxContribution":"183473,28","call":"H2020-MSCA-IF-2019","fundingScheme":"MSCA-IF-EF-ST","coordinator":"JRC -JOINT RESEARCH CENTRE- EUROPEAN COMMISSION","coordinatorCountry":"BE","participants":"","participantCountries":"","subjects":""} +{"rcn":"229265","id":"892890","acronym":"RhythmicPrediction","status":"SIGNED","programme":"H2020-EU.1.3.2.","topics":"MSCA-IF-2019","frameworkProgramme":"H2020","title":"Rhythmic prediction in speech perception: are our brain waves in sync with our native language?","startDate":"2021-01-01","endDate":"2022-12-31","projectUrl":"","objective":"Speech has rhythmic properties that widely differ across languages. When we listen to foreign languages, we may perceive them to be more musical, or rather more rap-like than our own. Even if we are unaware of it, the rhythm and melody of language, i.e. prosody, reflects its linguistic structure. On the one hand, prosody emphasizes content words and new information with stress and accents. On the other hand, it is aligned to phrase edges, marking them with boundary tones. Prosody hence helps the listener to focus on important words and to chunk sentences into phrases, and phrases into words. In fact, prosody is even used predictively, for instance to time the onset of the next word, the next piece of new information, or the total remaining length of the utterance, so the listener can seamlessly start their own speaking turn. \nSo, the listener, or rather their brain, is actively predicting when important speech events will happen, using prosody. How prosodic rhythms are exploited to predict speech timing, however, is unclear. No link between prosody and neural predictive processing has yet been empirically made. One hypothesis is that rhythm, such as the alternation of stressed and unstressed syllables, helps listeners time their attention. Similar behavior is best captured by the notion of an internal oscillator which can be set straight by attentional spikes. While neuroscientific evidence for the relation of neural oscillators to speech processing is starting to emerge, no link to the use of prosody nor predictive listening exists, yet. Furthermore, it is still unknown how native language knowledge affects cortical oscillations, and how oscillations are affected by cross-linguistic differences in rhythmic structure. The current project combines the standing knowledge of prosodic typology with the recent advances in neuroscience on cortical oscillations, to investigate the role of internal oscillators on native prosody perception, and active speech prediction.","totalCost":"191149,44","ecMaxContribution":"191149,44","call":"H2020-MSCA-IF-2019","fundingScheme":"MSCA-IF-EF-ST","coordinator":"UNIVERSITE DE GENEVE","coordinatorCountry":"CH","participants":"","participantCountries":"","subjects":""} +{"rcn":"229235","id":"886828","acronym":"ASAP","status":"SIGNED","programme":"H2020-EU.1.3.2.","topics":"MSCA-IF-2019","frameworkProgramme":"H2020","title":"Advanced Solutions for Asphalt Pavements","startDate":"2021-09-01","endDate":"2023-08-31","projectUrl":"","objective":"The Advanced Solutions for Asphalt Pavements (ASAP) project involves the development of a unique road paving technology which will use a bio-bitumen rejuvenator to rejuvenate aged asphalt bitumen. This technology will help to extend the lifespan of asphalt pavements (roads) and will reduce the environmental and economic impact of roads and road maintenance processes. Recycling and self-healing processes will replace fossil fuel dependent technology. Self-healing will involve rejuvenating aged asphalt bitumen using a bio-rejuvenator developed using microalgae oils (rejuvenating bio-oil). Microalgae has been selected because of its fast growth, versatility and ability to survive within hostile environments, such as wastewater. \n\nASAP will utilise microalgae, cultivated within the wastewater treatment process, as a source of the rejuvenating bio-oil. The solvent (Soxhlet) processes will be used to extract the oil from the microalgae. To ensure the efficiency of the oil extraction process, an ultrasonication process will be used to pre-treat the microalgae. The suitability of rejuvenating bio-oil as a replacement for the bitumen rejuvenator (fossil fuel based) will be ascertained via a series of standard bituminous and accelerated tests. A rejuvenator-binder diffusion numerical model will be developed, based on the Delft Lattice concrete diffusion model, to determine the conditions required for rejuvenation to occur and to ascertain the healing rate of the asphalt binder. These parameters will facilitate the selection and optimisation of the asphalt self-healing systems (specifically the amount of bio-oil rejuvenator and time required) to achieve full rejuvenation. \n\nThis novel approach will benchmark the effectiveness of this intervention against existing asphalt design and maintenance processes and assess feasibility. The ASAP project presents an opportunity to revolutionise road design and maintenance processes and reduce its environmental and financial costs.","totalCost":"187572,48","ecMaxContribution":"187572,48","call":"H2020-MSCA-IF-2019","fundingScheme":"MSCA-IF-EF-ST","coordinator":"NEDERLANDSE ORGANISATIE VOOR TOEGEPAST NATUURWETENSCHAPPELIJK ONDERZOEK TNO","coordinatorCountry":"NL","participants":"","participantCountries":"","subjects":""} +{"rcn":"229236","id":"886776","acronym":"BIOBESTicide","status":"SIGNED","programme":"H2020-EU.2.1.4.;H2020-EU.3.2.6.","topics":"BBI-2019-SO3-D4","frameworkProgramme":"H2020","title":"BIO-Based pESTicides production for sustainable agriculture management plan","startDate":"2020-05-01","endDate":"2023-04-30","projectUrl":"","objective":"The BIOBESTicide project will validate and demonstrate the production of an effective and cost-efficient biopesticide. The demonstration will be based on an innovative bio-based value chain starting from the valorisation of sustainable biomasses, i.e. beet pulp and sugar molasses and will exploit the properties of the oomycete Pythium oligandrum strain I-5180 to increase natural plant defenses, to produce an highly effective and eco-friendly biopesticide solution for vine plants protection. \nBIOVITIS, the project coordinator, has developed, at laboratory level (TRL4), an effective method to biocontrol one of the major causes of worldwide vineyards destruction, the Grapevine Trunk Diseases (GTDs). The protection system is based on the oomycete Pythium oligandrum strain I-5180 that, at applied at optimal time and concentration, colonises the root of vines and stimulates the natural plant defences against GTDs, providing a protection that ranges between 40% and 60%. \nBIOBESTicide project will respond to the increasing demands for innovative solutions for crop protection agents, transferring the technology to a DEMO Plant able to produce more than 10 T of a high-quality oomycete-based biopesticide product per year (TRL7). \nThe BIOBESTicide project will validate the efficiency of the formulated product on vineyards of different geographical areas.\nTo assure the safety of products under both health and environmental points of view, a full and complete approval dossier for Pythium oligandrum strain I-5180 will be submitted in all the European countries. \nA Life Cycle Sustainability Assessment (LCSA) will be conducted to assess the environmental, economic and social impacts of the developed products.\nThe adoption of the effective and cost-efficient biopesticide will have significant impacts with a potential ROI of 30 % in just 5 years and a total EBITDA of more than € 6,400,000.","totalCost":"4402772,5","ecMaxContribution":"3069653","call":"H2020-BBI-JTI-2019","fundingScheme":"BBI-IA-DEMO","coordinator":"BIOVITIS","coordinatorCountry":"FR","participants":"MERCIER FRERES SARL;FUNDACION TECNALIA RESEARCH & INNOVATION;LAMBERTI SPA;EURION CONSULTING;CIAOTECH Srl;STOWARZYSZENIE ZACHODNIOPOMORSKI KLASTER CHEMICZNY ZIELONA CHEMIA;NORDZUCKER AG;INSTITUT NATIONAL DE RECHERCHE POUR L'AGRICULTURE, L'ALIMENTATION ET L'ENVIRONNEMENT;INSTITUT FRANCAIS DE LA VIGNE ET DU VIN","participantCountries":"FR;ES;IT;PL;DE","subjects":""} +{"rcn":"229276","id":"895426","acronym":"DisMoBoH","status":"SIGNED","programme":"H2020-EU.1.3.2.","topics":"MSCA-IF-2019","frameworkProgramme":"H2020","title":"Dissecting the molecular building principles of locally formed transcriptional hubs","startDate":"2021-09-01","endDate":"2023-08-31","projectUrl":"","objective":"Numerous DNA variants have already been identified that modulate inter-individual molecular traits – most prominently gene expression. However, since finding mechanistic interpretations relating genotype to phenotype has proven challenging, the focus has shifted to higher-order regulatory features, i.e. chromatin accessibility, transcription factor (TF) binding and 3D chromatin interactions. This revealed at least two enhancer types: “lead” enhancers in which the presence of genetic variants modulates the activity of entire chromatin domains, and “dependent” ones in which variants induce subtle changes, affecting DNA accessibility, but not transcription. Although cell type-specific TFs are likely important, it remains unclear which sequence features are required to establish such enhancer hierarchies, and under which circumstances genetic variation results in altered enhancer-promoter contacts and differential gene expression. Here, we propose to investigate the molecular mechanisms that link DNA variation to TF binding, chromatin topology, and gene expression response. We will leverage data on enhancer hierarchy and sequence-specific TF binding to identify the sequence signatures that define “lead” enhancers. The results will guide the design of a synthetic locus that serves as an in vivo platform to systematically vary the building blocks of local transcriptional units: i) DNA sequence – including variations in TF binding site affinity and syntax, ii) molecular interactions between TFs, and iii) chromatin conformation. To validate our findings, we will perform optical reconstruction of chromatin architecture for a select number of DNA variants. By simultaneously perturbing co-dependent features, this proposal will provide novel mechanistic insights into the formation of local transcriptional hubs.","totalCost":"191149,44","ecMaxContribution":"191149,44","call":"H2020-MSCA-IF-2019","fundingScheme":"MSCA-IF-EF-RI","coordinator":"ECOLE POLYTECHNIQUE FEDERALE DE LAUSANNE","coordinatorCountry":"CH","participants":"","participantCountries":"","subjects":""} +{"rcn":"229288","id":"898218","acronym":"devUTRs","status":"SIGNED","programme":"H2020-EU.1.3.2.","topics":"MSCA-IF-2019","frameworkProgramme":"H2020","title":"Uncovering the roles of 5′UTRs in translational control during early zebrafish development","startDate":"2021-09-01","endDate":"2023-08-31","projectUrl":"","objective":"Following fertilisation, metazoan embryos are transcriptionally silent, and embryogenesis is controlled by maternally deposited factors. Developmental progression requires the synthesis of new mRNAs and proteins in a coordinated fashion. Many posttranscriptional mechanisms regulate the fate of maternal mRNAs, but it is less understood how translational control shapes early embryogenesis. In eukaryotes, translation starts at the mRNA 5′ end, consisting of the 5′ cap and 5′ untranslated region (UTR). Protein synthesis is primarily regulated at the translation initiation step by elements within the 5′UTR. However, the role of 5′UTRs in regulating the dynamics of mRNA translation during vertebrate embryogenesis remains unexplored. For example, all vertebrate ribosomal protein (RP) mRNAs harbor a conserved terminal oligopyrimidine tract (TOP) in their 5′UTR. RP levels must be tightly controlled to ensure proper organismal development, but if and how the TOP motif mediates RP mRNA translational regulation during embryogenesis is unclear. Overall, we lack a systematic understanding of the regulatory information contained in 5′UTRs. In this work, I aim to uncover the 5′UTR in vivo rules for mRNA translational regulation during zebrafish embryogenesis. I propose to apply imaging and biochemical approaches to characterise the role of the TOP motif in RP mRNA translational regulation during embryogenesis and identify the trans-acting factor(s) that bind(s) to it (Aim 1). To systematically assess the contribution of 5′UTRs to mRNA translational regulation during zebrafish embryogenesis, I will couple a massively parallel reporter assay of 5′UTRs to polysome profiling (Aim 2). By integrating the translational behaviour of 5′UTR reporters throughout embryogenesis with sequence-based regression models, I anticipate to uncover novel cis-regulatory elements in 5′UTRs with developmental roles.","totalCost":"191149,44","ecMaxContribution":"191149,44","call":"H2020-MSCA-IF-2019","fundingScheme":"MSCA-IF-EF-ST","coordinator":"UNIVERSITAT BASEL","coordinatorCountry":"CH","participants":"","participantCountries":"","subjects":""} +{"rcn":"229261","id":"893787","acronym":"HOLYHOST","status":"SIGNED","programme":"H2020-EU.1.3.2.","topics":"MSCA-IF-2019","frameworkProgramme":"H2020","title":"Welfare and Hosting buildings in the “Holy Land” between the 4th and the 7th c. AD","startDate":"2020-10-01","endDate":"2022-09-30","projectUrl":"","objective":"Between the 4th and the 7th century AD, many hospices dedicated to the poor, elderly, strangers and travelers were built in the countryside, along roads, around and inside cities. They were commissioned by the Church, rich pious men and women concerned by the redeem of their sins, as well as emperors who saw this as a guarantee of social stability. Welfare is thus an important phenomena of Late Antiquity, abundantly mentioned by ancient literary sources and inscriptions, particularly in the eastern part of the Empire. However, the buildings that provided shelter and care to the needy have not yet received sufficient attention from archaeologists. Except for buildings which were identified by their inventors as hostels dedicated to pilgrims, they are still invisible in the field. \nThe aim of the HOLYHOST research project is to bring this social history’s main topic on the field of archaeology. It will address the welfare issue through the archaeological and architectural survey and study of Ancient welfare and hosting establishments’ remains, in the Holy Land (Palestine and Jordan) and around. This work will contribute to a better understanding of the practices linked to hospitality, welfare, accommodation and care in Antiquity. Moreover, such establishments served as models for medieval and modern Islamic, Jewish and Christian waqf institutions (religious endowment), and welfare continues to be highly relevant nowadays, through issues still at the heart of contemporary challenges debated in Europe: poverty, social exclusion, migrant crisis, principle of reception and hospitality. This interdisciplinary and diachronic research project will thus offer many new research perspectives, in terms of history of architecture, evolution of care practices, social and political regulations.","totalCost":"196707,84","ecMaxContribution":"196707,84","call":"H2020-MSCA-IF-2019","fundingScheme":"MSCA-IF-EF-ST","coordinator":"UNIVERSITE PARIS I PANTHEON-SORBONNE","coordinatorCountry":"FR","participants":"","participantCountries":"","subjects":""} +{"rcn":"229282","id":"896189","acronym":"MICADO","status":"SIGNED","programme":"H2020-EU.1.3.2.","topics":"MSCA-IF-2019","frameworkProgramme":"H2020","title":"Microbial contribution to continental wetland carbon budget","startDate":"2021-01-04","endDate":"2023-01-03","projectUrl":"","objective":"Continental wetlands are major carbon dioxide sinks but the second largest source of methane. Monitoring of wetland methane emissions revealed large inter-site variability that is hard to explain in the framework of current biogeochemical theories. Methane production in wetlands is an anaerobic microbial driven process involving a complex set of microbial metabolisms depending on the availability of (i) energy (via the presence of specific redox couples), (ii) organic substrates and (iii) specific microbial communities. To understand the complexity of microbial drivers on wetland methane emissions and quantify their contribution, the MICADO project will set up a multidisciplinary approach linking isotope organic geochemistry and environmental microbiology to assess microbial functioning in situ. As an organic geochemist I have developed an innovative approach to trace in situ microbial activity via compound specific carbon isotope analysis of microbe macromolecules and organic metabolites. The host institution is a leader in France in environmental microbiology and biogeochemistry developing high-throughput metagenomics and microbial rate assessments, for which I will be trained during the MICADO project. These techniques are highly complementary and combined they will provide a comprehensive knowledge on microbial metabolisms involved in organic matter degradation encompassing their complexity and interactions. This will revisit the relationships between organic substrate availability and microbial communities and will contribute at estimating the impact of microbial activity on wetland methane emissions. This project will give me the opportunity to acquire fundamental knowledge and to develop original lines of research that will consolidate my position as an independent scientist in biogeochemistry.","totalCost":"196707,84","ecMaxContribution":"196707,84","call":"H2020-MSCA-IF-2019","fundingScheme":"MSCA-IF-EF-ST","coordinator":"CENTRE NATIONAL DE LA RECHERCHE SCIENTIFIQUE CNRS","coordinatorCountry":"FR","participants":"","participantCountries":"","subjects":""} +{"rcn":"229249","id":"891624","acronym":"CuTAN","status":"SIGNED","programme":"H2020-EU.1.3.2.","topics":"MSCA-IF-2019","frameworkProgramme":"H2020","title":"Copper-Catalyzed Multicomponent Reactions in Tandem Processes for Target Molecule Synthesis","startDate":"2021-02-01","endDate":"2023-01-31","projectUrl":"","objective":"The invention of processes that can form several bonds, stereocentres and rings in a single process is key to a sustainable future in synthetic chemistry. Multicomponent reactions and tandem procedures are two strategies that enable the rapid build-up of molecular complexity from simple reagents. By combining these two strategies into a single procedure, the diversity, complexity and value of products can be further enhanced along with the efficiency and economy of their construction. In this project, Dr Satpathi will develop novel copper-catalyzed multicomponent couplings of unsaturated hydrocarbons (e.g. allenes, enynes) with imines and boron reagents. These procedures will provide high-value amine products with universally high regio-, diastero- and enantiocontrol. The products will bear a variety of synthetic handles, for example, amino, alkynyl/alkenyl, and boryl groups, thus the products are primed for subsequent transformation. Dr Satpathi will exploit this functionality in tandem intramolecular couplings (e.g. intramolecular Suzuki/Buchwald-Hartwig reactions) to provide core cyclic structures of drug molecules and natural products. Thus, through a tandem procedure of; 1) copper-catalyzed borofunctionalization, and; 2) subsequent transition-metal catalyzed cyclization, he will gain efficient access to highly sought-after complex molecules. Overall, the process will provide high-value, chiral, cyclic motifs from abundant, achiral, linear substrates. Finally, Dr Satpathi has identified the phthalide-isoquinoline family of alkaloids as target molecules to display the power of his tandem methodology. Dr Satpathi has devised a novel route, which begins with our tandem multifunctionalization/cyclization reaction, to provide a range of these important alkaloids. The chosen alkaloids are of particular interest as they display a range of bioactivities – for example as natural products, receptor antagonists and on-market drugs.","totalCost":"212933,76","ecMaxContribution":"212933,76","call":"H2020-MSCA-IF-2019","fundingScheme":"MSCA-IF-EF-ST","coordinator":"THE UNIVERSITY OF MANCHESTER","coordinatorCountry":"UK","participants":"","participantCountries":"","subjects":""} +{"rcn":"229239","id":"887259","acronym":"ALEHOOP","status":"SIGNED","programme":"H2020-EU.2.1.4.;H2020-EU.3.2.6.","topics":"BBI-2019-SO3-D3","frameworkProgramme":"H2020","title":"Biorefineries for the valorisation of macroalgal residual biomass and legume processing by-products to obtain new protein value chains for high-value food and feed applications","startDate":"2020-06-01","endDate":"2024-05-31","projectUrl":"","objective":"ALEHOOP provides the demonstration at pilot scale of both sustainable macroalgae and legume-based biorefineries for the recovery of low-cost dietary proteins from alga-based and plant residual biomass and their validation to meet market requirements of consumers and industry in the food and feed sectors. In these sectors, consumers are demanding affordable functional natural proteins from alternative sources and industry is demanding low-cost bio-based protein formulations with better performance and higher sustainability. \nCurrent protein demand for the 7.3 billion inhabitants of the world is approximately 202 Mt. Due to the rise in meat consumption more proteins are therefore required for animal feeding. To satisfy the current protein demand, Europe imports over 30 Mt of soy from the Americas each year mainly for animal feeding, entailing 95% dependency of EU on imported soy. Current sources of proteins are becoming unsustainable from an economic and environmental perspective for Europe resulting in concerns for sustainability and food security and leading to search for new alternative proteins. \nALEHOOP addresses the obtaining of proteins from green macroalgal blooms, brown seaweed by-products from algae processors and legume processing by-products (peas, lupines, beans and lentils) as alternative protein sources for animal feeding (case of green seaweed) and food applications (case of brown seaweed and legume by-products), since they are low cost and under-exploited biomass that do not compete with traditional food crops for space and resources. This will reduce EU´s dependency on protein imports and contribute to our raw material security. The new proteins will be validated in foods for elderly, sporty and overweight people, vegetarians and healthy consumers as well as for animal feed creating cross-sectorial interconnection between these value chains and supporting the projected business plan.","totalCost":"6718370","ecMaxContribution":"5140274,41","call":"H2020-BBI-JTI-2019","fundingScheme":"BBI-IA-DEMO","coordinator":"CONTACTICA S.L.","coordinatorCountry":"ES","participants":"CENTIV GMBH;ALGINOR ASA;FUNDACION TECNALIA RESEARCH & INNOVATION;INDUKERN,S.A.;ASOCIACION NACIONAL DE FABRICANTES DE CONSERVAS DE PESCADOS Y MARISCOS-CENTRO TECNICO NACIONAL DE CONSERVACION DE PRODUCTOS DE LA PESCA;BIOZOON GMBH;EIGEN VERMOGEN VAN HET INSTITUUT VOOR LANDBOUW- EN VISSERIJONDERZOEK;BIOSURYA SL;VYZKUMNY USTAV VETERINARNIHO LEKARSTVI;NUTRITION SCIENCES;TECHNOLOGICAL UNIVERSITY DUBLIN;GARLAN, S.COOP.;ISANATUR SPAIN SL;UNIVERSIDAD DE VIGO;UNIVERSIDAD DE CADIZ","participantCountries":"DE;NO;ES;BE;CZ;IE","subjects":""} +{"rcn":"229258","id":"892834","acronym":"DENVPOC","status":"SIGNED","programme":"H2020-EU.1.3.2.","topics":"MSCA-IF-2019","frameworkProgramme":"H2020","title":"qPCR Microfluidics point-of-care platform for dengue diagnosis","startDate":"2020-05-18","endDate":"2022-05-17","projectUrl":"","objective":"As a result of Global climate change and fast urbanization, global outbreaks of Dengue (DENV)/ Zika(ZIKV)/Chikungunya(CHIKV) virus have the potential to occur. The most common pathway of these infections in humans is through the female Aedes mosquito vector. DENV is an exanthematous febrile disease with varied clinical manifestations and progressions . Due to similarities in symptoms between DENV and ZIKV and CHIKV, it is difficult to make a differential diagnosis, impeding appropriate, timely medical intervention. Furthermore, cross-reactivity with ZIKV, which was recently related to microcephaly, is a serious issue. In 2016, in Brazil alone, there were 4180 microcephaly cases reported instead of 163 cases, more in line with yearly expected projections , , Thus, the sooner an accurate diagnostic which differentiates DENV from the other manifestations is critical; most especially at the early stages of the infection, to have a reliable diagnosis in pregnant women. In 2016, the OMS emergency committee declared that the outbreaks and the potentially resultant neurological disorders in Brazil were an important international state of emergency in public health, as a result of the associated secondary effects; these diseases became a Global concern. This project allows developing a highly and fast Multiplex qPCR POC platform by using FASTGENE technology with a minimal amount of patient serotype. It would reduce the time of analysis (30 to 90’ for a standard) and costs. Additionally, the sample preprocessing and thermalization will shorten real-time PCR amplification time and will be integrated within the microfluidic systems. This platform can result in a commercialized product whereupon a main market target would be pregnant women and people living or traveling through/from outbreak risk areas.","totalCost":"196707,84","ecMaxContribution":"196707,84","call":"H2020-MSCA-IF-2019","fundingScheme":"MSCA-IF-EF-SE","coordinator":"BFORCURE","coordinatorCountry":"FR","participants":"","participantCountries":"","subjects":""} +{"rcn":"229280","id":"895716","acronym":"DoMiCoP","status":"SIGNED","programme":"H2020-EU.1.3.2.","topics":"MSCA-IF-2019","frameworkProgramme":"H2020","title":"The Diffusion of Migration Control Practice. Actors, Processes and Effects.","startDate":"2021-03-01","endDate":"2023-02-28","projectUrl":"","objective":"DoMiCoP develops new understandings and perspectives to study migration control in practice in the European Union by asking one main question: how and why do communities of practice develop and diffuse the knowledge required to put migration control into action? Unlike the nexus between expert knowledge, epistemic communities and policy formulation, the nexus between everyday knowledge, communities of practice and policy implementation has not yet received systematic scholarly attention. My project bridges that gap by focusing on intermediate arenas in which communities of practice take shape most notably the meetings and trainings that gather state and non-state actors involved in putting asylum, detention and removal into practice. By building on field-based methodologies (interviews and participant observations), DoMiCoP sheds ethnographic light on the role that ‘learning from abroad’ plays in the implementation of migration control in the EU. My project’s aim is threefold: 1) Identifying arenas at intermediate levels in which communities of practice take shape; 2) Analysing the communities of practice by focusing on the configurations of actors and organizations involved, the motivations underlying their involvement, the process of knowledge development in interaction, the conflicts and negotiations; 3) Revealing the role of non-state organizations (private for profit and not-for-profit). From a theoretical point of view, this project goes beyond the classical view of the implementation as a test to assess the effectiveness of policy transfers towards an analysis of policy transfer at that level of policy-making. From an empirical point of view, the project expands knowledge about less-studied venues of policy-making and provides original thick descriptions. From a methodological point of view, the project engages with qualitative methods for the study of policy diffusion and aims at responding to their main challenges through participant observation.","totalCost":"163673,28","ecMaxContribution":"163673,28","call":"H2020-MSCA-IF-2019","fundingScheme":"MSCA-IF-EF-ST","coordinator":"EUROPEAN UNIVERSITY INSTITUTE","coordinatorCountry":"IT","participants":"","participantCountries":"","subjects":""} +{"rcn":"229297","id":"954782","acronym":"MiniLLock","status":"SIGNED","programme":"H2020-EU.3.;H2020-EU.2.3.;H2020-EU.2.1.","topics":"EIC-SMEInst-2018-2020","frameworkProgramme":"H2020","title":"Mini Launch Lock devices for small satellites","startDate":"2020-05-01","endDate":"2022-04-30","projectUrl":"","objective":"Space industry is experiencing the most important paradigm shift in its history with the rise of small satellites and megaconstellations.\nSatellite miniaturization requires to reduce significantly production and orbit launching costs. To address the\nnew challenge of this manufacturing process and switch from craftsmanship to industrialization, space industry is turning\ntowards other domains looking for new solutions, disruptive technologies, and manufacturing process.\nMini Launch Lock devices for small satellites (MiniLLock) proposes innovative actuators on the cutting edge of customer\ndemand. They offer plug and play solutions that can directly be integrated into industry for satellites robotized production.\nMiniLLock is smaller, lighter, safer, with a longer lifetime and generates significantly less shocks and vibrations than\nstandard actuators such as electromagnet and pyrotechnics. MiniLLock offers performances which have never been reached\nwith any other materials.\nNimesis is the only company that can provide such cost-effective actuators suitable to small satellite with high performances\nand reliability, enabling features previously impossible.\nMiniLLock will accelerate and leverage the commercialization of Nimesis technology and ensure Europe worldwide\nleadership\nand independence in the new space emergent environment.\nNimesis ambitions to become the global leader of this domain with a turnover of € 26 million and a market share of 28% in\n2027.","totalCost":"2413543,75","ecMaxContribution":"1689480,63","call":"H2020-EIC-SMEInst-2018-2020-3","fundingScheme":"SME-2b","coordinator":"NIMESIS TECHNOLOGY SARL","coordinatorCountry":"FR","participants":"","participantCountries":"","subjects":""} +{"rcn":"229299","id":"101003374","acronym":"NOPHOS","status":"SIGNED","programme":"H2020-EU.4.","topics":"WF-02-2019","frameworkProgramme":"H2020","title":"Unravelling protein phosphorylation mechanisms and phosphoproteome changes under nitrosative stress conditions in E.coli","startDate":"2020-07-01","endDate":"2022-06-30","projectUrl":"","objective":"Currently, we face a global antibiotic resistance crisis aggravated by the slow development of more effective and anti-resistance promoting therapeutical solutions. Protein phosphorylation (PP) has recently emerged as one of the major post-translational modification in bacteria, involved in the regulation of multiple physiological processes. In this MSCA individual fellowship application we aim to bridge the current gap in the field for prokaryotes by unravelling the unknown regulatory role of PP on proteins involved in nitrosative stress (NS) detoxification in the model bacterium E.coli. We propose to examine for the first time both global protein modifications (e.g. phosphoproteomics) under nitrogen species stress, as well as characterize PP in individual proteins involved in NS response. We will construct a network model that reflect the phosphoproteomic changes upon NS in E.coli, that may pave the way for the design of new bacterial targets. Understanding how bacteria respond to the chemical weapons of the human innate system is fundamental to develop efficient therapies. We will pioneer research on the mechanism and the regulation of nitric oxide detoxification proteins already identified as phosphorylated, by analyzing how this modification influences their stability and activity in vitro and in vivo. This project opens up new research paths on bacterial detoxification systems and signalling in general, addressing for the first time the role of PP in these processes. The proposal brings together transversal and scientific skills that will enable the researcher to lead the development of this emerging field and position herself as an expert in the area, and aims at establishing the importance of PP in NO microbial response, a novelty in this field.","totalCost":"147815,04","ecMaxContribution":"147815,04","call":"H2020-WF-02-2019","fundingScheme":"MSCA-IF-EF-ST","coordinator":"UNIVERSIDADE NOVA DE LISBOA","coordinatorCountry":"PT","participants":"","participantCountries":"","subjects":""} \ No newline at end of file diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/project/whole_programme.json.gz b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/project/whole_programme.json.gz new file mode 100644 index 0000000000..85440ceed5 Binary files /dev/null and b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/project/whole_programme.json.gz differ diff --git a/dhp-workflows/dhp-blacklist/pom.xml b/dhp-workflows/dhp-blacklist/pom.xml index a3cc15b744..9c25f7b293 100644 --- a/dhp-workflows/dhp-blacklist/pom.xml +++ b/dhp-workflows/dhp-blacklist/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.2-SNAPSHOT + 1.2.4-SNAPSHOT 4.0.0 diff --git a/dhp-workflows/dhp-broker-events/pom.xml b/dhp-workflows/dhp-broker-events/pom.xml index 0e84d99a45..1a219c5c91 100644 --- a/dhp-workflows/dhp-broker-events/pom.xml +++ b/dhp-workflows/dhp-broker-events/pom.xml @@ -1,66 +1,68 @@ - - - dhp-workflows - eu.dnetlib.dhp - 1.2.2-SNAPSHOT - - 4.0.0 + + + dhp-workflows + eu.dnetlib.dhp + 1.2.4-SNAPSHOT + + 4.0.0 - dhp-broker-events + dhp-broker-events - + - - commons-io - commons-io - + + commons-io + commons-io + - - org.apache.spark - spark-core_2.11 - - - org.apache.spark - spark-sql_2.11 - - - org.apache.spark - spark-hive_2.11 - test - + + org.apache.spark + spark-core_2.11 + + + org.apache.spark + spark-sql_2.11 + + + org.elasticsearch + elasticsearch-hadoop + - - eu.dnetlib.dhp - dhp-common - ${project.version} - - - eu.dnetlib.dhp - dhp-schemas - ${project.version} - - - com.jayway.jsonpath - json-path - - - dom4j - dom4j - - - jaxen - jaxen - + + eu.dnetlib.dhp + dhp-common + ${project.version} + + + eu.dnetlib.dhp + dhp-schemas + ${project.version} + + + eu.dnetlib + dnet-pace-core + - - eu.dnetlib - dnet-openaire-broker-common - [2.0.0,3.0.0) - + + dom4j + dom4j + + + jaxen + jaxen + - + + eu.dnetlib.dhp + dnet-openaire-broker-common + [3.0.0-SNAPSHOT,) + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-broker-events/report.xml b/dhp-workflows/dhp-broker-events/report.xml new file mode 100644 index 0000000000..6e706f7230 --- /dev/null +++ b/dhp-workflows/dhp-broker-events/report.xml @@ -0,0 +1,37 @@ + + + + Feature Extraction + + + TCPFLOW + 1.5.0 + + 4.2.1 (4.2.1 Compatible Apple LLVM 11.0.0 (clang-1100.0.33.8)) + -D_THREAD_SAFE -pthread -I/usr/local/include -I/usr/local/include -DUTC_OFFSET=+0000 + -g -D_THREAD_SAFE -pthread -g -O3 -MD -Wpointer-arith -Wmissing-declarations -Wmissing-prototypes -Wshadow -Wwrite-strings -Wcast-align -Waggregate-return -Wbad-function-cast -Wcast-qual -Wundef -Wredundant-decls -Wdisabled-optimization -Wfloat-equal -Wmultichar -Wc++-compat -Wmissing-noreturn -Wall -Wstrict-prototypes -MD -D_FORTIFY_SOURCE=2 -Wpointer-arith -Wmissing-declarations -Wmissing-prototypes -Wshadow -Wwrite-strings -Wcast-align -Waggregate-return -Wbad-function-cast -Wcast-qual -Wundef -Wredundant-decls -Wdisabled-optimization -Wfloat-equal -Wmultichar -Wc++-compat -Wmissing-noreturn -Wall -Wstrict-prototypes + -g -D_THREAD_SAFE -pthread -g -O3 -Wall -MD -D_FORTIFY_SOURCE=2 -Wpointer-arith -Wshadow -Wwrite-strings -Wcast-align -Wredundant-decls -Wdisabled-optimization -Wfloat-equal -Wmultichar -Wmissing-noreturn -Woverloaded-virtual -Wsign-promo -funit-at-a-time -Weffc++ -std=c++11 -Wall -MD -D_FORTIFY_SOURCE=2 -Wpointer-arith -Wshadow -Wwrite-strings -Wcast-align -Wredundant-decls -Wdisabled-optimization -Wfloat-equal -Wmultichar -Wmissing-noreturn -Woverloaded-virtual -Wsign-promo -funit-at-a-time -Weffc++ + -L/usr/local/lib -L/usr/local/lib + -lpython2.7 -lpython2.7 -lpcap -lbz2 -lexpat -lsqlite3 -lcrypto -lssl -lcrypto -ldl -lz + 2019-10-11T01:16:58 + + + + + Darwin + 19.5.0 + Darwin Kernel Version 19.5.0: Tue May 26 20:41:44 PDT 2020; root:xnu-6153.121.2~2/RELEASE_X86_64 + Micheles-MBP.local + x86_64 + tcpflow + 501 + michele + 2020-06-15T14:55:03Z + + + + + 0 diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/model/Event.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/model/Event.java index 0512a38134..18950d98e5 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/model/Event.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/model/Event.java @@ -1,9 +1,14 @@ package eu.dnetlib.dhp.broker.model; -import java.util.Map; +import java.io.Serializable; -public class Event { +public class Event implements Serializable { + + /** + * + */ + private static final long serialVersionUID = -5936790326505781395L; private String eventId; @@ -19,7 +24,7 @@ public class Event { private boolean instantMessage; - private Map map; + private MappedFields map; public Event() { } @@ -27,7 +32,7 @@ public class Event { public Event(final String producerId, final String eventId, final String topic, final String payload, final Long creationDate, final Long expiryDate, final boolean instantMessage, - final Map map) { + final MappedFields map) { this.producerId = producerId; this.eventId = eventId; this.topic = topic; @@ -94,11 +99,11 @@ public class Event { this.instantMessage = instantMessage; } - public Map getMap() { + public MappedFields getMap() { return this.map; } - public void setMap(final Map map) { + public void setMap(final MappedFields map) { this.map = map; } } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/model/EventFactory.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/model/EventFactory.java index 9e5d986448..49e7506981 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/model/EventFactory.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/model/EventFactory.java @@ -3,21 +3,15 @@ package eu.dnetlib.dhp.broker.model; import java.text.ParseException; import java.util.Date; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.stream.Collectors; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.time.DateUtils; -import eu.dnetlib.broker.objects.OpenAireEventPayload; +import eu.dnetlib.broker.objects.OaBrokerMainEntity; import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; -import eu.dnetlib.dhp.schema.oaf.Author; -import eu.dnetlib.dhp.schema.oaf.KeyValue; -import eu.dnetlib.dhp.schema.oaf.Result; -import eu.dnetlib.dhp.schema.oaf.StructuredProperty; public class EventFactory { @@ -35,85 +29,54 @@ public class EventFactory { final Event res = new Event(); - final Map map = createMapFromResult(updateInfo); - - final String payload = createPayload(updateInfo); + final MappedFields map = createMapFromResult(updateInfo); final String eventId = calculateEventId( - updateInfo.getTopicPath(), updateInfo.getTarget().getOriginalId().get(0), - updateInfo.getHighlightValueAsString()); + updateInfo.getTopicPath(), updateInfo.getTarget().getOpenaireId(), updateInfo.getHighlightValueAsString()); res.setEventId(eventId); res.setProducerId(PRODUCER_ID); - res.setPayload(payload); + res.setPayload(updateInfo.asBrokerPayload().toJSON()); res.setMap(map); res.setTopic(updateInfo.getTopicPath()); res.setCreationDate(now); res.setExpiryDate(calculateExpiryDate(now)); res.setInstantMessage(false); + return res; } - private static String createPayload(final UpdateInfo updateInfo) { - final OpenAireEventPayload payload = new OpenAireEventPayload(); - // TODO + private static MappedFields createMapFromResult(final UpdateInfo updateInfo) { + final MappedFields map = new MappedFields(); - updateInfo.compileHighlight(payload); + final OaBrokerMainEntity source = updateInfo.getSource(); + final OaBrokerMainEntity target = updateInfo.getTarget(); - return payload.toJSON(); - } + map.setTargetDatasourceId(target.getCollectedFromId()); + map.setTargetDatasourceName(target.getCollectedFromName()); + map.setTargetDatasourceType(target.getCollectedFromType()); - private static Map createMapFromResult(final UpdateInfo updateInfo) { - final Map map = new HashMap<>(); + map.setTargetResultId(target.getOpenaireId()); - final Result source = updateInfo.getSource(); - final Result target = updateInfo.getTarget(); - - final List collectedFrom = target.getCollectedfrom(); - if (collectedFrom.size() == 1) { - map.put("target_datasource_id", collectedFrom.get(0).getKey()); - map.put("target_datasource_name", collectedFrom.get(0).getValue()); - } - - final List ids = target.getOriginalId(); - if (ids.size() > 0) { - map.put("target_publication_id", ids.get(0)); - } - - final List titles = target.getTitle(); + final List titles = target.getTitles(); if (titles.size() > 0) { - map.put("target_publication_title", titles.get(0)); + map.setTargetResultTitle(titles.get(0)); } - final long date = parseDateTolong(target.getDateofacceptance().getValue()); + final long date = parseDateTolong(target.getPublicationdate()); if (date > 0) { - map.put("target_dateofacceptance", date); + map.setTargetDateofacceptance(date); } - final List subjects = target.getSubject(); - if (subjects.size() > 0) { - map - .put( - "target_publication_subject_list", - subjects.stream().map(StructuredProperty::getValue).collect(Collectors.toList())); - } - - final List authors = target.getAuthor(); - if (authors.size() > 0) { - map - .put( - "target_publication_author_list", - authors.stream().map(Author::getFullname).collect(Collectors.toList())); - } + map.setTargetSubjects(target.getSubjects().stream().map(s -> s.getValue()).collect(Collectors.toList())); + map.setTargetAuthors(target.getCreators().stream().map(a -> a.getFullname()).collect(Collectors.toList())); // PROVENANCE INFO - map.put("trust", updateInfo.getTrust()); - final List sourceCollectedFrom = source.getCollectedfrom(); - if (sourceCollectedFrom.size() == 1) { - map.put("provenance_datasource_id", sourceCollectedFrom.get(0).getKey()); - map.put("provenance_datasource_name", sourceCollectedFrom.get(0).getValue()); - } - map.put("provenance_publication_id_list", source.getOriginalId()); + map.setTrust(updateInfo.getTrust()); + map.setProvenanceDatasourceId(source.getCollectedFromId()); + map.setProvenanceDatasourceName(source.getCollectedFromName()); + map.setProvenanceDatasourceType(source.getCollectedFromType()); + map.setProvenanceResultId(source.getOpenaireId()); return map; } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/model/MappedFields.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/model/MappedFields.java new file mode 100644 index 0000000000..2c1be3ba42 --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/model/MappedFields.java @@ -0,0 +1,136 @@ + +package eu.dnetlib.dhp.broker.model; + +import java.io.Serializable; +import java.util.List; + +public class MappedFields implements Serializable { + + /** + * + */ + private static final long serialVersionUID = -7999704113195802008L; + + private String targetDatasourceId; + private String targetDatasourceName; + private String targetDatasourceType; + private String targetResultId; + private String targetResultTitle; + private long targetDateofacceptance; + private List targetSubjects; + private List targetAuthors; + private float trust; + private String provenanceDatasourceId; + private String provenanceDatasourceName; + private String provenanceDatasourceType; + private String provenanceResultId; + + public String getTargetDatasourceId() { + return targetDatasourceId; + } + + public void setTargetDatasourceId(final String targetDatasourceId) { + this.targetDatasourceId = targetDatasourceId; + } + + public String getTargetDatasourceName() { + return targetDatasourceName; + } + + public void setTargetDatasourceName(final String targetDatasourceName) { + this.targetDatasourceName = targetDatasourceName; + } + + public String getTargetDatasourceType() { + return targetDatasourceType; + } + + public void setTargetDatasourceType(final String targetDatasourceType) { + this.targetDatasourceType = targetDatasourceType; + } + + public String getTargetResultId() { + return targetResultId; + } + + public void setTargetResultId(final String targetResultId) { + this.targetResultId = targetResultId; + } + + public String getTargetResultTitle() { + return targetResultTitle; + } + + public void setTargetResultTitle(final String targetResultTitle) { + this.targetResultTitle = targetResultTitle; + } + + public long getTargetDateofacceptance() { + return targetDateofacceptance; + } + + public void setTargetDateofacceptance(final long targetDateofacceptance) { + this.targetDateofacceptance = targetDateofacceptance; + } + + public List getTargetSubjects() { + return targetSubjects; + } + + public void setTargetSubjects(final List targetSubjects) { + this.targetSubjects = targetSubjects; + } + + public List getTargetAuthors() { + return targetAuthors; + } + + public void setTargetAuthors(final List targetAuthors) { + this.targetAuthors = targetAuthors; + } + + public float getTrust() { + return trust; + } + + public void setTrust(final float trust) { + this.trust = trust; + } + + public String getProvenanceDatasourceId() { + return provenanceDatasourceId; + } + + public void setProvenanceDatasourceId(final String provenanceDatasourceId) { + this.provenanceDatasourceId = provenanceDatasourceId; + } + + public String getProvenanceDatasourceName() { + return provenanceDatasourceName; + } + + public void setProvenanceDatasourceName(final String provenanceDatasourceName) { + this.provenanceDatasourceName = provenanceDatasourceName; + } + + public String getProvenanceDatasourceType() { + return provenanceDatasourceType; + } + + public void setProvenanceDatasourceType(final String provenanceDatasourceType) { + this.provenanceDatasourceType = provenanceDatasourceType; + } + + public String getProvenanceResultId() { + return provenanceResultId; + } + + public void setProvenanceResultId(final String provenanceResultId) { + this.provenanceResultId = provenanceResultId; + } + + public static long getSerialversionuid() { + return serialVersionUID; + } + +} diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/CheckDuplictedIdsJob.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/CheckDuplictedIdsJob.java new file mode 100644 index 0000000000..5ca865e8fc --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/CheckDuplictedIdsJob.java @@ -0,0 +1,112 @@ + +package eu.dnetlib.dhp.broker.oa; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.sql.Encoder; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.SaveMode; +import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.TypedColumn; +import org.apache.spark.sql.expressions.Aggregator; +import org.apache.spark.util.LongAccumulator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.broker.model.Event; +import eu.dnetlib.dhp.broker.oa.util.ClusterUtils; +import scala.Tuple2; + +public class CheckDuplictedIdsJob { + + private static final Logger log = LoggerFactory.getLogger(CheckDuplictedIdsJob.class); + + public static void main(final String[] args) throws Exception { + + final ArgumentApplicationParser parser = new ArgumentApplicationParser( + IOUtils + .toString( + CheckDuplictedIdsJob.class + .getResourceAsStream("/eu/dnetlib/dhp/broker/oa/common_params.json"))); + parser.parseArgument(args); + + final SparkConf conf = new SparkConf(); + + final String eventsPath = parser.get("workingPath") + "/events"; + log.info("eventsPath: {}", eventsPath); + + final String countPath = parser.get("workingPath") + "/counts"; + log.info("countPath: {}", countPath); + + final SparkSession spark = SparkSession.builder().config(conf).getOrCreate(); + + final LongAccumulator total = spark.sparkContext().longAccumulator("invaild_event_id"); + + final TypedColumn, Tuple2> agg = new CountAggregator().toColumn(); + + ClusterUtils + .readPath(spark, eventsPath, Event.class) + .map(e -> new Tuple2<>(e.getEventId(), 1l), Encoders.tuple(Encoders.STRING(), Encoders.LONG())) + .groupByKey(t -> t._1, Encoders.STRING()) + .agg(agg) + .map(t -> t._2, Encoders.tuple(Encoders.STRING(), Encoders.LONG())) + .filter(t -> t._2 > 1) + .map(o -> ClusterUtils.incrementAccumulator(o, total), Encoders.tuple(Encoders.STRING(), Encoders.LONG())) + .write() + .mode(SaveMode.Overwrite) + .json(countPath); + ; + + } + + private static String eventAsJsonString(final Event f) throws JsonProcessingException { + return new ObjectMapper().writeValueAsString(f); + } + +} + +class CountAggregator extends Aggregator, Tuple2, Tuple2> { + + /** + * + */ + private static final long serialVersionUID = 1395935985734672538L; + + @Override + public Encoder> bufferEncoder() { + return Encoders.tuple(Encoders.STRING(), Encoders.LONG()); + } + + @Override + public Tuple2 finish(final Tuple2 arg0) { + return arg0; + } + + @Override + public Tuple2 merge(final Tuple2 arg0, final Tuple2 arg1) { + final String s = StringUtils.defaultIfBlank(arg0._1, arg1._1); + return new Tuple2<>(s, arg0._2 + arg1._2); + } + + @Override + public Encoder> outputEncoder() { + return Encoders.tuple(Encoders.STRING(), Encoders.LONG()); + } + + @Override + public Tuple2 reduce(final Tuple2 arg0, final Tuple2 arg1) { + final String s = StringUtils.defaultIfBlank(arg0._1, arg1._1); + return new Tuple2<>(s, arg0._2 + arg1._2); + } + + @Override + public Tuple2 zero() { + return new Tuple2<>(null, 0l); + } + +} diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/GenerateEventsApplication.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/GenerateEventsApplication.java deleted file mode 100644 index d5e577972d..0000000000 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/GenerateEventsApplication.java +++ /dev/null @@ -1,280 +0,0 @@ - -package eu.dnetlib.dhp.broker.oa; - -import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Collectors; - -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.tuple.Pair; -import org.apache.hadoop.io.compress.GzipCodec; -import org.apache.spark.SparkConf; -import org.apache.spark.api.java.JavaRDD; -import org.apache.spark.api.java.JavaSparkContext; -import org.apache.spark.api.java.function.MapFunction; -import org.apache.spark.sql.Column; -import org.apache.spark.sql.Dataset; -import org.apache.spark.sql.Encoders; -import org.apache.spark.sql.Row; -import org.apache.spark.sql.SparkSession; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.fasterxml.jackson.databind.ObjectMapper; - -import eu.dnetlib.dhp.application.ArgumentApplicationParser; -import eu.dnetlib.dhp.broker.model.Event; -import eu.dnetlib.dhp.broker.model.EventFactory; -import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; -import eu.dnetlib.dhp.broker.oa.matchers.relatedDatasets.EnrichMissingDatasetIsReferencedBy; -import eu.dnetlib.dhp.broker.oa.matchers.relatedDatasets.EnrichMissingDatasetIsRelatedTo; -import eu.dnetlib.dhp.broker.oa.matchers.relatedDatasets.EnrichMissingDatasetIsSupplementedBy; -import eu.dnetlib.dhp.broker.oa.matchers.relatedDatasets.EnrichMissingDatasetIsSupplementedTo; -import eu.dnetlib.dhp.broker.oa.matchers.relatedDatasets.EnrichMissingDatasetReferences; -import eu.dnetlib.dhp.broker.oa.matchers.relatedProjects.EnrichMissingProject; -import eu.dnetlib.dhp.broker.oa.matchers.relatedProjects.EnrichMoreProject; -import eu.dnetlib.dhp.broker.oa.matchers.relatedPublications.EnrichMissingPublicationIsReferencedBy; -import eu.dnetlib.dhp.broker.oa.matchers.relatedPublications.EnrichMissingPublicationIsRelatedTo; -import eu.dnetlib.dhp.broker.oa.matchers.relatedPublications.EnrichMissingPublicationIsSupplementedBy; -import eu.dnetlib.dhp.broker.oa.matchers.relatedPublications.EnrichMissingPublicationIsSupplementedTo; -import eu.dnetlib.dhp.broker.oa.matchers.relatedPublications.EnrichMissingPublicationReferences; -import eu.dnetlib.dhp.broker.oa.matchers.simple.EnrichMissingAbstract; -import eu.dnetlib.dhp.broker.oa.matchers.simple.EnrichMissingAuthorOrcid; -import eu.dnetlib.dhp.broker.oa.matchers.simple.EnrichMissingOpenAccess; -import eu.dnetlib.dhp.broker.oa.matchers.simple.EnrichMissingPid; -import eu.dnetlib.dhp.broker.oa.matchers.simple.EnrichMissingPublicationDate; -import eu.dnetlib.dhp.broker.oa.matchers.simple.EnrichMissingSoftware; -import eu.dnetlib.dhp.broker.oa.matchers.simple.EnrichMissingSubject; -import eu.dnetlib.dhp.broker.oa.matchers.simple.EnrichMoreOpenAccess; -import eu.dnetlib.dhp.broker.oa.matchers.simple.EnrichMorePid; -import eu.dnetlib.dhp.broker.oa.matchers.simple.EnrichMoreSoftware; -import eu.dnetlib.dhp.broker.oa.matchers.simple.EnrichMoreSubject; -import eu.dnetlib.dhp.broker.oa.util.BrokerConstants; -import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; -import eu.dnetlib.dhp.common.HdfsSupport; -import eu.dnetlib.dhp.schema.oaf.OtherResearchProduct; -import eu.dnetlib.dhp.schema.oaf.Project; -import eu.dnetlib.dhp.schema.oaf.Publication; -import eu.dnetlib.dhp.schema.oaf.Relation; -import eu.dnetlib.dhp.schema.oaf.Result; -import eu.dnetlib.dhp.schema.oaf.Software; - -public class GenerateEventsApplication { - - private static final Logger log = LoggerFactory.getLogger(GenerateEventsApplication.class); - - // Simple Matchers - private static final UpdateMatcher enrichMissingAbstract = new EnrichMissingAbstract(); - private static final UpdateMatcher enrichMissingAuthorOrcid = new EnrichMissingAuthorOrcid(); - private static final UpdateMatcher enrichMissingOpenAccess = new EnrichMissingOpenAccess(); - private static final UpdateMatcher enrichMissingPid = new EnrichMissingPid(); - private static final UpdateMatcher enrichMissingPublicationDate = new EnrichMissingPublicationDate(); - private static final UpdateMatcher enrichMissingSubject = new EnrichMissingSubject(); - private static final UpdateMatcher enrichMoreOpenAccess = new EnrichMoreOpenAccess(); - private static final UpdateMatcher enrichMorePid = new EnrichMorePid(); - private static final UpdateMatcher enrichMoreSubject = new EnrichMoreSubject(); - - // Advanced matchers - private static final UpdateMatcher>, ?> enrichMissingProject = new EnrichMissingProject(); - private static final UpdateMatcher>, ?> enrichMoreProject = new EnrichMoreProject(); - - private static final UpdateMatcher>, ?> enrichMissingSoftware = new EnrichMissingSoftware(); - private static final UpdateMatcher>, ?> enrichMoreSoftware = new EnrichMoreSoftware(); - - private static final UpdateMatcher>, ?> enrichMisissingPublicationIsRelatedTo = new EnrichMissingPublicationIsRelatedTo(); - private static final UpdateMatcher>, ?> enrichMissingPublicationIsReferencedBy = new EnrichMissingPublicationIsReferencedBy(); - private static final UpdateMatcher>, ?> enrichMissingPublicationReferences = new EnrichMissingPublicationReferences(); - private static final UpdateMatcher>, ?> enrichMissingPublicationIsSupplementedTo = new EnrichMissingPublicationIsSupplementedTo(); - private static final UpdateMatcher>, ?> enrichMissingPublicationIsSupplementedBy = new EnrichMissingPublicationIsSupplementedBy(); - - private static final UpdateMatcher>, ?> enrichMisissingDatasetIsRelatedTo = new EnrichMissingDatasetIsRelatedTo(); - private static final UpdateMatcher>, ?> enrichMissingDatasetIsReferencedBy = new EnrichMissingDatasetIsReferencedBy(); - private static final UpdateMatcher>, ?> enrichMissingDatasetReferences = new EnrichMissingDatasetReferences(); - private static final UpdateMatcher>, ?> enrichMissingDatasetIsSupplementedTo = new EnrichMissingDatasetIsSupplementedTo(); - private static final UpdateMatcher>, ?> enrichMissingDatasetIsSupplementedBy = new EnrichMissingDatasetIsSupplementedBy(); - - public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - - public static void main(final String[] args) throws Exception { - final ArgumentApplicationParser parser = new ArgumentApplicationParser( - IOUtils - .toString( - GenerateEventsApplication.class - .getResourceAsStream("/eu/dnetlib/dhp/oa/graph/merge_claims_parameters.json"))); - parser.parseArgument(args); - - final Boolean isSparkSessionManaged = Optional - .ofNullable(parser.get("isSparkSessionManaged")) - .map(Boolean::valueOf) - .orElse(Boolean.TRUE); - log.info("isSparkSessionManaged: {}", isSparkSessionManaged); - - final String graphPath = parser.get("graphPath"); - log.info("graphPath: {}", graphPath); - - final String eventsPath = parser.get("eventsPath"); - log.info("eventsPath: {}", eventsPath); - - final SparkConf conf = new SparkConf(); - - runWithSparkSession(conf, isSparkSessionManaged, spark -> { - final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); - removeOutputDir(spark, eventsPath); - - final JavaRDD eventsRdd = sc.emptyRDD(); - - eventsRdd.union(generateSimpleEvents(spark, graphPath, Publication.class)); - eventsRdd.union(generateSimpleEvents(spark, graphPath, eu.dnetlib.dhp.schema.oaf.Dataset.class)); - eventsRdd.union(generateSimpleEvents(spark, graphPath, Software.class)); - eventsRdd.union(generateSimpleEvents(spark, graphPath, OtherResearchProduct.class)); - - eventsRdd.saveAsTextFile(eventsPath, GzipCodec.class); - }); - - } - - private static void removeOutputDir(final SparkSession spark, final String path) { - HdfsSupport.remove(path, spark.sparkContext().hadoopConfiguration()); - } - - private static JavaRDD generateSimpleEvents(final SparkSession spark, - final String graphPath, - final Class resultClazz) { - - final Dataset results = readPath( - spark, graphPath + "/" + resultClazz.getSimpleName().toLowerCase(), resultClazz) - .filter(r -> r.getDataInfo().getDeletedbyinference()); - - final Dataset rels = readPath(spark, graphPath + "/relation", Relation.class) - .filter(r -> r.getRelClass().equals(BrokerConstants.IS_MERGED_IN_CLASS)); - - final Column c = null; // TODO - - final Dataset aa = results - .joinWith(rels, results.col("id").equalTo(rels.col("source")), "inner") - .groupBy(rels.col("target")) - .agg(c) - .filter(x -> x.size() > 1) - // generateSimpleEvents(...) - // flatMap() - // toRdd() - ; - - return null; - - } - - private List generateSimpleEvents(final Collection children) { - final List> list = new ArrayList<>(); - - for (final Result target : children) { - list.addAll(enrichMissingAbstract.searchUpdatesForRecord(target, children)); - list.addAll(enrichMissingAuthorOrcid.searchUpdatesForRecord(target, children)); - list.addAll(enrichMissingOpenAccess.searchUpdatesForRecord(target, children)); - list.addAll(enrichMissingPid.searchUpdatesForRecord(target, children)); - list.addAll(enrichMissingPublicationDate.searchUpdatesForRecord(target, children)); - list.addAll(enrichMissingSubject.searchUpdatesForRecord(target, children)); - list.addAll(enrichMoreOpenAccess.searchUpdatesForRecord(target, children)); - list.addAll(enrichMorePid.searchUpdatesForRecord(target, children)); - list.addAll(enrichMoreSubject.searchUpdatesForRecord(target, children)); - } - - return list.stream().map(EventFactory::newBrokerEvent).collect(Collectors.toList()); - } - - private List generateProjectsEvents(final Collection>> childrenWithProjects) { - final List> list = new ArrayList<>(); - - for (final Pair> target : childrenWithProjects) { - list.addAll(enrichMissingProject.searchUpdatesForRecord(target, childrenWithProjects)); - list.addAll(enrichMoreProject.searchUpdatesForRecord(target, childrenWithProjects)); - } - - return list.stream().map(EventFactory::newBrokerEvent).collect(Collectors.toList()); - } - - private List generateSoftwareEvents(final Collection>> childrenWithSoftwares) { - final List> list = new ArrayList<>(); - - for (final Pair> target : childrenWithSoftwares) { - list.addAll(enrichMissingSoftware.searchUpdatesForRecord(target, childrenWithSoftwares)); - list.addAll(enrichMoreSoftware.searchUpdatesForRecord(target, childrenWithSoftwares)); - } - return list.stream().map(EventFactory::newBrokerEvent).collect(Collectors.toList()); - } - - private List generatePublicationRelatedEvents(final String relType, - final Collection>>> childrenWithRels) { - - final List> list = new ArrayList<>(); - - final List>> cleanedChildrens = childrenWithRels - .stream() - .filter(p -> p.getRight().containsKey(relType)) - .map(p -> Pair.of(p.getLeft(), p.getRight().get(relType))) - .filter(p -> p.getRight().size() > 0) - .collect(Collectors.toList()); - - for (final Pair> target : cleanedChildrens) { - if (relType.equals("isRelatedTo")) { - list.addAll(enrichMisissingPublicationIsRelatedTo.searchUpdatesForRecord(target, cleanedChildrens)); - } else if (relType.equals("references")) { - list.addAll(enrichMissingPublicationReferences.searchUpdatesForRecord(target, cleanedChildrens)); - } else if (relType.equals("isReferencedBy")) { - list.addAll(enrichMissingPublicationIsReferencedBy.searchUpdatesForRecord(target, cleanedChildrens)); - } else if (relType.equals("isSupplementedTo")) { - list.addAll(enrichMissingPublicationIsSupplementedTo.searchUpdatesForRecord(target, cleanedChildrens)); - } else if (relType.equals("isSupplementedBy")) { - list.addAll(enrichMissingPublicationIsSupplementedBy.searchUpdatesForRecord(target, cleanedChildrens)); - } - } - - return list.stream().map(EventFactory::newBrokerEvent).collect(Collectors.toList()); - - } - - private List generateDatasetRelatedEvents(final String relType, - final Collection>>> childrenWithRels) { - - final List> list = new ArrayList<>(); - - final List>> cleanedChildrens = childrenWithRels - .stream() - .filter(p -> p.getRight().containsKey(relType)) - .map(p -> Pair.of(p.getLeft(), p.getRight().get(relType))) - .filter(p -> p.getRight().size() > 0) - .collect(Collectors.toList()); - - for (final Pair> target : cleanedChildrens) { - if (relType.equals("isRelatedTo")) { - list.addAll(enrichMisissingDatasetIsRelatedTo.searchUpdatesForRecord(target, cleanedChildrens)); - } else if (relType.equals("references")) { - list.addAll(enrichMissingDatasetReferences.searchUpdatesForRecord(target, cleanedChildrens)); - } else if (relType.equals("isReferencedBy")) { - list.addAll(enrichMissingDatasetIsReferencedBy.searchUpdatesForRecord(target, cleanedChildrens)); - } else if (relType.equals("isSupplementedTo")) { - list.addAll(enrichMissingDatasetIsSupplementedTo.searchUpdatesForRecord(target, cleanedChildrens)); - } else if (relType.equals("isSupplementedBy")) { - list.addAll(enrichMissingDatasetIsSupplementedBy.searchUpdatesForRecord(target, cleanedChildrens)); - } - } - - return list.stream().map(EventFactory::newBrokerEvent).collect(Collectors.toList()); - - } - - public static Dataset readPath( - final SparkSession spark, - final String inputPath, - final Class clazz) { - return spark - .read() - .textFile(inputPath) - .map((MapFunction) value -> OBJECT_MAPPER.readValue(value, clazz), Encoders.bean(clazz)); - } -} diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/GenerateEventsJob.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/GenerateEventsJob.java new file mode 100644 index 0000000000..a2d92e149e --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/GenerateEventsJob.java @@ -0,0 +1,134 @@ + +package eu.dnetlib.dhp.broker.oa; + +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; + +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.SparkContext; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; +import org.apache.spark.util.LongAccumulator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.broker.model.Event; +import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; +import eu.dnetlib.dhp.broker.oa.util.ClusterUtils; +import eu.dnetlib.dhp.broker.oa.util.EventFinder; +import eu.dnetlib.dhp.broker.oa.util.EventGroup; +import eu.dnetlib.dhp.broker.oa.util.aggregators.simple.ResultGroup; +import eu.dnetlib.dhp.utils.ISLookupClientFactory; +import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; +import eu.dnetlib.pace.config.DedupConfig; + +public class GenerateEventsJob { + + private static final Logger log = LoggerFactory.getLogger(GenerateEventsJob.class); + + public static void main(final String[] args) throws Exception { + final ArgumentApplicationParser parser = new ArgumentApplicationParser( + IOUtils + .toString( + GenerateEventsJob.class + .getResourceAsStream("/eu/dnetlib/dhp/broker/oa/generate_events.json"))); + parser.parseArgument(args); + + final Boolean isSparkSessionManaged = Optional + .ofNullable(parser.get("isSparkSessionManaged")) + .map(Boolean::valueOf) + .orElse(Boolean.TRUE); + log.info("isSparkSessionManaged: {}", isSparkSessionManaged); + + final String workingPath = parser.get("workingPath"); + log.info("workingPath: {}", workingPath); + + final String isLookupUrl = parser.get("isLookupUrl"); + log.info("isLookupUrl: {}", isLookupUrl); + + final String dedupConfigProfileId = parser.get("dedupConfProfile"); + log.info("dedupConfigProfileId: {}", dedupConfigProfileId); + + final String eventsPath = workingPath + "/events"; + log.info("eventsPath: {}", eventsPath); + + final Set dsIdWhitelist = ClusterUtils.parseParamAsList(parser, "datasourceIdWhitelist"); + log.info("datasourceIdWhitelist: {}", StringUtils.join(dsIdWhitelist, ",")); + + final Set dsTypeWhitelist = ClusterUtils.parseParamAsList(parser, "datasourceTypeWhitelist"); + log.info("datasourceTypeWhitelist: {}", StringUtils.join(dsTypeWhitelist, ",")); + + final Set dsIdBlacklist = ClusterUtils.parseParamAsList(parser, "datasourceIdBlacklist"); + log.info("datasourceIdBlacklist: {}", StringUtils.join(dsIdBlacklist, ",")); + + final SparkConf conf = new SparkConf(); + + // TODO UNCOMMENT + // final DedupConfig dedupConfig = loadDedupConfig(isLookupUrl, dedupConfigProfileId); + final DedupConfig dedupConfig = null; + + runWithSparkSession(conf, isSparkSessionManaged, spark -> { + + ClusterUtils.removeDir(spark, eventsPath); + + final Map accumulators = prepareAccumulators(spark.sparkContext()); + + final LongAccumulator total = spark.sparkContext().longAccumulator("total_events"); + + final Dataset groups = ClusterUtils + .readPath(spark, workingPath + "/duplicates", ResultGroup.class); + + final Dataset dataset = groups + .map( + g -> EventFinder + .generateEvents(g, dsIdWhitelist, dsIdBlacklist, dsTypeWhitelist, dedupConfig, accumulators), + Encoders + .bean(EventGroup.class)) + .flatMap(g -> g.getData().iterator(), Encoders.bean(Event.class)); + + ClusterUtils.save(dataset, eventsPath, Event.class, total); + + }); + + } + + public static Map prepareAccumulators(final SparkContext sc) { + + return EventFinder + .getMatchers() + .stream() + .map(UpdateMatcher::accumulatorName) + .distinct() + .collect(Collectors.toMap(s -> s, s -> sc.longAccumulator(s))); + + } + + private static DedupConfig loadDedupConfig(final String isLookupUrl, final String profId) throws Exception { + + final ISLookUpService isLookUpService = ISLookupClientFactory.getLookUpService(isLookupUrl); + + final String conf = isLookUpService + .getResourceProfileByQuery( + String + .format( + "for $x in /RESOURCE_PROFILE[.//RESOURCE_IDENTIFIER/@value = '%s'] return $x//DEDUPLICATION/text()", + profId)); + + final DedupConfig dedupConfig = new ObjectMapper().readValue(conf, DedupConfig.class); + dedupConfig.getPace().initModel(); + dedupConfig.getPace().initTranslationMap(); + // dedupConfig.getWf().setConfigurationId("???"); + + return dedupConfig; + } + +} diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/IndexOnESJob.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/IndexOnESJob.java new file mode 100644 index 0000000000..806147bdd5 --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/IndexOnESJob.java @@ -0,0 +1,71 @@ + +package eu.dnetlib.dhp.broker.oa; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.io.IOUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.SparkSession; +import org.elasticsearch.spark.rdd.api.java.JavaEsSpark; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.broker.model.Event; +import eu.dnetlib.dhp.broker.oa.util.ClusterUtils; + +public class IndexOnESJob { + + private static final Logger log = LoggerFactory.getLogger(IndexOnESJob.class); + + public static void main(final String[] args) throws Exception { + + final ArgumentApplicationParser parser = new ArgumentApplicationParser( + IOUtils + .toString( + IndexOnESJob.class + .getResourceAsStream("/eu/dnetlib/dhp/broker/oa/index_es.json"))); + parser.parseArgument(args); + + final SparkConf conf = new SparkConf(); + + final String eventsPath = parser.get("workingPath") + "/events"; + log.info("eventsPath: {}", eventsPath); + + final String index = parser.get("index"); + log.info("index: {}", index); + + final String indexHost = parser.get("esHost"); + log.info("indexHost: {}", indexHost); + + final SparkSession spark = SparkSession.builder().config(conf).getOrCreate(); + + final JavaRDD inputRdd = ClusterUtils + .readPath(spark, eventsPath, Event.class) + // .limit(10000) // TODO REMOVE + .map(IndexOnESJob::eventAsJsonString, Encoders.STRING()) + .javaRDD(); + + final Map esCfg = new HashMap<>(); + // esCfg.put("es.nodes", "10.19.65.51, 10.19.65.52, 10.19.65.53, 10.19.65.54"); + esCfg.put("es.nodes", indexHost); + esCfg.put("es.mapping.id", "eventId"); // THE PRIMARY KEY + esCfg.put("es.batch.write.retry.count", "8"); + esCfg.put("es.batch.write.retry.wait", "60s"); + esCfg.put("es.batch.size.entries", "200"); + esCfg.put("es.nodes.wan.only", "true"); + + JavaEsSpark.saveJsonToEs(inputRdd, index, esCfg); + } + + private static String eventAsJsonString(final Event f) throws JsonProcessingException { + return new ObjectMapper().writeValueAsString(f); + } + +} diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/JoinStep0Job.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/JoinStep0Job.java new file mode 100644 index 0000000000..eb1825fa5a --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/JoinStep0Job.java @@ -0,0 +1,80 @@ + +package eu.dnetlib.dhp.broker.oa; + +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; + +import java.util.Optional; + +import org.apache.commons.io.IOUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.TypedColumn; +import org.apache.spark.util.LongAccumulator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import eu.dnetlib.broker.objects.OaBrokerMainEntity; +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.broker.oa.util.ClusterUtils; +import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.AddDatasourceTypeAggregator; +import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.SimpleDatasourceInfo; +import scala.Tuple2; + +public class JoinStep0Job { + + private static final Logger log = LoggerFactory.getLogger(JoinStep0Job.class); + + public static void main(final String[] args) throws Exception { + final ArgumentApplicationParser parser = new ArgumentApplicationParser( + IOUtils + .toString( + JoinStep0Job.class + .getResourceAsStream("/eu/dnetlib/dhp/broker/oa/common_params.json"))); + parser.parseArgument(args); + + final Boolean isSparkSessionManaged = Optional + .ofNullable(parser.get("isSparkSessionManaged")) + .map(Boolean::valueOf) + .orElse(Boolean.TRUE); + log.info("isSparkSessionManaged: {}", isSparkSessionManaged); + + final String graphPath = parser.get("graphPath"); + log.info("graphPath: {}", graphPath); + + final String workingPath = parser.get("workingPath"); + log.info("workingPath: {}", workingPath); + + final String outputPath = workingPath + "/joinedEntities_step0"; + log.info("outputPath: {}", outputPath); + + final SparkConf conf = new SparkConf(); + + runWithSparkSession(conf, isSparkSessionManaged, spark -> { + + ClusterUtils.removeDir(spark, outputPath); + + final LongAccumulator total = spark.sparkContext().longAccumulator("total_entities"); + + final Dataset sources = ClusterUtils + .readPath(spark, workingPath + "/simpleEntities", OaBrokerMainEntity.class); + + final Dataset datasources = ClusterUtils + .readPath(spark, workingPath + "/datasources", SimpleDatasourceInfo.class); + + final TypedColumn, OaBrokerMainEntity> aggr = new AddDatasourceTypeAggregator() + .toColumn(); + + final Dataset dataset = sources + .joinWith(datasources, sources.col("collectedFromId").equalTo(datasources.col("id")), "inner") + .groupByKey(t -> t._1.getOpenaireId(), Encoders.STRING()) + .agg(aggr) + .map(t -> t._2, Encoders.bean(OaBrokerMainEntity.class)); + + ClusterUtils.save(dataset, outputPath, OaBrokerMainEntity.class, total); + + }); + + } + +} diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/JoinStep1Job.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/JoinStep1Job.java new file mode 100644 index 0000000000..8e502f7366 --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/JoinStep1Job.java @@ -0,0 +1,80 @@ + +package eu.dnetlib.dhp.broker.oa; + +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; + +import java.util.Optional; + +import org.apache.commons.io.IOUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.function.MapFunction; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.TypedColumn; +import org.apache.spark.util.LongAccumulator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import eu.dnetlib.broker.objects.OaBrokerMainEntity; +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.broker.oa.util.ClusterUtils; +import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.RelatedProject; +import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.RelatedProjectAggregator; +import scala.Tuple2; + +public class JoinStep1Job { + + private static final Logger log = LoggerFactory.getLogger(JoinStep1Job.class); + + public static void main(final String[] args) throws Exception { + final ArgumentApplicationParser parser = new ArgumentApplicationParser( + IOUtils + .toString( + JoinStep1Job.class + .getResourceAsStream("/eu/dnetlib/dhp/broker/oa/common_params.json"))); + parser.parseArgument(args); + + final Boolean isSparkSessionManaged = Optional + .ofNullable(parser.get("isSparkSessionManaged")) + .map(Boolean::valueOf) + .orElse(Boolean.TRUE); + log.info("isSparkSessionManaged: {}", isSparkSessionManaged); + + final String workingPath = parser.get("workingPath"); + log.info("workingPath: {}", workingPath); + + final String joinedEntitiesPath = workingPath + "/joinedEntities_step1"; + log.info("joinedEntitiesPath: {}", joinedEntitiesPath); + + final SparkConf conf = new SparkConf(); + + runWithSparkSession(conf, isSparkSessionManaged, spark -> { + + ClusterUtils.removeDir(spark, joinedEntitiesPath); + + final LongAccumulator total = spark.sparkContext().longAccumulator("total_entities"); + + final Dataset sources = ClusterUtils + .readPath(spark, workingPath + "/joinedEntities_step0", OaBrokerMainEntity.class); + + final Dataset typedRels = ClusterUtils + .readPath(spark, workingPath + "/relatedProjects", RelatedProject.class); + + final TypedColumn, OaBrokerMainEntity> aggr = new RelatedProjectAggregator() + .toColumn(); + + final Dataset dataset = sources + .joinWith(typedRels, sources.col("openaireId").equalTo(typedRels.col("source")), "left_outer") + .groupByKey( + (MapFunction, String>) t -> t._1.getOpenaireId(), + Encoders.STRING()) + .agg(aggr) + .map(t -> t._2, Encoders.bean(OaBrokerMainEntity.class)); + + ClusterUtils.save(dataset, joinedEntitiesPath, OaBrokerMainEntity.class, total); + + }); + + } + +} diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/JoinStep2Job.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/JoinStep2Job.java new file mode 100644 index 0000000000..cdcf0add4a --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/JoinStep2Job.java @@ -0,0 +1,80 @@ + +package eu.dnetlib.dhp.broker.oa; + +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; + +import java.util.Optional; + +import org.apache.commons.io.IOUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.function.MapFunction; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.TypedColumn; +import org.apache.spark.util.LongAccumulator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import eu.dnetlib.broker.objects.OaBrokerMainEntity; +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.broker.oa.util.ClusterUtils; +import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.RelatedSoftware; +import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.RelatedSoftwareAggregator; +import scala.Tuple2; + +public class JoinStep2Job { + + private static final Logger log = LoggerFactory.getLogger(JoinStep2Job.class); + + public static void main(final String[] args) throws Exception { + final ArgumentApplicationParser parser = new ArgumentApplicationParser( + IOUtils + .toString( + JoinStep2Job.class + .getResourceAsStream("/eu/dnetlib/dhp/broker/oa/common_params.json"))); + parser.parseArgument(args); + + final Boolean isSparkSessionManaged = Optional + .ofNullable(parser.get("isSparkSessionManaged")) + .map(Boolean::valueOf) + .orElse(Boolean.TRUE); + log.info("isSparkSessionManaged: {}", isSparkSessionManaged); + + final String workingPath = parser.get("workingPath"); + log.info("workingPath: {}", workingPath); + + final String joinedEntitiesPath = workingPath + "/joinedEntities_step2"; + log.info("joinedEntitiesPath: {}", joinedEntitiesPath); + + final SparkConf conf = new SparkConf(); + + runWithSparkSession(conf, isSparkSessionManaged, spark -> { + + ClusterUtils.removeDir(spark, joinedEntitiesPath); + + final LongAccumulator total = spark.sparkContext().longAccumulator("total_entities"); + + final Dataset sources = ClusterUtils + .readPath(spark, workingPath + "/joinedEntities_step1", OaBrokerMainEntity.class); + + final Dataset typedRels = ClusterUtils + .readPath(spark, workingPath + "/relatedSoftwares", RelatedSoftware.class); + + final TypedColumn, OaBrokerMainEntity> aggr = new RelatedSoftwareAggregator() + .toColumn(); + + final Dataset dataset = sources + .joinWith(typedRels, sources.col("openaireId").equalTo(typedRels.col("source")), "left_outer") + .groupByKey( + (MapFunction, String>) t -> t._1.getOpenaireId(), + Encoders.STRING()) + .agg(aggr) + .map(t -> t._2, Encoders.bean(OaBrokerMainEntity.class)); + + ClusterUtils.save(dataset, joinedEntitiesPath, OaBrokerMainEntity.class, total); + + }); + + } + +} diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/JoinStep3Job.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/JoinStep3Job.java new file mode 100644 index 0000000000..4d06f6f133 --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/JoinStep3Job.java @@ -0,0 +1,80 @@ + +package eu.dnetlib.dhp.broker.oa; + +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; + +import java.util.Optional; + +import org.apache.commons.io.IOUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.function.MapFunction; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.TypedColumn; +import org.apache.spark.util.LongAccumulator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import eu.dnetlib.broker.objects.OaBrokerMainEntity; +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.broker.oa.util.ClusterUtils; +import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.RelatedDataset; +import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.RelatedDatasetAggregator; +import scala.Tuple2; + +public class JoinStep3Job { + + private static final Logger log = LoggerFactory.getLogger(JoinStep3Job.class); + + public static void main(final String[] args) throws Exception { + final ArgumentApplicationParser parser = new ArgumentApplicationParser( + IOUtils + .toString( + JoinStep3Job.class + .getResourceAsStream("/eu/dnetlib/dhp/broker/oa/common_params.json"))); + parser.parseArgument(args); + + final Boolean isSparkSessionManaged = Optional + .ofNullable(parser.get("isSparkSessionManaged")) + .map(Boolean::valueOf) + .orElse(Boolean.TRUE); + log.info("isSparkSessionManaged: {}", isSparkSessionManaged); + + final String workingPath = parser.get("workingPath"); + log.info("workingPath: {}", workingPath); + + final String joinedEntitiesPath = workingPath + "/joinedEntities_step3"; + log.info("joinedEntitiesPath: {}", joinedEntitiesPath); + + final SparkConf conf = new SparkConf(); + + runWithSparkSession(conf, isSparkSessionManaged, spark -> { + + ClusterUtils.removeDir(spark, joinedEntitiesPath); + + final LongAccumulator total = spark.sparkContext().longAccumulator("total_entities"); + + final Dataset sources = ClusterUtils + .readPath(spark, workingPath + "/joinedEntities_step2", OaBrokerMainEntity.class); + + final Dataset typedRels = ClusterUtils + .readPath(spark, workingPath + "/relatedDatasets", RelatedDataset.class); + + final TypedColumn, OaBrokerMainEntity> aggr = new RelatedDatasetAggregator() + .toColumn(); + + final Dataset dataset = sources + .joinWith(typedRels, sources.col("openaireId").equalTo(typedRels.col("source")), "left_outer") + .groupByKey( + (MapFunction, String>) t -> t._1.getOpenaireId(), + Encoders.STRING()) + .agg(aggr) + .map(t -> t._2, Encoders.bean(OaBrokerMainEntity.class)); + + ClusterUtils.save(dataset, joinedEntitiesPath, OaBrokerMainEntity.class, total); + + }); + + } + +} diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/JoinStep4Job.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/JoinStep4Job.java new file mode 100644 index 0000000000..b53d7e39b9 --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/JoinStep4Job.java @@ -0,0 +1,80 @@ + +package eu.dnetlib.dhp.broker.oa; + +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; + +import java.util.Optional; + +import org.apache.commons.io.IOUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.function.MapFunction; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.TypedColumn; +import org.apache.spark.util.LongAccumulator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import eu.dnetlib.broker.objects.OaBrokerMainEntity; +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.broker.oa.util.ClusterUtils; +import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.RelatedPublication; +import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.RelatedPublicationAggregator; +import scala.Tuple2; + +public class JoinStep4Job { + + private static final Logger log = LoggerFactory.getLogger(JoinStep4Job.class); + + public static void main(final String[] args) throws Exception { + final ArgumentApplicationParser parser = new ArgumentApplicationParser( + IOUtils + .toString( + JoinStep4Job.class + .getResourceAsStream("/eu/dnetlib/dhp/broker/oa/common_params.json"))); + parser.parseArgument(args); + + final Boolean isSparkSessionManaged = Optional + .ofNullable(parser.get("isSparkSessionManaged")) + .map(Boolean::valueOf) + .orElse(Boolean.TRUE); + log.info("isSparkSessionManaged: {}", isSparkSessionManaged); + + final String workingPath = parser.get("workingPath"); + log.info("workingPath: {}", workingPath); + + final String joinedEntitiesPath = workingPath + "/joinedEntities_step4"; + log.info("joinedEntitiesPath: {}", joinedEntitiesPath); + + final SparkConf conf = new SparkConf(); + + runWithSparkSession(conf, isSparkSessionManaged, spark -> { + + ClusterUtils.removeDir(spark, joinedEntitiesPath); + + final LongAccumulator total = spark.sparkContext().longAccumulator("total_entities"); + + final Dataset sources = ClusterUtils + .readPath(spark, workingPath + "/joinedEntities_step3", OaBrokerMainEntity.class); + + final Dataset typedRels = ClusterUtils + .readPath(spark, workingPath + "/relatedPublications", RelatedPublication.class); + + final TypedColumn, OaBrokerMainEntity> aggr = new RelatedPublicationAggregator() + .toColumn(); + + final Dataset dataset = sources + .joinWith(typedRels, sources.col("openaireId").equalTo(typedRels.col("source")), "left_outer") + .groupByKey( + (MapFunction, String>) t -> t._1.getOpenaireId(), + Encoders.STRING()) + .agg(aggr) + .map(t -> t._2, Encoders.bean(OaBrokerMainEntity.class)); + + ClusterUtils.save(dataset, joinedEntitiesPath, OaBrokerMainEntity.class, total); + + }); + + } + +} diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/PrepareGroupsJob.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/PrepareGroupsJob.java new file mode 100644 index 0000000000..eb9add00de --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/PrepareGroupsJob.java @@ -0,0 +1,86 @@ + +package eu.dnetlib.dhp.broker.oa; + +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; + +import java.util.Optional; + +import org.apache.commons.io.IOUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.function.MapFunction; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.TypedColumn; +import org.apache.spark.util.LongAccumulator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import eu.dnetlib.broker.objects.OaBrokerMainEntity; +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.broker.oa.util.BrokerConstants; +import eu.dnetlib.dhp.broker.oa.util.ClusterUtils; +import eu.dnetlib.dhp.broker.oa.util.aggregators.simple.ResultAggregator; +import eu.dnetlib.dhp.broker.oa.util.aggregators.simple.ResultGroup; +import eu.dnetlib.dhp.schema.oaf.Relation; +import scala.Tuple2; + +public class PrepareGroupsJob { + + private static final Logger log = LoggerFactory.getLogger(PrepareGroupsJob.class); + + public static void main(final String[] args) throws Exception { + final ArgumentApplicationParser parser = new ArgumentApplicationParser( + IOUtils + .toString( + PrepareGroupsJob.class + .getResourceAsStream("/eu/dnetlib/dhp/broker/oa/common_params.json"))); + parser.parseArgument(args); + + final Boolean isSparkSessionManaged = Optional + .ofNullable(parser.get("isSparkSessionManaged")) + .map(Boolean::valueOf) + .orElse(Boolean.TRUE); + log.info("isSparkSessionManaged: {}", isSparkSessionManaged); + + final String graphPath = parser.get("graphPath"); + log.info("graphPath: {}", graphPath); + + final String workingPath = parser.get("workingPath"); + log.info("workingPath: {}", workingPath); + + final String groupsPath = workingPath + "/duplicates"; + log.info("groupsPath: {}", groupsPath); + + final SparkConf conf = new SparkConf(); + + runWithSparkSession(conf, isSparkSessionManaged, spark -> { + + ClusterUtils.removeDir(spark, groupsPath); + + final LongAccumulator total = spark.sparkContext().longAccumulator("total_groups"); + + final Dataset results = ClusterUtils + .readPath(spark, workingPath + "/joinedEntities_step4", OaBrokerMainEntity.class); + + final Dataset mergedRels = ClusterUtils + .readPath(spark, graphPath + "/relation", Relation.class) + .filter(r -> r.getRelClass().equals(BrokerConstants.IS_MERGED_IN_CLASS)); + + final TypedColumn, ResultGroup> aggr = new ResultAggregator() + .toColumn(); + + final Dataset dataset = results + .joinWith(mergedRels, results.col("openaireId").equalTo(mergedRels.col("source")), "inner") + .groupByKey( + (MapFunction, String>) t -> t._2.getTarget(), + Encoders.STRING()) + .agg(aggr) + .map(t -> t._2, Encoders.bean(ResultGroup.class)) + .filter(rg -> rg.getData().size() > 1); + + ClusterUtils.save(dataset, groupsPath, ResultGroup.class, total); + + }); + } + +} diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/PrepareRelatedDatasetsJob.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/PrepareRelatedDatasetsJob.java new file mode 100644 index 0000000000..0cfc1adcbb --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/PrepareRelatedDatasetsJob.java @@ -0,0 +1,86 @@ + +package eu.dnetlib.dhp.broker.oa; + +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; + +import java.util.Optional; + +import org.apache.commons.io.IOUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; +import org.apache.spark.util.LongAccumulator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import eu.dnetlib.broker.objects.OaBrokerRelatedDataset; +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.broker.oa.util.ClusterUtils; +import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; +import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.RelatedDataset; +import eu.dnetlib.dhp.schema.common.ModelConstants; +import eu.dnetlib.dhp.schema.oaf.Relation; + +public class PrepareRelatedDatasetsJob { + + private static final Logger log = LoggerFactory.getLogger(PrepareRelatedDatasetsJob.class); + + public static void main(final String[] args) throws Exception { + final ArgumentApplicationParser parser = new ArgumentApplicationParser( + IOUtils + .toString( + PrepareRelatedDatasetsJob.class + .getResourceAsStream("/eu/dnetlib/dhp/broker/oa/common_params.json"))); + parser.parseArgument(args); + + final Boolean isSparkSessionManaged = Optional + .ofNullable(parser.get("isSparkSessionManaged")) + .map(Boolean::valueOf) + .orElse(Boolean.TRUE); + log.info("isSparkSessionManaged: {}", isSparkSessionManaged); + + final String graphPath = parser.get("graphPath"); + log.info("graphPath: {}", graphPath); + + final String workingPath = parser.get("workingPath"); + log.info("workingPath: {}", workingPath); + + final String relsPath = workingPath + "/relatedDatasets"; + log.info("relsPath: {}", relsPath); + + final SparkConf conf = new SparkConf(); + + runWithSparkSession(conf, isSparkSessionManaged, spark -> { + + ClusterUtils.removeDir(spark, relsPath); + + final LongAccumulator total = spark.sparkContext().longAccumulator("total_rels"); + + final Dataset datasets = ClusterUtils + .readPath(spark, graphPath + "/dataset", eu.dnetlib.dhp.schema.oaf.Dataset.class) + .filter(d -> !ClusterUtils.isDedupRoot(d.getId())) + .map(ConversionUtils::oafDatasetToBrokerDataset, Encoders.bean(OaBrokerRelatedDataset.class)); + + final Dataset rels = ClusterUtils + .readPath(spark, graphPath + "/relation", Relation.class) + .filter(r -> r.getDataInfo().getDeletedbyinference()) + .filter(r -> r.getRelType().equals(ModelConstants.RESULT_RESULT)) + .filter(r -> ClusterUtils.isValidResultResultClass(r.getRelClass())) + .filter(r -> !ClusterUtils.isDedupRoot(r.getSource())) + .filter(r -> !ClusterUtils.isDedupRoot(r.getTarget())); + + final Dataset dataset = rels + .joinWith(datasets, datasets.col("openaireId").equalTo(rels.col("target")), "inner") + .map(t -> { + final RelatedDataset rel = new RelatedDataset(t._1.getSource(), t._2); + rel.getRelDataset().setRelType(t._1.getRelClass()); + return rel; + }, Encoders.bean(RelatedDataset.class)); + + ClusterUtils.save(dataset, relsPath, RelatedDataset.class, total); + + }); + + } + +} diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/PrepareRelatedDatasourcesJob.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/PrepareRelatedDatasourcesJob.java new file mode 100644 index 0000000000..30f5ddac33 --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/PrepareRelatedDatasourcesJob.java @@ -0,0 +1,68 @@ + +package eu.dnetlib.dhp.broker.oa; + +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; + +import java.util.Optional; + +import org.apache.commons.io.IOUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; +import org.apache.spark.util.LongAccumulator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.broker.oa.util.ClusterUtils; +import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.SimpleDatasourceInfo; +import eu.dnetlib.dhp.schema.oaf.Datasource; + +public class PrepareRelatedDatasourcesJob { + + private static final Logger log = LoggerFactory.getLogger(PrepareRelatedDatasourcesJob.class); + + public static void main(final String[] args) throws Exception { + final ArgumentApplicationParser parser = new ArgumentApplicationParser( + IOUtils + .toString( + PrepareRelatedDatasourcesJob.class + .getResourceAsStream("/eu/dnetlib/dhp/broker/oa/common_params.json"))); + parser.parseArgument(args); + + final Boolean isSparkSessionManaged = Optional + .ofNullable(parser.get("isSparkSessionManaged")) + .map(Boolean::valueOf) + .orElse(Boolean.TRUE); + log.info("isSparkSessionManaged: {}", isSparkSessionManaged); + + final String graphPath = parser.get("graphPath"); + log.info("graphPath: {}", graphPath); + + final String workingPath = parser.get("workingPath"); + log.info("workingPath: {}", workingPath); + + final String relsPath = workingPath + "/datasources"; + log.info("relsPath: {}", relsPath); + + final SparkConf conf = new SparkConf(); + + runWithSparkSession(conf, isSparkSessionManaged, spark -> { + + ClusterUtils.removeDir(spark, relsPath); + + final LongAccumulator total = spark.sparkContext().longAccumulator("total_datasources"); + + final Dataset dataset = ClusterUtils + .readPath(spark, graphPath + "/datasource", Datasource.class) + .map( + ds -> new SimpleDatasourceInfo(ds.getId(), ds.getDatasourcetype().getClassid()), + Encoders.bean(SimpleDatasourceInfo.class)); + + ClusterUtils.save(dataset, relsPath, SimpleDatasourceInfo.class, total); + + }); + + } + +} diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/PrepareRelatedProjectsJob.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/PrepareRelatedProjectsJob.java new file mode 100644 index 0000000000..e988366c8c --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/PrepareRelatedProjectsJob.java @@ -0,0 +1,84 @@ + +package eu.dnetlib.dhp.broker.oa; + +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; + +import java.util.Optional; + +import org.apache.commons.io.IOUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; +import org.apache.spark.util.LongAccumulator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import eu.dnetlib.broker.objects.OaBrokerProject; +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.broker.oa.util.BrokerConstants; +import eu.dnetlib.dhp.broker.oa.util.ClusterUtils; +import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; +import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.RelatedProject; +import eu.dnetlib.dhp.schema.common.ModelConstants; +import eu.dnetlib.dhp.schema.oaf.Project; +import eu.dnetlib.dhp.schema.oaf.Relation; + +public class PrepareRelatedProjectsJob { + + private static final Logger log = LoggerFactory.getLogger(PrepareRelatedProjectsJob.class); + + public static void main(final String[] args) throws Exception { + final ArgumentApplicationParser parser = new ArgumentApplicationParser( + IOUtils + .toString( + PrepareRelatedProjectsJob.class + .getResourceAsStream("/eu/dnetlib/dhp/broker/oa/common_params.json"))); + parser.parseArgument(args); + + final Boolean isSparkSessionManaged = Optional + .ofNullable(parser.get("isSparkSessionManaged")) + .map(Boolean::valueOf) + .orElse(Boolean.TRUE); + log.info("isSparkSessionManaged: {}", isSparkSessionManaged); + + final String graphPath = parser.get("graphPath"); + log.info("graphPath: {}", graphPath); + + final String workingPath = parser.get("workingPath"); + log.info("workingPath: {}", workingPath); + + final String relsPath = workingPath + "/relatedProjects"; + log.info("relsPath: {}", relsPath); + + final SparkConf conf = new SparkConf(); + + runWithSparkSession(conf, isSparkSessionManaged, spark -> { + + ClusterUtils.removeDir(spark, relsPath); + + final LongAccumulator total = spark.sparkContext().longAccumulator("total_rels"); + + final Dataset projects = ClusterUtils + .readPath(spark, graphPath + "/project", Project.class) + .filter(p -> !ClusterUtils.isDedupRoot(p.getId())) + .map(ConversionUtils::oafProjectToBrokerProject, Encoders.bean(OaBrokerProject.class)); + + final Dataset rels = ClusterUtils + .readPath(spark, graphPath + "/relation", Relation.class) + .filter(r -> r.getDataInfo().getDeletedbyinference()) + .filter(r -> r.getRelType().equals(ModelConstants.RESULT_PROJECT)) + .filter(r -> !r.getRelClass().equals(BrokerConstants.IS_MERGED_IN_CLASS)) + .filter(r -> !ClusterUtils.isDedupRoot(r.getSource())) + .filter(r -> !ClusterUtils.isDedupRoot(r.getTarget())); + + final Dataset dataset = rels + .joinWith(projects, projects.col("openaireId").equalTo(rels.col("target")), "inner") + .map(t -> new RelatedProject(t._1.getSource(), t._2), Encoders.bean(RelatedProject.class)); + + ClusterUtils.save(dataset, relsPath, RelatedProject.class, total); + + }); + + } + +} diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/PrepareRelatedPublicationsJob.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/PrepareRelatedPublicationsJob.java new file mode 100644 index 0000000000..724acc4dce --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/PrepareRelatedPublicationsJob.java @@ -0,0 +1,89 @@ + +package eu.dnetlib.dhp.broker.oa; + +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; + +import java.util.Optional; + +import org.apache.commons.io.IOUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; +import org.apache.spark.util.LongAccumulator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import eu.dnetlib.broker.objects.OaBrokerRelatedPublication; +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.broker.oa.util.ClusterUtils; +import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; +import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.RelatedPublication; +import eu.dnetlib.dhp.schema.common.ModelConstants; +import eu.dnetlib.dhp.schema.oaf.Publication; +import eu.dnetlib.dhp.schema.oaf.Relation; + +public class PrepareRelatedPublicationsJob { + + private static final Logger log = LoggerFactory.getLogger(PrepareRelatedPublicationsJob.class); + + public static void main(final String[] args) throws Exception { + final ArgumentApplicationParser parser = new ArgumentApplicationParser( + IOUtils + .toString( + PrepareRelatedPublicationsJob.class + .getResourceAsStream("/eu/dnetlib/dhp/broker/oa/common_params.json"))); + parser.parseArgument(args); + + final Boolean isSparkSessionManaged = Optional + .ofNullable(parser.get("isSparkSessionManaged")) + .map(Boolean::valueOf) + .orElse(Boolean.TRUE); + log.info("isSparkSessionManaged: {}", isSparkSessionManaged); + + final String graphPath = parser.get("graphPath"); + log.info("graphPath: {}", graphPath); + + final String workingPath = parser.get("workingPath"); + log.info("workingPath: {}", workingPath); + + final String relsPath = workingPath + "/relatedPublications"; + log.info("relsPath: {}", relsPath); + + final SparkConf conf = new SparkConf(); + + runWithSparkSession(conf, isSparkSessionManaged, spark -> { + + ClusterUtils.removeDir(spark, relsPath); + + final LongAccumulator total = spark.sparkContext().longAccumulator("total_rels"); + + final Dataset pubs = ClusterUtils + .readPath(spark, graphPath + "/publication", Publication.class) + .filter(p -> !ClusterUtils.isDedupRoot(p.getId())) + .map( + ConversionUtils::oafPublicationToBrokerPublication, + Encoders.bean(OaBrokerRelatedPublication.class)); + + final Dataset rels = ClusterUtils + .readPath(spark, graphPath + "/relation", Relation.class) + .filter(r -> r.getDataInfo().getDeletedbyinference()) + .filter(r -> r.getRelType().equals(ModelConstants.RESULT_RESULT)) + .filter(r -> ClusterUtils.isValidResultResultClass(r.getRelClass())) + .filter(r -> !ClusterUtils.isDedupRoot(r.getSource())) + .filter(r -> !ClusterUtils.isDedupRoot(r.getTarget())); + + final Dataset dataset = rels + .joinWith(pubs, pubs.col("openaireId").equalTo(rels.col("target")), "inner") + .map(t -> { + final RelatedPublication rel = new RelatedPublication(t._1.getSource(), t._2); + rel.getRelPublication().setRelType(t._1.getRelClass()); + return rel; + }, Encoders.bean(RelatedPublication.class)); + + ClusterUtils.save(dataset, relsPath, RelatedPublication.class, total); + + }); + + } + +} diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/PrepareRelatedSoftwaresJob.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/PrepareRelatedSoftwaresJob.java new file mode 100644 index 0000000000..d15565d0d7 --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/PrepareRelatedSoftwaresJob.java @@ -0,0 +1,84 @@ + +package eu.dnetlib.dhp.broker.oa; + +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; + +import java.util.Optional; + +import org.apache.commons.io.IOUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; +import org.apache.spark.util.LongAccumulator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import eu.dnetlib.broker.objects.OaBrokerRelatedSoftware; +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.broker.oa.util.BrokerConstants; +import eu.dnetlib.dhp.broker.oa.util.ClusterUtils; +import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; +import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.RelatedSoftware; +import eu.dnetlib.dhp.schema.common.ModelConstants; +import eu.dnetlib.dhp.schema.oaf.Relation; +import eu.dnetlib.dhp.schema.oaf.Software; + +public class PrepareRelatedSoftwaresJob { + + private static final Logger log = LoggerFactory.getLogger(PrepareRelatedSoftwaresJob.class); + + public static void main(final String[] args) throws Exception { + final ArgumentApplicationParser parser = new ArgumentApplicationParser( + IOUtils + .toString( + PrepareRelatedSoftwaresJob.class + .getResourceAsStream("/eu/dnetlib/dhp/broker/oa/common_params.json"))); + parser.parseArgument(args); + + final Boolean isSparkSessionManaged = Optional + .ofNullable(parser.get("isSparkSessionManaged")) + .map(Boolean::valueOf) + .orElse(Boolean.TRUE); + log.info("isSparkSessionManaged: {}", isSparkSessionManaged); + + final String graphPath = parser.get("graphPath"); + log.info("graphPath: {}", graphPath); + + final String workingPath = parser.get("workingPath"); + log.info("workingPath: {}", workingPath); + + final String relsPath = workingPath + "/relatedSoftwares"; + log.info("relsPath: {}", relsPath); + + final SparkConf conf = new SparkConf(); + + runWithSparkSession(conf, isSparkSessionManaged, spark -> { + + ClusterUtils.removeDir(spark, relsPath); + + final LongAccumulator total = spark.sparkContext().longAccumulator("total_rels"); + + final Dataset softwares = ClusterUtils + .readPath(spark, graphPath + "/software", Software.class) + .filter(sw -> !ClusterUtils.isDedupRoot(sw.getId())) + .map(ConversionUtils::oafSoftwareToBrokerSoftware, Encoders.bean(OaBrokerRelatedSoftware.class)); + + final Dataset rels = ClusterUtils + .readPath(spark, graphPath + "/relation", Relation.class) + .filter(r -> r.getDataInfo().getDeletedbyinference()) + .filter(r -> r.getRelType().equals(ModelConstants.RESULT_RESULT)) + .filter(r -> !r.getRelClass().equals(BrokerConstants.IS_MERGED_IN_CLASS)) + .filter(r -> !ClusterUtils.isDedupRoot(r.getSource())) + .filter(r -> !ClusterUtils.isDedupRoot(r.getTarget())); + + final Dataset dataset = rels + .joinWith(softwares, softwares.col("openaireId").equalTo(rels.col("target")), "inner") + .map(t -> new RelatedSoftware(t._1.getSource(), t._2), Encoders.bean(RelatedSoftware.class)); + + ClusterUtils.save(dataset, relsPath, RelatedSoftware.class, total); + + }); + + } + +} diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/PrepareSimpleEntititiesJob.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/PrepareSimpleEntititiesJob.java new file mode 100644 index 0000000000..d3c7113ec0 --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/PrepareSimpleEntititiesJob.java @@ -0,0 +1,83 @@ + +package eu.dnetlib.dhp.broker.oa; + +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; + +import java.util.Optional; + +import org.apache.commons.io.IOUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.SparkSession; +import org.apache.spark.util.LongAccumulator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import eu.dnetlib.broker.objects.OaBrokerMainEntity; +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.broker.oa.util.ClusterUtils; +import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; +import eu.dnetlib.dhp.schema.oaf.OtherResearchProduct; +import eu.dnetlib.dhp.schema.oaf.Publication; +import eu.dnetlib.dhp.schema.oaf.Result; +import eu.dnetlib.dhp.schema.oaf.Software; + +public class PrepareSimpleEntititiesJob { + + private static final Logger log = LoggerFactory.getLogger(PrepareSimpleEntititiesJob.class); + + public static void main(final String[] args) throws Exception { + final ArgumentApplicationParser parser = new ArgumentApplicationParser( + IOUtils + .toString( + PrepareSimpleEntititiesJob.class + .getResourceAsStream("/eu/dnetlib/dhp/broker/oa/common_params.json"))); + parser.parseArgument(args); + + final Boolean isSparkSessionManaged = Optional + .ofNullable(parser.get("isSparkSessionManaged")) + .map(Boolean::valueOf) + .orElse(Boolean.TRUE); + log.info("isSparkSessionManaged: {}", isSparkSessionManaged); + + final String graphPath = parser.get("graphPath"); + log.info("graphPath: {}", graphPath); + + final String workingPath = parser.get("workingPath"); + log.info("workingPath: {}", workingPath); + + final String simpleEntitiesPath = workingPath + "/simpleEntities"; + log.info("simpleEntitiesPath: {}", simpleEntitiesPath); + + final SparkConf conf = new SparkConf(); + + runWithSparkSession(conf, isSparkSessionManaged, spark -> { + + ClusterUtils.removeDir(spark, simpleEntitiesPath); + + final LongAccumulator total = spark.sparkContext().longAccumulator("total_entities"); + + final Dataset dataset = prepareSimpleEntities(spark, graphPath, Publication.class) + .union(prepareSimpleEntities(spark, graphPath, eu.dnetlib.dhp.schema.oaf.Dataset.class)) + .union(prepareSimpleEntities(spark, graphPath, Software.class)) + .union(prepareSimpleEntities(spark, graphPath, OtherResearchProduct.class)); + + ClusterUtils.save(dataset, simpleEntitiesPath, OaBrokerMainEntity.class, total); + }); + + } + + private static Dataset prepareSimpleEntities( + final SparkSession spark, + final String graphPath, + final Class sourceClass) { + + return ClusterUtils + .readPath(spark, graphPath + "/" + sourceClass.getSimpleName().toLowerCase(), sourceClass) + .filter(r -> !ClusterUtils.isDedupRoot(r.getId())) + .filter(r -> r.getDataInfo().getDeletedbyinference()) + .map(ConversionUtils::oafResultToBrokerResult, Encoders.bean(OaBrokerMainEntity.class)); + } + +} diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/UpdateMatcher.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/UpdateMatcher.java index 5bfe108a56..af6ab30a1a 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/UpdateMatcher.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/UpdateMatcher.java @@ -1,68 +1,116 @@ package eu.dnetlib.dhp.broker.oa.matchers; -import java.util.Arrays; +import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.BiConsumer; +import java.util.function.Function; +import java.util.stream.Collectors; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.spark.util.LongAccumulator; +import eu.dnetlib.broker.objects.OaBrokerMainEntity; +import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; -import eu.dnetlib.dhp.schema.oaf.Field; +import eu.dnetlib.pace.config.DedupConfig; -public abstract class UpdateMatcher { +public abstract class UpdateMatcher { - private final boolean multipleUpdate; + private final int maxNumber; + private final Function topicFunction; + private final BiConsumer compileHighlightFunction; + private final Function highlightToStringFunction; - public UpdateMatcher(final boolean multipleUpdate) { - this.multipleUpdate = multipleUpdate; + public UpdateMatcher(final int maxNumber, final Function topicFunction, + final BiConsumer compileHighlightFunction, + final Function highlightToStringFunction) { + this.maxNumber = maxNumber; + this.topicFunction = topicFunction; + this.compileHighlightFunction = compileHighlightFunction; + this.highlightToStringFunction = highlightToStringFunction; } - public Collection> searchUpdatesForRecord(final K res, final Collection others) { + public Collection> searchUpdatesForRecord(final OaBrokerMainEntity res, + final Collection others, + final DedupConfig dedupConfig, + final Map accumulators) { final Map> infoMap = new HashMap<>(); - for (final K source : others) { + for (final OaBrokerMainEntity source : others) { if (source != res) { - for (final UpdateInfo info : findUpdates(source, res)) { - final String s = DigestUtils.md5Hex(info.getHighlightValueAsString()); - if (!infoMap.containsKey(s) || infoMap.get(s).getTrust() < info.getTrust()) { - } else { - infoMap.put(s, info); + for (final T hl : findDifferences(source, res)) { + final Topic topic = getTopicFunction().apply(hl); + if (topic != null) { + final UpdateInfo info = new UpdateInfo<>(topic, hl, source, res, + getCompileHighlightFunction(), + getHighlightToStringFunction(), dedupConfig); + + final String s = DigestUtils.md5Hex(info.getHighlightValueAsString()); + if (!infoMap.containsKey(s) || infoMap.get(s).getTrust() < info.getTrust()) { + infoMap.put(s, info); + } } } } } - final Collection> values = infoMap.values(); + final List> values = infoMap + .values() + .stream() + .sorted((o1, o2) -> Float.compare(o2.getTrust(), o1.getTrust())) // DESCENDING + .collect(Collectors.toList()); - if (values.isEmpty() || multipleUpdate) { - return values; + if (values.isEmpty()) { + return new ArrayList<>(); + } else if (values.size() > maxNumber) { + incrementAccumulator(accumulators, maxNumber); + return values.subList(0, maxNumber); } else { - final UpdateInfo v = values - .stream() - .sorted((o1, o2) -> Float.compare(o1.getTrust(), o2.getTrust())) - .findFirst() - .get(); - return Arrays.asList(v); + incrementAccumulator(accumulators, values.size()); + return values; } } - protected abstract List> findUpdates(K source, K target); + protected abstract List findDifferences(OaBrokerMainEntity source, OaBrokerMainEntity target); - protected abstract UpdateInfo generateUpdateInfo(final T highlightValue, - final K source, - final K target); - - protected static boolean isMissing(final List> list) { - return list == null || list.isEmpty() || StringUtils.isBlank(list.get(0).getValue()); + protected static boolean isMissing(final List list) { + return list == null || list.isEmpty() || StringUtils.isBlank(list.get(0)); } - protected boolean isMissing(final Field field) { - return field == null || StringUtils.isBlank(field.getValue()); + protected boolean isMissing(final String s) { + return StringUtils.isBlank(s); + } + + public int getMaxNumber() { + return maxNumber; + } + + public Function getTopicFunction() { + return topicFunction; + } + + public BiConsumer getCompileHighlightFunction() { + return compileHighlightFunction; + } + + public Function getHighlightToStringFunction() { + return highlightToStringFunction; + } + + public String accumulatorName() { + return "event_matcher_" + getClass().getSimpleName().toLowerCase(); + } + + public void incrementAccumulator(final Map accumulators, final long n) { + if (accumulators != null && accumulators.containsKey(accumulatorName())) { + accumulators.get(accumulatorName()).add(n); + } } } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/AbstractEnrichMissingDataset.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/AbstractEnrichMissingDataset.java index 321fd43184..2f73a24486 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/AbstractEnrichMissingDataset.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/AbstractEnrichMissingDataset.java @@ -1,63 +1,50 @@ package eu.dnetlib.dhp.broker.oa.matchers.relatedDatasets; +import java.util.ArrayList; import java.util.List; import java.util.Set; import java.util.stream.Collectors; -import org.apache.commons.lang3.tuple.Pair; - +import eu.dnetlib.broker.objects.OaBrokerMainEntity; +import eu.dnetlib.broker.objects.OaBrokerRelatedDataset; import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; -import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; -import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; -import eu.dnetlib.dhp.schema.oaf.Dataset; -import eu.dnetlib.dhp.schema.oaf.Result; +import eu.dnetlib.dhp.broker.oa.util.BrokerConstants; -public abstract class AbstractEnrichMissingDataset - extends UpdateMatcher>, eu.dnetlib.broker.objects.Dataset> { - - private final Topic topic; +public abstract class AbstractEnrichMissingDataset extends UpdateMatcher { public AbstractEnrichMissingDataset(final Topic topic) { - super(true); - this.topic = topic; + super(10, + rel -> topic, + (p, rel) -> p.getDatasets().add(rel), + rel -> rel.getOpenaireId()); } + protected abstract boolean filterByType(String relType); + @Override - protected final List> findUpdates( - final Pair> source, - final Pair> target) { + protected final List findDifferences(final OaBrokerMainEntity source, + final OaBrokerMainEntity target) { + + if (target.getDatasets().size() >= BrokerConstants.MAX_LIST_SIZE) { + return new ArrayList<>(); + } final Set existingDatasets = target - .getRight() + .getDatasets() .stream() - .map(Dataset::getId) + .filter(rel -> filterByType(rel.getRelType())) + .map(OaBrokerRelatedDataset::getOpenaireId) .collect(Collectors.toSet()); return source - .getRight() + .getDatasets() .stream() - .filter(d -> !existingDatasets.contains(d.getId())) - .map(ConversionUtils::oafDatasetToBrokerDataset) - .map(i -> generateUpdateInfo(i, source, target)) + .filter(rel -> filterByType(rel.getRelType())) + .filter(d -> !existingDatasets.contains(d.getOpenaireId())) .collect(Collectors.toList()); } - @Override - protected final UpdateInfo generateUpdateInfo( - final eu.dnetlib.broker.objects.Dataset highlightValue, - final Pair> source, - final Pair> target) { - return new UpdateInfo<>( - getTopic(), - highlightValue, source.getLeft(), target.getLeft(), - (p, rel) -> p.getDatasets().add(rel), - rel -> rel.getInstances().get(0).getUrl()); - } - - public Topic getTopic() { - return topic; - } } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/EnrichMissingDatasetIsReferencedBy.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/EnrichMissingDatasetIsReferencedBy.java index 74ce761f48..21786687ee 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/EnrichMissingDatasetIsReferencedBy.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/EnrichMissingDatasetIsReferencedBy.java @@ -9,4 +9,9 @@ public class EnrichMissingDatasetIsReferencedBy extends AbstractEnrichMissingDat super(Topic.ENRICH_MISSING_DATASET_IS_REFERENCED_BY); } + @Override + protected boolean filterByType(final String relType) { + return relType.equals("isReferencedBy"); + } + } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/EnrichMissingDatasetIsRelatedTo.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/EnrichMissingDatasetIsRelatedTo.java index 05a8910591..0f3739434e 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/EnrichMissingDatasetIsRelatedTo.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/EnrichMissingDatasetIsRelatedTo.java @@ -9,4 +9,9 @@ public class EnrichMissingDatasetIsRelatedTo extends AbstractEnrichMissingDatase super(Topic.ENRICH_MISSING_DATASET_IS_RELATED_TO); } + @Override + protected boolean filterByType(final String relType) { + return relType.equals("isRelatedTo"); + } + } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/EnrichMissingDatasetIsSupplementedBy.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/EnrichMissingDatasetIsSupplementedBy.java index 23bd68fa1b..cde227feed 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/EnrichMissingDatasetIsSupplementedBy.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/EnrichMissingDatasetIsSupplementedBy.java @@ -9,4 +9,9 @@ public class EnrichMissingDatasetIsSupplementedBy extends AbstractEnrichMissingD super(Topic.ENRICH_MISSING_DATASET_IS_SUPPLEMENTED_BY); } + @Override + protected boolean filterByType(final String relType) { + return relType.equals("isSupplementedBy"); + } + } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/EnrichMissingDatasetIsSupplementedTo.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/EnrichMissingDatasetIsSupplementedTo.java index 03160b6f00..750165ff5a 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/EnrichMissingDatasetIsSupplementedTo.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/EnrichMissingDatasetIsSupplementedTo.java @@ -9,4 +9,9 @@ public class EnrichMissingDatasetIsSupplementedTo extends AbstractEnrichMissingD super(Topic.ENRICH_MISSING_DATASET_IS_SUPPLEMENTED_TO); } + @Override + protected boolean filterByType(final String relType) { + return relType.equals("isSupplementedTo"); + } + } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/EnrichMissingDatasetReferences.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/EnrichMissingDatasetReferences.java index bf1df053d6..b1c0afe16f 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/EnrichMissingDatasetReferences.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/EnrichMissingDatasetReferences.java @@ -9,4 +9,9 @@ public class EnrichMissingDatasetReferences extends AbstractEnrichMissingDataset super(Topic.ENRICH_MISSING_DATASET_REFERENCES); } + @Override + protected boolean filterByType(final String relType) { + return relType.equals("references"); + } + } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedProjects/EnrichMissingProject.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedProjects/EnrichMissingProject.java index 461266d567..ab2735f2a8 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedProjects/EnrichMissingProject.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedProjects/EnrichMissingProject.java @@ -1,41 +1,29 @@ package eu.dnetlib.dhp.broker.oa.matchers.relatedProjects; -import java.util.Arrays; +import java.util.ArrayList; import java.util.List; -import org.apache.commons.lang3.tuple.Pair; - +import eu.dnetlib.broker.objects.OaBrokerMainEntity; +import eu.dnetlib.broker.objects.OaBrokerProject; import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; -import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; -import eu.dnetlib.dhp.schema.oaf.Project; -import eu.dnetlib.dhp.schema.oaf.Result; -public class EnrichMissingProject - extends UpdateMatcher>, eu.dnetlib.broker.objects.Project> { +public class EnrichMissingProject extends UpdateMatcher { public EnrichMissingProject() { - super(true); - } - - @Override - protected List> findUpdates(final Pair> source, - final Pair> target) { - // TODO - return Arrays.asList(); - } - - @Override - public UpdateInfo generateUpdateInfo( - final eu.dnetlib.broker.objects.Project highlightValue, - final Pair> source, - final Pair> target) { - return new UpdateInfo<>( - Topic.ENRICH_MISSING_PROJECT, - highlightValue, source.getLeft(), target.getLeft(), + super(20, + prj -> Topic.ENRICH_MISSING_PROJECT, (p, prj) -> p.getProjects().add(prj), - prj -> prj.getFunder() + "::" + prj.getFundingProgram() + prj.getCode()); + prj -> prj.getOpenaireId()); } + @Override + protected List findDifferences(final OaBrokerMainEntity source, final OaBrokerMainEntity target) { + if (target.getProjects().isEmpty()) { + return source.getProjects(); + } else { + return new ArrayList<>(); + } + } } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedProjects/EnrichMoreProject.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedProjects/EnrichMoreProject.java index d9bfb62d59..85086a6df8 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedProjects/EnrichMoreProject.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedProjects/EnrichMoreProject.java @@ -1,40 +1,45 @@ package eu.dnetlib.dhp.broker.oa.matchers.relatedProjects; -import java.util.Arrays; +import java.util.ArrayList; import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; -import org.apache.commons.lang3.tuple.Pair; - +import eu.dnetlib.broker.objects.OaBrokerMainEntity; +import eu.dnetlib.broker.objects.OaBrokerProject; import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; -import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; -import eu.dnetlib.dhp.schema.oaf.Project; -import eu.dnetlib.dhp.schema.oaf.Result; +import eu.dnetlib.dhp.broker.oa.util.BrokerConstants; -public class EnrichMoreProject extends UpdateMatcher>, eu.dnetlib.broker.objects.Project> { +public class EnrichMoreProject extends UpdateMatcher { public EnrichMoreProject() { - super(true); - } - - @Override - protected List> findUpdates(final Pair> source, - final Pair> target) { - // TODO - return Arrays.asList(); - } - - @Override - public UpdateInfo generateUpdateInfo( - final eu.dnetlib.broker.objects.Project highlightValue, - final Pair> source, - final Pair> target) { - return new UpdateInfo<>( - Topic.ENRICH_MORE_PROJECT, - highlightValue, source.getLeft(), target.getLeft(), + super(20, + prj -> Topic.ENRICH_MORE_PROJECT, (p, prj) -> p.getProjects().add(prj), - prj -> prj.getFunder() + "::" + prj.getFundingProgram() + prj.getCode()); + prj -> prj.getOpenaireId()); + } + + @Override + protected List findDifferences(final OaBrokerMainEntity source, + final OaBrokerMainEntity target) { + + if (target.getProjects().size() >= BrokerConstants.MAX_LIST_SIZE) { + return new ArrayList<>(); + } + + final Set existingProjects = target + .getProjects() + .stream() + .map(p -> p.getOpenaireId()) + .collect(Collectors.toSet()); + + return source + .getProjects() + .stream() + .filter(p -> !existingProjects.contains(p.getOpenaireId())) + .collect(Collectors.toList()); } } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/AbstractEnrichMissingPublication.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/AbstractEnrichMissingPublication.java index 75e77b3c6f..7ba3e5e029 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/AbstractEnrichMissingPublication.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/AbstractEnrichMissingPublication.java @@ -1,63 +1,51 @@ package eu.dnetlib.dhp.broker.oa.matchers.relatedPublications; +import java.util.ArrayList; import java.util.List; import java.util.Set; import java.util.stream.Collectors; -import org.apache.commons.lang3.tuple.Pair; - +import eu.dnetlib.broker.objects.OaBrokerMainEntity; +import eu.dnetlib.broker.objects.OaBrokerRelatedPublication; import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; -import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; -import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; -import eu.dnetlib.dhp.schema.oaf.Publication; -import eu.dnetlib.dhp.schema.oaf.Result; +import eu.dnetlib.dhp.broker.oa.util.BrokerConstants; -public abstract class AbstractEnrichMissingPublication - extends UpdateMatcher>, eu.dnetlib.broker.objects.Publication> { - - private final Topic topic; +public abstract class AbstractEnrichMissingPublication extends UpdateMatcher { public AbstractEnrichMissingPublication(final Topic topic) { - super(true); - this.topic = topic; + super(10, + rel -> topic, + (p, rel) -> p.getPublications().add(rel), + rel -> rel.getOpenaireId()); + } + protected abstract boolean filterByType(String relType); + @Override - protected final List> findUpdates( - final Pair> source, - final Pair> target) { + protected final List findDifferences( + final OaBrokerMainEntity source, + final OaBrokerMainEntity target) { + + if (target.getPublications().size() >= BrokerConstants.MAX_LIST_SIZE) { + return new ArrayList<>(); + } final Set existingPublications = target - .getRight() + .getPublications() .stream() - .map(Publication::getId) + .filter(rel -> filterByType(rel.getRelType())) + .map(OaBrokerRelatedPublication::getOpenaireId) .collect(Collectors.toSet()); return source - .getRight() + .getPublications() .stream() - .filter(d -> !existingPublications.contains(d.getId())) - .map(ConversionUtils::oafPublicationToBrokerPublication) - .map(i -> generateUpdateInfo(i, source, target)) + .filter(rel -> filterByType(rel.getRelType())) + .filter(p -> !existingPublications.contains(p.getOpenaireId())) .collect(Collectors.toList()); - } - @Override - protected final UpdateInfo generateUpdateInfo( - final eu.dnetlib.broker.objects.Publication highlightValue, - final Pair> source, - final Pair> target) { - return new UpdateInfo<>( - getTopic(), - highlightValue, source.getLeft(), target.getLeft(), - (p, rel) -> p.getPublications().add(rel), - rel -> rel.getInstances().get(0).getUrl()); - } - - public Topic getTopic() { - return topic; - } } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/EnrichMissingPublicationIsReferencedBy.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/EnrichMissingPublicationIsReferencedBy.java index 73fa8a45f6..eebb5c1a66 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/EnrichMissingPublicationIsReferencedBy.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/EnrichMissingPublicationIsReferencedBy.java @@ -9,4 +9,8 @@ public class EnrichMissingPublicationIsReferencedBy extends AbstractEnrichMissin super(Topic.ENRICH_MISSING_PUBLICATION_IS_REFERENCED_BY); } + @Override + protected boolean filterByType(final String relType) { + return relType.equals("isReferencedBy"); + } } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/EnrichMissingPublicationIsRelatedTo.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/EnrichMissingPublicationIsRelatedTo.java index 361ea3b342..a8aa550d44 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/EnrichMissingPublicationIsRelatedTo.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/EnrichMissingPublicationIsRelatedTo.java @@ -9,4 +9,9 @@ public class EnrichMissingPublicationIsRelatedTo extends AbstractEnrichMissingPu super(Topic.ENRICH_MISSING_PUBLICATION_IS_RELATED_TO); } + @Override + protected boolean filterByType(final String relType) { + return relType.equals("isRelatedTo"); + } + } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/EnrichMissingPublicationIsSupplementedBy.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/EnrichMissingPublicationIsSupplementedBy.java index 7e8863b1ed..762ac942e4 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/EnrichMissingPublicationIsSupplementedBy.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/EnrichMissingPublicationIsSupplementedBy.java @@ -9,4 +9,8 @@ public class EnrichMissingPublicationIsSupplementedBy extends AbstractEnrichMiss super(Topic.ENRICH_MISSING_PUBLICATION_IS_SUPPLEMENTED_BY); } + @Override + protected boolean filterByType(final String relType) { + return relType.equals("isSupplementedBy"); + } } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/EnrichMissingPublicationIsSupplementedTo.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/EnrichMissingPublicationIsSupplementedTo.java index dc4e513777..fc7196a015 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/EnrichMissingPublicationIsSupplementedTo.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/EnrichMissingPublicationIsSupplementedTo.java @@ -9,4 +9,9 @@ public class EnrichMissingPublicationIsSupplementedTo extends AbstractEnrichMiss super(Topic.ENRICH_MISSING_PUBLICATION_IS_SUPPLEMENTED_TO); } + @Override + protected boolean filterByType(final String relType) { + return relType.equals("isSupplementedTo"); + } + } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/EnrichMissingPublicationReferences.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/EnrichMissingPublicationReferences.java index 5198098bcf..da19944549 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/EnrichMissingPublicationReferences.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/EnrichMissingPublicationReferences.java @@ -9,4 +9,9 @@ public class EnrichMissingPublicationReferences extends AbstractEnrichMissingPub super(Topic.ENRICH_MISSING_PUBLICATION_REFERENCES); } + @Override + protected boolean filterByType(final String relType) { + return relType.equals("references"); + } + } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedSoftware/EnrichMissingSoftware.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedSoftware/EnrichMissingSoftware.java new file mode 100644 index 0000000000..a638024bc0 --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedSoftware/EnrichMissingSoftware.java @@ -0,0 +1,34 @@ + +package eu.dnetlib.dhp.broker.oa.matchers.relatedSoftware; + +import java.util.ArrayList; +import java.util.List; + +import eu.dnetlib.broker.objects.OaBrokerMainEntity; +import eu.dnetlib.broker.objects.OaBrokerRelatedSoftware; +import eu.dnetlib.dhp.broker.model.Topic; +import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; + +public class EnrichMissingSoftware + extends UpdateMatcher { + + public EnrichMissingSoftware() { + super(10, + s -> Topic.ENRICH_MISSING_SOFTWARE, + (p, s) -> p.getSoftwares().add(s), + s -> s.getOpenaireId()); + } + + @Override + protected List findDifferences( + final OaBrokerMainEntity source, + final OaBrokerMainEntity target) { + + if (target.getSoftwares().isEmpty()) { + return source.getSoftwares(); + } else { + return new ArrayList<>(); + } + } + +} diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedSoftware/EnrichMoreSoftware.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedSoftware/EnrichMoreSoftware.java new file mode 100644 index 0000000000..a6cd343590 --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedSoftware/EnrichMoreSoftware.java @@ -0,0 +1,46 @@ + +package eu.dnetlib.dhp.broker.oa.matchers.relatedSoftware; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import eu.dnetlib.broker.objects.OaBrokerMainEntity; +import eu.dnetlib.broker.objects.OaBrokerRelatedSoftware; +import eu.dnetlib.dhp.broker.model.Topic; +import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; +import eu.dnetlib.dhp.broker.oa.util.BrokerConstants; + +public class EnrichMoreSoftware extends UpdateMatcher { + + public EnrichMoreSoftware() { + super(10, + s -> Topic.ENRICH_MORE_SOFTWARE, + (p, s) -> p.getSoftwares().add(s), + s -> s.getOpenaireId()); + } + + @Override + protected List findDifferences( + final OaBrokerMainEntity source, + final OaBrokerMainEntity target) { + + if (target.getSoftwares().size() >= BrokerConstants.MAX_LIST_SIZE) { + return new ArrayList<>(); + } + + final Set existingSoftwares = source + .getSoftwares() + .stream() + .map(OaBrokerRelatedSoftware::getName) + .collect(Collectors.toSet()); + + return target + .getSoftwares() + .stream() + .filter(p -> !existingSoftwares.contains(p.getName())) + .collect(Collectors.toList()); + } + +} diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAbstract.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAbstract.java index a418b633ed..b61696e45a 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAbstract.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAbstract.java @@ -5,34 +5,26 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import eu.dnetlib.broker.objects.OaBrokerMainEntity; import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; -import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; -import eu.dnetlib.dhp.schema.oaf.Result; -public class EnrichMissingAbstract extends UpdateMatcher { +public class EnrichMissingAbstract extends UpdateMatcher { public EnrichMissingAbstract() { - super(false); - } - - @Override - protected List> findUpdates(final Result source, final Result target) { - if (isMissing(target.getDescription()) && !isMissing(source.getDescription())) { - return Arrays.asList(generateUpdateInfo(source.getDescription().get(0).getValue(), source, target)); - } - return new ArrayList<>(); - } - - @Override - public UpdateInfo generateUpdateInfo(final String highlightValue, - final Result source, - final Result target) { - return new UpdateInfo<>( - Topic.ENRICH_MISSING_ABSTRACT, - highlightValue, source, target, + super(1, + s -> Topic.ENRICH_MISSING_ABSTRACT, (p, s) -> p.getAbstracts().add(s), s -> s); } + @Override + protected List findDifferences(final OaBrokerMainEntity source, final OaBrokerMainEntity target) { + if (isMissing(target.getAbstracts()) && !isMissing(source.getAbstracts())) { + return Arrays.asList(source.getAbstracts().get(0)); + } else { + return new ArrayList<>(); + } + } + } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAuthorOrcid.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAuthorOrcid.java index b5c2f7e723..e834d1dde6 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAuthorOrcid.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAuthorOrcid.java @@ -1,36 +1,49 @@ package eu.dnetlib.dhp.broker.oa.matchers.simple; -import java.util.Arrays; +import java.util.ArrayList; import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; -import org.apache.commons.lang3.tuple.Pair; +import org.apache.commons.lang3.StringUtils; +import eu.dnetlib.broker.objects.OaBrokerAuthor; +import eu.dnetlib.broker.objects.OaBrokerMainEntity; import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; -import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; -import eu.dnetlib.dhp.schema.oaf.Result; +import eu.dnetlib.dhp.broker.oa.util.BrokerConstants; -public class EnrichMissingAuthorOrcid extends UpdateMatcher> { +public class EnrichMissingAuthorOrcid extends UpdateMatcher { public EnrichMissingAuthorOrcid() { - super(true); + super(40, + aut -> Topic.ENRICH_MISSING_AUTHOR_ORCID, + (p, aut) -> p.getCreators().add(aut), + aut -> aut.getOrcid()); } @Override - protected List>> findUpdates(final Result source, final Result target) { - // return Arrays.asList(new EnrichMissingAbstract("xxxxxxx", 0.9f)); - return Arrays.asList(); - } + protected List findDifferences(final OaBrokerMainEntity source, + final OaBrokerMainEntity target) { + + if (target.getCreators().size() >= BrokerConstants.MAX_LIST_SIZE) { + return new ArrayList<>(); + } + + final Set existingOrcids = target + .getCreators() + .stream() + .map(OaBrokerAuthor::getOrcid) + .filter(StringUtils::isNotBlank) + .collect(Collectors.toSet()); + + return source + .getCreators() + .stream() + .filter(a -> StringUtils.isNotBlank(a.getOrcid())) + .filter(a -> !existingOrcids.contains(a.getOrcid())) + .collect(Collectors.toList()); - @Override - public UpdateInfo> generateUpdateInfo(final Pair highlightValue, - final Result source, - final Result target) { - return new UpdateInfo<>( - Topic.ENRICH_MISSING_AUTHOR_ORCID, - highlightValue, source, target, - (p, pair) -> p.getCreators().add(pair.getLeft() + " - ORCID: " + pair.getRight()), - pair -> pair.getLeft() + "::" + pair.getRight()); } } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingOpenAccess.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingOpenAccess.java index c7e9dcbc11..8e4f2fcf47 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingOpenAccess.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingOpenAccess.java @@ -1,30 +1,38 @@ package eu.dnetlib.dhp.broker.oa.matchers.simple; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; -import eu.dnetlib.broker.objects.Instance; +import eu.dnetlib.broker.objects.OaBrokerInstance; +import eu.dnetlib.broker.objects.OaBrokerMainEntity; import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; import eu.dnetlib.dhp.broker.oa.util.BrokerConstants; -import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; -import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; -import eu.dnetlib.dhp.schema.oaf.Result; -public class EnrichMissingOpenAccess extends UpdateMatcher { +public class EnrichMissingOpenAccess extends UpdateMatcher { public EnrichMissingOpenAccess() { - super(true); + super(20, + i -> Topic.ENRICH_MISSING_OA_VERSION, + (p, i) -> p.getInstances().add(i), + OaBrokerInstance::getUrl); } @Override - protected List> findUpdates(final Result source, final Result target) { + protected List findDifferences(final OaBrokerMainEntity source, + final OaBrokerMainEntity target) { + + if (target.getInstances().size() >= BrokerConstants.MAX_LIST_SIZE) { + return new ArrayList<>(); + } + final long count = target - .getInstance() + .getInstances() .stream() - .map(i -> i.getAccessright().getClassid()) + .map(OaBrokerInstance::getLicense) .filter(right -> right.equals(BrokerConstants.OPEN_ACCESS)) .count(); @@ -33,24 +41,10 @@ public class EnrichMissingOpenAccess extends UpdateMatcher { } return source - .getInstance() + .getInstances() .stream() - .filter(i -> i.getAccessright().getClassid().equals(BrokerConstants.OPEN_ACCESS)) - .map(ConversionUtils::oafInstanceToBrokerInstances) - .flatMap(s -> s) - .map(i -> generateUpdateInfo(i, source, target)) + .filter(i -> i.getLicense().equals(BrokerConstants.OPEN_ACCESS)) .collect(Collectors.toList()); } - @Override - public UpdateInfo generateUpdateInfo(final Instance highlightValue, - final Result source, - final Result target) { - return new UpdateInfo<>( - Topic.ENRICH_MISSING_OA_VERSION, - highlightValue, source, target, - (p, i) -> p.getInstances().add(i), - Instance::getUrl); - } - } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingPid.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingPid.java index 522d46d402..4e40038908 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingPid.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingPid.java @@ -5,42 +5,32 @@ import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; -import eu.dnetlib.broker.objects.Pid; +import eu.dnetlib.broker.objects.OaBrokerMainEntity; +import eu.dnetlib.broker.objects.OaBrokerTypedValue; import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; -import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; -import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; -import eu.dnetlib.dhp.schema.oaf.Result; -public class EnrichMissingPid extends UpdateMatcher { +public class EnrichMissingPid extends UpdateMatcher { public EnrichMissingPid() { - super(true); - } - - @Override - protected List> findUpdates(final Result source, final Result target) { - final long count = target.getPid().size(); - - if (count > 0) { - return Arrays.asList(); - } - - return source - .getPid() - .stream() - .map(ConversionUtils::oafPidToBrokerPid) - .map(i -> generateUpdateInfo(i, source, target)) - .collect(Collectors.toList()); - } - - @Override - public UpdateInfo generateUpdateInfo(final Pid highlightValue, final Result source, final Result target) { - return new UpdateInfo<>( - Topic.ENRICH_MISSING_PID, - highlightValue, source, target, + super(10, + pid -> Topic.ENRICH_MISSING_PID, (p, pid) -> p.getPids().add(pid), pid -> pid.getType() + "::" + pid.getValue()); } + @Override + protected List findDifferences(final OaBrokerMainEntity source, + final OaBrokerMainEntity target) { + + if (target.getPids().size() > 0) { + return Arrays.asList(); + } + + return source + .getPids() + .stream() + .collect(Collectors.toList()); + } + } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingPublicationDate.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingPublicationDate.java index 197ace97c8..e7b65dad8a 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingPublicationDate.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingPublicationDate.java @@ -5,34 +5,28 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import eu.dnetlib.broker.objects.OaBrokerMainEntity; import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; -import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; -import eu.dnetlib.dhp.schema.oaf.Result; -public class EnrichMissingPublicationDate extends UpdateMatcher { +public class EnrichMissingPublicationDate extends UpdateMatcher { public EnrichMissingPublicationDate() { - super(false); - } - - @Override - protected List> findUpdates(final Result source, final Result target) { - if (isMissing(target.getDateofacceptance()) && !isMissing(source.getDateofacceptance())) { - return Arrays.asList(generateUpdateInfo(source.getDateofacceptance().getValue(), source, target)); - } - return new ArrayList<>(); - } - - @Override - public UpdateInfo generateUpdateInfo(final String highlightValue, - final Result source, - final Result target) { - return new UpdateInfo<>( - Topic.ENRICH_MISSING_PUBLICATION_DATE, - highlightValue, source, target, + super(1, + date -> Topic.ENRICH_MISSING_PUBLICATION_DATE, (p, date) -> p.setPublicationdate(date), s -> s); } + @Override + protected List findDifferences(final OaBrokerMainEntity source, + final OaBrokerMainEntity target) { + + if (isMissing(target.getPublicationdate()) && !isMissing(source.getPublicationdate())) { + return Arrays.asList(source.getPublicationdate()); + } else { + return new ArrayList<>(); + } + } + } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingSoftware.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingSoftware.java deleted file mode 100644 index 4fcba43a40..0000000000 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingSoftware.java +++ /dev/null @@ -1,42 +0,0 @@ - -package eu.dnetlib.dhp.broker.oa.matchers.simple; - -import java.util.Arrays; -import java.util.List; - -import org.apache.commons.lang3.tuple.Pair; - -import eu.dnetlib.dhp.broker.model.Topic; -import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; -import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; -import eu.dnetlib.dhp.schema.oaf.Result; -import eu.dnetlib.dhp.schema.oaf.Software; - -public class EnrichMissingSoftware - extends UpdateMatcher>, eu.dnetlib.broker.objects.Software> { - - public EnrichMissingSoftware() { - super(true); - } - - @Override - protected List> findUpdates( - final Pair> source, - final Pair> target) { - // TODO - return Arrays.asList(); - } - - @Override - public UpdateInfo generateUpdateInfo( - final eu.dnetlib.broker.objects.Software highlightValue, - final Pair> source, - final Pair> target) { - return new UpdateInfo<>( - Topic.ENRICH_MISSING_SOFTWARE, - highlightValue, source.getLeft(), target.getLeft(), - (p, s) -> p.getSoftwares().add(s), - s -> s.getName()); - } - -} diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingSubject.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingSubject.java index 290bad48b3..26ebbb7c03 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingSubject.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingSubject.java @@ -1,54 +1,49 @@ package eu.dnetlib.dhp.broker.oa.matchers.simple; +import java.util.ArrayList; import java.util.List; import java.util.Set; import java.util.stream.Collectors; -import org.apache.commons.lang3.tuple.Pair; - +import eu.dnetlib.broker.objects.OaBrokerMainEntity; +import eu.dnetlib.broker.objects.OaBrokerTypedValue; import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; -import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; -import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; -import eu.dnetlib.dhp.schema.oaf.Qualifier; -import eu.dnetlib.dhp.schema.oaf.Result; -import eu.dnetlib.dhp.schema.oaf.StructuredProperty; +import eu.dnetlib.dhp.broker.oa.util.BrokerConstants; -public class EnrichMissingSubject extends UpdateMatcher> { +public class EnrichMissingSubject extends UpdateMatcher { public EnrichMissingSubject() { - super(true); + super(20, + s -> Topic.fromPath("ENRICH/MISSING/SUBJECT/" + s.getType()), + (p, s) -> p.getSubjects().add(s), + s -> subjectAsString(s)); } @Override - protected List>> findUpdates(final Result source, final Result target) { - final Set existingTypes = target - .getSubject() + protected List findDifferences(final OaBrokerMainEntity source, + final OaBrokerMainEntity target) { + + if (target.getSubjects().size() >= BrokerConstants.MAX_LIST_SIZE) { + return new ArrayList<>(); + } + + final Set existingSubject = target + .getSubjects() .stream() - .map(StructuredProperty::getQualifier) - .map(Qualifier::getClassid) + .map(s -> subjectAsString(s)) .collect(Collectors.toSet()); return source - .getPid() + .getSubjects() .stream() - .filter(pid -> !existingTypes.contains(pid.getQualifier().getClassid())) - .map(ConversionUtils::oafSubjectToPair) - .map(i -> generateUpdateInfo(i, source, target)) + .filter(s -> !existingSubject.contains(subjectAsString(s))) .collect(Collectors.toList()); } - @Override - public UpdateInfo> generateUpdateInfo(final Pair highlightValue, - final Result source, - final Result target) { - - return new UpdateInfo<>( - Topic.fromPath("ENRICH/MISSING/SUBJECT/" + highlightValue.getLeft()), - highlightValue, source, target, - (p, pair) -> p.getSubjects().add(pair.getRight()), - pair -> pair.getLeft() + "::" + pair.getRight()); + private static String subjectAsString(final OaBrokerTypedValue s) { + return s.getType() + "::" + s.getValue(); } } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMoreOpenAccess.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMoreOpenAccess.java index c376da44d2..46f6fa80ca 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMoreOpenAccess.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMoreOpenAccess.java @@ -1,54 +1,47 @@ package eu.dnetlib.dhp.broker.oa.matchers.simple; +import java.util.ArrayList; import java.util.List; import java.util.Set; import java.util.stream.Collectors; -import eu.dnetlib.broker.objects.Instance; +import eu.dnetlib.broker.objects.OaBrokerInstance; +import eu.dnetlib.broker.objects.OaBrokerMainEntity; import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; import eu.dnetlib.dhp.broker.oa.util.BrokerConstants; -import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; -import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; -import eu.dnetlib.dhp.schema.oaf.Result; -public class EnrichMoreOpenAccess extends UpdateMatcher { +public class EnrichMoreOpenAccess extends UpdateMatcher { public EnrichMoreOpenAccess() { - super(true); + super(20, + i -> Topic.ENRICH_MORE_OA_VERSION, + (p, i) -> p.getInstances().add(i), + OaBrokerInstance::getUrl); } @Override - protected List> findUpdates(final Result source, final Result target) { + protected List findDifferences(final OaBrokerMainEntity source, + final OaBrokerMainEntity target) { + + if (target.getInstances().size() >= BrokerConstants.MAX_LIST_SIZE) { + return new ArrayList<>(); + } + final Set urls = target - .getInstance() + .getInstances() .stream() - .filter(i -> i.getAccessright().getClassid().equals(BrokerConstants.OPEN_ACCESS)) + .filter(i -> i.getLicense().equals(BrokerConstants.OPEN_ACCESS)) .map(i -> i.getUrl()) - .flatMap(List::stream) .collect(Collectors.toSet()); return source - .getInstance() + .getInstances() .stream() - .filter(i -> i.getAccessright().getClassid().equals(BrokerConstants.OPEN_ACCESS)) - .map(ConversionUtils::oafInstanceToBrokerInstances) - .flatMap(s -> s) + .filter(i -> i.getLicense().equals(BrokerConstants.OPEN_ACCESS)) .filter(i -> !urls.contains(i.getUrl())) - .map(i -> generateUpdateInfo(i, source, target)) .collect(Collectors.toList()); } - @Override - public UpdateInfo generateUpdateInfo(final Instance highlightValue, - final Result source, - final Result target) { - return new UpdateInfo<>( - Topic.ENRICH_MORE_OA_VERSION, - highlightValue, source, target, - (p, i) -> p.getInstances().add(i), - Instance::getUrl); - } - } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMorePid.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMorePid.java index 2ee327c831..609437b9d0 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMorePid.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMorePid.java @@ -1,47 +1,48 @@ package eu.dnetlib.dhp.broker.oa.matchers.simple; +import java.util.ArrayList; import java.util.List; import java.util.Set; import java.util.stream.Collectors; -import eu.dnetlib.broker.objects.Pid; +import eu.dnetlib.broker.objects.OaBrokerMainEntity; +import eu.dnetlib.broker.objects.OaBrokerTypedValue; import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; -import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; -import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; -import eu.dnetlib.dhp.schema.oaf.Result; +import eu.dnetlib.dhp.broker.oa.util.BrokerConstants; -public class EnrichMorePid extends UpdateMatcher { +public class EnrichMorePid extends UpdateMatcher { public EnrichMorePid() { - super(true); + super(20, + pid -> Topic.ENRICH_MORE_PID, + (p, pid) -> p.getPids().add(pid), + pid -> pidAsString(pid)); } @Override - protected List> findUpdates(final Result source, final Result target) { + protected List findDifferences(final OaBrokerMainEntity source, + final OaBrokerMainEntity target) { + + if (target.getPids().size() >= BrokerConstants.MAX_LIST_SIZE) { + return new ArrayList<>(); + } + final Set existingPids = target - .getPid() + .getPids() .stream() - .map(pid -> pid.getQualifier().getClassid() + "::" + pid.getValue()) + .map(pid -> pidAsString(pid)) .collect(Collectors.toSet()); return source - .getPid() + .getPids() .stream() - .filter(pid -> !existingPids.contains(pid.getQualifier().getClassid() + "::" + pid.getValue())) - .map(ConversionUtils::oafPidToBrokerPid) - .map(i -> generateUpdateInfo(i, source, target)) + .filter(pid -> !existingPids.contains(pidAsString(pid))) .collect(Collectors.toList()); } - @Override - public UpdateInfo generateUpdateInfo(final Pid highlightValue, final Result source, final Result target) { - return new UpdateInfo<>( - Topic.ENRICH_MORE_PID, - highlightValue, source, target, - (p, pid) -> p.getPids().add(pid), - pid -> pid.getType() + "::" + pid.getValue()); + private static String pidAsString(final OaBrokerTypedValue pid) { + return pid.getType() + "::" + pid.getValue(); } - } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMoreSoftware.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMoreSoftware.java deleted file mode 100644 index a1affff623..0000000000 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMoreSoftware.java +++ /dev/null @@ -1,42 +0,0 @@ - -package eu.dnetlib.dhp.broker.oa.matchers.simple; - -import java.util.Arrays; -import java.util.List; - -import org.apache.commons.lang3.tuple.Pair; - -import eu.dnetlib.dhp.broker.model.Topic; -import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; -import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; -import eu.dnetlib.dhp.schema.oaf.Result; -import eu.dnetlib.dhp.schema.oaf.Software; - -public class EnrichMoreSoftware - extends UpdateMatcher>, eu.dnetlib.broker.objects.Software> { - - public EnrichMoreSoftware() { - super(true); - } - - @Override - protected List> findUpdates( - final Pair> source, - final Pair> target) { - // TODO - return Arrays.asList(); - } - - @Override - public UpdateInfo generateUpdateInfo( - final eu.dnetlib.broker.objects.Software highlightValue, - final Pair> source, - final Pair> target) { - return new UpdateInfo<>( - Topic.ENRICH_MORE_SOFTWARE, - highlightValue, source.getLeft(), target.getLeft(), - (p, s) -> p.getSoftwares().add(s), - s -> s.getName()); - } - -} diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMoreSubject.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMoreSubject.java index b38445e885..bbe6609d75 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMoreSubject.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMoreSubject.java @@ -1,51 +1,48 @@ package eu.dnetlib.dhp.broker.oa.matchers.simple; +import java.util.ArrayList; import java.util.List; import java.util.Set; import java.util.stream.Collectors; -import org.apache.commons.lang3.tuple.Pair; - +import eu.dnetlib.broker.objects.OaBrokerMainEntity; +import eu.dnetlib.broker.objects.OaBrokerTypedValue; import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; -import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; -import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; -import eu.dnetlib.dhp.schema.oaf.Result; +import eu.dnetlib.dhp.broker.oa.util.BrokerConstants; -public class EnrichMoreSubject extends UpdateMatcher> { +public class EnrichMoreSubject extends UpdateMatcher { public EnrichMoreSubject() { - super(true); + super(20, + s -> Topic.fromPath("ENRICH/MORE/SUBJECT/" + s.getType()), + (p, s) -> p.getSubjects().add(s), + s -> subjectAsString(s)); } @Override - protected List>> findUpdates(final Result source, final Result target) { + protected List findDifferences(final OaBrokerMainEntity source, + final OaBrokerMainEntity target) { + + if (target.getSubjects().size() >= BrokerConstants.MAX_LIST_SIZE) { + return new ArrayList<>(); + } + final Set existingSubjects = target - .getSubject() + .getSubjects() .stream() - .map(pid -> pid.getQualifier().getClassid() + "::" + pid.getValue()) + .map(pid -> subjectAsString(pid)) .collect(Collectors.toSet()); return source - .getPid() + .getPids() .stream() - .filter(pid -> !existingSubjects.contains(pid.getQualifier().getClassid() + "::" + pid.getValue())) - .map(ConversionUtils::oafSubjectToPair) - .map(i -> generateUpdateInfo(i, source, target)) + .filter(s -> !existingSubjects.contains(subjectAsString(s))) .collect(Collectors.toList()); } - @Override - public UpdateInfo> generateUpdateInfo(final Pair highlightValue, - final Result source, - final Result target) { - - return new UpdateInfo<>( - Topic.fromPath("ENRICH/MORE/SUBJECT/" + highlightValue.getLeft()), - highlightValue, source, target, - (p, pair) -> p.getSubjects().add(pair.getRight()), - pair -> pair.getLeft() + "::" + pair.getRight()); + private static String subjectAsString(final OaBrokerTypedValue s) { + return s.getType() + "::" + s.getValue(); } - } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/BrokerConstants.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/BrokerConstants.java index 8e97192bfb..5308b9dffb 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/BrokerConstants.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/BrokerConstants.java @@ -1,9 +1,33 @@ package eu.dnetlib.dhp.broker.oa.util; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +import eu.dnetlib.dhp.broker.model.Event; +import eu.dnetlib.dhp.broker.oa.util.aggregators.simple.ResultGroup; +import eu.dnetlib.dhp.schema.common.ModelSupport; + public class BrokerConstants { - public final static String OPEN_ACCESS = "OPEN"; - public final static String IS_MERGED_IN_CLASS = "isMergedIn"; + public static final String OPEN_ACCESS = "OPEN"; + public static final String IS_MERGED_IN_CLASS = "isMergedIn"; + + public static final float MIN_TRUST = 0.25f; + public static final float MAX_TRUST = 1.00f; + + public static final int MAX_NUMBER_OF_RELS = 20; + + public static final int MAX_STRING_SIZE = 3000; + + public static final int MAX_LIST_SIZE = 50; + + public static Class[] getModelClasses() { + final Set> list = new HashSet<>(); + list.addAll(Arrays.asList(ModelSupport.getOafModelClasses())); + list.addAll(Arrays.asList(ResultGroup.class, Event.class)); + return list.toArray(new Class[] {}); + } } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/ClusterUtils.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/ClusterUtils.java new file mode 100644 index 0000000000..d8b8dd8079 --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/ClusterUtils.java @@ -0,0 +1,89 @@ + +package eu.dnetlib.dhp.broker.oa.util; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +import org.apache.commons.lang3.StringUtils; +import org.apache.spark.api.java.function.MapFunction; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.SaveMode; +import org.apache.spark.sql.SparkSession; +import org.apache.spark.util.LongAccumulator; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.common.HdfsSupport; + +public class ClusterUtils { + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + public static void createDirIfMissing(final SparkSession spark, final String path) { + HdfsSupport.remove(path, spark.sparkContext().hadoopConfiguration()); + } + + public static void removeDir(final SparkSession spark, final String path) { + HdfsSupport.remove(path, spark.sparkContext().hadoopConfiguration()); + } + + public static Dataset readPath( + final SparkSession spark, + final String inputPath, + final Class clazz) { + return spark + .read() + .textFile(inputPath) + .map((MapFunction) value -> OBJECT_MAPPER.readValue(value, clazz), Encoders.bean(clazz)); + } + + public static boolean isDedupRoot(final String id) { + return id.contains("dedup_wf_"); + } + + public static final boolean isValidResultResultClass(final String s) { + return s.equals("isReferencedBy") + || s.equals("isRelatedTo") + || s.equals("references") + || s.equals("isSupplementedBy") + || s.equals("isSupplementedTo"); + } + + public static T incrementAccumulator(final T o, final LongAccumulator acc) { + if (acc != null) { + acc.add(1); + } + return o; + } + + public static void save(final Dataset dataset, + final String path, + final Class clazz, + final LongAccumulator acc) { + dataset + .map(o -> ClusterUtils.incrementAccumulator(o, acc), Encoders.bean(clazz)) + .write() + .mode(SaveMode.Overwrite) + .json(path); + } + + public static Set parseParamAsList(final ArgumentApplicationParser parser, final String key) { + final String s = parser.get(key).trim(); + + final Set res = new HashSet<>(); + + if (s.length() > 1) { // A value of a single char (for example: '-') indicates an empty list + Arrays + .stream(s.split(",")) + .map(String::trim) + .filter(StringUtils::isNotBlank) + .forEach(res::add); + } + + return res; + } + +} diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/ConversionUtils.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/ConversionUtils.java index 2f87d0ee76..d00c5b817d 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/ConversionUtils.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/ConversionUtils.java @@ -1,49 +1,328 @@ package eu.dnetlib.dhp.broker.oa.util; -import java.util.stream.Stream; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; -import org.apache.commons.lang3.tuple.Pair; +import org.apache.commons.lang3.StringUtils; +import org.dom4j.Document; +import org.dom4j.DocumentException; +import org.dom4j.DocumentHelper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import eu.dnetlib.broker.objects.Instance; -import eu.dnetlib.broker.objects.Pid; +import com.google.common.base.Function; + +import eu.dnetlib.broker.objects.OaBrokerAuthor; +import eu.dnetlib.broker.objects.OaBrokerExternalReference; +import eu.dnetlib.broker.objects.OaBrokerInstance; +import eu.dnetlib.broker.objects.OaBrokerJournal; +import eu.dnetlib.broker.objects.OaBrokerMainEntity; +import eu.dnetlib.broker.objects.OaBrokerProject; +import eu.dnetlib.broker.objects.OaBrokerRelatedDataset; +import eu.dnetlib.broker.objects.OaBrokerRelatedPublication; +import eu.dnetlib.broker.objects.OaBrokerRelatedSoftware; +import eu.dnetlib.broker.objects.OaBrokerTypedValue; +import eu.dnetlib.dhp.schema.oaf.Author; import eu.dnetlib.dhp.schema.oaf.Dataset; +import eu.dnetlib.dhp.schema.oaf.ExternalReference; +import eu.dnetlib.dhp.schema.oaf.Field; +import eu.dnetlib.dhp.schema.oaf.Instance; +import eu.dnetlib.dhp.schema.oaf.Journal; +import eu.dnetlib.dhp.schema.oaf.KeyValue; +import eu.dnetlib.dhp.schema.oaf.Project; import eu.dnetlib.dhp.schema.oaf.Publication; +import eu.dnetlib.dhp.schema.oaf.Qualifier; +import eu.dnetlib.dhp.schema.oaf.Result; +import eu.dnetlib.dhp.schema.oaf.Software; import eu.dnetlib.dhp.schema.oaf.StructuredProperty; public class ConversionUtils { - public static Stream oafInstanceToBrokerInstances(final eu.dnetlib.dhp.schema.oaf.Instance i) { - return i.getUrl().stream().map(url -> { - final Instance r = new Instance(); - r.setUrl(url); - r.setInstancetype(i.getInstancetype().getClassid()); - r.setLicense(BrokerConstants.OPEN_ACCESS); - r.setHostedby(i.getHostedby().getValue()); - return r; + private static final Logger log = LoggerFactory.getLogger(ConversionUtils.class); + + public static List oafInstanceToBrokerInstances(final Instance i) { + if (i == null) { + return new ArrayList<>(); + } + + return mappedList(i.getUrl(), url -> { + final OaBrokerInstance res = new OaBrokerInstance(); + res.setUrl(url); + res.setInstancetype(classId(i.getInstancetype())); + res.setLicense(BrokerConstants.OPEN_ACCESS); + res.setHostedby(kvValue(i.getHostedby())); + return res; }); } - public static Pid oafPidToBrokerPid(final StructuredProperty sp) { - final Pid pid = new Pid(); - pid.setValue(sp.getValue()); - pid.setType(sp.getQualifier().getClassid()); - return pid; + public static OaBrokerTypedValue oafPidToBrokerPid(final StructuredProperty sp) { + return oafStructPropToBrokerTypedValue(sp); } - public static final Pair oafSubjectToPair(final StructuredProperty sp) { - return Pair.of(sp.getQualifier().getClassid(), sp.getValue()); + public static OaBrokerTypedValue oafStructPropToBrokerTypedValue(final StructuredProperty sp) { + return sp != null ? new OaBrokerTypedValue(classId(sp.getQualifier()), sp.getValue()) : null; } - public static final eu.dnetlib.broker.objects.Dataset oafDatasetToBrokerDataset(final Dataset d) { - final eu.dnetlib.broker.objects.Dataset res = new eu.dnetlib.broker.objects.Dataset(); - // TODO + public static final OaBrokerRelatedDataset oafDatasetToBrokerDataset(final Dataset d) { + if (d == null) { + return null; + } + + final OaBrokerRelatedDataset res = new OaBrokerRelatedDataset(); + res.setOpenaireId(d.getId()); + res.setOriginalId(first(d.getOriginalId())); + res.setTitle(structPropValue(d.getTitle())); + res.setPids(mappedList(d.getPid(), ConversionUtils::oafPidToBrokerPid)); + res.setInstances(flatMappedList(d.getInstance(), ConversionUtils::oafInstanceToBrokerInstances)); + res.setCollectedFrom(mappedFirst(d.getCollectedfrom(), KeyValue::getValue)); return res; } - public static final eu.dnetlib.broker.objects.Publication oafPublicationToBrokerPublication(final Publication d) { - final eu.dnetlib.broker.objects.Publication res = new eu.dnetlib.broker.objects.Publication(); - // TODO + public static OaBrokerRelatedPublication oafPublicationToBrokerPublication(final Publication p) { + if (p == null) { + return null; + } + + final OaBrokerRelatedPublication res = new OaBrokerRelatedPublication(); + res.setOpenaireId(p.getId()); + res.setOriginalId(first(p.getOriginalId())); + res.setTitle(structPropValue(p.getTitle())); + res.setPids(mappedList(p.getPid(), ConversionUtils::oafPidToBrokerPid)); + res.setInstances(flatMappedList(p.getInstance(), ConversionUtils::oafInstanceToBrokerInstances)); + res.setCollectedFrom(mappedFirst(p.getCollectedfrom(), KeyValue::getValue)); + return res; } + + public static final OaBrokerMainEntity oafResultToBrokerResult(final Result result) { + if (result == null) { + return null; + } + + final OaBrokerMainEntity res = new OaBrokerMainEntity(); + + res.setOpenaireId(result.getId()); + res.setOriginalId(first(result.getOriginalId())); + res.setTypology(classId(result.getResulttype())); + res.setTitles(structPropList(result.getTitle())); + res.setAbstracts(fieldList(result.getDescription())); + res.setLanguage(classId(result.getLanguage())); + res.setSubjects(structPropTypedList(result.getSubject())); + res.setCreators(mappedList(result.getAuthor(), ConversionUtils::oafAuthorToBrokerAuthor)); + res.setPublicationdate(fieldValue(result.getDateofacceptance())); + res.setPublisher(fieldValue(result.getPublisher())); + res.setEmbargoenddate(fieldValue(result.getEmbargoenddate())); + res.setContributor(fieldList(result.getContributor())); + res + .setJournal( + result instanceof Publication ? oafJournalToBrokerJournal(((Publication) result).getJournal()) : null); + res.setCollectedFromId(mappedFirst(result.getCollectedfrom(), KeyValue::getKey)); + res.setCollectedFromName(mappedFirst(result.getCollectedfrom(), KeyValue::getValue)); + res.setPids(mappedList(result.getPid(), ConversionUtils::oafPidToBrokerPid)); + res.setInstances(flatMappedList(result.getInstance(), ConversionUtils::oafInstanceToBrokerInstances)); + res + .setExternalReferences(mappedList(result.getExternalReference(), ConversionUtils::oafExtRefToBrokerExtRef)); + + return res; + } + + private static OaBrokerAuthor oafAuthorToBrokerAuthor(final Author author) { + if (author == null) { + return null; + } + + final String pids = author.getPid() != null ? author + .getPid() + .stream() + .filter(pid -> pid != null) + .filter(pid -> pid.getQualifier() != null) + .filter(pid -> pid.getQualifier().getClassid() != null) + .filter(pid -> pid.getQualifier().getClassid().equalsIgnoreCase("orcid")) + .map(pid -> pid.getValue()) + .map(pid -> cleanOrcid(pid)) + .filter(StringUtils::isNotBlank) + .findFirst() + .orElse(null) : null; + + return new OaBrokerAuthor(author.getFullname(), pids); + } + + private static String cleanOrcid(final String s) { + final String match = "//orcid.org/"; + return s.contains(match) ? StringUtils.substringAfter(s, match) : s; + } + + private static OaBrokerJournal oafJournalToBrokerJournal(final Journal journal) { + if (journal == null) { + return null; + } + + final OaBrokerJournal res = new OaBrokerJournal(); + res.setName(journal.getName()); + res.setIssn(journal.getIssnPrinted()); + res.setEissn(journal.getIssnOnline()); + res.setLissn(journal.getIssnLinking()); + + return res; + } + + private static OaBrokerExternalReference oafExtRefToBrokerExtRef(final ExternalReference ref) { + if (ref == null) { + return null; + } + + final OaBrokerExternalReference res = new OaBrokerExternalReference(); + res.setRefidentifier(ref.getRefidentifier()); + res.setSitename(ref.getSitename()); + res.setType(classId(ref.getQualifier())); + res.setUrl(ref.getUrl()); + return res; + } + + public static final OaBrokerProject oafProjectToBrokerProject(final Project p) { + if (p == null) { + return null; + } + + final OaBrokerProject res = new OaBrokerProject(); + res.setOpenaireId(p.getId()); + res.setTitle(fieldValue(p.getTitle())); + res.setAcronym(fieldValue(p.getAcronym())); + res.setCode(fieldValue(p.getCode())); + + final String ftree = fieldValue(p.getFundingtree()); + if (StringUtils.isNotBlank(ftree)) { + try { + final Document fdoc = DocumentHelper.parseText(ftree); + res.setFunder(fdoc.valueOf("/fundingtree/funder/shortname")); + res.setJurisdiction(fdoc.valueOf("/fundingtree/funder/jurisdiction")); + res.setFundingProgram(fdoc.valueOf("//funding_level_0/name")); + } catch (final DocumentException e) { + log.error("Error in record " + p.getId() + ": invalid fundingtree: " + ftree); + } + } + + return res; + } + + public static final OaBrokerRelatedSoftware oafSoftwareToBrokerSoftware(final Software sw) { + if (sw == null) { + return null; + } + + final OaBrokerRelatedSoftware res = new OaBrokerRelatedSoftware(); + res.setOpenaireId(sw.getId()); + res.setName(structPropValue(sw.getTitle())); + res.setDescription(fieldValue(sw.getDescription())); + res.setRepository(fieldValue(sw.getCodeRepositoryUrl())); + res.setLandingPage(fieldValue(sw.getDocumentationUrl())); + + return res; + } + + private static String first(final List list) { + return list != null && list.size() > 0 ? list.get(0) : null; + } + + private static String kvValue(final KeyValue kv) { + return kv != null ? kv.getValue() : null; + } + + private static String fieldValue(final Field f) { + return f != null ? f.getValue() : null; + } + + private static String fieldValue(final List> fl) { + return fl != null ? fl.stream().map(Field::getValue).filter(StringUtils::isNotBlank).findFirst().orElse(null) + : null; + } + + private static String classId(final Qualifier q) { + return q != null ? q.getClassid() : null; + } + + private static String structPropValue(final List props) { + return props != null + ? props.stream().map(StructuredProperty::getValue).filter(StringUtils::isNotBlank).findFirst().orElse(null) + : null; + } + + private static List fieldList(final List> fl) { + return fl != null + ? fl + .stream() + .map(Field::getValue) + .map(s -> StringUtils.abbreviate(s, BrokerConstants.MAX_STRING_SIZE)) + .filter(StringUtils::isNotBlank) + .limit(BrokerConstants.MAX_LIST_SIZE) + .collect(Collectors.toList()) + : new ArrayList<>(); + } + + private static List structPropList(final List props) { + return props != null + ? props + .stream() + .map(StructuredProperty::getValue) + .filter(StringUtils::isNotBlank) + .limit(BrokerConstants.MAX_LIST_SIZE) + .collect(Collectors.toList()) + : new ArrayList<>(); + } + + private static List structPropTypedList(final List list) { + if (list == null) { + return new ArrayList<>(); + } + + return list + .stream() + .map(ConversionUtils::oafStructPropToBrokerTypedValue) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + + private static List mappedList(final List list, final Function func) { + if (list == null) { + return new ArrayList<>(); + } + + return list + .stream() + .map(func::apply) + .filter(Objects::nonNull) + .limit(BrokerConstants.MAX_LIST_SIZE) + .collect(Collectors.toList()); + } + + private static List flatMappedList(final List list, final Function> func) { + if (list == null) { + return new ArrayList<>(); + } + + return list + .stream() + .map(func::apply) + .flatMap(List::stream) + .filter(Objects::nonNull) + .limit(BrokerConstants.MAX_LIST_SIZE) + .collect(Collectors.toList()); + } + + private static T mappedFirst(final List list, final Function func) { + if (list == null) { + return null; + } + + return list + .stream() + .map(func::apply) + .filter(Objects::nonNull) + .findFirst() + .orElse(null); + } + } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/EventFinder.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/EventFinder.java new file mode 100644 index 0000000000..593e66d434 --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/EventFinder.java @@ -0,0 +1,119 @@ + +package eu.dnetlib.dhp.broker.oa.util; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.spark.util.LongAccumulator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import eu.dnetlib.broker.objects.OaBrokerMainEntity; +import eu.dnetlib.dhp.broker.model.EventFactory; +import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; +import eu.dnetlib.dhp.broker.oa.matchers.relatedDatasets.EnrichMissingDatasetIsReferencedBy; +import eu.dnetlib.dhp.broker.oa.matchers.relatedDatasets.EnrichMissingDatasetIsRelatedTo; +import eu.dnetlib.dhp.broker.oa.matchers.relatedDatasets.EnrichMissingDatasetIsSupplementedBy; +import eu.dnetlib.dhp.broker.oa.matchers.relatedDatasets.EnrichMissingDatasetIsSupplementedTo; +import eu.dnetlib.dhp.broker.oa.matchers.relatedDatasets.EnrichMissingDatasetReferences; +import eu.dnetlib.dhp.broker.oa.matchers.relatedProjects.EnrichMissingProject; +import eu.dnetlib.dhp.broker.oa.matchers.relatedProjects.EnrichMoreProject; +import eu.dnetlib.dhp.broker.oa.matchers.relatedPublications.EnrichMissingPublicationIsReferencedBy; +import eu.dnetlib.dhp.broker.oa.matchers.relatedPublications.EnrichMissingPublicationIsRelatedTo; +import eu.dnetlib.dhp.broker.oa.matchers.relatedPublications.EnrichMissingPublicationIsSupplementedBy; +import eu.dnetlib.dhp.broker.oa.matchers.relatedPublications.EnrichMissingPublicationIsSupplementedTo; +import eu.dnetlib.dhp.broker.oa.matchers.relatedPublications.EnrichMissingPublicationReferences; +import eu.dnetlib.dhp.broker.oa.matchers.relatedSoftware.EnrichMissingSoftware; +import eu.dnetlib.dhp.broker.oa.matchers.relatedSoftware.EnrichMoreSoftware; +import eu.dnetlib.dhp.broker.oa.matchers.simple.EnrichMissingAbstract; +import eu.dnetlib.dhp.broker.oa.matchers.simple.EnrichMissingAuthorOrcid; +import eu.dnetlib.dhp.broker.oa.matchers.simple.EnrichMissingOpenAccess; +import eu.dnetlib.dhp.broker.oa.matchers.simple.EnrichMissingPid; +import eu.dnetlib.dhp.broker.oa.matchers.simple.EnrichMissingPublicationDate; +import eu.dnetlib.dhp.broker.oa.matchers.simple.EnrichMissingSubject; +import eu.dnetlib.dhp.broker.oa.matchers.simple.EnrichMoreOpenAccess; +import eu.dnetlib.dhp.broker.oa.matchers.simple.EnrichMorePid; +import eu.dnetlib.dhp.broker.oa.matchers.simple.EnrichMoreSubject; +import eu.dnetlib.dhp.broker.oa.util.aggregators.simple.ResultGroup; +import eu.dnetlib.pace.config.DedupConfig; + +public class EventFinder { + + private static final Logger log = LoggerFactory.getLogger(EventFinder.class); + + private static final List> matchers = new ArrayList<>(); + static { + matchers.add(new EnrichMissingAbstract()); + matchers.add(new EnrichMissingAuthorOrcid()); + matchers.add(new EnrichMissingOpenAccess()); + matchers.add(new EnrichMissingPid()); + matchers.add(new EnrichMissingPublicationDate()); + matchers.add(new EnrichMissingSubject()); + matchers.add(new EnrichMoreOpenAccess()); + matchers.add(new EnrichMorePid()); + matchers.add(new EnrichMoreSubject()); + + // Advanced matchers + matchers.add(new EnrichMissingProject()); + matchers.add(new EnrichMoreProject()); + matchers.add(new EnrichMissingSoftware()); + matchers.add(new EnrichMoreSoftware()); + matchers.add(new EnrichMissingPublicationIsRelatedTo()); + matchers.add(new EnrichMissingPublicationIsReferencedBy()); + matchers.add(new EnrichMissingPublicationReferences()); + matchers.add(new EnrichMissingPublicationIsSupplementedTo()); + matchers.add(new EnrichMissingPublicationIsSupplementedBy()); + matchers.add(new EnrichMissingDatasetIsRelatedTo()); + matchers.add(new EnrichMissingDatasetIsReferencedBy()); + matchers.add(new EnrichMissingDatasetReferences()); + matchers.add(new EnrichMissingDatasetIsSupplementedTo()); + matchers.add(new EnrichMissingDatasetIsSupplementedBy()); + } + + public static EventGroup generateEvents(final ResultGroup results, + final Set dsIdWhitelist, + final Set dsIdBlacklist, + final Set dsTypeWhitelist, + final DedupConfig dedupConfig, + final Map accumulators) { + + final List> list = new ArrayList<>(); + + for (final OaBrokerMainEntity target : results.getData()) { + if (verifyTarget(target, dsIdWhitelist, dsIdBlacklist, dsTypeWhitelist)) { + for (final UpdateMatcher matcher : matchers) { + list.addAll(matcher.searchUpdatesForRecord(target, results.getData(), dedupConfig, accumulators)); + } + } + } + + return asEventGroup(list); + } + + private static boolean verifyTarget(final OaBrokerMainEntity target, + final Set dsIdWhitelist, + final Set dsIdBlacklist, + final Set dsTypeWhitelist) { + + if (dsIdWhitelist.contains(target.getCollectedFromId())) { + return true; + } else if (dsIdBlacklist.contains(target.getCollectedFromId())) { + return false; + } else { + return dsTypeWhitelist.contains(target.getCollectedFromType()); + } + } + + private static EventGroup asEventGroup(final List> list) { + final EventGroup events = new EventGroup(); + list.stream().map(EventFactory::newBrokerEvent).forEach(events::addElement); + return events; + } + + public static List> getMatchers() { + return matchers; + } + +} diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/EventGroup.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/EventGroup.java new file mode 100644 index 0000000000..503e31ae1c --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/EventGroup.java @@ -0,0 +1,37 @@ + +package eu.dnetlib.dhp.broker.oa.util; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +import eu.dnetlib.dhp.broker.model.Event; + +public class EventGroup implements Serializable { + + /** + * + */ + private static final long serialVersionUID = 765977943803533130L; + + private List data = new ArrayList<>(); + + public List getData() { + return data; + } + + public void setData(final List data) { + this.data = data; + } + + public EventGroup addElement(final Event elem) { + data.add(elem); + return this; + } + + public EventGroup addGroup(final EventGroup group) { + data.addAll(group.getData()); + return this; + } + +} diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/TrustUtils.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/TrustUtils.java new file mode 100644 index 0000000000..5338d4f3d7 --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/TrustUtils.java @@ -0,0 +1,23 @@ + +package eu.dnetlib.dhp.broker.oa.util; + +public class TrustUtils { + + public static float rescale(final double score, final double threshold) { + if (score >= BrokerConstants.MAX_TRUST) { + return BrokerConstants.MAX_TRUST; + } + + final double val = (score - threshold) * (BrokerConstants.MAX_TRUST - BrokerConstants.MIN_TRUST) + / (BrokerConstants.MAX_TRUST - threshold); + + if (val < BrokerConstants.MIN_TRUST) { + return BrokerConstants.MIN_TRUST; + } + if (val > BrokerConstants.MAX_TRUST) { + return BrokerConstants.MAX_TRUST; + } + + return (float) val; + } +} diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/UpdateInfo.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/UpdateInfo.java index 5cc0d371db..0586b681ea 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/UpdateInfo.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/UpdateInfo.java @@ -4,10 +4,20 @@ package eu.dnetlib.dhp.broker.oa.util; import java.util.function.BiConsumer; import java.util.function.Function; -import eu.dnetlib.broker.objects.OpenAireEventPayload; -import eu.dnetlib.broker.objects.Publication; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.broker.objects.OaBrokerEventPayload; +import eu.dnetlib.broker.objects.OaBrokerInstance; +import eu.dnetlib.broker.objects.OaBrokerMainEntity; +import eu.dnetlib.broker.objects.OaBrokerProvenance; import eu.dnetlib.dhp.broker.model.Topic; -import eu.dnetlib.dhp.schema.oaf.Result; +import eu.dnetlib.pace.config.DedupConfig; +import eu.dnetlib.pace.model.MapDocument; +import eu.dnetlib.pace.tree.support.TreeProcessor; +import eu.dnetlib.pace.util.MapDocumentUtil; public final class UpdateInfo { @@ -15,43 +25,67 @@ public final class UpdateInfo { private final T highlightValue; - private final Result source; + private final OaBrokerMainEntity source; - private final Result target; + private final OaBrokerMainEntity target; - private final BiConsumer compileHighlight; + private final BiConsumer compileHighlight; private final Function highlightToString; private final float trust; - public UpdateInfo(final Topic topic, final T highlightValue, final Result source, final Result target, - final BiConsumer compileHighlight, - final Function highlightToString) { + private static final Logger log = LoggerFactory.getLogger(UpdateInfo.class); + + public UpdateInfo(final Topic topic, final T highlightValue, final OaBrokerMainEntity source, + final OaBrokerMainEntity target, + final BiConsumer compileHighlight, + final Function highlightToString, + final DedupConfig dedupConfig) { this.topic = topic; this.highlightValue = highlightValue; this.source = source; this.target = target; this.compileHighlight = compileHighlight; this.highlightToString = highlightToString; - this.trust = calculateTrust(source, target); + this.trust = calculateTrust(dedupConfig, source, target); } public T getHighlightValue() { return highlightValue; } - public Result getSource() { + public OaBrokerMainEntity getSource() { return source; } - public Result getTarget() { + public OaBrokerMainEntity getTarget() { return target; } - private float calculateTrust(final Result source, final Result target) { - // TODO - return 0.9f; + private float calculateTrust(final DedupConfig dedupConfig, + final OaBrokerMainEntity r1, + final OaBrokerMainEntity r2) { + + if (dedupConfig == null) { + return BrokerConstants.MIN_TRUST; + } + + try { + final ObjectMapper objectMapper = new ObjectMapper(); + final MapDocument doc1 = MapDocumentUtil + .asMapDocumentWithJPath(dedupConfig, objectMapper.writeValueAsString(r1)); + final MapDocument doc2 = MapDocumentUtil + .asMapDocumentWithJPath(dedupConfig, objectMapper.writeValueAsString(r2)); + + final double score = new TreeProcessor(dedupConfig).computeScore(doc1, doc2); + final double threshold = dedupConfig.getWf().getThreshold(); + + return TrustUtils.rescale(score, threshold); + } catch (final Exception e) { + log.error("Error computing score between results", e); + return BrokerConstants.MIN_TRUST; + } } protected Topic getTopic() { @@ -66,12 +100,38 @@ public final class UpdateInfo { return trust; } - public void compileHighlight(final OpenAireEventPayload payload) { - compileHighlight.accept(payload.getHighlight(), getHighlightValue()); - } - public String getHighlightValueAsString() { return highlightToString.apply(getHighlightValue()); } + public OaBrokerEventPayload asBrokerPayload() { + + compileHighlight.accept(target, getHighlightValue()); + + final OaBrokerMainEntity hl = new OaBrokerMainEntity(); + compileHighlight.accept(hl, getHighlightValue()); + + final String provId = getSource().getOpenaireId(); + final String provRepo = getSource().getCollectedFromName(); + final String provType = getSource().getCollectedFromType(); + + final String provUrl = getSource() + .getInstances() + .stream() + .map(OaBrokerInstance::getUrl) + .findFirst() + .orElse(null); + ; + + final OaBrokerProvenance provenance = new OaBrokerProvenance(provId, provRepo, provType, provUrl); + + final OaBrokerEventPayload res = new OaBrokerEventPayload(); + res.setResult(target); + res.setHighlight(hl); + res.setTrust(trust); + res.setProvenance(provenance); + + return res; + } + } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/simple/ResultAggregator.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/simple/ResultAggregator.java new file mode 100644 index 0000000000..ee1c8963e8 --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/simple/ResultAggregator.java @@ -0,0 +1,53 @@ + +package eu.dnetlib.dhp.broker.oa.util.aggregators.simple; + +import org.apache.spark.sql.Encoder; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.expressions.Aggregator; + +import eu.dnetlib.broker.objects.OaBrokerMainEntity; +import eu.dnetlib.dhp.schema.oaf.Relation; +import scala.Tuple2; + +public class ResultAggregator extends Aggregator, ResultGroup, ResultGroup> { + + /** + * + */ + private static final long serialVersionUID = -1492327874705585538L; + + @Override + public ResultGroup zero() { + return new ResultGroup(); + } + + @Override + public ResultGroup reduce(final ResultGroup group, final Tuple2 t) { + group.getData().add(t._1); + return group; + } + + @Override + public ResultGroup merge(final ResultGroup g1, final ResultGroup g2) { + g1.getData().addAll(g2.getData()); + return g1; + } + + @Override + public ResultGroup finish(final ResultGroup group) { + return group; + } + + @Override + public Encoder bufferEncoder() { + return Encoders.bean(ResultGroup.class); + + } + + @Override + public Encoder outputEncoder() { + return Encoders.bean(ResultGroup.class); + + } + +} diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/simple/ResultGroup.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/simple/ResultGroup.java new file mode 100644 index 0000000000..e718e0f1c4 --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/simple/ResultGroup.java @@ -0,0 +1,27 @@ + +package eu.dnetlib.dhp.broker.oa.util.aggregators.simple; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +import eu.dnetlib.broker.objects.OaBrokerMainEntity; + +public class ResultGroup implements Serializable { + + /** + * + */ + private static final long serialVersionUID = -3360828477088669296L; + + private List data = new ArrayList<>(); + + public List getData() { + return data; + } + + public void setData(final List data) { + this.data = data; + } + +} diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/AddDatasourceTypeAggregator.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/AddDatasourceTypeAggregator.java new file mode 100644 index 0000000000..ccd15c8c6e --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/AddDatasourceTypeAggregator.java @@ -0,0 +1,59 @@ + +package eu.dnetlib.dhp.broker.oa.util.aggregators.withRels; + +import org.apache.commons.lang3.StringUtils; +import org.apache.spark.sql.Encoder; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.expressions.Aggregator; + +import eu.dnetlib.broker.objects.OaBrokerMainEntity; +import scala.Tuple2; + +public class AddDatasourceTypeAggregator + extends Aggregator, OaBrokerMainEntity, OaBrokerMainEntity> { + + /** + * + */ + private static final long serialVersionUID = 8788588975496014728L; + + @Override + public OaBrokerMainEntity zero() { + return new OaBrokerMainEntity(); + } + + @Override + public OaBrokerMainEntity finish(final OaBrokerMainEntity g) { + return g; + } + + @Override + public OaBrokerMainEntity reduce(final OaBrokerMainEntity g, + final Tuple2 t) { + final OaBrokerMainEntity res = StringUtils.isNotBlank(g.getOpenaireId()) ? g : t._1; + if (t._2 != null && StringUtils.isNotBlank(t._2.getType())) { + res.setCollectedFromType(t._2.getType()); + } + return res; + + } + + @Override + public OaBrokerMainEntity merge(final OaBrokerMainEntity g1, final OaBrokerMainEntity g2) { + if (StringUtils.isNotBlank(g1.getOpenaireId()) && StringUtils.isNotBlank(g1.getCollectedFromType())) { + return g1; + } else { + return g2; + } + } + + @Override + public Encoder bufferEncoder() { + return Encoders.bean(OaBrokerMainEntity.class); + } + + @Override + public Encoder outputEncoder() { + return Encoders.bean(OaBrokerMainEntity.class); + } +} diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedDataset.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedDataset.java new file mode 100644 index 0000000000..0925e32912 --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedDataset.java @@ -0,0 +1,42 @@ + +package eu.dnetlib.dhp.broker.oa.util.aggregators.withRels; + +import java.io.Serializable; + +import eu.dnetlib.broker.objects.OaBrokerRelatedDataset; + +public class RelatedDataset implements Serializable { + + /** + * + */ + private static final long serialVersionUID = 774487705184038324L; + + private String source; + private OaBrokerRelatedDataset relDataset; + + public RelatedDataset() { + } + + public RelatedDataset(final String source, final OaBrokerRelatedDataset relDataset) { + this.source = source; + this.relDataset = relDataset; + } + + public String getSource() { + return source; + } + + public void setSource(final String source) { + this.source = source; + } + + public OaBrokerRelatedDataset getRelDataset() { + return relDataset; + } + + public void setRelDataset(final OaBrokerRelatedDataset relDataset) { + this.relDataset = relDataset; + } + +} diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedDatasetAggregator.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedDatasetAggregator.java new file mode 100644 index 0000000000..45000f6f3b --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedDatasetAggregator.java @@ -0,0 +1,68 @@ + +package eu.dnetlib.dhp.broker.oa.util.aggregators.withRels; + +import org.apache.commons.lang3.StringUtils; +import org.apache.spark.sql.Encoder; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.expressions.Aggregator; + +import eu.dnetlib.broker.objects.OaBrokerMainEntity; +import eu.dnetlib.dhp.broker.oa.util.BrokerConstants; +import scala.Tuple2; + +public class RelatedDatasetAggregator + extends Aggregator, OaBrokerMainEntity, OaBrokerMainEntity> { + + /** + * + */ + private static final long serialVersionUID = 6969761680131482557L; + + @Override + public OaBrokerMainEntity zero() { + return new OaBrokerMainEntity(); + } + + @Override + public OaBrokerMainEntity finish(final OaBrokerMainEntity g) { + return g; + } + + @Override + public OaBrokerMainEntity reduce(final OaBrokerMainEntity g, final Tuple2 t) { + final OaBrokerMainEntity res = StringUtils.isNotBlank(g.getOpenaireId()) ? g : t._1; + if (t._2 != null && res.getDatasets().size() < BrokerConstants.MAX_NUMBER_OF_RELS) { + res.getDatasets().add(t._2.getRelDataset()); + } + return res; + + } + + @Override + public OaBrokerMainEntity merge(final OaBrokerMainEntity g1, final OaBrokerMainEntity g2) { + if (StringUtils.isNotBlank(g1.getOpenaireId())) { + final int availables = BrokerConstants.MAX_NUMBER_OF_RELS - g1.getDatasets().size(); + if (availables > 0) { + if (g2.getDatasets().size() <= availables) { + g1.getDatasets().addAll(g2.getDatasets()); + } else { + g1.getDatasets().addAll(g2.getDatasets().subList(0, availables)); + } + } + return g1; + } else { + return g2; + } + } + + @Override + public Encoder bufferEncoder() { + return Encoders.bean(OaBrokerMainEntity.class); + } + + @Override + public Encoder outputEncoder() { + return Encoders.bean(OaBrokerMainEntity.class); + } + +} diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedProject.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedProject.java new file mode 100644 index 0000000000..74d19fe9df --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedProject.java @@ -0,0 +1,42 @@ + +package eu.dnetlib.dhp.broker.oa.util.aggregators.withRels; + +import java.io.Serializable; + +import eu.dnetlib.broker.objects.OaBrokerProject; + +public class RelatedProject implements Serializable { + + /** + * + */ + private static final long serialVersionUID = 4941437626549329870L; + + private String source; + private OaBrokerProject relProject; + + public RelatedProject() { + } + + public RelatedProject(final String source, final OaBrokerProject relProject) { + this.source = source; + this.relProject = relProject; + } + + public String getSource() { + return source; + } + + public void setSource(final String source) { + this.source = source; + } + + public OaBrokerProject getRelProject() { + return relProject; + } + + public void setRelProject(final OaBrokerProject relProject) { + this.relProject = relProject; + } + +} diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedProjectAggregator.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedProjectAggregator.java new file mode 100644 index 0000000000..7872178377 --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedProjectAggregator.java @@ -0,0 +1,68 @@ + +package eu.dnetlib.dhp.broker.oa.util.aggregators.withRels; + +import org.apache.commons.lang3.StringUtils; +import org.apache.spark.sql.Encoder; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.expressions.Aggregator; + +import eu.dnetlib.broker.objects.OaBrokerMainEntity; +import eu.dnetlib.dhp.broker.oa.util.BrokerConstants; +import scala.Tuple2; + +public class RelatedProjectAggregator + extends Aggregator, OaBrokerMainEntity, OaBrokerMainEntity> { + + /** + * + */ + private static final long serialVersionUID = 8559808519152275763L; + + @Override + public OaBrokerMainEntity zero() { + return new OaBrokerMainEntity(); + } + + @Override + public OaBrokerMainEntity finish(final OaBrokerMainEntity g) { + return g; + } + + @Override + public OaBrokerMainEntity reduce(final OaBrokerMainEntity g, final Tuple2 t) { + final OaBrokerMainEntity res = StringUtils.isNotBlank(g.getOpenaireId()) ? g : t._1; + if (t._2 != null && res.getProjects().size() < BrokerConstants.MAX_NUMBER_OF_RELS) { + res.getProjects().add(t._2.getRelProject()); + } + return res; + + } + + @Override + public OaBrokerMainEntity merge(final OaBrokerMainEntity g1, final OaBrokerMainEntity g2) { + if (StringUtils.isNotBlank(g1.getOpenaireId())) { + final int availables = BrokerConstants.MAX_NUMBER_OF_RELS - g1.getProjects().size(); + if (availables > 0) { + if (g2.getProjects().size() <= availables) { + g1.getProjects().addAll(g2.getProjects()); + } else { + g1.getProjects().addAll(g2.getProjects().subList(0, availables)); + } + } + return g1; + } else { + return g2; + } + } + + @Override + public Encoder bufferEncoder() { + return Encoders.bean(OaBrokerMainEntity.class); + } + + @Override + public Encoder outputEncoder() { + return Encoders.bean(OaBrokerMainEntity.class); + } + +} diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedPublication.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedPublication.java new file mode 100644 index 0000000000..ed6aeeab11 --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedPublication.java @@ -0,0 +1,42 @@ + +package eu.dnetlib.dhp.broker.oa.util.aggregators.withRels; + +import java.io.Serializable; + +import eu.dnetlib.broker.objects.OaBrokerRelatedPublication; + +public class RelatedPublication implements Serializable { + + /** + * + */ + private static final long serialVersionUID = 9021609640411395128L; + + private String source; + private OaBrokerRelatedPublication relPublication; + + public RelatedPublication() { + } + + public RelatedPublication(final String source, final OaBrokerRelatedPublication relPublication) { + this.source = source; + this.relPublication = relPublication; + } + + public String getSource() { + return source; + } + + public void setSource(final String source) { + this.source = source; + } + + public OaBrokerRelatedPublication getRelPublication() { + return relPublication; + } + + public void setRelPublication(final OaBrokerRelatedPublication relPublication) { + this.relPublication = relPublication; + } + +} diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedPublicationAggregator.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedPublicationAggregator.java new file mode 100644 index 0000000000..2289ebe367 --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedPublicationAggregator.java @@ -0,0 +1,70 @@ + +package eu.dnetlib.dhp.broker.oa.util.aggregators.withRels; + +import org.apache.commons.lang3.StringUtils; +import org.apache.spark.sql.Encoder; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.expressions.Aggregator; + +import eu.dnetlib.broker.objects.OaBrokerMainEntity; +import eu.dnetlib.dhp.broker.oa.util.BrokerConstants; +import scala.Tuple2; + +public class RelatedPublicationAggregator + extends Aggregator, OaBrokerMainEntity, OaBrokerMainEntity> { + + /** + * + */ + private static final long serialVersionUID = 4656934981558135919L; + + @Override + public OaBrokerMainEntity zero() { + return new OaBrokerMainEntity(); + } + + @Override + public OaBrokerMainEntity finish(final OaBrokerMainEntity g) { + return g; + } + + @Override + public OaBrokerMainEntity reduce(final OaBrokerMainEntity g, + final Tuple2 t) { + final OaBrokerMainEntity res = StringUtils.isNotBlank(g.getOpenaireId()) ? g : t._1; + if (t._2 != null && res.getPublications().size() < BrokerConstants.MAX_NUMBER_OF_RELS) { + res.getPublications().add(t._2.getRelPublication()); + } + return res; + + } + + @Override + public OaBrokerMainEntity merge(final OaBrokerMainEntity g1, final OaBrokerMainEntity g2) { + if (StringUtils.isNotBlank(g1.getOpenaireId())) { + final int availables = BrokerConstants.MAX_NUMBER_OF_RELS - g1.getPublications().size(); + if (availables > 0) { + if (g2.getPublications().size() <= availables) { + g1.getPublications().addAll(g2.getPublications()); + } else { + g1.getPublications().addAll(g2.getPublications().subList(0, availables)); + } + } + return g1; + + } else { + return g2; + } + } + + @Override + public Encoder bufferEncoder() { + return Encoders.bean(OaBrokerMainEntity.class); + } + + @Override + public Encoder outputEncoder() { + return Encoders.bean(OaBrokerMainEntity.class); + } + +} diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedSoftware.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedSoftware.java new file mode 100644 index 0000000000..0aa3a4045d --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedSoftware.java @@ -0,0 +1,42 @@ + +package eu.dnetlib.dhp.broker.oa.util.aggregators.withRels; + +import java.io.Serializable; + +import eu.dnetlib.broker.objects.OaBrokerRelatedSoftware; + +public class RelatedSoftware implements Serializable { + + /** + * + */ + private static final long serialVersionUID = 7573383356943300157L; + + private String source; + private OaBrokerRelatedSoftware relSoftware; + + public RelatedSoftware() { + } + + public RelatedSoftware(final String source, final OaBrokerRelatedSoftware relSoftware) { + this.source = source; + this.relSoftware = relSoftware; + } + + public String getSource() { + return source; + } + + public void setSource(final String source) { + this.source = source; + } + + public OaBrokerRelatedSoftware getRelSoftware() { + return relSoftware; + } + + public void setRelSoftware(final OaBrokerRelatedSoftware relSoftware) { + this.relSoftware = relSoftware; + } + +} diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedSoftwareAggregator.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedSoftwareAggregator.java new file mode 100644 index 0000000000..fedb3c9e98 --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedSoftwareAggregator.java @@ -0,0 +1,68 @@ + +package eu.dnetlib.dhp.broker.oa.util.aggregators.withRels; + +import org.apache.commons.lang3.StringUtils; +import org.apache.spark.sql.Encoder; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.expressions.Aggregator; + +import eu.dnetlib.broker.objects.OaBrokerMainEntity; +import eu.dnetlib.dhp.broker.oa.util.BrokerConstants; +import scala.Tuple2; + +public class RelatedSoftwareAggregator + extends Aggregator, OaBrokerMainEntity, OaBrokerMainEntity> { + + /** + * + */ + private static final long serialVersionUID = -8987959389106443702L; + + @Override + public OaBrokerMainEntity zero() { + return new OaBrokerMainEntity(); + } + + @Override + public OaBrokerMainEntity finish(final OaBrokerMainEntity g) { + return g; + } + + @Override + public OaBrokerMainEntity reduce(final OaBrokerMainEntity g, final Tuple2 t) { + final OaBrokerMainEntity res = StringUtils.isNotBlank(g.getOpenaireId()) ? g : t._1; + if (t._2 != null && res.getSoftwares().size() < BrokerConstants.MAX_NUMBER_OF_RELS) { + res.getSoftwares().add(t._2.getRelSoftware()); + } + return res; + + } + + @Override + public OaBrokerMainEntity merge(final OaBrokerMainEntity g1, final OaBrokerMainEntity g2) { + if (StringUtils.isNotBlank(g1.getOpenaireId())) { + final int availables = BrokerConstants.MAX_NUMBER_OF_RELS - g1.getSoftwares().size(); + if (availables > 0) { + if (g2.getSoftwares().size() <= availables) { + g1.getSoftwares().addAll(g2.getSoftwares()); + } else { + g1.getSoftwares().addAll(g2.getSoftwares().subList(0, availables)); + } + } + return g1; + } else { + return g2; + } + } + + @Override + public Encoder bufferEncoder() { + return Encoders.bean(OaBrokerMainEntity.class); + } + + @Override + public Encoder outputEncoder() { + return Encoders.bean(OaBrokerMainEntity.class); + } + +} diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/SimpleDatasourceInfo.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/SimpleDatasourceInfo.java new file mode 100644 index 0000000000..966f63fa0f --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/SimpleDatasourceInfo.java @@ -0,0 +1,40 @@ + +package eu.dnetlib.dhp.broker.oa.util.aggregators.withRels; + +import java.io.Serializable; + +public class SimpleDatasourceInfo implements Serializable { + + /** + * + */ + private static final long serialVersionUID = 2996609859416024734L; + + private String id; + private String type; + + public SimpleDatasourceInfo() { + } + + public SimpleDatasourceInfo(final String id, final String type) { + this.id = id; + this.type = type; + } + + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + + public String getType() { + return type; + } + + public void setType(final String type) { + this.type = type; + } + +} diff --git a/dhp-workflows/dhp-broker-events/src/main/resources/eu/dnetlib/dhp/broker/oa/common_params.json b/dhp-workflows/dhp-broker-events/src/main/resources/eu/dnetlib/dhp/broker/oa/common_params.json new file mode 100644 index 0000000000..adee1888a1 --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/resources/eu/dnetlib/dhp/broker/oa/common_params.json @@ -0,0 +1,14 @@ +[ + { + "paramName": "g", + "paramLongName": "graphPath", + "paramDescription": "the path where there the graph is stored", + "paramRequired": true + }, + { + "paramName": "o", + "paramLongName": "workingPath", + "paramDescription": "the path where the temporary data will be stored", + "paramRequired": true + } +] diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/raw_claims/config-default.xml b/dhp-workflows/dhp-broker-events/src/main/resources/eu/dnetlib/dhp/broker/oa/generate_all/oozie_app/config-default.xml similarity index 100% rename from dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/raw_claims/config-default.xml rename to dhp-workflows/dhp-broker-events/src/main/resources/eu/dnetlib/dhp/broker/oa/generate_all/oozie_app/config-default.xml diff --git a/dhp-workflows/dhp-broker-events/src/main/resources/eu/dnetlib/dhp/broker/oa/generate_all/oozie_app/workflow.xml b/dhp-workflows/dhp-broker-events/src/main/resources/eu/dnetlib/dhp/broker/oa/generate_all/oozie_app/workflow.xml new file mode 100644 index 0000000000..b85c60fdf7 --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/resources/eu/dnetlib/dhp/broker/oa/generate_all/oozie_app/workflow.xml @@ -0,0 +1,467 @@ + + + + + graphInputPath + the path where the graph is stored + + + workingPath + the path where the the generated data will be stored + + + isLookupUrl + the address of the lookUp service + + + dedupConfProfId + the id of a valid Dedup Configuration Profile + + + datasourceIdWhitelist + - + a white list (comma separeted, - for empty list) of datasource ids + + + datasourceTypeWhitelist + - + a white list (comma separeted, - for empty list) of datasource types + + + datasourceIdBlacklist + - + a black list (comma separeted, - for empty list) of datasource ids + + + esIndexName + the elasticsearch index name + + + esIndexHost + the elasticsearch host + + + sparkDriverMemory + memory for driver process + + + sparkExecutorMemory + memory for individual executor + + + sparkExecutorCores + number of cores used by single executor + + + oozieActionShareLibForSpark2 + oozie action sharelib for spark 2.* + + + spark2ExtraListeners + com.cloudera.spark.lineage.NavigatorAppListener + spark 2.* extra listeners classname + + + spark2SqlQueryExecutionListeners + com.cloudera.spark.lineage.NavigatorQueryListener + spark 2.* sql query execution listeners classname + + + spark2YarnHistoryServerAddress + spark 2.* yarn history server address + + + spark2EventLogDir + spark 2.* event log dir location + + + + + ${jobTracker} + ${nameNode} + + + mapreduce.job.queuename + ${queueName} + + + oozie.launcher.mapred.job.queue.name + ${oozieLauncherQueueName} + + + oozie.action.sharelib.for.spark + ${oozieActionShareLibForSpark2} + + + + + + + + Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] + + + + + + + + + + + + + + + + + + + + + + yarn + cluster + PrepareSimpleEntititiesJob + eu.dnetlib.dhp.broker.oa.PrepareSimpleEntititiesJob + dhp-broker-events-${projectVersion}.jar + + --executor-cores=${sparkExecutorCores} + --executor-memory=${sparkExecutorMemory} + --driver-memory=${sparkDriverMemory} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.sql.shuffle.partitions=3840 + + --graphPath${graphInputPath} + --workingPath${workingPath} + + + + + + + + yarn + cluster + PrepareRelatedDatasourcesJob + eu.dnetlib.dhp.broker.oa.PrepareRelatedDatasourcesJob + dhp-broker-events-${projectVersion}.jar + + --executor-cores=${sparkExecutorCores} + --executor-memory=${sparkExecutorMemory} + --driver-memory=${sparkDriverMemory} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.sql.shuffle.partitions=3840 + + --graphPath${graphInputPath} + --workingPath${workingPath} + + + + + + + + + yarn + cluster + PrepareRelatedDatasetsJob + eu.dnetlib.dhp.broker.oa.PrepareRelatedDatasetsJob + dhp-broker-events-${projectVersion}.jar + + --executor-cores=${sparkExecutorCores} + --executor-memory=${sparkExecutorMemory} + --driver-memory=${sparkDriverMemory} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.sql.shuffle.partitions=3840 + + --graphPath${graphInputPath} + --workingPath${workingPath} + + + + + + + + yarn + cluster + PrepareRelatedProjectsJob + eu.dnetlib.dhp.broker.oa.PrepareRelatedProjectsJob + dhp-broker-events-${projectVersion}.jar + + --executor-cores=${sparkExecutorCores} + --executor-memory=${sparkExecutorMemory} + --driver-memory=${sparkDriverMemory} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.sql.shuffle.partitions=3840 + + --graphPath${graphInputPath} + --workingPath${workingPath} + + + + + + + + yarn + cluster + PrepareRelatedPublicationsJob + eu.dnetlib.dhp.broker.oa.PrepareRelatedPublicationsJob + dhp-broker-events-${projectVersion}.jar + + --executor-cores=${sparkExecutorCores} + --executor-memory=${sparkExecutorMemory} + --driver-memory=${sparkDriverMemory} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.sql.shuffle.partitions=3840 + + --graphPath${graphInputPath} + --workingPath${workingPath} + + + + + + + + yarn + cluster + PrepareRelatedSoftwaresJob + eu.dnetlib.dhp.broker.oa.PrepareRelatedSoftwaresJob + dhp-broker-events-${projectVersion}.jar + + --executor-cores=${sparkExecutorCores} + --executor-memory=${sparkExecutorMemory} + --driver-memory=${sparkDriverMemory} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.sql.shuffle.partitions=3840 + + --graphPath${graphInputPath} + --workingPath${workingPath} + + + + + + + + + + yarn + cluster + JoinStep0 + eu.dnetlib.dhp.broker.oa.JoinStep0Job + dhp-broker-events-${projectVersion}.jar + + --executor-cores=${sparkExecutorCores} + --executor-memory=${sparkExecutorMemory} + --driver-memory=${sparkDriverMemory} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.sql.shuffle.partitions=3840 + + --graphPath${graphInputPath} + --workingPath${workingPath} + + + + + + + + yarn + cluster + JoinStep1 + eu.dnetlib.dhp.broker.oa.JoinStep1Job + dhp-broker-events-${projectVersion}.jar + + --executor-cores=${sparkExecutorCores} + --executor-memory=${sparkExecutorMemory} + --driver-memory=${sparkDriverMemory} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.sql.shuffle.partitions=3840 + + --graphPath${graphInputPath} + --workingPath${workingPath} + + + + + + + + yarn + cluster + JoinStep2 + eu.dnetlib.dhp.broker.oa.JoinStep2Job + dhp-broker-events-${projectVersion}.jar + + --executor-cores=${sparkExecutorCores} + --executor-memory=${sparkExecutorMemory} + --driver-memory=${sparkDriverMemory} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.sql.shuffle.partitions=3840 + + --graphPath${graphInputPath} + --workingPath${workingPath} + + + + + + + + yarn + cluster + JoinStep3 + eu.dnetlib.dhp.broker.oa.JoinStep3Job + dhp-broker-events-${projectVersion}.jar + + --executor-cores=${sparkExecutorCores} + --executor-memory=${sparkExecutorMemory} + --driver-memory=${sparkDriverMemory} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.sql.shuffle.partitions=3840 + + --graphPath${graphInputPath} + --workingPath${workingPath} + + + + + + + + yarn + cluster + JoinStep4 + eu.dnetlib.dhp.broker.oa.JoinStep4Job + dhp-broker-events-${projectVersion}.jar + + --executor-cores=${sparkExecutorCores} + --executor-memory=${sparkExecutorMemory} + --driver-memory=${sparkDriverMemory} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.sql.shuffle.partitions=3840 + + --graphPath${graphInputPath} + --workingPath${workingPath} + + + + + + + + yarn + cluster + PrepareGroupsJob + eu.dnetlib.dhp.broker.oa.PrepareGroupsJob + dhp-broker-events-${projectVersion}.jar + + --executor-cores=${sparkExecutorCores} + --executor-memory=${sparkExecutorMemory} + --driver-memory=${sparkDriverMemory} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.sql.shuffle.partitions=3840 + + --graphPath${graphInputPath} + --workingPath${workingPath} + + + + + + + + yarn + cluster + GenerateEventsJob + eu.dnetlib.dhp.broker.oa.GenerateEventsJob + dhp-broker-events-${projectVersion}.jar + + --executor-cores=${sparkExecutorCores} + --executor-memory=${sparkExecutorMemory} + --driver-memory=${sparkDriverMemory} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.sql.shuffle.partitions=3840 + + --workingPath${workingPath} + --isLookupUrl${isLookupUrl} + --dedupConfProfile${dedupConfProfId} + --datasourceIdWhitelist${datasourceIdWhitelist} + --datasourceTypeWhitelist${datasourceTypeWhitelist} + --datasourceIdBlacklist${datasourceIdBlacklist} + + + + + + + + yarn + cluster + IndexOnESJob + eu.dnetlib.dhp.broker.oa.IndexOnESJob + dhp-broker-events-${projectVersion}.jar + + --executor-memory=${sparkExecutorMemory} + --driver-memory=${sparkDriverMemory} + --conf spark.dynamicAllocation.maxExecutors="8" + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.sql.shuffle.partitions=3840 + + --workingPath${workingPath} + --index${esIndexName} + --esHost${esIndexHost} + + + + + + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-broker-events/src/main/resources/eu/dnetlib/dhp/broker/oa/generate_events.json b/dhp-workflows/dhp-broker-events/src/main/resources/eu/dnetlib/dhp/broker/oa/generate_events.json new file mode 100644 index 0000000000..c545884f9e --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/resources/eu/dnetlib/dhp/broker/oa/generate_events.json @@ -0,0 +1,38 @@ +[ + { + "paramName": "o", + "paramLongName": "workingPath", + "paramDescription": "the path where the generated events will be stored", + "paramRequired": true + }, + { + "paramName": "lu", + "paramLongName": "isLookupUrl", + "paramDescription": "the address of the ISLookUpService", + "paramRequired": true + }, + { + "paramName": "d", + "paramLongName": "dedupConfProfile", + "paramDescription": "the id of a valid Dedup Configuration Profile", + "paramRequired": true + }, + { + "paramName": "datasourceIdWhitelist", + "paramLongName": "datasourceIdWhitelist", + "paramDescription": "a white list (comma separeted, - for empty list) of datasource ids", + "paramRequired": true + }, + { + "paramName": "datasourceTypeWhitelist", + "paramLongName": "datasourceTypeWhitelist", + "paramDescription": "a white list (comma separeted, - for empty list) of datasource types", + "paramRequired": true + }, + { + "paramName": "datasourceIdBlacklist", + "paramLongName": "datasourceIdBlacklist", + "paramDescription": "a black list (comma separeted, - for empty list) of datasource ids", + "paramRequired": true + } +] diff --git a/dhp-workflows/dhp-broker-events/src/main/resources/eu/dnetlib/dhp/broker/oa/index_es.json b/dhp-workflows/dhp-broker-events/src/main/resources/eu/dnetlib/dhp/broker/oa/index_es.json new file mode 100644 index 0000000000..ac1dbf7867 --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/resources/eu/dnetlib/dhp/broker/oa/index_es.json @@ -0,0 +1,20 @@ +[ + { + "paramName": "o", + "paramLongName": "workingPath", + "paramDescription": "the workinh path", + "paramRequired": true + }, + { + "paramName": "idx", + "paramLongName": "index", + "paramDescription": "the ES index", + "paramRequired": true + }, + { + "paramName": "es", + "paramLongName": "esHost", + "paramDescription": "the ES host", + "paramRequired": true + } +] diff --git a/dhp-workflows/dhp-broker-events/src/main/resources/eu/dnetlib/dhp/broker/oa/partial/oozie_app/config-default.xml b/dhp-workflows/dhp-broker-events/src/main/resources/eu/dnetlib/dhp/broker/oa/partial/oozie_app/config-default.xml new file mode 100644 index 0000000000..2e0ed9aeea --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/resources/eu/dnetlib/dhp/broker/oa/partial/oozie_app/config-default.xml @@ -0,0 +1,18 @@ + + + jobTracker + yarnRM + + + nameNode + hdfs://nameservice1 + + + oozie.use.system.libpath + true + + + oozie.action.sharelib.for.spark + spark2 + + \ No newline at end of file diff --git a/dhp-workflows/dhp-broker-events/src/main/resources/eu/dnetlib/dhp/broker/oa/partial/oozie_app/workflow.xml b/dhp-workflows/dhp-broker-events/src/main/resources/eu/dnetlib/dhp/broker/oa/partial/oozie_app/workflow.xml new file mode 100644 index 0000000000..392271260e --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/resources/eu/dnetlib/dhp/broker/oa/partial/oozie_app/workflow.xml @@ -0,0 +1,109 @@ + + + + + graphInputPath + the path where the graph is stored + + + workingPath + the path where the the generated data will be stored + + + isLookupUrl + the address of the lookUp service + + + dedupConfProfId + the id of a valid Dedup Configuration Profile + + + + sparkDriverMemory + memory for driver process + + + sparkExecutorMemory + memory for individual executor + + + sparkExecutorCores + number of cores used by single executor + + + oozieActionShareLibForSpark2 + oozie action sharelib for spark 2.* + + + spark2ExtraListeners + com.cloudera.spark.lineage.NavigatorAppListener + spark 2.* extra listeners classname + + + spark2SqlQueryExecutionListeners + com.cloudera.spark.lineage.NavigatorQueryListener + spark 2.* sql query execution listeners classname + + + spark2YarnHistoryServerAddress + spark 2.* yarn history server address + + + spark2EventLogDir + spark 2.* event log dir location + + + + + ${jobTracker} + ${nameNode} + + + mapreduce.job.queuename + ${queueName} + + + oozie.launcher.mapred.job.queue.name + ${oozieLauncherQueueName} + + + oozie.action.sharelib.for.spark + ${oozieActionShareLibForSpark2} + + + + + + + + Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] + + + + + + yarn + cluster + Count + eu.dnetlib.dhp.broker.oa.CheckDuplictedIdsJob + dhp-broker-events-${projectVersion}.jar + + --executor-cores=${sparkExecutorCores} + --executor-memory=${sparkExecutorMemory} + --driver-memory=${sparkDriverMemory} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.sql.shuffle.partitions=3840 + + --graphPath${graphInputPath} + --workingPath${workingPath} + + + + + + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-broker-events/src/test/java/eu/dnetlib/dhp/broker/oa/matchers/UpdateMatcherTest.java b/dhp-workflows/dhp-broker-events/src/test/java/eu/dnetlib/dhp/broker/oa/matchers/UpdateMatcherTest.java new file mode 100644 index 0000000000..93bc5617fb --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/test/java/eu/dnetlib/dhp/broker/oa/matchers/UpdateMatcherTest.java @@ -0,0 +1,125 @@ + +package eu.dnetlib.dhp.broker.oa.matchers; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.Arrays; +import java.util.Collection; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import eu.dnetlib.broker.objects.OaBrokerMainEntity; +import eu.dnetlib.dhp.broker.oa.matchers.simple.EnrichMissingPublicationDate; +import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; + +class UpdateMatcherTest { + + UpdateMatcher matcher = new EnrichMissingPublicationDate(); + + @BeforeEach + void setUp() throws Exception { + } + + @Test + void testSearchUpdatesForRecord_1() { + final OaBrokerMainEntity res = new OaBrokerMainEntity(); + final OaBrokerMainEntity p1 = new OaBrokerMainEntity(); + final OaBrokerMainEntity p2 = new OaBrokerMainEntity(); + final OaBrokerMainEntity p3 = new OaBrokerMainEntity(); + final OaBrokerMainEntity p4 = new OaBrokerMainEntity(); + + final Collection> list = matcher + .searchUpdatesForRecord(res, Arrays.asList(p1, p2, p3, p4), null, null); + + assertTrue(list.isEmpty()); + } + + @Test + void testSearchUpdatesForRecord_2() { + final OaBrokerMainEntity res = new OaBrokerMainEntity(); + final OaBrokerMainEntity p1 = new OaBrokerMainEntity(); + final OaBrokerMainEntity p2 = new OaBrokerMainEntity(); + final OaBrokerMainEntity p3 = new OaBrokerMainEntity(); + final OaBrokerMainEntity p4 = new OaBrokerMainEntity(); + + res.setPublicationdate("2018"); + + final Collection> list = matcher + .searchUpdatesForRecord(res, Arrays.asList(p1, p2, p3, p4), null, null); + + assertTrue(list.isEmpty()); + } + + @Test + void testSearchUpdatesForRecord_3() { + final OaBrokerMainEntity res = new OaBrokerMainEntity(); + final OaBrokerMainEntity p1 = new OaBrokerMainEntity(); + final OaBrokerMainEntity p2 = new OaBrokerMainEntity(); + final OaBrokerMainEntity p3 = new OaBrokerMainEntity(); + final OaBrokerMainEntity p4 = new OaBrokerMainEntity(); + + p2.setPublicationdate("2018"); + + final Collection> list = matcher + .searchUpdatesForRecord(res, Arrays.asList(p1, p2, p3, p4), null, null); + + assertTrue(list.size() == 1); + } + + @Test + void testSearchUpdatesForRecord_4() { + final OaBrokerMainEntity res = new OaBrokerMainEntity(); + final OaBrokerMainEntity p1 = new OaBrokerMainEntity(); + final OaBrokerMainEntity p2 = new OaBrokerMainEntity(); + final OaBrokerMainEntity p3 = new OaBrokerMainEntity(); + final OaBrokerMainEntity p4 = new OaBrokerMainEntity(); + + res.setPublicationdate("2018"); + p2.setPublicationdate("2018"); + + final Collection> list = matcher + .searchUpdatesForRecord(res, Arrays.asList(p1, p2, p3, p4), null, null); + + assertTrue(list.isEmpty()); + } + + @Test + void testSearchUpdatesForRecord_5() { + final OaBrokerMainEntity res = new OaBrokerMainEntity(); + final OaBrokerMainEntity p1 = new OaBrokerMainEntity(); + final OaBrokerMainEntity p2 = new OaBrokerMainEntity(); + final OaBrokerMainEntity p3 = new OaBrokerMainEntity(); + final OaBrokerMainEntity p4 = new OaBrokerMainEntity(); + res.setPublicationdate("2018"); + p1.setPublicationdate("2018"); + p2.setPublicationdate("2018"); + p3.setPublicationdate("2018"); + p4.setPublicationdate("2018"); + + final Collection> list = matcher + .searchUpdatesForRecord(res, Arrays.asList(p1, p2, p3, p4), null, null); + + assertTrue(list.isEmpty()); + } + + @Test + void testSearchUpdatesForRecord_6() { + final OaBrokerMainEntity res = new OaBrokerMainEntity(); + final OaBrokerMainEntity p1 = new OaBrokerMainEntity(); + final OaBrokerMainEntity p2 = new OaBrokerMainEntity(); + final OaBrokerMainEntity p3 = new OaBrokerMainEntity(); + final OaBrokerMainEntity p4 = new OaBrokerMainEntity(); + + p1.setPublicationdate("2018"); + p2.setPublicationdate("2018"); + p3.setPublicationdate("2018"); + p4.setPublicationdate("2018"); + + final Collection> list = matcher + .searchUpdatesForRecord(res, Arrays.asList(p1, p2, p3, p4), null, null); + + assertTrue(list.size() == 1); + } + +} diff --git a/dhp-workflows/dhp-broker-events/src/test/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingPublicationDateTest.java b/dhp-workflows/dhp-broker-events/src/test/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingPublicationDateTest.java new file mode 100644 index 0000000000..77a19af4c1 --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/test/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingPublicationDateTest.java @@ -0,0 +1,57 @@ + +package eu.dnetlib.dhp.broker.oa.matchers.simple; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import eu.dnetlib.broker.objects.OaBrokerMainEntity; + +class EnrichMissingPublicationDateTest { + + final EnrichMissingPublicationDate matcher = new EnrichMissingPublicationDate(); + + @BeforeEach + void setUp() throws Exception { + } + + @Test + void testFindDifferences_1() { + final OaBrokerMainEntity source = new OaBrokerMainEntity(); + final OaBrokerMainEntity target = new OaBrokerMainEntity(); + final List list = matcher.findDifferences(source, target); + assertTrue(list.isEmpty()); + } + + @Test + void testFindDifferences_2() { + final OaBrokerMainEntity source = new OaBrokerMainEntity(); + final OaBrokerMainEntity target = new OaBrokerMainEntity(); + source.setPublicationdate("2018"); + final List list = matcher.findDifferences(source, target); + assertTrue(list.size() == 1); + } + + @Test + void testFindDifferences_3() { + final OaBrokerMainEntity source = new OaBrokerMainEntity(); + final OaBrokerMainEntity target = new OaBrokerMainEntity(); + target.setPublicationdate("2018"); + final List list = matcher.findDifferences(source, target); + assertTrue(list.isEmpty()); + } + + @Test + void testFindDifferences_4() { + final OaBrokerMainEntity source = new OaBrokerMainEntity(); + final OaBrokerMainEntity target = new OaBrokerMainEntity(); + source.setPublicationdate("2018"); + target.setPublicationdate("2018"); + final List list = matcher.findDifferences(source, target); + assertTrue(list.isEmpty()); + } + +} diff --git a/dhp-workflows/dhp-broker-events/src/test/java/eu/dnetlib/dhp/broker/oa/util/TrustUtilsTest.java b/dhp-workflows/dhp-broker-events/src/test/java/eu/dnetlib/dhp/broker/oa/util/TrustUtilsTest.java new file mode 100644 index 0000000000..bb23d6085e --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/test/java/eu/dnetlib/dhp/broker/oa/util/TrustUtilsTest.java @@ -0,0 +1,73 @@ + +package eu.dnetlib.dhp.broker.oa.util; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +public class TrustUtilsTest { + + private static final double THRESHOLD = 0.95; + + @Test + public void rescaleTest_1() { + verifyValue(-0.3, BrokerConstants.MIN_TRUST); + } + + @Test + public void rescaleTest_2() { + verifyValue(0.0, BrokerConstants.MIN_TRUST); + } + + @Test + public void rescaleTest_3() { + verifyValue(0.5, BrokerConstants.MIN_TRUST); + } + + @Test + public void rescaleTest_4() { + verifyValue(0.95, BrokerConstants.MIN_TRUST); + } + + @Test + public void rescaleTest_5() { + verifyValue(0.96, BrokerConstants.MIN_TRUST); + } + + @Test + public void rescaleTest_6() { + verifyValue(0.97, 0.3f); + } + + @Test + public void rescaleTest_7() { + verifyValue(0.98, 0.45f); + } + + @Test + public void rescaleTest_8() { + verifyValue(0.99, 0.6f); + } + + @Test + public void rescaleTest_9() { + verifyValue(1.00, BrokerConstants.MAX_TRUST); + } + + @Test + public void rescaleTest_10() { + verifyValue(1.01, BrokerConstants.MAX_TRUST); + } + + @Test + public void rescaleTest_11() { + verifyValue(2.00, BrokerConstants.MAX_TRUST); + } + + private void verifyValue(final double originalScore, final float expectedTrust) { + final float trust = TrustUtils.rescale(originalScore, THRESHOLD); + System.out.println(trust); + assertTrue(Math.abs(trust - expectedTrust) < 0.01); + } + +} diff --git a/dhp-workflows/dhp-dedup-openaire/pom.xml b/dhp-workflows/dhp-dedup-openaire/pom.xml index 44cf9e67cc..03ddbcf4c8 100644 --- a/dhp-workflows/dhp-dedup-openaire/pom.xml +++ b/dhp-workflows/dhp-dedup-openaire/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.2-SNAPSHOT + 1.2.4-SNAPSHOT 4.0.0 dhp-dedup-openaire diff --git a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/Deduper.java b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/Deduper.java index c72940deb8..180f9f8460 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/Deduper.java +++ b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/Deduper.java @@ -37,7 +37,7 @@ public class Deduper implements Serializable { public static JavaPairRDD createSortedBlocks( JavaPairRDD mapDocs, DedupConfig config) { final String of = config.getWf().getOrderField(); - final int maxQueueSize = config.getWf().getGroupMaxSize(); + final int maxQueueSize = config.getWf().getQueueMaxSize(); return mapDocs // the reduce is just to be sure that we haven't document with same id diff --git a/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/EntityMergerTest.java b/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/EntityMergerTest.java index f4b2c22522..513e14f073 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/EntityMergerTest.java +++ b/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/EntityMergerTest.java @@ -45,6 +45,16 @@ public class EntityMergerTest implements Serializable { } + @Test + public void softwareMergerTest() throws InstantiationException, IllegalAccessException { + List> softwares = readSample(testEntityBasePath + "/software_merge.json", Software.class); + + Software merged = DedupRecordFactory + .entityMerger(dedupId, softwares.iterator(), 0, dataInfo, Software.class); + + System.out.println(merged.getBestaccessright().getClassid()); + } + @Test public void publicationMergerTest() throws InstantiationException, IllegalAccessException { @@ -91,9 +101,9 @@ public class EntityMergerTest implements Serializable { assertEquals(pub_merged.getAuthor().size(), 9); assertEquals(AuthorMerger.countAuthorsPids(pub_merged.getAuthor()), 4); - //verify title + // verify title int count = 0; - for (StructuredProperty title: pub_merged.getTitle()){ + for (StructuredProperty title : pub_merged.getTitle()) { if (title.getQualifier().getClassid().equals("main title")) count++; } diff --git a/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkDedupTest.java b/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkDedupTest.java index 8dd00be979..88d5f24f90 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkDedupTest.java +++ b/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkDedupTest.java @@ -182,7 +182,7 @@ public class SparkDedupTest implements Serializable { .count(); assertEquals(3432, orgs_simrel); - assertEquals(7152, pubs_simrel); + assertEquals(7054, pubs_simrel); assertEquals(344, sw_simrel); assertEquals(458, ds_simrel); assertEquals(6750, orp_simrel); @@ -234,7 +234,7 @@ public class SparkDedupTest implements Serializable { .count(); assertEquals(1276, orgs_mergerel); - assertEquals(1442, pubs_mergerel); + assertEquals(1440, pubs_mergerel); assertEquals(288, sw_mergerel); assertEquals(472, ds_mergerel); assertEquals(718, orp_mergerel); @@ -423,7 +423,7 @@ public class SparkDedupTest implements Serializable { long relations = jsc.textFile(testDedupGraphBasePath + "/relation").count(); - assertEquals(4975, relations); + assertEquals(4971, relations); // check deletedbyinference final Dataset mergeRels = spark diff --git a/dhp-workflows/dhp-dedup-openaire/src/test/resources/eu/dnetlib/dhp/dedup/conf/ds.curr.conf.json b/dhp-workflows/dhp-dedup-openaire/src/test/resources/eu/dnetlib/dhp/dedup/conf/ds.curr.conf.json index 13b18e1c35..2469b2cc03 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/test/resources/eu/dnetlib/dhp/dedup/conf/ds.curr.conf.json +++ b/dhp-workflows/dhp-dedup-openaire/src/test/resources/eu/dnetlib/dhp/dedup/conf/ds.curr.conf.json @@ -6,10 +6,10 @@ "subEntityType" : "resulttype", "subEntityValue" : "dataset", "orderField" : "title", - "queueMaxSize" : "2000", + "queueMaxSize" : "800", "groupMaxSize" : "100", "maxChildren" : "100", - "slidingWindowSize" : "200", + "slidingWindowSize" : "80", "rootBuilder" : ["result", "resultProject_outcome_isProducedBy", "resultResult_publicationDataset_isRelatedTo", "resultResult_similarity_isAmongTopNSimilarDocuments", "resultResult_similarity_hasAmongTopNSimilarDocuments", "resultOrganization_affiliation_hasAuthorInstitution", "resultResult_part_hasPart", "resultResult_part_isPartOf", "resultResult_supplement_isSupplementTo", "resultResult_supplement_isSupplementedBy", "resultResult_version_isVersionOf" ], "includeChildren" : "true", "idPath" : "$.id", @@ -17,8 +17,7 @@ }, "pace" : { "clustering" : [ - { "name" : "ngrampairs", "fields" : [ "title" ], "params" : { "max" : "1", "ngramLen" : "3"} }, - { "name" : "suffixprefix", "fields" : [ "title" ], "params" : { "max" : "1", "len" : "3" } }, + { "name" : "wordssuffixprefix", "fields" : [ "title" ], "params" : { "max" : "2", "len" : "3" } }, { "name" : "lowercase", "fields" : [ "doi" ], "params" : { } } ], "decisionTree" : { diff --git a/dhp-workflows/dhp-dedup-openaire/src/test/resources/eu/dnetlib/dhp/dedup/conf/orp.curr.conf.json b/dhp-workflows/dhp-dedup-openaire/src/test/resources/eu/dnetlib/dhp/dedup/conf/orp.curr.conf.json index 5fb2a171ac..4adcc0439f 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/test/resources/eu/dnetlib/dhp/dedup/conf/orp.curr.conf.json +++ b/dhp-workflows/dhp-dedup-openaire/src/test/resources/eu/dnetlib/dhp/dedup/conf/orp.curr.conf.json @@ -6,10 +6,10 @@ "subEntityType" : "resulttype", "subEntityValue" : "otherresearchproduct", "orderField" : "title", - "queueMaxSize" : "2000", + "queueMaxSize" : "800", "groupMaxSize" : "100", "maxChildren" : "100", - "slidingWindowSize" : "200", + "slidingWindowSize" : "80", "rootBuilder" : [ "result", "resultProject_outcome_isProducedBy", "resultResult_publicationDataset_isRelatedTo", "resultResult_similarity_isAmongTopNSimilarDocuments", "resultResult_similarity_hasAmongTopNSimilarDocuments", "resultOrganization_affiliation_hasAuthorInstitution", "resultResult_part_hasPart", "resultResult_part_isPartOf", "resultResult_supplement_isSupplementTo", "resultResult_supplement_isSupplementedBy", "resultResult_version_isVersionOf" ], "includeChildren" : "true", "idPath" : "$.id", @@ -17,8 +17,7 @@ }, "pace" : { "clustering" : [ - { "name" : "ngrampairs", "fields" : [ "title" ], "params" : { "max" : "1", "ngramLen" : "3"} }, - { "name" : "suffixprefix", "fields" : [ "title" ], "params" : { "max" : "1", "len" : "3" } }, + { "name" : "wordssuffixprefix", "fields" : [ "title" ], "params" : { "max" : "2", "len" : "3" } }, { "name" : "lowercase", "fields" : [ "doi" ], "params" : { } } ], "decisionTree" : { diff --git a/dhp-workflows/dhp-dedup-openaire/src/test/resources/eu/dnetlib/dhp/dedup/conf/pub.curr.conf.json b/dhp-workflows/dhp-dedup-openaire/src/test/resources/eu/dnetlib/dhp/dedup/conf/pub.curr.conf.json index d471ccb89d..ef0b26af41 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/test/resources/eu/dnetlib/dhp/dedup/conf/pub.curr.conf.json +++ b/dhp-workflows/dhp-dedup-openaire/src/test/resources/eu/dnetlib/dhp/dedup/conf/pub.curr.conf.json @@ -6,10 +6,10 @@ "subEntityType": "resulttype", "subEntityValue": "publication", "orderField": "title", - "queueMaxSize": "2000", + "queueMaxSize": "800", "groupMaxSize": "100", "maxChildren": "100", - "slidingWindowSize": "200", + "slidingWindowSize": "80", "rootBuilder": [ "result", "resultProject_outcome_isProducedBy", @@ -29,8 +29,7 @@ }, "pace": { "clustering" : [ - { "name" : "ngrampairs", "fields" : [ "title" ], "params" : { "max" : "1", "ngramLen" : "3"} }, - { "name" : "suffixprefix", "fields" : [ "title" ], "params" : { "max" : "1", "len" : "3" } }, + { "name" : "wordssuffixprefix", "fields" : [ "title" ], "params" : { "max" : "2", "len" : "3" } }, { "name" : "lowercase", "fields" : [ "doi" ], "params" : { } } ], "decisionTree": { diff --git a/dhp-workflows/dhp-dedup-openaire/src/test/resources/eu/dnetlib/dhp/dedup/conf/sw.curr.conf.json b/dhp-workflows/dhp-dedup-openaire/src/test/resources/eu/dnetlib/dhp/dedup/conf/sw.curr.conf.json index f4a107c745..623abbf9f4 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/test/resources/eu/dnetlib/dhp/dedup/conf/sw.curr.conf.json +++ b/dhp-workflows/dhp-dedup-openaire/src/test/resources/eu/dnetlib/dhp/dedup/conf/sw.curr.conf.json @@ -6,10 +6,10 @@ "subEntityType" : "resulttype", "subEntityValue" : "software", "orderField" : "title", - "queueMaxSize" : "2000", + "queueMaxSize" : "800", "groupMaxSize" : "100", "maxChildren" : "100", - "slidingWindowSize" : "200", + "slidingWindowSize" : "80", "rootBuilder" : [ "result", "resultProject_outcome_isProducedBy", "resultResult_publicationDataset_isRelatedTo", "resultResult_similarity_isAmongTopNSimilarDocuments", "resultResult_similarity_hasAmongTopNSimilarDocuments", "resultOrganization_affiliation_hasAuthorInstitution", "resultResult_part_hasPart", "resultResult_part_isPartOf", "resultResult_supplement_isSupplementTo", "resultResult_supplement_isSupplementedBy", "resultResult_version_isVersionOf" ], "includeChildren" : "true", "idPath" : "$.id", @@ -17,8 +17,7 @@ }, "pace" : { "clustering" : [ - { "name" : "ngrampairs", "fields" : [ "title" ], "params" : { "max" : "1", "ngramLen" : "3"} }, - { "name" : "suffixprefix", "fields" : [ "title" ], "params" : { "max" : "1", "len" : "3" } }, + { "name" : "wordssuffixprefix", "fields" : [ "title" ], "params" : { "max" : "2", "len" : "3" } }, { "name" : "lowercase", "fields" : [ "doi", "url" ], "params" : { } } ], "decisionTree": { diff --git a/dhp-workflows/dhp-dedup-openaire/src/test/resources/eu/dnetlib/dhp/dedup/json/publication_merge.json b/dhp-workflows/dhp-dedup-openaire/src/test/resources/eu/dnetlib/dhp/dedup/json/publication_merge.json index 28548c5322..ae688e746b 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/test/resources/eu/dnetlib/dhp/dedup/json/publication_merge.json +++ b/dhp-workflows/dhp-dedup-openaire/src/test/resources/eu/dnetlib/dhp/dedup/json/publication_merge.json @@ -1,3 +1,3 @@ -{"context": [], "dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "sysimport:crosswalk:datasetarchive", "classname": "sysimport:crosswalk:datasetarchive", "schemename": "dnet:provenanceActions", "schemeid": "dnet:provenanceActions"}, "inferred": false, "inferenceprovenance": "dedup-similarity-result-levenstein", "invisible": false, "trust": "0.95"}, "resourcetype": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "pid": [], "contributor": [], "resulttype": {"classid": "publication", "classname": "publication", "schemename": "dnet:result_typologies", "schemeid": "dnet:result_typologies"}, "relevantdate": [], "collectedfrom": [{"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "Journal.fi", "key": "10|openaire____::6eef8049d0feedc089ee009abca55e35"}], "id": "50|a89337edbe55::4930db9e954866d70916cbfba9f81f97", "subject": [], "instance": [{"refereed": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "hostedby": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "Journal.fi", "key": "10|openaire____::6eef8049d0feedc089ee009abca55e35"}, "processingchargeamount": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "license": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "url": [], "distributionlocation": "", "processingchargecurrency": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "dateofacceptance": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "2016-01-01"}, "collectedfrom": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "Journal.fi", "key": "10|openaire____::6eef8049d0feedc089ee009abca55e35"}, "accessright": {"classid": "OPEN", "classname": "Open Access", "schemename": "dnet:access_modes", "schemeid": "dnet:access_modes"}, "instancetype": {"classid": "0001", "classname": "Article", "schemename": "dnet:dataCite_resource", "schemeid": "dnet:dataCite_resource"}}], "embargoenddate": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "lastupdatetimestamp": 0, "author": [{"surname": "Go\\u0308tz", "name": "Julia", "pid": [], "rank": 5, "affiliation": [], "fullname": "G\\u00f6tz, Julia"}, {"surname": "Wolff", "name": "Stephan", "pid": [], "rank": 6, "affiliation": [], "fullname": "Wolff, Stephan"}, {"surname": "Jansen", "name": "Olav", "pid": [], "rank": 7, "affiliation": [], "fullname": "Jansen, Olav"}, {"surname": "Dressler", "name": "Dirk", "pid": [{"qualifier": {"classid": "ORCID", "classname": "ORCID"}, "value": "0000-0000-0656-9999"},{"qualifier": {"classid": "id", "classname": "id"}, "value": "987654321"}], "rank": 8, "affiliation": [], "fullname": "Dressler, Dirk"}, {"surname": "Schneider", "name": "Susanne A.", "pid": [], "rank": 9, "affiliation": [], "fullname": "Schneider, Susanne A."}], "source": [], "dateofcollection": "2019-11-05T14:49:22.351Z", "fulltext": [], "dateoftransformation": "2019-11-05T16:10:58.988Z", "description": [], "format": [], "journal": {"issnPrinted": "1459-6067", "conferencedate": "", "conferenceplace": "", "name": "Agricultural and Food Science", "edition": "", "iss": "3", "sp": "", "vol": "27", "dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "issnOnline": "1795-1895", "ep": "", "issnLinking": ""}, "coverage": [], "publisher": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "language": {"classid": "eng", "classname": "English", "schemename": "dnet:languages", "schemeid": "dnet:languages"}, "bestaccessright": {"classid": "OPEN", "classname": "Open Access", "schemename": "dnet:access_modes", "schemeid": "dnet:access_modes"}, "country": [], "extraInfo": [], "originalId": [], "dateofacceptance": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "2018-09-30"}, "title": [{"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "qualifier": {"classid": "main title", "classname": "main title", "schemename": "dnet:dataCite_title", "schemeid": "dnet:dataCite_title"}, "value": "Altered brain activation in a reversal learning task unmasks adaptive changes in cognitive control in writer's cramp"}]} -{"context": [], "dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "sysimport:crosswalk:repository", "classname": "sysimport:crosswalk:repository", "schemename": "dnet:provenanceActions", "schemeid": "dnet:provenanceActions"}, "inferred": true, "inferenceprovenance": "dedup-similarity-result-levenstein", "invisible": false, "trust": "0.9"}, "resourcetype": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "pid": [{"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "qualifier": {"classid": "doi", "classname": "doi", "schemename": "dnet:pid_types", "schemeid": "dnet:pid_types"}, "value": "10.1016/j.nicl.2015.11.006"}], "contributor": [], "resulttype": {"classid": "publication", "classname": "publication", "schemename": "dnet:result_typologies", "schemeid": "dnet:result_typologies"}, "relevantdate": [], "collectedfrom": [{"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "BASE (Open Access Aggregator)", "key": "10|openaire____::df45502607927471ecf8a6ae83683ff5"}], "id": "50|base_oa_____::0968af610a356656706657e4f234b340", "subject": [], "instance": [{"refereed": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "hostedby": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "NeuroImage: Clinical", "key": "10|doajarticles::0c0e74daa5d95504eade9c81ebbd5b8a"}, "processingchargeamount": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "license": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "http://creativecommons.org/licenses/by-nc-nd/4.0/"}, "url": ["http://dx.doi.org/10.1016/j.nicl.2015.11.006"], "distributionlocation": "", "processingchargecurrency": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "dateofacceptance": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "2016-01-01"}, "collectedfrom": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "BASE (Open Access Aggregator)", "key": "10|openaire____::df45502607927471ecf8a6ae83683ff5"}, "accessright": {"classid": "OPEN", "classname": "Open Access", "schemename": "dnet:access_modes", "schemeid": "dnet:access_modes"}, "instancetype": {"classid": "0001", "classname": "Article", "schemename": "dnet:publication_resource", "schemeid": "dnet:publication_resource"}}], "embargoenddate": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "lastupdatetimestamp": 0, "author": [{"surname": "Zeuner", "name": "Kirsten E.", "pid": [], "rank": 1, "affiliation": [], "fullname": "Zeuner, Kirsten E."}, {"surname": "Knutzen", "name": "Arne", "pid": [], "rank": 2, "affiliation": [], "fullname": "Knutzen, Arne"}, {"surname": "Granert", "name": "Oliver", "pid": [{"qualifier": {"classid": "ORCID", "classname": "ORCID"}, "value": "0000-0002-0656-1023"}], "rank": 3, "affiliation": [], "fullname": "Granert, Oliver"}, {"surname": "Sablowsky", "name": "Simone", "pid": [], "rank": 4, "affiliation": [], "fullname": "Sablowsky, Simone"}, {"surname": "Go\\u0308tz", "name": "Julia", "pid": [], "rank": 5, "affiliation": [], "fullname": "G\\u00f6tz, Julia"}, {"surname": "Wolff", "name": "Stephan", "pid": [], "rank": 6, "affiliation": [], "fullname": "Wolff, Stephan"}, {"surname": "Jansen", "name": "Olav", "pid": [], "rank": 7, "affiliation": [], "fullname": "Jansen, Olav"}, {"surname": "Dressler", "name": "Dirk", "pid": [], "rank": 8, "affiliation": [], "fullname": "Dressler, Dirk"}, {"surname": "Schneider", "name": "Susanne A.", "pid": [], "rank": 9, "affiliation": [], "fullname": "Schneider, Susanne A."}, {"surname": "Klein", "name": "Christine", "pid": [], "rank": 10, "affiliation": [], "fullname": "Klein, Christine"}, {"surname": "Deuschl", "name": "Gu\\u0308nther", "pid": [], "rank": 11, "affiliation": [], "fullname": "Deuschl, G\\u00fcnther"}, {"surname": "Eimeren", "name": "Thilo", "pid": [], "rank": 12, "affiliation": [], "fullname": "van Eimeren, Thilo"}, {"surname": "Witt", "name": "Karsten", "pid": [], "rank": 13, "affiliation": [], "fullname": "Witt, Karsten"}], "source": [], "dateofcollection": "2017-07-27T19:04:09.131Z", "fulltext": [], "dateoftransformation": "2019-01-23T10:15:19.582Z", "description": [], "format": [], "journal": {"issnPrinted": "2213-1582", "conferencedate": "", "conferenceplace": "", "name": "NeuroImage: Clinical", "edition": "", "iss": "", "sp": "63", "vol": "10", "dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "issnOnline": "", "ep": "70", "issnLinking": ""}, "coverage": [], "publisher": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "Elsevier BV"}, "language": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "bestaccessright": {"classid": "OPEN", "classname": "Open Access", "schemename": "dnet:access_modes", "schemeid": "dnet:access_modes"}, "country": [{"classid": "IT", "classname": "Italy", "schemeid": "dnet:countries", "schemename": "dnet:countries"}], "extraInfo": [], "originalId": ["10.1016/j.nicl.2015.11.006"], "dateofacceptance": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "2016-01-01"}, "title": [{"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "qualifier": {"classid": "main title", "classname": "main title", "schemename": "dnet:dataCite_title", "schemeid": "dnet:dataCite_title"}, "value": "Altered brain activation in a reversal learning task unmasks adaptive changes in cognitive control in writer's cramp"}]} -{"context": [], "dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "sysimport:crosswalk:datasetarchive", "classname": "sysimport:crosswalk:datasetarchive", "schemename": "dnet:provenanceActions", "schemeid": "dnet:provenanceActions"}, "inferred": true, "inferenceprovenance": "dedup-similarity-result-levenstein", "invisible": false, "trust": "0.9"}, "resourcetype": {"classid": "0004", "classname": "Conference object", "schemename": "dnet:dataCite_resource", "schemeid": "dnet:dataCite_resource"}, "pid": [], "contributor": [], "resulttype": {"classid": "publication", "classname": "publication", "schemename": "dnet:result_typologies", "schemeid": "dnet:result_typologies"}, "relevantdate": [], "collectedfrom": [{"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "CRIS UNS (Current Research Information System University of Novi Sad)", "key": "10|CRIS_UNS____::f66f1bd369679b5b077dcdf006089556"}], "id": "50|CrisUnsNoviS::9f9d014eea45dab432cab636c4c9cf39", "subject": [], "instance": [{"refereed": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "hostedby": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "CRIS UNS (Current Research Information System University of Novi Sad)", "key": "10|CRIS_UNS____::f66f1bd369679b5b077dcdf006089556"}, "processingchargeamount": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "license": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "url": ["https://www.cris.uns.ac.rs/record.jsf?recordId=113444&source=OpenAIRE&language=en"], "distributionlocation": "", "processingchargecurrency": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "dateofacceptance": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "2019-01-01"}, "collectedfrom": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "CRIS UNS (Current Research Information System University of Novi Sad)", "key": "10|CRIS_UNS____::f66f1bd369679b5b077dcdf006089556"}, "accessright": {"classid": "UNKNOWN", "classname": "UNKNOWN", "schemename": "dnet:access_modes", "schemeid": "dnet:access_modes"}, "instancetype": {"classid": "0004", "classname": "Conference object", "schemename": "dnet:dataCite_resource", "schemeid": "dnet:dataCite_resource"}}], "embargoenddate": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "lastupdatetimestamp": 0, "author": [{"surname": "Zeuner", "name": "Kirsten E.", "pid": [], "rank": 1, "affiliation": [], "fullname": "Zeuner, Kirsten E."}, {"surname": "Knutzen", "name": "Arne", "pid": [], "rank": 2, "affiliation": [], "fullname": "Knutzen, Arne"}, {"surname": "Granert", "name": "Oliver", "pid": [{"qualifier": {"classid": "ORCID", "classname": "ORCID"}, "value": "0000-0002-0656-1023"}, {"qualifier": {"classid": "pubmed", "classname": "pubmed"}, "value": "pubmed.it"}], "rank": 3, "affiliation": [], "fullname": "Granert, Oliver"}, {"surname": "Sablowsky", "name": "Simone", "pid": [{"qualifier": {"classid": "id", "classname": "id"}, "value": "12345678"}], "rank": 4, "affiliation": [], "fullname": "Sablowsky, Simone"}, {"surname": "Go\\u0308tz", "name": "Julia", "pid": [], "rank": 5, "affiliation": [], "fullname": "G\\u00f6tz, Julia"}, {"surname": "Wolff", "name": "Stephan", "pid": [], "rank": 6, "affiliation": [], "fullname": "Wolff, Stephan"}, {"surname": "Jansen", "name": "Olav", "pid": [{"qualifier": {"classid": "ORCID", "classname": "ORCID"}, "value": "0000-0000-0656-1023"},{"qualifier": {"classid": "id", "classname": "id"}, "value": "987654321"}], "rank": 7, "affiliation": [], "fullname": "Jansen, Olav"}, {"surname": "Dressler", "name": "Dirk", "pid": [], "rank": 8, "affiliation": [], "fullname": "Dressler, Dirk"}, {"surname": "Schneider", "name": "Susanne A.", "pid": [], "rank": 9, "affiliation": [], "fullname": "Schneider, Susanne A."}], "source": [], "dateofcollection": "2020-03-10T15:05:38.685Z", "fulltext": [], "dateoftransformation": "2020-03-11T20:11:13.15Z", "description": [], "format": [], "journal": {"issnPrinted": "", "conferencedate": "", "conferenceplace": "", "name": "", "edition": "", "iss": "", "sp": "", "vol": "", "dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "issnOnline": "", "ep": "", "issnLinking": ""}, "coverage": [], "publisher": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "language": {"classid": "en", "classname": "en", "schemename": "dnet:languages", "schemeid": "dnet:languages"}, "bestaccessright": {"classid": "UNKNOWN", "classname": "not available", "schemename": "dnet:access_modes", "schemeid": "dnet:access_modes"}, "country": [{"classid": "FI", "classname": "Finland", "schemeid": "dnet:countries", "schemename": "dnet:countries"}], "extraInfo": [], "originalId": ["(BISIS)113444", "https://www.cris.uns.ac.rs/record.jsf?recordId=113444&source=OpenAIRE&language=en"], "dateofacceptance": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "2016-01-01"}, "title": [{"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "qualifier": {"classid": "test title", "classname": "test title", "schemename": "dnet:dataCite_title", "schemeid": "dnet:dataCite_title"}, "value": "Antichains of copies of ultrahomogeneous structures"}]} \ No newline at end of file +{"context": [], "dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "sysimport:crosswalk:datasetarchive", "classname": "sysimport:crosswalk:datasetarchive", "schemename": "dnet:provenanceActions", "schemeid": "dnet:provenanceActions"}, "inferred": false, "inferenceprovenance": "dedup-similarity-result-levenstein", "invisible": false, "trust": "0.95"}, "resourcetype": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "pid": [], "contributor": [], "resulttype": {"classid": "publication", "classname": "publication", "schemename": "dnet:result_typologies", "schemeid": "dnet:result_typologies"}, "relevantdate": [], "collectedfrom": [{"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "Journal.fi", "key": "10|openaire____::6eef8049d0feedc089ee009abca55e35"}], "id": "50|a89337edbe55::4930db9e954866d70916cbfba9f81f97", "subject": [], "instance": [{"refereed": null, "hostedby": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "Journal.fi", "key": "10|openaire____::6eef8049d0feedc089ee009abca55e35"}, "processingchargeamount": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "license": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "url": [], "distributionlocation": "", "processingchargecurrency": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "dateofacceptance": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "2016-01-01"}, "collectedfrom": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "Journal.fi", "key": "10|openaire____::6eef8049d0feedc089ee009abca55e35"}, "accessright": {"classid": "OPEN", "classname": "Open Access", "schemename": "dnet:access_modes", "schemeid": "dnet:access_modes"}, "instancetype": {"classid": "0001", "classname": "Article", "schemename": "dnet:dataCite_resource", "schemeid": "dnet:dataCite_resource"}}], "embargoenddate": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "lastupdatetimestamp": 0, "author": [{"surname": "Go\\u0308tz", "name": "Julia", "pid": [], "rank": 5, "affiliation": [], "fullname": "G\\u00f6tz, Julia"}, {"surname": "Wolff", "name": "Stephan", "pid": [], "rank": 6, "affiliation": [], "fullname": "Wolff, Stephan"}, {"surname": "Jansen", "name": "Olav", "pid": [], "rank": 7, "affiliation": [], "fullname": "Jansen, Olav"}, {"surname": "Dressler", "name": "Dirk", "pid": [{"qualifier": {"classid": "ORCID", "classname": "ORCID"}, "value": "0000-0000-0656-9999"},{"qualifier": {"classid": "id", "classname": "id"}, "value": "987654321"}], "rank": 8, "affiliation": [], "fullname": "Dressler, Dirk"}, {"surname": "Schneider", "name": "Susanne A.", "pid": [], "rank": 9, "affiliation": [], "fullname": "Schneider, Susanne A."}], "source": [], "dateofcollection": "2019-11-05T14:49:22.351Z", "fulltext": [], "dateoftransformation": "2019-11-05T16:10:58.988Z", "description": [], "format": [], "journal": {"issnPrinted": "1459-6067", "conferencedate": "", "conferenceplace": "", "name": "Agricultural and Food Science", "edition": "", "iss": "3", "sp": "", "vol": "27", "dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "issnOnline": "1795-1895", "ep": "", "issnLinking": ""}, "coverage": [], "publisher": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "language": {"classid": "eng", "classname": "English", "schemename": "dnet:languages", "schemeid": "dnet:languages"}, "bestaccessright": {"classid": "OPEN", "classname": "Open Access", "schemename": "dnet:access_modes", "schemeid": "dnet:access_modes"}, "country": [], "extraInfo": [], "originalId": [], "dateofacceptance": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "2018-09-30"}, "title": [{"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "qualifier": {"classid": "main title", "classname": "main title", "schemename": "dnet:dataCite_title", "schemeid": "dnet:dataCite_title"}, "value": "Altered brain activation in a reversal learning task unmasks adaptive changes in cognitive control in writer's cramp"}]} +{"context": [], "dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "sysimport:crosswalk:repository", "classname": "sysimport:crosswalk:repository", "schemename": "dnet:provenanceActions", "schemeid": "dnet:provenanceActions"}, "inferred": true, "inferenceprovenance": "dedup-similarity-result-levenstein", "invisible": false, "trust": "0.9"}, "resourcetype": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "pid": [{"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "qualifier": {"classid": "doi", "classname": "doi", "schemename": "dnet:pid_types", "schemeid": "dnet:pid_types"}, "value": "10.1016/j.nicl.2015.11.006"}], "contributor": [], "resulttype": {"classid": "publication", "classname": "publication", "schemename": "dnet:result_typologies", "schemeid": "dnet:result_typologies"}, "relevantdate": [], "collectedfrom": [{"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "BASE (Open Access Aggregator)", "key": "10|openaire____::df45502607927471ecf8a6ae83683ff5"}], "id": "50|base_oa_____::0968af610a356656706657e4f234b340", "subject": [], "instance": [{"refereed": null, "hostedby": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "NeuroImage: Clinical", "key": "10|doajarticles::0c0e74daa5d95504eade9c81ebbd5b8a"}, "processingchargeamount": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "license": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "http://creativecommons.org/licenses/by-nc-nd/4.0/"}, "url": ["http://dx.doi.org/10.1016/j.nicl.2015.11.006"], "distributionlocation": "", "processingchargecurrency": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "dateofacceptance": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "2016-01-01"}, "collectedfrom": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "BASE (Open Access Aggregator)", "key": "10|openaire____::df45502607927471ecf8a6ae83683ff5"}, "accessright": {"classid": "OPEN", "classname": "Open Access", "schemename": "dnet:access_modes", "schemeid": "dnet:access_modes"}, "instancetype": {"classid": "0001", "classname": "Article", "schemename": "dnet:publication_resource", "schemeid": "dnet:publication_resource"}}], "embargoenddate": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "lastupdatetimestamp": 0, "author": [{"surname": "Zeuner", "name": "Kirsten E.", "pid": [], "rank": 1, "affiliation": [], "fullname": "Zeuner, Kirsten E."}, {"surname": "Knutzen", "name": "Arne", "pid": [], "rank": 2, "affiliation": [], "fullname": "Knutzen, Arne"}, {"surname": "Granert", "name": "Oliver", "pid": [{"qualifier": {"classid": "ORCID", "classname": "ORCID"}, "value": "0000-0002-0656-1023"}], "rank": 3, "affiliation": [], "fullname": "Granert, Oliver"}, {"surname": "Sablowsky", "name": "Simone", "pid": [], "rank": 4, "affiliation": [], "fullname": "Sablowsky, Simone"}, {"surname": "Go\\u0308tz", "name": "Julia", "pid": [], "rank": 5, "affiliation": [], "fullname": "G\\u00f6tz, Julia"}, {"surname": "Wolff", "name": "Stephan", "pid": [], "rank": 6, "affiliation": [], "fullname": "Wolff, Stephan"}, {"surname": "Jansen", "name": "Olav", "pid": [], "rank": 7, "affiliation": [], "fullname": "Jansen, Olav"}, {"surname": "Dressler", "name": "Dirk", "pid": [], "rank": 8, "affiliation": [], "fullname": "Dressler, Dirk"}, {"surname": "Schneider", "name": "Susanne A.", "pid": [], "rank": 9, "affiliation": [], "fullname": "Schneider, Susanne A."}, {"surname": "Klein", "name": "Christine", "pid": [], "rank": 10, "affiliation": [], "fullname": "Klein, Christine"}, {"surname": "Deuschl", "name": "Gu\\u0308nther", "pid": [], "rank": 11, "affiliation": [], "fullname": "Deuschl, G\\u00fcnther"}, {"surname": "Eimeren", "name": "Thilo", "pid": [], "rank": 12, "affiliation": [], "fullname": "van Eimeren, Thilo"}, {"surname": "Witt", "name": "Karsten", "pid": [], "rank": 13, "affiliation": [], "fullname": "Witt, Karsten"}], "source": [], "dateofcollection": "2017-07-27T19:04:09.131Z", "fulltext": [], "dateoftransformation": "2019-01-23T10:15:19.582Z", "description": [], "format": [], "journal": {"issnPrinted": "2213-1582", "conferencedate": "", "conferenceplace": "", "name": "NeuroImage: Clinical", "edition": "", "iss": "", "sp": "63", "vol": "10", "dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "issnOnline": "", "ep": "70", "issnLinking": ""}, "coverage": [], "publisher": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "Elsevier BV"}, "language": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "bestaccessright": {"classid": "OPEN", "classname": "Open Access", "schemename": "dnet:access_modes", "schemeid": "dnet:access_modes"}, "country": [{"classid": "IT", "classname": "Italy", "schemeid": "dnet:countries", "schemename": "dnet:countries"}], "extraInfo": [], "originalId": ["10.1016/j.nicl.2015.11.006"], "dateofacceptance": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "2016-01-01"}, "title": [{"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "qualifier": {"classid": "main title", "classname": "main title", "schemename": "dnet:dataCite_title", "schemeid": "dnet:dataCite_title"}, "value": "Altered brain activation in a reversal learning task unmasks adaptive changes in cognitive control in writer's cramp"}]} +{"context": [], "dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "sysimport:crosswalk:datasetarchive", "classname": "sysimport:crosswalk:datasetarchive", "schemename": "dnet:provenanceActions", "schemeid": "dnet:provenanceActions"}, "inferred": true, "inferenceprovenance": "dedup-similarity-result-levenstein", "invisible": false, "trust": "0.9"}, "resourcetype": {"classid": "0004", "classname": "Conference object", "schemename": "dnet:dataCite_resource", "schemeid": "dnet:dataCite_resource"}, "pid": [], "contributor": [], "resulttype": {"classid": "publication", "classname": "publication", "schemename": "dnet:result_typologies", "schemeid": "dnet:result_typologies"}, "relevantdate": [], "collectedfrom": [{"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "CRIS UNS (Current Research Information System University of Novi Sad)", "key": "10|CRIS_UNS____::f66f1bd369679b5b077dcdf006089556"}], "id": "50|CrisUnsNoviS::9f9d014eea45dab432cab636c4c9cf39", "subject": [], "instance": [{"refereed": null, "hostedby": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "CRIS UNS (Current Research Information System University of Novi Sad)", "key": "10|CRIS_UNS____::f66f1bd369679b5b077dcdf006089556"}, "processingchargeamount": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "license": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "url": ["https://www.cris.uns.ac.rs/record.jsf?recordId=113444&source=OpenAIRE&language=en"], "distributionlocation": "", "processingchargecurrency": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "dateofacceptance": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "2019-01-01"}, "collectedfrom": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "CRIS UNS (Current Research Information System University of Novi Sad)", "key": "10|CRIS_UNS____::f66f1bd369679b5b077dcdf006089556"}, "accessright": {"classid": "UNKNOWN", "classname": "UNKNOWN", "schemename": "dnet:access_modes", "schemeid": "dnet:access_modes"}, "instancetype": {"classid": "0004", "classname": "Conference object", "schemename": "dnet:dataCite_resource", "schemeid": "dnet:dataCite_resource"}}], "embargoenddate": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "lastupdatetimestamp": 0, "author": [{"surname": "Zeuner", "name": "Kirsten E.", "pid": [], "rank": 1, "affiliation": [], "fullname": "Zeuner, Kirsten E."}, {"surname": "Knutzen", "name": "Arne", "pid": [], "rank": 2, "affiliation": [], "fullname": "Knutzen, Arne"}, {"surname": "Granert", "name": "Oliver", "pid": [{"qualifier": {"classid": "ORCID", "classname": "ORCID"}, "value": "0000-0002-0656-1023"}, {"qualifier": {"classid": "pubmed", "classname": "pubmed"}, "value": "pubmed.it"}], "rank": 3, "affiliation": [], "fullname": "Granert, Oliver"}, {"surname": "Sablowsky", "name": "Simone", "pid": [{"qualifier": {"classid": "id", "classname": "id"}, "value": "12345678"}], "rank": 4, "affiliation": [], "fullname": "Sablowsky, Simone"}, {"surname": "Go\\u0308tz", "name": "Julia", "pid": [], "rank": 5, "affiliation": [], "fullname": "G\\u00f6tz, Julia"}, {"surname": "Wolff", "name": "Stephan", "pid": [], "rank": 6, "affiliation": [], "fullname": "Wolff, Stephan"}, {"surname": "Jansen", "name": "Olav", "pid": [{"qualifier": {"classid": "ORCID", "classname": "ORCID"}, "value": "0000-0000-0656-1023"},{"qualifier": {"classid": "id", "classname": "id"}, "value": "987654321"}], "rank": 7, "affiliation": [], "fullname": "Jansen, Olav"}, {"surname": "Dressler", "name": "Dirk", "pid": [], "rank": 8, "affiliation": [], "fullname": "Dressler, Dirk"}, {"surname": "Schneider", "name": "Susanne A.", "pid": [], "rank": 9, "affiliation": [], "fullname": "Schneider, Susanne A."}], "source": [], "dateofcollection": "2020-03-10T15:05:38.685Z", "fulltext": [], "dateoftransformation": "2020-03-11T20:11:13.15Z", "description": [], "format": [], "journal": {"issnPrinted": "", "conferencedate": "", "conferenceplace": "", "name": "", "edition": "", "iss": "", "sp": "", "vol": "", "dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "issnOnline": "", "ep": "", "issnLinking": ""}, "coverage": [], "publisher": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "language": {"classid": "en", "classname": "en", "schemename": "dnet:languages", "schemeid": "dnet:languages"}, "bestaccessright": {"classid": "UNKNOWN", "classname": "not available", "schemename": "dnet:access_modes", "schemeid": "dnet:access_modes"}, "country": [{"classid": "FI", "classname": "Finland", "schemeid": "dnet:countries", "schemename": "dnet:countries"}], "extraInfo": [], "originalId": ["(BISIS)113444", "https://www.cris.uns.ac.rs/record.jsf?recordId=113444&source=OpenAIRE&language=en"], "dateofacceptance": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "2016-01-01"}, "title": [{"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "qualifier": {"classid": "test title", "classname": "test title", "schemename": "dnet:dataCite_title", "schemeid": "dnet:dataCite_title"}, "value": "Antichains of copies of ultrahomogeneous structures"}]} \ No newline at end of file diff --git a/dhp-workflows/dhp-dedup-openaire/src/test/resources/eu/dnetlib/dhp/dedup/json/software_merge.json b/dhp-workflows/dhp-dedup-openaire/src/test/resources/eu/dnetlib/dhp/dedup/json/software_merge.json new file mode 100644 index 0000000000..b146d61022 --- /dev/null +++ b/dhp-workflows/dhp-dedup-openaire/src/test/resources/eu/dnetlib/dhp/dedup/json/software_merge.json @@ -0,0 +1,3 @@ +{"context": [], "dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "sysimport:crosswalk:datasetarchive", "classname": "sysimport:crosswalk:datasetarchive", "schemename": "dnet:provenanceActions", "schemeid": "dnet:provenanceActions"}, "inferred": false, "inferenceprovenance": "dedup-similarity-result-levenstein", "invisible": false, "trust": "0.95"}, "resourcetype": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "pid": [], "contributor": [], "resulttype": {"classid": "software", "classname": "software", "schemename": "dnet:result_typologies", "schemeid": "dnet:result_typologies"}, "relevantdate": [], "collectedfrom": [{"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "Journal.fi", "key": "10|openaire____::6eef8049d0feedc089ee009abca55e35"}], "id": "50|a89337edbe55::4930db9e954866d70916cbfba9f81f97", "subject": [], "instance": [{"refereed": null, "hostedby": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "Journal.fi", "key": "10|openaire____::6eef8049d0feedc089ee009abca55e35"}, "processingchargeamount": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "license": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "url": [], "distributionlocation": "", "processingchargecurrency": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "dateofacceptance": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "2016-01-01"}, "collectedfrom": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "Journal.fi", "key": "10|openaire____::6eef8049d0feedc089ee009abca55e35"}, "accessright": {"classid": "OPEN", "classname": "Open Access", "schemename": "dnet:access_modes", "schemeid": "dnet:access_modes"}, "instancetype": {"classid": "0001", "classname": "Article", "schemename": "dnet:dataCite_resource", "schemeid": "dnet:dataCite_resource"}}], "embargoenddate": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "lastupdatetimestamp": 0, "author": [{"surname": "Go\\u0308tz", "name": "Julia", "pid": [], "rank": 5, "affiliation": [], "fullname": "G\\u00f6tz, Julia"}, {"surname": "Wolff", "name": "Stephan", "pid": [], "rank": 6, "affiliation": [], "fullname": "Wolff, Stephan"}, {"surname": "Jansen", "name": "Olav", "pid": [], "rank": 7, "affiliation": [], "fullname": "Jansen, Olav"}, {"surname": "Dressler", "name": "Dirk", "pid": [{"qualifier": {"classid": "ORCID", "classname": "ORCID"}, "value": "0000-0000-0656-9999"},{"qualifier": {"classid": "id", "classname": "id"}, "value": "987654321"}], "rank": 8, "affiliation": [], "fullname": "Dressler, Dirk"}, {"surname": "Schneider", "name": "Susanne A.", "pid": [], "rank": 9, "affiliation": [], "fullname": "Schneider, Susanne A."}], "source": [], "dateofcollection": "2019-11-05T14:49:22.351Z", "fulltext": [], "dateoftransformation": "2019-11-05T16:10:58.988Z", "description": [], "format": [], "coverage": [], "publisher": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "language": {"classid": "eng", "classname": "English", "schemename": "dnet:languages", "schemeid": "dnet:languages"}, "bestaccessright": {"classid": "OPEN SOURCE", "classname": "Open Source", "schemename": "dnet:access_modes", "schemeid": "dnet:access_modes"}, "country": [], "extraInfo": [], "originalId": [], "dateofacceptance": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "2018-09-30"}, "title": [{"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "qualifier": {"classid": "main title", "classname": "main title", "schemename": "dnet:dataCite_title", "schemeid": "dnet:dataCite_title"}, "value": "Altered brain activation in a reversal learning task unmasks adaptive changes in cognitive control in writer's cramp"}]} +{"context": [], "dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "sysimport:crosswalk:repository", "classname": "sysimport:crosswalk:repository", "schemename": "dnet:provenanceActions", "schemeid": "dnet:provenanceActions"}, "inferred": true, "inferenceprovenance": "dedup-similarity-result-levenstein", "invisible": false, "trust": "0.9"}, "resourcetype": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "pid": [{"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "qualifier": {"classid": "doi", "classname": "doi", "schemename": "dnet:pid_types", "schemeid": "dnet:pid_types"}, "value": "10.1016/j.nicl.2015.11.006"}], "contributor": [], "resulttype": {"classid": "software", "classname": "software", "schemename": "dnet:result_typologies", "schemeid": "dnet:result_typologies"}, "relevantdate": [], "collectedfrom": [{"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "BASE (Open Access Aggregator)", "key": "10|openaire____::df45502607927471ecf8a6ae83683ff5"}], "id": "50|base_oa_____::0968af610a356656706657e4f234b340", "subject": [], "instance": [{"refereed": null, "hostedby": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "NeuroImage: Clinical", "key": "10|doajarticles::0c0e74daa5d95504eade9c81ebbd5b8a"}, "processingchargeamount": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "license": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "http://creativecommons.org/licenses/by-nc-nd/4.0/"}, "url": ["http://dx.doi.org/10.1016/j.nicl.2015.11.006"], "distributionlocation": "", "processingchargecurrency": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "dateofacceptance": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "2016-01-01"}, "collectedfrom": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "BASE (Open Access Aggregator)", "key": "10|openaire____::df45502607927471ecf8a6ae83683ff5"}, "accessright": {"classid": "OPEN", "classname": "Open Access", "schemename": "dnet:access_modes", "schemeid": "dnet:access_modes"}, "instancetype": {"classid": "0001", "classname": "Article", "schemename": "dnet:publication_resource", "schemeid": "dnet:publication_resource"}}], "embargoenddate": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "lastupdatetimestamp": 0, "author": [{"surname": "Zeuner", "name": "Kirsten E.", "pid": [], "rank": 1, "affiliation": [], "fullname": "Zeuner, Kirsten E."}, {"surname": "Knutzen", "name": "Arne", "pid": [], "rank": 2, "affiliation": [], "fullname": "Knutzen, Arne"}, {"surname": "Granert", "name": "Oliver", "pid": [{"qualifier": {"classid": "ORCID", "classname": "ORCID"}, "value": "0000-0002-0656-1023"}], "rank": 3, "affiliation": [], "fullname": "Granert, Oliver"}, {"surname": "Sablowsky", "name": "Simone", "pid": [], "rank": 4, "affiliation": [], "fullname": "Sablowsky, Simone"}, {"surname": "Go\\u0308tz", "name": "Julia", "pid": [], "rank": 5, "affiliation": [], "fullname": "G\\u00f6tz, Julia"}, {"surname": "Wolff", "name": "Stephan", "pid": [], "rank": 6, "affiliation": [], "fullname": "Wolff, Stephan"}, {"surname": "Jansen", "name": "Olav", "pid": [], "rank": 7, "affiliation": [], "fullname": "Jansen, Olav"}, {"surname": "Dressler", "name": "Dirk", "pid": [], "rank": 8, "affiliation": [], "fullname": "Dressler, Dirk"}, {"surname": "Schneider", "name": "Susanne A.", "pid": [], "rank": 9, "affiliation": [], "fullname": "Schneider, Susanne A."}, {"surname": "Klein", "name": "Christine", "pid": [], "rank": 10, "affiliation": [], "fullname": "Klein, Christine"}, {"surname": "Deuschl", "name": "Gu\\u0308nther", "pid": [], "rank": 11, "affiliation": [], "fullname": "Deuschl, G\\u00fcnther"}, {"surname": "Eimeren", "name": "Thilo", "pid": [], "rank": 12, "affiliation": [], "fullname": "van Eimeren, Thilo"}, {"surname": "Witt", "name": "Karsten", "pid": [], "rank": 13, "affiliation": [], "fullname": "Witt, Karsten"}], "source": [], "dateofcollection": "2017-07-27T19:04:09.131Z", "fulltext": [], "dateoftransformation": "2019-01-23T10:15:19.582Z", "description": [], "format": [], "coverage": [], "publisher": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "Elsevier BV"}, "language": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "bestaccessright": {"classid": "OPEN SOURCE", "classname": "Open Source", "schemename": "dnet:access_modes", "schemeid": "dnet:access_modes"}, "country": [{"classid": "IT", "classname": "Italy", "schemeid": "dnet:countries", "schemename": "dnet:countries"}], "extraInfo": [], "originalId": ["10.1016/j.nicl.2015.11.006"], "dateofacceptance": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "2016-01-01"}, "title": [{"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "qualifier": {"classid": "main title", "classname": "main title", "schemename": "dnet:dataCite_title", "schemeid": "dnet:dataCite_title"}, "value": "Altered brain activation in a reversal learning task unmasks adaptive changes in cognitive control in writer's cramp"}]} +{"context": [], "dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "sysimport:crosswalk:datasetarchive", "classname": "sysimport:crosswalk:datasetarchive", "schemename": "dnet:provenanceActions", "schemeid": "dnet:provenanceActions"}, "inferred": true, "inferenceprovenance": "dedup-similarity-result-levenstein", "invisible": false, "trust": "0.9"}, "resourcetype": {"classid": "0004", "classname": "Conference object", "schemename": "dnet:dataCite_resource", "schemeid": "dnet:dataCite_resource"}, "pid": [], "contributor": [], "resulttype": {"classid": "software", "classname": "software", "schemename": "dnet:result_typologies", "schemeid": "dnet:result_typologies"}, "relevantdate": [], "collectedfrom": [{"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "CRIS UNS (Current Research Information System University of Novi Sad)", "key": "10|CRIS_UNS____::f66f1bd369679b5b077dcdf006089556"}], "id": "50|CrisUnsNoviS::9f9d014eea45dab432cab636c4c9cf39", "subject": [], "instance": [{"refereed": null, "hostedby": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "CRIS UNS (Current Research Information System University of Novi Sad)", "key": "10|CRIS_UNS____::f66f1bd369679b5b077dcdf006089556"}, "processingchargeamount": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "license": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "url": ["https://www.cris.uns.ac.rs/record.jsf?recordId=113444&source=OpenAIRE&language=en"], "distributionlocation": "", "processingchargecurrency": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "dateofacceptance": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "2019-01-01"}, "collectedfrom": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "CRIS UNS (Current Research Information System University of Novi Sad)", "key": "10|CRIS_UNS____::f66f1bd369679b5b077dcdf006089556"}, "accessright": {"classid": "UNKNOWN", "classname": "UNKNOWN", "schemename": "dnet:access_modes", "schemeid": "dnet:access_modes"}, "instancetype": {"classid": "0004", "classname": "Conference object", "schemename": "dnet:dataCite_resource", "schemeid": "dnet:dataCite_resource"}}], "embargoenddate": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "lastupdatetimestamp": 0, "author": [{"surname": "Zeuner", "name": "Kirsten E.", "pid": [], "rank": 1, "affiliation": [], "fullname": "Zeuner, Kirsten E."}, {"surname": "Knutzen", "name": "Arne", "pid": [], "rank": 2, "affiliation": [], "fullname": "Knutzen, Arne"}, {"surname": "Granert", "name": "Oliver", "pid": [{"qualifier": {"classid": "ORCID", "classname": "ORCID"}, "value": "0000-0002-0656-1023"}, {"qualifier": {"classid": "pubmed", "classname": "pubmed"}, "value": "pubmed.it"}], "rank": 3, "affiliation": [], "fullname": "Granert, Oliver"}, {"surname": "Sablowsky", "name": "Simone", "pid": [{"qualifier": {"classid": "id", "classname": "id"}, "value": "12345678"}], "rank": 4, "affiliation": [], "fullname": "Sablowsky, Simone"}, {"surname": "Go\\u0308tz", "name": "Julia", "pid": [], "rank": 5, "affiliation": [], "fullname": "G\\u00f6tz, Julia"}, {"surname": "Wolff", "name": "Stephan", "pid": [], "rank": 6, "affiliation": [], "fullname": "Wolff, Stephan"}, {"surname": "Jansen", "name": "Olav", "pid": [{"qualifier": {"classid": "ORCID", "classname": "ORCID"}, "value": "0000-0000-0656-1023"},{"qualifier": {"classid": "id", "classname": "id"}, "value": "987654321"}], "rank": 7, "affiliation": [], "fullname": "Jansen, Olav"}, {"surname": "Dressler", "name": "Dirk", "pid": [], "rank": 8, "affiliation": [], "fullname": "Dressler, Dirk"}, {"surname": "Schneider", "name": "Susanne A.", "pid": [], "rank": 9, "affiliation": [], "fullname": "Schneider, Susanne A."}], "source": [], "dateofcollection": "2020-03-10T15:05:38.685Z", "fulltext": [], "dateoftransformation": "2020-03-11T20:11:13.15Z", "description": [], "format": [], "coverage": [], "publisher": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "language": {"classid": "en", "classname": "en", "schemename": "dnet:languages", "schemeid": "dnet:languages"}, "bestaccessright": {"classid": "UNKNOWN", "classname": "unknown", "schemename": "dnet:access_modes", "schemeid": "dnet:access_modes"}, "country": [{"classid": "FI", "classname": "Finland", "schemeid": "dnet:countries", "schemename": "dnet:countries"}], "extraInfo": [], "originalId": ["(BISIS)113444", "https://www.cris.uns.ac.rs/record.jsf?recordId=113444&source=OpenAIRE&language=en"], "dateofacceptance": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "2016-01-01"}, "title": [{"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "qualifier": {"classid": "test title", "classname": "test title", "schemename": "dnet:dataCite_title", "schemeid": "dnet:dataCite_title"}, "value": "Antichains of copies of ultrahomogeneous structures"}]} \ No newline at end of file diff --git a/dhp-workflows/dhp-dedup-scholexplorer/pom.xml b/dhp-workflows/dhp-dedup-scholexplorer/pom.xml index 429c8a6486..aa4070b010 100644 --- a/dhp-workflows/dhp-dedup-scholexplorer/pom.xml +++ b/dhp-workflows/dhp-dedup-scholexplorer/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.2-SNAPSHOT + 1.2.4-SNAPSHOT 4.0.0 diff --git a/dhp-workflows/dhp-distcp/pom.xml b/dhp-workflows/dhp-distcp/pom.xml index 8454c29a4d..8c10538c01 100644 --- a/dhp-workflows/dhp-distcp/pom.xml +++ b/dhp-workflows/dhp-distcp/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.2-SNAPSHOT + 1.2.4-SNAPSHOT 4.0.0 diff --git a/dhp-workflows/dhp-doiboost/pom.xml b/dhp-workflows/dhp-doiboost/pom.xml new file mode 100644 index 0000000000..3299c1496c --- /dev/null +++ b/dhp-workflows/dhp-doiboost/pom.xml @@ -0,0 +1,92 @@ + + + + dhp-workflows + eu.dnetlib.dhp + 1.2.4-SNAPSHOT + + 4.0.0 + + dhp-doiboost + + + + + net.alchim31.maven + scala-maven-plugin + 4.0.1 + + + scala-compile-first + initialize + + add-source + compile + + + + scala-test-compile + process-test-resources + + testCompile + + + + + ${scala.version} + + + + + + + + + + + + org.apache.hadoop + hadoop-client + + + org.apache.httpcomponents + httpclient + 4.3.4 + + + eu.dnetlib.dhp + dhp-common + ${project.version} + + + org.apache.cxf + cxf-rt-transports-http + + + + + eu.dnetlib.dhp + dhp-schemas + ${project.version} + + + com.jayway.jsonpath + json-path + + + + org.apache.spark + spark-core_2.11 + + + + org.apache.spark + spark-sql_2.11 + + + + + + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/DoiBoostMappingUtil.scala b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/DoiBoostMappingUtil.scala new file mode 100644 index 0000000000..1a45defb06 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/DoiBoostMappingUtil.scala @@ -0,0 +1,388 @@ +package eu.dnetlib.doiboost + +import eu.dnetlib.dhp.schema.action.AtomicAction +import eu.dnetlib.dhp.schema.oaf.{DataInfo, Dataset, Field, Instance, KeyValue, Oaf, Organization, Publication, Qualifier, Relation, Result, StructuredProperty} +import eu.dnetlib.dhp.utils.DHPUtils +import org.apache.commons.lang3.StringUtils +import org.codehaus.jackson.map.ObjectMapper +import org.json4s +import org.json4s.DefaultFormats +import org.json4s.jackson.JsonMethods.parse +import org.slf4j.{Logger, LoggerFactory} + +import scala.collection.JavaConverters._ +import scala.io.Source + + +case class HostedByItemType(id: String, officialname: String, issn: String, eissn: String, lissn: String, openAccess: Boolean) {} + +case class DoiBoostAffiliation(PaperId:Long, AffiliationId:Long, GridId:Option[String], OfficialPage:Option[String], DisplayName:Option[String]){} + +object DoiBoostMappingUtil { + def getUnknownCountry(): Qualifier = { + createQualifier("UNKNOWN","UNKNOWN","dnet:countries","dnet:countries") + } + + + + def generateMAGAffiliationId(affId: String): String = { + s"20|microsoft___$SEPARATOR${DHPUtils.md5(affId)}" + } + + + val logger: Logger = LoggerFactory.getLogger(getClass) + + //STATIC STRING + val MAG = "microsoft" + val MAG_NAME = "Microsoft Academic Graph" + val ORCID = "ORCID" + val CROSSREF = "Crossref" + val UNPAYWALL = "UnpayWall" + val GRID_AC = "grid.ac" + val WIKPEDIA = "wikpedia" + val doiBoostNSPREFIX = "doiboost____" + val OPENAIRE_PREFIX = "openaire____" + val SEPARATOR = "::" + val DNET_LANGUAGES = "dnet:languages" + val PID_TYPES = "dnet:pid_types" + + val invalidName = List(",", "none none", "none, none", "none &na;", "(:null)", "test test test", "test test", "test", "&na; &na;") + + def toActionSet(item:Oaf) :(String, String) = { + val mapper = new ObjectMapper() + + item match { + case dataset: Dataset => + val a: AtomicAction[Dataset] = new AtomicAction[Dataset] + a.setClazz(classOf[Dataset]) + a.setPayload(dataset) + (dataset.getClass.getCanonicalName, mapper.writeValueAsString(a)) + case publication: Publication => + val a: AtomicAction[Publication] = new AtomicAction[Publication] + a.setClazz(classOf[Publication]) + a.setPayload(publication) + (publication.getClass.getCanonicalName, mapper.writeValueAsString(a)) + case organization: Organization => + val a: AtomicAction[Organization] = new AtomicAction[Organization] + a.setClazz(classOf[Organization]) + a.setPayload(organization) + (organization.getClass.getCanonicalName, mapper.writeValueAsString(a)) + case relation: Relation => + val a: AtomicAction[Relation] = new AtomicAction[Relation] + a.setClazz(classOf[Relation]) + a.setPayload(relation) + (relation.getClass.getCanonicalName, mapper.writeValueAsString(a)) + case _ => + null + } + + } + + + def toHostedByItem(input:String): (String, HostedByItemType) = { + implicit lazy val formats: DefaultFormats.type = org.json4s.DefaultFormats + + lazy val json: json4s.JValue = parse(input) + val c :Map[String,HostedByItemType] = json.extract[Map[String, HostedByItemType]] + (c.keys.head, c.values.head) + } + + + def toISSNPair(publication: Publication) : (String, Publication) = { + val issn = if (publication.getJournal == null) null else publication.getJournal.getIssnPrinted + val eissn =if (publication.getJournal == null) null else publication.getJournal.getIssnOnline + val lissn =if (publication.getJournal == null) null else publication.getJournal.getIssnLinking + + if (issn!= null && issn.nonEmpty) + (issn, publication) + else if(eissn!= null && eissn.nonEmpty) + (eissn, publication) + else if(lissn!= null && lissn.nonEmpty) + (lissn, publication) + else + (publication.getId, publication) + } + + + + + def generateGridAffiliationId(gridId:String) :String = { + s"20|grid________::${DHPUtils.md5(gridId.toLowerCase().trim())}" + } + + + def fixResult(result: Dataset) :Dataset = { + val instanceType = result.getInstance().asScala.find(i => i.getInstancetype != null && i.getInstancetype.getClassid.nonEmpty) + if (instanceType.isDefined) { + result.getInstance().asScala.foreach(i => i.setInstancetype(instanceType.get.getInstancetype)) + } + result.getInstance().asScala.foreach(i => { + i.setHostedby(getUbknownHostedBy()) + }) + result + } + + def getUbknownHostedBy():KeyValue = { + val hb = new KeyValue + hb.setValue("Unknown Repository") + hb.setKey(s"10|$OPENAIRE_PREFIX::55045bd2a65019fd8e6741a755395c8c") + hb + + } + + + def getOpenAccessQualifier():Qualifier = { + createQualifier("OPEN","Open Access","dnet:access_modes", "dnet:access_modes") + + } + + def getRestrictedQualifier():Qualifier = { + createQualifier("RESTRICTED","Restricted","dnet:access_modes", "dnet:access_modes") + + } + + def fixPublication(input:((String,Publication), (String,HostedByItemType))): Publication = { + + val publication = input._1._2 + + val item = if (input._2 != null) input._2._2 else null + + + val instanceType = publication.getInstance().asScala.find(i => i.getInstancetype != null && i.getInstancetype.getClassid.nonEmpty) + + if (instanceType.isDefined) { + publication.getInstance().asScala.foreach(i => i.setInstancetype(instanceType.get.getInstancetype)) + } + + + publication.getInstance().asScala.foreach(i => { + val hb = new KeyValue + if (item != null) { + hb.setValue(item.officialname) + hb.setKey(generateDSId(item.id)) + if (item.openAccess) + i.setAccessright(getOpenAccessQualifier()) + publication.setBestaccessright(getOpenAccessQualifier()) + } + else { + hb.setValue("Unknown Repository") + hb.setKey(s"10|$OPENAIRE_PREFIX::55045bd2a65019fd8e6741a755395c8c") + } + i.setHostedby(hb) + }) + + val ar = publication.getInstance().asScala.filter(i => i.getInstancetype != null && i.getAccessright!= null && i.getAccessright.getClassid!= null).map(f=> f.getAccessright.getClassid) + if (ar.nonEmpty) { + if(ar.contains("OPEN")){ + publication.setBestaccessright(getOpenAccessQualifier()) + } + else { + publication.setBestaccessright(getRestrictedQualifier()) + } + } + publication + } + + + def generateDSId(input: String): String = { + + val b = StringUtils.substringBefore(input, "::") + val a = StringUtils.substringAfter(input, "::") + s"10|${b}::${DHPUtils.md5(a)}" + } + + + def generateDataInfo(): DataInfo = { + generateDataInfo("0.9") + } + + + def filterPublication(publication: Publication): Boolean = { + + //Case empty publication + if (publication == null) + return false + + //Case publication with no title + if (publication.getTitle == null || publication.getTitle.size == 0) + return false + + + val s = publication.getTitle.asScala.count(p => p.getValue != null + && p.getValue.nonEmpty && !p.getValue.equalsIgnoreCase("[NO TITLE AVAILABLE]")) + + if (s == 0) + return false + + // fixes #4360 (test publisher) + val publisher = if (publication.getPublisher != null) publication.getPublisher.getValue else null + + if (publisher != null && (publisher.equalsIgnoreCase("Test accounts") || publisher.equalsIgnoreCase("CrossRef Test Account"))) { + return false; + } + + //Publication with no Author + if (publication.getAuthor == null || publication.getAuthor.size() == 0) + return false + + + //filter invalid author + val authors = publication.getAuthor.asScala.map(s => { + if (s.getFullname.nonEmpty) { + s.getFullname + } + else + s"${ + s.getName + } ${ + s.getSurname + }" + }) + + val c = authors.count(isValidAuthorName) + if (c == 0) + return false + + // fixes #4368 + if (authors.count(s => s.equalsIgnoreCase("Addie Jackson")) > 0 && "Elsevier BV".equalsIgnoreCase(publication.getPublisher.getValue)) + return false + + true + } + + + def isValidAuthorName(fullName: String): Boolean = { + if (fullName == null || fullName.isEmpty) + return false + if (invalidName.contains(fullName.toLowerCase.trim)) + return false + true + } + + + def generateDataInfo(trust: String): DataInfo = { + val di = new DataInfo + di.setDeletedbyinference(false) + di.setInferred(false) + di.setInvisible(false) + di.setTrust(trust) + di.setProvenanceaction(createQualifier("sysimport:actionset", "dnet:provenanceActions")) + di + } + + + + def createSP(value: String, classId: String,className:String, schemeId: String, schemeName:String): StructuredProperty = { + val sp = new StructuredProperty + sp.setQualifier(createQualifier(classId,className, schemeId, schemeName)) + sp.setValue(value) + sp + + } + + + + def createSP(value: String, classId: String,className:String, schemeId: String, schemeName:String, dataInfo: DataInfo): StructuredProperty = { + val sp = new StructuredProperty + sp.setQualifier(createQualifier(classId,className, schemeId, schemeName)) + sp.setValue(value) + sp.setDataInfo(dataInfo) + sp + + } + + def createSP(value: String, classId: String, schemeId: String): StructuredProperty = { + val sp = new StructuredProperty + sp.setQualifier(createQualifier(classId, schemeId)) + sp.setValue(value) + sp + + } + + + + def createSP(value: String, classId: String, schemeId: String, dataInfo: DataInfo): StructuredProperty = { + val sp = new StructuredProperty + sp.setQualifier(createQualifier(classId, schemeId)) + sp.setValue(value) + sp.setDataInfo(dataInfo) + sp + + } + + def createCrossrefCollectedFrom(): KeyValue = { + + val cf = new KeyValue + cf.setValue(CROSSREF) + cf.setKey("10|" + OPENAIRE_PREFIX + SEPARATOR + DHPUtils.md5(CROSSREF.toLowerCase)) + cf + + } + + + def createUnpayWallCollectedFrom(): KeyValue = { + + val cf = new KeyValue + cf.setValue(UNPAYWALL) + cf.setKey("10|" + OPENAIRE_PREFIX + SEPARATOR + DHPUtils.md5(UNPAYWALL.toLowerCase)) + cf + + } + + def createORIDCollectedFrom(): KeyValue = { + + val cf = new KeyValue + cf.setValue(ORCID) + cf.setKey("10|" + OPENAIRE_PREFIX + SEPARATOR + DHPUtils.md5(ORCID.toLowerCase)) + cf + + } + + + def generateIdentifier (oaf: Result, doi: String): String = { + val id = DHPUtils.md5 (doi.toLowerCase) + return s"50|${ + doiBoostNSPREFIX + }${ + SEPARATOR + }${ + id + }" + } + + + + + def createMAGCollectedFrom(): KeyValue = { + + val cf = new KeyValue + cf.setValue(MAG_NAME) + cf.setKey("10|" + OPENAIRE_PREFIX + SEPARATOR + DHPUtils.md5(MAG)) + cf + + } + + def createQualifier(clsName: String, clsValue: String, schName: String, schValue: String): Qualifier = { + val q = new Qualifier + q.setClassid(clsName) + q.setClassname(clsValue) + q.setSchemeid(schName) + q.setSchemename(schValue) + q + } + + def createQualifier(cls: String, sch: String): Qualifier = { + createQualifier(cls, cls, sch, sch) + } + + + def asField[T](value: T): Field[T] = { + val tmp = new Field[T] + tmp.setValue(value) + tmp + + + } + + +} diff --git a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/SparkGenerateDOIBoostActionSet.scala b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/SparkGenerateDOIBoostActionSet.scala new file mode 100644 index 0000000000..7a6cd3faaa --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/SparkGenerateDOIBoostActionSet.scala @@ -0,0 +1,80 @@ +package eu.dnetlib.doiboost + +import eu.dnetlib.dhp.application.ArgumentApplicationParser +import eu.dnetlib.dhp.schema.action.AtomicAction +import eu.dnetlib.dhp.schema.oaf.{Organization, Publication, Relation, Dataset => OafDataset} +import org.apache.commons.io.IOUtils +import org.apache.hadoop.io.Text +import org.apache.hadoop.io.compress.GzipCodec +import org.apache.hadoop.mapred.SequenceFileOutputFormat +import org.apache.spark.SparkConf +import org.apache.spark.sql.{Dataset, Encoder, Encoders, SaveMode, SparkSession} +import org.slf4j.{Logger, LoggerFactory} + +object SparkGenerateDOIBoostActionSet { + val logger: Logger = LoggerFactory.getLogger(getClass) + def main(args: Array[String]): Unit = { + + val conf: SparkConf = new SparkConf() + val parser = new ArgumentApplicationParser(IOUtils.toString(getClass.getResourceAsStream("/eu/dnetlib/dhp/doiboost/generate_doiboost_as_params.json"))) + parser.parseArgument(args) + val spark: SparkSession = + SparkSession + .builder() + .config(conf) + .appName(getClass.getSimpleName) + .master(parser.get("master")).getOrCreate() + + implicit val mapEncoderPub: Encoder[Publication] = Encoders.kryo[Publication] + implicit val mapEncoderOrg: Encoder[Organization] = Encoders.kryo[Organization] + implicit val mapEncoderDataset: Encoder[OafDataset] = Encoders.kryo[OafDataset] + implicit val mapEncoderRel: Encoder[Relation] = Encoders.kryo[Relation] + implicit val mapEncoderAS: Encoder[(String, String)] = Encoders.tuple(Encoders.STRING, Encoders.STRING) + + implicit val mapEncoderAtomiAction: Encoder[AtomicAction[OafDataset]] = Encoders.kryo[AtomicAction[OafDataset]] + + val dbPublicationPath = parser.get("dbPublicationPath") + val dbDatasetPath = parser.get("dbDatasetPath") + val crossRefRelation = parser.get("crossRefRelation") + val dbaffiliationRelationPath = parser.get("dbaffiliationRelationPath") + val dbOrganizationPath = parser.get("dbOrganizationPath") + val workingDirPath = parser.get("targetPath") + + spark.read.load(dbDatasetPath).as[OafDataset] + .map(d =>DoiBoostMappingUtil.fixResult(d)) + .map(d=>DoiBoostMappingUtil.toActionSet(d))(Encoders.tuple(Encoders.STRING, Encoders.STRING)) + .write.mode(SaveMode.Overwrite).save(s"$workingDirPath/actionSet") + + spark.read.load(dbPublicationPath).as[Publication] + .map(d=>DoiBoostMappingUtil.toActionSet(d))(Encoders.tuple(Encoders.STRING, Encoders.STRING)) + .write.mode(SaveMode.Append).save(s"$workingDirPath/actionSet") + + spark.read.load(dbOrganizationPath).as[Organization] + .map(d=>DoiBoostMappingUtil.toActionSet(d))(Encoders.tuple(Encoders.STRING, Encoders.STRING)) + .write.mode(SaveMode.Append).save(s"$workingDirPath/actionSet") + + + spark.read.load(crossRefRelation).as[Relation] + .map(d=>DoiBoostMappingUtil.toActionSet(d))(Encoders.tuple(Encoders.STRING, Encoders.STRING)) + .write.mode(SaveMode.Append).save(s"$workingDirPath/actionSet") + + spark.read.load(dbaffiliationRelationPath).as[Relation] + .map(d=>DoiBoostMappingUtil.toActionSet(d))(Encoders.tuple(Encoders.STRING, Encoders.STRING)) + .write.mode(SaveMode.Append).save(s"$workingDirPath/actionSet") + + + val d: Dataset[(String, String)] =spark.read.load(s"$workingDirPath/actionSet").as[(String,String)] + + d.rdd.map(s => (new Text(s._1), new Text(s._2))).saveAsHadoopFile(s"$workingDirPath/rawset", classOf[Text], classOf[Text], classOf[SequenceFileOutputFormat[Text,Text]], classOf[GzipCodec]) + + + + + + + + + + } + +} diff --git a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/SparkGenerateDoiBoost.scala b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/SparkGenerateDoiBoost.scala new file mode 100644 index 0000000000..772af010fd --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/SparkGenerateDoiBoost.scala @@ -0,0 +1,140 @@ +package eu.dnetlib.doiboost + +import eu.dnetlib.dhp.application.ArgumentApplicationParser +import eu.dnetlib.dhp.schema.oaf.{Publication, Relation, Dataset => OafDataset, Organization} +import eu.dnetlib.doiboost.mag.ConversionUtil +import org.apache.commons.io.IOUtils +import org.apache.spark.SparkConf +import org.apache.spark.sql.functions.col +import org.apache.spark.sql.{Dataset, Encoder, Encoders, SaveMode, SparkSession} +import org.slf4j.{Logger, LoggerFactory} +import scala.collection.JavaConverters._ + +object SparkGenerateDoiBoost { + + def main(args: Array[String]): Unit = { + + val logger: Logger = LoggerFactory.getLogger(getClass) + val conf: SparkConf = new SparkConf() + val parser = new ArgumentApplicationParser(IOUtils.toString(getClass.getResourceAsStream("/eu/dnetlib/dhp/doiboost/generate_doiboost_params.json"))) + parser.parseArgument(args) + val spark: SparkSession = + SparkSession + .builder() + .config(conf) + .appName(getClass.getSimpleName) + .master(parser.get("master")).getOrCreate() + + import spark.implicits._ + + val hostedByMapPath = parser.get("hostedByMapPath") + val workingDirPath = parser.get("workingDirPath") + + + implicit val mapEncoderPub: Encoder[Publication] = Encoders.kryo[Publication] + implicit val mapEncoderOrg: Encoder[Organization] = Encoders.kryo[Organization] + implicit val mapEncoderDataset: Encoder[OafDataset] = Encoders.kryo[OafDataset] + implicit val tupleForJoinEncoder: Encoder[(String, Publication)] = Encoders.tuple(Encoders.STRING, mapEncoderPub) + implicit val mapEncoderRel: Encoder[Relation] = Encoders.kryo[Relation] + + logger.info("Phase 2) Join Crossref with UnpayWall") + + val crossrefPublication: Dataset[(String, Publication)] = spark.read.load(s"$workingDirPath/crossrefPublication").as[Publication].map(p => (p.getId, p)) + val uwPublication: Dataset[(String, Publication)] = spark.read.load(s"$workingDirPath/uwPublication").as[Publication].map(p => (p.getId, p)) + + def applyMerge(item:((String, Publication), (String, Publication))) : Publication = + { + val crossrefPub = item._1._2 + if (item._2!= null) { + val otherPub = item._2._2 + if (otherPub != null) { + crossrefPub.mergeFrom(otherPub) + } + } + crossrefPub + } + crossrefPublication.joinWith(uwPublication, crossrefPublication("_1").equalTo(uwPublication("_1")), "left").map(applyMerge).write.mode(SaveMode.Overwrite).save(s"$workingDirPath/firstJoin") + logger.info("Phase 3) Join Result with ORCID") + val fj: Dataset[(String, Publication)] = spark.read.load(s"$workingDirPath/firstJoin").as[Publication].map(p => (p.getId, p)) + val orcidPublication: Dataset[(String, Publication)] = spark.read.load(s"$workingDirPath/orcidPublication").as[Publication].map(p => (p.getId, p)) + fj.joinWith(orcidPublication, fj("_1").equalTo(orcidPublication("_1")), "left").map(applyMerge).write.mode(SaveMode.Overwrite).save(s"$workingDirPath/secondJoin") + + logger.info("Phase 3) Join Result with MAG") + val sj: Dataset[(String, Publication)] = spark.read.load(s"$workingDirPath/secondJoin").as[Publication].map(p => (p.getId, p)) + + val magPublication: Dataset[(String, Publication)] = spark.read.load(s"$workingDirPath/magPublication").as[Publication].map(p => (p.getId, p)) + sj.joinWith(magPublication, sj("_1").equalTo(magPublication("_1")), "left").map(applyMerge).write.mode(SaveMode.Overwrite).save(s"$workingDirPath/doiBoostPublication") + + + val doiBoostPublication: Dataset[(String,Publication)] = spark.read.load(s"$workingDirPath/doiBoostPublication").as[Publication].filter(p=>DoiBoostMappingUtil.filterPublication(p)).map(DoiBoostMappingUtil.toISSNPair)(tupleForJoinEncoder) + + val hostedByDataset : Dataset[(String, HostedByItemType)] = spark.createDataset(spark.sparkContext.textFile(hostedByMapPath).map(DoiBoostMappingUtil.toHostedByItem)) + + + doiBoostPublication.joinWith(hostedByDataset, doiBoostPublication("_1").equalTo(hostedByDataset("_1")), "left") + .map(DoiBoostMappingUtil.fixPublication) + .write.mode(SaveMode.Overwrite).save(s"$workingDirPath/doiBoostPublicationFiltered") + + val affiliationPath = parser.get("affiliationPath") + val paperAffiliationPath = parser.get("paperAffiliationPath") + + val affiliation = spark.read.load(affiliationPath).select(col("AffiliationId"), col("GridId"), col("OfficialPage"), col("DisplayName")) + + val paperAffiliation = spark.read.load(paperAffiliationPath).select(col("AffiliationId").alias("affId"), col("PaperId")) + + + val a:Dataset[DoiBoostAffiliation] = paperAffiliation + .joinWith(affiliation, paperAffiliation("affId").equalTo(affiliation("AffiliationId"))) + .select(col("_1.PaperId"), col("_2.AffiliationId"), col("_2.GridId"), col("_2.OfficialPage"), col("_2.DisplayName")).as[DoiBoostAffiliation] + + + + val magPubs:Dataset[(String,Publication)]= spark.read.load(s"$workingDirPath/doiBoostPublicationFiltered").as[Publication] + .map(p => (ConversionUtil.extractMagIdentifier(p.getOriginalId.asScala), p))(tupleForJoinEncoder).filter(s =>s._1!= null ) + + + magPubs.joinWith(a,magPubs("_1").equalTo(a("PaperId"))).flatMap(item => { + val pub:Publication = item._1._2 + val affiliation = item._2 + val affId:String = if (affiliation.GridId.isDefined) DoiBoostMappingUtil.generateGridAffiliationId(affiliation.GridId.get) else DoiBoostMappingUtil.generateMAGAffiliationId(affiliation.AffiliationId.toString) + val r:Relation = new Relation + r.setSource(pub.getId) + r.setTarget(affId) + r.setRelType("resultOrganization") + r.setRelClass("hasAuthorInstitution") + r.setSubRelType("affiliation") + r.setDataInfo(pub.getDataInfo) + r.setCollectedfrom(List(DoiBoostMappingUtil.createMAGCollectedFrom()).asJava) + val r1:Relation = new Relation + r1.setTarget(pub.getId) + r1.setSource(affId) + r1.setRelType("resultOrganization") + r1.setRelClass("isAuthorInstitutionOf") + r1.setSubRelType("affiliation") + r1.setDataInfo(pub.getDataInfo) + r1.setCollectedfrom(List(DoiBoostMappingUtil.createMAGCollectedFrom()).asJava) + List(r, r1) + })(mapEncoderRel).write.mode(SaveMode.Overwrite).save(s"$workingDirPath/doiBoostPublicationAffiliation") + + + magPubs.joinWith(a,magPubs("_1").equalTo(a("PaperId"))).map( item => { + val affiliation = item._2 + if (affiliation.GridId.isEmpty) { + val o = new Organization + o.setCollectedfrom(List(DoiBoostMappingUtil.createMAGCollectedFrom()).asJava) + o.setDataInfo(DoiBoostMappingUtil.generateDataInfo()) + o.setId(DoiBoostMappingUtil.generateMAGAffiliationId(affiliation.AffiliationId.toString)) + o.setOriginalId(List(affiliation.AffiliationId.toString).asJava) + if (affiliation.DisplayName.nonEmpty) + o.setLegalname(DoiBoostMappingUtil.asField(affiliation.DisplayName.get)) + if (affiliation.OfficialPage.isDefined) + o.setWebsiteurl(DoiBoostMappingUtil.asField(affiliation.OfficialPage.get)) + o.setCountry(DoiBoostMappingUtil.getUnknownCountry()) + o + } + else + null + }).filter(o=> o!=null).write.mode(SaveMode.Overwrite).save(s"$workingDirPath/doiBoostOrganization") + } + +} diff --git a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala new file mode 100644 index 0000000000..f39dd5be8f --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala @@ -0,0 +1,445 @@ +package eu.dnetlib.doiboost.crossref + +import eu.dnetlib.dhp.schema.oaf._ +import eu.dnetlib.dhp.utils.DHPUtils +import eu.dnetlib.doiboost.DoiBoostMappingUtil._ +import org.apache.commons.lang.StringUtils +import org.json4s +import org.json4s.DefaultFormats +import org.json4s.JsonAST._ +import org.json4s.jackson.JsonMethods._ +import org.slf4j.{Logger, LoggerFactory} + +import scala.collection.JavaConverters._ +import scala.collection.mutable +import scala.util.matching.Regex + +case class mappingAffiliation(name: String) {} + +case class mappingAuthor(given: Option[String], family: String, ORCID: Option[String], affiliation: Option[mappingAffiliation]) {} + +case class mappingFunder(name: String, DOI: Option[String], award: Option[List[String]]) {} + + +case object Crossref2Oaf { + val logger: Logger = LoggerFactory.getLogger(Crossref2Oaf.getClass) + + val mappingCrossrefType = Map( + "book-section" -> "publication", + "book" -> "publication", + "book-chapter" -> "publication", + "book-part" -> "publication", + "book-series" -> "publication", + "book-set" -> "publication", + "book-track" -> "publication", + "edited-book" -> "publication", + "reference-book" -> "publication", + "monograph" -> "publication", + "journal-article" -> "publication", + "dissertation" -> "publication", + "other" -> "publication", + "peer-review" -> "publication", + "proceedings" -> "publication", + "proceedings-article" -> "publication", + "reference-entry" -> "publication", + "report" -> "publication", + "report-series" -> "publication", + "standard" -> "publication", + "standard-series" -> "publication", + "posted-content" -> "publication", + "dataset" -> "dataset" + ) + + + val mappingCrossrefSubType = Map( + "book-section" -> "0013 Part of book or chapter of book", + "book" -> "0002 Book", + "book-chapter" -> "0013 Part of book or chapter of book", + "book-part" -> "0013 Part of book or chapter of book", + "book-series" -> "0002 Book", + "book-set" -> "0002 Book", + "book-track" -> "0002 Book", + "edited-book" -> "0002 Book", + "reference-book" -> "0002 Book", + "monograph" -> "0002 Book", + "journal-article" -> "0001 Article", + "dissertation" -> "0006 Doctoral thesis", + "other" -> "0038 Other literature type", + "peer-review" -> "0015 Review", + "proceedings" -> "0004 Conference object", + "proceedings-article" -> "0004 Conference object", + "reference-entry" -> "0013 Part of book or chapter of book", + "report" -> "0017 Report", + "report-series" -> "0017 Report", + "standard" -> "0038 Other literature type", + "standard-series" -> "0038 Other literature type", + "dataset" -> "0021 Dataset", + "preprint" -> "0016 Preprint", + "report" -> "0017 Report" + ) + + def mappingResult(result: Result, json: JValue, cobjCategory: String): Result = { + implicit lazy val formats: DefaultFormats.type = org.json4s.DefaultFormats + + //MAPPING Crossref DOI into PID + val doi: String = (json \ "DOI").extract[String] + result.setPid(List(createSP(doi, "doi", PID_TYPES)).asJava) + + //MAPPING Crossref DOI into OriginalId + //and Other Original Identifier of dataset like clinical-trial-number + val clinicalTrialNumbers = for (JString(ctr) <- json \ "clinical-trial-number") yield ctr + val alternativeIds = for (JString(ids) <- json \ "alternative-id") yield ids + val tmp = clinicalTrialNumbers ::: alternativeIds ::: List(doi) + + result.setOriginalId(tmp.filter(id => id != null).asJava) + + //Set identifier as {50|60} | doiboost____::md5(DOI) + result.setId(generateIdentifier(result, doi)) + + // Add DataInfo + result.setDataInfo(generateDataInfo()) + + result.setLastupdatetimestamp((json \ "indexed" \ "timestamp").extract[Long]) + result.setDateofcollection((json \ "indexed" \ "date-time").extract[String]) + + result.setCollectedfrom(List(createCrossrefCollectedFrom()).asJava) + + // Publisher ( Name of work's publisher mapped into Result/Publisher) + val publisher = (json \ "publisher").extractOrElse[String](null) + if (publisher!= null && publisher.nonEmpty) + result.setPublisher(asField(publisher)) + + + // TITLE + val mainTitles = for {JString(title) <- json \ "title" if title.nonEmpty} yield createSP(title, "main title", "dnet:dataCite_title") + val originalTitles = for {JString(title) <- json \ "original-title" if title.nonEmpty} yield createSP(title, "alternative title", "dnet:dataCite_title") + val shortTitles = for {JString(title) <- json \ "short-title" if title.nonEmpty} yield createSP(title, "alternative title", "dnet:dataCite_title") + val subtitles = for {JString(title) <- json \ "subtitle" if title.nonEmpty} yield createSP(title, "subtitle", "dnet:dataCite_title") + result.setTitle((mainTitles ::: originalTitles ::: shortTitles ::: subtitles).asJava) + + // DESCRIPTION + val descriptionList = for {JString(description) <- json \ "abstract"} yield asField(description) + result.setDescription(descriptionList.asJava) + + // Source + val sourceList = for {JString(source) <- json \ "source" if source!= null && source.nonEmpty} yield asField(source) + result.setSource(sourceList.asJava) + + //RELEVANT DATE Mapping + val createdDate = generateDate((json \ "created" \ "date-time").extract[String], (json \ "created" \ "date-parts").extract[List[List[Int]]], "created", "dnet:dataCite_date") + val postedDate = generateDate((json \ "posted" \ "date-time").extractOrElse[String](null), (json \ "posted" \ "date-parts").extract[List[List[Int]]], "available", "dnet:dataCite_date") + val acceptedDate = generateDate((json \ "accepted" \ "date-time").extractOrElse[String](null), (json \ "accepted" \ "date-parts").extract[List[List[Int]]], "accepted", "dnet:dataCite_date") + val publishedPrintDate = generateDate((json \ "published-print" \ "date-time").extractOrElse[String](null), (json \ "published-print" \ "date-parts").extract[List[List[Int]]], "published-print", "dnet:dataCite_date") + val publishedOnlineDate = generateDate((json \ "published-online" \ "date-time").extractOrElse[String](null), (json \ "published-online" \ "date-parts").extract[List[List[Int]]], "published-online", "dnet:dataCite_date") + + val issuedDate = extractDate((json \ "issued" \ "date-time").extractOrElse[String](null), (json \ "issued" \ "date-parts").extract[List[List[Int]]]) + if (StringUtils.isNotBlank(issuedDate)) { + result.setDateofacceptance(asField(issuedDate)) + } + else { + result.setDateofacceptance(asField(createdDate.getValue)) + } + result.setRelevantdate(List(createdDate, postedDate, acceptedDate, publishedOnlineDate, publishedPrintDate).filter(p => p != null).asJava) + + //Mapping Subject + val subjectList:List[String] = (json \ "subject").extractOrElse[List[String]](List()) + + if (subjectList.nonEmpty) { + result.setSubject(subjectList.map(s=> createSP(s, "keywords", "dnet:subject_classification_typologies")).asJava) + } + + + + //Mapping Author + val authorList: List[mappingAuthor] = (json \ "author").extractOrElse[List[mappingAuthor]](List()) + result.setAuthor(authorList.map(a => generateAuhtor(a.given.orNull, a.family, a.ORCID.orNull)).asJava) + + // Mapping instance + val instance = new Instance() + val license = for { + JString(lic) <- json \ "license" \ "URL" + } yield asField(lic) + val l = license.filter(d => StringUtils.isNotBlank(d.getValue)) + if (l.nonEmpty) + instance.setLicense(l.head) + + + val has_review = (json \ "relation" \"has-review" \ "id") + + if(has_review != JNothing) { + instance.setRefereed( + createQualifier("0001", "peerReviewed", "dnet:review_levels", "dnet:review_levels")) + } + + + instance.setAccessright(getRestrictedQualifier()) + result.setInstance(List(instance).asJava) + instance.setInstancetype(createQualifier(cobjCategory.substring(0, 4), cobjCategory.substring(5), "dnet:publication_resource", "dnet:publication_resource")) + result.setResourcetype(createQualifier(cobjCategory.substring(0, 4),"dnet:dataCite_resource")) + + instance.setCollectedfrom(createCrossrefCollectedFrom()) + if (StringUtils.isNotBlank(issuedDate)) { + instance.setDateofacceptance(asField(issuedDate)) + } + else { + instance.setDateofacceptance(asField(createdDate.getValue)) + } + val s: String = (json \ "URL").extract[String] + val links: List[String] = ((for {JString(url) <- json \ "link" \ "URL"} yield url) ::: List(s)).filter(p => p != null).distinct + if (links.nonEmpty) + instance.setUrl(links.asJava) + result + } + + + def generateAuhtor(given: String, family: String, orcid: String): Author = { + val a = new Author + a.setName(given) + a.setSurname(family) + a.setFullname(s"$given $family") + if (StringUtils.isNotBlank(orcid)) + a.setPid(List(createSP(orcid, ORCID, PID_TYPES)).asJava) + + a + } + + def convert(input: String): List[Oaf] = { + implicit lazy val formats: DefaultFormats.type = org.json4s.DefaultFormats + lazy val json: json4s.JValue = parse(input) + + var resultList: List[Oaf] = List() + + + val objectType = (json \ "type").extractOrElse[String](null) + val objectSubType = (json \ "subtype").extractOrElse[String](null) + if (objectType == null) + return resultList + + + val result = generateItemFromType(objectType, objectSubType) + if (result == null) + return List() + val cOBJCategory = mappingCrossrefSubType.getOrElse(objectType, mappingCrossrefSubType.getOrElse(objectSubType, "0038 Other literature type")); + mappingResult(result, json, cOBJCategory) + + + val funderList: List[mappingFunder] = (json \ "funder").extractOrElse[List[mappingFunder]](List()) + + if (funderList.nonEmpty) { + resultList = resultList ::: mappingFunderToRelations(funderList, result.getId, createCrossrefCollectedFrom(), result.getDataInfo, result.getLastupdatetimestamp) + } + + + result match { + case publication: Publication => convertPublication(publication, json, cOBJCategory) + case dataset: Dataset => convertDataset(dataset) + } + + resultList = resultList ::: List(result) + resultList + } + + + def mappingFunderToRelations(funders: List[mappingFunder], sourceId: String, cf: KeyValue, di: DataInfo, ts: Long): List[Relation] = { + + val queue = new mutable.Queue[Relation] + + + def snsfRule(award:String): String = { + var tmp1 = StringUtils.substringAfter(award,"_") + val tmp2 = StringUtils.substringBefore(tmp1,"/") + logger.debug(s"From $award to $tmp2") + tmp2 + + + } + + + def extractECAward(award: String): String = { + val awardECRegex: Regex = "[0-9]{4,9}".r + if (awardECRegex.findAllIn(award).hasNext) + return awardECRegex.findAllIn(award).max + null + } + + + def generateRelation(sourceId:String, targetId:String, nsPrefix:String) :Relation = { + + val r = new Relation + r.setSource(sourceId) + r.setTarget(s"$nsPrefix::$targetId") + r.setRelType("resultProject") + r.setRelClass("isProducedBy") + r.setSubRelType("outcome") + r.setCollectedfrom(List(cf).asJava) + r.setDataInfo(di) + r.setLastupdatetimestamp(ts) + r + } + + + def generateSimpleRelationFromAward(funder: mappingFunder, nsPrefix: String, extractField: String => String): Unit = { + if (funder.award.isDefined && funder.award.get.nonEmpty) + funder.award.get.map(extractField).filter(a => a!= null && a.nonEmpty).foreach( + award => { + val targetId = DHPUtils.md5(award) + queue += generateRelation(sourceId, targetId, nsPrefix) + } + ) + } + + if (funders != null) + funders.foreach(funder => { + if (funder.DOI.isDefined && funder.DOI.get.nonEmpty) { + funder.DOI.get match { + case "10.13039/100010663" | + "10.13039/100010661" | + "10.13039/501100007601" | + "10.13039/501100000780" | + "10.13039/100010665" => generateSimpleRelationFromAward(funder, "corda__h2020", extractECAward) + case "10.13039/100011199" | + "10.13039/100004431" | + "10.13039/501100004963" | + "10.13039/501100000780" => generateSimpleRelationFromAward(funder, "corda_______", extractECAward) + case "10.13039/501100000781" => generateSimpleRelationFromAward(funder, "corda_______", extractECAward) + generateSimpleRelationFromAward(funder, "corda__h2020", extractECAward) + case "10.13039/100000001" => generateSimpleRelationFromAward(funder, "nsf_________", a => a) + case "10.13039/501100001665" => generateSimpleRelationFromAward(funder, "anr_________", a => a) + case "10.13039/501100002341" => generateSimpleRelationFromAward(funder, "aka_________", a => a) + case "10.13039/501100001602" => generateSimpleRelationFromAward(funder, "aka_________", a => a.replace("SFI", "")) + case "10.13039/501100000923" => generateSimpleRelationFromAward(funder, "arc_________", a => a) + case "10.13039/501100000038"=> queue += generateRelation(sourceId,"1e5e62235d094afd01cd56e65112fc63", "nserc_______" ) + case "10.13039/501100000155"=> queue += generateRelation(sourceId,"1e5e62235d094afd01cd56e65112fc63", "sshrc_______" ) + case "10.13039/501100000024"=> queue += generateRelation(sourceId,"1e5e62235d094afd01cd56e65112fc63", "cihr________" ) + case "10.13039/501100002848" => generateSimpleRelationFromAward(funder, "conicytf____", a => a) + case "10.13039/501100003448" => generateSimpleRelationFromAward(funder, "gsrt________", extractECAward) + case "10.13039/501100010198" => generateSimpleRelationFromAward(funder, "sgov________", a=>a) + case "10.13039/501100004564" => generateSimpleRelationFromAward(funder, "mestd_______", extractECAward) + case "10.13039/501100003407" => generateSimpleRelationFromAward(funder, "miur________", a=>a) + queue += generateRelation(sourceId,"1e5e62235d094afd01cd56e65112fc63", "miur________" ) + case "10.13039/501100006588" | + "10.13039/501100004488" => generateSimpleRelationFromAward(funder, "irb_hr______", a=>a.replaceAll("Project No.", "").replaceAll("HRZZ-","") ) + case "10.13039/501100006769"=> generateSimpleRelationFromAward(funder, "rsf_________", a=>a) + case "10.13039/501100001711"=> generateSimpleRelationFromAward(funder, "snsf________", snsfRule) + case "10.13039/501100004410"=> generateSimpleRelationFromAward(funder, "tubitakf____", a =>a) + case "10.10.13039/100004440"=> generateSimpleRelationFromAward(funder, "wt__________", a =>a) + case "10.13039/100004440"=> queue += generateRelation(sourceId,"1e5e62235d094afd01cd56e65112fc63", "wt__________" ) + case _ => logger.debug("no match for "+funder.DOI.get ) + + + } + + + } else { + funder.name match { + case "European Union’s Horizon 2020 research and innovation program" => generateSimpleRelationFromAward(funder, "corda__h2020", extractECAward) + case "European Union's" => + generateSimpleRelationFromAward(funder, "corda__h2020", extractECAward) + generateSimpleRelationFromAward(funder, "corda_______", extractECAward) + case "The French National Research Agency (ANR)" | + "The French National Research Agency" => generateSimpleRelationFromAward(funder, "anr_________", a => a) + case "CONICYT, Programa de Formación de Capital Humano Avanzado" => generateSimpleRelationFromAward(funder, "conicytf____", extractECAward) + case "Wellcome Trust Masters Fellowship" => queue += generateRelation(sourceId,"1e5e62235d094afd01cd56e65112fc63", "wt__________" ) + case _ => logger.debug("no match for "+funder.name ) + + } + } + + } + ) + queue.toList + } + + def convertDataset(dataset: Dataset): Unit = { + // TODO check if there are other info to map into the Dataset + } + + + def convertPublication(publication: Publication, json: JValue, cobjCategory: String): Unit = { + implicit lazy val formats: DefaultFormats.type = org.json4s.DefaultFormats + val containerTitles = for {JString(ct) <- json \ "container-title"} yield ct + + + //Mapping book + if (cobjCategory.toLowerCase.contains("book")) { + val ISBN = for {JString(isbn) <- json \ "ISBN"} yield isbn + if (ISBN.nonEmpty && containerTitles.nonEmpty) { + val source = s"${containerTitles.head} ISBN: ${ISBN.head}" + if (publication.getSource != null) { + val l: List[Field[String]] = publication.getSource.asScala.toList + val ll: List[Field[String]] = l ::: List(asField(source)) + publication.setSource(ll.asJava) + } + else + publication.setSource(List(asField(source)).asJava) + } + } else { + // Mapping Journal + + val issnInfos = for {JArray(issn_types) <- json \ "issn-type" + JObject(issn_type) <- issn_types + JField("type", JString(tp)) <- issn_type + JField("value", JString(vl)) <- issn_type + } yield Tuple2(tp, vl) + + val volume = (json \ "volume").extractOrElse[String](null) + if (containerTitles.nonEmpty) { + val journal = new Journal + journal.setName(containerTitles.head) + if (issnInfos.nonEmpty) { + + issnInfos.foreach(tp => { + tp._1 match { + case "electronic" => journal.setIssnOnline(tp._2) + case "print" => journal.setIssnPrinted(tp._2) + } + }) + } + journal.setVol(volume) + val page = (json \ "page").extractOrElse[String](null) + if (page != null) { + val pp = page.split("-") + if (pp.nonEmpty) + journal.setSp(pp.head) + if (pp.size > 1) + journal.setEp(pp(1)) + } + publication.setJournal(journal) + } + } + } + + def extractDate(dt: String, datePart: List[List[Int]]): String = { + if (StringUtils.isNotBlank(dt)) + return dt + if (datePart != null && datePart.size == 1) { + val res = datePart.head + if (res.size == 3) { + val dp = f"${res.head}-${res(1)}%02d-${res(2)}%02d" + if (dp.length == 10) { + return dp + } + } + } + null + + } + + def generateDate(dt: String, datePart: List[List[Int]], classId: String, schemeId: String): StructuredProperty = { + val dp = extractDate(dt, datePart) + if (StringUtils.isNotBlank(dp)) + return createSP(dp, classId, schemeId) + null + } + + def generateItemFromType(objectType: String, objectSubType: String): Result = { + if (mappingCrossrefType.contains(objectType)) { + if (mappingCrossrefType(objectType).equalsIgnoreCase("publication")) + return new Publication() + if (mappingCrossrefType(objectType).equalsIgnoreCase("dataset")) + return new Dataset() + } + null + } + +} diff --git a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/crossref/CrossrefImporter.java b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/crossref/CrossrefImporter.java new file mode 100644 index 0000000000..f69a05da1c --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/crossref/CrossrefImporter.java @@ -0,0 +1,103 @@ + +package eu.dnetlib.doiboost.crossref; + +import java.io.ByteArrayOutputStream; +import java.util.zip.Inflater; + +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.SequenceFile; +import org.apache.hadoop.io.Text; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import eu.dnetlib.dhp.application.ArgumentApplicationParser; + +public class CrossrefImporter { + + public static void main(String[] args) throws Exception { + + final ArgumentApplicationParser parser = new ArgumentApplicationParser( + IOUtils + .toString( + CrossrefImporter.class + .getResourceAsStream( + "/eu/dnetlib/dhp/doiboost/import_from_es.json"))); + + parser.parseArgument(args); + + final String hdfsuri = parser.get("namenode"); + System.out.println("HDFS URI" + hdfsuri); + Path hdfswritepath = new Path(parser.get("targetPath")); + System.out.println("TargetPath: " + hdfsuri); + + final Long timestamp = StringUtils.isNotBlank(parser.get("timestamp")) + ? Long.parseLong(parser.get("timestamp")) + : -1; + + if (timestamp > 0) + System.out.println("Timestamp added " + timestamp); + + // ====== Init HDFS File System Object + Configuration conf = new Configuration(); + // Set FileSystem URI + conf.set("fs.defaultFS", hdfsuri); + // Because of Maven + conf.set("fs.hdfs.impl", org.apache.hadoop.hdfs.DistributedFileSystem.class.getName()); + conf.set("fs.file.impl", org.apache.hadoop.fs.LocalFileSystem.class.getName()); + + ESClient client = timestamp > 0 + ? new ESClient("ip-90-147-167-25.ct1.garrservices.it", "crossref", timestamp) + : new ESClient("ip-90-147-167-25.ct1.garrservices.it", "crossref"); + + try (SequenceFile.Writer writer = SequenceFile + .createWriter( + conf, + SequenceFile.Writer.file(hdfswritepath), + SequenceFile.Writer.keyClass(IntWritable.class), + SequenceFile.Writer.valueClass(Text.class))) { + + int i = 0; + long start = System.currentTimeMillis(); + long end = 0; + final IntWritable key = new IntWritable(i); + final Text value = new Text(); + while (client.hasNext()) { + key.set(i++); + value.set(client.next()); + writer.append(key, value); + if (i % 100000 == 0) { + end = System.currentTimeMillis(); + final float time = (end - start) / 1000.0F; + System.out + .println( + String.format("Imported %d records last 100000 imported in %f seconds", i, time)); + start = System.currentTimeMillis(); + } + } + } + } + + public static String decompressBlob(final String blob) { + try { + byte[] byteArray = Base64.decodeBase64(blob.getBytes()); + final Inflater decompresser = new Inflater(); + decompresser.setInput(byteArray); + final ByteArrayOutputStream bos = new ByteArrayOutputStream(byteArray.length); + byte[] buffer = new byte[8192]; + while (!decompresser.finished()) { + int size = decompresser.inflate(buffer); + bos.write(buffer, 0, size); + } + byte[] unzippeddata = bos.toByteArray(); + decompresser.end(); + return new String(unzippeddata); + } catch (Throwable e) { + throw new RuntimeException("Wrong record:" + blob, e); + } + } +} diff --git a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/crossref/ESClient.java b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/crossref/ESClient.java new file mode 100644 index 0000000000..e31ccf399b --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/crossref/ESClient.java @@ -0,0 +1,114 @@ + +package eu.dnetlib.doiboost.crossref; + +import java.io.IOException; +import java.util.Iterator; +import java.util.List; + +import org.apache.commons.io.IOUtils; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.jayway.jsonpath.JsonPath; + +public class ESClient implements Iterator { + private static final Logger logger = LoggerFactory.getLogger(ESClient.class); + + static final String blobPath = "$.hits[*].hits[*]._source.blob"; + static final String scrollIdPath = "$._scroll_id"; + static final String JSON_NO_TS = "{\"size\":1000}"; + static final String JSON_WITH_TS = "{\"size\":1000, \"query\":{\"range\":{\"timestamp\":{\"gte\":%d}}}}"; + static final String JSON_SCROLL = "{\"scroll_id\":\"%s\",\"scroll\" : \"1m\"}"; + + private final String scrollId; + + private List buffer; + + private final String esHost; + + public ESClient(final String esHost, final String esIndex) throws IOException { + + this.esHost = esHost; + final String body = getResponse( + String.format("http://%s:9200/%s/_search?scroll=1m", esHost, esIndex), JSON_NO_TS); + scrollId = getJPathString(scrollIdPath, body); + buffer = getBlobs(body); + } + + public ESClient(final String esHost, final String esIndex, final long timestamp) + throws IOException { + this.esHost = esHost; + final String body = getResponse( + String.format("http://%s:9200/%s/_search?scroll=1m", esHost, esIndex), + String.format(JSON_WITH_TS, timestamp)); + scrollId = getJPathString(scrollIdPath, body); + buffer = getBlobs(body); + } + + private String getResponse(final String url, final String json) { + CloseableHttpClient client = HttpClients.createDefault(); + try { + + HttpPost httpPost = new HttpPost(url); + if (json != null) { + StringEntity entity = new StringEntity(json); + httpPost.setEntity(entity); + httpPost.setHeader("Accept", "application/json"); + httpPost.setHeader("Content-type", "application/json"); + } + CloseableHttpResponse response = client.execute(httpPost); + + return IOUtils.toString(response.getEntity().getContent()); + } catch (Throwable e) { + throw new RuntimeException("Error on executing request ", e); + } finally { + try { + client.close(); + } catch (IOException e) { + throw new RuntimeException("Unable to close client ", e); + } + } + } + + private String getJPathString(final String jsonPath, final String json) { + try { + Object o = JsonPath.read(json, jsonPath); + if (o instanceof String) + return (String) o; + return null; + } catch (Exception e) { + return ""; + } + } + + private List getBlobs(final String body) { + final List res = JsonPath.read(body, "$.hits.hits[*]._source.blob"); + return res; + } + + @Override + public boolean hasNext() { + return (buffer != null && !buffer.isEmpty()); + } + + @Override + public String next() { + final String nextItem = buffer.remove(0); + if (buffer.isEmpty()) { + + final String json_param = String.format(JSON_SCROLL, scrollId); + final String body = getResponse(String.format("http://%s:9200/_search/scroll", esHost), json_param); + try { + buffer = getBlobs(body); + } catch (Throwable e) { + logger.error("Error on get next page: body:" + body); + } + } + return nextItem; + } +} diff --git a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/crossref/SparkMapDumpIntoOAF.scala b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/crossref/SparkMapDumpIntoOAF.scala new file mode 100644 index 0000000000..fac4c90b47 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/crossref/SparkMapDumpIntoOAF.scala @@ -0,0 +1,98 @@ +package eu.dnetlib.doiboost.crossref + +import eu.dnetlib.dhp.application.ArgumentApplicationParser +import eu.dnetlib.dhp.schema.oaf +import eu.dnetlib.dhp.schema.oaf.{Oaf, Publication, Relation, Dataset => OafDataset} +import org.apache.commons.io.IOUtils +import org.apache.hadoop.io.{IntWritable, Text} +import org.apache.spark.SparkConf +import org.apache.spark.rdd.RDD +import org.apache.spark.sql.{Dataset, Encoder, Encoders, SaveMode, SparkSession} +import org.slf4j.{Logger, LoggerFactory} + + +case class Reference(author: String, firstPage: String) {} + +object SparkMapDumpIntoOAF { + + def main(args: Array[String]): Unit = { + + + val logger: Logger = LoggerFactory.getLogger(SparkMapDumpIntoOAF.getClass) + val conf: SparkConf = new SparkConf() + val parser = new ArgumentApplicationParser(IOUtils.toString(SparkMapDumpIntoOAF.getClass.getResourceAsStream("/eu/dnetlib/dhp/doiboost/convert_map_to_oaf_params.json"))) + parser.parseArgument(args) + val spark: SparkSession = + SparkSession + .builder() + .config(conf) + .appName(SparkMapDumpIntoOAF.getClass.getSimpleName) + .master(parser.get("master")).getOrCreate() + + implicit val mapEncoderPubs: Encoder[Publication] = Encoders.kryo[Publication] + implicit val mapEncoderRelatons: Encoder[Relation] = Encoders.kryo[Relation] + implicit val mapEncoderDatasets: Encoder[oaf.Dataset] = Encoders.kryo[OafDataset] + + val sc = spark.sparkContext + val targetPath = parser.get("targetPath") + + + sc.sequenceFile(parser.get("sourcePath"), classOf[IntWritable], classOf[Text]) + .map(k => k._2.toString).map(CrossrefImporter.decompressBlob) + .flatMap(k => Crossref2Oaf.convert(k)).saveAsObjectFile(s"${targetPath}/mixObject") + + val inputRDD = sc.objectFile[Oaf](s"${targetPath}/mixObject").filter(p=> p!= null) + + val distinctPubs:RDD[Publication] = inputRDD.filter(k => k != null && k.isInstanceOf[Publication]) + .map(k => k.asInstanceOf[Publication]).map { p: Publication => Tuple2(p.getId, p) }.reduceByKey { case (p1: Publication, p2: Publication) => + var r = if (p1 == null) p2 else p1 + if (p1 != null && p2 != null) { + if (p1.getLastupdatetimestamp != null && p2.getLastupdatetimestamp != null) { + if (p1.getLastupdatetimestamp < p2.getLastupdatetimestamp) + r = p2 + else + r = p1 + } else { + r = if (p1.getLastupdatetimestamp == null) p2 else p1 + } + } + r + }.map(_._2) + + val pubs:Dataset[Publication] = spark.createDataset(distinctPubs) + pubs.write.mode(SaveMode.Overwrite).save(s"${targetPath}/publication") + + + val distincDatasets:RDD[OafDataset] = inputRDD.filter(k => k != null && k.isInstanceOf[OafDataset]) + .map(k => k.asInstanceOf[OafDataset]).map(p => Tuple2(p.getId, p)).reduceByKey { case (p1: OafDataset, p2: OafDataset) => + var r = if (p1 == null) p2 else p1 + if (p1 != null && p2 != null) { + if (p1.getLastupdatetimestamp != null && p2.getLastupdatetimestamp != null) { + if (p1.getLastupdatetimestamp < p2.getLastupdatetimestamp) + r = p2 + else + r = p1 + } else { + r = if (p1.getLastupdatetimestamp == null) p2 else p1 + } + } + r + }.map(_._2) + + spark.createDataset(distincDatasets).write.mode(SaveMode.Overwrite).save(s"${targetPath}/dataset") + + + + val distinctRels =inputRDD.filter(k => k != null && k.isInstanceOf[Relation]) + .map(k => k.asInstanceOf[Relation]).map(r=> (s"${r.getSource}::${r.getTarget}",r)) + .reduceByKey { case (p1: Relation, p2: Relation) => + if (p1 == null) p2 else p1 + }.map(_._2) + + val rels: Dataset[Relation] = spark.createDataset(distinctRels) + + rels.write.mode(SaveMode.Overwrite).save(s"${targetPath}/relations") + } + + +} diff --git a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/mag/MagDataModel.scala b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/mag/MagDataModel.scala new file mode 100644 index 0000000000..7bb4686cf5 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/mag/MagDataModel.scala @@ -0,0 +1,316 @@ +package eu.dnetlib.doiboost.mag + + +import eu.dnetlib.dhp.schema.oaf.{Instance, Journal, Publication, StructuredProperty} +import eu.dnetlib.doiboost.DoiBoostMappingUtil +import org.json4s +import org.json4s.DefaultFormats +import org.json4s.jackson.JsonMethods.parse +import eu.dnetlib.doiboost.DoiBoostMappingUtil._ + +import scala.collection.JavaConverters._ +import scala.collection.mutable +import scala.util.matching.Regex + + +case class MagPapers(PaperId: Long, Rank: Integer, Doi: String, + DocType: String, PaperTitle: String, OriginalTitle: String, + BookTitle: String, Year: Option[Integer], Date: Option[java.sql.Timestamp], Publisher: String, + JournalId: Option[Long], ConferenceSeriesId: Option[Long], ConferenceInstanceId: Option[Long], + Volume: String, Issue: String, FirstPage: String, LastPage: String, + ReferenceCount: Option[Long], CitationCount: Option[Long], EstimatedCitation: Option[Long], + OriginalVenue: String, FamilyId: Option[Long], CreatedDate: java.sql.Timestamp) {} + + +case class MagPaperAbstract(PaperId: Long, IndexedAbstract: String) {} + +case class MagAuthor(AuthorId: Long, Rank: Option[Int], NormalizedName: Option[String], DisplayName: Option[String], LastKnownAffiliationId: Option[Long], PaperCount: Option[Long], CitationCount: Option[Long], CreatedDate: Option[java.sql.Timestamp]) {} + +case class MagAffiliation(AffiliationId: Long, Rank: Int, NormalizedName: String, DisplayName: String, GridId: String, OfficialPage: String, WikiPage: String, PaperCount: Long, CitationCount: Long, Latitude: Option[Float], Longitude: Option[Float], CreatedDate: java.sql.Timestamp) {} + +case class MagPaperAuthorAffiliation(PaperId: Long, AuthorId: Long, AffiliationId: Option[Long], AuthorSequenceNumber: Int, OriginalAuthor: String, OriginalAffiliation: String) {} + + +case class MagAuthorAffiliation(author: MagAuthor, affiliation:String) + +case class MagPaperWithAuthorList(PaperId: Long, authors: List[MagAuthorAffiliation]) {} + +case class MagPaperAuthorDenormalized(PaperId: Long, author: MagAuthor, affiliation:String) {} + +case class MagPaperUrl(PaperId: Long, SourceType: Option[Int], SourceUrl: Option[String], LanguageCode: Option[String]) {} + +case class MagUrlInstance(SourceUrl:String){} + +case class MagUrl(PaperId: Long, instances: List[MagUrlInstance]) + +case class MagSubject(FieldOfStudyId:Long, DisplayName:String, MainType:Option[String], Score:Float){} + +case class MagFieldOfStudy(PaperId:Long, subjects:List[MagSubject]) {} + +case class MagJournal(JournalId: Long, Rank: Option[Int], NormalizedName: Option[String], DisplayName: Option[String], Issn: Option[String], Publisher: Option[String], Webpage: Option[String], PaperCount: Option[Long], CitationCount: Option[Long], CreatedDate: Option[java.sql.Timestamp]) {} + + +case class MagConferenceInstance(ci:Long, DisplayName:Option[String], Location:Option[String], StartDate:Option[java.sql.Timestamp], EndDate:Option[java.sql.Timestamp], PaperId:Long){} + +case object ConversionUtil { + + def extractMagIdentifier(pids:mutable.Buffer[String]) :String ={ + val magIDRegex: Regex = "^[0-9]+$".r + val s =pids.filter(p=> magIDRegex.findAllIn(p).hasNext) + + if (s.nonEmpty) + return s.head + null + } + + + def mergePublication(a: Publication, b:Publication) : Publication = { + if ((a != null) && (b != null)) { + a.mergeFrom(b) + a + } else { + if (a == null) b else a + } + + + } + + def choiceLatestMagArtitcle(p1: MagPapers, p2:MagPapers) :MagPapers = { + var r = if (p1 == null) p2 else p1 + if (p1 != null && p2 != null) { + if (p1.CreatedDate != null && p2.CreatedDate != null) { + if (p1.CreatedDate.before(p2.CreatedDate)) + r = p2 + else + r = p1 + } else { + r = if (p1.CreatedDate == null) p2 else p1 + } + } + r + + } + + + def updatePubsWithDescription(inputItem:((String, Publication), MagPaperAbstract)) : Publication = { + val pub = inputItem._1._2 + val abst = inputItem._2 + if (abst != null) { + pub.setDescription(List(asField(abst.IndexedAbstract)).asJava) + } + pub + + } + + + def updatePubsWithConferenceInfo(inputItem:((String, Publication), MagConferenceInstance)) : Publication = { + val publication:Publication= inputItem._1._2 + val ci:MagConferenceInstance = inputItem._2 + + if (ci!= null){ + + val j:Journal = new Journal + if (ci.Location.isDefined) + j.setConferenceplace(ci.Location.get) + j.setName(ci.DisplayName.get) + if (ci.StartDate.isDefined && ci.EndDate.isDefined) + { + j.setConferencedate(s"${ci.StartDate.get.toString.substring(0,10)} - ${ci.EndDate.get.toString.substring(0,10)}") + } + + publication.setJournal(j) + } + publication + } + + def updatePubsWithSubject(item:((String, Publication), MagFieldOfStudy)) : Publication = { + + val publication = item._1._2 + val fieldOfStudy = item._2 + if (fieldOfStudy != null && fieldOfStudy.subjects != null && fieldOfStudy.subjects.nonEmpty) { + val p: List[StructuredProperty] = fieldOfStudy.subjects.flatMap(s => { + val s1 = createSP(s.DisplayName, "MAG","Microsoft Academic Graph classification", "dnet:subject_classification_typologies", "dnet:subject_classification_typologies") + val di = DoiBoostMappingUtil.generateDataInfo(s.Score.toString) + var resList: List[StructuredProperty] = List(s1) + if (s.MainType.isDefined) { + val maintp = s.MainType.get + val s2 = createSP(s.MainType.get, "MAG","Microsoft Academic Graph classification", "dnet:subject_classification_typologies", "dnet:subject_classification_typologies") + s2.setDataInfo(di) + resList = resList ::: List(s2) + if (maintp.contains(".")) { + val s3 = createSP(maintp.split("\\.").head, "MAG","Microsoft Academic Graph classification", "dnet:subject_classification_typologies", "dnet:subject_classification_typologies") + s3.setDataInfo(di) + resList = resList ::: List(s3) + } + } + resList + }) + publication.setSubject(p.asJava) + } + publication + } + + + + def addInstances(a: (Publication, MagUrl)): Publication = { + val pub = a._1 + val urls = a._2 + + + + val i = new Instance + + + if (urls!= null) { + + val l:List[String] = urls.instances.filter(k=>k.SourceUrl.nonEmpty).map(k=>k.SourceUrl):::List(s"https://academic.microsoft.com/#/detail/${extractMagIdentifier(pub.getOriginalId.asScala)}") + + i.setUrl(l.asJava) + } + else + i.setUrl(List(s"https://academic.microsoft.com/#/detail/${extractMagIdentifier(pub.getOriginalId.asScala)}").asJava) + + i.setCollectedfrom(createMAGCollectedFrom()) + pub.setInstance(List(i).asJava) + pub + } + + + def transformPaperAbstract(input: MagPaperAbstract): MagPaperAbstract = { + MagPaperAbstract(input.PaperId, convertInvertedIndexString(input.IndexedAbstract)) + } + + + def createOAFFromJournalAuthorPaper(inputParams: ((MagPapers, MagJournal), MagPaperWithAuthorList)): Publication = { + val paper = inputParams._1._1 + val journal = inputParams._1._2 + val authors = inputParams._2 + + val pub = new Publication + pub.setPid(List(createSP(paper.Doi.toLowerCase, "doi", PID_TYPES)).asJava) + pub.setOriginalId(List(paper.PaperId.toString, paper.Doi.toLowerCase).asJava) + + //Set identifier as 50|doiboost____::md5(DOI) + pub.setId(generateIdentifier(pub, paper.Doi.toLowerCase)) + + val mainTitles = createSP(paper.PaperTitle, "main title", "dnet:dataCite_title") + val originalTitles = createSP(paper.OriginalTitle, "alternative title", "dnet:dataCite_title") + pub.setTitle(List(mainTitles, originalTitles).asJava) + + pub.setSource(List(asField(paper.BookTitle)).asJava) + + val authorsOAF = authors.authors.map { f: MagAuthorAffiliation => + + val a: eu.dnetlib.dhp.schema.oaf.Author = new eu.dnetlib.dhp.schema.oaf.Author + + a.setFullname(f.author.DisplayName.get) + + if(f.affiliation!= null) + a.setAffiliation(List(asField(f.affiliation)).asJava) + a.setPid(List(createSP(s"https://academic.microsoft.com/#/detail/${f.author.AuthorId}", "URL", PID_TYPES)).asJava) + a + } + pub.setAuthor(authorsOAF.asJava) + + + if (paper.Date != null && paper.Date.isDefined) { + pub.setDateofacceptance(asField(paper.Date.get.toString.substring(0,10))) + } + pub.setPublisher(asField(paper.Publisher)) + + + if (journal != null && journal.DisplayName.isDefined) { + val j = new Journal + + j.setName(journal.DisplayName.get) + j.setSp(paper.FirstPage) + j.setEp(paper.LastPage) + if (journal.Publisher.isDefined) + pub.setPublisher(asField(journal.Publisher.get)) + if (journal.Issn.isDefined) + j.setIssnPrinted(journal.Issn.get) + j.setVol(paper.Volume) + j.setIss(paper.Issue) + pub.setJournal(j) + } + pub.setCollectedfrom(List(createMAGCollectedFrom()).asJava) + pub.setDataInfo(generateDataInfo()) + pub + } + + + def createOAF(inputParams: ((MagPapers, MagPaperWithAuthorList), MagPaperAbstract)): Publication = { + + val paper = inputParams._1._1 + val authors = inputParams._1._2 + val description = inputParams._2 + + val pub = new Publication + pub.setPid(List(createSP(paper.Doi.toLowerCase, "doi", PID_TYPES)).asJava) + pub.setOriginalId(List(paper.PaperId.toString, paper.Doi.toLowerCase).asJava) + + //Set identifier as 50 | doiboost____::md5(DOI) + pub.setId(generateIdentifier(pub, paper.Doi.toLowerCase)) + + val mainTitles = createSP(paper.PaperTitle, "main title", "dnet:dataCite_title") + val originalTitles = createSP(paper.OriginalTitle, "alternative title", "dnet:dataCite_title") + pub.setTitle(List(mainTitles, originalTitles).asJava) + + pub.setSource(List(asField(paper.BookTitle)).asJava) + + + if (description != null) { + pub.setDescription(List(asField(description.IndexedAbstract)).asJava) + } + + + val authorsOAF = authors.authors.map { f: MagAuthorAffiliation => + + val a: eu.dnetlib.dhp.schema.oaf.Author = new eu.dnetlib.dhp.schema.oaf.Author + + a.setFullname(f.author.DisplayName.get) + + if(f.affiliation!= null) + a.setAffiliation(List(asField(f.affiliation)).asJava) + + + a.setPid(List(createSP(s"https://academic.microsoft.com/#/detail/${f.author.AuthorId}", "URL", PID_TYPES)).asJava) + + a + + } + + + if (paper.Date != null) { + pub.setDateofacceptance(asField(paper.Date.toString.substring(0,10))) + } + + pub.setAuthor(authorsOAF.asJava) + + + pub + + } + + + def convertInvertedIndexString(json_input: String): String = { + implicit lazy val formats: DefaultFormats.type = org.json4s.DefaultFormats + lazy val json: json4s.JValue = parse(json_input) + val idl = (json \ "IndexLength").extract[Int] + if (idl > 0) { + val res = Array.ofDim[String](idl) + + val iid = (json \ "InvertedIndex").extract[Map[String, List[Int]]] + + for {(k: String, v: List[Int]) <- iid} { + v.foreach(item => res(item) = k) + } + (0 until idl).foreach(i => { + if (res(i) == null) + res(i) = "" + }) + return res.mkString(" ") + } + "" + } +} diff --git a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/mag/SparkImportMagIntoDataset.scala b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/mag/SparkImportMagIntoDataset.scala new file mode 100644 index 0000000000..f291a92f99 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/mag/SparkImportMagIntoDataset.scala @@ -0,0 +1,92 @@ +package eu.dnetlib.doiboost.mag + +import eu.dnetlib.dhp.application.ArgumentApplicationParser +import org.apache.commons.io.IOUtils +import org.apache.spark.SparkConf +import org.apache.spark.sql.{SaveMode, SparkSession} +import org.apache.spark.sql.types._ +import org.slf4j.{Logger, LoggerFactory} +import org.apache.spark.sql.functions._ + +object SparkImportMagIntoDataset { + val datatypedict = Map( + "int" -> IntegerType, + "uint" -> IntegerType, + "long" -> LongType, + "ulong" -> LongType, + "float" -> FloatType, + "string" -> StringType, + "DateTime" -> DateType + ) + + + val stream = Map( + "Affiliations" -> Tuple2("mag/Affiliations.txt", Seq("AffiliationId:long", "Rank:uint", "NormalizedName:string", "DisplayName:string", "GridId:string", "OfficialPage:string", "WikiPage:string", "PaperCount:long", "CitationCount:long", "Latitude:float?", "Longitude:float?", "CreatedDate:DateTime")), + "Authors" -> Tuple2("mag/Authors.txt", Seq("AuthorId:long", "Rank:uint", "NormalizedName:string", "DisplayName:string", "LastKnownAffiliationId:long?", "PaperCount:long", "CitationCount:long", "CreatedDate:DateTime")), + "ConferenceInstances" -> Tuple2("mag/ConferenceInstances.txt", Seq("ConferenceInstanceId:long", "NormalizedName:string", "DisplayName:string", "ConferenceSeriesId:long", "Location:string", "OfficialUrl:string", "StartDate:DateTime?", "EndDate:DateTime?", "AbstractRegistrationDate:DateTime?", "SubmissionDeadlineDate:DateTime?", "NotificationDueDate:DateTime?", "FinalVersionDueDate:DateTime?", "PaperCount:long", "CitationCount:long", "Latitude:float?", "Longitude:float?", "CreatedDate:DateTime")), + "ConferenceSeries" -> Tuple2("mag/ConferenceSeries.txt", Seq("ConferenceSeriesId:long", "Rank:uint", "NormalizedName:string", "DisplayName:string", "PaperCount:long", "CitationCount:long", "CreatedDate:DateTime")), + "EntityRelatedEntities" -> Tuple2("advanced/EntityRelatedEntities.txt", Seq("EntityId:long", "EntityType:string", "RelatedEntityId:long", "RelatedEntityType:string", "RelatedType:int", "Score:float")), + "FieldOfStudyChildren" -> Tuple2("advanced/FieldOfStudyChildren.txt", Seq("FieldOfStudyId:long", "ChildFieldOfStudyId:long")), + "FieldOfStudyExtendedAttributes" -> Tuple2("advanced/FieldOfStudyExtendedAttributes.txt", Seq("FieldOfStudyId:long", "AttributeType:int", "AttributeValue:string")), + "FieldsOfStudy" -> Tuple2("advanced/FieldsOfStudy.txt", Seq("FieldOfStudyId:long", "Rank:uint", "NormalizedName:string", "DisplayName:string", "MainType:string", "Level:int", "PaperCount:long", "CitationCount:long", "CreatedDate:DateTime")), + "Journals" -> Tuple2("mag/Journals.txt", Seq("JournalId:long", "Rank:uint", "NormalizedName:string", "DisplayName:string", "Issn:string", "Publisher:string", "Webpage:string", "PaperCount:long", "CitationCount:long", "CreatedDate:DateTime")), + "PaperAbstractsInvertedIndex" -> Tuple2("nlp/PaperAbstractsInvertedIndex.txt.*", Seq("PaperId:long", "IndexedAbstract:string")), + "PaperAuthorAffiliations" -> Tuple2("mag/PaperAuthorAffiliations.txt", Seq("PaperId:long", "AuthorId:long", "AffiliationId:long?", "AuthorSequenceNumber:uint", "OriginalAuthor:string", "OriginalAffiliation:string")), + "PaperCitationContexts" -> Tuple2("nlp/PaperCitationContexts.txt", Seq("PaperId:long", "PaperReferenceId:long", "CitationContext:string")), + "PaperExtendedAttributes" -> Tuple2("mag/PaperExtendedAttributes.txt", Seq("PaperId:long", "AttributeType:int", "AttributeValue:string")), + "PaperFieldsOfStudy" -> Tuple2("advanced/PaperFieldsOfStudy.txt", Seq("PaperId:long", "FieldOfStudyId:long", "Score:float")), + "PaperRecommendations" -> Tuple2("advanced/PaperRecommendations.txt", Seq("PaperId:long", "RecommendedPaperId:long", "Score:float")), + "PaperReferences" -> Tuple2("mag/PaperReferences.txt", Seq("PaperId:long", "PaperReferenceId:long")), + "PaperResources" -> Tuple2("mag/PaperResources.txt", Seq("PaperId:long", "ResourceType:int", "ResourceUrl:string", "SourceUrl:string", "RelationshipType:int")), + "PaperUrls" -> Tuple2("mag/PaperUrls.txt", Seq("PaperId:long", "SourceType:int?", "SourceUrl:string", "LanguageCode:string")), + "Papers" -> Tuple2("mag/Papers.txt", Seq("PaperId:long", "Rank:uint", "Doi:string", "DocType:string", "PaperTitle:string", "OriginalTitle:string", "BookTitle:string", "Year:int?", "Date:DateTime?", "Publisher:string", "JournalId:long?", "ConferenceSeriesId:long?", "ConferenceInstanceId:long?", "Volume:string", "Issue:string", "FirstPage:string", "LastPage:string", "ReferenceCount:long", "CitationCount:long", "EstimatedCitation:long", "OriginalVenue:string", "FamilyId:long?", "CreatedDate:DateTime")), + "RelatedFieldOfStudy" -> Tuple2("advanced/RelatedFieldOfStudy.txt", Seq("FieldOfStudyId1:long", "Type1:string", "FieldOfStudyId2:long", "Type2:string", "Rank:float")) + ) + + + def getSchema(streamName: String): StructType = { + var schema = new StructType() + val d: Seq[String] = stream(streamName)._2 + d.foreach { case t => + val currentType = t.split(":") + val fieldName: String = currentType.head + var fieldType: String = currentType.last + val nullable: Boolean = fieldType.endsWith("?") + if (nullable) + fieldType = fieldType.replace("?", "") + schema = schema.add(StructField(fieldName, datatypedict(fieldType), nullable)) + } + schema + } + + + def main(args: Array[String]): Unit = { + val logger: Logger = LoggerFactory.getLogger(getClass) + val conf: SparkConf = new SparkConf() + val parser = new ArgumentApplicationParser(IOUtils.toString(getClass.getResourceAsStream("/eu/dnetlib/dhp/doiboost/mag/convert_mag_to_oaf_params.json"))) + parser.parseArgument(args) + val spark: SparkSession = + SparkSession + .builder() + .config(conf) + .appName(getClass.getSimpleName) + .master(parser.get("master")).getOrCreate() + + + + stream.foreach { case (k, v) => + val s: StructType = getSchema(k) + val df = spark.read + .option("header", "false") + .option("charset", "UTF8") + .option("delimiter", "\t") + .schema(s) + .csv(s"${parser.get("sourcePath")}/${v._1}") + logger.info(s"Converting $k") + + df.write.mode(SaveMode.Overwrite).save(s"${parser.get("targetPath")}/$k") + } + + } + +} diff --git a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/mag/SparkPreProcessMAG.scala b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/mag/SparkPreProcessMAG.scala new file mode 100644 index 0000000000..a24f0e6bb4 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/mag/SparkPreProcessMAG.scala @@ -0,0 +1,157 @@ +package eu.dnetlib.doiboost.mag + +import eu.dnetlib.dhp.application.ArgumentApplicationParser +import eu.dnetlib.dhp.schema.oaf.Publication +import org.apache.commons.io.IOUtils +import org.apache.spark.SparkConf +import org.apache.spark.rdd.RDD +import org.apache.spark.sql.functions._ +import org.apache.spark.sql._ +import org.slf4j.{Logger, LoggerFactory} + +import scala.collection.JavaConverters._ + +object SparkPreProcessMAG { + def main(args: Array[String]): Unit = { + + val logger: Logger = LoggerFactory.getLogger(getClass) + val conf: SparkConf = new SparkConf() + val parser = new ArgumentApplicationParser(IOUtils.toString(getClass.getResourceAsStream("/eu/dnetlib/dhp/doiboost/mag/preprocess_mag_params.json"))) + parser.parseArgument(args) + val spark: SparkSession = + SparkSession + .builder() + .config(conf) + .appName(getClass.getSimpleName) + .master(parser.get("master")).getOrCreate() + + val sourcePath = parser.get("sourcePath") + import spark.implicits._ + implicit val mapEncoderPubs: Encoder[Publication] = org.apache.spark.sql.Encoders.kryo[Publication] + implicit val tupleForJoinEncoder: Encoder[(String, Publication)] = Encoders.tuple(Encoders.STRING, mapEncoderPubs) + + logger.info("Phase 1) make uninque DOI in Papers:") + val d: Dataset[MagPapers] = spark.read.load(s"${parser.get("sourcePath")}/Papers").as[MagPapers] + + // Filtering Papers with DOI, and since for the same DOI we have multiple version of item with different PapersId we get the last one + val result: RDD[MagPapers] = d.where(col("Doi").isNotNull) + .rdd + .map{ p: MagPapers => Tuple2(p.Doi, p) } + .reduceByKey((p1:MagPapers,p2:MagPapers) => ConversionUtil.choiceLatestMagArtitcle(p1,p2)) + .map(_._2) + + val distinctPaper: Dataset[MagPapers] = spark.createDataset(result) + distinctPaper.write.mode(SaveMode.Overwrite).save(s"${parser.get("targetPath")}/Papers_distinct") + + logger.info("Phase 0) Enrich Publication with description") + val pa = spark.read.load(s"${parser.get("sourcePath")}/PaperAbstractsInvertedIndex").as[MagPaperAbstract] + pa.map(ConversionUtil.transformPaperAbstract).write.mode(SaveMode.Overwrite).save(s"${parser.get("targetPath")}/PaperAbstract") + + logger.info("Phase 3) Group Author by PaperId") + val authors = spark.read.load(s"$sourcePath/Authors").as[MagAuthor] + + val affiliation = spark.read.load(s"$sourcePath/Affiliations").as[MagAffiliation] + val paperAuthorAffiliation = spark.read.load(s"$sourcePath/PaperAuthorAffiliations").as[MagPaperAuthorAffiliation] + + paperAuthorAffiliation.joinWith(authors, paperAuthorAffiliation("AuthorId").equalTo(authors("AuthorId"))) + .map { case (a: MagPaperAuthorAffiliation, b: MagAuthor) => (a.AffiliationId, MagPaperAuthorDenormalized(a.PaperId, b, null)) } + .joinWith(affiliation, affiliation("AffiliationId").equalTo(col("_1")), "left") + .map(s => { + val mpa = s._1._2 + val af = s._2 + if (af != null) { + MagPaperAuthorDenormalized(mpa.PaperId, mpa.author, af.DisplayName) + } else + mpa + }).groupBy("PaperId").agg(collect_list(struct($"author", $"affiliation")).as("authors")) + .write.mode(SaveMode.Overwrite).save(s"${parser.get("targetPath")}/merge_step_1_paper_authors") + + logger.info("Phase 4) create First Version of publication Entity with Paper Journal and Authors") + + val journals = spark.read.load(s"$sourcePath/Journals").as[MagJournal] + + val papers = spark.read.load((s"${parser.get("targetPath")}/Papers_distinct")).as[MagPapers] + + val paperWithAuthors = spark.read.load(s"${parser.get("targetPath")}/merge_step_1_paper_authors").as[MagPaperWithAuthorList] + + val firstJoin = papers.joinWith(journals, papers("JournalId").equalTo(journals("JournalId")), "left") + firstJoin.joinWith(paperWithAuthors, firstJoin("_1.PaperId").equalTo(paperWithAuthors("PaperId")), "left") + .map { a => ConversionUtil.createOAFFromJournalAuthorPaper(a) } + .write.mode(SaveMode.Overwrite).save(s"${parser.get("targetPath")}/merge_step_2") + + + var magPubs: Dataset[(String, Publication)] = + spark.read.load(s"${parser.get("targetPath")}/merge_step_2").as[Publication] + .map(p => (ConversionUtil.extractMagIdentifier(p.getOriginalId.asScala), p)).as[(String, Publication)] + + + val conference = spark.read.load(s"$sourcePath/ConferenceInstances") + .select($"ConferenceInstanceId".as("ci"), $"DisplayName", $"Location", $"StartDate",$"EndDate" ) + val conferenceInstance = conference.joinWith(papers, papers("ConferenceInstanceId").equalTo(conference("ci"))) + .select($"_1.ci", $"_1.DisplayName", $"_1.Location", $"_1.StartDate",$"_1.EndDate", $"_2.PaperId").as[MagConferenceInstance] + + + magPubs.joinWith(conferenceInstance, col("_1").equalTo(conferenceInstance("PaperId")), "left") + .map(item => ConversionUtil.updatePubsWithConferenceInfo(item)) + .write + .mode(SaveMode.Overwrite) + .save(s"${parser.get("targetPath")}/merge_step_2_conference") + + + magPubs= spark.read.load(s"${parser.get("targetPath")}/merge_step_2_conference").as[Publication] + .map(p => (ConversionUtil.extractMagIdentifier(p.getOriginalId.asScala), p)).as[(String, Publication)] + + val paperUrlDataset = spark.read.load(s"$sourcePath/PaperUrls").as[MagPaperUrl].groupBy("PaperId").agg(collect_list(struct("sourceUrl")).as("instances")).as[MagUrl] + + + logger.info("Phase 5) enrich publication with URL and Instances") + magPubs.joinWith(paperUrlDataset, col("_1").equalTo(paperUrlDataset("PaperId")), "left") + .map { a: ((String, Publication), MagUrl) => ConversionUtil.addInstances((a._1._2, a._2)) } + .write.mode(SaveMode.Overwrite) + .save(s"${parser.get("targetPath")}/merge_step_3") + + +// logger.info("Phase 6) Enrich Publication with description") +// val pa = spark.read.load(s"${parser.get("sourcePath")}/PaperAbstractsInvertedIndex").as[MagPaperAbstract] +// pa.map(ConversionUtil.transformPaperAbstract).write.mode(SaveMode.Overwrite).save(s"${parser.get("targetPath")}/PaperAbstract") + + val paperAbstract = spark.read.load((s"${parser.get("targetPath")}/PaperAbstract")).as[MagPaperAbstract] + + + magPubs = spark.read.load(s"${parser.get("targetPath")}/merge_step_3").as[Publication] + .map(p => (ConversionUtil.extractMagIdentifier(p.getOriginalId.asScala), p)).as[(String, Publication)] + + magPubs.joinWith(paperAbstract, col("_1").equalTo(paperAbstract("PaperId")), "left") + .map(item => ConversionUtil.updatePubsWithDescription(item) + ).write.mode(SaveMode.Overwrite).save(s"${parser.get("targetPath")}/merge_step_4") + + + logger.info("Phase 7) Enrich Publication with FieldOfStudy") + + magPubs = spark.read.load(s"${parser.get("targetPath")}/merge_step_4").as[Publication] + .map(p => (ConversionUtil.extractMagIdentifier(p.getOriginalId.asScala), p)).as[(String, Publication)] + + val fos = spark.read.load(s"$sourcePath/FieldsOfStudy").select($"FieldOfStudyId".alias("fos"), $"DisplayName", $"MainType") + + val pfos = spark.read.load(s"$sourcePath/PaperFieldsOfStudy") + + val paperField = pfos.joinWith(fos, fos("fos").equalTo(pfos("FieldOfStudyId"))) + .select($"_1.FieldOfStudyId", $"_2.DisplayName", $"_2.MainType", $"_1.PaperId", $"_1.Score") + .groupBy($"PaperId").agg(collect_list(struct($"FieldOfStudyId", $"DisplayName", $"MainType", $"Score")).as("subjects")) + .as[MagFieldOfStudy] + + magPubs.joinWith(paperField, col("_1") + .equalTo(paperField("PaperId")), "left") + .map(item => ConversionUtil.updatePubsWithSubject(item)) + .write.mode(SaveMode.Overwrite) + .save(s"${parser.get("targetPath")}/mag_publication") + + + val s:RDD[Publication] = spark.read.load(s"${parser.get("targetPath")}/mag_publication").as[Publication] + .map(p=>Tuple2(p.getId, p)).rdd.reduceByKey((a:Publication, b:Publication) => ConversionUtil.mergePublication(a,b)) + .map(_._2) + + spark.createDataset(s).as[Publication].write.mode(SaveMode.Overwrite).save(s"${parser.get("targetPath")}/mag_publication_u") + + } +} diff --git a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/ActivitiesDecompressor.java b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/ActivitiesDecompressor.java new file mode 100644 index 0000000000..570fdef17c --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/ActivitiesDecompressor.java @@ -0,0 +1,146 @@ + +package eu.dnetlib.doiboost.orcid; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URI; + +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.io.SequenceFile; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.compress.CompressionCodec; +import org.apache.hadoop.io.compress.CompressionCodecFactory; +import org.mortbay.log.Log; + +import eu.dnetlib.doiboost.orcid.json.JsonWriter; +import eu.dnetlib.doiboost.orcid.model.WorkData; +import eu.dnetlib.doiboost.orcid.xml.XMLRecordParser; + +public class ActivitiesDecompressor { + + private static final int MAX_XML_WORKS_PARSED = -1; + private static final int XML_WORKS_PARSED_COUNTER_LOG_INTERVAL = 100000; + + public static void parseGzActivities(Configuration conf, String inputUri, Path outputPath) + throws Exception { + String uri = inputUri; + FileSystem fs = FileSystem.get(URI.create(uri), conf); + Path inputPath = new Path(uri); + CompressionCodecFactory factory = new CompressionCodecFactory(conf); + CompressionCodec codec = factory.getCodec(inputPath); + if (codec == null) { + System.err.println("No codec found for " + uri); + System.exit(1); + } + CompressionCodecFactory.removeSuffix(uri, codec.getDefaultExtension()); + InputStream gzipInputStream = null; + try { + gzipInputStream = codec.createInputStream(fs.open(inputPath)); + parseTarActivities(fs, conf, gzipInputStream, outputPath); + + } finally { + Log.debug("Closing gzip stream"); + IOUtils.closeStream(gzipInputStream); + } + } + + private static void parseTarActivities( + FileSystem fs, Configuration conf, InputStream gzipInputStream, Path outputPath) { + int counter = 0; + int doiFound = 0; + int errorFromOrcidFound = 0; + int xmlParserErrorFound = 0; + try (TarArchiveInputStream tais = new TarArchiveInputStream(gzipInputStream)) { + TarArchiveEntry entry = null; + + try (SequenceFile.Writer writer = SequenceFile + .createWriter( + conf, + SequenceFile.Writer.file(outputPath), + SequenceFile.Writer.keyClass(Text.class), + SequenceFile.Writer.valueClass(Text.class))) { + while ((entry = tais.getNextTarEntry()) != null) { + String filename = entry.getName(); + + try { + if (entry.isDirectory() || !filename.contains("works")) { + + } else { + Log.debug("XML work entry name: " + entry.getName()); + counter++; + BufferedReader br = new BufferedReader(new InputStreamReader(tais)); // Read directly from + // tarInput + String line; + StringBuffer buffer = new StringBuffer(); + while ((line = br.readLine()) != null) { + buffer.append(line); + } + WorkData workData = XMLRecordParser.VTDParseWorkData(buffer.toString().getBytes()); + if (workData != null) { + if (workData.getErrorCode() != null) { + errorFromOrcidFound += 1; + Log + .debug( + "error from Orcid with code " + + workData.getErrorCode() + + " for entry " + + entry.getName()); + continue; + } + if (workData.isDoiFound()) { + String jsonData = JsonWriter.create(workData); + Log.debug("oid: " + workData.getOid() + " data: " + jsonData); + + final Text key = new Text(workData.getOid()); + final Text value = new Text(jsonData); + + try { + writer.append(key, value); + } catch (IOException e) { + Log.debug("Writing to sequence file: " + e.getMessage()); + Log.debug(e); + throw new RuntimeException(e); + } + doiFound += 1; + } + + } else { + Log.warn("Data not retrievable [" + entry.getName() + "] " + buffer.toString()); + xmlParserErrorFound += 1; + } + } + } catch (Exception e) { + Log + .warn( + "Parsing work from tar archive and xml work: " + filename + " " + e.getMessage()); + Log.warn(e); + } + + if ((counter % XML_WORKS_PARSED_COUNTER_LOG_INTERVAL) == 0) { + Log.info("Current xml works parsed: " + counter); + } + + if ((MAX_XML_WORKS_PARSED > -1) && (counter > MAX_XML_WORKS_PARSED)) { + break; + } + } + } + } catch (IOException e) { + Log.warn("Parsing work from gzip archive: " + e.getMessage()); + Log.warn(e); + throw new RuntimeException(e); + } + Log.info("Activities parse completed"); + Log.info("Total XML works parsed: " + counter); + Log.info("Total doi found: " + doiFound); + Log.info("Error from Orcid found: " + errorFromOrcidFound); + Log.info("Error parsing xml work found: " + xmlParserErrorFound); + } +} diff --git a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/ORCIDToOAF.scala b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/ORCIDToOAF.scala new file mode 100644 index 0000000000..f230c604f7 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/ORCIDToOAF.scala @@ -0,0 +1,81 @@ +package eu.dnetlib.doiboost.orcid + +import eu.dnetlib.dhp.schema.oaf.{Author, Publication} +import eu.dnetlib.doiboost.DoiBoostMappingUtil +import eu.dnetlib.doiboost.DoiBoostMappingUtil.{ORCID, PID_TYPES, createSP, generateDataInfo, generateIdentifier} +import org.apache.commons.lang.StringUtils +import org.codehaus.jackson.map.ObjectMapper +import org.slf4j.{Logger, LoggerFactory} + +import scala.collection.JavaConverters._ + + +case class ORCIDItem(oid:String,name:String,surname:String,creditName:String,errorCode:String){} + + + +case class ORCIDElement(doi:String, authors:List[ORCIDItem]) {} +object ORCIDToOAF { + val logger: Logger = LoggerFactory.getLogger(ORCIDToOAF.getClass) + val mapper = new ObjectMapper + + def isJsonValid(inputStr: String): Boolean = { + import java.io.IOException + try { + mapper.readTree(inputStr) + true + } catch { + case e: IOException => + false + } + } + + def extractValueFromInputString(input: String): (String, String) = { + val i = input.indexOf('[') + if (i <5) { + return null + } + val orcidList = input.substring(i, input.length - 1) + val doi = input.substring(1, i - 1) + if (isJsonValid(orcidList)) { + (doi, orcidList) + } else null + } + + + def convertTOOAF(input:ORCIDElement) :Publication = { + val doi = input.doi + val pub:Publication = new Publication + pub.setPid(List(createSP(doi, "doi", PID_TYPES)).asJava) + pub.setDataInfo(generateDataInfo()) + pub.setId(generateIdentifier(pub, doi.toLowerCase)) + try{ + pub.setAuthor(input.authors.map(a=> { + generateAuthor(a.name, a.surname, a.creditName, a.oid) + }).asJava) + pub.setCollectedfrom(List(DoiBoostMappingUtil.createORIDCollectedFrom()).asJava) + pub.setDataInfo(DoiBoostMappingUtil.generateDataInfo()) + pub + } catch { + case e: Throwable => + logger.info(s"ERROR ON GENERATE Publication from $input") + null + } + } + + def generateAuthor(given: String, family: String, fullName:String, orcid: String): Author = { + val a = new Author + a.setName(given) + a.setSurname(family) + if (fullName!= null && fullName.nonEmpty) + a.setFullname(fullName) + else + a.setFullname(s"$given $family") + if (StringUtils.isNotBlank(orcid)) + a.setPid(List(createSP(orcid, ORCID, PID_TYPES)).asJava) + + a + } + + +} diff --git a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/OrcidAuthorsDOIsDataGen.java b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/OrcidAuthorsDOIsDataGen.java new file mode 100644 index 0000000000..70528a8f60 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/OrcidAuthorsDOIsDataGen.java @@ -0,0 +1,51 @@ + +package eu.dnetlib.doiboost.orcid; + +import java.io.IOException; + +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.mortbay.log.Log; + +import eu.dnetlib.dhp.application.ArgumentApplicationParser; + +public class OrcidAuthorsDOIsDataGen extends OrcidDSManager { + + private String activitiesFileNameTarGz; + private String outputAuthorsDOIsPath; + + public static void main(String[] args) throws IOException, Exception { + OrcidAuthorsDOIsDataGen orcidAuthorsDOIsDataGen = new OrcidAuthorsDOIsDataGen(); + orcidAuthorsDOIsDataGen.loadArgs(args); + orcidAuthorsDOIsDataGen.generateAuthorsDOIsData(); + } + + public void generateAuthorsDOIsData() throws Exception { + Configuration conf = initConfigurationObject(); + FileSystem fs = initFileSystemObject(conf); + String tarGzUri = hdfsServerUri.concat(hdfsOrcidDefaultPath).concat(activitiesFileNameTarGz); + Path outputPath = new Path(hdfsServerUri.concat(hdfsOrcidDefaultPath).concat(outputAuthorsDOIsPath)); + ActivitiesDecompressor.parseGzActivities(conf, tarGzUri, outputPath); + } + + private void loadArgs(String[] args) throws IOException, Exception { + final ArgumentApplicationParser parser = new ArgumentApplicationParser( + IOUtils + .toString( + OrcidAuthorsDOIsDataGen.class + .getResourceAsStream( + "/eu/dnetlib/dhp/doiboost/create_orcid_authors_dois_data.json"))); + parser.parseArgument(args); + + hdfsServerUri = parser.get("hdfsServerUri"); + Log.info("HDFS URI: " + hdfsServerUri); + hdfsOrcidDefaultPath = parser.get("hdfsOrcidDefaultPath"); + Log.info("Default Path: " + hdfsOrcidDefaultPath); + activitiesFileNameTarGz = parser.get("activitiesFileNameTarGz"); + Log.info("Activities File Name: " + activitiesFileNameTarGz); + outputAuthorsDOIsPath = parser.get("outputAuthorsDOIsPath"); + Log.info("Output Authors DOIs Data: " + outputAuthorsDOIsPath); + } +} diff --git a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/OrcidDSManager.java b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/OrcidDSManager.java new file mode 100644 index 0000000000..4f846bdf3c --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/OrcidDSManager.java @@ -0,0 +1,81 @@ + +package eu.dnetlib.doiboost.orcid; + +import java.io.IOException; +import java.net.URI; + +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.mortbay.log.Log; + +import eu.dnetlib.dhp.application.ArgumentApplicationParser; + +public class OrcidDSManager { + + protected String hdfsServerUri; + protected String hdfsOrcidDefaultPath; + private String summariesFileNameTarGz; + private String outputAuthorsPath; + + public static void main(String[] args) throws IOException, Exception { + OrcidDSManager orcidDSManager = new OrcidDSManager(); + orcidDSManager.loadArgs(args); + orcidDSManager.generateAuthors(); + } + + public void generateAuthors() throws Exception { + Configuration conf = initConfigurationObject(); + FileSystem fs = initFileSystemObject(conf); + String tarGzUri = hdfsServerUri.concat(hdfsOrcidDefaultPath).concat(summariesFileNameTarGz); + Path outputPath = new Path( + hdfsServerUri + .concat(hdfsOrcidDefaultPath) + .concat(outputAuthorsPath) + .concat("authors.seq")); + SummariesDecompressor.parseGzSummaries(conf, tarGzUri, outputPath); + } + + protected Configuration initConfigurationObject() { + // ====== Init HDFS File System Object + Configuration conf = new Configuration(); + // Set FileSystem URI + conf.set("fs.defaultFS", hdfsServerUri.concat(hdfsOrcidDefaultPath)); + // Because of Maven + conf.set("fs.hdfs.impl", org.apache.hadoop.hdfs.DistributedFileSystem.class.getName()); + conf.set("fs.file.impl", org.apache.hadoop.fs.LocalFileSystem.class.getName()); + return conf; + } + + protected FileSystem initFileSystemObject(Configuration conf) { + // Get the filesystem - HDFS + FileSystem fs = null; + try { + fs = FileSystem.get(URI.create(hdfsServerUri.concat(hdfsOrcidDefaultPath)), conf); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + return fs; + } + + private void loadArgs(String[] args) throws IOException, Exception { + final ArgumentApplicationParser parser = new ArgumentApplicationParser( + IOUtils + .toString( + OrcidDSManager.class + .getResourceAsStream( + "/eu/dnetlib/dhp/doiboost/create_orcid_authors_data.json"))); + parser.parseArgument(args); + + hdfsServerUri = parser.get("hdfsServerUri"); + Log.info("HDFS URI: " + hdfsServerUri); + hdfsOrcidDefaultPath = parser.get("hdfsOrcidDefaultPath"); + Log.info("Default Path: " + hdfsOrcidDefaultPath); + summariesFileNameTarGz = parser.get("summariesFileNameTarGz"); + Log.info("Summaries File Name: " + summariesFileNameTarGz); + outputAuthorsPath = parser.get("outputAuthorsPath"); + Log.info("Output Authors Data: " + outputAuthorsPath); + } +} diff --git a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/OrcidDownloader.java b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/OrcidDownloader.java new file mode 100644 index 0000000000..2e1a199da9 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/OrcidDownloader.java @@ -0,0 +1,203 @@ + +package eu.dnetlib.doiboost.orcid; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.text.SimpleDateFormat; +import java.util.Arrays; +import java.util.Date; +import java.util.List; + +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.SequenceFile; +import org.apache.hadoop.io.Text; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.mortbay.log.Log; + +import eu.dnetlib.dhp.application.ArgumentApplicationParser; + +public class OrcidDownloader extends OrcidDSManager { + + static final int REQ_LIMIT = 24; +// static final int REQ_MAX_TEST = 100; + static final int RECORD_PARSED_COUNTER_LOG_INTERVAL = 10000; + static final String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss"; + static final String lastUpdate = "2019-09-30 00:00:00"; + private String lambdaFileName; + private String outputPath; + private String token; + + public static void main(String[] args) throws IOException, Exception { + OrcidDownloader orcidDownloader = new OrcidDownloader(); + orcidDownloader.loadArgs(args); + orcidDownloader.parseLambdaFile(); + } + + private String downloadRecord(String orcidId) { + try (CloseableHttpClient client = HttpClients.createDefault()) { + HttpGet httpGet = new HttpGet("https://api.orcid.org/v3.0/" + orcidId + "/record"); + httpGet.addHeader("Accept", "application/vnd.orcid+xml"); + httpGet.addHeader("Authorization", String.format("Bearer %s", token)); + CloseableHttpResponse response = client.execute(httpGet); + if (response.getStatusLine().getStatusCode() != 200) { + Log + .warn( + "Downloading " + orcidId + " status code: " + response.getStatusLine().getStatusCode()); + return new String(""); + } + return IOUtils.toString(response.getEntity().getContent()); + + } catch (Throwable e) { + Log.warn("Downloading " + orcidId, e.getMessage()); + + } + return new String(""); + } + + public void parseLambdaFile() throws Exception { + int parsedRecordsCounter = 0; + int downloadedRecordsCounter = 0; + int savedRecordsCounter = 0; + long startDownload = 0; + Configuration conf = initConfigurationObject(); + FileSystem fs = initFileSystemObject(conf); + String lambdaFileUri = hdfsServerUri.concat(hdfsOrcidDefaultPath).concat(lambdaFileName); + Path hdfsreadpath = new Path(lambdaFileUri); + FSDataInputStream lambdaFileStream = fs.open(hdfsreadpath); + Path hdfsoutputPath = new Path( + hdfsServerUri + .concat(hdfsOrcidDefaultPath) + .concat(outputPath) + .concat("orcid_records.seq")); + + try (SequenceFile.Writer writer = SequenceFile + .createWriter( + conf, + SequenceFile.Writer.file(hdfsoutputPath), + SequenceFile.Writer.keyClass(Text.class), + SequenceFile.Writer.valueClass(Text.class))) { + + try (BufferedReader br = new BufferedReader(new InputStreamReader(lambdaFileStream))) { + String line; + int nReqTmp = 0; + startDownload = System.currentTimeMillis(); + long startReqTmp = System.currentTimeMillis(); + while ((line = br.readLine()) != null) { + parsedRecordsCounter++; + // skip headers line + if (parsedRecordsCounter == 1) { + continue; + } + String[] values = line.split(","); + List recordInfo = Arrays.asList(values); + String orcidId = recordInfo.get(0); + if (isModified(orcidId, recordInfo.get(3))) { + String record = downloadRecord(orcidId); + downloadedRecordsCounter++; + if (!record.isEmpty()) { + String compressRecord = ArgumentApplicationParser.compressArgument(record); + final Text key = new Text(recordInfo.get(0)); + final Text value = new Text(compressRecord); + + try { + writer.append(key, value); + savedRecordsCounter++; + } catch (IOException e) { + Log.warn("Writing to sequence file: " + e.getMessage()); + Log.warn(e); + throw new RuntimeException(e); + } + } + } + long endReq = System.currentTimeMillis(); + nReqTmp++; + if (nReqTmp == REQ_LIMIT) { + long reqSessionDuration = endReq - startReqTmp; + if (reqSessionDuration <= 1000) { + Log + .warn( + "\nreqSessionDuration: " + + reqSessionDuration + + " nReqTmp: " + + nReqTmp + + " wait ...."); + Thread.sleep(1000 - reqSessionDuration); + } else { + nReqTmp = 0; + startReqTmp = System.currentTimeMillis(); + } + } + +// if (parsedRecordsCounter > REQ_MAX_TEST) { +// break; +// } + if ((parsedRecordsCounter % RECORD_PARSED_COUNTER_LOG_INTERVAL) == 0) { + Log + .info( + "Current parsed: " + + parsedRecordsCounter + + " downloaded: " + + downloadedRecordsCounter + + " saved: " + + savedRecordsCounter); +// if (parsedRecordsCounter > REQ_MAX_TEST) { +// break; +// } + } + } + long endDownload = System.currentTimeMillis(); + long downloadTime = endDownload - startDownload; + Log.info("Download time: " + ((downloadTime / 1000) / 60) + " minutes"); + } + } + lambdaFileStream.close(); + Log.info("Download started at: " + new Date(startDownload).toString()); + Log.info("Parsed Records Counter: " + parsedRecordsCounter); + Log.info("Downloaded Records Counter: " + downloadedRecordsCounter); + Log.info("Saved Records Counter: " + savedRecordsCounter); + } + + private void loadArgs(String[] args) throws IOException, Exception { + final ArgumentApplicationParser parser = new ArgumentApplicationParser( + IOUtils + .toString( + OrcidDownloader.class + .getResourceAsStream( + "/eu/dnetlib/dhp/doiboost/download_orcid_data.json"))); + parser.parseArgument(args); + + hdfsServerUri = parser.get("hdfsServerUri"); + Log.info("HDFS URI: " + hdfsServerUri); + hdfsOrcidDefaultPath = parser.get("hdfsOrcidDefaultPath"); + Log.info("Default Path: " + hdfsOrcidDefaultPath); + lambdaFileName = parser.get("lambdaFileName"); + Log.info("Lambda File Name: " + lambdaFileName); + outputPath = parser.get("outputPath"); + Log.info("Output Data: " + outputPath); + token = parser.get("token"); + } + + private boolean isModified(String orcidId, String modifiedDate) { + Date modifiedDateDt = null; + Date lastUpdateDt = null; + try { + if (modifiedDate.length() != 19) { + modifiedDate = modifiedDate.substring(0, 19); + } + modifiedDateDt = new SimpleDateFormat(DATE_FORMAT).parse(modifiedDate); + lastUpdateDt = new SimpleDateFormat(DATE_FORMAT).parse(lastUpdate); + } catch (Exception e) { + Log.warn("[" + orcidId + "] Parsing date: ", e.getMessage()); + return true; + } + return modifiedDateDt.after(lastUpdateDt); + } +} diff --git a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/SparkConvertORCIDToOAF.scala b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/SparkConvertORCIDToOAF.scala new file mode 100644 index 0000000000..1cd9ba4d42 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/SparkConvertORCIDToOAF.scala @@ -0,0 +1,44 @@ +package eu.dnetlib.doiboost.orcid + +import eu.dnetlib.dhp.application.ArgumentApplicationParser +import eu.dnetlib.dhp.schema.oaf.Publication +import eu.dnetlib.doiboost.mag.ConversionUtil +import org.apache.commons.io.IOUtils +import org.apache.spark.SparkConf +import org.apache.spark.rdd.RDD +import org.apache.spark.sql.{Dataset, Encoder, Encoders, SaveMode, SparkSession} +import org.slf4j.{Logger, LoggerFactory} + +object SparkConvertORCIDToOAF { + + + + def main(args: Array[String]): Unit = { + + val logger: Logger = LoggerFactory.getLogger(SparkConvertORCIDToOAF.getClass) + val conf: SparkConf = new SparkConf() + val parser = new ArgumentApplicationParser(IOUtils.toString(SparkConvertORCIDToOAF.getClass.getResourceAsStream("/eu/dnetlib/dhp/doiboost/convert_map_to_oaf_params.json"))) + parser.parseArgument(args) + val spark: SparkSession = + SparkSession + .builder() + .config(conf) + .appName(getClass.getSimpleName) + .master(parser.get("master")).getOrCreate() + + implicit val mapEncoderPubs: Encoder[Publication] = Encoders.kryo[Publication] + implicit val tupleForJoinEncoder: Encoder[(String, Publication)] = Encoders.tuple(Encoders.STRING, mapEncoderPubs) + import spark.implicits._ + val sourcePath = parser.get("sourcePath") + val targetPath = parser.get("targetPath") + val dataset:Dataset[ORCIDElement] = spark.read.json(sourcePath).as[ORCIDElement] + + + logger.info("Converting ORCID to OAF") + val d:RDD[Publication] = dataset.map(o => ORCIDToOAF.convertTOOAF(o)).filter(p=>p!=null).map(p=>(p.getId,p)).rdd.reduceByKey(ConversionUtil.mergePublication) + .map(_._2) + + spark.createDataset(d).as[Publication].write.mode(SaveMode.Overwrite).save(targetPath) + } + +} diff --git a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/SparkGenerateDoiAuthorList.java b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/SparkGenerateDoiAuthorList.java new file mode 100644 index 0000000000..b4239bba29 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/SparkGenerateDoiAuthorList.java @@ -0,0 +1,180 @@ + +package eu.dnetlib.doiboost.orcid; + +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.compress.GzipCodec; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaPairRDD; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.api.java.function.Function; +import org.apache.spark.api.java.function.MapFunction; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.esotericsoftware.minlog.Log; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.gson.JsonElement; +import com.google.gson.JsonParser; + +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.doiboost.orcid.model.AuthorData; +import eu.dnetlib.doiboost.orcid.model.WorkData; +import scala.Tuple2; + +public class SparkGenerateDoiAuthorList { + + public static void main(String[] args) throws IOException, Exception { + Logger logger = LoggerFactory.getLogger(SparkGenerateDoiAuthorList.class); + logger.info("[ SparkGenerateDoiAuthorList STARTED]"); + + final ArgumentApplicationParser parser = new ArgumentApplicationParser( + IOUtils + .toString( + SparkGenerateDoiAuthorList.class + .getResourceAsStream( + "/eu/dnetlib/dhp/doiboost/gen_doi_author_list_orcid_parameters.json"))); + parser.parseArgument(args); + Boolean isSparkSessionManaged = Optional + .ofNullable(parser.get("isSparkSessionManaged")) + .map(Boolean::valueOf) + .orElse(Boolean.TRUE); + logger.info("isSparkSessionManaged: {}", isSparkSessionManaged); + final String workingPath = parser.get("workingPath"); + logger.info("workingPath: ", workingPath); + final String outputDoiAuthorListPath = parser.get("outputDoiAuthorListPath"); + logger.info("outputDoiAuthorListPath: ", outputDoiAuthorListPath); + + SparkConf conf = new SparkConf(); + runWithSparkSession( + conf, + isSparkSessionManaged, + spark -> { + JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); + + JavaPairRDD summariesRDD = sc + .sequenceFile(workingPath + "../orcid_summaries/output/authors.seq", Text.class, Text.class); + Dataset summariesDataset = spark + .createDataset( + summariesRDD.map(seq -> loadAuthorFromJson(seq._1(), seq._2())).rdd(), + Encoders.bean(AuthorData.class)); + + JavaPairRDD activitiesRDD = sc + .sequenceFile(workingPath + "/output/*.seq", Text.class, Text.class); + Dataset activitiesDataset = spark + .createDataset( + activitiesRDD.map(seq -> loadWorkFromJson(seq._1(), seq._2())).rdd(), + Encoders.bean(WorkData.class)); + + Function, Tuple2>> toAuthorListFunction = data -> { + try { + String doi = data._1(); + if (doi == null) { + return null; + } + AuthorData author = data._2(); + if (author == null) { + return null; + } + List toAuthorList = Arrays.asList(author); + return new Tuple2<>(doi, toAuthorList); + } catch (Exception e) { + Log.error("toAuthorListFunction ERROR", e); + return null; + } + }; + + JavaRDD>> doisRDD = activitiesDataset + .joinWith( + summariesDataset, + activitiesDataset.col("oid").equalTo(summariesDataset.col("oid")), "inner") + .map( + (MapFunction, Tuple2>) value -> { + WorkData w = value._1; + AuthorData a = value._2; + return new Tuple2<>(w.getDoi(), a); + }, + Encoders.tuple(Encoders.STRING(), Encoders.bean(AuthorData.class))) + .filter(Objects::nonNull) + .toJavaRDD() + .map(toAuthorListFunction); + + JavaPairRDD + .fromJavaRDD(doisRDD) + .reduceByKey((d1, d2) -> { + try { + if (d1 != null && d2 != null) { + Stream mergedStream = Stream + .concat( + d1.stream(), + d2.stream()); + List mergedAuthors = mergedStream.collect(Collectors.toList()); + return mergedAuthors; + } + if (d1 != null) { + return d1; + } + if (d2 != null) { + return d2; + } + } catch (Exception e) { + Log.error("mergeAuthorsFunction ERROR", e); + return null; + } + return null; + }) + .mapToPair( + s -> { + ObjectMapper mapper = new ObjectMapper(); + return new Tuple2<>(s._1(), mapper.writeValueAsString(s._2())); + }) + .repartition(10) + .saveAsTextFile(workingPath + outputDoiAuthorListPath); + }); + + } + + private static AuthorData loadAuthorFromJson(Text orcidId, Text json) { + AuthorData authorData = new AuthorData(); + authorData.setOid(orcidId.toString()); + JsonElement jElement = new JsonParser().parse(json.toString()); + authorData.setName(getJsonValue(jElement, "name")); + authorData.setSurname(getJsonValue(jElement, "surname")); + authorData.setCreditName(getJsonValue(jElement, "creditname")); + return authorData; + } + + private static WorkData loadWorkFromJson(Text orcidId, Text json) { + WorkData workData = new WorkData(); + workData.setOid(orcidId.toString()); + JsonElement jElement = new JsonParser().parse(json.toString()); + workData.setDoi(getJsonValue(jElement, "doi")); + return workData; + } + + private static String getJsonValue(JsonElement jElement, String property) { + if (jElement.getAsJsonObject().has(property)) { + JsonElement name = null; + name = jElement.getAsJsonObject().get(property); + if (name != null && !name.isJsonNull()) { + return name.getAsString(); + } + } + return null; + } +} diff --git a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/SparkOrcidGenerateAuthors.java b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/SparkOrcidGenerateAuthors.java new file mode 100644 index 0000000000..4e18ab8401 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/SparkOrcidGenerateAuthors.java @@ -0,0 +1,165 @@ + +package eu.dnetlib.doiboost.orcid; + +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; + +import java.io.IOException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.List; +import java.util.Optional; + +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.io.Text; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.api.java.function.Function; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.SaveMode; +import org.apache.spark.util.LongAccumulator; +import org.mortbay.log.Log; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.doiboost.orcid.model.DownloadedRecordData; +import scala.Tuple2; + +public class SparkOrcidGenerateAuthors { + + static final String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss"; + static final String lastUpdate = "2019-09-30 00:00:00"; + + public static void main(String[] args) throws IOException, Exception { + Logger logger = LoggerFactory.getLogger(SparkOrcidGenerateAuthors.class); + logger.info("[ SparkOrcidGenerateAuthors STARTED]"); + + final ArgumentApplicationParser parser = new ArgumentApplicationParser( + IOUtils + .toString( + SparkOrcidGenerateAuthors.class + .getResourceAsStream( + "/eu/dnetlib/dhp/doiboost/gen_orcid_authors_parameters.json"))); + parser.parseArgument(args); + Boolean isSparkSessionManaged = Optional + .ofNullable(parser.get("isSparkSessionManaged")) + .map(Boolean::valueOf) + .orElse(Boolean.TRUE); + logger.info("isSparkSessionManaged: {}", isSparkSessionManaged); + final String workingPath = parser.get("workingPath"); + logger.info("workingPath: ", workingPath); + final String outputAuthorsPath = parser.get("outputAuthorsPath"); + logger.info("outputAuthorsPath: ", outputAuthorsPath); + final String token = parser.get("token"); + + SparkConf conf = new SparkConf(); + runWithSparkSession( + conf, + isSparkSessionManaged, + spark -> { + JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); + + LongAccumulator parsedRecordsAcc = sc.sc().longAccumulator("parsedRecords"); + LongAccumulator modifiedRecordsAcc = sc.sc().longAccumulator("modifiedRecords"); + LongAccumulator downloadedRecordsAcc = sc.sc().longAccumulator("downloadedRecords"); + LongAccumulator alreadyDownloadedRecords = sc.sc().longAccumulator("alreadyDownloadedRecords"); + JavaRDD lamdaFileRDD = sc.textFile(workingPath + "lamdafiles"); + + JavaRDD downloadedRDD = sc.textFile(workingPath + "downloaded"); + Function getOrcidIdFunction = line -> { + try { + String[] values = line.split(","); + return values[0].substring(1); + } catch (Exception e) { + return new String(""); + } + }; + List downloadedRecords = downloadedRDD.map(getOrcidIdFunction).collect(); + + Function isModifiedAfterFilter = line -> { + String[] values = line.split(","); + String orcidId = values[0]; + parsedRecordsAcc.add(1); + if (isModified(orcidId, values[3])) { + modifiedRecordsAcc.add(1); + return true; + } + return false; + }; + Function isNotDownloadedFilter = line -> { + String[] values = line.split(","); + String orcidId = values[0]; + if (downloadedRecords.contains(orcidId)) { + alreadyDownloadedRecords.add(1); + return false; + } + return true; + }; + Function> downloadRecordFunction = line -> { + String[] values = line.split(","); + String orcidId = values[0]; + String modifiedDate = values[3]; + return downloadRecord(orcidId, modifiedDate, token, downloadedRecordsAcc); + }; + + lamdaFileRDD + .filter(isModifiedAfterFilter) + .filter(isNotDownloadedFilter) + .map(downloadRecordFunction) + .rdd() + .saveAsTextFile(workingPath.concat(outputAuthorsPath)); + }); + + } + + private static boolean isModified(String orcidId, String modifiedDate) { + Date modifiedDateDt = null; + Date lastUpdateDt = null; + try { + if (modifiedDate.length() != 19) { + modifiedDate = modifiedDate.substring(0, 19); + } + modifiedDateDt = new SimpleDateFormat(DATE_FORMAT).parse(modifiedDate); + lastUpdateDt = new SimpleDateFormat(DATE_FORMAT).parse(lastUpdate); + } catch (Exception e) { + Log.warn("[" + orcidId + "] Parsing date: ", e.getMessage()); + return true; + } + return modifiedDateDt.after(lastUpdateDt); + } + + private static Tuple2 downloadRecord(String orcidId, String modifiedDate, String token, + LongAccumulator downloadedRecordsAcc) { + final DownloadedRecordData data = new DownloadedRecordData(); + data.setOrcidId(orcidId); + data.setModifiedDate(modifiedDate); + try (CloseableHttpClient client = HttpClients.createDefault()) { + HttpGet httpGet = new HttpGet("https://api.orcid.org/v3.0/" + orcidId + "/record"); + httpGet.addHeader("Accept", "application/vnd.orcid+xml"); + httpGet.addHeader("Authorization", String.format("Bearer %s", token)); + CloseableHttpResponse response = client.execute(httpGet); + int statusCode = response.getStatusLine().getStatusCode(); + data.setStatusCode(statusCode); + if (statusCode != 200) { + Log + .warn( + "Downloading " + orcidId + " status code: " + response.getStatusLine().getStatusCode()); + return data.toTuple2(); + } + downloadedRecordsAcc.add(1); + data + .setCompressedData( + ArgumentApplicationParser.compressArgument(IOUtils.toString(response.getEntity().getContent()))); + } catch (Throwable e) { + Log.warn("Downloading " + orcidId, e.getMessage()); + data.setErrorMessage(e.getMessage()); + return data.toTuple2(); + } + return data.toTuple2(); + } +} diff --git a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/SparkPartitionLambdaFile.java b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/SparkPartitionLambdaFile.java new file mode 100644 index 0000000000..ca6f0f6c4c --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/SparkPartitionLambdaFile.java @@ -0,0 +1,50 @@ + +package eu.dnetlib.doiboost.orcid; + +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; + +import java.io.IOException; +import java.util.Optional; + +import org.apache.commons.io.IOUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import eu.dnetlib.dhp.application.ArgumentApplicationParser; + +public class SparkPartitionLambdaFile { + + public static void main(String[] args) throws IOException, Exception { + Logger logger = LoggerFactory.getLogger(SparkOrcidGenerateAuthors.class); + + final ArgumentApplicationParser parser = new ArgumentApplicationParser( + IOUtils + .toString( + SparkOrcidGenerateAuthors.class + .getResourceAsStream( + "/eu/dnetlib/dhp/doiboost/gen_orcid_authors_parameters.json"))); + parser.parseArgument(args); + Boolean isSparkSessionManaged = Optional + .ofNullable(parser.get("isSparkSessionManaged")) + .map(Boolean::valueOf) + .orElse(Boolean.TRUE); + final String workingPath = parser.get("workingPath"); + + SparkConf conf = new SparkConf(); + runWithSparkSession( + conf, + isSparkSessionManaged, + spark -> { + JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); + JavaRDD lamdaFileRDD = sc.textFile(workingPath + "last_modified.csv"); + + lamdaFileRDD + .repartition(20) + .saveAsTextFile(workingPath.concat("lamdafiles")); + }); + } + +} diff --git a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/SummariesDecompressor.java b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/SummariesDecompressor.java new file mode 100644 index 0000000000..f0bbb5c327 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/SummariesDecompressor.java @@ -0,0 +1,158 @@ + +package eu.dnetlib.doiboost.orcid; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URI; + +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.io.SequenceFile; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.compress.CompressionCodec; +import org.apache.hadoop.io.compress.CompressionCodecFactory; +import org.mortbay.log.Log; + +import eu.dnetlib.doiboost.orcid.json.JsonWriter; +import eu.dnetlib.doiboost.orcid.model.AuthorData; +import eu.dnetlib.doiboost.orcid.xml.XMLRecordParser; + +public class SummariesDecompressor { + + private static final int MAX_XML_RECORDS_PARSED = -1; + + public static void parseGzSummaries(Configuration conf, String inputUri, Path outputPath) + throws Exception { + String uri = inputUri; + FileSystem fs = FileSystem.get(URI.create(uri), conf); + Path inputPath = new Path(uri); + CompressionCodecFactory factory = new CompressionCodecFactory(conf); + CompressionCodec codec = factory.getCodec(inputPath); + if (codec == null) { + System.err.println("No codec found for " + uri); + System.exit(1); + } + CompressionCodecFactory.removeSuffix(uri, codec.getDefaultExtension()); + InputStream gzipInputStream = null; + try { + gzipInputStream = codec.createInputStream(fs.open(inputPath)); + parseTarSummaries(fs, conf, gzipInputStream, outputPath); + + } finally { + Log.debug("Closing gzip stream"); + IOUtils.closeStream(gzipInputStream); + } + } + + private static void parseTarSummaries( + FileSystem fs, Configuration conf, InputStream gzipInputStream, Path outputPath) { + int counter = 0; + int nameFound = 0; + int surnameFound = 0; + int creditNameFound = 0; + int errorFromOrcidFound = 0; + int xmlParserErrorFound = 0; + try (TarArchiveInputStream tais = new TarArchiveInputStream(gzipInputStream)) { + TarArchiveEntry entry = null; + + try (SequenceFile.Writer writer = SequenceFile + .createWriter( + conf, + SequenceFile.Writer.file(outputPath), + SequenceFile.Writer.keyClass(Text.class), + SequenceFile.Writer.valueClass(Text.class))) { + while ((entry = tais.getNextTarEntry()) != null) { + String filename = entry.getName(); + try { + if (entry.isDirectory()) { + Log.debug("Directory entry name: " + entry.getName()); + } else { + Log.debug("XML record entry name: " + entry.getName()); + counter++; + BufferedReader br = new BufferedReader(new InputStreamReader(tais)); // Read directly from + // tarInput + String line; + StringBuffer buffer = new StringBuffer(); + while ((line = br.readLine()) != null) { + buffer.append(line); + } + AuthorData authorData = XMLRecordParser.VTDParseAuthorData(buffer.toString().getBytes()); + if (authorData != null) { + if (authorData.getErrorCode() != null) { + errorFromOrcidFound += 1; + Log + .debug( + "error from Orcid with code " + + authorData.getErrorCode() + + " for oid " + + entry.getName()); + continue; + } + String jsonData = JsonWriter.create(authorData); + Log.debug("oid: " + authorData.getOid() + " data: " + jsonData); + + final Text key = new Text(authorData.getOid()); + final Text value = new Text(jsonData); + + try { + writer.append(key, value); + } catch (IOException e) { + Log.debug("Writing to sequence file: " + e.getMessage()); + Log.debug(e); + throw new RuntimeException(e); + } + + if (authorData.getName() != null) { + nameFound += 1; + } + if (authorData.getSurname() != null) { + surnameFound += 1; + } + if (authorData.getCreditName() != null) { + creditNameFound += 1; + } + + } else { + Log.warn("Data not retrievable [" + entry.getName() + "] " + buffer.toString()); + xmlParserErrorFound += 1; + } + } + } catch (Exception e) { + Log + .warn( + "Parsing record from tar archive and xml record: " + + filename + + " " + + e.getMessage()); + Log.warn(e); + } + + if ((counter % 100000) == 0) { + Log.info("Current xml records parsed: " + counter); + } + + if ((MAX_XML_RECORDS_PARSED > -1) && (counter > MAX_XML_RECORDS_PARSED)) { + break; + } + } + } + } catch (IOException e) { + Log.warn("Parsing record from gzip archive: " + e.getMessage()); + Log.warn(e); + throw new RuntimeException(e); + } + Log.info("Summaries parse completed"); + Log.info("Total XML records parsed: " + counter); + Log.info("Name found: " + nameFound); + Log.info("Surname found: " + surnameFound); + Log.info("Credit name found: " + creditNameFound); + Log.info("Error from Orcid found: " + errorFromOrcidFound); + Log.info("Error parsing xml record found: " + xmlParserErrorFound); + } +} diff --git a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/json/JsonWriter.java b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/json/JsonWriter.java new file mode 100644 index 0000000000..35676d5bab --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/json/JsonWriter.java @@ -0,0 +1,28 @@ + +package eu.dnetlib.doiboost.orcid.json; + +import com.google.gson.JsonObject; + +import eu.dnetlib.doiboost.orcid.model.AuthorData; +import eu.dnetlib.doiboost.orcid.model.WorkData; + +public class JsonWriter { + + public static String create(AuthorData authorData) { + JsonObject author = new JsonObject(); + author.addProperty("oid", authorData.getOid()); + author.addProperty("name", authorData.getName()); + author.addProperty("surname", authorData.getSurname()); + if (authorData.getCreditName() != null) { + author.addProperty("creditname", authorData.getCreditName()); + } + return author.toString(); + } + + public static String create(WorkData workData) { + JsonObject work = new JsonObject(); + work.addProperty("oid", workData.getOid()); + work.addProperty("doi", workData.getDoi()); + return work.toString(); + } +} diff --git a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/model/AuthorData.java b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/model/AuthorData.java new file mode 100644 index 0000000000..29551c347a --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/model/AuthorData.java @@ -0,0 +1,53 @@ + +package eu.dnetlib.doiboost.orcid.model; + +import java.io.Serializable; + +public class AuthorData implements Serializable { + + private String oid; + private String name; + private String surname; + private String creditName; + private String errorCode; + + public String getErrorCode() { + return errorCode; + } + + public void setErrorCode(String errorCode) { + this.errorCode = errorCode; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getSurname() { + return surname; + } + + public void setSurname(String surname) { + this.surname = surname; + } + + public String getCreditName() { + return creditName; + } + + public void setCreditName(String creditName) { + this.creditName = creditName; + } + + public String getOid() { + return oid; + } + + public void setOid(String oid) { + this.oid = oid; + } +} diff --git a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/model/DownloadedRecordData.java b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/model/DownloadedRecordData.java new file mode 100644 index 0000000000..f66ef82a2f --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/model/DownloadedRecordData.java @@ -0,0 +1,76 @@ + +package eu.dnetlib.doiboost.orcid.model; + +import java.io.Serializable; + +import org.apache.hadoop.io.Text; + +import com.google.gson.JsonObject; + +import scala.Tuple2; + +public class DownloadedRecordData implements Serializable { + + private String orcidId; + private String modifiedDate; + private String statusCode; + private String compressedData; + private String errorMessage; + + public Tuple2 toTuple2() { + JsonObject data = new JsonObject(); + data.addProperty("statusCode", getStatusCode()); + data.addProperty("modifiedDate", getModifiedDate()); + if (getCompressedData() != null) { + data.addProperty("compressedData", getCompressedData()); + } + if (getErrorMessage() != null) { + data.addProperty("errorMessage", getErrorMessage()); + } + return new Tuple2<>(orcidId, data.toString()); + } + + public String getErrorMessage() { + return errorMessage; + } + + public void setErrorMessage(String errorMessage) { + this.errorMessage = errorMessage; + } + + public String getOrcidId() { + return orcidId; + } + + public void setOrcidId(String orcidId) { + this.orcidId = orcidId; + } + + public int getStatusCode() { + try { + return Integer.parseInt(statusCode); + } catch (Exception e) { + return -2; + } + } + + public void setStatusCode(int statusCode) { + this.statusCode = Integer.toString(statusCode); + } + + public String getCompressedData() { + return compressedData; + } + + public void setCompressedData(String compressedData) { + this.compressedData = compressedData; + } + + public String getModifiedDate() { + return modifiedDate; + } + + public void setModifiedDate(String modifiedDate) { + this.modifiedDate = modifiedDate; + } +} diff --git a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/model/WorkData.java b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/model/WorkData.java new file mode 100644 index 0000000000..db1728a9b4 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/model/WorkData.java @@ -0,0 +1,45 @@ + +package eu.dnetlib.doiboost.orcid.model; + +import java.io.Serializable; + +public class WorkData implements Serializable { + + private String oid; + private String doi; + private boolean doiFound = false; + + public boolean isDoiFound() { + return doiFound; + } + + public void setDoiFound(boolean doiFound) { + this.doiFound = doiFound; + } + + public String getOid() { + return oid; + } + + public void setOid(String oid) { + this.oid = oid; + } + + public String getDoi() { + return doi; + } + + public void setDoi(String doi) { + this.doi = doi; + } + + public String getErrorCode() { + return errorCode; + } + + public void setErrorCode(String errorCode) { + this.errorCode = errorCode; + } + + private String errorCode; +} diff --git a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/xml/XMLRecordParser.java b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/xml/XMLRecordParser.java new file mode 100644 index 0000000000..2e43f4d3e6 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/xml/XMLRecordParser.java @@ -0,0 +1,123 @@ + +package eu.dnetlib.doiboost.orcid.xml; + +import java.util.Arrays; +import java.util.List; + +import com.ximpleware.AutoPilot; +import com.ximpleware.EOFException; +import com.ximpleware.EncodingException; +import com.ximpleware.EntityException; +import com.ximpleware.ParseException; +import com.ximpleware.VTDGen; +import com.ximpleware.VTDNav; + +import eu.dnetlib.dhp.parser.utility.VtdException; +import eu.dnetlib.dhp.parser.utility.VtdUtilityParser; +import eu.dnetlib.doiboost.orcid.model.AuthorData; +import eu.dnetlib.doiboost.orcid.model.WorkData; + +public class XMLRecordParser { + + private static final String NS_COMMON_URL = "http://www.orcid.org/ns/common"; + private static final String NS_COMMON = "common"; + private static final String NS_PERSON_URL = "http://www.orcid.org/ns/person"; + private static final String NS_PERSON = "person"; + private static final String NS_DETAILS_URL = "http://www.orcid.org/ns/personal-details"; + private static final String NS_DETAILS = "personal-details"; + private static final String NS_OTHER_URL = "http://www.orcid.org/ns/other-name"; + private static final String NS_OTHER = "other-name"; + private static final String NS_RECORD_URL = "http://www.orcid.org/ns/record"; + private static final String NS_RECORD = "record"; + private static final String NS_ERROR_URL = "http://www.orcid.org/ns/error"; + + private static final String NS_WORK = "work"; + private static final String NS_WORK_URL = "http://www.orcid.org/ns/work"; + + private static final String NS_ERROR = "error"; + + public static AuthorData VTDParseAuthorData(byte[] bytes) + throws VtdException, EncodingException, EOFException, EntityException, ParseException { + final VTDGen vg = new VTDGen(); + vg.setDoc(bytes); + vg.parse(true); + final VTDNav vn = vg.getNav(); + final AutoPilot ap = new AutoPilot(vn); + ap.declareXPathNameSpace(NS_COMMON, NS_COMMON_URL); + ap.declareXPathNameSpace(NS_PERSON, NS_PERSON_URL); + ap.declareXPathNameSpace(NS_DETAILS, NS_DETAILS_URL); + ap.declareXPathNameSpace(NS_OTHER, NS_OTHER_URL); + ap.declareXPathNameSpace(NS_RECORD, NS_RECORD_URL); + ap.declareXPathNameSpace(NS_ERROR, NS_ERROR_URL); + + AuthorData authorData = new AuthorData(); + final List errors = VtdUtilityParser.getTextValue(ap, vn, "//error:response-code"); + if (!errors.isEmpty()) { + authorData.setErrorCode(errors.get(0)); + return authorData; + } + + List recordNodes = VtdUtilityParser + .getTextValuesWithAttributes( + ap, vn, "//record:record", Arrays.asList("path")); + if (!recordNodes.isEmpty()) { + final String oid = (recordNodes.get(0).getAttributes().get("path")).substring(1); + authorData.setOid(oid); + } else { + return null; + } + + final List names = VtdUtilityParser.getTextValue(ap, vn, "//personal-details:given-names"); + if (!names.isEmpty()) { + authorData.setName(names.get(0)); + } + + final List surnames = VtdUtilityParser.getTextValue(ap, vn, "//personal-details:family-name"); + if (!surnames.isEmpty()) { + authorData.setSurname(surnames.get(0)); + } + + final List creditNames = VtdUtilityParser.getTextValue(ap, vn, "//personal-details:credit-name"); + if (!creditNames.isEmpty()) { + authorData.setCreditName(creditNames.get(0)); + } + return authorData; + } + + public static WorkData VTDParseWorkData(byte[] bytes) + throws VtdException, EncodingException, EOFException, EntityException, ParseException { + final VTDGen vg = new VTDGen(); + vg.setDoc(bytes); + vg.parse(true); + final VTDNav vn = vg.getNav(); + final AutoPilot ap = new AutoPilot(vn); + ap.declareXPathNameSpace(NS_COMMON, NS_COMMON_URL); + ap.declareXPathNameSpace(NS_WORK, NS_WORK_URL); + ap.declareXPathNameSpace(NS_ERROR, NS_ERROR_URL); + + WorkData workData = new WorkData(); + final List errors = VtdUtilityParser.getTextValue(ap, vn, "//error:response-code"); + if (!errors.isEmpty()) { + workData.setErrorCode(errors.get(0)); + return workData; + } + + List workNodes = VtdUtilityParser + .getTextValuesWithAttributes(ap, vn, "//work:work", Arrays.asList("path")); + if (!workNodes.isEmpty()) { + final String oid = (workNodes.get(0).getAttributes().get("path")).split("/")[1]; + workData.setOid(oid); + } else { + return null; + } + + final List dois = VtdUtilityParser + .getTextValue( + ap, vn, "//common:external-id-type[text()=\"doi\"]/../common:external-id-value"); + if (!dois.isEmpty()) { + workData.setDoi(dois.get(0)); + workData.setDoiFound(true); + } + return workData; + } +} diff --git a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/uw/SparkMapUnpayWallToOAF.scala b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/uw/SparkMapUnpayWallToOAF.scala new file mode 100644 index 0000000000..a72e4b0d6e --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/uw/SparkMapUnpayWallToOAF.scala @@ -0,0 +1,43 @@ +package eu.dnetlib.doiboost.uw + +import eu.dnetlib.dhp.application.ArgumentApplicationParser + +import eu.dnetlib.dhp.schema.oaf.Publication +import eu.dnetlib.doiboost.crossref.SparkMapDumpIntoOAF +import org.apache.commons.io.IOUtils +import org.apache.spark.SparkConf +import org.apache.spark.rdd.RDD +import org.apache.spark.sql.{Dataset, Encoder, Encoders, SaveMode, SparkSession} +import org.slf4j.{Logger, LoggerFactory} + + +object SparkMapUnpayWallToOAF { + + def main(args: Array[String]): Unit = { + + + val logger: Logger = LoggerFactory.getLogger(SparkMapDumpIntoOAF.getClass) + val conf: SparkConf = new SparkConf() + val parser = new ArgumentApplicationParser(IOUtils.toString(SparkMapDumpIntoOAF.getClass.getResourceAsStream("/eu/dnetlib/dhp/doiboost/convert_map_to_oaf_params.json"))) + parser.parseArgument(args) + val spark: SparkSession = + SparkSession + .builder() + .config(conf) + .appName(getClass.getSimpleName) + .master(parser.get("master")).getOrCreate() + + implicit val mapEncoderPubs: Encoder[Publication] = Encoders.kryo[Publication] + + + val sourcePath = parser.get("sourcePath") + val targetPath = parser.get("targetPath") + val inputRDD:RDD[String] = spark.sparkContext.textFile(s"$sourcePath") + + logger.info("Converting UnpayWall to OAF") + + val d:Dataset[Publication] = spark.createDataset(inputRDD.map(UnpayWallToOAF.convertToOAF).filter(p=>p!=null)).as[Publication] + d.write.mode(SaveMode.Overwrite).save(targetPath) + } + +} diff --git a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/uw/UnpayWallToOAF.scala b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/uw/UnpayWallToOAF.scala new file mode 100644 index 0000000000..08cd4ee8ee --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/uw/UnpayWallToOAF.scala @@ -0,0 +1,63 @@ +package eu.dnetlib.doiboost.uw + +import eu.dnetlib.dhp.schema.oaf.{Instance, Publication} +import org.json4s +import org.json4s.DefaultFormats +import org.json4s.jackson.JsonMethods.parse +import org.slf4j.{Logger, LoggerFactory} + +import scala.collection.JavaConverters._ +import eu.dnetlib.doiboost.DoiBoostMappingUtil._ + + + +case class OALocation(evidence:Option[String], host_type:Option[String], is_best:Option[Boolean], license: Option[String], pmh_id:Option[String], updated:Option[String], + url:Option[String], url_for_landing_page:Option[String], url_for_pdf:Option[String], version:Option[String]) {} + + + + +object UnpayWallToOAF { + val logger: Logger = LoggerFactory.getLogger(getClass) + + def convertToOAF(input:String):Publication = { + val pub = new Publication + + implicit lazy val formats: DefaultFormats.type = org.json4s.DefaultFormats + lazy val json: json4s.JValue = parse(input) + + val doi = (json \"doi").extract[String] + + + val is_oa = (json\ "is_oa").extract[Boolean] + + val oaLocation:OALocation = (json \ "best_oa_location").extractOrElse[OALocation](null) + pub.setPid(List(createSP(doi, "doi", PID_TYPES)).asJava) + pub.setId(generateIdentifier(pub, doi.toLowerCase)) + + pub.setCollectedfrom(List(createUnpayWallCollectedFrom()).asJava) + pub.setDataInfo(generateDataInfo()) + + if (!is_oa) + return null + + if(oaLocation== null || oaLocation.url.isEmpty) + return null + val i :Instance= new Instance() + + i.setCollectedfrom(createUnpayWallCollectedFrom()) + i.setAccessright(getOpenAccessQualifier()) + i.setUrl(List(oaLocation.url.get).asJava) + + if (oaLocation.license.isDefined) + i.setLicense(asField(oaLocation.license.get)) + pub.setInstance(List(i).asJava) + + pub + + } + + + + +} diff --git a/dhp-workflows/dhp-doiboost/src/main/resources/eu.dnetlib.dhp.doiboost.mappings/crossref_mapping.csv b/dhp-workflows/dhp-doiboost/src/main/resources/eu.dnetlib.dhp.doiboost.mappings/crossref_mapping.csv new file mode 100644 index 0000000000..6a5fc3f871 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/resources/eu.dnetlib.dhp.doiboost.mappings/crossref_mapping.csv @@ -0,0 +1,62 @@ +Crossref Field,Type,Required,Description (from Crossref),OAF field,Comments +publisher,String,Yes,Name of work's publisher,Result/Publisher, +title,Array of String,Yes,"Work titles, including translated titles","Result/Title with Qualifier(""main title"", ""dnet:dataCite_title"")", +original-title,Array of String,No,Work titles in the work's original publication language,"Result/Title with Qualifier(""alternative title"", ""dnet:dataCite_title"")", +short-title,Array of String,No,Short or abbreviated work titles,"Result/Title with Qualifier(""alternative title"", ""dnet:dataCite_title"")", +abstract,XML String,No,Abstract as a JSON string or a JATS XML snippet encoded into a JSON string,Result/description, +reference-count,Number,Yes,Deprecated Same as references-count,"- ", +references-count,Number,Yes,Count of outbound references deposited with Crossref,N/A, +is-referenced-by-count,Number,Yes,Count of inbound references deposited with Crossref,N/A, +source,String,Yes,Currently always Crossref,Result/source, +prefix,String,Yes,DOI prefix identifier of the form http://id.crossref.org/prefix/DOI_PREFIX,N/A, +DOI,String,Yes,DOI of the work,OafEntity/originalId, +,,,,OafEntity/PID, +,,,,"Oaf/id ",Use to generate the OpenAIRE id in the form 50|doiboost____::md5(DOI) +URL,URL,Yes,URL form of the work's DOI,Instance/url, +member,String,Yes,Member identifier of the form http://id.crossref.org/member/MEMBER_ID,N/A, +type,String,Yes,"Enumeration, one of the type ids from https://api.crossref.org/v1/types",Instance/instancetype,Also use to map the record as OAF Publication or Dataset according to the mapping defined in eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala +created,Date,Yes,Date on which the DOI was first registered,"Result/relevantDate with Qualifier(""created"", ""dnet:dataCite_date"")", +,,,,"Result/dateofacceptance +Instance/dateofacceptance",If crossref.issued is blank +deposited,Date,Yes,Date on which the work metadata was most recently updated,N/A, +indexed,Date,Yes,"Date on which the work metadata was most recently indexed. Re-indexing does not imply a metadata change, see deposited for the most recent metadata change date",Result/lastupdatetimestamp, +issued,Partial Date,Yes,Earliest of published-print and published-online,Result/dateofacceptance,OAF dateofacceptance is used also for the publishing date. It's the date visualised in the OpenAIRE EXPLORE portal. +,,,,Instance/dateofacceptance, +posted,Partial Date,No,Date on which posted content was made available online,"Result/relevantDate with Qualifier(""available"", ""dnet:dataCite_date"")", +accepted,Partial Date,No,"Date on which a work was accepted, after being submitted, during a submission process","Result/relevantDate with Qualifier(""accepted"", ""dnet:dataCite_date"")", +subtitle,Array of String,No,"Work subtitles, including original language and translated","Result/Title with Qualifier(""subtitle"", ""dnet:dataCite_title"")", +container-title,Array of String,No,Full titles of the containing work (usually a book or journal),Publication/Journal/name only in case of Journal title for book title see ISBN Mapping, +short-container-title,Array of String,No,Abbreviated titles of the containing work,N/A, +group-title,String,No,Group title for posted content,N/A, +issue,String,No,Issue number of an article's journal,Publication/Journal/iss, +volume,String,No,Volume number of an article's journal,Publication/Journal/vol, +page,String,No,Pages numbers of an article within its journal,"Publication/Journal/sp +Publication/Journal/ep",Obtain start and end page by splitting by '-' +article-number,String,No,,N/A, +published-print,Partial Date,No,Date on which the work was published in print,"Result/relevantDate with Qualifier(""published-print"", ""dnet:dataCite_date"")", +published-online,Partial Date,No,Date on which the work was published online,"Result/relevantDate with Qualifier(""published-online"", ""dnet:dataCite_date"")", +subject,Array of String,No,"Subject category names, a controlled vocabulary from Sci-Val. Available for most journal articles","Result/subject with Qualifier(""keywords"", ""dnet:subject_classification_typologies""). ","Future improvements: map the controlled vocabulary instead of using the generic ""keywords"" qualifier" +ISSN,Array of String,No,,"Publication/Journal/issn +Publication/Journal/lissn +Publication/Journal/eissn",The mapping depends on the value of issn-type +issn-type,Array of ISSN with Type,No,List of ISSNs with ISSN type information,N/A,Its value guides the setting of the properties in Journal (see row above) +ISBN,Array of String,No,,Publication/source,"In case of Book We can map ISBN and container title on Publication/source using this syntax container-title + ""ISBN: "" + ISBN" +archive,Array of String,No,,N/A, +license,Array of License,No,,Result/Instance/License, +funder,Array of Funder,No,,Relation,Whenever we are able to link to a funder or project integrated into OpenAIRE. Mapping to OpenAIRE funders and projects is in eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala.generateSimpleRelationFromAward +assertion,Array of Assertion,No,,N/A, +author,Array of Contributor,No,,Result/author (with orcid if available), +editor,Array of Contributor,No,,N/A, +chair,Array of Contributor,No,,N/A, +translator,Array of Contributor,No,,N/A, +update-to,Array of Update,No,,N/A, +update-policy,URL,No,Link to an update policy covering Crossmark updates for this work,N/A, +link,Array of Resource Link,No,URLs to full-text locations,Result/Instance/url, +clinical-trial-number,Array of Clinical Trial Number,No,,OafEntity/originalId, +alternative-id,String,No,Other identifiers for the work provided by the depositing member,OafEntity/originalId, +reference,Array of Reference,No,List of references made by the work,,Future improvement: map to references +content-domain,Content Domain,No,Information on domains that support Crossmark for this work,N/A, +relation,Relations,No,Relations to other works,Result/Instance/refereed,"if(relation.has-review) instance.refereed = ""peerReviewed"". " +review,Review,No,Peer review metadata,N/A, +funder,Array of Funder,No,,Relation between Result and Project,"The mapping between Crossref funder elements and OpenAIRE projects is implemented in eu.dnetlib.doiboost.crossref.Crossref2Oaf.mappingFunderToRelations. +The matching is based on Funder DOIs, Funder names, and grant codes. Different mapping approaches are applied to cover the different cases in Crossref records." diff --git a/dhp-workflows/dhp-doiboost/src/main/resources/eu.dnetlib.dhp.doiboost.mappings/mag_mapping.csv b/dhp-workflows/dhp-doiboost/src/main/resources/eu.dnetlib.dhp.doiboost.mappings/mag_mapping.csv new file mode 100644 index 0000000000..35ee651770 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/resources/eu.dnetlib.dhp.doiboost.mappings/mag_mapping.csv @@ -0,0 +1,37 @@ +Micorsoft Academic Graph Field,OAF field,Comments +Papers/DOI,OafEntity/originalId, +,OafEntity/PID, +,Oaf/id in the form 50|doiboost____::md5(DOI), +Papers/PaperId,OafEntity/originalId, +Papers/PaperTitle,"Result/Title with Qualifier(""main title"", ""dnet:dataCite_title"")", +Papers/OriginalTitle,"Result/Title with Qualifier(""alternative title"", ""dnet:dataCite_title"")", +Papers/BookTitle,Publication/Source, +Papers/Year,N/A, +Papers/Date,Result/dateofacceptance, +Papers/Publisher,Result/Publisher,Possibly overridden by Journal/Publisher +Papers/JournalId,N/A, +Journal/Rank,N/A, +Journal/NormalizedName,N/A, +Journal/DisplayName,Publication/Journal/Name, +Journal/Issn,Publication/Journal/issnPrinted, +Journal/Publisher,Result/publisher,"If avalable, it overrides value in Papers/Publisher" +Journal/Webpage,N/A, +Journal/PaperCount,N/A, +Journal/CitationCount,N/A, +Journal/CreatedDate,N/A, +ConferenceInstances/DisplayName,Publication/Journal/Name, +ConferenceInstances/Location,Publication/Journal/Conferenceplace, +ConferenceInstances/StartDate,Publication/Journal/Conferencedate,subjectTo be constructed as StartDate - EndDate (only the first 10 chars ofthe dates are kept) +ConferenceInstances/EndDate,Publication/Journal/Conferencedate,To be constructed as StartDate - EndDate (only the first 10 chars ofthe dates are kept) +Papers/Volume,Publication/Journal/vol, +Papers/Issue,Publication/Journal/iss, +Papers/FirstPage,Publication/Journal/sp, +Papers/LastPage,Publication/Journal/ep, +Papers/ReferenceCount,N/A, +Papers/CitationCount,N/A, +Papers/EstimatedCitation,N/A, +Papers/OriginalVenue,N/A, +Papers/FamilyId,N/A, +Papers/CreatedDate,Result/LastUpdateTimeStamp, +Papers/FieldOfStudy/DisplayName,"Result/subject with Qualifier(""keyword"", ""dnet:subject_classification_typologies"")",Some FieldOfStudy come from the UMLS controlled vocabulary. Future releases of DOIBoost will include such terms with a specific Qualifier (i.e. not as generic keywords) +Papers/FieldOfStudy/MainType,"Result/subject with Qualifier(""keyword"", ""dnet:subject_classification_typologies"")","If MainType is splittable on . (like 'food.type_of_dish', 'aviation.airport', 'soccer.team'), then the first token (i.e. food, aviation, soccer) is also added as additional subject." \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/main/resources/eu.dnetlib.dhp.doiboost.mappings/orcid_mapping.csv b/dhp-workflows/dhp-doiboost/src/main/resources/eu.dnetlib.dhp.doiboost.mappings/orcid_mapping.csv new file mode 100644 index 0000000000..71020d5c34 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/resources/eu.dnetlib.dhp.doiboost.mappings/orcid_mapping.csv @@ -0,0 +1,6 @@ +ORCID field,OAF mapping,Comment +doi,Result/pid, +oid,Result/Author/pid, +name,Result/Author/name, +surname,Result/Author/surname, +creditName,N/A,Result/Author/fullname is generated by concatenating name and surname \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/main/resources/eu.dnetlib.dhp.doiboost.mappings/unpaywall_mapping.csv b/dhp-workflows/dhp-doiboost/src/main/resources/eu.dnetlib.dhp.doiboost.mappings/unpaywall_mapping.csv new file mode 100644 index 0000000000..0c891b35a6 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/resources/eu.dnetlib.dhp.doiboost.mappings/unpaywall_mapping.csv @@ -0,0 +1,11 @@ +Unpaywall field,Description,OAF Mapping,Comment +doi,The DOI of this resource.,Result/pid, +is_oa,Is there an OA copy of this resource.,N/A,Add a Result/Instance only if is_oa is true +best_oa_location,The best OA Location Object we could find for this DOI.,,Create one Result/Instance with Open Access right +best_oa_location.url,The url_for_pdf if there is one; otherwise landing page URL.,Result/instance/url, +best_oa_location.license,The license under which this copy is published.,Result/instance/license, +oa_status,"The OA status, or color, of this resource. Classifies OA resources by location and license terms as one of: gold, hybrid, bronze, green or closed.",N/A,New field to be introduced in the OpenAIRE data model +published_date,"The date this resource was published. As reported by the publishers, who unfortunately have inconsistent definitions of what counts as officially ""published.""",N/A,"Not used in the mapping. Could be used to improve the coverage of dates, if needed." +title,The title of this resource.,N/A, +oa_locations,List of all the OA Location objects associated with this resource.,N/A,"Coud be useful if we plan to ""patch"" existing instances. If the instance URL is a URL of a oa_location, then it's Open. +" \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/convert_map_to_oaf_params.json b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/convert_map_to_oaf_params.json new file mode 100644 index 0000000000..312bd0751e --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/convert_map_to_oaf_params.json @@ -0,0 +1,6 @@ +[ + {"paramName":"s", "paramLongName":"sourcePath", "paramDescription": "the path of the sequencial file to read", "paramRequired": true}, + {"paramName":"t", "paramLongName":"targetPath", "paramDescription": "the working dir path", "paramRequired": true}, + {"paramName":"m", "paramLongName":"master", "paramDescription": "the master name", "paramRequired": true} + +] \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/create_orcid_authors_data.json b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/create_orcid_authors_data.json new file mode 100644 index 0000000000..bf992b5086 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/create_orcid_authors_data.json @@ -0,0 +1,6 @@ +[ + {"paramName":"n", "paramLongName":"hdfsServerUri", "paramDescription": "the server uri", "paramRequired": true}, + {"paramName":"d", "paramLongName":"hdfsOrcidDefaultPath", "paramDescription": "the default work path", "paramRequired": true}, + {"paramName":"f", "paramLongName":"summariesFileNameTarGz", "paramDescription": "the name of the summaries orcid file", "paramRequired": true}, + {"paramName":"o", "paramLongName":"outputAuthorsPath", "paramDescription": "the relative folder of the sequencial file to write", "paramRequired": true} +] \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/create_orcid_authors_dois_data.json b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/create_orcid_authors_dois_data.json new file mode 100644 index 0000000000..131c301257 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/create_orcid_authors_dois_data.json @@ -0,0 +1,6 @@ +[ + {"paramName":"n", "paramLongName":"hdfsServerUri", "paramDescription": "the server uri", "paramRequired": true}, + {"paramName":"d", "paramLongName":"hdfsOrcidDefaultPath", "paramDescription": "the default work path", "paramRequired": true}, + {"paramName":"f", "paramLongName":"activitiesFileNameTarGz", "paramDescription": "the name of the activities orcid file", "paramRequired": true}, + {"paramName":"o", "paramLongName":"outputAuthorsDOIsPath", "paramDescription": "the relative folder of the sequencial file to write", "paramRequired": true} +] \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/crossref/oozie_app/config-default.xml b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/crossref/oozie_app/config-default.xml new file mode 100644 index 0000000000..508202e301 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/crossref/oozie_app/config-default.xml @@ -0,0 +1,42 @@ + + + jobTracker + yarnRM + + + nameNode + hdfs://nameservice1 + + + oozie.use.system.libpath + true + + + oozie.action.sharelib.for.spark + spark2 + + + oozie.launcher.mapreduce.user.classpath.first + true + + + hive_metastore_uris + thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083 + + + spark2YarnHistoryServerAddress + http://iis-cdh5-test-gw.ocean.icm.edu.pl:18089 + + + spark2EventLogDir + /user/spark/spark2ApplicationHistory + + + spark2ExtraListeners + "com.cloudera.spark.lineage.NavigatorAppListener" + + + spark2SqlQueryExecutionListeners + "com.cloudera.spark.lineage.NavigatorQueryListener" + + \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/crossref/oozie_app/workflow.xml b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/crossref/oozie_app/workflow.xml new file mode 100644 index 0000000000..db4ac96f94 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/crossref/oozie_app/workflow.xml @@ -0,0 +1,80 @@ + + + + workingPath + the working dir base path + + + sparkDriverMemory + memory for driver process + + + sparkExecutorMemory + memory for individual executor + + + sparkExecutorCores + number of cores used by single executor + + + timestamp + Timestamp for incremental Harvesting + + + + + + + + + Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] + + + + + + + + + + + + + + + + ${jobTracker} + ${nameNode} + eu.dnetlib.doiboost.crossref.CrossrefImporter + -t${workingPath}/input/crossref/index_dump + -n${nameNode} + -ts${timestamp} + + + + + + + + + yarn-cluster + cluster + ExtractCrossrefToOAF + eu.dnetlib.doiboost.crossref.SparkMapDumpIntoOAF + dhp-doiboost-${projectVersion}.jar + + --executor-memory=${sparkExecutorMemory} + --executor-cores=${sparkExecutorCores} + --driver-memory=${sparkDriverMemory} + ${sparkExtraOPT} + + --sourcePath${workingPath}/input/crossref/index_dump,${workingPath}/crossref/index_dump + --targetPath${workingPath}/input/crossref + --masteryarn-cluster + + + + + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/download_orcid_data.json b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/download_orcid_data.json new file mode 100644 index 0000000000..444e487f7b --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/download_orcid_data.json @@ -0,0 +1,7 @@ +[ + {"paramName":"n", "paramLongName":"hdfsServerUri", "paramDescription": "the server uri", "paramRequired": true}, + {"paramName":"d", "paramLongName":"hdfsOrcidDefaultPath", "paramDescription": "the default work path", "paramRequired": true}, + {"paramName":"f", "paramLongName":"lambdaFileName", "paramDescription": "the name of the lambda file", "paramRequired": true}, + {"paramName":"o", "paramLongName":"outputPath", "paramDescription": "the relative folder of the sequencial file to write", "paramRequired": true}, + {"paramName":"t", "paramLongName":"token", "paramDescription": "token to grant access", "paramRequired": true} +] \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/gen_doi_author_list_orcid_parameters.json b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/gen_doi_author_list_orcid_parameters.json new file mode 100644 index 0000000000..b894177b3b --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/gen_doi_author_list_orcid_parameters.json @@ -0,0 +1,3 @@ +[{"paramName":"w", "paramLongName":"workingPath", "paramDescription": "the working path", "paramRequired": true}, + {"paramName":"o", "paramLongName":"outputDoiAuthorListPath", "paramDescription": "the relative folder of the sequencial file to write the data", "paramRequired": true} +] \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/gen_orcid_authors_parameters.json b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/gen_orcid_authors_parameters.json new file mode 100644 index 0000000000..35bfe1b415 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/gen_orcid_authors_parameters.json @@ -0,0 +1,4 @@ +[{"paramName":"w", "paramLongName":"workingPath", "paramDescription": "the working path", "paramRequired": true}, + {"paramName":"t", "paramLongName":"token", "paramDescription": "token to grant access", "paramRequired": true}, + {"paramName":"o", "paramLongName":"outputAuthorsPath", "paramDescription": "the relative folder of the sequencial file to write the authors data", "paramRequired": true} +] \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/generate_doiboost_as_params.json b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/generate_doiboost_as_params.json new file mode 100644 index 0000000000..6eb1ec6f16 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/generate_doiboost_as_params.json @@ -0,0 +1,9 @@ +[ + {"paramName": "m", "paramLongName":"master", "paramDescription": "the master name", "paramRequired": true}, + {"paramName": "dp", "paramLongName":"dbPublicationPath", "paramDescription": "the Crossref Publication Path", "paramRequired": true}, + {"paramName": "dd", "paramLongName":"dbDatasetPath", "paramDescription": "the Crossref Dataset Path", "paramRequired": true}, + {"paramName": "cr", "paramLongName":"crossRefRelation", "paramDescription": "the UnpayWall Publication Path", "paramRequired": true}, + {"paramName": "da", "paramLongName":"dbaffiliationRelationPath", "paramDescription": "the MAG Publication Path", "paramRequired": true}, + {"paramName": "do", "paramLongName":"dbOrganizationPath", "paramDescription": "the MAG Publication Path", "paramRequired": true}, + {"paramName": "w", "paramLongName":"targetPath", "paramDescription": "the Working Path", "paramRequired": true} +] diff --git a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/generate_doiboost_params.json b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/generate_doiboost_params.json new file mode 100644 index 0000000000..ea08f47d43 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/generate_doiboost_params.json @@ -0,0 +1,7 @@ +[ + {"paramName": "m", "paramLongName":"master", "paramDescription": "the master name", "paramRequired": true}, + {"paramName": "hb", "paramLongName":"hostedByMapPath", "paramDescription": "the hosted By Map Path", "paramRequired": true}, + {"paramName": "ap", "paramLongName":"affiliationPath", "paramDescription": "the Affliation Path", "paramRequired": true}, + {"paramName": "pa", "paramLongName":"paperAffiliationPath", "paramDescription": "the paperAffiliation Path", "paramRequired": true}, + {"paramName": "w", "paramLongName":"workingDirPath", "paramDescription": "the Working Path", "paramRequired": true} +] diff --git a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/import_from_es.json b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/import_from_es.json new file mode 100644 index 0000000000..87a138d525 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/import_from_es.json @@ -0,0 +1,5 @@ +[ + {"paramName":"t", "paramLongName":"targetPath", "paramDescription": "the path of the sequencial file to write", "paramRequired": true}, + {"paramName":"n", "paramLongName":"namenode", "paramDescription": "the hive metastore uris", "paramRequired": true}, + {"paramName":"ts", "paramLongName":"timestamp", "paramDescription": "timestamp", "paramRequired": false} +] \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/intersection/oozie_app/config-default.xml b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/intersection/oozie_app/config-default.xml new file mode 100644 index 0000000000..cf617a84c4 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/intersection/oozie_app/config-default.xml @@ -0,0 +1,38 @@ + + + jobTracker + yarnRM + + + nameNode + hdfs://nameservice1 + + + oozie.use.system.libpath + true + + + oozie.action.sharelib.for.spark + spark2 + + + hive_metastore_uris + thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083 + + + spark2YarnHistoryServerAddress + http://iis-cdh5-test-gw.ocean.icm.edu.pl:18089 + + + spark2EventLogDir + /user/spark/spark2ApplicationHistory + + + spark2ExtraListeners + "com.cloudera.spark.lineage.NavigatorAppListener" + + + spark2SqlQueryExecutionListeners + "com.cloudera.spark.lineage.NavigatorQueryListener" + + \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/intersection/oozie_app/workflow.xml b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/intersection/oozie_app/workflow.xml new file mode 100644 index 0000000000..bf91958cfa --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/intersection/oozie_app/workflow.xml @@ -0,0 +1,103 @@ + + + + hostedByMapPath + the Hosted By Map Path + + + affiliationPath + the Affliation Path + + + paperAffiliationPath + the paperAffiliation Path + + + workingDirPath + the Working Path + + + sparkDriverMemory + memory for driver process + + + sparkExecutorMemory + memory for individual executor + + + sparkExecutorCores + number of cores used by single executor + + + + + + + + + + Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] + + + + + + + + + + + + + + yarn-cluster + cluster + Create DOIBoost Infospace + eu.dnetlib.doiboost.SparkGenerateDoiBoost + dhp-doiboost-${projectVersion}.jar + + --executor-memory=${sparkExecutorMemory} + --executor-cores=${sparkExecutorCores} + --driver-memory=${sparkDriverMemory} + --conf spark.sql.shuffle.partitions=3840 + ${sparkExtraOPT} + + --hostedByMapPath${hostedByMapPath} + --affiliationPath${affiliationPath} + --paperAffiliationPath${paperAffiliationPath} + --workingDirPath${workingDirPath} + --masteryarn-cluster + + + + + + + + + yarn-cluster + cluster + Generate DOIBoost ActionSet + eu.dnetlib.doiboost.SparkGenerateDOIBoostActionSet + dhp-doiboost-${projectVersion}.jar + + --executor-memory=${sparkExecutorMemory} + --executor-cores=${sparkExecutorCores} + --driver-memory=${sparkDriverMemory} + --conf spark.sql.shuffle.partitions=3840 + ${sparkExtraOPT} + + --dbPublicationPath${workingDirPath}/doiBoostPublicationFiltered + --dbDatasetPath${workingDirPath}/crossrefDataset + --crossRefRelation/data/doiboost/input/crossref/relations + --dbaffiliationRelationPath${workingDirPath}/doiBoostPublicationAffiliation + -do${workingDirPath}/doiBoostOrganization + --targetPath${workingDirPath}/actionDataSet + --masteryarn-cluster + + + + + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/mag/convert_mag_to_oaf_params.json b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/mag/convert_mag_to_oaf_params.json new file mode 100644 index 0000000000..bf0b80f69f --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/mag/convert_mag_to_oaf_params.json @@ -0,0 +1,6 @@ +[ + {"paramName":"s", "paramLongName":"sourcePath", "paramDescription": "the base path of MAG input", "paramRequired": true}, + {"paramName":"t", "paramLongName":"targetPath", "paramDescription": "the working dir path", "paramRequired": true}, + {"paramName":"m", "paramLongName":"master", "paramDescription": "the master name", "paramRequired": true} + +] \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/mag/oozie_app/config-default.xml b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/mag/oozie_app/config-default.xml new file mode 100644 index 0000000000..59e5c059fc --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/mag/oozie_app/config-default.xml @@ -0,0 +1,42 @@ + + + jobTracker + yarnRM + + + nameNode + hdfs://nameservice1 + + + oozie.use.system.libpath + true + + + oozie.action.sharelib.for.spark + spark2 + + + oozie.wf.rerun.failnodes + false + + + hive_metastore_uris + thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083 + + + spark2YarnHistoryServerAddress + http://iis-cdh5-test-gw.ocean.icm.edu.pl:18089 + + + spark2EventLogDir + /user/spark/spark2ApplicationHistory + + + spark2ExtraListeners + "com.cloudera.spark.lineage.NavigatorAppListener" + + + spark2SqlQueryExecutionListeners + "com.cloudera.spark.lineage.NavigatorQueryListener" + + \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/mag/oozie_app/workflow.xml b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/mag/oozie_app/workflow.xml new file mode 100644 index 0000000000..2277b79b0f --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/mag/oozie_app/workflow.xml @@ -0,0 +1,87 @@ + + + + sourcePath + the working dir base path + + + targetPath + the working dir base path + + + sparkDriverMemory + memory for driver process + + + sparkExecutorMemory + memory for individual executor + + + sparkExecutorCores + number of cores used by single executor + + + + + + + + Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] + + + + + + + + + + + + + + yarn-cluster + cluster + Convert Mag to Dataset + eu.dnetlib.doiboost.mag.SparkImportMagIntoDataset + dhp-doiboost-${projectVersion}.jar + + --executor-memory=${sparkExecutorMemory} + --executor-cores=${sparkExecutorCores} + --driver-memory=${sparkDriverMemory} + ${sparkExtraOPT} + + --sourcePath${sourcePath} + --targetPath${targetPath} + --masteryarn-cluster + + + + + + + + + + yarn-cluster + cluster + Convert Mag to Dataset + eu.dnetlib.doiboost.mag.SparkPreProcessMAG + dhp-doiboost-${projectVersion}.jar + + --executor-memory=${sparkExecutorMemory} + --executor-cores=${sparkExecutorCores} + --driver-memory=${sparkDriverMemory} + --conf spark.sql.shuffle.partitions=3840 + ${sparkExtraOPT} + + --sourcePath${sourcePath} + --targetPath${targetPath} + --masteryarn-cluster + + + + + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/mag/preprocess_mag_params.json b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/mag/preprocess_mag_params.json new file mode 100644 index 0000000000..bf0b80f69f --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/mag/preprocess_mag_params.json @@ -0,0 +1,6 @@ +[ + {"paramName":"s", "paramLongName":"sourcePath", "paramDescription": "the base path of MAG input", "paramRequired": true}, + {"paramName":"t", "paramLongName":"targetPath", "paramDescription": "the working dir path", "paramRequired": true}, + {"paramName":"m", "paramLongName":"master", "paramDescription": "the master name", "paramRequired": true} + +] \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/orcid/oozie_app/config-default.xml b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/orcid/oozie_app/config-default.xml new file mode 100644 index 0000000000..fe14bb8cb5 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/orcid/oozie_app/config-default.xml @@ -0,0 +1,42 @@ + + + jobTracker + hadoop-rm3.garr-pa1.d4science.org:8032 + + + nameNode + hdfs://hadoop-rm1.garr-pa1.d4science.org:8020 + + + oozie.use.system.libpath + true + + + oozie.action.sharelib.for.spark + spark2 + + + oozie.launcher.mapreduce.user.classpath.first + true + + + hive_metastore_uris + thrift://hadoop-edge2.garr-pa1.d4science.org:9083 + + + spark2YarnHistoryServerAddress + http://hadoop-edge1.garr-pa1.d4science.org:18089/ + + + spark2EventLogDir + /user/spark/spark2ApplicationHistory + + + spark2ExtraListeners + "com.cloudera.spark.lineage.NavigatorAppListener" + + + spark2SqlQueryExecutionListeners + "com.cloudera.spark.lineage.NavigatorQueryListener" + + \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/orcid/oozie_app/workflow.xml b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/orcid/oozie_app/workflow.xml new file mode 100644 index 0000000000..7a8d041870 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/orcid/oozie_app/workflow.xml @@ -0,0 +1,41 @@ + + + + workingPath + the working dir base path + + + + + + + + Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] + + + + + + + + + + + + + + + + ${jobTracker} + ${nameNode} + eu.dnetlib.doiboost.orcid.OrcidDSManager + -d${workingPath}/ + -n${nameNode} + -fORCID_2019_summaries.tar.gz + -ooutput/ + + + + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/orcid_authors_dois_data/oozie_app/config-default.xml b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/orcid_authors_dois_data/oozie_app/config-default.xml new file mode 100644 index 0000000000..5621415d97 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/orcid_authors_dois_data/oozie_app/config-default.xml @@ -0,0 +1,22 @@ + + + jobTracker + yarnRM + + + nameNode + hdfs://nameservice1 + + + oozie.action.sharelib.for.java + spark2 + + + oozie.launcher.mapreduce.user.classpath.first + true + + + oozie.launcher.mapreduce.map.java.opts + -Xmx4g + + \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/orcid_authors_dois_data/oozie_app/workflow.xml b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/orcid_authors_dois_data/oozie_app/workflow.xml new file mode 100644 index 0000000000..1c2ae89ddc --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/orcid_authors_dois_data/oozie_app/workflow.xml @@ -0,0 +1,505 @@ + + + + workingPath_activities + the working dir base path + + + shell_cmd_0 + wget -O /tmp/ORCID_2019_activites_0.tar.gz https://orcid.figshare.com/ndownloader/files/18017660 ; hdfs dfs -copyFromLocal /tmp/ORCID_2019_activites_0.tar.gz /data/orcid_activities/ORCID_2019_activites_0.tar.gz + + the shell command that downloads and puts to hdfs orcid activity file 0 + + + shell_cmd_1 + wget -O /tmp/ORCID_2019_activites_1.tar.gz https://orcid.figshare.com/ndownloader/files/18017675 ; hdfs dfs -copyFromLocal /tmp/ORCID_2019_activites_1.tar.gz /data/orcid_activities/ORCID_2019_activites_1.tar.gz + + the shell command that downloads and puts to hdfs orcid activity file 1 + + + shell_cmd_2 + wget -O /tmp/ORCID_2019_activites_2.tar.gz https://orcid.figshare.com/ndownloader/files/18017717 ; hdfs dfs -copyFromLocal /tmp/ORCID_2019_activites_2.tar.gz /data/orcid_activities/ORCID_2019_activites_2.tar.gz + + the shell command that downloads and puts to hdfs orcid activity file 2 + + + shell_cmd_3 + wget -O /tmp/ORCID_2019_activites_3.tar.gz https://orcid.figshare.com/ndownloader/files/18017765 ; hdfs dfs -copyFromLocal /tmp/ORCID_2019_activites_3.tar.gz /data/orcid_activities/ORCID_2019_activites_3.tar.gz + + the shell command that downloads and puts to hdfs orcid activity file 3 + + + shell_cmd_4 + wget -O /tmp/ORCID_2019_activites_4.tar.gz https://orcid.figshare.com/ndownloader/files/18017831 ; hdfs dfs -copyFromLocal /tmp/ORCID_2019_activites_4.tar.gz /data/orcid_activities/ORCID_2019_activites_4.tar.gz + + the shell command that downloads and puts to hdfs orcid activity file 4 + + + shell_cmd_5 + wget -O /tmp/ORCID_2019_activites_5.tar.gz https://orcid.figshare.com/ndownloader/files/18017987 ; hdfs dfs -copyFromLocal /tmp/ORCID_2019_activites_5.tar.gz /data/orcid_activities/ORCID_2019_activites_5.tar.gz + + the shell command that downloads and puts to hdfs orcid activity file 5 + + + shell_cmd_6 + wget -O /tmp/ORCID_2019_activites_6.tar.gz https://orcid.figshare.com/ndownloader/files/18018053 ; hdfs dfs -copyFromLocal /tmp/ORCID_2019_activites_6.tar.gz /data/orcid_activities/ORCID_2019_activites_6.tar.gz + + the shell command that downloads and puts to hdfs orcid activity file 6 + + + shell_cmd_7 + wget -O /tmp/ORCID_2019_activites_7.tar.gz https://orcid.figshare.com/ndownloader/files/18018023 ; hdfs dfs -copyFromLocal /tmp/ORCID_2019_activites_7.tar.gz /data/orcid_activities/ORCID_2019_activites_7.tar.gz + + the shell command that downloads and puts to hdfs orcid activity file 7 + + + shell_cmd_8 + wget -O /tmp/ORCID_2019_activites_8.tar.gz https://orcid.figshare.com/ndownloader/files/18018248 ; hdfs dfs -copyFromLocal /tmp/ORCID_2019_activites_8.tar.gz /data/orcid_activities/ORCID_2019_activites_8.tar.gz + + the shell command that downloads and puts to hdfs orcid activity file 8 + + + shell_cmd_9 + wget -O /tmp/ORCID_2019_activites_9.tar.gz https://orcid.figshare.com/ndownloader/files/18018029 ; hdfs dfs -copyFromLocal /tmp/ORCID_2019_activites_9.tar.gz /data/orcid_activities/ORCID_2019_activites_9.tar.gz + + the shell command that downloads and puts to hdfs orcid activity file 9 + + + shell_cmd_X + wget -O /tmp/ORCID_2019_activites_X.tar.gz https://orcid.figshare.com/ndownloader/files/18018182 ; hdfs dfs -copyFromLocal /tmp/ORCID_2019_activites_X.tar.gz /data/orcid_activities/ORCID_2019_activites_X.tar.gz + + the shell command that downloads and puts to hdfs orcid activity file X + + + + + + + + Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ${fs:exists(concat(workingPath_activities,'/ORCID_2019_activites_0.tar.gz'))} + + + + + + + + ${jobTracker} + ${nameNode} + bash + -c + ${shell_cmd_0} + + + + + + + + + ${jobTracker} + ${nameNode} + eu.dnetlib.doiboost.orcid.OrcidAuthorsDOIsDataGen + -d${workingPath_activities}/ + -n${nameNode} + -fORCID_2019_activites_0.tar.gz + -ooutput/authors_dois_0.seq + + + + + + + + + ${fs:exists(concat(workingPath_activities,'/ORCID_2019_activites_1.tar.gz'))} + + + + + + + + ${jobTracker} + ${nameNode} + bash + -c + ${shell_cmd_1} + + + + + + + + + ${jobTracker} + ${nameNode} + eu.dnetlib.doiboost.orcid.OrcidAuthorsDOIsDataGen + -d${workingPath_activities}/ + -n${nameNode} + -fORCID_2019_activites_1.tar.gz + -ooutput/authors_dois_1.seq + + + + + + + + + ${fs:exists(concat(workingPath_activities,'/ORCID_2019_activites_2.tar.gz'))} + + + + + + + + ${jobTracker} + ${nameNode} + bash + -c + ${shell_cmd_2} + + + + + + + + + ${jobTracker} + ${nameNode} + eu.dnetlib.doiboost.orcid.OrcidAuthorsDOIsDataGen + -d${workingPath_activities}/ + -n${nameNode} + -fORCID_2019_activites_2.tar.gz + -ooutput/authors_dois_2.seq + + + + + + + + + ${fs:exists(concat(workingPath_activities,'/ORCID_2019_activites_3.tar.gz'))} + + + + + + + + ${jobTracker} + ${nameNode} + bash + -c + ${shell_cmd_3} + + + + + + + + + ${jobTracker} + ${nameNode} + eu.dnetlib.doiboost.orcid.OrcidAuthorsDOIsDataGen + -d${workingPath_activities}/ + -n${nameNode} + -fORCID_2019_activites_3.tar.gz + -ooutput/authors_dois_3.seq + + + + + + + + + ${fs:exists(concat(workingPath_activities,'/ORCID_2019_activites_4.tar.gz'))} + + + + + + + + ${jobTracker} + ${nameNode} + bash + -c + ${shell_cmd_4} + + + + + + + + + ${jobTracker} + ${nameNode} + eu.dnetlib.doiboost.orcid.OrcidAuthorsDOIsDataGen + -d${workingPath_activities}/ + -n${nameNode} + -fORCID_2019_activites_4.tar.gz + -ooutput/authors_dois_4.seq + + + + + + + + + ${fs:exists(concat(workingPath_activities,'/ORCID_2019_activites_5.tar.gz'))} + + + + + + + + ${jobTracker} + ${nameNode} + bash + -c + ${shell_cmd_5} + + + + + + + + + ${jobTracker} + ${nameNode} + eu.dnetlib.doiboost.orcid.OrcidAuthorsDOIsDataGen + -d${workingPath_activities}/ + -n${nameNode} + -fORCID_2019_activites_5.tar.gz + -ooutput/authors_dois_5.seq + + + + + + + + + ${fs:exists(concat(workingPath_activities,'/ORCID_2019_activites_6.tar.gz'))} + + + + + + + + ${jobTracker} + ${nameNode} + bash + -c + ${shell_cmd_6} + + + + + + + + + ${jobTracker} + ${nameNode} + eu.dnetlib.doiboost.orcid.OrcidAuthorsDOIsDataGen + -d${workingPath_activities}/ + -n${nameNode} + -fORCID_2019_activites_6.tar.gz + -ooutput/authors_dois_6.seq + + + + + + + + + + ${fs:exists(concat(workingPath_activities,'/ORCID_2019_activites_7.tar.gz'))} + + + + + + + + ${jobTracker} + ${nameNode} + bash + -c + ${shell_cmd_7} + + + + + + + + + ${jobTracker} + ${nameNode} + eu.dnetlib.doiboost.orcid.OrcidAuthorsDOIsDataGen + -d${workingPath_activities}/ + -n${nameNode} + -fORCID_2019_activites_7.tar.gz + -ooutput/authors_dois_7.seq + + + + + + + + + ${fs:exists(concat(workingPath_activities,'/ORCID_2019_activites_8.tar.gz'))} + + + + + + + + ${jobTracker} + ${nameNode} + bash + -c + ${shell_cmd_8} + + + + + + + + + ${jobTracker} + ${nameNode} + eu.dnetlib.doiboost.orcid.OrcidAuthorsDOIsDataGen + -d${workingPath_activities}/ + -n${nameNode} + -fORCID_2019_activites_8.tar.gz + -ooutput/authors_dois_8.seq + + + + + + + + + ${fs:exists(concat(workingPath_activities,'/ORCID_2019_activites_9.tar.gz'))} + + + + + + + + ${jobTracker} + ${nameNode} + bash + -c + ${shell_cmd_9} + + + + + + + + + ${jobTracker} + ${nameNode} + eu.dnetlib.doiboost.orcid.OrcidAuthorsDOIsDataGen + -d${workingPath_activities}/ + -n${nameNode} + -fORCID_2019_activites_9.tar.gz + -ooutput/authors_dois_9.seq + + + + + + + + + ${fs:exists(concat(workingPath_activities,'/ORCID_2019_activites_X.tar.gz'))} + + + + + + + + ${jobTracker} + ${nameNode} + bash + -c + ${shell_cmd_X} + + + + + + + + + ${jobTracker} + ${nameNode} + eu.dnetlib.doiboost.orcid.OrcidAuthorsDOIsDataGen + -d${workingPath_activities}/ + -n${nameNode} + -fORCID_2019_activites_X.tar.gz + -ooutput/authors_dois_X.seq + + + + + + + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/orcid_doi_author_list/oozie_app/config-default.xml b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/orcid_doi_author_list/oozie_app/config-default.xml new file mode 100644 index 0000000000..3726022cbf --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/orcid_doi_author_list/oozie_app/config-default.xml @@ -0,0 +1,18 @@ + + + jobTracker + hadoop-rm3.garr-pa1.d4science.org:8032 + + + nameNode + hdfs://hadoop-rm1.garr-pa1.d4science.org:8020 + + + queueName + default + + + oozie.action.sharelib.for.spark + spark2 + + \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/orcid_doi_author_list/oozie_app/workflow.xml b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/orcid_doi_author_list/oozie_app/workflow.xml new file mode 100644 index 0000000000..21d092a83e --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/orcid_doi_author_list/oozie_app/workflow.xml @@ -0,0 +1,55 @@ + + + + workingPath + the working dir base path + + + sparkDriverMemory + memory for driver process + + + sparkExecutorMemory + memory for individual executor + + + sparkExecutorCores + number of cores used by single executor + + + + + + + + Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] + + + + + + + + + + + + + ${jobTracker} + ${nameNode} + yarn + cluster + Gen_Doi_Author_List + eu.dnetlib.doiboost.orcid.SparkGenerateDoiAuthorList + dhp-doiboost-1.2.1-SNAPSHOT.jar + --num-executors 10 --conf spark.yarn.jars="hdfs://hadoop-rm1.garr-pa1.d4science.org:8020/user/oozie/share/lib/lib_20180405103059/spark2" --executor-memory=${sparkExecutorMemory} --executor-cores=${sparkExecutorCores} --driver-memory=${sparkDriverMemory} + + -w${workingPath}/ + -odoi_author_list/ + + + + + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/orcid_download/oozie_app/config-default.xml b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/orcid_download/oozie_app/config-default.xml new file mode 100644 index 0000000000..5621415d97 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/orcid_download/oozie_app/config-default.xml @@ -0,0 +1,22 @@ + + + jobTracker + yarnRM + + + nameNode + hdfs://nameservice1 + + + oozie.action.sharelib.for.java + spark2 + + + oozie.launcher.mapreduce.user.classpath.first + true + + + oozie.launcher.mapreduce.map.java.opts + -Xmx4g + + \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/orcid_download/oozie_app/workflow.xml b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/orcid_download/oozie_app/workflow.xml new file mode 100644 index 0000000000..1f9adeb4d8 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/orcid_download/oozie_app/workflow.xml @@ -0,0 +1,45 @@ + + + + workingPathOrcid + the working dir base path + + + token + access token + + + + + + + + Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] + + + + + + + + + + + + + + ${jobTracker} + ${nameNode} + eu.dnetlib.doiboost.orcid.OrcidDownloader + -d${workingPathOrcid}/ + -n${nameNode} + -flast_modified.csv + -odownload/ + -t${token} + + + + + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/orcid_gen_authors/oozie_app/config-default.xml b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/orcid_gen_authors/oozie_app/config-default.xml new file mode 100644 index 0000000000..a720e75923 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/orcid_gen_authors/oozie_app/config-default.xml @@ -0,0 +1,22 @@ + + + jobTracker + hadoop-rm3.garr-pa1.d4science.org:8032 + + + nameNode + hdfs://hadoop-rm1.garr-pa1.d4science.org:8020 + + + queueName + default + + + oozie.use.system.libpath + true + + + oozie.action.sharelib.for.spark + spark2 + + \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/orcid_gen_authors/oozie_app/workflow.xml b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/orcid_gen_authors/oozie_app/workflow.xml new file mode 100644 index 0000000000..7ebc5f0a0b --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/orcid_gen_authors/oozie_app/workflow.xml @@ -0,0 +1,83 @@ + + + + workingPath + the working dir base path + + + token + access token + + + sparkDriverMemory + memory for driver process + + + sparkExecutorMemory + memory for individual executor + + + sparkExecutorCores + number of cores used by single executor + + + outputPath + the working dir base path + + + + + + + + Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] + + + + + + + + + + + + + ${jobTracker} + ${nameNode} + yarn + cluster + Split_Lambda_File + eu.dnetlib.doiboost.orcid.SparkPartitionLambdaFile + dhp-doiboost-1.2.1-SNAPSHOT.jar + --num-executors 24 --conf spark.yarn.jars="hdfs://hadoop-rm1.garr-pa1.d4science.org:8020/user/oozie/share/lib/lib_20180405103059/spark2" --executor-memory=${sparkExecutorMemory} --executor-cores=${sparkExecutorCores} --driver-memory=${sparkDriverMemory} + + -w${workingPath}/ + -oauthors/ + -t${token} + + + + + + + + ${jobTracker} + ${nameNode} + yarn + cluster + Gen_Orcid_Authors + eu.dnetlib.doiboost.orcid.SparkOrcidGenerateAuthors + dhp-doiboost-1.2.1-SNAPSHOT.jar + --num-executors 20 --conf spark.yarn.jars="hdfs://hadoop-rm1.garr-pa1.d4science.org:8020/user/oozie/share/lib/lib_20180405103059/spark2" --executor-memory=${sparkExecutorMemory} --executor-cores=${sparkExecutorCores} --driver-memory=${sparkDriverMemory} + + -w${workingPath}/ + -oauthors/ + -t${token} + + + + + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/orcid_oaf/oozie_app/config-default.xml b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/orcid_oaf/oozie_app/config-default.xml new file mode 100644 index 0000000000..cf617a84c4 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/orcid_oaf/oozie_app/config-default.xml @@ -0,0 +1,38 @@ + + + jobTracker + yarnRM + + + nameNode + hdfs://nameservice1 + + + oozie.use.system.libpath + true + + + oozie.action.sharelib.for.spark + spark2 + + + hive_metastore_uris + thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083 + + + spark2YarnHistoryServerAddress + http://iis-cdh5-test-gw.ocean.icm.edu.pl:18089 + + + spark2EventLogDir + /user/spark/spark2ApplicationHistory + + + spark2ExtraListeners + "com.cloudera.spark.lineage.NavigatorAppListener" + + + spark2SqlQueryExecutionListeners + "com.cloudera.spark.lineage.NavigatorQueryListener" + + \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/orcid_oaf/oozie_app/workflow.xml b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/orcid_oaf/oozie_app/workflow.xml new file mode 100644 index 0000000000..bffde793ba --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/orcid_oaf/oozie_app/workflow.xml @@ -0,0 +1,55 @@ + + + + sourcePath + the working dir base path + + + targetPath + the working dir base path + + + sparkDriverMemory + memory for driver process + + + sparkExecutorMemory + memory for individual executor + + + sparkExecutorCores + number of cores used by single executor + + + + + + + + Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] + + + + + yarn-cluster + cluster + Convert ORCID to Dataset + eu.dnetlib.doiboost.orcid.SparkConvertORCIDToOAF + dhp-doiboost-${projectVersion}.jar + + --executor-memory=${sparkExecutorMemory} + --executor-cores=${sparkExecutorCores} + --driver-memory=${sparkDriverMemory} + --conf spark.sql.shuffle.partitions=3840 + ${sparkExtraOPT} + + --sourcePath${sourcePath} + --targetPath${targetPath} + --masteryarn-cluster + + + + + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/unpaywall/oozie_app/config-default.xml b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/unpaywall/oozie_app/config-default.xml new file mode 100644 index 0000000000..cf617a84c4 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/unpaywall/oozie_app/config-default.xml @@ -0,0 +1,38 @@ + + + jobTracker + yarnRM + + + nameNode + hdfs://nameservice1 + + + oozie.use.system.libpath + true + + + oozie.action.sharelib.for.spark + spark2 + + + hive_metastore_uris + thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083 + + + spark2YarnHistoryServerAddress + http://iis-cdh5-test-gw.ocean.icm.edu.pl:18089 + + + spark2EventLogDir + /user/spark/spark2ApplicationHistory + + + spark2ExtraListeners + "com.cloudera.spark.lineage.NavigatorAppListener" + + + spark2SqlQueryExecutionListeners + "com.cloudera.spark.lineage.NavigatorQueryListener" + + \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/unpaywall/oozie_app/workflow.xml b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/unpaywall/oozie_app/workflow.xml new file mode 100644 index 0000000000..d2a69752ea --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/unpaywall/oozie_app/workflow.xml @@ -0,0 +1,55 @@ + + + + sourcePath + the working dir base path + + + targetPath + the working dir base path + + + sparkDriverMemory + memory for driver process + + + sparkExecutorMemory + memory for individual executor + + + sparkExecutorCores + number of cores used by single executor + + + + + + + + Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] + + + + + yarn-cluster + cluster + Convert UnpayWall to Dataset + eu.dnetlib.doiboost.uw.SparkMapUnpayWallToOAF + dhp-doiboost-${projectVersion}.jar + + --executor-memory=${sparkExecutorMemory} + --executor-cores=${sparkExecutorCores} + --driver-memory=${sparkDriverMemory} + --conf spark.sql.shuffle.partitions=3840 + ${sparkExtraOPT} + + --sourcePath${sourcePath}/uw_extracted + --targetPath${targetPath} + --masteryarn-cluster + + + + + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/test/java/eu/dnetlib/dhp/doiboost/DoiBoostHostedByMapTest.scala b/dhp-workflows/dhp-doiboost/src/test/java/eu/dnetlib/dhp/doiboost/DoiBoostHostedByMapTest.scala new file mode 100644 index 0000000000..4912648bea --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/test/java/eu/dnetlib/dhp/doiboost/DoiBoostHostedByMapTest.scala @@ -0,0 +1,70 @@ +package eu.dnetlib.dhp.doiboost + +import eu.dnetlib.dhp.schema.oaf.{Publication, Dataset => OafDataset} +import eu.dnetlib.doiboost.{DoiBoostMappingUtil, HostedByItemType} +import eu.dnetlib.doiboost.SparkGenerateDoiBoost.getClass +import eu.dnetlib.doiboost.mag.ConversionUtil +import eu.dnetlib.doiboost.orcid.ORCIDElement +import org.apache.spark.SparkConf +import org.apache.spark.rdd.RDD +import org.apache.spark.sql.{Dataset, Encoder, Encoders, SaveMode, SparkSession} +import org.codehaus.jackson.map.{ObjectMapper, SerializationConfig} +import org.junit.jupiter.api.Test + +import scala.io.Source + +class DoiBoostHostedByMapTest { + + + +// @Test +// def testMerge():Unit = { +// val conf: SparkConf = new SparkConf() +// val spark: SparkSession = +// SparkSession +// .builder() +// .config(conf) +// .appName(getClass.getSimpleName) +// .master("local[*]").getOrCreate() +// +// +// +// implicit val mapEncoderPub: Encoder[Publication] = Encoders.kryo[Publication] +// implicit val mapEncoderDataset: Encoder[OafDataset] = Encoders.kryo[OafDataset] +// implicit val tupleForJoinEncoder: Encoder[(String, Publication)] = Encoders.tuple(Encoders.STRING, mapEncoderPub) +// +// +// import spark.implicits._ +// val dataset:RDD[String]= spark.sparkContext.textFile("/home/sandro/Downloads/hbMap.gz") +// +// +// val hbMap:Dataset[(String, HostedByItemType)] =spark.createDataset(dataset.map(DoiBoostMappingUtil.toHostedByItem)) +// +// +// hbMap.show() +// +// +// +// +// +// +// +// +// +// +// } + + + @Test + def idDSGeneration():Unit = { + val s ="doajarticles::0066-782X" + + + + println(DoiBoostMappingUtil.generateDSId(s)) + + + } + + +} diff --git a/dhp-workflows/dhp-doiboost/src/test/java/eu/dnetlib/doiboost/crossref/CrossrefMappingTest.scala b/dhp-workflows/dhp-doiboost/src/test/java/eu/dnetlib/doiboost/crossref/CrossrefMappingTest.scala new file mode 100644 index 0000000000..f62ac2b67c --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/test/java/eu/dnetlib/doiboost/crossref/CrossrefMappingTest.scala @@ -0,0 +1,389 @@ +package eu.dnetlib.doiboost.crossref + +import eu.dnetlib.dhp.schema.oaf._ +import eu.dnetlib.dhp.utils.DHPUtils +import org.codehaus.jackson.map.{ObjectMapper, SerializationConfig} +import org.junit.jupiter.api.Assertions._ +import org.junit.jupiter.api.Test +import org.slf4j.{Logger, LoggerFactory} + +import scala.collection.JavaConverters._ +import scala.io.Source +import scala.util.matching.Regex + + +class CrossrefMappingTest { + + val logger: Logger = LoggerFactory.getLogger(Crossref2Oaf.getClass) + val mapper = new ObjectMapper() + + + + + + @Test + def testFunderRelationshipsMapping(): Unit = { + val template = Source.fromInputStream(getClass.getResourceAsStream("article_funder_template.json")).mkString + val funder_doi = Source.fromInputStream(getClass.getResourceAsStream("funder_doi")).mkString + val funder_name = Source.fromInputStream(getClass.getResourceAsStream("funder_doi")).mkString + + + for (line <- funder_doi.lines) { + val json = template.replace("%s", line) + val resultList: List[Oaf] = Crossref2Oaf.convert(json) + assertTrue(resultList.nonEmpty) + checkRelation(resultList) + } + for (line <- funder_name.lines) { + val json = template.replace("%s", line) + val resultList: List[Oaf] = Crossref2Oaf.convert(json) + assertTrue(resultList.nonEmpty) + checkRelation(resultList) + } + } + + def checkRelation(generatedOAF: List[Oaf]): Unit = { + + val rels: List[Relation] = generatedOAF.filter(p => p.isInstanceOf[Relation]).asInstanceOf[List[Relation]] + assertFalse(rels.isEmpty) + rels.foreach(relation => { + val relJson = mapper.writeValueAsString(relation) + + assertNotNull(relation.getSource, s"Source of relation null $relJson") + assertNotNull(relation.getTarget, s"Target of relation null $relJson") + assertFalse(relation.getTarget.isEmpty, s"Target is empty: $relJson") + assertFalse(relation.getRelClass.isEmpty, s"RelClass is empty: $relJson") + assertFalse(relation.getRelType.isEmpty, s"RelType is empty: $relJson") + assertFalse(relation.getSubRelType.isEmpty, s"SubRelType is empty: $relJson") + + }) + + } + + + @Test + def testOrcidID() :Unit = { + val json = Source.fromInputStream(getClass.getResourceAsStream("orcid_data.json")).mkString + + + assertNotNull(json) + assertFalse(json.isEmpty); + + val resultList: List[Oaf] = Crossref2Oaf.convert(json) + + assertTrue(resultList.nonEmpty) + + val items = resultList.filter(p => p.isInstanceOf[Result]) + + + mapper.getSerializationConfig.enable(SerializationConfig.Feature.INDENT_OUTPUT) + items.foreach(p => println(mapper.writeValueAsString(p))) + + + } + + @Test + def testEmptyTitle() :Unit = { + val json = Source.fromInputStream(getClass.getResourceAsStream("empty_title.json")).mkString + + + assertNotNull(json) + assertFalse(json.isEmpty); + + val resultList: List[Oaf] = Crossref2Oaf.convert(json) + + assertTrue(resultList.nonEmpty) + + val items = resultList.filter(p => p.isInstanceOf[Result]) + + + mapper.getSerializationConfig.enable(SerializationConfig.Feature.INDENT_OUTPUT) + items.foreach(p => println(mapper.writeValueAsString(p))) + + + } + + + @Test + def testPeerReviewed(): Unit = { + val json = Source.fromInputStream(getClass.getResourceAsStream("prwTest.json")).mkString + mapper.getSerializationConfig.enable(SerializationConfig.Feature.INDENT_OUTPUT) + + assertNotNull(json) + assertFalse(json.isEmpty); + + val resultList: List[Oaf] = Crossref2Oaf.convert(json) + + assertTrue(resultList.nonEmpty) + + val items = resultList.filter(p => p.isInstanceOf[Result]) + + + items.foreach(p => logger.info(mapper.writeValueAsString(p))) + + + + + } + + def extractECAward(award: String): String = { + val awardECRegex: Regex = "[0-9]{4,9}".r + if (awardECRegex.findAllIn(award).hasNext) + return awardECRegex.findAllIn(award).max + null + } + + + @Test + def extractECTest(): Unit = { + val s = "FP7/2007-2013" + val awardExtracted = extractECAward(s) + println(awardExtracted) + + println(DHPUtils.md5(awardExtracted)) + + + } + + @Test + def testJournalRelation(): Unit = { + val json = Source.fromInputStream(getClass.getResourceAsStream("awardTest.json")).mkString + assertNotNull(json) + + assertFalse(json.isEmpty) + + val resultList: List[Oaf] = Crossref2Oaf.convert(json) + + assertTrue(resultList.nonEmpty) + val rels:List[Relation] = resultList.filter(p => p.isInstanceOf[Relation]).map(r=> r.asInstanceOf[Relation]) + + + + rels.foreach(s => logger.info(s.getTarget)) + assertEquals(rels.size, 3 ) + + + } + + + @Test + def testConvertBookFromCrossRef2Oaf(): Unit = { + val json = Source.fromInputStream(getClass.getResourceAsStream("book.json")).mkString + assertNotNull(json) + + assertFalse(json.isEmpty); + + val resultList: List[Oaf] = Crossref2Oaf.convert(json) + + assertTrue(resultList.nonEmpty) + + val items = resultList.filter(p => p.isInstanceOf[Result]) + + assert(items.nonEmpty) + assert(items.size == 1) + val result: Result = items.head.asInstanceOf[Result] + assertNotNull(result) + + logger.info(mapper.writeValueAsString(result)); + + assertNotNull(result.getDataInfo, "Datainfo test not null Failed"); + assertNotNull( + result.getDataInfo.getProvenanceaction, + "DataInfo/Provenance test not null Failed"); + assertFalse( + result.getDataInfo.getProvenanceaction.getClassid.isEmpty, + "DataInfo/Provenance/classId test not null Failed"); + assertFalse( + result.getDataInfo.getProvenanceaction.getClassname.isEmpty, + "DataInfo/Provenance/className test not null Failed"); + assertFalse( + result.getDataInfo.getProvenanceaction.getSchemeid.isEmpty, + "DataInfo/Provenance/SchemeId test not null Failed"); + assertFalse( + result.getDataInfo.getProvenanceaction.getSchemename.isEmpty, + "DataInfo/Provenance/SchemeName test not null Failed"); + + assertNotNull(result.getCollectedfrom, "CollectedFrom test not null Failed"); + assertFalse(result.getCollectedfrom.isEmpty); + + val collectedFromList = result.getCollectedfrom.asScala + assert(collectedFromList.exists(c => c.getKey.equalsIgnoreCase("10|openaire____::081b82f96300b6a6e3d282bad31cb6e2")), "Wrong collected from assertion") + + assert(collectedFromList.exists(c => c.getValue.equalsIgnoreCase("crossref")), "Wrong collected from assertion") + + + val relevantDates = result.getRelevantdate.asScala + + assert(relevantDates.exists(d => d.getQualifier.getClassid.equalsIgnoreCase("created")), "Missing relevant date of type created") + assert(relevantDates.exists(d => d.getQualifier.getClassid.equalsIgnoreCase("published-online")), "Missing relevant date of type published-online") + assert(relevantDates.exists(d => d.getQualifier.getClassid.equalsIgnoreCase("published-print")), "Missing relevant date of type published-print") + val rels = resultList.filter(p => p.isInstanceOf[Relation]) + assert(rels.isEmpty) + } + + + @Test + def testConvertPreprintFromCrossRef2Oaf(): Unit = { + val json = Source.fromInputStream(getClass.getResourceAsStream("preprint.json")).mkString + assertNotNull(json) + + assertFalse(json.isEmpty); + + val resultList: List[Oaf] = Crossref2Oaf.convert(json) + + assertTrue(resultList.nonEmpty) + + val items = resultList.filter(p => p.isInstanceOf[Publication]) + + assert(items.nonEmpty) + assert(items.size == 1) + val result: Result = items.head.asInstanceOf[Publication] + assertNotNull(result) + + logger.info(mapper.writeValueAsString(result)); + + assertNotNull(result.getDataInfo, "Datainfo test not null Failed"); + assertNotNull( + result.getDataInfo.getProvenanceaction, + "DataInfo/Provenance test not null Failed"); + assertFalse( + result.getDataInfo.getProvenanceaction.getClassid.isEmpty, + "DataInfo/Provenance/classId test not null Failed"); + assertFalse( + result.getDataInfo.getProvenanceaction.getClassname.isEmpty, + "DataInfo/Provenance/className test not null Failed"); + assertFalse( + result.getDataInfo.getProvenanceaction.getSchemeid.isEmpty, + "DataInfo/Provenance/SchemeId test not null Failed"); + assertFalse( + result.getDataInfo.getProvenanceaction.getSchemename.isEmpty, + "DataInfo/Provenance/SchemeName test not null Failed"); + + assertNotNull(result.getCollectedfrom, "CollectedFrom test not null Failed"); + assertFalse(result.getCollectedfrom.isEmpty); + + val collectedFromList = result.getCollectedfrom.asScala + assert(collectedFromList.exists(c => c.getKey.equalsIgnoreCase("10|openaire____::081b82f96300b6a6e3d282bad31cb6e2")), "Wrong collected from assertion") + + assert(collectedFromList.exists(c => c.getValue.equalsIgnoreCase("crossref")), "Wrong collected from assertion") + + + val relevantDates = result.getRelevantdate.asScala + + assert(relevantDates.exists(d => d.getQualifier.getClassid.equalsIgnoreCase("created")), "Missing relevant date of type created") + assert(relevantDates.exists(d => d.getQualifier.getClassid.equalsIgnoreCase("available")), "Missing relevant date of type available") + assert(relevantDates.exists(d => d.getQualifier.getClassid.equalsIgnoreCase("accepted")), "Missing relevant date of type accepted") + assert(relevantDates.exists(d => d.getQualifier.getClassid.equalsIgnoreCase("published-online")), "Missing relevant date of type published-online") + assert(relevantDates.exists(d => d.getQualifier.getClassid.equalsIgnoreCase("published-print")), "Missing relevant date of type published-print") + val rels = resultList.filter(p => p.isInstanceOf[Relation]) + assert(rels.isEmpty) + } + + + @Test + def testConvertDatasetFromCrossRef2Oaf(): Unit = { + val json = Source.fromInputStream(getClass.getResourceAsStream("dataset.json")).mkString + assertNotNull(json) + + assertFalse(json.isEmpty); + + val resultList: List[Oaf] = Crossref2Oaf.convert(json) + + assertTrue(resultList.nonEmpty) + + val items = resultList.filter(p => p.isInstanceOf[Dataset]) + + assert(items.nonEmpty) + assert(items.size == 1) + val result: Result = items.head.asInstanceOf[Dataset] + assertNotNull(result) + + logger.info(mapper.writeValueAsString(result)); + + assertNotNull(result.getDataInfo, "Datainfo test not null Failed"); + assertNotNull( + result.getDataInfo.getProvenanceaction, + "DataInfo/Provenance test not null Failed"); + assertFalse( + result.getDataInfo.getProvenanceaction.getClassid.isEmpty, + "DataInfo/Provenance/classId test not null Failed"); + assertFalse( + result.getDataInfo.getProvenanceaction.getClassname.isEmpty, + "DataInfo/Provenance/className test not null Failed"); + assertFalse( + result.getDataInfo.getProvenanceaction.getSchemeid.isEmpty, + "DataInfo/Provenance/SchemeId test not null Failed"); + assertFalse( + result.getDataInfo.getProvenanceaction.getSchemename.isEmpty, + "DataInfo/Provenance/SchemeName test not null Failed"); + + assertNotNull(result.getCollectedfrom, "CollectedFrom test not null Failed"); + assertFalse(result.getCollectedfrom.isEmpty); + } + + @Test + def testConvertArticleFromCrossRef2Oaf(): Unit = { + val json = Source.fromInputStream(getClass.getResourceAsStream("article.json")).mkString + assertNotNull(json) + + assertFalse(json.isEmpty); + + val resultList: List[Oaf] = Crossref2Oaf.convert(json) + + assertTrue(resultList.nonEmpty) + + val items = resultList.filter(p => p.isInstanceOf[Publication]) + + assert(items.nonEmpty) + assert(items.size == 1) + val result: Result = items.head.asInstanceOf[Publication] + assertNotNull(result) + + logger.info(mapper.writeValueAsString(result)); + + assertNotNull(result.getDataInfo, "Datainfo test not null Failed"); + assertNotNull( + result.getDataInfo.getProvenanceaction, + "DataInfo/Provenance test not null Failed"); + assertFalse( + result.getDataInfo.getProvenanceaction.getClassid.isEmpty, + "DataInfo/Provenance/classId test not null Failed"); + assertFalse( + result.getDataInfo.getProvenanceaction.getClassname.isEmpty, + "DataInfo/Provenance/className test not null Failed"); + assertFalse( + result.getDataInfo.getProvenanceaction.getSchemeid.isEmpty, + "DataInfo/Provenance/SchemeId test not null Failed"); + assertFalse( + result.getDataInfo.getProvenanceaction.getSchemename.isEmpty, + "DataInfo/Provenance/SchemeName test not null Failed"); + + assertNotNull(result.getCollectedfrom, "CollectedFrom test not null Failed"); + assertFalse(result.getCollectedfrom.isEmpty); + + val collectedFromList = result.getCollectedfrom.asScala + assert(collectedFromList.exists(c => c.getKey.equalsIgnoreCase("10|openaire____::081b82f96300b6a6e3d282bad31cb6e2")), "Wrong collected from assertion") + + assert(collectedFromList.exists(c => c.getValue.equalsIgnoreCase("crossref")), "Wrong collected from assertion") + + + val relevantDates = result.getRelevantdate.asScala + + assert(relevantDates.exists(d => d.getQualifier.getClassid.equalsIgnoreCase("created")), "Missing relevant date of type created") + + val rels = resultList.filter(p => p.isInstanceOf[Relation]).asInstanceOf[List[Relation]] + assertFalse(rels.isEmpty) + rels.foreach(relation => { + assertNotNull(relation) + assertFalse(relation.getSource.isEmpty) + assertFalse(relation.getTarget.isEmpty) + assertFalse(relation.getRelClass.isEmpty) + assertFalse(relation.getRelType.isEmpty) + assertFalse(relation.getSubRelType.isEmpty) + + }) + + + } + +} diff --git a/dhp-workflows/dhp-doiboost/src/test/java/eu/dnetlib/doiboost/mag/MAGMappingTest.scala b/dhp-workflows/dhp-doiboost/src/test/java/eu/dnetlib/doiboost/mag/MAGMappingTest.scala new file mode 100644 index 0000000000..88b1669f48 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/test/java/eu/dnetlib/doiboost/mag/MAGMappingTest.scala @@ -0,0 +1,69 @@ +package eu.dnetlib.doiboost.mag + +import java.sql.Timestamp + +import eu.dnetlib.dhp.schema.oaf.Publication +import org.apache.htrace.fasterxml.jackson.databind.SerializationFeature +import org.apache.spark.SparkConf +import org.apache.spark.api.java.function.MapFunction +import org.apache.spark.sql.{Dataset, Encoder, Encoders, SaveMode, SparkSession} +import org.codehaus.jackson.map.{ObjectMapper, SerializationConfig} +import org.junit.jupiter.api.Test +import org.slf4j.{Logger, LoggerFactory} +import org.junit.jupiter.api.Assertions._ +import org.apache.spark.sql.functions._ + +import scala.collection.JavaConverters._ +import scala.io.Source +import scala.reflect.ClassTag +import scala.util.matching.Regex + + + +class MAGMappingTest { + + val logger: Logger = LoggerFactory.getLogger(getClass) + val mapper = new ObjectMapper() + + + + + @Test + def testSplitter():Unit = { + val s = "sports.team" + + + if (s.contains(".")) { + println(s.split("\\.")head) + } + + } + + + + @Test + def testDate() :Unit = { + + val p:Timestamp = Timestamp.valueOf("2011-10-02 00:00:00") + + println(p.toString.substring(0,10)) + + } + + + + @Test + def buildInvertedIndexTest(): Unit = { + val json_input = Source.fromInputStream(getClass.getResourceAsStream("invertedIndex.json")).mkString + val description = ConversionUtil.convertInvertedIndexString(json_input) + assertNotNull(description) + assertTrue(description.nonEmpty) + + logger.debug(description) + + } + + +} + + diff --git a/dhp-workflows/dhp-doiboost/src/test/java/eu/dnetlib/doiboost/orcid/ElasticSearchTest.java b/dhp-workflows/dhp-doiboost/src/test/java/eu/dnetlib/doiboost/orcid/ElasticSearchTest.java new file mode 100644 index 0000000000..69a2547fd9 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/test/java/eu/dnetlib/doiboost/orcid/ElasticSearchTest.java @@ -0,0 +1,192 @@ + +package eu.dnetlib.doiboost.orcid; + +import java.io.IOException; +import java.net.ConnectException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import org.apache.commons.io.IOUtils; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.junit.jupiter.api.Test; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +public class ElasticSearchTest { + + private static final String BASE_CFG_URL = "http://%s:9200/%s/%s/%s?pretty"; + + private String indexHost; + private String indexName; + private String indexType; + private String record; + private int readTimeout = 30000; + + private int nThreads = 4; + private int nTasks = 150; + private ExecutorService executorService = Executors.newFixedThreadPool(nThreads); + private List> resList = Lists.newArrayList(); + + public void setup() { + indexHost = "ip-90-147-167-25.ct1.garrservices.it"; + indexName = "orcid_update"; + indexType = "record"; + record = "{\n" + + " \"timestamp\": 1540825815212,\n" + + " \"pid\": \"0000-0002-5214-479X\",\n" + + " \"blob\": \"H4sIAAAAAAAAAO19zXLcOLbm/j4Fw4spdYSRAkiQBDQuV9iyyla3y65rqW7/bCZAEpR4nUnmJTPtUq36HWZ1I2Z2s+l36F29ST/JAGQyCclCiqJIZjKT3SXbygRxDvFzzvcdHAAvfvh1NjW+8DSLkvj7Z2gCnxk89pMgiq++f/bL5Y+APDOyBYsDNk1i/v2zG549++Hlv71IuZ+kwUnxlzFni+vvnx1D8T8gfkxgmwgD7NK/PDOEgDg7ieIFT2M2/f7Z9WIxPzk+/vr16yRJ/SgQf14dx9lxWaJ8ggdLny1ytXSPrIuUzwRRtohif/NTSqH1c3wu3oQteLDhsXWZ8qlkcc1TELMZ1z9VlSmfmvGZJ9r7Oprrn6rKrFsjTZN0Q0vIr8uyfjKbbWqA4vuydNGF+tLF92XpuVArEb0EAr5g0TTTP3e3ZFnDZ37zdaPAVYH1m8/E0xveXH69LvtrMYZAFPB4EYUR39Rm3xYu6wmXcTEDdM+uCqxbJeUhT8XE4ZsapCpUPseCIOXZhmdWBapZ9CUSAxDMkyzaPMjvlixrEC37Wf+U/LYseS2mSZLe6AuvClT9NJ8mNzPRlJs6qyxTPvVfSzYVTf/QTL9VrHw24+mXyN8w+VYFbo/dh0bsum+EhRBtGG3q0qpMNZsyzlJfTvplumHY3i5XaSh+T/mXiH/dpOa6UPmct5xu6FX57V39RAVZskw3td43RZ+9/DdD/O9FYT9O8sLK5Cm+VUss0+ilrD0T1VdV3+MlXhwrT3xTjfQuLzc9lRf45rHrJFu8XItdF84/Ll7keOObvFAmrDrDFUlqiWnisyl/yeMXx/d8vBK4scoXqxlVTj1FUPmNn/J8CoAZX1wnwcs3kTDPixfHuu+/rSFberMok+4eBMKZvTQhcgG0ATIvITnB9olNJo5j/q2q8+4T3zT0lGULMBOYQbRdUNZqQlkrdC8hPBH/mXCCkPO3dZPf88w9bztl0YwHLxfpkivvuPr02/ICxxT15U7hzmN3vtzw9DyNZiy92VzL7UKr7r23A18UdmVlfjZgpdIA1WxhRAESVdBL0zzB5AS6E4pRzRZeqSRxifElyiIvmkYLYevnS28a+c9WSt6H56pKVAXzcaeophlS95W+t777X7isFYkhZZ6YeGLDmkNKeekKkZxcRV94nIOz7OVPLI2Y8SnJ2ItvoMutgg/UGbJZNL3Jy758FS+Sq5j99tt9daoFH6hTtFcQLYqilZ7GxurVZ6p+P1Y6Xvm4QqkKqM02DVWl2LNjpabbfu2OO9xY452imoFWcyY42HzEwNiotDFfLoBgRMJTIoIdE5nPNkyZOq92vK5HUJH5lN2AKA74r2K+3XnpTTNs8yvrZ1jXjanWX8CGb7//tgzwp5HwvsIH319afeJ+VFFUcPzq55+B/Zd3/355Zlo//+J+fHVxSe9HF7rqczCxsZrbcENXz4Pw45sHjx/VJHcaMJ/RfxKc5LvMOE2mU37FjfdJHCTx3Xq/tTj3SL+nwJ1ZIn42CDXmaRJGUy6mxZVwn7pn60hZd/ZnfzpfpnzJFhPxzwnzJ8vPx/MkXbDpMY9Xpk2QZ2khQSosJGBrC3kUEjPwCA/FkBXQBHs+BAyGCPguDngQWA7x0R8m14vZ9D5t7xiMuyVu/6ra2832UCmZ44iC7G40kwXfVU3uiiOWZHKz3S4JpVrBinCXzHzj82WZWxXcw6Tvo+Kb3+ue8k/zAbXR0GPeQfEG1LRsCh/tDO570eOyric4BAIgAojI97cF6iITZJGOHMKjWlatf3QI++QQVt8rAxosbub85YWfzJeZ8WopKGhqnL9Z1/RNyVp1fmHTJX9p26ZJHYRdCO+tryhVq8LSp6yCHlmu7kQUPGZC5Sg4LjD0RJT7geUvcR58/5D8b5yETnrKpzlFl9HdlxmfhvdWd6vUHedT01SpbqiuhV7HKVTKuvqwCnYpsTGQLWeSB2/0OVWUrHHwwLZJTWqrqKlE+jf6HrXcbdda1bVea9jsxdaltPWsQ6APuPl1scd6QVOY69wL0BNkTdxHMSG1j0OB36IimnSVJst5A7exVoWc2HBiEruZ21BGanZ8T7GqsZTmrUbm2mM7pg2Ri+5xs/W64nhdwT0+f7NRvsdbk8qn2nRiQ6uut+6j6VUZm7z2t+XqeG71qY68tyriqR5crevRXjx/+HGe/J5Gfao3v0eLzZ0uXo7F0W/5/H9Yyxrq6fVSK1rxk3p94Yv59/LD8atq3sgP6j2aLONFevPy7evq4dVHD/fig0qWUoSdYTMvulrms7lek9Z7XvXz2EHOWrd6jzxWunx8NWo+nX94+/bj+zf3SVSKPdyKj20cZblmU9EXxxvdwV0wVd/jdeYcH++n1frbco42dFxq282d47qCJztHN28cLBvHJOK/CYW0G+fYrOlVGaNzNEbnuFnL0TneK2V0jhtacUjO8VY5hbjeT4BXqVMb2W9ZRkei72Y3bazsm8K6Wqusu431KcV0NSkpOhurUsvp6rqV+rSxttsldfV9k9ZTa1W2Kq2rd5VstbG2soyuDpl/trGCvMBjAiJPiCTdVa4JznuieLV+Fedt9k5KyYet8Dcx2ZWnlC0tPntE7HZTzUV01ifcJYFrA4c4JsCMQkA5gQBDhBw/9ELPp/cK1AR3N0mMk3QmpsNvPDAWKYuzKE9MlMk8z5rrUVX6OGWeGvNd133PY12OhSCJWh0ACE4QRtYxtQjtpacfFjiMLr2v1D1W4IWctbkVvS/6SRDBmGzmc/LZ46rkfWubNdYo1Ua7Q+6kRUTAyumX5Z5YeOLQ2uuUar1d2VtVxkjulLr2ntzlk2cRLaY1erwo9pqL1wmfG5/E31dRssyM8hMWB8aPLFpcr0VvqPjF8UOi64MATel6Xdw+GNDV3i0g0EndGijQKdSWF8nrrw0ONNo0HCPNQIKuypaAgq767sGCTvJ2uromcFCVfkzKXQ4JCoWPvdz0yTcobCFYfSBMIQilKTyqM8XKHDxFmU3WWo6x/xRGSr4ZSxeRP+WlNdUOv7VzrLTfgAXUJ24EF85DyGv98k8efE78yC0sZP1Y8cHDITJ28xKiKijENAG4ypvXeqWi7cpmK9zOJ0FzswUT6M74MZomWRJGbNWStwveg02/QZ0b4kgPLKy0QLIRmRC3hUyDgZJsFmJKUWCKNvEZwBxx4LkhBizgvsUwZJ6D+qBejfUYBiNrbyx0QbIhtY6TX8MkDa69Y+oSYbKoi7Fr2hOIrAl2+hgBzRUZxhBoh5Q70CVmLVJeluyGlJvSgkpSbp1YdGJTuytS3sQ+qzJGUq7UNZJyRbmi2JkvQGAWJdPk6ua5ceYvZzyOslnByS+TKU/z2T8S8y2ACJ3UrQEJnUIjMe8ZXOjEbg9g6DTah6ERZV7c+ti41fp9jYGHhbbW13OWLkCyd3EbrvhLwEt3mYdtFmtveVTHCjeI3XhJ8hn412wu3rKHwE0VgdEHbloIplxec+NjbpSMd6Id5UsaSWi855EXR78NNKxiTxDCux1W2QaVxrtCpe9XZBhUur0h0GFkjYQEObYVCGZOCMA+poBRV9Ann9jc9JBneo/am9h0MDTWYxhjob2wCqJ1wyqyZD9hlfZ2ebVgn1UZY1hFqWsMqyjKFcXybIY8hvKJs2yMn2hq79ZV6KRuzV3oFNoHkryt+EkjRKkTuz1UqdNoH4bGGD+pJ/kQ4id5bkseLklzx3hUx+COoZJ9DpVYE9dpYTvvvoVKGhm1Lpxac0O3fXrc3hDoMFTiYQpDy+bC8PkuwA7zAfMCKP5g3KLQtIQp7GMwNNZjGGOhxVAJqR0qIX2FSmBnoZIG9lmVMYZKlLrGUImiXFHs5+s8g3Z+LSDR1BAIJ19by2Mnp9epPGyMxcabxF+kUcyzMZCyBUeik7o1Z6JTaB/Y8rYCKb2NjG4xp06jfRga3QdS7n21MZKyi5GUueo3wWLlN/PQil/6TRCUfvOojj0e4yx7HGeB9sQ03d2OszQ0cB1Gibdj17ZPltvr0w4DJ3YYEu4jBGzumQBbPgMUcRvIDZLYtSkitJe9O431GMZYaCNwYkOLEGLVyDGpSrYfOEFE9AmA+bn/lnliOhMLd7Z1p4nBVWWMgROlrjFwoihXD6SMgZLeHYdO6tach06hkQ13BRh11e8IGd7HszQE9wVFMAlcrwyh4PtgWhjCozoTrAHzlTcGCj8upe0P8b3jT7JhUl3kTqhJd5vqdkiLsBtyxjkBbugTMeCxAxhiDgg8aBNomQ5Ffh+0qLEeh0OLRCM4FqHmw7SoKtkBLbpzV6w1cezuUu8bTE5VxkiLlLpGWqQot6ZFSVqsH/+csl+jzIjikhT96+//nZVLzKvl5mR5db0YydIW3IlO6tZcik6hkSxputYCNpI4Gjs2sPoaApaQmYu8X+K4bPggWRLmMV8mnOfmUWCGkipl5TLiakkxt41Hdebc7q8cuv0QqJWjMaKZ8T7yrxfcCHi6djk8NobJqCCdELPhPbZ7wKjcIKDIEraOhQKZY8hdQLBcSzdxACFxfOw1sn+PtX2N9RgGo2pvLHS0kExcRBxs9reQvEngMPq0RZZs1WbJ1uBZchODq8oYWbJS18iSFeVu4ZSR+G7B6+ukbs3z6xQaiW9XaEBXffeIQCd571cJy+XAOjOpIZ3tgcZWqQCPOky/8poz/WH6j6C/u8ZPkT3BpnWw/NR3oMUsh4DAtMSY9m0XMM/3ADUDC5nchtz2+iAtjfU4HC6DTSwaiNRIhKxKdpIICa0yEZKeYDShLurwsK1HT05VxshllLpGLqMoVxRDLnKMf/39f99a5ZODzJA+cqQ4W3AcOqlbcx46hfYe9srJAaoFntz2ymlxVKe5d39Zpw4ebm9ZR1gWY8oXvy14JoyNx+PsP9l1OswlHUQnFMGDhcxW4KPADCEgLKQAMxICBj0b+Jy7DAUoYAHvAzI31uOAIDPC1HbrJMlVJbuBzGYOmS05fTCeUKu7a38aTE5VxgiZlbpGyKwoVxR7mwTPjVM5ZAWOyIrbfj7wREyPRSIvNFBy5kb8vAUvopO6NU+iU2hcItAvEViYuMjGFuxzI9EDQsf0uM106SoJgF/axTxLLlbMopIrd1RnujXgT34ShzzlsTBaczbfJw714+//TI1lnPHUeDv9/Z/+ZyORuXEBy8rfRXOLVx8qnyITbDu7zad4yyTKBJk5ETPAFugYOsQ03T4oU02pwyBI7XVuh2SZUmSGtu8AAnkobyBngHHXAq5j84C7GDu8n6S4pnoc2ljo5MBi6Mqj2oAFLESBRYgFAf5fvfT7Y8UPo7tbTI3EtVMj8eBTI5s4WlXGGBtR6hpjI4pydw6kLTYRXviRRONjROSh2ruFCjqpW4MLOoX2ISLSjCfoqmyTK+hk9MQXdOL3odM7O3i4PdyoE9Y3dtTpsQ/DoLNoqGj6vOV7u7HrYaGDjIZqFNrN7jYp6fNM6Urq/vR3j9Hv9ZnSxQbxrIC+atS7DqQas4bKY6S5QNrxgseSs59lGbvJjCQ2FtfcUEjG/2Cz+f+UP2uqkYTG28mfJyXfGO6ecZvu+B3oXd7q5EBqQlE5h1Aem0AhYIE8RcElPHA9j/rwfvvR9q1OTfUYRhCtvbHQ1eIIpg41TYSsXs6iril1GJ3bRoTUcS3XwqRGhLQq2c2VXQggO99w4Z5Ac+LCLjePP9ryqjLGCKlS1xghVZSrTp5+J6xiKlARv0pSPhOz3TfO4zCKo0WVZDTGR/uDBTqpW4MGOoX2IUbSZXz0aXBBJ6MnyKATv/fbbOQZ1NffmEQBFgqTeFRnqjXgzSUlZKmQNuU9UOcqrfAxG9BRFWudtbIBvWqHOxt1zs5ffzj/m/Hp7D/Oz/48TOKMrIlpkd0mzh0kmGDLxDJKLw9ncB0To/tjeO2nltQUPAzK1F4Xd7n5igXIczwTWCGyAQ7l3kMm/oAB90Lf8jyb9pJJ2FiPQxsLHcZGIKYU475jI5ukDqNzW8wec2pnjzl9ZY/hDq8zf7RzVWWMsRGlrjE2oih3a6e40Kw4aVku5mRLL1swubIjE8quo8yY8QWbX99kkZ/JAnJFKIp9JqaxNA1jAGUL2EEndWv4QafQGEDpElPoZPSEK3Ti96HTO0owa4c36sT0xx11GuxD13eVZSSa3XUwMc0+T+F8QOggk4w0Cu1adyOLuBSJf/XZ3ZuFDrK7e4yM37pbJL2RNzOu8XCeZSbwMFDwsCwgo+kKHj6qg7N2P+usWegc1gqdq88F7OYlQuun5K+dhdsT/3PBaspctSQ2LsruHWj4HU9s/Mh7u/sOv3cXjzMxQaJFet3I/YDUYcTj2uvcTjbvEngMBT0hLiETYYvwhLpYtHhPSyyPk39oHd7hSktgQWIG1AfQCgOAWWgCYtsOsDnktmk5iONG+zEeOwYa6zGMsdBioqJdO1HR7itRsbtj7hq4WlXGGIxX6hqD8YpyVaLiax7zMFoYi0TddLEKul8slkH+y/kiM95F2UIwlDH6vgV/opO6NZ+iU2gfonHdRt+fwiB0MnpiETrx+9DpnW3vbpFZ6KT1zi50ihxEHqtXeEywSICyIXQVlsukx5S/RItMhu6kxzyqY4n3Jru1hbDZ6zRaRNm18cfiUyNM0hyOrBCIBCQKVhlmGA06EwfTgw2j2eL/gqX1HEbbJHUYTLq9zu0gjGYTFx17zGfBcXEbn2s6NhYjHUJL/MBeshgb63FoA6DDsBqyqR8izwLYC6BweL4NPIwpgL5tu45rIdPtJ7TaVI9hjIUWc1xJ7RxX0leOa2dhtSauV5UxhtWUusawmqLc3RzXN4m/SKM4P7LkMpnydMxf1dferevQSd2a+9AptA/BlI4jaE8gDzoZPREInfh96PSOImidkAqd1K0RC51C+zAwOrxjHrkOxHZvsfSHhQ4y3VGj0O5192q69dvdm4QOsru3kd0arJBwHidfI+GjOghr9zNXq2XUR2WuVjmos/qZq1aFOLvMXP0gr3hiU+M9+5rnr1bsRZ70LoPxZyyd3hhn8TS6ul7weCbmzUAj8XjiOgcbiYfCuDkU9nKYRE2pwwi+tde5HQZimcl46GICfGz6AFvyPFrPNIEXWDRkNkOm00sqc2M9hjEWWgvEUljjGt+qZD+BWKezQGwDy6vKGAOxSl1jIFZRrih2PjV8gazSyFsuEiPg02liyASNKDF4/is3eBD9JgxAJKaNseDZIpLfJP5SwplIQB3xvfibTafMkLkHUXKVsjBiRpinH4QCXBnAiIQcliXGdLwAZ0Pt3XokndSteSWdQvsQ0ek2vvsUSKqT0RMs1Ynf+8S4aAoqUwtyUwsKUwt4/isHpakFQQRyUyu/KU0tWJlaIE0tUE0tqEwtAFKMsLTr+xbqzOBBZNbVCSG0QOs/Cd6cLZj0dj+umpXJm+3BhZ9I9CX92TBZPHImLkW7zeK7PDIwcJ3Q8RyAPUYBxqYFCEeiibDrYR+KAeb1QvEb6zEMotfeWGg7/e7Nx/OTPNebWmLizdMkPBHTe7VMRR0HQsvsOw/vCcoMYzi0wftXuxVrHDJYlexnX6Pb2b7GBrZalTHyfqWukfcryhXFflofl8KmxtkXQc8/cTllxAgeufkWQIRO6taAhE6hfeDmXaThdA4udOK3CzB0Wu3DMOkqSwNAgCgougD0eexc1fFjpkaDAM5M8ZlAeMupeJmVzzyqY4v3I0ujjcwJ/lW5zHKVC/6vv/93Zlxec4Fx/ZthBligLQ3qbgdYukuTcAmGyDZ7TpPYKHUYFLm9zu0weuYzEzkcuoAiKwSYmj6gpuUCAm3fFa6FYMfvo+cb6zGMsdBimgSsnSYBh79frYHlVWWM4RKlrjFcoihXFPs5jfJJNmMLMS2fG3Pxu4ANX7iYYlmW/518lV/IRNB8aAtQOb0xprLYOif0/Zj8sKH2bv2MTurWfI1OoX1gzt0mPzwFaOpk9AQ2deL3PvlhXhhQUBhQsLafYGU/QW4/89O8K/MJKvMpQMg6o6HOtBwzGireLI8ojNjK+ywiFrNh0msEJ9B+5KXke5S/4Fph4Pt2CJBrIYA9SAAhNACUB4zjwDO9oJdV68Z6HBgDI5TWPjGEDp6BNZmcqoyRgSl1jQxMUW6VlsdZlsTPjZR/WU30gmyx9CpPj8zWR+C94WIOjRxrC55EJ3Vr3kSn0D5wrM72kGNkuy4m1O5rCDwsdFyZ3Myu0tw4gso25kRqbRqBMI350auBNI1HdabcuFZZpoMnS/FXIDzhaTKbs1iG8haJcZZv6RaNe22cij+XAv5V560OlF7RCbIOl175mAeB6dmAUuIAHMhzFhyCADQD5NoOIybr5SzWxnocDr3CJrYcQmsscFUlO6BXBEALQJLnA9MTjCbUtTqjVw0mpypjpFdKXSO9UpQrip2J4RRNjXmSiWkylVt/DT9NopQbmbBPhig7T3kcpPyHkVltwYnopG7NkegU2vuFDLkrcwrKeSIQNSjmCZDzBFTz5KhO8+8+yrb6Qdmnog0FMYuloZlxmRX4hS8XPwwTSEM8ge6OpwF2CKSxb9PADEIByEICcEgY8GyXA8dEPseexwK/l3WKxnocDpBe5X/h2plieB8O1Hn05FRljEBaqWsE0opyt042z7PZA+Vs8+pExxNjfSeS3EnwXC5aJNPkKt+KJ5c05im7momivlHE9bIRcm/B3eikbs3l6BQaFzM2LGasz6a999XGE3F3kWI9cCIuuGU+gWI981WPtfUEK+t5VGdWjkzs8E61RXBC6eEueIRBiCEReN91uID6DkeAIscTTRRyhzNCQr/R5tTHGszGehwOTyuWMUiNG6iqkr0seDjdLXg0mJyqjJGnKXWNPE1R7rafkz7uYjmXM7P44KebTMxT4WJG1rUF56GTujUHolNo7xc6VvMhx9SZMkHArJwgR3UafgTWKzj8lmeimQVgNgKeySTVbLbMJKY+W6bJfKhbMtyJC8luQ+gObvHND9VJfg2TNLj2yiN1bGxTF03EgJpA9R7wDqHzU1QZBnpubxh0yKSga7shRCagzJKHv1g2YH4YAt8PEXNhQMIA9TEcGusxjLHQ4oqXVXvFyxr8ilcTG63KGJmUUtfIpBTlVhnS+eQW5iObjZRpC15CJ3VrnkKn0D4sVHV0g2tXgFIneJugUqfTPgyPDtcxV+3f7zLmJqHjMuYDm7Iqv3hUx96OkZJV1OPymhsfc+NjvGNxIF8yz6ARH7+L5L0nN/LXs8V15GfDDJtAPLGtHQ+bdMiXOSEBdW0CsOVyORsY8JDLACecOBiRkLFe+HJjPYbBl9sbC52E0JB7fAGh5VALOghJCk5pH71eCs9qCT+0rv6aZMu2w2OioSF1TWLLdoZ99HEdkcPo2RaDXnbtoJc9+KBXEw+ryhiDXkpdY9BLUW51IKiag1gkyilZ3NXh5hK0XqZRPB5Jo6u9W0iok7o1WKhTaB+iH42xg67W1vCDTkAfGEInex96vLNwaFvkQCeqX4Kg02Lvk4hup+rLVCI1XZ/nbhIkcX440aJwk0d1zG/TWJm8sIV/7SFUViWc6UNl6nPiR6AxVPmV4oOuQmwXpx8vL88v3hl//PjLpw+v3hsffzQu3519fP/x7V8HGlJzJwQe7uW2HsWMmqYHiON7gtA5BBDuiibybSd0A2iZ8H6BLRPvxnoMg423NxYarhRtXCUCUDS7TV1gIxs02/HWZJkolylEjrvp8+BJjWtqq5L9hFk6u6a2idVVZYxhFqWuMcyiKFcUe7e8Soy3abKIltlzGWIxLtPl4rpcDjy9TqNMnmhvfOLT6CpSVB6DLP2BBJ3UrQEFnUL7QLk7vJX0qeBBJ6N7AKGTvPcE+1oYSHBVGMgVixb2Md8wL37xS/soXzG3j0d15l0Dds0DeQ0qkCR7R9h1Cyz57l74W3vejdOpvLpmsCkozgRZh3tIme2YNrSIBagnz8AmkAHKbAZCx+GhQ0PbJr2c9ttYj4OjVW5tWuUOn1Y1mJyqjJFWKXWNtEpRrsqwPE3iUNgQMUUkm/qRRYJWfZV/FDetlKe9/Ovv/+dNlPlCfmb8nP7+D3klW8yilP/r7/+3pGHVrd0j+erd4+ikbs3r6BQayZeefFmC/UBKEe2VeFVSxwT/BtwrZ1hrKyo5VyitKJBWdHX4mLy2Mr94ZWVCwVwSsZUFLVnaYmU+j+pMy93fJYD6IWfvuS9vCc0MvhC+SrzaXHin4rLQLD9iQSZhRfKfwkf9/g/Zwr//Iz/Y/u3E+POkuox5kOyNTCCiu83eml0CrDeTq2t4XUpt6riu1ewC4MeayZpSh0HH2uvcDqm5YweOQyEHISI2wCG0AHGQA1wv9E3Mqc1xoxvAH9vzjfUYxlhoj5ojVJeay5L9UHPSGTVvYHlVGSM1V+oaqbmiXFHsfGqkbC4RZ2KEPOAgZTJuz42YT405lyYqTUrwErE4GUn3FnyJTurW/IlOoX0g3c2ApK7KNsGkTkZPgFInfu9XPaMpKI0kkEZyZSOBsJGgtJFgWtnIozpzrwGzLlkjSxeRP+U9kOtqG0K3t54KrpotmABmxo95BncYMeMDT8CFn0jEI2odJnFG5oRguNvEuZstuCYRGNwl4l/K1bkd0qc6IofBlNrr2S6vr3UC4iFKQRB6qNg6QbjlAZNAEtgO9zzey5kKjfUYxlhokTXXuL62Kjlw1tzE7KoyRtas1DWyZkW5W7dunRivk+DmuXGx9AR6iX3+3PgpiVllaUeO3J/n0EndmvfQKbQPHLm7rddPxY06AX1gR53svSfIK+4LPGEOBQJZWUPhcYUxPKozyXZ+g21PRPjDMvZlavUfi0+Nj2G+C6U8h078euGLkevzgdJhZ4JNc7fpcJekift+GBIIaGC7AENsA0a4Cwj2rRCZhNGeSFNTPQ6NNEFaOwuYDp80NZicqoyRNCl1jaRJUe7bq4p/4gs2v76R+1nWWb3jsVUbau/Wgeikbs2J6BTaB+7U2Y5KaiOMEMSot6vQhNQHhI5JvTUvH55VFnGdqLs6oajOTNv9PN2eGNTqXMSbYgdlfnfwTwLRpbGxPkPxxjhCtg0Bcm34h2ESKWhPCHV3m0h1tK7oOpaLrF7XFR8SOQxa1F7PdkiRA+LCgDECOA5CgE2CgOdaDDimFWIzsE2b9nLVXWM9Dm0sdHJWO4GSb1uWS8U/TRtSy+ol+/5xwofR1S1GQ2pcCF6VHPqe6AYeVpUxRkOUusZoiKLc6kTvNFkkV4yzMeCxBTigk7o1SKBTaB8CHh0uFj+RDOgE9EEIdLL3occ7O6e7LWCoE9UvONRpsff5AvPS+R3VMapjckDh+F99+PDq/YU8O/vi9Pzsw+nZMGNXCE3MA76NjjkODSgJAbJtuTfEcQD1/BBQi5mcWgHFpJc7qxrrcWi0F+HamdO4L9pLO0sCaDA5VRkj7VXqGmmvolxR7HWUiekoBtM0yFdootmMy/l+YiRpdBXFeToAMz7zG8NPYp/PF+XJYDxfy5kVazlB4svVMS5LL+PoC08zNjWuWSokjSkE23A/Oqlbc0E6hfaBX3WWQmBhgi3HMXFvdFqeC7ZZ6JhCsJlMeZVJza88Kk0qWFlUmU7AgLCoYGVRy1PCcosKCosKSosqS68tKlhZ1KM6M3XnUxAg7YenXcx//3+LlMcsyrI8D766vZFLb/aGLxeZfz2V/g/ZLgTIseGJcRYvvv7+zzTkxt++RuJ7HhvvljMWR9lsmRm/iMJ/SuKQZ9nqmveIp8v46rnx8fPn5XQhSl+mLIikvuLfsviFf72crvNKPg+ULcKJSezdZotdXK4DBcMg0EEU9naxzlrigGlfe53aYQiAeJSHrs8AC7mwpr4bAI+GASA2YQ4lNnRpL6eBN9ZjGGOhxRCAVTsEYA0/BNDA4qoyxhCAUtcYAlCUu7N5+lVsnMcLLrTzF0tB319HyVUqwcpI4LfgPHRSt+ZAdAqNBL4z1Kirv3vkqJO890uh5dZpJg/vrowh8EpjeFRnkjXk3sPj3DtHT+kE24d7rxHlLnMC6ICAegRgaHIxLCVUDUPMTWw6HuklXbuxHsNgMu2NhY42ZTiCj1BHJkT10t11RA6jZ1vkqGZtjmoOf696A7Oryhg5qlLXyFEV5aobq1Y8FbzhmfE6yTIu1UxTns2FksUZLCNN7R0Z6KRuDR3oFNoHmtph5vYTEYNOQB+oQSd777mqXDRe8dWAZ8DLjaIAIapRPKoz2XY9o7evleLXqSCi2bVRHvcVJmme61Se95WE6q71gS7g2hPHxQfLkOVRd7ZtMtEgHAHsUHluA8bAIuJ3z6IWdFgflKmxHsPgUe2NhU4W8y3hcCB0sdvbYv5a4oA7tUVybNcmx/bwF3AbWFxVxkiOlbpGcqwod2sB10hi9eC2PJ37PPaZmKhy8p+UNzvLzz/xLyubUKZzX4hP4gUXP9fGqfhzmY7LvtvAETqpW8MSOoX2gU93tuz7NHyhq797jKGTvPdUulz2TWL1qLcif7uyoeW1zvLjdG1Cy/ztrDKhwC9M6FGdqbm7i8XVVur9XCyGE9va8dugOrpG2bEIpiZCxL7/8vdurlF+SOowaFB7ndthnMN1bYsGHgWe4whWxWxb/MsOge0ExPZd4lCrl55vrMcwxkKLlNipTYmd4VPiBpZXlTFSYqWukRIryhXFTsUv4nUyyYlfBWyWjVx2C75CJ3Vr/kKn0D5w2Q6vSX4iWNTJ6Akw6sTvQ6d3EsCAiLqmg03Y3xlumyWOO843By/8lbeT0Qsmvd1RHSu6+/vDqzhWt7dd5w/JLdyz58bPYhQkco93Hid/mwQDXeWHE9c53Dx427cR9VwIXJuL8W9yF1BumgAx1xM/RPz0crNXYz0Ojv3WPssa9XWWtQM7ZL+PnpyqjJH9KnWN7FdRbnUwZXq1LOhvmcB29qs8lCb28wO6hFM7ya+xPE3iRRSLgmxqnC3TZM6ZPA3FEwNw5Mtb8C46qVvzMDqF9n45kJUzCIgZVBzOVM4geSDTVRLkH/rVBAJ8NYFAkE+gozods/sQ3OkHgufWiM28NAqubqXbnkVX17ezUpQM3GFCc+hOEDrcBFxGPOzQEIMQ4wBg4lJAAoJkeroVBlBMDJ/2Ac0b63Fw0NytDc3dwUPzJpNTlTFCc6WuEZoryq1iTfwLZ1MuMzCn0VWeljki8SecntuRM9FJ3ZpD0Sm090g8XU0Y+Q75hHkQeNfphxF4HyLwJhMbHi7wdnyHWi5lgPk0BNgjUMwLRoDDGDM9RAI77GXnW2M9hgG82xsLHZ0NY5o2Nm35r17uNakjchg92yKlorUpFR0+pWpgdlUZI6VS6hoplaLcarXDEL/OWSw3sy2S2/eSzNeQZTLyqC1gA53UreEDnUL7kAzW4ekwT8QMOgF94Aad7L1nzgys7SJYJLdvF6ns4lGd6bbz58NUeYOdsuWf352//3jxUfx1+uq98e+/vPp0efbp/V+HSYeRObGps9t0uNnFtXpblt8Zi5zjCwgtCiyHoCNo/UEaGAR6WXUqFchqKzAMbtRel3e04VG0NzEtSrHZRzfXE3poXdthcMtCnFAmKmfEowCHpg2YFVLgcEwowQJJkEYXUz+24xvrcWhjoZvgFiKWaVIkgyq93NhTR+Qwera94JYJ6wa3ZMl+gluos1TeBiBKlTEGt5S6xuCWotzts53kZhSZyDsXbQFOk7mYooLCxOVlu3xi/Cm5SX//hyEpVxaJAT7Gu7YAF3RStwYZdAqN8a4uYYROQB9QQid7H3q8wz3OT+OHOhH9cESd9H3o8mYRIF2VHUWBdOL6jwTpNNn7aHd1b1exNUMCJX+Nk8ortDn4LGASB2uUdFTH+zYIf5exYJYuIn/KewiBVx5DHwJXnxM/ApcreWbFB12Fzi8uf3lzfnZhnH8w3p1fXH789Ffj1Yc3xjqi/lfj44/Gxen52YfTs2HG06E7sfGOnybXYQQOuj73LGQDlxEXYCaaxuMhB27IKMOMQT/oZct1Yz2GEadpbyx0FYGDxLIkhkW9nC9XR+Sh9WwXq2aWbR1fz68nwtMI1AppL3O5vuBhdHGLQVZUO8iKBh9kbeJZVRljkFWpawyyKsrdCrIKzX7iC3nHbxb52YlxvsiMj2l0FcVZHn99I098TuZyX7hxVKS2/GGMsm4BEuqkbg0W6hTahwBMl1HWp0FFnYA+4KJO9j70eFcht1YgpE5MfzBSp8GhhNgyMKu8JIjksYOFk8xjb0HlJFfpn0d1TO/ehNhaCJX98eMvnz68ei8DYpfvztbhMvFrFS0bZogM2RNquQcbIvM9HIQ8QHIOQIAh8gG1GAQuMuUZt8y0EG9iEx9rDxvrcXAsG9dm2bgvlm12eE3doyenKmNk2UpdI8tWlLvFskfGvAUPoZO6NS+hU+hQQPRRndbd/VNKzH4Q8ZvoN3lCdxolxnks3kx4VT+aTyPxERcu1rjwxcD9jRnc+JEHfKDY2JyY5uGe2E0cK/AJd4Bt+fKiPC5IostsQELMLM4s7Nu9YOPGehwcNrZqY2Nr+Ni4weRUZYzYWKlrxMaKct+m+a9OBlxmxmUy5Wk+7UfQvAXXoZO6NfehU+hQQPPq3t3V9ACL9fQ4qtPsgwgx9wSoX/109un89NUH4/TV5buP789PDc3RBsaJcScc/dCjry4uPp6ev7o8//hhoEAcT5BzuMcEUtviDvQswKlMiPZcDqgjl2wsTF2fmJ6D77cvLQPxxnocHBA3awNxc/CpYE0mpypjBOJKXSMQV5Qriv3EsiyaJcZPy6uYRScy3JQmwVKGn7jBplNmhPnJSWHEZOCpxO1Hn8Z0sA21d+tZdFK35l10Cu09Tp8VswfM8tkj/Ek1eYCcPGA9eUAQgRLVr9JE6nTJiOErIP06FZA3uzb+WHy6vutLOcS7OrHbGCgWRxPsHG7CCPNc7EMSAj+0fYBDDwMWmAHgvmdDzDAN3V422TTW4+CwuF0bi9vDD4o3mJyqjBGLK3WNWFxRbrXqy+XkkIiCceMsFh+KIZX9YLyWN/GlIZ8Geby82r3xMV4k0+Qqd32fStswQvItOBid1K05GZ1Cew/JA2USAV5OIuCt51AeVF+ndierKSRvuVy/9VGdrhkENEc1oLn63Cw/EwFViH7W6SEJ78/OX384/5vx6ew/zs/+PEzIDsmE0B0/BqGjM2YdTDFFDkZ2L6i8ptRhYPD2Orejcw0c1zYtS6L6fo6xqCHy0Hq2m9PA3fwcKIxRftMAtLH9lz46uBSe3RH+69jV3QZVuGNSQh0KPIdDgAkmgMndaTZlPnaZaYeW3ccAaKzHMMZCi0EVUjuoQvoKqlgd3pb1aPikyhiDKkpdY1BFUe52pmHAjTc8MS7TKE5OqnuyBG+ZGiybc3+RyTBKFVzxk9jnc2kZ5OdyoUE+K6bZGGDZgrPRSd2aw9EptA8HInR4BMYTWYVOQB/MQid7H3q8w4OGnxgk0MnoKVCgE78Pnd7dUcMtUUydqH5ppk6LvQ+ll/kqARf/JWAhoRO4hZzACjnJ6Pk6pF4BJ/mxPJt4UQCnozoOeW/C6i2Exz+dvT9/e/7xlwtjdZrwMCPkCE0oMnc7Qt5JqI3AY0gdSFwCEYIQ2xZyewu11RY+jPBKe13d4YV7GFsm7fvCvQ1CD61rO4yiOj4hgUNtwH3iA2wxAoiLfeCTIPQs12Uw7KXjG+sxjLHQYhTVrR1FdQcfRW3iYlUZYxRVqWuMoirKFcUur2X4NOQC2ZfR0J9uZEaNgIh3wqPrq9vOY5+JGSytwon40Dj7lc3mU347yvrdR1E0/c74xFk2bvvW1N6tB9JJ3ZoX0im0DzGXjm/0ag5AdSL6AaE66fvQ5Z2F2dqilzpR/VJMnRZ7H2aTEbKg8K1lwGxW+tY7EbT1bV9R5VvFZ4AXrvVWGC6RjlW0i/SrR3Xs9Rh5q2JqB7HXzJkgh+x2WK5DQo8RCblpM+D6rimmBAwAY9ACnHsYYur5PsF9EPrGehwcoXdqE3pn+HvNGkxOVcZI6JW6RkKvKFcR+g/8q3HKBCRJrpYVrWfxMvPTaF7kQ11eJzOWCXoejResb8Ob6KRuzaPoFDoImB7zr8AvZ8warFczpvhIzhjxplFwVKcnRtitLnifvxn4WjedOK5zsKDaswm3TA8C4nMx4G0XAmaZDPDQsSD0A9NyejnVuLEeBweqaW1QTYe/StZgcqoyRlCt1DWCakW51V6DKAj49LnxZhn7bFzM2oaj0EndmrPQKbQPKxtR5sWtjwxEbBs6LsSkt7WMzRJb6+W5wOuCIewbMZrmVg8EudE7qjOZGlCeIPIX+ZUoN/JQj/SmB84Da3Ae9blZflQGrKj1TH92hvpcwG5eomojkvy1K371S/w5Tr7Gw6RWkExM+3DXK5Bn2xblDnCwawNMbAt4zOUCO1uBh3zHplYv6xWN9Tg0amXButRKlhw4tWoyOVUZI7VS6hqplaJcUex8aqTRlMV+lBgBv5pGRrZYBpEhw6wsjgwWGK88ngacj7xrG15EJ3VrnkSn0N6vU0RTUM4UkM8UkM8UsJopgAWArWbKUZ0e2P1LGesA9RbA84/rg/C5cbqcLpYpM2I+NS74YsF9Ma4TQ6YNsTiO/GSgENueYGfHz7LrEGJbjh0wikPAbdsDmGITEO4gQDzL8gITce73chZaYz0ODmLXvgrG6u0qGNwZxG4wOVUZI8RW6hohtqJcUey789mMywn+Xb6F5zs+m/M08q+TlGdR9t3EkDlD4keel5sfm/QpSYokoXcsFdXcGFGsOat6hORb8Do6qVvzPDqFxqUQTddSl1CICbIty6R9DYGHhY4LIg8wsJUdzXdr3LaixWaOyoaCVNpQmTZ2XdhQAUQ0Z5Uf1ZmbO07WEKW0H7ImfdVfk2V8ZahXDV9HmbJv47mBHOwA5Mhz9gdJ15wJcQ+XrmGHhxRbAoC7kAPsEAdQ7CIQhsSjEHI/dHqha431GAZda28sNHSSG6ATxi7EyIK9HKvzgLRh9GaL5BvVJt9o+OS7galVZYzkW6lrJN+KcqsLyqcC893izEYSG2+T4PlqV86VAIWpGGbr4zV+isQbxHG0nI3segsgQSd1a0BBp9DIrjsBD7q6OwYQOrF7v6zJcgOpEmIgSOFVEqz2YJX2cX1EwmxtH4/qzLuGzHl4jPlJJPbWd7KKVW/f+rz6ZyXgxXHK/SQNToq/Xv7b/weirs3DHDIDAA==\"\n" + + " }"; + } + + // @Test + public void testSingleFeed() throws Exception { + setup(); + String url = String.format(BASE_CFG_URL, indexHost, indexName, indexType, "0000-0002-5214-479X"); + System.out.println("Sending to: " + url); + System.out.println("record: " + record); + + System.out.println("response from ES: " + getResponse(url, record)); + } + +// @Test + public void testMultipleFeed() throws Exception { + setup(); + String recordTemplate = "{\n" + + " \"timestamp\": 1540825815212,\n" + + " \"pid\": \"0000-0000-0000-%s\",\n" + + " \"blob\": \"H4sIAAAAAAAAAO19zXLcOLbm/j4Fw4spdYSRAkiQBDQuV9iyyla3y65rqW7/bCZAEpR4nUnmJTPtUq36HWZ1I2Z2s+l36F29ST/JAGQyCclCiqJIZjKT3SXbygRxDvFzzvcdHAAvfvh1NjW+8DSLkvj7Z2gCnxk89pMgiq++f/bL5Y+APDOyBYsDNk1i/v2zG549++Hlv71IuZ+kwUnxlzFni+vvnx1D8T8gfkxgmwgD7NK/PDOEgDg7ieIFT2M2/f7Z9WIxPzk+/vr16yRJ/SgQf14dx9lxWaJ8ggdLny1ytXSPrIuUzwRRtohif/NTSqH1c3wu3oQteLDhsXWZ8qlkcc1TELMZ1z9VlSmfmvGZJ9r7Oprrn6rKrFsjTZN0Q0vIr8uyfjKbbWqA4vuydNGF+tLF92XpuVArEb0EAr5g0TTTP3e3ZFnDZ37zdaPAVYH1m8/E0xveXH69LvtrMYZAFPB4EYUR39Rm3xYu6wmXcTEDdM+uCqxbJeUhT8XE4ZsapCpUPseCIOXZhmdWBapZ9CUSAxDMkyzaPMjvlixrEC37Wf+U/LYseS2mSZLe6AuvClT9NJ8mNzPRlJs6qyxTPvVfSzYVTf/QTL9VrHw24+mXyN8w+VYFbo/dh0bsum+EhRBtGG3q0qpMNZsyzlJfTvplumHY3i5XaSh+T/mXiH/dpOa6UPmct5xu6FX57V39RAVZskw3td43RZ+9/DdD/O9FYT9O8sLK5Cm+VUss0+ilrD0T1VdV3+MlXhwrT3xTjfQuLzc9lRf45rHrJFu8XItdF84/Ll7keOObvFAmrDrDFUlqiWnisyl/yeMXx/d8vBK4scoXqxlVTj1FUPmNn/J8CoAZX1wnwcs3kTDPixfHuu+/rSFberMok+4eBMKZvTQhcgG0ATIvITnB9olNJo5j/q2q8+4T3zT0lGULMBOYQbRdUNZqQlkrdC8hPBH/mXCCkPO3dZPf88w9bztl0YwHLxfpkivvuPr02/ICxxT15U7hzmN3vtzw9DyNZiy92VzL7UKr7r23A18UdmVlfjZgpdIA1WxhRAESVdBL0zzB5AS6E4pRzRZeqSRxifElyiIvmkYLYevnS28a+c9WSt6H56pKVAXzcaeophlS95W+t777X7isFYkhZZ6YeGLDmkNKeekKkZxcRV94nIOz7OVPLI2Y8SnJ2ItvoMutgg/UGbJZNL3Jy758FS+Sq5j99tt9daoFH6hTtFcQLYqilZ7GxurVZ6p+P1Y6Xvm4QqkKqM02DVWl2LNjpabbfu2OO9xY452imoFWcyY42HzEwNiotDFfLoBgRMJTIoIdE5nPNkyZOq92vK5HUJH5lN2AKA74r2K+3XnpTTNs8yvrZ1jXjanWX8CGb7//tgzwp5HwvsIH319afeJ+VFFUcPzq55+B/Zd3/355Zlo//+J+fHVxSe9HF7rqczCxsZrbcENXz4Pw45sHjx/VJHcaMJ/RfxKc5LvMOE2mU37FjfdJHCTx3Xq/tTj3SL+nwJ1ZIn42CDXmaRJGUy6mxZVwn7pn60hZd/ZnfzpfpnzJFhPxzwnzJ8vPx/MkXbDpMY9Xpk2QZ2khQSosJGBrC3kUEjPwCA/FkBXQBHs+BAyGCPguDngQWA7x0R8m14vZ9D5t7xiMuyVu/6ra2832UCmZ44iC7G40kwXfVU3uiiOWZHKz3S4JpVrBinCXzHzj82WZWxXcw6Tvo+Kb3+ue8k/zAbXR0GPeQfEG1LRsCh/tDO570eOyric4BAIgAojI97cF6iITZJGOHMKjWlatf3QI++QQVt8rAxosbub85YWfzJeZ8WopKGhqnL9Z1/RNyVp1fmHTJX9p26ZJHYRdCO+tryhVq8LSp6yCHlmu7kQUPGZC5Sg4LjD0RJT7geUvcR58/5D8b5yETnrKpzlFl9HdlxmfhvdWd6vUHedT01SpbqiuhV7HKVTKuvqwCnYpsTGQLWeSB2/0OVWUrHHwwLZJTWqrqKlE+jf6HrXcbdda1bVea9jsxdaltPWsQ6APuPl1scd6QVOY69wL0BNkTdxHMSG1j0OB36IimnSVJst5A7exVoWc2HBiEruZ21BGanZ8T7GqsZTmrUbm2mM7pg2Ri+5xs/W64nhdwT0+f7NRvsdbk8qn2nRiQ6uut+6j6VUZm7z2t+XqeG71qY68tyriqR5crevRXjx/+HGe/J5Gfao3v0eLzZ0uXo7F0W/5/H9Yyxrq6fVSK1rxk3p94Yv59/LD8atq3sgP6j2aLONFevPy7evq4dVHD/fig0qWUoSdYTMvulrms7lek9Z7XvXz2EHOWrd6jzxWunx8NWo+nX94+/bj+zf3SVSKPdyKj20cZblmU9EXxxvdwV0wVd/jdeYcH++n1frbco42dFxq282d47qCJztHN28cLBvHJOK/CYW0G+fYrOlVGaNzNEbnuFnL0TneK2V0jhtacUjO8VY5hbjeT4BXqVMb2W9ZRkei72Y3bazsm8K6Wqusu431KcV0NSkpOhurUsvp6rqV+rSxttsldfV9k9ZTa1W2Kq2rd5VstbG2soyuDpl/trGCvMBjAiJPiCTdVa4JznuieLV+Fedt9k5KyYet8Dcx2ZWnlC0tPntE7HZTzUV01ifcJYFrA4c4JsCMQkA5gQBDhBw/9ELPp/cK1AR3N0mMk3QmpsNvPDAWKYuzKE9MlMk8z5rrUVX6OGWeGvNd133PY12OhSCJWh0ACE4QRtYxtQjtpacfFjiMLr2v1D1W4IWctbkVvS/6SRDBmGzmc/LZ46rkfWubNdYo1Ua7Q+6kRUTAyumX5Z5YeOLQ2uuUar1d2VtVxkjulLr2ntzlk2cRLaY1erwo9pqL1wmfG5/E31dRssyM8hMWB8aPLFpcr0VvqPjF8UOi64MATel6Xdw+GNDV3i0g0EndGijQKdSWF8nrrw0ONNo0HCPNQIKuypaAgq767sGCTvJ2uromcFCVfkzKXQ4JCoWPvdz0yTcobCFYfSBMIQilKTyqM8XKHDxFmU3WWo6x/xRGSr4ZSxeRP+WlNdUOv7VzrLTfgAXUJ24EF85DyGv98k8efE78yC0sZP1Y8cHDITJ28xKiKijENAG4ypvXeqWi7cpmK9zOJ0FzswUT6M74MZomWRJGbNWStwveg02/QZ0b4kgPLKy0QLIRmRC3hUyDgZJsFmJKUWCKNvEZwBxx4LkhBizgvsUwZJ6D+qBejfUYBiNrbyx0QbIhtY6TX8MkDa69Y+oSYbKoi7Fr2hOIrAl2+hgBzRUZxhBoh5Q70CVmLVJeluyGlJvSgkpSbp1YdGJTuytS3sQ+qzJGUq7UNZJyRbmi2JkvQGAWJdPk6ua5ceYvZzyOslnByS+TKU/z2T8S8y2ACJ3UrQEJnUIjMe8ZXOjEbg9g6DTah6ERZV7c+ti41fp9jYGHhbbW13OWLkCyd3EbrvhLwEt3mYdtFmtveVTHCjeI3XhJ8hn412wu3rKHwE0VgdEHbloIplxec+NjbpSMd6Id5UsaSWi855EXR78NNKxiTxDCux1W2QaVxrtCpe9XZBhUur0h0GFkjYQEObYVCGZOCMA+poBRV9Ann9jc9JBneo/am9h0MDTWYxhjob2wCqJ1wyqyZD9hlfZ2ebVgn1UZY1hFqWsMqyjKFcXybIY8hvKJs2yMn2hq79ZV6KRuzV3oFNoHkryt+EkjRKkTuz1UqdNoH4bGGD+pJ/kQ4id5bkseLklzx3hUx+COoZJ9DpVYE9dpYTvvvoVKGhm1Lpxac0O3fXrc3hDoMFTiYQpDy+bC8PkuwA7zAfMCKP5g3KLQtIQp7GMwNNZjGGOhxVAJqR0qIX2FSmBnoZIG9lmVMYZKlLrGUImiXFHs5+s8g3Z+LSDR1BAIJ19by2Mnp9epPGyMxcabxF+kUcyzMZCyBUeik7o1Z6JTaB/Y8rYCKb2NjG4xp06jfRga3QdS7n21MZKyi5GUueo3wWLlN/PQil/6TRCUfvOojj0e4yx7HGeB9sQ03d2OszQ0cB1Gibdj17ZPltvr0w4DJ3YYEu4jBGzumQBbPgMUcRvIDZLYtSkitJe9O431GMZYaCNwYkOLEGLVyDGpSrYfOEFE9AmA+bn/lnliOhMLd7Z1p4nBVWWMgROlrjFwoihXD6SMgZLeHYdO6tach06hkQ13BRh11e8IGd7HszQE9wVFMAlcrwyh4PtgWhjCozoTrAHzlTcGCj8upe0P8b3jT7JhUl3kTqhJd5vqdkiLsBtyxjkBbugTMeCxAxhiDgg8aBNomQ5Ffh+0qLEeh0OLRCM4FqHmw7SoKtkBLbpzV6w1cezuUu8bTE5VxkiLlLpGWqQot6ZFSVqsH/+csl+jzIjikhT96+//nZVLzKvl5mR5db0YydIW3IlO6tZcik6hkSxputYCNpI4Gjs2sPoaApaQmYu8X+K4bPggWRLmMV8mnOfmUWCGkipl5TLiakkxt41Hdebc7q8cuv0QqJWjMaKZ8T7yrxfcCHi6djk8NobJqCCdELPhPbZ7wKjcIKDIEraOhQKZY8hdQLBcSzdxACFxfOw1sn+PtX2N9RgGo2pvLHS0kExcRBxs9reQvEngMPq0RZZs1WbJ1uBZchODq8oYWbJS18iSFeVu4ZSR+G7B6+ukbs3z6xQaiW9XaEBXffeIQCd571cJy+XAOjOpIZ3tgcZWqQCPOky/8poz/WH6j6C/u8ZPkT3BpnWw/NR3oMUsh4DAtMSY9m0XMM/3ADUDC5nchtz2+iAtjfU4HC6DTSwaiNRIhKxKdpIICa0yEZKeYDShLurwsK1HT05VxshllLpGLqMoVxRDLnKMf/39f99a5ZODzJA+cqQ4W3AcOqlbcx46hfYe9srJAaoFntz2ymlxVKe5d39Zpw4ebm9ZR1gWY8oXvy14JoyNx+PsP9l1OswlHUQnFMGDhcxW4KPADCEgLKQAMxICBj0b+Jy7DAUoYAHvAzI31uOAIDPC1HbrJMlVJbuBzGYOmS05fTCeUKu7a38aTE5VxgiZlbpGyKwoVxR7mwTPjVM5ZAWOyIrbfj7wREyPRSIvNFBy5kb8vAUvopO6NU+iU2hcItAvEViYuMjGFuxzI9EDQsf0uM106SoJgF/axTxLLlbMopIrd1RnujXgT34ShzzlsTBaczbfJw714+//TI1lnPHUeDv9/Z/+ZyORuXEBy8rfRXOLVx8qnyITbDu7zad4yyTKBJk5ETPAFugYOsQ03T4oU02pwyBI7XVuh2SZUmSGtu8AAnkobyBngHHXAq5j84C7GDu8n6S4pnoc2ljo5MBi6Mqj2oAFLESBRYgFAf5fvfT7Y8UPo7tbTI3EtVMj8eBTI5s4WlXGGBtR6hpjI4pydw6kLTYRXviRRONjROSh2ruFCjqpW4MLOoX2ISLSjCfoqmyTK+hk9MQXdOL3odM7O3i4PdyoE9Y3dtTpsQ/DoLNoqGj6vOV7u7HrYaGDjIZqFNrN7jYp6fNM6Urq/vR3j9Hv9ZnSxQbxrIC+atS7DqQas4bKY6S5QNrxgseSs59lGbvJjCQ2FtfcUEjG/2Cz+f+UP2uqkYTG28mfJyXfGO6ecZvu+B3oXd7q5EBqQlE5h1Aem0AhYIE8RcElPHA9j/rwfvvR9q1OTfUYRhCtvbHQ1eIIpg41TYSsXs6iril1GJ3bRoTUcS3XwqRGhLQq2c2VXQggO99w4Z5Ac+LCLjePP9ryqjLGCKlS1xghVZSrTp5+J6xiKlARv0pSPhOz3TfO4zCKo0WVZDTGR/uDBTqpW4MGOoX2IUbSZXz0aXBBJ6MnyKATv/fbbOQZ1NffmEQBFgqTeFRnqjXgzSUlZKmQNuU9UOcqrfAxG9BRFWudtbIBvWqHOxt1zs5ffzj/m/Hp7D/Oz/48TOKMrIlpkd0mzh0kmGDLxDJKLw9ncB0To/tjeO2nltQUPAzK1F4Xd7n5igXIczwTWCGyAQ7l3kMm/oAB90Lf8jyb9pJJ2FiPQxsLHcZGIKYU475jI5ukDqNzW8wec2pnjzl9ZY/hDq8zf7RzVWWMsRGlrjE2oih3a6e40Kw4aVku5mRLL1swubIjE8quo8yY8QWbX99kkZ/JAnJFKIp9JqaxNA1jAGUL2EEndWv4QafQGEDpElPoZPSEK3Ti96HTO0owa4c36sT0xx11GuxD13eVZSSa3XUwMc0+T+F8QOggk4w0Cu1adyOLuBSJf/XZ3ZuFDrK7e4yM37pbJL2RNzOu8XCeZSbwMFDwsCwgo+kKHj6qg7N2P+usWegc1gqdq88F7OYlQuun5K+dhdsT/3PBaspctSQ2LsruHWj4HU9s/Mh7u/sOv3cXjzMxQaJFet3I/YDUYcTj2uvcTjbvEngMBT0hLiETYYvwhLpYtHhPSyyPk39oHd7hSktgQWIG1AfQCgOAWWgCYtsOsDnktmk5iONG+zEeOwYa6zGMsdBioqJdO1HR7itRsbtj7hq4WlXGGIxX6hqD8YpyVaLiax7zMFoYi0TddLEKul8slkH+y/kiM95F2UIwlDH6vgV/opO6NZ+iU2gfonHdRt+fwiB0MnpiETrx+9DpnW3vbpFZ6KT1zi50ihxEHqtXeEywSICyIXQVlsukx5S/RItMhu6kxzyqY4n3Jru1hbDZ6zRaRNm18cfiUyNM0hyOrBCIBCQKVhlmGA06EwfTgw2j2eL/gqX1HEbbJHUYTLq9zu0gjGYTFx17zGfBcXEbn2s6NhYjHUJL/MBeshgb63FoA6DDsBqyqR8izwLYC6BweL4NPIwpgL5tu45rIdPtJ7TaVI9hjIUWc1xJ7RxX0leOa2dhtSauV5UxhtWUusawmqLc3RzXN4m/SKM4P7LkMpnydMxf1dferevQSd2a+9AptA/BlI4jaE8gDzoZPREInfh96PSOImidkAqd1K0RC51C+zAwOrxjHrkOxHZvsfSHhQ4y3VGj0O5192q69dvdm4QOsru3kd0arJBwHidfI+GjOghr9zNXq2XUR2WuVjmos/qZq1aFOLvMXP0gr3hiU+M9+5rnr1bsRZ70LoPxZyyd3hhn8TS6ul7weCbmzUAj8XjiOgcbiYfCuDkU9nKYRE2pwwi+tde5HQZimcl46GICfGz6AFvyPFrPNIEXWDRkNkOm00sqc2M9hjEWWgvEUljjGt+qZD+BWKezQGwDy6vKGAOxSl1jIFZRrih2PjV8gazSyFsuEiPg02liyASNKDF4/is3eBD9JgxAJKaNseDZIpLfJP5SwplIQB3xvfibTafMkLkHUXKVsjBiRpinH4QCXBnAiIQcliXGdLwAZ0Pt3XokndSteSWdQvsQ0ek2vvsUSKqT0RMs1Ynf+8S4aAoqUwtyUwsKUwt4/isHpakFQQRyUyu/KU0tWJlaIE0tUE0tqEwtAFKMsLTr+xbqzOBBZNbVCSG0QOs/Cd6cLZj0dj+umpXJm+3BhZ9I9CX92TBZPHImLkW7zeK7PDIwcJ3Q8RyAPUYBxqYFCEeiibDrYR+KAeb1QvEb6zEMotfeWGg7/e7Nx/OTPNebWmLizdMkPBHTe7VMRR0HQsvsOw/vCcoMYzi0wftXuxVrHDJYlexnX6Pb2b7GBrZalTHyfqWukfcryhXFflofl8KmxtkXQc8/cTllxAgeufkWQIRO6taAhE6hfeDmXaThdA4udOK3CzB0Wu3DMOkqSwNAgCgougD0eexc1fFjpkaDAM5M8ZlAeMupeJmVzzyqY4v3I0ujjcwJ/lW5zHKVC/6vv/93Zlxec4Fx/ZthBligLQ3qbgdYukuTcAmGyDZ7TpPYKHUYFLm9zu0weuYzEzkcuoAiKwSYmj6gpuUCAm3fFa6FYMfvo+cb6zGMsdBimgSsnSYBh79frYHlVWWM4RKlrjFcoihXFPs5jfJJNmMLMS2fG3Pxu4ANX7iYYlmW/518lV/IRNB8aAtQOb0xprLYOif0/Zj8sKH2bv2MTurWfI1OoX1gzt0mPzwFaOpk9AQ2deL3PvlhXhhQUBhQsLafYGU/QW4/89O8K/MJKvMpQMg6o6HOtBwzGireLI8ojNjK+ywiFrNh0msEJ9B+5KXke5S/4Fph4Pt2CJBrIYA9SAAhNACUB4zjwDO9oJdV68Z6HBgDI5TWPjGEDp6BNZmcqoyRgSl1jQxMUW6VlsdZlsTPjZR/WU30gmyx9CpPj8zWR+C94WIOjRxrC55EJ3Vr3kSn0D5wrM72kGNkuy4m1O5rCDwsdFyZ3Myu0tw4gso25kRqbRqBMI350auBNI1HdabcuFZZpoMnS/FXIDzhaTKbs1iG8haJcZZv6RaNe22cij+XAv5V560OlF7RCbIOl175mAeB6dmAUuIAHMhzFhyCADQD5NoOIybr5SzWxnocDr3CJrYcQmsscFUlO6BXBEALQJLnA9MTjCbUtTqjVw0mpypjpFdKXSO9UpQrip2J4RRNjXmSiWkylVt/DT9NopQbmbBPhig7T3kcpPyHkVltwYnopG7NkegU2vuFDLkrcwrKeSIQNSjmCZDzBFTz5KhO8+8+yrb6Qdmnog0FMYuloZlxmRX4hS8XPwwTSEM8ge6OpwF2CKSxb9PADEIByEICcEgY8GyXA8dEPseexwK/l3WKxnocDpBe5X/h2plieB8O1Hn05FRljEBaqWsE0opyt042z7PZA+Vs8+pExxNjfSeS3EnwXC5aJNPkKt+KJ5c05im7momivlHE9bIRcm/B3eikbs3l6BQaFzM2LGasz6a999XGE3F3kWI9cCIuuGU+gWI981WPtfUEK+t5VGdWjkzs8E61RXBC6eEueIRBiCEReN91uID6DkeAIscTTRRyhzNCQr/R5tTHGszGehwOTyuWMUiNG6iqkr0seDjdLXg0mJyqjJGnKXWNPE1R7rafkz7uYjmXM7P44KebTMxT4WJG1rUF56GTujUHolNo7xc6VvMhx9SZMkHArJwgR3UafgTWKzj8lmeimQVgNgKeySTVbLbMJKY+W6bJfKhbMtyJC8luQ+gObvHND9VJfg2TNLj2yiN1bGxTF03EgJpA9R7wDqHzU1QZBnpubxh0yKSga7shRCagzJKHv1g2YH4YAt8PEXNhQMIA9TEcGusxjLHQ4oqXVXvFyxr8ilcTG63KGJmUUtfIpBTlVhnS+eQW5iObjZRpC15CJ3VrnkKn0D4sVHV0g2tXgFIneJugUqfTPgyPDtcxV+3f7zLmJqHjMuYDm7Iqv3hUx96OkZJV1OPymhsfc+NjvGNxIF8yz6ARH7+L5L0nN/LXs8V15GfDDJtAPLGtHQ+bdMiXOSEBdW0CsOVyORsY8JDLACecOBiRkLFe+HJjPYbBl9sbC52E0JB7fAGh5VALOghJCk5pH71eCs9qCT+0rv6aZMu2w2OioSF1TWLLdoZ99HEdkcPo2RaDXnbtoJc9+KBXEw+ryhiDXkpdY9BLUW51IKiag1gkyilZ3NXh5hK0XqZRPB5Jo6u9W0iok7o1WKhTaB+iH42xg67W1vCDTkAfGEInex96vLNwaFvkQCeqX4Kg02Lvk4hup+rLVCI1XZ/nbhIkcX440aJwk0d1zG/TWJm8sIV/7SFUViWc6UNl6nPiR6AxVPmV4oOuQmwXpx8vL88v3hl//PjLpw+v3hsffzQu3519fP/x7V8HGlJzJwQe7uW2HsWMmqYHiON7gtA5BBDuiibybSd0A2iZ8H6BLRPvxnoMg423NxYarhRtXCUCUDS7TV1gIxs02/HWZJkolylEjrvp8+BJjWtqq5L9hFk6u6a2idVVZYxhFqWuMcyiKFcUe7e8Soy3abKIltlzGWIxLtPl4rpcDjy9TqNMnmhvfOLT6CpSVB6DLP2BBJ3UrQEFnUL7QLk7vJX0qeBBJ6N7AKGTvPcE+1oYSHBVGMgVixb2Md8wL37xS/soXzG3j0d15l0Dds0DeQ0qkCR7R9h1Cyz57l74W3vejdOpvLpmsCkozgRZh3tIme2YNrSIBagnz8AmkAHKbAZCx+GhQ0PbJr2c9ttYj4OjVW5tWuUOn1Y1mJyqjJFWKXWNtEpRrsqwPE3iUNgQMUUkm/qRRYJWfZV/FDetlKe9/Ovv/+dNlPlCfmb8nP7+D3klW8yilP/r7/+3pGHVrd0j+erd4+ikbs3r6BQayZeefFmC/UBKEe2VeFVSxwT/BtwrZ1hrKyo5VyitKJBWdHX4mLy2Mr94ZWVCwVwSsZUFLVnaYmU+j+pMy93fJYD6IWfvuS9vCc0MvhC+SrzaXHin4rLQLD9iQSZhRfKfwkf9/g/Zwr//Iz/Y/u3E+POkuox5kOyNTCCiu83eml0CrDeTq2t4XUpt6riu1ewC4MeayZpSh0HH2uvcDqm5YweOQyEHISI2wCG0AHGQA1wv9E3Mqc1xoxvAH9vzjfUYxlhoj5ojVJeay5L9UHPSGTVvYHlVGSM1V+oaqbmiXFHsfGqkbC4RZ2KEPOAgZTJuz42YT405lyYqTUrwErE4GUn3FnyJTurW/IlOoX0g3c2ApK7KNsGkTkZPgFInfu9XPaMpKI0kkEZyZSOBsJGgtJFgWtnIozpzrwGzLlkjSxeRP+U9kOtqG0K3t54KrpotmABmxo95BncYMeMDT8CFn0jEI2odJnFG5oRguNvEuZstuCYRGNwl4l/K1bkd0qc6IofBlNrr2S6vr3UC4iFKQRB6qNg6QbjlAZNAEtgO9zzey5kKjfUYxlhokTXXuL62Kjlw1tzE7KoyRtas1DWyZkW5W7dunRivk+DmuXGx9AR6iX3+3PgpiVllaUeO3J/n0EndmvfQKbQPHLm7rddPxY06AX1gR53svSfIK+4LPGEOBQJZWUPhcYUxPKozyXZ+g21PRPjDMvZlavUfi0+Nj2G+C6U8h078euGLkevzgdJhZ4JNc7fpcJekift+GBIIaGC7AENsA0a4Cwj2rRCZhNGeSFNTPQ6NNEFaOwuYDp80NZicqoyRNCl1jaRJUe7bq4p/4gs2v76R+1nWWb3jsVUbau/Wgeikbs2J6BTaB+7U2Y5KaiOMEMSot6vQhNQHhI5JvTUvH55VFnGdqLs6oajOTNv9PN2eGNTqXMSbYgdlfnfwTwLRpbGxPkPxxjhCtg0Bcm34h2ESKWhPCHV3m0h1tK7oOpaLrF7XFR8SOQxa1F7PdkiRA+LCgDECOA5CgE2CgOdaDDimFWIzsE2b9nLVXWM9Dm0sdHJWO4GSb1uWS8U/TRtSy+ol+/5xwofR1S1GQ2pcCF6VHPqe6AYeVpUxRkOUusZoiKLc6kTvNFkkV4yzMeCxBTigk7o1SKBTaB8CHh0uFj+RDOgE9EEIdLL3occ7O6e7LWCoE9UvONRpsff5AvPS+R3VMapjckDh+F99+PDq/YU8O/vi9Pzsw+nZMGNXCE3MA76NjjkODSgJAbJtuTfEcQD1/BBQi5mcWgHFpJc7qxrrcWi0F+HamdO4L9pLO0sCaDA5VRkj7VXqGmmvolxR7HWUiekoBtM0yFdootmMy/l+YiRpdBXFeToAMz7zG8NPYp/PF+XJYDxfy5kVazlB4svVMS5LL+PoC08zNjWuWSokjSkE23A/Oqlbc0E6hfaBX3WWQmBhgi3HMXFvdFqeC7ZZ6JhCsJlMeZVJza88Kk0qWFlUmU7AgLCoYGVRy1PCcosKCosKSosqS68tKlhZ1KM6M3XnUxAg7YenXcx//3+LlMcsyrI8D766vZFLb/aGLxeZfz2V/g/ZLgTIseGJcRYvvv7+zzTkxt++RuJ7HhvvljMWR9lsmRm/iMJ/SuKQZ9nqmveIp8v46rnx8fPn5XQhSl+mLIikvuLfsviFf72crvNKPg+ULcKJSezdZotdXK4DBcMg0EEU9naxzlrigGlfe53aYQiAeJSHrs8AC7mwpr4bAI+GASA2YQ4lNnRpL6eBN9ZjGGOhxRCAVTsEYA0/BNDA4qoyxhCAUtcYAlCUu7N5+lVsnMcLLrTzF0tB319HyVUqwcpI4LfgPHRSt+ZAdAqNBL4z1Kirv3vkqJO890uh5dZpJg/vrowh8EpjeFRnkjXk3sPj3DtHT+kE24d7rxHlLnMC6ICAegRgaHIxLCVUDUPMTWw6HuklXbuxHsNgMu2NhY42ZTiCj1BHJkT10t11RA6jZ1vkqGZtjmoOf696A7Oryhg5qlLXyFEV5aobq1Y8FbzhmfE6yTIu1UxTns2FksUZLCNN7R0Z6KRuDR3oFNoHmtph5vYTEYNOQB+oQSd777mqXDRe8dWAZ8DLjaIAIapRPKoz2XY9o7evleLXqSCi2bVRHvcVJmme61Se95WE6q71gS7g2hPHxQfLkOVRd7ZtMtEgHAHsUHluA8bAIuJ3z6IWdFgflKmxHsPgUe2NhU4W8y3hcCB0sdvbYv5a4oA7tUVybNcmx/bwF3AbWFxVxkiOlbpGcqwod2sB10hi9eC2PJ37PPaZmKhy8p+UNzvLzz/xLyubUKZzX4hP4gUXP9fGqfhzmY7LvtvAETqpW8MSOoX2gU93tuz7NHyhq797jKGTvPdUulz2TWL1qLcif7uyoeW1zvLjdG1Cy/ztrDKhwC9M6FGdqbm7i8XVVur9XCyGE9va8dugOrpG2bEIpiZCxL7/8vdurlF+SOowaFB7ndthnMN1bYsGHgWe4whWxWxb/MsOge0ExPZd4lCrl55vrMcwxkKLlNipTYmd4VPiBpZXlTFSYqWukRIryhXFTsUv4nUyyYlfBWyWjVx2C75CJ3Vr/kKn0D5w2Q6vSX4iWNTJ6Akw6sTvQ6d3EsCAiLqmg03Y3xlumyWOO843By/8lbeT0Qsmvd1RHSu6+/vDqzhWt7dd5w/JLdyz58bPYhQkco93Hid/mwQDXeWHE9c53Dx427cR9VwIXJuL8W9yF1BumgAx1xM/RPz0crNXYz0Ojv3WPssa9XWWtQM7ZL+PnpyqjJH9KnWN7FdRbnUwZXq1LOhvmcB29qs8lCb28wO6hFM7ya+xPE3iRRSLgmxqnC3TZM6ZPA3FEwNw5Mtb8C46qVvzMDqF9n45kJUzCIgZVBzOVM4geSDTVRLkH/rVBAJ8NYFAkE+gozods/sQ3OkHgufWiM28NAqubqXbnkVX17ezUpQM3GFCc+hOEDrcBFxGPOzQEIMQ4wBg4lJAAoJkeroVBlBMDJ/2Ac0b63Fw0NytDc3dwUPzJpNTlTFCc6WuEZoryq1iTfwLZ1MuMzCn0VWeljki8SecntuRM9FJ3ZpD0Sm090g8XU0Y+Q75hHkQeNfphxF4HyLwJhMbHi7wdnyHWi5lgPk0BNgjUMwLRoDDGDM9RAI77GXnW2M9hgG82xsLHZ0NY5o2Nm35r17uNakjchg92yKlorUpFR0+pWpgdlUZI6VS6hoplaLcarXDEL/OWSw3sy2S2/eSzNeQZTLyqC1gA53UreEDnUL7kAzW4ekwT8QMOgF94Aad7L1nzgys7SJYJLdvF6ns4lGd6bbz58NUeYOdsuWf352//3jxUfx1+uq98e+/vPp0efbp/V+HSYeRObGps9t0uNnFtXpblt8Zi5zjCwgtCiyHoCNo/UEaGAR6WXUqFchqKzAMbtRel3e04VG0NzEtSrHZRzfXE3poXdthcMtCnFAmKmfEowCHpg2YFVLgcEwowQJJkEYXUz+24xvrcWhjoZvgFiKWaVIkgyq93NhTR+Qwera94JYJ6wa3ZMl+gluos1TeBiBKlTEGt5S6xuCWotzts53kZhSZyDsXbQFOk7mYooLCxOVlu3xi/Cm5SX//hyEpVxaJAT7Gu7YAF3RStwYZdAqN8a4uYYROQB9QQid7H3q8wz3OT+OHOhH9cESd9H3o8mYRIF2VHUWBdOL6jwTpNNn7aHd1b1exNUMCJX+Nk8ortDn4LGASB2uUdFTH+zYIf5exYJYuIn/KewiBVx5DHwJXnxM/ApcreWbFB12Fzi8uf3lzfnZhnH8w3p1fXH789Ffj1Yc3xjqi/lfj44/Gxen52YfTs2HG06E7sfGOnybXYQQOuj73LGQDlxEXYCaaxuMhB27IKMOMQT/oZct1Yz2GEadpbyx0FYGDxLIkhkW9nC9XR+Sh9WwXq2aWbR1fz68nwtMI1AppL3O5vuBhdHGLQVZUO8iKBh9kbeJZVRljkFWpawyyKsrdCrIKzX7iC3nHbxb52YlxvsiMj2l0FcVZHn99I098TuZyX7hxVKS2/GGMsm4BEuqkbg0W6hTahwBMl1HWp0FFnYA+4KJO9j70eFcht1YgpE5MfzBSp8GhhNgyMKu8JIjksYOFk8xjb0HlJFfpn0d1TO/ehNhaCJX98eMvnz68ei8DYpfvztbhMvFrFS0bZogM2RNquQcbIvM9HIQ8QHIOQIAh8gG1GAQuMuUZt8y0EG9iEx9rDxvrcXAsG9dm2bgvlm12eE3doyenKmNk2UpdI8tWlLvFskfGvAUPoZO6NS+hU+hQQPRRndbd/VNKzH4Q8ZvoN3lCdxolxnks3kx4VT+aTyPxERcu1rjwxcD9jRnc+JEHfKDY2JyY5uGe2E0cK/AJd4Bt+fKiPC5IostsQELMLM4s7Nu9YOPGehwcNrZqY2Nr+Ni4weRUZYzYWKlrxMaKct+m+a9OBlxmxmUy5Wk+7UfQvAXXoZO6NfehU+hQQPPq3t3V9ACL9fQ4qtPsgwgx9wSoX/109un89NUH4/TV5buP789PDc3RBsaJcScc/dCjry4uPp6ev7o8//hhoEAcT5BzuMcEUtviDvQswKlMiPZcDqgjl2wsTF2fmJ6D77cvLQPxxnocHBA3awNxc/CpYE0mpypjBOJKXSMQV5Qriv3EsiyaJcZPy6uYRScy3JQmwVKGn7jBplNmhPnJSWHEZOCpxO1Hn8Z0sA21d+tZdFK35l10Cu09Tp8VswfM8tkj/Ek1eYCcPGA9eUAQgRLVr9JE6nTJiOErIP06FZA3uzb+WHy6vutLOcS7OrHbGCgWRxPsHG7CCPNc7EMSAj+0fYBDDwMWmAHgvmdDzDAN3V422TTW4+CwuF0bi9vDD4o3mJyqjBGLK3WNWFxRbrXqy+XkkIiCceMsFh+KIZX9YLyWN/GlIZ8Geby82r3xMV4k0+Qqd32fStswQvItOBid1K05GZ1Cew/JA2USAV5OIuCt51AeVF+ndierKSRvuVy/9VGdrhkENEc1oLn63Cw/EwFViH7W6SEJ78/OX384/5vx6ew/zs/+PEzIDsmE0B0/BqGjM2YdTDFFDkZ2L6i8ptRhYPD2Orejcw0c1zYtS6L6fo6xqCHy0Hq2m9PA3fwcKIxRftMAtLH9lz46uBSe3RH+69jV3QZVuGNSQh0KPIdDgAkmgMndaTZlPnaZaYeW3ccAaKzHMMZCi0EVUjuoQvoKqlgd3pb1aPikyhiDKkpdY1BFUe52pmHAjTc8MS7TKE5OqnuyBG+ZGiybc3+RyTBKFVzxk9jnc2kZ5OdyoUE+K6bZGGDZgrPRSd2aw9EptA8HInR4BMYTWYVOQB/MQid7H3q8w4OGnxgk0MnoKVCgE78Pnd7dUcMtUUydqH5ppk6LvQ+ll/kqARf/JWAhoRO4hZzACjnJ6Pk6pF4BJ/mxPJt4UQCnozoOeW/C6i2Exz+dvT9/e/7xlwtjdZrwMCPkCE0oMnc7Qt5JqI3AY0gdSFwCEYIQ2xZyewu11RY+jPBKe13d4YV7GFsm7fvCvQ1CD61rO4yiOj4hgUNtwH3iA2wxAoiLfeCTIPQs12Uw7KXjG+sxjLHQYhTVrR1FdQcfRW3iYlUZYxRVqWuMoirKFcUur2X4NOQC2ZfR0J9uZEaNgIh3wqPrq9vOY5+JGSytwon40Dj7lc3mU347yvrdR1E0/c74xFk2bvvW1N6tB9JJ3ZoX0im0DzGXjm/0ag5AdSL6AaE66fvQ5Z2F2dqilzpR/VJMnRZ7H2aTEbKg8K1lwGxW+tY7EbT1bV9R5VvFZ4AXrvVWGC6RjlW0i/SrR3Xs9Rh5q2JqB7HXzJkgh+x2WK5DQo8RCblpM+D6rimmBAwAY9ACnHsYYur5PsF9EPrGehwcoXdqE3pn+HvNGkxOVcZI6JW6RkKvKFcR+g/8q3HKBCRJrpYVrWfxMvPTaF7kQ11eJzOWCXoejResb8Ob6KRuzaPoFDoImB7zr8AvZ8warFczpvhIzhjxplFwVKcnRtitLnifvxn4WjedOK5zsKDaswm3TA8C4nMx4G0XAmaZDPDQsSD0A9NyejnVuLEeBweqaW1QTYe/StZgcqoyRlCt1DWCakW51V6DKAj49LnxZhn7bFzM2oaj0EndmrPQKbQPKxtR5sWtjwxEbBs6LsSkt7WMzRJb6+W5wOuCIewbMZrmVg8EudE7qjOZGlCeIPIX+ZUoN/JQj/SmB84Da3Ae9blZflQGrKj1TH92hvpcwG5eomojkvy1K371S/w5Tr7Gw6RWkExM+3DXK5Bn2xblDnCwawNMbAt4zOUCO1uBh3zHplYv6xWN9Tg0amXButRKlhw4tWoyOVUZI7VS6hqplaJcUex8aqTRlMV+lBgBv5pGRrZYBpEhw6wsjgwWGK88ngacj7xrG15EJ3VrnkSn0N6vU0RTUM4UkM8UkM8UsJopgAWArWbKUZ0e2P1LGesA9RbA84/rg/C5cbqcLpYpM2I+NS74YsF9Ma4TQ6YNsTiO/GSgENueYGfHz7LrEGJbjh0wikPAbdsDmGITEO4gQDzL8gITce73chZaYz0ODmLXvgrG6u0qGNwZxG4wOVUZI8RW6hohtqJcUey789mMywn+Xb6F5zs+m/M08q+TlGdR9t3EkDlD4keel5sfm/QpSYokoXcsFdXcGFGsOat6hORb8Do6qVvzPDqFxqUQTddSl1CICbIty6R9DYGHhY4LIg8wsJUdzXdr3LaixWaOyoaCVNpQmTZ2XdhQAUQ0Z5Uf1ZmbO07WEKW0H7ImfdVfk2V8ZahXDV9HmbJv47mBHOwA5Mhz9gdJ15wJcQ+XrmGHhxRbAoC7kAPsEAdQ7CIQhsSjEHI/dHqha431GAZda28sNHSSG6ATxi7EyIK9HKvzgLRh9GaL5BvVJt9o+OS7galVZYzkW6lrJN+KcqsLyqcC893izEYSG2+T4PlqV86VAIWpGGbr4zV+isQbxHG0nI3segsgQSd1a0BBp9DIrjsBD7q6OwYQOrF7v6zJcgOpEmIgSOFVEqz2YJX2cX1EwmxtH4/qzLuGzHl4jPlJJPbWd7KKVW/f+rz6ZyXgxXHK/SQNToq/Xv7b/weirs3DHDIDAA==\"\n" + + " }"; + for (int i = 0; i < 10000; i++) { + String orcidId = String.format("0000-0000-0000-%s", Integer.toString(i)); + String url = String.format(BASE_CFG_URL, indexHost, indexName, indexType, orcidId); + String recordTs = String.format(recordTemplate, Integer.toString(i)); + getResponse(url, recordTs); + } + System.out.println(""); + } + + private String getResponse(final String url, final String json) { + CloseableHttpClient client = HttpClients.createDefault(); + try { + + HttpPost httpPost = new HttpPost(url); + if (json != null) { + StringEntity entity = new StringEntity(json); + httpPost.setEntity(entity); + httpPost.setHeader("Accept", "application/json"); + httpPost.setHeader("Content-type", "application/json"); + } + CloseableHttpResponse response = client.execute(httpPost); + + return IOUtils.toString(response.getEntity().getContent()); + } catch (Throwable e) { + throw new RuntimeException("Error on executing request ", e); + } finally { + try { + client.close(); + } catch (IOException e) { + throw new RuntimeException("Unable to close client ", e); + } + } + } + +// @Test + public void testMultiThreadFeed() throws Exception { + setup(); + int countAll = 0; + int countOk = 0; + int partial = 0; + String recordTemplate = "{\n" + + " \"timestamp\": 1540825815212,\n" + + " \"pid\": \"%s\",\n" + + " \"blob\": \"H4sIAAAAAAAAAO19zXLcOLbm/j4Fw4spdYSRAkiQBDQuV9iyyla3y65rqW7/bCZAEpR4nUnmJTPtUq36HWZ1I2Z2s+l36F29ST/JAGQyCclCiqJIZjKT3SXbygRxDvFzzvcdHAAvfvh1NjW+8DSLkvj7Z2gCnxk89pMgiq++f/bL5Y+APDOyBYsDNk1i/v2zG549++Hlv71IuZ+kwUnxlzFni+vvnx1D8T8gfkxgmwgD7NK/PDOEgDg7ieIFT2M2/f7Z9WIxPzk+/vr16yRJ/SgQf14dx9lxWaJ8ggdLny1ytXSPrIuUzwRRtohif/NTSqH1c3wu3oQteLDhsXWZ8qlkcc1TELMZ1z9VlSmfmvGZJ9r7Oprrn6rKrFsjTZN0Q0vIr8uyfjKbbWqA4vuydNGF+tLF92XpuVArEb0EAr5g0TTTP3e3ZFnDZ37zdaPAVYH1m8/E0xveXH69LvtrMYZAFPB4EYUR39Rm3xYu6wmXcTEDdM+uCqxbJeUhT8XE4ZsapCpUPseCIOXZhmdWBapZ9CUSAxDMkyzaPMjvlixrEC37Wf+U/LYseS2mSZLe6AuvClT9NJ8mNzPRlJs6qyxTPvVfSzYVTf/QTL9VrHw24+mXyN8w+VYFbo/dh0bsum+EhRBtGG3q0qpMNZsyzlJfTvplumHY3i5XaSh+T/mXiH/dpOa6UPmct5xu6FX57V39RAVZskw3td43RZ+9/DdD/O9FYT9O8sLK5Cm+VUss0+ilrD0T1VdV3+MlXhwrT3xTjfQuLzc9lRf45rHrJFu8XItdF84/Ll7keOObvFAmrDrDFUlqiWnisyl/yeMXx/d8vBK4scoXqxlVTj1FUPmNn/J8CoAZX1wnwcs3kTDPixfHuu+/rSFberMok+4eBMKZvTQhcgG0ATIvITnB9olNJo5j/q2q8+4T3zT0lGULMBOYQbRdUNZqQlkrdC8hPBH/mXCCkPO3dZPf88w9bztl0YwHLxfpkivvuPr02/ICxxT15U7hzmN3vtzw9DyNZiy92VzL7UKr7r23A18UdmVlfjZgpdIA1WxhRAESVdBL0zzB5AS6E4pRzRZeqSRxifElyiIvmkYLYevnS28a+c9WSt6H56pKVAXzcaeophlS95W+t777X7isFYkhZZ6YeGLDmkNKeekKkZxcRV94nIOz7OVPLI2Y8SnJ2ItvoMutgg/UGbJZNL3Jy758FS+Sq5j99tt9daoFH6hTtFcQLYqilZ7GxurVZ6p+P1Y6Xvm4QqkKqM02DVWl2LNjpabbfu2OO9xY452imoFWcyY42HzEwNiotDFfLoBgRMJTIoIdE5nPNkyZOq92vK5HUJH5lN2AKA74r2K+3XnpTTNs8yvrZ1jXjanWX8CGb7//tgzwp5HwvsIH319afeJ+VFFUcPzq55+B/Zd3/355Zlo//+J+fHVxSe9HF7rqczCxsZrbcENXz4Pw45sHjx/VJHcaMJ/RfxKc5LvMOE2mU37FjfdJHCTx3Xq/tTj3SL+nwJ1ZIn42CDXmaRJGUy6mxZVwn7pn60hZd/ZnfzpfpnzJFhPxzwnzJ8vPx/MkXbDpMY9Xpk2QZ2khQSosJGBrC3kUEjPwCA/FkBXQBHs+BAyGCPguDngQWA7x0R8m14vZ9D5t7xiMuyVu/6ra2832UCmZ44iC7G40kwXfVU3uiiOWZHKz3S4JpVrBinCXzHzj82WZWxXcw6Tvo+Kb3+ue8k/zAbXR0GPeQfEG1LRsCh/tDO570eOyric4BAIgAojI97cF6iITZJGOHMKjWlatf3QI++QQVt8rAxosbub85YWfzJeZ8WopKGhqnL9Z1/RNyVp1fmHTJX9p26ZJHYRdCO+tryhVq8LSp6yCHlmu7kQUPGZC5Sg4LjD0RJT7geUvcR58/5D8b5yETnrKpzlFl9HdlxmfhvdWd6vUHedT01SpbqiuhV7HKVTKuvqwCnYpsTGQLWeSB2/0OVWUrHHwwLZJTWqrqKlE+jf6HrXcbdda1bVea9jsxdaltPWsQ6APuPl1scd6QVOY69wL0BNkTdxHMSG1j0OB36IimnSVJst5A7exVoWc2HBiEruZ21BGanZ8T7GqsZTmrUbm2mM7pg2Ri+5xs/W64nhdwT0+f7NRvsdbk8qn2nRiQ6uut+6j6VUZm7z2t+XqeG71qY68tyriqR5crevRXjx/+HGe/J5Gfao3v0eLzZ0uXo7F0W/5/H9Yyxrq6fVSK1rxk3p94Yv59/LD8atq3sgP6j2aLONFevPy7evq4dVHD/fig0qWUoSdYTMvulrms7lek9Z7XvXz2EHOWrd6jzxWunx8NWo+nX94+/bj+zf3SVSKPdyKj20cZblmU9EXxxvdwV0wVd/jdeYcH++n1frbco42dFxq282d47qCJztHN28cLBvHJOK/CYW0G+fYrOlVGaNzNEbnuFnL0TneK2V0jhtacUjO8VY5hbjeT4BXqVMb2W9ZRkei72Y3bazsm8K6Wqusu431KcV0NSkpOhurUsvp6rqV+rSxttsldfV9k9ZTa1W2Kq2rd5VstbG2soyuDpl/trGCvMBjAiJPiCTdVa4JznuieLV+Fedt9k5KyYet8Dcx2ZWnlC0tPntE7HZTzUV01ifcJYFrA4c4JsCMQkA5gQBDhBw/9ELPp/cK1AR3N0mMk3QmpsNvPDAWKYuzKE9MlMk8z5rrUVX6OGWeGvNd133PY12OhSCJWh0ACE4QRtYxtQjtpacfFjiMLr2v1D1W4IWctbkVvS/6SRDBmGzmc/LZ46rkfWubNdYo1Ua7Q+6kRUTAyumX5Z5YeOLQ2uuUar1d2VtVxkjulLr2ntzlk2cRLaY1erwo9pqL1wmfG5/E31dRssyM8hMWB8aPLFpcr0VvqPjF8UOi64MATel6Xdw+GNDV3i0g0EndGijQKdSWF8nrrw0ONNo0HCPNQIKuypaAgq767sGCTvJ2uromcFCVfkzKXQ4JCoWPvdz0yTcobCFYfSBMIQilKTyqM8XKHDxFmU3WWo6x/xRGSr4ZSxeRP+WlNdUOv7VzrLTfgAXUJ24EF85DyGv98k8efE78yC0sZP1Y8cHDITJ28xKiKijENAG4ypvXeqWi7cpmK9zOJ0FzswUT6M74MZomWRJGbNWStwveg02/QZ0b4kgPLKy0QLIRmRC3hUyDgZJsFmJKUWCKNvEZwBxx4LkhBizgvsUwZJ6D+qBejfUYBiNrbyx0QbIhtY6TX8MkDa69Y+oSYbKoi7Fr2hOIrAl2+hgBzRUZxhBoh5Q70CVmLVJeluyGlJvSgkpSbp1YdGJTuytS3sQ+qzJGUq7UNZJyRbmi2JkvQGAWJdPk6ua5ceYvZzyOslnByS+TKU/z2T8S8y2ACJ3UrQEJnUIjMe8ZXOjEbg9g6DTah6ERZV7c+ti41fp9jYGHhbbW13OWLkCyd3EbrvhLwEt3mYdtFmtveVTHCjeI3XhJ8hn412wu3rKHwE0VgdEHbloIplxec+NjbpSMd6Id5UsaSWi855EXR78NNKxiTxDCux1W2QaVxrtCpe9XZBhUur0h0GFkjYQEObYVCGZOCMA+poBRV9Ann9jc9JBneo/am9h0MDTWYxhjob2wCqJ1wyqyZD9hlfZ2ebVgn1UZY1hFqWsMqyjKFcXybIY8hvKJs2yMn2hq79ZV6KRuzV3oFNoHkryt+EkjRKkTuz1UqdNoH4bGGD+pJ/kQ4id5bkseLklzx3hUx+COoZJ9DpVYE9dpYTvvvoVKGhm1Lpxac0O3fXrc3hDoMFTiYQpDy+bC8PkuwA7zAfMCKP5g3KLQtIQp7GMwNNZjGGOhxVAJqR0qIX2FSmBnoZIG9lmVMYZKlLrGUImiXFHs5+s8g3Z+LSDR1BAIJ19by2Mnp9epPGyMxcabxF+kUcyzMZCyBUeik7o1Z6JTaB/Y8rYCKb2NjG4xp06jfRga3QdS7n21MZKyi5GUueo3wWLlN/PQil/6TRCUfvOojj0e4yx7HGeB9sQ03d2OszQ0cB1Gibdj17ZPltvr0w4DJ3YYEu4jBGzumQBbPgMUcRvIDZLYtSkitJe9O431GMZYaCNwYkOLEGLVyDGpSrYfOEFE9AmA+bn/lnliOhMLd7Z1p4nBVWWMgROlrjFwoihXD6SMgZLeHYdO6tach06hkQ13BRh11e8IGd7HszQE9wVFMAlcrwyh4PtgWhjCozoTrAHzlTcGCj8upe0P8b3jT7JhUl3kTqhJd5vqdkiLsBtyxjkBbugTMeCxAxhiDgg8aBNomQ5Ffh+0qLEeh0OLRCM4FqHmw7SoKtkBLbpzV6w1cezuUu8bTE5VxkiLlLpGWqQot6ZFSVqsH/+csl+jzIjikhT96+//nZVLzKvl5mR5db0YydIW3IlO6tZcik6hkSxputYCNpI4Gjs2sPoaApaQmYu8X+K4bPggWRLmMV8mnOfmUWCGkipl5TLiakkxt41Hdebc7q8cuv0QqJWjMaKZ8T7yrxfcCHi6djk8NobJqCCdELPhPbZ7wKjcIKDIEraOhQKZY8hdQLBcSzdxACFxfOw1sn+PtX2N9RgGo2pvLHS0kExcRBxs9reQvEngMPq0RZZs1WbJ1uBZchODq8oYWbJS18iSFeVu4ZSR+G7B6+ukbs3z6xQaiW9XaEBXffeIQCd571cJy+XAOjOpIZ3tgcZWqQCPOky/8poz/WH6j6C/u8ZPkT3BpnWw/NR3oMUsh4DAtMSY9m0XMM/3ADUDC5nchtz2+iAtjfU4HC6DTSwaiNRIhKxKdpIICa0yEZKeYDShLurwsK1HT05VxshllLpGLqMoVxRDLnKMf/39f99a5ZODzJA+cqQ4W3AcOqlbcx46hfYe9srJAaoFntz2ymlxVKe5d39Zpw4ebm9ZR1gWY8oXvy14JoyNx+PsP9l1OswlHUQnFMGDhcxW4KPADCEgLKQAMxICBj0b+Jy7DAUoYAHvAzI31uOAIDPC1HbrJMlVJbuBzGYOmS05fTCeUKu7a38aTE5VxgiZlbpGyKwoVxR7mwTPjVM5ZAWOyIrbfj7wREyPRSIvNFBy5kb8vAUvopO6NU+iU2hcItAvEViYuMjGFuxzI9EDQsf0uM106SoJgF/axTxLLlbMopIrd1RnujXgT34ShzzlsTBaczbfJw714+//TI1lnPHUeDv9/Z/+ZyORuXEBy8rfRXOLVx8qnyITbDu7zad4yyTKBJk5ETPAFugYOsQ03T4oU02pwyBI7XVuh2SZUmSGtu8AAnkobyBngHHXAq5j84C7GDu8n6S4pnoc2ljo5MBi6Mqj2oAFLESBRYgFAf5fvfT7Y8UPo7tbTI3EtVMj8eBTI5s4WlXGGBtR6hpjI4pydw6kLTYRXviRRONjROSh2ruFCjqpW4MLOoX2ISLSjCfoqmyTK+hk9MQXdOL3odM7O3i4PdyoE9Y3dtTpsQ/DoLNoqGj6vOV7u7HrYaGDjIZqFNrN7jYp6fNM6Urq/vR3j9Hv9ZnSxQbxrIC+atS7DqQas4bKY6S5QNrxgseSs59lGbvJjCQ2FtfcUEjG/2Cz+f+UP2uqkYTG28mfJyXfGO6ecZvu+B3oXd7q5EBqQlE5h1Aem0AhYIE8RcElPHA9j/rwfvvR9q1OTfUYRhCtvbHQ1eIIpg41TYSsXs6iril1GJ3bRoTUcS3XwqRGhLQq2c2VXQggO99w4Z5Ac+LCLjePP9ryqjLGCKlS1xghVZSrTp5+J6xiKlARv0pSPhOz3TfO4zCKo0WVZDTGR/uDBTqpW4MGOoX2IUbSZXz0aXBBJ6MnyKATv/fbbOQZ1NffmEQBFgqTeFRnqjXgzSUlZKmQNuU9UOcqrfAxG9BRFWudtbIBvWqHOxt1zs5ffzj/m/Hp7D/Oz/48TOKMrIlpkd0mzh0kmGDLxDJKLw9ncB0To/tjeO2nltQUPAzK1F4Xd7n5igXIczwTWCGyAQ7l3kMm/oAB90Lf8jyb9pJJ2FiPQxsLHcZGIKYU475jI5ukDqNzW8wec2pnjzl9ZY/hDq8zf7RzVWWMsRGlrjE2oih3a6e40Kw4aVku5mRLL1swubIjE8quo8yY8QWbX99kkZ/JAnJFKIp9JqaxNA1jAGUL2EEndWv4QafQGEDpElPoZPSEK3Ti96HTO0owa4c36sT0xx11GuxD13eVZSSa3XUwMc0+T+F8QOggk4w0Cu1adyOLuBSJf/XZ3ZuFDrK7e4yM37pbJL2RNzOu8XCeZSbwMFDwsCwgo+kKHj6qg7N2P+usWegc1gqdq88F7OYlQuun5K+dhdsT/3PBaspctSQ2LsruHWj4HU9s/Mh7u/sOv3cXjzMxQaJFet3I/YDUYcTj2uvcTjbvEngMBT0hLiETYYvwhLpYtHhPSyyPk39oHd7hSktgQWIG1AfQCgOAWWgCYtsOsDnktmk5iONG+zEeOwYa6zGMsdBioqJdO1HR7itRsbtj7hq4WlXGGIxX6hqD8YpyVaLiax7zMFoYi0TddLEKul8slkH+y/kiM95F2UIwlDH6vgV/opO6NZ+iU2gfonHdRt+fwiB0MnpiETrx+9DpnW3vbpFZ6KT1zi50ihxEHqtXeEywSICyIXQVlsukx5S/RItMhu6kxzyqY4n3Jru1hbDZ6zRaRNm18cfiUyNM0hyOrBCIBCQKVhlmGA06EwfTgw2j2eL/gqX1HEbbJHUYTLq9zu0gjGYTFx17zGfBcXEbn2s6NhYjHUJL/MBeshgb63FoA6DDsBqyqR8izwLYC6BweL4NPIwpgL5tu45rIdPtJ7TaVI9hjIUWc1xJ7RxX0leOa2dhtSauV5UxhtWUusawmqLc3RzXN4m/SKM4P7LkMpnydMxf1dferevQSd2a+9AptA/BlI4jaE8gDzoZPREInfh96PSOImidkAqd1K0RC51C+zAwOrxjHrkOxHZvsfSHhQ4y3VGj0O5192q69dvdm4QOsru3kd0arJBwHidfI+GjOghr9zNXq2XUR2WuVjmos/qZq1aFOLvMXP0gr3hiU+M9+5rnr1bsRZ70LoPxZyyd3hhn8TS6ul7weCbmzUAj8XjiOgcbiYfCuDkU9nKYRE2pwwi+tde5HQZimcl46GICfGz6AFvyPFrPNIEXWDRkNkOm00sqc2M9hjEWWgvEUljjGt+qZD+BWKezQGwDy6vKGAOxSl1jIFZRrih2PjV8gazSyFsuEiPg02liyASNKDF4/is3eBD9JgxAJKaNseDZIpLfJP5SwplIQB3xvfibTafMkLkHUXKVsjBiRpinH4QCXBnAiIQcliXGdLwAZ0Pt3XokndSteSWdQvsQ0ek2vvsUSKqT0RMs1Ynf+8S4aAoqUwtyUwsKUwt4/isHpakFQQRyUyu/KU0tWJlaIE0tUE0tqEwtAFKMsLTr+xbqzOBBZNbVCSG0QOs/Cd6cLZj0dj+umpXJm+3BhZ9I9CX92TBZPHImLkW7zeK7PDIwcJ3Q8RyAPUYBxqYFCEeiibDrYR+KAeb1QvEb6zEMotfeWGg7/e7Nx/OTPNebWmLizdMkPBHTe7VMRR0HQsvsOw/vCcoMYzi0wftXuxVrHDJYlexnX6Pb2b7GBrZalTHyfqWukfcryhXFflofl8KmxtkXQc8/cTllxAgeufkWQIRO6taAhE6hfeDmXaThdA4udOK3CzB0Wu3DMOkqSwNAgCgougD0eexc1fFjpkaDAM5M8ZlAeMupeJmVzzyqY4v3I0ujjcwJ/lW5zHKVC/6vv/93Zlxec4Fx/ZthBligLQ3qbgdYukuTcAmGyDZ7TpPYKHUYFLm9zu0weuYzEzkcuoAiKwSYmj6gpuUCAm3fFa6FYMfvo+cb6zGMsdBimgSsnSYBh79frYHlVWWM4RKlrjFcoihXFPs5jfJJNmMLMS2fG3Pxu4ANX7iYYlmW/518lV/IRNB8aAtQOb0xprLYOif0/Zj8sKH2bv2MTurWfI1OoX1gzt0mPzwFaOpk9AQ2deL3PvlhXhhQUBhQsLafYGU/QW4/89O8K/MJKvMpQMg6o6HOtBwzGireLI8ojNjK+ywiFrNh0msEJ9B+5KXke5S/4Fph4Pt2CJBrIYA9SAAhNACUB4zjwDO9oJdV68Z6HBgDI5TWPjGEDp6BNZmcqoyRgSl1jQxMUW6VlsdZlsTPjZR/WU30gmyx9CpPj8zWR+C94WIOjRxrC55EJ3Vr3kSn0D5wrM72kGNkuy4m1O5rCDwsdFyZ3Myu0tw4gso25kRqbRqBMI350auBNI1HdabcuFZZpoMnS/FXIDzhaTKbs1iG8haJcZZv6RaNe22cij+XAv5V560OlF7RCbIOl175mAeB6dmAUuIAHMhzFhyCADQD5NoOIybr5SzWxnocDr3CJrYcQmsscFUlO6BXBEALQJLnA9MTjCbUtTqjVw0mpypjpFdKXSO9UpQrip2J4RRNjXmSiWkylVt/DT9NopQbmbBPhig7T3kcpPyHkVltwYnopG7NkegU2vuFDLkrcwrKeSIQNSjmCZDzBFTz5KhO8+8+yrb6Qdmnog0FMYuloZlxmRX4hS8XPwwTSEM8ge6OpwF2CKSxb9PADEIByEICcEgY8GyXA8dEPseexwK/l3WKxnocDpBe5X/h2plieB8O1Hn05FRljEBaqWsE0opyt042z7PZA+Vs8+pExxNjfSeS3EnwXC5aJNPkKt+KJ5c05im7momivlHE9bIRcm/B3eikbs3l6BQaFzM2LGasz6a999XGE3F3kWI9cCIuuGU+gWI981WPtfUEK+t5VGdWjkzs8E61RXBC6eEueIRBiCEReN91uID6DkeAIscTTRRyhzNCQr/R5tTHGszGehwOTyuWMUiNG6iqkr0seDjdLXg0mJyqjJGnKXWNPE1R7rafkz7uYjmXM7P44KebTMxT4WJG1rUF56GTujUHolNo7xc6VvMhx9SZMkHArJwgR3UafgTWKzj8lmeimQVgNgKeySTVbLbMJKY+W6bJfKhbMtyJC8luQ+gObvHND9VJfg2TNLj2yiN1bGxTF03EgJpA9R7wDqHzU1QZBnpubxh0yKSga7shRCagzJKHv1g2YH4YAt8PEXNhQMIA9TEcGusxjLHQ4oqXVXvFyxr8ilcTG63KGJmUUtfIpBTlVhnS+eQW5iObjZRpC15CJ3VrnkKn0D4sVHV0g2tXgFIneJugUqfTPgyPDtcxV+3f7zLmJqHjMuYDm7Iqv3hUx96OkZJV1OPymhsfc+NjvGNxIF8yz6ARH7+L5L0nN/LXs8V15GfDDJtAPLGtHQ+bdMiXOSEBdW0CsOVyORsY8JDLACecOBiRkLFe+HJjPYbBl9sbC52E0JB7fAGh5VALOghJCk5pH71eCs9qCT+0rv6aZMu2w2OioSF1TWLLdoZ99HEdkcPo2RaDXnbtoJc9+KBXEw+ryhiDXkpdY9BLUW51IKiag1gkyilZ3NXh5hK0XqZRPB5Jo6u9W0iok7o1WKhTaB+iH42xg67W1vCDTkAfGEInex96vLNwaFvkQCeqX4Kg02Lvk4hup+rLVCI1XZ/nbhIkcX440aJwk0d1zG/TWJm8sIV/7SFUViWc6UNl6nPiR6AxVPmV4oOuQmwXpx8vL88v3hl//PjLpw+v3hsffzQu3519fP/x7V8HGlJzJwQe7uW2HsWMmqYHiON7gtA5BBDuiibybSd0A2iZ8H6BLRPvxnoMg423NxYarhRtXCUCUDS7TV1gIxs02/HWZJkolylEjrvp8+BJjWtqq5L9hFk6u6a2idVVZYxhFqWuMcyiKFcUe7e8Soy3abKIltlzGWIxLtPl4rpcDjy9TqNMnmhvfOLT6CpSVB6DLP2BBJ3UrQEFnUL7QLk7vJX0qeBBJ6N7AKGTvPcE+1oYSHBVGMgVixb2Md8wL37xS/soXzG3j0d15l0Dds0DeQ0qkCR7R9h1Cyz57l74W3vejdOpvLpmsCkozgRZh3tIme2YNrSIBagnz8AmkAHKbAZCx+GhQ0PbJr2c9ttYj4OjVW5tWuUOn1Y1mJyqjJFWKXWNtEpRrsqwPE3iUNgQMUUkm/qRRYJWfZV/FDetlKe9/Ovv/+dNlPlCfmb8nP7+D3klW8yilP/r7/+3pGHVrd0j+erd4+ikbs3r6BQayZeefFmC/UBKEe2VeFVSxwT/BtwrZ1hrKyo5VyitKJBWdHX4mLy2Mr94ZWVCwVwSsZUFLVnaYmU+j+pMy93fJYD6IWfvuS9vCc0MvhC+SrzaXHin4rLQLD9iQSZhRfKfwkf9/g/Zwr//Iz/Y/u3E+POkuox5kOyNTCCiu83eml0CrDeTq2t4XUpt6riu1ewC4MeayZpSh0HH2uvcDqm5YweOQyEHISI2wCG0AHGQA1wv9E3Mqc1xoxvAH9vzjfUYxlhoj5ojVJeay5L9UHPSGTVvYHlVGSM1V+oaqbmiXFHsfGqkbC4RZ2KEPOAgZTJuz42YT405lyYqTUrwErE4GUn3FnyJTurW/IlOoX0g3c2ApK7KNsGkTkZPgFInfu9XPaMpKI0kkEZyZSOBsJGgtJFgWtnIozpzrwGzLlkjSxeRP+U9kOtqG0K3t54KrpotmABmxo95BncYMeMDT8CFn0jEI2odJnFG5oRguNvEuZstuCYRGNwl4l/K1bkd0qc6IofBlNrr2S6vr3UC4iFKQRB6qNg6QbjlAZNAEtgO9zzey5kKjfUYxlhokTXXuL62Kjlw1tzE7KoyRtas1DWyZkW5W7dunRivk+DmuXGx9AR6iX3+3PgpiVllaUeO3J/n0EndmvfQKbQPHLm7rddPxY06AX1gR53svSfIK+4LPGEOBQJZWUPhcYUxPKozyXZ+g21PRPjDMvZlavUfi0+Nj2G+C6U8h078euGLkevzgdJhZ4JNc7fpcJekift+GBIIaGC7AENsA0a4Cwj2rRCZhNGeSFNTPQ6NNEFaOwuYDp80NZicqoyRNCl1jaRJUe7bq4p/4gs2v76R+1nWWb3jsVUbau/Wgeikbs2J6BTaB+7U2Y5KaiOMEMSot6vQhNQHhI5JvTUvH55VFnGdqLs6oajOTNv9PN2eGNTqXMSbYgdlfnfwTwLRpbGxPkPxxjhCtg0Bcm34h2ESKWhPCHV3m0h1tK7oOpaLrF7XFR8SOQxa1F7PdkiRA+LCgDECOA5CgE2CgOdaDDimFWIzsE2b9nLVXWM9Dm0sdHJWO4GSb1uWS8U/TRtSy+ol+/5xwofR1S1GQ2pcCF6VHPqe6AYeVpUxRkOUusZoiKLc6kTvNFkkV4yzMeCxBTigk7o1SKBTaB8CHh0uFj+RDOgE9EEIdLL3occ7O6e7LWCoE9UvONRpsff5AvPS+R3VMapjckDh+F99+PDq/YU8O/vi9Pzsw+nZMGNXCE3MA76NjjkODSgJAbJtuTfEcQD1/BBQi5mcWgHFpJc7qxrrcWi0F+HamdO4L9pLO0sCaDA5VRkj7VXqGmmvolxR7HWUiekoBtM0yFdootmMy/l+YiRpdBXFeToAMz7zG8NPYp/PF+XJYDxfy5kVazlB4svVMS5LL+PoC08zNjWuWSokjSkE23A/Oqlbc0E6hfaBX3WWQmBhgi3HMXFvdFqeC7ZZ6JhCsJlMeZVJza88Kk0qWFlUmU7AgLCoYGVRy1PCcosKCosKSosqS68tKlhZ1KM6M3XnUxAg7YenXcx//3+LlMcsyrI8D766vZFLb/aGLxeZfz2V/g/ZLgTIseGJcRYvvv7+zzTkxt++RuJ7HhvvljMWR9lsmRm/iMJ/SuKQZ9nqmveIp8v46rnx8fPn5XQhSl+mLIikvuLfsviFf72crvNKPg+ULcKJSezdZotdXK4DBcMg0EEU9naxzlrigGlfe53aYQiAeJSHrs8AC7mwpr4bAI+GASA2YQ4lNnRpL6eBN9ZjGGOhxRCAVTsEYA0/BNDA4qoyxhCAUtcYAlCUu7N5+lVsnMcLLrTzF0tB319HyVUqwcpI4LfgPHRSt+ZAdAqNBL4z1Kirv3vkqJO890uh5dZpJg/vrowh8EpjeFRnkjXk3sPj3DtHT+kE24d7rxHlLnMC6ICAegRgaHIxLCVUDUPMTWw6HuklXbuxHsNgMu2NhY42ZTiCj1BHJkT10t11RA6jZ1vkqGZtjmoOf696A7Oryhg5qlLXyFEV5aobq1Y8FbzhmfE6yTIu1UxTns2FksUZLCNN7R0Z6KRuDR3oFNoHmtph5vYTEYNOQB+oQSd777mqXDRe8dWAZ8DLjaIAIapRPKoz2XY9o7evleLXqSCi2bVRHvcVJmme61Se95WE6q71gS7g2hPHxQfLkOVRd7ZtMtEgHAHsUHluA8bAIuJ3z6IWdFgflKmxHsPgUe2NhU4W8y3hcCB0sdvbYv5a4oA7tUVybNcmx/bwF3AbWFxVxkiOlbpGcqwod2sB10hi9eC2PJ37PPaZmKhy8p+UNzvLzz/xLyubUKZzX4hP4gUXP9fGqfhzmY7LvtvAETqpW8MSOoX2gU93tuz7NHyhq797jKGTvPdUulz2TWL1qLcif7uyoeW1zvLjdG1Cy/ztrDKhwC9M6FGdqbm7i8XVVur9XCyGE9va8dugOrpG2bEIpiZCxL7/8vdurlF+SOowaFB7ndthnMN1bYsGHgWe4whWxWxb/MsOge0ExPZd4lCrl55vrMcwxkKLlNipTYmd4VPiBpZXlTFSYqWukRIryhXFTsUv4nUyyYlfBWyWjVx2C75CJ3Vr/kKn0D5w2Q6vSX4iWNTJ6Akw6sTvQ6d3EsCAiLqmg03Y3xlumyWOO843By/8lbeT0Qsmvd1RHSu6+/vDqzhWt7dd5w/JLdyz58bPYhQkco93Hid/mwQDXeWHE9c53Dx427cR9VwIXJuL8W9yF1BumgAx1xM/RPz0crNXYz0Ojv3WPssa9XWWtQM7ZL+PnpyqjJH9KnWN7FdRbnUwZXq1LOhvmcB29qs8lCb28wO6hFM7ya+xPE3iRRSLgmxqnC3TZM6ZPA3FEwNw5Mtb8C46qVvzMDqF9n45kJUzCIgZVBzOVM4geSDTVRLkH/rVBAJ8NYFAkE+gozods/sQ3OkHgufWiM28NAqubqXbnkVX17ezUpQM3GFCc+hOEDrcBFxGPOzQEIMQ4wBg4lJAAoJkeroVBlBMDJ/2Ac0b63Fw0NytDc3dwUPzJpNTlTFCc6WuEZoryq1iTfwLZ1MuMzCn0VWeljki8SecntuRM9FJ3ZpD0Sm090g8XU0Y+Q75hHkQeNfphxF4HyLwJhMbHi7wdnyHWi5lgPk0BNgjUMwLRoDDGDM9RAI77GXnW2M9hgG82xsLHZ0NY5o2Nm35r17uNakjchg92yKlorUpFR0+pWpgdlUZI6VS6hoplaLcarXDEL/OWSw3sy2S2/eSzNeQZTLyqC1gA53UreEDnUL7kAzW4ekwT8QMOgF94Aad7L1nzgys7SJYJLdvF6ns4lGd6bbz58NUeYOdsuWf352//3jxUfx1+uq98e+/vPp0efbp/V+HSYeRObGps9t0uNnFtXpblt8Zi5zjCwgtCiyHoCNo/UEaGAR6WXUqFchqKzAMbtRel3e04VG0NzEtSrHZRzfXE3poXdthcMtCnFAmKmfEowCHpg2YFVLgcEwowQJJkEYXUz+24xvrcWhjoZvgFiKWaVIkgyq93NhTR+Qwera94JYJ6wa3ZMl+gluos1TeBiBKlTEGt5S6xuCWotzts53kZhSZyDsXbQFOk7mYooLCxOVlu3xi/Cm5SX//hyEpVxaJAT7Gu7YAF3RStwYZdAqN8a4uYYROQB9QQid7H3q8wz3OT+OHOhH9cESd9H3o8mYRIF2VHUWBdOL6jwTpNNn7aHd1b1exNUMCJX+Nk8ortDn4LGASB2uUdFTH+zYIf5exYJYuIn/KewiBVx5DHwJXnxM/ApcreWbFB12Fzi8uf3lzfnZhnH8w3p1fXH789Ffj1Yc3xjqi/lfj44/Gxen52YfTs2HG06E7sfGOnybXYQQOuj73LGQDlxEXYCaaxuMhB27IKMOMQT/oZct1Yz2GEadpbyx0FYGDxLIkhkW9nC9XR+Sh9WwXq2aWbR1fz68nwtMI1AppL3O5vuBhdHGLQVZUO8iKBh9kbeJZVRljkFWpawyyKsrdCrIKzX7iC3nHbxb52YlxvsiMj2l0FcVZHn99I098TuZyX7hxVKS2/GGMsm4BEuqkbg0W6hTahwBMl1HWp0FFnYA+4KJO9j70eFcht1YgpE5MfzBSp8GhhNgyMKu8JIjksYOFk8xjb0HlJFfpn0d1TO/ehNhaCJX98eMvnz68ei8DYpfvztbhMvFrFS0bZogM2RNquQcbIvM9HIQ8QHIOQIAh8gG1GAQuMuUZt8y0EG9iEx9rDxvrcXAsG9dm2bgvlm12eE3doyenKmNk2UpdI8tWlLvFskfGvAUPoZO6NS+hU+hQQPRRndbd/VNKzH4Q8ZvoN3lCdxolxnks3kx4VT+aTyPxERcu1rjwxcD9jRnc+JEHfKDY2JyY5uGe2E0cK/AJd4Bt+fKiPC5IostsQELMLM4s7Nu9YOPGehwcNrZqY2Nr+Ni4weRUZYzYWKlrxMaKct+m+a9OBlxmxmUy5Wk+7UfQvAXXoZO6NfehU+hQQPPq3t3V9ACL9fQ4qtPsgwgx9wSoX/109un89NUH4/TV5buP789PDc3RBsaJcScc/dCjry4uPp6ev7o8//hhoEAcT5BzuMcEUtviDvQswKlMiPZcDqgjl2wsTF2fmJ6D77cvLQPxxnocHBA3awNxc/CpYE0mpypjBOJKXSMQV5Qriv3EsiyaJcZPy6uYRScy3JQmwVKGn7jBplNmhPnJSWHEZOCpxO1Hn8Z0sA21d+tZdFK35l10Cu09Tp8VswfM8tkj/Ek1eYCcPGA9eUAQgRLVr9JE6nTJiOErIP06FZA3uzb+WHy6vutLOcS7OrHbGCgWRxPsHG7CCPNc7EMSAj+0fYBDDwMWmAHgvmdDzDAN3V422TTW4+CwuF0bi9vDD4o3mJyqjBGLK3WNWFxRbrXqy+XkkIiCceMsFh+KIZX9YLyWN/GlIZ8Geby82r3xMV4k0+Qqd32fStswQvItOBid1K05GZ1Cew/JA2USAV5OIuCt51AeVF+ndierKSRvuVy/9VGdrhkENEc1oLn63Cw/EwFViH7W6SEJ78/OX384/5vx6ew/zs/+PEzIDsmE0B0/BqGjM2YdTDFFDkZ2L6i8ptRhYPD2Orejcw0c1zYtS6L6fo6xqCHy0Hq2m9PA3fwcKIxRftMAtLH9lz46uBSe3RH+69jV3QZVuGNSQh0KPIdDgAkmgMndaTZlPnaZaYeW3ccAaKzHMMZCi0EVUjuoQvoKqlgd3pb1aPikyhiDKkpdY1BFUe52pmHAjTc8MS7TKE5OqnuyBG+ZGiybc3+RyTBKFVzxk9jnc2kZ5OdyoUE+K6bZGGDZgrPRSd2aw9EptA8HInR4BMYTWYVOQB/MQid7H3q8w4OGnxgk0MnoKVCgE78Pnd7dUcMtUUydqH5ppk6LvQ+ll/kqARf/JWAhoRO4hZzACjnJ6Pk6pF4BJ/mxPJt4UQCnozoOeW/C6i2Exz+dvT9/e/7xlwtjdZrwMCPkCE0oMnc7Qt5JqI3AY0gdSFwCEYIQ2xZyewu11RY+jPBKe13d4YV7GFsm7fvCvQ1CD61rO4yiOj4hgUNtwH3iA2wxAoiLfeCTIPQs12Uw7KXjG+sxjLHQYhTVrR1FdQcfRW3iYlUZYxRVqWuMoirKFcUur2X4NOQC2ZfR0J9uZEaNgIh3wqPrq9vOY5+JGSytwon40Dj7lc3mU347yvrdR1E0/c74xFk2bvvW1N6tB9JJ3ZoX0im0DzGXjm/0ag5AdSL6AaE66fvQ5Z2F2dqilzpR/VJMnRZ7H2aTEbKg8K1lwGxW+tY7EbT1bV9R5VvFZ4AXrvVWGC6RjlW0i/SrR3Xs9Rh5q2JqB7HXzJkgh+x2WK5DQo8RCblpM+D6rimmBAwAY9ACnHsYYur5PsF9EPrGehwcoXdqE3pn+HvNGkxOVcZI6JW6RkKvKFcR+g/8q3HKBCRJrpYVrWfxMvPTaF7kQ11eJzOWCXoejResb8Ob6KRuzaPoFDoImB7zr8AvZ8warFczpvhIzhjxplFwVKcnRtitLnifvxn4WjedOK5zsKDaswm3TA8C4nMx4G0XAmaZDPDQsSD0A9NyejnVuLEeBweqaW1QTYe/StZgcqoyRlCt1DWCakW51V6DKAj49LnxZhn7bFzM2oaj0EndmrPQKbQPKxtR5sWtjwxEbBs6LsSkt7WMzRJb6+W5wOuCIewbMZrmVg8EudE7qjOZGlCeIPIX+ZUoN/JQj/SmB84Da3Ae9blZflQGrKj1TH92hvpcwG5eomojkvy1K371S/w5Tr7Gw6RWkExM+3DXK5Bn2xblDnCwawNMbAt4zOUCO1uBh3zHplYv6xWN9Tg0amXButRKlhw4tWoyOVUZI7VS6hqplaJcUex8aqTRlMV+lBgBv5pGRrZYBpEhw6wsjgwWGK88ngacj7xrG15EJ3VrnkSn0N6vU0RTUM4UkM8UkM8UsJopgAWArWbKUZ0e2P1LGesA9RbA84/rg/C5cbqcLpYpM2I+NS74YsF9Ma4TQ6YNsTiO/GSgENueYGfHz7LrEGJbjh0wikPAbdsDmGITEO4gQDzL8gITce73chZaYz0ODmLXvgrG6u0qGNwZxG4wOVUZI8RW6hohtqJcUey789mMywn+Xb6F5zs+m/M08q+TlGdR9t3EkDlD4keel5sfm/QpSYokoXcsFdXcGFGsOat6hORb8Do6qVvzPDqFxqUQTddSl1CICbIty6R9DYGHhY4LIg8wsJUdzXdr3LaixWaOyoaCVNpQmTZ2XdhQAUQ0Z5Uf1ZmbO07WEKW0H7ImfdVfk2V8ZahXDV9HmbJv47mBHOwA5Mhz9gdJ15wJcQ+XrmGHhxRbAoC7kAPsEAdQ7CIQhsSjEHI/dHqha431GAZda28sNHSSG6ATxi7EyIK9HKvzgLRh9GaL5BvVJt9o+OS7galVZYzkW6lrJN+KcqsLyqcC893izEYSG2+T4PlqV86VAIWpGGbr4zV+isQbxHG0nI3segsgQSd1a0BBp9DIrjsBD7q6OwYQOrF7v6zJcgOpEmIgSOFVEqz2YJX2cX1EwmxtH4/qzLuGzHl4jPlJJPbWd7KKVW/f+rz6ZyXgxXHK/SQNToq/Xv7b/weirs3DHDIDAA==\"\n" + + " }"; + Map errors = Maps.newHashMap(); + PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(); + cm.setMaxTotal(nThreads); + CloseableHttpClient client = HttpClients.custom().setConnectionManager(cm).build(); + for (int i = 40000; i < 41000; i++) { + String orcidId = String.format("%s", i); + String record = String.format(recordTemplate, orcidId); + countAll++; + if (partial == nTasks) { + System.out + .println( + "Waiting for tasks to complete before resubmitting to executor (countAll = " + countAll + + ") . . . "); + System.out.println("Getting replies"); + long startWait = System.currentTimeMillis(); + for (Future res : resList) { + if (res.get() == 200) + countOk++; + } + resList.clear(); + partial = 0; + System.out + .println(". . . Ready to submit again after " + (System.currentTimeMillis() - startWait) + " ms"); + } + partial++; + Future res = executorService.submit(() -> { + CloseableHttpResponse responsePPOST = null; + try { + + String url = String.format(BASE_CFG_URL, indexHost, indexName, indexType, orcidId); + HttpPost post = new HttpPost(url); + post.setHeader("Accept", "application/json"); + post.setHeader("Content-type", "application/json"); + StringEntity entity = new StringEntity(record); + post.setEntity(entity); + responsePPOST = client.execute(post); + int statusCode = responsePPOST.getStatusLine().getStatusCode(); + switch (statusCode) { + case 200: + case 201: + return statusCode; + default: + System.out + .println( + responsePPOST.getStatusLine().getStatusCode() + ": " + + responsePPOST.getStatusLine().getReasonPhrase()); + System.out.println("Source record causing error: " + record); + errors.merge(statusCode, 1, Integer::sum); + return statusCode; + } + } catch (ConnectException ce) { + throw ce; + } catch (IOException e) { + e.printStackTrace(); + errors.merge(-1, 1, Integer::sum); + } finally { + if (responsePPOST != null) + responsePPOST.close(); + } + return -1; + }); + resList.add(res); + } + executorService.shutdown(); + + // now let's wait for the results. We can block ourselves here: we have nothing else to do + System.out.println("Waiting for responses"); + for (Future res : resList) { + if (res.get() == 200) + countOk++; + } + client.close(); + cm.shutdown(); + + System.out.println("countOk: " + countOk); + System.out.println("countAll: " + countAll); + System.out.println("errors count: " + errors.size()); + } +} diff --git a/dhp-workflows/dhp-doiboost/src/test/java/eu/dnetlib/doiboost/orcid/MappingORCIDToOAFTest.scala b/dhp-workflows/dhp-doiboost/src/test/java/eu/dnetlib/doiboost/orcid/MappingORCIDToOAFTest.scala new file mode 100644 index 0000000000..5b82409421 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/test/java/eu/dnetlib/doiboost/orcid/MappingORCIDToOAFTest.scala @@ -0,0 +1,28 @@ +package eu.dnetlib.doiboost.orcid + +import org.codehaus.jackson.map.ObjectMapper +import org.junit.jupiter.api.Assertions._ +import org.junit.jupiter.api.Test +import org.slf4j.{Logger, LoggerFactory} + +import scala.io.Source + +class MappingORCIDToOAFTest { + val logger: Logger = LoggerFactory.getLogger(ORCIDToOAF.getClass) + val mapper = new ObjectMapper() + + @Test + def testExtractData():Unit ={ + val json = Source.fromInputStream(getClass.getResourceAsStream("dataOutput")).mkString + assertNotNull(json) + assertFalse(json.isEmpty) + json.lines.foreach(s => { + assertNotNull(ORCIDToOAF.extractValueFromInputString(s)) + }) + } + + + + + +} diff --git a/dhp-workflows/dhp-doiboost/src/test/java/eu/dnetlib/doiboost/orcid/OrcidClientTest.java b/dhp-workflows/dhp-doiboost/src/test/java/eu/dnetlib/doiboost/orcid/OrcidClientTest.java new file mode 100644 index 0000000000..75f857ca44 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/test/java/eu/dnetlib/doiboost/orcid/OrcidClientTest.java @@ -0,0 +1,136 @@ + +package eu.dnetlib.doiboost.orcid; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Arrays; +import java.util.Date; +import java.util.List; + +import org.apache.commons.io.IOUtils; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.junit.jupiter.api.Test; + +import eu.dnetlib.dhp.application.ArgumentApplicationParser; + +public class OrcidClientTest { + final String orcidId = "0000-0001-7291-3210"; + final int REQ_LIMIT = 24; + final int REQ_MAX_TEST = 100; + final int RECORD_DOWNLOADED_COUNTER_LOG_INTERVAL = 10; + final String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss"; + final String toRetrieveDate = "2020-05-06 23:59:46.031145"; + String toNotRetrieveDate = "2019-09-29 23:59:59.000000"; + String lastUpdate = "2019-09-30 00:00:00"; + String shortDate = "2020-05-06 16:06:11"; + +// curl -i -H "Accept: application/vnd.orcid+xml" +// -H 'Authorization: Bearer 78fdb232-7105-4086-8570-e153f4198e3d' +// 'https://api.orcid.org/v3.0/0000-0001-7291-3210/record' + + public String testDownloadRecord(String orcidId) throws Exception { + try (CloseableHttpClient client = HttpClients.createDefault()) { + HttpGet httpGet = new HttpGet("https://api.orcid.org/v3.0/" + orcidId + "/record"); + httpGet.addHeader("Accept", "application/vnd.orcid+xml"); + httpGet.addHeader("Authorization", "Bearer 78fdb232-7105-4086-8570-e153f4198e3d"); + CloseableHttpResponse response = client.execute(httpGet); + if (response.getStatusLine().getStatusCode() != 200) { + System.out + .println("Downloading " + orcidId + " status code: " + response.getStatusLine().getStatusCode()); + } + return IOUtils.toString(response.getEntity().getContent()); + } catch (Throwable e) { + e.printStackTrace(); + } + return new String(""); + } + +// @Test + public void testLambdaFileParser() throws Exception { + try (BufferedReader br = new BufferedReader( + new InputStreamReader(this.getClass().getResourceAsStream("last_modified.csv")))) { + String line; + int counter = 0; + int nReqTmp = 0; + long startDownload = System.currentTimeMillis(); + long startReqTmp = System.currentTimeMillis(); + while ((line = br.readLine()) != null) { + counter++; +// skip headers line + if (counter == 1) { + continue; + } + String[] values = line.split(","); + List recordInfo = Arrays.asList(values); + testDownloadRecord(recordInfo.get(0)); + long endReq = System.currentTimeMillis(); + nReqTmp++; + if (nReqTmp == REQ_LIMIT) { + long reqSessionDuration = endReq - startReqTmp; + if (reqSessionDuration <= 1000) { + System.out + .println( + "\nreqSessionDuration: " + reqSessionDuration + " nReqTmp: " + nReqTmp + " wait ...."); + Thread.sleep(1000 - reqSessionDuration); + } else { + nReqTmp = 0; + startReqTmp = System.currentTimeMillis(); + } + } + + if (counter > REQ_MAX_TEST) { + break; + } + if ((counter % RECORD_DOWNLOADED_COUNTER_LOG_INTERVAL) == 0) { + System.out.println("Current record downloaded: " + counter); + } + } + long endDownload = System.currentTimeMillis(); + long downloadTime = endDownload - startDownload; + System.out.println("Download time: " + ((downloadTime / 1000) / 60) + " minutes"); + } + } + +// @Test + public void getRecordDatestamp() throws ParseException { + Date toRetrieveDateDt = new SimpleDateFormat(DATE_FORMAT).parse(toRetrieveDate); + Date toNotRetrieveDateDt = new SimpleDateFormat(DATE_FORMAT).parse(toNotRetrieveDate); + Date lastUpdateDt = new SimpleDateFormat(DATE_FORMAT).parse(lastUpdate); + assertTrue(toRetrieveDateDt.after(lastUpdateDt)); + assertTrue(!toNotRetrieveDateDt.after(lastUpdateDt)); + } + + public void testDate(String value) throws ParseException { + System.out.println(value.toString()); + if (value.length() != 19) { + value = value.substring(0, 19); + } + Date valueDt = new SimpleDateFormat(DATE_FORMAT).parse(value); + System.out.println(valueDt.toString()); + } + +// @Test + public void testModifiedDate() throws ParseException { + testDate(toRetrieveDate); + testDate(toNotRetrieveDate); + testDate(shortDate); + } + +// @Test + public void testReadBase64CompressedRecord() throws Exception { + final String base64CompressedRecord = IOUtils + .toString(getClass().getResourceAsStream("0000-0001-6645-509X.compressed.base64")); + final String recordFromSeqFile = ArgumentApplicationParser.decompressValue(base64CompressedRecord); + System.out.println(recordFromSeqFile); + final String downloadedRecord = testDownloadRecord("0000-0001-6645-509X"); + assertTrue(recordFromSeqFile.equals(downloadedRecord)); + } +} diff --git a/dhp-workflows/dhp-doiboost/src/test/java/eu/dnetlib/doiboost/orcid/xml/XMLRecordParserTest.java b/dhp-workflows/dhp-doiboost/src/test/java/eu/dnetlib/doiboost/orcid/xml/XMLRecordParserTest.java new file mode 100644 index 0000000000..d5da4eec02 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/test/java/eu/dnetlib/doiboost/orcid/xml/XMLRecordParserTest.java @@ -0,0 +1,58 @@ + +package eu.dnetlib.doiboost.orcid.xml; + +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import org.apache.commons.io.IOUtils; +import org.junit.jupiter.api.Test; + +import eu.dnetlib.doiboost.orcid.model.AuthorData; +import eu.dnetlib.doiboost.orcid.model.WorkData; + +public class XMLRecordParserTest { + + @Test + public void testOrcidAuthorDataXMLParser() throws Exception { + + String xml = IOUtils.toString(this.getClass().getResourceAsStream("summary_0000-0001-6828-479X.xml")); + + XMLRecordParser p = new XMLRecordParser(); + + AuthorData authorData = p.VTDParseAuthorData(xml.getBytes()); + assertNotNull(authorData); + assertNotNull(authorData.getName()); + System.out.println("name: " + authorData.getName()); + assertNotNull(authorData.getSurname()); + System.out.println("surname: " + authorData.getSurname()); + } + + @Test + public void testOrcidXMLErrorRecordParser() throws Exception { + + String xml = IOUtils.toString(this.getClass().getResourceAsStream("summary_error.xml")); + + XMLRecordParser p = new XMLRecordParser(); + + AuthorData authorData = p.VTDParseAuthorData(xml.getBytes()); + assertNotNull(authorData); + assertNotNull(authorData.getErrorCode()); + System.out.println("error: " + authorData.getErrorCode()); + } + + @Test + public void testOrcidWorkDataXMLParser() throws Exception { + + String xml = IOUtils + .toString( + this.getClass().getResourceAsStream("activity_work_0000-0002-5982-8983.xml")); + + XMLRecordParser p = new XMLRecordParser(); + + WorkData workData = p.VTDParseWorkData(xml.getBytes()); + assertNotNull(workData); + assertNotNull(workData.getOid()); + System.out.println("oid: " + workData.getOid()); + assertNotNull(workData.getDoi()); + System.out.println("doi: " + workData.getDoi()); + } +} diff --git a/dhp-workflows/dhp-doiboost/src/test/java/eu/dnetlib/doiboost/uw/UnpayWallMappingTest.scala b/dhp-workflows/dhp-doiboost/src/test/java/eu/dnetlib/doiboost/uw/UnpayWallMappingTest.scala new file mode 100644 index 0000000000..8a6dd279b6 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/test/java/eu/dnetlib/doiboost/uw/UnpayWallMappingTest.scala @@ -0,0 +1,49 @@ +package eu.dnetlib.doiboost.uw + +import org.codehaus.jackson.map.{ObjectMapper, SerializationConfig} +import org.junit.jupiter.api.Test + +import scala.io.Source +import org.junit.jupiter.api.Assertions._ +import org.slf4j.{Logger, LoggerFactory} + +class UnpayWallMappingTest { + + val logger: Logger = LoggerFactory.getLogger(getClass) + val mapper = new ObjectMapper() + + + @Test + def testMappingToOAF():Unit ={ + + val Ilist = Source.fromInputStream(getClass.getResourceAsStream("input.json")).mkString + + + for (line <-Ilist.lines) { + + + val p = UnpayWallToOAF.convertToOAF(line) + + if(p!= null) { + assertTrue(p.getPid.size()==1) + logger.info(p.getId) + } + assertNotNull(line) + assertTrue(line.nonEmpty) + } + mapper.getSerializationConfig.enable(SerializationConfig.Feature.INDENT_OUTPUT) + + + val l = Ilist.lines.next() + + logger.info(mapper.writeValueAsString(UnpayWallToOAF.convertToOAF(l))) + + + + + + + + } + +} diff --git a/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/dhp/doiboost/publicationInput.json b/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/dhp/doiboost/publicationInput.json new file mode 100644 index 0000000000..4e6db2a124 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/dhp/doiboost/publicationInput.json @@ -0,0 +1,20 @@ +[{"journal":{"issnOnline":null,"vol":"37","issnLinking":null,"edition":null,"issnPrinted":"2415-8038","dataInfo":null,"conferencedate":null,"iss":null,"ep":"34","sp":"30","conferenceplace":null,"name":"Scientific Herald of Uzhhorod University.Series Physics"},"format":[],"source":[{"dataInfo":null,"value":"Crossref"},{"dataInfo":null,"value":null}],"description":[],"subject":[{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Instability"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Softening"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Composite material"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Self-organization"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Materials science"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Crystalline materials"}],"author":[{"surname":"Мар’ян","affiliation":null,"fullname":"М. І. Мар’ян","rank":null,"pid":null,"name":"М. І."},{"surname":"Юркович","affiliation":null,"fullname":"Н. В. Юркович","rank":null,"pid":null,"name":"Н. В."}],"resulttype":{"blank":false,"classid":"publication","classname":"publication","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"title":[{"qualifier":{"blank":false,"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":null,"value":"Dynamic instability and self-organization processes in the temperature interval of softening non-crystalline materials"},{"qualifier":{"blank":false,"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":null,"value":"dynamic instability and self organization processes in the temperature interval of softening non crystalline materials"},{"qualifier":{"blank":false,"classid":"alternative title","classname":"alternative title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":null,"value":"Dynamic instability and self-organization processes in the temperature interval of softening non-crystalline materials"}],"relevantdate":[{"qualifier":{"blank":false,"classid":"created","classname":"created","schemeid":"dnet:dataCite_date","schemename":"dnet:dataCite_date"},"dataInfo":null,"value":"2017-09-29T13:48:31Z"},{"qualifier":{"blank":false,"classid":"published-online","classname":"published-online","schemeid":"dnet:dataCite_date","schemename":"dnet:dataCite_date"},"dataInfo":null,"value":"2015-06-01"}],"dateofacceptance":{"dataInfo":null,"value":"2015-06-01"},"publisher":{"dataInfo":null,"value":"Uzhhorod National University"},"embargoenddate":null,"fulltext":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"externalReference":[],"language":null,"instance":[{"license":null,"instancetype":{"blank":false,"classid":"0001","classname":"Article","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":null,"distributionlocation":null,"accessright":{"blank":false,"classid":"Restricted","classname":"Restricted","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"url":["http://dx.doi.org/10.24144/2415-8038.2015.37.30-34"],"dateofacceptance":{"dataInfo":null,"value":"2015-06-01"},"collectedfrom":{"blank":false,"dataInfo":null,"value":"Crossref","key":"10|openaire____::081b82f96300b6a6e3d282bad31cb6e2"},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null},{"license":null,"instancetype":null,"hostedby":null,"distributionlocation":null,"accessright":null,"url":["https://academic.microsoft.com/#/detail/2760156017"],"dateofacceptance":null,"collectedfrom":{"blank":false,"dataInfo":null,"value":"microsoft","key":"10|openaire____::5f532a3fc4f1ea403f37070f59a7a53a"},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"context":[],"country":[],"originalId":["10.24144/2415-8038.2015.37.30-34","2760156017"],"pid":[{"qualifier":{"blank":false,"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"dataInfo":null,"value":"10.24144/2415-8038.2015.37.30-34"}],"dateofcollection":"2019-02-18T14:04:00Z","dateoftransformation":null,"extraInfo":[],"oaiprovenance":null,"id":"50|doiboost____::000212f083136806ae4d90079543ef29","collectedfrom":[{"blank":false,"dataInfo":null,"value":"Crossref","key":"10|openaire____::081b82f96300b6a6e3d282bad31cb6e2"},{"blank":false,"dataInfo":null,"value":"microsoft","key":"10|openaire____::5f532a3fc4f1ea403f37070f59a7a53a"}],"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1550498640287} +,{"journal":{"issnOnline":null,"vol":"58","issnLinking":null,"edition":null,"issnPrinted":"1554-0138","dataInfo":null,"conferencedate":null,"iss":null,"ep":null,"sp":null,"conferenceplace":null,"name":"PsycCRITIQUES"},"format":[],"source":[{"dataInfo":null,"value":"Crossref"},{"dataInfo":null,"value":null}],"description":[],"subject":[{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Self"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Trajectory"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Cognitive psychology"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Psychology"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Applied psychology"}],"author":[{"surname":"Cook","affiliation":null,"fullname":"Mary Ann Cook","rank":null,"pid":null,"name":"Mary Ann"}],"resulttype":{"blank":false,"classid":"publication","classname":"publication","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"title":[{"qualifier":{"blank":false,"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":null,"value":"The Trajectory of the Self"},{"qualifier":{"blank":false,"classid":"alternative title","classname":"alternative title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":null,"value":"The trajectory of the self."}],"relevantdate":[{"qualifier":{"blank":false,"classid":"created","classname":"created","schemeid":"dnet:dataCite_date","schemename":"dnet:dataCite_date"},"dataInfo":null,"value":"2013-09-03T15:05:24Z"},{"qualifier":{"blank":false,"classid":"published-online","classname":"published-online","schemeid":"dnet:dataCite_date","schemename":"dnet:dataCite_date"},"dataInfo":null,"value":"2013-09-04"}],"dateofacceptance":{"dataInfo":null,"value":"2013-09-03T15:05:24Z"},"publisher":{"dataInfo":null,"value":"Portico"},"embargoenddate":null,"fulltext":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"externalReference":[],"language":null,"instance":[{"license":null,"instancetype":{"blank":false,"classid":"0001","classname":"Article","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":null,"distributionlocation":null,"accessright":{"blank":false,"classid":"Restricted","classname":"Restricted","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"url":["http://dx.doi.org/10.1037/a0033963"],"dateofacceptance":null,"collectedfrom":{"blank":false,"dataInfo":null,"value":"Crossref","key":"10|openaire____::081b82f96300b6a6e3d282bad31cb6e2"},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null},{"license":null,"instancetype":null,"hostedby":null,"distributionlocation":null,"accessright":null,"url":["http://content.apa.org/reviews/a0033963","https://academic.microsoft.com/#/detail/2332131917"],"dateofacceptance":null,"collectedfrom":{"blank":false,"dataInfo":null,"value":"microsoft","key":"10|openaire____::5f532a3fc4f1ea403f37070f59a7a53a"},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"context":[],"country":[],"originalId":["10.1037/a0033963","2332131917"],"pid":[{"qualifier":{"blank":false,"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"dataInfo":null,"value":"10.1037/a0033963"}],"dateofcollection":"2019-11-23T21:36:18Z","dateoftransformation":null,"extraInfo":[],"oaiprovenance":null,"id":"50|doiboost____::0004e9dc4961ffe3db4f3b29869447b2","collectedfrom":[{"blank":false,"dataInfo":null,"value":"Crossref","key":"10|openaire____::081b82f96300b6a6e3d282bad31cb6e2"},{"blank":false,"dataInfo":null,"value":"microsoft","key":"10|openaire____::5f532a3fc4f1ea403f37070f59a7a53a"}],"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1574544978268} +,{"journal":{"issnOnline":null,"vol":"27","issnLinking":null,"edition":null,"issnPrinted":"0904-2512","dataInfo":null,"conferencedate":null,"iss":null,"ep":"129","sp":"124","conferenceplace":null,"name":"Journal of Oral Pathology & Medicine"},"format":[],"source":[{"dataInfo":null,"value":"Crossref"},{"dataInfo":null,"value":null}],"description":[{"dataInfo":null,"value":"Previous studies on the frequency of mast cells (MCs) in recurrent aphthous ulcers (RAU) have yielded conflicting results. Monoclonal antibodies specific for tryptase (AA1) and anti-IgE (polyclonal antibody) were used to identify density and distribution of MCs in an immunohistochemical study of RAU (n=15), induced oral traumatic ulcers (TUs) (n=9), and control clinically healthy oral mucosa (n=15). Results were quantified by means of a VIDAS image analyzer. In all sections studied, IgE-positive cells showed similar frequency and distribution to tryptase-positive MCs. In RAU lesions, numerous tryptase-positive MCs were found in the sub-epithelial lamina propria, but MC numbers in the epithelium were low and present only in some RAU biopsies. MCs were also more numerous in RAU-inflammatory infiltrates (118+/-31 cells/mm2) than those seen in TU-inflammatory infiltrates (75+/-18 cells/mm2, P<0.001). MC activation/degranulation, as judged by diffuse extracellular tryptase staining, was a common feature within RAU-inflammatory infiltrates and at RAU-inflammatory infiltrates-connective tissue interfaces, which were often associated with connective tissue disruption. MC counts in the RAU connective tissue, lateral to the inflammatory infiltrates, were significantly greater than in the connective tissue of TUs and of control biopsies (124+/-36 vs 73+/-13 vs 69+/-21 cells/mm2, respectively; P<0.001). Overall, MCs were significantly increased in aphthae (116+/-26 cells/mm2) compared with TU lesions (72+/-11 cells/mm2, P<0.001) and controls (71+/-16 cells/mm2, P<0.001). In conclusion, MC numbers are increased in a typical topographical pattern, and the local MCs show signs of activation/degranulation suggesting active involvement of this cell type in RAU pathogenesis."}],"subject":[{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Pathology and Forensic Medicine"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Cancer Research"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Oral Surgery"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Otorhinolaryngology"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Periodontics"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Tryptase"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.5225649"},"value":"biology.protein"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.5225649"},"value":"biology"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Biopsy"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.48158374"},"value":"medicine.diagnostic_test"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.48158374"},"value":"medicine"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Oral mucosa"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.51747984"},"value":"medicine.anatomical_structure"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Lamina propria"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Connective tissue"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Immunoglobulin E"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Basement membrane"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Degranulation"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.32854074"},"value":"business.industry"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.32854074"},"value":"business"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Pathology"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.43155256"},"value":"medicine.medical_specialty"}],"author":[{"surname":"Natah","affiliation":null,"fullname":"Sirajedin S. Natah","rank":null,"pid":null,"name":"Sirajedin S."},{"surname":"Häyrinen-Immonen","affiliation":null,"fullname":"Ritva Häyrinen-Immonen","rank":null,"pid":null,"name":"Ritva"},{"surname":"Hietanen","affiliation":null,"fullname":"Jarkko Hietanen","rank":null,"pid":null,"name":"Jarkko"},{"surname":"Malmström","affiliation":null,"fullname":"Maria Malmström","rank":null,"pid":null,"name":"Maria"},{"surname":"Konttinen","affiliation":null,"fullname":"Yrjö T. Konttinen","rank":null,"pid":null,"name":"Yrjö T."}],"resulttype":{"blank":false,"classid":"publication","classname":"publication","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"title":[{"qualifier":{"blank":false,"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":null,"value":"Quantitative assessment of mast cells in recurrent aphthous ulcers (RAU)"},{"qualifier":{"blank":false,"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":null,"value":"quantitative assessment of mast cells in recurrent aphthous ulcers rau"},{"qualifier":{"blank":false,"classid":"alternative title","classname":"alternative title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":null,"value":"Quantitative assessment of mast cells in recurrent aphthous ulcers (RAU)."}],"relevantdate":[{"qualifier":{"blank":false,"classid":"created","classname":"created","schemeid":"dnet:dataCite_date","schemename":"dnet:dataCite_date"},"dataInfo":null,"value":"2010-08-12T11:53:06Z"},{"qualifier":{"blank":false,"classid":"published-online","classname":"published-online","schemeid":"dnet:dataCite_date","schemename":"dnet:dataCite_date"},"dataInfo":null,"value":"2007-02-27"}],"dateofacceptance":{"dataInfo":null,"value":"2007-02-27"},"publisher":{"dataInfo":null,"value":"Wiley"},"embargoenddate":null,"fulltext":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"externalReference":[],"language":null,"instance":[{"license":{"dataInfo":null,"value":"http://doi.wiley.com/10.1002/tdm_license_1.1"},"instancetype":{"blank":false,"classid":"0001","classname":"Article","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":null,"distributionlocation":null,"accessright":{"blank":false,"classid":"Restricted","classname":"Restricted","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"url":["https://api.wiley.com/onlinelibrary/tdm/v1/articles/10.1111%2Fj.1600-0714.1998.tb01927.x","http://dx.doi.org/10.1111/j.1600-0714.1998.tb01927.x"],"dateofacceptance":{"dataInfo":null,"value":"2007-02-27"},"collectedfrom":{"blank":false,"dataInfo":null,"value":"Crossref","key":"10|openaire____::081b82f96300b6a6e3d282bad31cb6e2"},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null},{"license":null,"instancetype":null,"hostedby":null,"distributionlocation":null,"accessright":null,"url":["http://onlinelibrary.wiley.com/doi/10.1111/j.1600-0714.1998.tb01927.x/abstract","https://www.ncbi.nlm.nih.gov/pubmed/9563804","http://europepmc.org/abstract/MED/9563804","https://academic.microsoft.com/#/detail/2166339459"],"dateofacceptance":null,"collectedfrom":{"blank":false,"dataInfo":null,"value":"microsoft","key":"10|openaire____::5f532a3fc4f1ea403f37070f59a7a53a"},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"context":[],"country":[],"originalId":["10.1111/j.1600-0714.1998.tb01927.x","2166339459"],"pid":[{"qualifier":{"blank":false,"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"dataInfo":null,"value":"10.1111/j.1600-0714.1998.tb01927.x"}],"dateofcollection":"2019-03-21T12:03:27Z","dateoftransformation":null,"extraInfo":[],"oaiprovenance":null,"id":"50|doiboost____::0007c6d579668132b5bdb5e340f114d2","collectedfrom":[{"blank":false,"dataInfo":null,"value":"Crossref","key":"10|openaire____::081b82f96300b6a6e3d282bad31cb6e2"},{"blank":false,"dataInfo":null,"value":"microsoft","key":"10|openaire____::5f532a3fc4f1ea403f37070f59a7a53a"}],"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1553169807945} +,{"journal":{"issnOnline":"1933-169X","vol":"11","issnLinking":null,"edition":null,"issnPrinted":"1933-1681","dataInfo":null,"conferencedate":null,"iss":null,"ep":"328","sp":"309","conferenceplace":null,"name":"Journal of Information Technology & Politics"},"format":[],"source":[{"dataInfo":null,"value":"Crossref"},{"dataInfo":null,"value":null}],"description":[{"dataInfo":null,"value":"AbstractThis study examines the role of online networking in a grassroots movement in China. Drawing on Manuel Castells’s theory of communication power in the network society, we argue that microblogs can facilitate China’s mass self-communication in a network environment, even under authoritarian control, and are able to challenge the power of agenda setting, which has been mainly dominated by the state and the state media. We study a grassroots movement in China and examine the ways in which messages were communicated and people were connected into a network. Thus we investigate the role of online communication in reconfiguring the balance of power between the authority and Chinese citizens. Using systematic data collection and social network analysis, we characterize the microbloggers who contributed to the process, the network configuration, and the interplays between different stakeholders."}],"subject":[{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"General Computer Science"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Sociology and Political Science"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Public Administration"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Social network analysis"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Data collection"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Social media"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Microblogging"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Authoritarianism"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Communication theory"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"China"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Sociology"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Grassroots"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Social science"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Network society"}],"author":[{"surname":"Fu","affiliation":null,"fullname":"King-wa Fu","rank":null,"pid":null,"name":"King-wa"},{"surname":"Chau","affiliation":null,"fullname":"Michael Chau","rank":null,"pid":null,"name":"Michael"}],"resulttype":{"blank":false,"classid":"publication","classname":"publication","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"title":[{"qualifier":{"blank":false,"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":null,"value":"Use of Microblogs in Grassroots Movements in China: Exploring the Role of Online Networking in Agenda Setting"},{"qualifier":{"blank":false,"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":null,"value":"use of microblogs in grassroots movements in china exploring the role of online networking in agenda setting"}],"relevantdate":[{"qualifier":{"blank":false,"classid":"created","classname":"created","schemeid":"dnet:dataCite_date","schemename":"dnet:dataCite_date"},"dataInfo":null,"value":"2014-04-07T20:00:40Z"},{"qualifier":{"blank":false,"classid":"published-print","classname":"published-print","schemeid":"dnet:dataCite_date","schemename":"dnet:dataCite_date"},"dataInfo":null,"value":"2014-07-03"}],"dateofacceptance":{"dataInfo":null,"value":"2014-07-03"},"publisher":{"dataInfo":null,"value":"Informa UK Limited"},"embargoenddate":null,"fulltext":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"externalReference":[],"language":null,"instance":[{"license":null,"instancetype":{"blank":false,"classid":"0001","classname":"Article","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":null,"distributionlocation":null,"accessright":{"blank":false,"classid":"Restricted","classname":"Restricted","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"url":["http://www.tandfonline.com/doi/pdf/10.1080/19331681.2014.909344","http://dx.doi.org/10.1080/19331681.2014.909344"],"dateofacceptance":{"dataInfo":null,"value":"2014-07-03"},"collectedfrom":{"blank":false,"dataInfo":null,"value":"Crossref","key":"10|openaire____::081b82f96300b6a6e3d282bad31cb6e2"},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null},{"license":null,"instancetype":null,"hostedby":null,"distributionlocation":null,"accessright":null,"url":["https://www.tandfonline.com/doi/abs/10.1080/19331681.2014.909344","https://core.ac.uk/display/38056217","http://hub.hku.hk/handle/10722/203542","http://www.tandfonline.com/doi/full/10.1080/19331681.2014.909344","https://academic.microsoft.com/#/detail/2070515339"],"dateofacceptance":null,"collectedfrom":{"blank":false,"dataInfo":null,"value":"microsoft","key":"10|openaire____::5f532a3fc4f1ea403f37070f59a7a53a"},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"context":[],"country":[],"originalId":["10.1080/19331681.2014.909344","2070515339"],"pid":[{"qualifier":{"blank":false,"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"dataInfo":null,"value":"10.1080/19331681.2014.909344"}],"dateofcollection":"2019-11-25T18:51:18Z","dateoftransformation":null,"extraInfo":[],"oaiprovenance":null,"id":"50|doiboost____::00091e97b547d64358f7286c6003a70f","collectedfrom":[{"blank":false,"dataInfo":null,"value":"Crossref","key":"10|openaire____::081b82f96300b6a6e3d282bad31cb6e2"},{"blank":false,"dataInfo":null,"value":"ORCID","key":"10|openaire____::806360c771262b4d6770e7cdf04b5c5a"},{"blank":false,"dataInfo":null,"value":"microsoft","key":"10|openaire____::5f532a3fc4f1ea403f37070f59a7a53a"}],"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1574707878233} +,{"journal":{"issnOnline":"1573-9171","vol":"20","issnLinking":null,"edition":null,"issnPrinted":"0568-5230","dataInfo":null,"conferencedate":null,"iss":null,"ep":"1654","sp":"1650","conferenceplace":null,"name":"Bulletin of the Academy of Sciences of the USSR Division of Chemical Science"},"format":[],"source":[{"dataInfo":null,"value":"Crossref"},{"dataInfo":null,"value":null}],"description":[{"dataInfo":null,"value":"1.\r\n\r\nA study was made of the sulfonation of the 2-phenyl-, 2-(4′-alkylphenyl)-2-(4′-methoxyphenyl)-, 2-(3′,4′-dimethoxyphenyl)-, and 2-(4′-hydroxyphenyl)-3-hydroxypyridines.\r\n\r\n\r\n\r\n\r\n2.\r\n\r\nIn the case of 2-phenyl-3-hydroxypyridine the substitution was directed to the p-position of the phenyl ring, whereas in the p-aryl-substituted 3-hydroxypyridines the sulfo group substitutes in the mposition of the phenyl ring.\r\n\r\n\r\n\r\n\r\n3.\r\n\r\nA comparison of the conditions for the sulfonation of benzene and 2-phenyl-3-hydroxypyridine discloses that the presence of a heterocyclic ring as a substituent in the phenyl ring lowers the reactivity of the latter in sulfonation reactions.\r\n\r\n\r\n\r\n\r\n4.\r\n\r\nThe insertion of alkyl groups (with the exception of tert-butyl), methoxy, and hydroxy groups in the phenyl ring facilitates sulfonation.\r\n\r\n\r\n\r\n\r\n5.\r\n\r\n2-(3′-Hydroxy-4′-alkylphenyl)-3-hydroxypyridines were synthesized by the alkaline fusion of sulfo acids of the 2-(4′-alkylphenyl)-3-hydroxypyridine series."}],"subject":[{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"General Chemistry"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Substituent"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.56475383"},"value":"chemistry.chemical_compound"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.56475383"},"value":"chemistry"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Polymer chemistry"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Organic chemistry"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Alkyl"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.5366165"},"value":"chemistry.chemical_classification"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Benzene"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Aryl"}],"author":[{"surname":"Smirnov","affiliation":null,"fullname":"L. D. Smirnov","rank":null,"pid":null,"name":"L. D."},{"surname":"Kuz'min","affiliation":null,"fullname":"V. I. Kuz'min","rank":null,"pid":null,"name":"V. I."},{"surname":"Mikhailova","affiliation":null,"fullname":"L. M. Mikhailova","rank":null,"pid":null,"name":"L. M."},{"surname":"Lezina","affiliation":null,"fullname":"V. P. Lezina","rank":null,"pid":null,"name":"V. P."},{"surname":"Dyumaev","affiliation":null,"fullname":"K. M. Dyumaev","rank":null,"pid":null,"name":"K. M."}],"resulttype":{"blank":false,"classid":"publication","classname":"publication","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"title":[{"qualifier":{"blank":false,"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":null,"value":"Chemistry of 2-aryl-substituted 3-hydroxypyridines communication 3. Sulfonation of 2-aryl-3-hydroxypyridines"},{"qualifier":{"blank":false,"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":null,"value":"chemistry of 2 aryl substituted 3 hydroxypyridines communication 3 sulfonation of 2 aryl 3 hydroxypyridines"}],"relevantdate":[{"qualifier":{"blank":false,"classid":"created","classname":"created","schemeid":"dnet:dataCite_date","schemename":"dnet:dataCite_date"},"dataInfo":null,"value":"2005-01-08T02:10:56Z"}],"dateofacceptance":{"dataInfo":null,"value":"2005-01-08T02:10:56Z"},"publisher":{"dataInfo":null,"value":"Springer Science and Business Media LLC"},"embargoenddate":null,"fulltext":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"externalReference":[],"language":null,"instance":[{"license":{"dataInfo":null,"value":"http://www.springer.com/tdm"},"instancetype":{"blank":false,"classid":"0001","classname":"Article","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":null,"distributionlocation":null,"accessright":{"blank":false,"classid":"Restricted","classname":"Restricted","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"url":["http://link.springer.com/content/pdf/10.1007/BF00860021.pdf","http://link.springer.com/article/10.1007/BF00860021/fulltext.html","http://link.springer.com/content/pdf/10.1007/BF00860021","http://dx.doi.org/10.1007/bf00860021"],"dateofacceptance":null,"collectedfrom":{"blank":false,"dataInfo":null,"value":"Crossref","key":"10|openaire____::081b82f96300b6a6e3d282bad31cb6e2"},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null},{"license":null,"instancetype":null,"hostedby":null,"distributionlocation":null,"accessright":null,"url":["https://link.springer.com/article/10.1007/BF00860021","https://academic.microsoft.com/#/detail/214895358"],"dateofacceptance":null,"collectedfrom":{"blank":false,"dataInfo":null,"value":"microsoft","key":"10|openaire____::5f532a3fc4f1ea403f37070f59a7a53a"},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"context":[],"country":[],"originalId":["BF00860021","10.1007/bf00860021","214895358"],"pid":[{"qualifier":{"blank":false,"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"dataInfo":null,"value":"10.1007/bf00860021"}],"dateofcollection":"2019-04-26T20:11:56Z","dateoftransformation":null,"extraInfo":[],"oaiprovenance":null,"id":"50|doiboost____::00120b3e4e3ad02db2318298308c46d1","collectedfrom":[{"blank":false,"dataInfo":null,"value":"Crossref","key":"10|openaire____::081b82f96300b6a6e3d282bad31cb6e2"},{"blank":false,"dataInfo":null,"value":"microsoft","key":"10|openaire____::5f532a3fc4f1ea403f37070f59a7a53a"}],"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1556309516404} +,{"journal":{"issnOnline":"1521-3870","vol":"49","issnLinking":null,"edition":null,"issnPrinted":"0942-5616","dataInfo":null,"conferencedate":null,"iss":null,"ep":"108","sp":"101","conferenceplace":null,"name":"MLQ"},"format":[],"source":[{"dataInfo":null,"value":"Crossref"},{"dataInfo":null,"value":null}],"description":[{"dataInfo":null,"value":"Continuing the earlier research in [1] and [4] we work out a class of interstices in countable arithmetically saturated models of PA in which selective types are realized and a class of interstices in which 2-indiscernible types are realized (and hence, there exist elements whose stabilizers are strongly maximal)"}],"subject":[{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Discrete mathematics"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Countable set"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Mathematics"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Peano axioms"}],"author":[{"surname":"Bigorajska","affiliation":null,"fullname":"Teresa Bigorajska","rank":null,"pid":null,"name":"Teresa"}],"resulttype":{"blank":false,"classid":"publication","classname":"publication","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"title":[{"qualifier":{"blank":false,"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":null,"value":"Strongly maximal subgroups determined by elements in interstices"}],"relevantdate":[{"qualifier":{"blank":false,"classid":"created","classname":"created","schemeid":"dnet:dataCite_date","schemename":"dnet:dataCite_date"},"dataInfo":null,"value":"2003-01-17T16:43:11Z"}],"dateofacceptance":{"dataInfo":null,"value":"2003-01-17T16:43:11Z"},"publisher":{"dataInfo":null,"value":"Wiley"},"embargoenddate":null,"fulltext":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"externalReference":[],"language":null,"instance":[{"license":{"dataInfo":null,"value":"http://doi.wiley.com/10.1002/tdm_license_1.1"},"instancetype":{"blank":false,"classid":"0001","classname":"Article","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":null,"distributionlocation":null,"accessright":{"blank":false,"classid":"Restricted","classname":"Restricted","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"url":["https://api.wiley.com/onlinelibrary/tdm/v1/articles/10.1002%2Fmalq.200310010","https://onlinelibrary.wiley.com/doi/full/10.1002/malq.200310010","http://dx.doi.org/10.1002/malq.200310010"],"dateofacceptance":null,"collectedfrom":{"blank":false,"dataInfo":null,"value":"Crossref","key":"10|openaire____::081b82f96300b6a6e3d282bad31cb6e2"},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null},{"license":null,"instancetype":null,"hostedby":null,"distributionlocation":null,"accessright":null,"url":["https://onlinelibrary.wiley.com/doi/abs/10.1002/malq.200310010","https://academic.microsoft.com/#/detail/2093896874"],"dateofacceptance":null,"collectedfrom":{"blank":false,"dataInfo":null,"value":"microsoft","key":"10|openaire____::5f532a3fc4f1ea403f37070f59a7a53a"},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"context":[],"country":[],"originalId":["10.1002/malq.200310010","2093896874"],"pid":[{"qualifier":{"blank":false,"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"dataInfo":null,"value":"10.1002/malq.200310010"}],"dateofcollection":"2019-02-13T17:19:35Z","dateoftransformation":null,"extraInfo":[],"oaiprovenance":null,"id":"50|doiboost____::0012695bd163a2fa841fee8c9002bb50","collectedfrom":[{"blank":false,"dataInfo":null,"value":"Crossref","key":"10|openaire____::081b82f96300b6a6e3d282bad31cb6e2"},{"blank":false,"dataInfo":null,"value":"microsoft","key":"10|openaire____::5f532a3fc4f1ea403f37070f59a7a53a"}],"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1550078375948} +,{"journal":{"issnOnline":"1808-057X","vol":null,"issnLinking":null,"edition":null,"issnPrinted":"1519-7077","dataInfo":null,"conferencedate":null,"iss":null,"ep":null,"sp":null,"conferenceplace":null,"name":"Revista Contabilidade & Finanças"},"format":[],"source":[{"dataInfo":null,"value":"Crossref"},{"dataInfo":null,"value":null}],"description":[{"dataInfo":null,"value":"Abstract This essay presents recommendations in regard to accounting for operations that involve bitcoins, in compliance with the International Financial Reporting Standards (IFRS), and analyzes their main tax aspects. There is no specific pronouncement on the part of the International Accounting Standards Board (IASB) or from the Brazilian Accounting Pronouncements Committee (CPC) regarding the accounting treatment to be applied in operations that use these currencies. Bitcoin is of interest to economists as a virtual currency with the potential to disrupt existing payment systems and even monetary systems. This essay offers a contribution for standard-setters and the tax authority (fisco) by providing the basis for possible guidelines to be issued on the accounting treatment of bitcoin operations, as well as by defining the appropriate tax treatment; in addition, it makes a contribution for accounting professionals by suggesting the accounting policy to be adopted in these operations. Here, the analysis of the characteristics of bitcoins is compared with the guidelines and concepts of IFRS, in order to elaborate the recommendation for accounting treatment, and it suggests that the most adequate procedure would be that of foreign currency, which would go against the tax treatment adopted up until now by the Brazilian Internal Revenue Service (Receita Federal) or the Internal Revenue Service (IRS) of the United States of America (USA), which suggest treating virtual currencies as goods and not as currencies. It warrants mentioning that this contradiction may cause tax risks for taxpayers."}],"subject":[{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Economics"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Accounting"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.44534323"},"value":"business.industry"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.44534323"},"value":"business"}],"author":[{"surname":"Pelucio-Grecco","affiliation":null,"fullname":"Marta Cristina Pelucio-Grecco","rank":null,"pid":[{"qualifier":{"blank":false,"classid":"ORCID","classname":"ORCID","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"dataInfo":null,"value":"http://orcid.org/0000-0001-6994-4219"}],"name":"Marta Cristina"},{"surname":"Santos Neto","affiliation":null,"fullname":"Jacinto Pedro dos Santos Neto","rank":null,"pid":[{"qualifier":{"blank":false,"classid":"ORCID","classname":"ORCID","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"dataInfo":null,"value":"http://orcid.org/0000-0002-2106-8800"}],"name":"Jacinto Pedro dos"},{"surname":"Constancio","affiliation":null,"fullname":"Diego Constancio","rank":null,"pid":[{"qualifier":{"blank":false,"classid":"ORCID","classname":"ORCID","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"dataInfo":null,"value":"http://orcid.org/0000-0002-2659-0884"}],"name":"Diego"}],"resulttype":{"blank":false,"classid":"publication","classname":"publication","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"title":[{"qualifier":{"blank":false,"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":null,"value":"Accounting for bitcoins in light of IFRS and tax aspects"}],"relevantdate":[{"qualifier":{"blank":false,"classid":"created","classname":"created","schemeid":"dnet:dataCite_date","schemename":"dnet:dataCite_date"},"dataInfo":null,"value":"2020-01-31T09:33:16Z"},{"qualifier":{"blank":false,"classid":"published-online","classname":"published-online","schemeid":"dnet:dataCite_date","schemename":"dnet:dataCite_date"},"dataInfo":null,"value":"2020-01-31"}],"dateofacceptance":{"dataInfo":null,"value":"2020-01-31"},"publisher":{"dataInfo":null,"value":"FapUNIFESP (SciELO)"},"embargoenddate":null,"fulltext":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"externalReference":[],"language":null,"instance":[{"license":{"dataInfo":null,"value":"http://creativecommons.org/licenses/by/4.0/"},"instancetype":{"blank":false,"classid":"0001","classname":"Article","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":null,"distributionlocation":null,"accessright":{"blank":false,"classid":"Restricted","classname":"Restricted","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"url":["http://www.scielo.br/pdf/rcf/2020nahead/1808-057X-rcf-1808-057x201909110.pdf","http://dx.doi.org/10.1590/1808-057x201909110"],"dateofacceptance":{"dataInfo":null,"value":"2020-01-31"},"collectedfrom":{"blank":false,"dataInfo":null,"value":"Crossref","key":"10|openaire____::081b82f96300b6a6e3d282bad31cb6e2"},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null},{"license":{"dataInfo":null,"value":"cc-by"},"instancetype":null,"hostedby":null,"distributionlocation":null,"accessright":{"blank":false,"classid":"Open","classname":"Open","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"url":["http://www.scielo.br/pdf/rcf/2020nahead/1808-057X-rcf-1808-057x201909110.pdf"],"dateofacceptance":null,"collectedfrom":{"blank":false,"dataInfo":null,"value":"UnpayWall","key":"10|openaire____::8ac8380272269217cb09a928c8caa993"},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null},{"license":null,"instancetype":null,"hostedby":null,"distributionlocation":null,"accessright":null,"url":["http://www.scielo.br/scielo.php?script=sci_arttext&pid=S1519-70772020005001202&lng=pt&tlng=pt","https://academic.microsoft.com/#/detail/3003918216"],"dateofacceptance":null,"collectedfrom":{"blank":false,"dataInfo":null,"value":"microsoft","key":"10|openaire____::5f532a3fc4f1ea403f37070f59a7a53a"},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"context":[],"country":[],"originalId":["S1519-70772020005001202","10.1590/1808-057x201909110","3003918216"],"pid":[{"qualifier":{"blank":false,"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"dataInfo":null,"value":"10.1590/1808-057x201909110"}],"dateofcollection":"2020-03-23T23:40:24Z","dateoftransformation":null,"extraInfo":[],"oaiprovenance":null,"id":"50|doiboost____::0012dde1ec8595555eb3f0117a5f0ce6","collectedfrom":[{"blank":false,"dataInfo":null,"value":"Crossref","key":"10|openaire____::081b82f96300b6a6e3d282bad31cb6e2"},{"blank":false,"dataInfo":null,"value":"UnpayWall","key":"10|openaire____::8ac8380272269217cb09a928c8caa993"},{"blank":false,"dataInfo":null,"value":"microsoft","key":"10|openaire____::5f532a3fc4f1ea403f37070f59a7a53a"}],"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1585006824804} +,{"journal":{"issnOnline":null,"vol":"184","issnLinking":null,"edition":null,"issnPrinted":"0927-7765","dataInfo":null,"conferencedate":null,"iss":null,"ep":null,"sp":"110503","conferenceplace":null,"name":"Colloids and Surfaces B: Biointerfaces"},"format":[],"source":[{"dataInfo":null,"value":"Crossref"},{"dataInfo":null,"value":null}],"description":[{"dataInfo":null,"value":"Abstract In this work, highly lubricous hyaluronic acid-silica (HA-SiO2) nanohybrid coatings were fabricated through a sequential process consisting of a sol-gel followed by electrophoretic deposition (EPD). SiO2 nanoparticles were uniformly distributed in the coating layers, and the coating thickness was identified as approximately 1–2 μm regardless of the amount of SiO2. Incorporation of SiO2 into the HA polymer matrix enhanced the mechanical stability of the nanohybrid coatings, indicating greater interfacial bonding strength compared to HA coating layers alone. In addition, due to improved stability, the nanohybrid coatings showed excellent biolubrication properties, which were evaluated with a tribological experiment. These results indicate that the nanohybrid coatings have great potential to be used in biomedical applications that require superior biolubrication properties."}],"subject":[{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Biotechnology"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Physical and Theoretical Chemistry"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Colloid and Surface Chemistry"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Surfaces and Interfaces"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"General Medicine"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Organic chemistry"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Polymer"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.4843148"},"value":"chemistry.chemical_classification"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.4843148"},"value":"chemistry"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Coating"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.53945994"},"value":"engineering.material"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.53945994"},"value":"engineering"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Mechanical stability"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Nanoparticle"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Chemical engineering"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Hyaluronic acid"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.41162413"},"value":"chemistry.chemical_compound"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Electrophoretic deposition"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Tribology"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Self-healing hydrogels"}],"author":[{"surname":"Hwang","affiliation":null,"fullname":"Changha Hwang","rank":null,"pid":null,"name":"Changha"},{"surname":"Min","affiliation":null,"fullname":"YouJin Min","rank":null,"pid":null,"name":"YouJin"},{"surname":"Seong","affiliation":null,"fullname":"Yun-Jeong Seong","rank":null,"pid":null,"name":"Yun-Jeong"},{"surname":"Kim","affiliation":null,"fullname":"Dae-Eun Kim","rank":null,"pid":null,"name":"Dae-Eun"},{"surname":"Kim","affiliation":null,"fullname":"Hyoun-Ee Kim","rank":null,"pid":null,"name":"Hyoun-Ee"},{"surname":"Jeong","affiliation":null,"fullname":"Seol-Ha Jeong","rank":null,"pid":[{"qualifier":{"blank":false,"classid":"ORCID","classname":"ORCID","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"dataInfo":null,"value":"http://orcid.org/0000-0003-3210-9580"}],"name":"Seol-Ha"}],"resulttype":{"blank":false,"classid":"publication","classname":"publication","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"title":[{"qualifier":{"blank":false,"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":null,"value":"Enhanced biolubrication on biomedical devices using hyaluronic acid-silica nanohybrid hydrogels"},{"qualifier":{"blank":false,"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":null,"value":"enhanced biolubrication on biomedical devices using hyaluronic acid silica nanohybrid hydrogels"}],"relevantdate":[{"qualifier":{"blank":false,"classid":"created","classname":"created","schemeid":"dnet:dataCite_date","schemename":"dnet:dataCite_date"},"dataInfo":null,"value":"2019-09-26T21:12:12Z"}],"dateofacceptance":{"dataInfo":null,"value":"2019-09-26T21:12:12Z"},"publisher":{"dataInfo":null,"value":"Elsevier BV"},"embargoenddate":null,"fulltext":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"externalReference":[],"language":null,"instance":[{"license":{"dataInfo":null,"value":"https://www.elsevier.com/tdm/userlicense/1.0/"},"instancetype":{"blank":false,"classid":"0001","classname":"Article","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":null,"distributionlocation":null,"accessright":{"blank":false,"classid":"Restricted","classname":"Restricted","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"url":["https://api.elsevier.com/content/article/PII:S0927776519306472?httpAccept=text/xml","https://api.elsevier.com/content/article/PII:S0927776519306472?httpAccept=text/plain","http://dx.doi.org/10.1016/j.colsurfb.2019.110503"],"dateofacceptance":null,"collectedfrom":{"blank":false,"dataInfo":null,"value":"Crossref","key":"10|openaire____::081b82f96300b6a6e3d282bad31cb6e2"},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null},{"license":null,"instancetype":null,"hostedby":null,"distributionlocation":null,"accessright":null,"url":["https://www.sciencedirect.com/science/article/pii/S0927776519306472","https://academic.microsoft.com/#/detail/2976422802"],"dateofacceptance":null,"collectedfrom":{"blank":false,"dataInfo":null,"value":"microsoft","key":"10|openaire____::5f532a3fc4f1ea403f37070f59a7a53a"},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"context":[],"country":[],"originalId":["S0927776519306472","10.1016/j.colsurfb.2019.110503","2976422802"],"pid":[{"qualifier":{"blank":false,"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"dataInfo":null,"value":"10.1016/j.colsurfb.2019.110503"}],"dateofcollection":"2019-12-23T12:10:24Z","dateoftransformation":null,"extraInfo":[],"oaiprovenance":null,"id":"50|doiboost____::001311f85afceb7f157e9c756c0bc817","collectedfrom":[{"blank":false,"dataInfo":null,"value":"Crossref","key":"10|openaire____::081b82f96300b6a6e3d282bad31cb6e2"},{"blank":false,"dataInfo":null,"value":"ORCID","key":"10|openaire____::806360c771262b4d6770e7cdf04b5c5a"},{"blank":false,"dataInfo":null,"value":"microsoft","key":"10|openaire____::5f532a3fc4f1ea403f37070f59a7a53a"}],"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1577103024594} +,{"journal":{"issnOnline":"1680-7324","vol":"16","issnLinking":null,"edition":null,"issnPrinted":null,"dataInfo":null,"conferencedate":null,"iss":null,"ep":"3902","sp":"3881","conferenceplace":null,"name":"Atmospheric Chemistry and Physics"},"format":[],"source":[{"dataInfo":null,"value":"Crossref"},{"dataInfo":null,"value":null}],"description":[{"dataInfo":null,"value":"Abstract. We use the Weather Research and Forecast (WRF) model to simulate a large-scale tropical tropopause layer (TTL) cirrus in order to understand the formation and life cycle of the cloud. This cirrus event has been previously described through satellite observations by Taylor et al. (2011). Comparisons of the simulated and observed cirrus show a fair agreement and validate the reference simulation regarding cloud extension, location and life time. The validated simulation is used to understand the causes of cloud formation. It is shown that several cirrus clouds successively form in the region due to adiabatic cooling and large-scale uplift rather than from convective anvils. The structure of the uplift is tied to the equatorial response (equatorial wave excitation) to a potential vorticity intrusion from the midlatitudes. Sensitivity tests are then performed to assess the relative importance of the choice of the microphysics parameterization and of the initial and boundary conditions. The initial dynamical conditions (wind and temperature) essentially control the horizontal location and area of the cloud. However, the choice of the microphysics scheme influences the ice water content and the cloud vertical position. Last, the fair agreement with the observations allows to estimate the cloud impact in the TTL in the simulations. The cirrus clouds have a small but not negligible impact on the radiative budget of the local TTL. However, for this particular case, the cloud radiative heating does not significantly influence the simulated dynamics. This result is due to (1) the lifetime of air parcels in the cloud system, which is too short to significantly influence the dynamics, and (2) the fact that induced vertical motions would be comparable to or smaller than the typical mesoscale motions present. Finally, the simulation also provides an estimate of the vertical redistribution of water by the cloud and the results emphasize the importance in our case of both rehydration and dehydration in the vicinity of the cirrus.\n "}],"subject":[{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Atmospheric Science"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Cirrus"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Weather Research and Forecasting Model"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Cloud height"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Chemistry"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Atmospheric sciences"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Meteorology"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Microphysics"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Mesoscale meteorology"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Radiative transfer"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Liquid water content"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Climatology"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Potential vorticity"}],"author":[{"surname":"Podglajen","affiliation":null,"fullname":"Aurélien Podglajen","rank":null,"pid":[{"qualifier":{"blank":false,"classid":"ORCID","classname":"ORCID","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"dataInfo":null,"value":"http://orcid.org/0000-0001-9768-3511"}],"name":"Aurélien"},{"surname":"Plougonven","affiliation":null,"fullname":"Riwal Plougonven","rank":null,"pid":[{"qualifier":{"blank":false,"classid":"ORCID","classname":"ORCID","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"dataInfo":null,"value":"http://orcid.org/0000-0003-3310-8280"}],"name":"Riwal"},{"surname":"Hertzog","affiliation":null,"fullname":"Albert Hertzog","rank":null,"pid":null,"name":"Albert"},{"surname":"Legras","affiliation":null,"fullname":"Bernard Legras","rank":null,"pid":[{"qualifier":{"blank":false,"classid":"ORCID","classname":"ORCID","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"dataInfo":null,"value":"http://orcid.org/0000-0002-3756-7794"}],"name":"Bernard"}],"resulttype":{"blank":false,"classid":"publication","classname":"publication","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"title":[{"qualifier":{"blank":false,"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":null,"value":"A modelling case study of a large-scale cirrus in the tropical tropopause layer"},{"qualifier":{"blank":false,"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":null,"value":"a modelling case study of a large scale cirrus in the tropical tropopause layer"}],"relevantdate":[{"qualifier":{"blank":false,"classid":"created","classname":"created","schemeid":"dnet:dataCite_date","schemename":"dnet:dataCite_date"},"dataInfo":null,"value":"2016-03-23T09:58:39Z"},{"qualifier":{"blank":false,"classid":"published-online","classname":"published-online","schemeid":"dnet:dataCite_date","schemename":"dnet:dataCite_date"},"dataInfo":null,"value":"2016-03-23"}],"dateofacceptance":{"dataInfo":null,"value":"2016-03-23"},"publisher":{"dataInfo":null,"value":"Copernicus GmbH"},"embargoenddate":null,"fulltext":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"externalReference":[],"language":null,"instance":[{"license":{"dataInfo":null,"value":"https://creativecommons.org/licenses/by/3.0/"},"instancetype":{"blank":false,"classid":"0001","classname":"Article","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":null,"distributionlocation":null,"accessright":{"blank":false,"classid":"Restricted","classname":"Restricted","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"url":["https://www.atmos-chem-phys.net/16/3881/2016/acp-16-3881-2016.pdf","http://dx.doi.org/10.5194/acp-16-3881-2016"],"dateofacceptance":{"dataInfo":null,"value":"2016-03-23"},"collectedfrom":{"blank":false,"dataInfo":null,"value":"Crossref","key":"10|openaire____::081b82f96300b6a6e3d282bad31cb6e2"},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null},{"license":{"dataInfo":null,"value":"cc-by"},"instancetype":null,"hostedby":null,"distributionlocation":null,"accessright":{"blank":false,"classid":"Open","classname":"Open","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"url":["https://www.atmos-chem-phys.net/16/3881/2016/acp-16-3881-2016.pdf"],"dateofacceptance":null,"collectedfrom":{"blank":false,"dataInfo":null,"value":"UnpayWall","key":"10|openaire____::8ac8380272269217cb09a928c8caa993"},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null},{"license":null,"instancetype":null,"hostedby":null,"distributionlocation":null,"accessright":null,"url":["https://www.atmos-chem-phys.net/16/3881/2016/acp-16-3881-2016.pdf","https://www.atmos-chem-phys.net/16/3881/2016/","https://ui.adsabs.harvard.edu/abs/2015ACPD...1531089P/abstract","https://noa.gwlb.de/receive/cop_mods_00043829","https://academic.microsoft.com/#/detail/2135547990"],"dateofacceptance":null,"collectedfrom":{"blank":false,"dataInfo":null,"value":"microsoft","key":"10|openaire____::5f532a3fc4f1ea403f37070f59a7a53a"},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"context":[],"country":[],"originalId":["10.5194/acp-16-3881-2016","2135547990"],"pid":[{"qualifier":{"blank":false,"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"dataInfo":null,"value":"10.5194/acp-16-3881-2016"}],"dateofcollection":"2019-10-25T10:40:21Z","dateoftransformation":null,"extraInfo":[],"oaiprovenance":null,"id":"50|doiboost____::00182898f6c9a64b1256cbafaad5db16","collectedfrom":[{"blank":false,"dataInfo":null,"value":"Crossref","key":"10|openaire____::081b82f96300b6a6e3d282bad31cb6e2"},{"blank":false,"dataInfo":null,"value":"UnpayWall","key":"10|openaire____::8ac8380272269217cb09a928c8caa993"},{"blank":false,"dataInfo":null,"value":"ORCID","key":"10|openaire____::806360c771262b4d6770e7cdf04b5c5a"},{"blank":false,"dataInfo":null,"value":"microsoft","key":"10|openaire____::5f532a3fc4f1ea403f37070f59a7a53a"}],"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1572000021163} +,{"journal":{"issnOnline":"2067-9440","vol":"9","issnLinking":null,"edition":null,"issnPrinted":null,"dataInfo":null,"conferencedate":null,"iss":null,"ep":"58","sp":"47","conferenceplace":null,"name":"Valahian Journal of Economic Studies"},"format":[],"source":[{"dataInfo":null,"value":"Crossref"},{"dataInfo":null,"value":null}],"description":[{"dataInfo":null,"value":"Abstract\nProcurement management is a function for B2B market and a complex concept based on a continuous buying cycle and supposes: determining the need, taking the decision of buying, identification of sources, selecting and choosing the right suppliers, contracting, and maintaining the relationships with the future contractors.\nSupplier selection is a challenging task along supply chain and is based on some important criteria in order to choose the right supplier and maintain with it a long and prosper relationship. In order to select the right supplier, the buyer may use quantitative and qualitative methods. In this article we present the importance of supplier selection along supply chain and the used criteria to do this. We also present a supplier selection method based on mathematical models, especially on regression function and time series analysis, in order to show that they bring important benefits for the companies which used them."}],"subject":[{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Industrial organization"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Procurement"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Supply chain"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Procurement process"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Business informatics"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Economics"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Marketing"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Business management"}],"author":[{"surname":"Duica","affiliation":null,"fullname":"Mircea Constantin Duica","rank":null,"pid":null,"name":"Mircea Constantin"},{"surname":"Florea","affiliation":null,"fullname":"Nicoleta Valentina Florea","rank":null,"pid":null,"name":"Nicoleta Valentina"},{"surname":"Duica","affiliation":null,"fullname":"Anisoara Duica","rank":null,"pid":null,"name":"Anisoara"}],"resulttype":{"blank":false,"classid":"publication","classname":"publication","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"title":[{"qualifier":{"blank":false,"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":null,"value":"Selecting the Right Suppliers in Procurement Process along Supply Chain-a Mathematical Modeling Approach"},{"qualifier":{"blank":false,"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":null,"value":"selecting the right suppliers in procurement process along supply chain a mathematical modeling approach"}],"relevantdate":[{"qualifier":{"blank":false,"classid":"created","classname":"created","schemeid":"dnet:dataCite_date","schemename":"dnet:dataCite_date"},"dataInfo":null,"value":"2018-12-04T13:44:24Z"},{"qualifier":{"blank":false,"classid":"published-print","classname":"published-print","schemeid":"dnet:dataCite_date","schemename":"dnet:dataCite_date"},"dataInfo":null,"value":"2018-04-01"}],"dateofacceptance":{"dataInfo":null,"value":"2018-04-01"},"publisher":{"dataInfo":null,"value":"Walter de Gruyter GmbH"},"embargoenddate":null,"fulltext":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"externalReference":[],"language":null,"instance":[{"license":{"dataInfo":null,"value":"http://creativecommons.org/licenses/by-nc-nd/3.0"},"instancetype":{"blank":false,"classid":"0001","classname":"Article","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":null,"distributionlocation":null,"accessright":{"blank":false,"classid":"Restricted","classname":"Restricted","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"url":["https://content.sciendo.com/view/journals/vjes/9/1/article-p47.xml","http://dx.doi.org/10.2478/vjes-2018-0005"],"dateofacceptance":{"dataInfo":null,"value":"2018-04-01"},"collectedfrom":{"blank":false,"dataInfo":null,"value":"Crossref","key":"10|openaire____::081b82f96300b6a6e3d282bad31cb6e2"},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null},{"license":{"dataInfo":null,"value":"cc-by-nc-nd"},"instancetype":null,"hostedby":null,"distributionlocation":null,"accessright":{"blank":false,"classid":"Open","classname":"Open","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"url":["https://content.sciendo.com/downloadpdf/journals/vjes/9/1/article-p47.pdf"],"dateofacceptance":null,"collectedfrom":{"blank":false,"dataInfo":null,"value":"UnpayWall","key":"10|openaire____::8ac8380272269217cb09a928c8caa993"},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null},{"license":null,"instancetype":null,"hostedby":null,"distributionlocation":null,"accessright":null,"url":["https://content.sciendo.com/view/journals/vjes/9/1/article-p47.xml?lang=en","https://ideas.repec.org/a/vrs/vaecst/v9y2018i1p47-58n5.html","https://academic.microsoft.com/#/detail/2902900408"],"dateofacceptance":null,"collectedfrom":{"blank":false,"dataInfo":null,"value":"microsoft","key":"10|openaire____::5f532a3fc4f1ea403f37070f59a7a53a"},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"context":[],"country":[],"originalId":["10.2478/vjes-2018-0005","2902900408"],"pid":[{"qualifier":{"blank":false,"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"dataInfo":null,"value":"10.2478/vjes-2018-0005"}],"dateofcollection":"2019-10-01T15:05:24Z","dateoftransformation":null,"extraInfo":[],"oaiprovenance":null,"id":"50|doiboost____::001ad8f51e226b3f37d4544647524460","collectedfrom":[{"blank":false,"dataInfo":null,"value":"Crossref","key":"10|openaire____::081b82f96300b6a6e3d282bad31cb6e2"},{"blank":false,"dataInfo":null,"value":"UnpayWall","key":"10|openaire____::8ac8380272269217cb09a928c8caa993"},{"blank":false,"dataInfo":null,"value":"microsoft","key":"10|openaire____::5f532a3fc4f1ea403f37070f59a7a53a"}],"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1569942324370} +,{"journal":{"issnOnline":"1529-2401","vol":"25","issnLinking":null,"edition":null,"issnPrinted":"0270-6474","dataInfo":null,"conferencedate":null,"iss":null,"ep":"2061","sp":"2050","conferenceplace":null,"name":"Journal of Neuroscience"},"format":[],"source":[{"dataInfo":null,"value":"Crossref"},{"dataInfo":null,"value":null}],"description":[{"dataInfo":null,"value":"Neurogenesis requires factors that regulate the decision of dividing progenitors to leave the cell cycle and activate the neuronal differentiation program. It is shown here that the murine runt -related gene Runx1 is expressed in proliferating cells on the basal side of the olfactory epithelium. These include both Mash1+ olfactory receptor neuron (ORN) progenitors and NeuroD+ ORN precursors. Disruption of Runx1 function in vivo does not cause a change in Mash1 expression but leads to a decrease in the number of NeuroD+ neuronal precursors and an increase in differentiated ORNs. These effects result in premature and ectopic ORN differentiation. It is shown further that exogenous Runx1 expression in cultured olfactory neural progenitors causes an expansion of the mitotic cell population. In agreement with these findings, exogenous Runx1 expression also promotes cortical neural progenitor cell proliferation without inhibiting neuronal differentiation. These effects are phenocopied by a chimeric protein containing ETO, the eight twenty one transcriptional repressor, fused to the Runx1 DNA-binding domain, which suggests the involvement of transcription repression mechanisms. Consistent with this possibility, Runx1 represses transcription driven by the promoter of the cell cycle inhibitor p21 Cip 1 in cortical progenitors. Together, these findings suggest a previously unrecognized role for Runx1 in coordinating the proliferation and neuronal differentiation of selected populations of neural progenitors."}],"subject":[{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"General Neuroscience"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Cancer research"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Progenitor cell"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"RUNX1"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.51708895"},"value":"chemistry.chemical_compound"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.51708895"},"value":"chemistry"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Cell cycle"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Olfactory epithelium"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.53877425"},"value":"medicine.anatomical_structure"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.53877425"},"value":"medicine"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Population"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.50842315"},"value":"education.field_of_study"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.50842315"},"value":"education"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Neurogenesis"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"NeuroD"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Anatomy"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Olfactory receptor neuron"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Biology"}],"author":[{"surname":"Theriault","affiliation":null,"fullname":"F. M. Theriault","rank":null,"pid":null,"name":"F. M."}],"resulttype":{"blank":false,"classid":"publication","classname":"publication","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"title":[{"qualifier":{"blank":false,"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":null,"value":"Role for Runx1 in the Proliferation and Neuronal Differentiation of Selected Progenitor Cells in the Mammalian Nervous System"}],"relevantdate":[{"qualifier":{"blank":false,"classid":"created","classname":"created","schemeid":"dnet:dataCite_date","schemename":"dnet:dataCite_date"},"dataInfo":null,"value":"2005-02-23T22:18:03Z"},{"qualifier":{"blank":false,"classid":"published-print","classname":"published-print","schemeid":"dnet:dataCite_date","schemename":"dnet:dataCite_date"},"dataInfo":null,"value":"2005-02-23"}],"dateofacceptance":{"dataInfo":null,"value":"2005-02-23"},"publisher":{"dataInfo":null,"value":"Society for Neuroscience"},"embargoenddate":null,"fulltext":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"externalReference":[],"language":null,"instance":[{"license":null,"instancetype":{"blank":false,"classid":"0001","classname":"Article","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":null,"distributionlocation":null,"accessright":{"blank":false,"classid":"Restricted","classname":"Restricted","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"url":["http://dx.doi.org/10.1523/jneurosci.5108-04.2005"],"dateofacceptance":{"dataInfo":null,"value":"2005-02-23"},"collectedfrom":{"blank":false,"dataInfo":null,"value":"Crossref","key":"10|openaire____::081b82f96300b6a6e3d282bad31cb6e2"},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null},{"license":null,"instancetype":null,"hostedby":null,"distributionlocation":null,"accessright":{"blank":false,"classid":"Open","classname":"Open","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"url":["http://www.jneurosci.org/content/jneuro/25/8/2050.full.pdf"],"dateofacceptance":null,"collectedfrom":{"blank":false,"dataInfo":null,"value":"UnpayWall","key":"10|openaire____::8ac8380272269217cb09a928c8caa993"},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null},{"license":null,"instancetype":null,"hostedby":null,"distributionlocation":null,"accessright":null,"url":["https://www.jneurosci.org/content/25/8/2050","https://www.jneurosci.org/content/jneuro/25/8/2050.full.pdf","https://www.ncbi.nlm.nih.gov/pubmed/15728845","http://www.jneurosci.org/content/25/8/2050.full","https://academic.microsoft.com/#/detail/2083575840"],"dateofacceptance":null,"collectedfrom":{"blank":false,"dataInfo":null,"value":"microsoft","key":"10|openaire____::5f532a3fc4f1ea403f37070f59a7a53a"},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"context":[],"country":[],"originalId":["10.1523/jneurosci.5108-04.2005","2083575840"],"pid":[{"qualifier":{"blank":false,"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"dataInfo":null,"value":"10.1523/jneurosci.5108-04.2005"}],"dateofcollection":"2019-11-18T22:18:00Z","dateoftransformation":null,"extraInfo":[],"oaiprovenance":null,"id":"50|doiboost____::001aefea47c1b834ee7b9653a69dc497","collectedfrom":[{"blank":false,"dataInfo":null,"value":"Crossref","key":"10|openaire____::081b82f96300b6a6e3d282bad31cb6e2"},{"blank":false,"dataInfo":null,"value":"UnpayWall","key":"10|openaire____::8ac8380272269217cb09a928c8caa993"},{"blank":false,"dataInfo":null,"value":"ORCID","key":"10|openaire____::806360c771262b4d6770e7cdf04b5c5a"},{"blank":false,"dataInfo":null,"value":"microsoft","key":"10|openaire____::5f532a3fc4f1ea403f37070f59a7a53a"}],"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1574115480119} +,{"journal":{"issnOnline":"2372-9708","vol":"39","issnLinking":null,"edition":null,"issnPrinted":"0160-5976","dataInfo":null,"conferencedate":null,"iss":null,"ep":"343","sp":"342","conferenceplace":null,"name":"Humanity & Society"},"format":[],"source":[{"dataInfo":null,"value":"Crossref"},{"dataInfo":null,"value":null}],"description":[],"subject":[{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Media studies"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Social science"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Environmental justice"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Sociology"}],"author":[{"surname":"Denning","affiliation":null,"fullname":"C. Holly Denning","rank":null,"pid":null,"name":"C. Holly"}],"resulttype":{"blank":false,"classid":"publication","classname":"publication","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"title":[{"qualifier":{"blank":false,"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":null,"value":"Book Review: Our Roots Run as Deep as Ironweed: Appalachian Women and the Fight for Environmental Justice"},{"qualifier":{"blank":false,"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":null,"value":"book review our roots run as deep as ironweed appalachian women and the fight for environmental justicebellelizabeth shannon 2013 our roots run as deep as ironweed appalachian women and the fight for environmental justice champaign il university of illinois press 210 pp 25 00 isbn 978 0 252 07946 7"},{"qualifier":{"blank":false,"classid":"alternative title","classname":"alternative title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":null,"value":"Book Review: Our Roots Run as Deep as Ironweed: Appalachian Women and the Fight for Environmental JusticeBellElizabeth Shannon. 2013. Our Roots Run as Deep as Ironweed: Appalachian Women and the Fight for Environmental Justice. Champaign, IL: University of Illinois Press. 210 pp.$25.00. ISBN: 978-0-252-07946-7."}],"relevantdate":[{"qualifier":{"blank":false,"classid":"created","classname":"created","schemeid":"dnet:dataCite_date","schemename":"dnet:dataCite_date"},"dataInfo":null,"value":"2015-04-15T00:11:03Z"},{"qualifier":{"blank":false,"classid":"published-online","classname":"published-online","schemeid":"dnet:dataCite_date","schemename":"dnet:dataCite_date"},"dataInfo":null,"value":"2015-04-13"}],"dateofacceptance":{"dataInfo":null,"value":"2015-04-13"},"publisher":{"dataInfo":null,"value":"SAGE Publications"},"embargoenddate":null,"fulltext":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"externalReference":[],"language":null,"instance":[{"license":{"dataInfo":null,"value":"http://journals.sagepub.com/page/policies/text-and-data-mining-license"},"instancetype":{"blank":false,"classid":"0001","classname":"Article","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":null,"distributionlocation":null,"accessright":{"blank":false,"classid":"Restricted","classname":"Restricted","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"url":["http://journals.sagepub.com/doi/pdf/10.1177/0160597615579703","http://journals.sagepub.com/doi/full-xml/10.1177/0160597615579703","http://dx.doi.org/10.1177/0160597615579703"],"dateofacceptance":{"dataInfo":null,"value":"2015-04-13"},"collectedfrom":{"blank":false,"dataInfo":null,"value":"Crossref","key":"10|openaire____::081b82f96300b6a6e3d282bad31cb6e2"},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null},{"license":null,"instancetype":null,"hostedby":null,"distributionlocation":null,"accessright":null,"url":["http://journals.sagepub.com/doi/10.1177/0160597615579703","https://academic.microsoft.com/#/detail/2318726924"],"dateofacceptance":null,"collectedfrom":{"blank":false,"dataInfo":null,"value":"microsoft","key":"10|openaire____::5f532a3fc4f1ea403f37070f59a7a53a"},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"context":[],"country":[],"originalId":["10.1177/0160597615579703","2318726924"],"pid":[{"qualifier":{"blank":false,"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"dataInfo":null,"value":"10.1177/0160597615579703"}],"dateofcollection":"2019-02-14T22:24:27Z","dateoftransformation":null,"extraInfo":[],"oaiprovenance":null,"id":"50|doiboost____::001fc2f1ea9b2bcd3163d68a983127c2","collectedfrom":[{"blank":false,"dataInfo":null,"value":"Crossref","key":"10|openaire____::081b82f96300b6a6e3d282bad31cb6e2"},{"blank":false,"dataInfo":null,"value":"microsoft","key":"10|openaire____::5f532a3fc4f1ea403f37070f59a7a53a"}],"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1550183067815} +,{"journal":{"issnOnline":"1432-122X","vol":"21","issnLinking":null,"edition":null,"issnPrinted":"0170-6012","dataInfo":null,"conferencedate":null,"iss":null,"ep":"88","sp":"84","conferenceplace":null,"name":"Informatik-Spektrum"},"format":[],"source":[{"dataInfo":null,"value":"Crossref"},{"dataInfo":null,"value":null}],"description":[],"subject":[{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Information Systems"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Computer Science Applications"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Software engineering"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.40795562"},"value":"business.industry"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.40795562"},"value":"business"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Computer science"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Von Neumann architecture"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.58637065"},"value":"symbols.namesake"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.58637065"},"value":"symbols"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Programming language"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.4103477"},"value":"computer.software_genre"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.4103477"},"value":"computer"}],"author":[{"surname":"Bauer","affiliation":null,"fullname":"Friedrich L. Bauer","rank":null,"pid":null,"name":"Friedrich L."}],"resulttype":{"blank":false,"classid":"publication","classname":"publication","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"title":[{"qualifier":{"blank":false,"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":null,"value":"Wer erfand den von-Neumann-Rechner?"},{"qualifier":{"blank":false,"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":null,"value":"wer erfand den von neumann rechner"}],"relevantdate":[{"qualifier":{"blank":false,"classid":"created","classname":"created","schemeid":"dnet:dataCite_date","schemename":"dnet:dataCite_date"},"dataInfo":null,"value":"2002-08-25T08:03:26Z"},{"qualifier":{"blank":false,"classid":"published-print","classname":"published-print","schemeid":"dnet:dataCite_date","schemename":"dnet:dataCite_date"},"dataInfo":null,"value":"1998-04-20"}],"dateofacceptance":{"dataInfo":null,"value":"1998-04-20"},"publisher":{"dataInfo":null,"value":"Springer Science and Business Media LLC"},"embargoenddate":null,"fulltext":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"externalReference":[],"language":null,"instance":[{"license":{"dataInfo":null,"value":"http://www.springer.com/tdm"},"instancetype":{"blank":false,"classid":"0001","classname":"Article","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":null,"distributionlocation":null,"accessright":{"blank":false,"classid":"Restricted","classname":"Restricted","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"url":["http://link.springer.com/content/pdf/10.1007/s002870050091.pdf","http://link.springer.com/article/10.1007/s002870050091/fulltext.html","http://link.springer.com/content/pdf/10.1007/s002870050091","http://dx.doi.org/10.1007/s002870050091"],"dateofacceptance":{"dataInfo":null,"value":"1998-04-20"},"collectedfrom":{"blank":false,"dataInfo":null,"value":"Crossref","key":"10|openaire____::081b82f96300b6a6e3d282bad31cb6e2"},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null},{"license":null,"instancetype":null,"hostedby":null,"distributionlocation":null,"accessright":null,"url":["https://link.springer.com/article/10.1007%2Fs002870050091","https://rd.springer.com/article/10.1007/s002870050091","https://academic.microsoft.com/#/detail/146127379"],"dateofacceptance":null,"collectedfrom":{"blank":false,"dataInfo":null,"value":"microsoft","key":"10|openaire____::5f532a3fc4f1ea403f37070f59a7a53a"},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"context":[],"country":[],"originalId":["309TM8JWRUL76A3R","10.1007/s002870050091","146127379"],"pid":[{"qualifier":{"blank":false,"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"dataInfo":null,"value":"10.1007/s002870050091"}],"dateofcollection":"2019-11-18T18:08:53Z","dateoftransformation":null,"extraInfo":[],"oaiprovenance":null,"id":"50|doiboost____::0023d75771ba51300426f53f3d9944ea","collectedfrom":[{"blank":false,"dataInfo":null,"value":"Crossref","key":"10|openaire____::081b82f96300b6a6e3d282bad31cb6e2"},{"blank":false,"dataInfo":null,"value":"microsoft","key":"10|openaire____::5f532a3fc4f1ea403f37070f59a7a53a"}],"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1574100533227} +,{"journal":{"issnOnline":null,"vol":null,"issnLinking":null,"edition":null,"issnPrinted":null,"dataInfo":null,"conferencedate":null,"iss":null,"ep":null,"sp":null,"conferenceplace":null,"name":"Canadian Conference on Electrical and Computer Engineering 2004 (IEEE Cat. No.04CH37513)"},"format":[],"source":[{"dataInfo":null,"value":"Crossref"},{"dataInfo":null,"value":null}],"description":[{"dataInfo":null,"value":"In this paper, we develop an algorithm to cope with the sensor placement problem for target location under constraints of cost limitation and complete coverage. We adopt the grid-based placement scenario that deploys exact one sensor in one grid point at most. A target in a grid point can be positioned by a set of sensors whose transmission radius covers the grid point. The optimal sensor placement for target location is to find a sensor deployment such that targets can be positioned in any grid point of the sensor fields. However, due to the cost limitation, the optimal sensor deployment cannot be achieved frequently. Consequently, the positioning accuracy is the major issue of the problem. The distance error is one of the most natural criteria to measure the positioning accuracy. In this paper, the distance error of two indistinguishable grid points is defined as the Euclidean distance between them. We formulate the sensor placement problem as a combinatorial optimization problem for minimizing the maximum distance error in the sensor field under constraints. The sensor placement problem is NP-complete for arbitrary sensor fields. We present an efficient algorithm that is based on the simulated annealing approach to address the problem. We first compare our algorithm with the brute force approach in the case of smaller sensor fields. The evidence indicates that our algorithm can find the optimal sensor placement under the minimum cost limitation. Moreover, the simulation results also show that the proposed algorithm is very superior in terms of the positioning accuracy even in the case of larger sensor fields."}],"subject":[{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Intelligent sensor"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Algorithm"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Computer science"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Brooks–Iyengar algorithm"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Euclidean distance"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Grid"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Electronic engineering"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Simulated annealing"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Mathematical optimization"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Wireless sensor network"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Software deployment"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Constrained optimization"}],"author":[{"surname":"Chiu","affiliation":null,"fullname":"P.L. Chiu","rank":null,"pid":null,"name":"P.L."},{"surname":"Lin","affiliation":null,"fullname":"F.Y.S. Lin","rank":null,"pid":null,"name":"F.Y.S."}],"resulttype":{"blank":false,"classid":"publication","classname":"publication","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"title":[{"qualifier":{"blank":false,"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":null,"value":"A simulated annealing algorithm to support the sensor placement for target location"}],"relevantdate":[{"qualifier":{"blank":false,"classid":"created","classname":"created","schemeid":"dnet:dataCite_date","schemename":"dnet:dataCite_date"},"dataInfo":null,"value":"2004-11-13T00:14:59Z"}],"dateofacceptance":{"dataInfo":null,"value":"2004-11-13T00:14:59Z"},"publisher":{"dataInfo":null,"value":"IEEE"},"embargoenddate":null,"fulltext":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"externalReference":[],"language":null,"instance":[{"license":null,"instancetype":{"blank":false,"classid":"0004","classname":"Conference object","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":null,"distributionlocation":null,"accessright":{"blank":false,"classid":"Restricted","classname":"Restricted","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"url":["http://xplorestaging.ieee.org/ielx5/9317/29619/01345252.pdf?arnumber=1345252","http://dx.doi.org/10.1109/ccece.2004.1345252"],"dateofacceptance":null,"collectedfrom":{"blank":false,"dataInfo":null,"value":"Crossref","key":"10|openaire____::081b82f96300b6a6e3d282bad31cb6e2"},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null},{"license":null,"instancetype":null,"hostedby":null,"distributionlocation":null,"accessright":null,"url":["https://ieeexplore.ieee.org/document/1345252/","http://ieeexplore.ieee.org/iel5/9317/29619/01345252.pdf?arnumber=1345252","https://academic.microsoft.com/#/detail/2158831329"],"dateofacceptance":null,"collectedfrom":{"blank":false,"dataInfo":null,"value":"microsoft","key":"10|openaire____::5f532a3fc4f1ea403f37070f59a7a53a"},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"context":[],"country":[],"originalId":["10.1109/ccece.2004.1345252","2158831329"],"pid":[{"qualifier":{"blank":false,"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"dataInfo":null,"value":"10.1109/ccece.2004.1345252"}],"dateofcollection":"2019-02-14T15:17:01Z","dateoftransformation":null,"extraInfo":[],"oaiprovenance":null,"id":"50|doiboost____::0029fa2e4d799451deaf3c562ce98d3b","collectedfrom":[{"blank":false,"dataInfo":null,"value":"Crossref","key":"10|openaire____::081b82f96300b6a6e3d282bad31cb6e2"},{"blank":false,"dataInfo":null,"value":"microsoft","key":"10|openaire____::5f532a3fc4f1ea403f37070f59a7a53a"}],"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1550157421738} +,{"journal":{"issnOnline":"1476-3567","vol":"59","issnLinking":null,"edition":null,"issnPrinted":"0365-0340","dataInfo":null,"conferencedate":null,"iss":null,"ep":"1321","sp":"1311","conferenceplace":null,"name":"Archives of Agronomy and Soil Science"},"format":[],"source":[{"dataInfo":null,"value":"Crossref"},{"dataInfo":null,"value":null}],"description":[{"dataInfo":null,"value":"Nutrient management plays an important role in yield and quality of tomatoes. A field experiment was conducted during the years 2008–09 and 2009–10 to analyze the effect of organic and conventional sources of fertilizer on yield and quality of tomatoes in an acid lateritic soil of India. The organic sources of fertilizer were vermicompost (VC), crop residue (CR), vermiwash (VW) and biofertilizer (BF). The conventional input was chemical fertilizer (CF) applied at recommended dose of 100:80:60 kg ha−1 of N:P2O5:K2O for the tomato. The organic source VC was used to supply 100% N recommendation as single source or 50% N recommendation when combined with CF or organic sources. Maximum fruit yield was recorded when CF was applied at full dose, which was on a par with VC at full recommendation. A higher percentage of large-size fruits (>7 cm) was obtained in VC-based treatments compared with CF treatment. Vermicompost at full dose increased ascorbic acid, beta carotene, total soluble solids and color value comp..."}],"subject":[{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Agronomy and Crop Science"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Soil Science"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Agronomy"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Nutrient management"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Agroforestry"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Field experiment"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Biofertilizer"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Fertilizer"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.5152762"},"value":"engineering.material"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.5152762"},"value":"engineering"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Crop residue"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Ascorbic acid"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Vermicompost"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Biology"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Organic fertilizer"}],"author":[{"surname":"Murmu","affiliation":null,"fullname":"Kanu Murmu","rank":null,"pid":null,"name":"Kanu"},{"surname":"Ghosh","affiliation":null,"fullname":"Bijoy Chandra Ghosh","rank":null,"pid":null,"name":"Bijoy Chandra"},{"surname":"Swain","affiliation":null,"fullname":"Dillip Kumar Swain","rank":null,"pid":null,"name":"Dillip Kumar"}],"resulttype":{"blank":false,"classid":"publication","classname":"publication","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"title":[{"qualifier":{"blank":false,"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":null,"value":"Yield and quality of tomato grown under organic and conventional nutrient management"}],"relevantdate":[{"qualifier":{"blank":false,"classid":"created","classname":"created","schemeid":"dnet:dataCite_date","schemename":"dnet:dataCite_date"},"dataInfo":null,"value":"2012-08-14T07:07:44Z"}],"dateofacceptance":{"dataInfo":null,"value":"2012-08-14T07:07:44Z"},"publisher":{"dataInfo":null,"value":"Informa UK Limited"},"embargoenddate":null,"fulltext":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"externalReference":[],"language":null,"instance":[{"license":null,"instancetype":{"blank":false,"classid":"0001","classname":"Article","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":null,"distributionlocation":null,"accessright":{"blank":false,"classid":"Restricted","classname":"Restricted","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"url":["http://www.tandfonline.com/doi/pdf/10.1080/03650340.2012.711472","http://dx.doi.org/10.1080/03650340.2012.711472"],"dateofacceptance":null,"collectedfrom":{"blank":false,"dataInfo":null,"value":"Crossref","key":"10|openaire____::081b82f96300b6a6e3d282bad31cb6e2"},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null},{"license":null,"instancetype":null,"hostedby":null,"distributionlocation":null,"accessright":null,"url":["https://www.tandfonline.com/doi/abs/10.1080/03650340.2012.711472","https://academic.microsoft.com/#/detail/2031565734"],"dateofacceptance":null,"collectedfrom":{"blank":false,"dataInfo":null,"value":"microsoft","key":"10|openaire____::5f532a3fc4f1ea403f37070f59a7a53a"},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"context":[],"country":[],"originalId":["10.1080/03650340.2012.711472","2031565734"],"pid":[{"qualifier":{"blank":false,"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"dataInfo":null,"value":"10.1080/03650340.2012.711472"}],"dateofcollection":"2019-07-02T21:10:18Z","dateoftransformation":null,"extraInfo":[],"oaiprovenance":null,"id":"50|doiboost____::002ad37d986a5e92b346e8da92096ad5","collectedfrom":[{"blank":false,"dataInfo":null,"value":"Crossref","key":"10|openaire____::081b82f96300b6a6e3d282bad31cb6e2"},{"blank":false,"dataInfo":null,"value":"ORCID","key":"10|openaire____::806360c771262b4d6770e7cdf04b5c5a"},{"blank":false,"dataInfo":null,"value":"microsoft","key":"10|openaire____::5f532a3fc4f1ea403f37070f59a7a53a"}],"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1562101818154} +,{"journal":{"issnOnline":"2044-5377","vol":"94-B","issnLinking":null,"edition":null,"issnPrinted":"0301-620X","dataInfo":null,"conferencedate":null,"iss":null,"ep":"847","sp":"842","conferenceplace":null,"name":"The Journal of Bone and Joint Surgery. British volume"},"format":[],"source":[{"dataInfo":null,"value":"Crossref"},{"dataInfo":null,"value":null}],"description":[{"dataInfo":null,"value":"We present our early experience of arthroscopic reduction of the dislocated hip in very young infants with developmental dysplasia of the hip (DDH).\n\nEight dislocated hips, which had failed attempts at closed reduction, were treated by arthroscopy of the hip in five children with a mean age of 5.8 months (4 to 7). A two-portal technique was used, with a medial sub-adductor portal for a 2.7 mm cannulated system with a 70° arthroscope and an anterolateral portal for the instruments. Following evaluation of the key intra-articular structures, the hypertrophic ligamentum teres and acetabular pulvinar were resected, and a limited release of the capsule was performed prior to reduction of the hip. All hips were reduced by a single arthroscopic procedure, the reduction being confirmed on MRI scan. None of the hips had an inverted labrum. The greatest obstacle to reduction was a constriction of the capsule. At a mean follow-up of 13.2 months (9 to 24), all eight hips remained stable. Three developed avascular necrosis. The mean acetabular index decreased from 35.5° (30° to 40°) pre-operatively to 23.3° (17° to 28°).\n\nThis study demonstrates that arthroscopic reduction is feasible using two standardised portals. Longer follow-up studies are necessary to evaluate the functional results."}],"subject":[{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Surgery"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Orthopedics and Sports Medicine"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.43935013"},"value":"medicine.medical_specialty"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.43935013"},"value":"medicine"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Avascular necrosis"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.5198782"},"value":"medicine.disease"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Hip arthroscopy"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Capsule"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Arthroscopy"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.528666"},"value":"medicine.diagnostic_test"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Constriction"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Young infants"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Dysplasia"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.35628062"},"value":"business.industry"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.35628062"},"value":"business"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Labrum"}],"author":[{"surname":"Eberhardt","affiliation":null,"fullname":"O. Eberhardt","rank":null,"pid":null,"name":"O."},{"surname":"Fernandez","affiliation":null,"fullname":"F. F. Fernandez","rank":null,"pid":null,"name":"F. F."},{"surname":"Wirth","affiliation":null,"fullname":"T. Wirth","rank":null,"pid":null,"name":"T."}],"resulttype":{"blank":false,"classid":"publication","classname":"publication","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"title":[{"qualifier":{"blank":false,"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":null,"value":"Arthroscopic reduction of the dislocated hip in infants"}],"relevantdate":[{"qualifier":{"blank":false,"classid":"created","classname":"created","schemeid":"dnet:dataCite_date","schemename":"dnet:dataCite_date"},"dataInfo":null,"value":"2012-05-24T15:25:59Z"}],"dateofacceptance":{"dataInfo":null,"value":"2012-05-24T15:25:59Z"},"publisher":{"dataInfo":null,"value":"British Editorial Society of Bone & Joint Surgery"},"embargoenddate":null,"fulltext":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"externalReference":[],"language":null,"instance":[{"license":null,"instancetype":{"blank":false,"classid":"0001","classname":"Article","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":null,"distributionlocation":null,"accessright":{"blank":false,"classid":"Restricted","classname":"Restricted","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"url":["https://syndication.highwire.org/content/doi/10.1302/0301-620X.94B6.28161","http://dx.doi.org/10.1302/0301-620x.94b6.28161"],"dateofacceptance":null,"collectedfrom":{"blank":false,"dataInfo":null,"value":"Crossref","key":"10|openaire____::081b82f96300b6a6e3d282bad31cb6e2"},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null},{"license":null,"instancetype":null,"hostedby":null,"distributionlocation":null,"accessright":null,"url":["https://www.ncbi.nlm.nih.gov/pubmed/22628603","http://online.boneandjoint.org.uk/doi/10.1302/0301-620X.94B6.28161","https://academic.microsoft.com/#/detail/2016293216"],"dateofacceptance":null,"collectedfrom":{"blank":false,"dataInfo":null,"value":"microsoft","key":"10|openaire____::5f532a3fc4f1ea403f37070f59a7a53a"},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"context":[],"country":[],"originalId":["10.1302/0301-620X.94B6.28161","10.1302/0301-620x.94b6.28161","2016293216"],"pid":[{"qualifier":{"blank":false,"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"dataInfo":null,"value":"10.1302/0301-620x.94b6.28161"}],"dateofcollection":"2019-09-05T05:23:31Z","dateoftransformation":null,"extraInfo":[],"oaiprovenance":null,"id":"50|doiboost____::0040a9baf091fd13ed2deccf705271dd","collectedfrom":[{"blank":false,"dataInfo":null,"value":"Crossref","key":"10|openaire____::081b82f96300b6a6e3d282bad31cb6e2"},{"blank":false,"dataInfo":null,"value":"microsoft","key":"10|openaire____::5f532a3fc4f1ea403f37070f59a7a53a"}],"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1567661011266} +,{"journal":{"issnOnline":"1748-880X","vol":"77","issnLinking":null,"edition":null,"issnPrinted":"0007-1285","dataInfo":null,"conferencedate":null,"iss":null,"ep":"S175","sp":"S167","conferenceplace":null,"name":"The British Journal of Radiology"},"format":[],"source":[{"dataInfo":null,"value":"Crossref"},{"dataInfo":null,"value":null}],"description":[{"dataInfo":null,"value":"fMRI (functional magnetic resonance imaging) is a powerful non-invasive tool in the study of the function of the brain, used, for example, by psychologists, psychiatrists and neurologists. fMRI can give high quality visualization of the location of activity in the brain resulting from sensory stimulation or cognitive function. It therefore allows the study of how the healthy brain functions, how it is affected by different diseases, how it attempts to recover after damage and how drugs can modulate activity or post-damage recovery. After an fMRI experiment has been designed and carried out, the resulting data must be passed through various analysis steps before the experimenter can get answers to questions about experimentally related activations at the individual or multi-subject level. This paper gives a brief overview of the most commonly used analysis pipeline: data pre-processing, temporal linear modelling and activation thresholding. For more details, see Jezzard et al [1]. fMRI Data In a typical fMRI session a low-resolution functional volume is acquired every few seconds. (MR volumes are often also referred to as ‘‘images’’ or ‘‘scans’’). Over the course of the experiment, 100 volumes or more are typically recorded. In the simplest possible experiment, some images will be taken whilst stimulation (for the remainder of this chapter, reference to ‘‘stimulation’’ should be taken to include also the carrying out of physical or cognitive activity) is applied, and some will be taken with the subject at rest. Because the images are taken using an MR sequence which is sensitive to changes in local blood oxygenation level (BOLD imaging; see Chapters 2 and 3 in Jezzard et al [1]), parts of the images taken during stimulation should show increased intensity, compared with those taken whilst at rest. The parts of these images that show increased intensity should correspond to the brain areas which are activated by the stimulation. The goal of fMRI analysis is to detect, in a robust, sensitive and valid way, those parts of the brain that show increased intensity at the points in time that stimulation was applied. A single volume is made up of individual cuboid elements called voxels (Figure 1). An fMRI data set from a single session can either be thought of as t volumes, one taken every few seconds, or as v voxels, each with an associated time series of t time points. It is important to be able to conceptualize both of these representations, as some analysis steps make more sense when thinking of the data in one way, and others make more sense the other way. An example time-series from a single voxel is shown in Figure 2. Image intensity is shown on the y axis, and time (in scans) on the x axis. As described above, for some of the time points, stimulation was applied, (the higher intensity periods), and at some time points the subject was at rest. As well as the effect of the stimulation being clear, the high frequency noise is also apparent. The aim of fMRI analysis is to identify in which voxels’ time-series the signal of interest is significantly greater than the noise level."}],"subject":[{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Radiology"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.32073238"},"value":"medicine.medical_specialty"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.32073238"},"value":"medicine"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Voxel"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.51576644"},"value":"computer.software_genre"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.51576644"},"value":"computer"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Nuclear medicine"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.32511315"},"value":"business.industry"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.32511315"},"value":"business"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Sensory stimulation therapy"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Visualization"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Cognition"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Functional magnetic resonance imaging"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.5011593"},"value":"medicine.diagnostic_test"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Artificial intelligence"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Pattern recognition"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Stimulation"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Cuboid"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Thresholding"}],"author":[{"surname":"Smith","affiliation":null,"fullname":"S M Smith","rank":null,"pid":null,"name":"S M"}],"resulttype":{"blank":false,"classid":"publication","classname":"publication","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"title":[{"qualifier":{"blank":false,"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":null,"value":"Overview of fMRI analysis"},{"qualifier":{"blank":false,"classid":"alternative title","classname":"alternative title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":null,"value":"Overview of fMRI analysis."}],"relevantdate":[{"qualifier":{"blank":false,"classid":"created","classname":"created","schemeid":"dnet:dataCite_date","schemename":"dnet:dataCite_date"},"dataInfo":null,"value":"2005-01-27T19:53:02Z"}],"dateofacceptance":{"dataInfo":null,"value":"2005-01-27T19:53:02Z"},"publisher":{"dataInfo":null,"value":"British Institute of Radiology"},"embargoenddate":null,"fulltext":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"externalReference":[],"language":null,"instance":[{"license":null,"instancetype":{"blank":false,"classid":"0001","classname":"Article","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":null,"distributionlocation":null,"accessright":{"blank":false,"classid":"Restricted","classname":"Restricted","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"url":["http://www.birpublications.org/doi/pdf/10.1259/bjr/33553595","http://dx.doi.org/10.1259/bjr/33553595"],"dateofacceptance":null,"collectedfrom":{"blank":false,"dataInfo":null,"value":"Crossref","key":"10|openaire____::081b82f96300b6a6e3d282bad31cb6e2"},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null},{"license":null,"instancetype":null,"hostedby":null,"distributionlocation":null,"accessright":null,"url":["https://cfn.upenn.edu/stslab/wiki/lib/exe/fetch.php/fmri_club:preprocess1:smith_2004_brjrad.pdf","https://www.birpublications.org/doi/abs/10.1259/bjr/33553595","https://www.ncbi.nlm.nih.gov/pubmed/15677358","https://www.neuroscience.ox.ac.uk/publications/116696","https://www.ndcn.ox.ac.uk/publications/116696","http://www.birpublications.org/doi/figure/10.1259/bjr/33553595","https://academic.microsoft.com/#/detail/2033777218"],"dateofacceptance":null,"collectedfrom":{"blank":false,"dataInfo":null,"value":"microsoft","key":"10|openaire____::5f532a3fc4f1ea403f37070f59a7a53a"},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"context":[],"country":[],"originalId":["10.1259/bjr/33553595","2033777218"],"pid":[{"qualifier":{"blank":false,"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"dataInfo":null,"value":"10.1259/bjr/33553595"}],"dateofcollection":"2019-12-20T11:23:26Z","dateoftransformation":null,"extraInfo":[],"oaiprovenance":null,"id":"50|doiboost____::0041552fcee851b1e7dfe74119ee44f0","collectedfrom":[{"blank":false,"dataInfo":null,"value":"Crossref","key":"10|openaire____::081b82f96300b6a6e3d282bad31cb6e2"},{"blank":false,"dataInfo":null,"value":"ORCID","key":"10|openaire____::806360c771262b4d6770e7cdf04b5c5a"},{"blank":false,"dataInfo":null,"value":"microsoft","key":"10|openaire____::5f532a3fc4f1ea403f37070f59a7a53a"}],"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1576841006413} +,{"journal":{"issnOnline":"1573-4803","vol":"29","issnLinking":null,"edition":null,"issnPrinted":"0022-2461","dataInfo":null,"conferencedate":null,"iss":null,"ep":"1718","sp":"1714","conferenceplace":null,"name":"Journal of Materials Science"},"format":[],"source":[{"dataInfo":null,"value":"Crossref"},{"dataInfo":null,"value":null}],"description":[{"dataInfo":null,"value":"Chemical and structural properties of the system (1−x)Fe2O3 + xNd2O3, 0≤x≤1, were investigated using X-ray diffraction, 57Fe Mossbauer spectroscopy and Fourier transform-infrared (FT-IR) spectroscopy. The samples were prepared by the chemical coprecipitation and thermal treatment of Fe(OH)3/Nd(OH)3 coprecipitates. X-ray diffraction showed the presence of oxide phases α-Fe2O3 + NdFeO3 in the Fe2O3-rich region, and the oxide phases Nd2O3 + NdFeO3 in the Nd2O3-rich region. 57Fe Mossbauer spectra were characterized with one sextet of spectral lines at room temperature. Mathematical evaluation of the Mossbauer spectra showed distinct regularities in the changes of Mossbauer parameters, thus indicating the presence of two subspectra with very similar spectral behaviour. High sensitivity of the Nd2O3 phase to the moisture and atmosphere CO2 was demonstrated by FT-IR spectroscopy."}],"subject":[{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Mechanical Engineering"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"General Materials Science"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Mechanics of Materials"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Coprecipitation"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Materials science"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Spectral line"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Mössbauer spectroscopy"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"X-ray crystallography"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Solid solution"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Infrared spectroscopy"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Oxide"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.50058734"},"value":"chemistry.chemical_compound"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.50058734"},"value":"chemistry"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Spectroscopy"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Analytical chemistry"}],"author":[{"surname":"Musić","affiliation":null,"fullname":"S. Musić","rank":null,"pid":null,"name":"S."},{"surname":"Popović","affiliation":null,"fullname":"S. Popović","rank":null,"pid":null,"name":"S."},{"surname":"Ristić","affiliation":null,"fullname":"M. Ristić","rank":null,"pid":null,"name":"M."},{"surname":"Sepiol","affiliation":null,"fullname":"B. Sepiol","rank":null,"pid":null,"name":"B."}],"resulttype":{"blank":false,"classid":"publication","classname":"publication","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"title":[{"qualifier":{"blank":false,"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":null,"value":"Chemical and structural properties of the system Fe2O3-Nd2O3"},{"qualifier":{"blank":false,"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":null,"value":"chemical and structural properties of the system fe2o3 in2o3"},{"qualifier":{"blank":false,"classid":"alternative title","classname":"alternative title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":null,"value":"Chemical and structural properties of the system Fe2O3-In2O3"}],"relevantdate":[{"qualifier":{"blank":false,"classid":"created","classname":"created","schemeid":"dnet:dataCite_date","schemename":"dnet:dataCite_date"},"dataInfo":null,"value":"2004-10-30T20:17:56Z"},{"qualifier":{"blank":false,"classid":"published-online","classname":"published-online","schemeid":"dnet:dataCite_date","schemename":"dnet:dataCite_date"},"dataInfo":null,"value":"2004-09-12"}],"dateofacceptance":{"dataInfo":null,"value":"2004-10-30T20:17:56Z"},"publisher":{"dataInfo":null,"value":"Springer Science and Business Media LLC"},"embargoenddate":null,"fulltext":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"externalReference":[],"language":null,"instance":[{"license":{"dataInfo":null,"value":"http://www.springer.com/tdm"},"instancetype":{"blank":false,"classid":"0001","classname":"Article","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":null,"distributionlocation":null,"accessright":{"blank":false,"classid":"Restricted","classname":"Restricted","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"url":["http://link.springer.com/content/pdf/10.1007/BF00351287.pdf","http://link.springer.com/article/10.1007/BF00351287/fulltext.html","http://link.springer.com/content/pdf/10.1007/BF00351287","http://dx.doi.org/10.1007/bf00351287"],"dateofacceptance":null,"collectedfrom":{"blank":false,"dataInfo":null,"value":"Crossref","key":"10|openaire____::081b82f96300b6a6e3d282bad31cb6e2"},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null},{"license":null,"instancetype":null,"hostedby":null,"distributionlocation":null,"accessright":null,"url":["https://ui.adsabs.harvard.edu/abs/1993JMatS..28..632M/abstract","https://link.springer.com/article/10.1007/BF02402973","https://rd.springer.com/article/10.1007/BF00351287","https://academic.microsoft.com/#/detail/2039099087"],"dateofacceptance":null,"collectedfrom":{"blank":false,"dataInfo":null,"value":"microsoft","key":"10|openaire____::5f532a3fc4f1ea403f37070f59a7a53a"},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"context":[],"country":[],"originalId":["BF00351287","10.1007/bf00351287","2039099087"],"pid":[{"qualifier":{"blank":false,"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"dataInfo":null,"value":"10.1007/bf00351287"}],"dateofcollection":"2019-09-13T05:40:02Z","dateoftransformation":null,"extraInfo":[],"oaiprovenance":null,"id":"50|doiboost____::0042dc76d02068f81d6a764a71d146df","collectedfrom":[{"blank":false,"dataInfo":null,"value":"Crossref","key":"10|openaire____::081b82f96300b6a6e3d282bad31cb6e2"},{"blank":false,"dataInfo":null,"value":"ORCID","key":"10|openaire____::806360c771262b4d6770e7cdf04b5c5a"},{"blank":false,"dataInfo":null,"value":"microsoft","key":"10|openaire____::5f532a3fc4f1ea403f37070f59a7a53a"}],"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1568353202504} +,{"journal":null,"format":[],"source":[{"dataInfo":null,"value":"Crossref"},{"dataInfo":null,"value":"Unfettering Poetry ISBN: 9781349528738"},{"dataInfo":null,"value":null}],"description":[{"dataInfo":null,"value":"Philosophers, theorists, reviewers, and literary critics from the French Revolution to the present have sought to devise strategies to displace or disarm the power of the Fancy. This skepticism, or what I have called “fanciphobia,” adopts or takes several forms: (1) The Fancy is trivialized (associated with escape and unreality). (2) The Fancy is subordinated to the Imagination as a poetic faculty (i.e., it is honored but kept under control). (3) It is associated with the carefree nature of childhood or with an apparently simpler historical past (and thus it becomes irrelevant to the present). (4) After childhood and adolescence, the Fancy becomes the object of a “mature” person’s skepticism about its viability in the real world of getting and spending. Skepticism toward the Fancy’s dominance in mental life expresses the successful maturation of the adult. (5) It is considered a gatherer of “raw materials” that are to be shaped into something permanent and beautiful by the Imagination."}],"subject":[{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Literary criticism"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Aesthetics"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Skepticism"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.601421"},"value":"media_common.quotation_subject"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.601421"},"value":"media_common"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"French revolution"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Mental life"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Art"},{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Poetry"}],"author":[{"surname":"Robinson","affiliation":null,"fullname":"Jeffrey C. Robinson","rank":null,"pid":null,"name":"Jeffrey C."}],"resulttype":{"blank":false,"classid":"publication","classname":"publication","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"title":[{"qualifier":{"blank":false,"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":null,"value":"Fanciphobia: A History of Skepticism about the Fancy"},{"qualifier":{"blank":false,"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":null,"value":"fanciphobia a history of skepticism about the fancy"}],"relevantdate":[{"qualifier":{"blank":false,"classid":"created","classname":"created","schemeid":"dnet:dataCite_date","schemename":"dnet:dataCite_date"},"dataInfo":null,"value":"2016-05-21T05:28:48Z"}],"dateofacceptance":{"dataInfo":null,"value":"2016-05-21T05:28:48Z"},"publisher":{"dataInfo":null,"value":"Palgrave Macmillan US"},"embargoenddate":null,"fulltext":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"externalReference":[],"language":null,"instance":[{"license":{"dataInfo":null,"value":"http://www.springer.com/tdm"},"instancetype":{"blank":false,"classid":"0013","classname":"Part of book or chapter of book","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":null,"distributionlocation":null,"accessright":{"blank":false,"classid":"Restricted","classname":"Restricted","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"url":["http://link.springer.com/content/pdf/10.1057/9781403982834_2","http://dx.doi.org/10.1057/9781403982834_2"],"dateofacceptance":null,"collectedfrom":{"blank":false,"dataInfo":null,"value":"Crossref","key":"10|openaire____::081b82f96300b6a6e3d282bad31cb6e2"},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null},{"license":null,"instancetype":null,"hostedby":null,"distributionlocation":null,"accessright":null,"url":["https://link.springer.com/chapter/10.1057/9781403982834_2","https://academic.microsoft.com/#/detail/2501403633"],"dateofacceptance":null,"collectedfrom":{"blank":false,"dataInfo":null,"value":"microsoft","key":"10|openaire____::5f532a3fc4f1ea403f37070f59a7a53a"},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"context":[],"country":[],"originalId":["10.1057/9781403982834_2","2501403633"],"pid":[{"qualifier":{"blank":false,"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"dataInfo":null,"value":"10.1057/9781403982834_2"}],"dateofcollection":"2019-05-08T15:42:45Z","dateoftransformation":null,"extraInfo":[],"oaiprovenance":null,"id":"50|doiboost____::0046ca097e625d3c0dea887b63aa5105","collectedfrom":[{"blank":false,"dataInfo":null,"value":"Crossref","key":"10|openaire____::081b82f96300b6a6e3d282bad31cb6e2"},{"blank":false,"dataInfo":null,"value":"microsoft","key":"10|openaire____::5f532a3fc4f1ea403f37070f59a7a53a"}],"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1557330165026} +,{"journal":{"issnOnline":"2005-8330","vol":"19","issnLinking":null,"edition":null,"issnPrinted":"1229-6929","dataInfo":null,"conferencedate":null,"iss":null,"ep":null,"sp":"368","conferenceplace":null,"name":"Korean Journal of Radiology"},"format":[],"source":[{"dataInfo":null,"value":"Crossref"}],"description":[],"subject":[{"qualifier":{"blank":false,"classid":"keywords","classname":"keywords","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":null,"value":"Radiology Nuclear Medicine and imaging"}],"author":[{"surname":"Chae","affiliation":null,"fullname":"Kum Ju Chae","rank":null,"pid":null,"name":"Kum Ju"},{"surname":"Goo","affiliation":null,"fullname":"Jin Mo Goo","rank":null,"pid":null,"name":"Jin Mo"},{"surname":"Ahn","affiliation":null,"fullname":"Su Yeon Ahn","rank":null,"pid":null,"name":"Su Yeon"},{"surname":"Yoo","affiliation":null,"fullname":"Jin Young Yoo","rank":null,"pid":null,"name":"Jin Young"},{"surname":"Yoon","affiliation":null,"fullname":"Soon Ho Yoon","rank":null,"pid":null,"name":"Soon Ho"}],"resulttype":{"blank":false,"classid":"publication","classname":"publication","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"title":[{"qualifier":{"blank":false,"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":null,"value":"Erratum: Application of Deconvolution Algorithm of Point Spread Function in Improving Image Quality: An Observer Preference Study on Chest Radiography"}],"relevantdate":[{"qualifier":{"blank":false,"classid":"created","classname":"created","schemeid":"dnet:dataCite_date","schemename":"dnet:dataCite_date"},"dataInfo":null,"value":"2018-03-02T12:04:55Z"}],"dateofacceptance":{"dataInfo":null,"value":"2018-03-02T12:04:55Z"},"publisher":{"dataInfo":null,"value":"The Korean Society of Radiology (KAMJE)"},"embargoenddate":null,"fulltext":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"externalReference":[],"language":null,"instance":[{"license":{"dataInfo":null,"value":"http://creativecommons.org/licenses/by-nc/4.0/"},"instancetype":{"blank":false,"classid":"0001","classname":"Article","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":null,"distributionlocation":null,"accessright":{"blank":false,"classid":"Restricted","classname":"Restricted","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"url":["https://synapse.koreamed.org/pdf/10.3348/kjr.2018.19.2.368","https://synapse.koreamed.org/DOIx.php?id=10.3348/kjr.2018.19.2.368","http://dx.doi.org/10.3348/kjr.2018.19.2.368"],"dateofacceptance":null,"collectedfrom":{"blank":false,"dataInfo":null,"value":"Crossref","key":"10|openaire____::081b82f96300b6a6e3d282bad31cb6e2"},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null},{"license":{"dataInfo":null,"value":"cc-by-nc"},"instancetype":null,"hostedby":null,"distributionlocation":null,"accessright":{"blank":false,"classid":"Open","classname":"Open","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"url":["http://europepmc.org/articles/pmc5840068?pdf=render"],"dateofacceptance":null,"collectedfrom":{"blank":false,"dataInfo":null,"value":"UnpayWall","key":"10|openaire____::8ac8380272269217cb09a928c8caa993"},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"context":[],"country":[],"originalId":["10.3348/kjr.2018.19.2.368"],"pid":[{"qualifier":{"blank":false,"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"dataInfo":null,"value":"10.3348/kjr.2018.19.2.368"}],"dateofcollection":"2019-11-26T21:53:29Z","dateoftransformation":null,"extraInfo":[],"oaiprovenance":null,"id":"50|doiboost____::00474983edc459df6a662f3970b8430b","collectedfrom":[{"blank":false,"dataInfo":null,"value":"Crossref","key":"10|openaire____::081b82f96300b6a6e3d282bad31cb6e2"},{"blank":false,"dataInfo":null,"value":"UnpayWall","key":"10|openaire____::8ac8380272269217cb09a928c8caa993"}],"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"inferenceprovenance":null,"provenanceaction":{"blank":false,"classid":"sysimport:actionset","classname":"sysimport:actionset","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1574805209002}] \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/dhp/doiboost/publication_merge b/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/dhp/doiboost/publication_merge new file mode 100644 index 0000000000..f47449f8d2 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/dhp/doiboost/publication_merge @@ -0,0 +1,3 @@ +{"context": [], "dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "sysimport:crosswalk:datasetarchive", "classname": "sysimport:crosswalk:datasetarchive", "schemename": "dnet:provenanceActions", "schemeid": "dnet:provenanceActions"}, "inferred": false, "inferenceprovenance": "dedup-similarity-result-levenstein", "invisible": false, "trust": "0.95"}, "resourcetype": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "pid": [], "contributor": [], "resulttype": {"classid": "publication", "classname": "publication", "schemename": "dnet:result_typologies", "schemeid": "dnet:result_typologies"}, "relevantdate": [], "collectedfrom": [{"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "Journal.fi", "key": "10|openaire____::6eef8049d0feedc089ee009abca55e35"}], "id": "50|a89337edbe55::4930db9e954866d70916cbfba9f81f97", "subject": [], "instance": [{"refereed": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "hostedby": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "Journal.fi", "key": "10|openaire____::6eef8049d0feedc089ee009abca55e35"}, "processingchargeamount": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "license": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "url": [], "distributionlocation": "", "processingchargecurrency": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "dateofacceptance": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "2016-01-01"}, "collectedfrom": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "Journal.fi", "key": "10|openaire____::6eef8049d0feedc089ee009abca55e35"}, "accessright": {"classid": "OPEN", "classname": "Open Access", "schemename": "dnet:access_modes", "schemeid": "dnet:access_modes"}, "instancetype": {"classid": "0001", "classname": "Article", "schemename": "dnet:dataCite_resource", "schemeid": "dnet:dataCite_resource"}}], "embargoenddate": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "lastupdatetimestamp": 0, "author": [{"surname": "Go\\u0308tz", "name": "Julia", "pid": [], "rank": 5, "affiliation": [], "fullname": "G\\u00f6tz, Julia"}, {"surname": "Wolff", "name": "Stephan", "pid": [], "rank": 6, "affiliation": [], "fullname": "Wolff, Stephan"}, {"surname": "Jansen", "name": "Olav", "pid": [], "rank": 7, "affiliation": [], "fullname": "Jansen, Olav"}, {"surname": "Dressler", "name": "Dirk", "pid": [{"qualifier": {"classid": "ORCID", "classname": "ORCID"}, "value": "0000-0000-0656-9999"},{"qualifier": {"classid": "id", "classname": "id"}, "value": "987654321"}], "rank": 8, "affiliation": [], "fullname": "Dressler, Dirk"}, {"surname": "Schneider", "name": "Susanne A.", "pid": [], "rank": 9, "affiliation": [], "fullname": "Schneider, Susanne A."}], "source": [], "dateofcollection": "2019-11-05T14:49:22.351Z", "fulltext": [], "dateoftransformation": "2019-11-05T16:10:58.988Z", "description": [], "format": [], "journal": {"issnPrinted": "1459-6067", "conferencedate": "", "conferenceplace": "", "name": "Agricultural and Food Science", "edition": "", "iss": "3", "sp": "", "vol": "27", "dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "issnOnline": "1795-1895", "ep": "", "issnLinking": ""}, "coverage": [], "publisher": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "language": {"classid": "eng", "classname": "English", "schemename": "dnet:languages", "schemeid": "dnet:languages"}, "bestaccessright": {"classid": "OPEN", "classname": "Open Access", "schemename": "dnet:access_modes", "schemeid": "dnet:access_modes"}, "country": [], "extraInfo": [], "originalId": [], "dateofacceptance": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "2018-09-30"}, "title": [{"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "qualifier": {"classid": "main title", "classname": "main title", "schemename": "dnet:dataCite_title", "schemeid": "dnet:dataCite_title"}, "value": null}]} +{"context": [], "dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "sysimport:crosswalk:repository", "classname": "sysimport:crosswalk:repository", "schemename": "dnet:provenanceActions", "schemeid": "dnet:provenanceActions"}, "inferred": true, "inferenceprovenance": "dedup-similarity-result-levenstein", "invisible": false, "trust": "0.9"}, "resourcetype": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "pid": [{"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "qualifier": {"classid": "doi", "classname": "doi", "schemename": "dnet:pid_types", "schemeid": "dnet:pid_types"}, "value": "10.1016/j.nicl.2015.11.006"}], "contributor": [], "resulttype": {"classid": "publication", "classname": "publication", "schemename": "dnet:result_typologies", "schemeid": "dnet:result_typologies"}, "relevantdate": [], "collectedfrom": [{"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "BASE (Open Access Aggregator)", "key": "10|openaire____::df45502607927471ecf8a6ae83683ff5"}], "id": "50|a89337edbe55::4930db9e954866d70916cbfba9f81f97", "subject": [], "instance": [{"refereed": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "hostedby": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "NeuroImage: Clinical", "key": "10|doajarticles::0c0e74daa5d95504eade9c81ebbd5b8a"}, "processingchargeamount": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "license": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "http://creativecommons.org/licenses/by-nc-nd/4.0/"}, "url": ["http://dx.doi.org/10.1016/j.nicl.2015.11.006"], "distributionlocation": "", "processingchargecurrency": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "dateofacceptance": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "2016-01-01"}, "collectedfrom": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "BASE (Open Access Aggregator)", "key": "10|openaire____::df45502607927471ecf8a6ae83683ff5"}, "accessright": {"classid": "OPEN", "classname": "Open Access", "schemename": "dnet:access_modes", "schemeid": "dnet:access_modes"}, "instancetype": {"classid": "0001", "classname": "Article", "schemename": "dnet:publication_resource", "schemeid": "dnet:publication_resource"}}], "embargoenddate": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "lastupdatetimestamp": 0, "author": [{"surname": "Zeuner", "name": "Kirsten E.", "pid": [], "rank": 1, "affiliation": [], "fullname": "Zeuner, Kirsten E."}, {"surname": "Knutzen", "name": "Arne", "pid": [], "rank": 2, "affiliation": [], "fullname": "Knutzen, Arne"}, {"surname": "Granert", "name": "Oliver", "pid": [{"qualifier": {"classid": "ORCID", "classname": "ORCID"}, "value": "0000-0002-0656-1023"}], "rank": 3, "affiliation": [], "fullname": "Granert, Oliver"}, {"surname": "Sablowsky", "name": "Simone", "pid": [], "rank": 4, "affiliation": [], "fullname": "Sablowsky, Simone"}, {"surname": "Go\\u0308tz", "name": "Julia", "pid": [], "rank": 5, "affiliation": [], "fullname": "G\\u00f6tz, Julia"}, {"surname": "Wolff", "name": "Stephan", "pid": [], "rank": 6, "affiliation": [], "fullname": "Wolff, Stephan"}, {"surname": "Jansen", "name": "Olav", "pid": [], "rank": 7, "affiliation": [], "fullname": "Jansen, Olav"}, {"surname": "Dressler", "name": "Dirk", "pid": [], "rank": 8, "affiliation": [], "fullname": "Dressler, Dirk"}, {"surname": "Schneider", "name": "Susanne A.", "pid": [], "rank": 9, "affiliation": [], "fullname": "Schneider, Susanne A."}, {"surname": "Klein", "name": "Christine", "pid": [], "rank": 10, "affiliation": [], "fullname": "Klein, Christine"}, {"surname": "Deuschl", "name": "Gu\\u0308nther", "pid": [], "rank": 11, "affiliation": [], "fullname": "Deuschl, G\\u00fcnther"}, {"surname": "Eimeren", "name": "Thilo", "pid": [], "rank": 12, "affiliation": [], "fullname": "van Eimeren, Thilo"}, {"surname": "Witt", "name": "Karsten", "pid": [], "rank": 13, "affiliation": [], "fullname": "Witt, Karsten"}], "source": [], "dateofcollection": "2017-07-27T19:04:09.131Z", "fulltext": [], "dateoftransformation": "2019-01-23T10:15:19.582Z", "description": [], "format": [], "journal": {"issnPrinted": "2213-1582", "conferencedate": "", "conferenceplace": "", "name": "NeuroImage: Clinical", "edition": "", "iss": "", "sp": "63", "vol": "10", "dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "issnOnline": "", "ep": "70", "issnLinking": ""}, "coverage": [], "publisher": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "Elsevier BV"}, "language": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "bestaccessright": {"classid": "OPEN", "classname": "Open Access", "schemename": "dnet:access_modes", "schemeid": "dnet:access_modes"}, "country": [{"classid": "IT", "classname": "Italy", "schemeid": "dnet:countries", "schemename": "dnet:countries"}], "extraInfo": [], "originalId": ["10.1016/j.nicl.2015.11.006"], "dateofacceptance": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "2016-01-01"}, "title": [{"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "qualifier": {"classid": "main title", "classname": "main title", "schemename": "dnet:dataCite_title", "schemeid": "dnet:dataCite_title"}, "value": null}]} +{"context": [], "dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "sysimport:crosswalk:datasetarchive", "classname": "sysimport:crosswalk:datasetarchive", "schemename": "dnet:provenanceActions", "schemeid": "dnet:provenanceActions"}, "inferred": true, "inferenceprovenance": "dedup-similarity-result-levenstein", "invisible": false, "trust": "0.9"}, "resourcetype": {"classid": "0004", "classname": "Conference object", "schemename": "dnet:dataCite_resource", "schemeid": "dnet:dataCite_resource"}, "pid": [], "contributor": [], "resulttype": {"classid": "publication", "classname": "publication", "schemename": "dnet:result_typologies", "schemeid": "dnet:result_typologies"}, "relevantdate": [], "collectedfrom": [{"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "CRIS UNS (Current Research Information System University of Novi Sad)", "key": "10|CRIS_UNS____::f66f1bd369679b5b077dcdf006089556"}], "id": "50|a89337edbe55::4930db9e954866d70916cbfba9f81f97", "subject": [], "instance": [{"refereed": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "hostedby": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "CRIS UNS (Current Research Information System University of Novi Sad)", "key": "10|CRIS_UNS____::f66f1bd369679b5b077dcdf006089556"}, "processingchargeamount": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "license": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "url": ["https://www.cris.uns.ac.rs/record.jsf?recordId=113444&source=OpenAIRE&language=en"], "distributionlocation": "", "processingchargecurrency": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "dateofacceptance": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "2019-01-01"}, "collectedfrom": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "CRIS UNS (Current Research Information System University of Novi Sad)", "key": "10|CRIS_UNS____::f66f1bd369679b5b077dcdf006089556"}, "accessright": {"classid": "UNKNOWN", "classname": "UNKNOWN", "schemename": "dnet:access_modes", "schemeid": "dnet:access_modes"}, "instancetype": {"classid": "0004", "classname": "Conference object", "schemename": "dnet:dataCite_resource", "schemeid": "dnet:dataCite_resource"}}], "embargoenddate": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "lastupdatetimestamp": 0, "author": [{"surname": "Zeuner", "name": "Kirsten E.", "pid": [], "rank": 1, "affiliation": [], "fullname": "Zeuner, Kirsten E."}, {"surname": "Knutzen", "name": "Arne", "pid": [], "rank": 2, "affiliation": [], "fullname": "Knutzen, Arne"}, {"surname": "Granert", "name": "Oliver", "pid": [{"qualifier": {"classid": "ORCID", "classname": "ORCID"}, "value": "0000-0002-0656-1023"}, {"qualifier": {"classid": "pubmed", "classname": "pubmed"}, "value": "pubmed.it"}], "rank": 3, "affiliation": [], "fullname": "Granert, Oliver"}, {"surname": "Sablowsky", "name": "Simone", "pid": [{"qualifier": {"classid": "id", "classname": "id"}, "value": "12345678"}], "rank": 4, "affiliation": [], "fullname": "Sablowsky, Simone"}, {"surname": "Go\\u0308tz", "name": "Julia", "pid": [], "rank": 5, "affiliation": [], "fullname": "G\\u00f6tz, Julia"}, {"surname": "Wolff", "name": "Stephan", "pid": [], "rank": 6, "affiliation": [], "fullname": "Wolff, Stephan"}, {"surname": "Jansen", "name": "Olav", "pid": [{"qualifier": {"classid": "ORCID", "classname": "ORCID"}, "value": "0000-0000-0656-1023"},{"qualifier": {"classid": "id", "classname": "id"}, "value": "987654321"}], "rank": 7, "affiliation": [], "fullname": "Jansen, Olav"}, {"surname": "Dressler", "name": "Dirk", "pid": [], "rank": 8, "affiliation": [], "fullname": "Dressler, Dirk"}, {"surname": "Schneider", "name": "Susanne A.", "pid": [], "rank": 9, "affiliation": [], "fullname": "Schneider, Susanne A."}], "source": [], "dateofcollection": "2020-03-10T15:05:38.685Z", "fulltext": [], "dateoftransformation": "2020-03-11T20:11:13.15Z", "description": [], "format": [], "journal": {"issnPrinted": "", "conferencedate": "", "conferenceplace": "", "name": "", "edition": "", "iss": "", "sp": "", "vol": "", "dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "issnOnline": "", "ep": "", "issnLinking": ""}, "coverage": [], "publisher": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "language": {"classid": "en", "classname": "en", "schemename": "dnet:languages", "schemeid": "dnet:languages"}, "bestaccessright": {"classid": "UNKNOWN", "classname": "not available", "schemename": "dnet:access_modes", "schemeid": "dnet:access_modes"}, "country": [{"classid": "FI", "classname": "Finland", "schemeid": "dnet:countries", "schemename": "dnet:countries"}], "extraInfo": [], "originalId": ["(BISIS)113444", "https://www.cris.uns.ac.rs/record.jsf?recordId=113444&source=OpenAIRE&language=en"], "dateofacceptance": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "2016-01-01"}, "title": [{"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "qualifier": {"classid": "test title", "classname": "test title", "schemename": "dnet:dataCite_title", "schemeid": "dnet:dataCite_title"}, "value": "Antichains of copies of ultrahomogeneous structures"}]} \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/crossref/article.json b/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/crossref/article.json new file mode 100644 index 0000000000..e0dc0db39d --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/crossref/article.json @@ -0,0 +1,175 @@ +{ + "DOI": "10.26850/1678-4618eqj.v35.1.2010.p41-46", + "issued": { + "date-parts": [ + [ + 2018, + 1, + 15 + ] + ] + }, + "abstract": "A qualitative spot-test and tandem quantitative analysis of dipyrone in the bulk drugand in pharmaceutical preparations is proposed. The formation of a reddish-violet\u00a0 color indicates a positive result. In sequence a quantitative procedure can be performed in the same flask. The quantitative results obtained were statistically compared with those obtained with the method indicated by the Brazilian\u00a0 Pharmacopoeia, using the Student\u2019s t and the F tests. Considering the concentration in a 100 \u03bcL aliquot, the qualitative visual limit of detection is about 5\u00d710-6 g; instrumental LOD \u2245 1.4\u00d710-4 mol L-1 ; LOQ \u2245 4.5\u00d710-4 mol L-1.", + "prefix": "10.26850", + "author": [ + { + "authenticated-orcid": false, + "given": "Matthieu", + "family": "Tubino", + "sequence": "first", + "affiliation": [], + "ORCID": "http://orcid.org/0000-0002-1987-3907" + }, + { + "affiliation": [], + "given": "A. C.", + "family": "Biondo", + "sequence": "additional" + }, + { + "authenticated-orcid": false, + "given": "Marta Maria Duarte Carvalho", + "family": "Vila", + "sequence": "additional", + "affiliation": [], + "ORCID": "http://orcid.org/0000-0002-0198-7076" + }, + { + "authenticated-orcid": false, + "given": "Leonardo", + "family": "Pezza", + "sequence": "additional", + "affiliation": [], + "ORCID": "http://orcid.org/0000-0003-0197-7369" + }, + { + "authenticated-orcid": false, + "given": "Helena Redigolo", + "family": "Pezza", + "sequence": "additional", + "affiliation": [], + "ORCID": "http://orcid.org/0000-0001-5564-1639" + } + ], + "reference-count": 0, + "ISSN": [ + "1678-4618" + ], + "member": "11395", + "source": "Crossref", + "score": 1.0, + "deposited": { + "timestamp": 1540823529000, + "date-time": "2018-10-29T14:32:09Z", + "date-parts": [ + [ + 2018, + 10, + 29 + ] + ] + }, + "indexed": { + "timestamp": 1540825815212, + "date-time": "2018-10-29T15:10:15Z", + "date-parts": [ + [ + 2018, + 10, + 29 + ] + ] + }, + "type": "journal-article", + "published-online": { + "date-parts": [ + [ + 2018, + 1, + 15 + ] + ] + }, + "URL": "http://dx.doi.org/10.26850/1678-4618eqj.v35.1.2010.p41-46", + "is-referenced-by-count": 0, + "volume": "35", + "issn-type": [ + { + "type": "electronic", + "value": "1678-4618" + } + ], + "link": [ + { + "URL": "http://revista.iq.unesp.br/ojs/index.php/ecletica/article/viewFile/191/149", + "intended-application": "text-mining", + "content-version": "vor", + "content-type": "application/pdf" + }, + { + "URL": "http://revista.iq.unesp.br/ojs/index.php/ecletica/article/viewFile/191/149", + "intended-application": "similarity-checking", + "content-version": "vor", + "content-type": "unspecified" + } + ], + "journal-issue": { + "issue": "1", + "published-online": { + "date-parts": [ + [ + 2018, + 1, + 15 + ] + ] + } + }, + "references-count": 0, + "short-container-title": [ + "Eclet. Quim. J." + ], + "publisher": "Ecletica Quimica Journal", + "content-domain": { + "domain": [], + "crossmark-restriction": false + }, + "license": [ + { + "URL": "http://creativecommons.org/licenses/by/4.0", + "start": { + "timestamp": 1515974400000, + "date-time": "2018-01-15T00:00:00Z", + "date-parts": [ + [ + 2018, + 1, + 15 + ] + ] + }, + "content-version": "unspecified", + "delay-in-days": 0 + } + ], + "created": { + "timestamp": 1517590842000, + "date-time": "2018-02-02T17:00:42Z", + "date-parts": [ + [ + 2018, + 2, + 2 + ] + ] + }, + "issue": "1", + "title": [ + "Spot-test identification and rapid quantitative sequential analys is of dipyrone" + ], + "container-title": [ + "Ecl\u00e9tica Qu\u00edmica Journal" + ], + "page": "41-50", + "funder": [{"DOI": "10.13039/100010663","name": "H2020 European Research Council","doi-asserted-by": "publisher","award": ["677749"]}] +} \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/crossref/article_funder_template.json b/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/crossref/article_funder_template.json new file mode 100644 index 0000000000..1a49109ecd --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/crossref/article_funder_template.json @@ -0,0 +1,175 @@ +{ + "DOI": "10.26850/1678-4618eqj.v35.1.2010.p41-46", + "issued": { + "date-parts": [ + [ + 2018, + 1, + 15 + ] + ] + }, + "abstract": "A qualitative spot-test and tandem quantitative analysis of dipyrone in the bulk drugand in pharmaceutical preparations is proposed. The formation of a reddish-violet\u00a0 color indicates a positive result. In sequence a quantitative procedure can be performed in the same flask. The quantitative results obtained were statistically compared with those obtained with the method indicated by the Brazilian\u00a0 Pharmacopoeia, using the Student\u2019s t and the F tests. Considering the concentration in a 100 \u03bcL aliquot, the qualitative visual limit of detection is about 5\u00d710-6 g; instrumental LOD \u2245 1.4\u00d710-4 mol L-1 ; LOQ \u2245 4.5\u00d710-4 mol L-1.", + "prefix": "10.26850", + "author": [ + { + "authenticated-orcid": false, + "given": "Matthieu", + "family": "Tubino", + "sequence": "first", + "affiliation": [], + "ORCID": "http://orcid.org/0000-0002-1987-3907" + }, + { + "affiliation": [], + "given": "A. C.", + "family": "Biondo", + "sequence": "additional" + }, + { + "authenticated-orcid": false, + "given": "Marta Maria Duarte Carvalho", + "family": "Vila", + "sequence": "additional", + "affiliation": [], + "ORCID": "http://orcid.org/0000-0002-0198-7076" + }, + { + "authenticated-orcid": false, + "given": "Leonardo", + "family": "Pezza", + "sequence": "additional", + "affiliation": [], + "ORCID": "http://orcid.org/0000-0003-0197-7369" + }, + { + "authenticated-orcid": false, + "given": "Helena Redigolo", + "family": "Pezza", + "sequence": "additional", + "affiliation": [], + "ORCID": "http://orcid.org/0000-0001-5564-1639" + } + ], + "reference-count": 0, + "ISSN": [ + "1678-4618" + ], + "member": "11395", + "source": "Crossref", + "score": 1.0, + "deposited": { + "timestamp": 1540823529000, + "date-time": "2018-10-29T14:32:09Z", + "date-parts": [ + [ + 2018, + 10, + 29 + ] + ] + }, + "indexed": { + "timestamp": 1540825815212, + "date-time": "2018-10-29T15:10:15Z", + "date-parts": [ + [ + 2018, + 10, + 29 + ] + ] + }, + "type": "journal-article", + "published-online": { + "date-parts": [ + [ + 2018, + 1, + 15 + ] + ] + }, + "URL": "http://dx.doi.org/10.26850/1678-4618eqj.v35.1.2010.p41-46", + "is-referenced-by-count": 0, + "volume": "35", + "issn-type": [ + { + "type": "electronic", + "value": "1678-4618" + } + ], + "link": [ + { + "URL": "http://revista.iq.unesp.br/ojs/index.php/ecletica/article/viewFile/191/149", + "intended-application": "text-mining", + "content-version": "vor", + "content-type": "application/pdf" + }, + { + "URL": "http://revista.iq.unesp.br/ojs/index.php/ecletica/article/viewFile/191/149", + "intended-application": "similarity-checking", + "content-version": "vor", + "content-type": "unspecified" + } + ], + "journal-issue": { + "issue": "1", + "published-online": { + "date-parts": [ + [ + 2018, + 1, + 15 + ] + ] + } + }, + "references-count": 0, + "short-container-title": [ + "Eclet. Quim. J." + ], + "publisher": "Ecletica Quimica Journal", + "content-domain": { + "domain": [], + "crossmark-restriction": false + }, + "license": [ + { + "URL": "http://creativecommons.org/licenses/by/4.0", + "start": { + "timestamp": 1515974400000, + "date-time": "2018-01-15T00:00:00Z", + "date-parts": [ + [ + 2018, + 1, + 15 + ] + ] + }, + "content-version": "unspecified", + "delay-in-days": 0 + } + ], + "created": { + "timestamp": 1517590842000, + "date-time": "2018-02-02T17:00:42Z", + "date-parts": [ + [ + 2018, + 2, + 2 + ] + ] + }, + "issue": "1", + "title": [ + "Spot-test identification and rapid quantitative sequential analys is of dipyrone" + ], + "container-title": [ + "Ecl\u00e9tica Qu\u00edmica Journal" + ], + %s + "page": "41-50" +} \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/crossref/awardTest.json b/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/crossref/awardTest.json new file mode 100644 index 0000000000..7c6e7beb92 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/crossref/awardTest.json @@ -0,0 +1,193 @@ +{ + "DOI": "10.1016/j.infbeh.2016.11.001", + "issued": { + "date-parts": [ + [ + 2017, + 8 + ] + ] + }, + "update-policy": "http://dx.doi.org/10.1016/elsevier_cm_policy", + "prefix": "10.1016", + "subject": [ + "Developmental and Educational Psychology" + ], + "author": [ + { + "affiliation": [], + "given": "Dora", + "family": "Kampis", + "sequence": "first" + }, + { + "affiliation": [], + "given": "D\u00f3ra", + "family": "Fogd", + "sequence": "additional" + }, + { + "affiliation": [], + "given": "\u00c1gnes Melinda", + "family": "Kov\u00e1cs", + "sequence": "additional" + } + ], + "reference-count": 109, + "ISSN": [ + "0163-6383" + ], + "assertion": [ + { + "name": "publisher", + "value": "Elsevier", + "label": "This article is maintained by" + }, + { + "name": "articletitle", + "value": "Nonverbal components of Theory of Mind in typical and atypical development", + "label": "Article Title" + }, + { + "name": "journaltitle", + "value": "Infant Behavior and Development", + "label": "Journal Title" + }, + { + "name": "articlelink", + "value": "https://doi.org/10.1016/j.infbeh.2016.11.001", + "label": "CrossRef DOI link to publisher maintained version" + }, + { + "name": "content_type", + "value": "article", + "label": "Content Type" + }, + { + "name": "copyright", + "value": "\u00a9 2016 Elsevier Inc. All rights reserved.", + "label": "Copyright" + } + ], + "member": "78", + "source": "Crossref", + "score": 1.0, + "deposited": { + "timestamp": 1565383284000, + "date-parts": [ + [ + 2019, + 8, + 9 + ] + ], + "date-time": "2019-08-09T20:41:24Z" + }, + "indexed": { + "timestamp": 1565385055278, + "date-parts": [ + [ + 2019, + 8, + 9 + ] + ], + "date-time": "2019-08-09T21:10:55Z" + }, + "type": "journal-article", + "URL": "http://dx.doi.org/10.1016/j.infbeh.2016.11.001", + "is-referenced-by-count": 1, + "volume": "48", + "issn-type": [ + { + "type": "print", + "value": "0163-6383" + } + ], + "link": [ + { + "URL": "https://api.elsevier.com/content/article/PII:S0163638315300059?httpAccept=text/xml", + "intended-application": "text-mining", + "content-version": "vor", + "content-type": "text/xml" + }, + { + "URL": "https://api.elsevier.com/content/article/PII:S0163638315300059?httpAccept=text/plain", + "intended-application": "text-mining", + "content-version": "vor", + "content-type": "text/plain" + } + ], + "published-print": { + "date-parts": [ + [ + 2017, + 8 + ] + ] + }, + "references-count": 109, + "short-container-title": [ + "Infant Behavior and Development" + ], + "publisher": "Elsevier BV", + "content-domain": { + "domain": [ + "elsevier.com", + "sciencedirect.com" + ], + "crossmark-restriction": true + }, + "license": [ + { + "URL": "https://www.elsevier.com/tdm/userlicense/1.0/", + "start": { + "timestamp": 1501545600000, + "date-parts": [ + [ + 2017, + 8, + 1 + ] + ], + "date-time": "2017-08-01T00:00:00Z" + }, + "content-version": "tdm", + "delay-in-days": 0 + } + ], + "language": "en", + "created": { + "timestamp": 1479142046000, + "date-parts": [ + [ + 2016, + 11, + 14 + ] + ], + "date-time": "2016-11-14T16:47:26Z" + }, + "title": [ + "Nonverbal components of Theory of Mind in typical and atypical development" + ], + "alternative-id": [ + "S0163638315300059" + ], + "container-title": [ + "Infant Behavior and Development" + ], + "funder": [ + { + "DOI": "10.13039/501100001711", + "name": "Swiss National Science Foundation (Schweizerische Nationalfonds)", + "doi-asserted-by": "publisher", + "award": [ + "CR32I3_156724", + "31003A_173281/1", + "200021_165850" + ] + } + ], + "page": "54-62" +} diff --git a/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/crossref/book.json b/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/crossref/book.json new file mode 100644 index 0000000000..ab422b6f3d --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/crossref/book.json @@ -0,0 +1,104 @@ +{ + "DOI": "10.17848/9780880992299.vol1ch4", + "ISBN": [ + "9780880992299" + ], + "issued": { + "date-parts": [ + [ + 2001, + 12, + 1 + ] + ] + }, + "prefix": "10.17848", + "author": [ + { + "affiliation": [ + ], + "given": "William E.", + "family": "Even", + "authenticated-orcid": false + }, + { + "affiliation": [ + ], + "given": "David A.", + "family": "Macpherson" + } + ], + "reference-count": 0, + "member": "7312", + "source": "Crossref", + "score": 1.0, + "deposited": { + "timestamp": 1461687244000, + "date-parts": [ + [ + 2016, + 4, + 26 + ] + ], + "date-time": "2016-04-26T16:14:04Z" + }, + "indexed": { + "timestamp": 1502548826285, + "date-parts": [ + [ + 2017, + 8, + 12 + ] + ], + "date-time": "2017-08-12T14:40:26Z" + }, + "type": "book-chapter", + "published-online": { + "date-parts": [ + [ + 2010, + 5, + 27 + ] + ] + }, + "URL": "http://dx.doi.org/10.17848/9780880992299.vol1ch4", + "is-referenced-by-count": 0, + "download_ts": 1508079092.874343, + "published-print": { + "date-parts": [ + [ + 2001, + 12, + 1 + ] + ] + }, + "references-count": 0, + "publisher": "W.E. Upjohn Institute", + "content-domain": { + "domain": [ + ], + "crossmark-restriction": false + }, + "created": { + "timestamp": 1434034139000, + "date-parts": [ + [ + 2015, + 6, + 11 + ] + ], + "date-time": "2015-06-11T14:48:59Z" + }, + "title": [ + "Children\\'s Effects on Women\\'s Labor Market Attachment and Earnings" + ], + "container-title": [ + "Working Time in Comparative Perspective - Volume II: Life-Cycle Working Time and Nonstandard Hours" + ], + "page": "99-128" +} \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/crossref/dataset.json b/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/crossref/dataset.json new file mode 100644 index 0000000000..5c4b8c5a2a --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/crossref/dataset.json @@ -0,0 +1,105 @@ +{ + "DOI": "10.1037/e522512014-096", + "subtitle": [ + "(522512014-096)" + ], + "issued": { + "date-parts": [ + [ + 2012 + ] + ] + }, + "prefix": "10.1037", + "author": [ + { + "affiliation": [], + "given": "Jessica", + "family": "Trudeau", + "sequence": "first" + }, + { + "affiliation": [], + "given": "Amy", + "family": "McShane", + "sequence": "additional" + }, + { + "affiliation": [], + "given": "Renee", + "family": "McDonald", + "sequence": "additional" + } + ], + "reference-count": 0, + "member": "15", + "source": "Crossref", + "score": 1.0, + "deposited": { + "timestamp": 1413827035000, + "date-parts": [ + [ + 2014, + 10, + 20 + ] + ], + "date-time": "2014-10-20T17:43:55Z" + }, + "indexed": { + "timestamp": 1550142454710, + "date-parts": [ + [ + 2019, + 2, + 14 + ] + ], + "date-time": "2019-02-14T11:07:34Z" + }, + "type": "dataset", + "URL": "http://dx.doi.org/10.1037/e522512014-096", + "is-referenced-by-count": 0, + "published-print": { + "date-parts": [ + [ + 2012 + ] + ] + }, + "references-count": 0, + "institution": { + "acronym": [ + "APA" + ], + "place": [ + "-" + ], + "name": "American Psychological Association" + }, + "publisher": "American Psychological Association (APA)", + "content-domain": { + "domain": [], + "crossmark-restriction": false + }, + "created": { + "timestamp": 1413826121000, + "date-parts": [ + [ + 2014, + 10, + 20 + ] + ], + "date-time": "2014-10-20T17:28:41Z" + }, + "title": [ + "Project Support: A Randomized Control Study to Evaluate the Translation of an Evidence- Based Program" + ], + "alternative-id": [ + "522512014-096" + ], + "container-title": [ + "PsycEXTRA Dataset" + ] +} \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/crossref/empty_title.json b/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/crossref/empty_title.json new file mode 100644 index 0000000000..2d274f8856 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/crossref/empty_title.json @@ -0,0 +1,121 @@ +{ + "indexed": { + "date-parts": [ + [ + 2020, + 4, + 7 + ] + ], + "date-time": "2020-04-07T15:54:28Z", + "timestamp": 1586274868901 + }, + "reference-count": 0, + "publisher": "Japan Society of Mechanical Engineers", + "issue": "432", + "content-domain": { + "domain": [], + "crossmark-restriction": false + }, + "short-container-title": [ + "JSMET" + ], + "published-print": { + "date-parts": [ + [ + 1982 + ] + ] + }, + "DOI": "10.1299\/kikaib.48.1474", + "type": "journal-article", + "created": { + "date-parts": [ + [ + 2011, + 9, + 13 + ] + ], + "date-time": "2011-09-13T05:59:01Z", + "timestamp": 1315893541000 + }, + "page": "1474-1482", + "source": "Crossref", + "is-referenced-by-count": 0, + "title": [ + "" + ], + "prefix": "10.1299", + "volume": "48", + "author": [ + { + "given": "Hiroshi", + "family": "KATO", + "sequence": "first", + "affiliation": [] + }, + { + "given": "Yoshichika", + "family": "MIZUNO", + "sequence": "additional", + "affiliation": [] + } + ], + "member": "124", + "container-title": [ + "Transactions of the Japan Society of Mechanical Engineers Series B" + ], + "original-title": [ + "\u5e0c\u8584\u9ad8\u5206\u5b50\u6eb6\u6db2\u4e2d\u306e\u6709\u9650\u9577\u5186\u67f1\u306e\u62b5\u6297" + ], + "language": "ja", + "deposited": { + "date-parts": [ + [ + 2011, + 9, + 13 + ] + ], + "date-time": "2011-09-13T06:01:33Z", + "timestamp": 1315893693000 + }, + "score": 1.0, + "subtitle": [], + "short-title": [], + "issued": { + "date-parts": [ + [ + 1982 + ] + ] + }, + "references-count": 0, + "journal-issue": { + "published-print": { + "date-parts": [ + [ + 1982 + ] + ] + }, + "issue": "432" + }, + "URL": "http:\/\/dx.doi.org\/10.1299\/kikaib.48.1474", + "relation": {}, + "ISSN": [ + "0387-5016", + "1884-8346" + ], + "issn-type": [ + { + "value": "0387-5016", + "type": "print" + }, + { + "value": "1884-8346", + "type": "electronic" + } + ] +} \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/crossref/funder_doi b/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/crossref/funder_doi new file mode 100644 index 0000000000..4183b3b2d1 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/crossref/funder_doi @@ -0,0 +1,34 @@ +"funder": [{"DOI": "10.13039/100010663","name": "H2020 European Research Council","doi-asserted-by": "publisher","award": ["677749"]}], +"funder": [{"name": "European Union’s Horizon 2020 research and innovation program","award": ["296801","304995","675395"]}], +"funder": [{"DOI": "10.13039/100010661","name": "Horizon 2020 Framework Programme","doi-asserted-by": "publisher","award": ["722467", "H2020-FETOPEN-2015-CSA 712689","773830 (2018–2022)"]}], +"funder": [{"DOI": "10.13039/501100007601","name": "Horizon 2020","doi-asserted-by": "publisher","award": ["645119"]}], +"funder": [{"DOI": "10.13039/100010665","name": "H2020 Marie Skłodowska-Curie Actions","doi-asserted-by": "publisher","award": ["840267"]}], +"funder": [{"DOI": "10.13039/100011199","name": "FP7 Ideas: European Research Council","doi-asserted-by": "publisher","award": ["226438"]}], +"funder": [{"DOI": "10.13039/100004431","name": "Directorate-General for Research and Innovation","doi-asserted-by": "publisher","award": ["321427"]}], +"funder": [{"DOI": "10.13039/501100004963","name": "Seventh Framework Programme","doi-asserted-by": "publisher","award": ["287818","612538"]}], +"funder": [{"DOI": "10.13039/501100000781","name": "European Research Council","doi-asserted-by": "publisher","award": ["340185"]}], +"funder": [{"name": "European Union's","award": ["763909"]}], +"funder": [{"DOI": "10.13039/501100000780","name": "European Commission","doi-asserted-by": "publisher","award": ["645119", "H2020-FETOPEN-2015-CSA_712689"]}], +"funder": [{"DOI": "10.13039/100000001","name": "National Science Foundation","doi-asserted-by": "publisher","award": ["1639552,1634422","ID0EEMBI7182"]}], +"funder": [{"name": "The French National Research Agency (ANR)","award": ["ID0E4QBI7183","ANR-11-JS56-01501","ID0E3VBI7184","ANR-13-BS06-0008"]}], +"funder": [{"DOI": "10.13039/501100001665","name": "Agence Nationale de la Recherche","doi-asserted-by": "publisher","award": ["ANR-14-ASTR-0004-01"]}], +"funder": [{"DOI": "10.13039/501100002341","name": "Academy of Finland","doi-asserted-by": "publisher","award": ["294337","292335","31444","250114","292482"]}], +"funder": [{"DOI": "10.13039/501100001602","name": "Science Foundation Ireland","doi-asserted-by": "publisher","award": ["16/SP/3829","12/RC/2302_P2","SFI/09/IN.I/12974"]}], +"funder": [{"DOI": "10.13039/501100000923","name": "Australian Research Council","doi-asserted-by": "publisher","award": ["LP110200134"]}], +"funder": [{"DOI": "10.13039/501100000038","name": "NSERC","doi-asserted-by": "crossref","award": []}], +"funder": [{"DOI": "10.13039/501100000155","name": "Social Sciences and Humanities Research Council of Canada","doi-asserted-by": "publisher","award": []}], +"funder": [{"DOI": "10.13039/501100000024","name": "Canadian Institutes for Health Research","doi-asserted-by": "crossref","award": ["HIB-126784","HHP-111405"]}], +"funder": [{"DOI": "10.13039/501100002848","name": "Comisión Nacional de Investigación Científica y Tecnológica","doi-asserted-by": "publisher","award": ["15130011"]}], +"funder": [{"DOI": "10.13039/501100003448","name": "General Secretariat for Research and Technology","doi-asserted-by": "publisher","award": ["MIS: 380170"]}], +"funder": [{"DOI": "10.13039/501100010198","name": "Ministerio de Economía, Industria y Competitividad, Gobierno de España","doi-asserted-by": "publisher","award": ["ECO2017-89715-P"]}], +"funder": [{"DOI": "10.13039/501100004564","name": "Ministarstvo Prosvete, Nauke i Tehnološkog Razvoja","doi-asserted-by": "publisher","award": ["TR34008"]}], +"funder": [{"DOI": "10.13039/501100003407","name": "MIUR","doi-asserted-by": "publisher","award": ["20158A9CBM"]}], +"funder": [{"DOI": "10.13039/501100003407","name": "MIUR","doi-asserted-by": "publisher","award": []}], +"funder": [{"DOI": "10.13039/501100006588","name": "Ministarstvo Znanosti, Obrazovanja i Sporta","doi-asserted-by": "publisher","award": ["037-0372790-2799", "Project No. 125-1253008-1350"]}], +"funder": [{"DOI": "10.13039/501100006588","name": "Ministry of Science, Education and Sports","doi-asserted-by": "publisher","award": ["181-1811096-1093"]}], +"funder": [{"DOI": "10.13039/501100004488","name": "Hrvatska Zaklada za Znanost","doi-asserted-by": "publisher","award": ["HRZZ-IP-2013-11-3013", "UIP-2014-09-4744"]}], +"funder": [{"DOI": "10.13039/501100006769","name": "Russian Science Foundation","doi-asserted-by": "publisher","award": ["17-11-01027"]}], +"funder": [{"DOI": "10.13039/501100001711","name": "Swiss National Science Foundation (Schweizerische Nationalfonds)","doi-asserted-by": "publisher","award": ["CR32I3_156724", "31003A_173281/1"]}], +"funder": [{"DOI": "10.13039/501100004410","name": "Türkiye Bilimsel ve Teknolojik Araştirma Kurumu","doi-asserted-by": "publisher","award": ["113M552"]}], +"funder": [{"DOI": "10.13039/100004440","name": "Wellcome Trust","doi-asserted-by": "crossref","award": ["095127","079080"]}], +"funder": [{"DOI": "10.13039/100004440","name": "Wellcome Trust","doi-asserted-by": "crossref","award": []}], \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/crossref/funder_name b/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/crossref/funder_name new file mode 100644 index 0000000000..b5c9232acf --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/crossref/funder_name @@ -0,0 +1,5 @@ +"funder": [{"name": "Wellcome Trust Masters Fellowship","award": ["090633"]}], +"funder": [{"name": "CONICYT, Programa de Formación de Capital Humano Avanzado","award": ["#72140079"]}], +"funder": [{"name": "European Union's","award": ["763909"]}], +"funder": [{"name": "European Union’s Horizon 2020 research and innovation program","award": ["296801","304995","675395"]}], +"funder": [{"name": "The French National Research Agency (ANR)","award": ["ID0E4QBI7183","ANR-11-JS56-01501","ID0E3VBI7184","ANR-13-BS06-0008"]}], diff --git a/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/crossref/orcid_data.json b/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/crossref/orcid_data.json new file mode 100644 index 0000000000..def546ddb4 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/crossref/orcid_data.json @@ -0,0 +1,271 @@ +{ + "DOI":"10.1016/j.carbpol.2020.115930", + "issued":{ + "date-parts":[ + [ + 2020, + 4 + ] + ] + }, + "published-print":{ + "date-parts":[ + [ + 2020, + 4 + ] + ] + }, + "prefix":"10.1016", + "subject":[ + "Organic Chemistry", + "Materials Chemistry", + "Polymers and Plastics" + ], + "author":[ + { + "affiliation":[ + + ], + "given":"Lei", + "family":"Fang", + "sequence":"first" + }, + { + "affiliation":[ + + ], + "given":"Hua", + "family":"Lin", + "sequence":"additional" + }, + { + "affiliation":[ + + ], + "given":"Zhenfeng", + "family":"Wu", + "sequence":"additional" + }, + { + "affiliation":[ + + ], + "given":"Zhen", + "family":"Wang", + "sequence":"additional" + }, + { + "affiliation":[ + + ], + "given":"Xinxin", + "family":"Fan", + "sequence":"additional" + }, + { + "affiliation":[ + + ], + "given":"Ziting", + "family":"Cheng", + "sequence":"additional" + }, + { + "affiliation":[ + + ], + "given":"Xiaoya", + "family":"Hou", + "sequence":"additional" + }, + { + "authenticated-orcid":false, + "given":"Daquan", + "family":"Chen", + "sequence":"additional", + "affiliation":[ + + ], + "ORCID":"http://orcid.org/0000-0002-6796-0204" + } + ], + "reference-count":41, + "ISSN":[ + "0144-8617" + ], + "assertion":[ + { + "name":"publisher", + "value":"Elsevier", + "label":"This article is maintained by" + }, + { + "name":"articletitle", + "value":"In vitro/vivo evaluation of novel mitochondrial targeting charge-reversal polysaccharide-based antitumor nanoparticle", + "label":"Article Title" + }, + { + "name":"journaltitle", + "value":"Carbohydrate Polymers", + "label":"Journal Title" + }, + { + "name":"articlelink", + "value":"https://doi.org/10.1016/j.carbpol.2020.115930", + "label":"CrossRef DOI link to publisher maintained version" + }, + { + "name":"content_type", + "value":"article", + "label":"Content Type" + }, + { + "name":"copyright", + "value":"\\u00a9 2020 Elsevier Ltd. All rights reserved.", + "label":"Copyright" + } + ], + "member":"78", + "source":"Crossref", + "score":1.0, + "deposited":{ + "timestamp":1584590965000, + "date-time":"2020-03-19T04:09:25Z", + "date-parts":[ + [ + 2020, + 3, + 19 + ] + ] + }, + "indexed":{ + "timestamp":1584592912467, + "date-time":"2020-03-19T04:41:52Z", + "date-parts":[ + [ + 2020, + 3, + 19 + ] + ] + }, + "type":"journal-article", + "URL":"http://dx.doi.org/10.1016/j.carbpol.2020.115930", + "is-referenced-by-count":0, + "volume":"234", + "issn-type":[ + { + "type":"print", + "value":"0144-8617" + } + ], + "link":[ + { + "URL":"https://api.elsevier.com/content/article/PII:S0144861720301041?httpAccept=text/xml", + "intended-application":"text-mining", + "content-version":"vor", + "content-type":"text/xml" + }, + { + "URL":"https://api.elsevier.com/content/article/PII:S0144861720301041?httpAccept=text/plain", + "intended-application":"text-mining", + "content-version":"vor", + "content-type":"text/plain" + } + ], + "update-policy":"http://dx.doi.org/10.1016/elsevier_cm_policy", + "references-count":41, + "short-container-title":[ + "Carbohydrate Polymers" + ], + "publisher":"Elsevier BV", + "content-domain":{ + "domain":[ + "elsevier.com", + "sciencedirect.com" + ], + "crossmark-restriction":true + }, + "language":"en", + "license":[ + { + "URL":"https://www.elsevier.com/tdm/userlicense/1.0/", + "start":{ + "timestamp":1585699200000, + "date-time":"2020-04-01T00:00:00Z", + "date-parts":[ + [ + 2020, + 4, + 1 + ] + ] + }, + "content-version":"tdm", + "delay-in-days":0 + } + ], + "created":{ + "timestamp":1581759678000, + "date-time":"2020-02-15T09:41:18Z", + "date-parts":[ + [ + 2020, + 2, + 15 + ] + ] + }, + "title":[ + "In vitro/vivo evaluation of novel mitochondrial targeting charge-reversal polysaccharide-based antitumor nanoparticle" + ], + "alternative-id":[ + "S0144861720301041" + ], + "container-title":[ + "Carbohydrate Polymers" + ], + "funder":[ + { + "doi-asserted-by":"publisher", + "DOI":"10.13039/501100007129", + "name":"Natural Science Foundation of Shandong Province", + "award":[ + "ZR2019ZD24", + "ZR2019YQ30" + ] + }, + { + "doi-asserted-by":"publisher", + "DOI":"10.13039/100010449", + "name":"Ministry of Education, Libya", + "award":[ + + ] + }, + { + "doi-asserted-by":"publisher", + "DOI":"10.13039/501100012249", + "name":"Jiangxi University of Traditional Chinese Medicine", + "award":[ + "TCM-0906" + ] + }, + { + "name":"Taishan Scholar Program", + "award":[ + "qnts20161035" + ] + }, + { + "name":"Open fund project of Key Laboratory of Modern Preparation of TCM", + "award":[ + + ] + } + ], + "page":"115930", + "article-number":"115930" +} \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/crossref/preprint.json b/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/crossref/preprint.json new file mode 100644 index 0000000000..13b2fea841 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/crossref/preprint.json @@ -0,0 +1,146 @@ +{ + "DOI": "10.1101/030080", + "issued": { + "date-parts": [ + [ + 2015, + 10, + 28 + ] + ] + }, + "abstract": "Abstract Key MessageAgrobacterium tumefacienswas used to transform radiata pine shoots and to efficiently produce stable genetically modified pine plants. Abstract Micropropagated shoot explants fromPinus radiataD. Don were used to produce stable transgenic plants byAgrobacterium tumefaciensmediated transformation. Using this method any genotype that can be micropropagated could produce stable transgenic lines. As over 80% ofP. radiatagenotypes tested can be micropropagated, this effectively means that any line chosen for superior characteristics could be transformed. There are well established protocols for progressing such germplasm to field deployment. Here we used open and control pollinated seed lines and embryogenic clones. The method developed was faster than other methods previously developed using mature cotyledons. PCR positive shoots could be obtain within 6 months ofAgrobacteriumcocultivation compared with 12 months for cotyledon methods. Transformed shoots were obtained using either kanamycin or geneticin as the selectable marker gene. Shoots were recovered from selection, were tested and were not chimeric, indicating that the selection pressure was optimal for this explant type. GFP was used as a vital marker, and the bar gene, (for resistance to the herbicide Buster\\u00ae/Basta\\u00ae) was used to produce lines that could potentially be used in commercial application. As expected, a range of expression phenotypes were identified for both these reporter genes and the analyses for expression were relatively easy.", + "prefix": "10.1101", + "author": [ + { + "affiliation": [], + "given": "Jan E", + "family": "Grant", + "sequence": "first" + }, + { + "affiliation": [], + "given": "Pauline A", + "family": "Cooper", + "sequence": "additional" + }, + { + "affiliation": [], + "given": "Tracy M", + "family": "Dale", + "sequence": "additional" + } + ], + "reference-count": 0, + "member": "246", + "source": "Crossref", + "score": 1.0, + "deposited": { + "timestamp": 1483495053000, + "date-time": "2017-01-04T01:57:33Z", + "date-parts": [ + [ + 2017, + 1, + 4 + ] + ] + }, + "indexed": { + "timestamp": 1550234353119, + "date-time": "2019-02-15T12:39:13Z", + "date-parts": [ + [ + 2019, + 2, + 15 + ] + ] + }, + "type": "posted-content", + "URL": "http://dx.doi.org/10.1101/030080", + "is-referenced-by-count": 2, + "link": [ + { + "URL": "https://syndication.highwire.org/content/doi/10.1101/030080", + "intended-application": "similarity-checking", + "content-version": "vor", + "content-type": "unspecified" + } + ], + "accepted": { + "date-parts": [ + [ + 2015, + 10, + 28 + ] + ] + }, + "references-count": 0, + "institution": { + "acronym": [ + "-" + ], + "place": [ + "-" + ], + "name": "bioRxiv" + }, + "posted": { + "date-parts": [ + [ + 2015, + 10, + 28 + ] + ] + }, + "publisher": "Cold Spring Harbor Laboratory", + "content-domain": { + "domain": [], + "crossmark-restriction": false + }, + "created": { + "timestamp": 1446095513000, + "date-time": "2015-10-29T05:11:53Z", + "date-parts": [ + [ + 2015, + 10, + 29 + ] + ] + }, + "published-print": { + "timestamp": 1446095513000, + "date-time": "2015-10-29T05:11:53Z", + "date-parts": [ + [ + 2015, + 2, + 29 + ] + ] + }, + "published-online": { + "date-parts": [ + [ + 2015, + 2, + 2 + ] + ] + }, + "title": [ + "Genetic transformation of micropropagated shoots ofPinus radiataD.Don" + ], + "original-title": [ + "OR TITLE" + ], + "short-title": [ + "SHORT TITLE" + ], + "group-title": "Plant Biology", + "subtype": "preprint" +} \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/crossref/prwTest.json b/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/crossref/prwTest.json new file mode 100644 index 0000000000..756ee6ddac --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/crossref/prwTest.json @@ -0,0 +1,1029 @@ +{ + "indexed": { + "date-parts": [ + [ + 2020, + 5, + 5 + ] + ], + "date-time": "2020-05-05T11:24:12Z", + "timestamp": 1588677852676 + }, + "reference-count": 59, + "publisher": "eLife Sciences Publications, Ltd", + "license": [ + { + "URL": "http:\/\/creativecommons.org\/licenses\/by\/3.0\/", + "start": { + "date-parts": [ + [ + 2014, + 3, + 18 + ] + ], + "date-time": "2014-03-18T00:00:00Z", + "timestamp": 1395100800000 + }, + "delay-in-days": 0, + "content-version": "vor" + }, + { + "URL": "http:\/\/creativecommons.org\/licenses\/by\/3.0\/", + "start": { + "date-parts": [ + [ + 2014, + 3, + 18 + ] + ], + "date-time": "2014-03-18T00:00:00Z", + "timestamp": 1395100800000 + }, + "delay-in-days": 0, + "content-version": "am" + }, + { + "URL": "http:\/\/creativecommons.org\/licenses\/by\/3.0\/", + "start": { + "date-parts": [ + [ + 2014, + 3, + 18 + ] + ], + "date-time": "2014-03-18T00:00:00Z", + "timestamp": 1395100800000 + }, + "delay-in-days": 0, + "content-version": "tdm" + }, + { + "URL": "http:\/\/creativecommons.org\/licenses\/by\/3.0\/", + "start": { + "date-parts": [ + [ + 2014, + 3, + 18 + ] + ], + "date-time": "2014-03-18T00:00:00Z", + "timestamp": 1395100800000 + }, + "delay-in-days": 0, + "content-version": "vor" + }, + { + "URL": "http:\/\/creativecommons.org\/licenses\/by\/3.0\/", + "start": { + "date-parts": [ + [ + 2014, + 3, + 18 + ] + ], + "date-time": "2014-03-18T00:00:00Z", + "timestamp": 1395100800000 + }, + "delay-in-days": 0, + "content-version": "am" + }, + { + "URL": "http:\/\/creativecommons.org\/licenses\/by\/3.0\/", + "start": { + "date-parts": [ + [ + 2014, + 3, + 18 + ] + ], + "date-time": "2014-03-18T00:00:00Z", + "timestamp": 1395100800000 + }, + "delay-in-days": 0, + "content-version": "tdm" + } + ], + "funder": [ + { + "DOI": "10.13039\/100000002", + "name": "National Institutes of Health", + "doi-asserted-by": "crossref", + "award": [ + "GM56169", + "AG39420", + "T32 GM007276" + ] + }, + { + "DOI": "10.13039\/100000968", + "name": "American Heart Association", + "doi-asserted-by": "crossref", + "award": [ + "Postdoctoral Fellowship" + ] + }, + { + "DOI": "10.13039\/501100000024", + "name": "Canadian Institutes of Health Research", + "doi-asserted-by": "crossref", + "award": [ + "Postdoctoral Fellowship" + ] + }, + { + "DOI": "10.13039\/100000002", + "name": "National Institutes of Health", + "doi-asserted-by": "publisher", + "award": [ + "GM56169", + "AG39420", + "T32 GM007276" + ] + }, + { + "DOI": "10.13039\/100000968", + "name": "American Heart Association", + "doi-asserted-by": "publisher", + "award": [ + "Postdoctoral Fellowship" + ] + }, + { + "DOI": "10.13039\/501100000024", + "name": "Canadian Institutes of Health Research", + "doi-asserted-by": "publisher", + "award": [ + "Postdoctoral Fellowship" + ] + } + ], + "content-domain": { + "domain": [ + "www.elifesciences.org" + ], + "crossmark-restriction": false + }, + "short-container-title": [], + "abstract": "Glycogen synthase kinase-3 (GSK-3) is a key regulator of many cellular signaling pathways. Unlike most kinases, GSK-3 is controlled by inhibition rather than by specific activation. In the insulin and several other signaling pathways, phosphorylation of a serine present in a conserved sequence near the amino terminus of GSK-3 generates an auto-inhibitory peptide. In contrast, Wnt\/\u03b2-catenin signal transduction requires phosphorylation of Ser\/Pro rich sequences present in the Wnt co-receptors LRP5\/6, and these motifs inhibit GSK-3 activity. We present crystal structures of GSK-3 bound to its phosphorylated N-terminus and to two of the phosphorylated LRP6 motifs. A conserved loop unique to GSK-3 undergoes a dramatic conformational change that clamps the bound pseudo-substrate peptides, and reveals the mechanism of primed substrate recognition. The structures rationalize target sequence preferences and suggest avenues for the design of inhibitors selective for a subset of pathways regulated by GSK-3.<\/jats:p>", + "DOI": "10.7554\/elife.01998", + "type": "journal-article", + "created": { + "date-parts": [ + [ + 2014, + 3, + 18 + ] + ], + "date-time": "2014-03-18T15:23:32Z", + "timestamp": 1395156212000 + }, + "source": "Crossref", + "is-referenced-by-count": 56, + "title": [ + "Structural basis of GSK-3 inhibition by N-terminal phosphorylation and by the Wnt receptor LRP6" + ], + "prefix": "10.7554", + "volume": "3", + "author": [ + { + "given": "Jennifer L", + "family": "Stamos", + "sequence": "first", + "affiliation": [ + { + "name": "Department of Structural Biology, Stanford University, Stanford, United States" + }, + { + "name": "Department of Molecular and Cellular Physiology, Stanford University, Stanford, United States" + } + ] + }, + { + "given": "Matthew Ling-Hon", + "family": "Chu", + "sequence": "additional", + "affiliation": [ + { + "name": "Department of Structural Biology, Stanford University, Stanford, United States" + }, + { + "name": "Department of Molecular and Cellular Physiology, Stanford University, Stanford, United States" + } + ] + }, + { + "given": "Michael D", + "family": "Enos", + "sequence": "additional", + "affiliation": [ + { + "name": "Department of Structural Biology, Stanford University, Stanford, United States" + }, + { + "name": "Department of Molecular and Cellular Physiology, Stanford University, Stanford, United States" + } + ] + }, + { + "given": "Niket", + "family": "Shah", + "sequence": "additional", + "affiliation": [ + { + "name": "Department of Structural Biology, Stanford University, Stanford, United States" + }, + { + "name": "Department of Molecular and Cellular Physiology, Stanford University, Stanford, United States" + } + ] + }, + { + "given": "William I", + "family": "Weis", + "sequence": "additional", + "affiliation": [ + { + "name": "Department of Structural Biology, Stanford University, Stanford, United States" + }, + { + "name": "Department of Molecular and Cellular Physiology, Stanford University, Stanford, United States" + } + ] + } + ], + "member": "4374", + "published-online": { + "date-parts": [ + [ + 2014, + 3, + 18 + ] + ] + }, + "reference": [ + { + "key": "bib1", + "doi-asserted-by": "publisher", + "first-page": "213", + "DOI": "10.1107\/S0907444909052925", + "article-title": "PHENIX: a comprehensive Python-based system for macromolecular structure solution", + "volume": "66", + "author": "Adams", + "year": "2010", + "journal-title": "Acta Crystallographica Section D Biological Crystallography" + }, + { + "key": "bib2", + "doi-asserted-by": "publisher", + "first-page": "2527", + "DOI": "10.1021\/cr000110o", + "article-title": "Glycogen synthase kinase-3: properties, functions, and regulation", + "volume": "101", + "author": "Ali", + "year": "2001", + "journal-title": "Chemical Reviews" + }, + { + "key": "bib3", + "doi-asserted-by": "publisher", + "first-page": "2264", + "DOI": "10.2174\/138161211797052484", + "article-title": "The possible involvement of glycogen synthase kinase-3 (GSK-3) in diabetes, cancer and central nervous system diseases", + "volume": "17", + "author": "Amar", + "year": "2011", + "journal-title": "Current Pharmaceutical Design" + }, + { + "key": "bib4", + "doi-asserted-by": "publisher", + "first-page": "1143", + "DOI": "10.1016\/S0969-2126(01)00679-7", + "article-title": "The structure of phosphorylated GSK-3beta complexed with a peptide, FRATtide, that inhibits beta-catenin phosphorylation", + "volume": "9", + "author": "Bax", + "year": "2001", + "journal-title": "Structure" + }, + { + "key": "bib5", + "doi-asserted-by": "publisher", + "first-page": "393", + "DOI": "10.1016\/j.jmb.2003.08.031", + "article-title": "Structural characterization of the GSK-3beta active site using selective and non-selective ATP-mimetic inhibitors", + "volume": "333", + "author": "Bertrand", + "year": "2003", + "journal-title": "Journal of Molecular Biology" + }, + { + "key": "bib6", + "doi-asserted-by": "publisher", + "first-page": "1619", + "DOI": "10.1126\/science.1137065", + "article-title": "Wnt induces LRP6 signalosomes and promotes dishevelled-dependent LRP6 phosphorylation", + "volume": "316", + "author": "Bilic", + "year": "2007", + "journal-title": "Science" + }, + { + "key": "bib7", + "doi-asserted-by": "publisher", + "first-page": "438", + "DOI": "10.1038\/15674", + "article-title": "The structural basis for specificity of substrate and recruitment peptides for cyclin-dependent kinases", + "volume": "1", + "author": "Brown", + "year": "1999", + "journal-title": "Nature Cell Biology" + }, + { + "key": "bib8", + "doi-asserted-by": "publisher", + "first-page": "179", + "DOI": "10.1128\/MCB.00550-10", + "article-title": "Mitogen-activated protein kinases promote WNT\/\u03b2-catenin signaling via phosphorylation of LRP6", + "volume": "31", + "author": "Cervenka", + "year": "2011", + "journal-title": "Molecular and Cellular Biology" + }, + { + "key": "bib9", + "doi-asserted-by": "publisher", + "first-page": "35040", + "DOI": "10.1074\/jbc.M109.047456", + "article-title": "G Protein-coupled receptor kinases phosphorylate LRP6 in the Wnt pathway", + "volume": "284", + "author": "Chen", + "year": "2009", + "journal-title": "The Journal of Biological Chemistry" + }, + { + "key": "bib10", + "doi-asserted-by": "publisher", + "first-page": "12", + "DOI": "10.1107\/S0907444909042073", + "article-title": "MolProbity: all-atom structure validation for macromolecular crystallography", + "volume": "66", + "author": "Chen", + "year": "2010", + "journal-title": "Acta Crystallographica Section D Biological Crystallography" + }, + { + "key": "bib11", + "doi-asserted-by": "publisher", + "first-page": "1188", + "DOI": "10.1101\/gr.849004", + "article-title": "WebLogo: a sequence logo generator", + "volume": "14", + "author": "Crooks", + "year": "2004", + "journal-title": "Genome Research" + }, + { + "key": "bib12", + "doi-asserted-by": "publisher", + "first-page": "785", + "DOI": "10.1038\/378785a0", + "article-title": "Inhibition of glycogen synthase kinase-3 by insulin mediated by protein kinase B", + "volume": "378", + "author": "Cross", + "year": "1995", + "journal-title": "Nature" + }, + { + "key": "bib13", + "doi-asserted-by": "publisher", + "first-page": "494", + "DOI": "10.1093\/emboj\/cdg068", + "article-title": "Structural basis for recruitment of glycogen synthase kinase 3beta to the axin-APC scaffold complex", + "volume": "22", + "author": "Dajani", + "year": "2003", + "journal-title": "The EMBO Journal" + }, + { + "key": "bib14", + "doi-asserted-by": "publisher", + "first-page": "721", + "DOI": "10.1016\/S0092-8674(01)00374-9", + "article-title": "Crystal structure of glycogen synthase kinase 3 beta: structural basis for phosphate-primed substrate specificity and autoinhibition", + "volume": "105", + "author": "Dajani", + "year": "2001", + "journal-title": "Cell" + }, + { + "key": "bib15", + "doi-asserted-by": "publisher", + "first-page": "867", + "DOI": "10.1038\/nature04170", + "article-title": "Casein kinase 1 gamma couples Wnt receptor activation to cytoplasmic signal transduction", + "volume": "438", + "author": "Davidson", + "year": "2005", + "journal-title": "Nature" + }, + { + "key": "bib16", + "doi-asserted-by": "publisher", + "first-page": "4395", + "DOI": "10.1111\/j.1742-4658.2009.07147.x", + "article-title": "Structural recognition of an optimized substrate for the ephrin family of receptor tyrosine kinaes", + "volume": "276", + "author": "Davis", + "year": "2009", + "journal-title": "The FEBS Journal" + }, + { + "key": "bib17", + "doi-asserted-by": "publisher", + "first-page": "32475", + "DOI": "10.1074\/jbc.M005342200", + "article-title": "Differential regulation of glycogen synthase kinase 3beta by insulin and Wnt signaling", + "volume": "275", + "author": "Ding", + "year": "2000", + "journal-title": "The Journal of Biological Chemistry" + }, + { + "key": "bib18", + "doi-asserted-by": "publisher", + "first-page": "957", + "DOI": "10.1016\/j.devcel.2007.04.001", + "article-title": "Functional redundancy of GSK-3alpha and GSK-3beta in Wnt\/beta-catenin signaling shown by using an allelic series of embryonic stem cell lines", + "volume": "12", + "author": "Doble", + "year": "2007", + "journal-title": "Developmental Cell" + }, + { + "key": "bib19", + "doi-asserted-by": "publisher", + "first-page": "2126", + "DOI": "10.1107\/S0907444904019158", + "article-title": "Coot: model-building tools for molecular graphics", + "volume": "60", + "author": "Emsley", + "year": "2004", + "journal-title": "Acta Crystallographica Section D Biological Crystallography" + }, + { + "key": "bib20", + "doi-asserted-by": "publisher", + "first-page": "1204", + "DOI": "10.1107\/S0907444913000061", + "article-title": "How good are my data and what is the resolution?", + "volume": "69", + "author": "Evans", + "year": "2013", + "journal-title": "Acta Crystallographica Section D Biological Crystallography" + }, + { + "key": "bib21", + "first-page": "14042", + "article-title": "Formation of protein kinase recognition sites by covalent modification of the substrate. Molecular mechanism for the synergistic action of casein kinase II and glycogen synthase kinase 3", + "volume": "262", + "author": "Fiol", + "year": "1987", + "journal-title": "The Journal of Biological Chemistry" + }, + { + "key": "bib22", + "doi-asserted-by": "publisher", + "first-page": "1", + "DOI": "10.1042\/0264-6021:3590001", + "article-title": "GSK3 takes centre stage more than 20 years after its discovery", + "volume": "359", + "author": "Frame", + "year": "2001", + "journal-title": "Biochemical Journal" + }, + { + "key": "bib23", + "doi-asserted-by": "publisher", + "first-page": "1321", + "DOI": "10.1016\/j.febslet.2006.06.035", + "article-title": "A common phosphate binding site explains the unique substrate specificity of GSK3 and its inactivation by phosphorylation", + "volume": "7", + "author": "Frame", + "year": "2001", + "journal-title": "Molecular Cell" + }, + { + "key": "bib24", + "doi-asserted-by": "publisher", + "first-page": "5572", + "DOI": "10.1093\/emboj\/16.18.5572", + "article-title": "Crystal structure of the activated insulin receptor tyrosine kinase in complex with peptide substrate and ATP analog", + "volume": "16", + "author": "Hubbard", + "year": "1997", + "journal-title": "The EMBO Journal" + }, + { + "key": "bib25", + "doi-asserted-by": "publisher", + "first-page": "30621", + "DOI": "10.1074\/jbc.M604633200", + "article-title": "Identification of novel glycogen synthase kinase-3beta substrate-interacting residues suggests a common mechanism for substrate recognition", + "volume": "281", + "author": "Ilouz", + "year": "2006", + "journal-title": "The Journal of Biological Chemistry" + }, + { + "key": "bib26", + "doi-asserted-by": "publisher", + "first-page": "95", + "DOI": "10.1016\/j.tibs.2003.12.004", + "article-title": "The glamour and gloom of glycogen synthase kinase-3", + "volume": "29", + "author": "Jope", + "year": "2004", + "journal-title": "Trends in Biochemical Sciences" + }, + { + "key": "bib27", + "doi-asserted-by": "publisher", + "first-page": "9", + "DOI": "10.1016\/j.molcel.2011.03.004", + "article-title": "Catalytic control in the EGF receptor and its connection to general kinase regulatory mechanisms", + "volume": "42", + "author": "Jura", + "year": "2011", + "journal-title": "Molecular Cell" + }, + { + "key": "bib28", + "doi-asserted-by": "publisher", + "first-page": "125", + "DOI": "10.1107\/S0907444909047337", + "article-title": "Xds", + "volume": "66", + "author": "Kabsch", + "year": "2010", + "journal-title": "Acta Crystallographica Section D Biological Crystallography" + }, + { + "key": "bib29", + "doi-asserted-by": "publisher", + "first-page": "35", + "DOI": "10.1186\/1756-6606-2-35", + "article-title": "Abnormalities in brain structure and behavior in GSK-3alpha mutant mice", + "volume": "2", + "author": "Kaidanovich-Beilin", + "year": "2009", + "journal-title": "Molecular Brain" + }, + { + "key": "bib30", + "doi-asserted-by": "publisher", + "first-page": "40", + "DOI": "10.3389\/fnmol.2011.00040", + "article-title": "GSK-3: functional insights from cell biology and animal models", + "volume": "4", + "author": "Kaidanovich-Beilin", + "year": "2011", + "journal-title": "Frontiers in Molecular Neuroscience" + }, + { + "key": "bib31", + "doi-asserted-by": "publisher", + "first-page": "735", + "DOI": "10.1016\/j.bbrc.2013.03.103", + "article-title": "Development of Akt-activated GSK3beta inhibitory peptide", + "volume": "434", + "author": "Kim", + "year": "2013a", + "journal-title": "Biochemical and Biophysical Research" + }, + { + "key": "bib32", + "doi-asserted-by": "publisher", + "first-page": "867", + "DOI": "10.1126\/science.1232389", + "article-title": "Wnt stabilization of beta-catenin reveals principles for morphogen receptor-scaffold assemblies", + "volume": "340", + "author": "Kim", + "year": "2013b", + "journal-title": "Science" + }, + { + "key": "bib33", + "doi-asserted-by": "publisher", + "first-page": "3359", + "DOI": "10.1016\/j.bmcl.2008.04.034", + "article-title": "Synthesis and structure based optimization of novel Akt inhibitors", + "volume": "18", + "author": "Lippa", + "year": "2008", + "journal-title": "Bioorganic & Medicinal Chemistry Letters" + }, + { + "key": "bib34", + "doi-asserted-by": "publisher", + "first-page": "627", + "DOI": "10.1016\/j.molcel.2006.10.009", + "article-title": "A chaperone-dependent GSK3beta transitional intermediate mediates activation-loop autophosphorylation", + "volume": "24", + "author": "Lochhead", + "year": "2006", + "journal-title": "Molecular Cell" + }, + { + "key": "bib35", + "doi-asserted-by": "publisher", + "first-page": "3171", + "DOI": "10.1096\/fj.09-143743", + "article-title": "New potent dual inhibitors of CK2 and Pim kinases: discovery and structural insights", + "volume": "24", + "author": "Lopez-Ramos", + "year": "2010", + "journal-title": "The FASEB Journal" + }, + { + "key": "bib36", + "doi-asserted-by": "publisher", + "first-page": "6646", + "DOI": "10.1093\/emboj\/16.22.6646", + "article-title": "The crystal structure of a phosphorylase kinase peptide substrate complex: kinase substrate recognition", + "volume": "16", + "author": "Lowe", + "year": "1997", + "journal-title": "The EMBO Journal" + }, + { + "key": "bib37", + "doi-asserted-by": "publisher", + "first-page": "a007880", + "DOI": "10.1101\/cshperspect.a007880", + "article-title": "Frizzled and LRP5\/6 receptors for Wnt\/beta-catenin signaling", + "volume": "4", + "author": "MacDonald", + "year": "2012", + "journal-title": "Cold Spring Harbour Perspectives in Biology" + }, + { + "key": "bib38", + "doi-asserted-by": "publisher", + "first-page": "16115", + "DOI": "10.1074\/jbc.M800327200", + "article-title": "Wnt signal amplification via activity, cooperativity, and regulation of multiple intracellular PPPSP motifs in the Wnt co-receptor LRP6", + "volume": "283", + "author": "MacDonald", + "year": "2008", + "journal-title": "The Journal of Biological Chemistry" + }, + { + "key": "bib39", + "doi-asserted-by": "publisher", + "first-page": "273", + "DOI": "10.1038\/nsb780", + "article-title": "Crystal structure of a transition state mimic of the catalytic subunit of cAMP-dependent protein kinase", + "volume": "9", + "author": "Madhusudan", + "year": "2002", + "journal-title": "Nature Structural Biology" + }, + { + "key": "bib40", + "doi-asserted-by": "publisher", + "first-page": "176", + "DOI": "10.1002\/pro.5560030203", + "article-title": "cAMP-dependent protein kinase: crystallographic insights into substrate recognition and phosphotransfer", + "volume": "3", + "author": "Madhusudan", + "year": "1994", + "journal-title": "Protein Science" + }, + { + "key": "bib41", + "doi-asserted-by": "publisher", + "first-page": "821", + "DOI": "10.1038\/nchembio.452", + "article-title": "Dynamics connect substrate recognition to catalysis in protein kinase A", + "volume": "6", + "author": "Masterson", + "year": "2010", + "journal-title": "Nature Chemical Biology" + }, + { + "key": "bib42", + "doi-asserted-by": "publisher", + "first-page": "6969", + "DOI": "10.1073\/pnas.1102701108", + "article-title": "Dynamically committed, uncommitted, and quenched states encoded in protein kinase A revealed by NMR spectroscopy", + "volume": "108", + "author": "Masterson", + "year": "2011", + "journal-title": "Proceedings of the National Academy of Sciences of the United States of America" + }, + { + "key": "bib43", + "doi-asserted-by": "publisher", + "first-page": "1571", + "DOI": "10.1038\/sj.emboj.7600633", + "article-title": "Role that phosphorylation of GSK3 plays in insulin and Wnt signalling defined by knockin analysis", + "volume": "24", + "author": "McManus", + "year": "2005", + "journal-title": "The EMBO Journal" + }, + { + "key": "bib44", + "first-page": "533", + "article-title": "Glycogen synthase kinase-3 (GSK-3) inhibitors reach the clinic", + "volume": "11", + "author": "Medina", + "year": "2008", + "journal-title": "Current Opinion in Drug Discovery and Development" + }, + { + "key": "bib45", + "doi-asserted-by": "publisher", + "first-page": "921", + "DOI": "10.1016\/S0969-2126(97)00246-3", + "article-title": "A binary complex of the catalytic subunit of cAMP-dependent protein kinase and adenosine further defines conformational flexibility", + "volume": "5", + "author": "Narayana", + "year": "1997", + "journal-title": "Structure" + }, + { + "key": "bib46", + "doi-asserted-by": "publisher", + "first-page": "2551", + "DOI": "10.1007\/s00018-010-0329-3", + "article-title": "Regulation of Lrp6 phosphorylation", + "volume": "67", + "author": "Niehrs", + "year": "2010", + "journal-title": "Cellular and Molecular Life Sciences" + }, + { + "key": "bib47", + "doi-asserted-by": "publisher", + "first-page": "6314", + "DOI": "10.1128\/MCB.00763-08", + "article-title": "Tissue-specific role of glycogen synthase kinase 3beta in glucose homeostasis and insulin action", + "volume": "28", + "author": "Patel", + "year": "2008", + "journal-title": "Molecular and Cellular Biology" + }, + { + "key": "bib48", + "doi-asserted-by": "publisher", + "first-page": "e4046", + "DOI": "10.1371\/journal.pone.0004046", + "article-title": "Direct inhibition of GSK3beta by the phosphorylated cytoplasmic domain of LRP6 in Wnt\/beta-catenin signaling", + "volume": "3", + "author": "Piao", + "year": "2008", + "journal-title": "PLOS ONE" + }, + { + "key": "bib49", + "doi-asserted-by": "publisher", + "first-page": "986", + "DOI": "10.1016\/j.str.2013.03.012", + "article-title": "Structures of Down syndrome kinases, DYRKs, reveal mechanisms of kinase activation and substrate recognition", + "volume": "21", + "author": "Soundararajan", + "year": "2013", + "journal-title": "Structure" + }, + { + "key": "bib50", + "doi-asserted-by": "publisher", + "first-page": "a007898", + "DOI": "10.1101\/cshperspect.a007898", + "article-title": "The beta-catenin destruction complex", + "volume": "5", + "author": "Stamos", + "year": "2013", + "journal-title": "Cold Spring Harbour Perspectives in Biology" + }, + { + "key": "bib51", + "doi-asserted-by": "publisher", + "first-page": "505607", + "DOI": "10.4061\/2011\/505607", + "article-title": "What are the bona fide GSK3 substrates?", + "volume": "2011", + "author": "Sutherland", + "year": "2011", + "journal-title": "International Journal of Alzheimer\u2019s Disease" + }, + { + "key": "bib52", + "doi-asserted-by": "publisher", + "first-page": "9312", + "DOI": "10.1021\/jm301034u", + "article-title": "Selectivity, cocrystal structures, and neuroprotective properties of leucettines, a family of protein kinase inhibitors derived from the marine sponge alkaloid leucettamine B", + "volume": "55", + "author": "Tahtouh", + "year": "2012", + "journal-title": "Journal of Medicinal Chemistry" + }, + { + "key": "bib53", + "doi-asserted-by": "publisher", + "first-page": "179", + "DOI": "10.1254\/jphs.08R28FM", + "article-title": "Drug development targeting the glycogen synthase kinase-3beta (GSK-3beta)-mediated signal transduction pathway: inhibitors of the Wnt\/beta-catenin signaling pathway as novel anticancer drugs", + "volume": "109", + "author": "Takahashi-Yanaga", + "year": "2009", + "journal-title": "Journal of Pharmacological Sciences" + }, + { + "key": "bib54", + "doi-asserted-by": "publisher", + "first-page": "593", + "DOI": "10.1038\/89624", + "article-title": "Structure of GSK3beta reveals a primed phosphorylation mechanism", + "volume": "8", + "author": "ter Haar", + "year": "2001", + "journal-title": "Nature Structural Biology" + }, + { + "key": "bib55", + "doi-asserted-by": "publisher", + "first-page": "1558", + "DOI": "10.2741\/3324", + "article-title": "GSK-3 inhibitors and insulin receptor signaling in health, disease, and therapeutics", + "volume": "14", + "author": "Wada", + "year": "2009", + "journal-title": "Frontiers in Bioscience" + }, + { + "key": "bib56", + "first-page": "14566", + "article-title": "Glycogen synthase kinase-3 beta is a dual specificity kinase differentially regulated by tyrosine and serine\/threonine phosphorylation", + "volume": "269", + "author": "Wang", + "year": "1994", + "journal-title": "The Journal of Biological Chemistry" + }, + { + "key": "bib57", + "doi-asserted-by": "publisher", + "first-page": "e4926", + "DOI": "10.1371\/journal.pone.0004926", + "article-title": "Inhibition of GSK3 phosphorylation of beta-catenin via phosphorylated PPPSPXS motifs of Wnt coreceptor LRP6", + "volume": "4", + "author": "Wu", + "year": "2009", + "journal-title": "PLOS ONE" + }, + { + "key": "bib58", + "doi-asserted-by": "publisher", + "first-page": "873", + "DOI": "10.1038\/nature04185", + "article-title": "A dual-kinase mechanism for Wnt co-receptor phosphorylation and activation", + "volume": "438", + "author": "Zeng", + "year": "2005", + "journal-title": "Nature" + }, + { + "key": "bib59", + "doi-asserted-by": "publisher", + "first-page": "7797", + "DOI": "10.1074\/jbc.272.12.7797", + "article-title": "Target protease specificity of the viral serpin CrmA. Analysis of five caspases", + "volume": "272", + "author": "Zhou", + "year": "1997", + "journal-title": "The Journal of Biological Chemistry" + } + ], + "container-title": [ + "eLife" + ], + "original-title": [], + "language": "en", + "link": [ + { + "URL": "https:\/\/cdn.elifesciences.org\/articles\/01998\/elife-01998-v1.pdf", + "content-type": "application\/pdf", + "content-version": "vor", + "intended-application": "text-mining" + }, + { + "URL": "https:\/\/cdn.elifesciences.org\/articles\/01998\/elife-01998-v1.xml", + "content-type": "application\/xml", + "content-version": "vor", + "intended-application": "text-mining" + } + ], + "deposited": { + "date-parts": [ + [ + 2018, + 8, + 23 + ] + ], + "date-time": "2018-08-23T13:49:30Z", + "timestamp": 1535032170000 + }, + "score": 1.0, + "subtitle": [], + "short-title": [], + "issued": { + "date-parts": [ + [ + 2014, + 3, + 18 + ] + ] + }, + "references-count": 59, + "alternative-id": [ + "10.7554\/eLife.01998" + ], + "URL": "http:\/\/dx.doi.org\/10.7554\/elife.01998", + "archive": [ + "CLOCKSS" + ], + "relation": { + "has-review": [ + { + "id-type": "doi", + "id": "10.7554\/eLife.01998.014", + "asserted-by": "object" + }, + { + "id-type": "doi", + "id": "10.7554\/eLife.01998.015", + "asserted-by": "object" + } + ], + "cites": [] + }, + "ISSN": [ + "2050-084X" + ], + "issn-type": [ + { + "value": "2050-084X", + "type": "electronic" + } + ], + "subject": [ + "General Biochemistry, Genetics and Molecular Biology", + "General Immunology and Microbiology", + "General Neuroscience", + "General Medicine" + ], + "assertion": [ + { + "value": "2013-12-03", + "order": 0, + "name": "received", + "label": "Received", + "group": { + "name": "publication_history", + "label": "Publication History" + } + }, + { + "value": "2014-02-17", + "order": 1, + "name": "accepted", + "label": "Accepted", + "group": { + "name": "publication_history", + "label": "Publication History" + } + }, + { + "value": "2014-03-18", + "order": 2, + "name": "published", + "label": "Published", + "group": { + "name": "publication_history", + "label": "Publication History" + } + } + ] +} diff --git a/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/crossref/response.json b/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/crossref/response.json new file mode 100644 index 0000000000..3249e312f8 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/crossref/response.json @@ -0,0 +1,127 @@ +{ + "_scroll_id": "DnF1ZXJ5VGhlbkZldGNo8AEAAAAAAAJmvhYzbFlNS3k5a1RoT2REVkVvYlhGeUJRAAAAAAAAB38Wa3YzMVZsNkdSYnFEQ2x3a0R6NENVUQAAAAAAAAeBFmt2MzFWbDZHUmJxRENsd2tEejRDVVEAAAAAAAAHiRZrdjMxVmw2R1JicURDbHdrRHo0Q1VRAAAAAAAAB4QWa3YzMVZsNkdSYnFEQ2x3a0R6NENVUQAAAAAAAAeAFmt2MzFWbDZHUmJxRENsd2tEejRDVVEAAAAAAAAHkRZrdjMxVmw2R1JicURDbHdrRHo0Q1VRAAAAAAAAB4IWa3YzMVZsNkdSYnFEQ2x3a0R6NENVUQAAAAAAAAd-Fmt2MzFWbDZHUmJxRENsd2tEejRDVVEAAAAAAAAHhxZrdjMxVmw2R1JicURDbHdrRHo0Q1VRAAAAAAACZr8WM2xZTUt5OWtUaE9kRFZFb2JYRnlCUQAAAAAAAAeDFmt2MzFWbDZHUmJxRENsd2tEejRDVVEAAAAAAAJmwBYzbFlNS3k5a1RoT2REVkVvYlhGeUJRAAAAAAACZsEWM2xZTUt5OWtUaE9kRFZFb2JYRnlCUQAAAAAAAAk7Fmlza1d4VjV2VDVLYy1VWDQ1aERWb0EAAAAAAAAHiBZrdjMxVmw2R1JicURDbHdrRHo0Q1VRAAAAAAAAB4UWa3YzMVZsNkdSYnFEQ2x3a0R6NENVUQAAAAAAAAeGFmt2MzFWbDZHUmJxRENsd2tEejRDVVEAAAAAAAJmxRYzbFlNS3k5a1RoT2REVkVvYlhGeUJRAAAAAAAAB44Wa3YzMVZsNkdSYnFEQ2x3a0R6NENVUQAAAAAAAAeKFmt2MzFWbDZHUmJxRENsd2tEejRDVVEAAAAAAAAJPRZpc2tXeFY1dlQ1S2MtVVg0NWhEVm9BAAAAAAACZscWM2xZTUt5OWtUaE9kRFZFb2JYRnlCUQAAAAAAAmbEFjNsWU1LeTlrVGhPZERWRW9iWEZ5QlEAAAAAAAAHjBZrdjMxVmw2R1JicURDbHdrRHo0Q1VRAAAAAAAAB4sWa3YzMVZsNkdSYnFEQ2x3a0R6NENVUQAAAAAAAAeNFmt2MzFWbDZHUmJxRENsd2tEejRDVVEAAAAAAAAJRBZpc2tXeFY1dlQ1S2MtVVg0NWhEVm9BAAAAAAAAB5cWa3YzMVZsNkdSYnFEQ2x3a0R6NENVUQAAAAAAAAeSFmt2MzFWbDZHUmJxRENsd2tEejRDVVEAAAAAAAAJPxZpc2tXeFY1dlQ1S2MtVVg0NWhEVm9BAAAAAAACZskWM2xZTUt5OWtUaE9kRFZFb2JYRnlCUQAAAAAAAAeTFmt2MzFWbDZHUmJxRENsd2tEejRDVVEAAAAAAAJmyxYzbFlNS3k5a1RoT2REVkVvYlhGeUJRAAAAAAAAB5UWa3YzMVZsNkdSYnFEQ2x3a0R6NENVUQAAAAAAAmbGFjNsWU1LeTlrVGhPZERWRW9iWEZ5QlEAAAAAAAJmyBYzbFlNS3k5a1RoT2REVkVvYlhGeUJRAAAAAAAAB5QWa3YzMVZsNkdSYnFEQ2x3a0R6NENVUQAAAAAAAAlAFmlza1d4VjV2VDVLYy1VWDQ1aERWb0EAAAAAAAJmzBYzbFlNS3k5a1RoT2REVkVvYlhGeUJRAAAAAAAACTwWaXNrV3hWNXZUNUtjLVVYNDVoRFZvQQAAAAAAAAmrFkR4bkItaG5ZUzhXVWFrVnlDdks3WlEAAAAAAAAJrhZEeG5CLWhuWVM4V1Vha1Z5Q3ZLN1pRAAAAAAAACbEWRHhuQi1obllTOFdVYWtWeUN2SzdaUQAAAAAAAmbCFjNsWU1LeTlrVGhPZERWRW9iWEZ5QlEAAAAAAAAJrBZEeG5CLWhuWVM4V1Vha1Z5Q3ZLN1pRAAAAAAAACbAWRHhuQi1obllTOFdVYWtWeUN2SzdaUQAAAAAAAmbDFjNsWU1LeTlrVGhPZERWRW9iWEZ5QlEAAAAAAAAJqxZwaDlETnJ4alJDeUZYdGlLck9XQ3dBAAAAAAAACbIWRHhuQi1obllTOFdVYWtWeUN2SzdaUQAAAAAAAAmtFkR4bkItaG5ZUzhXVWFrVnlDdks3WlEAAAAAAAAHjxZrdjMxVmw2R1JicURDbHdrRHo0Q1VRAAAAAAAAB5AWa3YzMVZsNkdSYnFEQ2x3a0R6NENVUQAAAAAAAAeWFmt2MzFWbDZHUmJxRENsd2tEejRDVVEAAAAAAAAJPhZpc2tXeFY1dlQ1S2MtVVg0NWhEVm9BAAAAAAACZsoWM2xZTUt5OWtUaE9kRFZFb2JYRnlCUQAAAAAAAAmsFnBoOUROcnhqUkN5Rlh0aUtyT1dDd0EAAAAAAAALfRY5R0Y2TXNyRFM3bUJkMHE4SkVQZmNRAAAAAAAACa8WRHhuQi1obllTOFdVYWtWeUN2SzdaUQAAAAAAAmWHFlRHZW4xZGhFUnVPbXBCWlNhWF9kYWcAAAAAAAAJQRZpc2tXeFY1dlQ1S2MtVVg0NWhEVm9BAAAAAAACZZEWVEdlbjFkaEVSdU9tcEJaU2FYX2RhZwAAAAAAAAlCFmlza1d4VjV2VDVLYy1VWDQ1aERWb0EAAAAAAAJmzRYzbFlNS3k5a1RoT2REVkVvYlhGeUJRAAAAAAAACa0WcGg5RE5yeGpSQ3lGWHRpS3JPV0N3QQAAAAAAAAt-FjlHRjZNc3JEUzdtQmQwcThKRVBmY1EAAAAAAAAJsxZEeG5CLWhuWVM4V1Vha1Z5Q3ZLN1pRAAAAAAAAB5gWa3YzMVZsNkdSYnFEQ2x3a0R6NENVUQAAAAAAAArHFjJyVjB4UWROUlV1NEZvcGFSNUFFNEEAAAAAAAJljBZUR2VuMWRoRVJ1T21wQlpTYVhfZGFnAAAAAAAACUMWaXNrV3hWNXZUNUtjLVVYNDVoRFZvQQAAAAAAAAeZFmt2MzFWbDZHUmJxRENsd2tEejRDVVEAAAAAAAAJrhZwaDlETnJ4alJDeUZYdGlLck9XQ3dBAAAAAAAAC4EWOUdGNk1zckRTN21CZDBxOEpFUGZjUQAAAAAAAAm1FkR4bkItaG5ZUzhXVWFrVnlDdks3WlEAAAAAAAJliBZUR2VuMWRoRVJ1T21wQlpTYVhfZGFnAAAAAAAACskWMnJWMHhRZE5SVXU0Rm9wYVI1QUU0QQAAAAAAAmWLFlRHZW4xZGhFUnVPbXBCWlNhWF9kYWcAAAAAAAAJRRZpc2tXeFY1dlQ1S2MtVVg0NWhEVm9BAAAAAAACZs4WM2xZTUt5OWtUaE9kRFZFb2JYRnlCUQAAAAAAAAmvFnBoOUROcnhqUkN5Rlh0aUtyT1dDd0EAAAAAAAALfxY5R0Y2TXNyRFM3bUJkMHE4SkVQZmNRAAAAAAAACbQWRHhuQi1obllTOFdVYWtWeUN2SzdaUQAAAAAAAAeaFmt2MzFWbDZHUmJxRENsd2tEejRDVVEAAAAAAAAKyBYyclYweFFkTlJVdTRGb3BhUjVBRTRBAAAAAAACZYkWVEdlbjFkaEVSdU9tcEJaU2FYX2RhZwAAAAAAAAlGFmlza1d4VjV2VDVLYy1VWDQ1aERWb0EAAAAAAAJmzxYzbFlNS3k5a1RoT2REVkVvYlhGeUJRAAAAAAAACbAWcGg5RE5yeGpSQ3lGWHRpS3JPV0N3QQAAAAAAAAuAFjlHRjZNc3JEUzdtQmQwcThKRVBmY1EAAAAAAAAJthZEeG5CLWhuWVM4V1Vha1Z5Q3ZLN1pRAAAAAAAACbIWcGg5RE5yeGpSQ3lGWHRpS3JPV0N3QQAAAAAAAArMFjJyVjB4UWROUlV1NEZvcGFSNUFFNEEAAAAAAAJlihZUR2VuMWRoRVJ1T21wQlpTYVhfZGFnAAAAAAAACUcWaXNrV3hWNXZUNUtjLVVYNDVoRFZvQQAAAAAAAAmxFnBoOUROcnhqUkN5Rlh0aUtyT1dDd0EAAAAAAAAJsxZwaDlETnJ4alJDeUZYdGlLck9XQ3dBAAAAAAAAC4IWOUdGNk1zckRTN21CZDBxOEpFUGZjUQAAAAAAAAm3FkR4bkItaG5ZUzhXVWFrVnlDdks3WlEAAAAAAAAK0hYyclYweFFkTlJVdTRGb3BhUjVBRTRBAAAAAAAACsoWMnJWMHhRZE5SVXU0Rm9wYVI1QUU0QQAAAAAAAmWPFlRHZW4xZGhFUnVPbXBCWlNhWF9kYWcAAAAAAAAJSBZpc2tXeFY1dlQ1S2MtVVg0NWhEVm9BAAAAAAAAC4MWOUdGNk1zckRTN21CZDBxOEpFUGZjUQAAAAAAAAm0FnBoOUROcnhqUkN5Rlh0aUtyT1dDd0EAAAAAAAALhBY5R0Y2TXNyRFM3bUJkMHE4SkVQZmNRAAAAAAAACbgWRHhuQi1obllTOFdVYWtWeUN2SzdaUQAAAAAAAAuFFjlHRjZNc3JEUzdtQmQwcThKRVBmY1EAAAAAAAAKyxYyclYweFFkTlJVdTRGb3BhUjVBRTRBAAAAAAACZZIWVEdlbjFkaEVSdU9tcEJaU2FYX2RhZwAAAAAAAAlJFmlza1d4VjV2VDVLYy1VWDQ1aERWb0EAAAAAAAJm0BYzbFlNS3k5a1RoT2REVkVvYlhGeUJRAAAAAAAACbYWcGg5RE5yeGpSQ3lGWHRpS3JPV0N3QQAAAAAAAAuGFjlHRjZNc3JEUzdtQmQwcThKRVBmY1EAAAAAAAAJuRZEeG5CLWhuWVM4V1Vha1Z5Q3ZLN1pRAAAAAAACZZAWVEdlbjFkaEVSdU9tcEJaU2FYX2RhZwAAAAAAAArQFjJyVjB4UWROUlV1NEZvcGFSNUFFNEEAAAAAAAJljRZUR2VuMWRoRVJ1T21wQlpTYVhfZGFnAAAAAAAACUoWaXNrV3hWNXZUNUtjLVVYNDVoRFZvQQAAAAAAAmbRFjNsWU1LeTlrVGhPZERWRW9iWEZ5QlEAAAAAAAAJtRZwaDlETnJ4alJDeUZYdGlLck9XQ3dBAAAAAAAAC4cWOUdGNk1zckRTN21CZDBxOEpFUGZjUQAAAAAAAArPFjJyVjB4UWROUlV1NEZvcGFSNUFFNEEAAAAAAAAKzRYyclYweFFkTlJVdTRGb3BhUjVBRTRBAAAAAAAACs4WMnJWMHhRZE5SVXU0Rm9wYVI1QUU0QQAAAAAAAmWdFlRHZW4xZGhFUnVPbXBCWlNhWF9kYWcAAAAAAAJljhZUR2VuMWRoRVJ1T21wQlpTYVhfZGFnAAAAAAAACbcWcGg5RE5yeGpSQ3lGWHRpS3JPV0N3QQAAAAAAAAm4FnBoOUROcnhqUkN5Rlh0aUtyT1dDd0EAAAAAAAALiBY5R0Y2TXNyRFM3bUJkMHE4SkVQZmNRAAAAAAAACboWRHhuQi1obllTOFdVYWtWeUN2SzdaUQAAAAAAAAm7FkR4bkItaG5ZUzhXVWFrVnlDdks3WlEAAAAAAAAK0RYyclYweFFkTlJVdTRGb3BhUjVBRTRBAAAAAAACZZQWVEdlbjFkaEVSdU9tcEJaU2FYX2RhZwAAAAAAAAlLFmlza1d4VjV2VDVLYy1VWDQ1aERWb0EAAAAAAAJm0hYzbFlNS3k5a1RoT2REVkVvYlhGeUJRAAAAAAAACbkWcGg5RE5yeGpSQ3lGWHRpS3JPV0N3QQAAAAAAAAuJFjlHRjZNc3JEUzdtQmQwcThKRVBmY1EAAAAAAAAJvBZEeG5CLWhuWVM4V1Vha1Z5Q3ZLN1pRAAAAAAAAC4oWOUdGNk1zckRTN21CZDBxOEpFUGZjUQAAAAAAAArUFjJyVjB4UWROUlV1NEZvcGFSNUFFNEEAAAAAAAJlmRZUR2VuMWRoRVJ1T21wQlpTYVhfZGFnAAAAAAAACUwWaXNrV3hWNXZUNUtjLVVYNDVoRFZvQQAAAAAAAmbTFjNsWU1LeTlrVGhPZERWRW9iWEZ5QlEAAAAAAAAJuhZwaDlETnJ4alJDeUZYdGlLck9XQ3dBAAAAAAAAC4sWOUdGNk1zckRTN21CZDBxOEpFUGZjUQAAAAAAAmWTFlRHZW4xZGhFUnVPbXBCWlNhWF9kYWcAAAAAAAALjBY5R0Y2TXNyRFM3bUJkMHE4SkVQZmNRAAAAAAAACtUWMnJWMHhRZE5SVXU0Rm9wYVI1QUU0QQAAAAAAAmWfFlRHZW4xZGhFUnVPbXBCWlNhWF9kYWcAAAAAAAAJvRZEeG5CLWhuWVM4V1Vha1Z5Q3ZLN1pRAAAAAAAACb4WRHhuQi1obllTOFdVYWtWeUN2SzdaUQAAAAAAAAm7FnBoOUROcnhqUkN5Rlh0aUtyT1dDd0EAAAAAAAALjRY5R0Y2TXNyRFM3bUJkMHE4SkVQZmNRAAAAAAAACb8WRHhuQi1obllTOFdVYWtWeUN2SzdaUQAAAAAAAAm8FnBoOUROcnhqUkN5Rlh0aUtyT1dDd0EAAAAAAAAK0xYyclYweFFkTlJVdTRGb3BhUjVBRTRBAAAAAAACZZUWVEdlbjFkaEVSdU9tcEJaU2FYX2RhZwAAAAAAAAlNFmlza1d4VjV2VDVLYy1VWDQ1aERWb0EAAAAAAAAJvRZwaDlETnJ4alJDeUZYdGlLck9XQ3dBAAAAAAAACb4WcGg5RE5yeGpSQ3lGWHRpS3JPV0N3QQAAAAAAAAuOFjlHRjZNc3JEUzdtQmQwcThKRVBmY1EAAAAAAAAJwBZEeG5CLWhuWVM4V1Vha1Z5Q3ZLN1pRAAAAAAAACb8WcGg5RE5yeGpSQ3lGWHRpS3JPV0N3QQAAAAAAAArYFjJyVjB4UWROUlV1NEZvcGFSNUFFNEEAAAAAAAJllxZUR2VuMWRoRVJ1T21wQlpTYVhfZGFnAAAAAAAACU4WaXNrV3hWNXZUNUtjLVVYNDVoRFZvQQAAAAAAAmbUFjNsWU1LeTlrVGhPZERWRW9iWEZ5QlEAAAAAAAAJwBZwaDlETnJ4alJDeUZYdGlLck9XQ3dBAAAAAAAAC48WOUdGNk1zckRTN21CZDBxOEpFUGZjUQAAAAAAAAnBFkR4bkItaG5ZUzhXVWFrVnlDdks3WlEAAAAAAAALkBY5R0Y2TXNyRFM3bUJkMHE4SkVQZmNRAAAAAAAACtYWMnJWMHhRZE5SVXU0Rm9wYVI1QUU0QQAAAAAAAmWWFlRHZW4xZGhFUnVPbXBCWlNhWF9kYWcAAAAAAAAJTxZpc2tXeFY1dlQ1S2MtVVg0NWhEVm9BAAAAAAACZtUWM2xZTUt5OWtUaE9kRFZFb2JYRnlCUQAAAAAAAAnBFnBoOUROcnhqUkN5Rlh0aUtyT1dDd0EAAAAAAAALkRY5R0Y2TXNyRFM3bUJkMHE4SkVQZmNRAAAAAAAACcIWRHhuQi1obllTOFdVYWtWeUN2SzdaUQAAAAAAAAuSFjlHRjZNc3JEUzdtQmQwcThKRVBmY1EAAAAAAAAK1xYyclYweFFkTlJVdTRGb3BhUjVBRTRBAAAAAAACZZgWVEdlbjFkaEVSdU9tcEJaU2FYX2RhZwAAAAAAAAlQFmlza1d4VjV2VDVLYy1VWDQ1aERWb0EAAAAAAAJm1hYzbFlNS3k5a1RoT2REVkVvYlhGeUJRAAAAAAAACcIWcGg5RE5yeGpSQ3lGWHRpS3JPV0N3QQAAAAAAAAuTFjlHRjZNc3JEUzdtQmQwcThKRVBmY1EAAAAAAAJlmhZUR2VuMWRoRVJ1T21wQlpTYVhfZGFnAAAAAAAAB5sWa3YzMVZsNkdSYnFEQ2x3a0R6NENVUQAAAAAAAArZFjJyVjB4UWROUlV1NEZvcGFSNUFFNEEAAAAAAAJlmxZUR2VuMWRoRVJ1T21wQlpTYVhfZGFnAAAAAAAACVEWaXNrV3hWNXZUNUtjLVVYNDVoRFZvQQAAAAAAAAuUFjlHRjZNc3JEUzdtQmQwcThKRVBmY1EAAAAAAAAJwxZwaDlETnJ4alJDeUZYdGlLck9XQ3dBAAAAAAAAC5UWOUdGNk1zckRTN21CZDBxOEpFUGZjUQAAAAAAAAnDFkR4bkItaG5ZUzhXVWFrVnlDdks3WlEAAAAAAAAJUxZpc2tXeFY1dlQ1S2MtVVg0NWhEVm9BAAAAAAAACtsWMnJWMHhRZE5SVXU0Rm9wYVI1QUU0QQAAAAAAAmWcFlRHZW4xZGhFUnVPbXBCWlNhWF9kYWcAAAAAAAAJUhZpc2tXeFY1dlQ1S2MtVVg0NWhEVm9BAAAAAAAACtoWMnJWMHhRZE5SVXU0Rm9wYVI1QUU0QQAAAAAAAAnEFnBoOUROcnhqUkN5Rlh0aUtyT1dDd0EAAAAAAAALlhY5R0Y2TXNyRFM3bUJkMHE4SkVQZmNRAAAAAAAACcQWRHhuQi1obllTOFdVYWtWeUN2SzdaUQAAAAAAAArcFjJyVjB4UWROUlV1NEZvcGFSNUFFNEEAAAAAAAAK3RYyclYweFFkTlJVdTRGb3BhUjVBRTRBAAAAAAACZZ4WVEdlbjFkaEVSdU9tcEJaU2FYX2RhZwAAAAAAAAlUFmlza1d4VjV2VDVLYy1VWDQ1aERWb0EAAAAAAAJm1xYzbFlNS3k5a1RoT2REVkVvYlhGeUJRAAAAAAAACcUWcGg5RE5yeGpSQ3lGWHRpS3JPV0N3QQAAAAAAAAuXFjlHRjZNc3JEUzdtQmQwcThKRVBmY1EAAAAAAAAJxRZEeG5CLWhuWVM4V1Vha1Z5Q3ZLN1pRAAAAAAAACt4WMnJWMHhRZE5SVXU0Rm9wYVI1QUU0QQAAAAAAAArfFjJyVjB4UWROUlV1NEZvcGFSNUFFNEEAAAAAAAJloBZUR2VuMWRoRVJ1T21wQlpTYVhfZGFnAAAAAAAACVUWaXNrV3hWNXZUNUtjLVVYNDVoRFZvQQAAAAAAAArgFjJyVjB4UWROUlV1NEZvcGFSNUFFNEEAAAAAAAAJxhZwaDlETnJ4alJDeUZYdGlLck9XQ3dBAAAAAAAAC5gWOUdGNk1zckRTN21CZDBxOEpFUGZjUQAAAAAAAAnGFkR4bkItaG5ZUzhXVWFrVnlDdks3WlEAAAAAAAAJxxZEeG5CLWhuWVM4V1Vha1Z5Q3ZLN1pRAAAAAAAACuQWMnJWMHhRZE5SVXU0Rm9wYVI1QUU0QQAAAAAAAmWhFlRHZW4xZGhFUnVPbXBCWlNhWF9kYWcAAAAAAAAJVhZpc2tXeFY1dlQ1S2MtVVg0NWhEVm9BAAAAAAACZtgWM2xZTUt5OWtUaE9kRFZFb2JYRnlCUQAAAAAAAAnHFnBoOUROcnhqUkN5Rlh0aUtyT1dDd0EAAAAAAAALmRY5R0Y2TXNyRFM3bUJkMHE4SkVQZmNRAAAAAAAACuIWMnJWMHhRZE5SVXU0Rm9wYVI1QUU0QQAAAAAAAmWiFlRHZW4xZGhFUnVPbXBCWlNhWF9kYWcAAAAAAAAK4RYyclYweFFkTlJVdTRGb3BhUjVBRTRBAAAAAAACZaMWVEdlbjFkaEVSdU9tcEJaU2FYX2RhZwAAAAAAAAlXFmlza1d4VjV2VDVLYy1VWDQ1aERWb0EAAAAAAAJm2RYzbFlNS3k5a1RoT2REVkVvYlhGeUJRAAAAAAAACcgWcGg5RE5yeGpSQ3lGWHRpS3JPV0N3QQAAAAAAAAuaFjlHRjZNc3JEUzdtQmQwcThKRVBmY1EAAAAAAAAJyBZEeG5CLWhuWVM4V1Vha1Z5Q3ZLN1pRAAAAAAACZtoWM2xZTUt5OWtUaE9kRFZFb2JYRnlCUQAAAAAAAArjFjJyVjB4UWROUlV1NEZvcGFSNUFFNEEAAAAAAAJlpBZUR2VuMWRoRVJ1T21wQlpTYVhfZGFnAAAAAAAACVgWaXNrV3hWNXZUNUtjLVVYNDVoRFZvQQAAAAAAAmbbFjNsWU1LeTlrVGhPZERWRW9iWEZ5QlE=", + "took": 89, + "timed_out": false, + "_shards": { + "total": 240, + "successful": 240, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": 113109351, + "max_score": 1.0, + "hits": [ + { + "_index": "crossref", + "_type": "item", + "_id": "10.26850/1678-4618eqj.v35.1.2010.p41-46", + "_score": 1.0, + "_source": { + "timestamp": 1540825815212, + "pid": "10.26850/1678-4618eqj.v35.1.2010.p41-46", + "blob": "eF69V9tyE0cQ/ZUpPXsvs9rVDSpVYIeKKSdc7OQhhIfRbksae2+emRUWFP+e07OSLYFNMFWEMiDP9HSf7j590afByavTwUwMZBwmo0kWR3I0ngTpSE7o+jJcD7NQhkmM2zaVOB4ciYG2tqMCjz4NCuUoaJVxFr++ewfByZGQ+Mnev/8MUTW3zqjcsYWnl8rZWfvLM3HdqVI75fSahG0bFziyTqi6EA7/UMUCtdtJqFqVG6utaBai0O3GNDUJXQu3IjHvyitRmG7Jj3HWrpSpVE6d07kqRWsI4GCoqa2AhtY0bWOpCMUFHi8ayPIda1bCUFFouwrWuinJ/dPFsYpF3pSNgeYC+oASYlCgPXJDtitdKE5rYem6ozonXB9Ah72cis6QyFUt5iRaMmyVPFh2wKoKQEplr3pMB897C3B87pSu8egDQZXlyFnvYLkBwAou8p12K6iEe3vy/RmJinDDRns/CjHfePPPjfqoS63qrbuv+/ghSqTVkeisrpde8Nx1BdWIClI8tWKbLVy8EJw8G4pjxFgXZHYv8gYBqZF9H2G4q4SMYwFDw3l+JsCA665xR177PiHW2oIeotSVdj7l5CjvdSD886ZzImO0xVjGwUgsn0A3SNZVMIZ3Z69OYCNJ0kzIML0VTEXV4DKQ4glE3uxE0vBO161I+DTaUpXZDg4t9M1+ifCp6hBQw6T/5D/DuGdIETQm11wcC1VaguQSXKn5+e/KuZWmjp8vVKXLDZ9edHNdN3y2IxGfLrSxzttZLDg/7D8be4+jV2+PT09YaOVcO4sibzBszDKK8SfA3ySQ08k4GE7j8QBVCIRfa7mF9QypCw8xPYe14itMCuXBOFS5Vfq9bhunxO/KaCVOOrQKEsfKrFW58hbuIvGXLtWXcdiz+ePBAGMnwTgejx6F+4zgqunDcAfyNX38+DNQDgOgHAfj4Wj6KJS/UUm1Em+p0Eu0qsOI/iywMsiyURrI0RBgmZIoEXQm1HuQN13N3T7G6en5+R9M2sHtSBmwcEXVnLh0BlIOp5lPedOZnPjo2DTWQp0/zRvDhzJkbQX5zrsdPE5XaDuqavk+S+NJMsySKbjPkjyUWIAV8kgK0CmS6YVMZ8NkFk//ZuX3Ty48T6b96EKvpJsHrWUTmSUy+Za1bCbjmcy+z5rbtB7uJSKBCgtQJzoviZG23bzEXOLWUpcYAxD778H759szVrdtEcVNWDTa94hHzfngNrFFMN8cJHfdlGi5bGPoU4iloA62XqAn7vwBPXOHia1zFkLVd/7NHSM8f+DWFY7x7hC3IUwDp0J9HXY12Tacm6i5tJHPTdiu2ogQJO680TZg0VrThxe6pEhOZSTTqcdWO8KLIlBtW3Kb7rvpwNGNCypdY2SxGCYW5FywJmO3Emv0+L2bnVd7eqK2WPQV+79CtxiPJXqqQ1ZWlF892oWuti3leqFBcZ+DHfP8duc5tvs0kD9GQ14AbwlkD9hjMTwdTjCywWiDanUgO/eKXzmhoXjT6SoUL0PfMXYV4JuGF0ASvQj//7Ivmf08FU0FvX2h7D762Zlze6mUuQKzsTRov1hAzg9rxgt6UG09lC/JmBvyKys2rgq7jq+mrbiN5psoDWPGAL4a7oCogYMeJbPpOE15RN/fo2IZyOwijmf+5xtd4267voew+2kFmIJKtQl0HRRqwyt67FPtPbm3tclxNo0nafIgyCSIkws5ZpBp8jBINMZk20a3dOpJdJfn89u9H0sjlqfFti791wCjWl0crtL9duQ0Vrz+CwHv83vfCDxRHiAUb4E03ZLG/1IcMMdzTC19Z0rl4PO/owMhhQ==" + } + }, + { + "_index": "crossref", + "_type": "item", + "_id": "10.18410/jebmh/2018/575", + "_score": 1.0, + "_source": { + "timestamp": 1541495459193, + "pid": "10.18410/jebmh/2018/575", + "blob": "eF6dVNtO4zAQ/RXLz02TlKa0fQslQFYlrXrRakEImcRpDUmctZ0KVPHvO5NQKKsu2903ezyey5kzZ0vPJyEdEuo6bbffdR37kT/ka7vjuH3bO/Voi1ChdcUTcNrShBlulUwZDdfbW/RqkUGLdLy7u1dwLRVPxfN+PAzAKrOWCn9sKUtTkQlmhCzQcAfPK7HheKG+2ogiwR8py0X2grYZy9Gg+c+KFzFHUyqUNvQVv0I6rtBuxbIqDLw6YA3n8wiD085Jd2B1vF4HQ7xdTh2KP3OeP3CsiZ6ednt1ClmpJsFISa0hdG2NpcKsbhsjJ7yUWpg3NIzIuTYsL/Hd67rdQdfr9Byn9kSk0AFTIE6W61pOb+H0h5437PRuMPhBOF23RXoNnIAGf/5jMq/rDdzByVfJBkPXGXqDo5KZl7Iu9hFwKFhmwZRFnHGss6weMqHXPLFkkYkC3f7OheVsjOHWxpRD206e24kUbalW9pdUs95HmlgPL5/GupFZ1eC5o2VhvRUNxNqVXyoBRICiNyyrGvTfWQAU/XDkGY+NkoWID3kDTWqGQbtPyKUt3etHQ0P1nrRjmdtMa260DcNk92WS2r5iyON7oO69dX8RRv64DXbMAqVxmGlisbLMRLxbA6oF8J0pYaDjNY+fRLFC91iiv7E2XOlmYegGFmnvZdd1VeiSxyIVwJa67o+RNYAcM7Hd5OuFr2e8O9Gu8988+OdKUEjeWaA/UUAqsRJITiMMUBN3PChWyM16qTXIjIEPhWHAUrXnVU+r9tmVU+/+mG94RpYlOas0/NCajABv0IU9iBOZQ7SG8rtjrVsxykTO1BNwVhsl4jdRS1mmOTYRKw4q0CjnQa3wjtIK76j1/W1UHwD5ZL5Ynv8gk4iMxmEUjvwxmc4mF+E4IH50Tq79yL8MroNoQSYXZOovQjjO8XwVzKbBnNxM5otgRibTq8WVP74OR8s5CSNyvpxd+tPlrEW+B/MFOQuiS39cQ3xoAN8aVSEyJcFGJKjZ5IxpnpBrnogYwCesSMgVZ5lZxww0tx4WWzUr3Hf6VqfvuvT1F2Qb/lQ=" + } + }, + { + "_index": "crossref", + "_type": "item", + "_id": "10.26686/lew.v0i0.1330", + "_score": 1.0, + "_source": { + "timestamp": 1543961469345, + "pid": "10.26686/lew.v0i0.1330", + "blob": "eF61Vu9v2zYQ/VcIoQU2wLIl+UdtY9iAdR3aIeuAOl2AFvlASyeLiURqJGVHC/q/7x1tZ07gdNuHAf4gU6e7d+/e3fE++um3d9FSRGkyzGaz+WxU0264TVQyTMfjJBqISDnXUQGb+6iQnuJWWu/w9/PndPEqGYgUv+vrL7CUa+etzD37++5Gerdsv7+sSLynnfhEspa6EI3SqukasZMbEhb+RCWdsJST9nUv6K4lq0jnVAgpXOe8VBrPfGoKYUqxsWbnK+Er6UVtzK0TjrzwRuRGe6U7Ep0uyMKARN5ZC7/iQq5NZ+MajjZmS1Y3OB2KlUIckSVJFqxl0dX+GYCwtCQdHKx7kc1fitKaRryYC86pgnNGcBoypAa4L5jY6VBcVgppKkdC6WD4iAjmwHQe9Km25dQBkgliltxA7CqVV/jwBEM6fSk4leDLyYYOFB0itRKMic6RCwanFXhrcFyZujjQIn42FjSsOrulPuSjvBPvdG7gdNW1bU1MFyeoCjyosn8MfmfsLVkXPi3I5VatiaMqi7+N2VjZAn94fWPWIq8kiwQFdV7lbohQgjWl8q6WdhDw7uHTnQRLSAGfaBBxpO6JB6b5EZtHQGvyOyK9rzCHR6lRil9PJXiwHRzJRJX2WhsIaaFQqqVXW4I0e9PpjfgmkC6h2roMFqhQIdJZnE1FT9K6bweitVQYIJJB0iU1siYUETQpeOBcY69ALkNiF7W65QAgGMRR09amh0/oRIoV2a3KkTrbruCHewWQ8N7kedcCnIEdF20vqw+EjqnDyVvjWuVlrXyPtwV6CZ0Fvn88RwvTDig7CFHi2RLFJWvkKLqjf4fqBWGHBuYwDo1t+5Aeq+CgEcUqdmfrcr4CHIDrI3aMw9VqUzF7wMB8I05jnlJ1ZBZKBakIuFOYDNzPZ0X1AL2RFkwUh8og4cagLmclVJhuzWODAZwwpJ/n5aybQ3HeYggCITt7bZqm01yahxIfatRjoOXe2OF3o8MM5SkMTZXq7nRS86nsfGUsD+P7SJalqlUQBB9c4/UGyuU/0S/S4RQnJTqq7vnoUjWA6vjQ0R8dj1w+LpV1PvrCXyMiSoHzOIf0eapj2kfvVqv37D/KJrNxnM2SJGLjhpo1MZIoTbNsEtxiLO6dvrbGObgLpznqCLt0yN4Kao1TkDNO7iPuCkz8puX308l4MV3MX02TJFjy7mEDjpEl6TxOsziZXGbpcjJZjqef2PmTBcVm2E7ZQEz2GwoU091zwWbpZLYYT6ZfC5Yt03SZLP5VMN+3AewNeNCyjsOMqylUE6pSrqIiNrrGiAvZ/8N2/fjhgr1V3rfL0ai4GxZGDY3djL62uuOHIhbxun9USOx1HR8wQj5HtFRDfdZolTPQray7PeMP1Q7aAOhboMF3J7AccJkbN8TEgn6VHMp8qP8cXby5Gh1yHxVmp2sjixHfLkZputhfMbQnVKaIJbaNyo8ajjzd+ZgbSm8YDG94LKAYE9jtVR5tof6TN8ckTvyM2qKMcDv5f5E6hb6SFv0c5xXlPOj/G+JOu5ZyVWIuPek+96hqDv3ucaLDxciiIzwkxf0ImkMnHrUVmvH3QynER41ZANowcDC8r6hGBTceu+NCrS3m9ylaLC8432vy+BjmSc59jOl5C1nxNskPw6aUtSO+AfIF5XwzZ+NFOklm8+eaOZnE6YybOXu1nMyf768J+nm2b+a/Uz9u9CveSVfHRaRPrz1LcVWZsG0xxvsfAlNnWQxXxYF4E9ZwuPjwuGanvKFOLlLR9Ze/AK09uI8=" + } + }, + { + "_index": "crossref", + "_type": "item", + "_id": "10.30707/etd2017.mckinnis.r", + "_score": 1.0, + "_source": { + "timestamp": 1544632883454, + "pid": "10.30707/etd2017.mckinnis.r", + "blob": "eF69VE1v2zAM/SuCz3FiO9++DdulWIoWSXtZkINi0wlRWdIkOYgR5L+PtNMCLdZtpwE2LNOk3uPToy+RbfYK/RFclIvoTinUBr3YBBlAPGs8gfMY2mggosLoADrEpaklakq/RG/L7Y4TnPG+lu4lduCDwyKg4bxKKg9XSvj2cMcoaTIcJ/NkPoJQZkk6H9bFC2qNfugYx0EFDnQBPi5MowOVJBQu4eAA6GUb3W8ixpNNOBrmvb1EsqpQobwhdnQORJ7hozWgUq1YyVYDA1SyRtXyl/viewfMUQ8/G0bleIXOh+jKuzyvVxw5hmDz0ag8D0uDQ+MOo790gd43UFIpqURaxla64JnrVjdK7Xasx1un7xotHFBBXxqwJillbakynU4ms3EynmZJ0gnC23IC8yMdF3Ga0fWUTvPpMk+zH9zVB2xOG4g047vngJ5O6yZ4Ge/bd1Sktc6cft8Gn9xALAciW/Y71VDvexulWTqbd5qaxvWKfmVvEBBHLT3xzKxfNexyC+O4lXTYn7Y1ZLz/J4Mu4fwpWrZYjCfTyZ9En+WTNM/G/yZ6wKB6K2+KI5QNTd1BrIEs73PxCKSZZSd78VCJVaOLozOmFpvGgjuhN84PxBNIKuXVF12KR4e6QCtVNxihtZ0pSjIhOBplnkOKo/aE3NyGhIaGJla3NbsyirtKq2R3Xq/vWvb2+vzHcL3+ApKoSns=" + } + }, + { + "_index": "crossref", + "_type": "item", + "_id": "10.5209/rev_esmp.2001.v17.n1.13", + "_score": 1.0, + "_source": { + "timestamp": 1502718217884, + "pid": "10.5209/rev_esmp.2001.v17.n1.13", + "blob": "eF6NlG9v0zAQxr+KlVcgNZmdpM2ftxsvJm2AGBMSo5rc+Lp5OHZkO2XTtO/OXdptFIrgXes73z1+7nd5TE4+nCYtSwTP5jlvjjxsriH0Q5ZzLrKNqDIrMlEkM5boEEZQmPyYKBkhHaSPAf9eXeVciBkTfMbK5fIJUwcPa33/S126H8bVHXSRbiTHru9HqzsZtbPJEqNyjLfOU/Axkeu1NnobwwMK3+gNWCp4Aisvqdxa9to80NEXCFEakzxRInYGD7aDtHOjpW4oKzm9uHg/NRZNXaf5ollQCSGKMhWLvJkU9NCvgBQkhVg0k2I3+g7o5Ni7ELD0dNo5T4cio8oKBhd03BkTdU9i+oHiZV4v8qqaN5xPmX+YNkfDyLcltZ/CdJ/6oaPzlKM4/pmLludt0XxNyFltFdwfajbneSXqXFR1jTUPNKtmrMZmOKEDzaqU16koP4uiLYu2qLbN4sMwqblDH6w0KQ5cdwbIhGFcGR1uQaXOGm0p7d9YXH46o3K3MQ7t0ZG6z5TTmfM3R/9FX/oyWpWuHvbGu3Fm3Bonqh2qNt2pR56e3zF4jUSg+o004/SyVwLQ3NdEMAiqdwjofvYLPBNqyv2wxkl1Pa2BmPOa13neNBmfl1WOCS+Cw57agKBHPLFRonMehx7RVKLzXYijytg52CDvgH0Er53KJjqfDZ8AvbS4Dj5oJRXDVRrMGPEKMAXsXCqvFXtzeXz+lsRTG7AxVa7HbtsxPf+cNqsjtHvpv6O/IXrdTSvZsrU0AQi5zgOiuV38Pb4LUdfzxaJEvA/zvfsoiOogcwLpTkVFzPGy5XwHOH1lptGQ+ldrziRzK9g+Wdu1tFEbBpYZyfBrg34xCIP8NnK+Fs7IybO/WqxdYMGtPDAwv7lNFUAFBN0ly6eflnaEwg==" + } + }, + { + "_index": "crossref", + "_type": "item", + "_id": "10.1345/1542-6270(2001)035<1096:rahcri>2.0.co;2", + "_score": 1.0, + "_source": { + "timestamp": 1502583760457, + "pid": "10.1345/1542-6270(2001)035<1096:rahcri>2.0.co;2", + "blob": "eF6dVG1r2zAQ/ivCn1qIUsmvsTcGIRttx7qWJGOwUoYiK7U6xzKS3CWU/vfdOUlLRtfCEn+wT/fo3p7nHoKPl+dBQQLOhjyKkxOexCFNw4wdhYzxYxYl7znL08KKSlr9IRyyoTTvwmBAAu1cp0oAPwSl8Iq2wnoHn9fXCB2Q/ObmEdxaq5Z6vY/BswyxrlvcKenRO7iqhF0JaWpzuyFHK1VqKerj4AbcROcrY9HrIRDLpa618No0aMDjW32v8CO4MJUAIBnj3Uux0vUGzZcrYQPI4XXwZ4A6cnUI/a5rB4HeBM8FxBBkdoiemHXwiBlC7cqqRioqTddgvQys57PZ1750zlJGWThiCH9qfV/6Sq0WCksPeJb3LTOdlQoNE2ucg5t7qzQWjXyIF5eqNU773VS8hsK8WLV4HoejNIyilMEPPf+eGE8GJO6Htj9FOIYLGU8oiynL52FUxHkRsh/YmEA3pVq/FCthYTKKspTFSfZiLLCOBoRHLwbLKBtRHs0ZK8KwiHfB/Kbts7mDNjSipkA2LWuFPfg2/YInlfdtcXJSroel0UNjb0/+i9T0aWYlXWye5ga9Ce5N3W17EiUYGBTQ0F1iwNB9iq3VMGr0F3XXJ/08555Qe0dVgwasabQ89N5rcMuh0vxuaiPKn726eMJG+A/5cJSDZHOI03aLWrtKlXQb+i1JPlXoDmi5v6Zn3Wx8+olc4cWy15zDFKVpvGo8Lc1KaFQeSH//2itSIjVBEb+gi85bLXdyXYraKeSMtAqYt90aB/xkcZamuDj+wU8WDQg8PH6JMyyi8PB4zrOCR0XItwT12gNDUGjTSiwg042pN047IpqSnF2c0okZk6kqO+mFU+S8qfRCe2PddvnUXgHVPCwZqjFj1OvrW3I6PptMz/stObmELdm3BHoGzVKWPqczrxQZN0BjR8yS7Begr5QV7aZHteJ2x5w8pZyzLHj8A8nNo4g=" + } + }, + { + "_index": "crossref", + "_type": "item", + "_id": "10.15707/disem.2015.26.3.002", + "_score": 1.0, + "_source": { + "timestamp": 1502556112306, + "pid": "10.15707/disem.2015.26.3.002", + "blob": "eF6NVE1vnDAQ/SsWh54W1jYYA7eq6SHql9SkhzZZVcY2CQ1gBCbNKsp/74x3Nx/qVukFwczzm8fMG99HJ19Oo4pEjCZMSCrXpp1tn3DKRMLzJE0o5dGKRO08L9YA8j4yytt4VJOf4fPiAqErUmw2DwAbJ9u0d88J8bBa/LWbEH0fqaZpu1b51g0Y2EC6UX3bbfHQdzVcrciZczfxJ2sjYHzlwOWiC5pdLnVqSniXUv/fIckpwEuNz5yy6AF1gHY72UHbWLtl8CCIQvT07OwzKo1YKfM4KySLENzbvrb4T1GeAQFEZrdM2mLk3eTmGdhCVLsJgyxBMmNHN7d+30nf9nb2qh8xn4mC84yXOaUB+VeX8xVJV4TJDZYPaTyP9WAEeUzTmMlzllWsqFj+A/sQtYOxd8eKCcqFyBnjKQXaI8UkjBSK8aPFZEyLmPFzllcZqwTfFfPbMaj5BX0YVBeDQ1rdWWzCt68fMXPt/Vit1+YuMa5N3HS1ft128eNUTFxvX0zm1nXLvgFi79Eh3qsApx30jFMLwwQVt6pbgsKnSYa5G/d76JwyP4OjmaAFlTCGLEmLXHAGiHGpu3a+tibekb22Bo+a5xeCZ9gCD5HBq3awE8zPQ3/QW7B1qoa98Fvbj53b9hYko7ZD5eC0D2AlRd4/IsjbK7DrljRuIv7akhNk6axZP4Oc2FvbuREJyekwQ8nFh5mgDAjGxvWgBuhhsw+vYS01urhX0w2MYPZTq/c726hutuguDWqOWzmjAjv4TyvjhbEiPD/qLoHu4vk5LStKK3qwMl5A2IQUxT91Drde6cvF5AKetW7gNtCFhMVWuhEQF6IkT7tucOMLhtBUA1RpSjEtFWC1yDEtZYFPUWBCaEjUdYq0nEHCiAKuGiDXmGaBymYhrTII1aWGEEhghxpaGKxR1w3yWgHPmpeIxbsHGZUgFXwqbcOFpPAuKxqkgcvliDzEGszXBf61lmXQvVO8k8+NwZpBmTIC8nUpg+SSQU3NbTiY7kmC2Y45M3gqOJO8eWa9nTnV1W4kIs7K6OEPZADnVg==" + } + }, + { + "_index": "crossref", + "_type": "item", + "_id": "10.2202/1234-1234.1004", + "_score": 1.0, + "_source": { + "timestamp": 1502749524652, + "pid": "10.2202/1234-1234.1004", + "blob": "eF6VVGFP2zAQ/StWPuxT09pO0ib5yhAwsYFWpklD1eQmbmNw7Mh2gArx33fnliEm2DSpSpPz+d7zu3d+TD5enCU1SRidck75jPEsT/ExZZTmyYQkyvtRtpDzmLQiyHQQLnj4vL7mlC4mhMFvtXqCzMHJjXo4VGMFK3C7H9c3sgm4IVnaRllttzsiTEsurVZBNUKTZaOkaSSmn4v7ZAX/YgyddbjrMRGbjdJKBGUNBnB5q+4kfiSfRq0kOepU2u1igY3old7h0nK8TZ4wGWhJhwBpY0eDVChEz5bLL5EVq7Iy5UVeROBe9muJwEm2iOf3dnTADQJHznoPxWK0sQ6DbIq1WjlYr8JBpqB66YPoB1zPqzzL5gsQl8bMPyVkIGExIdUKweMqbkc4TtkipUVKqyuW1RmvKf+RoM7KtPLhLayC8kVeFTyfF/w9rBLalb8HVqYsv+K85kXN8z1Y2A2RzQ3IYIROofuq0VHqYVxr5TvZptZoZTDtnx759vUcq3UhDPVs1j5MW6um1m1nfzNg+ruDbbreverindXjXi52MKtJD5TBOc/kpQYLOmtUg0l3Qo/xTC+djz5p7b3RVrQ/o79ZQUtazvN5OZ1nOSugTy9O8q9IeLBqgIgJAmRw0MEACqG3wM7kAzkOnWo8sRtyOvbCkK9q2wEGYj5rGB33XeggHWklOXHjDl9P+vUpUsba0oS0tT1A7IV+fo0D0aA3e+FuQSsfnGoO07IR2ks0TeMkmGs/x68MyhYFzXlWVu8YFGc8mxBevmUaCg7NUl5eUVpnVZ1XB4finREVRvIvchzDOWyvGnIxDCDZaOACkD7eBqGT5NLZAI2COUetPitjnQo7ci7MdhRbuVfs/2V++gUcDnPL" + } + }, + { + "_index": "crossref", + "_type": "item", + "_id": "10.2202/1234-1234.1006", + "_score": 1.0, + "_source": { + "timestamp": 1502749524653, + "pid": "10.2202/1234-1234.1006", + "blob": "eF6VVF1r2zAU/SvCD3uqEkmWEsdvYxttofsg6RishKHYSqJOloIsdw2l/333KmlLR7sxCI59dHXP8blHvivefz4valJwNhKCiTEXpaR4GXHGJsUJKWzfD6aFmrui1cnQnY6ph8erK8HY9IRw+C2X91C5i2Ztb4/duOIKt/fD6to0CTcUi9DY4MJmT7RvyZfgbLKNdmTRWOMbg+UX+lexhH89pG2IuOuu0Ou1dVYnGzwCuLyxNwYfircOQADWurNuj8iZjsYV91gFekzEzrQJg0cNDNDzxeJTlsNnZUWFkiozdqZbGWQsyqnMysMQQRQA72Loe2iW0SZEBPkIe7VmF3qbjv4k25k+6W6H63ImS8lLMInlyj+94+CdOiGzJZLnVdyOdILxKWWKstkll7Wsasm+F2iw9a25fYlLMTGVMyXkRJWvcVUwJ/kaWUW5vBSiFqoW8kCW9rus5hps8NpRGLttXB7Sblg5229NS4N31mPZP8PxdX6B3bYp7erxuL0dtcGOQtyM/5Y8+jjBlq72z6Z4E9xwsIsfU+rpUTJE5kG8cZC9GLxtsOhGuyG/09Pkc07a8Mu7oNsfOdhcsYpVEzmpRpNSqukEKh519M9E9JDRBIhPGmyIMMEEDmG2IMfkDfmQtrbpSViTs6HTnsztZgscyPngYU7cN+2SiaQ15DQOe7w97VZnKBl7G59oGzqgOBj9cJtPQoPZ7HT8CV71KdrmeEzW2vUGQ9NEA+E6HOBnAeVTxaSQTL0SUDzckCZRvRQaBgktqaguGYN41kwdE4ofi+wwin+yY242g4MT7DfkY2hB6J7ODQAGPgNRg+bGHGz5fy/vfwMeHmcs" + } + }, + { + "_index": "crossref", + "_type": "item", + "_id": "10.5433/1980-511.2008v3n1p107", + "_score": 1.0, + "_source": { + "timestamp": 1502757844220, + "pid": "10.5433/1980-511.2008v3n1p107", + "blob": "eF6NVNtu2zAM/RXBz7Ej2Vac+G1r91Cg2IpegGFdMCiWkmqTJU+Ss17Qfx9pJ72h7fZgQKYoHvLwkHfJ4ZejpCYJoxkvi2LKFnOacsaynNL5trCsY7RKJiTRIfRKgutdIkVUaSd8DPB7eYmeE1JNCOPL5T24dl6t9fWTqPhe9PHKeXxwl4j1WhstonYWDUu43uitwp/kwOsQtbCOfPQ6Ony6Fq02N3h5rG0gBy5EkQDQ+4E+WOm/95SqBTl3RskXoU6d9HrTq9vkHvEhZeWVbVTauN5GAKNgPTo7+4wZJntavibo3Kp2pbCWpOCcY4rB9b5RY/4uBIg2WBvn0cgyDCZV54KOOxKjbhXU0XZ4X7JZnlcMmKSD50uCWQnsMviqJeIP9xgAAXPKypSxlFXnLK9zXuf0G7KTaCvV9WtonOYVr+Zlmeevo0EvoaPYzlfAqpTOU8bPKa3LsqblCBZvuiGbn0CEFSYFdejGKGSh61dGhyslU2eNtuj2bw1dnB5juKsYu3o6ldeZdDpzfjP9D6GmD82U6ermWUO3zvQjbcVO1DbdpQ663BehjGqid1Y36LQVph9qexTBIBnp/ljjhPwxzAHjdE7nszmbZbSYFXwGHg95hGdJBBiECBYbBdDhoZMRmEKZnaptRg61z8gJkta4QW57AgfFXViYFB+0FFKRT6Ag2QtD4HzsQPDaCkwZYysbU+lagBgJ3x+HcWs8qLQV/hdwFaLXzW4W18IEheJpvAKRjfP+TKrFgi/yGePsHakWE5K/pVRapDkotagZrznbKRV3y8AwJv9Ix8FugCUsAixRtzBBcAxu5RXpvJN9dIGA0HssQhh9KyQYwNOGvoVT41oizLAG5O/eRUFulR9ZfYN/2D6CSIddUAh7go/X4kk3xGbMFfbi/V9eupFL" + } + } + ] + } +} diff --git a/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/crossref/s.json b/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/crossref/s.json new file mode 100644 index 0000000000..a201d51a03 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/crossref/s.json @@ -0,0 +1,4 @@ +{ + "scroll_id": "DnF1ZXJ5VGhlbkZldGNo8AEAAAAAAAJmBRYzbFlNS3k5a1RoT2REVkVvYlhGeUJRAAAAAAAAByUWa3YzMVZsNkdSYnFEQ2x3a0R6NENVUQAAAAAAAAckFmt2MzFWbDZHUmJxRENsd2tEejRDVVEAAAAAAAAHPBZrdjMxVmw2R1JicURDbHdrRHo0Q1VRAAAAAAAABykWa3YzMVZsNkdSYnFEQ2x3a0R6NENVUQAAAAAAAAcqFmt2MzFWbDZHUmJxRENsd2tEejRDVVEAAAAAAAAHKBZrdjMxVmw2R1JicURDbHdrRHo0Q1VRAAAAAAAABycWa3YzMVZsNkdSYnFEQ2x3a0R6NENVUQAAAAAAAAc6Fmt2MzFWbDZHUmJxRENsd2tEejRDVVEAAAAAAAAHJhZrdjMxVmw2R1JicURDbHdrRHo0Q1VRAAAAAAACZgcWM2xZTUt5OWtUaE9kRFZFb2JYRnlCUQAAAAAAAAcrFmt2MzFWbDZHUmJxRENsd2tEejRDVVEAAAAAAAJmBhYzbFlNS3k5a1RoT2REVkVvYlhGeUJRAAAAAAACZggWM2xZTUt5OWtUaE9kRFZFb2JYRnlCUQAAAAAAAAjjFmlza1d4VjV2VDVLYy1VWDQ1aERWb0EAAAAAAAAHLRZrdjMxVmw2R1JicURDbHdrRHo0Q1VRAAAAAAAABywWa3YzMVZsNkdSYnFEQ2x3a0R6NENVUQAAAAAAAAcvFmt2MzFWbDZHUmJxRENsd2tEejRDVVEAAAAAAAJmCxYzbFlNS3k5a1RoT2REVkVvYlhGeUJRAAAAAAAABz0Wa3YzMVZsNkdSYnFEQ2x3a0R6NENVUQAAAAAAAAcuFmt2MzFWbDZHUmJxRENsd2tEejRDVVEAAAAAAAAI4hZpc2tXeFY1dlQ1S2MtVVg0NWhEVm9BAAAAAAACZgkWM2xZTUt5OWtUaE9kRFZFb2JYRnlCUQAAAAAAAmYPFjNsWU1LeTlrVGhPZERWRW9iWEZ5QlEAAAAAAAAHMxZrdjMxVmw2R1JicURDbHdrRHo0Q1VRAAAAAAAABzAWa3YzMVZsNkdSYnFEQ2x3a0R6NENVUQAAAAAAAAc0Fmt2MzFWbDZHUmJxRENsd2tEejRDVVEAAAAAAAAI5BZpc2tXeFY1dlQ1S2MtVVg0NWhEVm9BAAAAAAAABzEWa3YzMVZsNkdSYnFEQ2x3a0R6NENVUQAAAAAAAAcyFmt2MzFWbDZHUmJxRENsd2tEejRDVVEAAAAAAAAI5RZpc2tXeFY1dlQ1S2MtVVg0NWhEVm9BAAAAAAACZg4WM2xZTUt5OWtUaE9kRFZFb2JYRnlCUQAAAAAAAAc1Fmt2MzFWbDZHUmJxRENsd2tEejRDVVEAAAAAAAJmERYzbFlNS3k5a1RoT2REVkVvYlhGeUJRAAAAAAAABzYWa3YzMVZsNkdSYnFEQ2x3a0R6NENVUQAAAAAAAmYSFjNsWU1LeTlrVGhPZERWRW9iWEZ5QlEAAAAAAAJmDBYzbFlNS3k5a1RoT2REVkVvYlhGeUJRAAAAAAAABzcWa3YzMVZsNkdSYnFEQ2x3a0R6NENVUQAAAAAAAAjmFmlza1d4VjV2VDVLYy1VWDQ1aERWb0EAAAAAAAJmExYzbFlNS3k5a1RoT2REVkVvYlhGeUJRAAAAAAAACOEWaXNrV3hWNXZUNUtjLVVYNDVoRFZvQQAAAAAAAAlVFkR4bkItaG5ZUzhXVWFrVnlDdks3WlEAAAAAAAAJUhZEeG5CLWhuWVM4V1Vha1Z5Q3ZLN1pRAAAAAAAACVEWRHhuQi1obllTOFdVYWtWeUN2SzdaUQAAAAAAAmYKFjNsWU1LeTlrVGhPZERWRW9iWEZ5QlEAAAAAAAAJUxZEeG5CLWhuWVM4V1Vha1Z5Q3ZLN1pRAAAAAAAACVcWRHhuQi1obllTOFdVYWtWeUN2SzdaUQAAAAAAAmYNFjNsWU1LeTlrVGhPZERWRW9iWEZ5QlEAAAAAAAAJURZwaDlETnJ4alJDeUZYdGlLck9XQ3dBAAAAAAAACVQWRHhuQi1obllTOFdVYWtWeUN2SzdaUQAAAAAAAAlWFkR4bkItaG5ZUzhXVWFrVnlDdks3WlEAAAAAAAAHOBZrdjMxVmw2R1JicURDbHdrRHo0Q1VRAAAAAAAABzkWa3YzMVZsNkdSYnFEQ2x3a0R6NENVUQAAAAAAAAc7Fmt2MzFWbDZHUmJxRENsd2tEejRDVVEAAAAAAAAI5xZpc2tXeFY1dlQ1S2MtVVg0NWhEVm9BAAAAAAACZhAWM2xZTUt5OWtUaE9kRFZFb2JYRnlCUQAAAAAAAAlSFnBoOUROcnhqUkN5Rlh0aUtyT1dDd0EAAAAAAAALIxY5R0Y2TXNyRFM3bUJkMHE4SkVQZmNRAAAAAAAACVoWRHhuQi1obllTOFdVYWtWeUN2SzdaUQAAAAAAAmTOFlRHZW4xZGhFUnVPbXBCWlNhWF9kYWcAAAAAAAAI6BZpc2tXeFY1dlQ1S2MtVVg0NWhEVm9BAAAAAAACZM8WVEdlbjFkaEVSdU9tcEJaU2FYX2RhZwAAAAAAAAjpFmlza1d4VjV2VDVLYy1VWDQ1aERWb0EAAAAAAAJmFBYzbFlNS3k5a1RoT2REVkVvYlhGeUJRAAAAAAAACVMWcGg5RE5yeGpSQ3lGWHRpS3JPV0N3QQAAAAAAAAskFjlHRjZNc3JEUzdtQmQwcThKRVBmY1EAAAAAAAAJWBZEeG5CLWhuWVM4V1Vha1Z5Q3ZLN1pRAAAAAAAABz4Wa3YzMVZsNkdSYnFEQ2x3a0R6NENVUQAAAAAAAAptFjJyVjB4UWROUlV1NEZvcGFSNUFFNEEAAAAAAAJk0BZUR2VuMWRoRVJ1T21wQlpTYVhfZGFnAAAAAAAACOoWaXNrV3hWNXZUNUtjLVVYNDVoRFZvQQAAAAAAAAc_Fmt2MzFWbDZHUmJxRENsd2tEejRDVVEAAAAAAAAJVRZwaDlETnJ4alJDeUZYdGlLck9XQ3dBAAAAAAAACyUWOUdGNk1zckRTN21CZDBxOEpFUGZjUQAAAAAAAAlZFkR4bkItaG5ZUzhXVWFrVnlDdks3WlEAAAAAAAJk0hZUR2VuMWRoRVJ1T21wQlpTYVhfZGFnAAAAAAAACm4WMnJWMHhRZE5SVXU0Rm9wYVI1QUU0QQAAAAAAAmTRFlRHZW4xZGhFUnVPbXBCWlNhWF9kYWcAAAAAAAAI6xZpc2tXeFY1dlQ1S2MtVVg0NWhEVm9BAAAAAAACZhUWM2xZTUt5OWtUaE9kRFZFb2JYRnlCUQAAAAAAAAlUFnBoOUROcnhqUkN5Rlh0aUtyT1dDd0EAAAAAAAALJhY5R0Y2TXNyRFM3bUJkMHE4SkVQZmNRAAAAAAAACVsWRHhuQi1obllTOFdVYWtWeUN2SzdaUQAAAAAAAAdAFmt2MzFWbDZHUmJxRENsd2tEejRDVVEAAAAAAAAKbxYyclYweFFkTlJVdTRGb3BhUjVBRTRBAAAAAAACZNMWVEdlbjFkaEVSdU9tcEJaU2FYX2RhZwAAAAAAAAjsFmlza1d4VjV2VDVLYy1VWDQ1aERWb0EAAAAAAAJmFhYzbFlNS3k5a1RoT2REVkVvYlhGeUJRAAAAAAAACVYWcGg5RE5yeGpSQ3lGWHRpS3JPV0N3QQAAAAAAAAsnFjlHRjZNc3JEUzdtQmQwcThKRVBmY1EAAAAAAAAJXBZEeG5CLWhuWVM4V1Vha1Z5Q3ZLN1pRAAAAAAAACVcWcGg5RE5yeGpSQ3lGWHRpS3JPV0N3QQAAAAAAAApwFjJyVjB4UWROUlV1NEZvcGFSNUFFNEEAAAAAAAJk1BZUR2VuMWRoRVJ1T21wQlpTYVhfZGFnAAAAAAAACO0WaXNrV3hWNXZUNUtjLVVYNDVoRFZvQQAAAAAAAAlYFnBoOUROcnhqUkN5Rlh0aUtyT1dDd0EAAAAAAAAJWRZwaDlETnJ4alJDeUZYdGlLck9XQ3dBAAAAAAAACygWOUdGNk1zckRTN21CZDBxOEpFUGZjUQAAAAAAAAldFkR4bkItaG5ZUzhXVWFrVnlDdks3WlEAAAAAAAAKchYyclYweFFkTlJVdTRGb3BhUjVBRTRBAAAAAAAACnEWMnJWMHhRZE5SVXU0Rm9wYVI1QUU0QQAAAAAAAmTVFlRHZW4xZGhFUnVPbXBCWlNhWF9kYWcAAAAAAAAI7hZpc2tXeFY1dlQ1S2MtVVg0NWhEVm9BAAAAAAAACykWOUdGNk1zckRTN21CZDBxOEpFUGZjUQAAAAAAAAlbFnBoOUROcnhqUkN5Rlh0aUtyT1dDd0EAAAAAAAALLRY5R0Y2TXNyRFM3bUJkMHE4SkVQZmNRAAAAAAAACV4WRHhuQi1obllTOFdVYWtWeUN2SzdaUQAAAAAAAAssFjlHRjZNc3JEUzdtQmQwcThKRVBmY1EAAAAAAAAKcxYyclYweFFkTlJVdTRGb3BhUjVBRTRBAAAAAAACZNYWVEdlbjFkaEVSdU9tcEJaU2FYX2RhZwAAAAAAAAjvFmlza1d4VjV2VDVLYy1VWDQ1aERWb0EAAAAAAAJmFxYzbFlNS3k5a1RoT2REVkVvYlhGeUJRAAAAAAAACVoWcGg5RE5yeGpSQ3lGWHRpS3JPV0N3QQAAAAAAAAsqFjlHRjZNc3JEUzdtQmQwcThKRVBmY1EAAAAAAAAJXxZEeG5CLWhuWVM4V1Vha1Z5Q3ZLN1pRAAAAAAACZNcWVEdlbjFkaEVSdU9tcEJaU2FYX2RhZwAAAAAAAAp0FjJyVjB4UWROUlV1NEZvcGFSNUFFNEEAAAAAAAJk2BZUR2VuMWRoRVJ1T21wQlpTYVhfZGFnAAAAAAAACPAWaXNrV3hWNXZUNUtjLVVYNDVoRFZvQQAAAAAAAmYYFjNsWU1LeTlrVGhPZERWRW9iWEZ5QlEAAAAAAAAJXBZwaDlETnJ4alJDeUZYdGlLck9XQ3dBAAAAAAAACysWOUdGNk1zckRTN21CZDBxOEpFUGZjUQAAAAAAAAp1FjJyVjB4UWROUlV1NEZvcGFSNUFFNEEAAAAAAAAKdxYyclYweFFkTlJVdTRGb3BhUjVBRTRBAAAAAAAACnYWMnJWMHhRZE5SVXU0Rm9wYVI1QUU0QQAAAAAAAmTZFlRHZW4xZGhFUnVPbXBCWlNhWF9kYWcAAAAAAAJk2hZUR2VuMWRoRVJ1T21wQlpTYVhfZGFnAAAAAAAACV0WcGg5RE5yeGpSQ3lGWHRpS3JPV0N3QQAAAAAAAAleFnBoOUROcnhqUkN5Rlh0aUtyT1dDd0EAAAAAAAALLhY5R0Y2TXNyRFM3bUJkMHE4SkVQZmNRAAAAAAAACWEWRHhuQi1obllTOFdVYWtWeUN2SzdaUQAAAAAAAAlgFkR4bkItaG5ZUzhXVWFrVnlDdks3WlEAAAAAAAAKeRYyclYweFFkTlJVdTRGb3BhUjVBRTRBAAAAAAACZNsWVEdlbjFkaEVSdU9tcEJaU2FYX2RhZwAAAAAAAAjxFmlza1d4VjV2VDVLYy1VWDQ1aERWb0EAAAAAAAJmGRYzbFlNS3k5a1RoT2REVkVvYlhGeUJRAAAAAAAACV8WcGg5RE5yeGpSQ3lGWHRpS3JPV0N3QQAAAAAAAAsvFjlHRjZNc3JEUzdtQmQwcThKRVBmY1EAAAAAAAAJYhZEeG5CLWhuWVM4V1Vha1Z5Q3ZLN1pRAAAAAAAACzAWOUdGNk1zckRTN21CZDBxOEpFUGZjUQAAAAAAAAp4FjJyVjB4UWROUlV1NEZvcGFSNUFFNEEAAAAAAAJk3BZUR2VuMWRoRVJ1T21wQlpTYVhfZGFnAAAAAAAACPIWaXNrV3hWNXZUNUtjLVVYNDVoRFZvQQAAAAAAAmYaFjNsWU1LeTlrVGhPZERWRW9iWEZ5QlEAAAAAAAAJYBZwaDlETnJ4alJDeUZYdGlLck9XQ3dBAAAAAAAACzEWOUdGNk1zckRTN21CZDBxOEpFUGZjUQAAAAAAAmTfFlRHZW4xZGhFUnVPbXBCWlNhWF9kYWcAAAAAAAALMhY5R0Y2TXNyRFM3bUJkMHE4SkVQZmNRAAAAAAAACnoWMnJWMHhRZE5SVXU0Rm9wYVI1QUU0QQAAAAAAAmTdFlRHZW4xZGhFUnVPbXBCWlNhWF9kYWcAAAAAAAAJYxZEeG5CLWhuWVM4V1Vha1Z5Q3ZLN1pRAAAAAAAACWQWRHhuQi1obllTOFdVYWtWeUN2SzdaUQAAAAAAAAlhFnBoOUROcnhqUkN5Rlh0aUtyT1dDd0EAAAAAAAALMxY5R0Y2TXNyRFM3bUJkMHE4SkVQZmNRAAAAAAAACWUWRHhuQi1obllTOFdVYWtWeUN2SzdaUQAAAAAAAAliFnBoOUROcnhqUkN5Rlh0aUtyT1dDd0EAAAAAAAAKexYyclYweFFkTlJVdTRGb3BhUjVBRTRBAAAAAAACZN4WVEdlbjFkaEVSdU9tcEJaU2FYX2RhZwAAAAAAAAjzFmlza1d4VjV2VDVLYy1VWDQ1aERWb0EAAAAAAAAJZhZwaDlETnJ4alJDeUZYdGlLck9XQ3dBAAAAAAAACWMWcGg5RE5yeGpSQ3lGWHRpS3JPV0N3QQAAAAAAAAs0FjlHRjZNc3JEUzdtQmQwcThKRVBmY1EAAAAAAAAJZhZEeG5CLWhuWVM4V1Vha1Z5Q3ZLN1pRAAAAAAAACWQWcGg5RE5yeGpSQ3lGWHRpS3JPV0N3QQAAAAAAAAp8FjJyVjB4UWROUlV1NEZvcGFSNUFFNEEAAAAAAAJk4BZUR2VuMWRoRVJ1T21wQlpTYVhfZGFnAAAAAAAACPQWaXNrV3hWNXZUNUtjLVVYNDVoRFZvQQAAAAAAAmYbFjNsWU1LeTlrVGhPZERWRW9iWEZ5QlEAAAAAAAAJZRZwaDlETnJ4alJDeUZYdGlLck9XQ3dBAAAAAAAACzUWOUdGNk1zckRTN21CZDBxOEpFUGZjUQAAAAAAAAlnFkR4bkItaG5ZUzhXVWFrVnlDdks3WlEAAAAAAAALNhY5R0Y2TXNyRFM3bUJkMHE4SkVQZmNRAAAAAAAACn0WMnJWMHhRZE5SVXU0Rm9wYVI1QUU0QQAAAAAAAmThFlRHZW4xZGhFUnVPbXBCWlNhWF9kYWcAAAAAAAAI9RZpc2tXeFY1dlQ1S2MtVVg0NWhEVm9BAAAAAAACZhwWM2xZTUt5OWtUaE9kRFZFb2JYRnlCUQAAAAAAAAlnFnBoOUROcnhqUkN5Rlh0aUtyT1dDd0EAAAAAAAALNxY5R0Y2TXNyRFM3bUJkMHE4SkVQZmNRAAAAAAAACWgWRHhuQi1obllTOFdVYWtWeUN2SzdaUQAAAAAAAAs4FjlHRjZNc3JEUzdtQmQwcThKRVBmY1EAAAAAAAAKfhYyclYweFFkTlJVdTRGb3BhUjVBRTRBAAAAAAACZOUWVEdlbjFkaEVSdU9tcEJaU2FYX2RhZwAAAAAAAAj2Fmlza1d4VjV2VDVLYy1VWDQ1aERWb0EAAAAAAAJmHRYzbFlNS3k5a1RoT2REVkVvYlhGeUJRAAAAAAAACWgWcGg5RE5yeGpSQ3lGWHRpS3JPV0N3QQAAAAAAAAs5FjlHRjZNc3JEUzdtQmQwcThKRVBmY1EAAAAAAAJk4xZUR2VuMWRoRVJ1T21wQlpTYVhfZGFnAAAAAAAAB0EWa3YzMVZsNkdSYnFEQ2x3a0R6NENVUQAAAAAAAAp_FjJyVjB4UWROUlV1NEZvcGFSNUFFNEEAAAAAAAJk4hZUR2VuMWRoRVJ1T21wQlpTYVhfZGFnAAAAAAAACPcWaXNrV3hWNXZUNUtjLVVYNDVoRFZvQQAAAAAAAAs6FjlHRjZNc3JEUzdtQmQwcThKRVBmY1EAAAAAAAAJaRZwaDlETnJ4alJDeUZYdGlLck9XQ3dBAAAAAAAACzsWOUdGNk1zckRTN21CZDBxOEpFUGZjUQAAAAAAAAlpFkR4bkItaG5ZUzhXVWFrVnlDdks3WlEAAAAAAAAI-BZpc2tXeFY1dlQ1S2MtVVg0NWhEVm9BAAAAAAAACoAWMnJWMHhRZE5SVXU0Rm9wYVI1QUU0QQAAAAAAAmTkFlRHZW4xZGhFUnVPbXBCWlNhWF9kYWcAAAAAAAAI-RZpc2tXeFY1dlQ1S2MtVVg0NWhEVm9BAAAAAAAACoEWMnJWMHhRZE5SVXU0Rm9wYVI1QUU0QQAAAAAAAAlqFnBoOUROcnhqUkN5Rlh0aUtyT1dDd0EAAAAAAAALPBY5R0Y2TXNyRFM3bUJkMHE4SkVQZmNRAAAAAAAACWoWRHhuQi1obllTOFdVYWtWeUN2SzdaUQAAAAAAAAqCFjJyVjB4UWROUlV1NEZvcGFSNUFFNEEAAAAAAAAKgxYyclYweFFkTlJVdTRGb3BhUjVBRTRBAAAAAAACZOYWVEdlbjFkaEVSdU9tcEJaU2FYX2RhZwAAAAAAAAj6Fmlza1d4VjV2VDVLYy1VWDQ1aERWb0EAAAAAAAJmHhYzbFlNS3k5a1RoT2REVkVvYlhGeUJRAAAAAAAACWsWcGg5RE5yeGpSQ3lGWHRpS3JPV0N3QQAAAAAAAAs9FjlHRjZNc3JEUzdtQmQwcThKRVBmY1EAAAAAAAAJaxZEeG5CLWhuWVM4V1Vha1Z5Q3ZLN1pRAAAAAAAACoQWMnJWMHhRZE5SVXU0Rm9wYVI1QUU0QQAAAAAAAAqFFjJyVjB4UWROUlV1NEZvcGFSNUFFNEEAAAAAAAJk5xZUR2VuMWRoRVJ1T21wQlpTYVhfZGFnAAAAAAAACPsWaXNrV3hWNXZUNUtjLVVYNDVoRFZvQQAAAAAAAAqKFjJyVjB4UWROUlV1NEZvcGFSNUFFNEEAAAAAAAAJbBZwaDlETnJ4alJDeUZYdGlLck9XQ3dBAAAAAAAACz4WOUdGNk1zckRTN21CZDBxOEpFUGZjUQAAAAAAAAlsFkR4bkItaG5ZUzhXVWFrVnlDdks3WlEAAAAAAAAJbRZEeG5CLWhuWVM4V1Vha1Z5Q3ZLN1pRAAAAAAAACoYWMnJWMHhRZE5SVXU0Rm9wYVI1QUU0QQAAAAAAAmToFlRHZW4xZGhFUnVPbXBCWlNhWF9kYWcAAAAAAAAI_BZpc2tXeFY1dlQ1S2MtVVg0NWhEVm9BAAAAAAACZh8WM2xZTUt5OWtUaE9kRFZFb2JYRnlCUQAAAAAAAAltFnBoOUROcnhqUkN5Rlh0aUtyT1dDd0EAAAAAAAALPxY5R0Y2TXNyRFM3bUJkMHE4SkVQZmNRAAAAAAAACocWMnJWMHhRZE5SVXU0Rm9wYVI1QUU0QQAAAAAAAmTpFlRHZW4xZGhFUnVPbXBCWlNhWF9kYWcAAAAAAAAKiRYyclYweFFkTlJVdTRGb3BhUjVBRTRBAAAAAAACZOoWVEdlbjFkaEVSdU9tcEJaU2FYX2RhZwAAAAAAAAj9Fmlza1d4VjV2VDVLYy1VWDQ1aERWb0EAAAAAAAJmIBYzbFlNS3k5a1RoT2REVkVvYlhGeUJRAAAAAAAACW4WcGg5RE5yeGpSQ3lGWHRpS3JPV0N3QQAAAAAAAAtAFjlHRjZNc3JEUzdtQmQwcThKRVBmY1EAAAAAAAAJbhZEeG5CLWhuWVM4V1Vha1Z5Q3ZLN1pRAAAAAAACZiEWM2xZTUt5OWtUaE9kRFZFb2JYRnlCUQAAAAAAAAqIFjJyVjB4UWROUlV1NEZvcGFSNUFFNEEAAAAAAAJk6xZUR2VuMWRoRVJ1T21wQlpTYVhfZGFnAAAAAAAACP4WaXNrV3hWNXZUNUtjLVVYNDVoRFZvQQAAAAAAAmYiFjNsWU1LeTlrVGhPZERWRW9iWEZ5QlE=", + "size": 100 +} diff --git a/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/mag/invertedIndex.json b/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/mag/invertedIndex.json new file mode 100644 index 0000000000..0a84e330d2 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/mag/invertedIndex.json @@ -0,0 +1,334 @@ +{ + "IndexLength": 139, + "InvertedIndex": { + "The": [ + 0, + 23, + 47 + ], + "invention": [ + 1, + 53 + ], + "discloses": [ + 2 + ], + "a": [ + 3, + 10, + 71, + 81, + 121 + ], + "treatment": [ + 4, + 69, + 85, + 96 + ], + "method": [ + 5, + 24, + 49 + ], + "of": [ + 6, + 9, + 19, + 57, + 84, + 117, + 120 + ], + "waste": [ + 7, + 118 + ], + "mash": [ + 8, + 119 + ], + "cane": [ + 11, + 122 + ], + "sugar": [ + 12, + 123 + ], + "factory,": [ + 13 + ], + "belonging": [ + 14 + ], + "to": [ + 15 + ], + "the": [ + 16, + 26, + 52, + 55, + 66, + 93, + 115, + 135 + ], + "technical": [ + 17, + 48 + ], + "field": [ + 18 + ], + "industrial": [ + 20 + ], + "wastewater": [ + 21 + ], + "treatment.": [ + 22 + ], + "comprises": [ + 25 + ], + "following": [ + 27 + ], + "steps": [ + 28 + ], + "of:": [ + 29 + ], + "(1)": [ + 30 + ], + "pretreatment;": [ + 31 + ], + "(2)": [ + 32 + ], + "primary": [ + 33 + ], + "concentration;": [ + 34 + ], + "(3)": [ + 35 + ], + "cooling": [ + 36 + ], + "sedimentation": [ + 37 + ], + "and": [ + 38, + 45, + 62, + 80, + 86, + 114, + 134 + ], + "dense": [ + 39 + ], + "slurry": [ + 40 + ], + "drying;": [ + 41 + ], + "(4)": [ + 42 + ], + "secondary": [ + 43 + ], + "concentration": [ + 44 + ], + "drying.": [ + 46 + ], + "disclosed": [ + 50 + ], + "by": [ + 51 + ], + "has": [ + 54 + ], + "advantages": [ + 56 + ], + "small": [ + 58 + ], + "investment,": [ + 59 + ], + "simple": [ + 60 + ], + "equipment": [ + 61 + ], + "easiness": [ + 63 + ], + "in": [ + 64, + 132 + ], + "popularization;": [ + 65 + ], + "product": [ + 67 + ], + "after": [ + 68 + ], + "is": [ + 70, + 91, + 98, + 102, + 112, + 130, + 137 + ], + "high-quality": [ + 72 + ], + "high": [ + 73 + ], + "value-added": [ + 74 + ], + "(fully": [ + 75 + ], + "water-soluble)": [ + 76 + ], + "potassium": [ + 77 + ], + "humate": [ + 78 + ], + "product,": [ + 79 + ], + "new": [ + 82 + ], + "mode": [ + 83 + ], + "profit": [ + 87 + ], + "enabling": [ + 88 + ], + "sustainable": [ + 89 + ], + "development": [ + 90 + ], + "realized;": [ + 92 + ], + "environmental": [ + 94 + ], + "protection": [ + 95 + ], + "effect": [ + 97 + ], + "good,": [ + 99 + ], + "water": [ + 100, + 106 + ], + "balance": [ + 101 + ], + "realized": [ + 103 + ], + "through": [ + 104 + ], + "final": [ + 105 + ], + "quality": [ + 107 + ], + "treatment,": [ + 108 + ], + "real": [ + 109 + ], + "zero": [ + 110 + ], + "emission": [ + 111 + ], + "realized,": [ + 113 + ], + "problem": [ + 116 + ], + "factory": [ + 124 + ], + "can": [ + 125 + ], + "be": [ + 126 + ], + "solved": [ + 127 + ], + "fundamentally;": [ + 128 + ], + "energy": [ + 129 + ], + "saved": [ + 131 + ], + "operation,": [ + 133 + ], + "feasibility": [ + 136 + ], + "high.": [ + 138 + ] + } +} \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/orcid/0000-0001-6645-509X.compressed.base64 b/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/orcid/0000-0001-6645-509X.compressed.base64 new file mode 100644 index 0000000000..1b088e061c --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/orcid/0000-0001-6645-509X.compressed.base64 @@ -0,0 +1 @@ +H4sIAAAAAAAAAO1a227bOBB9z1cIepd18SW24aho0wTbAgEWjRdY9I2RaJtbSdSSkhP165eURIm6kHa2SbCLNkBiWDxzhhxyZg7tbN49xZFxhIQinFyZ7sQxDZgEOETJ/sr8Y3trLU2DZiAJQYQTeGUWkJrv/IsNgQEm4bp6MVKQHa5M22E/Fvt1rcViNrfmzupP02AOErpGSQZJAqIr85Bl6dq2Hx8fJ5gEKGR/93ZCbYEQFjDMA5CV01KZNBBhEyKaoSTQW0mgxg6mbCUgg6HGrMEIK5wdILESEEO1VYsRVjGMH1i8DyhVW7WYJhqEYKKJBB8W2ADHsS4A1bhAV1uoRlfjAp2yaWG2S1YIM4AiqrbrIwXDN1g8ah3WgGblMbPWrJwPN9in6gxZKIRJhnYI6mI2BAueXZ5UGaCyrQFNVAjcQcISB+oC0oKEHQhDAqnGpga0WXRE7ABaKaZIf8j7SMHAIvtNbcVHBfLA0gSTQg2uAe0+pREuYhZK3WYJjLD6OwcRC/2pTO/AhC2F5IgCTfLVgO7ZPXVim71hFYLFEOm2tMW02UQhIAFP+pxojm0X186QvSfwiOCjbpoNSNg95JFmV/lof36MgOKc6KI3gJr+hcF+NlX9WJdgKXmqURmRE+RzdsroW+qRLrGxJYsBDe8uvs6qBAzMDphmfuO2AZePq4XY2pVspISVM1zyJCMiHIAI+jDZ2COPa4dayk2dUSL1JEdiJCCwTAErhtkBh/5d2SiskonAcGOrgEMqmj/EiPK+b4Wsq/me464sZ2l53tadrmeLtXc58ZbLry1n32IQ8QjQzIqZeGBBDAWrx7Ztbrnu1puu59P11JksPfdrE/sRm5FlRwDFMPQzkkNpjfXTIZ4Jmoqv7A49s96gxjolKAak0LN0QfU+j+7kpiowdR3SiCZRieSTVplyIWEcEUUPKEIZK85p/hChwKzJxgRYSyJvVXk+2k0abv187rWb1EGP8o1u/QlW3dZLi24lxHqPjjAp1RT1twgkRb4Z6IwO6ATfDsQoKkqs/xmBETIZ0e6GLW2H9LgVe5I2pLqNlmCmLTF120Ovq2gZe9AOa3lEK0Gl5ag0lWxZ6xAhWPSLEqJFJqhFnVB/WnuB6c59qNbG5J5+XSN44aTZ0+qlftg2eEkPWDSPecprY9Aqg2fUyZnlTLfObD2brZ3pZHm5OLNOStOUbjfaWMi47la3XM39Sh/VBqXkaWTfiWPXwFRMte7W0giMiqMvjbVkA7CKtb2yafkkmIpJ0ndaKhmn4uroZi1bF6niG2jCs2pRi1bx1kpdyyYwKg5+edESlABFP3zplOxPbk9wnnaHX9u9zC9VPjpEKZDjQAXYyooU+iFGzfwGg8+iO4Ioh77rTFzXWdnvr69v7u8nPCYTb7X0PNcZ9VNZPctRgknMjv53GBoZAQlF5Q2Wiz2zcQ8Cdu7oafct1/PmwDp1c1FiISyvSc9dOud4llMCoyrZWTHyKYx2o7Qd1PjJGTEbOYkjqJGjuOFJWqZy22XzzApwyG6qly67kCxWjnkqy+0WOSaWWe9LI1BYKAnhE1PNpj4lelqZp+XUmjpbz1szYTt3JjP38hyt3Od9raSXfVR19/TBqHBWEPHjr8192Wr8gl+RSJuzWi5nlrtyp+P3fJ2H3t1/yNS9++uoTn4eMGpsPztAvZCWd4Rrgillt/Q+XfcCoXGsAJXZkqEsOmOLK9g9K1CR9ZFdnBN+kzdu2WnNCTTuQEbQk3HNMp3VvlIXGnflZwfGDhPjI6y+FDC+wBQyJnbHMm7Ze0iMO3yElba7JTg2biIYZATzzzXSA4jwnoDYuEd7lvK0WZRmyhv71KLOb2oK9Hnn5YWam4ryVRqcytlbNznVPF690akcv1SzK/nPangq5An99W8jpIxKXSP4Gf2LlRI+CUAyFERQZJry+DZFuOyb1eeJ6pYjWxRM95fNrJlf+UQfpPPcVOsRS6nKxKebmxvjfXl+60V1x0fUyEBn9LS7rRfvP6rt64/GVlt3vnYXa8ebLJz5T6jt53ObB8OeLl2m2WZvJurP8fviav4cpz+BjF+4znzqzd3TMr5FvryMP5GBPyjjXyC/ZR+/ZPwvGd+Rzh8IQIl1jWOWVkyDf+L/PLMDATSuDyBJYGTdQ67DuYq/ZxUwg/vC+AAoq4fsyXuWtwVF1MA74+bIA/GFlwc2+BHSIgkOBCfoe1kvjC1OuYRPD4WBSi78DRq/szGu+H/p+ddqaiovb9bYVBN4veam8vj/l+6q0PwnNbu7OkOzy3bslxf3ZWNWPThpF4LC91or/va17gefq3e83v0GQZQdAkCgcZPsUQIhQcn+DW4NnbHyqwjxxaP2S0b/YmN3/tnSv/gH9+klwrUpAAA= \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/orcid/dataOutput b/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/orcid/dataOutput new file mode 100644 index 0000000000..54ed4adae8 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/orcid/dataOutput @@ -0,0 +1,309 @@ +(10.1109/JPHOT.2018.2880319,[{"oid":"0000-0002-1473-2224","name":"zhong","surname":"chen","creditName":null,"errorCode":null},{"oid":"0000-0002-5739-2127","name":"Yue","surname":"Lin","creditName":null,"errorCode":null}]) +(10.1073/pnas.1816459115,[{"oid":"0000-0001-6260-4326","name":"Igor","surname":"Sokolov","creditName":null,"errorCode":null},{"oid":"0000-0001-6260-4326","name":"Igor","surname":"Sokolov","creditName":null,"errorCode":null}]) +(10.1016/j.mjafi.2016.05.007,[{"oid":"0000-0002-4342-6656","name":"Anoop","surname":"Sharma","creditName":null,"errorCode":null}]) +(10.1152/japplphysiol.00476.2016,[{"oid":"0000-0001-9039-0302","name":"Graeme","surname":"Zosky","creditName":"Graeme R. Zosky","errorCode":null},{"oid":"0000-0001-8355-7362","name":"Stephen","surname":"Dubsky","creditName":"Stephen Eric Dubsky","errorCode":null}]) +(10.1111/j.1365-2621.1995.tb09798.x,[{"oid":"0000-0001-6457-4044","name":"jose martin","surname":"yañez limon","creditName":null,"errorCode":null},{"oid":"0000-0002-7108-8447","name":"Orlando","surname":"Zelaya-Angel","creditName":null,"errorCode":null},{"oid":"0000-0002-9267-4621","name":"Jean","surname":"Martinez","creditName":null,"errorCode":null},{"oid":"0000-0003-3060-2638","name":"Juan Jose","surname":"Alvarado-Gil","creditName":null,"errorCode":null}]) +(10.1007/s10461-019-02391-1,[{"oid":"0000-0003-4375-7452","name":"Cathy","surname":"Reback","creditName":null,"errorCode":null}]) +(10.1039/C4TA03030C,[{"oid":"0000-0003-1615-5034","name":"Jihong","surname":"Yu","creditName":null,"errorCode":null}]) +(10.1016/0306-4522(89)90381-3,[{"oid":"0000-0002-7272-8370","name":"PAOLA","surname":"D'ASCANIO","creditName":null,"errorCode":null}]) +(10.1016/S0921-4526(01)00945-0,[{"oid":"0000-0002-5360-1008","name":"Tomasz","surname":"Nowak","creditName":null,"errorCode":null},{"oid":"0000-0001-5554-8178","name":"Pawel","surname":"Olko","creditName":null,"errorCode":null},{"oid":"0000-0002-1993-9320","name":"Michael","surname":"Waligorski","creditName":null,"errorCode":null},{"oid":"0000-0002-1993-9320","name":"Michael","surname":"Waligorski","creditName":null,"errorCode":null}]) +(10.1074/jbc.m312875200,[{"oid":"0000-0003-3631-2252","name":"Isabelle","surname":"Dugail","creditName":null,"errorCode":null},{"oid":"0000-0001-6577-9009","name":"Bruno","surname":"Feve","creditName":null,"errorCode":null},{"oid":"0000-0002-4502-3543","name":"Martine","surname":"GLORIAN","creditName":"GLORIAN","errorCode":null},{"oid":"0000-0002-1275-9861","name":"KHADIJA","surname":"EL HADRI","creditName":null,"errorCode":null}]) +(10.1038/nrc3802,[{"oid":"0000-0001-7427-4651","name":"William","surname":"Foulkes","creditName":null,"errorCode":null},{"oid":"0000-0001-7427-4651","name":"William","surname":"Foulkes","creditName":null,"errorCode":null}]) +(10.1186/s12974-018-1125-5,[{"oid":"0000-0001-5270-9888","name":"I-Chen","surname":"Yu","creditName":null,"errorCode":null}]) +(10.4997/JRCPE.2016.310,[{"oid":"0000-0001-7658-1209","name":"David","surname":"Burn","creditName":null,"errorCode":null}]) +(10.13140/RG.2.2.20772.48006,[{"oid":"0000-0003-2825-5884","name":"Abdelkader","surname":"Mezghani","creditName":null,"errorCode":null}]) +(10.1039/c8ay00816g,[{"oid":"0000-0002-5444-7276","name":"Torbjörn","surname":"Pettersson","creditName":null,"errorCode":null}]) +(10.2147/dhps.s6226,[{"oid":"0000-0001-8558-3396","name":"Mario Francisco","surname":"Juruena","creditName":"JURUENA OR JURUENA M OR JURUENA M F OR OR JURUENA MARIO OR JURUENA MARIO F OR JURUENA MARIO FRANCISCO OR JURUENA MF","errorCode":null}]) +(10.1007/s00415-004-0351-1,[{"oid":"0000-0002-2737-3662","name":"Gian Domenico","surname":"Borasio","creditName":null,"errorCode":null}]) +(10.1088/0967-3334/23/1/318,[{"oid":"0000-0002-3549-4309","name":"Gavin","surname":"Screaton","creditName":null,"errorCode":null},{"oid":"0000-0002-0124-5072","name":"Carlos-Augusto","surname":"Gonzalez-Correa","creditName":null,"errorCode":null}]) +(10.1111/aogs.13316,[{"oid":"0000-0002-1241-9875","name":"Lena","surname":"Liljestrom","creditName":null,"errorCode":null}]) +(10.1038/sj.thj.6200111,[{"oid":"0000-0001-6115-8790","name":"Felipe","surname":"Prosper","creditName":null,"errorCode":null},{"oid":"0000-0001-6115-8790","name":"Felipe","surname":"Prosper","creditName":null,"errorCode":null},{"oid":"0000-0002-9467-932X","name":"Maria J","surname":"Terol","creditName":null,"errorCode":null},{"oid":"0000-0001-9622-1649","name":"María del Mar","surname":"Tormo Díaz","creditName":"Mar Tormo","errorCode":null},{"oid":"0000-0002-7938-3950","name":"Jose A","surname":"Martinez-Climent","creditName":null,"errorCode":null}]) +(10.1021/jo052299f,[{"oid":"0000-0002-2419-0705","name":"Per-Ola","surname":"Norrby","creditName":null,"errorCode":null},{"oid":"0000-0002-2548-7025","name":"Mogens","surname":"Johannsen","creditName":null,"errorCode":null},{"oid":"0000-0002-2548-7025","name":"Mogens","surname":"Johannsen","creditName":null,"errorCode":null}]) +(10.1182/blood-2014-04-569392,[{"oid":"0000-0003-1680-5295","name":"X. Long","surname":"Zheng","creditName":null,"errorCode":null},{"oid":"0000-0003-1680-5295","name":"X. Long","surname":"Zheng","creditName":null,"errorCode":null}]) +(10.1194/jlr.M030049,[{"oid":"0000-0003-4488-7734","name":"Sander","surname":"Kersten","creditName":null,"errorCode":null}]) +(10.1006/jcis.1998.6070,[{"oid":"0000-0003-1407-6498","name":"Anna Luisa","surname":"Costa","creditName":null,"errorCode":null},{"oid":"0000-0002-7892-2836","name":"Carmen","surname":"Galassi","creditName":null,"errorCode":null}]) +(10.1002/ange.201707070,[{"oid":"0000-0002-2516-084X","name":"tao","surname":"xiong","creditName":null,"errorCode":null}]) +(10.1056/NEJMc1502191,[{"oid":"0000-0003-0893-9015","name":"David","surname":"Stoltz","creditName":null,"errorCode":null},{"oid":"0000-0003-1552-3253","name":"David","surname":"Meyerholz","creditName":"David K. Meyerholz","errorCode":null}]) +(10.1300/J017v21n03_01,[{"oid":"0000-0002-6132-5883","name":"Stephen","surname":"Butler","creditName":null,"errorCode":null}]) +(10.4028/www.scientific.net/SSP.227.47,[{"oid":"0000-0001-5648-222X","name":"Pawel","surname":"Rokicki","creditName":null,"errorCode":null}]) +(10.1002/1521-3935(20000801)201:12<1329::AID-MACP1329>3.0.CO;2-8,[{"oid":"0000-0003-4175-4741","name":"Zhiyuan","surname":"Zhong","creditName":null,"errorCode":null}]) +(10.1111/j.1365-2966.2006.11355.x,[{"oid":"0000-0002-1967-2849","name":"Sunil","surname":"Maharaj","creditName":null,"errorCode":null}]) +(10.1021/acs.jpca.6b01563,[{"oid":"0000-0002-5301-6730","name":"Stefano","surname":"Falcinelli","creditName":null,"errorCode":null},{"oid":"0000-0002-5301-6730","name":"Stefano","surname":"Falcinelli","creditName":null,"errorCode":null},{"oid":"0000-0003-1934-7891","name":"Piergiorgio","surname":"Casavecchia","creditName":null,"errorCode":null},{"oid":"0000-0002-3961-5710","name":"Astrid","surname":"Bergeat","creditName":null,"errorCode":null},{"oid":"0000-0001-5121-5683","name":"Nadia","surname":"Balucani","creditName":null,"errorCode":null}]) +(10.1017/S1431927608089332,[{"oid":"0000-0003-4670-4516","name":"Sónia","surname":"Simões","creditName":"Sónia Simões","errorCode":null},{"oid":"0000-0003-4670-4516","name":"Sónia","surname":"Simões","creditName":"Sónia Simões","errorCode":null},{"oid":"0000-0002-2894-771X","name":"Filomena","surname":"Viana","creditName":"F. Viana","errorCode":null},{"oid":"0000-0002-8486-5436","name":"Ana Sofia","surname":"Ramos","creditName":"A.S. Ramos","errorCode":null},{"oid":"0000-0002-8486-5436","name":"Ana Sofia","surname":"Ramos","creditName":"A.S. Ramos","errorCode":null}]) +(10.1063/1.1452737,[{"oid":"0000-0002-4041-7631","name":"Thierry","surname":"Visart de Bocarmé","creditName":null,"errorCode":null}]) +(10.1056/nejmp1014255,[{"oid":"0000-0002-9354-5389","name":"Salal","surname":"Humair","creditName":null,"errorCode":null}]) +(10.1007/978-3-531-19249-9,[{"oid":"0000-0002-3497-6947","name":"Jan W.","surname":"van Deth","creditName":null,"errorCode":null}]) +(10.1080/00397910903291129,[{"oid":"0000-0003-3602-7400","name":"Hayreddin","surname":"Gezegen","creditName":null,"errorCode":null}]) +(10.1002/lite.201400037,[{"oid":"0000-0001-8347-8496","name":"James","surname":"Kenar","creditName":null,"errorCode":null},{"oid":"0000-0001-8347-8496","name":"James","surname":"Kenar","creditName":null,"errorCode":null}]) +(10.1109/ICACCS.2017.8014639,[{"oid":"0000-0001-9570-1077","name":"Gadadhar","surname":"Sahoo","creditName":null,"errorCode":null}]) +(10.18632/oncotarget.4840,[{"oid":"0000-0002-5664-7781","name":"Brian","surname":"McStay","creditName":null,"errorCode":null}]) +(10.1016/j.palaeo.2010.12.015,[{"oid":"0000-0001-6088-3261","name":"Michael","surname":"Joachimski","creditName":null,"errorCode":null}]) +(10.1016/S0927-0256(97)00020-7,[{"oid":"0000-0002-0604-6590","name":"Ilja","surname":"Turek","creditName":null,"errorCode":null}]) +(10.1074/jbc.M111.222364,[{"oid":"0000-0002-5441-5709","name":"Khortnal","surname":"Delvecchio","creditName":null,"errorCode":null}]) +(10.1007/s00468-014-1099-6,[{"oid":"0000-0002-7441-2070","name":"Tingfa","surname":"Dong","creditName":"Tingfa Dong","errorCode":null}]) +(10.1055/s-0029-1241171,[{"oid":"0000-0003-2795-6013","name":"Alessandro","surname":"Mussa","creditName":null,"errorCode":null}]) +(10.1016/j.conb.2013.01.010,[{"oid":"0000-0003-3044-9565","name":"Jonathan","surname":"Britt","creditName":null,"errorCode":null},{"oid":"0000-0003-3044-9565","name":"Jonathan","surname":"Britt","creditName":null,"errorCode":null},{"oid":"0000-0003-3044-9565","name":"Jonathan","surname":"Britt","creditName":null,"errorCode":null}]) +(10.1061/(ASCE)CC.1943-5614.0000977,[{"oid":"0000-0003-1663-7535","name":"CHRISTIAN","surname":"CARLONI","creditName":null,"errorCode":null}]) +(10.1177/0146167211399101,[{"oid":"0000-0001-6060-8083","name":"Kristof","surname":"Dhont","creditName":null,"errorCode":null},{"oid":"0000-0001-5814-1189","name":"Arne","surname":"Roets","creditName":null,"errorCode":null}]) +(10.1007/s00441-005-1096-6,[{"oid":"0000-0002-0240-7416","name":"Wee-Ming","surname":"Boon","creditName":null,"errorCode":null},{"oid":"0000-0002-8085-0034","name":"Karen","surname":"Moritz","creditName":null,"errorCode":null}]) +(10.1080/01468039308204227,[{"oid":"0000-0002-5148-6624","name":"Miguel A.","surname":"Muriel","creditName":"Miguel A. Muriel","errorCode":null},{"oid":"0000-0002-5148-6624","name":"Miguel A.","surname":"Muriel","creditName":"Miguel A. Muriel","errorCode":null}]) +(10.1111/ijfs.14302,[{"oid":"0000-0002-5053-6378","name":"Mariana","surname":"Blanco Massani","creditName":null,"errorCode":null},{"oid":"0000-0002-5053-6378","name":"Mariana","surname":"Blanco Massani","creditName":null,"errorCode":null}]) +(10.1016/j.nanoen.2016.08.017,[{"oid":"0000-0002-4115-3287","name":"Xiaofeng","surname":"Li","creditName":null,"errorCode":null}]) +(10.1073/pnas.1402739111,[{"oid":"0000-0001-5933-6463","name":"Anselm","surname":"Enders","creditName":null,"errorCode":null},{"oid":"0000-0003-3922-6376","name":"Dan","surname":"Andrews","creditName":null,"errorCode":null}]) +(10.1128/JVI.01432-15,[{"oid":"0000-0002-9141-8001","name":"Markus","surname":"Cornberg","creditName":null,"errorCode":null},{"oid":"0000-0002-6993-4333","name":"Liisa","surname":"Selin","creditName":null,"errorCode":null}]) +(10.1097/PCC.0000000000001178,[{"oid":"0000-0003-3089-0318","name":"Sapna","surname":"Kudchadkar","creditName":"Sapna R. Kudchadkar","errorCode":null}]) +(10.3866/pku.whxb20001113,[{"oid":"0000-0002-6469-0376","name":"jingfang","surname":"zhou","creditName":null,"errorCode":null}]) +(10.1109/JSEN.2019.2912827,[{"oid":"0000-0003-4870-8473","name":"Fengde","surname":"Jia","creditName":null,"errorCode":null}]) +(10.1007/978-3-642-23430-9_65,[{"oid":"0000-0002-0663-3437","name":"Heppenstall","surname":"Alison","creditName":null,"errorCode":null},{"oid":"0000-0002-0650-6606","name":"Dianna","surname":"Smith","creditName":"Dianna M Smith","errorCode":null}]) +(10.3389/fenrg.2019.00056,[{"oid":"0000-0003-4222-6975","name":"andrea","surname":"saltelli","creditName":null,"errorCode":null},{"oid":"0000-0002-2625-483X","name":"Samuele","surname":"Lo Piano","creditName":null,"errorCode":null}]) +(10.1007/s00701-015-2639-6,[{"oid":"0000-0001-8914-5086","name":"Samuel","surname":"Lenell","creditName":null,"errorCode":null}]) +(10.4111/kju.2013.54.9.638,[{"oid":"0000-0002-7467-5954","name":"Jae Min","surname":"Chung","creditName":null,"errorCode":null}]) +(10.1136/eb-2015-102104,[{"oid":"0000-0001-6330-3314","name":"Paulo","surname":"Menezes","creditName":null,"errorCode":null},{"oid":"0000-0002-3403-5792","name":"Andrea","surname":"Silva","creditName":"Andréa Tenório C da Silva","errorCode":null}]) +(10.1016/j.compositesb.2016.03.046,[{"oid":"0000-0003-4867-1404","name":"Sergey","surname":"Vakhrushev","creditName":null,"errorCode":null},{"oid":"0000-0003-1929-032X","name":"Alexander","surname":"Naberezhnov","creditName":null,"errorCode":null},{"oid":"0000-0002-0830-272X","name":"Ewa","surname":"Rysiakiewicz-Pasek","creditName":null,"errorCode":null}]) +(10.1038/s41598-018-33547-z,[{"oid":"0000-0002-2080-1695","name":"Nadeem Ahmad","surname":"Sheikh","creditName":null,"errorCode":null}]) +(10.1016/j.ecl.2012.04.016,[{"oid":"0000-0001-8107-7947","name":"Ignacio","surname":"Torres Aleman","creditName":null,"errorCode":null},{"oid":"0000-0001-8107-7947","name":"Ignacio","surname":"Torres Aleman","creditName":null,"errorCode":null}]) +(10.4102/hts.v64i1.33,[{"oid":"0000-0003-3810-4190","name":"Ernest","surname":"van Eck","creditName":null,"errorCode":null}]) +(10.3233/JAD-2010-100675,[{"oid":"0000-0003-2298-615X","name":"Monica","surname":"Di Luca","creditName":null,"errorCode":null},{"oid":"0000-0003-4598-5563","name":"Fabrizio","surname":"Gardoni","creditName":null,"errorCode":null}]) +(10.1039/c3dt51431e,[{"oid":"0000-0002-0675-2057","name":"Balasubramanian","surname":"Murugesapandian","creditName":null,"errorCode":null},{"oid":"0000-0001-5221-9459","name":"Mrituanjay D","surname":"Pandey","creditName":null,"errorCode":null}]) +(10.1159/000356772,[{"oid":"0000-0002-8803-4496","name":"Takashi","surname":"Matsukura","creditName":null,"errorCode":null}]) +(10.1111/j.1469-8137.2011.03973.x,[{"oid":"0000-0003-4801-4412","name":"Julianne","surname":"O'Reilly-Wapstra","creditName":null,"errorCode":null},{"oid":"0000-0003-4801-4412","name":"Julianne","surname":"O'Reilly-Wapstra","creditName":null,"errorCode":null},{"oid":"0000-0002-9383-667X","name":"Mark","surname":"Genung","creditName":null,"errorCode":null},{"oid":"0000-0001-8249-8057","name":"Jennifer","surname":"Rowntree","creditName":null,"errorCode":null},{"oid":"0000-0001-6244-289X","name":"Brad","surname":"Potts","creditName":"Brad M Potts","errorCode":null}]) +(10.1103/physreva.78.021401,[{"oid":"0000-0002-1228-5029","name":"Michael","surname":"Martins","creditName":null,"errorCode":null}]) +(10.1109/tencon.2011.6129079,[{"oid":"0000-0003-2519-0130","name":"Mustarum","surname":"Musaruddin","creditName":null,"errorCode":null}]) +(10.1016/s0956-053x(01)00047-2,[{"oid":"0000-0002-5719-8076","name":"Michael","surname":"Frisch","creditName":null,"errorCode":null}]) +(10.1200/JCO.2005.04.3216,[{"oid":"0000-0001-8530-780X","name":"Claus","surname":"Garbe","creditName":null,"errorCode":null}]) +(10.1016/j.carbon.2018.04.084,[{"oid":"0000-0003-2149-4479","name":"Vincent","surname":"Chan","creditName":null,"errorCode":null}]) +(10.1037/0278-7393.28.3.497,[{"oid":"0000-0002-8868-7067","name":"Norbert","surname":"Schwarz","creditName":null,"errorCode":null},{"oid":"0000-0002-8868-7067","name":"Norbert","surname":"Schwarz","creditName":null,"errorCode":null}]) +(10.1039/c3ra47116k,[{"oid":"0000-0002-6626-0599","name":"Ying-Shi","surname":"Guan","creditName":null,"errorCode":null}]) +(10.1039/C8DT01469H,[{"oid":"0000-0002-2225-7072","name":"Hong-Bin","surname":"Luo","creditName":null,"errorCode":null}]) +(10.1088/1475-7516/2019/02/007,[{"oid":"0000-0002-4487-8742","name":"Miguel","surname":"Escudero","creditName":null,"errorCode":null}]) +(10.1109/DICTA.2009.27,[{"oid":"0000-0001-7782-1956","name":"Wojciech","surname":"Chojnacki","creditName":null,"errorCode":null},{"oid":"0000-0001-9612-5884","name":"Michael","surname":"Brooks","creditName":"M.J. Brooks","errorCode":null}]) +(10.1007/s11103-017-0629-1,[{"oid":"0000-0001-8226-0700","name":"Rose Adele","surname":"Monteiro","creditName":null,"errorCode":null},{"oid":"0000-0002-8553-0718","name":"Caroline","surname":"Kukolj","creditName":null,"errorCode":null},{"oid":"0000-0003-1882-1512","name":"Glaucio","surname":"Valdameri","creditName":null,"errorCode":null}]) +(10.1136/jnnp-2012-302993,[{"oid":"0000-0003-1258-5678","name":"Maeike","surname":"Zijlmans","creditName":null,"errorCode":null}]) +(10.1002/1521-3773(20000804)39:15<2707::aid-anie2707>3.0.co;2-m,[{"oid":"0000-0002-1443-8818","name":"Michael","surname":"Anderson","creditName":"Michael W Anderson","errorCode":null}]) +(10.1080/19401493.2017.1354070,[{"oid":"0000-0002-4248-6788","name":"Jérôme","surname":"Kämpf","creditName":null,"errorCode":null},{"oid":"0000-0002-1186-4299","name":"Clayton","surname":"Miller","creditName":null,"errorCode":null},{"oid":"0000-0002-1186-4299","name":"Clayton","surname":"Miller","creditName":null,"errorCode":null}]) +(10.1080/1062936x.2015.1032347,[{"oid":"0000-0003-4145-9590","name":"Ayako","surname":"Furuhama","creditName":null,"errorCode":null}]) +(10.1155/2016/9578308,[{"oid":"0000-0002-8874-1473","name":"Murat","surname":"Gunay","creditName":"Gunay M","errorCode":null}]) +(10.1016/j.gexplo.2015.09.011,[{"oid":"0000-0001-8503-4266","name":"Sérgio Benjamin","surname":"Baggio","creditName":null,"errorCode":null},{"oid":"0000-0001-7863-5071","name":"Léo","surname":"Hartmann","creditName":"Hartmann, L.A.","errorCode":null}]) +(10.1051/0004-6361:20053947,[{"oid":"0000-0003-3603-394X","name":"Christophe","surname":"Letellier","creditName":null,"errorCode":null},{"oid":"0000-0002-2746-5102","name":"Luis","surname":"Aguirre","creditName":null,"errorCode":null}]) +(10.1186/s12959-018-0180-6,[{"oid":"0000-0002-0936-1609","name":"Andrew","surname":"Kotaska","creditName":null,"errorCode":null}]) +(https://doi.org/10.1016/j.jct.2018.08.026,[{"oid":"0000-0001-8381-2466","name":"Jason","surname":"Calvin","creditName":null,"errorCode":null}]) +(10.1016/j.ceb.2010.04.009,[{"oid":"0000-0002-3329-9032","name":"Steven","surname":"Kosak","creditName":null,"errorCode":null}]) +(10.1002/asi.20042,[{"oid":"0000-0002-9989-6681","name":"Aboul Ella","surname":"Hassanien","creditName":"Aboul Ella Hassanien","errorCode":null}]) +(10.2105/AJPH.92.6.897,[{"oid":"0000-0002-8214-1776","name":"Kenneth","surname":"Warner","creditName":null,"errorCode":null}]) +(10.1016/j.foodhyd.2019.02.010,[{"oid":"0000-0002-4557-4580","name":"Diana","surname":"Gawkowska","creditName":null,"errorCode":null},{"oid":"0000-0002-8038-3050","name":"Jolanta","surname":"Cieśla","creditName":null,"errorCode":null},{"oid":"0000-0003-3323-4535","name":"Justyna","surname":"Cybulska","creditName":null,"errorCode":null},{"oid":"0000-0001-9395-1486","name":"Artur","surname":"Zdunek","creditName":null,"errorCode":null}]) +(10.1016/j.jbiosc.2019.03.005,[{"oid":"0000-0002-8601-6657","name":"Sachiyo","surname":"Aburatani","creditName":null,"errorCode":null}]) +(10.1016/j.idm.2017.06.004,[{"oid":"0000-0002-0946-2741","name":"Orou G.","surname":"Gaoue","creditName":null,"errorCode":null}]) +(10.1017/cbo9780511599194.007,[{"oid":"0000-0002-1683-4486","name":"Barry","surname":"Eichengreen","creditName":null,"errorCode":null}]) +(10.1134/S0006297915090060,[{"oid":"0000-0002-0530-4244","name":"Richard","surname":"Beckett","creditName":null,"errorCode":null},{"oid":"0000-0002-0530-4244","name":"Richard","surname":"Beckett","creditName":null,"errorCode":null},{"oid":"0000-0002-0313-7921","name":"Andrei","surname":"Chasov","creditName":null,"errorCode":null},{"oid":"0000-0002-0313-7921","name":"Andrei","surname":"Chasov","creditName":null,"errorCode":null}]) +(10.31229/osf.io/jdw7f,[{"oid":"0000-0001-7849-1282","name":"Adib Rifqi","surname":"Setiawan","creditName":"Alobatnic","errorCode":null}]) +(10.1016/j.proci.2018.06.150,[{"oid":"0000-0001-7058-6498","name":"Wolfgang","surname":"Polifke","creditName":null,"errorCode":null},{"oid":"0000-0001-5286-0756","name":"Matthias","surname":"Haeringer","creditName":null,"errorCode":null}]) +(10.3891/acta.chem.scand.39a-0411,[{"oid":"0000-0002-1061-7536","name":"Ingmar","surname":"Persson","creditName":null,"errorCode":null}]) +(10.1038/sj.ejcn.1601726,[{"oid":"0000-0001-8424-2864","name":"Agneta","surname":"Sjöberg","creditName":null,"errorCode":null}]) +(10.1093/brain/aww278,[{"oid":"0000-0001-8683-8741","name":"Arturo","surname":"Cardenas-Blanco","creditName":"Arturo Cardenas-Blanco","errorCode":null},{"oid":"0000-0003-1174-5983","name":"Julio","surname":"Acosta-Cabronero","creditName":"Julio Acosta-Cabronero","errorCode":null},{"oid":"0000-0003-1174-5983","name":"Julio","surname":"Acosta-Cabronero","creditName":"Julio Acosta-Cabronero","errorCode":null},{"oid":"0000-0002-2840-4678","name":"Matthew","surname":"Betts","creditName":null,"errorCode":null},{"oid":"0000-0002-5860-5921","name":"Peter","surname":"Nestor","creditName":null,"errorCode":null},{"oid":"0000-0002-5860-5921","name":"Peter","surname":"Nestor","creditName":null,"errorCode":null}]) +(10.1016/j.drugpo.2015.05.017,[{"oid":"0000-0002-9402-8682","name":"Judith","surname":"Aldridge","creditName":null,"errorCode":null}]) +(10.1088/1742-6596/340/1/011001,[{"oid":"0000-0003-1298-2120","name":"Vladimir","surname":"Sechovsky","creditName":null,"errorCode":null}]) +(10.1016/j.scitotenv.2019.06.168,[{"oid":"0000-0002-0543-2641","name":"Jordi","surname":"Garcia-Orellana","creditName":null,"errorCode":null}]) +(10.1007/978-3-319-05215-1_5,[{"oid":"0000-0002-6231-8415","name":"Siddhartha","surname":"Jana","creditName":null,"errorCode":null}]) +(10.1007/978-981-287-736-9_85,[{"oid":"0000-0002-7913-9712","name":"Oleg","surname":"Lupan","creditName":"Oleg LUPAN","errorCode":null},{"oid":"0000-0002-7913-9712","name":"Oleg","surname":"Lupan","creditName":"Oleg LUPAN","errorCode":null}]) +(10.1175/2009JCLI3061.1,[{"oid":"0000-0001-6935-4112","name":"David","surname":"Randall","creditName":null,"errorCode":null}]) +(10.3987/COM-05-10376,[{"oid":"0000-0003-3638-2517","name":"Tecla","surname":"Gasperi","creditName":null,"errorCode":null}]) +(10.1016/j.bbrc.2009.07.098,[{"oid":"0000-0003-1537-0437","name":"A. M.","surname":"Stalcup","creditName":null,"errorCode":null}]) +(10.1179/1753555713Y.0000000065,[{"oid":"0000-0002-0233-1407","name":"Pimpa","surname":"Hormnirun","creditName":null,"errorCode":null}]) +(10.1109/OCEANS.2018.8604762,[{"oid":"0000-0002-5734-2699","name":"Gregory","surname":"Murad Reis","creditName":null,"errorCode":null}]) +(10.1063/1.4963346,[{"oid":"0000-0002-7504-031X","name":"Lin","surname":"Gu","creditName":null,"errorCode":null}]) +(10.1016/j.jviromet.2014.06.023,[{"oid":"0000-0001-8276-1804","name":"MARGHERITA","surname":"PROFITI","creditName":null,"errorCode":null},{"oid":"0000-0002-8315-0316","name":"Sergio","surname":"Rosati","creditName":null,"errorCode":null},{"oid":"0000-0002-0344-3101","name":"Stefano","surname":"Nardelli","creditName":null,"errorCode":null},{"oid":"0000-0001-7883-1283","name":"Esterina","surname":"De Carlo","creditName":null,"errorCode":null}]) +(10.1590/S1678-58782011000500002,[{"oid":"0000-0003-1537-5430","name":"Mario Henrique","surname":"Macagnan","creditName":null,"errorCode":null}]) +(10.1080/17425255.2017.1290080,[{"oid":"0000-0003-4652-7089","name":"Michael","surname":"Durkin","creditName":null,"errorCode":null},{"oid":"0000-0003-4652-7089","name":"Michael","surname":"Durkin","creditName":null,"errorCode":null},{"oid":"0000-0003-4652-7089","name":"Michael","surname":"Durkin","creditName":null,"errorCode":null}]) +(10.1111/j.2047-6310.2013.00183.x,[{"oid":"0000-0002-0252-9923","name":"Thomas","surname":"Willis","creditName":"Thomas Andrew Willis","errorCode":null},{"oid":"0000-0002-0252-9923","name":"Thomas","surname":"Willis","creditName":"Thomas Andrew Willis","errorCode":null},{"oid":"0000-0002-4065-4397","name":"Charlotte","surname":"Evans","creditName":null,"errorCode":null}]) +(10.1098/rstb.2018.0138,[{"oid":"0000-0003-2308-2603","name":"Peter","surname":"Bossaerts","creditName":null,"errorCode":null}]) +(10.12660/GVCASOSV4N1N4,[{"oid":"0000-0003-2183-8112","name":"Pelayo Munhoz","surname":"Olea","creditName":"Pelayo Munhoz Olea","errorCode":null}]) +(10.1016/0306-4522(96)00157-1,[{"oid":"0000-0002-3525-4671","name":"Elek","surname":"Molnár","creditName":null,"errorCode":null}]) +(10.1016/j.jclinane.2018.12.004,[{"oid":"0000-0002-4776-3211","name":"Hiroshi","surname":"Otake","creditName":null,"errorCode":null}]) +(10.1098/rsta.1996.0131,[{"oid":"0000-0002-8781-6154","name":"Ekhard","surname":"Salje","creditName":"Salje","errorCode":null}]) +(10.1074/jbc.M113.526178,[{"oid":"0000-0002-7945-4050","name":"Jaime","surname":"Nagy","creditName":null,"errorCode":null},{"oid":"0000-0002-8302-6905","name":"Xiaolan","surname":"Zhao","creditName":null,"errorCode":null}]) +(10.1016/j.biomaterials.2009.08.014,[{"oid":"0000-0002-9774-7412","name":"Liam","surname":"Grover","creditName":null,"errorCode":null},{"oid":"0000-0001-6412-2371","name":"nicola","surname":"hunt","creditName":null,"errorCode":null}]) +(10.1016/j.physb.2011.09.063,[{"oid":"0000-0001-5768-0244","name":"Anne","surname":"Henry","creditName":null,"errorCode":null},{"oid":"0000-0001-7721-5091","name":"Erik","surname":"Janzen","creditName":null,"errorCode":null},{"oid":"0000-0003-4579-3529","name":"Stefano","surname":"Leone","creditName":null,"errorCode":null},{"oid":"0000-0003-4579-3529","name":"Stefano","surname":"Leone","creditName":null,"errorCode":null},{"oid":"0000-0002-7171-5383","name":"Henrik","surname":"Pedersen","creditName":"Henrik Pedersen","errorCode":null}]) +(http://dx.doi.org/10.1080/00958970412331295219,[{"oid":"0000-0002-3226-2959","name":"Magdi","surname":"Iskander","creditName":null,"errorCode":null}]) +(10.1166/sam.2012.1300,[{"oid":"0000-0002-0260-1059","name":"Haibo","surname":"Zeng","creditName":null,"errorCode":null},{"oid":"0000-0002-0281-3617","name":"Haibo","surname":"Zeng","creditName":null,"errorCode":null},{"oid":"0000-0002-0281-3617","name":"Haibo","surname":"Zeng","creditName":null,"errorCode":null},{"oid":"0000-0002-0281-3617","name":"Haibo","surname":"Zeng","creditName":null,"errorCode":null}]) +(10.2217/fmb.11.70,[{"oid":"0000-0003-4042-7466","name":"Grzegorz","surname":"Wegrzyn","creditName":null,"errorCode":null}]) +(10.1109/SIU.2008.4632665,[{"oid":"0000-0002-6335-459X","name":"AHMET","surname":"SAYAR","creditName":null,"errorCode":null}]) +(10.1002/sim.910,[{"oid":"0000-0002-1829-8664","name":"Noah","surname":"Rosenberg","creditName":null,"errorCode":null}]) +(10.1007/S12603-011-0071-Z,[{"oid":"0000-0001-7593-3081","name":"Jean","surname":"Woo","creditName":null,"errorCode":null}]) +(10.1364/PHOTONICS.2016.Th3A.11,[{"oid":"0000-0003-2418-8418","name":"Narayan","surname":"Krishnaswamy","creditName":null,"errorCode":null}]) +(10.1134/S0020168510050109,[{"oid":"0000-0002-2562-6427","name":"Nikolay","surname":"Kuznetsov","creditName":null,"errorCode":null}]) +(10.1128/AEM.71.8.4539-4547.2005,[{"oid":"0000-0003-0501-2556","name":"Martin W.","surname":"Hahn","creditName":null,"errorCode":null},{"oid":"0000-0003-0501-2556","name":"Martin W.","surname":"Hahn","creditName":null,"errorCode":null}]) +(10.1162/artl_a_00211,[{"oid":"0000-0003-0910-743X","name":"Tim","surname":"Taylor","creditName":"Timothy John Taylor","errorCode":null}]) +(10.1080/10934529.2015.1047675,[{"oid":"0000-0003-2020-9824","name":"Bogdan","surname":"Skwarzec","creditName":null,"errorCode":null},{"oid":"0000-0001-7900-6517","name":"Dagmara","surname":"Struminska-Parulska","creditName":"Dagmara Struminska-Parulska","errorCode":null}]) +(10.3109/16066359.2011.588352,[{"oid":"0000-0002-6792-916X","name":"David","surname":"Best","creditName":null,"errorCode":null}]) +(10.1007/s001090050204,[{"oid":"0000-0003-3051-1285","name":"Thierry","surname":"Calandra","creditName":null,"errorCode":null}]) +(10.1016/j.ajog.2007.10.519,[{"oid":"0000-0003-3159-6321","name":"Martin","surname":"Frasch","creditName":null,"errorCode":null}]) +(10.1111/phpr.12304,[{"oid":"0000-0001-6180-457X","name":"Douglas","surname":"Portmore","creditName":"Douglas W. Portmore","errorCode":null}]) +(10.2305/iucn.uk.2017-2.rlts.t22486174a22486850.en,[{"oid":"0000-0002-9847-2035","name":"Emma","surname":"Williams","creditName":null,"errorCode":null}]) +(10.3791/50878,[{"oid":"0000-0002-8699-6253","name":"Tessa","surname":"Durham Brooks","creditName":null,"errorCode":null},{"oid":"0000-0002-8699-6253","name":"Tessa","surname":"Durham Brooks","creditName":null,"errorCode":null}]) +(10.1007/s00340-009-3580-2,[{"oid":"0000-0002-2652-5134","name":"Ci-Ling","surname":"Pan","creditName":null,"errorCode":null},{"oid":"0000-0002-2652-5134","name":"Ci-Ling","surname":"Pan","creditName":null,"errorCode":null}]) +(10.1007/978-3-319-19216-1_38,[{"oid":"0000-0001-9922-0350","name":"Dariusz","surname":"Laskowski","creditName":null,"errorCode":null},{"oid":"0000-0002-1748-5070","name":"ŁUBKOWSKI","surname":"PIOTR","creditName":null,"errorCode":null}]) +(10.1007/s00192-017-3446-9,[{"oid":"0000-0002-9064-8454","name":"Katja","surname":"Stenström Bohlin","creditName":null,"errorCode":null}]) +(10.1088/0022-3700/13/6/001,[{"oid":"0000-0002-4357-6048","name":"Jacek","surname":"Migdalek","creditName":null,"errorCode":null}]) +(10.12776/qip.v16i2.67,[{"oid":"0000-0002-9155-189X","name":"Jostein","surname":"Langstrand","creditName":null,"errorCode":null}]) +(10.1016/j.ecoenv.2018.07.109,[{"oid":"0000-0003-0926-4849","name":"Galia","surname":null,"creditName":null,"errorCode":null},{"oid":"0000-0002-8857-4353","name":"Vladimír","surname":"Žlábek","creditName":null,"errorCode":null},{"oid":"0000-0003-3091-7824","name":"Sidika","surname":"Sakalli","creditName":null,"errorCode":null}]) +(10.1080/09585192.2013.870290,[{"oid":"0000-0001-8766-3314","name":"Andreas","surname":"Hirschi","creditName":null,"errorCode":null}]) +(10.1016/j.optcom.2010.07.033,[{"oid":"0000-0001-8387-8131","name":"Esteban","surname":"Vera","creditName":null,"errorCode":null},{"oid":"0000-0002-4834-7543","name":"Alberto","surname":"Lencina","creditName":null,"errorCode":null}]) +(10.1002/adma.201807383,[{"oid":"0000-0002-3042-0335","name":"Romain","surname":"Gautier","creditName":null,"errorCode":null},{"oid":"0000-0002-8671-0630","name":"Michael","surname":"PARIS","creditName":null,"errorCode":null}]) +(http://dx.doi.org/10.18561/2179-5746/biotaamazonia.v4n2p55-63,[{"oid":"0000-0003-2173-6968","name":"Marcelo","surname":"Moreira de Carvalho","creditName":"Carvalho M. M.","errorCode":null}]) +(10.1016/j.ehbc.2004.02.015,[{"oid":"0000-0001-7724-0668","name":"Eddy","surname":"Perez-Then","creditName":null,"errorCode":null},{"oid":"0000-0001-7724-0668","name":"Eddy","surname":"Perez-Then","creditName":null,"errorCode":null}]) +(10.1016/S0001-6519(08)75117-3,[{"oid":"0000-0001-5560-8097","name":"Francesc Xavier","surname":"Avilés-Jurado","creditName":null,"errorCode":null},{"oid":"0000-0001-5560-8097","name":"Francesc Xavier","surname":"Avilés-Jurado","creditName":null,"errorCode":null}]) +(10.1074/jbc.M409535200,[{"oid":"0000-0001-7811-7784","name":"Chihiro","surname":"Hisatsune","creditName":null,"errorCode":null},{"oid":"0000-0001-7713-9471","name":"Yukiko","surname":"Kuroda","creditName":"Yukiko Kuroda","errorCode":null}]) +(10.1063/1.98280,[{"oid":"0000-0002-1856-474X","name":"Brian","surname":"Skromme","creditName":null,"errorCode":null}]) +(10.1002/ajmg.a.37755,[{"oid":"0000-0003-3602-5704","name":null,"surname":null,"creditName":null,"errorCode":null},{"oid":"0000-0003-3602-5704","name":null,"surname":null,"creditName":null,"errorCode":null},{"oid":"0000-0003-4990-8784","name":"Anne","surname":"Trinh","creditName":null,"errorCode":null},{"oid":"0000-0003-4472-6893","name":"Stefano","surname":"Lise","creditName":null,"errorCode":null},{"oid":"0000-0002-1361-9174","name":"Caroline","surname":"Gorvin","creditName":null,"errorCode":null},{"oid":"0000-0002-1361-9174","name":"Caroline","surname":"Gorvin","creditName":null,"errorCode":null},{"oid":"0000-0002-1361-9174","name":"Caroline","surname":"Gorvin","creditName":null,"errorCode":null},{"oid":"0000-0002-3586-7654","name":"Sian","surname":"Piret","creditName":"Sian E Piret","errorCode":null}]) +(10.1002/jmor.10934,[{"oid":"0000-0002-5006-6679","name":"Miriam","surname":"Zelditch","creditName":null,"errorCode":null},{"oid":"0000-0002-5006-6679","name":"Miriam","surname":"Zelditch","creditName":null,"errorCode":null}]) +(http://dx.doi.org/10.1016/j.nucengdes.2010.12.007,[{"oid":"0000-0003-2653-3430","name":"Luca","surname":"Ammirabile","creditName":null,"errorCode":null}]) +(10.1016/S0009-2614(99)01368-8,[{"oid":"0000-0001-6672-2186","name":"Cleber","surname":"Mendonca","creditName":null,"errorCode":null},{"oid":"0000-0001-6624-8453","name":"Lino","surname":"Misoguti","creditName":null,"errorCode":null},{"oid":"0000-0003-2372-5509","name":"José Alberto","surname":"Giacometti","creditName":null,"errorCode":null}]) +(10.1038/ncomms2249,[{"oid":"0000-0002-1088-5565","name":"Santiago","surname":"Munne","creditName":null,"errorCode":null},{"oid":"0000-0002-8285-0222","name":"Shawn","surname":"Chavez","creditName":null,"errorCode":null},{"oid":"0000-0002-6487-1329","name":"Renee","surname":"Reijo Pera","creditName":null,"errorCode":null},{"oid":"0000-0002-6487-1329","name":"Renee","surname":"Reijo Pera","creditName":null,"errorCode":null}]) +(https://doi.org/10.1016/j.jclepro.2017.12.156,[{"oid":"0000-0003-4311-5185","name":"Hao","surname":"Zheng","creditName":null,"errorCode":null}]) +(10.1177/0022022112466590,[{"oid":"0000-0003-3764-3571","name":"Kenneth","surname":"Locke","creditName":null,"errorCode":null},{"oid":"0000-0002-4174-6955","name":"Guy","surname":"Curtis","creditName":null,"errorCode":null},{"oid":"0000-0002-4174-6955","name":"Guy","surname":"Curtis","creditName":null,"errorCode":null}]) +(10.3390/app8091453,[{"oid":"0000-0001-5102-0077","name":"Liu","surname":"Yuting","creditName":null,"errorCode":null},{"oid":"0000-0001-5102-0077","name":"Liu","surname":"Yuting","creditName":null,"errorCode":null},{"oid":"0000-0002-5578-408X","name":"Huanan","surname":"Liu","creditName":null,"errorCode":null}]) +(10.1063/1.3225247,[{"oid":"0000-0003-0574-1513","name":"Antonio","surname":"Garcia-Loureiro","creditName":null,"errorCode":null},{"oid":"0000-0002-6794-3893","name":"Manuel","surname":"Aldegunde","creditName":null,"errorCode":null},{"oid":"0000-0003-0973-461X","name":"Natalia","surname":"Seoane","creditName":null,"errorCode":null}]) +(10.1200/jco.2015.33.7_suppl.41,[{"oid":"0000-0002-2500-5030","name":"Julien","surname":"Boudon","creditName":null,"errorCode":null}]) +(10.1037/e592712011-023,[{"oid":"0000-0003-1574-585X","name":"Samuel","surname":"Posner","creditName":null,"errorCode":null}]) +(10.1007/s00605-013-0526-x,[{"oid":"0000-0003-2371-4818","name":"Ákos","surname":"G.Horváth","creditName":null,"errorCode":null}]) +(10.1371/journal.pone.0139754,[{"oid":"0000-0002-9109-0958","name":"Julien","surname":"Louis","creditName":null,"errorCode":null}]) +(10.1021/jp054948x,[{"oid":"0000-0002-2033-5284","name":"Victor","surname":"Climent","creditName":null,"errorCode":null},{"oid":"0000-0003-4751-3279","name":"Juan","surname":"Feliu","creditName":null,"errorCode":null},{"oid":"0000-0002-5231-8032","name":"Roberto","surname":"Gómez","creditName":"Roberto Gómez","errorCode":null}]) +(10.1155/2016/9828517,[{"oid":"0000-0002-2030-0356","name":"Ewa","surname":"Jasek-Gajda","creditName":null,"errorCode":null},{"oid":"0000-0002-4541-2864","name":"Ewa","surname":"Siucinska","creditName":null,"errorCode":null},{"oid":"0000-0002-8102-6039","name":"Malgorzata","surname":"Jasinska","creditName":null,"errorCode":null},{"oid":"0000-0002-8102-6039","name":"Malgorzata","surname":"Jasinska","creditName":null,"errorCode":null},{"oid":"0000-0001-7808-1101","name":"Malgorzata","surname":"Kossut","creditName":null,"errorCode":null}]) +(10.1002/pi.2097,[{"oid":"0000-0002-4375-5580","name":"Graeme","surname":"Moad","creditName":"Graeme Moad","errorCode":null},{"oid":"0000-0003-2849-229X","name":"John","surname":"Forsythe","creditName":null,"errorCode":null}]) +(10.1117/12.432956,[{"oid":"0000-0002-4209-1372","name":"Michael","surname":"Bakunov","creditName":null,"errorCode":null}]) +(10.2144/000114087,[{"oid":"0000-0001-8479-6269","name":"HERNAN GUILLERMO","surname":"HERNANDEZ","creditName":null,"errorCode":null},{"oid":"0000-0001-9175-3363","name":"Diego","surname":"Forero","creditName":"Forero DA","errorCode":null}]) +(10.1016/j.bcp.2015.08.003,[{"oid":"0000-0002-8247-9494","name":"Istvan","surname":"Czikora","creditName":null,"errorCode":null},{"oid":"0000-0003-3805-8868","name":"Rudolf","surname":"Lucas","creditName":null,"errorCode":null}]) +(10.1016/j.foreco.2016.08.036,[{"oid":"0000-0001-7999-1692","name":"Dugald","surname":"Close","creditName":null,"errorCode":null}]) +(10.2903/j.efsa.2017.4782,[{"oid":"0000-0002-4720-2940","name":"Margaret","surname":"Good","creditName":null,"errorCode":null},{"oid":"0000-0002-2634-3138","name":"Antonio","surname":"Velarde","creditName":null,"errorCode":null},{"oid":"0000-0003-2417-0787","name":"Søren","surname":"Nielsen","creditName":"Søren Saxmose Nielsen","errorCode":null},{"oid":"0000-0002-5904-8397","name":"Andrew","surname":"Butterworth","creditName":"Andrew Butterworth","errorCode":null}]) +(10.1002/ejoc.201300745,[{"oid":"0000-0003-0168-9295","name":"Emanuela","surname":"Licandro","creditName":null,"errorCode":null},{"oid":"0000-0001-9252-6218","name":"Alberto","surname":"Bossi","creditName":null,"errorCode":null},{"oid":"0000-0001-9252-6218","name":"Alberto","surname":"Bossi","creditName":null,"errorCode":null},{"oid":"0000-0001-9252-6218","name":"Alberto","surname":"Bossi","creditName":null,"errorCode":null},{"oid":"0000-0003-4063-1563","name":"Patrizia Romana","surname":"Mussini","creditName":null,"errorCode":null},{"oid":"0000-0003-4063-1563","name":"Patrizia Romana","surname":"Mussini","creditName":null,"errorCode":null},{"oid":"0000-0002-9540-9073","name":"SILVIA","surname":"CAUTERUCCIO","creditName":null,"errorCode":null},{"oid":"0000-0002-9540-9073","name":"SILVIA","surname":"CAUTERUCCIO","creditName":null,"errorCode":null}]) +(10.1038/s41431-018-0183-6,[{"oid":"0000-0002-4475-4117","name":"Simone","surname":"Baldovino","creditName":null,"errorCode":null},{"oid":"0000-0001-6133-9831","name":"Marcella","surname":"Neri","creditName":null,"errorCode":null},{"oid":"0000-0003-4187-8816","name":"GIOVANNA","surname":"FLORIDIA","creditName":null,"errorCode":null},{"oid":"0000-0001-9406-5734","name":"Paraskevas","surname":"Iatropoulos","creditName":null,"errorCode":null}]) +(10.1038/sj.ijo.0800401,[{"oid":"0000-0002-7916-4619","name":"Frederic","surname":"Fumeron","creditName":null,"errorCode":null}]) +(10.1080/10934529.2012.668029,[{"oid":"0000-0002-6255-1153","name":"David","surname":"Stuckey","creditName":null,"errorCode":null},{"oid":"0000-0002-6255-1153","name":"David","surname":"Stuckey","creditName":null,"errorCode":null},{"oid":"0000-0002-2880-708X","name":"Antoine","surname":"Trzcinski","creditName":null,"errorCode":null}]) +(10.1371/journal.pmed.1001427,[{"oid":"0000-0001-6857-8931","name":"Etheldreda","surname":"Nakimuli-Mpungu","creditName":null,"errorCode":null}]) +(10.1186/S13660-015-0787-0,[{"oid":"0000-0003-1769-2800","name":"Ke","surname":"Wang","creditName":"Ke Wang","errorCode":null}]) +(10.2174/157018011794578204,[{"oid":"0000-0002-8902-2876","name":"Gajendra","surname":"Raghava","creditName":"G. P. S. Raghava","errorCode":null}]) +(10.1016/j.jtho.2016.11.002,[{"oid":"0000-0002-3832-8744","name":"Chandana","surname":"Reddy","creditName":null,"errorCode":null}]) +(10.26502/jesph.96120058,[{"oid":"0000-0002-1942-7384","name":"Anahi","surname":"Aguilera","creditName":null,"errorCode":null}]) +(10.1007/BF00749305,[{"oid":"0000-0003-4546-002X","name":"Oleg","surname":"Ponomarev","creditName":null,"errorCode":null},{"oid":"0000-0002-0362-4121","name":"Farid","surname":"Enikeev","creditName":null,"errorCode":null}]) +(10.1016/j.engfracmech.2015.01.008,[{"oid":"0000-0003-1117-9619","name":"Xian-Fang","surname":"Li","creditName":"Xian-Fang Li","errorCode":null}]) +(10.1007/s10898-015-0341-0,[{"oid":"0000-0001-6340-385X","name":"Semu Mitiku","surname":"Kassa","creditName":null,"errorCode":null},{"oid":"0000-0001-5494-040X","name":"Semu Mitiku","surname":"Kassa","creditName":null,"errorCode":null}]) +(10.1177/0091270010395591,[{"oid":"0000-0002-4227-5817","name":"David","surname":"Le Couteur","creditName":null,"errorCode":null},{"oid":"0000-0002-5970-1501","name":"Sarah","surname":"Hilmer","creditName":null,"errorCode":null}]) +(10.1016/j.cellimm.2016.09.003,[{"oid":"0000-0002-7412-9507","name":"Joseph","surname":"Mattapallil","creditName":null,"errorCode":null}]) +(10.1158/0008-5472.CAN-09-1091,[{"oid":"0000-0002-6093-0395","name":"Rajiv","surname":"Kumar","creditName":null,"errorCode":null},{"oid":"0000-0003-0674-7757","name":"Giuseppe","surname":"MATULLO","creditName":null,"errorCode":null},{"oid":"0000-0001-9599-309X","name":"Alessandra","surname":"Allione","creditName":null,"errorCode":null},{"oid":"0000-0003-1901-9513","name":"Heather","surname":"Nelson","creditName":null,"errorCode":null},{"oid":"0000-0002-8752-8785","name":"Tim","surname":"Bishop","creditName":"D T Bishop","errorCode":null},{"oid":"0000-0002-0787-3969","name":"Gunnar","surname":"Steineck","creditName":null,"errorCode":null},{"oid":"0000-0001-7208-2912","name":"Anne","surname":"Kiltie","creditName":null,"errorCode":null},{"oid":"0000-0002-4669-3995","name":"Zuo-Feng","surname":"Zhang","creditName":null,"errorCode":null},{"oid":"0000-0002-4620-3108","name":"Jian-Min","surname":"Yuan","creditName":null,"errorCode":null},{"oid":"0000-0002-3561-6580","name":"Ananya","surname":"Choudhury","creditName":null,"errorCode":null},{"oid":"0000-0002-3561-6580","name":"Ananya","surname":"Choudhury","creditName":null,"errorCode":null},{"oid":"0000-0002-1720-7724","name":"Jenny","surname":"Barrett","creditName":null,"errorCode":null},{"oid":"0000-0002-5277-8477","name":"Marcello","surname":"Campagna","creditName":null,"errorCode":null},{"oid":"0000-0003-2538-3784","name":"Nuria","surname":"Malats","creditName":null,"errorCode":null}]) +(10.1016/j.intell.2007.04.003,[{"oid":"0000-0003-3159-9370","name":"Alasdair","surname":"MacLullich","creditName":null,"errorCode":null}]) +(10.1021/acs.orglett.7b00261,[{"oid":"0000-0002-7178-2981","name":"Jun Yong","surname":"Kang","creditName":null,"errorCode":null},{"oid":"0000-0002-9732-1454","name":"Hai","surname":"Huang","creditName":null,"errorCode":null}]) +(10.1088/0004-637X/778/2/175,[{"oid":"0000-0003-0489-0920","name":"Alexei","surname":"Pevtsov","creditName":null,"errorCode":null},{"oid":"0000-0003-0489-0920","name":"Alexei","surname":"Pevtsov","creditName":null,"errorCode":null}]) +(10.22201/iis.01882503p.2018.4.57796,[{"oid":"0000-0002-7334-1936","name":"Yanina","surname":"Welp","creditName":null,"errorCode":null},{"oid":"0000-0001-9905-0777","name":"Flavia","surname":"Freidenberg","creditName":null,"errorCode":null},{"oid":"0000-0001-9905-0777","name":"Flavia","surname":"Freidenberg","creditName":null,"errorCode":null}]) +(10.1109/IWAGPR.2017.7996109,[{"oid":"0000-0003-1698-0800","name":"Miro","surname":"Govedarica","creditName":null,"errorCode":null}]) +(10.1207/s15328015tlm1603_6,[{"oid":"0000-0002-0577-1440","name":"Theodor","surname":"Adla","creditName":null,"errorCode":null}]) +(10.1109/CBMI.2009.10,[{"oid":"0000-0002-1364-218X","name":"Marco","surname":"Bertini","creditName":"Marco Bertini","errorCode":null},{"oid":"0000-0002-4269-4501","name":"Giuseppe","surname":"Serra","creditName":null,"errorCode":null},{"oid":"0000-0002-1052-8322","name":"ALBERTO","surname":"DEL BIMBO","creditName":null,"errorCode":null},{"oid":"0000-0003-0819-851X","name":"Lamberto","surname":"Ballan","creditName":null,"errorCode":null}]) +(10.1017/S003329171200147X,[{"oid":"0000-0001-8198-9022","name":"Amanda","surname":"Baxter","creditName":null,"errorCode":null},{"oid":"0000-0003-4667-6623","name":"Harvey","surname":"Whiteford","creditName":null,"errorCode":null},{"oid":"0000-0003-4667-6623","name":"Harvey","surname":"Whiteford","creditName":null,"errorCode":null}]) +(10.1088/0305-4470/30/2/021,[{"oid":"0000-0001-6097-1202","name":"Andrés","surname":"Riaguas","creditName":null,"errorCode":null}]) +(10.4103/1658-354X.169476,[{"oid":"0000-0002-1009-0869","name":"KUSAI","surname":"BAROUDI","creditName":null,"errorCode":null},{"oid":"0000-0002-1009-0869","name":"KUSAI","surname":"BAROUDI","creditName":null,"errorCode":null}]) +(10.1007/s12414-015-0042-0,[{"oid":"0000-0003-3752-7837","name":"Matthijs","surname":"Janssen","creditName":null,"errorCode":null}]) +(10.1007/978-3-319-08491-6_5,[{"oid":"0000-0002-4728-689X","name":"Agnieszka","surname":"Landowska","creditName":null,"errorCode":null},{"oid":"0000-0002-1117-903X","name":"Michał","surname":"Wróbel","creditName":null,"errorCode":null}]) +(10.3390/inorganics7010008,[{"oid":"0000-0003-4502-5212","name":"Ian","surname":"Dance","creditName":null,"errorCode":null}]) +(10.4028/www.scientific.net/DDF.297-301.814,[{"oid":"0000-0002-6970-0406","name":"Jaroslav","surname":"Kováčik","creditName":null,"errorCode":null}]) +(10.1117/12.2257557,[{"oid":"0000-0002-7409-3305","name":"Jan","surname":"Kulawik","creditName":null,"errorCode":null}]) +(10.1109/SIBGRAPI.2011.20,[{"oid":"0000-0001-8612-5805","name":"Fabio","surname":"Miranda","creditName":null,"errorCode":null}]) +(10.1089/fpd.2012.1448,[{"oid":"0000-0001-7995-7427","name":"Beilei","surname":"Ge","creditName":null,"errorCode":null}]) +(10.1016/j.lungcan.2013.04.006,[{"oid":"0000-0001-8100-7914","name":"Ruey-Long","surname":"Hong","creditName":null,"errorCode":null}]) +(10.1109/IROS.2005.1545338,[{"oid":"0000-0002-4612-3554","name":"Rui P.","surname":"Rocha","creditName":"Rui P. Rocha","errorCode":null},{"oid":"0000-0003-3729-5263","name":"Adriano","surname":"Carvalho","creditName":null,"errorCode":null},{"oid":"0000-0002-2725-8867","name":"Jorge","surname":"Dias","creditName":"Jorge Dias","errorCode":null}]) +(10.3182/20100901-3-IT-2016.00036,[{"oid":"0000-0001-8247-7790","name":"Hélène","surname":"Piet-Lahanier","creditName":null,"errorCode":null}]) +(10.1016/S0925-9635(99)00257-5,[{"oid":"0000-0002-2400-1759","name":"BRAMBILLA","surname":"Andrea","creditName":null,"errorCode":null},{"oid":"0000-0002-0647-9134","name":"Philippe","surname":"Bergonzo","creditName":null,"errorCode":null}]) +(10.1016/j.jsg.2011.08.008,[{"oid":"0000-0002-4647-8224","name":"Bernhard","surname":"Grasemann","creditName":null,"errorCode":null},{"oid":"0000-0002-4647-8224","name":"Bernhard","surname":"Grasemann","creditName":null,"errorCode":null}]) +(10.1016/S1387-6473(03)00072-1,[{"oid":"0000-0001-5273-9817","name":"James","surname":"Lovell","creditName":null,"errorCode":null}]) +(10.1039/C9TA02405K,[{"oid":"0000-0003-3847-2620","name":"Ch Venkata","surname":"Surya Kumar","creditName":null,"errorCode":null},{"oid":"0000-0001-6353-8877","name":"Ki Tae","surname":"Nam","creditName":null,"errorCode":null}]) +(10.1007/s00216-008-2520-z,[{"oid":"0000-0002-5134-7130","name":"Helmut","surname":"Ehrenberg","creditName":null,"errorCode":null},{"oid":"0000-0002-5134-7130","name":"Helmut","surname":"Ehrenberg","creditName":null,"errorCode":null},{"oid":"0000-0002-5106-9214","name":"Kristian","surname":"Nikolowski","creditName":null,"errorCode":null}]) +(10.1007/s13760-018-0889-9,[{"oid":"0000-0002-6644-9148","name":"Ahmet","surname":"yildirim","creditName":null,"errorCode":null}]) +(10.1016/j.enpol.2012.10.037,[{"oid":"0000-0002-0476-2315","name":"Anu","surname":"Ramaswami","creditName":null,"errorCode":null}]) +(10.1007/978-981-13-2517-5_65,[{"oid":"0000-0002-1182-0552","name":"Tiago","surname":"Benetti","creditName":null,"errorCode":null}]) +(10.1179/1973947814Y.0000000172,[{"oid":"0000-0001-5610-7745","name":"Fernanda","surname":"Ruiz Larrea","creditName":null,"errorCode":null},{"oid":"0000-0003-0023-3069","name":"Vanesa","surname":"Estepa","creditName":null,"errorCode":null},{"oid":"0000-0003-3709-1690","name":"Carmen","surname":"Torres","creditName":null,"errorCode":null}]) +(10.1016/s0378-1119(00)00530-8,[{"oid":"0000-0001-7584-3721","name":"Tzung-Fu","surname":"Hsieh","creditName":null,"errorCode":null}]) +(10.1149/1.3567401,[{"oid":"0000-0002-9655-6155","name":"Mohammad Reza","surname":"Hantehzadeh","creditName":null,"errorCode":null}]) +(10.1029/2012jd018226,[{"oid":"0000-0002-1520-4386","name":"永红","surname":"胡","creditName":"胡永红","errorCode":null},{"oid":"0000-0001-5950-9555","name":"Gensuo","surname":"Jia","creditName":null,"errorCode":null}]) +(10.1186/1477-7517-11-31,[{"oid":"0000-0002-2191-7833","name":"Tim","surname":"Mackey","creditName":null,"errorCode":null},{"oid":"0000-0003-0726-0676","name":"Leo","surname":"Beletsky","creditName":null,"errorCode":null},{"oid":"0000-0003-3376-152X","name":"Maria Gudelia","surname":"Rangel","creditName":null,"errorCode":null},{"oid":"0000-0002-7724-691X","name":"Steffanie","surname":"Strathdee","creditName":null,"errorCode":null}]) +(10.1023/A:1022686622752,[{"oid":"0000-0002-0534-7797","name":"Stéphane","surname":"Le Crom","creditName":null,"errorCode":null},{"oid":"0000-0001-9327-5166","name":"Marika","surname":"Kapsimali","creditName":null,"errorCode":null}]) +(10.1524/zpch.2012.0224,[{"oid":"0000-0003-1563-9176","name":"Paul","surname":"Heitjans","creditName":null,"errorCode":null},{"oid":"0000-0001-9706-4892","name":"H. Martin R.","surname":"Wilkening","creditName":"Prof. Dr. Martin Wilkening","errorCode":null},{"oid":"0000-0001-9706-4892","name":"H. Martin R.","surname":"Wilkening","creditName":"Prof. Dr. Martin Wilkening","errorCode":null},{"oid":"0000-0001-9706-4892","name":"H. Martin R.","surname":"Wilkening","creditName":"Prof. Dr. Martin Wilkening","errorCode":null}]) +(10.1080/03155986.2017.1393730,[{"oid":"0000-0002-8203-0689","name":"Linyan","surname":"Zhang","creditName":null,"errorCode":null}]) +(10.1016/j.jbiotec.2011.06.040,[{"oid":"0000-0001-7635-1075","name":"Cecilia","surname":"Faraloni","creditName":null,"errorCode":null},{"oid":"0000-0001-7981-0847","name":"Alberto","surname":"Scoma","creditName":"Alberto Scoma","errorCode":null}]) +(10.1055/s-0033-1361119,[{"oid":"0000-0001-9369-5136","name":"Farnaz","surname":"Monajjemzadeh","creditName":null,"errorCode":null}]) +(10.1016/j.ijporl.2013.03.025,[{"oid":"0000-0003-4084-2301","name":"Chul Ho","surname":"Jang","creditName":null,"errorCode":null}]) +(10.3758/s13423-011-0203-9,[{"oid":"0000-0002-7584-2275","name":"Patrik","surname":"Sörqvist","creditName":null,"errorCode":null}]) +(10.1016/s0306-4522(02)00200-2,[{"oid":"0000-0002-8323-8211","name":"BS","surname":"Shankaranarayana Rao","creditName":null,"errorCode":null}]) +(10.1021/tx030036f,[{"oid":"0000-0003-4194-6401","name":"Nico P.E.","surname":"Vermeulen","creditName":"N.P.E. Vermeulen","errorCode":null},{"oid":"0000-0002-8358-9425","name":"Angel","surname":"Messeguer","creditName":null,"errorCode":null}]) +(10.1063/1.4819076,[{"oid":"0000-0003-2367-3825","name":"Timothy","surname":"Hele","creditName":null,"errorCode":null}]) +(10.1049/el.2014.2651,[{"oid":"0000-0002-2826-6367","name":"Ivan","surname":"Lee","creditName":null,"errorCode":null}]) +(10.1300/J137v15n02_02,[{"oid":"0000-0001-8374-2277","name":"Donna","surname":"Ryan","creditName":null,"errorCode":null}]) +(10.1590/S1413-78522013000100003,[{"oid":"0000-0003-3584-3342","name":"Aldo José","surname":"Fontes Pereira","creditName":"Pereira, A.J.F","errorCode":null}]) +(10.1039/c5lc01356a,[{"oid":"0000-0002-3084-0823","name":"Anna","surname":"Marsano","creditName":null,"errorCode":null},{"oid":"0000-0001-7115-0151","name":"Emiliano","surname":"Votta","creditName":null,"errorCode":null}]) +(10.1007/s10903-018-0840-4,[{"oid":"0000-0003-1710-3578","name":"Ahsan","surname":"Saleem","creditName":null,"errorCode":null}]) +(10.1007/s00216-015-8936-3,[{"oid":"0000-0003-4741-8650","name":"Nina","surname":"Kroepfl","creditName":null,"errorCode":null}]) +(10.1109/ISCAS.2018.8351770,[{"oid":"0000-0002-7158-1426","name":"Panu","surname":"Sjövall","creditName":null,"errorCode":null}]) +(10.1021/jm100732t,[{"oid":"0000-0002-1767-7072","name":"Matthew","surname":"Fuchter","creditName":null,"errorCode":null},{"oid":"0000-0002-1767-7072","name":"Matthew","surname":"Fuchter","creditName":null,"errorCode":null},{"oid":"0000-0002-5658-8486","name":"Paul","surname":"Freemont","creditName":null,"errorCode":null},{"oid":"0000-0002-5658-8486","name":"Paul","surname":"Freemont","creditName":null,"errorCode":null}]) +(10.11588/BJB.2004.0.29842,[{"oid":"0000-0002-0555-3451","name":"Frank","surname":"Siegmund","creditName":null,"errorCode":null}]) +(10.1006/jctb.1999.1911,[{"oid":"0000-0002-2692-9198","name":"John","surname":"Caughman","creditName":null,"errorCode":null},{"oid":"0000-0002-2692-9198","name":"John","surname":"Caughman","creditName":null,"errorCode":null}]) +(10.1007/BF02125350,[{"oid":"0000-0002-4281-1843","name":"Gábor","surname":"Tardos","creditName":null,"errorCode":null}]) +(10.1039/C9SC01502G,[{"oid":"0000-0003-3527-7379","name":"Andrzej","surname":"Sienkiewicz","creditName":null,"errorCode":null},{"oid":"0000-0002-7515-7593","name":"Farzaneh","surname":"Fadaei Tirani","creditName":"Farzaneh Fadaei Tirani","errorCode":null}]) +(10.1063/1.119082,[{"oid":"0000-0001-8187-7469","name":"Hans","surname":"Christen","creditName":null,"errorCode":null}]) +(10.1109/OCEANSAP.2016.7485389,[{"oid":"0000-0001-7619-8015","name":"Horst","surname":"Hellbrück","creditName":null,"errorCode":null}]) +(10.1071/AN11006,[{"oid":"0000-0001-7750-0570","name":"Michael","surname":"Friend","creditName":null,"errorCode":null},{"oid":"0000-0001-5129-2216","name":"Susan","surname":"Robertson","creditName":null,"errorCode":null},{"oid":"0000-0001-5639-9581","name":"John","surname":"Broster","creditName":null,"errorCode":null}]) +(10.1186/1746-4811-4-1,[{"oid":"0000-0002-1872-2998","name":"Wilhelm","surname":"Gruissem","creditName":null,"errorCode":null},{"oid":"0000-0002-6645-1862","name":"Lars","surname":"Hennig","creditName":null,"errorCode":null},{"oid":"0000-0002-6645-1862","name":"Lars","surname":"Hennig","creditName":null,"errorCode":null},{"oid":"0000-0002-4802-754X","name":"Matthias","surname":"Hirsch-Hoffmann","creditName":null,"errorCode":null}]) +(10.1109/WCNC.2014.6951929,[{"oid":"0000-0003-1802-5022","name":"Michael","surname":"Heimlich","creditName":null,"errorCode":null},{"oid":"0000-0003-1802-5022","name":"Michael","surname":"Heimlich","creditName":null,"errorCode":null}]) +(10.1210/jc.2006-0075,[{"oid":"0000-0003-0637-1577","name":"Raj","surname":"Vuppalanchi","creditName":null,"errorCode":null}]) +(10.1590/S1678-91992005000300012,[{"oid":"0000-0001-9319-7516","name":"Rodrigo","surname":"Silva","creditName":"Da Silva, Rodrigo Costa","errorCode":null}]) +(10.1088/0953-2048/24/7/075025,[{"oid":"0000-0001-7488-066X","name":"Qing-Ping","surname":"Ding","creditName":null,"errorCode":null},{"oid":"0000-0003-1796-6840","name":"Toshihiro","surname":"Taen","creditName":null,"errorCode":null}]) +(10.1016/j.apenergy.2015.11.058,[{"oid":"0000-0003-2692-8299","name":"Zia","surname":"Wadud","creditName":null,"errorCode":null}]) +(10.1002/ange.201106310,[{"oid":"0000-0002-7898-317X","name":"Belén","surname":"Martín-Matute","creditName":null,"errorCode":null},{"oid":"0000-0002-1534-2690","name":"Agnieszka","surname":"Bartoszewicz","creditName":null,"errorCode":null}]) +(10.1016/s0002-9440(10)64486-0,[{"oid":"0000-0001-6151-4257","name":"Markus","surname":"Britschgi","creditName":"Markus Britschgi","errorCode":null}]) +(10.1021/ac0715672,[{"oid":"0000-0002-3664-5711","name":"Maria Jose","surname":"Gomez Ramos","creditName":null,"errorCode":null},{"oid":"0000-0002-3664-5711","name":"Maria Jose","surname":"Gomez Ramos","creditName":null,"errorCode":null},{"oid":"0000-0003-4321-6795","name":"Maria Jesus","surname":"Martinez Bueno","creditName":null,"errorCode":null},{"oid":"0000-0002-1963-8106","name":"Dolores","surname":"Hernando","creditName":null,"errorCode":null},{"oid":"0000-0003-2649-6772","name":"Ana","surname":"Agüera","creditName":null,"errorCode":null},{"oid":"0000-0001-9158-0271","name":"JUAN F","surname":"GARCÍA-REYES","creditName":null,"errorCode":null}]) +(10.5020/18061230.2014.319,[{"oid":"0000-0002-9554-1736","name":"Thiago","surname":"Vasconcelos","creditName":null,"errorCode":null}]) +(10.1109/ICoAC.2017.7951738,[{"oid":"0000-0001-8438-0175","name":"Nedunchezhian","surname":"Raju","creditName":null,"errorCode":null},{"oid":"0000-0002-3769-657X","name":"Vijayakumar","surname":"Velusamy","creditName":null,"errorCode":null},{"oid":"0000-0002-3769-657X","name":"Vijayakumar","surname":"Velusamy","creditName":null,"errorCode":null}]) +(10.1126/scitranslmed.3002785,[{"oid":"0000-0003-3619-1657","name":"Christian","surname":"Ottensmeier","creditName":null,"errorCode":null}]) +(10.1007/s00245-010-9130-9,[{"oid":"0000-0002-8510-7477","name":"Catherine","surname":"Donnelly","creditName":null,"errorCode":null}]) +(10.1530/endoabs.37.ep899,[{"oid":"0000-0002-9205-9530","name":"Volodymyr","surname":"Pankiv","creditName":null,"errorCode":null}]) +(10.1029/2010GL045928,[{"oid":"0000-0003-1378-8434","name":"Shari","surname":"Yvon-Lewis","creditName":"Shari A Yvon-Lewis","errorCode":null},{"oid":"0000-0003-1097-6800","name":"John","surname":"Kessler","creditName":null,"errorCode":null},{"oid":"0000-0003-1097-6800","name":"John","surname":"Kessler","creditName":null,"errorCode":null}]) +(10.1016/S0952-7915(00)00127-8,[{"oid":"0000-0001-6479-5330","name":"Gunther","surname":"Eysenbach","creditName":null,"errorCode":null}]) +(10.1103/PhysRevB.83.064508,[{"oid":"0000-0001-7991-3918","name":"Jianxin","surname":"Zhu","creditName":null,"errorCode":null}]) +(10.1002/ETT.2664,[{"oid":"0000-0001-6868-6860","name":"Qianbin","surname":null,"creditName":null,"errorCode":null}]) +(10.1252/jcej.12we249,[{"oid":"0000-0002-7423-5561","name":"Takayoshi","surname":"Ishimoto","creditName":null,"errorCode":null},{"oid":"0000-0003-4347-9923","name":"Michihisa","surname":"Koyama","creditName":null,"errorCode":null}]) +(10.1088/0305-4470/37/13/006,[{"oid":"0000-0002-2295-8055","name":"Epifanio Guido","surname":"Virga","creditName":null,"errorCode":null}]) +(10.14419/IJET.V7I4.44.26872,[{"oid":"0000-0002-3164-5157","name":"Edy","surname":"Budiman","creditName":null,"errorCode":null}]) +(10.1117/12.667839,[{"oid":"0000-0002-1551-6646","name":"Mikhail","surname":"Bryushinin","creditName":null,"errorCode":null}]) +(10.1021/acs.jafc.6b03279,[{"oid":"0000-0002-4145-5031","name":"Manuel","surname":"Mota","creditName":"Mota, MM","errorCode":null},{"oid":"0000-0002-4145-5031","name":"Manuel","surname":"Mota","creditName":"Mota, MM","errorCode":null},{"oid":"0000-0003-2817-7943","name":"Jorge","surname":"Faria","creditName":"Jorge M. S. Faria","errorCode":null},{"oid":"0000-0003-2817-7943","name":"Jorge","surname":"Faria","creditName":"Jorge M. S. Faria","errorCode":null},{"oid":"0000-0003-3257-9777","name":"Ana M.","surname":"Rodrigues","creditName":null,"errorCode":null}]) +(10.1117/12.2228432,[{"oid":"0000-0003-2590-034X","name":"Peppino","surname":"Fazio","creditName":null,"errorCode":null},{"oid":"0000-0003-0593-5254","name":"Mauro","surname":"Tropea","creditName":null,"errorCode":null}]) +(10.1186/s40349-016-0049-8,[{"oid":"0000-0002-4271-7670","name":"Marc Nicola","surname":"Gallay","creditName":null,"errorCode":null}]) +(10.1055/s-0029-1214397,[{"oid":"0000-0002-5286-7322","name":"Eva","surname":"Münster","creditName":null,"errorCode":null},{"oid":"0000-0002-5286-7322","name":"Eva","surname":"Münster","creditName":null,"errorCode":null}]) +(10.1108/00197850610671991,[{"oid":"0000-0003-1239-8733","name":"viki","surname":"holton","creditName":null,"errorCode":null},{"oid":"0000-0003-1239-8733","name":"viki","surname":"holton","creditName":null,"errorCode":null}]) +(10.1186/s12863-017-0480-z,[{"oid":"0000-0001-7747-0930","name":"Puthen Veettil","surname":"Jithesh","creditName":null,"errorCode":null}]) +(10.1109/jsen.2011.2106156,[{"oid":"0000-0002-4213-9575","name":"Yael","surname":"Hanein","creditName":null,"errorCode":null}]) +(10.1115/FEDSM2012-72228,[{"oid":"0000-0001-6692-858X","name":"Yogesh","surname":"Joshi","creditName":null,"errorCode":null}]) +(10.1039/c5cp03696h,[{"oid":"0000-0002-0664-2536","name":"François","surname":"LIQUE","creditName":null,"errorCode":null},{"oid":"0000-0002-7407-303X","name":"Jacek","surname":"Klos","creditName":null,"errorCode":null}]) +(10.1016/0308-8146(95)00112-3,[{"oid":"0000-0002-3438-0852","name":"José María","surname":"Fresno-Baro","creditName":null,"errorCode":null},{"oid":"0000-0002-3438-0852","name":"José María","surname":"Fresno-Baro","creditName":null,"errorCode":null},{"oid":"0000-0002-3737-9830","name":"Javier","surname":"Carballo","creditName":null,"errorCode":null},{"oid":"0000-0002-1697-178X","name":"María Josefa","surname":"González Prieto","creditName":null,"errorCode":null},{"oid":"0000-0002-5658-6720","name":"Ana","surname":"Bernardo Álvarez","creditName":null,"errorCode":null}]) +(10.1016/j.jmaa.2005.05.047,[{"oid":"0000-0002-8320-3596","name":"Kazimierz","surname":"Wlodarczyk","creditName":null,"errorCode":null},{"oid":"0000-0001-6635-0781","name":"Dariusz","surname":"Wardowski","creditName":null,"errorCode":null}]) +(10.1056/NEJM199312233292602,[{"oid":"0000-0002-9631-1254","name":"Martin","surname":"Larson","creditName":null,"errorCode":null}]) +(10.1016/j.apergo.2008.01.001,[{"oid":"0000-0001-7844-4881","name":"Vilhelm F","surname":"Koefoed","creditName":null,"errorCode":null}]) +(10.1007/s00421-011-2121-y,[{"oid":"0000-0001-6864-7706","name":"Taku","surname":"Wakahara","creditName":null,"errorCode":null}]) +(10.1016/j.agrformet.2007.05.012,[{"oid":"0000-0001-5779-2764","name":"Rachid","surname":"Hadria","creditName":null,"errorCode":null},{"oid":"0000-0002-3905-7560","name":"Gilles","surname":"Boulet","creditName":null,"errorCode":null}]) +(10.1393/ncb/i2007-10061-0,[{"oid":"0000-0002-7349-1109","name":"dafne","surname":"guetta","creditName":null,"errorCode":null}]) +(10.1016/j.ajog.2015.11.023,[{"oid":"0000-0001-7122-6187","name":"Katie","surname":"Propst","creditName":null,"errorCode":null},{"oid":"0000-0001-7122-6187","name":"Katie","surname":"Propst","creditName":null,"errorCode":null},{"oid":"0000-0002-2044-1693","name":"Tyler","surname":"Muffly","creditName":null,"errorCode":null},{"oid":"0000-0002-2044-1693","name":"Tyler","surname":"Muffly","creditName":null,"errorCode":null}]) +(doi:10.1166/jnn.2018.15274,[{"oid":"0000-0003-0582-575X","name":"David","surname":"Rojas","creditName":"D. Rojas","errorCode":null}]) +(10.21061/cc.v2i1.a.10,[{"oid":"0000-0001-7242-6762","name":"Jacob","surname":"Bruggeman","creditName":null,"errorCode":null}]) +(10.3390/nu10050622,[{"oid":"0000-0001-6890-5250","name":"Marjory","surname":"Moodie","creditName":null,"errorCode":null},{"oid":"0000-0002-5957-6931","name":"Jaithri","surname":"Ananthapavan","creditName":null,"errorCode":null},{"oid":"0000-0001-9736-1539","name":"Gary","surname":"Sacks","creditName":null,"errorCode":null},{"oid":"0000-0001-9736-1539","name":"Gary","surname":"Sacks","creditName":null,"errorCode":null},{"oid":"0000-0001-9736-1539","name":"Gary","surname":"Sacks","creditName":null,"errorCode":null},{"oid":"0000-0002-3206-8232","name":"Lennert","surname":"Veerman","creditName":null,"errorCode":null},{"oid":"0000-0002-3323-575X","name":"kathryn","surname":"backholer","creditName":null,"errorCode":null},{"oid":"0000-0003-2891-9476","name":"Vicki","surname":"Brown","creditName":null,"errorCode":null}]) +(10.5194/bg-2018-344,[{"oid":"0000-0001-6469-7167","name":"Neus","surname":"Garcias-Bonet","creditName":null,"errorCode":null}]) +(10.1038/srep01933,[{"oid":"0000-0001-8948-466X","name":"Yichi","surname":"Zhang","creditName":null,"errorCode":null}]) +(10.1306/05131615170,[{"oid":"0000-0003-0397-8744","name":"Sarada","surname":"Mohanty","creditName":"Sarada P Mohanty","errorCode":null}]) +(10.1080/14658011.2017.1399531,[{"oid":"0000-0003-2296-5623","name":"Janak","surname":"Sapkota","creditName":null,"errorCode":null},{"oid":"0000-0003-2296-5623","name":"Janak","surname":"Sapkota","creditName":null,"errorCode":null},{"oid":"0000-0003-4737-9823","name":"Joamin","surname":"Gonzalez-Gutierrez","creditName":null,"errorCode":null},{"oid":"0000-0001-5149-7895","name":"Clemens","surname":"Holzer","creditName":null,"errorCode":null},{"oid":"0000-0001-5149-7895","name":"Clemens","surname":"Holzer","creditName":null,"errorCode":null}]) +(10.4322/acr.2015.007,[{"oid":"0000-0002-8414-4161","name":"Jussara Bianchi","surname":"Castelli","creditName":null,"errorCode":null},{"oid":"0000-0002-5092-0505","name":"Benoit","surname":"Bibas","creditName":null,"errorCode":null}]) +(10.1186/s12888-017-1560-3,[{"oid":"0000-0002-9742-1359","name":"Alexandra","surname":"Pitman","creditName":null,"errorCode":null},{"oid":"0000-0003-2519-1539","name":"David","surname":"Osborn","creditName":null,"errorCode":null}]) +(10.1007/s10741-019-09829-7,[{"oid":"0000-0001-7175-0464","name":"Pierpaolo","surname":"Pellicori","creditName":null,"errorCode":null}]) +(10.1007/s00542-012-1613-y,[{"oid":"0000-0001-5535-9473","name":"Shota","surname":"Yabui","creditName":null,"errorCode":null},{"oid":"0000-0003-0623-5938","name":"Takenori","surname":"Atsumi","creditName":null,"errorCode":null}]) +(10.1039/C9SC02399B,[{"oid":"0000-0002-1034-0856","name":"Marvin","surname":"Vega","creditName":"Marvin M Vega","errorCode":null},{"oid":"0000-0003-0994-7789","name":"Li","surname":"Fu","creditName":null,"errorCode":null}]) +(10.1039/C8DT04698K,[{"oid":"0000-0002-3213-1350","name":"Likun","surname":"Tan","creditName":null,"errorCode":null}]) +(10.1080/09595230802093802,[{"oid":"0000-0001-9580-1545","name":"Simon","surname":"Moyes","creditName":null,"errorCode":null}]) +(10.1371/journal.pone.0136586,[{"oid":"0000-0002-6296-1427","name":"Scot","surname":"Dowd","creditName":null,"errorCode":null},{"oid":"0000-0002-2176-6932","name":"Jan","surname":"Suchodolski","creditName":null,"errorCode":null},{"oid":"0000-0002-9557-3068","name":"Cole","surname":"Mcqueen","creditName":null,"errorCode":null}]) +(10.3233/RNN-2009-0473,[{"oid":"0000-0002-6746-1034","name":"Iris-Katharina","surname":"Penner","creditName":null,"errorCode":null}]) +(10.1016/j.jcpa.2015.04.004,[{"oid":"0000-0001-5281-0521","name":"Antonio","surname":"Fernandez","creditName":null,"errorCode":null},{"oid":"0000-0003-3749-8845","name":"Eva","surname":"Sierra","creditName":null,"errorCode":null},{"oid":"0000-0002-1623-5010","name":"Manuel","surname":"Arbelo","creditName":"marbelo","errorCode":null}]) +(10.1007/s11270-011-1042-z,[{"oid":"0000-0003-2811-1979","name":"Irvan","surname":"Dahlan","creditName":null,"errorCode":null}]) +(10.1054/bjoc.2001.2096,[{"oid":"0000-0001-9484-9977","name":"Sasiwarang","surname":"Wannamethee","creditName":null,"errorCode":null}]) +(10.1137/140988541,[{"oid":"0000-0001-5645-5854","name":"Ilse","surname":"Ipsen","creditName":null,"errorCode":null}]) \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/orcid/xml/activity_work_0000-0002-5982-8983.xml b/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/orcid/xml/activity_work_0000-0002-5982-8983.xml new file mode 100644 index 0000000000..63b4405f15 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/orcid/xml/activity_work_0000-0002-5982-8983.xml @@ -0,0 +1,79 @@ + + + 2018-11-01T19:49:45.562Z + 2018-11-01T19:49:45.562Z + + + https://orcid.org/client/0000-0002-5982-8983 + 0000-0002-5982-8983 + orcid.org + + Scopus - Elsevier + + + "Calling Out" in class: Degrees of candor in addressing social injustices in + racially homogenous and heterogeneous U.S. history classrooms + + Journal of Social Studies Research + + bibtex + @article{Massaro2018,title = {{"}Calling Out{"} in class: Degrees of + candor in addressing social injustices in racially homogenous and heterogeneous U.S. + history classrooms},journal = {Journal of Social Studies Research},year = {2018},author + = {Parkhouse, H. and Massaro, V.R.}} + + journal-article + + 2018 + + + + doi + 10.1016/j.jssr.2018.01.004 + 10.1016/j.jssr.2018.01.004 + self + + + eid + 2-s2.0-85041949043 + 2-s2.0-85041949043 + self + + + http://www.scopus.com/inward/record.url?eid=2-s2.0-85041949043&partnerID=MN8TOARS + + + Parkhouse, H. + + + Massaro, V.R. + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/orcid/xml/summary_0000-0001-6828-479X.xml b/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/orcid/xml/summary_0000-0001-6828-479X.xml new file mode 100644 index 0000000000..5593527510 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/orcid/xml/summary_0000-0001-6828-479X.xml @@ -0,0 +1,71 @@ + + + + https://orcid.org/0000-0001-6828-479X + 0000-0001-6828-479X + orcid.org + + + en + + + Member-referred + 2017-02-17T06:16:06.428Z + 2017-10-04T04:38:43.529Z + true + true + true + + + + 2017-02-17T06:16:06.428Z + 2017-02-17T06:16:06.652Z + Masahide + Terazima + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/orcid/xml/summary_error.xml b/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/orcid/xml/summary_error.xml new file mode 100644 index 0000000000..a5eaff8a03 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/orcid/xml/summary_error.xml @@ -0,0 +1,33 @@ + + + 409 + 409 Conflict: The ORCID record is locked and cannot be edited. ORCID + https://orcid.org/0000-0002-9716-679X + The ORCID record is locked. + 9018 + https://members.orcid.org/api/resources/troubleshooting + \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/uw/input.json b/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/uw/input.json new file mode 100644 index 0000000000..33d4dbc3c7 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/uw/input.json @@ -0,0 +1,160 @@ +{"doi": "10.1038/2211089b0", "year": 1969, "genre": "journal-article", "is_oa": true, "title": "Planning: Trees in Danger", "doi_url": "https://doi.org/10.1038/2211089b0", "updated": "2020-02-06T13:51:15.164623", "oa_status": "bronze", "publisher": "Springer Nature", "z_authors": [{"name": "Our Planning Correspondent"}], "is_paratext": false, "journal_name": "Nature", "oa_locations": [{"url": "http://www.nature.com/articles/2211089b0.pdf", "pmh_id": null, "is_best": true, "license": null, "updated": "2018-07-11T09:19:40.598930", "version": "publishedVersion", "evidence": "open (via free pdf)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "http://www.nature.com/articles/2211089b0.pdf", "url_for_landing_page": "https://doi.org/10.1038/2211089b0", "repository_institution": null}], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0028-0836,1476-4687", "journal_issn_l": "0028-0836", "published_date": "1969-03-01", "best_oa_location": {"url": "http://www.nature.com/articles/2211089b0.pdf", "pmh_id": null, "is_best": true, "license": null, "updated": "2018-07-11T09:19:40.598930", "version": "publishedVersion", "evidence": "open (via free pdf)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "http://www.nature.com/articles/2211089b0.pdf", "url_for_landing_page": "https://doi.org/10.1038/2211089b0", "repository_institution": null}, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1021/acs.bioconjchem.8b00058.s001", "year": null, "genre": "component", "is_oa": true, "title": "Engineering Reversible CellCell Interactions with Lipid Anchored Prosthetic Receptors", "doi_url": "https://doi.org/10.1021/acs.bioconjchem.8b00058.s001", "updated": "2020-04-04T21:15:41.966773", "oa_status": "bronze", "publisher": "American Chemical Society (ACS)", "z_authors": null, "is_paratext": false, "journal_name": null, "oa_locations": [{"url": "https://doi.org/10.1021/acs.bioconjchem.8b00058.s001", "pmh_id": null, "is_best": true, "license": null, "updated": "2020-04-04T21:13:39.352965", "version": "publishedVersion", "evidence": "open (via free pdf)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "https://doi.org/10.1021/acs.bioconjchem.8b00058.s001", "url_for_landing_page": null, "repository_institution": null}], "data_standard": 2, "journal_is_oa": false, "journal_issns": null, "journal_issn_l": null, "published_date": null, "best_oa_location": {"url": "https://doi.org/10.1021/acs.bioconjchem.8b00058.s001", "pmh_id": null, "is_best": true, "license": null, "updated": "2020-04-04T21:13:39.352965", "version": "publishedVersion", "evidence": "open (via free pdf)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "https://doi.org/10.1021/acs.bioconjchem.8b00058.s001", "url_for_landing_page": null, "repository_institution": null}, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1021/acs.bioconjchem.8b00086.s001", "year": null, "genre": "component", "is_oa": true, "title": "Rapid, Stoichiometric, Site-Specific Modification of Aldehyde-Containing Proteins Using a Tandem Knoevenagel-Intra Michael Addition Reaction", "doi_url": "https://doi.org/10.1021/acs.bioconjchem.8b00086.s001", "updated": "2020-04-04T21:24:50.688286", "oa_status": "bronze", "publisher": "American Chemical Society (ACS)", "z_authors": null, "is_paratext": false, "journal_name": null, "oa_locations": [{"url": "https://doi.org/10.1021/acs.bioconjchem.8b00086.s001", "pmh_id": null, "is_best": true, "license": null, "updated": "2020-04-04T21:22:19.694440", "version": "publishedVersion", "evidence": "open (via free pdf)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "https://doi.org/10.1021/acs.bioconjchem.8b00086.s001", "url_for_landing_page": null, "repository_institution": null}], "data_standard": 2, "journal_is_oa": false, "journal_issns": null, "journal_issn_l": null, "published_date": null, "best_oa_location": {"url": "https://doi.org/10.1021/acs.bioconjchem.8b00086.s001", "pmh_id": null, "is_best": true, "license": null, "updated": "2020-04-04T21:22:19.694440", "version": "publishedVersion", "evidence": "open (via free pdf)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "https://doi.org/10.1021/acs.bioconjchem.8b00086.s001", "url_for_landing_page": null, "repository_institution": null}, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1192/bjp.89.375.270", "year": 1943, "genre": "journal-article", "is_oa": false, "title": "Unusual Pituitary Activity in a Case of Anorexia Nervosa", "doi_url": "https://doi.org/10.1192/bjp.89.375.270", "updated": "2020-03-09T08:54:12.827623", "oa_status": "closed", "publisher": "Royal College of Psychiatrists", "z_authors": [{"given": "M.", "family": "Reiss", "sequence": "first"}], "is_paratext": false, "journal_name": "Journal of Mental Science", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0368-315X,2514-9946", "journal_issn_l": "0368-315X", "published_date": "1943-04-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1016/s0167-7012(99)00056-1", "year": 1999, "genre": "journal-article", "is_oa": false, "title": "Development of radiographic and microscopic techniques for the characterization of bacterial transport in intact sediment cores from Oyster, Virginia", "doi_url": "https://doi.org/10.1016/s0167-7012(99)00056-1", "updated": "2020-04-05T11:15:40.634599", "oa_status": "closed", "publisher": "Elsevier BV", "z_authors": [{"given": "Hailiang", "family": "Dong", "sequence": "first"}, {"given": "Tullis C.", "family": "Onstott", "sequence": "additional"}, {"given": "Mary F.", "family": "DeFlaun", "sequence": "additional"}, {"given": "Mark E.", "family": "Fuller", "sequence": "additional"}, {"given": "Kathleen M.", "family": "Gillespie", "sequence": "additional"}, {"given": "James K.", "family": "Fredrickson", "sequence": "additional"}], "is_paratext": false, "journal_name": "Journal of Microbiological Methods", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0167-7012", "journal_issn_l": "0167-7012", "published_date": "1999-08-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1086/mp.1905.2.issue-3", "year": 1905, "genre": "journal-issue", "is_oa": false, "title": null, "doi_url": "https://doi.org/10.1086/mp.1905.2.issue-3", "updated": "2020-02-07T15:51:44.560109", "oa_status": "closed", "publisher": "University of Chicago Press", "z_authors": null, "is_paratext": false, "journal_name": "Modern Philology", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0026-8232,1545-6951", "journal_issn_l": "0026-8232", "published_date": "1905-01-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1007/978-1-349-11096-4_7", "year": 1989, "genre": "book-chapter", "is_oa": false, "title": "Interpreting Drawings, Specifications and Data", "doi_url": "https://doi.org/10.1007/978-1-349-11096-4_7", "updated": "2020-03-15T12:03:50.450290", "oa_status": "closed", "publisher": "Palgrave Macmillan UK", "z_authors": [{"given": "Jack", "family": "Hirst", "sequence": "first"}, {"given": "John", "family": "Whipp", "sequence": "additional"}, {"given": "Roy", "family": "Brooks", "sequence": "additional"}], "is_paratext": false, "journal_name": "Repair and Servicing of Road Vehicles", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": null, "journal_issn_l": null, "published_date": "1989-01-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1111/an.2002.43.9.43.2", "year": 2002, "genre": "journal-article", "is_oa": false, "title": "Association for Africanist Anthropology", "doi_url": "https://doi.org/10.1111/an.2002.43.9.43.2", "updated": "2020-03-21T19:33:30.089926", "oa_status": "closed", "publisher": "Wiley", "z_authors": [{"given": "Michael", "family": "Lambert", "sequence": "first"}], "is_paratext": false, "journal_name": "Anthropology News", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "1541-6151", "journal_issn_l": "1541-6151", "published_date": "2002-12-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.2217/14750708.2.5.745", "year": 2005, "genre": "journal-article", "is_oa": false, "title": "Is there a relationship between estrogen serum level and symptom severity throughout the menstrual cycle of patients with schizophrenia?", "doi_url": "https://doi.org/10.2217/14750708.2.5.745", "updated": "2020-03-24T04:08:45.741510", "oa_status": "closed", "publisher": "Future Medicine Ltd", "z_authors": [{"given": "Shahin", "family": "Akhondzadeh", "sequence": "first"}, {"given": "Kamran", "family": "Mokhberi", "sequence": "additional"}, {"given": "Homayoun", "family": "Amini", "sequence": "additional"}, {"given": "Bagher", "family": "Larijani", "sequence": "additional"}, {"given": "Ladan", "family": "Kashani", "sequence": "additional"}, {"given": "Ladan", "family": "Hashemi", "sequence": "additional"}, {"given": "Ali-Akbar", "family": "Nejatisafa", "sequence": "additional"}, {"given": "Ahmad-Reza", "family": "Shafaei", "sequence": "additional"}], "is_paratext": false, "journal_name": "Therapy", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "1475-0708,1744-831X", "journal_issn_l": "1475-0708", "published_date": "2005-09-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1007/978-1-4757-0797-7_13", "year": 1985, "genre": "book-chapter", "is_oa": false, "title": "Diseases and Induced Lesions of the Neuromuscular Endplate", "doi_url": "https://doi.org/10.1007/978-1-4757-0797-7_13", "updated": "2020-03-15T12:03:49.875280", "oa_status": "closed", "publisher": "Springer US", "z_authors": [{"given": "Edith", "family": "Heilbronn", "sequence": "first"}], "is_paratext": false, "journal_name": "Pathological Neurochemistry", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": null, "journal_issn_l": null, "published_date": "1985-01-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1088/0256-307x/27/2/026804", "year": 2010, "genre": "journal-article", "is_oa": false, "title": "Effects of Rapid Thermal Processing on Microstructure and Optical Properties of As-Deposited Ag2O Films by Direct-Current Reactive Magnetron Sputtering", "doi_url": "https://doi.org/10.1088/0256-307x/27/2/026804", "updated": "2020-03-08T13:45:58.996113", "oa_status": "closed", "publisher": "IOP Publishing", "z_authors": [{"given": "Gao", "family": "Xiao-Yong", "sequence": "first"}, {"given": "Feng", "family": "Hong-Liang", "sequence": "additional"}, {"given": "Zhang", "family": "Zeng-Yuan", "sequence": "additional"}, {"given": "Ma", "family": "Jiao-Min", "sequence": "additional"}, {"given": "Lu", "family": "Jing-Xiao", "sequence": "additional"}], "is_paratext": false, "journal_name": "Chinese Physics Letters", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0256-307X,1741-3540", "journal_issn_l": "0256-307X", "published_date": "2010-02-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1371/journal.pone.0209587.g009", "year": null, "genre": "component", "is_oa": true, "title": null, "doi_url": "https://doi.org/10.1371/journal.pone.0209587.g009", "updated": "2020-02-05T00:02:31.742178", "oa_status": "gold", "publisher": "Public Library of Science (PLoS)", "z_authors": null, "is_paratext": false, "journal_name": null, "oa_locations": [{"url": "https://doi.org/10.1371/journal.pone.0209587.g009", "pmh_id": null, "is_best": true, "license": null, "updated": "2020-04-22T18:12:56.284881", "version": "publishedVersion", "evidence": "oa journal (via publisher name)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": null, "url_for_landing_page": "https://doi.org/10.1371/journal.pone.0209587.g009", "repository_institution": null}], "data_standard": 2, "journal_is_oa": true, "journal_issns": null, "journal_issn_l": null, "published_date": null, "best_oa_location": {"url": "https://doi.org/10.1371/journal.pone.0209587.g009", "pmh_id": null, "is_best": true, "license": null, "updated": "2020-04-22T18:12:56.284881", "version": "publishedVersion", "evidence": "oa journal (via publisher name)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": null, "url_for_landing_page": "https://doi.org/10.1371/journal.pone.0209587.g009", "repository_institution": null}, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1089/10430340050015301", "year": 2000, "genre": "journal-article", "is_oa": false, "title": "Improved Gene Transfer to Neuroblastoma Cells by a Monoclonal Antibody Targeting RET, a Receptor Tyrosine Kinase", "doi_url": "https://doi.org/10.1089/10430340050015301", "updated": "2020-03-08T13:45:50.039000", "oa_status": "closed", "publisher": "Mary Ann Liebert Inc", "z_authors": [{"given": "Lisa", "family": "Yano", "sequence": "first"}, {"given": "Mari", "family": "Shimura", "sequence": "additional"}, {"given": "Mika", "family": "Taniguchi", "sequence": "additional"}, {"given": "Yasuhide", "family": "Hayashi", "sequence": "additional"}, {"given": "Toshimitsu", "family": "Suzuki", "sequence": "additional"}, {"given": "Kiyohiko", "family": "Hatake", "sequence": "additional"}, {"given": "Fumimaro", "family": "Takaku", "sequence": "additional"}, {"given": "Yukihito", "family": "Ishizaka", "sequence": "additional"}], "is_paratext": false, "journal_name": "Human Gene Therapy", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "1043-0342,1557-7422", "journal_issn_l": "1043-0342", "published_date": "2000-05-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1182/blood.v124.21.2082.2082", "year": 2014, "genre": "journal-article", "is_oa": false, "title": "Thymidine Kinase-Deleted, let7a-Regulated Vaccinia Virus Specifically Infects and Lyses Myeloma Cells in a Mouse Myeloma Model", "doi_url": "https://doi.org/10.1182/blood.v124.21.2082.2082", "updated": "2020-02-04T16:57:14.305643", "oa_status": "closed", "publisher": "American Society of Hematology", "z_authors": [{"given": "Muneyoshi", "family": "Futami", "sequence": "first", "affiliation": [{"name": "University of Tokyo, Tokyo, Japan"}]}, {"given": "Kota", "family": "Sato", "sequence": "additional", "affiliation": [{"name": "University of Tokyo, Tokyo, Japan"}]}, {"given": "Takafumi", "family": "Nakamura", "sequence": "additional", "affiliation": [{"name": "Tottori University, Yonago, Japan"}]}, {"given": "Arinobu", "family": "Tojo", "sequence": "additional", "affiliation": [{"name": "University of Tokyo, Tokyo, Japan"}]}], "is_paratext": false, "journal_name": "Blood", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0006-4971,1528-0020", "journal_issn_l": "0006-4971", "published_date": "2014-12-06", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.31826/9781463218355-007", "year": 2010, "genre": "book-chapter", "is_oa": false, "title": "LETTER V. TO OSMUND", "doi_url": "https://doi.org/10.31826/9781463218355-007", "updated": "2020-02-04T16:58:11.205863", "oa_status": "closed", "publisher": "Gorgias Press", "z_authors": null, "is_paratext": false, "journal_name": "The Letters, I", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": null, "journal_issn_l": null, "published_date": "2010-12-31", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.4000/rechercheformation.639", "year": 2008, "genre": "journal-article", "is_oa": true, "title": "La formation à la recherche des enseignants au Québec", "doi_url": "https://doi.org/10.4000/rechercheformation.639", "updated": "2020-03-19T20:54:54.338264", "oa_status": "bronze", "publisher": "OpenEdition", "z_authors": [{"given": "Jean-Marie", "family": "Van der Maren", "sequence": "first"}], "is_paratext": false, "journal_name": "Recherche & formation", "oa_locations": [{"url": "http://journals.openedition.org/rechercheformation/pdf/639", "pmh_id": null, "is_best": true, "license": null, "updated": "2018-05-14T18:32:37.769791", "version": "publishedVersion", "evidence": "open (via free pdf)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "http://journals.openedition.org/rechercheformation/pdf/639", "url_for_landing_page": "https://doi.org/10.4000/rechercheformation.639", "repository_institution": null}, {"url": "http://pdfs.semanticscholar.org/fc66/288fc0488953310a918e145b34656a07c991.pdf", "pmh_id": null, "is_best": false, "license": null, "updated": "2019-10-01T00:00:00", "version": "submittedVersion", "evidence": "oa repository (semantic scholar lookup)", "host_type": "repository", "endpoint_id": null, "url_for_pdf": "http://pdfs.semanticscholar.org/fc66/288fc0488953310a918e145b34656a07c991.pdf", "url_for_landing_page": "https://semanticscholar.org/paper/fc66288fc0488953310a918e145b34656a07c991", "repository_institution": null}], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0988-1824,1968-3936", "journal_issn_l": "0988-1824", "published_date": "2008-12-01", "best_oa_location": {"url": "http://journals.openedition.org/rechercheformation/pdf/639", "pmh_id": null, "is_best": true, "license": null, "updated": "2018-05-14T18:32:37.769791", "version": "publishedVersion", "evidence": "open (via free pdf)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "http://journals.openedition.org/rechercheformation/pdf/639", "url_for_landing_page": "https://doi.org/10.4000/rechercheformation.639", "repository_institution": null}, "journal_is_in_doaj": false, "has_repository_copy": true} +{"doi": "10.1021/acsami.8b00889.s001", "year": null, "genre": "component", "is_oa": true, "title": "In Situ Exsolution of Bimetallic RhNi Nanoalloys: a Highly Efficient Catalyst for CO2 Methanation", "doi_url": "https://doi.org/10.1021/acsami.8b00889.s001", "updated": "2020-04-04T21:37:46.317043", "oa_status": "bronze", "publisher": "American Chemical Society (ACS)", "z_authors": null, "is_paratext": false, "journal_name": null, "oa_locations": [{"url": "https://doi.org/10.1021/acsami.8b00889.s001", "pmh_id": null, "is_best": true, "license": null, "updated": "2020-04-04T21:36:02.136386", "version": "publishedVersion", "evidence": "open (via free pdf)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "https://doi.org/10.1021/acsami.8b00889.s001", "url_for_landing_page": null, "repository_institution": null}], "data_standard": 2, "journal_is_oa": false, "journal_issns": null, "journal_issn_l": null, "published_date": null, "best_oa_location": {"url": "https://doi.org/10.1021/acsami.8b00889.s001", "pmh_id": null, "is_best": true, "license": null, "updated": "2020-04-04T21:36:02.136386", "version": "publishedVersion", "evidence": "open (via free pdf)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "https://doi.org/10.1021/acsami.8b00889.s001", "url_for_landing_page": null, "repository_institution": null}, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.4103/1673-5374.280315", "year": 2020, "genre": "journal-article", "is_oa": true, "title": "Limited therapeutic potential of astrocyte elevated gene-1 transduction in an animal model of Parkinson's disease", "doi_url": "https://doi.org/10.4103/1673-5374.280315", "updated": "2020-04-03T21:03:00.926369", "oa_status": "gold", "publisher": "Medknow", "z_authors": [{"given": "SangRyong", "family": "Kim", "sequence": "first"}, {"given": "Eunju", "family": "Leem", "sequence": "additional"}], "is_paratext": false, "journal_name": "Neural Regeneration Research", "oa_locations": [{"url": "https://doi.org/10.4103/1673-5374.280315", "pmh_id": null, "is_best": true, "license": "cc-by-nc-sa", "updated": "2020-04-26T20:55:28.594777", "version": "publishedVersion", "evidence": "oa journal (via doaj)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": null, "url_for_landing_page": "https://doi.org/10.4103/1673-5374.280315", "repository_institution": null}], "data_standard": 2, "journal_is_oa": true, "journal_issns": "1673-5374", "journal_issn_l": "1673-5374", "published_date": "2020-01-01", "best_oa_location": {"url": "https://doi.org/10.4103/1673-5374.280315", "pmh_id": null, "is_best": true, "license": "cc-by-nc-sa", "updated": "2020-04-26T20:55:28.594777", "version": "publishedVersion", "evidence": "oa journal (via doaj)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": null, "url_for_landing_page": "https://doi.org/10.4103/1673-5374.280315", "repository_institution": null}, "journal_is_in_doaj": true, "has_repository_copy": false} +{"doi": "10.1086/ahr/101.1.170-a", "year": 1996, "genre": "journal-article", "is_oa": false, "title": "Robin Chapman Stacey. The Road to Judgment: From Custom to Court in Medieval Ireland and Wales. (Middle Ages Series.) Philadelphia: University of Pennsylvania Press. 1994. Pp. xvi, 342. $46.95", "doi_url": "https://doi.org/10.1086/ahr/101.1.170-a", "updated": "2020-02-10T19:06:57.426463", "oa_status": "closed", "publisher": "Oxford University Press (OUP)", "z_authors": null, "is_paratext": false, "journal_name": "The American Historical Review", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "1937-5239", "journal_issn_l": "0002-8762", "published_date": "1996-02-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1088/0256-307x/24/8/018", "year": 2007, "genre": "journal-article", "is_oa": false, "title": "Nuclear Potential and Fusion Cross Sections for Synthesizing Super-Heavy Elements in Di-nuclear Systems", "doi_url": "https://doi.org/10.1088/0256-307x/24/8/018", "updated": "2020-03-10T18:27:03.812257", "oa_status": "closed", "publisher": "IOP Publishing", "z_authors": [{"given": "Wang", "family": "Nan", "sequence": "first"}, {"given": "Li", "family": "Jun-Qing", "sequence": "additional"}, {"given": "Zhao", "family": "En-Guang", "sequence": "additional"}, {"given": "Feng", "family": "Zhao-Qing", "sequence": "additional"}], "is_paratext": false, "journal_name": "Chinese Physics Letters", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0256-307X,1741-3540", "journal_issn_l": "0256-307X", "published_date": "2007-07-26", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1089/omi.2019.0018", "year": 2019, "genre": "journal-article", "is_oa": false, "title": "The Genetics of Warfarin Dose–Response Variability in Africans: An Expert Perspective on Past, Present, and Future", "doi_url": "https://doi.org/10.1089/omi.2019.0018", "updated": "2020-02-07T15:52:11.808933", "oa_status": "closed", "publisher": "Mary Ann Liebert Inc", "z_authors": [{"given": "Arinao", "family": "Ndadza", "sequence": "first", "affiliation": [{"name": "Pharmacogenomics and Drug Metabolism Research Group, Division of Human Genetics, Department of Pathology, Institute of Infectious Disease and Molecular Medicine (IDM), Faculty of Health Sciences, University of Cape Town, Cape Town, South Africa."}]}, {"given": "Nicholas Ekow", "family": "Thomford", "sequence": "additional", "affiliation": [{"name": "Pharmacogenomics and Drug Metabolism Research Group, Division of Human Genetics, Department of Pathology, Institute of Infectious Disease and Molecular Medicine (IDM), Faculty of Health Sciences, University of Cape Town, Cape Town, South Africa."}]}, {"given": "Stanley", "family": "Mukanganyama", "sequence": "additional", "affiliation": [{"name": "Department of Biochemistry, University of Zimbabwe, Harare, Zimbabwe."}]}, {"given": "Ambroise", "family": "Wonkam", "sequence": "additional", "affiliation": [{"name": "Pharmacogenomics and Drug Metabolism Research Group, Division of Human Genetics, Department of Pathology, Institute of Infectious Disease and Molecular Medicine (IDM), Faculty of Health Sciences, University of Cape Town, Cape Town, South Africa."}]}, {"given": "Mpiko", "family": "Ntsekhe", "sequence": "additional", "affiliation": [{"name": "Division of Cardiology, Department of Medicine, Faculty of Health Sciences, University of Cape Town, Cape Town, South Africa."}]}, {"given": "Collet", "family": "Dandara", "sequence": "additional", "affiliation": [{"name": "Pharmacogenomics and Drug Metabolism Research Group, Division of Human Genetics, Department of Pathology, Institute of Infectious Disease and Molecular Medicine (IDM), Faculty of Health Sciences, University of Cape Town, Cape Town, South Africa."}]}], "is_paratext": false, "journal_name": "OMICS: A Journal of Integrative Biology", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "1557-8100", "journal_issn_l": "1536-2310", "published_date": "2019-03-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1787/sti_scoreboard-2013-graph205-en", "year": 2013, "genre": "component", "is_oa": false, "title": "Global trade in ICT goods and top ten exporters, 2000 and 2011", "doi_url": "https://doi.org/10.1787/sti_scoreboard-2013-graph205-en", "updated": "2020-03-19T20:54:49.500840", "oa_status": "closed", "publisher": "Organisation for Economic Co-Operation and Development (OECD)", "z_authors": [{"name": "OECD", "sequence": "first"}], "is_paratext": false, "journal_name": null, "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": null, "journal_issn_l": null, "published_date": "2013-10-23", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.19173/irrodl.v16i1.1814", "year": 2015, "genre": "journal-article", "is_oa": true, "title": "From the classroom to the keyboard: How seven teachers created their online teacher identities", "doi_url": "https://doi.org/10.19173/irrodl.v16i1.1814", "updated": "2020-03-13T20:09:41.402407", "oa_status": "gold", "publisher": "Athabasca University Press", "z_authors": [{"given": "Jennifer C", "family": "Richardson", "sequence": "first"}, {"given": "Janet", "family": "Alsup", "sequence": "additional"}], "is_paratext": false, "journal_name": "The International Review of Research in Open and Distributed Learning", "oa_locations": [{"url": "http://www.irrodl.org/index.php/irrodl/article/download/1814/3253", "pmh_id": null, "is_best": true, "license": "cc-by", "updated": "2019-01-05T13:59:38.399777", "version": "publishedVersion", "evidence": "open (via page says license)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "http://www.irrodl.org/index.php/irrodl/article/download/1814/3253", "url_for_landing_page": "https://doi.org/10.19173/irrodl.v16i1.1814", "repository_institution": null}, {"url": "https://doi.org/10.19173/irrodl.v16i1.1814", "pmh_id": null, "is_best": false, "license": "cc-by", "updated": "2020-04-24T06:25:33.267893", "version": "publishedVersion", "evidence": "oa journal (via doaj)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": null, "url_for_landing_page": "https://doi.org/10.19173/irrodl.v16i1.1814", "repository_institution": null}, {"url": "http://www.erudit.org/fr/revues/irrodl/2015-v16-n1-irrodl04978/1065932ar.pdf", "pmh_id": "oai:erudit.org:1065932ar", "is_best": false, "license": "cc-by", "updated": "2020-04-04T14:21:32.895968", "version": "publishedVersion", "evidence": "oa repository (via OAI-PMH doi match)", "host_type": "repository", "endpoint_id": "a7ded957df6faebd849", "url_for_pdf": "http://www.erudit.org/fr/revues/irrodl/2015-v16-n1-irrodl04978/1065932ar.pdf", "url_for_landing_page": "http://id.erudit.org/iderudit/1065932ar", "repository_institution": null}, {"url": "http://www.irrodl.org/index.php/irrodl/article/download/1814/3253/", "pmh_id": "oai:CiteSeerX.psu:10.1.1.867.5215", "is_best": false, "license": "cc-by", "updated": "2020-01-18T21:21:04.741502", "version": "submittedVersion", "evidence": "oa repository (via OAI-PMH title and first author match)", "host_type": "repository", "endpoint_id": "7e4fec42e1147584aae", "url_for_pdf": "http://www.irrodl.org/index.php/irrodl/article/download/1814/3253/", "url_for_landing_page": "http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.867.5215", "repository_institution": "The Pennsylvania State University - CiteSeer X"}], "data_standard": 2, "journal_is_oa": true, "journal_issns": "1492-3831", "journal_issn_l": "1492-3831", "published_date": "2015-01-20", "best_oa_location": {"url": "http://www.irrodl.org/index.php/irrodl/article/download/1814/3253", "pmh_id": null, "is_best": true, "license": "cc-by", "updated": "2019-01-05T13:59:38.399777", "version": "publishedVersion", "evidence": "open (via page says license)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "http://www.irrodl.org/index.php/irrodl/article/download/1814/3253", "url_for_landing_page": "https://doi.org/10.19173/irrodl.v16i1.1814", "repository_institution": null}, "journal_is_in_doaj": true, "has_repository_copy": true} +{"doi": "10.1093/nq/146.31.92b", "year": 1924, "genre": "journal-article", "is_oa": false, "title": "The Belton Estate, The Three Dervishes and other Persion Tales and Legends", "doi_url": "https://doi.org/10.1093/nq/146.31.92b", "updated": "2020-02-10T19:05:06.598079", "oa_status": "closed", "publisher": "Oxford University Press (OUP)", "z_authors": null, "is_paratext": false, "journal_name": "Notes and Queries", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "1471-6941,0029-3970", "journal_issn_l": "0029-3970", "published_date": "1924-02-02", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1182/blood.v116.21.1124.1124", "year": 2010, "genre": "journal-article", "is_oa": false, "title": "Combinatorial Antimicrobicidal Peptides Exhibit Enhanced Activity Against Bacterial Contaminants In Stored Platelet Concentrates (PCs).", "doi_url": "https://doi.org/10.1182/blood.v116.21.1124.1124", "updated": "2020-02-04T16:57:49.044181", "oa_status": "closed", "publisher": "American Society of Hematology", "z_authors": [{"given": "Krishna Mohan V.", "family": "Ketha", "sequence": "first", "affiliation": [{"name": "Lab of Cellular Hematology, CBER (Food and Drug Administration), Bethesda, MD, USA,"}]}, {"given": "Shilpakala Sainath", "family": "Rao", "sequence": "additional", "affiliation": [{"name": "Division of Hematology, CBER/FDA, Bethesda, MD, USA"}]}, {"given": "Chintamani D", "family": "Atreya", "sequence": "additional", "affiliation": [{"name": "Lab of Cellular Hematology, CBER (Food and Drug Administration), Bethesda, MD, USA,"}]}], "is_paratext": false, "journal_name": "Blood", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0006-4971,1528-0020", "journal_issn_l": "0006-4971", "published_date": "2010-11-19", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1109/vlhcc.2009.5295296", "year": 2009, "genre": "proceedings-article", "is_oa": true, "title": "Revealing the copy and paste habits of end users", "doi_url": "https://doi.org/10.1109/vlhcc.2009.5295296", "updated": "2020-03-20T15:53:06.707752", "oa_status": "green", "publisher": "IEEE", "z_authors": [{"given": "Kathryn T.", "family": "Stolee", "sequence": "first"}, {"given": "Sebastian", "family": "Elbaum", "sequence": "additional"}, {"given": "Gregg", "family": "Rothermel", "sequence": "additional"}], "is_paratext": false, "journal_name": "2009 IEEE Symposium on Visual Languages and Human-Centric Computing (VL/HCC)", "oa_locations": [{"url": "https://digitalcommons.unl.edu/cgi/viewcontent.cgi?article=1139&context=cseconfwork", "pmh_id": "oai:digitalcommons.unl.edu:cseconfwork-1139", "is_best": true, "license": null, "updated": "2020-03-25T14:30:10.522478", "version": "publishedVersion", "evidence": "oa repository (via OAI-PMH title and first author match)", "host_type": "repository", "endpoint_id": "5ae9f225287388f2391", "url_for_pdf": "https://digitalcommons.unl.edu/cgi/viewcontent.cgi?article=1139&context=cseconfwork", "url_for_landing_page": "https://digitalcommons.unl.edu/cseconfwork/133", "repository_institution": null}, {"url": "http://cse.unl.edu/~grother/papers/vlhcc09.pdf", "pmh_id": "oai:CiteSeerX.psu:10.1.1.153.4117", "is_best": false, "license": null, "updated": "2017-10-21T09:33:23.797406", "version": "submittedVersion", "evidence": "oa repository (via OAI-PMH doi match)", "host_type": "repository", "endpoint_id": "CiteSeerX.psu", "url_for_pdf": "http://cse.unl.edu/~grother/papers/vlhcc09.pdf", "url_for_landing_page": "http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.153.4117", "repository_institution": "CiteSeerX.psu"}], "data_standard": 2, "journal_is_oa": false, "journal_issns": null, "journal_issn_l": null, "published_date": "2009-09-01", "best_oa_location": {"url": "https://digitalcommons.unl.edu/cgi/viewcontent.cgi?article=1139&context=cseconfwork", "pmh_id": "oai:digitalcommons.unl.edu:cseconfwork-1139", "is_best": true, "license": null, "updated": "2020-03-25T14:30:10.522478", "version": "publishedVersion", "evidence": "oa repository (via OAI-PMH title and first author match)", "host_type": "repository", "endpoint_id": "5ae9f225287388f2391", "url_for_pdf": "https://digitalcommons.unl.edu/cgi/viewcontent.cgi?article=1139&context=cseconfwork", "url_for_landing_page": "https://digitalcommons.unl.edu/cseconfwork/133", "repository_institution": null}, "journal_is_in_doaj": false, "has_repository_copy": true} +{"doi": "10.1371/journal.pone.0211321", "year": 2019, "genre": "journal-article", "is_oa": true, "title": "Nearest transfer effects of working memory training: A comparison of two programs focused on working memory updating", "doi_url": "https://doi.org/10.1371/journal.pone.0211321", "updated": "2020-02-05T00:01:53.986492", "oa_status": "gold", "publisher": "Public Library of Science (PLoS)", "z_authors": [{"given": "Rocío", "family": "Linares", "sequence": "first"}, {"given": "Erika", "family": "Borella", "sequence": "additional"}, {"ORCID": "http://orcid.org/0000-0003-3511-0149", "given": "Mª Teresa", "family": "Lechuga", "sequence": "additional", "authenticated-orcid": true}, {"given": "Barbara", "family": "Carretti", "sequence": "additional"}, {"ORCID": "http://orcid.org/0000-0002-4441-8965", "given": "Santiago", "family": "Pelegrina", "sequence": "additional", "authenticated-orcid": true}], "is_paratext": false, "journal_name": "PLOS ONE", "oa_locations": [{"url": "https://journals.plos.org/plosone/article/file?id=10.1371/journal.pone.0211321&type=printable", "pmh_id": null, "is_best": true, "license": "cc-by", "updated": "2019-11-25T05:07:17.964115", "version": "publishedVersion", "evidence": "open (via page says license)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "https://journals.plos.org/plosone/article/file?id=10.1371/journal.pone.0211321&type=printable", "url_for_landing_page": "https://doi.org/10.1371/journal.pone.0211321", "repository_institution": null}, {"url": "https://doi.org/10.1371/journal.pone.0211321", "pmh_id": null, "is_best": false, "license": "cc-by", "updated": "2020-04-22T18:12:45.106623", "version": "publishedVersion", "evidence": "oa journal (via doaj)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": null, "url_for_landing_page": "https://doi.org/10.1371/journal.pone.0211321", "repository_institution": null}, {"url": "https://www.research.unipd.it/bitstream/11577/3296260/1/journal.pone.0211321.pdf", "pmh_id": "oai:www.research.unipd.it:11577/3296260", "is_best": false, "license": "cc-by", "updated": "2020-03-29T16:56:36.110076", "version": "publishedVersion", "evidence": "oa repository (via OAI-PMH title and first author match)", "host_type": "repository", "endpoint_id": "3067c07bad2a6056a48", "url_for_pdf": "https://www.research.unipd.it/bitstream/11577/3296260/1/journal.pone.0211321.pdf", "url_for_landing_page": "http://hdl.handle.net/11577/3296260", "repository_institution": null}, {"url": "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6373913", "pmh_id": null, "is_best": false, "license": null, "updated": "2020-04-22T18:12:45.106854", "version": "publishedVersion", "evidence": "oa repository (via pmcid lookup)", "host_type": "repository", "endpoint_id": null, "url_for_pdf": null, "url_for_landing_page": "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6373913", "repository_institution": null}, {"url": "http://pdfs.semanticscholar.org/e4d5/3ea7f72924165fd7366d81c1b5d17ad17a2e.pdf", "pmh_id": null, "is_best": false, "license": null, "updated": "2019-10-01T00:00:00", "version": "submittedVersion", "evidence": "oa repository (semantic scholar lookup)", "host_type": "repository", "endpoint_id": null, "url_for_pdf": "http://pdfs.semanticscholar.org/e4d5/3ea7f72924165fd7366d81c1b5d17ad17a2e.pdf", "url_for_landing_page": "https://semanticscholar.org/paper/e4d53ea7f72924165fd7366d81c1b5d17ad17a2e", "repository_institution": null}], "data_standard": 2, "journal_is_oa": true, "journal_issns": "1932-6203", "journal_issn_l": "1932-6203", "published_date": "2019-02-13", "best_oa_location": {"url": "https://journals.plos.org/plosone/article/file?id=10.1371/journal.pone.0211321&type=printable", "pmh_id": null, "is_best": true, "license": "cc-by", "updated": "2019-11-25T05:07:17.964115", "version": "publishedVersion", "evidence": "open (via page says license)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "https://journals.plos.org/plosone/article/file?id=10.1371/journal.pone.0211321&type=printable", "url_for_landing_page": "https://doi.org/10.1371/journal.pone.0211321", "repository_institution": null}, "journal_is_in_doaj": true, "has_repository_copy": true} +{"doi": "10.1016/j.celrep.2015.06.074", "year": 2015, "genre": "journal-article", "is_oa": true, "title": "miR-302 Is Required for Timing of Neural Differentiation, Neural Tube Closure, and Embryonic Viability", "doi_url": "https://doi.org/10.1016/j.celrep.2015.06.074", "updated": "2020-03-18T15:03:29.371139", "oa_status": "gold", "publisher": "Elsevier BV", "z_authors": [{"given": "Ronald J.", "family": "Parchem", "sequence": "first"}, {"given": "Nicole", "family": "Moore", "sequence": "additional"}, {"given": "Jennifer L.", "family": "Fish", "sequence": "additional"}, {"given": "Jacqueline G.", "family": "Parchem", "sequence": "additional"}, {"given": "Tarcio T.", "family": "Braga", "sequence": "additional"}, {"given": "Archana", "family": "Shenoy", "sequence": "additional"}, {"given": "Michael C.", "family": "Oldham", "sequence": "additional"}, {"given": "John L.R.", "family": "Rubenstein", "sequence": "additional"}, {"given": "Richard A.", "family": "Schneider", "sequence": "additional"}, {"given": "Robert", "family": "Blelloch", "sequence": "additional"}], "is_paratext": false, "journal_name": "Cell Reports", "oa_locations": [{"url": "http://www.cell.com/article/S2211124715007123/pdf", "pmh_id": null, "is_best": true, "license": "cc-by-nc-nd", "updated": "2020-01-20T09:52:46.185494", "version": "publishedVersion", "evidence": "open (via page says license)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "http://www.cell.com/article/S2211124715007123/pdf", "url_for_landing_page": "https://doi.org/10.1016/j.celrep.2015.06.074", "repository_institution": null}, {"url": "https://doi.org/10.1016/j.celrep.2015.06.074", "pmh_id": null, "is_best": false, "license": "cc-by-nc-nd", "updated": "2020-04-23T02:55:57.701989", "version": "publishedVersion", "evidence": "oa journal (via doaj)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": null, "url_for_landing_page": "https://doi.org/10.1016/j.celrep.2015.06.074", "repository_institution": null}, {"url": "http://europepmc.org/articles/pmc4741278?pdf=render", "pmh_id": "oai:pubmedcentral.nih.gov:4741278", "is_best": false, "license": "cc-by-nc-nd", "updated": "2017-10-21T18:16:06.923793", "version": "acceptedVersion", "evidence": "oa repository (via OAI-PMH doi match)", "host_type": "repository", "endpoint_id": "pubmedcentral.nih.gov", "url_for_pdf": "http://europepmc.org/articles/pmc4741278?pdf=render", "url_for_landing_page": "http://europepmc.org/articles/pmc4741278", "repository_institution": "pubmedcentral.nih.gov"}, {"url": "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4741278", "pmh_id": null, "is_best": false, "license": null, "updated": "2020-04-23T02:55:57.702453", "version": "acceptedVersion", "evidence": "oa repository (via pmcid lookup)", "host_type": "repository", "endpoint_id": null, "url_for_pdf": null, "url_for_landing_page": "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4741278", "repository_institution": null}], "data_standard": 2, "journal_is_oa": true, "journal_issns": "2211-1247", "journal_issn_l": "2211-1247", "published_date": "2015-08-01", "best_oa_location": {"url": "http://www.cell.com/article/S2211124715007123/pdf", "pmh_id": null, "is_best": true, "license": "cc-by-nc-nd", "updated": "2020-01-20T09:52:46.185494", "version": "publishedVersion", "evidence": "open (via page says license)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "http://www.cell.com/article/S2211124715007123/pdf", "url_for_landing_page": "https://doi.org/10.1016/j.celrep.2015.06.074", "repository_institution": null}, "journal_is_in_doaj": true, "has_repository_copy": true} +{"doi": "10.1177/1558689816674564", "year": 2016, "genre": "journal-article", "is_oa": true, "title": "Process Evaluation of a Retreat for Scholars in the First Cohort: The NIH Mixed Methods Research Training Program for the Health Sciences", "doi_url": "https://doi.org/10.1177/1558689816674564", "updated": "2020-03-14T18:40:51.946726", "oa_status": "green", "publisher": "SAGE Publications", "z_authors": [{"given": "Timothy C.", "family": "Guetterman", "sequence": "first", "affiliation": [{"name": "University of Michigan, Ann Arbor, MI, USA"}]}, {"given": "John W.", "family": "Creswell", "sequence": "additional", "affiliation": [{"name": "University of Michigan, Ann Arbor, MI, USA"}]}, {"given": "Charles", "family": "Deutsch", "sequence": "additional", "affiliation": [{"name": "Harvard University, Boston, MA, USA"}]}, {"given": "Joseph J.", "family": "Gallo", "sequence": "additional", "affiliation": [{"name": "Johns Hopkins University, Baltimore, MD, USA"}]}], "is_paratext": false, "journal_name": "Journal of Mixed Methods Research", "oa_locations": [{"url": "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6322415", "pmh_id": null, "is_best": true, "license": null, "updated": "2020-04-25T06:14:50.485049", "version": "acceptedVersion", "evidence": "oa repository (via pmcid lookup)", "host_type": "repository", "endpoint_id": null, "url_for_pdf": null, "url_for_landing_page": "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6322415", "repository_institution": null}], "data_standard": 2, "journal_is_oa": false, "journal_issns": "1558-6898,1558-6901", "journal_issn_l": "1558-6898", "published_date": "2016-10-26", "best_oa_location": {"url": "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6322415", "pmh_id": null, "is_best": true, "license": null, "updated": "2020-04-25T06:14:50.485049", "version": "acceptedVersion", "evidence": "oa repository (via pmcid lookup)", "host_type": "repository", "endpoint_id": null, "url_for_pdf": null, "url_for_landing_page": "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6322415", "repository_institution": null}, "journal_is_in_doaj": false, "has_repository_copy": true} +{"doi": "10.1002/(sici)1097-0029(19960901)35:1<87::aid-jemt8>3.3.co;2-b", "year": 1996, "genre": "journal-article", "is_oa": false, "title": "Collagen fibril surface: TMAFM, FEG‐SEM and freeze‐etching observations", "doi_url": "https://doi.org/10.1002/(sici)1097-0029(19960901)35:1<87::aid-jemt8>3.3.co;2-b", "updated": "2020-02-08T08:22:40.551625", "oa_status": "closed", "publisher": "Wiley-Blackwell", "z_authors": [{"given": "Mario", "family": "Raspanti"}, {"given": "Andrea", "family": "Alessandrini"}, {"given": "Pietro", "family": "Gobbi"}, {"given": "Alessandro", "family": "Ruggeri"}], "is_paratext": false, "journal_name": "Microscopy Research and Technique", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "1059-910X,1097-0029", "journal_issn_l": "1059-910X", "published_date": "1996-09-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1093/benz/9780199773787.article.b00167498", "year": 2011, "genre": "reference-entry", "is_oa": false, "title": "Seminario, Enrique José", "doi_url": "https://doi.org/10.1093/benz/9780199773787.article.b00167498", "updated": "2020-02-07T15:51:16.702243", "oa_status": "closed", "publisher": "Oxford University Press", "z_authors": null, "is_paratext": false, "journal_name": "Benezit Dictionary of Artists", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": null, "journal_issn_l": null, "published_date": "2011-10-31", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.2165/00128415-200812090-00025", "year": 2008, "genre": "journal-article", "is_oa": false, "title": "Antineoplastics", "doi_url": "https://doi.org/10.2165/00128415-200812090-00025", "updated": "2020-03-19T17:56:38.037237", "oa_status": "closed", "publisher": "Springer Science and Business Media LLC", "z_authors": [{"family": "&NA;", "sequence": "first"}], "is_paratext": false, "journal_name": "Reactions Weekly", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0114-9954", "journal_issn_l": "0114-9954", "published_date": "2008-07-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1002/1520-6807(197710)14:4<426::aid-pits2310140408>3.0.co;2-e", "year": 1977, "genre": "journal-article", "is_oa": false, "title": "An abbreviated form of the WISC-R: Is it valid?", "doi_url": "https://doi.org/10.1002/1520-6807(197710)14:4<426::aid-pits2310140408>3.0.co;2-e", "updated": "2020-03-27T13:45:45.585683", "oa_status": "closed", "publisher": "Wiley", "z_authors": [{"given": "Robert J.", "family": "Resnick", "sequence": "first"}], "is_paratext": false, "journal_name": "Psychology in the Schools", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0033-3085,1520-6807", "journal_issn_l": "0033-3085", "published_date": "1977-10-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.7748/nop2003.02.14.10.14.c2198", "year": 2003, "genre": "journal-article", "is_oa": true, "title": "Care provision in Scotland: background, policy context and research", "doi_url": "https://doi.org/10.7748/nop2003.02.14.10.14.c2198", "updated": "2020-03-07T08:58:53.694845", "oa_status": "green", "publisher": "RCN Publishing Ltd.", "z_authors": [{"given": "Belinda Jane", "family": "Dewar", "sequence": "first"}, {"given": "Fiona", "family": "O’May", "sequence": "additional"}, {"given": "Esther", "family": "Walker", "sequence": "additional"}], "is_paratext": false, "journal_name": "Nursing Older People", "oa_locations": [{"url": "http://pdfs.semanticscholar.org/192c/6986a7e4b9895e2a9d905da7c5b04ebd9a28.pdf", "pmh_id": null, "is_best": true, "license": null, "updated": "2019-10-01T00:00:00", "version": "submittedVersion", "evidence": "oa repository (semantic scholar lookup)", "host_type": "repository", "endpoint_id": null, "url_for_pdf": "http://pdfs.semanticscholar.org/192c/6986a7e4b9895e2a9d905da7c5b04ebd9a28.pdf", "url_for_landing_page": "https://semanticscholar.org/paper/192c6986a7e4b9895e2a9d905da7c5b04ebd9a28", "repository_institution": null}], "data_standard": 2, "journal_is_oa": false, "journal_issns": "1472-0795,2047-8941", "journal_issn_l": "1472-0795", "published_date": "2003-02-01", "best_oa_location": {"url": "http://pdfs.semanticscholar.org/192c/6986a7e4b9895e2a9d905da7c5b04ebd9a28.pdf", "pmh_id": null, "is_best": true, "license": null, "updated": "2019-10-01T00:00:00", "version": "submittedVersion", "evidence": "oa repository (semantic scholar lookup)", "host_type": "repository", "endpoint_id": null, "url_for_pdf": "http://pdfs.semanticscholar.org/192c/6986a7e4b9895e2a9d905da7c5b04ebd9a28.pdf", "url_for_landing_page": "https://semanticscholar.org/paper/192c6986a7e4b9895e2a9d905da7c5b04ebd9a28", "repository_institution": null}, "journal_is_in_doaj": false, "has_repository_copy": true} +{"doi": "10.1200/jco.1985.3.6.858", "year": 1985, "genre": "journal-article", "is_oa": false, "title": "Extraskeletal mesenchymal chondrosarcoma: case report and review of the literature.", "doi_url": "https://doi.org/10.1200/jco.1985.3.6.858", "updated": "2020-03-02T17:59:08.300613", "oa_status": "closed", "publisher": "American Society of Clinical Oncology (ASCO)", "z_authors": [{"given": "C", "family": "Louvet", "sequence": "first"}, {"given": "A", "family": "de Gramont", "sequence": "additional"}, {"given": "M", "family": "Krulik", "sequence": "additional"}, {"given": "M", "family": "Jagueux", "sequence": "additional"}, {"given": "D", "family": "Hubert", "sequence": "additional"}, {"given": "P", "family": "Brissaud", "sequence": "additional"}, {"given": "A", "family": "Sirinelli", "sequence": "additional"}, {"given": "B", "family": "Augereau", "sequence": "additional"}, {"given": "J M", "family": "Tubiana", "sequence": "additional"}, {"given": "J", "family": "Debray", "sequence": "additional"}], "is_paratext": false, "journal_name": "Journal of Clinical Oncology", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0732-183X,1527-7755", "journal_issn_l": "0732-183X", "published_date": "1985-06-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1097/00005053-194010000-00011", "year": 1940, "genre": "journal-article", "is_oa": false, "title": "Cushingʼs Syndrome", "doi_url": "https://doi.org/10.1097/00005053-194010000-00011", "updated": "2020-02-10T19:08:16.356439", "oa_status": "closed", "publisher": "Ovid Technologies (Wolters Kluwer Health)", "z_authors": [{"given": "L. R.", "family": "Brostex", "sequence": "first"}], "is_paratext": false, "journal_name": "The Journal of Nervous and Mental Disease", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0022-3018", "journal_issn_l": "0022-3018", "published_date": "1940-10-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1002/asia.200600037", "year": 2006, "genre": "journal-article", "is_oa": false, "title": "Pyrene-Based Dual-Mode Fluorescence Switches and Logic Gates That Function in Solution and Film", "doi_url": "https://doi.org/10.1002/asia.200600037", "updated": "2020-02-08T08:21:32.418903", "oa_status": "closed", "publisher": "Wiley-Blackwell", "z_authors": [{"given": "Weidong", "family": "Zhou"}, {"given": "Yongjun", "family": "Li"}, {"given": "Yuliang", "family": "Li"}, {"given": "Huibiao", "family": "Liu"}, {"given": "Shu", "family": "Wang"}, {"given": "Cuihong", "family": "Li"}, {"given": "Mingjian", "family": "Yuan"}, {"given": "Xiaofeng", "family": "Liu"}, {"given": "Daoben", "family": "Zhu"}], "is_paratext": false, "journal_name": "Chemistry – An Asian Journal", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "1861-4728,1861-471X", "journal_issn_l": "1861-471X", "published_date": "2006-07-17", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1016/s1067-991x(03)70006-6", "year": 2003, "genre": "journal-article", "is_oa": false, "title": "Use of the autolaunch method of dispatching a helicopter", "doi_url": "https://doi.org/10.1016/s1067-991x(03)70006-6", "updated": "2020-03-12T07:24:35.659404", "oa_status": "closed", "publisher": "Elsevier BV", "z_authors": [{"given": "Kathleen S.", "family": "Berns", "sequence": "first"}, {"given": "Jeffery J.", "family": "Caniglia", "sequence": "additional"}, {"given": "Daniel G.", "family": "Hankins", "sequence": "additional"}, {"given": "Scott P.", "family": "Zietlow", "sequence": "additional"}], "is_paratext": false, "journal_name": "Air Medical Journal", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "1067-991X", "journal_issn_l": "1067-991X", "published_date": "2003-05-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1016/j.clinimag.2015.12.002", "year": 2016, "genre": "journal-article", "is_oa": false, "title": "Imaging findings, diagnosis, and clinical outcomes in patients with mycotic aneurysms: single center experience", "doi_url": "https://doi.org/10.1016/j.clinimag.2015.12.002", "updated": "2020-03-12T17:56:16.049536", "oa_status": "closed", "publisher": "Elsevier BV", "z_authors": [{"given": "Amy R.", "family": "Deipolyi", "sequence": "first"}, {"given": "Alexander", "family": "Bailin", "sequence": "additional"}, {"given": "Ali", "family": "Khademhosseini", "sequence": "additional"}, {"ORCID": "http://orcid.org/0000-0003-4984-1778", "given": "Rahmi", "family": "Oklu", "sequence": "additional", "authenticated-orcid": false}], "is_paratext": false, "journal_name": "Clinical Imaging", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0899-7071", "journal_issn_l": "0899-7071", "published_date": "2016-05-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1016/j.biocel.2013.05.012", "year": 2013, "genre": "journal-article", "is_oa": false, "title": "MAVS-mediated host cell defense is inhibited by Borna disease virus", "doi_url": "https://doi.org/10.1016/j.biocel.2013.05.012", "updated": "2020-03-09T20:49:25.975316", "oa_status": "closed", "publisher": "Elsevier BV", "z_authors": [{"given": "Yujun", "family": "Li", "sequence": "first"}, {"given": "Wuqi", "family": "Song", "sequence": "additional"}, {"given": "Jing", "family": "Wu", "sequence": "additional"}, {"given": "Qingmeng", "family": "Zhang", "sequence": "additional"}, {"given": "Junming", "family": "He", "sequence": "additional"}, {"given": "Aimei", "family": "Li", "sequence": "additional"}, {"given": "Jun", "family": "Qian", "sequence": "additional"}, {"given": "Aixia", "family": "Zhai", "sequence": "additional"}, {"given": "Yunlong", "family": "Hu", "sequence": "additional"}, {"given": "Wenping", "family": "Kao", "sequence": "additional"}, {"given": "Lanlan", "family": "Wei", "sequence": "additional"}, {"given": "Fengmin", "family": "Zhang", "sequence": "additional"}, {"given": "Dakang", "family": "Xu", "sequence": "additional"}], "is_paratext": false, "journal_name": "The International Journal of Biochemistry & Cell Biology", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "1357-2725", "journal_issn_l": "1357-2725", "published_date": "2013-08-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1021/acsami.8b01074.s004", "year": null, "genre": "component", "is_oa": false, "title": "Solution Coating of Pharmaceutical Nanothin Films and Multilayer Nanocomposites with Controlled Morphology and Polymorphism", "doi_url": "https://doi.org/10.1021/acsami.8b01074.s004", "updated": "2020-04-04T21:02:07.815195", "oa_status": "closed", "publisher": "American Chemical Society (ACS)", "z_authors": null, "is_paratext": false, "journal_name": null, "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": null, "journal_issn_l": null, "published_date": null, "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1093/nar/18.18.5552", "year": 1990, "genre": "journal-article", "is_oa": true, "title": "Nucleotide sequence of LTR-gag region of Rous sarcoma virus adapted to semi-permissive host", "doi_url": "https://doi.org/10.1093/nar/18.18.5552", "updated": "2020-02-07T07:59:06.754183", "oa_status": "green", "publisher": "Oxford University Press (OUP)", "z_authors": [{"given": "Vladimir I.", "family": "Kashuba", "sequence": "first"}, {"given": "Serge V.", "family": "Zubak", "sequence": "additional"}, {"given": "Vadim M.", "family": "Kavsan", "sequence": "additional"}, {"given": "Alla V.", "family": "Rynditch", "sequence": "additional"}, {"given": "Ivo", "family": "Hlozanek", "sequence": "additional"}], "is_paratext": false, "journal_name": "Nucleic Acids Research", "oa_locations": [{"url": "http://europepmc.org/articles/pmc332244?pdf=render", "pmh_id": "oai:pubmedcentral.nih.gov:332244", "is_best": true, "license": null, "updated": "2017-10-22T11:38:23.025497", "version": "publishedVersion", "evidence": "oa repository (via OAI-PMH doi match)", "host_type": "repository", "endpoint_id": "pubmedcentral.nih.gov", "url_for_pdf": "http://europepmc.org/articles/pmc332244?pdf=render", "url_for_landing_page": "http://europepmc.org/articles/pmc332244", "repository_institution": "pubmedcentral.nih.gov"}, {"url": "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC332244", "pmh_id": null, "is_best": false, "license": null, "updated": "2020-04-24T18:18:02.810779", "version": "publishedVersion", "evidence": "oa repository (via pmcid lookup)", "host_type": "repository", "endpoint_id": null, "url_for_pdf": null, "url_for_landing_page": "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC332244", "repository_institution": null}], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0305-1048,1362-4962", "journal_issn_l": "0305-1048", "published_date": "1990-01-01", "best_oa_location": {"url": "http://europepmc.org/articles/pmc332244?pdf=render", "pmh_id": "oai:pubmedcentral.nih.gov:332244", "is_best": true, "license": null, "updated": "2017-10-22T11:38:23.025497", "version": "publishedVersion", "evidence": "oa repository (via OAI-PMH doi match)", "host_type": "repository", "endpoint_id": "pubmedcentral.nih.gov", "url_for_pdf": "http://europepmc.org/articles/pmc332244?pdf=render", "url_for_landing_page": "http://europepmc.org/articles/pmc332244", "repository_institution": "pubmedcentral.nih.gov"}, "journal_is_in_doaj": false, "has_repository_copy": true} +{"doi": "10.1021/acsami.8b01294.s001", "year": null, "genre": "component", "is_oa": true, "title": "Highly Elastic Biodegradable Single-Network Hydrogel for Cell Printing", "doi_url": "https://doi.org/10.1021/acsami.8b01294.s001", "updated": "2020-04-04T22:12:53.813586", "oa_status": "bronze", "publisher": "American Chemical Society (ACS)", "z_authors": null, "is_paratext": false, "journal_name": null, "oa_locations": [{"url": "https://doi.org/10.1021/acsami.8b01294.s001", "pmh_id": null, "is_best": true, "license": null, "updated": "2020-04-04T22:11:06.757648", "version": "publishedVersion", "evidence": "open (via free pdf)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "https://doi.org/10.1021/acsami.8b01294.s001", "url_for_landing_page": null, "repository_institution": null}, {"url": "http://europepmc.org/articles/pmc5876623?pdf=render", "pmh_id": "oai:pubmedcentral.nih.gov:5876623", "is_best": false, "license": "acs-specific: authorchoice/editors choice usage agreement", "updated": "2020-02-19T13:50:59.876849", "version": "publishedVersion", "evidence": "oa repository (via OAI-PMH title match)", "host_type": "repository", "endpoint_id": "ac9de7698155b820de7", "url_for_pdf": "http://europepmc.org/articles/pmc5876623?pdf=render", "url_for_landing_page": "http://europepmc.org/articles/pmc5876623", "repository_institution": "National Institutes of Health (USA) - US National Library of Medicine"}], "data_standard": 2, "journal_is_oa": false, "journal_issns": null, "journal_issn_l": null, "published_date": null, "best_oa_location": {"url": "https://doi.org/10.1021/acsami.8b01294.s001", "pmh_id": null, "is_best": true, "license": null, "updated": "2020-04-04T22:11:06.757648", "version": "publishedVersion", "evidence": "open (via free pdf)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "https://doi.org/10.1021/acsami.8b01294.s001", "url_for_landing_page": null, "repository_institution": null}, "journal_is_in_doaj": false, "has_repository_copy": true} +{"doi": "10.1097/scs.0b013e3181ef67ba", "year": 2010, "genre": "journal-article", "is_oa": false, "title": "Anomaly of the Internal Carotid Artery Detected During Tonsillectomy", "doi_url": "https://doi.org/10.1097/scs.0b013e3181ef67ba", "updated": "2020-02-10T19:05:26.462040", "oa_status": "closed", "publisher": "Ovid Technologies (Wolters Kluwer Health)", "z_authors": [{"given": "Serdar", "family": "Ceylan", "sequence": "first"}, {"given": "Serkan", "family": "Salman", "sequence": "additional"}, {"given": "Fatih", "family": "Bora", "sequence": "additional"}], "is_paratext": false, "journal_name": "Journal of Craniofacial Surgery", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "1049-2275", "journal_issn_l": "1049-2275", "published_date": "2010-09-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1002/pds.880", "year": 2004, "genre": "journal-article", "is_oa": false, "title": "Antibiotic use profile at paediatric clinics in two transitional countries", "doi_url": "https://doi.org/10.1002/pds.880", "updated": "2020-04-12T23:58:25.353974", "oa_status": "closed", "publisher": "Wiley", "z_authors": [{"given": "Goran", "family": "Palčevski", "sequence": "first"}, {"given": "Vladimir", "family": "Ahel", "sequence": "additional"}, {"given": "Vera", "family": "Vlahović-Palčevski", "sequence": "additional"}, {"given": "Svetlana", "family": "Ratchina", "sequence": "additional"}, {"given": "Vesna", "family": "Rosovic-Bazijanac", "sequence": "additional"}, {"given": "L.", "family": "Averchenkova", "sequence": "additional"}], "is_paratext": false, "journal_name": "Pharmacoepidemiology and Drug Safety", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "1053-8569,1099-1557", "journal_issn_l": "1053-8569", "published_date": "2004-03-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1387/zer.19161", "year": 2018, "genre": "journal-article", "is_oa": true, "title": "Representaciones de la inmigración latinoamericana en la cinematográfica española (1996-2008)", "doi_url": "https://doi.org/10.1387/zer.19161", "updated": "2020-02-29T09:33:29.323932", "oa_status": "hybrid", "publisher": "UPV/EHU Press", "z_authors": [{"given": "Maíra", "family": "Dias Pereira", "sequence": "first"}], "is_paratext": false, "journal_name": "ZER - Revista de Estudios de Comunicación", "oa_locations": [{"url": "https://www.ehu.eus/ojs/index.php/Zer/article/download/19161/18230", "pmh_id": null, "is_best": true, "license": "cc-by-sa", "updated": "2020-01-03T11:50:30.101874", "version": "publishedVersion", "evidence": "open (via page says license)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "https://www.ehu.eus/ojs/index.php/Zer/article/download/19161/18230", "url_for_landing_page": "https://doi.org/10.1387/zer.19161", "repository_institution": null}, {"url": "https://addi.ehu.es/bitstream/10810/41276/1/19161-77528-1-PB.pdf", "pmh_id": "oai:addi.ehu.es:10810/41276", "is_best": false, "license": null, "updated": "2020-04-05T18:21:31.860362", "version": "publishedVersion", "evidence": "oa repository (via OAI-PMH doi match)", "host_type": "repository", "endpoint_id": "12746c530f3574451c6", "url_for_pdf": "https://addi.ehu.es/bitstream/10810/41276/1/19161-77528-1-PB.pdf", "url_for_landing_page": "http://hdl.handle.net/10810/41276", "repository_institution": "University of the Basque Country - Communities in ADDI"}], "data_standard": 2, "journal_is_oa": false, "journal_issns": "1989-631X,1137-1102", "journal_issn_l": "1137-1102", "published_date": "2018-11-30", "best_oa_location": {"url": "https://www.ehu.eus/ojs/index.php/Zer/article/download/19161/18230", "pmh_id": null, "is_best": true, "license": "cc-by-sa", "updated": "2020-01-03T11:50:30.101874", "version": "publishedVersion", "evidence": "open (via page says license)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "https://www.ehu.eus/ojs/index.php/Zer/article/download/19161/18230", "url_for_landing_page": "https://doi.org/10.1387/zer.19161", "repository_institution": null}, "journal_is_in_doaj": false, "has_repository_copy": true} +{"doi": "10.1093/gmo/9781561592630.article.l2286021", "year": 2015, "genre": "reference-book", "is_oa": false, "title": "Magbomboyo", "doi_url": "https://doi.org/10.1093/gmo/9781561592630.article.l2286021", "updated": "2020-02-26T00:38:46.900330", "oa_status": "closed", "publisher": "Oxford University Press", "z_authors": [{"given": "Ferdinand J. de", "family": "Hen", "sequence": "first"}], "is_paratext": false, "journal_name": "Oxford Music Online", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": null, "journal_issn_l": null, "published_date": "2015-09-22", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1016/j.bpj.2015.11.364", "year": 2016, "genre": "journal-article", "is_oa": true, "title": "Generalized Allostery in Thrombin", "doi_url": "https://doi.org/10.1016/j.bpj.2015.11.364", "updated": "2020-03-15T03:08:08.676899", "oa_status": "hybrid", "publisher": "Elsevier BV", "z_authors": [{"given": "Jiajie", "family": "Xiao", "sequence": "first"}, {"given": "Freddie R.", "family": "Salsbury", "suffix": "Jr", "sequence": "additional"}], "is_paratext": false, "journal_name": "Biophysical Journal", "oa_locations": [{"url": "http://www.cell.com/article/S0006349515015477/pdf", "pmh_id": null, "is_best": true, "license": "elsevier-specific: oa user license", "updated": "2020-04-25T14:44:51.025898", "version": "publishedVersion", "evidence": "open (via free pdf)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "http://www.cell.com/article/S0006349515015477/pdf", "url_for_landing_page": "https://doi.org/10.1016/j.bpj.2015.11.364", "repository_institution": null}], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0006-3495", "journal_issn_l": "0006-3495", "published_date": "2016-02-01", "best_oa_location": {"url": "http://www.cell.com/article/S0006349515015477/pdf", "pmh_id": null, "is_best": true, "license": "elsevier-specific: oa user license", "updated": "2020-04-25T14:44:51.025898", "version": "publishedVersion", "evidence": "open (via free pdf)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "http://www.cell.com/article/S0006349515015477/pdf", "url_for_landing_page": "https://doi.org/10.1016/j.bpj.2015.11.364", "repository_institution": null}, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1093/ww/9780199540884.013.11624", "year": 2007, "genre": "reference-entry", "is_oa": false, "title": "Comyns, Jacqueline Roberta, (born 27 April 1943), a District Judge (Magistrates’ Courts) (formerly a Metropolitan Stipendiary Magistrate), 1982–2013", "doi_url": "https://doi.org/10.1093/ww/9780199540884.013.11624", "updated": "2020-02-26T00:38:42.330690", "oa_status": "closed", "publisher": "Oxford University Press", "z_authors": null, "is_paratext": false, "journal_name": "Who's Who", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": null, "journal_issn_l": null, "published_date": "2007-12-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1002/chin.197234068", "year": 1972, "genre": "journal-article", "is_oa": false, "title": "ChemInform Abstract: NACHBARGRUPPEN-EFFEKTE BEI (1)H-NMR-SHIFTS DER TERT.-BUTYLGRUPPE, ROTATIONS-KONFORM. BEI DEN DIASTEREOMEREN VON 1,3-DI-TERT.-BUTYL-PROPARGYL-2-PHENYLPROPIONAT", "doi_url": "https://doi.org/10.1002/chin.197234068", "updated": "2020-03-15T06:52:11.575168", "oa_status": "closed", "publisher": "Wiley", "z_authors": [{"given": "R. S.", "family": "MACOMBER", "sequence": "first"}], "is_paratext": false, "journal_name": "Chemischer Informationsdienst", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0009-2975", "journal_issn_l": "0009-2975", "published_date": "1972-08-22", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1002/eet.3320020306", "year": 2007, "genre": "journal-article", "is_oa": false, "title": "Gis and environmental management", "doi_url": "https://doi.org/10.1002/eet.3320020306", "updated": "2020-03-21T09:14:36.158938", "oa_status": "closed", "publisher": "Wiley", "z_authors": [{"given": "Donald A.", "family": "Davidson", "sequence": "first"}], "is_paratext": false, "journal_name": "European Environment", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0961-0405,1099-0976", "journal_issn_l": "0961-0405", "published_date": "2007-07-06", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1103/physrevlett.58.2102", "year": 1987, "genre": "journal-article", "is_oa": false, "title": "Hydrodynamic Screening and Diffusion in Entangled Polymer Solutions", "doi_url": "https://doi.org/10.1103/physrevlett.58.2102", "updated": "2020-04-04T03:05:50.896467", "oa_status": "closed", "publisher": "American Physical Society (APS)", "z_authors": [{"given": "Y.", "family": "Shiwa", "sequence": "first"}], "is_paratext": false, "journal_name": "Physical Review Letters", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0031-9007", "journal_issn_l": "0031-9007", "published_date": "1987-05-18", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1007/978-3-030-34312-5", "year": 2020, "genre": "book", "is_oa": false, "title": "Optimal Districting and Territory Design", "doi_url": "https://doi.org/10.1007/978-3-030-34312-5", "updated": "2020-02-04T21:06:27.756631", "oa_status": "closed", "publisher": "Springer International Publishing", "z_authors": null, "is_paratext": false, "journal_name": "International Series in Operations Research & Management Science", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0884-8289,2214-7934", "journal_issn_l": "0884-8289", "published_date": "2020-01-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1007/978-3-319-01544-6_2", "year": 2013, "genre": "book-chapter", "is_oa": false, "title": "Coombe Hill, Apperley, Deerhurst, and Tewkesbury", "doi_url": "https://doi.org/10.1007/978-3-319-01544-6_2", "updated": "2020-03-06T03:29:01.970019", "oa_status": "closed", "publisher": "Springer International Publishing", "z_authors": [{"given": "Bernard", "family": "Michaux", "sequence": "first"}], "is_paratext": false, "journal_name": "Tewkesbury Walks", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": null, "journal_issn_l": null, "published_date": "2013-09-19", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1109/aieepas.1953.4498653", "year": 1953, "genre": "journal-article", "is_oa": false, "title": "Short-Time Thermal and Mechanical Limits for Transformers and Reactors [includes discussion]", "doi_url": "https://doi.org/10.1109/aieepas.1953.4498653", "updated": "2020-03-22T18:48:18.612422", "oa_status": "closed", "publisher": "Institute of Electrical and Electronics Engineers (IEEE)", "z_authors": [{"given": "J. E.", "family": "Clem", "sequence": "first"}], "is_paratext": false, "journal_name": "Transactions of the American Institute of Electrical Engineers. Part III: Power Apparatus and Systems", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0097-2460", "journal_issn_l": "0097-2460", "published_date": "1953-06-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1300/j358v06n01_10", "year": 1990, "genre": "journal-article", "is_oa": false, "title": "So Far, Yet So Near:", "doi_url": "https://doi.org/10.1300/j358v06n01_10", "updated": "2020-03-23T06:22:40.080466", "oa_status": "closed", "publisher": "The Haworth Press", "z_authors": [{"given": "John", "family": "Duryee", "sequence": "first"}], "is_paratext": false, "journal_name": "The Psychotherapy Patient", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0738-6176", "journal_issn_l": "0738-6176", "published_date": "1990-02-26", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1080/03605302.2011.556695", "year": 2011, "genre": "journal-article", "is_oa": true, "title": "Stability of Solitary Waves and Global Existence of a Generalized Two-Component Camassa–Holm System", "doi_url": "https://doi.org/10.1080/03605302.2011.556695", "updated": "2020-04-19T18:37:01.605875", "oa_status": "green", "publisher": "Informa UK Limited", "z_authors": [{"given": "Robin Ming", "family": "Chen", "sequence": "first"}, {"given": "Yue", "family": "Liu", "sequence": "additional"}, {"given": "Zhijun", "family": "Qiao", "sequence": "additional"}], "is_paratext": false, "journal_name": "Communications in Partial Differential Equations", "oa_locations": [{"url": "http://arxiv.org/pdf/1009.2449.pdf", "pmh_id": "oai:CiteSeerX.psu:10.1.1.767.774", "is_best": true, "license": null, "updated": "2020-04-15T11:50:31.231257", "version": "submittedVersion", "evidence": "oa repository (via OAI-PMH title and first author match)", "host_type": "repository", "endpoint_id": "7e4fec42e1147584aae", "url_for_pdf": "http://arxiv.org/pdf/1009.2449.pdf", "url_for_landing_page": "http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.767.774", "repository_institution": "The Pennsylvania State University - CiteSeer X"}], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0360-5302,1532-4133", "journal_issn_l": "0360-5302", "published_date": "2011-12-01", "best_oa_location": {"url": "http://arxiv.org/pdf/1009.2449.pdf", "pmh_id": "oai:CiteSeerX.psu:10.1.1.767.774", "is_best": true, "license": null, "updated": "2020-04-15T11:50:31.231257", "version": "submittedVersion", "evidence": "oa repository (via OAI-PMH title and first author match)", "host_type": "repository", "endpoint_id": "7e4fec42e1147584aae", "url_for_pdf": "http://arxiv.org/pdf/1009.2449.pdf", "url_for_landing_page": "http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.767.774", "repository_institution": "The Pennsylvania State University - CiteSeer X"}, "journal_is_in_doaj": false, "has_repository_copy": true} +{"doi": "10.1109/iccd.1988.25673", "year": null, "genre": "proceedings-article", "is_oa": false, "title": "Extension of a transistor level digital timing simulator to include first order analog behavior", "doi_url": "https://doi.org/10.1109/iccd.1988.25673", "updated": "2020-02-10T19:06:00.976825", "oa_status": "closed", "publisher": "IEEE Comput. Soc. Press", "z_authors": [{"given": "R.", "family": "Chadha"}, {"given": "C.-F.", "family": "Chen"}], "is_paratext": false, "journal_name": "Proceedings 1988 IEEE International Conference on Computer Design: VLSI", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": null, "journal_issn_l": null, "published_date": null, "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1306/f4c90164-1712-11d7-8645000102c1865d", "year": 1992, "genre": "journal-article", "is_oa": false, "title": "Kinematics of the Eastern Flank of the Beartooth Mountains, Montana and Wyoming", "doi_url": "https://doi.org/10.1306/f4c90164-1712-11d7-8645000102c1865d", "updated": "2020-02-04T01:02:18.977971", "oa_status": null, "publisher": "CrossRef Test Account", "z_authors": [{"family": "O'CONNELL, PATRICK, Conoco Inc., Ca"}], "is_paratext": false, "journal_name": "AAPG Bulletin", "oa_locations": [], "data_standard": 1, "journal_is_oa": false, "journal_issns": "0149-1423", "journal_issn_l": "0149-1423", "published_date": "1992-01-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.7551/mitpress/11839.003.0014", "year": 2019, "genre": "book-chapter", "is_oa": false, "title": "Acknowledgments", "doi_url": "https://doi.org/10.7551/mitpress/11839.003.0014", "updated": "2020-02-06T04:42:45.146779", "oa_status": "closed", "publisher": "The MIT Press", "z_authors": null, "is_paratext": false, "journal_name": "The Evolving Animal Orchestra", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": null, "journal_issn_l": null, "published_date": "2019-01-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1109/ictai.2017.00033", "year": 2017, "genre": "proceedings-article", "is_oa": false, "title": "Using Transitional Bottlenecks to Improve Learning in Nearest Sequence Memory Algorithm", "doi_url": "https://doi.org/10.1109/ictai.2017.00033", "updated": "2020-02-10T19:06:47.677349", "oa_status": "closed", "publisher": "IEEE", "z_authors": [{"given": "Huseyin", "family": "Aydin", "sequence": "first"}, {"given": "Erkin", "family": "Cilden", "sequence": "additional"}, {"given": "Faruk", "family": "Polat", "sequence": "additional"}], "is_paratext": false, "journal_name": "2017 IEEE 29th International Conference on Tools with Artificial Intelligence (ICTAI)", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": null, "journal_issn_l": null, "published_date": "2017-11-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1097/00000446-195009000-00032", "year": 1950, "genre": "journal-article", "is_oa": false, "title": "Crossed Eyes in Children", "doi_url": "https://doi.org/10.1097/00000446-195009000-00032", "updated": "2020-03-20T07:41:37.594309", "oa_status": "closed", "publisher": "Ovid Technologies (Wolters Kluwer Health)", "z_authors": [{"given": "Walter B.", "family": "Lancaster", "sequence": "first"}], "is_paratext": false, "journal_name": "AJN, American Journal of Nursing", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0002-936X", "journal_issn_l": "0002-936X", "published_date": "1950-09-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1109/ieem.2016.7798041", "year": 2016, "genre": "proceedings-article", "is_oa": false, "title": "Biogas use as fuel in spark ignition engines", "doi_url": "https://doi.org/10.1109/ieem.2016.7798041", "updated": "2020-03-16T17:00:53.219630", "oa_status": "closed", "publisher": "IEEE", "z_authors": [{"given": "T. O.", "family": "Kukoyi", "sequence": "first"}, {"given": "E.", "family": "Muzenda", "sequence": "additional"}, {"given": "E. T.", "family": "Akinlabi", "sequence": "additional"}, {"given": "A.", "family": "Mashamba", "sequence": "additional"}, {"given": "C.", "family": "Mbohwa", "sequence": "additional"}, {"given": "T.", "family": "Mahlatsi", "sequence": "additional"}], "is_paratext": false, "journal_name": "2016 IEEE International Conference on Industrial Engineering and Engineering Management (IEEM)", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": null, "journal_issn_l": null, "published_date": "2016-12-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1109/jphotov.2017.2756452", "year": 2017, "genre": "journal-article", "is_oa": false, "title": "Detecting Photovoltaic Module Failures in the Field During Daytime With Ultraviolet Fluorescence Module Inspection", "doi_url": "https://doi.org/10.1109/jphotov.2017.2756452", "updated": "2020-02-28T00:00:02.565714", "oa_status": "closed", "publisher": "Institute of Electrical and Electronics Engineers (IEEE)", "z_authors": [{"ORCID": "http://orcid.org/0000-0003-0206-9522", "given": "Arnaud", "family": "Morlier", "sequence": "first", "authenticated-orcid": false}, {"given": "Michael", "family": "Siebert", "sequence": "additional"}, {"given": "Iris", "family": "Kunze", "sequence": "additional"}, {"given": "Gerhard", "family": "Mathiak", "sequence": "additional"}, {"ORCID": "http://orcid.org/0000-0001-6565-6842", "given": "Marc", "family": "Kontges", "sequence": "additional", "authenticated-orcid": false}], "is_paratext": false, "journal_name": "IEEE Journal of Photovoltaics", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "2156-3381,2156-3403", "journal_issn_l": "2156-3403", "published_date": "2017-11-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.7551/mitpress/3653.003.0014", "year": 1994, "genre": "book-chapter", "is_oa": false, "title": "References", "doi_url": "https://doi.org/10.7551/mitpress/3653.003.0014", "updated": "2020-02-06T04:42:05.211183", "oa_status": "closed", "publisher": "The MIT Press", "z_authors": null, "is_paratext": false, "journal_name": "Image And Brain", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": null, "journal_issn_l": null, "published_date": "1994-01-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1016/s0378-5173(02)00044-3", "year": 2002, "genre": "journal-article", "is_oa": false, "title": "Targetability and intracellular delivery of anti-BCG antibody-modified, pH-sensitive fusogenic immunoliposomes to tumor cells", "doi_url": "https://doi.org/10.1016/s0378-5173(02)00044-3", "updated": "2020-04-06T03:56:21.653819", "oa_status": "closed", "publisher": "Elsevier BV", "z_authors": [{"given": "Toshiro", "family": "Mizoue", "sequence": "first"}, {"given": "Toshiya", "family": "Horibe", "sequence": "additional"}, {"given": "Kazuo", "family": "Maruyama", "sequence": "additional"}, {"given": "Tomoko", "family": "Takizawa", "sequence": "additional"}, {"given": "Motoharu", "family": "Iwatsuru", "sequence": "additional"}, {"given": "Kenji", "family": "Kono", "sequence": "additional"}, {"given": "Hironobu", "family": "Yanagie", "sequence": "additional"}, {"given": "Fuminori", "family": "Moriyasu", "sequence": "additional"}], "is_paratext": false, "journal_name": "International Journal of Pharmaceutics", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0378-5173", "journal_issn_l": "0378-5173", "published_date": "2002-04-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1109/sas.2019.8705987", "year": 2019, "genre": "proceedings-article", "is_oa": false, "title": "Bring your own Sensor: Use your Android Smartphone as a Sensing Platform", "doi_url": "https://doi.org/10.1109/sas.2019.8705987", "updated": "2020-02-08T04:59:33.784144", "oa_status": "closed", "publisher": "IEEE", "z_authors": [{"given": "Gilles", "family": "Callebaut", "sequence": "first"}, {"given": "Geoffrey", "family": "Ottoy", "sequence": "additional"}, {"given": "Lieven De", "family": "Strycker", "sequence": "additional"}], "is_paratext": false, "journal_name": "2019 IEEE Sensors Applications Symposium (SAS)", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": null, "journal_issn_l": null, "published_date": "2019-03-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.4236/oalib.1100444", "year": 2014, "genre": "journal-article", "is_oa": true, "title": "A Case Study of the Communication Pattern and Participation in Life of a 51-Year Aphasic Adult in Natural Setting", "doi_url": "https://doi.org/10.4236/oalib.1100444", "updated": "2020-03-04T13:16:45.656538", "oa_status": "bronze", "publisher": "Scientific Research Publishing, Inc,", "z_authors": [{"given": "Ojinga Gideon", "family": "Omiunu", "sequence": "first"}], "is_paratext": false, "journal_name": "OALib", "oa_locations": [{"url": "https://doi.org/10.4236/oalib.1100444", "pmh_id": null, "is_best": true, "license": null, "updated": "2019-02-07T00:23:33.492183", "version": "publishedVersion", "evidence": "open (via free pdf)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "https://doi.org/10.4236/oalib.1100444", "url_for_landing_page": null, "repository_institution": null}, {"url": "http://pdfs.semanticscholar.org/1a42/b64948d3fcd70eee591726b5bf80b0c37cea.pdf", "pmh_id": null, "is_best": false, "license": null, "updated": "2019-10-01T00:00:00", "version": "submittedVersion", "evidence": "oa repository (semantic scholar lookup)", "host_type": "repository", "endpoint_id": null, "url_for_pdf": "http://pdfs.semanticscholar.org/1a42/b64948d3fcd70eee591726b5bf80b0c37cea.pdf", "url_for_landing_page": "https://semanticscholar.org/paper/1a42b64948d3fcd70eee591726b5bf80b0c37cea", "repository_institution": null}], "data_standard": 2, "journal_is_oa": false, "journal_issns": "2333-9705,2333-9721", "journal_issn_l": "2333-9705", "published_date": "2014-01-01", "best_oa_location": {"url": "https://doi.org/10.4236/oalib.1100444", "pmh_id": null, "is_best": true, "license": null, "updated": "2019-02-07T00:23:33.492183", "version": "publishedVersion", "evidence": "open (via free pdf)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "https://doi.org/10.4236/oalib.1100444", "url_for_landing_page": null, "repository_institution": null}, "journal_is_in_doaj": false, "has_repository_copy": true} +{"doi": "10.1007/978-1-4302-4201-7_5", "year": 2012, "genre": "book-chapter", "is_oa": false, "title": "Grouping and Summarizing", "doi_url": "https://doi.org/10.1007/978-1-4302-4201-7_5", "updated": "2020-03-21T09:15:39.647605", "oa_status": "closed", "publisher": "Apress", "z_authors": [{"given": "Jason", "family": "Brimhall", "sequence": "first"}, {"given": "David", "family": "Dye", "sequence": "additional"}, {"given": "Jonathan", "family": "Gennick", "sequence": "additional"}, {"given": "Andy", "family": "Roberts", "sequence": "additional"}, {"given": "Wayne", "family": "Sheffield", "sequence": "additional"}], "is_paratext": false, "journal_name": "SQL Server 2012 T-SQL Recipes", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": null, "journal_issn_l": null, "published_date": "2012-01-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1111/j.1365-313x.1992.tb00152.x", "year": 1992, "genre": "journal-article", "is_oa": false, "title": "In-situlocalization of cyanogenic β-glucosidase (linamarase) gene expression in leaves of cassava (Manihot esculentaCranz) using non-isotopic riboprobes", "doi_url": "https://doi.org/10.1111/j.1365-313x.1992.tb00152.x", "updated": "2020-03-21T19:33:26.355324", "oa_status": "closed", "publisher": "Wiley", "z_authors": [{"given": "A. Pancoro and M.A.", "family": "Hughes", "sequence": "first"}], "is_paratext": false, "journal_name": "The Plant Journal", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0960-7412", "journal_issn_l": "0960-7412", "published_date": "1992-09-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1109/twc.2009.080612", "year": 2009, "genre": "journal-article", "is_oa": false, "title": "Transmitter optimization and performance gain for multiple-input single-output systems with finite-rate direction feedback", "doi_url": "https://doi.org/10.1109/twc.2009.080612", "updated": "2020-03-22T18:48:25.947545", "oa_status": "closed", "publisher": "Institute of Electrical and Electronics Engineers (IEEE)", "z_authors": [{"given": "Zhengdao", "family": "Wang", "sequence": "first"}, {"given": "Shengli", "family": "Zhou", "sequence": "additional"}, {"given": "Jinhong", "family": "Wu", "sequence": "additional"}], "is_paratext": false, "journal_name": "IEEE Transactions on Wireless Communications", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "1536-1276", "journal_issn_l": "1536-1276", "published_date": "2009-05-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1097/00005392-199904020-00076", "year": 1999, "genre": "journal-article", "is_oa": false, "title": "EFFECTIVE SUPPRESSION OF DIHYDROTESTOSTERONE (DHT) BY GI198745, A NOVEL, DUAL 5 ALPHA REDUCTASE INHIBITOR", "doi_url": "https://doi.org/10.1097/00005392-199904020-00076", "updated": "2020-03-08T04:45:15.181154", "oa_status": "closed", "publisher": "Ovid Technologies (Wolters Kluwer Health)", "z_authors": [{"given": "Richard V", "family": "Clark", "sequence": "first"}, {"given": "David J", "family": "Hermann", "sequence": "additional"}, {"given": "Hoda", "family": "Gabriel", "sequence": "additional"}, {"given": "Timothy H", "family": "Wilson", "sequence": "additional"}, {"given": "Betsy B", "family": "Morrill", "sequence": "additional"}, {"given": "Stuart", "family": "Hobbs", "sequence": "additional"}], "is_paratext": false, "journal_name": "The Journal of Urology", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0022-5347", "journal_issn_l": "0022-5347", "published_date": "1999-04-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1016/j.physbeh.2011.12.022", "year": 2012, "genre": "journal-article", "is_oa": false, "title": "Dopamine and food reward: Effects of acute tyrosine/phenylalanine depletion on appetite", "doi_url": "https://doi.org/10.1016/j.physbeh.2011.12.022", "updated": "2020-03-23T23:39:10.242485", "oa_status": "closed", "publisher": "Elsevier BV", "z_authors": [{"given": "Charlotte A.", "family": "Hardman", "sequence": "first"}, {"given": "Vanessa M.B.", "family": "Herbert", "sequence": "additional"}, {"given": "Jeffrey M.", "family": "Brunstrom", "sequence": "additional"}, {"given": "Marcus R.", "family": "Munafò", "sequence": "additional"}, {"given": "Peter J.", "family": "Rogers", "sequence": "additional"}], "is_paratext": false, "journal_name": "Physiology & Behavior", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0031-9384", "journal_issn_l": "0031-9384", "published_date": "2012-03-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.16980/jitc.12.5.201610", "year": 2016, "genre": "journal-issue", "is_oa": false, "title": null, "doi_url": "https://doi.org/10.16980/jitc.12.5.201610", "updated": "2020-03-26T15:38:15.506226", "oa_status": "closed", "publisher": "Korea International Trade Research Institute", "z_authors": null, "is_paratext": false, "journal_name": "Korea International Trade Research Institute", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": null, "journal_issn_l": null, "published_date": "2016-10-31", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.4025/actascihealthsci.v33i2.7956", "year": 2011, "genre": "journal-article", "is_oa": true, "title": "Desigualdades nos indicadores de saúde da mãe e do recém-nascido, no Estado do Paraná", "doi_url": "https://doi.org/10.4025/actascihealthsci.v33i2.7956", "updated": "2020-03-19T20:54:56.924528", "oa_status": "gold", "publisher": "Universidade Estadual de Maringa", "z_authors": [{"given": "Kelen Marja", "family": "Predebon", "sequence": "first"}, {"given": "Thais Aidar de Freitas", "family": "Mathias", "sequence": "additional"}], "is_paratext": false, "journal_name": "Acta Scientiarum. Health Science", "oa_locations": [{"url": "http://periodicos.uem.br/ojs/index.php/ActaSciHealthSci/article/download/7956/pdf", "pmh_id": null, "is_best": true, "license": "cc-by", "updated": "2019-02-26T23:46:42.509576", "version": "publishedVersion", "evidence": "open (via free pdf)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "http://periodicos.uem.br/ojs/index.php/ActaSciHealthSci/article/download/7956/pdf", "url_for_landing_page": "https://doi.org/10.4025/actascihealthsci.v33i2.7956", "repository_institution": null}, {"url": "https://doi.org/10.4025/actascihealthsci.v33i2.7956", "pmh_id": null, "is_best": false, "license": "cc-by", "updated": "2020-04-24T06:25:28.186984", "version": "publishedVersion", "evidence": "oa journal (via doaj)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": null, "url_for_landing_page": "https://doi.org/10.4025/actascihealthsci.v33i2.7956", "repository_institution": null}], "data_standard": 2, "journal_is_oa": true, "journal_issns": "1807-8648,1679-9291", "journal_issn_l": "1679-9291", "published_date": "2011-09-30", "best_oa_location": {"url": "http://periodicos.uem.br/ojs/index.php/ActaSciHealthSci/article/download/7956/pdf", "pmh_id": null, "is_best": true, "license": "cc-by", "updated": "2019-02-26T23:46:42.509576", "version": "publishedVersion", "evidence": "open (via free pdf)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "http://periodicos.uem.br/ojs/index.php/ActaSciHealthSci/article/download/7956/pdf", "url_for_landing_page": "https://doi.org/10.4025/actascihealthsci.v33i2.7956", "repository_institution": null}, "journal_is_in_doaj": true, "has_repository_copy": false} +{"doi": "10.1016/j.cor.2018.11.002", "year": 2019, "genre": "journal-article", "is_oa": false, "title": "Optimal facility layout and material handling network design", "doi_url": "https://doi.org/10.1016/j.cor.2018.11.002", "updated": "2020-02-29T17:57:40.122166", "oa_status": "closed", "publisher": "Elsevier BV", "z_authors": [{"given": "Armin", "family": "Klausnitzer", "sequence": "first"}, {"given": "Rainer", "family": "Lasch", "sequence": "additional"}], "is_paratext": false, "journal_name": "Computers & Operations Research", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0305-0548", "journal_issn_l": "0305-0548", "published_date": "2019-03-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1016/j.ejca.2014.02.023", "year": 2014, "genre": "journal-article", "is_oa": true, "title": "Adverse surgical outcomes in screen-detected ductal carcinoma in situ of the breast", "doi_url": "https://doi.org/10.1016/j.ejca.2014.02.023", "updated": "2020-03-06T14:50:28.823394", "oa_status": "green", "publisher": "Elsevier BV", "z_authors": [{"given": "Jeremy", "family": "Thomas", "sequence": "first"}, {"given": "Andrew", "family": "Hanby", "sequence": "additional"}, {"given": "Sarah E.", "family": "Pinder", "sequence": "additional"}, {"given": "Graham", "family": "Ball", "sequence": "additional"}, {"given": "Gill", "family": "Lawrence", "sequence": "additional"}, {"given": "Anthony", "family": "Maxwell", "sequence": "additional"}, {"given": "Matthew", "family": "Wallis", "sequence": "additional"}, {"given": "Andrew", "family": "Evans", "sequence": "additional"}, {"given": "Hilary", "family": "Dobson", "sequence": "additional"}, {"given": "Karen", "family": "Clements", "sequence": "additional"}, {"given": "Alastair", "family": "Thompson", "sequence": "additional"}], "is_paratext": false, "journal_name": "European Journal of Cancer", "oa_locations": [{"url": "http://irep.ntu.ac.uk/id/eprint/26926/1/PubSub4291_Ball.pdf", "pmh_id": "oai:irep.ntu.ac.uk:26926", "is_best": true, "license": null, "updated": "2020-03-30T03:10:47.341539", "version": "submittedVersion", "evidence": "oa repository (via OAI-PMH doi match)", "host_type": "repository", "endpoint_id": "2bec808aad12bb6c794", "url_for_pdf": "http://irep.ntu.ac.uk/id/eprint/26926/1/PubSub4291_Ball.pdf", "url_for_landing_page": "http://irep.ntu.ac.uk/id/eprint/26926/1/PubSub4291_Ball.pdf", "repository_institution": "Nottingham Trent Repository - Nottingham Trent University's Institutional Repository"}], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0959-8049", "journal_issn_l": "0959-8049", "published_date": "2014-07-01", "best_oa_location": {"url": "http://irep.ntu.ac.uk/id/eprint/26926/1/PubSub4291_Ball.pdf", "pmh_id": "oai:irep.ntu.ac.uk:26926", "is_best": true, "license": null, "updated": "2020-03-30T03:10:47.341539", "version": "submittedVersion", "evidence": "oa repository (via OAI-PMH doi match)", "host_type": "repository", "endpoint_id": "2bec808aad12bb6c794", "url_for_pdf": "http://irep.ntu.ac.uk/id/eprint/26926/1/PubSub4291_Ball.pdf", "url_for_landing_page": "http://irep.ntu.ac.uk/id/eprint/26926/1/PubSub4291_Ball.pdf", "repository_institution": "Nottingham Trent Repository - Nottingham Trent University's Institutional Repository"}, "journal_is_in_doaj": false, "has_repository_copy": true} +{"doi": "10.1142/9789812796691_0016", "year": 2003, "genre": "book-chapter", "is_oa": false, "title": "Role of Iron Metabolism in Multiple Sclerosis", "doi_url": "https://doi.org/10.1142/9789812796691_0016", "updated": "2020-03-19T16:22:00.811858", "oa_status": "closed", "publisher": "WORLD SCIENTIFIC", "z_authors": [{"given": "Maritha J.", "family": "Kotze", "sequence": "first", "affiliation": [{"name": "Division of Human Genetics, Faculty of Health Sciences, University of Stellenbosch, Tygerberg 7500, South Africa"}]}, {"given": "J. Nico P.", "family": "De Villiers", "sequence": "additional", "affiliation": [{"name": "Division of Human Genetics, Faculty of Health Sciences, University of Stellenbosch, Tygerberg 7500, South Africa"}]}, {"given": "Monique G.", "family": "Zaahl", "sequence": "additional", "affiliation": [{"name": "MRC Unit of Molecular Hematology, Weatherall Institute of Molecular Medicine, John Radcliffe Hospital, Headington, Oxford OX3 9DS, UK"}]}, {"given": "Kathryn J. H.", "family": "Robson", "sequence": "additional", "affiliation": [{"name": "MRC Unit of Molecular Hematology, Weatherall Institute of Molecular Medicine, John Radcliffe Hospital, Headington, Oxford OX3 9DS, UK"}]}], "is_paratext": false, "journal_name": "Metal Ions and Neurodegenerative Disorders", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": null, "journal_issn_l": null, "published_date": "2003-11-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.4028/www.scientific.net/kem.86-87.237", "year": 1993, "genre": "journal-article", "is_oa": false, "title": "Development of a Digital Speckle Correlation System for Use in the Non-Destructive Testing of Advanced Engineering Ceramics", "doi_url": "https://doi.org/10.4028/www.scientific.net/kem.86-87.237", "updated": "2020-03-25T23:48:39.254328", "oa_status": "closed", "publisher": "Trans Tech Publications, Ltd.", "z_authors": [{"given": "D.", "family": "Coburn", "sequence": "first"}, {"given": "J.A.", "family": "Slevin", "sequence": "additional"}], "is_paratext": false, "journal_name": "Key Engineering Materials", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "1662-9795", "journal_issn_l": "1013-9826", "published_date": "1993-07-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1203/00006450-199704001-00348", "year": 1997, "genre": "journal-article", "is_oa": true, "title": "Inositol 1, 4, 5 Triphosphate (IP3) Receptor Modification During Hypoxia in Cerebral Cortical Nuclei of the Guinea Pig Fetus. ♦ 328", "doi_url": "https://doi.org/10.1203/00006450-199704001-00348", "updated": "2020-03-27T00:50:24.766701", "oa_status": "bronze", "publisher": "Springer Science and Business Media LLC", "z_authors": [{"given": "Om P.", "family": "Mishra", "sequence": "first"}, {"given": "Santina", "family": "Zanelli", "sequence": "additional"}, {"given": "Joanna", "family": "Kubin", "sequence": "additional"}, {"given": "Woo-Taek", "family": "Kim", "sequence": "additional"}, {"given": "Endla K.", "family": "Anday", "sequence": "additional"}, {"given": "Maria", "family": "Delivoria-Papadopoulos", "sequence": "additional"}], "is_paratext": false, "journal_name": "Pediatric Research", "oa_locations": [{"url": "https://www.nature.com/articles/pr1997507.pdf", "pmh_id": null, "is_best": true, "license": null, "updated": "2018-06-17T14:57:25.976059", "version": "publishedVersion", "evidence": "open (via free pdf)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "https://www.nature.com/articles/pr1997507.pdf", "url_for_landing_page": "https://doi.org/10.1203/00006450-199704001-00348", "repository_institution": null}], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0031-3998,1530-0447", "journal_issn_l": "0031-3998", "published_date": "1997-04-01", "best_oa_location": {"url": "https://www.nature.com/articles/pr1997507.pdf", "pmh_id": null, "is_best": true, "license": null, "updated": "2018-06-17T14:57:25.976059", "version": "publishedVersion", "evidence": "open (via free pdf)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "https://www.nature.com/articles/pr1997507.pdf", "url_for_landing_page": "https://doi.org/10.1203/00006450-199704001-00348", "repository_institution": null}, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1109/wisa.2015.60", "year": 2015, "genre": "proceedings-article", "is_oa": false, "title": "A MMDB Cluster Based Cloud Platform for Query-Intensive Web Information Systems", "doi_url": "https://doi.org/10.1109/wisa.2015.60", "updated": "2020-03-16T17:00:42.799926", "oa_status": "closed", "publisher": "IEEE", "z_authors": [{"given": "Aipeng", "family": "Li", "sequence": "first"}, {"given": "Xing", "family": "Chen", "sequence": "additional"}, {"given": "Yuanbin", "family": "Xu", "sequence": "additional"}, {"given": "Xuee", "family": "Zeng", "sequence": "additional"}, {"given": "Xingtu", "family": "Lan", "sequence": "additional"}, {"given": "Wenzhong", "family": "Guo", "sequence": "additional"}], "is_paratext": false, "journal_name": "2015 12th Web Information System and Application Conference (WISA)", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": null, "journal_issn_l": null, "published_date": "2015-09-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1080/09751122.2010.11889997", "year": 2010, "genre": "journal-article", "is_oa": false, "title": "The Effect of Analogical Reasoning and Extended Wait Time on Achievement in Biology", "doi_url": "https://doi.org/10.1080/09751122.2010.11889997", "updated": "2020-03-09T00:35:42.006395", "oa_status": "closed", "publisher": "Kamla Raj Enterprises", "z_authors": [{"given": "Janet Omolayo", "family": "Olajide", "sequence": "first", "affiliation": [{"name": "Samaru College of Agriculture, Division of Agricultural Colleges, Ahmadu Bello University, Zaria, Kaduna State, Nigeria"}]}, {"given": "Femi Adetunji", "family": "Adeoye", "sequence": "additional", "affiliation": [{"name": "School of Education, National Open University of Nigeria, Lagos, Lagos State, Nigeria"}]}], "is_paratext": false, "journal_name": "International Journal of Educational Sciences", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0975-1122", "journal_issn_l": "0975-1122", "published_date": "2010-07-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1007/978-1-4614-0539-9_1169", "year": 2013, "genre": "book-chapter", "is_oa": false, "title": "3α-Isobutyroyloxy-furanoeremophil-9-one (Euryopsonol-isobutyrat)", "doi_url": "https://doi.org/10.1007/978-1-4614-0539-9_1169", "updated": "2020-02-08T08:21:06.309896", "oa_status": "closed", "publisher": "Springer New York", "z_authors": null, "is_paratext": false, "journal_name": "Natural Compounds", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": null, "journal_issn_l": null, "published_date": "2013-01-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.2110/jsr.2012.12", "year": 2012, "genre": "journal-article", "is_oa": false, "title": "Constructing Sandstone Provenance and Classification Ternary Diagrams Using An Electronic Spreadsheet", "doi_url": "https://doi.org/10.2110/jsr.2012.12", "updated": "2020-03-22T09:27:09.023104", "oa_status": "closed", "publisher": "Society for Sedimentary Geology", "z_authors": [{"given": "K. M.", "family": "Zahid", "sequence": "first"}, {"given": "D. L.", "family": "Barbeau", "sequence": "additional"}], "is_paratext": false, "journal_name": "Journal of Sedimentary Research", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "1527-1404", "journal_issn_l": "1527-1404", "published_date": "2012-02-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1016/s0021-9673(00)84953-5", "year": 1980, "genre": "journal-article", "is_oa": false, "title": "Comparative assessment of gas—liquid chromatography and high-performance liquid chromatography for the separation of tin tetraalkyls and alkyltin halides", "doi_url": "https://doi.org/10.1016/s0021-9673(00)84953-5", "updated": "2020-03-08T10:05:15.727727", "oa_status": "closed", "publisher": "Elsevier BV", "z_authors": [{"given": "D.Thorburn", "family": "Burns", "sequence": "first"}, {"given": "F.", "family": "Glockling", "sequence": "additional"}, {"given": "M.", "family": "Harriott", "sequence": "additional"}], "is_paratext": false, "journal_name": "Journal of Chromatography A", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0021-9673", "journal_issn_l": "0021-9673", "published_date": "1980-11-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.15623/ijret.2017.0605018", "year": 2017, "genre": "journal-article", "is_oa": true, "title": "FACTORS INFLUENCING ACADEMIC PERFORMANCE OF THE STUDENTS AT UNIVERSITY LEVEL EXAM: A LITERATURE REVIEW", "doi_url": "https://doi.org/10.15623/ijret.2017.0605018", "updated": "2020-03-12T03:51:09.911478", "oa_status": "bronze", "publisher": "eSAT Publishing House", "z_authors": [{"given": "Radheshyam H. Gajghat", "family": ".", "sequence": "first"}], "is_paratext": false, "journal_name": "International Journal of Research in Engineering and Technology", "oa_locations": [{"url": "https://doi.org/10.15623/ijret.2017.0605018", "pmh_id": null, "is_best": true, "license": null, "updated": "2019-10-14T07:56:48.623304", "version": "publishedVersion", "evidence": "open (via free pdf)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "https://doi.org/10.15623/ijret.2017.0605018", "url_for_landing_page": null, "repository_institution": null}], "data_standard": 2, "journal_is_oa": false, "journal_issns": "2321-7308,2319-1163", "journal_issn_l": "2319-1163", "published_date": "2017-05-25", "best_oa_location": {"url": "https://doi.org/10.15623/ijret.2017.0605018", "pmh_id": null, "is_best": true, "license": null, "updated": "2019-10-14T07:56:48.623304", "version": "publishedVersion", "evidence": "open (via free pdf)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "https://doi.org/10.15623/ijret.2017.0605018", "url_for_landing_page": null, "repository_institution": null}, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1109/eptc.2018.8654391", "year": 2018, "genre": "proceedings-article", "is_oa": false, "title": "Via Interconnections for Half-Inch Sized Package Fabricated by Minimal Fab", "doi_url": "https://doi.org/10.1109/eptc.2018.8654391", "updated": "2020-02-07T15:50:36.911685", "oa_status": "closed", "publisher": "IEEE", "z_authors": [{"given": "Fumito", "family": "Imura", "sequence": "first"}, {"given": "Michihiro", "family": "Inoue", "sequence": "additional"}, {"given": "Sommawan", "family": "Khumpuang", "sequence": "additional"}, {"given": "Shiro", "family": "Hara", "sequence": "additional"}], "is_paratext": false, "journal_name": "2018 IEEE 20th Electronics Packaging Technology Conference (EPTC)", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": null, "journal_issn_l": null, "published_date": "2018-12-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1109/ica-symp.2019.8646146", "year": 2019, "genre": "proceedings-article", "is_oa": false, "title": "Trajectory Tracking Control for Omnidirectional Mobile Robots Using Direct Adaptive Neural Network Dynamic Surface Controller", "doi_url": "https://doi.org/10.1109/ica-symp.2019.8646146", "updated": "2020-02-07T15:50:59.175637", "oa_status": "closed", "publisher": "IEEE", "z_authors": [{"given": "Duyen-Ha", "family": "Thi Kim", "sequence": "first"}, {"given": "Tien-Ngo", "family": "Manh", "sequence": "additional"}, {"given": "Ngoc-Pham", "family": "Van Bach", "sequence": "additional"}, {"given": "Tuan-Pham", "family": "Duc", "sequence": "additional"}], "is_paratext": false, "journal_name": "2019 First International Symposium on Instrumentation, Control, Artificial Intelligence, and Robotics (ICA-SYMP)", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": null, "journal_issn_l": null, "published_date": "2019-01-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1590/2179-8087.019316", "year": 2018, "genre": "journal-article", "is_oa": true, "title": "Allometric models to biomass in restoration areas in the Atlantic rain forest", "doi_url": "https://doi.org/10.1590/2179-8087.019316", "updated": "2020-03-06T03:27:56.314870", "oa_status": "gold", "publisher": "FapUNIFESP (SciELO)", "z_authors": [{"given": "Emanuel José Gomes de", "family": "Araújo", "sequence": "first", "affiliation": [{"name": "Universidade Federal Rural do Rio de Janeiro, Brazil"}]}, {"given": "Gabrielle Hambrecht", "family": "Loureiro", "sequence": "additional", "affiliation": [{"name": "Suzano Papel e Celulose, Brazil"}]}, {"given": "Carlos Roberto", "family": "Sanquetta", "sequence": "additional", "affiliation": [{"name": "Universidade Federal do Paraná, Brazil"}]}, {"given": "Mateus Niroh Inoue", "family": "Sanquetta", "sequence": "additional", "affiliation": [{"name": "Universidade Federal do Paraná, Brazil"}]}, {"given": "Ana Paula Dalla", "family": "Corte", "sequence": "additional", "affiliation": [{"name": "Universidade Federal do Paraná, Brazil"}]}, {"given": "Sylvio", "family": "Péllico Netto", "sequence": "additional", "affiliation": [{"name": "Universidade Federal do Paraná, Brazil"}]}, {"given": "Alexandre", "family": "Behling", "sequence": "additional", "affiliation": [{"name": "Universidade Federal do Paraná, Brazil"}]}], "is_paratext": false, "journal_name": "Floresta e Ambiente", "oa_locations": [{"url": "http://www.scielo.br/pdf/floram/v25n1/2179-8087-floram-25-1-e20160193.pdf", "pmh_id": null, "is_best": true, "license": "cc-by", "updated": "2019-12-25T00:08:04.010040", "version": "publishedVersion", "evidence": "open (via page says license)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "http://www.scielo.br/pdf/floram/v25n1/2179-8087-floram-25-1-e20160193.pdf", "url_for_landing_page": "https://doi.org/10.1590/2179-8087.019316", "repository_institution": null}, {"url": "https://doi.org/10.1590/2179-8087.019316", "pmh_id": null, "is_best": false, "license": "cc-by", "updated": "2020-04-22T14:49:36.486394", "version": "publishedVersion", "evidence": "oa journal (via doaj)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": null, "url_for_landing_page": "https://doi.org/10.1590/2179-8087.019316", "repository_institution": null}, {"url": "http://pdfs.semanticscholar.org/cbf0/a7b5759bcb7632c00cddf58e258938cb2510.pdf", "pmh_id": null, "is_best": false, "license": null, "updated": "2019-10-01T00:00:00", "version": "submittedVersion", "evidence": "oa repository (semantic scholar lookup)", "host_type": "repository", "endpoint_id": null, "url_for_pdf": "http://pdfs.semanticscholar.org/cbf0/a7b5759bcb7632c00cddf58e258938cb2510.pdf", "url_for_landing_page": "https://semanticscholar.org/paper/cbf0a7b5759bcb7632c00cddf58e258938cb2510", "repository_institution": null}], "data_standard": 2, "journal_is_oa": true, "journal_issns": "2179-8087,1415-0980", "journal_issn_l": "1415-0980", "published_date": "2018-02-01", "best_oa_location": {"url": "http://www.scielo.br/pdf/floram/v25n1/2179-8087-floram-25-1-e20160193.pdf", "pmh_id": null, "is_best": true, "license": "cc-by", "updated": "2019-12-25T00:08:04.010040", "version": "publishedVersion", "evidence": "open (via page says license)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "http://www.scielo.br/pdf/floram/v25n1/2179-8087-floram-25-1-e20160193.pdf", "url_for_landing_page": "https://doi.org/10.1590/2179-8087.019316", "repository_institution": null}, "journal_is_in_doaj": true, "has_repository_copy": true} +{"doi": "10.1038/clpt.1992.21", "year": 1992, "genre": "journal-article", "is_oa": false, "title": "The method of relative drug accumulation: A simple method for illustrating the effects of different drug dosing regimens and variability in drug elimination on time courses of drug concentrations", "doi_url": "https://doi.org/10.1038/clpt.1992.21", "updated": "2020-03-20T12:16:35.441442", "oa_status": "closed", "publisher": "Springer Science and Business Media LLC", "z_authors": [{"given": "Thorir D", "family": "Bjornsson", "sequence": "first"}], "is_paratext": false, "journal_name": "Clinical Pharmacology and Therapeutics", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0009-9236,1532-6535", "journal_issn_l": "0009-9236", "published_date": "1992-03-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.7551/mitpress/3926.003.0008", "year": 1998, "genre": "book-chapter", "is_oa": false, "title": "Fuzzy Relations and Their Calculus", "doi_url": "https://doi.org/10.7551/mitpress/3926.003.0008", "updated": "2020-02-06T04:42:47.095604", "oa_status": "closed", "publisher": "The MIT Press", "z_authors": null, "is_paratext": false, "journal_name": "An Introduction to Fuzzy Sets", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": null, "journal_issn_l": null, "published_date": "1998-01-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1007/978-3-642-29482-2_1", "year": 2014, "genre": "book-chapter", "is_oa": false, "title": "The Upper Mantle Seismic Velocity Structure of South-Central Africa and the Seismic Architecture of Precambrian Lithosphere Beneath the Congo Basin", "doi_url": "https://doi.org/10.1007/978-3-642-29482-2_1", "updated": "2020-03-15T06:51:45.897859", "oa_status": "closed", "publisher": "Springer Berlin Heidelberg", "z_authors": [{"given": "Andriamiranto", "family": "Raveloson", "sequence": "first"}, {"given": "Andrew", "family": "Nyblade", "sequence": "additional"}, {"given": "Stewart", "family": "Fishwick", "sequence": "additional"}, {"given": "Azangi", "family": "Mangongolo", "sequence": "additional"}, {"given": "Sharad", "family": "Master", "sequence": "additional"}], "is_paratext": false, "journal_name": "Geology and Resource Potential of the Congo Basin", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": null, "journal_issn_l": null, "published_date": "2014-12-16", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1111/j.1536-7150.1997.tb03456.x", "year": 1997, "genre": "journal-article", "is_oa": false, "title": "Organization and Distributional Equality in a Network of Communes: The Shakers", "doi_url": "https://doi.org/10.1111/j.1536-7150.1997.tb03456.x", "updated": "2020-02-07T15:27:53.029310", "oa_status": "closed", "publisher": "Wiley-Blackwell", "z_authors": [{"given": "Metin M.", "family": "Cosgel"}, {"given": "Thomas J.", "family": "Miceli"}, {"given": "John E.", "family": "Murray"}], "is_paratext": false, "journal_name": "American Journal of Economics and Sociology", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0002-9246,1536-7150", "journal_issn_l": "0002-9246", "published_date": "1997-04-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1080/10811449708414413", "year": 1997, "genre": "journal-article", "is_oa": false, "title": "Parental grief of a perinatal loss: A comparison of individual and relationship variables", "doi_url": "https://doi.org/10.1080/10811449708414413", "updated": "2020-03-21T05:42:27.269033", "oa_status": "closed", "publisher": "Informa UK Limited", "z_authors": [{"given": "Volker", "family": "Thomas", "sequence": "first"}, {"given": "Phil", "family": "Striegel", "sequence": "additional"}, {"given": "Dorothea", "family": "Dudley", "sequence": "additional"}, {"given": "Jane", "family": "Wilkins", "sequence": "additional"}, {"given": "Darlene", "family": "Gibson", "sequence": "additional"}], "is_paratext": false, "journal_name": "Journal of Personal and Interpersonal Loss", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "1081-1443", "journal_issn_l": "1081-1443", "published_date": "1997-04-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1007/978-1-349-04912-7_9", "year": 1977, "genre": "book-chapter", "is_oa": false, "title": "The primitive ungulates", "doi_url": "https://doi.org/10.1007/978-1-349-04912-7_9", "updated": "2020-03-14T09:43:27.933850", "oa_status": "closed", "publisher": "Macmillan Education UK", "z_authors": [{"given": "J. E.", "family": "Webb", "sequence": "first"}, {"given": "J. A.", "family": "Wallwork", "sequence": "additional"}, {"given": "J. H.", "family": "Elgood", "sequence": "additional"}], "is_paratext": false, "journal_name": "Guide to Living Mammals", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": null, "journal_issn_l": null, "published_date": "1977-01-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1017/9781108304429.008", "year": null, "genre": "book-chapter", "is_oa": false, "title": "Capital Flows and Macroprudential Policy", "doi_url": "https://doi.org/10.1017/9781108304429.008", "updated": "2020-02-05T00:03:44.205302", "oa_status": "closed", "publisher": "Cambridge University Press", "z_authors": [{"given": "Matteo F.", "family": "Ghilardi", "sequence": "first"}, {"given": "Shanaka J.", "family": "Peiris", "sequence": "additional"}], "is_paratext": false, "journal_name": "Macroprudential Policy and Practice", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": null, "journal_issn_l": null, "published_date": null, "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1007/978-3-662-47152-4_10", "year": 2015, "genre": "book-chapter", "is_oa": false, "title": "Raumzeitgeometrie", "doi_url": "https://doi.org/10.1007/978-3-662-47152-4_10", "updated": "2020-03-15T06:51:43.234472", "oa_status": "closed", "publisher": "Springer Berlin Heidelberg", "z_authors": [{"given": "N. David", "family": "Mermin", "sequence": "first"}], "is_paratext": false, "journal_name": "Es ist an der Zeit", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": null, "journal_issn_l": null, "published_date": "2015-09-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1016/0013-4694(95)00147-q", "year": 1995, "genre": "journal-article", "is_oa": false, "title": "Investigation of EEG non-linearity in dementia and Parkinson's disease", "doi_url": "https://doi.org/10.1016/0013-4694(95)00147-q", "updated": "2020-03-02T07:05:03.111050", "oa_status": "closed", "publisher": "Elsevier BV", "z_authors": [{"given": "C.J.", "family": "Stam", "sequence": "first"}, {"given": "B.", "family": "Jelles", "sequence": "additional"}, {"given": "H.A.M.", "family": "Achtereekte", "sequence": "additional"}, {"given": "S.A.R.B.", "family": "Rombouts", "sequence": "additional"}, {"given": "J.P.J.", "family": "Slaets", "sequence": "additional"}, {"given": "R.W.M.", "family": "Keunen", "sequence": "additional"}], "is_paratext": false, "journal_name": "Electroencephalography and Clinical Neurophysiology", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0013-4694", "journal_issn_l": "0013-4694", "published_date": "1995-11-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1109/icip.1999.822894", "year": null, "genre": "proceedings-article", "is_oa": false, "title": "Document image data hiding technique using character spacing width sequence coding", "doi_url": "https://doi.org/10.1109/icip.1999.822894", "updated": "2020-02-07T15:51:57.307572", "oa_status": "closed", "publisher": "IEEE", "z_authors": [{"given": "N.", "family": "Chotikakamthorn"}], "is_paratext": false, "journal_name": "Proceedings 1999 International Conference on Image Processing (Cat. 99CH36348)", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": null, "journal_issn_l": null, "published_date": null, "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1016/j.geb.2007.07.006", "year": 2008, "genre": "journal-article", "is_oa": false, "title": "Regret minimization in repeated matrix games with variable stage duration", "doi_url": "https://doi.org/10.1016/j.geb.2007.07.006", "updated": "2020-02-29T17:57:38.292553", "oa_status": "closed", "publisher": "Elsevier BV", "z_authors": [{"given": "Shie", "family": "Mannor", "sequence": "first"}, {"given": "Nahum", "family": "Shimkin", "sequence": "additional"}], "is_paratext": false, "journal_name": "Games and Economic Behavior", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0899-8256", "journal_issn_l": "0899-8256", "published_date": "2008-05-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1111/1467-8489.00090", "year": 1999, "genre": "journal-article", "is_oa": true, "title": "The Travel Cost Method: an Empirical Investigation of Randall's Difficulty", "doi_url": "https://doi.org/10.1111/1467-8489.00090", "updated": "2020-04-18T22:33:49.873021", "oa_status": "green", "publisher": "Wiley-Blackwell", "z_authors": [{"given": "M.", "family": "Common"}, {"given": "T.", "family": "Bull"}, {"given": "N.", "family": "Stoeckl"}], "is_paratext": false, "journal_name": "The Australian Journal of Agricultural and Resource Economics", "oa_locations": [{"url": "https://researchonline.jcu.edu.au/14892/1/RDIFF.pdf", "pmh_id": "oai:researchonline.jcu.edu.au:14892", "is_best": true, "license": null, "updated": "2020-03-24T18:30:04.313502", "version": "acceptedVersion", "evidence": "oa repository (via OAI-PMH doi match)", "host_type": "repository", "endpoint_id": "13c188e0ff8a8e0a2d9", "url_for_pdf": "https://researchonline.jcu.edu.au/14892/1/RDIFF.pdf", "url_for_landing_page": "https://researchonline.jcu.edu.au/14892/1/RDIFF.pdf", "repository_institution": "James Cook University - ResearchOnline at James Cook University"}, {"url": "http://ageconsearch.umn.edu/record/117209/files/1467-8489.00090.pdf", "pmh_id": "oai:ageconsearch.umn.edu:117209", "is_best": false, "license": null, "updated": "2020-03-24T19:27:07.682309", "version": "submittedVersion", "evidence": "oa repository (via OAI-PMH title and first author match)", "host_type": "repository", "endpoint_id": "mf79cjfoywym4fwvffbh", "url_for_pdf": "http://ageconsearch.umn.edu/record/117209/files/1467-8489.00090.pdf", "url_for_landing_page": "http://ageconsearch.umn.edu/record/117209/files/1467-8489.00090.pdf", "repository_institution": "University of Minnesota, USA - AgEcon Search"}, {"url": "https://ageconsearch.umn.edu/record/117209/files/1467-8489.00090.pdf", "pmh_id": "oai:ageconsearch.umn.edu:117209", "is_best": false, "license": null, "updated": "2020-03-24T19:41:27.303215", "version": "submittedVersion", "evidence": "oa repository (via OAI-PMH title and first author match)", "host_type": "repository", "endpoint_id": "mf79cjfoywym4fwvffbh", "url_for_pdf": "https://ageconsearch.umn.edu/record/117209/files/1467-8489.00090.pdf", "url_for_landing_page": "http://ageconsearch.umn.edu/record/117209", "repository_institution": "University of Minnesota, USA - AgEcon Search"}, {"url": "https://openresearch-repository.anu.edu.au/bitstream/1885/40909/2/eep9705.pdf", "pmh_id": "oai:openresearch-repository.anu.edu.au:1885/40909", "is_best": false, "license": null, "updated": "2020-04-01T02:34:52.876231", "version": "submittedVersion", "evidence": "oa repository (via OAI-PMH title and first author match)", "host_type": "repository", "endpoint_id": "2nhihxmth4mwfyiuunth", "url_for_pdf": "https://openresearch-repository.anu.edu.au/bitstream/1885/40909/2/eep9705.pdf", "url_for_landing_page": "http://hdl.handle.net/1885/40909", "repository_institution": "Australian National University - ANU Open Research"}], "data_standard": 2, "journal_is_oa": false, "journal_issns": "1364-985X,1467-8489", "journal_issn_l": "1364-985X", "published_date": "1999-12-01", "best_oa_location": {"url": "https://researchonline.jcu.edu.au/14892/1/RDIFF.pdf", "pmh_id": "oai:researchonline.jcu.edu.au:14892", "is_best": true, "license": null, "updated": "2020-03-24T18:30:04.313502", "version": "acceptedVersion", "evidence": "oa repository (via OAI-PMH doi match)", "host_type": "repository", "endpoint_id": "13c188e0ff8a8e0a2d9", "url_for_pdf": "https://researchonline.jcu.edu.au/14892/1/RDIFF.pdf", "url_for_landing_page": "https://researchonline.jcu.edu.au/14892/1/RDIFF.pdf", "repository_institution": "James Cook University - ResearchOnline at James Cook University"}, "journal_is_in_doaj": false, "has_repository_copy": true} +{"doi": "10.7551/mitpress/4069.001.0001", "year": 2000, "genre": "book", "is_oa": false, "title": "Knowledge and Mind", "doi_url": "https://doi.org/10.7551/mitpress/4069.001.0001", "updated": "2020-02-06T04:42:33.300473", "oa_status": "closed", "publisher": "The MIT Press", "z_authors": [{"given": "Andrew", "family": "Brook", "sequence": "first"}, {"given": "Robert J.", "family": "Stainton", "sequence": "additional"}], "is_paratext": false, "journal_name": null, "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": null, "journal_issn_l": null, "published_date": "2000-01-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1515/9783110912173.151", "year": null, "genre": "book-chapter", "is_oa": false, "title": "9. Salient morphological features of nominal anglicisms: gender, plural, and genitive case", "doi_url": "https://doi.org/10.1515/9783110912173.151", "updated": "2020-02-05T00:02:53.301460", "oa_status": "closed", "publisher": "DE GRUYTER", "z_authors": null, "is_paratext": false, "journal_name": "Anglicisms in German", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": null, "journal_issn_l": null, "published_date": null, "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1007/978-3-8348-9292-8_4", "year": 2008, "genre": "book-chapter", "is_oa": false, "title": "Die z-Transformation", "doi_url": "https://doi.org/10.1007/978-3-8348-9292-8_4", "updated": "2020-03-21T09:14:33.289824", "oa_status": "closed", "publisher": "Vieweg+Teubner", "z_authors": [{"given": "Thomas", "family": "Frey", "sequence": "first"}, {"given": "Martin", "family": "Bossert", "sequence": "additional"}], "is_paratext": false, "journal_name": "Signal- und Systemtheorie", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": null, "journal_issn_l": null, "published_date": "2008-01-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1111/j.1365-3032.1984.tb00710.x", "year": 1984, "genre": "journal-article", "is_oa": false, "title": "A preparation of the stick insect Carausius morosus for recording intracellularly from identified neurones during walking", "doi_url": "https://doi.org/10.1111/j.1365-3032.1984.tb00710.x", "updated": "2020-03-22T18:48:15.121727", "oa_status": "closed", "publisher": "Wiley", "z_authors": [{"given": "D. H.", "family": "GODDEN", "sequence": "first"}, {"given": "D.", "family": "GRAHAM", "sequence": "additional"}], "is_paratext": false, "journal_name": "Physiological Entomology", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0307-6962,1365-3032", "journal_issn_l": "0307-6962", "published_date": "1984-09-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1001/jama.2009.909", "year": 2009, "genre": "journal-article", "is_oa": false, "title": "Clinical Practice Guidelines and Scientific Evidence", "doi_url": "https://doi.org/10.1001/jama.2009.909", "updated": "2020-03-23T23:40:16.508417", "oa_status": "closed", "publisher": "American Medical Association (AMA)", "z_authors": [{"given": "Finlay A.", "family": "McAlister", "sequence": "first"}], "is_paratext": false, "journal_name": "JAMA", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0098-7484", "journal_issn_l": "0098-7484", "published_date": "2009-07-08", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1111/j.1746-1561.2010.00533.x", "year": 2010, "genre": "journal-article", "is_oa": false, "title": "A Structured, Interactive Method for Youth Participation in a School District-University Partnership to Prevent Obesity", "doi_url": "https://doi.org/10.1111/j.1746-1561.2010.00533.x", "updated": "2020-03-22T18:48:52.768791", "oa_status": "closed", "publisher": "Wiley", "z_authors": [{"given": "Janet C.", "family": "Meininger", "sequence": "first"}, {"given": "Lisa R.", "family": "Reyes", "sequence": "additional"}, {"given": "Beatrice J.", "family": "Selwyn", "sequence": "additional"}, {"given": "Sandra L.", "family": "Upchurch", "sequence": "additional"}, {"given": "Christine A.", "family": "Brosnan", "sequence": "additional"}, {"given": "Wendell C.", "family": "Taylor", "sequence": "additional"}, {"given": "Evangelina", "family": "Villagomez", "sequence": "additional"}, {"given": "Vianey", "family": "Quintana", "sequence": "additional"}, {"given": "Bridgette", "family": "Pullis", "sequence": "additional"}, {"given": "Denise", "family": "Caudill", "sequence": "additional"}, {"given": "Sharon", "family": "Sterchy", "sequence": "additional"}, {"given": "Melinda", "family": "Phillips", "sequence": "additional"}], "is_paratext": false, "journal_name": "Journal of School Health", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0022-4391", "journal_issn_l": "0022-4391", "published_date": "2010-09-14", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.19177/reen.v4e12011133-157", "year": 2011, "genre": "journal-article", "is_oa": true, "title": "Uma Análise da Evolução da Produção Científica sobre Inovação no Brasil", "doi_url": "https://doi.org/10.19177/reen.v4e12011133-157", "updated": "2020-03-13T20:09:55.529364", "oa_status": "gold", "publisher": "Universidade do Sul de Santa Catarina - UNISUL", "z_authors": [{"given": "Jordana Marques", "family": "Kneipp", "sequence": "first"}, {"given": "Luciana Aparecida", "family": "Barbieri da Rosa", "sequence": "additional"}, {"given": "Roberto", "family": "Schoproni Bichueti", "sequence": "additional"}, {"given": "Ana Paula", "family": "Perlin", "sequence": "additional"}, {"given": "Vitor Francisco", "family": "Schuch Júnior", "sequence": "additional"}], "is_paratext": false, "journal_name": "Revista Eletrônica de Estratégia & Negócios", "oa_locations": [{"url": "http://portaldeperiodicos.unisul.br/index.php/EeN/article/download/597/634", "pmh_id": null, "is_best": true, "license": "cc-by-nc-nd", "updated": "2018-05-12T20:57:42.562759", "version": "publishedVersion", "evidence": "open (via page says license)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "http://portaldeperiodicos.unisul.br/index.php/EeN/article/download/597/634", "url_for_landing_page": "https://doi.org/10.19177/reen.v4e12011133-157", "repository_institution": null}, {"url": "https://doi.org/10.19177/reen.v4e12011133-157", "pmh_id": null, "is_best": false, "license": "cc-by-nc-nd", "updated": "2020-04-24T06:25:31.284557", "version": "publishedVersion", "evidence": "oa journal (via doaj)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": null, "url_for_landing_page": "https://doi.org/10.19177/reen.v4e12011133-157", "repository_institution": null}], "data_standard": 2, "journal_is_oa": true, "journal_issns": "1984-3372,1984-3372", "journal_issn_l": "1984-3372", "published_date": "2011-10-21", "best_oa_location": {"url": "http://portaldeperiodicos.unisul.br/index.php/EeN/article/download/597/634", "pmh_id": null, "is_best": true, "license": "cc-by-nc-nd", "updated": "2018-05-12T20:57:42.562759", "version": "publishedVersion", "evidence": "open (via page says license)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "http://portaldeperiodicos.unisul.br/index.php/EeN/article/download/597/634", "url_for_landing_page": "https://doi.org/10.19177/reen.v4e12011133-157", "repository_institution": null}, "journal_is_in_doaj": true, "has_repository_copy": false} +{"doi": "10.3847/1538-4357/aacb27", "year": 2018, "genre": "journal-article", "is_oa": true, "title": "A Magnetic Field Connecting the Galactic Center Circumnuclear Disk with Streamers and Mini-spiral: Implications from 850 μm Polarization Data", "doi_url": "https://doi.org/10.3847/1538-4357/aacb27", "updated": "2020-03-15T03:06:31.646925", "oa_status": "hybrid", "publisher": "American Astronomical Society", "z_authors": [{"ORCID": "http://orcid.org/0000-0001-9155-3978", "given": "Pei-Ying", "family": "Hsieh", "sequence": "first", "authenticated-orcid": false}, {"ORCID": "http://orcid.org/0000-0003-2777-5861", "given": "Patrick M.", "family": "Koch", "sequence": "additional", "authenticated-orcid": false}, {"ORCID": "http://orcid.org/0000-0003-4625-229X", "given": "Woong-Tae", "family": "Kim", "sequence": "additional", "authenticated-orcid": false}, {"given": "Paul T. P.", "family": "Ho", "sequence": "additional"}, {"ORCID": "http://orcid.org/0000-0002-0675-276X", "given": "Ya-Wen", "family": "Tang", "sequence": "additional", "authenticated-orcid": false}, {"ORCID": "http://orcid.org/0000-0001-8436-8803", "given": "Hsiang-Hsu", "family": "Wang", "sequence": "additional", "authenticated-orcid": false}], "is_paratext": false, "journal_name": "The Astrophysical Journal", "oa_locations": [{"url": "https://doi.org/10.3847/1538-4357/aacb27", "pmh_id": null, "is_best": true, "license": "cc-by", "updated": "2020-04-25T14:42:53.593061", "version": "publishedVersion", "evidence": "open (via crossref license)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": null, "url_for_landing_page": "https://doi.org/10.3847/1538-4357/aacb27", "repository_institution": null}, {"url": "http://arxiv.org/pdf/1806.02719", "pmh_id": "oai:arXiv.org:1806.02719", "is_best": false, "license": null, "updated": "2020-04-05T08:57:32.566555", "version": "submittedVersion", "evidence": "oa repository (via OAI-PMH doi match)", "host_type": "repository", "endpoint_id": "ca8f8d56758a80a4f86", "url_for_pdf": "http://arxiv.org/pdf/1806.02719", "url_for_landing_page": "http://arxiv.org/abs/1806.02719", "repository_institution": "Cornell University - arXiv"}], "data_standard": 2, "journal_is_oa": false, "journal_issns": "1538-4357", "journal_issn_l": "0004-637X", "published_date": "2018-08-02", "best_oa_location": {"url": "https://doi.org/10.3847/1538-4357/aacb27", "pmh_id": null, "is_best": true, "license": "cc-by", "updated": "2020-04-25T14:42:53.593061", "version": "publishedVersion", "evidence": "open (via crossref license)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": null, "url_for_landing_page": "https://doi.org/10.3847/1538-4357/aacb27", "repository_institution": null}, "journal_is_in_doaj": false, "has_repository_copy": true} +{"doi": "10.1186/1744-859x-7-s1-s4", "year": 2008, "genre": "journal-article", "is_oa": true, "title": "Structure and function of the therapy department of the Open Psychotherapeutic Center", "doi_url": "https://doi.org/10.1186/1744-859x-7-s1-s4", "updated": "2020-03-21T01:58:55.137980", "oa_status": "gold", "publisher": "Springer Science and Business Media LLC", "z_authors": [{"given": "Dimitris", "family": "Moschonas", "sequence": "first"}], "is_paratext": false, "journal_name": "Annals of General Psychiatry", "oa_locations": [{"url": "https://annals-general-psychiatry.biomedcentral.com/track/pdf/10.1186/1744-859X-7-S1-S4", "pmh_id": null, "is_best": true, "license": null, "updated": "2018-05-28T10:39:08.999070", "version": "publishedVersion", "evidence": "open (via free pdf)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "https://annals-general-psychiatry.biomedcentral.com/track/pdf/10.1186/1744-859X-7-S1-S4", "url_for_landing_page": "https://doi.org/10.1186/1744-859x-7-s1-s4", "repository_institution": null}, {"url": "https://doi.org/10.1186/1744-859x-7-s1-s4", "pmh_id": null, "is_best": false, "license": "cc-by", "updated": "2020-04-25T11:09:14.468648", "version": "publishedVersion", "evidence": "oa journal (via doaj)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": null, "url_for_landing_page": "https://doi.org/10.1186/1744-859x-7-s1-s4", "repository_institution": null}], "data_standard": 2, "journal_is_oa": true, "journal_issns": "1744-859X", "journal_issn_l": "1744-859X", "published_date": "2008-04-01", "best_oa_location": {"url": "https://annals-general-psychiatry.biomedcentral.com/track/pdf/10.1186/1744-859X-7-S1-S4", "pmh_id": null, "is_best": true, "license": null, "updated": "2018-05-28T10:39:08.999070", "version": "publishedVersion", "evidence": "open (via free pdf)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "https://annals-general-psychiatry.biomedcentral.com/track/pdf/10.1186/1744-859X-7-S1-S4", "url_for_landing_page": "https://doi.org/10.1186/1744-859x-7-s1-s4", "repository_institution": null}, "journal_is_in_doaj": true, "has_repository_copy": false} +{"doi": "10.1007/978-3-319-74942-6_8", "year": 2018, "genre": "book-chapter", "is_oa": false, "title": "Molecular Cytology Applications on Gynecological Cytology", "doi_url": "https://doi.org/10.1007/978-3-319-74942-6_8", "updated": "2020-02-04T21:07:34.658661", "oa_status": "closed", "publisher": "Springer International Publishing", "z_authors": [{"given": "Francesca", "family": "Carozzi", "sequence": "first"}, {"given": "Giovanni", "family": "Negri", "sequence": "additional"}, {"given": "Cristina", "family": "Sani", "sequence": "additional"}], "is_paratext": false, "journal_name": "Molecular Applications in Cytology", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": null, "journal_issn_l": null, "published_date": "2018-01-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1007/978-94-015-8601-6_1", "year": 1997, "genre": "book-chapter", "is_oa": false, "title": "Contemporary Philosophy and the Problem of Truth", "doi_url": "https://doi.org/10.1007/978-94-015-8601-6_1", "updated": "2020-03-21T09:15:03.036736", "oa_status": "closed", "publisher": "Springer Netherlands", "z_authors": [{"given": "Jaakko", "family": "Hintikka", "sequence": "first"}], "is_paratext": false, "journal_name": "Lingua Universalis vs. Calculus Ratiocinator", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": null, "journal_issn_l": null, "published_date": "1997-01-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.29081/jesr.v19i4.97", "year": 2017, "genre": "journal-article", "is_oa": true, "title": "MATEMATICAL MODELING OF THE MICROCLIMATE IN BUILDINGS WITH GREEN ROOFS", "doi_url": "https://doi.org/10.29081/jesr.v19i4.97", "updated": "2020-03-23T02:25:25.009697", "oa_status": "gold", "publisher": "Vasile Alecsandri University of Bacau", "z_authors": [{"given": "BORIS", "family": "EVSTAVIEV", "sequence": "first"}, {"given": "KATERINA", "family": "GABROVSKA", "sequence": "additional"}], "is_paratext": false, "journal_name": "Journal of Engineering Studies and Research", "oa_locations": [{"url": "https://doi.org/10.29081/jesr.v19i4.97", "pmh_id": null, "is_best": true, "license": "cc-by", "updated": "2020-04-27T10:17:08.408480", "version": "publishedVersion", "evidence": "oa journal (via doaj)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": null, "url_for_landing_page": "https://doi.org/10.29081/jesr.v19i4.97", "repository_institution": null}, {"url": "http://pdfs.semanticscholar.org/f5b0/570f63b2f679bbf79cf1dbfc6157690b8fb5.pdf", "pmh_id": null, "is_best": false, "license": null, "updated": "2019-10-01T00:00:00", "version": "submittedVersion", "evidence": "oa repository (semantic scholar lookup)", "host_type": "repository", "endpoint_id": null, "url_for_pdf": "http://pdfs.semanticscholar.org/f5b0/570f63b2f679bbf79cf1dbfc6157690b8fb5.pdf", "url_for_landing_page": "https://semanticscholar.org/paper/f5b0570f63b2f679bbf79cf1dbfc6157690b8fb5", "repository_institution": null}], "data_standard": 2, "journal_is_oa": true, "journal_issns": "2068-7559", "journal_issn_l": "2068-7559", "published_date": "2017-03-15", "best_oa_location": {"url": "https://doi.org/10.29081/jesr.v19i4.97", "pmh_id": null, "is_best": true, "license": "cc-by", "updated": "2020-04-27T10:17:08.408480", "version": "publishedVersion", "evidence": "oa journal (via doaj)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": null, "url_for_landing_page": "https://doi.org/10.29081/jesr.v19i4.97", "repository_institution": null}, "journal_is_in_doaj": true, "has_repository_copy": true} +{"doi": "10.1016/j.phymed.2012.02.003", "year": 2012, "genre": "journal-article", "is_oa": false, "title": "Evodiamine, a dual catalytic inhibitor of type I and II topoisomerases, exhibits enhanced inhibition against camptothecin resistant cells", "doi_url": "https://doi.org/10.1016/j.phymed.2012.02.003", "updated": "2020-02-04T14:08:20.345835", "oa_status": "closed", "publisher": "Elsevier BV", "z_authors": [{"given": "Xiaobei", "family": "Pan", "sequence": "first"}, {"given": "Janet M.", "family": "Hartley", "sequence": "additional"}, {"given": "John A.", "family": "Hartley", "sequence": "additional"}, {"given": "Kenneth N.", "family": "White", "sequence": "additional"}, {"given": "Zhengtao", "family": "Wang", "sequence": "additional"}, {"given": "S.W. Annie", "family": "Bligh", "sequence": "additional"}], "is_paratext": false, "journal_name": "Phytomedicine", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0944-7113", "journal_issn_l": "0944-7113", "published_date": "2012-05-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1017/9781108645270.010", "year": null, "genre": "book-chapter", "is_oa": false, "title": "Index", "doi_url": "https://doi.org/10.1017/9781108645270.010", "updated": "2020-02-05T00:03:06.370681", "oa_status": "closed", "publisher": "Cambridge University Press", "z_authors": null, "is_paratext": false, "journal_name": "Russia's Turn to Persia", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": null, "journal_issn_l": null, "published_date": null, "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1109/isscc.2019.8662478", "year": 2019, "genre": "proceedings-article", "is_oa": false, "title": "4.9 A 60GHz CMOS Power Amplifier with Cascaded Asymmetric Distributed-Active-Transformer Achieving Watt-Level Peak Output Power with 20.8% PAE and Supporting 2Gsym/s 64-QAM Modulation", "doi_url": "https://doi.org/10.1109/isscc.2019.8662478", "updated": "2020-02-07T15:50:43.780477", "oa_status": "closed", "publisher": "IEEE", "z_authors": [{"given": "Huy Thong", "family": "Nguyen", "sequence": "first"}, {"given": "Doohwan", "family": "Jung", "sequence": "additional"}, {"given": "Hua", "family": "Wang", "sequence": "additional"}], "is_paratext": false, "journal_name": "2019 IEEE International Solid- State Circuits Conference - (ISSCC)", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": null, "journal_issn_l": null, "published_date": "2019-02-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1587/elex.10.20130108", "year": 2013, "genre": "journal-article", "is_oa": true, "title": "Effect of double-patterning and double-etching on the line-edge-roughness of multi-gate bulk MOSFETs", "doi_url": "https://doi.org/10.1587/elex.10.20130108", "updated": "2020-03-09T20:50:30.215724", "oa_status": "bronze", "publisher": "Institute of Electronics, Information and Communications Engineers (IEICE)", "z_authors": [{"given": "In Jun", "family": "Park", "sequence": "first"}, {"given": "Changhwan", "family": "Shin", "sequence": "additional"}], "is_paratext": false, "journal_name": "IEICE Electronics Express", "oa_locations": [{"url": "https://www.jstage.jst.go.jp/article/elex/10/5/10_10.20130108/_pdf", "pmh_id": null, "is_best": true, "license": null, "updated": "2018-10-20T13:32:22.623811", "version": "publishedVersion", "evidence": "open (via free pdf)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "https://www.jstage.jst.go.jp/article/elex/10/5/10_10.20130108/_pdf", "url_for_landing_page": "https://doi.org/10.1587/elex.10.20130108", "repository_institution": null}, {"url": "http://pdfs.semanticscholar.org/0967/4d80c8a095e135720888f646588a4e4c518b.pdf", "pmh_id": null, "is_best": false, "license": null, "updated": "2019-10-01T00:00:00", "version": "submittedVersion", "evidence": "oa repository (semantic scholar lookup)", "host_type": "repository", "endpoint_id": null, "url_for_pdf": "http://pdfs.semanticscholar.org/0967/4d80c8a095e135720888f646588a4e4c518b.pdf", "url_for_landing_page": "https://semanticscholar.org/paper/09674d80c8a095e135720888f646588a4e4c518b", "repository_institution": null}], "data_standard": 2, "journal_is_oa": false, "journal_issns": "1349-2543", "journal_issn_l": "1349-2543", "published_date": "2013-01-01", "best_oa_location": {"url": "https://www.jstage.jst.go.jp/article/elex/10/5/10_10.20130108/_pdf", "pmh_id": null, "is_best": true, "license": null, "updated": "2018-10-20T13:32:22.623811", "version": "publishedVersion", "evidence": "open (via free pdf)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "https://www.jstage.jst.go.jp/article/elex/10/5/10_10.20130108/_pdf", "url_for_landing_page": "https://doi.org/10.1587/elex.10.20130108", "repository_institution": null}, "journal_is_in_doaj": false, "has_repository_copy": true} +{"doi": "10.4271/910125", "year": 1991, "genre": "proceedings-article", "is_oa": false, "title": "The Calculation of Motorcycle Speeds from Sliding Distances", "doi_url": "https://doi.org/10.4271/910125", "updated": "2020-03-22T11:59:21.305908", "oa_status": "closed", "publisher": "SAE International", "z_authors": [{"given": "Richard F.", "family": "Lambourn", "sequence": "first"}], "is_paratext": false, "journal_name": "SAE Technical Paper Series", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": null, "journal_issn_l": null, "published_date": "1991-02-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1182/blood.v124.21.2256.2256", "year": 2014, "genre": "journal-article", "is_oa": false, "title": "Toxicity of Very Prolonged Pegasparaginase and Erwinia Asparaginase Courses in Relation to Asparaginase Activity Levels with a Special Focus on Dyslipidemia", "doi_url": "https://doi.org/10.1182/blood.v124.21.2256.2256", "updated": "2020-02-04T16:57:28.725514", "oa_status": "closed", "publisher": "American Society of Hematology", "z_authors": [{"given": "Wing H.", "family": "Tong", "sequence": "first", "affiliation": [{"name": "Erasmus MC - Sophia Children's Hospital, Rotterdam, Netherlands"}]}, {"given": "Rob", "family": "Pieters", "sequence": "additional", "affiliation": [{"name": "Dutch Childhood Oncology Group (DCOG; Netherlands), Rotterdam, Netherlands"}, {"name": "Princess Máxima Center for Pediatric Oncology, Utrecht, Netherlands"}]}, {"given": "Hester A.", "family": "de Groot-Kruseman", "sequence": "additional", "affiliation": [{"name": "Dutch Childhood Oncology Group (DCOG), The Hague, Netherlands"}]}, {"given": "Wim C.J.", "family": "Hop", "sequence": "additional", "affiliation": [{"name": "Erasmus MC - University Medical Center, Rotterdam, Netherlands"}]}, {"given": "Joachim", "family": "Boos", "sequence": "additional", "affiliation": [{"name": "University Children's Hospital, Muenster, Germany"}]}, {"given": "Wim J", "family": "Tissing", "sequence": "additional", "affiliation": [{"name": "University Medical Center Groningen, Groningen, Netherlands"}]}, {"given": "Inge M.", "family": "van der Sluis", "sequence": "additional", "affiliation": [{"name": "Erasmus MC - Sophia Children's Hospital, Rotterdam, Netherlands"}]}], "is_paratext": false, "journal_name": "Blood", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0006-4971,1528-0020", "journal_issn_l": "0006-4971", "published_date": "2014-12-06", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.18535/jmscr/v4i11.24", "year": 2016, "genre": "journal-article", "is_oa": true, "title": "Personality Traits in Patients Affected with Flood - A Hospital Based Study", "doi_url": "https://doi.org/10.18535/jmscr/v4i11.24", "updated": "2020-03-18T04:42:29.088697", "oa_status": "gold", "publisher": "Valley International", "z_authors": [{"given": "Junaid", "family": "Nabi", "sequence": "first"}, {"name": "Department of Psychiatry, SKIMS MC, Srinagar", "sequence": "first"}], "is_paratext": false, "journal_name": "Journal of Medical Science And clinical Research", "oa_locations": [{"url": "https://doi.org/10.18535/jmscr/v4i11.24", "pmh_id": null, "is_best": true, "license": null, "updated": "2020-01-19T14:04:13.601407", "version": "publishedVersion", "evidence": "open (via free pdf)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "https://doi.org/10.18535/jmscr/v4i11.24", "url_for_landing_page": null, "repository_institution": null}], "data_standard": 2, "journal_is_oa": true, "journal_issns": "2347-176X", "journal_issn_l": "2347-176X", "published_date": "2016-11-04", "best_oa_location": {"url": "https://doi.org/10.18535/jmscr/v4i11.24", "pmh_id": null, "is_best": true, "license": null, "updated": "2020-01-19T14:04:13.601407", "version": "publishedVersion", "evidence": "open (via free pdf)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "https://doi.org/10.18535/jmscr/v4i11.24", "url_for_landing_page": null, "repository_institution": null}, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.2307/2004084", "year": 1962, "genre": "journal-article", "is_oa": false, "title": "Numerical Methods of Curve Fitting.", "doi_url": "https://doi.org/10.2307/2004084", "updated": "2020-04-14T10:58:17.855143", "oa_status": "closed", "publisher": "JSTOR", "z_authors": [{"given": "N. Donald", "family": "Ylvisaker", "sequence": "first"}, {"given": "P. G.", "family": "Guest", "sequence": "additional"}], "is_paratext": false, "journal_name": "Mathematics of Computation", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0025-5718", "journal_issn_l": "0025-5718", "published_date": "1962-07-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1097/00024382-199911000-00004", "year": 1999, "genre": "journal-article", "is_oa": false, "title": "THE SRC FAMILY-SELECTIVE TYROSINE KINASE INHIBITOR PP1 BLOCKS LPS AND IFN-α-MEDIATED TNF AND iNOS PRODUCTION IN MURINE MACROPHAGES", "doi_url": "https://doi.org/10.1097/00024382-199911000-00004", "updated": "2020-02-07T08:00:15.599938", "oa_status": "closed", "publisher": "Ovid Technologies (Wolters Kluwer Health)", "z_authors": [{"given": "Shari L.", "family": "Orlicek", "sequence": "first"}, {"given": "Jeffrey H.", "family": "Hanke", "sequence": "additional"}, {"given": "B. Keith", "family": "English", "sequence": "additional"}], "is_paratext": false, "journal_name": "Shock", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "1073-2322", "journal_issn_l": "1073-2322", "published_date": "1999-11-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1088/0032-1028/25/12/303", "year": 1983, "genre": "journal-article", "is_oa": false, "title": "On radiative power from impurities in plasmas", "doi_url": "https://doi.org/10.1088/0032-1028/25/12/303", "updated": "2020-03-09T00:34:56.115746", "oa_status": "closed", "publisher": "IOP Publishing", "z_authors": [{"given": "H P", "family": "Summers", "sequence": "first"}, {"given": "M B", "family": "Hooper", "sequence": "additional"}], "is_paratext": false, "journal_name": "Plasma Physics", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0032-1028", "journal_issn_l": "0032-1028", "published_date": "1983-12-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1056/nejm195801232580406", "year": 1958, "genre": "journal-article", "is_oa": false, "title": "Pathways of Carbohydrate Metabolism in Normal and Neoplastic Cells", "doi_url": "https://doi.org/10.1056/nejm195801232580406", "updated": "2020-03-20T21:08:13.850776", "oa_status": "closed", "publisher": "Massachusetts Medical Society", "z_authors": [{"given": "B. L.", "family": "Horecker", "sequence": "first"}, {"given": "Howard H.", "family": "Hiatt", "sequence": "additional"}], "is_paratext": false, "journal_name": "New England Journal of Medicine", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0028-4793,1533-4406", "journal_issn_l": "0028-4793", "published_date": "1958-01-23", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1016/j.healun.2005.11.143", "year": 2006, "genre": "journal-article", "is_oa": false, "title": "137", "doi_url": "https://doi.org/10.1016/j.healun.2005.11.143", "updated": "2020-02-05T09:13:11.146775", "oa_status": "closed", "publisher": "Elsevier BV", "z_authors": [{"given": "M.R.", "family": "Ahsan"}, {"given": "D.T.", "family": "Keating"}, {"given": "R.", "family": "Linehan"}, {"given": "D.", "family": "Healy"}, {"given": "L.", "family": "Nolke"}, {"given": "F.", "family": "Wood"}, {"given": "J.", "family": "McCArthy"}, {"given": "P.", "family": "Doran"}, {"given": "J.J.", "family": "Egan"}], "is_paratext": false, "journal_name": "The Journal of Heart and Lung Transplantation", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "1053-2498", "journal_issn_l": "1053-2498", "published_date": "2006-02-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1590/0102-4698134352", "year": 2015, "genre": "journal-article", "is_oa": true, "title": "A EXPERIÊNCIA ESCOLAR E A SOCIALIZAÇÃO PRÉ-PROFISSIONAL DE PROFESSORES DE EDUCAÇÃO FÍSICA", "doi_url": "https://doi.org/10.1590/0102-4698134352", "updated": "2020-03-15T23:20:37.847520", "oa_status": "gold", "publisher": "FapUNIFESP (SciELO)", "z_authors": [{"given": "José Ângelo", "family": "Gariglio", "sequence": "first", "affiliation": [{"name": "Universidade Federal de Minas Gerais, Brazil"}]}], "is_paratext": false, "journal_name": "Educação em Revista", "oa_locations": [{"url": "http://www.scielo.br/pdf/edur/v31n2/0102-4698-edur-31-02-00229.pdf", "pmh_id": null, "is_best": true, "license": "cc-by-nc", "updated": "2019-02-05T07:19:06.946914", "version": "publishedVersion", "evidence": "open (via page says license)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "http://www.scielo.br/pdf/edur/v31n2/0102-4698-edur-31-02-00229.pdf", "url_for_landing_page": "https://doi.org/10.1590/0102-4698134352", "repository_institution": null}, {"url": "https://doi.org/10.1590/0102-4698134352", "pmh_id": null, "is_best": false, "license": "cc-by-nc", "updated": "2020-04-26T10:33:27.352912", "version": "publishedVersion", "evidence": "oa journal (via doaj)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": null, "url_for_landing_page": "https://doi.org/10.1590/0102-4698134352", "repository_institution": null}], "data_standard": 2, "journal_is_oa": true, "journal_issns": "0102-4698", "journal_issn_l": "0102-4698", "published_date": "2015-06-01", "best_oa_location": {"url": "http://www.scielo.br/pdf/edur/v31n2/0102-4698-edur-31-02-00229.pdf", "pmh_id": null, "is_best": true, "license": "cc-by-nc", "updated": "2019-02-05T07:19:06.946914", "version": "publishedVersion", "evidence": "open (via page says license)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "http://www.scielo.br/pdf/edur/v31n2/0102-4698-edur-31-02-00229.pdf", "url_for_landing_page": "https://doi.org/10.1590/0102-4698134352", "repository_institution": null}, "journal_is_in_doaj": true, "has_repository_copy": false} +{"doi": "10.1007/978-1-62703-324-4_14", "year": 2013, "genre": "book-chapter", "is_oa": false, "title": "Gestational Restless Legs Syndrome", "doi_url": "https://doi.org/10.1007/978-1-62703-324-4_14", "updated": "2020-03-06T22:42:53.571377", "oa_status": "closed", "publisher": "Humana Press", "z_authors": [{"given": "Mari", "family": "Viola-Saltzman", "sequence": "first"}], "is_paratext": false, "journal_name": "Sleep Disorders in Women", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": null, "journal_issn_l": null, "published_date": "2013-01-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1016/b978-0-12-436860-6.50018-1", "year": 1982, "genre": "book-chapter", "is_oa": false, "title": "More on Sclerophylly and Xeromorphy", "doi_url": "https://doi.org/10.1016/b978-0-12-436860-6.50018-1", "updated": "2020-03-02T22:35:58.362682", "oa_status": "closed", "publisher": "Elsevier", "z_authors": [{"given": "James A.", "family": "Larsen", "sequence": "first"}], "is_paratext": false, "journal_name": "Ecology of the Northern Lowland Bogs and Conifer Forests", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": null, "journal_issn_l": null, "published_date": "1982-01-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1101/2019.12.20.881474", "year": 2019, "genre": "posted-content", "is_oa": true, "title": "Inferring the Demographic History of Inbred Species from Genome-Wide SNP Frequency Data", "doi_url": "https://doi.org/10.1101/2019.12.20.881474", "updated": "2020-03-02T06:17:58.061866", "oa_status": "hybrid", "publisher": "Cold Spring Harbor Laboratory", "z_authors": [{"ORCID": "http://orcid.org/0000-0001-9177-8958", "given": "Paul D.", "family": "Blischak", "sequence": "first", "authenticated-orcid": false}, {"ORCID": "http://orcid.org/0000-0001-7173-1319", "given": "Michael S.", "family": "Barker", "sequence": "additional", "authenticated-orcid": false}, {"ORCID": "http://orcid.org/0000-0002-8659-0579", "given": "Ryan N.", "family": "Gutenkunst", "sequence": "additional", "authenticated-orcid": false}], "is_paratext": false, "journal_name": null, "oa_locations": [{"url": "https://www.biorxiv.org/content/biorxiv/early/2019/12/20/2019.12.20.881474.full.pdf", "pmh_id": null, "is_best": true, "license": "cc-by", "updated": "2020-02-25T15:24:14.502903", "version": "publishedVersion", "evidence": "open (via page says license)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "https://www.biorxiv.org/content/biorxiv/early/2019/12/20/2019.12.20.881474.full.pdf", "url_for_landing_page": "https://doi.org/10.1101/2019.12.20.881474", "repository_institution": null}], "data_standard": 2, "journal_is_oa": false, "journal_issns": null, "journal_issn_l": null, "published_date": "2019-12-20", "best_oa_location": {"url": "https://www.biorxiv.org/content/biorxiv/early/2019/12/20/2019.12.20.881474.full.pdf", "pmh_id": null, "is_best": true, "license": "cc-by", "updated": "2020-02-25T15:24:14.502903", "version": "publishedVersion", "evidence": "open (via page says license)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "https://www.biorxiv.org/content/biorxiv/early/2019/12/20/2019.12.20.881474.full.pdf", "url_for_landing_page": "https://doi.org/10.1101/2019.12.20.881474", "repository_institution": null}, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1002/jctb.5000600405", "year": 1941, "genre": "journal-article", "is_oa": false, "title": "Effect of temperature on the exchange capacities of some base-exchange materials used in water softening", "doi_url": "https://doi.org/10.1002/jctb.5000600405", "updated": "2020-03-24T11:32:22.574740", "oa_status": "closed", "publisher": "Wiley", "z_authors": [{"given": "H.", "family": "Ingleson", "sequence": "first"}, {"given": "A.", "family": "Harrison", "sequence": "additional"}], "is_paratext": false, "journal_name": "Journal of the Society of Chemical Industry", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0368-4075,1934-9971", "journal_issn_l": "0368-4075", "published_date": "1941-04-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1016/j.ijhydene.2008.05.025", "year": 2008, "genre": "journal-article", "is_oa": false, "title": "Effects of ethylene on carbon formation in diesel autothermal reforming", "doi_url": "https://doi.org/10.1016/j.ijhydene.2008.05.025", "updated": "2020-02-29T17:57:49.257884", "oa_status": "closed", "publisher": "Elsevier BV", "z_authors": [{"given": "Sangho", "family": "Yoon", "sequence": "first"}, {"given": "Inyong", "family": "Kang", "sequence": "additional"}, {"given": "Joongmyeon", "family": "Bae", "sequence": "additional"}], "is_paratext": false, "journal_name": "International Journal of Hydrogen Energy", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0360-3199", "journal_issn_l": "0360-3199", "published_date": "2008-09-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.2307/2033829", "year": 1963, "genre": "journal-article", "is_oa": true, "title": "Rapidly Increasing Kernels", "doi_url": "https://doi.org/10.2307/2033829", "updated": "2020-02-08T16:03:34.179453", "oa_status": "bronze", "publisher": "JSTOR", "z_authors": [{"given": "Harold", "family": "Widom"}], "is_paratext": false, "journal_name": "Proceedings of the American Mathematical Society", "oa_locations": [{"url": "https://www.ams.org/proc/1963-014-03/S0002-9939-1963-0151806-5/S0002-9939-1963-0151806-5.pdf", "pmh_id": null, "is_best": true, "license": null, "updated": "2019-09-18T14:21:09.631891", "version": "publishedVersion", "evidence": "open (via free pdf)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "https://www.ams.org/proc/1963-014-03/S0002-9939-1963-0151806-5/S0002-9939-1963-0151806-5.pdf", "url_for_landing_page": "https://doi.org/10.2307/2033829", "repository_institution": null}], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0002-9939", "journal_issn_l": "0002-9939", "published_date": "1963-06-01", "best_oa_location": {"url": "https://www.ams.org/proc/1963-014-03/S0002-9939-1963-0151806-5/S0002-9939-1963-0151806-5.pdf", "pmh_id": null, "is_best": true, "license": null, "updated": "2019-09-18T14:21:09.631891", "version": "publishedVersion", "evidence": "open (via free pdf)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "https://www.ams.org/proc/1963-014-03/S0002-9939-1963-0151806-5/S0002-9939-1963-0151806-5.pdf", "url_for_landing_page": "https://doi.org/10.2307/2033829", "repository_institution": null}, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1016/s0894-7317(96)90227-8", "year": 1996, "genre": "journal-article", "is_oa": false, "title": "Ultrasonic tissue characterization: Applicability and sensitivity of detection of serial changes in the myocardium", "doi_url": "https://doi.org/10.1016/s0894-7317(96)90227-8", "updated": "2020-02-05T21:32:37.455113", "oa_status": "closed", "publisher": "Elsevier BV", "z_authors": null, "is_paratext": false, "journal_name": "Journal of the American Society of Echocardiography", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0894-7317", "journal_issn_l": "0894-7317", "published_date": "1996-05-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1016/j.chroma.2016.12.078", "year": 2017, "genre": "journal-article", "is_oa": false, "title": "Microfluidic membrane suppressor module design and evaluation for capillary ion chromatography", "doi_url": "https://doi.org/10.1016/j.chroma.2016.12.078", "updated": "2020-03-15T23:21:03.082031", "oa_status": "closed", "publisher": "Elsevier BV", "z_authors": [{"given": "Sam", "family": "Wouters", "sequence": "first"}, {"given": "Cees", "family": "Bruggink", "sequence": "additional"}, {"given": "Yury", "family": "Agroskin", "sequence": "additional"}, {"given": "Christopher", "family": "Pohl", "sequence": "additional"}, {"given": "Sebastiaan", "family": "Eeltink", "sequence": "additional"}], "is_paratext": false, "journal_name": "Journal of Chromatography A", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0021-9673", "journal_issn_l": "0021-9673", "published_date": "2017-02-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.22456/1984-1191.9325", "year": 2008, "genre": "journal-article", "is_oa": true, "title": "A desterritorialização dos saberes e fazeres antropológicos e o desentendimento no corpo de verdade da letra", "doi_url": "https://doi.org/10.22456/1984-1191.9325", "updated": "2020-02-04T21:07:23.738645", "oa_status": "bronze", "publisher": "Universidade Federal do Rio Grande do Sul", "z_authors": [{"given": "Priscila Farfan", "family": "Barroso", "sequence": "first"}, {"given": "Rafael", "family": "Lopo", "sequence": "additional"}, {"given": "Ana Luiza Carvalho da", "family": "Rocha", "sequence": "additional"}, {"given": "Viviane", "family": "Vedana", "sequence": "additional"}], "is_paratext": false, "journal_name": "ILUMINURAS", "oa_locations": [{"url": "https://seer.ufrgs.br/iluminuras/article/download/9325/5393", "pmh_id": null, "is_best": true, "license": null, "updated": "2019-06-12T10:56:48.093740", "version": "publishedVersion", "evidence": "open (via free pdf)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "https://seer.ufrgs.br/iluminuras/article/download/9325/5393", "url_for_landing_page": "https://doi.org/10.22456/1984-1191.9325", "repository_institution": null}, {"url": "http://pdfs.semanticscholar.org/3fa9/e9fc7244b54232e580ea34f049cd52da6f58.pdf", "pmh_id": null, "is_best": false, "license": null, "updated": "2019-10-01T00:00:00", "version": "submittedVersion", "evidence": "oa repository (semantic scholar lookup)", "host_type": "repository", "endpoint_id": null, "url_for_pdf": "http://pdfs.semanticscholar.org/3fa9/e9fc7244b54232e580ea34f049cd52da6f58.pdf", "url_for_landing_page": "https://semanticscholar.org/paper/3fa9e9fc7244b54232e580ea34f049cd52da6f58", "repository_institution": null}], "data_standard": 2, "journal_is_oa": false, "journal_issns": "1984-1191", "journal_issn_l": "1984-1191", "published_date": "2008-12-03", "best_oa_location": {"url": "https://seer.ufrgs.br/iluminuras/article/download/9325/5393", "pmh_id": null, "is_best": true, "license": null, "updated": "2019-06-12T10:56:48.093740", "version": "publishedVersion", "evidence": "open (via free pdf)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "https://seer.ufrgs.br/iluminuras/article/download/9325/5393", "url_for_landing_page": "https://doi.org/10.22456/1984-1191.9325", "repository_institution": null}, "journal_is_in_doaj": false, "has_repository_copy": true} +{"doi": "10.1601/nm.1454", "year": 2003, "genre": "dataset", "is_oa": false, "title": "Nomenclature Abstract for Methylocella.", "doi_url": "https://doi.org/10.1601/nm.1454", "updated": "2020-02-27T06:48:10.755838", "oa_status": "closed", "publisher": "NamesforLife, LLC", "z_authors": [{"ORCID": "http://orcid.org/0000-0002-7436-3176", "given": "Charles Thomas", "family": "Parker", "suffix": "Jr", "sequence": "first", "affiliation": [{"name": "NamesforLife, LLC"}], "authenticated-orcid": false}, {"ORCID": "http://orcid.org/0000-0001-6386-9107", "given": "Sarah", "family": "Wigley", "sequence": "additional", "affiliation": [{"name": "NamesforLife, LLC"}], "authenticated-orcid": false}, {"ORCID": "http://orcid.org/0000-0002-4465-7034", "given": "George M", "family": "Garrity", "sequence": "additional", "affiliation": [{"name": "NamesforLife, LLC"}], "authenticated-orcid": false}], "is_paratext": false, "journal_name": "The NamesforLife Abstracts", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": null, "journal_issn_l": null, "published_date": "2003-01-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.3897/zookeys.617.9850.figures20-29", "year": null, "genre": "component", "is_oa": true, "title": "Figures 20–29 from: Skuhravá M, Massa B, Cerasa G (2016) Rediscovery and identity of Pumilomyia protrahenda De Stefani (Diptera, Cecidomyiidae) in Sicily with redescription and reassessment of its taxonomic position. ZooKeys 617: 129-137. https://doi.org/10.3897/zookeys.617.9850", "doi_url": "https://doi.org/10.3897/zookeys.617.9850.figures20-29", "updated": "2020-02-04T18:56:04.431773", "oa_status": "bronze", "publisher": "Pensoft Publishers", "z_authors": null, "is_paratext": false, "journal_name": null, "oa_locations": [{"url": "https://zookeys.pensoft.net/article/9850/download/pdf/", "pmh_id": null, "is_best": true, "license": null, "updated": "2019-01-22T23:56:51.024092", "version": "publishedVersion", "evidence": "open (via free pdf)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "https://zookeys.pensoft.net/article/9850/download/pdf/", "url_for_landing_page": "https://doi.org/10.3897/zookeys.617.9850.figures20-29", "repository_institution": null}], "data_standard": 2, "journal_is_oa": false, "journal_issns": null, "journal_issn_l": null, "published_date": null, "best_oa_location": {"url": "https://zookeys.pensoft.net/article/9850/download/pdf/", "pmh_id": null, "is_best": true, "license": null, "updated": "2019-01-22T23:56:51.024092", "version": "publishedVersion", "evidence": "open (via free pdf)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "https://zookeys.pensoft.net/article/9850/download/pdf/", "url_for_landing_page": "https://doi.org/10.3897/zookeys.617.9850.figures20-29", "repository_institution": null}, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1007/bf03295311", "year": 1988, "genre": "journal-article", "is_oa": false, "title": "Digoxin-specific antibodies in digitalis toxicity", "doi_url": "https://doi.org/10.1007/bf03295311", "updated": "2020-03-08T01:06:18.447104", "oa_status": "closed", "publisher": "Springer Science and Business Media LLC", "z_authors": null, "is_paratext": false, "journal_name": "Reactions", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0157-7271", "journal_issn_l": "0157-7271", "published_date": "1988-07-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.2524/jtappij.52.1707", "year": 1998, "genre": "journal-article", "is_oa": true, "title": "Wet End Starches for Paper Making.", "doi_url": "https://doi.org/10.2524/jtappij.52.1707", "updated": "2020-03-20T01:02:32.720421", "oa_status": "bronze", "publisher": "Japan Technical Association of the Pulp and Paper Industry", "z_authors": [{"given": "Atsushi", "family": "Sakakibara", "sequence": "first"}], "is_paratext": false, "journal_name": "JAPAN TAPPI JOURNAL", "oa_locations": [{"url": "https://www.jstage.jst.go.jp/article/jtappij1955/52/12/52_12_1707/_pdf", "pmh_id": null, "is_best": true, "license": null, "updated": "2018-07-29T08:16:30.437455", "version": "publishedVersion", "evidence": "open (via free pdf)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "https://www.jstage.jst.go.jp/article/jtappij1955/52/12/52_12_1707/_pdf", "url_for_landing_page": "https://doi.org/10.2524/jtappij.52.1707", "repository_institution": null}], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0022-815X,1881-1000", "journal_issn_l": "0022-815X", "published_date": "1998-01-01", "best_oa_location": {"url": "https://www.jstage.jst.go.jp/article/jtappij1955/52/12/52_12_1707/_pdf", "pmh_id": null, "is_best": true, "license": null, "updated": "2018-07-29T08:16:30.437455", "version": "publishedVersion", "evidence": "open (via free pdf)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "https://www.jstage.jst.go.jp/article/jtappij1955/52/12/52_12_1707/_pdf", "url_for_landing_page": "https://doi.org/10.2524/jtappij.52.1707", "repository_institution": null}, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.5585/cpg.v8n0.2099", "year": 2010, "genre": "journal-article", "is_oa": true, "title": "Cooperativas como forma gestora de cursos menos onerosos no ensino superior privado", "doi_url": "https://doi.org/10.5585/cpg.v8n0.2099", "updated": "2020-02-07T02:40:32.077936", "oa_status": "gold", "publisher": "University Nove de Julho", "z_authors": [{"given": "Sérgio Macedo", "family": "Oliveira", "sequence": "first"}], "is_paratext": false, "journal_name": "Cadernos de Pós-graduação", "oa_locations": [{"url": "http://periodicos.uninove.br/index.php?journal=cadernosdepos&page=article&op=download&path%5B%5D=2099&path%5B%5D=1576", "pmh_id": null, "is_best": true, "license": "cc-by-nc", "updated": "2018-08-21T21:01:53.801808", "version": "publishedVersion", "evidence": "open (via page says license)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "http://periodicos.uninove.br/index.php?journal=cadernosdepos&page=article&op=download&path%5B%5D=2099&path%5B%5D=1576", "url_for_landing_page": "https://doi.org/10.5585/cpg.v8n0.2099", "repository_institution": null}, {"url": "https://doi.org/10.5585/cpg.v8n0.2099", "pmh_id": null, "is_best": false, "license": "cc-by-nc", "updated": "2020-04-24T14:47:48.737146", "version": "publishedVersion", "evidence": "oa journal (via doaj)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": null, "url_for_landing_page": "https://doi.org/10.5585/cpg.v8n0.2099", "repository_institution": null}, {"url": "http://pdfs.semanticscholar.org/8f6b/c7e1c59d33b3a18742378100e5153368dd14.pdf", "pmh_id": null, "is_best": false, "license": null, "updated": "2019-10-01T00:00:00", "version": "submittedVersion", "evidence": "oa repository (semantic scholar lookup)", "host_type": "repository", "endpoint_id": null, "url_for_pdf": "http://pdfs.semanticscholar.org/8f6b/c7e1c59d33b3a18742378100e5153368dd14.pdf", "url_for_landing_page": "https://semanticscholar.org/paper/8f6bc7e1c59d33b3a18742378100e5153368dd14", "repository_institution": null}], "data_standard": 2, "journal_is_oa": true, "journal_issns": "2525-3514", "journal_issn_l": "1678-4103", "published_date": "2010-01-22", "best_oa_location": {"url": "http://periodicos.uninove.br/index.php?journal=cadernosdepos&page=article&op=download&path%5B%5D=2099&path%5B%5D=1576", "pmh_id": null, "is_best": true, "license": "cc-by-nc", "updated": "2018-08-21T21:01:53.801808", "version": "publishedVersion", "evidence": "open (via page says license)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "http://periodicos.uninove.br/index.php?journal=cadernosdepos&page=article&op=download&path%5B%5D=2099&path%5B%5D=1576", "url_for_landing_page": "https://doi.org/10.5585/cpg.v8n0.2099", "repository_institution": null}, "journal_is_in_doaj": true, "has_repository_copy": true} +{"doi": "10.1016/j.bpj.2019.11.2723", "year": 2020, "genre": "journal-article", "is_oa": false, "title": "The Stickers and Spacers Framework for Describing Phase Behavior of Multivalent Intrinsically Disordered Proteins", "doi_url": "https://doi.org/10.1016/j.bpj.2019.11.2723", "updated": "2020-02-07T21:05:29.508395", "oa_status": "closed", "publisher": "Elsevier BV", "z_authors": [{"given": "Jeong-Mo", "family": "Choi", "sequence": "first"}, {"given": "Rohit V.", "family": "Pappu", "sequence": "additional"}], "is_paratext": false, "journal_name": "Biophysical Journal", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0006-3495", "journal_issn_l": "0006-3495", "published_date": "2020-02-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1128/9781555816728.ch100", "year": 2011, "genre": "book-chapter", "is_oa": false, "title": "Human Herpesviruses 6, 7, and 8", "doi_url": "https://doi.org/10.1128/9781555816728.ch100", "updated": "2020-03-04T20:28:22.992053", "oa_status": "closed", "publisher": "American Society of Microbiology", "z_authors": null, "is_paratext": false, "journal_name": "Manual of Clinical Microbiology, 10th Edition", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": null, "journal_issn_l": null, "published_date": "2011-01-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1182/blood.v116.21.3313.3313", "year": 2010, "genre": "journal-article", "is_oa": false, "title": "Temozolomide In Acute Myeloid Leukemia: A MGMT Promoter Methylation Status–Based Treatment Stratification", "doi_url": "https://doi.org/10.1182/blood.v116.21.3313.3313", "updated": "2020-02-04T16:58:20.872245", "oa_status": "closed", "publisher": "American Society of Hematology", "z_authors": [{"given": "Bruno C.", "family": "Medeiros", "sequence": "first", "affiliation": [{"name": "Stanford University School of Medicine, Stanford, CA, USA,"}]}, {"given": "Holbrook E", "family": "Kohrt", "sequence": "additional", "affiliation": [{"name": "Hematology, Stanford University School of Medicine, Stanford, CA, USA,"}]}, {"given": "Richa", "family": "Rajwanshi", "sequence": "additional", "affiliation": [{"name": "Stanford U School of Medicine, Stanford, CA,"}]}, {"given": "Jason", "family": "Gotlib", "sequence": "additional", "affiliation": [{"name": "Division of Hematology, Stanford University School of Medicine, Stanford, CA, USA,"}]}, {"given": "Steven", "family": "Coutre", "sequence": "additional", "affiliation": [{"name": "Stanford University School of Medicine, Stanford, CA,"}]}, {"given": "Michaela", "family": "Liedtke", "sequence": "additional", "affiliation": [{"name": "Division of Hematology, Stanford University School of Medicine, Stanford, CA, USA,"}]}, {"given": "Caroline", "family": "Berube", "sequence": "additional", "affiliation": [{"name": "Hematology, Stanford University School of Medicine, Stanford, CA, USA,"}]}, {"given": "Melody", "family": "Zhang", "sequence": "additional", "affiliation": [{"name": "Division of Hematology, Stanford University School of Medicine, Stanford, CA, USA,"}]}, {"given": "Daniel A.", "family": "Arber", "sequence": "additional", "affiliation": [{"name": "Dept. of Clinical Laboratory, Stanford Univ. Medical Center, Stanford, CA, USA,"}]}, {"given": "James L.", "family": "Zehnder", "sequence": "additional", "affiliation": [{"name": "Department of Pathology, Stanford University School of Medicine, Stanford, CA, USA"}]}], "is_paratext": false, "journal_name": "Blood", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0006-4971,1528-0020", "journal_issn_l": "0006-4971", "published_date": "2010-11-19", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1680/macr.1953.5.13.1", "year": 1953, "genre": "journal-article", "is_oa": true, "title": "Editorial comment", "doi_url": "https://doi.org/10.1680/macr.1953.5.13.1", "updated": "2020-02-09T01:06:04.670147", "oa_status": "bronze", "publisher": "Thomas Telford Ltd.", "z_authors": null, "is_paratext": false, "journal_name": "Magazine of Concrete Research", "oa_locations": [{"url": "https://www.icevirtuallibrary.com/doi/pdf/10.1680/macr.1953.5.13.1", "pmh_id": null, "is_best": true, "license": null, "updated": "2019-01-20T23:32:12.804739", "version": "publishedVersion", "evidence": "open (via free pdf)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "https://www.icevirtuallibrary.com/doi/pdf/10.1680/macr.1953.5.13.1", "url_for_landing_page": "https://doi.org/10.1680/macr.1953.5.13.1", "repository_institution": null}], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0024-9831,1751-763X", "journal_issn_l": "0024-9831", "published_date": "1953-08-01", "best_oa_location": {"url": "https://www.icevirtuallibrary.com/doi/pdf/10.1680/macr.1953.5.13.1", "pmh_id": null, "is_best": true, "license": null, "updated": "2019-01-20T23:32:12.804739", "version": "publishedVersion", "evidence": "open (via free pdf)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "https://www.icevirtuallibrary.com/doi/pdf/10.1680/macr.1953.5.13.1", "url_for_landing_page": "https://doi.org/10.1680/macr.1953.5.13.1", "repository_institution": null}, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.26502/jppd.2572-519x0089", "year": 2020, "genre": "journal-article", "is_oa": false, "title": "Mick Jagger, Capitalism and the Pursuit of Happiness", "doi_url": "https://doi.org/10.26502/jppd.2572-519x0089", "updated": "2020-04-04T21:02:13.395420", "oa_status": "closed", "publisher": "Fortune Journals", "z_authors": [{"given": "David L", "family": "Ryan MD", "sequence": "first"}], "is_paratext": false, "journal_name": "Journal of Psychiatry and Psychiatric Disorders", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "2572-519X", "journal_issn_l": "2572-519X", "published_date": "2020-01-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1149/1.2096387", "year": 1989, "genre": "journal-article", "is_oa": false, "title": "Deposition Profile Simulation Using the Direct Simulation Monte Carlo Method", "doi_url": "https://doi.org/10.1149/1.2096387", "updated": "2020-03-04T09:34:29.488606", "oa_status": "closed", "publisher": "The Electrochemical Society", "z_authors": [{"given": "Masato", "family": "Ikegawa", "sequence": "first"}], "is_paratext": false, "journal_name": "Journal of The Electrochemical Society", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0013-4651", "journal_issn_l": "0013-4651", "published_date": "1989-01-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.20472/iac.2018.040.048", "year": 2018, "genre": "proceedings-article", "is_oa": true, "title": "USING IMPORTANCE-PERFORMANCE ANALYSIS (IPA) TO EVALUATE FACTORS AFFECTING THE LIVING IN ELDERLY CONDOMINIUM", "doi_url": "https://doi.org/10.20472/iac.2018.040.048", "updated": "2020-02-05T00:02:24.990581", "oa_status": "bronze", "publisher": "International Institute of Social and Economic Sciences", "z_authors": [{"given": "Supeecha", "family": "Panichpathom", "sequence": "first"}, {"given": "Yongyuth", "family": "Suphotgamkul", "sequence": "additional"}], "is_paratext": false, "journal_name": "Proceedings of the 40th International Academic Conference, Stockholm", "oa_locations": [{"url": "http://iises.net/proceedings/40th-international-academic-conference-stockholm/table-of-content?cid=65&iid=048&rid=8956", "pmh_id": null, "is_best": true, "license": null, "updated": "2019-10-20T11:17:47.678931", "version": "publishedVersion", "evidence": "open (via free pdf)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "http://iises.net/proceedings/40th-international-academic-conference-stockholm/table-of-content?cid=65&iid=048&rid=8956", "url_for_landing_page": "https://doi.org/10.20472/iac.2018.040.048", "repository_institution": null}], "data_standard": 2, "journal_is_oa": false, "journal_issns": null, "journal_issn_l": null, "published_date": "2018-01-01", "best_oa_location": {"url": "http://iises.net/proceedings/40th-international-academic-conference-stockholm/table-of-content?cid=65&iid=048&rid=8956", "pmh_id": null, "is_best": true, "license": null, "updated": "2019-10-20T11:17:47.678931", "version": "publishedVersion", "evidence": "open (via free pdf)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "http://iises.net/proceedings/40th-international-academic-conference-stockholm/table-of-content?cid=65&iid=048&rid=8956", "url_for_landing_page": "https://doi.org/10.20472/iac.2018.040.048", "repository_institution": null}, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.4157/grj1984b.69.21", "year": 1996, "genre": "journal-article", "is_oa": true, "title": "Studies in the Historical Geography of Japan, 1988-1995", "doi_url": "https://doi.org/10.4157/grj1984b.69.21", "updated": "2020-03-01T19:51:39.763390", "oa_status": "bronze", "publisher": "The Association of Japanese Geographers", "z_authors": [{"given": "Tsunetoshl", "family": "MIZOGUCHI", "sequence": "first"}], "is_paratext": false, "journal_name": "Geographical review of Japan, Series B.", "oa_locations": [{"url": "https://www.jstage.jst.go.jp/article/grj1984b/69/1/69_1_21/_pdf", "pmh_id": null, "is_best": true, "license": null, "updated": "2018-12-18T07:51:31.642606", "version": "publishedVersion", "evidence": "open (via free pdf)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "https://www.jstage.jst.go.jp/article/grj1984b/69/1/69_1_21/_pdf", "url_for_landing_page": "https://doi.org/10.4157/grj1984b.69.21", "repository_institution": null}], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0289-6001,2185-1700", "journal_issn_l": "0289-6001", "published_date": "1996-01-01", "best_oa_location": {"url": "https://www.jstage.jst.go.jp/article/grj1984b/69/1/69_1_21/_pdf", "pmh_id": null, "is_best": true, "license": null, "updated": "2018-12-18T07:51:31.642606", "version": "publishedVersion", "evidence": "open (via free pdf)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "https://www.jstage.jst.go.jp/article/grj1984b/69/1/69_1_21/_pdf", "url_for_landing_page": "https://doi.org/10.4157/grj1984b.69.21", "repository_institution": null}, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.5301/tj.5000521", "year": 2016, "genre": "journal-article", "is_oa": false, "title": "A case of pseudoachalasia hiding a malignant pleural mesothelioma", "doi_url": "https://doi.org/10.5301/tj.5000521", "updated": "2020-03-13T20:09:57.872436", "oa_status": "closed", "publisher": "SAGE Publications", "z_authors": [{"given": "Federica", "family": "Branchi", "sequence": "first", "affiliation": [{"name": "Department of Pathophysiology and Transplantation, Università degli Studi di Milano, Milan - Italy"}, {"name": "Gastroenterology and Endoscopy Unit, Fondazione IRCCS Ca’ Granda - Ospedale Maggiore Policlinico, Milan - Italy"}]}, {"given": "Andrea", "family": "Tenca", "sequence": "additional", "affiliation": [{"name": "Department of Pathophysiology and Transplantation, Università degli Studi di Milano, Milan - Italy"}, {"name": "Gastroenterology and Endoscopy Unit, Fondazione IRCCS Ca’ Granda - Ospedale Maggiore Policlinico, Milan - Italy"}]}, {"given": "Claudia", "family": "Bareggi", "sequence": "additional", "affiliation": [{"name": "Oncology Unit, Fondazione IRCCS Ca’ Granda - Ospedale Maggiore Policlinico, Milan - Italy"}]}, {"given": "Carolina", "family": "Mensi", "sequence": "additional", "affiliation": [{"name": "Epidemiology Unit, Fondazione IRCCS Ca’ Granda - Ospedale Maggiore Policlinico, Milan - Italy"}]}, {"given": "Aurelio", "family": "Mauro", "sequence": "additional", "affiliation": [{"name": "Department of Pathophysiology and Transplantation, Università degli Studi di Milano, Milan - Italy"}, {"name": "Gastroenterology and Endoscopy Unit, Fondazione IRCCS Ca’ Granda - Ospedale Maggiore Policlinico, Milan - Italy"}]}, {"given": "Dario", "family": "Conte", "sequence": "additional", "affiliation": [{"name": "Department of Pathophysiology and Transplantation, Università degli Studi di Milano, Milan - Italy"}, {"name": "Gastroenterology and Endoscopy Unit, Fondazione IRCCS Ca’ Granda - Ospedale Maggiore Policlinico, Milan - Italy"}]}, {"given": "Roberto", "family": "Penagini", "sequence": "additional", "affiliation": [{"name": "Department of Pathophysiology and Transplantation, Università degli Studi di Milano, Milan - Italy"}, {"name": "Gastroenterology and Endoscopy Unit, Fondazione IRCCS Ca’ Granda - Ospedale Maggiore Policlinico, Milan - Italy"}]}], "is_paratext": false, "journal_name": "Tumori Journal", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0300-8916,2038-2529", "journal_issn_l": "0300-8916", "published_date": "2016-11-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1097/pec.0b013e3182072458", "year": 2010, "genre": "journal-article", "is_oa": false, "title": "ECGs in the ED", "doi_url": "https://doi.org/10.1097/pec.0b013e3182072458", "updated": "2020-02-06T12:10:53.918344", "oa_status": "closed", "publisher": "Ovid Technologies (Wolters Kluwer Health)", "z_authors": [{"given": "Ronn E.", "family": "Tanel", "sequence": "first"}], "is_paratext": false, "journal_name": "Pediatric Emergency Care", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0749-5161", "journal_issn_l": "0749-5161", "published_date": "2010-12-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.5840/iabsproc20011256", "year": 2001, "genre": "journal-article", "is_oa": false, "title": "Sustainable Leaming and American Liberal Arts Education", "doi_url": "https://doi.org/10.5840/iabsproc20011256", "updated": "2020-02-07T02:41:46.126585", "oa_status": "closed", "publisher": "Philosophy Documentation Center", "z_authors": [{"given": "Daniel R.", "family": "Gilbert", "sequence": "first"}, {"name": "International Association for Business and Society", "sequence": "additional"}], "is_paratext": false, "journal_name": "Proceedings of the International Association for Business and Society", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "2155-2568", "journal_issn_l": "2155-2568", "published_date": "2001-01-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.4103/1319-2442.118094", "year": 2013, "genre": "journal-article", "is_oa": true, "title": "Incidence and clinical outcome of renal amyloidosis: A retrospective study", "doi_url": "https://doi.org/10.4103/1319-2442.118094", "updated": "2020-03-05T23:59:03.463335", "oa_status": "gold", "publisher": "Medknow", "z_authors": [{"given": "Emad", "family": "Abdallah", "sequence": "first"}, {"given": "Emam", "family": "Waked", "sequence": "additional"}], "is_paratext": false, "journal_name": "Saudi Journal of Kidney Diseases and Transplantation", "oa_locations": [{"url": "https://doi.org/10.4103/1319-2442.118094", "pmh_id": null, "is_best": true, "license": "cc-by-nc-sa", "updated": "2020-04-22T10:43:51.612077", "version": "publishedVersion", "evidence": "oa journal (via doaj)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": null, "url_for_landing_page": "https://doi.org/10.4103/1319-2442.118094", "repository_institution": null}], "data_standard": 2, "journal_is_oa": true, "journal_issns": "1319-2442", "journal_issn_l": "1319-2442", "published_date": "2013-01-01", "best_oa_location": {"url": "https://doi.org/10.4103/1319-2442.118094", "pmh_id": null, "is_best": true, "license": "cc-by-nc-sa", "updated": "2020-04-22T10:43:51.612077", "version": "publishedVersion", "evidence": "oa journal (via doaj)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": null, "url_for_landing_page": "https://doi.org/10.4103/1319-2442.118094", "repository_institution": null}, "journal_is_in_doaj": true, "has_repository_copy": false} +{"doi": "10.1299/kikai1938.35.936", "year": 1969, "genre": "journal-article", "is_oa": true, "title": "Investigation on Low Cycle Fatigue at Elevated Temperature : 3rd Report Influence of Temperature and Plastic Strain Range on the Behavior of Fatigue Cracks", "doi_url": "https://doi.org/10.1299/kikai1938.35.936", "updated": "2020-03-20T19:56:33.522813", "oa_status": "bronze", "publisher": "Japan Society of Mechanical Engineers", "z_authors": [{"given": "Kenji", "family": "KANAZAWA", "sequence": "first"}, {"given": "Yukio", "family": "NISHIMURA", "sequence": "additional"}, {"given": "Takeshi", "family": "KNIO", "sequence": "additional"}, {"given": "Kanetoshi", "family": "IWAMOTO", "sequence": "additional"}, {"given": "Teruyuki", "family": "UEDA", "sequence": "additional"}], "is_paratext": false, "journal_name": "Transactions of the Japan Society of Mechanical Engineers", "oa_locations": [{"url": "https://www.jstage.jst.go.jp/article/kikai1938/35/273/35_273_936/_pdf", "pmh_id": null, "is_best": true, "license": null, "updated": "2018-07-26T19:53:57.351085", "version": "publishedVersion", "evidence": "open (via free pdf)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "https://www.jstage.jst.go.jp/article/kikai1938/35/273/35_273_936/_pdf", "url_for_landing_page": "https://doi.org/10.1299/kikai1938.35.936", "repository_institution": null}], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0029-0270,2185-9485", "journal_issn_l": "0029-0270", "published_date": "1969-01-01", "best_oa_location": {"url": "https://www.jstage.jst.go.jp/article/kikai1938/35/273/35_273_936/_pdf", "pmh_id": null, "is_best": true, "license": null, "updated": "2018-07-26T19:53:57.351085", "version": "publishedVersion", "evidence": "open (via free pdf)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "https://www.jstage.jst.go.jp/article/kikai1938/35/273/35_273_936/_pdf", "url_for_landing_page": "https://doi.org/10.1299/kikai1938.35.936", "repository_institution": null}, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1136/jnnp.2008.167494", "year": 2010, "genre": "journal-article", "is_oa": false, "title": "Central apnoea associated with subcortical haemorrhage in the left temporal lobe", "doi_url": "https://doi.org/10.1136/jnnp.2008.167494", "updated": "2020-03-22T18:47:52.542640", "oa_status": "closed", "publisher": "BMJ", "z_authors": [{"given": "T.", "family": "Nezu", "sequence": "first"}, {"given": "M.", "family": "Naganuma", "sequence": "additional"}, {"given": "Y.", "family": "Shono", "sequence": "additional"}, {"given": "K.", "family": "Toyoda", "sequence": "additional"}, {"given": "K.", "family": "Minematsu", "sequence": "additional"}], "is_paratext": false, "journal_name": "Journal of Neurology, Neurosurgery & Psychiatry", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0022-3050", "journal_issn_l": "0022-3050", "published_date": "2010-02-25", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1016/s0950-3536(05)80269-0", "year": 1995, "genre": "journal-article", "is_oa": false, "title": "2 Plasminogen activators and plasminogen activator inhibitors: biochemical aspects", "doi_url": "https://doi.org/10.1016/s0950-3536(05)80269-0", "updated": "2020-03-25T02:09:34.927917", "oa_status": "closed", "publisher": "Elsevier BV", "z_authors": [{"given": "Dingeman C.", "family": "Rijken", "sequence": "first"}], "is_paratext": false, "journal_name": "Baillière's Clinical Haematology", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0950-3536", "journal_issn_l": "0950-3536", "published_date": "1995-06-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1139/o55-117", "year": 1955, "genre": "journal-article", "is_oa": false, "title": "ON BODY FAT AND BODY WATER IN RATS", "doi_url": "https://doi.org/10.1139/o55-117", "updated": "2020-02-28T00:00:10.798997", "oa_status": "closed", "publisher": "Canadian Science Publishing", "z_authors": [{"given": "Louis-Marie", "family": "Babineau", "sequence": "first"}, {"given": "Edouard", "family": "Pagé", "sequence": "additional"}], "is_paratext": false, "journal_name": "Canadian Journal of Biochemistry and Physiology", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0576-5544", "journal_issn_l": "0576-5544", "published_date": "1955-11-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.2307/2033958", "year": 1963, "genre": "journal-article", "is_oa": false, "title": "Invariant Subspaces in L 1", "doi_url": "https://doi.org/10.2307/2033958", "updated": "2020-04-14T10:58:20.125191", "oa_status": "closed", "publisher": "JSTOR", "z_authors": [{"given": "Frank", "family": "Forelli", "sequence": "first"}], "is_paratext": false, "journal_name": "Proceedings of the American Mathematical Society", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0002-9939", "journal_issn_l": "0002-9939", "published_date": "1963-02-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.4287/jsprs1962.13.4_8", "year": 1974, "genre": "journal-article", "is_oa": true, "title": "", "doi_url": "https://doi.org/10.4287/jsprs1962.13.4_8", "updated": "2020-03-22T11:59:23.244855", "oa_status": "bronze", "publisher": "Japan Society of Photogrammetry and Remote Sensing", "z_authors": [{"given": "Yukio", "family": "Ozaki", "sequence": "first"}], "is_paratext": false, "journal_name": "Journal of the Japan society of photogrammetry", "oa_locations": [{"url": "https://www.jstage.jst.go.jp/article/jsprs1962/13/4/13_4_8/_pdf", "pmh_id": null, "is_best": true, "license": null, "updated": "2018-07-22T12:12:32.891516", "version": "publishedVersion", "evidence": "open (via free pdf)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "https://www.jstage.jst.go.jp/article/jsprs1962/13/4/13_4_8/_pdf", "url_for_landing_page": "https://doi.org/10.4287/jsprs1962.13.4_8", "repository_institution": null}], "data_standard": 2, "journal_is_oa": false, "journal_issns": "1884-3980,0549-4451", "journal_issn_l": "0549-4451", "published_date": "1974-01-01", "best_oa_location": {"url": "https://www.jstage.jst.go.jp/article/jsprs1962/13/4/13_4_8/_pdf", "pmh_id": null, "is_best": true, "license": null, "updated": "2018-07-22T12:12:32.891516", "version": "publishedVersion", "evidence": "open (via free pdf)", "host_type": "publisher", "endpoint_id": null, "url_for_pdf": "https://www.jstage.jst.go.jp/article/jsprs1962/13/4/13_4_8/_pdf", "url_for_landing_page": "https://doi.org/10.4287/jsprs1962.13.4_8", "repository_institution": null}, "journal_is_in_doaj": false, "has_repository_copy": false} +{"doi": "10.1016/j.biopsycho.2012.02.013", "year": 2012, "genre": "journal-article", "is_oa": true, "title": "Heart rate responses to parental behavior in depressed adolescents", "doi_url": "https://doi.org/10.1016/j.biopsycho.2012.02.013", "updated": "2020-03-21T01:08:24.519658", "oa_status": "green", "publisher": "Elsevier BV", "z_authors": [{"given": "Nicholas B.", "family": "Allen", "sequence": "first"}, {"given": "Peter", "family": "Kuppens", "sequence": "additional"}, {"given": "Lisa B.", "family": "Sheeber", "sequence": "additional"}], "is_paratext": false, "journal_name": "Biological Psychology", "oa_locations": [{"url": "http://europepmc.org/articles/pmc3934559?pdf=render", "pmh_id": "oai:pubmedcentral.nih.gov:3934559", "is_best": true, "license": null, "updated": "2017-10-20T16:21:24.823564", "version": "acceptedVersion", "evidence": "oa repository (via OAI-PMH doi match)", "host_type": "repository", "endpoint_id": "pubmedcentral.nih.gov", "url_for_pdf": "http://europepmc.org/articles/pmc3934559?pdf=render", "url_for_landing_page": "http://europepmc.org/articles/pmc3934559", "repository_institution": "pubmedcentral.nih.gov"}, {"url": "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC3934559", "pmh_id": null, "is_best": false, "license": null, "updated": "2020-04-25T10:29:56.934849", "version": "acceptedVersion", "evidence": "oa repository (via pmcid lookup)", "host_type": "repository", "endpoint_id": null, "url_for_pdf": null, "url_for_landing_page": "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC3934559", "repository_institution": null}], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0301-0511", "journal_issn_l": "0301-0511", "published_date": "2012-04-01", "best_oa_location": {"url": "http://europepmc.org/articles/pmc3934559?pdf=render", "pmh_id": "oai:pubmedcentral.nih.gov:3934559", "is_best": true, "license": null, "updated": "2017-10-20T16:21:24.823564", "version": "acceptedVersion", "evidence": "oa repository (via OAI-PMH doi match)", "host_type": "repository", "endpoint_id": "pubmedcentral.nih.gov", "url_for_pdf": "http://europepmc.org/articles/pmc3934559?pdf=render", "url_for_landing_page": "http://europepmc.org/articles/pmc3934559", "repository_institution": "pubmedcentral.nih.gov"}, "journal_is_in_doaj": false, "has_repository_copy": true} +{"doi": "10.1119/1.1996235", "year": 1959, "genre": "journal-article", "is_oa": false, "title": "American Physical Society, Southeastern Section", "doi_url": "https://doi.org/10.1119/1.1996235", "updated": "2020-03-11T14:52:21.252730", "oa_status": "closed", "publisher": "American Association of Physics Teachers (AAPT)", "z_authors": [{"given": "Howard", "family": "Carr", "sequence": "first"}], "is_paratext": false, "journal_name": "American Journal of Physics", "oa_locations": [], "data_standard": 2, "journal_is_oa": false, "journal_issns": "0002-9505,1943-2909", "journal_issn_l": "0002-9505", "published_date": "1959-10-01", "best_oa_location": null, "journal_is_in_doaj": false, "has_repository_copy": false} \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/test/resources/log4j.properties b/dhp-workflows/dhp-doiboost/src/test/resources/log4j.properties new file mode 100644 index 0000000000..20f56e38dd --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/test/resources/log4j.properties @@ -0,0 +1,11 @@ +# Set root logger level to DEBUG and its only appender to A1. +log4j.rootLogger=INFO, A1 + +# A1 is set to be a ConsoleAppender. +log4j.appender.A1=org.apache.log4j.ConsoleAppender + +# A1 uses PatternLayout. +log4j.logger.org = ERROR +log4j.logger.eu.dnetlib = DEBUG +log4j.appender.A1.layout=org.apache.log4j.PatternLayout +log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n \ No newline at end of file diff --git a/dhp-workflows/dhp-enrichment/pom.xml b/dhp-workflows/dhp-enrichment/pom.xml index 2dc0f24366..d0ab77cc55 100644 --- a/dhp-workflows/dhp-enrichment/pom.xml +++ b/dhp-workflows/dhp-enrichment/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.2-SNAPSHOT + 1.2.4-SNAPSHOT 4.0.0 diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java index 1c65e8adec..4800def0ae 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java @@ -4,6 +4,7 @@ package eu.dnetlib.dhp.bulktag; import static eu.dnetlib.dhp.PropagationConstant.removeOutputDir; import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; +import java.util.ArrayList; import java.util.Optional; import org.apache.commons.io.IOUtils; @@ -100,6 +101,7 @@ public class SparkBulkTagJob { ResultTagger resultTagger = new ResultTagger(); readPath(spark, inputPath, resultClazz) + .map(patchResult(), Encoders.bean(resultClazz)) .map( (MapFunction) value -> resultTagger .enrichContextCriteria( @@ -119,4 +121,17 @@ public class SparkBulkTagJob { .map((MapFunction) value -> OBJECT_MAPPER.readValue(value, clazz), Encoders.bean(clazz)); } + // TODO remove this hack as soon as the values fixed by this method will be provided as NON null + private static MapFunction patchResult() { + return (MapFunction) r -> { + if (r.getDataInfo().getDeletedbyinference() == null) { + r.getDataInfo().setDeletedbyinference(false); + } + if (r.getContext() == null) { + r.setContext(new ArrayList<>()); + } + return r; + }; + } + } diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/CommunityConfigurationFactory.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/CommunityConfigurationFactory.java index 607315f3f8..e3662e04c0 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/CommunityConfigurationFactory.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/CommunityConfigurationFactory.java @@ -110,13 +110,13 @@ public class CommunityConfigurationFactory { } private static List parseZenodoCommunities(final Node node) { - final Node oacommunitynode = node.selectSingleNode("./oacommunity"); - String oacommunity = null; - if (oacommunitynode != null) { - String tmp = oacommunitynode.getText(); - if (StringUtils.isNotBlank(tmp)) - oacommunity = tmp; - } +// final Node oacommunitynode = node.selectSingleNode("./oacommunity"); +// String oacommunity = null; +// if (oacommunitynode != null) { +// String tmp = oacommunitynode.getText(); +// if (StringUtils.isNotBlank(tmp)) +// oacommunity = tmp; +// } final List list = node.selectNodes("./zenodocommunities/zenodocommunity"); final List zenodoCommunityList = new ArrayList<>(); @@ -127,11 +127,11 @@ public class CommunityConfigurationFactory { zenodoCommunityList.add(zc); } - if (oacommunity != null) { - ZenodoCommunity zc = new ZenodoCommunity(); - zc.setZenodoCommunityId(oacommunity); - zenodoCommunityList.add(zc); - } +// if (oacommunity != null) { +// ZenodoCommunity zc = new ZenodoCommunity(); +// zc.setZenodoCommunityId(oacommunity); +// zenodoCommunityList.add(zc); +// } log.info("size of the zenodo community list " + zenodoCommunityList.size()); return zenodoCommunityList; } diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/Provider.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/Provider.java index b9c37f4dcc..a9427b5941 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/Provider.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/Provider.java @@ -44,7 +44,7 @@ public class Provider implements Serializable { } private void setSelCriteria(String json, VerbResolver resolver) { - log.info("Selection constraints for datasource = " + json); + log.debug("Selection constraints for datasource = " + json); selectionConstraints = new Gson().fromJson(json, SelectionConstraints.class); selectionConstraints.setSelection(resolver); @@ -54,7 +54,7 @@ public class Provider implements Serializable { try { setSelCriteria(n.getText(), resolver); } catch (Exception e) { - log.info("not set selection criteria... "); + log.debug("not set selection criteria... "); selectionConstraints = null; } } diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/QueryInformationSystem.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/QueryInformationSystem.java index 7ec2f916fb..6a40bc2e29 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/QueryInformationSystem.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/QueryInformationSystem.java @@ -17,6 +17,8 @@ public class QueryInformationSystem { + " let $datasources := $x//CONFIGURATION/context/category[./@id=concat($x//CONFIGURATION/context/@id,'::contentproviders')]/concept " + " let $organizations := $x//CONFIGURATION/context/category[./@id=concat($x//CONFIGURATION/context/@id,'::resultorganizations')]/concept " + " let $communities := $x//CONFIGURATION/context/category[./@id=concat($x//CONFIGURATION/context/@id,'::zenodocommunities')]/concept " + + + "let $zenodo := $x//param[./@name='zenodoCommunity']/text() " + " where $x//CONFIGURATION/context[./@type='community' or ./@type='ri'] " + " return " + " " @@ -38,8 +40,15 @@ public class QueryInformationSystem { + " {$d/param[./@name='selcriteria']/text()} " + " " + " } " - + " " - + " " + + " " + + " " + + "{for $zc in $zenodo " + + "return " + + " " + + " " + + "{$zc} " + + " " + + "}" + " {for $zc in $communities " + " return " + " " diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/ResultTagger.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/ResultTagger.java index f5a985d15f..d58704b753 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/ResultTagger.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/ResultTagger.java @@ -71,10 +71,9 @@ public class ResultTagger implements Serializable { // tagging for Subject final Set subjects = new HashSet<>(); - Optional> oresultsubj = Optional.ofNullable(result.getSubject()); - if (oresultsubj.isPresent()) { - oresultsubj - .get() + + if (Objects.nonNull(result.getSubject())){ + result.getSubject() .stream() .map(subject -> subject.getValue()) .filter(StringUtils::isNotBlank) @@ -90,15 +89,23 @@ public class ResultTagger implements Serializable { final Set datasources = new HashSet<>(); final Set tmp = new HashSet<>(); - Optional> oresultinstance = Optional.ofNullable(result.getInstance()); - if (oresultinstance.isPresent()) { - for (Instance i : oresultinstance.get()) { - tmp.add(StringUtils.substringAfter(i.getCollectedfrom().getKey(), "|")); - tmp.add(StringUtils.substringAfter(i.getHostedby().getKey(), "|")); + if (Objects.nonNull(result.getInstance())) { + for (Instance i : result.getInstance()) { + if(Objects.nonNull(i.getCollectedfrom())){ + if(Objects.nonNull(i.getCollectedfrom().getKey())){ + tmp.add(StringUtils.substringAfter(i.getCollectedfrom().getKey(), "|")); + } + } + if(Objects.nonNull(i.getHostedby())){ + if(Objects.nonNull(i.getHostedby().getKey())){ + tmp.add(StringUtils.substringAfter(i.getHostedby().getKey(), "|")); + } + } + } - oresultinstance - .get() + result + .getInstance() .stream() .map(i -> new Pair<>(i.getCollectedfrom().getKey(), i.getHostedby().getKey())) .flatMap(p -> Stream.of(p.getFst(), p.getSnd())) diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/communityconfiguration/tagging_conf.xml b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/communityconfiguration/tagging_conf.xml index a44372e4de..d4c83438b0 100644 --- a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/communityconfiguration/tagging_conf.xml +++ b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/communityconfiguration/tagging_conf.xml @@ -257,6 +257,9 @@ bodhgaya + + oac_dh-ch + diff --git a/dhp-workflows/dhp-graph-mapper/pom.xml b/dhp-workflows/dhp-graph-mapper/pom.xml index aee3d27c1d..0439c2ba32 100644 --- a/dhp-workflows/dhp-graph-mapper/pom.xml +++ b/dhp-workflows/dhp-graph-mapper/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.2-SNAPSHOT + 1.2.4-SNAPSHOT 4.0.0 diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleanGraphSparkJob.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleanGraphSparkJob.java new file mode 100644 index 0000000000..fd707e949a --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleanGraphSparkJob.java @@ -0,0 +1,208 @@ + +package eu.dnetlib.dhp.oa.graph.clean; + +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; + +import java.io.BufferedInputStream; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collectors; + +import eu.dnetlib.dhp.oa.graph.raw.AbstractMdRecordToOafMapper; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.function.MapFunction; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.SaveMode; +import org.apache.spark.sql.SparkSession; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.common.HdfsSupport; +import eu.dnetlib.dhp.oa.graph.raw.common.OafMapperUtils; +import eu.dnetlib.dhp.oa.graph.raw.common.VocabularyGroup; +import eu.dnetlib.dhp.schema.common.ModelConstants; +import eu.dnetlib.dhp.schema.oaf.*; +import eu.dnetlib.dhp.utils.ISLookupClientFactory; +import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; + +public class CleanGraphSparkJob { + + private static final Logger log = LoggerFactory.getLogger(CleanGraphSparkJob.class); + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + public static void main(String[] args) throws Exception { + + String jsonConfiguration = IOUtils + .toString( + CleanGraphSparkJob.class + .getResourceAsStream( + "/eu/dnetlib/dhp/oa/graph/input_clean_graph_parameters.json")); + final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); + parser.parseArgument(args); + + Boolean isSparkSessionManaged = Optional + .ofNullable(parser.get("isSparkSessionManaged")) + .map(Boolean::valueOf) + .orElse(Boolean.TRUE); + log.info("isSparkSessionManaged: {}", isSparkSessionManaged); + + String inputPath = parser.get("inputPath"); + log.info("inputPath: {}", inputPath); + + String outputPath = parser.get("outputPath"); + log.info("outputPath: {}", outputPath); + + String isLookupUrl = parser.get("isLookupUrl"); + log.info("isLookupUrl: {}", isLookupUrl); + + String graphTableClassName = parser.get("graphTableClassName"); + log.info("graphTableClassName: {}", graphTableClassName); + + Class entityClazz = (Class) Class.forName(graphTableClassName); + + final ISLookUpService isLookupService = ISLookupClientFactory.getLookUpService(isLookupUrl); + final VocabularyGroup vocs = VocabularyGroup.loadVocsFromIS(isLookupService); + + SparkConf conf = new SparkConf(); + runWithSparkSession( + conf, + isSparkSessionManaged, + spark -> { + removeOutputDir(spark, outputPath); + fixGraphTable(spark, vocs, inputPath, entityClazz, outputPath); + }); + } + + private static void fixGraphTable( + SparkSession spark, + VocabularyGroup vocs, + String inputPath, + Class clazz, + String outputPath) { + + final CleaningRuleMap mapping = CleaningRuleMap.create(vocs); + + readTableFromPath(spark, inputPath, clazz) + .map((MapFunction) value -> OafCleaner.apply(value, mapping), Encoders.bean(clazz)) + .map((MapFunction) value -> fixDefaults(value), Encoders.bean(clazz)) + .write() + .mode(SaveMode.Overwrite) + .option("compression", "gzip") + .json(outputPath); + } + + protected static T fixDefaults(T value) { + if (value instanceof Datasource) { + // nothing to clean here + } else if (value instanceof Project) { + // nothing to clean here + } else if (value instanceof Organization) { + Organization o = (Organization) value; + if (Objects.isNull(o.getCountry()) || StringUtils.isBlank(o.getCountry().getClassid())) { + o.setCountry(qualifier("UNKNOWN", "Unknown", ModelConstants.DNET_COUNTRY_TYPE)); + } + } else if (value instanceof Relation) { + // nothing to clean here + } else if (value instanceof Result) { + + Result r = (Result) value; + if (Objects.isNull(r.getLanguage()) || StringUtils.isBlank(r.getLanguage().getClassid())) { + r + .setLanguage( + qualifier("und", "Undetermined", ModelConstants.DNET_LANGUAGES)); + } + if (Objects.nonNull(r.getSubject())) { + r + .setSubject( + r + .getSubject() + .stream() + .filter(Objects::nonNull) + .filter(sp -> StringUtils.isNotBlank(sp.getValue())) + .filter(sp -> Objects.nonNull(sp.getQualifier())) + .filter(sp -> StringUtils.isNotBlank(sp.getQualifier().getClassid())) + .collect(Collectors.toList())); + } + if (Objects.isNull(r.getResourcetype()) || StringUtils.isBlank(r.getResourcetype().getClassid())) { + r + .setResourcetype( + qualifier("UNKNOWN", "Unknown", ModelConstants.DNET_DATA_CITE_RESOURCE)); + } + if (Objects.nonNull(r.getInstance())) { + for (Instance i : r.getInstance()) { + if (Objects.isNull(i.getAccessright()) || StringUtils.isBlank(i.getAccessright().getClassid())) { + i.setAccessright(qualifier("UNKNOWN", "not available", ModelConstants.DNET_ACCESS_MODES)); + } + if (Objects.isNull(i.getHostedby()) || StringUtils.isBlank(i.getHostedby().getKey())) { + i.setHostedby(ModelConstants.UNKNOWN_REPOSITORY); + } + if (Objects.isNull(i.getRefereed())) { + i.setRefereed(qualifier("0000", "Unknown", ModelConstants.DNET_REVIEW_LEVELS)); + } + } + } + if (Objects.isNull(r.getBestaccessright()) || StringUtils.isBlank(r.getBestaccessright().getClassid())) { + Qualifier bestaccessrights = AbstractMdRecordToOafMapper.createBestAccessRights(r.getInstance()); + if (Objects.isNull(bestaccessrights)) { + r.setBestaccessright( + qualifier("UNKNOWN", "not available", ModelConstants.DNET_ACCESS_MODES)); + } else { + r.setBestaccessright(bestaccessrights); + } + } + if (Objects.nonNull(r.getAuthor())) { + boolean nullRank = r + .getAuthor() + .stream() + .anyMatch(a -> Objects.isNull(a.getRank())); + if (nullRank) { + int i = 1; + for (Author author : r.getAuthor()) { + author.setRank(i++); + } + } + } + if (value instanceof Publication) { + + } else if (value instanceof eu.dnetlib.dhp.schema.oaf.Dataset) { + + } else if (value instanceof OtherResearchProduct) { + + } else if (value instanceof Software) { + + } + } + + return value; + } + + private static Qualifier qualifier(String classid, String classname, String scheme) { + return OafMapperUtils + .qualifier( + classid, classname, scheme, scheme); + } + + private static Dataset readTableFromPath( + SparkSession spark, String inputEntityPath, Class clazz) { + + log.info("Reading Graph table from: {}", inputEntityPath); + return spark + .read() + .textFile(inputEntityPath) + .map( + (MapFunction) value -> OBJECT_MAPPER.readValue(value, clazz), + Encoders.bean(clazz)); + } + + private static void removeOutputDir(SparkSession spark, String path) { + HdfsSupport.remove(path, spark.sparkContext().hadoopConfiguration()); + } + +} diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRuleMap.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRuleMap.java new file mode 100644 index 0000000000..d2d4e118fd --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRuleMap.java @@ -0,0 +1,44 @@ + +package eu.dnetlib.dhp.oa.graph.clean; + +import java.io.Serializable; +import java.util.HashMap; + +import org.apache.commons.lang3.StringUtils; + +import eu.dnetlib.dhp.common.FunctionalInterfaceSupport.SerializableConsumer; +import eu.dnetlib.dhp.oa.graph.raw.common.VocabularyGroup; +import eu.dnetlib.dhp.schema.common.ModelConstants; +import eu.dnetlib.dhp.schema.oaf.Country; +import eu.dnetlib.dhp.schema.oaf.Qualifier; + +public class CleaningRuleMap extends HashMap> implements Serializable { + + /** + * Creates the mapping for the Oaf types subject to cleaning + * + * @param vocabularies + */ + public static CleaningRuleMap create(VocabularyGroup vocabularies) { + CleaningRuleMap mapping = new CleaningRuleMap(); + mapping.put(Qualifier.class, o -> cleanQualifier(vocabularies, (Qualifier) o)); + mapping.put(Country.class, o -> { + final Country c = (Country) o; + if (StringUtils.isBlank(c.getSchemeid())) { + c.setSchemeid(ModelConstants.DNET_COUNTRY_TYPE); + c.setSchemename(ModelConstants.DNET_COUNTRY_TYPE); + } + cleanQualifier(vocabularies, c); + }); + return mapping; + } + + private static void cleanQualifier(VocabularyGroup vocabularies, Q q) { + if (vocabularies.vocabularyExists(q.getSchemeid())) { + Qualifier newValue = vocabularies.lookup(q.getSchemeid(), q.getClassid()); + q.setClassid(newValue.getClassid()); + q.setClassname(newValue.getClassname()); + } + } + +} diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/OafCleaner.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/OafCleaner.java new file mode 100644 index 0000000000..9ba153ba5e --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/OafCleaner.java @@ -0,0 +1,82 @@ + +package eu.dnetlib.dhp.oa.graph.clean; + +import java.io.Serializable; +import java.lang.reflect.Field; +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; +import java.util.Objects; + +import eu.dnetlib.dhp.schema.oaf.Oaf; + +public class OafCleaner implements Serializable { + + public static E apply(E oaf, CleaningRuleMap mapping) { + try { + navigate(oaf, mapping); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } + return oaf; + } + + private static void navigate(Object o, CleaningRuleMap mapping) throws IllegalAccessException { + if (isPrimitive(o)) { + return; + } else if (isIterable(o.getClass())) { + for (final Object elem : (Iterable) o) { + navigate(elem, mapping); + } + } else if (hasMapping(o, mapping)) { + mapping.get(o.getClass()).accept(o); + } else { + for (final Field f : getAllFields(o.getClass())) { + f.setAccessible(true); + final Object val = f.get(o); + if (!isPrimitive(val) && hasMapping(val, mapping)) { + mapping.get(val.getClass()).accept(val); + } else { + navigate(f.get(o), mapping); + } + } + } + } + + private static boolean hasMapping(Object o, CleaningRuleMap mapping) { + return mapping.containsKey(o.getClass()); + } + + private static boolean isIterable(final Class cl) { + return Iterable.class.isAssignableFrom(cl); + } + + private static boolean isPrimitive(Object o) { + return Objects.isNull(o) + || o.getClass().isPrimitive() + || o instanceof Class + || o instanceof Integer + || o instanceof Double + || o instanceof Float + || o instanceof Long + || o instanceof Boolean + || o instanceof String + || o instanceof Byte; + } + + private static List getAllFields(Class clazz) { + return getAllFields(new LinkedList<>(), clazz); + } + + private static List getAllFields(List fields, Class clazz) { + fields.addAll(Arrays.asList(clazz.getDeclaredFields())); + + final Class superclass = clazz.getSuperclass(); + if (Objects.nonNull(superclass) && superclass.getPackage().equals(Oaf.class.getPackage())) { + getAllFields(fields, superclass); + } + + return fields; + } + +} diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/AbstractMdRecordToOafMapper.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/AbstractMdRecordToOafMapper.java index 5c89d5096f..c43ee29fe0 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/AbstractMdRecordToOafMapper.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/AbstractMdRecordToOafMapper.java @@ -10,10 +10,27 @@ import static eu.dnetlib.dhp.oa.graph.raw.common.OafMapperUtils.listFields; import static eu.dnetlib.dhp.oa.graph.raw.common.OafMapperUtils.oaiIProvenance; import static eu.dnetlib.dhp.oa.graph.raw.common.OafMapperUtils.qualifier; import static eu.dnetlib.dhp.oa.graph.raw.common.OafMapperUtils.structuredProperty; -import static eu.dnetlib.dhp.schema.common.ModelConstants.*; +import static eu.dnetlib.dhp.schema.common.ModelConstants.DATASET_DEFAULT_RESULTTYPE; import static eu.dnetlib.dhp.schema.common.ModelConstants.DNET_ACCESS_MODES; +import static eu.dnetlib.dhp.schema.common.ModelConstants.DNET_PID_TYPES; +import static eu.dnetlib.dhp.schema.common.ModelConstants.IS_PRODUCED_BY; +import static eu.dnetlib.dhp.schema.common.ModelConstants.NOT_AVAILABLE; +import static eu.dnetlib.dhp.schema.common.ModelConstants.ORP_DEFAULT_RESULTTYPE; +import static eu.dnetlib.dhp.schema.common.ModelConstants.OUTCOME; +import static eu.dnetlib.dhp.schema.common.ModelConstants.PRODUCES; +import static eu.dnetlib.dhp.schema.common.ModelConstants.PUBLICATION_DEFAULT_RESULTTYPE; +import static eu.dnetlib.dhp.schema.common.ModelConstants.REPOSITORY_PROVENANCE_ACTIONS; +import static eu.dnetlib.dhp.schema.common.ModelConstants.RESULT_PROJECT; +import static eu.dnetlib.dhp.schema.common.ModelConstants.SOFTWARE_DEFAULT_RESULTTYPE; +import static eu.dnetlib.dhp.schema.common.ModelConstants.UNKNOWN; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; import org.apache.commons.lang3.StringUtils; import org.dom4j.Document; @@ -21,6 +38,7 @@ import org.dom4j.DocumentFactory; import org.dom4j.DocumentHelper; import org.dom4j.Node; +import eu.dnetlib.dhp.oa.graph.raw.common.VocabularyGroup; import eu.dnetlib.dhp.schema.common.LicenseComparator; import eu.dnetlib.dhp.schema.oaf.Author; import eu.dnetlib.dhp.schema.oaf.Context; @@ -43,7 +61,9 @@ import eu.dnetlib.dhp.schema.oaf.StructuredProperty; public abstract class AbstractMdRecordToOafMapper { - protected final Map code2name; + protected final VocabularyGroup vocs; + + private final boolean invisible; protected static final String DATACITE_SCHEMA_KERNEL_4 = "http://datacite.org/schema/kernel-4"; protected static final String DATACITE_SCHEMA_KERNEL_3 = "http://datacite.org/schema/kernel-3"; @@ -67,8 +87,9 @@ public abstract class AbstractMdRecordToOafMapper { protected static final Qualifier MAIN_TITLE_QUALIFIER = qualifier( "main title", "main title", "dnet:dataCite_title", "dnet:dataCite_title"); - protected AbstractMdRecordToOafMapper(final Map code2name) { - this.code2name = code2name; + protected AbstractMdRecordToOafMapper(final VocabularyGroup vocs, final boolean invisible) { + this.vocs = vocs; + this.invisible = invisible; } public List processMdRecord(final String xml) { @@ -94,7 +115,7 @@ public abstract class AbstractMdRecordToOafMapper { return null; } - final DataInfo info = prepareDataInfo(doc); + final DataInfo info = prepareDataInfo(doc, invisible); final long lastUpdateTimestamp = new Date().getTime(); return createOafs(doc, type, collectedFrom, hostedBy, info, lastUpdateTimestamp); @@ -247,10 +268,7 @@ public abstract class AbstractMdRecordToOafMapper { r.setId(createOpenaireId(50, doc.valueOf("//dri:objIdentifier"), false)); r.setOriginalId(Arrays.asList(doc.valueOf("//dri:objIdentifier"))); r.setCollectedfrom(Arrays.asList(collectedFrom)); - r - .setPid( - prepareListStructProps( - doc, "//oaf:identifier", "@identifierType", "dnet:pid_types", "dnet:pid_types", info)); + r.setPid(prepareResultPids(doc, info)); r.setDateofcollection(doc.valueOf("//dr:dateOfCollection")); r.setDateoftransformation(doc.valueOf("//dr:dateOfTransformation")); r.setExtraInfo(new ArrayList<>()); // NOT PRESENT IN MDSTORES @@ -278,6 +296,8 @@ public abstract class AbstractMdRecordToOafMapper { r.setBestaccessright(getBestAccessRights(instances)); } + protected abstract List prepareResultPids(Document doc, DataInfo info); + private List prepareContexts(final Document doc, final DataInfo info) { final List list = new ArrayList<>(); for (final Object o : doc.selectNodes("//oaf:concept")) { @@ -358,7 +378,11 @@ public abstract class AbstractMdRecordToOafMapper { protected abstract Field prepareDatasetStorageDate(Document doc, DataInfo info); - protected static Qualifier getBestAccessRights(List instanceList) { + public static Qualifier createBestAccessRights(final List instanceList) { + return getBestAccessRights(instanceList); + } + + protected static Qualifier getBestAccessRights(final List instanceList) { if (instanceList != null) { final Optional min = instanceList .stream() @@ -405,14 +429,12 @@ public abstract class AbstractMdRecordToOafMapper { return null; } - protected Qualifier prepareQualifier( - final Node node, - final String xpath, - final String schemeId, - final String schemeName) { - final String classId = node.valueOf(xpath); - final String className = code2name.get(classId); - return qualifier(classId, className, schemeId, schemeName); + protected Qualifier prepareQualifier(final Node node, final String xpath, final String schemeId) { + return prepareQualifier(node.valueOf(xpath).trim(), schemeId); + } + + protected Qualifier prepareQualifier(final String classId, final String schemeId) { + return vocs.getTermAsQualifier(schemeId, classId); } protected List prepareListStructProps( @@ -420,14 +442,31 @@ public abstract class AbstractMdRecordToOafMapper { final String xpath, final String xpathClassId, final String schemeId, - final String schemeName, final DataInfo info) { final List res = new ArrayList<>(); + for (final Object o : node.selectNodes(xpath)) { final Node n = (Node) o; - final String classId = n.valueOf(xpathClassId); - final String className = code2name.get(classId); - res.add(structuredProperty(n.getText(), classId, className, schemeId, schemeName, info)); + final String classId = n.valueOf(xpathClassId).trim(); + res.add(structuredProperty(n.getText(), prepareQualifier(classId, schemeId), info)); + } + return res; + } + + protected List prepareListStructPropsWithValidQualifier( + final Node node, + final String xpath, + final String xpathClassId, + final String schemeId, + final DataInfo info) { + final List res = new ArrayList<>(); + + for (final Object o : node.selectNodes(xpath)) { + final Node n = (Node) o; + final String classId = n.valueOf(xpathClassId).trim(); + if (vocs.termExists(schemeId, classId)) { + res.add(structuredProperty(n.getText(), vocs.getTermAsQualifier(schemeId, classId), info)); + } } return res; } @@ -478,11 +517,11 @@ public abstract class AbstractMdRecordToOafMapper { return oaiIProvenance(identifier, baseURL, metadataNamespace, altered, datestamp, harvestDate); } - protected DataInfo prepareDataInfo(final Document doc) { + protected DataInfo prepareDataInfo(final Document doc, final boolean invisible) { final Node n = doc.selectSingleNode("//oaf:datainfo"); if (n == null) { - return dataInfo(false, null, false, false, REPOSITORY_PROVENANCE_ACTIONS, "0.9"); + return dataInfo(false, null, false, invisible, REPOSITORY_PROVENANCE_ACTIONS, "0.9"); } final String paClassId = n.valueOf("./oaf:provenanceaction/@classid"); @@ -496,7 +535,7 @@ public abstract class AbstractMdRecordToOafMapper { final String trust = n.valueOf("./oaf:trust"); return dataInfo( - deletedbyinference, inferenceprovenance, inferred, false, + deletedbyinference, inferenceprovenance, inferred, invisible, qualifier(paClassId, paClassName, paSchemeId, paSchemeName), trust); } diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/GenerateEntitiesApplication.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/GenerateEntitiesApplication.java index 739c7a4629..3568dc52aa 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/GenerateEntitiesApplication.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/GenerateEntitiesApplication.java @@ -4,8 +4,10 @@ package eu.dnetlib.dhp.oa.graph.raw; import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; import java.io.IOException; -import java.sql.SQLException; -import java.util.*; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.Optional; import java.util.stream.Collectors; import org.apache.commons.io.IOUtils; @@ -24,10 +26,21 @@ import org.slf4j.LoggerFactory; import com.fasterxml.jackson.databind.ObjectMapper; import eu.dnetlib.dhp.application.ArgumentApplicationParser; -import eu.dnetlib.dhp.common.DbClient; import eu.dnetlib.dhp.common.HdfsSupport; +import eu.dnetlib.dhp.oa.graph.raw.common.VocabularyGroup; import eu.dnetlib.dhp.schema.common.ModelSupport; -import eu.dnetlib.dhp.schema.oaf.*; +import eu.dnetlib.dhp.schema.oaf.Dataset; +import eu.dnetlib.dhp.schema.oaf.Datasource; +import eu.dnetlib.dhp.schema.oaf.Oaf; +import eu.dnetlib.dhp.schema.oaf.OafEntity; +import eu.dnetlib.dhp.schema.oaf.Organization; +import eu.dnetlib.dhp.schema.oaf.OtherResearchProduct; +import eu.dnetlib.dhp.schema.oaf.Project; +import eu.dnetlib.dhp.schema.oaf.Publication; +import eu.dnetlib.dhp.schema.oaf.Relation; +import eu.dnetlib.dhp.schema.oaf.Software; +import eu.dnetlib.dhp.utils.ISLookupClientFactory; +import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; import scala.Tuple2; public class GenerateEntitiesApplication { @@ -40,44 +53,43 @@ public class GenerateEntitiesApplication { final ArgumentApplicationParser parser = new ArgumentApplicationParser( IOUtils .toString( - MigrateMongoMdstoresApplication.class - .getResourceAsStream( - "/eu/dnetlib/dhp/oa/graph/generate_entities_parameters.json"))); + GenerateEntitiesApplication.class + .getResourceAsStream("/eu/dnetlib/dhp/oa/graph/generate_entities_parameters.json"))); parser.parseArgument(args); - Boolean isSparkSessionManaged = Optional + final Boolean isSparkSessionManaged = Optional .ofNullable(parser.get("isSparkSessionManaged")) .map(Boolean::valueOf) .orElse(Boolean.TRUE); log.info("isSparkSessionManaged: {}", isSparkSessionManaged); final String sourcePaths = parser.get("sourcePaths"); + log.info("sourcePaths: {}", sourcePaths); + final String targetPath = parser.get("targetPath"); + log.info("targetPath: {}", targetPath); - final String dbUrl = parser.get("postgresUrl"); - final String dbUser = parser.get("postgresUser"); - final String dbPassword = parser.get("postgresPassword"); + final String isLookupUrl = parser.get("isLookupUrl"); + log.info("isLookupUrl: {}", isLookupUrl); - final Map code2name = loadClassNames(dbUrl, dbUser, dbPassword); + final ISLookUpService isLookupService = ISLookupClientFactory.getLookUpService(isLookupUrl); + final VocabularyGroup vocs = VocabularyGroup.loadVocsFromIS(isLookupService); - SparkConf conf = new SparkConf(); - runWithSparkSession( - conf, - isSparkSessionManaged, - spark -> { - removeOutputDir(spark, targetPath); - generateEntities(spark, code2name, sourcePaths, targetPath); - }); + final SparkConf conf = new SparkConf(); + runWithSparkSession(conf, isSparkSessionManaged, spark -> { + removeOutputDir(spark, targetPath); + generateEntities(spark, vocs, sourcePaths, targetPath); + }); } private static void generateEntities( final SparkSession spark, - final Map code2name, + final VocabularyGroup vocs, final String sourcePaths, final String targetPath) { - JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); + final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); final List existingSourcePaths = Arrays .stream(sourcePaths.split(",")) .filter(p -> exists(sc, p)) @@ -94,7 +106,7 @@ public class GenerateEntitiesApplication { sc .sequenceFile(sp, Text.class, Text.class) .map(k -> new Tuple2<>(k._1().toString(), k._2().toString())) - .map(k -> convertToListOaf(k._1(), k._2(), code2name)) + .map(k -> convertToListOaf(k._1(), k._2(), vocs)) .filter(Objects::nonNull) .flatMap(list -> list.iterator())); } @@ -110,7 +122,7 @@ public class GenerateEntitiesApplication { .saveAsTextFile(targetPath, GzipCodec.class); } - private static Oaf merge(Oaf o1, Oaf o2) { + private static Oaf merge(final Oaf o1, final Oaf o2) { if (ModelSupport.isSubClass(o1, OafEntity.class)) { ((OafEntity) o1).mergeFrom((OafEntity) o2); } else if (ModelSupport.isSubClass(o1, Relation.class)) { @@ -122,14 +134,22 @@ public class GenerateEntitiesApplication { } private static List convertToListOaf( - final String id, final String s, final Map code2name) { + final String id, + final String s, + final VocabularyGroup vocs) { final String type = StringUtils.substringAfter(id, ":"); switch (type.toLowerCase()) { - case "native_oaf": - return new OafToOafMapper(code2name).processMdRecord(s); - case "native_odf": - return new OdfToOafMapper(code2name).processMdRecord(s); + case "oaf-store-cleaned": + case "oaf-store-claim": + return new OafToOafMapper(vocs, false).processMdRecord(s); + case "odf-store-cleaned": + case "odf-store-claim": + return new OdfToOafMapper(vocs, false).processMdRecord(s); + case "oaf-store-intersection": + return new OafToOafMapper(vocs, true).processMdRecord(s); + case "odf-store-intersection": + return new OdfToOafMapper(vocs, true).processMdRecord(s); case "datasource": return Arrays.asList(convertFromJson(s, Datasource.class)); case "organization": @@ -151,31 +171,6 @@ public class GenerateEntitiesApplication { } } - private static Map loadClassNames( - final String dbUrl, final String dbUser, final String dbPassword) throws IOException { - - log.info("Loading vocabulary terms from db..."); - - final Map map = new HashMap<>(); - - try (DbClient dbClient = new DbClient(dbUrl, dbUser, dbPassword)) { - dbClient - .processResults( - "select code, name from class", - rs -> { - try { - map.put(rs.getString("code"), rs.getString("name")); - } catch (final SQLException e) { - e.printStackTrace(); - } - }); - } - - log.info("Found " + map.size() + " terms."); - - return map; - } - private static Oaf convertFromJson(final String s, final Class clazz) { try { return OBJECT_MAPPER.readValue(s, clazz); @@ -196,7 +191,7 @@ public class GenerateEntitiesApplication { } } - private static void removeOutputDir(SparkSession spark, String path) { + private static void removeOutputDir(final SparkSession spark, final String path) { HdfsSupport.remove(path, spark.sparkContext().hadoopConfiguration()); } } diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateDbEntitiesApplication.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateDbEntitiesApplication.java index 5b8296c19b..da2ba47237 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateDbEntitiesApplication.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateDbEntitiesApplication.java @@ -10,7 +10,28 @@ import static eu.dnetlib.dhp.oa.graph.raw.common.OafMapperUtils.listFields; import static eu.dnetlib.dhp.oa.graph.raw.common.OafMapperUtils.listKeyValues; import static eu.dnetlib.dhp.oa.graph.raw.common.OafMapperUtils.qualifier; import static eu.dnetlib.dhp.oa.graph.raw.common.OafMapperUtils.structuredProperty; -import static eu.dnetlib.dhp.schema.common.ModelConstants.*; +import static eu.dnetlib.dhp.schema.common.ModelConstants.DATASET_DEFAULT_RESULTTYPE; +import static eu.dnetlib.dhp.schema.common.ModelConstants.DATASOURCE_ORGANIZATION; +import static eu.dnetlib.dhp.schema.common.ModelConstants.DNET_PROVENANCE_ACTIONS; +import static eu.dnetlib.dhp.schema.common.ModelConstants.ENTITYREGISTRY_PROVENANCE_ACTION; +import static eu.dnetlib.dhp.schema.common.ModelConstants.HAS_PARTICIPANT; +import static eu.dnetlib.dhp.schema.common.ModelConstants.IS_PARTICIPANT; +import static eu.dnetlib.dhp.schema.common.ModelConstants.IS_PRODUCED_BY; +import static eu.dnetlib.dhp.schema.common.ModelConstants.IS_PROVIDED_BY; +import static eu.dnetlib.dhp.schema.common.ModelConstants.IS_RELATED_TO; +import static eu.dnetlib.dhp.schema.common.ModelConstants.ORP_DEFAULT_RESULTTYPE; +import static eu.dnetlib.dhp.schema.common.ModelConstants.OUTCOME; +import static eu.dnetlib.dhp.schema.common.ModelConstants.PARTICIPATION; +import static eu.dnetlib.dhp.schema.common.ModelConstants.PRODUCES; +import static eu.dnetlib.dhp.schema.common.ModelConstants.PROJECT_ORGANIZATION; +import static eu.dnetlib.dhp.schema.common.ModelConstants.PROVIDES; +import static eu.dnetlib.dhp.schema.common.ModelConstants.PROVISION; +import static eu.dnetlib.dhp.schema.common.ModelConstants.PUBLICATION_DEFAULT_RESULTTYPE; +import static eu.dnetlib.dhp.schema.common.ModelConstants.RELATIONSHIP; +import static eu.dnetlib.dhp.schema.common.ModelConstants.RESULT_PROJECT; +import static eu.dnetlib.dhp.schema.common.ModelConstants.RESULT_RESULT; +import static eu.dnetlib.dhp.schema.common.ModelConstants.SOFTWARE_DEFAULT_RESULTTYPE; +import static eu.dnetlib.dhp.schema.common.ModelConstants.USER_CLAIM; import java.io.Closeable; import java.io.IOException; @@ -26,12 +47,13 @@ import java.util.function.Function; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.common.DbClient; import eu.dnetlib.dhp.oa.graph.raw.common.AbstractMigrationApplication; +import eu.dnetlib.dhp.oa.graph.raw.common.VocabularyGroup; import eu.dnetlib.dhp.schema.oaf.Context; import eu.dnetlib.dhp.schema.oaf.DataInfo; import eu.dnetlib.dhp.schema.oaf.Dataset; @@ -49,10 +71,11 @@ import eu.dnetlib.dhp.schema.oaf.Relation; import eu.dnetlib.dhp.schema.oaf.Result; import eu.dnetlib.dhp.schema.oaf.Software; import eu.dnetlib.dhp.schema.oaf.StructuredProperty; +import eu.dnetlib.dhp.utils.ISLookupClientFactory; public class MigrateDbEntitiesApplication extends AbstractMigrationApplication implements Closeable { - private static final Log log = LogFactory.getLog(MigrateDbEntitiesApplication.class); + private static final Logger log = LoggerFactory.getLogger(MigrateDbEntitiesApplication.class); public static final String SOURCE_TYPE = "source_type"; public static final String TARGET_TYPE = "target_type"; @@ -61,6 +84,8 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i private final long lastUpdateTimestamp; + private final VocabularyGroup vocs; + public static void main(final String[] args) throws Exception { final ArgumentApplicationParser parser = new ArgumentApplicationParser( IOUtils @@ -71,15 +96,28 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i parser.parseArgument(args); final String dbUrl = parser.get("postgresUrl"); + log.info("postgresUrl: {}", dbUrl); + final String dbUser = parser.get("postgresUser"); + log.info("postgresUser: {}", dbUser); + final String dbPassword = parser.get("postgresPassword"); + log.info("postgresPassword: xxx"); + + final String dbSchema = parser.get("dbschema"); + log.info("dbSchema {}: " + dbSchema); + + final String isLookupUrl = parser.get("isLookupUrl"); + log.info("isLookupUrl: {}", isLookupUrl); final String hdfsPath = parser.get("hdfsPath"); + log.info("hdfsPath: {}", hdfsPath); final boolean processClaims = parser.get("action") != null && parser.get("action").equalsIgnoreCase("claims"); + log.info("processClaims: {}", processClaims); try (final MigrateDbEntitiesApplication smdbe = new MigrateDbEntitiesApplication(hdfsPath, dbUrl, dbUser, - dbPassword)) { + dbPassword, isLookupUrl)) { if (processClaims) { log.info("Processing claims..."); smdbe.execute("queryClaims.sql", smdbe::processClaims); @@ -88,7 +126,11 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i smdbe.execute("queryDatasources.sql", smdbe::processDatasource); log.info("Processing projects..."); - smdbe.execute("queryProjects.sql", smdbe::processProject); + if (dbSchema.equalsIgnoreCase("beta")) { + smdbe.execute("queryProjects.sql", smdbe::processProject); + } else { + smdbe.execute("queryProjects_production.sql", smdbe::processProject); + } log.info("Processing orgs..."); smdbe.execute("queryOrganizations.sql", smdbe::processOrganization); @@ -103,18 +145,21 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i } } - protected MigrateDbEntitiesApplication() { // ONLY FOR UNIT TEST + protected MigrateDbEntitiesApplication(final VocabularyGroup vocs) { // ONLY FOR UNIT TEST super(); this.dbClient = null; this.lastUpdateTimestamp = new Date().getTime(); + this.vocs = vocs; } public MigrateDbEntitiesApplication( - final String hdfsPath, final String dbUrl, final String dbUser, final String dbPassword) + final String hdfsPath, final String dbUrl, final String dbUser, final String dbPassword, + final String isLookupUrl) throws Exception { super(hdfsPath); this.dbClient = new DbClient(dbUrl, dbUser, dbPassword); this.lastUpdateTimestamp = new Date().getTime(); + this.vocs = VocabularyGroup.loadVocsFromIS(ISLookupClientFactory.getLookUpService(isLookupUrl)); } public void execute(final String sqlFile, final Function> producer) @@ -133,7 +178,7 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i final Datasource ds = new Datasource(); ds.setId(createOpenaireId(10, rs.getString("datasourceid"), true)); - ds.setOriginalId(Arrays.asList(rs.getString("datasourceid"))); + ds.setOriginalId(Arrays.asList((String[]) rs.getArray("identities").getArray())); ds .setCollectedfrom( listKeyValues( @@ -453,12 +498,7 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i final Boolean inferred = rs.getBoolean("inferred"); final String trust = rs.getString("trust"); return dataInfo( - deletedbyinference, - inferenceprovenance, - inferred, - false, - ENTITYREGISTRY_PROVENANCE_ACTION, - trust); + deletedbyinference, inferenceprovenance, inferred, false, ENTITYREGISTRY_PROVENANCE_ACTION, trust); } private Qualifier prepareQualifierSplitting(final String s) { @@ -466,7 +506,7 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i return null; } final String[] arr = s.split("@@@"); - return arr.length == 4 ? qualifier(arr[0], arr[1], arr[2], arr[3]) : null; + return arr.length == 2 ? vocs.getTermAsQualifier(arr[1], arr[0]) : null; } private List> prepareListFields(final Array array, final DataInfo info) { @@ -485,8 +525,8 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i if (parts.length == 2) { final String value = parts[0]; final String[] arr = parts[1].split("@@@"); - if (arr.length == 4) { - return structuredProperty(value, arr[0], arr[1], arr[2], arr[3], dataInfo); + if (arr.length == 2) { + return structuredProperty(value, vocs.getTermAsQualifier(arr[1], arr[0]), dataInfo); } } return null; diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateMongoMdstoresApplication.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateMongoMdstoresApplication.java index 00c1dc4bb4..e7703bf72d 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateMongoMdstoresApplication.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateMongoMdstoresApplication.java @@ -26,8 +26,7 @@ public class MigrateMongoMdstoresApplication extends AbstractMigrationApplicatio IOUtils .toString( MigrateMongoMdstoresApplication.class - .getResourceAsStream( - "/eu/dnetlib/dhp/oa/graph/migrate_mongo_mstores_parameters.json"))); + .getResourceAsStream("/eu/dnetlib/dhp/oa/graph/migrate_mongo_mstores_parameters.json"))); parser.parseArgument(args); final String mongoBaseUrl = parser.get("mongoBaseUrl"); @@ -60,7 +59,7 @@ public class MigrateMongoMdstoresApplication extends AbstractMigrationApplicatio final String currentColl = entry.getValue(); for (final String xml : mdstoreClient.listRecords(currentColl)) { - emit(xml, "native_" + format); + emit(xml, String.format("%s-%s-%s", format, layout, interpretation)); } } } diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OafToOafMapper.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OafToOafMapper.java index 53c0913c23..dea80fabda 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OafToOafMapper.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OafToOafMapper.java @@ -4,16 +4,10 @@ package eu.dnetlib.dhp.oa.graph.raw; import static eu.dnetlib.dhp.oa.graph.raw.common.OafMapperUtils.createOpenaireId; import static eu.dnetlib.dhp.oa.graph.raw.common.OafMapperUtils.field; import static eu.dnetlib.dhp.oa.graph.raw.common.OafMapperUtils.structuredProperty; -import static eu.dnetlib.dhp.schema.common.ModelConstants.DNET_ACCESS_MODES; -import static eu.dnetlib.dhp.schema.common.ModelConstants.DNET_LANGUAGES; -import static eu.dnetlib.dhp.schema.common.ModelConstants.DNET_PUBLICATION_RESOURCE; -import static eu.dnetlib.dhp.schema.common.ModelConstants.IS_RELATED_TO; -import static eu.dnetlib.dhp.schema.common.ModelConstants.PUBLICATION_DATASET; -import static eu.dnetlib.dhp.schema.common.ModelConstants.RESULT_RESULT; +import static eu.dnetlib.dhp.schema.common.ModelConstants.*; import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; @@ -24,6 +18,7 @@ import org.dom4j.Node; import com.google.common.collect.Lists; import eu.dnetlib.dhp.common.PacePerson; +import eu.dnetlib.dhp.oa.graph.raw.common.VocabularyGroup; import eu.dnetlib.dhp.schema.oaf.Author; import eu.dnetlib.dhp.schema.oaf.DataInfo; import eu.dnetlib.dhp.schema.oaf.Field; @@ -36,8 +31,8 @@ import eu.dnetlib.dhp.schema.oaf.StructuredProperty; public class OafToOafMapper extends AbstractMdRecordToOafMapper { - public OafToOafMapper(final Map code2name) { - super(code2name); + public OafToOafMapper(final VocabularyGroup vocs, final boolean invisible) { + super(vocs, invisible); } @Override @@ -83,7 +78,7 @@ public class OafToOafMapper extends AbstractMdRecordToOafMapper { @Override protected Qualifier prepareLanguages(final Document doc) { - return prepareQualifier(doc, "//dc:language", DNET_LANGUAGES, DNET_LANGUAGES); + return prepareQualifier(doc, "//dc:language", DNET_LANGUAGES); } @Override @@ -130,16 +125,15 @@ public class OafToOafMapper extends AbstractMdRecordToOafMapper { final Instance instance = new Instance(); instance - .setInstancetype( - prepareQualifier(doc, "//dr:CobjCategory", DNET_PUBLICATION_RESOURCE, DNET_PUBLICATION_RESOURCE)); + .setInstancetype(prepareQualifier(doc, "//dr:CobjCategory", DNET_PUBLICATION_RESOURCE)); instance.setCollectedfrom(collectedfrom); instance.setHostedby(hostedby); instance.setDateofacceptance(field(doc.valueOf("//oaf:dateAccepted"), info)); instance.setDistributionlocation(doc.valueOf("//oaf:distributionlocation")); instance - .setAccessright(prepareQualifier(doc, "//oaf:accessrights", DNET_ACCESS_MODES, DNET_ACCESS_MODES)); + .setAccessright(prepareQualifier(doc, "//oaf:accessrights", DNET_ACCESS_MODES)); instance.setLicense(field(doc.valueOf("//oaf:license"), info)); - instance.setRefereed(field(doc.valueOf("//oaf:refereed"), info)); + instance.setRefereed(prepareQualifier(doc, "//oaf:refereed", DNET_REVIEW_LEVELS)); instance .setProcessingchargeamount(field(doc.valueOf("//oaf:processingchargeamount"), info)); instance @@ -281,12 +275,12 @@ public class OafToOafMapper extends AbstractMdRecordToOafMapper { res .add( getRelation( - docId, otherId, RESULT_RESULT, PUBLICATION_DATASET, IS_RELATED_TO, collectedFrom, info, + docId, otherId, RESULT_RESULT, RELATIONSHIP, IS_RELATED_TO, collectedFrom, info, lastUpdateTimestamp)); res .add( getRelation( - otherId, docId, RESULT_RESULT, PUBLICATION_DATASET, IS_RELATED_TO, collectedFrom, info, + otherId, docId, RESULT_RESULT, RELATIONSHIP, IS_RELATED_TO, collectedFrom, info, lastUpdateTimestamp)); } } @@ -297,4 +291,10 @@ public class OafToOafMapper extends AbstractMdRecordToOafMapper { protected Qualifier prepareResourceType(final Document doc, final DataInfo info) { return null; // NOT PRESENT IN OAF } + + @Override + protected List prepareResultPids(final Document doc, final DataInfo info) { + return prepareListStructPropsWithValidQualifier( + doc, "//oaf:identifier", "@identifierType", DNET_PID_TYPES, info); + } } diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OdfToOafMapper.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OdfToOafMapper.java index b9a1599174..62f8123bb3 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OdfToOafMapper.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OdfToOafMapper.java @@ -4,24 +4,12 @@ package eu.dnetlib.dhp.oa.graph.raw; import static eu.dnetlib.dhp.oa.graph.raw.common.OafMapperUtils.createOpenaireId; import static eu.dnetlib.dhp.oa.graph.raw.common.OafMapperUtils.field; import static eu.dnetlib.dhp.oa.graph.raw.common.OafMapperUtils.structuredProperty; -import static eu.dnetlib.dhp.schema.common.ModelConstants.DNET_ACCESS_MODES; -import static eu.dnetlib.dhp.schema.common.ModelConstants.DNET_DATA_CITE_DATE; -import static eu.dnetlib.dhp.schema.common.ModelConstants.DNET_DATA_CITE_RESOURCE; -import static eu.dnetlib.dhp.schema.common.ModelConstants.DNET_LANGUAGES; -import static eu.dnetlib.dhp.schema.common.ModelConstants.DNET_PUBLICATION_RESOURCE; -import static eu.dnetlib.dhp.schema.common.ModelConstants.HAS_PARTS; -import static eu.dnetlib.dhp.schema.common.ModelConstants.IS_PART_OF; -import static eu.dnetlib.dhp.schema.common.ModelConstants.IS_SUPPLEMENTED_BY; -import static eu.dnetlib.dhp.schema.common.ModelConstants.IS_SUPPLEMENT_TO; -import static eu.dnetlib.dhp.schema.common.ModelConstants.PART; -import static eu.dnetlib.dhp.schema.common.ModelConstants.RESULT_RESULT; -import static eu.dnetlib.dhp.schema.common.ModelConstants.SUPPLEMENT; +import static eu.dnetlib.dhp.schema.common.ModelConstants.*; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; import org.apache.commons.lang3.StringUtils; @@ -29,6 +17,7 @@ import org.dom4j.Document; import org.dom4j.Node; import eu.dnetlib.dhp.common.PacePerson; +import eu.dnetlib.dhp.oa.graph.raw.common.VocabularyGroup; import eu.dnetlib.dhp.schema.oaf.Author; import eu.dnetlib.dhp.schema.oaf.DataInfo; import eu.dnetlib.dhp.schema.oaf.Field; @@ -43,8 +32,8 @@ public class OdfToOafMapper extends AbstractMdRecordToOafMapper { public static final String HTTP_DX_DOI_PREIFX = "http://dx.doi.org/"; - public OdfToOafMapper(final Map code2name) { - super(code2name); + public OdfToOafMapper(final VocabularyGroup vocs, final boolean invisible) { + super(vocs, invisible); } @Override @@ -120,16 +109,15 @@ public class OdfToOafMapper extends AbstractMdRecordToOafMapper { final Instance instance = new Instance(); instance - .setInstancetype( - prepareQualifier(doc, "//dr:CobjCategory", DNET_PUBLICATION_RESOURCE, DNET_PUBLICATION_RESOURCE)); + .setInstancetype(prepareQualifier(doc, "//dr:CobjCategory", DNET_PUBLICATION_RESOURCE)); instance.setCollectedfrom(collectedfrom); instance.setHostedby(hostedby); instance.setDateofacceptance(field(doc.valueOf("//oaf:dateAccepted"), info)); instance.setDistributionlocation(doc.valueOf("//oaf:distributionlocation")); instance - .setAccessright(prepareQualifier(doc, "//oaf:accessrights", DNET_ACCESS_MODES, DNET_ACCESS_MODES)); + .setAccessright(prepareQualifier(doc, "//oaf:accessrights", DNET_ACCESS_MODES)); instance.setLicense(field(doc.valueOf("//oaf:license"), info)); - instance.setRefereed(field(doc.valueOf("//oaf:refereed"), info)); + instance.setRefereed(prepareQualifier(doc, "//oaf:refereed", DNET_REVIEW_LEVELS)); instance.setProcessingchargeamount(field(doc.valueOf("//oaf:processingchargeamount"), info)); instance .setProcessingchargecurrency(field(doc.valueOf("//oaf:processingchargeamount/@currency"), info)); @@ -138,9 +126,16 @@ public class OdfToOafMapper extends AbstractMdRecordToOafMapper { for (final Object o : doc.selectNodes("//datacite:alternateIdentifier[@alternateIdentifierType='URL']")) { url.add(((Node) o).getText().trim()); } + for (final Object o : doc + .selectNodes("//datacite:alternateIdentifier[@alternateIdentifierType='landingPage']")) { + url.add(((Node) o).getText().trim()); + } for (final Object o : doc.selectNodes("//datacite:identifier[@identifierType='URL']")) { url.add(((Node) o).getText().trim()); } + for (final Object o : doc.selectNodes("//datacite:identifier[@identifierType='landingPage']")) { + url.add(((Node) o).getText().trim()); + } for (final Object o : doc.selectNodes("//datacite:alternateIdentifier[@alternateIdentifierType='DOI']")) { url.add(HTTP_DX_DOI_PREIFX + ((Node) o).getText().trim()); } @@ -211,7 +206,7 @@ public class OdfToOafMapper extends AbstractMdRecordToOafMapper { @Override protected Qualifier prepareLanguages(final Document doc) { - return prepareQualifier(doc, "//datacite:language", DNET_LANGUAGES, DNET_LANGUAGES); + return prepareQualifier(doc, "//datacite:language", DNET_LANGUAGES); } @Override @@ -239,7 +234,7 @@ public class OdfToOafMapper extends AbstractMdRecordToOafMapper { @Override protected Qualifier prepareSoftwareProgrammingLanguage(final Document doc, final DataInfo info) { - return prepareQualifier(doc, "//datacite:format", "dnet:programming_languages", "dnet:programming_languages"); + return prepareQualifier(doc, "//datacite:format", "dnet:programming_languages"); } @Override @@ -366,7 +361,28 @@ public class OdfToOafMapper extends AbstractMdRecordToOafMapper { @Override protected Qualifier prepareResourceType(final Document doc, final DataInfo info) { return prepareQualifier( - doc, "//*[local-name() = 'resource']//*[local-name() = 'resourceType']", DNET_DATA_CITE_RESOURCE, - DNET_DATA_CITE_RESOURCE); + doc, "//*[local-name() = 'resource']//*[local-name() = 'resourceType']", DNET_DATA_CITE_RESOURCE); } + + @Override + protected List prepareResultPids(final Document doc, final DataInfo info) { + final List res = new ArrayList<>(); + res + .addAll( + prepareListStructPropsWithValidQualifier( + doc, "//oaf:identifier", "@identifierType", DNET_PID_TYPES, info)); + res + .addAll( + prepareListStructPropsWithValidQualifier( + doc, "//datacite:identifier[@identifierType != 'URL' and @identifierType != 'landingPage']", + "@identifierType", DNET_PID_TYPES, info)); + res + .addAll( + prepareListStructPropsWithValidQualifier( + doc, + "//datacite:alternateIdentifier[@alternateIdentifierType != 'URL' and @alternateIdentifierType != 'landingPage']", + "@alternateIdentifierType", DNET_PID_TYPES, info)); + return res; + } + } diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/OafMapperUtils.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/OafMapperUtils.java index 9beed28371..8ede407731 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/OafMapperUtils.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/OafMapperUtils.java @@ -9,7 +9,15 @@ import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; -import eu.dnetlib.dhp.schema.oaf.*; +import eu.dnetlib.dhp.schema.oaf.DataInfo; +import eu.dnetlib.dhp.schema.oaf.ExtraInfo; +import eu.dnetlib.dhp.schema.oaf.Field; +import eu.dnetlib.dhp.schema.oaf.Journal; +import eu.dnetlib.dhp.schema.oaf.KeyValue; +import eu.dnetlib.dhp.schema.oaf.OAIProvenance; +import eu.dnetlib.dhp.schema.oaf.OriginDescription; +import eu.dnetlib.dhp.schema.oaf.Qualifier; +import eu.dnetlib.dhp.schema.oaf.StructuredProperty; import eu.dnetlib.dhp.utils.DHPUtils; public class OafMapperUtils { @@ -60,6 +68,10 @@ public class OafMapperUtils { .collect(Collectors.toList()); } + public static Qualifier unknown(final String schemeid, final String schemename) { + return qualifier("UNKNOWN", "Unknown", schemeid, schemename); + } + public static Qualifier qualifier( final String classid, final String classname, @@ -85,7 +97,9 @@ public class OafMapperUtils { } public static StructuredProperty structuredProperty( - final String value, final Qualifier qualifier, final DataInfo dataInfo) { + final String value, + final Qualifier qualifier, + final DataInfo dataInfo) { if (value == null) { return null; } @@ -188,8 +202,12 @@ public class OafMapperUtils { } public static String createOpenaireId( - final int prefix, final String originalId, final boolean to_md5) { - if (to_md5) { + final int prefix, + final String originalId, + final boolean to_md5) { + if (StringUtils.isBlank(originalId)) { + return null; + } else if (to_md5) { final String nsPrefix = StringUtils.substringBefore(originalId, "::"); final String rest = StringUtils.substringAfter(originalId, "::"); return String.format("%s|%s::%s", prefix, nsPrefix, DHPUtils.md5(rest)); @@ -199,7 +217,9 @@ public class OafMapperUtils { } public static String createOpenaireId( - final String type, final String originalId, final boolean to_md5) { + final String type, + final String originalId, + final boolean to_md5) { switch (type) { case "datasource": return createOpenaireId(10, originalId, to_md5); diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/Vocabulary.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/Vocabulary.java new file mode 100644 index 0000000000..9bf198c8b7 --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/Vocabulary.java @@ -0,0 +1,86 @@ + +package eu.dnetlib.dhp.oa.graph.raw.common; + +import java.io.Serializable; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +import org.apache.commons.lang3.StringUtils; + +import com.google.common.collect.Maps; + +import eu.dnetlib.dhp.schema.oaf.Qualifier; + +public class Vocabulary implements Serializable { + + private final String id; + private final String name; + + /** + * Code to Term mappings for this Vocabulary. + */ + private final Map terms = new HashMap<>(); + + /** + * Synonym to Code mappings for this Vocabulary. + */ + private final Map synonyms = Maps.newHashMap(); + + public Vocabulary(final String id, final String name) { + this.id = id; + this.name = name; + } + + public String getId() { + return id; + } + + public String getName() { + return name; + } + + protected Map getTerms() { + return terms; + } + + public VocabularyTerm getTerm(final String id) { + return Optional.ofNullable(id).map(s -> s.toLowerCase()).map(s -> terms.get(s)).orElse(null); + } + + protected void addTerm(final String id, final String name) { + terms.put(id.toLowerCase(), new VocabularyTerm(id, name)); + } + + protected boolean termExists(final String id) { + return terms.containsKey(id.toLowerCase()); + } + + protected void addSynonym(final String syn, final String termCode) { + synonyms.put(syn, termCode.toLowerCase()); + } + + public VocabularyTerm getTermBySynonym(final String syn) { + return getTerm(synonyms.get(syn.toLowerCase())); + } + + public Qualifier getTermAsQualifier(final String termId) { + if (StringUtils.isBlank(termId)) { + return OafMapperUtils.unknown(getId(), getName()); + } else if (termExists(termId)) { + final VocabularyTerm t = getTerm(termId); + return OafMapperUtils.qualifier(t.getId(), t.getName(), getId(), getName()); + } else { + return OafMapperUtils.qualifier(termId, termId, getId(), getName()); + } + } + + public Qualifier getSynonymAsQualifier(final String syn) { + return Optional + .ofNullable(getTermBySynonym(syn)) + .map(term -> getTermAsQualifier(term.getId())) + .orElse(null); + // .orElse(OafMapperUtils.unknown(getId(), getName())); + } + +} diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/VocabularyGroup.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/VocabularyGroup.java new file mode 100644 index 0000000000..334339d3be --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/VocabularyGroup.java @@ -0,0 +1,144 @@ + +package eu.dnetlib.dhp.oa.graph.raw.common; + +import java.io.Serializable; +import java.util.*; +import java.util.stream.Collectors; + +import org.apache.commons.lang3.StringUtils; + +import eu.dnetlib.dhp.schema.oaf.Qualifier; +import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException; +import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; + +public class VocabularyGroup implements Serializable { + + public static final String VOCABULARIES_XQUERY = "for $x in collection('/db/DRIVER/VocabularyDSResources/VocabularyDSResourceType') \n" + + + "let $vocid := $x//VOCABULARY_NAME/@code\n" + + "let $vocname := $x//VOCABULARY_NAME/text()\n" + + "for $term in ($x//TERM)\n" + + "return concat($vocid,' @=@ ',$vocname,' @=@ ',$term/@code,' @=@ ',$term/@english_name)"; + + public static final String VOCABULARY_SYNONYMS_XQUERY = "for $x in collection('/db/DRIVER/VocabularyDSResources/VocabularyDSResourceType')\n" + + + "let $vocid := $x//VOCABULARY_NAME/@code\n" + + "let $vocname := $x//VOCABULARY_NAME/text()\n" + + "for $term in ($x//TERM)\n" + + "for $syn in ($term//SYNONYM/@term)\n" + + "return concat($vocid,' @=@ ',$term/@code,' @=@ ', $syn)\n"; + + public static VocabularyGroup loadVocsFromIS(ISLookUpService isLookUpService) throws ISLookUpException { + + final VocabularyGroup vocs = new VocabularyGroup(); + + for (final String s : isLookUpService.quickSearchProfile(VOCABULARIES_XQUERY)) { + final String[] arr = s.split("@=@"); + if (arr.length == 4) { + final String vocId = arr[0].trim(); + final String vocName = arr[1].trim(); + final String termId = arr[2].trim(); + final String termName = arr[3].trim(); + + if (!vocs.vocabularyExists(vocId)) { + vocs.addVocabulary(vocId, vocName); + } + + vocs.addTerm(vocId, termId, termName); + // vocs.addSynonyms(vocId, termId, termId); + } + } + + for (final String s : isLookUpService.quickSearchProfile(VOCABULARY_SYNONYMS_XQUERY)) { + final String[] arr = s.split("@=@"); + if (arr.length == 3) { + final String vocId = arr[0].trim(); + final String termId = arr[1].trim(); + final String syn = arr[2].trim(); + + vocs.addSynonyms(vocId, termId, syn); + // vocs.addSynonyms(vocId, termId, termId); + } + } + + return vocs; + } + + private final Map vocs = new HashMap<>(); + + public void addVocabulary(final String id, final String name) { + vocs.put(id.toLowerCase(), new Vocabulary(id, name)); + } + + public void addTerm(final String vocId, final String id, final String name) { + if (vocabularyExists(vocId)) { + vocs.get(vocId.toLowerCase()).addTerm(id, name); + } + } + + public VocabularyTerm getTerm(final String vocId, final String id) { + if (termExists(vocId, id)) { + return vocs.get(vocId.toLowerCase()).getTerm(id); + } else { + return new VocabularyTerm(id, id); + } + } + + public Set getTerms(String vocId) { + if (!vocabularyExists(vocId)) { + return new HashSet<>(); + } + return vocs + .get(vocId.toLowerCase()) + .getTerms() + .values() + .stream() + .map(t -> t.getId()) + .collect(Collectors.toCollection(HashSet::new)); + } + + public Qualifier lookup(String vocId, String id) { + return Optional + .ofNullable(getSynonymAsQualifier(vocId, id)) + .orElse(getTermAsQualifier(vocId, id)); + } + + public Qualifier getTermAsQualifier(final String vocId, final String id) { + if (vocabularyExists(vocId)) { + return vocs.get(vocId.toLowerCase()).getTermAsQualifier(id); + } + return OafMapperUtils.qualifier(id, id, "", ""); + } + + public Qualifier getSynonymAsQualifier(final String vocId, final String syn) { + if (StringUtils.isBlank(vocId)) { + return OafMapperUtils.unknown("", ""); + } + return vocs.get(vocId.toLowerCase()).getSynonymAsQualifier(syn); + } + + public boolean termExists(final String vocId, final String id) { + return vocabularyExists(vocId) && vocs.get(vocId.toLowerCase()).termExists(id); + } + + public boolean vocabularyExists(final String vocId) { + return Optional + .ofNullable(vocId) + .map(String::toLowerCase) + .map(id -> vocs.containsKey(id)) + .orElse(false); + } + + private void addSynonyms(final String vocId, final String termId, final String syn) { + String id = Optional + .ofNullable(vocId) + .map(s -> s.toLowerCase()) + .orElseThrow( + () -> new IllegalArgumentException(String.format("empty vocabulary id for [term:%s, synonym:%s]"))); + Optional + .ofNullable(vocs.get(id)) + .orElseThrow(() -> new IllegalArgumentException("missing vocabulary id: " + vocId)) + .addSynonym(syn.toLowerCase(), termId); + } + +} diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/VocabularyTerm.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/VocabularyTerm.java new file mode 100644 index 0000000000..1aa1b82539 --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/VocabularyTerm.java @@ -0,0 +1,24 @@ + +package eu.dnetlib.dhp.oa.graph.raw.common; + +import java.io.Serializable; + +public class VocabularyTerm implements Serializable { + + private final String id; + private final String name; + + public VocabularyTerm(final String id, final String name) { + this.id = id; + this.name = name; + } + + public String getId() { + return id; + } + + public String getName() { + return name; + } + +} diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/parser/DatasetScholexplorerParser.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/parser/DatasetScholexplorerParser.java index f49163c87a..afba57bb8a 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/parser/DatasetScholexplorerParser.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/parser/DatasetScholexplorerParser.java @@ -159,7 +159,7 @@ public class DatasetScholexplorerParser extends AbstractScholexplorerParser { .setDescription( descs .stream() - .map(it -> it.length() < 10000 ? it : it.substring(0, 10000)) +// .map(it -> it.length() < 10000 ? it : it.substring(0, 10000)) .map( it -> { final Field d = new Field<>(); diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/parser/PublicationScholexplorerParser.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/parser/PublicationScholexplorerParser.java index edbb444db7..bf59a6f0ef 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/parser/PublicationScholexplorerParser.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/parser/PublicationScholexplorerParser.java @@ -213,10 +213,10 @@ public class PublicationScholexplorerParser extends AbstractScholexplorerParser .setValue( VtdUtilityParser.getSingleValue(ap, vn, "//*[local-name()='description']")); - if (StringUtils.isNotBlank(description.getValue()) - && description.getValue().length() > 10000) { - description.setValue(description.getValue().substring(0, 10000)); - } +// if (StringUtils.isNotBlank(description.getValue()) +// && description.getValue().length() > 10000) { +// description.setValue(description.getValue().substring(0, 10000)); +// } parsedObject.setDescription(Collections.singletonList(description)); diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/clean/oozie_app/config-default.xml b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/clean/oozie_app/config-default.xml new file mode 100644 index 0000000000..2e0ed9aeea --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/clean/oozie_app/config-default.xml @@ -0,0 +1,18 @@ + + + jobTracker + yarnRM + + + nameNode + hdfs://nameservice1 + + + oozie.use.system.libpath + true + + + oozie.action.sharelib.for.spark + spark2 + + \ No newline at end of file diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/clean/oozie_app/workflow.xml b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/clean/oozie_app/workflow.xml new file mode 100644 index 0000000000..7329df29a4 --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/clean/oozie_app/workflow.xml @@ -0,0 +1,281 @@ + + + + + graphInputPath + the input path to read graph content + + + graphOutputPath + the target path to store cleaned graph + + + isLookupUrl + the address of the lookUp service + + + + sparkDriverMemory + memory for driver process + + + sparkExecutorMemory + memory for individual executor + + + sparkExecutorCores + number of cores used by single executor + + + oozieActionShareLibForSpark2 + oozie action sharelib for spark 2.* + + + spark2ExtraListeners + com.cloudera.spark.lineage.NavigatorAppListener + spark 2.* extra listeners classname + + + spark2SqlQueryExecutionListeners + com.cloudera.spark.lineage.NavigatorQueryListener + spark 2.* sql query execution listeners classname + + + spark2YarnHistoryServerAddress + spark 2.* yarn history server address + + + spark2EventLogDir + spark 2.* event log dir location + + + + + + + Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] + + + + + + + + + + + + + + + + yarn + cluster + Clean publications + eu.dnetlib.dhp.oa.graph.clean.CleanGraphSparkJob + dhp-graph-mapper-${projectVersion}.jar + + --executor-cores=${sparkExecutorCores} + --executor-memory=${sparkExecutorMemory} + --driver-memory=${sparkDriverMemory} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.sql.shuffle.partitions=7680 + + --inputPath${graphInputPath}/publication + --outputPath${graphOutputPath}/publication + --graphTableClassNameeu.dnetlib.dhp.schema.oaf.Publication + --isLookupUrl${isLookupUrl} + + + + + + + + yarn + cluster + Clean datasets + eu.dnetlib.dhp.oa.graph.clean.CleanGraphSparkJob + dhp-graph-mapper-${projectVersion}.jar + + --executor-cores=${sparkExecutorCores} + --executor-memory=${sparkExecutorMemory} + --driver-memory=${sparkDriverMemory} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.sql.shuffle.partitions=7680 + + --inputPath${graphInputPath}/dataset + --outputPath${graphOutputPath}/dataset + --graphTableClassNameeu.dnetlib.dhp.schema.oaf.Dataset + --isLookupUrl${isLookupUrl} + + + + + + + + yarn + cluster + Clean otherresearchproducts + eu.dnetlib.dhp.oa.graph.clean.CleanGraphSparkJob + dhp-graph-mapper-${projectVersion}.jar + + --executor-cores=${sparkExecutorCores} + --executor-memory=${sparkExecutorMemory} + --driver-memory=${sparkDriverMemory} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.sql.shuffle.partitions=7680 + + --inputPath${graphInputPath}/otherresearchproduct + --outputPath${graphOutputPath}/otherresearchproduct + --graphTableClassNameeu.dnetlib.dhp.schema.oaf.OtherResearchProduct + --isLookupUrl${isLookupUrl} + + + + + + + + yarn + cluster + Clean softwares + eu.dnetlib.dhp.oa.graph.clean.CleanGraphSparkJob + dhp-graph-mapper-${projectVersion}.jar + + --executor-cores=${sparkExecutorCores} + --executor-memory=${sparkExecutorMemory} + --driver-memory=${sparkDriverMemory} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.sql.shuffle.partitions=7680 + + --inputPath${graphInputPath}/software + --outputPath${graphOutputPath}/software + --graphTableClassNameeu.dnetlib.dhp.schema.oaf.Software + --isLookupUrl${isLookupUrl} + + + + + + + + yarn + cluster + Clean datasources + eu.dnetlib.dhp.oa.graph.clean.CleanGraphSparkJob + dhp-graph-mapper-${projectVersion}.jar + + --executor-cores=${sparkExecutorCores} + --executor-memory=${sparkExecutorMemory} + --driver-memory=${sparkDriverMemory} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.sql.shuffle.partitions=7680 + + --inputPath${graphInputPath}/datasource + --outputPath${graphOutputPath}/datasource + --graphTableClassNameeu.dnetlib.dhp.schema.oaf.Datasource + --isLookupUrl${isLookupUrl} + + + + + + + + yarn + cluster + Clean organizations + eu.dnetlib.dhp.oa.graph.clean.CleanGraphSparkJob + dhp-graph-mapper-${projectVersion}.jar + + --executor-cores=${sparkExecutorCores} + --executor-memory=${sparkExecutorMemory} + --driver-memory=${sparkDriverMemory} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.sql.shuffle.partitions=7680 + + --inputPath${graphInputPath}/organization + --outputPath${graphOutputPath}/organization + --graphTableClassNameeu.dnetlib.dhp.schema.oaf.Organization + --isLookupUrl${isLookupUrl} + + + + + + + + yarn + cluster + Clean projects + eu.dnetlib.dhp.oa.graph.clean.CleanGraphSparkJob + dhp-graph-mapper-${projectVersion}.jar + + --executor-cores=${sparkExecutorCores} + --executor-memory=${sparkExecutorMemory} + --driver-memory=${sparkDriverMemory} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.sql.shuffle.partitions=7680 + + --inputPath${graphInputPath}/project + --outputPath${graphOutputPath}/project + --graphTableClassNameeu.dnetlib.dhp.schema.oaf.Project + --isLookupUrl${isLookupUrl} + + + + + + + + yarn + cluster + Clean relations + eu.dnetlib.dhp.oa.graph.clean.CleanGraphSparkJob + dhp-graph-mapper-${projectVersion}.jar + + --executor-cores=${sparkExecutorCores} + --executor-memory=${sparkExecutorMemory} + --driver-memory=${sparkDriverMemory} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.sql.shuffle.partitions=7680 + + --inputPath${graphInputPath}/relation + --outputPath${graphOutputPath}/relation + --graphTableClassNameeu.dnetlib.dhp.schema.oaf.Relation + --isLookupUrl${isLookupUrl} + + + + + + + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/generate_entities_parameters.json b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/generate_entities_parameters.json index 293bf041b9..8342dde959 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/generate_entities_parameters.json +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/generate_entities_parameters.json @@ -18,22 +18,9 @@ "paramRequired": true }, { - "paramName": "pgurl", - "paramLongName": "postgresUrl", - "paramDescription": "postgres url, example: jdbc:postgresql://localhost:5432/testdb", + "paramName": "isu", + "paramLongName": "isLookupUrl", + "paramDescription": "the url of the ISLookupService", "paramRequired": true - }, - { - "paramName": "pguser", - "paramLongName": "postgresUser", - "paramDescription": "postgres user", - "paramRequired": false - }, - { - "paramName": "pgpasswd", - "paramLongName": "postgresPassword", - "paramDescription": "postgres password", - "paramRequired": false } - ] \ No newline at end of file diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/hive/oozie_app/lib/scripts/postprocessing.sql b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/hive/oozie_app/lib/scripts/postprocessing.sql index 6c49679cdf..7bec2fe040 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/hive/oozie_app/lib/scripts/postprocessing.sql +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/hive/oozie_app/lib/scripts/postprocessing.sql @@ -1,10 +1,10 @@ DROP VIEW IF EXISTS ${hiveDbName}.result; CREATE VIEW IF NOT EXISTS result as - select id, dateofcollection, title, publisher, bestaccessright, datainfo, collectedfrom, pid, author, resulttype, language, country, subject, description, dateofacceptance, embargoenddate, resourcetype, context, instance from ${hiveDbName}.publication p + select id, dateofcollection, title, publisher, bestaccessright, datainfo, collectedfrom, pid, author, resulttype, language, country, subject, description, dateofacceptance, embargoenddate, resourcetype, context, externalreference, instance from ${hiveDbName}.publication p union all - select id, dateofcollection, title, publisher, bestaccessright, datainfo, collectedfrom, pid, author, resulttype, language, country, subject, description, dateofacceptance, embargoenddate, resourcetype, context, instance from ${hiveDbName}.dataset d + select id, dateofcollection, title, publisher, bestaccessright, datainfo, collectedfrom, pid, author, resulttype, language, country, subject, description, dateofacceptance, embargoenddate, resourcetype, context, externalreference, instance from ${hiveDbName}.dataset d union all - select id, dateofcollection, title, publisher, bestaccessright, datainfo, collectedfrom, pid, author, resulttype, language, country, subject, description, dateofacceptance, embargoenddate, resourcetype, context, instance from ${hiveDbName}.software s + select id, dateofcollection, title, publisher, bestaccessright, datainfo, collectedfrom, pid, author, resulttype, language, country, subject, description, dateofacceptance, embargoenddate, resourcetype, context, externalreference, instance from ${hiveDbName}.software s union all - select id, dateofcollection, title, publisher, bestaccessright, datainfo, collectedfrom, pid, author, resulttype, language, country, subject, description, dateofacceptance, embargoenddate, resourcetype, context, instance from ${hiveDbName}.otherresearchproduct o; + select id, dateofcollection, title, publisher, bestaccessright, datainfo, collectedfrom, pid, author, resulttype, language, country, subject, description, dateofacceptance, embargoenddate, resourcetype, context, externalreference, instance from ${hiveDbName}.otherresearchproduct o; diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/input_clean_graph_parameters.json b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/input_clean_graph_parameters.json new file mode 100644 index 0000000000..9cfed1e915 --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/input_clean_graph_parameters.json @@ -0,0 +1,32 @@ +[ + { + "paramName": "issm", + "paramLongName": "isSparkSessionManaged", + "paramDescription": "when true will stop SparkSession after job execution", + "paramRequired": false + }, + { + "paramName": "in", + "paramLongName": "inputPath", + "paramDescription": "the path to the graph data dump to read", + "paramRequired": true + }, + { + "paramName": "out", + "paramLongName": "outputPath", + "paramDescription": "the path to store the output graph", + "paramRequired": true + }, + { + "paramName": "isu", + "paramLongName": "isLookupUrl", + "paramDescription": "url to the ISLookup Service", + "paramRequired": true + }, + { + "paramName": "class", + "paramLongName": "graphTableClassName", + "paramDescription": "class name moelling the graph table", + "paramRequired": true + } +] diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/migrate_db_entities_parameters.json b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/migrate_db_entities_parameters.json index cb13ff0242..6dfef32db0 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/migrate_db_entities_parameters.json +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/migrate_db_entities_parameters.json @@ -28,5 +28,17 @@ "paramLongName": "action", "paramDescription": "process claims", "paramRequired": false + }, + { + "paramName": "isu", + "paramLongName": "isLookupUrl", + "paramDescription": "the url of the ISLookupService", + "paramRequired": true + }, + { + "paramName": "dbschema", + "paramLongName": "dbschema", + "paramDescription": "the database schema according to the D-Net infrastructure (beta or production)", + "paramRequired": true } ] \ No newline at end of file diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/raw_all/oozie_app/workflow.xml b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/raw_all/oozie_app/workflow.xml index fa015499c9..d8b61b5ead 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/raw_all/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/raw_all/oozie_app/workflow.xml @@ -25,6 +25,11 @@ postgresPassword the password postgres + + + dbSchema + beta + the database schema according to the D-Net infrastructure (beta or production) mongoURL @@ -34,6 +39,10 @@ mongoDb mongo database + + isLookupUrl + the address of the lookUp service + sparkDriverMemory @@ -119,7 +128,9 @@ --postgresUrl${postgresURL} --postgresUser${postgresUser} --postgresPassword${postgresPassword} + --isLookupUrl${isLookupUrl} --actionclaims + --dbschema${dbSchema} @@ -169,6 +180,8 @@ --postgresUrl${postgresURL} --postgresUser${postgresUser} --postgresPassword${postgresPassword} + --isLookupUrl${isLookupUrl} + --dbschema${dbSchema} @@ -204,6 +217,23 @@ --mdLayoutstore --mdInterpretationcleaned + + + + + + + + + + eu.dnetlib.dhp.oa.graph.raw.MigrateMongoMdstoresApplication + --hdfsPath${contentPath}/oaf_records_invisible + --mongoBaseUrl${mongoURL} + --mongoDb${mongoDb} + --mdFormatOAF + --mdLayoutstore + --mdInterpretationintersection + @@ -231,11 +261,9 @@ --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - --sourcePaths${contentPath}/db_claims,${contentPath}/oaf_claims,${contentPath}/odf_claims + --sourcePaths${contentPath}/db_claims,${contentPath}/oaf_claims,${contentPath}/odf_claims,${contentPath}/oaf_records_invisible --targetPath${workingDir}/entities_claim - --postgresUrl${postgresURL} - --postgresUser${postgresUser} - --postgresPassword${postgresPassword} + --isLookupUrl${isLookupUrl} @@ -282,9 +310,7 @@ --sourcePaths${contentPath}/db_records,${contentPath}/oaf_records,${contentPath}/odf_records --targetPath${workingDir}/entities - --postgresUrl${postgresURL} - --postgresUser${postgresUser} - --postgresPassword${postgresPassword} + --isLookupUrl${isLookupUrl} diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/raw_claims/oozie_app/config-default.xml b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/raw_claims/oozie_app/config-default.xml new file mode 100644 index 0000000000..2e0ed9aeea --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/raw_claims/oozie_app/config-default.xml @@ -0,0 +1,18 @@ + + + jobTracker + yarnRM + + + nameNode + hdfs://nameservice1 + + + oozie.use.system.libpath + true + + + oozie.action.sharelib.for.spark + spark2 + + \ No newline at end of file diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/raw_claims/oozie_app/workflow.xml b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/raw_claims/oozie_app/workflow.xml new file mode 100644 index 0000000000..66eaeeb263 --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/raw_claims/oozie_app/workflow.xml @@ -0,0 +1,157 @@ + + + + reuseContent + false + should import content from the aggregator or reuse a previous version + + + contentPath + path location to store (or reuse) content from the aggregator + + + postgresURL + the postgres URL to access to the database + + + postgresUser + the user postgres + + + postgresPassword + the password postgres + + + dbSchema + beta + the database schema according to the D-Net infrastructure (beta or production) + + + mongoURL + mongoDB url, example: mongodb://[username:password@]host[:port] + + + mongoDb + mongo database + + + isLookupUrl + the address of the lookUp service + + + + sparkDriverMemory + memory for driver process + + + sparkExecutorMemory + memory for individual executor + + + sparkExecutorCores + number of cores used by single executor + + + oozieActionShareLibForSpark2 + oozie action sharelib for spark 2.* + + + spark2ExtraListeners + com.cloudera.spark.lineage.NavigatorAppListener + spark 2.* extra listeners classname + + + spark2SqlQueryExecutionListeners + com.cloudera.spark.lineage.NavigatorQueryListener + spark 2.* sql query execution listeners classname + + + spark2YarnHistoryServerAddress + spark 2.* yarn history server address + + + spark2EventLogDir + spark 2.* event log dir location + + + + + ${jobTracker} + ${nameNode} + + + mapreduce.job.queuename + ${queueName} + + + oozie.launcher.mapred.job.queue.name + ${oozieLauncherQueueName} + + + oozie.action.sharelib.for.spark + ${oozieActionShareLibForSpark2} + + + + + + + + Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] + + + + + + + + eu.dnetlib.dhp.oa.graph.raw.MigrateDbEntitiesApplication + --hdfsPath${contentPath}/db_claims + --postgresUrl${postgresURL} + --postgresUser${postgresUser} + --postgresPassword${postgresPassword} + --isLookupUrl${isLookupUrl} + --actionclaims + --dbschema${dbSchema} + + + + + + + + + + + eu.dnetlib.dhp.oa.graph.raw.MigrateMongoMdstoresApplication + -p${contentPath}/odf_claims + -mongourl${mongoURL} + -mongodb${mongoDb} + -fODF + -lstore + -iclaim + + + + + + + + + + + eu.dnetlib.dhp.oa.graph.raw.MigrateMongoMdstoresApplication + -p${contentPath}/oaf_claims + -mongourl${mongoURL} + -mongodb${mongoDb} + -fOAF + -lstore + -iclaim + + + + + + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/raw_claims/workflow.xml b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/raw_claims/workflow.xml deleted file mode 100644 index 1ac456976d..0000000000 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/raw_claims/workflow.xml +++ /dev/null @@ -1,169 +0,0 @@ - - - - migrationClaimsPathStep1 - the base path to store hdfs file - - - migrationClaimsPathStep2 - the temporary path to store entities before dispatching - - - migrationClaimsPathStep3 - the graph Raw base path - - - postgresURL - the postgres URL to access to the database - - - postgresUser - the user postgres - - - postgresPassword - the password postgres - - - mongoURL - mongoDB url, example: mongodb://[username:password@]host[:port] - - - mongoDb - mongo database - - - sparkDriverMemory - memory for driver process - - - sparkExecutorMemory - memory for individual executor - - - sparkExecutorCores - number of cores used by single executor - - - - - - - Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] - - - - - - - - - - - - - - ${jobTracker} - ${nameNode} - eu.dnetlib.dhp.migration.step1.MigrateDbEntitiesApplication - -p${migrationClaimsPathStep1}/db_claims - -pgurl${postgresURL} - -pguser${postgresUser} - -pgpasswd${postgresPassword} - -aclaims - - - - - - - - ${jobTracker} - ${nameNode} - eu.dnetlib.dhp.migration.step1.MigrateMongoMdstoresApplication - -p${migrationClaimsPathStep1}/odf_claims - -mongourl${mongoURL} - -mongodb${mongoDb} - -fODF - -lstore - -iclaim - - - - - - - - ${jobTracker} - ${nameNode} - eu.dnetlib.dhp.migration.step1.MigrateMongoMdstoresApplication - -p${migrationClaimsPathStep1}/oaf_claims - -mongourl${mongoURL} - -mongodb${mongoDb} - -fOAF - -lstore - -iclaim - - - - - - - - - - - - - - - - - ${jobTracker} - ${nameNode} - yarn-cluster - cluster - GenerateClaimEntities - eu.dnetlib.dhp.migration.step2.GenerateEntitiesApplication - dhp-aggregation-${projectVersion}.jar - --executor-memory ${sparkExecutorMemory} --executor-cores ${sparkExecutorCores} --driver-memory=${sparkDriverMemory} --conf spark.extraListeners="com.cloudera.spark.lineage.NavigatorAppListener" --conf spark.sql.queryExecutionListeners="com.cloudera.spark.lineage.NavigatorQueryListener" --conf spark.sql.warehouse.dir="/user/hive/warehouse" - -mt yarn-cluster - -s${migrationClaimsPathStep1}/db_claims,${migrationClaimsPathStep1}/oaf_claims,${migrationClaimsPathStep1}/odf_claims - -t${migrationClaimsPathStep2}/claim_entities - -pgurl${postgresURL} - -pguser${postgresUser} - -pgpasswd${postgresPassword} - - - - - - - - - - - - - - - - - ${jobTracker} - ${nameNode} - yarn-cluster - cluster - GenerateClaimGraph - eu.dnetlib.dhp.migration.step3.DispatchEntitiesApplication - dhp-aggregation-${projectVersion}.jar - --executor-memory ${sparkExecutorMemory} --executor-cores ${sparkExecutorCores} --driver-memory=${sparkDriverMemory} --conf spark.extraListeners="com.cloudera.spark.lineage.NavigatorAppListener" --conf spark.sql.queryExecutionListeners="com.cloudera.spark.lineage.NavigatorQueryListener" --conf spark.sql.warehouse.dir="/user/hive/warehouse" - -mt yarn-cluster - -s${migrationClaimsPathStep2}/claim_entities - -g${migrationClaimsPathStep3} - - - - - - - \ No newline at end of file diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/raw_db/oozie_app/workflow.xml b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/raw_db/oozie_app/workflow.xml index 05b85a5615..575f9229ed 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/raw_db/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/raw_db/oozie_app/workflow.xml @@ -16,6 +16,10 @@ postgresPassword the password postgres + + isLookupUrl + the address of the lookUp service + sparkDriverMemory @@ -88,6 +92,7 @@ --postgresUrl${postgresURL} --postgresUser${postgresUser} --postgresPassword${postgresPassword} + --isLookupUrl${isLookupUrl} @@ -103,6 +108,7 @@ --postgresUrl${postgresURL} --postgresUser${postgresUser} --postgresPassword${postgresPassword} + --isLookupUrl${isLookupUrl} --actionclaims diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/raw_step1/oozie_app/workflow.xml b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/raw_step1/oozie_app/workflow.xml index f16e22f957..8684181526 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/raw_step1/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/raw_step1/oozie_app/workflow.xml @@ -24,6 +24,10 @@ mongoDb mongo database + + isLookupUrl + the address of the lookUp service + sparkDriverMemory memory for driver process @@ -62,6 +66,7 @@ -pgurl${postgresURL} -pguser${postgresUser} -pgpasswd${postgresPassword} + -islookup${isLookupUrl} diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/raw_step2/oozie_app/workflow.xml b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/raw_step2/oozie_app/workflow.xml index cd0a4025e8..f6485ea9c3 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/raw_step2/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/raw_step2/oozie_app/workflow.xml @@ -9,17 +9,10 @@ the temporary path to store entities before dispatching - postgresURL - the postgres URL to access to the database - - - postgresUser - the user postgres - - - postgresPassword - the password postgres + isLookupUrl + the address of the lookUp service + sparkDriverMemory memory for driver process @@ -62,9 +55,7 @@ -mt yarn-cluster -s${migrationPathStep1}/db_records,${migrationPathStep1}/oaf_records,${migrationPathStep1}/odf_records -t${migrationPathStep2}/all_entities - -pgurl${postgresURL} - -pguser${postgresUser} - -pgpasswd${postgresPassword} + --islookup${isLookupUrl} diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryDatasourceOrganization.sql b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryDatasourceOrganization.sql index 745f83971a..687377aa48 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryDatasourceOrganization.sql +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryDatasourceOrganization.sql @@ -1,17 +1,16 @@ SELECT - dor.datasource AS datasource, - dor.organization AS organization, - NULL AS startdate, - NULL AS enddate, - false AS inferred, - false AS deletedbyinference, - 0.9 AS trust, - NULL AS inferenceprovenance, - dc.id AS collectedfromid, - dc.officialname AS collectedfromname, - 'providedBy@@@provided by@@@dnet:datasources_organizations_typologies@@@dnet:datasources_organizations_typologies' AS semantics, - d.provenanceaction || '@@@' || d.provenanceaction || '@@@dnet:provenanceActions@@@dnet:provenanceActions' AS provenanceaction - + dor.datasource AS datasource, + dor.organization AS organization, + NULL AS startdate, + NULL AS enddate, + false AS inferred, + false AS deletedbyinference, + 0.9 AS trust, + NULL AS inferenceprovenance, + dc.id AS collectedfromid, + dc.officialname AS collectedfromname, + 'providedBy@@@dnet:datasources_organizations_typologies' AS semantics, + d.provenanceaction || '@@@dnet:provenanceActions' AS provenanceaction FROM dsm_datasource_organization dor - LEFT OUTER JOIN dsm_datasources d ON (dor.datasource = d.id) + LEFT OUTER JOIN dsm_datasources d ON (dor.datasource = d.id) LEFT OUTER JOIN dsm_datasources dc ON (dc.id = d.collectedfrom) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryDatasources.sql b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryDatasources.sql index ff1178c711..43b0f8f4b1 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryDatasources.sql +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryDatasources.sql @@ -7,36 +7,36 @@ SELECT CASE WHEN (array_agg(DISTINCT COALESCE (a.compatibility_override, a.compatibility):: TEXT) @> ARRAY ['openaire-cris_1.1']) THEN - 'openaire-cris_1.1@@@OpenAIRE CRIS v1.1@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel' + 'openaire-cris_1.1@@@dnet:datasourceCompatibilityLevel' WHEN (array_agg(DISTINCT COALESCE (a.compatibility_override, a.compatibility):: TEXT) @> ARRAY ['openaire4.0']) THEN - 'openaire4.0@@@OpenAIRE 4.0@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel' + 'openaire4.0@@@dnet:datasourceCompatibilityLevel' WHEN (array_agg(DISTINCT COALESCE (a.compatibility_override, a.compatibility):: TEXT) @> ARRAY ['driver', 'openaire2.0']) THEN - 'driver-openaire2.0@@@OpenAIRE 2.0+ (DRIVER OA, EC funding)@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel' + 'driver-openaire2.0@@@dnet:datasourceCompatibilityLevel' WHEN (array_agg(DISTINCT COALESCE (a.compatibility_override, a.compatibility) :: TEXT) @> ARRAY ['driver']) THEN - 'driver@@@OpenAIRE Basic (DRIVER OA)@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel' + 'driver@@@dnet:datasourceCompatibilityLevel' WHEN (array_agg(DISTINCT COALESCE (a.compatibility_override, a.compatibility) :: TEXT) @> ARRAY ['openaire2.0']) THEN - 'openaire2.0@@@OpenAIRE 2.0 (EC funding)@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel' + 'openaire2.0@@@dnet:datasourceCompatibilityLevel' WHEN (array_agg(DISTINCT COALESCE (a.compatibility_override, a.compatibility) :: TEXT) @> ARRAY ['openaire3.0']) THEN - 'openaire3.0@@@OpenAIRE 3.0 (OA, funding)@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel' + 'openaire3.0@@@dnet:datasourceCompatibilityLevel' WHEN (array_agg(DISTINCT COALESCE (a.compatibility_override, a.compatibility) :: TEXT) @> ARRAY ['openaire2.0_data']) THEN - 'openaire2.0_data@@@OpenAIRE Data (funded, referenced datasets)@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel' + 'openaire2.0_data@@@dnet:datasourceCompatibilityLevel' WHEN (array_agg(DISTINCT COALESCE (a.compatibility_override, a.compatibility) :: TEXT) @> ARRAY ['native']) THEN - 'native@@@proprietary@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel' + 'native@@@dnet:datasourceCompatibilityLevel' WHEN (array_agg(DISTINCT COALESCE (a.compatibility_override, a.compatibility) :: TEXT) @> ARRAY ['hostedBy']) THEN - 'hostedBy@@@collected from a compatible aggregator@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel' + 'hostedBy@@@dnet:datasourceCompatibilityLevel' WHEN (array_agg(DISTINCT COALESCE (a.compatibility_override, a.compatibility) :: TEXT) @> ARRAY ['notCompatible']) THEN - 'notCompatible@@@under validation@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel' + 'notCompatible@@@dnet:datasourceCompatibilityLevel' ELSE - 'UNKNOWN@@@not available@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel' + 'UNKNOWN@@@dnet:datasourceCompatibilityLevel' END AS openairecompatibility, d.websiteurl AS websiteurl, d.logourl AS logourl, @@ -47,7 +47,7 @@ SELECT NULL AS odnumberofitems, NULL AS odnumberofitemsdate, - (SELECT array_agg(s|| '###keywords@@@keywords@@@dnet:subject_classification_typologies@@@dnet:subject_classification_typologies') + (SELECT array_agg(s|| '###keywords@@@dnet:subject_classification_typologies') FROM UNNEST( ARRAY( SELECT trim(s) @@ -83,32 +83,9 @@ SELECT ARRAY[]::text[] AS policies, dc.id AS collectedfromid, dc.officialname AS collectedfromname, - d.typology || '@@@' || CASE - WHEN (d.typology = 'crissystem') THEN 'CRIS System' - WHEN (d.typology = 'datarepository::unknown') THEN 'Data Repository' - WHEN (d.typology = 'aggregator::datarepository') THEN 'Data Repository Aggregator' - WHEN (d.typology = 'infospace') THEN 'Information Space' - WHEN (d.typology = 'pubsrepository::institutional') THEN 'Institutional Repository' - WHEN (d.typology = 'aggregator::pubsrepository::institutional') THEN 'Institutional Repository Aggregator' - WHEN (d.typology = 'pubsrepository::journal') THEN 'Journal' - WHEN (d.typology = 'aggregator::pubsrepository::journals') THEN 'Journal Aggregator/Publisher' - WHEN (d.typology = 'pubsrepository::mock') THEN 'Other' - WHEN (d.typology = 'pubscatalogue::unknown') THEN 'Publication Catalogue' - WHEN (d.typology = 'pubsrepository::unknown') THEN 'Publication Repository' - WHEN (d.typology = 'aggregator::pubsrepository::unknown') THEN 'Publication Repository Aggregator' - WHEN (d.typology = 'entityregistry') THEN 'Registry' - WHEN (d.typology = 'scholarcomminfra') THEN 'Scholarly Comm. Infrastructure' - WHEN (d.typology = 'pubsrepository::thematic') THEN 'Thematic Repository' - WHEN (d.typology = 'websource') THEN 'Web Source' - WHEN (d.typology = 'entityregistry::projects') THEN 'Funder database' - WHEN (d.typology = 'entityregistry::repositories') THEN 'Registry of repositories' - WHEN (d.typology = 'softwarerepository') THEN 'Software Repository' - WHEN (d.typology = 'aggregator::softwarerepository') THEN 'Software Repository Aggregator' - WHEN (d.typology = 'orprepository') THEN 'Repository' - ELSE 'Other' - END || '@@@dnet:datasource_typologies@@@dnet:datasource_typologies' AS datasourcetype, - 'sysimport:crosswalk:entityregistry@@@sysimport:crosswalk:entityregistry@@@dnet:provenance_actions@@@dnet:provenance_actions' AS provenanceaction, - CONCAT(d.issn, ' @@@ ', d.eissn, ' @@@ ', d.lissn) AS journal + d.typology||'@@@dnet:datasource_typologies' AS datasourcetype, + 'sysimport:crosswalk:entityregistry@@@dnet:provenance_actions' AS provenanceaction, + d.issn || ' @@@ ' || d.eissn || ' @@@ ' || d.lissn AS journal FROM dsm_datasources d diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryOrganizations.sql b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryOrganizations.sql index d13bd43425..3e5de80711 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryOrganizations.sql +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryOrganizations.sql @@ -22,13 +22,12 @@ SELECT '' AS inferenceprovenance, d.id AS collectedfromid, d.officialname AS collectedfromname, - o.country || '@@@' || COALESCE(cntr.name,o.country) || '@@@dnet:countries@@@dnet:countries' AS country, - 'sysimport:crosswalk:entityregistry@@@sysimport:crosswalk:entityregistry@@@dnet:provenance_actions@@@dnet:provenance_actions' AS provenanceaction, + o.country || '@@@dnet:countries' AS country, + 'sysimport:crosswalk:entityregistry@@@dnet:provenance_actions' AS provenanceaction, ARRAY[]::text[] AS pid FROM dsm_organizations o LEFT OUTER JOIN dsm_datasources d ON (d.id = o.collectedfrom) - LEFT OUTER JOIN class cntr ON (cntr.code = o.country) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryOrganizationsFromOpenOrgsDB.sql b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryOrganizationsFromOpenOrgsDB.sql index 99c8e04b4b..3396f365c9 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryOrganizationsFromOpenOrgsDB.sql +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryOrganizationsFromOpenOrgsDB.sql @@ -11,8 +11,8 @@ SELECT '' AS inferenceprovenance, 'openaire____::openorgs' AS collectedfromid, 'OpenOrgs Database' AS collectedfromname, - o.country || '@@@' || o.country || '@@@dnet:countries@@@dnet:countries' AS country, - 'sysimport:crosswalk:entityregistry@@@sysimport:crosswalk:entityregistry@@@dnet:provenance_actions@@@dnet:provenance_actions' AS provenanceaction, + o.country || '@@@dnet:countries' AS country, + 'sysimport:crosswalk:entityregistry@@@dnet:provenance_actions' AS provenanceaction, array_agg(DISTINCT i.otherid || '###' || i.type || '@@@dnet:pid_types') AS pid FROM organizations o LEFT OUTER JOIN acronyms a ON (a.id = o.id) @@ -40,8 +40,8 @@ SELECT '' AS inferenceprovenance, 'openaire____::openorgs' AS collectedfromid, 'OpenOrgs Database' AS collectedfromname, - o.country || '@@@' || o.country || '@@@dnet:countries@@@dnet:countries' AS country, - 'sysimport:crosswalk:entityregistry@@@sysimport:crosswalk:entityregistry@@@dnet:provenance_actions@@@dnet:provenance_actions' AS provenanceaction, + o.country || '@@@dnet:countries' AS country, + 'sysimport:crosswalk:entityregistry@@@dnet:provenance_actions' AS provenanceaction, array_agg(DISTINCT i.otherid || '###' || i.type || '@@@dnet:pid_types') AS pid FROM other_names n LEFT OUTER JOIN organizations o ON (n.id = o.id) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryProjectOrganization.sql b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryProjectOrganization.sql index 4c06ca5b9a..13cfca8715 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryProjectOrganization.sql +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryProjectOrganization.sql @@ -11,8 +11,8 @@ SELECT NULL AS inferenceprovenance, dc.id AS collectedfromid, dc.officialname AS collectedfromname, - po.semanticclass || '@@@' || po.semanticclass || '@@@dnet:project_organization_relations@@@dnet:project_organization_relations' AS semantics, - 'sysimport:crosswalk:entityregistry@@@sysimport:crosswalk:entityregistry@@@dnet:provenance_actions@@@dnet:provenance_actions' AS provenanceaction + po.semanticclass || '@@@dnet:project_organization_relations' AS semantics, + 'sysimport:crosswalk:entityregistry@@@dnet:provenance_actions' AS provenanceaction FROM project_organization po LEFT OUTER JOIN projects p ON (p.id = po.project) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryProjects.sql b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryProjects.sql index 685b57ab65..db0da83f77 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryProjects.sql +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryProjects.sql @@ -31,17 +31,14 @@ SELECT p.fundedamount AS fundedamount, dc.id AS collectedfromid, dc.officialname AS collectedfromname, - p.contracttype || '@@@' || p.contracttypename || '@@@' || p.contracttypescheme || '@@@' || p.contracttypescheme AS contracttype, - pac.code || '@@@' || pac.name || '@@@' || pas.code || '@@@' || pas.name AS provenanceaction, + p.contracttype || '@@@' || p.contracttypescheme AS contracttype, + p.provenanceactionclass || '@@@' || p.provenanceactionscheme AS provenanceaction, array_agg(DISTINCT i.pid || '###' || i.issuertype) AS pid, - array_agg(DISTINCT s.name || '###' || sc.code || '@@@' || sc.name || '@@@' || ss.code || '@@@' || ss.name) AS subjects, + array_agg(DISTINCT s.name || '###' || s.semanticclass || '@@@' || s.semanticscheme) AS subjects, array_agg(DISTINCT fp.path) AS fundingtree FROM projects p - LEFT OUTER JOIN class pac ON (pac.code = p.provenanceactionclass) - LEFT OUTER JOIN scheme pas ON (pas.code = p.provenanceactionscheme) - LEFT OUTER JOIN projectpids pp ON (pp.project = p.id) LEFT OUTER JOIN dsm_identities i ON (i.pid = pp.pid) @@ -53,9 +50,6 @@ SELECT LEFT OUTER JOIN project_subject ps ON (ps.project = p.id) LEFT OUTER JOIN subjects s ON (s.id = ps.subject) - LEFT OUTER JOIN class sc ON (sc.code = s.semanticclass) - LEFT OUTER JOIN scheme ss ON (ss.code = s.semanticscheme) - GROUP BY p.id, p.code, @@ -85,5 +79,6 @@ SELECT p.fundedamount, dc.id, dc.officialname, - pac.code, pac.name, pas.code, pas.name, - p.contracttype , p.contracttypename, p.contracttypescheme; \ No newline at end of file + p.contracttype, + p.contracttypescheme; + \ No newline at end of file diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryProjects_production.sql b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryProjects_production.sql index 6cff188756..234bb7c3eb 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryProjects_production.sql +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryProjects_production.sql @@ -28,18 +28,15 @@ SELECT p.summary AS summary, p.currency AS currency, p.totalcost AS totalcost, - p.fundedamount AS fundedamount, + p.fundedamount AS fundedamount, dc.id AS collectedfromid, dc.officialname AS collectedfromname, - ctc.code || '@@@' || ctc.name || '@@@' || cts.code || '@@@' || cts.name AS contracttype, - pac.code || '@@@' || pac.name || '@@@' || pas.code || '@@@' || pas.name AS provenanceaction, + p.contracttypeclass || '@@@' || p.contracttypescheme AS contracttype, + p.provenanceactionclass || '@@@' || p.provenanceactionscheme AS provenanceaction, array_agg(DISTINCT i.pid || '###' || i.issuertype) AS pid, - array_agg(DISTINCT s.name || '###' || sc.code || '@@@' || sc.name || '@@@' || ss.code || '@@@' || ss.name) AS subjects, + array_agg(DISTINCT s.name || '###' || s.semanticclass || '@@@' || s.semanticscheme) AS subjects, array_agg(DISTINCT fp.path) AS fundingtree FROM projects p - LEFT OUTER JOIN class pac ON (pac.code = p.provenanceactionclass) - LEFT OUTER JOIN scheme pas ON (pas.code = p.provenanceactionscheme) - LEFT OUTER JOIN projectpids pp ON (pp.project = p.id) LEFT OUTER JOIN dsm_identities i ON (i.pid = pp.pid) @@ -51,12 +48,6 @@ SELECT LEFT OUTER JOIN project_subject ps ON (ps.project = p.id) LEFT OUTER JOIN subjects s ON (s.id = ps.subject) - LEFT OUTER JOIN class sc ON (sc.code = s.semanticclass) - LEFT OUTER JOIN scheme ss ON (ss.code = s.semanticscheme) - - LEFT OUTER JOIN class ctc ON (ctc.code = p.contracttypeclass) - LEFT OUTER JOIN scheme cts ON (cts.code = p.contracttypescheme) - GROUP BY p.id, p.code, @@ -85,6 +76,6 @@ SELECT p.totalcost, p.fundedamount, dc.id, - dc.officialname, - pac.code, pac.name, pas.code, pas.name, - ctc.code, ctc.name, cts.code, cts.name; \ No newline at end of file + dc.officialname + + \ No newline at end of file diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/xquery/load_vocabularies.xquery b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/xquery/load_vocabularies.xquery new file mode 100644 index 0000000000..4938c0aa42 --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/xquery/load_vocabularies.xquery @@ -0,0 +1,5 @@ +for $x in collection(' /db/DRIVER/VocabularyDSResources/VocabularyDSResourceType') + let $vocid := $x//VOCABULARY_NAME/@code + let $vocname := $x//VOCABULARY_NAME/text() + for $term in ($x//TERM) + return concat($vocid,' @=@ ',$vocname,' @=@ ',$term/@code,' @=@ ',$term/@english_name) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/xquery/load_vocabularies_synonyms.xquery b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/xquery/load_vocabularies_synonyms.xquery new file mode 100644 index 0000000000..f4f8cb45de --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/xquery/load_vocabularies_synonyms.xquery @@ -0,0 +1,6 @@ +for $x in collection('/db/DRIVER/VocabularyDSResources/VocabularyDSResourceType') +let $vocid := $x//VOCABULARY_NAME/@code +let $vocname := $x//VOCABULARY_NAME/text() + for $term in ($x//TERM) + for $syn in ($term//SYNONYM/@term) + return concat($vocid,' @=@ ',$term/@code,' @=@ ', $syn) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/graph/step1/oozie_app/workflow.xml b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/graph/step1/oozie_app/workflow.xml index ce00eff7b3..d74d68663f 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/graph/step1/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/graph/step1/oozie_app/workflow.xml @@ -47,8 +47,8 @@ - ${wf:conf('reuseContent') eq false} - ${wf:conf('reuseContent') eq true} + ${wf:conf('reuseContent') eq false} + ${wf:conf('reuseContent') eq true} diff --git a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/clean/CleaningFunctionTest.java b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/clean/CleaningFunctionTest.java new file mode 100644 index 0000000000..559a30b1ea --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/clean/CleaningFunctionTest.java @@ -0,0 +1,118 @@ + +package eu.dnetlib.dhp.oa.graph.clean; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.lenient; + +import java.io.IOException; +import java.util.List; +import java.util.Set; +import java.util.stream.Stream; + +import org.apache.commons.io.IOUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.dhp.oa.graph.raw.common.VocabularyGroup; +import eu.dnetlib.dhp.schema.oaf.Publication; +import eu.dnetlib.dhp.schema.oaf.Qualifier; +import eu.dnetlib.dhp.schema.oaf.Result; +import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException; +import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; + +@ExtendWith(MockitoExtension.class) +public class CleaningFunctionTest { + + public static final ObjectMapper MAPPER = new ObjectMapper(); + + @Mock + private ISLookUpService isLookUpService; + + private VocabularyGroup vocabularies; + + private CleaningRuleMap mapping; + + @BeforeEach + public void setUp() throws ISLookUpException, IOException { + lenient().when(isLookUpService.quickSearchProfile(VocabularyGroup.VOCABULARIES_XQUERY)).thenReturn(vocs()); + lenient() + .when(isLookUpService.quickSearchProfile(VocabularyGroup.VOCABULARY_SYNONYMS_XQUERY)) + .thenReturn(synonyms()); + + vocabularies = VocabularyGroup.loadVocsFromIS(isLookUpService); + mapping = CleaningRuleMap.create(vocabularies); + } + + @Test + public void testCleaning() throws Exception { + + assertNotNull(vocabularies); + assertNotNull(mapping); + + String json = IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/dhp/oa/graph/clean/result.json")); + Publication p_in = MAPPER.readValue(json, Publication.class); + + assertNull(p_in.getBestaccessright()); + + assertTrue(p_in instanceof Result); + assertTrue(p_in instanceof Publication); + + Publication p_out = OafCleaner.apply(p_in, mapping); + + assertNotNull(p_out); + + assertEquals("und", p_out.getLanguage().getClassid()); + assertEquals("Undetermined", p_out.getLanguage().getClassname()); + + assertEquals("DE", p_out.getCountry().get(0).getClassid()); + assertEquals("Germany", p_out.getCountry().get(0).getClassname()); + + assertEquals("0018", p_out.getInstance().get(0).getInstancetype().getClassid()); + assertEquals("Annotation", p_out.getInstance().get(0).getInstancetype().getClassname()); + + assertEquals("CLOSED", p_out.getInstance().get(0).getAccessright().getClassid()); + assertEquals("Closed Access", p_out.getInstance().get(0).getAccessright().getClassname()); + + Set pidTerms = vocabularies.getTerms("dnet:pid_types"); + assertTrue( + p_out + .getPid() + .stream() + .map(p -> p.getQualifier()) + .allMatch(q -> pidTerms.contains(q.getClassid()))); + + Publication p_defaults = CleanGraphSparkJob.fixDefaults(p_out); + assertEquals("CLOSED", p_defaults.getBestaccessright().getClassid()); + + // TODO add more assertions to verity the cleaned values + System.out.println(MAPPER.writeValueAsString(p_out)); + + /* + * assertTrue( p_out .getPid() .stream() .allMatch(sp -> StringUtils.isNotBlank(sp.getValue()))); + */ + } + + private Stream getAuthorPidTypes(Publication pub) { + return pub + .getAuthor() + .stream() + .map(a -> a.getPid()) + .flatMap(p -> p.stream()) + .map(s -> s.getQualifier()); + } + + private List vocs() throws IOException { + return IOUtils + .readLines(CleaningFunctionTest.class.getResourceAsStream("/eu/dnetlib/dhp/oa/graph/clean/terms.txt")); + } + + private List synonyms() throws IOException { + return IOUtils + .readLines(CleaningFunctionTest.class.getResourceAsStream("/eu/dnetlib/dhp/oa/graph/clean/synonyms.txt")); + } +} diff --git a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MappersTest.java b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MappersTest.java index b9da9fb297..b1f0ecf0dd 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MappersTest.java +++ b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MappersTest.java @@ -2,6 +2,7 @@ package eu.dnetlib.dhp.oa.graph.raw; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.anyString; @@ -9,7 +10,6 @@ import static org.mockito.Mockito.when; import java.io.IOException; import java.util.List; -import java.util.Map; import java.util.Optional; import org.apache.commons.io.IOUtils; @@ -20,6 +20,8 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import eu.dnetlib.dhp.oa.graph.raw.common.OafMapperUtils; +import eu.dnetlib.dhp.oa.graph.raw.common.VocabularyGroup; import eu.dnetlib.dhp.schema.common.ModelConstants; import eu.dnetlib.dhp.schema.oaf.Author; import eu.dnetlib.dhp.schema.oaf.Dataset; @@ -34,18 +36,27 @@ import eu.dnetlib.dhp.schema.oaf.StructuredProperty; public class MappersTest { @Mock - private Map code2name; + private VocabularyGroup vocs; @BeforeEach public void setUp() throws Exception { - when(code2name.get(anyString())).thenAnswer(invocation -> invocation.getArgument(0)); + when(vocs.getTermAsQualifier(anyString(), anyString())) + .thenAnswer( + invocation -> OafMapperUtils + .qualifier( + invocation.getArgument(1), invocation.getArgument(1), invocation.getArgument(0), + invocation.getArgument(0))); + + when(vocs.termExists(anyString(), anyString())).thenReturn(true); + } @Test void testPublication() throws IOException { + final String xml = IOUtils.toString(getClass().getResourceAsStream("oaf_record.xml")); - final List list = new OafToOafMapper(code2name).processMdRecord(xml); + final List list = new OafToOafMapper(vocs, false).processMdRecord(xml); assertEquals(3, list.size()); assertTrue(list.get(0) instanceof Publication); @@ -59,6 +70,7 @@ public class MappersTest { assertValidId(p.getId()); assertValidId(p.getCollectedfrom().get(0).getKey()); assertTrue(StringUtils.isNotBlank(p.getTitle().get(0).getValue())); + assertFalse(p.getDataInfo().getInvisible()); assertTrue(p.getAuthor().size() > 0); final Optional author = p @@ -86,6 +98,10 @@ public class MappersTest { assertTrue(StringUtils.isNotBlank(p.getJournal().getIssnOnline())); assertTrue(StringUtils.isNotBlank(p.getJournal().getName())); + assertTrue(p.getPid().size() > 0); + assertEquals(p.getPid().get(0).getValue(), "10.3897/oneeco.2.e13718"); + assertEquals(p.getPid().get(0).getQualifier().getClassid(), "doi"); + assertNotNull(p.getInstance()); assertTrue(p.getInstance().size() > 0); p @@ -95,6 +111,7 @@ public class MappersTest { assertNotNull(i.getAccessright()); assertEquals("OPEN", i.getAccessright().getClassid()); }); + assertEquals("0001", p.getInstance().get(0).getRefereed().getClassid()); assertNotNull(p.getBestaccessright()); assertEquals("OPEN", p.getBestaccessright().getClassid()); @@ -115,15 +132,32 @@ public class MappersTest { assertTrue(StringUtils.isNotBlank(r1.getRelType())); assertTrue(StringUtils.isNotBlank(r2.getRelType())); + // System.out.println(new ObjectMapper().writeValueAsString(p)); // System.out.println(new ObjectMapper().writeValueAsString(r1)); // System.out.println(new ObjectMapper().writeValueAsString(r2)); } + @Test + void testPublicationInvisible() throws IOException { + + final String xml = IOUtils.toString(getClass().getResourceAsStream("oaf_record.xml")); + + final List list = new OafToOafMapper(vocs, true).processMdRecord(xml); + + assertTrue(list.size() > 0); + assertTrue(list.get(0) instanceof Publication); + + final Publication p = (Publication) list.get(0); + + assertTrue(p.getDataInfo().getInvisible()); + + } + @Test void testDataset() throws IOException { final String xml = IOUtils.toString(getClass().getResourceAsStream("odf_dataset.xml")); - final List list = new OdfToOafMapper(code2name).processMdRecord(xml); + final List list = new OdfToOafMapper(vocs, false).processMdRecord(xml); assertEquals(3, list.size()); assertTrue(list.get(0) instanceof Dataset); @@ -184,6 +218,7 @@ public class MappersTest { assertNotNull(i.getAccessright()); assertEquals("OPEN", i.getAccessright().getClassid()); }); + assertEquals("0001", d.getInstance().get(0).getRefereed().getClassid()); assertValidId(r1.getSource()); assertValidId(r1.getTarget()); @@ -205,7 +240,7 @@ public class MappersTest { void testSoftware() throws IOException { final String xml = IOUtils.toString(getClass().getResourceAsStream("odf_software.xml")); - final List list = new OdfToOafMapper(code2name).processMdRecord(xml); + final List list = new OdfToOafMapper(vocs, false).processMdRecord(xml); assertEquals(1, list.size()); assertTrue(list.get(0) instanceof Software); diff --git a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MigrateDbEntitiesApplicationTest.java b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MigrateDbEntitiesApplicationTest.java index 1bbe57ee83..22fcb36c98 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MigrateDbEntitiesApplicationTest.java +++ b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MigrateDbEntitiesApplicationTest.java @@ -4,6 +4,8 @@ package eu.dnetlib.dhp.oa.graph.raw; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.lenient; import java.io.IOException; import java.sql.Array; @@ -25,6 +27,8 @@ import org.mockito.junit.jupiter.MockitoExtension; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; +import eu.dnetlib.dhp.oa.graph.raw.common.OafMapperUtils; +import eu.dnetlib.dhp.oa.graph.raw.common.VocabularyGroup; import eu.dnetlib.dhp.schema.oaf.Datasource; import eu.dnetlib.dhp.schema.oaf.Oaf; import eu.dnetlib.dhp.schema.oaf.Organization; @@ -40,9 +44,22 @@ public class MigrateDbEntitiesApplicationTest { @Mock private ResultSet rs; + @Mock + private VocabularyGroup vocs; + @BeforeEach public void setUp() { - this.app = new MigrateDbEntitiesApplication(); + lenient() + .when(vocs.getTermAsQualifier(anyString(), anyString())) + .thenAnswer( + invocation -> OafMapperUtils + .qualifier( + invocation.getArgument(1), invocation.getArgument(1), invocation.getArgument(0), + invocation.getArgument(0))); + + lenient().when(vocs.termExists(anyString(), anyString())).thenReturn(true); + + this.app = new MigrateDbEntitiesApplication(vocs); } @Test @@ -61,8 +78,7 @@ public class MigrateDbEntitiesApplicationTest { assertEquals(ds.getContactemail().getValue(), getValueAsString("contactemail", fields)); assertEquals(ds.getWebsiteurl().getValue(), getValueAsString("websiteurl", fields)); assertEquals(ds.getNamespaceprefix().getValue(), getValueAsString("namespaceprefix", fields)); - assertEquals( - ds.getCollectedfrom().get(0).getValue(), getValueAsString("collectedfromname", fields)); + assertEquals(ds.getCollectedfrom().get(0).getValue(), getValueAsString("collectedfromname", fields)); } @Test @@ -78,8 +94,7 @@ public class MigrateDbEntitiesApplicationTest { assertValidId(p.getCollectedfrom().get(0).getKey()); assertEquals(p.getAcronym().getValue(), getValueAsString("acronym", fields)); assertEquals(p.getTitle().getValue(), getValueAsString("title", fields)); - assertEquals( - p.getCollectedfrom().get(0).getValue(), getValueAsString("collectedfromname", fields)); + assertEquals(p.getCollectedfrom().get(0).getValue(), getValueAsString("collectedfromname", fields)); } @Test @@ -99,13 +114,10 @@ public class MigrateDbEntitiesApplicationTest { assertEquals(o.getLegalname().getValue(), getValueAsString("legalname", fields)); assertEquals(o.getWebsiteurl().getValue(), getValueAsString("websiteurl", fields)); assertEquals(o.getCountry().getClassid(), getValueAsString("country", fields).split("@@@")[0]); - assertEquals( - o.getCountry().getClassname(), getValueAsString("country", fields).split("@@@")[1]); - assertEquals(o.getCountry().getSchemeid(), getValueAsString("country", fields).split("@@@")[2]); - assertEquals( - o.getCountry().getSchemename(), getValueAsString("country", fields).split("@@@")[3]); - assertEquals( - o.getCollectedfrom().get(0).getValue(), getValueAsString("collectedfromname", fields)); + assertEquals(o.getCountry().getClassname(), getValueAsString("country", fields).split("@@@")[0]); + assertEquals(o.getCountry().getSchemeid(), getValueAsString("country", fields).split("@@@")[1]); + assertEquals(o.getCountry().getSchemename(), getValueAsString("country", fields).split("@@@")[1]); + assertEquals(o.getCollectedfrom().get(0).getValue(), getValueAsString("collectedfromname", fields)); } @Test diff --git a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/reflections/ReflectionTest.java b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/reflections/ReflectionTest.java new file mode 100644 index 0000000000..110fabf450 --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/reflections/ReflectionTest.java @@ -0,0 +1,200 @@ + +package eu.dnetlib.dhp.oa.graph.reflections; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.List; + +import org.junit.jupiter.api.Test; + +public class ReflectionTest { + + private final Cleaner cleaner = new Cleaner(); + + @Test + void testObject() throws Exception { + final Publication pub = new Publication(); + pub.setTitle("openaire guidelines"); + pub.getAuthors().add(new Author("Michele Artini", new Prop("aa-001", "orcid"))); + pub.getAuthors().add(new Author("Claudio Atzori", new Prop("aa-002", "orcid"))); + pub.getAuthors().add(new Author("Alessia Bardi", new Prop("aa-003", "orcid"))); + pub.getSubjects().add(new Prop("infrastructures", "keyword")); + pub.getSubjects().add(new Prop("digital libraries", "keyword")); + + cleaner.clean(pub); + + System.out.println(pub); + + assertEquals("OPENAIRE GUIDELINES", pub.getTitle()); + + assertEquals("MICHELE ARTINI", pub.getAuthors().get(0).getName()); + assertEquals("CLAUDIO ATZORI", pub.getAuthors().get(1).getName()); + assertEquals("ALESSIA BARDI", pub.getAuthors().get(2).getName()); + + assertEquals("dnet:aa-001", pub.getAuthors().get(0).getId().getId()); + assertEquals("dnet:aa-002", pub.getAuthors().get(1).getId().getId()); + assertEquals("dnet:aa-003", pub.getAuthors().get(2).getId().getId()); + assertEquals("dnet:orcid", pub.getAuthors().get(0).getId().getName()); + assertEquals("dnet:orcid", pub.getAuthors().get(1).getId().getName()); + assertEquals("dnet:orcid", pub.getAuthors().get(2).getId().getName()); + + assertEquals("dnet:infrastructures", pub.getSubjects().get(0).getId()); + assertEquals("dnet:keyword", pub.getSubjects().get(0).getName()); + assertEquals("dnet:digital libraries", pub.getSubjects().get(1).getId()); + assertEquals("dnet:keyword", pub.getSubjects().get(1).getName()); + } + +} + +class Cleaner { + + public void clean(final Object o) throws IllegalArgumentException, IllegalAccessException { + if (isPrimitive(o)) { + return; + } else if (isIterable(o.getClass())) { + for (final Object elem : (Iterable) o) { + clean(elem); + } + } else if (hasMapping(o)) { + mapObject(o); + } else { + for (final Field f : o.getClass().getDeclaredFields()) { + f.setAccessible(true); + final Object val = f.get(o); + if (isPrimitive(val)) { + f.set(o, cleanValue(f.get(o))); + } else if (hasMapping(val)) { + mapObject(val); + } else { + clean(f.get(o)); + } + } + } + } + + private boolean hasMapping(final Object o) { + return o.getClass() == Prop.class; + } + + private void mapObject(final Object o) { + if (o.getClass() == Prop.class) { + ((Prop) o).setId("dnet:" + ((Prop) o).getId()); + ((Prop) o).setName("dnet:" + ((Prop) o).getName()); + } + + } + + private Object cleanValue(final Object o) { + if (o.getClass() == String.class) { + return ((String) o).toUpperCase(); + } else { + return o; + } + + } + + private boolean isIterable(final Class cl) { + return Iterable.class.isAssignableFrom(cl); + } + + private boolean isPrimitive(final Object o) { + return o.getClass() == String.class; + } +} + +class Publication { + + private String title; + private final List authors = new ArrayList<>(); + private final List subjects = new ArrayList<>(); + + public String getTitle() { + return title; + } + + public void setTitle(final String title) { + this.title = title; + } + + public List getAuthors() { + return authors; + } + + public List getSubjects() { + return subjects; + } + + @Override + public String toString() { + return String.format("Publication [title=%s, authors=%s, subjects=%s]", title, authors, subjects); + } + +} + +class Prop { + + private String id; + private String name; + + public Prop(final String id, final String name) { + this.id = id; + this.name = name; + } + + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + @Override + public String toString() { + return String.format("Prop [id=%s, name=%s]", id, name); + } + +} + +class Author { + + private String name; + private Prop id; + + public Author(final String name, final Prop id) { + this.name = name; + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + public Prop getId() { + return id; + } + + public void setId(final Prop id) { + this.id = id; + } + + @Override + public String toString() { + return String.format("Author [name=%s, id=%s]", name, id); + } + +} diff --git a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/result.json b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/result.json new file mode 100644 index 0000000000..5d0c0d1ed8 --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/result.json @@ -0,0 +1,758 @@ +{ + "author": [ + { + "affiliation": [ + ], + "fullname": "Brien, Tom", + "name": "Tom", + "pid": [ + { + "dataInfo": { + "deletedbyinference": false, + "inferenceprovenance": "", + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:datasetarchive", + "classname": "sysimport:crosswalk:datasetarchive", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "qualifier": { + "classid": "ORCID12", + "classname": "ORCID12", + "schemeid": "dnet:pid_types", + "schemename": "dnet:pid_types" + }, + "value": "0000-0001-9613-6639" + } + ], + "rank": 1, + "surname": "Brien" + }, + { + "affiliation": [ + ], + "fullname": "Ade, Peter", + "name": "Peter", + "pid": [ + { + "dataInfo": { + "deletedbyinference": false, + "inferenceprovenance": "", + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:datasetarchive", + "classname": "sysimport:crosswalk:datasetarchive", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "qualifier": { + "classid": "xyz", + "classname": "XYZ", + "schemeid": "dnet:pid_types", + "schemename": "dnet:pid_types" + }, + "value": "qwerty" + } + ], + "rank": 2, + "surname": "Ade" + }, + { + "affiliation": [ + ], + "fullname": "Barry, Peter S.", + "name": "Peter S.", + "pid": [ + ], + "rank": 3, + "surname": "Barry" + }, + { + "affiliation": [ + ], + "fullname": "Dunscombe, Chris J.", + "name": "Chris J.", + "pid": [ + ], + "rank": 4, + "surname": "Dunscombe" + }, + { + "affiliation": [ + ], + "fullname": "Leadley, David R.", + "name": "David R.", + "pid": [ + ], + "rank": 5, + "surname": "Leadley" + }, + { + "affiliation": [ + ], + "fullname": "Morozov, Dmitry V.", + "name": "Dmitry V.", + "pid": [ + ], + "rank": 6, + "surname": "Morozov" + }, + { + "affiliation": [ + ], + "fullname": "Myronov, Maksym", + "name": "Maksym", + "pid": [ + ], + "rank": 7, + "surname": "Myronov" + }, + { + "affiliation": [ + ], + "fullname": "Parker, Evan", + "name": "Evan", + "pid": [ + ], + "rank": 8, + "surname": "Parker" + }, + { + "affiliation": [ + ], + "fullname": "Prest, Martin J.", + "name": "Martin J.", + "pid": [ + ], + "rank": 9, + "surname": "Prest" + }, + { + "affiliation": [ + ], + "fullname": "Prunnila, Mika", + "name": "Mika", + "pid": [ + ], + "rank": 10, + "surname": "Prunnila" + }, + { + "affiliation": [ + ], + "fullname": "Sudiwala, Rashmi V.", + "name": "Rashmi V.", + "pid": [ + ], + "rank": 11, + "surname": "Sudiwala" + }, + { + "affiliation": [ + ], + "fullname": "Whall, Terry E.", + "name": "Terry E.", + "pid": [ + ], + "rank": 12, + "surname": "Whall" + }, + { + "affiliation": [ + ], + "fullname": "Mauskopf", + "name": "", + "pid": [ + ], + "rank": 13, + "surname": "" + }, + { + "affiliation": [ + ], + "fullname": " P. D. ", + "name": "", + "pid": [ + ], + "rank": 14, + "surname": "" + } + ], + "bestaccessright": null, + "collectedfrom": [ + { + "key": "10|CSC_________::a2b9ce8435390bcbfc05f3cae3948747", + "value": "VIRTA" + } + ], + "context": [ + ], + "contributor": [ + ], + "country": [ + { + "classid": "DE", + "classname": "DE", + "schemeid": "dnet:countries", + "schemename": "dnet:countries" + } + ], + "coverage": [ + ], + "dataInfo": { + "deletedbyinference": false, + "inferenceprovenance": "", + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:datasetarchive", + "classname": "sysimport:crosswalk:datasetarchive", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "dateofacceptance": { + "dataInfo": { + "deletedbyinference": false, + "inferenceprovenance": "", + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:datasetarchive", + "classname": "sysimport:crosswalk:datasetarchive", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "value": "2016-01-01" + }, + "dateofcollection": "", + "dateoftransformation": "2020-04-22T12:34:08.009Z", + "description": [ + ], + "externalReference": [ + ], + "extraInfo": [ + ], + "format": [ + ], + "fulltext": [ + ], + "id": "50|CSC_________::2250a70c903c6ac6e4c01438259e9375", + "instance": [ + { + "accessright": { + "classid": "CLOSED", + "classname": "CLOSED", + "schemeid": "dnet:access_modes", + "schemename": "dnet:access_modes" + }, + "collectedfrom": { + "key": "10|CSC_________::a2b9ce8435390bcbfc05f3cae3948747", + "value": "VIRTA" + }, + "dateofacceptance": { + "dataInfo": { + "deletedbyinference": false, + "inferenceprovenance": "", + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:datasetarchive", + "classname": "sysimport:crosswalk:datasetarchive", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "value": "2016-01-01" + }, + "distributionlocation": "", + "hostedby": { + "key": "10|CSC_________::a2b9ce8435390bcbfc05f3cae3948747", + "value": "VIRTA" + }, + "instancetype": { + "classid": "Comment/debate", + "classname": "Comment/debate", + "schemeid": "dnet:publication_resource", + "schemename": "dnet:publication_resource" + }, + "url": [ + "http://juuli.fi/Record/0275158616", + "http://dx.doi.org/10.1007/s109090161569x" + ] + } + ], + "journal": { + "dataInfo": { + "deletedbyinference": false, + "inferenceprovenance": "", + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:datasetarchive", + "classname": "sysimport:crosswalk:datasetarchive", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "edition": "", + "ep": " 7", + "iss": "9 March", + "issnLinking": "", + "issnOnline": "", + "issnPrinted": "0022-2291", + "name": "Journal of Low Temperature Physics - Early Acces", + "sp": "1 ", + "vol": "" + }, + "language": { + "classid": "UNKNOWN", + "classname": "UNKNOWN", + "schemeid": "dnet:languages", + "schemename": "dnet:languages" + }, + "lastupdatetimestamp": 1591283286319, + "oaiprovenance": { + "originDescription": { + "altered": true, + "baseURL": "https%3A%2F%2Fvirta-jtp.csc.fi%2Fapi%2Fcerif", + "datestamp": "2019-07-30", + "harvestDate": "2020-04-22T11:04:38.685Z", + "identifier": "oai:virta-jtp.csc.fi:Publications/0275158616", + "metadataNamespace": "" + } + }, + "originalId": [ + "CSC_________::2250a70c903c6ac6e4c01438259e9375" + ], + "pid": [ + { + "dataInfo": { + "deletedbyinference": false, + "inferenceprovenance": "", + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:datasetarchive", + "classname": "sysimport:crosswalk:datasetarchive", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "qualifier": { + "classid": "doi", + "classname": "doi", + "schemeid": "dnet:pid_types", + "schemename": "dnet:pid_types" + }, + "value": "10.1007/s109090161569x" + }, + { + "dataInfo": { + "deletedbyinference": false, + "inferenceprovenance": "", + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:datasetarchive", + "classname": "sysimport:crosswalk:datasetarchive", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "qualifier": { + "classid": "doi", + "classname": "doi", + "schemeid": "dnet:pid_types", + "schemename": "dnet:pid_types" + }, + "value": "10.1007/s109090161569x" + }, + { + "dataInfo": { + "deletedbyinference": false, + "inferenceprovenance": "", + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:datasetarchive", + "classname": "sysimport:crosswalk:datasetarchive", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "qualifier": { + "classid": "doi", + "classname": "doi", + "schemeid": "dnet:pid_types", + "schemename": "dnet:pid_types" + }, + "value": "" + } + ], + "relevantdate": [ + ], + "resourcetype": { + "classid": "0001", + "classname": "0001", + "schemeid": "dnet:dataCite_resource", + "schemename": "dnet:dataCite_resource" + }, + "resulttype": { + "classid": "publication", + "classname": "publication", + "schemeid": "dnet:result_typologies", + "schemename": "dnet:result_typologies" + }, + "source": [ + ], + "subject": [ + { + "dataInfo": { + "deletedbyinference": false, + "inferenceprovenance": "", + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:datasetarchive", + "classname": "sysimport:crosswalk:datasetarchive", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "qualifier": { + "classid": "", + "classname": "", + "schemeid": "", + "schemename": "" + }, + "value": "ta213" + }, + { + "dataInfo": { + "deletedbyinference": false, + "inferenceprovenance": "", + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:datasetarchive", + "classname": "sysimport:crosswalk:datasetarchive", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "qualifier": { + "classid": "", + "classname": "", + "schemeid": "", + "schemename": "" + }, + "value": "infrared detectors" + }, + { + "dataInfo": { + "deletedbyinference": false, + "inferenceprovenance": "", + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:datasetarchive", + "classname": "sysimport:crosswalk:datasetarchive", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "qualifier": { + "classid": "", + "classname": "", + "schemeid": "", + "schemename": "" + }, + "value": "lens antennas" + }, + { + "dataInfo": { + "deletedbyinference": false, + "inferenceprovenance": "", + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:datasetarchive", + "classname": "sysimport:crosswalk:datasetarchive", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "qualifier": { + "classid": "", + "classname": "", + "schemeid": "", + "schemename": "" + }, + "value": "silicon" + }, + { + "dataInfo": { + "deletedbyinference": false, + "inferenceprovenance": "", + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:datasetarchive", + "classname": "sysimport:crosswalk:datasetarchive", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "qualifier": { + "classid": "", + "classname": "", + "schemeid": "", + "schemename": "" + }, + "value": "slot antennas" + }, + { + "dataInfo": { + "deletedbyinference": false, + "inferenceprovenance": "", + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:datasetarchive", + "classname": "sysimport:crosswalk:datasetarchive", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "qualifier": { + "classid": "", + "classname": "", + "schemeid": "", + "schemename": "" + }, + "value": "strained silicon" + }, + { + "dataInfo": { + "deletedbyinference": false, + "inferenceprovenance": "", + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:datasetarchive", + "classname": "sysimport:crosswalk:datasetarchive", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "qualifier": { + "classid": "", + "classname": "", + "schemeid": "", + "schemename": "" + }, + "value": "cold electron bolometers" + }, + { + "dataInfo": { + "deletedbyinference": false, + "inferenceprovenance": "", + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:datasetarchive", + "classname": "sysimport:crosswalk:datasetarchive", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "qualifier": { + "classid": "", + "classname": "", + "schemeid": "", + "schemename": "" + }, + "value": "doped silicon" + }, + { + "dataInfo": { + "deletedbyinference": false, + "inferenceprovenance": "", + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:datasetarchive", + "classname": "sysimport:crosswalk:datasetarchive", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "qualifier": { + "classid": "", + "classname": "", + "schemeid": "", + "schemename": "" + }, + "value": "measure noise" + }, + { + "dataInfo": { + "deletedbyinference": false, + "inferenceprovenance": "", + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:datasetarchive", + "classname": "sysimport:crosswalk:datasetarchive", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "qualifier": { + "classid": "", + "classname": "", + "schemeid": "", + "schemename": "" + }, + "value": "noise equivalent power" + }, + { + "dataInfo": { + "deletedbyinference": false, + "inferenceprovenance": "", + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:datasetarchive", + "classname": "sysimport:crosswalk:datasetarchive", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "qualifier": { + "classid": "", + "classname": "", + "schemeid": "", + "schemename": "" + }, + "value": "optical characterisation" + }, + { + "dataInfo": { + "deletedbyinference": false, + "inferenceprovenance": "", + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:datasetarchive", + "classname": "sysimport:crosswalk:datasetarchive", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "qualifier": { + "classid": "", + "classname": "", + "schemeid": "", + "schemename": "" + }, + "value": "optical response" + }, + { + "dataInfo": { + "deletedbyinference": false, + "inferenceprovenance": "", + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:datasetarchive", + "classname": "sysimport:crosswalk:datasetarchive", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "qualifier": { + "classid": "", + "classname": "", + "schemeid": "", + "schemename": "" + }, + "value": "photon noise" + }, + { + "dataInfo": { + "deletedbyinference": false, + "inferenceprovenance": "", + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:datasetarchive", + "classname": "sysimport:crosswalk:datasetarchive", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "qualifier": { + "classid": "", + "classname": "", + "schemeid": "", + "schemename": "" + }, + "value": "silicon absorbers" + } + ], + "title": [ + { + "dataInfo": { + "deletedbyinference": false, + "inferenceprovenance": "", + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:datasetarchive", + "classname": "sysimport:crosswalk:datasetarchive", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "qualifier": { + "classid": "main title", + "classname": "main title", + "schemeid": "dnet:dataCite_title", + "schemename": "dnet:dataCite_title" + }, + "value": "Optical response of strained- and unstrained-silicon cold-electron bolometers" + } + ] +} \ No newline at end of file diff --git a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/synonyms.txt b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/synonyms.txt new file mode 100644 index 0000000000..05484c8e50 --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/synonyms.txt @@ -0,0 +1,1233 @@ +dnet:access_modes @=@ CLOSED @=@ http://purl.org/coar/access_right/c_14cb +dnet:access_modes @=@ CLOSED @=@ info:eu-repo/semantics/closedAccess +dnet:access_modes @=@ EMBARGO @=@ http://purl.org/coar/access_right/c_f1cf +dnet:access_modes @=@ EMBARGO @=@ info:eu-repo/semantics/embargoedAccess +dnet:access_modes @=@ OPEN @=@ Creative Commons License [CC BY-NC-ND] http://creativecommons.org/licenses/by-nc-nd/3.0/de/ +dnet:access_modes @=@ OPEN @=@ Creative commons +dnet:access_modes @=@ OPEN @=@ http://creativecommons.org/licenses/by-nc-nd/3.0/ +dnet:access_modes @=@ OPEN @=@ http://creativecommons.org/licenses/by-nc/3.0/ +dnet:access_modes @=@ OPEN @=@ http://creativecommons.org/licenses/by-sa/3.0/ +dnet:access_modes @=@ OPEN @=@ http://creativecommons.org/licenses/by-sa/4.0/ +dnet:access_modes @=@ OPEN @=@ http://creativecommons.org/licenses/by/3.0/ +dnet:access_modes @=@ OPEN @=@ http://creativecommons.org/licenses/by/3.0/us/ +dnet:access_modes @=@ OPEN @=@ http://creativecommons.org/licenses/by/4.0/ +dnet:access_modes @=@ OPEN @=@ http://creativecommons.org/publicdomain/zero/1.0/ +dnet:access_modes @=@ OPEN @=@ http://creativecommons.org/publicdomain/zero/1.0/ & http://www.canadensys.net/norms +dnet:access_modes @=@ OPEN @=@ http://purl.org/coar/access_right/c_abf2 +dnet:access_modes @=@ OPEN @=@ https://creativecommons.org/licenses/by-nc/4.0/ +dnet:access_modes @=@ OPEN @=@ info:eu-repo/semantics/openAccess +dnet:access_modes @=@ OPEN @=@ open_access +dnet:access_modes @=@ RESTRICTED @=@ http://purl.org/coar/access_right/c_16ec +dnet:access_modes @=@ RESTRICTED @=@ info:eu-repo/semantics/restrictedAccess +dnet:compatibilityLevel @=@ openaire-pub_4.0 @=@ openaire4.0 +dnet:subject_classification_typologies @=@ jel @=@ jelElement +dnet:publication_resource @=@ 0018 @=@ Comment/debate +dnet:publication_resource @=@ 0018 @=@ http://purl.org/coar/resource_type/c_1162 +dnet:publication_resource @=@ 0018 @=@ info:eu-repo/semantics/annotation +dnet:publication_resource @=@ 0001 @=@ A1 Alkuperäisartikkeli tieteellisessä aikakauslehdessä +dnet:publication_resource @=@ 0001 @=@ Article +dnet:publication_resource @=@ 0001 @=@ Article (author) +dnet:publication_resource @=@ 0001 @=@ Article - letter to the editor +dnet:publication_resource @=@ 0001 @=@ Article / Letter to editor +dnet:publication_resource @=@ 0001 @=@ Article / Letter to the editor +dnet:publication_resource @=@ 0001 @=@ Article / Newspaper +dnet:publication_resource @=@ 0001 @=@ Article in journal +dnet:publication_resource @=@ 0001 @=@ Article in monograph or in proceedings +dnet:publication_resource @=@ 0001 @=@ Article in proceedings +dnet:publication_resource @=@ 0001 @=@ Article-letter to the editor +dnet:publication_resource @=@ 0001 @=@ Article/Letter to editor +dnet:publication_resource @=@ 0001 @=@ Articolo +dnet:publication_resource @=@ 0001 @=@ Artículo +dnet:publication_resource @=@ 0001 @=@ Aufsatz +dnet:publication_resource @=@ 0001 @=@ Clinical Study +dnet:publication_resource @=@ 0001 @=@ Institutional Series +dnet:publication_resource @=@ 0001 @=@ International Journal +dnet:publication_resource @=@ 0001 @=@ International Journal Abstract +dnet:publication_resource @=@ 0001 @=@ International Journal ISI/JCR +dnet:publication_resource @=@ 0001 @=@ Journal (full / special issue) +dnet:publication_resource @=@ 0001 @=@ Journal Article/Review +dnet:publication_resource @=@ 0001 @=@ Journal article +dnet:publication_resource @=@ 0001 @=@ Journal article (on-line or printed) +dnet:publication_resource @=@ 0001 @=@ Journal articles +dnet:publication_resource @=@ 0001 @=@ Journal paper +dnet:publication_resource @=@ 0001 @=@ National Journal +dnet:publication_resource @=@ 0001 @=@ Original article (non peer-reviewed) +dnet:publication_resource @=@ 0001 @=@ Original article (peer-reviewed) +dnet:publication_resource @=@ 0001 @=@ Peer-reviewed Article +dnet:publication_resource @=@ 0001 @=@ Published Journal Article +dnet:publication_resource @=@ 0001 @=@ Research Article +dnet:publication_resource @=@ 0001 @=@ Review article (non peer-reviewed) +dnet:publication_resource @=@ 0001 @=@ Review article (peer-reviewed) +dnet:publication_resource @=@ 0001 @=@ Volumes Edited / Special Issues +dnet:publication_resource @=@ 0001 @=@ article in non peer-reviewed journal +dnet:publication_resource @=@ 0001 @=@ article in peer-reviewed journal +dnet:publication_resource @=@ 0001 @=@ article-commentary +dnet:publication_resource @=@ 0001 @=@ article_site_web +dnet:publication_resource @=@ 0001 @=@ doc-type:Journal Article +dnet:publication_resource @=@ 0001 @=@ doc-type:article +dnet:publication_resource @=@ 0001 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 +dnet:publication_resource @=@ 0001 @=@ http://purl.org/coar/resource_type/c_545b +dnet:publication_resource @=@ 0001 @=@ http://purl.org/coar/resource_type/c_6501 +dnet:publication_resource @=@ 0001 @=@ http://purl.org/coar/resource_type/c_7877 +dnet:publication_resource @=@ 0001 @=@ in-brief +dnet:publication_resource @=@ 0001 @=@ info:eu-repo/semantics/article +dnet:publication_resource @=@ 0001 @=@ journal-article +dnet:publication_resource @=@ 0001 @=@ journalArticle +dnet:publication_resource @=@ 0001 @=@ journal_article +dnet:publication_resource @=@ 0001 @=@ letter +dnet:publication_resource @=@ 0001 @=@ non peer-reviewed article +dnet:publication_resource @=@ 0001 @=@ partial-retraction +dnet:publication_resource @=@ 0001 @=@ proceeding with peer review +dnet:publication_resource @=@ 0001 @=@ publication-article +dnet:publication_resource @=@ 0001 @=@ rapid-communication +dnet:publication_resource @=@ 0001 @=@ reply +dnet:publication_resource @=@ 0001 @=@ research-article +dnet:publication_resource @=@ 0001 @=@ retraction +dnet:publication_resource @=@ 0001 @=@ review-article +dnet:publication_resource @=@ 0001 @=@ text (article) +dnet:publication_resource @=@ 0001 @=@ Статья +dnet:publication_resource @=@ 0001 @=@ ArticleArtikel +dnet:publication_resource @=@ 0033 @=@ AUDIOVISUAL_DOCUMENT +dnet:publication_resource @=@ 0033 @=@ Audiovisual/Audiovisual +dnet:publication_resource @=@ 0033 @=@ http://purl.org/coar/resource_type/c_c513 +dnet:publication_resource @=@ 0008 @=@ Bachelor's +dnet:publication_resource @=@ 0008 @=@ Bachelor's Degree +dnet:publication_resource @=@ 0008 @=@ Bachelors Thesis +dnet:publication_resource @=@ 0008 @=@ Proyecto fin de carrera +dnet:publication_resource @=@ 0008 @=@ Undergraduate Thesis +dnet:publication_resource @=@ 0008 @=@ http://purl.org/coar/resource_type/c_7a1f +dnet:publication_resource @=@ 0008 @=@ info:eu-repo/semantics/bachelorThesis +dnet:publication_resource @=@ 0008 @=@ выпускная бакалаврская работа +dnet:publication_resource @=@ 0002 @=@ Book (monograph) +dnet:publication_resource @=@ 0002 @=@ Book (non peer-reviewed) +dnet:publication_resource @=@ 0002 @=@ Book (peer-reviewed) +dnet:publication_resource @=@ 0002 @=@ Book - monograph - editorial book +dnet:publication_resource @=@ 0002 @=@ Book Section +dnet:publication_resource @=@ 0002 @=@ Book as author +dnet:publication_resource @=@ 0002 @=@ Buch +dnet:publication_resource @=@ 0002 @=@ International Book/Monograph +dnet:publication_resource @=@ 0002 @=@ Libro +dnet:publication_resource @=@ 0002 @=@ Monografia +dnet:publication_resource @=@ 0002 @=@ Monograph +dnet:publication_resource @=@ 0002 @=@ National Book/Monograph +dnet:publication_resource @=@ 0002 @=@ atlas +dnet:publication_resource @=@ 0002 @=@ book +dnet:publication_resource @=@ 0002 @=@ book-series +dnet:publication_resource @=@ 0002 @=@ book-set +dnet:publication_resource @=@ 0002 @=@ book-track +dnet:publication_resource @=@ 0002 @=@ book_series +dnet:publication_resource @=@ 0002 @=@ book_title +dnet:publication_resource @=@ 0002 @=@ doc-type:book +dnet:publication_resource @=@ 0002 @=@ edited-book +dnet:publication_resource @=@ 0002 @=@ http://purl.org/coar/resource_type/c_2f33 +dnet:publication_resource @=@ 0002 @=@ info:eu-repo/semantics/book +dnet:publication_resource @=@ 0002 @=@ ouvrage +dnet:publication_resource @=@ 0002 @=@ publication-book +dnet:publication_resource @=@ 0002 @=@ reference-book +dnet:publication_resource @=@ 0002 @=@ scientific book +dnet:publication_resource @=@ 0002 @=@ Монография +dnet:publication_resource @=@ 0002 @=@ Учебник +dnet:publication_resource @=@ 0037 @=@ clinicalTrial +dnet:publication_resource @=@ 0037 @=@ http://purl.org/coar/resource_type/c_cb28 +dnet:publication_resource @=@ 0022 @=@ collection +dnet:publication_resource @=@ 0004 @=@ A4 Artikkeli konferenssijulkaisussa +dnet:publication_resource @=@ 0004 @=@ Comunicación de congreso +dnet:publication_resource @=@ 0004 @=@ Conference Paper +dnet:publication_resource @=@ 0004 @=@ Conference Paper/Proceeding/Abstract +dnet:publication_resource @=@ 0004 @=@ Conference Proceedings +dnet:publication_resource @=@ 0004 @=@ Conference article +dnet:publication_resource @=@ 0004 @=@ Conference contribution +dnet:publication_resource @=@ 0004 @=@ Conference lecture +dnet:publication_resource @=@ 0004 @=@ Conference or Workshop Item +dnet:publication_resource @=@ 0004 @=@ Conference paper, poster, etc. +dnet:publication_resource @=@ 0004 @=@ Conference papers +dnet:publication_resource @=@ 0004 @=@ Conference report +dnet:publication_resource @=@ 0004 @=@ International Conference +dnet:publication_resource @=@ 0004 @=@ International Conference Abstract/Poster +dnet:publication_resource @=@ 0004 @=@ International Conference ISI/JCR +dnet:publication_resource @=@ 0004 @=@ International Conference communication/abstract/poster +dnet:publication_resource @=@ 0004 @=@ National Conference +dnet:publication_resource @=@ 0004 @=@ National Conference Abstract/Poster +dnet:publication_resource @=@ 0004 @=@ National Conference communication/abstract/poster +dnet:publication_resource @=@ 0004 @=@ PREFACE_PROCEEDINGS +dnet:publication_resource @=@ 0004 @=@ PROCEEDING_PAPER +dnet:publication_resource @=@ 0004 @=@ Papers in Conference Proceedings +dnet:publication_resource @=@ 0004 @=@ Presentación +dnet:publication_resource @=@ 0004 @=@ Proceedings (peer-reviewed) +dnet:publication_resource @=@ 0004 @=@ Proceedings of a Conference +dnet:publication_resource @=@ 0004 @=@ Proceedings paper +dnet:publication_resource @=@ 0004 @=@ Póster +dnet:publication_resource @=@ 0004 @=@ actes_congres +dnet:publication_resource @=@ 0004 @=@ communication_avec_actes +dnet:publication_resource @=@ 0004 @=@ communication_invitee +dnet:publication_resource @=@ 0004 @=@ communication_par_affiche +dnet:publication_resource @=@ 0004 @=@ communication_sans_actes +dnet:publication_resource @=@ 0004 @=@ conference +dnet:publication_resource @=@ 0004 @=@ conference item +dnet:publication_resource @=@ 0004 @=@ conference proceeding +dnet:publication_resource @=@ 0004 @=@ conferenceObject +dnet:publication_resource @=@ 0004 @=@ conference_paper +dnet:publication_resource @=@ 0004 @=@ doc-type:conferenceObject +dnet:publication_resource @=@ 0004 @=@ http://purl.org/coar/resource_type/c_18co +dnet:publication_resource @=@ 0004 @=@ http://purl.org/coar/resource_type/c_18cp +dnet:publication_resource @=@ 0004 @=@ http://purl.org/coar/resource_type/c_5794 +dnet:publication_resource @=@ 0004 @=@ http://purl.org/coar/resource_type/c_6670 +dnet:publication_resource @=@ 0004 @=@ http://purl.org/coar/resource_type/c_c94f +dnet:publication_resource @=@ 0004 @=@ http://purl.org/coar/resource_type/c_f744 +dnet:publication_resource @=@ 0004 @=@ info:eu-repo/semantics/conferenceItem +dnet:publication_resource @=@ 0004 @=@ info:eu-repo/semantics/conferenceObject +dnet:publication_resource @=@ 0004 @=@ invited conference talk +dnet:publication_resource @=@ 0004 @=@ poster +dnet:publication_resource @=@ 0004 @=@ presentation +dnet:publication_resource @=@ 0004 @=@ proceeding, seminar, workshop without peer review +dnet:publication_resource @=@ 0004 @=@ proceedings +dnet:publication_resource @=@ 0004 @=@ proceedings-article +dnet:publication_resource @=@ 0004 @=@ publication-conferencepaper +dnet:publication_resource @=@ 0004 @=@ научный доклад +dnet:publication_resource @=@ 0005 @=@ Newspaper or magazine article +dnet:publication_resource @=@ 0005 @=@ http://purl.org/coar/resource_type/c_998f +dnet:publication_resource @=@ 0005 @=@ info:eu-repo/semantics/contributionToPeriodical +dnet:publication_resource @=@ 0045 @=@ Data Management Plan +dnet:publication_resource @=@ 0045 @=@ Data Management Plan (NSF Generic) +dnet:publication_resource @=@ 0045 @=@ http://purl.org/coar/resource_type/c_ab20 +dnet:publication_resource @=@ 0045 @=@ http://purl.org/spar/fabio/DataManagementPolicy +dnet:publication_resource @=@ 0045 @=@ http://purl.org/spar/fabio/DataManagementPolicyDocument +dnet:publication_resource @=@ 0045 @=@ http://purl.org/spar/fabio/DataMangementPlan +dnet:publication_resource @=@ 0045 @=@ plan de gestión de datos +dnet:publication_resource @=@ 0045 @=@ publication-datamanagementplan +dnet:publication_resource @=@ 0031 @=@ Data Descriptor +dnet:publication_resource @=@ 0031 @=@ DataPaper +dnet:publication_resource @=@ 0031 @=@ data-article +dnet:publication_resource @=@ 0031 @=@ http://purl.org/coar/resource_type/c_beb9 +dnet:publication_resource @=@ 0021 @=@ Dataset/Dataset +dnet:publication_resource @=@ 0021 @=@ Research Data +dnet:publication_resource @=@ 0021 @=@ dataset +dnet:publication_resource @=@ 0021 @=@ http://purl.org/coar/resource_type/c_ddb1 +dnet:publication_resource @=@ 0021 @=@ info:eu-repo/semantics/DDIInstance +dnet:publication_resource @=@ 0021 @=@ info:eu-repo/semantics/datafile +dnet:publication_resource @=@ 0021 @=@ info:eu-repo/semantics/dataset +dnet:publication_resource @=@ 0021 @=@ info:eu-repo/semantics/enhancedObjectFile +dnet:publication_resource @=@ 0006 @=@ Diss +dnet:publication_resource @=@ 0006 @=@ Dissertation +dnet:publication_resource @=@ 0006 @=@ Doctoral +dnet:publication_resource @=@ 0006 @=@ DoctoralThesis +dnet:publication_resource @=@ 0006 @=@ PhD thesis +dnet:publication_resource @=@ 0006 @=@ Tesis +dnet:publication_resource @=@ 0006 @=@ Text.Thesis.Doctoral +dnet:publication_resource @=@ 0006 @=@ Theses +dnet:publication_resource @=@ 0006 @=@ Thesis +dnet:publication_resource @=@ 0006 @=@ Thesis or Dissertation +dnet:publication_resource @=@ 0006 @=@ Thesis.Doctoral +dnet:publication_resource @=@ 0006 @=@ doc-type:doctoralThesis +dnet:publication_resource @=@ 0006 @=@ http://purl.org/coar/resource_type/c_db06 +dnet:publication_resource @=@ 0006 @=@ info:eu-repo/semantics/doctoralThesis +dnet:publication_resource @=@ 0006 @=@ publication-thesis +dnet:publication_resource @=@ 0006 @=@ these +dnet:publication_resource @=@ 0006 @=@ these exercice +dnet:publication_resource @=@ 0023 @=@ Event/Event +dnet:publication_resource @=@ 0023 @=@ event +dnet:publication_resource @=@ 0009 @=@ Departmental Technical Report +dnet:publication_resource @=@ 0009 @=@ Informe Técnico +dnet:publication_resource @=@ 0009 @=@ RESEARCH_REPORT +dnet:publication_resource @=@ 0009 @=@ Tech-Report +dnet:publication_resource @=@ 0009 @=@ Technical Report +dnet:publication_resource @=@ 0009 @=@ http://purl.org/coar/resource_type/c_18gh +dnet:publication_resource @=@ 0009 @=@ publication-technicalnote +dnet:publication_resource @=@ 0009 @=@ research report +dnet:publication_resource @=@ 0024 @=@ Video +dnet:publication_resource @=@ 0024 @=@ film +dnet:publication_resource @=@ 0024 @=@ http://purl.org/coar/resource_type/c_12ce +dnet:publication_resource @=@ 0024 @=@ http://purl.org/coar/resource_type/c_8a7e +dnet:publication_resource @=@ 0025 @=@ Diagram +dnet:publication_resource @=@ 0025 @=@ Drawing +dnet:publication_resource @=@ 0025 @=@ Figure +dnet:publication_resource @=@ 0025 @=@ Image/Image +dnet:publication_resource @=@ 0025 @=@ Imagen +dnet:publication_resource @=@ 0025 @=@ Photo +dnet:publication_resource @=@ 0025 @=@ Plot +dnet:publication_resource @=@ 0025 @=@ fotó +dnet:publication_resource @=@ 0025 @=@ grafika +dnet:publication_resource @=@ 0025 @=@ http://purl.org/coar/resource_type/c_ecc8 +dnet:publication_resource @=@ 0025 @=@ image +dnet:publication_resource @=@ 0025 @=@ image-diagram +dnet:publication_resource @=@ 0025 @=@ image-drawing +dnet:publication_resource @=@ 0025 @=@ image-figure +dnet:publication_resource @=@ 0025 @=@ image-other +dnet:publication_resource @=@ 0025 @=@ image-photo +dnet:publication_resource @=@ 0025 @=@ image-plot +dnet:publication_resource @=@ 0026 @=@ http://purl.org/coar/resource_type/c_e9a0 +dnet:publication_resource @=@ 0026 @=@ interactiveResource +dnet:publication_resource @=@ 0011 @=@ Internal note +dnet:publication_resource @=@ 0011 @=@ http://purl.org/coar/resource_type/c_18ww +dnet:publication_resource @=@ 0043 @=@ http://purl.org/coar/resource_type/c_0640 +dnet:publication_resource @=@ 0010 @=@ Inaugural lecture +dnet:publication_resource @=@ 0010 @=@ Material didáctico +dnet:publication_resource @=@ 0010 @=@ Public-Lecture +dnet:publication_resource @=@ 0010 @=@ http://purl.org/coar/resource_type/c_8544 +dnet:publication_resource @=@ 0010 @=@ info:eu-repo/semantics/lecture +dnet:publication_resource @=@ 0010 @=@ lesson +dnet:publication_resource @=@ 0010 @=@ Учебный материал +dnet:publication_resource @=@ 0007 @=@ Diploma Project +dnet:publication_resource @=@ 0007 @=@ MSc Thesis +dnet:publication_resource @=@ 0007 @=@ Master Degree +dnet:publication_resource @=@ 0007 @=@ Master's +dnet:publication_resource @=@ 0007 @=@ Masterarbeit u.a. +dnet:publication_resource @=@ 0007 @=@ Masters (Taught) +dnet:publication_resource @=@ 0007 @=@ Masters thesis +dnet:publication_resource @=@ 0007 @=@ Masters-Thesis.Magister +dnet:publication_resource @=@ 0007 @=@ Tesina +dnet:publication_resource @=@ 0007 @=@ Thesis.Master +dnet:publication_resource @=@ 0007 @=@ Trabajo fin de Máster +dnet:publication_resource @=@ 0007 @=@ doc-type:masterThesis +dnet:publication_resource @=@ 0007 @=@ hdr +dnet:publication_resource @=@ 0007 @=@ http://purl.org/coar/resource_type/c_bdcc +dnet:publication_resource @=@ 0007 @=@ info:eu-repo/semantics/masterThesis +dnet:publication_resource @=@ 0007 @=@ masterThesis +dnet:publication_resource @=@ 0007 @=@ memoire +dnet:publication_resource @=@ 0027 @=@ Model/Model +dnet:publication_resource @=@ 0027 @=@ model +dnet:publication_resource @=@ 0020 @=@ Exhibition +dnet:publication_resource @=@ 0020 @=@ Learning Object +dnet:publication_resource @=@ 0020 @=@ Mapa +dnet:publication_resource @=@ 0020 @=@ Modelo de utilidad +dnet:publication_resource @=@ 0020 @=@ PEDAGOGICAL_DOCUMENT +dnet:publication_resource @=@ 0020 @=@ Partitura +dnet:publication_resource @=@ 0020 @=@ Sitio web +dnet:publication_resource @=@ 0020 @=@ Trabajo de divulgación +dnet:publication_resource @=@ 0020 @=@ Web publication/site +dnet:publication_resource @=@ 0020 @=@ application +dnet:publication_resource @=@ 0020 @=@ artefact +dnet:publication_resource @=@ 0020 @=@ carte +dnet:publication_resource @=@ 0020 @=@ composition +dnet:publication_resource @=@ 0020 @=@ document_audiovisuel +dnet:publication_resource @=@ 0020 @=@ http://purl.org/coar/resource_type/c_12cc +dnet:publication_resource @=@ 0020 @=@ http://purl.org/coar/resource_type/c_12cd +dnet:publication_resource @=@ 0020 @=@ http://purl.org/coar/resource_type/c_1843 +dnet:publication_resource @=@ 0020 @=@ http://purl.org/coar/resource_type/c_18cd +dnet:publication_resource @=@ 0020 @=@ http://purl.org/coar/resource_type/c_18cw +dnet:publication_resource @=@ 0020 @=@ http://purl.org/coar/resource_type/c_26e4 +dnet:publication_resource @=@ 0020 @=@ http://purl.org/coar/resource_type/c_7ad9 +dnet:publication_resource @=@ 0020 @=@ http://purl.org/coar/resource_type/c_e059 +dnet:publication_resource @=@ 0020 @=@ info:eu-repo/semantics/other +dnet:publication_resource @=@ 0020 @=@ learningObject +dnet:publication_resource @=@ 0020 @=@ map +dnet:publication_resource @=@ 0020 @=@ misc +dnet:publication_resource @=@ 0020 @=@ other +dnet:publication_resource @=@ 0020 @=@ revue +dnet:publication_resource @=@ 0038 @=@ Abstract +dnet:publication_resource @=@ 0038 @=@ Blog +dnet:publication_resource @=@ 0038 @=@ Book Prospectus +dnet:publication_resource @=@ 0038 @=@ Dictionary Entry +dnet:publication_resource @=@ 0038 @=@ Disclosure +dnet:publication_resource @=@ 0038 @=@ Editorial +dnet:publication_resource @=@ 0038 @=@ Editorial ISI/JCR +dnet:publication_resource @=@ 0038 @=@ Editors +dnet:publication_resource @=@ 0038 @=@ Editors (non peer-reviewed) +dnet:publication_resource @=@ 0038 @=@ Editors (peer-reviewed) +dnet:publication_resource @=@ 0038 @=@ Encyclopedia Entry +dnet:publication_resource @=@ 0038 @=@ Entrada de blog +dnet:publication_resource @=@ 0038 @=@ Funding Submission +dnet:publication_resource @=@ 0038 @=@ HabilitationThesis +dnet:publication_resource @=@ 0038 @=@ License +dnet:publication_resource @=@ 0038 @=@ Manual +dnet:publication_resource @=@ 0038 @=@ Manuscript +dnet:publication_resource @=@ 0038 @=@ Manuscrito +dnet:publication_resource @=@ 0038 @=@ Other publication (non peer-review) +dnet:publication_resource @=@ 0038 @=@ Other publication (peer-review) +dnet:publication_resource @=@ 0038 @=@ Revista +dnet:publication_resource @=@ 0038 @=@ Supervised Student Publication +dnet:publication_resource @=@ 0038 @=@ Tesis/trabajos de grado – Thesis +dnet:publication_resource @=@ 0038 @=@ Text +dnet:publication_resource @=@ 0038 @=@ Text/Text +dnet:publication_resource @=@ 0038 @=@ Trademark +dnet:publication_resource @=@ 0038 @=@ Translation +dnet:publication_resource @=@ 0038 @=@ afterword +dnet:publication_resource @=@ 0038 @=@ avantpropos +dnet:publication_resource @=@ 0038 @=@ bibliography +dnet:publication_resource @=@ 0038 @=@ chronique +dnet:publication_resource @=@ 0038 @=@ compte rendu +dnet:publication_resource @=@ 0038 @=@ correction +dnet:publication_resource @=@ 0038 @=@ foreword +dnet:publication_resource @=@ 0038 @=@ habilitation à diriger des recherches +dnet:publication_resource @=@ 0038 @=@ historicalDocument +dnet:publication_resource @=@ 0038 @=@ http://purl.org/coar/resource_type/c_0040 +dnet:publication_resource @=@ 0038 @=@ http://purl.org/coar/resource_type/c_0857 +dnet:publication_resource @=@ 0038 @=@ http://purl.org/coar/resource_type/c_18cf +dnet:publication_resource @=@ 0038 @=@ http://purl.org/coar/resource_type/c_18wz +dnet:publication_resource @=@ 0038 @=@ http://purl.org/coar/resource_type/c_3e5a +dnet:publication_resource @=@ 0038 @=@ http://purl.org/coar/resource_type/c_46ec +dnet:publication_resource @=@ 0038 @=@ http://purl.org/coar/resource_type/c_6947 +dnet:publication_resource @=@ 0038 @=@ http://purl.org/coar/resource_type/c_7acd +dnet:publication_resource @=@ 0038 @=@ http://purl.org/coar/resource_type/c_86bc +dnet:publication_resource @=@ 0038 @=@ http://purl.org/coar/resource_type/c_b239 +dnet:publication_resource @=@ 0038 @=@ note de lecture +dnet:publication_resource @=@ 0038 @=@ notedelecture +dnet:publication_resource @=@ 0038 @=@ other publication +dnet:publication_resource @=@ 0038 @=@ postface +dnet:publication_resource @=@ 0038 @=@ publication-other +dnet:publication_resource @=@ 0038 @=@ revuedepresse +dnet:publication_resource @=@ 0038 @=@ sa_component +dnet:publication_resource @=@ 0038 @=@ standard +dnet:publication_resource @=@ 0038 @=@ standard-series +dnet:publication_resource @=@ 0013 @=@ A3 Kirjan tai muun kokoomateoksen osa +dnet:publication_resource @=@ 0013 @=@ Book Part (author) +dnet:publication_resource @=@ 0013 @=@ Book Section / Chapter +dnet:publication_resource @=@ 0013 @=@ Book chapter or Essay in book +dnet:publication_resource @=@ 0013 @=@ Book editorial +dnet:publication_resource @=@ 0013 @=@ Book section +dnet:publication_resource @=@ 0013 @=@ Book_Chapter +dnet:publication_resource @=@ 0013 @=@ Buchbeitrag +dnet:publication_resource @=@ 0013 @=@ Capítulo de libro +dnet:publication_resource @=@ 0013 @=@ Contribution to International Book/Monograph +dnet:publication_resource @=@ 0013 @=@ Contribution to International Book/Monograph ISI/JCR +dnet:publication_resource @=@ 0013 @=@ Contribution to National Book/Monograph +dnet:publication_resource @=@ 0013 @=@ Contribution to book (non peer-reviewed) +dnet:publication_resource @=@ 0013 @=@ Contribution to book (peer-reviewed) +dnet:publication_resource @=@ 0013 @=@ Part of book - chapter +dnet:publication_resource @=@ 0013 @=@ book chapter +dnet:publication_resource @=@ 0013 @=@ book-part +dnet:publication_resource @=@ 0013 @=@ bookPart +dnet:publication_resource @=@ 0013 @=@ book_content +dnet:publication_resource @=@ 0013 @=@ chapitre_ouvrage +dnet:publication_resource @=@ 0013 @=@ chapter +dnet:publication_resource @=@ 0013 @=@ doc-type:bookPart +dnet:publication_resource @=@ 0013 @=@ http://purl.org/coar/resource_type/c_3248 +dnet:publication_resource @=@ 0013 @=@ info:eu-repo/semantics/bookPart +dnet:publication_resource @=@ 0013 @=@ publication-section +dnet:publication_resource @=@ 0013 @=@ reference-entry +dnet:publication_resource @=@ 0013 @=@ reference_entry +dnet:publication_resource @=@ 0013 @=@ scientific book chapter +dnet:publication_resource @=@ 0013 @=@ Глава монографии +dnet:publication_resource @=@ 0019 @=@ H1 Myönnetty patentti +dnet:publication_resource @=@ 0019 @=@ Patent +dnet:publication_resource @=@ 0019 @=@ Patente +dnet:publication_resource @=@ 0019 @=@ Solicitud de patente +dnet:publication_resource @=@ 0019 @=@ Traducción de patente +dnet:publication_resource @=@ 0019 @=@ brevet +dnet:publication_resource @=@ 0019 @=@ http://purl.org/coar/resource_type/c_15cd +dnet:publication_resource @=@ 0019 @=@ info:eu-repo/semantics/patent +dnet:publication_resource @=@ 0019 @=@ publication-patent +dnet:publication_resource @=@ 0028 @=@ Service +dnet:publication_resource @=@ 0028 @=@ physicalObject +dnet:publication_resource @=@ 0016 @=@ Pre Print +dnet:publication_resource @=@ 0016 @=@ Pre-print +dnet:publication_resource @=@ 0016 @=@ http://purl.org/coar/resource_type/c_816b +dnet:publication_resource @=@ 0016 @=@ info:eu-repo/semantics/preprint +dnet:publication_resource @=@ 0016 @=@ publication-preprint +dnet:publication_resource @=@ 0016 @=@ Препринт +dnet:publication_resource @=@ 0034 @=@ Project deliverable +dnet:publication_resource @=@ 0034 @=@ http://purl.org/coar/resource_type/c_18op +dnet:publication_resource @=@ 0034 @=@ publication-deliverable +dnet:publication_resource @=@ 0035 @=@ Project milestone +dnet:publication_resource @=@ 0035 @=@ publication-milestone +dnet:publication_resource @=@ 0036 @=@ Proposal +dnet:publication_resource @=@ 0036 @=@ http://purl.org/coar/resource_type/c_baaf +dnet:publication_resource @=@ 0036 @=@ research-proposal +dnet:publication_resource @=@ 0017 @=@ ACTIVITY_REPORT +dnet:publication_resource @=@ 0017 @=@ Commissioned report +dnet:publication_resource @=@ 0017 @=@ D4 Julkaistu kehittämis- tai tutkimusraportti tai -selvitys +dnet:publication_resource @=@ 0017 @=@ Deliverable +dnet:publication_resource @=@ 0017 @=@ Documento tecnico +dnet:publication_resource @=@ 0017 @=@ Project Report +dnet:publication_resource @=@ 0017 @=@ Software documentation +dnet:publication_resource @=@ 0017 @=@ brief-report +dnet:publication_resource @=@ 0017 @=@ case-report +dnet:publication_resource @=@ 0017 @=@ chapitre_rapport +dnet:publication_resource @=@ 0017 @=@ doc-type:report +dnet:publication_resource @=@ 0017 @=@ document_institutionnel +dnet:publication_resource @=@ 0017 @=@ document_technique +dnet:publication_resource @=@ 0017 @=@ http://purl.org/coar/resource_type/c_186u +dnet:publication_resource @=@ 0017 @=@ http://purl.org/coar/resource_type/c_18hj +dnet:publication_resource @=@ 0017 @=@ http://purl.org/coar/resource_type/c_18wq +dnet:publication_resource @=@ 0017 @=@ http://purl.org/coar/resource_type/c_18ws +dnet:publication_resource @=@ 0017 @=@ http://purl.org/coar/resource_type/c_71bd +dnet:publication_resource @=@ 0017 @=@ http://purl.org/coar/resource_type/c_93fc +dnet:publication_resource @=@ 0017 @=@ http://purl.org/coar/resource_type/c_ba1f +dnet:publication_resource @=@ 0017 @=@ info:eu-repo/semantics/report +dnet:publication_resource @=@ 0017 @=@ publication-report +dnet:publication_resource @=@ 0017 @=@ publication-softwaredocumentation +dnet:publication_resource @=@ 0017 @=@ rapport_expertise +dnet:publication_resource @=@ 0017 @=@ rapport_mission +dnet:publication_resource @=@ 0017 @=@ report +dnet:publication_resource @=@ 0017 @=@ report-paper +dnet:publication_resource @=@ 0017 @=@ report-paper_title +dnet:publication_resource @=@ 0017 @=@ report-series +dnet:publication_resource @=@ 0017 @=@ support_cours +dnet:publication_resource @=@ 0014 @=@ Arbeitspapier +dnet:publication_resource @=@ 0014 @=@ Departmental Bulletin Paper +dnet:publication_resource @=@ 0014 @=@ Documento de trabajo +dnet:publication_resource @=@ 0014 @=@ Paper +dnet:publication_resource @=@ 0014 @=@ Project description +dnet:publication_resource @=@ 0014 @=@ Research-Paper +dnet:publication_resource @=@ 0014 @=@ ResearchPaper +dnet:publication_resource @=@ 0014 @=@ Working / discussion paper +dnet:publication_resource @=@ 0014 @=@ Working Paper +dnet:publication_resource @=@ 0014 @=@ Working Paper / Technical Report +dnet:publication_resource @=@ 0014 @=@ doc-type:workingPaper +dnet:publication_resource @=@ 0014 @=@ http://purl.org/coar/resource_type/c_8042 +dnet:publication_resource @=@ 0014 @=@ info:eu-repo/semantics/paper +dnet:publication_resource @=@ 0014 @=@ info:eu-repo/semantics/workingPaper +dnet:publication_resource @=@ 0014 @=@ publication-workingpaper +dnet:publication_resource @=@ 0014 @=@ workingPaper +dnet:publication_resource @=@ 0015 @=@ A2 Katsausartikkeli tieteellisessä aikakauslehdessä +dnet:publication_resource @=@ 0015 @=@ Book Review +dnet:publication_resource @=@ 0015 @=@ Book/Film/Article review +dnet:publication_resource @=@ 0015 @=@ Literature review +dnet:publication_resource @=@ 0015 @=@ Peer review +dnet:publication_resource @=@ 0015 @=@ Reseña bibliográfica +dnet:publication_resource @=@ 0015 @=@ Review Article +dnet:publication_resource @=@ 0015 @=@ RezensionReview +dnet:publication_resource @=@ 0015 @=@ book-review +dnet:publication_resource @=@ 0015 @=@ http://purl.org/coar/resource_type/c_ba08 +dnet:publication_resource @=@ 0015 @=@ http://purl.org/coar/resource_type/c_dcae04bc +dnet:publication_resource @=@ 0015 @=@ http://purl.org/coar/resource_type/c_efa0 +dnet:publication_resource @=@ 0015 @=@ info:eu-repo/semantics/review +dnet:publication_resource @=@ 0015 @=@ peer-review +dnet:publication_resource @=@ 0029 @=@ Software +dnet:publication_resource @=@ 0029 @=@ Software/Software +dnet:publication_resource @=@ 0029 @=@ Workflow +dnet:publication_resource @=@ 0029 @=@ Workflow/Workflow +dnet:publication_resource @=@ 0029 @=@ http://purl.org/coar/resource_type/c_393c +dnet:publication_resource @=@ 0029 @=@ http://purl.org/coar/resource_type/c_5ce6 +dnet:publication_resource @=@ 0029 @=@ http://purl.org/coar/resource_type/c_c950 +dnet:publication_resource @=@ 0032 @=@ http://purl.org/coar/resource_type/c_7bab +dnet:publication_resource @=@ 0030 @=@ http://purl.org/coar/resource_type/c_18cc +dnet:publication_resource @=@ 0030 @=@ sound +dnet:publication_resource @=@ 0044 @=@ Graduate diploma +dnet:publication_resource @=@ 0044 @=@ Undergraduate diploma +dnet:publication_resource @=@ 0000 @=@ UNKNOWN +dnet:publication_resource @=@ 0042 @=@ EGI Virtual Appliance +dnet:languages @=@ abk @=@ ab +dnet:languages @=@ aar @=@ aa +dnet:languages @=@ afr @=@ af +dnet:languages @=@ alb/sqi @=@ sq +dnet:languages @=@ amh @=@ am +dnet:languages @=@ ara @=@ ar +dnet:languages @=@ arm/hye @=@ hy +dnet:languages @=@ asm @=@ as +dnet:languages @=@ ina @=@ ia +dnet:languages @=@ aym @=@ ay +dnet:languages @=@ aze @=@ az +dnet:languages @=@ bak @=@ ba +dnet:languages @=@ baq/eus @=@ eu +dnet:languages @=@ bel @=@ be +dnet:languages @=@ ben @=@ bn +dnet:languages @=@ bih @=@ bh +dnet:languages @=@ bis @=@ bi +dnet:languages @=@ bre @=@ br +dnet:languages @=@ bul @=@ bg +dnet:languages @=@ bur/mya @=@ my +dnet:languages @=@ cat @=@ ca +dnet:languages @=@ chi/zho @=@ zh +dnet:languages @=@ cos @=@ co +dnet:languages @=@ hrv @=@ hr +dnet:languages @=@ hrv @=@ hr +dnet:languages @=@ hrv @=@ scr/hrv +dnet:languages @=@ ces/cze @=@ cs +dnet:languages @=@ dan @=@ da +dnet:languages @=@ dut/nld @=@ dut/nla +dnet:languages @=@ dut/nld @=@ dutdut +dnet:languages @=@ dut/nld @=@ nl +dnet:languages @=@ dut/nld @=@ nl_be +dnet:languages @=@ dut/nld @=@ nl_nl +dnet:languages @=@ dut/nld @=@ nld +dnet:languages @=@ dzo @=@ dz +dnet:languages @=@ eng @=@ en +dnet:languages @=@ eng @=@ en_au +dnet:languages @=@ eng @=@ en_en +dnet:languages @=@ eng @=@ en_gb +dnet:languages @=@ eng @=@ en_nz +dnet:languages @=@ eng @=@ en_us +dnet:languages @=@ eng @=@ english +dnet:languages @=@ eng @=@ en-us +dnet:languages @=@ eng @=@ en-US +dnet:languages @=@ eng @=@ English +dnet:languages @=@ eng @=@ EN +dnet:languages @=@ eng @=@ en angielski +dnet:languages @=@ eng @=@ en-GB +dnet:languages @=@ eng @=@ Englisch +dnet:languages @=@ epo @=@ eo +dnet:languages @=@ est @=@ et +dnet:languages @=@ fao @=@ fo +dnet:languages @=@ fij @=@ fj +dnet:languages @=@ fin @=@ fi +dnet:languages @=@ fin @=@ Finnish +dnet:languages @=@ fra/fre @=@ fr +dnet:languages @=@ fra/fre @=@ FR +dnet:languages @=@ fra/fre @=@ fr_be +dnet:languages @=@ fra/fre @=@ fr_fr +dnet:languages @=@ fra/fre @=@ fre/fra +dnet:languages @=@ fra/fre @=@ fra +dnet:languages @=@ fry @=@ fy +dnet:languages @=@ glg @=@ gl +dnet:languages @=@ geo/kat @=@ ka +dnet:languages @=@ deu/ger @=@ de +dnet:languages @=@ deu/ger @=@ ger/deu +dnet:languages @=@ deu/ger @=@ german +dnet:languages @=@ deu/ger @=@ ger +dnet:languages @=@ deu/ger @=@ deu +dnet:languages @=@ deu/ger @=@ DE-de +dnet:languages @=@ ell/gre @=@ el +dnet:languages @=@ ell/gre @=@ gr +dnet:languages @=@ ell/gre @=@ el-GR +dnet:languages @=@ kal @=@ kl +dnet:languages @=@ grn @=@ gn +dnet:languages @=@ guj @=@ gu +dnet:languages @=@ hau @=@ ha +dnet:languages @=@ heb @=@ he +dnet:languages @=@ hin @=@ hi +dnet:languages @=@ hun @=@ hu +dnet:languages @=@ ice/isl @=@ is +dnet:languages @=@ ine @=@ - +dnet:languages @=@ ind @=@ id +dnet:languages @=@ iku @=@ iu +dnet:languages @=@ ipk @=@ ik +dnet:languages @=@ gai/iri @=@ ga +dnet:languages @=@ gai/iri @=@ gle +dnet:languages @=@ ita @=@ it +dnet:languages @=@ jpn @=@ ja +dnet:languages @=@ jav @=@ jv +dnet:languages @=@ jav @=@ jv/jw +dnet:languages @=@ jav @=@ jw +dnet:languages @=@ kan @=@ kn +dnet:languages @=@ kas @=@ ks +dnet:languages @=@ kaz @=@ kk +dnet:languages @=@ khm @=@ km +dnet:languages @=@ kin @=@ rw +dnet:languages @=@ kir @=@ ky +dnet:languages @=@ kor @=@ ko +dnet:languages @=@ kur @=@ ku +dnet:languages @=@ lao @=@ lo +dnet:languages @=@ lat @=@ la +dnet:languages @=@ lav @=@ lv +dnet:languages @=@ lin @=@ ln +dnet:languages @=@ lit @=@ lt +dnet:languages @=@ mac/mak @=@ mk +dnet:languages @=@ mlg @=@ mg +dnet:languages @=@ may/msa @=@ ms +dnet:languages @=@ mlt @=@ ml +dnet:languages @=@ mao/mri @=@ mi +dnet:languages @=@ mar @=@ mr +dnet:languages @=@ mol @=@ mo +dnet:languages @=@ mon @=@ mn +dnet:languages @=@ nau @=@ na +dnet:languages @=@ nep @=@ ne +dnet:languages @=@ nor @=@ no +dnet:languages @=@ oci @=@ oc +dnet:languages @=@ ori @=@ or +dnet:languages @=@ orm @=@ om +dnet:languages @=@ pan @=@ pa +dnet:languages @=@ fas/per @=@ fa +dnet:languages @=@ pol @=@ pl +dnet:languages @=@ por @=@ pt +dnet:languages @=@ por @=@ pt_pt +dnet:languages @=@ pus @=@ ps +dnet:languages @=@ que @=@ qu +dnet:languages @=@ roh @=@ rm +dnet:languages @=@ ron/rum @=@ ro +dnet:languages @=@ run @=@ rn +dnet:languages @=@ rus @=@ ru +dnet:languages @=@ smo @=@ sm +dnet:languages @=@ sag @=@ sg +dnet:languages @=@ san @=@ sa +dnet:languages @=@ srp @=@ scc/srp +dnet:languages @=@ srp @=@ sr +dnet:languages @=@ scr @=@ sh +dnet:languages @=@ sna @=@ sn +dnet:languages @=@ snd @=@ sd +dnet:languages @=@ sin @=@ si +dnet:languages @=@ sit @=@ - +dnet:languages @=@ slk/slo @=@ sk +dnet:languages @=@ slv @=@ sl +dnet:languages @=@ som @=@ so +dnet:languages @=@ sot @=@ st +dnet:languages @=@ esl/spa @=@ es +dnet:languages @=@ sun @=@ su +dnet:languages @=@ swa @=@ sw +dnet:languages @=@ ssw @=@ ss +dnet:languages @=@ swe @=@ sv +dnet:languages @=@ swe @=@ sve/swe +dnet:languages @=@ tgl @=@ tl +dnet:languages @=@ tgk @=@ tg +dnet:languages @=@ tam @=@ ta +dnet:languages @=@ tat @=@ tt +dnet:languages @=@ tel @=@ te +dnet:languages @=@ tha @=@ th +dnet:languages @=@ tha @=@ thai +dnet:languages @=@ bod/tib @=@ bo +dnet:languages @=@ tir @=@ ti +dnet:languages @=@ tog @=@ to +dnet:languages @=@ tso @=@ ts +dnet:languages @=@ tsn @=@ tn +dnet:languages @=@ tur @=@ tr +dnet:languages @=@ tuk @=@ tk +dnet:languages @=@ twi @=@ tw +dnet:languages @=@ uig @=@ ug +dnet:languages @=@ ukr @=@ uk +dnet:languages @=@ und @=@ UNKNOWN +dnet:languages @=@ und @=@ none +dnet:languages @=@ urd @=@ ur +dnet:languages @=@ uzb @=@ uz +dnet:languages @=@ vie @=@ vi +dnet:languages @=@ vol @=@ vo +dnet:languages @=@ wln @=@ wa +dnet:languages @=@ cym/wel @=@ cy +dnet:languages @=@ wol @=@ wo +dnet:languages @=@ xho @=@ xh +dnet:languages @=@ yid @=@ yi +dnet:languages @=@ yor @=@ yo +dnet:languages @=@ zha @=@ za +dnet:languages @=@ zul @=@ zu +dnet:result_typologies @=@ dataset @=@ 0021 +dnet:result_typologies @=@ dataset @=@ 0024 +dnet:result_typologies @=@ dataset @=@ 0025 +dnet:result_typologies @=@ dataset @=@ 0030 +dnet:result_typologies @=@ dataset @=@ 0033 +dnet:result_typologies @=@ dataset @=@ 0037 +dnet:result_typologies @=@ dataset @=@ 0039 +dnet:result_typologies @=@ dataset @=@ 0046 +dnet:result_typologies @=@ other @=@ 0000 +dnet:result_typologies @=@ other @=@ 0010 +dnet:result_typologies @=@ other @=@ 0018 +dnet:result_typologies @=@ other @=@ 0020 +dnet:result_typologies @=@ other @=@ 0022 +dnet:result_typologies @=@ other @=@ 0023 +dnet:result_typologies @=@ other @=@ 0026 +dnet:result_typologies @=@ other @=@ 0027 +dnet:result_typologies @=@ other @=@ 0028 +dnet:result_typologies @=@ other @=@ 0042 +dnet:result_typologies @=@ publication @=@ 0001 +dnet:result_typologies @=@ publication @=@ 0002 +dnet:result_typologies @=@ publication @=@ 0004 +dnet:result_typologies @=@ publication @=@ 0005 +dnet:result_typologies @=@ publication @=@ 0006 +dnet:result_typologies @=@ publication @=@ 0007 +dnet:result_typologies @=@ publication @=@ 0008 +dnet:result_typologies @=@ publication @=@ 0009 +dnet:result_typologies @=@ publication @=@ 0011 +dnet:result_typologies @=@ publication @=@ 0012 +dnet:result_typologies @=@ publication @=@ 0013 +dnet:result_typologies @=@ publication @=@ 0014 +dnet:result_typologies @=@ publication @=@ 0015 +dnet:result_typologies @=@ publication @=@ 0016 +dnet:result_typologies @=@ publication @=@ 0017 +dnet:result_typologies @=@ publication @=@ 0019 +dnet:result_typologies @=@ publication @=@ 0031 +dnet:result_typologies @=@ publication @=@ 0032 +dnet:result_typologies @=@ publication @=@ 0034 +dnet:result_typologies @=@ publication @=@ 0035 +dnet:result_typologies @=@ publication @=@ 0036 +dnet:result_typologies @=@ publication @=@ 0038 +dnet:result_typologies @=@ publication @=@ 0044 +dnet:result_typologies @=@ publication @=@ 0045 +dnet:result_typologies @=@ software @=@ 0029 +dnet:result_typologies @=@ software @=@ 0040 +dnet:countries @=@ AF @=@ AFG +dnet:countries @=@ AF @=@ Afghanistan +dnet:countries @=@ AD @=@ Andorra +dnet:countries @=@ AO @=@ Angola +dnet:countries @=@ AR @=@ ARG +dnet:countries @=@ AR @=@ Argentina +dnet:countries @=@ AU @=@ AUS +dnet:countries @=@ AU @=@ Australia +dnet:countries @=@ AT @=@ AUT +dnet:countries @=@ AT @=@ Austria +dnet:countries @=@ AZ @=@ AZE +dnet:countries @=@ BD @=@ Bangladesh +dnet:countries @=@ BY @=@ Belarus +dnet:countries @=@ BE @=@ BEL +dnet:countries @=@ BE @=@ Belgium +dnet:countries @=@ BJ @=@ BEN +dnet:countries @=@ BO @=@ Bolivia, Plurinational State of +dnet:countries @=@ BA @=@ BIH +dnet:countries @=@ BA @=@ Bosnia-Hercegovina +dnet:countries @=@ BR @=@ BRA +dnet:countries @=@ BR @=@ Brazil +dnet:countries @=@ BG @=@ Bulgaria +dnet:countries @=@ BF @=@ BFA +dnet:countries @=@ KH @=@ Cambodia +dnet:countries @=@ KH @=@ Cambogia +dnet:countries @=@ KH @=@ Campuchea +dnet:countries @=@ CM @=@ CMR +dnet:countries @=@ CA @=@ CAN +dnet:countries @=@ CA @=@ Canada +dnet:countries @=@ CV @=@ Cape Verde +dnet:countries @=@ CL @=@ CHL +dnet:countries @=@ CL @=@ Chile +dnet:countries @=@ CN @=@ CHN +dnet:countries @=@ CN @=@ China +dnet:countries @=@ CO @=@ COL +dnet:countries @=@ CO @=@ Colombia +dnet:countries @=@ CD @=@ Congo +dnet:countries @=@ CD @=@ Congo Democratic Republic (formerly Zaire) +dnet:countries @=@ CD @=@ Congo, Republic +dnet:countries @=@ CD @=@ Congo, the Democratic Republic of the +dnet:countries @=@ CD @=@ Zaire +dnet:countries @=@ CR @=@ CRI +dnet:countries @=@ CI @=@ CIV +dnet:countries @=@ CI @=@ Ivory Coast +dnet:countries @=@ HR @=@ Croatia +dnet:countries @=@ HR @=@ HRV +dnet:countries @=@ CY @=@ CYP +dnet:countries @=@ CY @=@ Cyprus +dnet:countries @=@ CZ @=@ CZE +dnet:countries @=@ CZ @=@ Czech Republic +dnet:countries @=@ CZ @=@ Czechia +dnet:countries @=@ CZ @=@ Czechoslovakia +dnet:countries @=@ DK @=@ DNK +dnet:countries @=@ DK @=@ Denmark +dnet:countries @=@ EC @=@ Ecuador +dnet:countries @=@ EG @=@ EGY +dnet:countries @=@ EG @=@ Egypt +dnet:countries @=@ SV @=@ SLV +dnet:countries @=@ EE @=@ EST +dnet:countries @=@ EE @=@ Estonia +dnet:countries @=@ ET @=@ ETH +dnet:countries @=@ EU @=@ EEC +dnet:countries @=@ FJ @=@ FJI +dnet:countries @=@ FI @=@ FIN +dnet:countries @=@ FI @=@ Finland +dnet:countries @=@ MK @=@ Macedonia +dnet:countries @=@ MK @=@ Macedonia, the Former Yugoslav Republic Of +dnet:countries @=@ MK @=@ North Macedonia +dnet:countries @=@ FR @=@ FRA +dnet:countries @=@ FR @=@ France +dnet:countries @=@ PF @=@ French Polynesia +dnet:countries @=@ PF @=@ PYF +dnet:countries @=@ TF @=@ French Southern Territories +dnet:countries @=@ GE @=@ Georgia +dnet:countries @=@ DE @=@ DEU +dnet:countries @=@ DE @=@ Germany +dnet:countries @=@ DE @=@ Germany, Berlin +dnet:countries @=@ GH @=@ GHA +dnet:countries @=@ GR @=@ EL +dnet:countries @=@ GR @=@ GRC +dnet:countries @=@ GL @=@ GRL +dnet:countries @=@ GN @=@ Guinea +dnet:countries @=@ GW @=@ Guinea-Bissau +dnet:countries @=@ VA @=@ Vatican State +dnet:countries @=@ HK @=@ HKG +dnet:countries @=@ HK @=@ Hong Kong +dnet:countries @=@ HK @=@ Hongkong +dnet:countries @=@ HU @=@ HUN +dnet:countries @=@ HU @=@ Hungary +dnet:countries @=@ IS @=@ ISL +dnet:countries @=@ IN @=@ IND +dnet:countries @=@ IN @=@ India +dnet:countries @=@ ID @=@ IDN +dnet:countries @=@ ID @=@ Indonesia +dnet:countries @=@ IR @=@ Iran +dnet:countries @=@ IR @=@ Iran, Islamic Republic of +dnet:countries @=@ IE @=@ IRL +dnet:countries @=@ IE @=@ Ireland +dnet:countries @=@ IL @=@ ISR +dnet:countries @=@ IL @=@ Israel +dnet:countries @=@ IT @=@ ITA +dnet:countries @=@ IT @=@ Italy +dnet:countries @=@ JM @=@ Jamaica +dnet:countries @=@ JP @=@ JPN +dnet:countries @=@ JP @=@ Japan +dnet:countries @=@ KZ @=@ KAZ +dnet:countries @=@ KZ @=@ Kazakistan +dnet:countries @=@ KZ @=@ Kazakstan +dnet:countries @=@ KE @=@ KEN +dnet:countries @=@ KE @=@ Kenya +dnet:countries @=@ KR @=@ KOR +dnet:countries @=@ KR @=@ Korea, Republic of +dnet:countries @=@ KR @=@ Korean Republic (South Korea) +dnet:countries @=@ KP @=@ PRK +dnet:countries @=@ LV @=@ LVA +dnet:countries @=@ LY @=@ Libya +dnet:countries @=@ LT @=@ LTU +dnet:countries @=@ LU @=@ LUX +dnet:countries @=@ LU @=@ Luxembourg +dnet:countries @=@ MO @=@ Macao +dnet:countries @=@ MG @=@ Madagascar +dnet:countries @=@ MY @=@ Malaysia +dnet:countries @=@ ML @=@ Mali +dnet:countries @=@ MT @=@ Malta +dnet:countries @=@ MU @=@ Mauritius +dnet:countries @=@ MX @=@ MEX +dnet:countries @=@ MX @=@ Mexico +dnet:countries @=@ FM @=@ Micronesia +dnet:countries @=@ MD @=@ Moldova +dnet:countries @=@ MD @=@ Moldova, Republic of +dnet:countries @=@ MN @=@ Mongolia +dnet:countries @=@ MA @=@ Morocco +dnet:countries @=@ MZ @=@ Mozambique +dnet:countries @=@ NA @=@ NAM +dnet:countries @=@ NL @=@ NLD +dnet:countries @=@ NL @=@ Netherlands +dnet:countries @=@ AN @=@ Netherlands Antilles +dnet:countries @=@ NC @=@ NCL +dnet:countries @=@ NZ @=@ NZL +dnet:countries @=@ NZ @=@ New Zealand +dnet:countries @=@ NO @=@ NOR +dnet:countries @=@ NO @=@ Norway +dnet:countries @=@ OC @=@ Australasia +dnet:countries @=@ OM @=@ Oman +dnet:countries @=@ PK @=@ PAK +dnet:countries @=@ PK @=@ Pakistan +dnet:countries @=@ PS @=@ Palestin, State of +dnet:countries @=@ PS @=@ Palestine, State of +dnet:countries @=@ PS @=@ Palestinian Territory, Occupied +dnet:countries @=@ PA @=@ PAN +dnet:countries @=@ PA @=@ Panama +dnet:countries @=@ PG @=@ PapuaNew Guinea +dnet:countries @=@ PE @=@ PER +dnet:countries @=@ PH @=@ PHL +dnet:countries @=@ PH @=@ Philippines +dnet:countries @=@ PL @=@ POL +dnet:countries @=@ PL @=@ Poland +dnet:countries @=@ PT @=@ PRT +dnet:countries @=@ PT @=@ Portugal +dnet:countries @=@ PR @=@ Puerto Rico +dnet:countries @=@ RO @=@ ROU +dnet:countries @=@ RO @=@ Romania +dnet:countries @=@ RU @=@ RUS +dnet:countries @=@ RU @=@ Russia +dnet:countries @=@ RU @=@ Russian Federation +dnet:countries @=@ RE @=@ Réunion +dnet:countries @=@ KN @=@ Saint Kitts And Nevis +dnet:countries @=@ SA @=@ Saudi Arabia +dnet:countries @=@ SN @=@ SEN +dnet:countries @=@ RS @=@ SRB +dnet:countries @=@ CS @=@ Serbia and Montenegro +dnet:countries @=@ SG @=@ SGP +dnet:countries @=@ SG @=@ Singapore +dnet:countries @=@ SK @=@ SVK +dnet:countries @=@ SI @=@ SVN +dnet:countries @=@ SI @=@ Slovenia +dnet:countries @=@ ZA @=@ South Africa +dnet:countries @=@ ZA @=@ ZAF +dnet:countries @=@ ES @=@ ESP +dnet:countries @=@ ES @=@ Spain +dnet:countries @=@ LK @=@ LKA +dnet:countries @=@ LK @=@ Sri Lanka +dnet:countries @=@ SD @=@ SDN +dnet:countries @=@ SR @=@ Suriname +dnet:countries @=@ SE @=@ SWE +dnet:countries @=@ SE @=@ Sweden +dnet:countries @=@ CH @=@ CHE +dnet:countries @=@ CH @=@ Switzerland +dnet:countries @=@ SY @=@ Syria +dnet:countries @=@ ST @=@ Sao Tome and Principe +dnet:countries @=@ TW @=@ TWN +dnet:countries @=@ TW @=@ Taiwan +dnet:countries @=@ TW @=@ Taiwan, Province of China +dnet:countries @=@ TZ @=@ Tanzania +dnet:countries @=@ TZ @=@ Tanzania, United Republic of +dnet:countries @=@ TH @=@ THA +dnet:countries @=@ TH @=@ Thailand +dnet:countries @=@ TL @=@ East Timor +dnet:countries @=@ TN @=@ TUN +dnet:countries @=@ TN @=@ Tunisia +dnet:countries @=@ TR @=@ TUR +dnet:countries @=@ TR @=@ Turkey +dnet:countries @=@ UNKNOWN @=@ AAA +dnet:countries @=@ UNKNOWN @=@ [Unknown] +dnet:countries @=@ UNKNOWN @=@ _? +dnet:countries @=@ UA @=@ UKR +dnet:countries @=@ UA @=@ Ukraine +dnet:countries @=@ AE @=@ United Arab Emirates +dnet:countries @=@ GB @=@ England +dnet:countries @=@ GB @=@ GBR +dnet:countries @=@ GB @=@ Great Britain +dnet:countries @=@ GB @=@ Great Britain and Northern Ireland +dnet:countries @=@ GB @=@ Scotland +dnet:countries @=@ GB @=@ UK +dnet:countries @=@ GB @=@ United Kingdom +dnet:countries @=@ US @=@ USA +dnet:countries @=@ US @=@ United States +dnet:countries @=@ US @=@ United States of America +dnet:countries @=@ UY @=@ Uruguay +dnet:countries @=@ UZ @=@ Uzbekistan +dnet:countries @=@ VE @=@ Venezuela, Bolivarian Republic of +dnet:countries @=@ VN @=@ Vietnam +dnet:countries @=@ VG @=@ British Virgin Islands +dnet:countries @=@ YU @=@ Jugoslavia +dnet:countries @=@ YU @=@ Yugoslavia +dnet:countries @=@ ZW @=@ ABW +dnet:protocols @=@ oai @=@ OAI-PMH +dnet:protocols @=@ oai @=@ OAI_PMH +dnet:pid_types @=@ orcid @=@ ORCID12 +dnet:review_levels @=@ 0000 @=@ UNKNOWN +dnet:review_levels @=@ 0002 @=@ 80 大阪経大学会「Working Paper」 +dnet:review_levels @=@ 0002 @=@ AO +dnet:review_levels @=@ 0002 @=@ ARTICLE SANS COMITE DE LECTURE (ASCL) +dnet:review_levels @=@ 0002 @=@ Arbeitspapier +dnet:review_levels @=@ 0002 @=@ Arbeitspapier [workingPaper] +dnet:review_levels @=@ 0002 @=@ Article (author) +dnet:review_levels @=@ 0002 @=@ Article type: preprint +dnet:review_levels @=@ 0002 @=@ Article(author version) +dnet:review_levels @=@ 0002 @=@ Article, not peer-reviewed +dnet:review_levels @=@ 0002 @=@ Articulo no evaluado +dnet:review_levels @=@ 0002 @=@ Artigo Solicitado e Não Avaliado por Pares +dnet:review_levels @=@ 0002 @=@ Artigo não avaliado pelos pares +dnet:review_levels @=@ 0002 @=@ Artigo não avaliado por pares +dnet:review_levels @=@ 0002 @=@ Artigo não avaliado por pres +dnet:review_levels @=@ 0002 @=@ Artikkeli|Artikkeli ammattilehdessä. Ei vertaisarvioitu +dnet:review_levels @=@ 0002 @=@ Artículo no evaluado +dnet:review_levels @=@ 0002 @=@ Book (non peer-reviewed) +dnet:review_levels @=@ 0002 @=@ Book Part (author) +dnet:review_levels @=@ 0002 @=@ Book item; Non-peer-reviewed +dnet:review_levels @=@ 0002 @=@ Conference preprint +dnet:review_levels @=@ 0002 @=@ Contribution to book (non peer-reviewed) +dnet:review_levels @=@ 0002 @=@ Discussion Paper +dnet:review_levels @=@ 0002 @=@ Document de travail (Working Paper) +dnet:review_levels @=@ 0002 @=@ Documento de trabajo +dnet:review_levels @=@ 0002 @=@ Documento de trabajo de investigaci??n +dnet:review_levels @=@ 0002 @=@ Draft +dnet:review_levels @=@ 0002 @=@ E-pub ahead of print +dnet:review_levels @=@ 0002 @=@ Editorial de revista, no evaluado por pares +dnet:review_levels @=@ 0002 @=@ Editorial de revista, não avaliado por pares +dnet:review_levels @=@ 0002 @=@ Editorial não avaliado pelos pares +dnet:review_levels @=@ 0002 @=@ Editors (non peer-reviewed) +dnet:review_levels @=@ 0002 @=@ Epub ahead of print +dnet:review_levels @=@ 0002 @=@ Hakemlik Sürecinden Geçmiş Makale +dnet:review_levels @=@ 0002 @=@ Hakemlik sürecindeki makale +dnet:review_levels @=@ 0002 @=@ Hakemlik sürecinden geçmemiş kitap değerlendirmesi +dnet:review_levels @=@ 0002 @=@ Journal Article (author version) +dnet:review_levels @=@ 0002 @=@ Journal Article Preprint +dnet:review_levels @=@ 0002 @=@ Journal Editorial, not peer-reviewed +dnet:review_levels @=@ 0002 @=@ Journal article; Non-peer-reviewed +dnet:review_levels @=@ 0002 @=@ Journal:WorkingPaper +dnet:review_levels @=@ 0002 @=@ Manuscript (preprint) +dnet:review_levels @=@ 0002 @=@ Monográfico (Informes, Documentos de trabajo, etc.) +dnet:review_levels @=@ 0002 @=@ NOTE INTERNE OU DE TRAVAIL +dnet:review_levels @=@ 0002 @=@ Nicht begutachteter Beitrag +dnet:review_levels @=@ 0002 @=@ No evaluado por pares +dnet:review_levels @=@ 0002 @=@ Non-Refereed +dnet:review_levels @=@ 0002 @=@ Non-refeered article +dnet:review_levels @=@ 0002 @=@ Non-refereed Article +dnet:review_levels @=@ 0002 @=@ Non-refereed Book Review +dnet:review_levels @=@ 0002 @=@ Non-refereed Review +dnet:review_levels @=@ 0002 @=@ Non-refereed Text +dnet:review_levels @=@ 0002 @=@ NonPeerReviewed +dnet:review_levels @=@ 0002 @=@ Not Peer reviewed +dnet:review_levels @=@ 0002 @=@ Not Reviewed +dnet:review_levels @=@ 0002 @=@ Not peer-reviewed +dnet:review_levels @=@ 0002 @=@ Não Avaliado por Pares +dnet:review_levels @=@ 0002 @=@ Não avaliada pelos pares +dnet:review_levels @=@ 0002 @=@ Não avaliado pelos pares +dnet:review_levels @=@ 0002 @=@ Original article (non peer-reviewed) +dnet:review_levels @=@ 0002 @=@ Other publication (non peer-review) +dnet:review_levels @=@ 0002 @=@ Pre Print +dnet:review_levels @=@ 0002 @=@ Pre-print +dnet:review_levels @=@ 0002 @=@ Preprint Article +dnet:review_levels @=@ 0002 @=@ Preprints +dnet:review_levels @=@ 0002 @=@ Preprints, Working Papers, ... +dnet:review_levels @=@ 0002 @=@ Rapporto tecnico / Working Paper / Rapporto di progetto +dnet:review_levels @=@ 0002 @=@ Resumo Não Avaliado por Pares +dnet:review_levels @=@ 0002 @=@ Review article (non peer-reviewed) +dnet:review_levels @=@ 0002 @=@ SMUR +dnet:review_levels @=@ 0002 @=@ Submissão dos artigos +dnet:review_levels @=@ 0002 @=@ Submitted version +dnet:review_levels @=@ 0002 @=@ Vertaisarvioimaton kirjan tai muun kokoomateoksen osa +dnet:review_levels @=@ 0002 @=@ Vorabdruck +dnet:review_levels @=@ 0002 @=@ Wetensch. publ. non-refereed +dnet:review_levels @=@ 0002 @=@ Working / discussion paper +dnet:review_levels @=@ 0002 @=@ Working Document +dnet:review_levels @=@ 0002 @=@ Working Notes +dnet:review_levels @=@ 0002 @=@ Working Paper +dnet:review_levels @=@ 0002 @=@ Working Paper / Technical Report +dnet:review_levels @=@ 0002 @=@ Working Papers +dnet:review_levels @=@ 0002 @=@ WorkingPaper +dnet:review_levels @=@ 0002 @=@ article in non peer-reviewed journal +dnet:review_levels @=@ 0002 @=@ articolo preliminare +dnet:review_levels @=@ 0002 @=@ articulo preliminar +dnet:review_levels @=@ 0002 @=@ articulo sin revision por pares +dnet:review_levels @=@ 0002 @=@ artigo preliminar +dnet:review_levels @=@ 0002 @=@ artigo sem revisão +dnet:review_levels @=@ 0002 @=@ artículo preliminar +dnet:review_levels @=@ 0002 @=@ artículo sin revisión por pares +dnet:review_levels @=@ 0002 @=@ bookchapter (author version) +dnet:review_levels @=@ 0002 @=@ borrador +dnet:review_levels @=@ 0002 @=@ column (author version) +dnet:review_levels @=@ 0002 @=@ communication_invitee +dnet:review_levels @=@ 0002 @=@ doc-type:preprint +dnet:review_levels @=@ 0002 @=@ doc-type:workingPaper +dnet:review_levels @=@ 0002 @=@ draf +dnet:review_levels @=@ 0002 @=@ eu-repo/semantics/submittedVersion +dnet:review_levels @=@ 0002 @=@ http://purl.org/coar/resource_type/c_8042 +dnet:review_levels @=@ 0002 @=@ http://purl.org/coar/resource_type/c_816b +dnet:review_levels @=@ 0002 @=@ http://purl.org/coar/version/c_71e4c1898caa6e32 +dnet:review_levels @=@ 0002 @=@ http://purl.org/coar/version/c_b1a7d7d4d402bcce +dnet:review_levels @=@ 0002 @=@ http://purl.org/eprint/type/SubmittedBookItem +dnet:review_levels @=@ 0002 @=@ http://purl.org/eprint/type/SubmittedJournalArticle +dnet:review_levels @=@ 0002 @=@ http://purl.org/info:eu-repo/semantics/authorVersion +dnet:review_levels @=@ 0002 @=@ http://purl.org/info:eu-repo/semantics/submittedVersion +dnet:review_levels @=@ 0002 @=@ http://purl.org/spar/fabio/Preprint +dnet:review_levels @=@ 0002 @=@ http://purl.org/spar/fabio/WorkingPaper +dnet:review_levels @=@ 0002 @=@ https://dictionary.casrai.org/Preprint +dnet:review_levels @=@ 0002 @=@ info:ar-repo/semantics/documento de trabajo +dnet:review_levels @=@ 0002 @=@ info:ar-repo/semantics/documentoDeTrabajo +dnet:review_levels @=@ 0002 @=@ info:eu repo/semantics/draft +dnet:review_levels @=@ 0002 @=@ info:eu-repo/semantics/authorVersion +dnet:review_levels @=@ 0002 @=@ info:eu-repo/semantics/draft +dnet:review_levels @=@ 0002 @=@ info:eu-repo/semantics/preprint +dnet:review_levels @=@ 0002 @=@ info:eu-repo/semantics/submitedVersion +dnet:review_levels @=@ 0002 @=@ info:eu-repo/semantics/submittedVersion +dnet:review_levels @=@ 0002 @=@ info:eu-repo/semantics/unReviewed +dnet:review_levels @=@ 0002 @=@ info:eu-repo/semantics/updatedVersion +dnet:review_levels @=@ 0002 @=@ info:eu-repo/semantics/workingPaper +dnet:review_levels @=@ 0002 @=@ info:eu-repo/submittedVersion +dnet:review_levels @=@ 0002 @=@ info:ulb-repo/semantics/articleNonPeerReview +dnet:review_levels @=@ 0002 @=@ info:ulb-repo/semantics/openurl/vlink-workingpaper +dnet:review_levels @=@ 0002 @=@ info:ulb-repo/semantics/workingPaper +dnet:review_levels @=@ 0002 @=@ non peer-reviewed article +dnet:review_levels @=@ 0002 @=@ non-refereed review article +dnet:review_levels @=@ 0002 @=@ não avaliado +dnet:review_levels @=@ 0002 @=@ preprint +dnet:review_levels @=@ 0002 @=@ prepublicación +dnet:review_levels @=@ 0002 @=@ proceeding, seminar, workshop without peer review +dnet:review_levels @=@ 0002 @=@ proceedings (author version) +dnet:review_levels @=@ 0002 @=@ pré-print +dnet:review_levels @=@ 0002 @=@ pré-publication +dnet:review_levels @=@ 0002 @=@ préprint +dnet:review_levels @=@ 0002 @=@ prépublication +dnet:review_levels @=@ 0002 @=@ publicació preliminar +dnet:review_levels @=@ 0002 @=@ publication-preprint +dnet:review_levels @=@ 0002 @=@ publication-workingpaper +dnet:review_levels @=@ 0002 @=@ submitedVersion +dnet:review_levels @=@ 0002 @=@ submittedVersion +dnet:review_levels @=@ 0002 @=@ voordruk +dnet:review_levels @=@ 0002 @=@ workingPaper +dnet:review_levels @=@ 0002 @=@ ön baskı +dnet:review_levels @=@ 0002 @=@ Препринт +dnet:review_levels @=@ 0002 @=@ предпечатная версия публикации +dnet:review_levels @=@ 0002 @=@ препринт статьи +dnet:review_levels @=@ 0002 @=@ ディスカッション／ワーキング・ペーパー DP/WP +dnet:review_levels @=@ 0002 @=@ プレプリント +dnet:review_levels @=@ 0002 @=@ プレプリント Preprint +dnet:review_levels @=@ 0002 @=@ プレプリント（Preprint） +dnet:review_levels @=@ 0002 @=@ 印刷物/電子媒体-その他（査読無し） +dnet:review_levels @=@ 0002 @=@ 印刷物/電子媒体-テクニカルレポート類（査読無し） +dnet:review_levels @=@ 0002 @=@ 印刷物/電子媒体-会議発表論文（査読無し） +dnet:review_levels @=@ 0002 @=@ 印刷物/電子媒体-図書（査読無し） +dnet:review_levels @=@ 0002 @=@ 印刷物/電子媒体-学術雑誌論文（査読無し） +dnet:review_levels @=@ 0002 @=@ 印刷物/電子媒体-紀要論文（査読無し） +dnet:review_levels @=@ 0002 @=@ 印刷物/電子媒体-雑誌記事（査読無し） +dnet:review_levels @=@ 0002 @=@ 预印本 +dnet:review_levels @=@ 0001 @=@ ##rt.metadata.pkp.peerReviewed## +dnet:review_levels @=@ 0001 @=@ A1 Alkuperäisartikkeli tieteellisessä aikakauslehdessä +dnet:review_levels @=@ 0001 @=@ Art?culo revisado por pares +dnet:review_levels @=@ 0001 @=@ Article revisat per persones expertes +dnet:review_levels @=@ 0001 @=@ Article type: peer review +dnet:review_levels @=@ 0001 @=@ Article évalué par les pairs +dnet:review_levels @=@ 0001 @=@ Article évalué par des pairs +dnet:review_levels @=@ 0001 @=@ Article évalué par les pairs +dnet:review_levels @=@ 0001 @=@ Articolo valutato secondo i criteri della peer review +dnet:review_levels @=@ 0001 @=@ Articulo evaluado por dos pares +dnet:review_levels @=@ 0001 @=@ Articulo revisado por pares +dnet:review_levels @=@ 0001 @=@ Artigo Avaliado pelos Pares +dnet:review_levels @=@ 0001 @=@ Artigo Revisto por Pares +dnet:review_levels @=@ 0001 @=@ Artigo avaliado por blind peer review +dnet:review_levels @=@ 0001 @=@ Artigo avaliado por pares +dnet:review_levels @=@ 0001 @=@ Artigo de convidado. Avaliado pelos pares +dnet:review_levels @=@ 0001 @=@ Artigos; Avaliado pelos pares +dnet:review_levels @=@ 0001 @=@ Artículo de investigación, Investigaciones originales, Artículo evaluado por pares, Investigaciones empíricas +dnet:review_levels @=@ 0001 @=@ Artículo evaluado por pares +dnet:review_levels @=@ 0001 @=@ Artículo evaluado por pares, Ensayos de investigación +dnet:review_levels @=@ 0001 @=@ Artículo evaluado por pares, Investigaciones empíricas, Artículos de investigación +dnet:review_levels @=@ 0001 @=@ Artículo revisado +dnet:review_levels @=@ 0001 @=@ Artículo revisado por pares +dnet:review_levels @=@ 0001 @=@ Artículos de estudiantes, Artículo evaluado por pares, Artículos de investigación +dnet:review_levels @=@ 0001 @=@ Artículos de investigación evaluados por doble ciego +dnet:review_levels @=@ 0001 @=@ Artículos evaluadores por doble ciego +dnet:review_levels @=@ 0001 @=@ Artículos evaluados por pares +dnet:review_levels @=@ 0001 @=@ Artículos evaluados por pares académicos +dnet:review_levels @=@ 0001 @=@ Artículos revisados por pares +dnet:review_levels @=@ 0001 @=@ Avaliadas pelos pares +dnet:review_levels @=@ 0001 @=@ Avaliado anonimamente por pares +dnet:review_levels @=@ 0001 @=@ Avaliado em duplo cego por pares +dnet:review_levels @=@ 0001 @=@ Avaliado pela Editoria +dnet:review_levels @=@ 0001 @=@ Avaliado pela Editoria. Avaliado pelos pares. +dnet:review_levels @=@ 0001 @=@ Avaliado pelo Editoria +dnet:review_levels @=@ 0001 @=@ Avaliado pelo pares +dnet:review_levels @=@ 0001 @=@ Avaliado pelos Editores +dnet:review_levels @=@ 0001 @=@ Avaliado pelos pares +dnet:review_levels @=@ 0001 @=@ Avaliado pelos pares, Artigo de convidado +dnet:review_levels @=@ 0001 @=@ Avaliado pelos pares, Artigos Originais +dnet:review_levels @=@ 0001 @=@ Avaliado pelos pares, Artigos Originais, Artigos de Revisão +dnet:review_levels @=@ 0001 @=@ Avaliado pelos pares. Avaliado pelo Editoria +dnet:review_levels @=@ 0001 @=@ Avaliado po Pares +dnet:review_levels @=@ 0001 @=@ Avaliado por Editor +dnet:review_levels @=@ 0001 @=@ Avaliado por pares +dnet:review_levels @=@ 0001 @=@ Avaliados pelos pares +dnet:review_levels @=@ 0001 @=@ Avaliados por Pares +dnet:review_levels @=@ 0001 @=@ Blind Peer-reviewed Article +dnet:review_levels @=@ 0001 @=@ Book (peer-reviewed) +dnet:review_levels @=@ 0001 @=@ Comentario de libros, Comentario de revistas, Comentario de conferencias, Artículo evaluado por pares, Artículo de investigación +dnet:review_levels @=@ 0001 @=@ Conference paper; Peer-reviewed +dnet:review_levels @=@ 0001 @=@ Contribution to book (peer-reviewed) +dnet:review_levels @=@ 0001 @=@ Documento Avaliado por Pares +dnet:review_levels @=@ 0001 @=@ Double blind evaluation articles +dnet:review_levels @=@ 0001 @=@ Double blind peer review +dnet:review_levels @=@ 0001 @=@ Editors (peer-reviewed) +dnet:review_levels @=@ 0001 @=@ Evaluación por pares +dnet:review_levels @=@ 0001 @=@ Evaluado por pares +dnet:review_levels @=@ 0001 @=@ Evaluados por los pares +dnet:review_levels @=@ 0001 @=@ Hakem sürecinden geçmiş makale +dnet:review_levels @=@ 0001 @=@ Hakemli makale +dnet:review_levels @=@ 0001 @=@ Hakemlik Sürecinden Geçmiş +dnet:review_levels @=@ 0001 @=@ Invited Peer-Reviewed Article +dnet:review_levels @=@ 0001 @=@ Journal article; Peer-reviewed +dnet:review_levels @=@ 0001 @=@ Original article (peer-reviewed) +dnet:review_levels @=@ 0001 @=@ Other publication (peer-review) +dnet:review_levels @=@ 0001 @=@ Paper peer-reviewed +dnet:review_levels @=@ 0001 @=@ Papers evaluated by academic peers +dnet:review_levels @=@ 0001 @=@ Peer reviewed +dnet:review_levels @=@ 0001 @=@ Peer reviewed article +dnet:review_levels @=@ 0001 @=@ Peer reviewed invited commentry +dnet:review_levels @=@ 0001 @=@ Peer-Reviewed Protocol +dnet:review_levels @=@ 0001 @=@ Peer-reviewd Article +dnet:review_levels @=@ 0001 @=@ Peer-reviewed +dnet:review_levels @=@ 0001 @=@ Peer-reviewed Article +dnet:review_levels @=@ 0001 @=@ Peer-reviewed Paper +dnet:review_levels @=@ 0001 @=@ Peer-reviewed Review +dnet:review_levels @=@ 0001 @=@ Peer-reviewed Review Article +dnet:review_levels @=@ 0001 @=@ Peer-reviewed Text +dnet:review_levels @=@ 0001 @=@ Peer-reviewed communication +dnet:review_levels @=@ 0001 @=@ Peer-reviewed conference proceedings +dnet:review_levels @=@ 0001 @=@ Peer-reviewed research article +dnet:review_levels @=@ 0001 @=@ Peer-reviewed short communication +dnet:review_levels @=@ 0001 @=@ PeerReviewed +dnet:review_levels @=@ 0001 @=@ Proceedings (peer-reviewed) +dnet:review_levels @=@ 0001 @=@ Refereed +dnet:review_levels @=@ 0001 @=@ Refereed Article +dnet:review_levels @=@ 0001 @=@ Research articles evaluated by double blind +dnet:review_levels @=@ 0001 @=@ Resenha avaliada pelos pares +dnet:review_levels @=@ 0001 @=@ Review article (peer-reviewed) +dnet:review_levels @=@ 0001 @=@ Reviewed by peers +dnet:review_levels @=@ 0001 @=@ Revisión por Expertos +dnet:review_levels @=@ 0001 @=@ Revisto por Pares +dnet:review_levels @=@ 0001 @=@ SBBq abstracts / peer-reviewed +dnet:review_levels @=@ 0001 @=@ SBBq resúmenes - revisada por pares +dnet:review_levels @=@ 0001 @=@ Scholarly publ. Refereed +dnet:review_levels @=@ 0001 @=@ Scientific Publ (refereed) +dnet:review_levels @=@ 0001 @=@ Vertaisarvioimaton kirjoitus tieteellisessä aikakauslehdessä +dnet:review_levels @=@ 0001 @=@ Vertaisarvioitu alkuperäisartikkeli tieteellisessä aikakauslehdessä +dnet:review_levels @=@ 0001 @=@ Vertaisarvioitu artikkeli konferenssijulkaisussa +dnet:review_levels @=@ 0001 @=@ Vertaisarvioitu artikkeli tieteellisessä aikakauslehdessä +dnet:review_levels @=@ 0001 @=@ Vertaisarvioitu kirjan tai muun kokoomateoksen osa +dnet:review_levels @=@ 0001 @=@ Wetensch. publ. Refereed +dnet:review_levels @=@ 0001 @=@ article in peer-reviewed journal +dnet:review_levels @=@ 0001 @=@ articles validés +dnet:review_levels @=@ 0001 @=@ avaliado por pares, temas livres +dnet:review_levels @=@ 0001 @=@ info:eu-repo/semantics/peerReviewed +dnet:review_levels @=@ 0001 @=@ info:ulb-repo/semantics/articlePeerReview +dnet:review_levels @=@ 0001 @=@ proceeding with peer review +dnet:review_levels @=@ 0001 @=@ refereed_publications +dnet:review_levels @=@ 0001 @=@ ul_published_reviewed +dnet:review_levels @=@ 0001 @=@ Άρθρο που έχει αξιολογηθεί από ομότιμους ειδικούς +dnet:review_levels @=@ 0001 @=@ Άρθρο το οποίο έχει περάσει από ομότιμη αξιολόγηση +dnet:review_levels @=@ 0001 @=@ レフェリー付き論文 +dnet:review_levels @=@ 0001 @=@ 印刷物/電子媒体-テクニカルレポート類（査読有り） +dnet:review_levels @=@ 0001 @=@ 印刷物/電子媒体-会議発表論文（査読有り） +dnet:review_levels @=@ 0001 @=@ 印刷物/電子媒体-図書（査読有り） +dnet:review_levels @=@ 0001 @=@ 印刷物/電子媒体-学術雑誌論文（査読有り） +dnet:review_levels @=@ 0001 @=@ 印刷物/電子媒体-紀要論文（査読有り） +dnet:review_levels @=@ 0001 @=@ 印刷物/電子媒体-雑誌記事（査読有り） +dnet:review_levels @=@ 0001 @=@ 原著論文（査読有り） +dnet:review_levels @=@ 0001 @=@ 査読論文 \ No newline at end of file diff --git a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/terms.txt b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/terms.txt new file mode 100644 index 0000000000..59bed7c3a0 --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/terms.txt @@ -0,0 +1,1079 @@ +ModularUiLabels @=@ ModularUiLabels @=@ PendingRepositoryResources @=@ Pending datasource +ModularUiLabels @=@ ModularUiLabels @=@ RepositoryServiceResources @=@ Valid datasource +dnet:content_description_typologies @=@ D-Net Content Description Typologies @=@ file::EuropePMC @=@ file::EuropePMC +dnet:content_description_typologies @=@ D-Net Content Description Typologies @=@ file::PDF @=@ file::PDF +dnet:content_description_typologies @=@ D-Net Content Description Typologies @=@ file::WoS @=@ file::WoS +dnet:content_description_typologies @=@ D-Net Content Description Typologies @=@ metadata @=@ metadata +dnet:content_description_typologies @=@ D-Net Content Description Typologies @=@ file::hybrid @=@ file::hybrid +dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:crosswalk:cris @=@ Harvested +dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:actionset:orcidworks-no-doi @=@ Harvested +dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:crosswalk:infospace @=@ Harvested +dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:crosswalk @=@ Harvested +dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:crosswalk:aggregator @=@ Harvested +dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:crosswalk:datasetarchive @=@ Harvested +dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:actionset @=@ Harvested +dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:crosswalk:entityregistry @=@ Harvested +dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:crosswalk:repository @=@ Harvested +dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:mining:aggregator @=@ Inferred by OpenAIRE +dnet:provenanceActions @=@ dnet:provenanceActions @=@ community:subject @=@ Inferred by OpenAIRE +dnet:provenanceActions @=@ dnet:provenanceActions @=@ community:zenodocommunity @=@ Inferred by OpenAIRE +dnet:provenanceActions @=@ dnet:provenanceActions @=@ iis @=@ Inferred by OpenAIRE +dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:mining:entityregistry @=@ Inferred by OpenAIRE +dnet:provenanceActions @=@ dnet:provenanceActions @=@ community:organization @=@ Inferred by OpenAIRE +dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:mining:infospace @=@ Inferred by OpenAIRE +dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:dedup @=@ Inferred by OpenAIRE +dnet:provenanceActions @=@ dnet:provenanceActions @=@ community:datasource @=@ Inferred by OpenAIRE +dnet:provenanceActions @=@ dnet:provenanceActions @=@ propagation:project:semrel @=@ Inferred by OpenAIRE +dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:mining:cris @=@ Inferred by OpenAIRE +dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:mining:repository @=@ Inferred by OpenAIRE +dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:mining:datasetarchive @=@ Inferred by OpenAIRE +dnet:provenanceActions @=@ dnet:provenanceActions @=@ community:semrel @=@ Inferred by OpenAIRE +dnet:provenanceActions @=@ dnet:provenanceActions @=@ user:claim @=@ Linked by user +dnet:provenanceActions @=@ dnet:provenanceActions @=@ user:claim:pid @=@ Linked by user +dnet:provenanceActions @=@ dnet:provenanceActions @=@ user:insert @=@ Linked by user +dnet:provenanceActions @=@ dnet:provenanceActions @=@ user:claim:search @=@ Linked by user +dnet:provenanceActions @=@ dnet:provenanceActions @=@ UNKNOWN @=@ UNKNOWN +dnet:provenanceActions @=@ dnet:provenanceActions @=@ country:instrepos @=@ Inferred by OpenAIRE +dnet:access_modes @=@ dnet:access_modes @=@ 12MONTHS @=@ 12 Months Embargo +dnet:access_modes @=@ dnet:access_modes @=@ 6MONTHS @=@ 6 Months Embargo +dnet:access_modes @=@ dnet:access_modes @=@ CLOSED @=@ Closed Access +dnet:access_modes @=@ dnet:access_modes @=@ EMBARGO @=@ Embargo +dnet:access_modes @=@ dnet:access_modes @=@ OPEN @=@ Open Access +dnet:access_modes @=@ dnet:access_modes @=@ OPEN SOURCE @=@ Open Source +dnet:access_modes @=@ dnet:access_modes @=@ OTHER @=@ Other +dnet:access_modes @=@ dnet:access_modes @=@ RESTRICTED @=@ Restricted +dnet:access_modes @=@ dnet:access_modes @=@ UNKNOWN @=@ not available +fct:funding_typologies @=@ fct:funding_typologies @=@ fct:program @=@ fct:program +dnet:compatibilityLevel @=@ dnet:compatibilityLevel @=@ openaire2.0 @=@ OpenAIRE 2.0 (EC funding) +dnet:compatibilityLevel @=@ dnet:compatibilityLevel @=@ openaire3.0 @=@ OpenAIRE 3.0 (OA, funding) +dnet:compatibilityLevel @=@ dnet:compatibilityLevel @=@ driver @=@ OpenAIRE Basic (DRIVER OA) +dnet:compatibilityLevel @=@ dnet:compatibilityLevel @=@ openaire-cris_1.1 @=@ OpenAIRE CRIS v1.1 +dnet:compatibilityLevel @=@ dnet:compatibilityLevel @=@ openaire2.0_data @=@ OpenAIRE Data (funded, referenced datasets) +dnet:compatibilityLevel @=@ dnet:compatibilityLevel @=@ openaire-pub_4.0 @=@ OpenAIRE PubRepos v4.0 +dnet:compatibilityLevel @=@ dnet:compatibilityLevel @=@ hostedBy @=@ collected from a compatible aggregator +dnet:compatibilityLevel @=@ dnet:compatibilityLevel @=@ files @=@ files +dnet:compatibilityLevel @=@ dnet:compatibilityLevel @=@ native @=@ native +dnet:compatibilityLevel @=@ dnet:compatibilityLevel @=@ UNKNOWN @=@ not available +dnet:compatibilityLevel @=@ dnet:compatibilityLevel @=@ notCompatible @=@ under validation +dnet:dataCite_date @=@ dnet:dataCite_date @=@ UNKNOWN @=@ UNKNOWN +dnet:dataCite_date @=@ dnet:dataCite_date @=@ available @=@ available +dnet:dataCite_date @=@ dnet:dataCite_date @=@ copyrighted @=@ copyrighted +dnet:dataCite_date @=@ dnet:dataCite_date @=@ created @=@ created +dnet:dataCite_date @=@ dnet:dataCite_date @=@ endDate @=@ endDate +dnet:dataCite_date @=@ dnet:dataCite_date @=@ issued @=@ issued +dnet:dataCite_date @=@ dnet:dataCite_date @=@ startDate @=@ startDate +dnet:dataCite_date @=@ dnet:dataCite_date @=@ submitted @=@ submitted +dnet:dataCite_date @=@ dnet:dataCite_date @=@ updated @=@ updated +dnet:dataCite_date @=@ dnet:dataCite_date @=@ valid @=@ valid +dnet:dataCite_date @=@ dnet:dataCite_date @=@ published-print @=@ published-print +dnet:dataCite_date @=@ dnet:dataCite_date @=@ published-online @=@ published-online +dnet:dataCite_date @=@ dnet:dataCite_date @=@ accepted @=@ accepted +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ crissystem @=@ CRIS System +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ datarepository::unknown @=@ Data Repository +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ aggregator::datarepository @=@ Data Repository Aggregator +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ entityregistry::projects @=@ Funder database +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ infospace @=@ Information Space +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ pubsrepository::institutional @=@ Institutional Repository +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ aggregator::pubsrepository::institutional @=@ Institutional Repository Aggregator +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ pubsrepository::journal @=@ Journal +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ aggregator::pubsrepository::journals @=@ Journal Aggregator/Publisher +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ pubsrepository::mock @=@ Other +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ pubscatalogue::unknown @=@ Publication Catalogue +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ pubsrepository::unknown @=@ Publication Repository +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ aggregator::pubsrepository::unknown @=@ Publication Repository Aggregator +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ entityregistry @=@ Registry +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ entityregistry::repositories @=@ Registry of repositories +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ entityregistry::products @=@ Registry of research products +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ entityregistry::researchers @=@ Registry of researchers +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ entityregistry::organizations @=@ Registry of organizations +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ scholarcomminfra @=@ Scholarly Comm. Infrastructure +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ softwarerepository @=@ Software Repository +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ pubsrepository::thematic @=@ Thematic Repository +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ websource @=@ Web Source +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ aggregator::softwarerepository @=@ Software Repository Aggregator +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ orprepository @=@ Repository +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ researchgraph @=@ Research Graph +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ ACM @=@ ACM Computing Classification System +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ agrovoc @=@ AGROVOC +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ bicssc @=@ BIC standard subject categories +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ DFG @=@ DFG Classification +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ ddc @=@ Dewey Decimal Classification +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ nsf:fieldOfApplication @=@ Field of Application (NSF) +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ gok @=@ Göttingen Online Classification +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ ec:h2020topics @=@ Horizon 2020 Topics +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ IPC @=@ International Patent Classification +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ jel @=@ JEL Classification +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ lcsh @=@ Library of Congress Subject Headings +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ msc @=@ Mathematics Subject Classification +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ mesheuropmc @=@ Medical Subject Headings +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ mesh @=@ Medical Subject Headings +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ bk @=@ Nederlandse basisclassificatie +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ dnet:od_subjects @=@ OpenDOAR subjects +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ ocis @=@ Optics Classification and Indexing Scheme +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ pacs @=@ Physics and Astronomy Classification Scheme +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ rvk @=@ Regensburger Verbundklassifikation +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ UNKNOWN @=@ UNKNOWN +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ udc @=@ Universal Decimal Classification +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ wos @=@ Web of Science Subject Areas +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ arxiv @=@ arXiv +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ keyword @=@ keyword +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ MAG @=@ Microsoft Academic Graph classification +fct:contractTypes @=@ fct:contractTypes @=@ UNKNOWN @=@ UNKNOWN +dnet:publication_resource @=@ dnet:publication_resource @=@ 0018 @=@ Annotation +dnet:publication_resource @=@ dnet:publication_resource @=@ 0001 @=@ Article +dnet:publication_resource @=@ dnet:publication_resource @=@ 0033 @=@ Audiovisual +dnet:publication_resource @=@ dnet:publication_resource @=@ 0008 @=@ Bachelor thesis +dnet:publication_resource @=@ dnet:publication_resource @=@ 0046 @=@ Bioentity +dnet:publication_resource @=@ dnet:publication_resource @=@ 0002 @=@ Book +dnet:publication_resource @=@ dnet:publication_resource @=@ 0037 @=@ Clinical Trial +dnet:publication_resource @=@ dnet:publication_resource @=@ 0022 @=@ Collection +dnet:publication_resource @=@ dnet:publication_resource @=@ 0004 @=@ Conference object +dnet:publication_resource @=@ dnet:publication_resource @=@ 0005 @=@ Contribution for newspaper or weekly magazine +dnet:publication_resource @=@ dnet:publication_resource @=@ 0045 @=@ Data Management Plan +dnet:publication_resource @=@ dnet:publication_resource @=@ 0031 @=@ Data Paper +dnet:publication_resource @=@ dnet:publication_resource @=@ 0021 @=@ Dataset +dnet:publication_resource @=@ dnet:publication_resource @=@ 0006 @=@ Doctoral thesis +dnet:publication_resource @=@ dnet:publication_resource @=@ 0023 @=@ Event +dnet:publication_resource @=@ dnet:publication_resource @=@ 0009 @=@ External research report +dnet:publication_resource @=@ dnet:publication_resource @=@ 0024 @=@ Film +dnet:publication_resource @=@ dnet:publication_resource @=@ 0025 @=@ Image +dnet:publication_resource @=@ dnet:publication_resource @=@ 0026 @=@ InteractiveResource +dnet:publication_resource @=@ dnet:publication_resource @=@ 0011 @=@ Internal report +dnet:publication_resource @=@ dnet:publication_resource @=@ 0043 @=@ Journal +dnet:publication_resource @=@ dnet:publication_resource @=@ 0010 @=@ Lecture +dnet:publication_resource @=@ dnet:publication_resource @=@ 0007 @=@ Master thesis +dnet:publication_resource @=@ dnet:publication_resource @=@ 0027 @=@ Model +dnet:publication_resource @=@ dnet:publication_resource @=@ 0012 @=@ Newsletter +dnet:publication_resource @=@ dnet:publication_resource @=@ 0020 @=@ Other ORP type +dnet:publication_resource @=@ dnet:publication_resource @=@ 0039 @=@ Other dataset type +dnet:publication_resource @=@ dnet:publication_resource @=@ 0038 @=@ Other literature type +dnet:publication_resource @=@ dnet:publication_resource @=@ 0040 @=@ Other software type +dnet:publication_resource @=@ dnet:publication_resource @=@ 0013 @=@ Part of book or chapter of book +dnet:publication_resource @=@ dnet:publication_resource @=@ 0019 @=@ Patent +dnet:publication_resource @=@ dnet:publication_resource @=@ 0028 @=@ PhysicalObject +dnet:publication_resource @=@ dnet:publication_resource @=@ 0016 @=@ Preprint +dnet:publication_resource @=@ dnet:publication_resource @=@ 0034 @=@ Project deliverable +dnet:publication_resource @=@ dnet:publication_resource @=@ 0035 @=@ Project milestone +dnet:publication_resource @=@ dnet:publication_resource @=@ 0036 @=@ Project proposal +dnet:publication_resource @=@ dnet:publication_resource @=@ 0017 @=@ Report +dnet:publication_resource @=@ dnet:publication_resource @=@ 0014 @=@ Research +dnet:publication_resource @=@ dnet:publication_resource @=@ 0015 @=@ Review +dnet:publication_resource @=@ dnet:publication_resource @=@ 0029 @=@ Software +dnet:publication_resource @=@ dnet:publication_resource @=@ 0032 @=@ Software Paper +dnet:publication_resource @=@ dnet:publication_resource @=@ 0030 @=@ Sound +dnet:publication_resource @=@ dnet:publication_resource @=@ 0044 @=@ Thesis +dnet:publication_resource @=@ dnet:publication_resource @=@ 0000 @=@ Unknown +dnet:publication_resource @=@ dnet:publication_resource @=@ 0042 @=@ Virtual Appliance +ec:funding_typologies @=@ ec:funding_typologies @=@ ec:frameworkprogram @=@ frameworkprogram +ec:funding_typologies @=@ ec:funding_typologies @=@ ec:program @=@ program +ec:funding_typologies @=@ ec:funding_typologies @=@ ec:specificprogram @=@ specificprogram +ec:FP7contractTypes @=@ ec:FP7contractTypes @=@ 171 @=@ Article 171 of the Treaty +ec:FP7contractTypes @=@ ec:FP7contractTypes @=@ BSG @=@ Research for the benefit of specific groups +ec:FP7contractTypes @=@ ec:FP7contractTypes @=@ CIP-EIP-TN @=@ CIP-Eco-Innovation - CIP-Thematic Network +ec:FP7contractTypes @=@ ec:FP7contractTypes @=@ CP @=@ Collaborative project +ec:FP7contractTypes @=@ ec:FP7contractTypes @=@ CP-CSA @=@ Combination of CP & CSA +ec:FP7contractTypes @=@ ec:FP7contractTypes @=@ CSA @=@ Coordination and support action +ec:FP7contractTypes @=@ ec:FP7contractTypes @=@ ERC @=@ Support for frontier research (ERC) +ec:FP7contractTypes @=@ ec:FP7contractTypes @=@ MC @=@ Support for training and career development of researchers (Marie Curie) +ec:FP7contractTypes @=@ ec:FP7contractTypes @=@ NoE @=@ Network of Excellence +wt:funding_relations @=@ wt:funding_relations @=@ wt:hasParentFunding @=@ wt:hasParentFunding +dnet:languages @=@ dnet:languages @=@ abk @=@ Abkhazian +dnet:languages @=@ dnet:languages @=@ ace @=@ Achinese +dnet:languages @=@ dnet:languages @=@ ach @=@ Acoli +dnet:languages @=@ dnet:languages @=@ ada @=@ Adangme +dnet:languages @=@ dnet:languages @=@ aar @=@ Afar +dnet:languages @=@ dnet:languages @=@ afh @=@ Afrihili +dnet:languages @=@ dnet:languages @=@ afr @=@ Afrikaans +dnet:languages @=@ dnet:languages @=@ afa @=@ Afro-Asiatic +dnet:languages @=@ dnet:languages @=@ aka @=@ Akan +dnet:languages @=@ dnet:languages @=@ akk @=@ Akkadian +dnet:languages @=@ dnet:languages @=@ alb/sqi @=@ Albanian +dnet:languages @=@ dnet:languages @=@ ale @=@ Aleut +dnet:languages @=@ dnet:languages @=@ alg @=@ Algonquian languages +dnet:languages @=@ dnet:languages @=@ tut @=@ Altaic +dnet:languages @=@ dnet:languages @=@ amh @=@ Amharic +dnet:languages @=@ dnet:languages @=@ egy @=@ Ancient Egyptian +dnet:languages @=@ dnet:languages @=@ grc @=@ Ancient Greek +dnet:languages @=@ dnet:languages @=@ apa @=@ Apache +dnet:languages @=@ dnet:languages @=@ ara @=@ Arabic +dnet:languages @=@ dnet:languages @=@ arg @=@ Aragonese +dnet:languages @=@ dnet:languages @=@ arc @=@ Aramaic +dnet:languages @=@ dnet:languages @=@ arp @=@ Arapaho +dnet:languages @=@ dnet:languages @=@ arn @=@ Araucanian +dnet:languages @=@ dnet:languages @=@ arw @=@ Arawak +dnet:languages @=@ dnet:languages @=@ arm/hye @=@ Armenian +dnet:languages @=@ dnet:languages @=@ art @=@ Artificial +dnet:languages @=@ dnet:languages @=@ asm @=@ Assamese +dnet:languages @=@ dnet:languages @=@ ath @=@ Athapascan +dnet:languages @=@ dnet:languages @=@ map @=@ Austronesian +dnet:languages @=@ dnet:languages @=@ ina @=@ Auxiliary Language Association) +dnet:languages @=@ dnet:languages @=@ ava @=@ Avaric +dnet:languages @=@ dnet:languages @=@ ave @=@ Avestan +dnet:languages @=@ dnet:languages @=@ awa @=@ Awadhi +dnet:languages @=@ dnet:languages @=@ aym @=@ Aymara +dnet:languages @=@ dnet:languages @=@ aze @=@ Azerbaijani +dnet:languages @=@ dnet:languages @=@ nah @=@ Aztec +dnet:languages @=@ dnet:languages @=@ ban @=@ Balinese +dnet:languages @=@ dnet:languages @=@ bat @=@ Baltic +dnet:languages @=@ dnet:languages @=@ bal @=@ Baluchi +dnet:languages @=@ dnet:languages @=@ bam @=@ Bambara +dnet:languages @=@ dnet:languages @=@ bai @=@ Bamileke +dnet:languages @=@ dnet:languages @=@ bad @=@ Banda +dnet:languages @=@ dnet:languages @=@ bnt @=@ Bantu +dnet:languages @=@ dnet:languages @=@ bas @=@ Basa +dnet:languages @=@ dnet:languages @=@ bak @=@ Bashkir +dnet:languages @=@ dnet:languages @=@ baq/eus @=@ Basque +dnet:languages @=@ dnet:languages @=@ bej @=@ Beja +dnet:languages @=@ dnet:languages @=@ bel @=@ Belarusian +dnet:languages @=@ dnet:languages @=@ bem @=@ Bemba +dnet:languages @=@ dnet:languages @=@ ben @=@ Bengali +dnet:languages @=@ dnet:languages @=@ ber @=@ Berber +dnet:languages @=@ dnet:languages @=@ bho @=@ Bhojpuri +dnet:languages @=@ dnet:languages @=@ bih @=@ Bihari +dnet:languages @=@ dnet:languages @=@ bik @=@ Bikol +dnet:languages @=@ dnet:languages @=@ bin @=@ Bini +dnet:languages @=@ dnet:languages @=@ bis @=@ Bislama +dnet:languages @=@ dnet:languages @=@ nob @=@ Bokmål, Norwegian; Norwegian Bokmål +dnet:languages @=@ dnet:languages @=@ bos @=@ Bosnian +dnet:languages @=@ dnet:languages @=@ bra @=@ Braj +dnet:languages @=@ dnet:languages @=@ bre @=@ Breton +dnet:languages @=@ dnet:languages @=@ bug @=@ Buginese +dnet:languages @=@ dnet:languages @=@ bul @=@ Bulgarian +dnet:languages @=@ dnet:languages @=@ bua @=@ Buriat +dnet:languages @=@ dnet:languages @=@ bur/mya @=@ Burmese +dnet:languages @=@ dnet:languages @=@ cad @=@ Caddo +dnet:languages @=@ dnet:languages @=@ car @=@ Carib +dnet:languages @=@ dnet:languages @=@ cat @=@ Catalan; Valencian +dnet:languages @=@ dnet:languages @=@ cau @=@ Caucasian +dnet:languages @=@ dnet:languages @=@ ceb @=@ Cebuano +dnet:languages @=@ dnet:languages @=@ cel @=@ Celtic +dnet:languages @=@ dnet:languages @=@ cai @=@ Central American Indian +dnet:languages @=@ dnet:languages @=@ chg @=@ Chagatai +dnet:languages @=@ dnet:languages @=@ cha @=@ Chamorro +dnet:languages @=@ dnet:languages @=@ che @=@ Chechen +dnet:languages @=@ dnet:languages @=@ chr @=@ Cherokee +dnet:languages @=@ dnet:languages @=@ nya @=@ Chewa; Chichewa; Nyanja +dnet:languages @=@ dnet:languages @=@ chy @=@ Cheyenne +dnet:languages @=@ dnet:languages @=@ chb @=@ Chibcha +dnet:languages @=@ dnet:languages @=@ chi/zho @=@ Chinese +dnet:languages @=@ dnet:languages @=@ chn @=@ Chinook jargon +dnet:languages @=@ dnet:languages @=@ cho @=@ Choctaw +dnet:languages @=@ dnet:languages @=@ chu @=@ Church Slavic; Slavonic; Church Slavonic; Old Bulgarian; Old Church Slavonic +dnet:languages @=@ dnet:languages @=@ chv @=@ Chuvash +dnet:languages @=@ dnet:languages @=@ cop @=@ Coptic +dnet:languages @=@ dnet:languages @=@ cor @=@ Cornish +dnet:languages @=@ dnet:languages @=@ cos @=@ Corsican +dnet:languages @=@ dnet:languages @=@ cre @=@ Cree +dnet:languages @=@ dnet:languages @=@ mus @=@ Creek +dnet:languages @=@ dnet:languages @=@ crp @=@ Creoles and Pidgins +dnet:languages @=@ dnet:languages @=@ hrv @=@ Croatian +dnet:languages @=@ dnet:languages @=@ cus @=@ Cushitic +dnet:languages @=@ dnet:languages @=@ ces/cze @=@ Czech +dnet:languages @=@ dnet:languages @=@ dak @=@ Dakota +dnet:languages @=@ dnet:languages @=@ dan @=@ Danish +dnet:languages @=@ dnet:languages @=@ del @=@ Delaware +dnet:languages @=@ dnet:languages @=@ din @=@ Dinka +dnet:languages @=@ dnet:languages @=@ div @=@ Divehi +dnet:languages @=@ dnet:languages @=@ doi @=@ Dogri +dnet:languages @=@ dnet:languages @=@ dra @=@ Dravidian +dnet:languages @=@ dnet:languages @=@ dua @=@ Duala +dnet:languages @=@ dnet:languages @=@ dut/nld @=@ Dutch; Flemish +dnet:languages @=@ dnet:languages @=@ dyu @=@ Dyula +dnet:languages @=@ dnet:languages @=@ dzo @=@ Dzongkha +dnet:languages @=@ dnet:languages @=@ efi @=@ Efik +dnet:languages @=@ dnet:languages @=@ eka @=@ Ekajuk +dnet:languages @=@ dnet:languages @=@ elx @=@ Elamite +dnet:languages @=@ dnet:languages @=@ eng @=@ English +dnet:languages @=@ dnet:languages @=@ cpe @=@ English-based Creoles and Pidgins +dnet:languages @=@ dnet:languages @=@ esk @=@ Eskimo +dnet:languages @=@ dnet:languages @=@ epo @=@ Esperanto +dnet:languages @=@ dnet:languages @=@ est @=@ Estonian +dnet:languages @=@ dnet:languages @=@ ewe @=@ Ewe +dnet:languages @=@ dnet:languages @=@ ewo @=@ Ewondo +dnet:languages @=@ dnet:languages @=@ fan @=@ Fang +dnet:languages @=@ dnet:languages @=@ fat @=@ Fanti +dnet:languages @=@ dnet:languages @=@ fao @=@ Faroese +dnet:languages @=@ dnet:languages @=@ fij @=@ Fijian +dnet:languages @=@ dnet:languages @=@ fin @=@ Finnish +dnet:languages @=@ dnet:languages @=@ fiu @=@ Finno-Ugrian +dnet:languages @=@ dnet:languages @=@ fon @=@ Fon +dnet:languages @=@ dnet:languages @=@ fra/fre @=@ French +dnet:languages @=@ dnet:languages @=@ cpf @=@ French-based Creoles and Pidgins +dnet:languages @=@ dnet:languages @=@ fry @=@ Frisian +dnet:languages @=@ dnet:languages @=@ ful @=@ Fulah +dnet:languages @=@ dnet:languages @=@ gaa @=@ Ga +dnet:languages @=@ dnet:languages @=@ gae/gdh @=@ Gaelic +dnet:languages @=@ dnet:languages @=@ gla @=@ Gaelic; Scottish Gaelic +dnet:languages @=@ dnet:languages @=@ glg @=@ Galician +dnet:languages @=@ dnet:languages @=@ lug @=@ Ganda +dnet:languages @=@ dnet:languages @=@ gay @=@ Gayo +dnet:languages @=@ dnet:languages @=@ gez @=@ Geez +dnet:languages @=@ dnet:languages @=@ geo/kat @=@ Georgian +dnet:languages @=@ dnet:languages @=@ deu/ger @=@ German +dnet:languages @=@ dnet:languages @=@ gem @=@ Germanic +dnet:languages @=@ dnet:languages @=@ kik @=@ Gikuyu; Kikuyu +dnet:languages @=@ dnet:languages @=@ gil @=@ Gilbertese +dnet:languages @=@ dnet:languages @=@ gon @=@ Gondi +dnet:languages @=@ dnet:languages @=@ got @=@ Gothic +dnet:languages @=@ dnet:languages @=@ grb @=@ Grebo +dnet:languages @=@ dnet:languages @=@ ell/gre @=@ Greek +dnet:languages @=@ dnet:languages @=@ gre/ell @=@ Greek, Modern (1453-) +dnet:languages @=@ dnet:languages @=@ kal @=@ Greenlandic; Kalaallisut +dnet:languages @=@ dnet:languages @=@ grn @=@ Guarani +dnet:languages @=@ dnet:languages @=@ guj @=@ Gujarati +dnet:languages @=@ dnet:languages @=@ hai @=@ Haida +dnet:languages @=@ dnet:languages @=@ hat @=@ Haitian; Haitian Creole +dnet:languages @=@ dnet:languages @=@ hau @=@ Hausa +dnet:languages @=@ dnet:languages @=@ haw @=@ Hawaiian +dnet:languages @=@ dnet:languages @=@ heb @=@ Hebrew +dnet:languages @=@ dnet:languages @=@ her @=@ Herero +dnet:languages @=@ dnet:languages @=@ hil @=@ Hiligaynon +dnet:languages @=@ dnet:languages @=@ him @=@ Himachali +dnet:languages @=@ dnet:languages @=@ hin @=@ Hindi +dnet:languages @=@ dnet:languages @=@ hmo @=@ Hiri Motu +dnet:languages @=@ dnet:languages @=@ hun @=@ Hungarian +dnet:languages @=@ dnet:languages @=@ hup @=@ Hupa +dnet:languages @=@ dnet:languages @=@ iba @=@ Iban +dnet:languages @=@ dnet:languages @=@ ice/isl @=@ Icelandic +dnet:languages @=@ dnet:languages @=@ ido @=@ Ido +dnet:languages @=@ dnet:languages @=@ ibo @=@ Igbo +dnet:languages @=@ dnet:languages @=@ ijo @=@ Ijo +dnet:languages @=@ dnet:languages @=@ ilo @=@ Iloko +dnet:languages @=@ dnet:languages @=@ inc @=@ Indic +dnet:languages @=@ dnet:languages @=@ ine @=@ Indo-European +dnet:languages @=@ dnet:languages @=@ ind @=@ Indonesian +dnet:languages @=@ dnet:languages @=@ ile @=@ Interlingue +dnet:languages @=@ dnet:languages @=@ iku @=@ Inuktitut +dnet:languages @=@ dnet:languages @=@ ipk @=@ Inupiaq +dnet:languages @=@ dnet:languages @=@ ira @=@ Iranian +dnet:languages @=@ dnet:languages @=@ gai/iri @=@ Irish +dnet:languages @=@ dnet:languages @=@ iro @=@ Iroquoian +dnet:languages @=@ dnet:languages @=@ ita @=@ Italian +dnet:languages @=@ dnet:languages @=@ jpn @=@ Japanese +dnet:languages @=@ dnet:languages @=@ jav @=@ Javanese +dnet:languages @=@ dnet:languages @=@ jrb @=@ Judeo-Arabic +dnet:languages @=@ dnet:languages @=@ jpr @=@ Judeo-Persian +dnet:languages @=@ dnet:languages @=@ kab @=@ Kabyle +dnet:languages @=@ dnet:languages @=@ kac @=@ Kachin +dnet:languages @=@ dnet:languages @=@ kam @=@ Kamba +dnet:languages @=@ dnet:languages @=@ kan @=@ Kannada +dnet:languages @=@ dnet:languages @=@ kau @=@ Kanuri +dnet:languages @=@ dnet:languages @=@ kaa @=@ Kara-Kalpak +dnet:languages @=@ dnet:languages @=@ kar @=@ Karen +dnet:languages @=@ dnet:languages @=@ kas @=@ Kashmiri +dnet:languages @=@ dnet:languages @=@ kaw @=@ Kawi +dnet:languages @=@ dnet:languages @=@ kaz @=@ Kazakh +dnet:languages @=@ dnet:languages @=@ kha @=@ Khasi +dnet:languages @=@ dnet:languages @=@ khm @=@ Khmer +dnet:languages @=@ dnet:languages @=@ khi @=@ Khoisan +dnet:languages @=@ dnet:languages @=@ kho @=@ Khotanese +dnet:languages @=@ dnet:languages @=@ kin @=@ Kinyarwanda +dnet:languages @=@ dnet:languages @=@ kir @=@ Kirghiz +dnet:languages @=@ dnet:languages @=@ kom @=@ Komi +dnet:languages @=@ dnet:languages @=@ kon @=@ Kongo +dnet:languages @=@ dnet:languages @=@ kok @=@ Konkani +dnet:languages @=@ dnet:languages @=@ kor @=@ Korean +dnet:languages @=@ dnet:languages @=@ kpe @=@ Kpelle +dnet:languages @=@ dnet:languages @=@ kro @=@ Kru +dnet:languages @=@ dnet:languages @=@ kua @=@ Kuanyama; Kwanyama +dnet:languages @=@ dnet:languages @=@ kum @=@ Kumyk +dnet:languages @=@ dnet:languages @=@ kur @=@ Kurdish +dnet:languages @=@ dnet:languages @=@ kru @=@ Kurukh +dnet:languages @=@ dnet:languages @=@ kus @=@ Kusaie +dnet:languages @=@ dnet:languages @=@ kut @=@ Kutenai +dnet:languages @=@ dnet:languages @=@ lad @=@ Ladino +dnet:languages @=@ dnet:languages @=@ lah @=@ Lahnda +dnet:languages @=@ dnet:languages @=@ lam @=@ Lamba +dnet:languages @=@ dnet:languages @=@ lao @=@ Lao +dnet:languages @=@ dnet:languages @=@ lat @=@ Latin +dnet:languages @=@ dnet:languages @=@ lav @=@ Latvian +dnet:languages @=@ dnet:languages @=@ ltz @=@ Letzeburgesch; Luxembourgish +dnet:languages @=@ dnet:languages @=@ lez @=@ Lezghian +dnet:languages @=@ dnet:languages @=@ lim @=@ Limburgan; Limburger; Limburgish +dnet:languages @=@ dnet:languages @=@ lin @=@ Lingala +dnet:languages @=@ dnet:languages @=@ lit @=@ Lithuanian +dnet:languages @=@ dnet:languages @=@ loz @=@ Lozi +dnet:languages @=@ dnet:languages @=@ lub @=@ Luba-Katanga +dnet:languages @=@ dnet:languages @=@ lui @=@ Luiseno +dnet:languages @=@ dnet:languages @=@ lun @=@ Lunda +dnet:languages @=@ dnet:languages @=@ luo @=@ Luo +dnet:languages @=@ dnet:languages @=@ mac/mak @=@ Macedonian +dnet:languages @=@ dnet:languages @=@ mad @=@ Madurese +dnet:languages @=@ dnet:languages @=@ mag @=@ Magahi +dnet:languages @=@ dnet:languages @=@ mai @=@ Maithili +dnet:languages @=@ dnet:languages @=@ mak @=@ Makasar +dnet:languages @=@ dnet:languages @=@ mlg @=@ Malagasy +dnet:languages @=@ dnet:languages @=@ may/msa @=@ Malay +dnet:languages @=@ dnet:languages @=@ mal @=@ Malayalam +dnet:languages @=@ dnet:languages @=@ mlt @=@ Maltese +dnet:languages @=@ dnet:languages @=@ man @=@ Mandingo +dnet:languages @=@ dnet:languages @=@ mni @=@ Manipuri +dnet:languages @=@ dnet:languages @=@ mno @=@ Manobo +dnet:languages @=@ dnet:languages @=@ glv @=@ Manx +dnet:languages @=@ dnet:languages @=@ mao/mri @=@ Maori +dnet:languages @=@ dnet:languages @=@ mar @=@ Marathi +dnet:languages @=@ dnet:languages @=@ chm @=@ Mari +dnet:languages @=@ dnet:languages @=@ mah @=@ Marshallese +dnet:languages @=@ dnet:languages @=@ mwr @=@ Marwari +dnet:languages @=@ dnet:languages @=@ mas @=@ Masai +dnet:languages @=@ dnet:languages @=@ myn @=@ Mayan +dnet:languages @=@ dnet:languages @=@ men @=@ Mende +dnet:languages @=@ dnet:languages @=@ mic @=@ Micmac +dnet:languages @=@ dnet:languages @=@ dum @=@ Middle Dutch +dnet:languages @=@ dnet:languages @=@ enm @=@ Middle English +dnet:languages @=@ dnet:languages @=@ frm @=@ Middle French +dnet:languages @=@ dnet:languages @=@ gmh @=@ Middle High German +dnet:languages @=@ dnet:languages @=@ mga @=@ Middle Irish +dnet:languages @=@ dnet:languages @=@ min @=@ Minangkabau +dnet:languages @=@ dnet:languages @=@ mis @=@ Miscellaneous +dnet:languages @=@ dnet:languages @=@ moh @=@ Mohawk +dnet:languages @=@ dnet:languages @=@ mol @=@ Moldavian +dnet:languages @=@ dnet:languages @=@ mkh @=@ Mon-Kmer +dnet:languages @=@ dnet:languages @=@ lol @=@ Mongo +dnet:languages @=@ dnet:languages @=@ mon @=@ Mongolian +dnet:languages @=@ dnet:languages @=@ mos @=@ Mossi +dnet:languages @=@ dnet:languages @=@ mul @=@ Multiple languages +dnet:languages @=@ dnet:languages @=@ mun @=@ Munda +dnet:languages @=@ dnet:languages @=@ nau @=@ Nauru +dnet:languages @=@ dnet:languages @=@ nav @=@ Navajo; Navaho +dnet:languages @=@ dnet:languages @=@ nde @=@ Ndebele, North +dnet:languages @=@ dnet:languages @=@ nbl @=@ Ndebele, South +dnet:languages @=@ dnet:languages @=@ ndo @=@ Ndonga +dnet:languages @=@ dnet:languages @=@ nep @=@ Nepali +dnet:languages @=@ dnet:languages @=@ new @=@ Newari +dnet:languages @=@ dnet:languages @=@ nic @=@ Niger-Kordofanian +dnet:languages @=@ dnet:languages @=@ ssa @=@ Nilo-Saharan +dnet:languages @=@ dnet:languages @=@ niu @=@ Niuean +dnet:languages @=@ dnet:languages @=@ non @=@ Norse +dnet:languages @=@ dnet:languages @=@ nai @=@ North American Indian +dnet:languages @=@ dnet:languages @=@ sme @=@ Northern Sami +dnet:languages @=@ dnet:languages @=@ nor @=@ Norwegian +dnet:languages @=@ dnet:languages @=@ nno @=@ Norwegian Nynorsk; Nynorsk, Norwegian +dnet:languages @=@ dnet:languages @=@ nub @=@ Nubian +dnet:languages @=@ dnet:languages @=@ nym @=@ Nyamwezi +dnet:languages @=@ dnet:languages @=@ nyn @=@ Nyankole +dnet:languages @=@ dnet:languages @=@ nyo @=@ Nyoro +dnet:languages @=@ dnet:languages @=@ nzi @=@ Nzima +dnet:languages @=@ dnet:languages @=@ oci @=@ Occitan (post 1500); Provençal +dnet:languages @=@ dnet:languages @=@ oji @=@ Ojibwa +dnet:languages @=@ dnet:languages @=@ ang @=@ Old English +dnet:languages @=@ dnet:languages @=@ fro @=@ Old French +dnet:languages @=@ dnet:languages @=@ goh @=@ Old High German +dnet:languages @=@ dnet:languages @=@ ori @=@ Oriya +dnet:languages @=@ dnet:languages @=@ orm @=@ Oromo +dnet:languages @=@ dnet:languages @=@ osa @=@ Osage +dnet:languages @=@ dnet:languages @=@ oss @=@ Ossetian; Ossetic +dnet:languages @=@ dnet:languages @=@ oto @=@ Otomian +dnet:languages @=@ dnet:languages @=@ ota @=@ Ottoman +dnet:languages @=@ dnet:languages @=@ pal @=@ Pahlavi +dnet:languages @=@ dnet:languages @=@ pau @=@ Palauan +dnet:languages @=@ dnet:languages @=@ pli @=@ Pali +dnet:languages @=@ dnet:languages @=@ pam @=@ Pampanga +dnet:languages @=@ dnet:languages @=@ pag @=@ Pangasinan +dnet:languages @=@ dnet:languages @=@ pan @=@ Panjabi; Punjabi +dnet:languages @=@ dnet:languages @=@ pap @=@ Papiamento +dnet:languages @=@ dnet:languages @=@ paa @=@ Papuan-Australian +dnet:languages @=@ dnet:languages @=@ fas/per @=@ Persian +dnet:languages @=@ dnet:languages @=@ peo @=@ Persian, Old (ca 600 - 400 B.C.) +dnet:languages @=@ dnet:languages @=@ phn @=@ Phoenician +dnet:languages @=@ dnet:languages @=@ pol @=@ Polish +dnet:languages @=@ dnet:languages @=@ pon @=@ Ponape +dnet:languages @=@ dnet:languages @=@ por @=@ Portuguese +dnet:languages @=@ dnet:languages @=@ cpp @=@ Portuguese-based Creoles and Pidgins +dnet:languages @=@ dnet:languages @=@ pra @=@ Prakrit +dnet:languages @=@ dnet:languages @=@ pro @=@ Provencal +dnet:languages @=@ dnet:languages @=@ pus @=@ Pushto +dnet:languages @=@ dnet:languages @=@ que @=@ Quechua +dnet:languages @=@ dnet:languages @=@ roh @=@ Raeto-Romance +dnet:languages @=@ dnet:languages @=@ raj @=@ Rajasthani +dnet:languages @=@ dnet:languages @=@ rar @=@ Rarotongan +dnet:languages @=@ dnet:languages @=@ roa @=@ Romance +dnet:languages @=@ dnet:languages @=@ ron/rum @=@ Romanian +dnet:languages @=@ dnet:languages @=@ rom @=@ Romany +dnet:languages @=@ dnet:languages @=@ run @=@ Rundi +dnet:languages @=@ dnet:languages @=@ rus @=@ Russian +dnet:languages @=@ dnet:languages @=@ sal @=@ Salishan +dnet:languages @=@ dnet:languages @=@ sam @=@ Samaritan +dnet:languages @=@ dnet:languages @=@ smi @=@ Sami +dnet:languages @=@ dnet:languages @=@ smo @=@ Samoan +dnet:languages @=@ dnet:languages @=@ sad @=@ Sandawe +dnet:languages @=@ dnet:languages @=@ sag @=@ Sango +dnet:languages @=@ dnet:languages @=@ san @=@ Sanskrit +dnet:languages @=@ dnet:languages @=@ srd @=@ Sardinian +dnet:languages @=@ dnet:languages @=@ sco @=@ Scots +dnet:languages @=@ dnet:languages @=@ sel @=@ Selkup +dnet:languages @=@ dnet:languages @=@ sem @=@ Semitic +dnet:languages @=@ dnet:languages @=@ srp @=@ Serbian +dnet:languages @=@ dnet:languages @=@ scr @=@ Serbo-Croatian +dnet:languages @=@ dnet:languages @=@ srr @=@ Serer +dnet:languages @=@ dnet:languages @=@ shn @=@ Shan +dnet:languages @=@ dnet:languages @=@ sna @=@ Shona +dnet:languages @=@ dnet:languages @=@ iii @=@ Sichuan Yi +dnet:languages @=@ dnet:languages @=@ sid @=@ Sidamo +dnet:languages @=@ dnet:languages @=@ bla @=@ Siksika +dnet:languages @=@ dnet:languages @=@ snd @=@ Sindhi +dnet:languages @=@ dnet:languages @=@ sin @=@ Sinhala; Sinhalese +dnet:languages @=@ dnet:languages @=@ sit @=@ Sino-Tibetan +dnet:languages @=@ dnet:languages @=@ sio @=@ Siouan +dnet:languages @=@ dnet:languages @=@ sla @=@ Slavic +dnet:languages @=@ dnet:languages @=@ slk/slo @=@ Slovak +dnet:languages @=@ dnet:languages @=@ slv @=@ Slovenian +dnet:languages @=@ dnet:languages @=@ sog @=@ Sogdian +dnet:languages @=@ dnet:languages @=@ som @=@ Somali +dnet:languages @=@ dnet:languages @=@ son @=@ Songhai +dnet:languages @=@ dnet:languages @=@ wen @=@ Sorbian +dnet:languages @=@ dnet:languages @=@ nso @=@ Sotho +dnet:languages @=@ dnet:languages @=@ sot @=@ Sotho, Southern +dnet:languages @=@ dnet:languages @=@ sai @=@ South American Indian +dnet:languages @=@ dnet:languages @=@ esl/spa @=@ Spanish +dnet:languages @=@ dnet:languages @=@ spa @=@ Spanish; Castilian +dnet:languages @=@ dnet:languages @=@ suk @=@ Sukuma +dnet:languages @=@ dnet:languages @=@ sux @=@ Sumerian +dnet:languages @=@ dnet:languages @=@ sun @=@ Sundanese +dnet:languages @=@ dnet:languages @=@ sus @=@ Susu +dnet:languages @=@ dnet:languages @=@ swa @=@ Swahili +dnet:languages @=@ dnet:languages @=@ ssw @=@ Swati +dnet:languages @=@ dnet:languages @=@ swe @=@ Swedish +dnet:languages @=@ dnet:languages @=@ syr @=@ Syriac +dnet:languages @=@ dnet:languages @=@ tgl @=@ Tagalog +dnet:languages @=@ dnet:languages @=@ tah @=@ Tahitian +dnet:languages @=@ dnet:languages @=@ tgk @=@ Tajik +dnet:languages @=@ dnet:languages @=@ tmh @=@ Tamashek +dnet:languages @=@ dnet:languages @=@ tam @=@ Tamil +dnet:languages @=@ dnet:languages @=@ tat @=@ Tatar +dnet:languages @=@ dnet:languages @=@ tel @=@ Telugu +dnet:languages @=@ dnet:languages @=@ ter @=@ Tereno +dnet:languages @=@ dnet:languages @=@ tha @=@ Thai +dnet:languages @=@ dnet:languages @=@ bod/tib @=@ Tibetan +dnet:languages @=@ dnet:languages @=@ tig @=@ Tigre +dnet:languages @=@ dnet:languages @=@ tir @=@ Tigrinya +dnet:languages @=@ dnet:languages @=@ tem @=@ Timne +dnet:languages @=@ dnet:languages @=@ tiv @=@ Tivi +dnet:languages @=@ dnet:languages @=@ tli @=@ Tlingit +dnet:languages @=@ dnet:languages @=@ ton @=@ Tonga (Tonga Islands) +dnet:languages @=@ dnet:languages @=@ tog @=@ Tonga(Nyasa) +dnet:languages @=@ dnet:languages @=@ tru @=@ Truk +dnet:languages @=@ dnet:languages @=@ tsi @=@ Tsimshian +dnet:languages @=@ dnet:languages @=@ tso @=@ Tsonga +dnet:languages @=@ dnet:languages @=@ tsn @=@ Tswana +dnet:languages @=@ dnet:languages @=@ tum @=@ Tumbuka +dnet:languages @=@ dnet:languages @=@ tur @=@ Turkish +dnet:languages @=@ dnet:languages @=@ tuk @=@ Turkmen +dnet:languages @=@ dnet:languages @=@ tyv @=@ Tuvinian +dnet:languages @=@ dnet:languages @=@ twi @=@ Twi +dnet:languages @=@ dnet:languages @=@ uga @=@ Ugaritic +dnet:languages @=@ dnet:languages @=@ uig @=@ Uighur; Uyghur +dnet:languages @=@ dnet:languages @=@ ukr @=@ Ukrainian +dnet:languages @=@ dnet:languages @=@ umb @=@ Umbundu +dnet:languages @=@ dnet:languages @=@ und @=@ Undetermined +dnet:languages @=@ dnet:languages @=@ urd @=@ Urdu +dnet:languages @=@ dnet:languages @=@ uzb @=@ Uzbek +dnet:languages @=@ dnet:languages @=@ vai @=@ Vai +dnet:languages @=@ dnet:languages @=@ ven @=@ Venda +dnet:languages @=@ dnet:languages @=@ vie @=@ Vietnamese +dnet:languages @=@ dnet:languages @=@ vol @=@ Volapük +dnet:languages @=@ dnet:languages @=@ vot @=@ Votic +dnet:languages @=@ dnet:languages @=@ wak @=@ Wakashan +dnet:languages @=@ dnet:languages @=@ wal @=@ Walamo +dnet:languages @=@ dnet:languages @=@ wln @=@ Walloon +dnet:languages @=@ dnet:languages @=@ war @=@ Waray +dnet:languages @=@ dnet:languages @=@ was @=@ Washo +dnet:languages @=@ dnet:languages @=@ cym/wel @=@ Welsh +dnet:languages @=@ dnet:languages @=@ wol @=@ Wolof +dnet:languages @=@ dnet:languages @=@ xho @=@ Xhosa +dnet:languages @=@ dnet:languages @=@ sah @=@ Yakut +dnet:languages @=@ dnet:languages @=@ yao @=@ Yao +dnet:languages @=@ dnet:languages @=@ yap @=@ Yap +dnet:languages @=@ dnet:languages @=@ yid @=@ Yiddish +dnet:languages @=@ dnet:languages @=@ yor @=@ Yoruba +dnet:languages @=@ dnet:languages @=@ zap @=@ Zapotec +dnet:languages @=@ dnet:languages @=@ zen @=@ Zenaga +dnet:languages @=@ dnet:languages @=@ zha @=@ Zhuang; Chuang +dnet:languages @=@ dnet:languages @=@ zul @=@ Zulu +dnet:languages @=@ dnet:languages @=@ zun @=@ Zuni +dnet:languages @=@ dnet:languages @=@ sga @=@ old Irish +nsf:contractTypes @=@ NSF Contract Types @=@ BOA/Task Order @=@ BOA/Task Order +nsf:contractTypes @=@ NSF Contract Types @=@ Continuing grant @=@ Continuing grant +nsf:contractTypes @=@ NSF Contract Types @=@ Contract @=@ Contract +nsf:contractTypes @=@ NSF Contract Types @=@ Contract Interagency Agreement @=@ Contract Interagency Agreement +nsf:contractTypes @=@ NSF Contract Types @=@ Cooperative Agreement @=@ Cooperative Agreement +nsf:contractTypes @=@ NSF Contract Types @=@ Fellowship @=@ Fellowship +nsf:contractTypes @=@ NSF Contract Types @=@ Fixed Price Award @=@ Fixed Price Award +nsf:contractTypes @=@ NSF Contract Types @=@ GAA @=@ GAA +nsf:contractTypes @=@ NSF Contract Types @=@ Interagency Agreement @=@ Interagency Agreement +nsf:contractTypes @=@ NSF Contract Types @=@ Intergovernmental Personnel Award @=@ Intergovernmental Personnel Award +nsf:contractTypes @=@ NSF Contract Types @=@ Personnel Agreement @=@ Personnel Agreement +nsf:contractTypes @=@ NSF Contract Types @=@ Standard Grant @=@ Standard Grant +ec:funding_relations @=@ ec:funding_relations @=@ ec:hasframeworkprogram @=@ hasframeworkprogram +ec:funding_relations @=@ ec:funding_relations @=@ ec:hasprogram @=@ hasprogram +ec:funding_relations @=@ ec:funding_relations @=@ ec:hasspecificprogram @=@ hasspecificprogram +dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ UNKNOWN @=@ UNKNOWN +dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ collection @=@ collection +dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ dataset @=@ dataset +dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ event @=@ event +dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ film @=@ film +dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ image @=@ image +dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ interactiveResource @=@ interactiveResource +dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ model @=@ model +dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ physicalObject @=@ physicalObject +dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ service @=@ service +dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ software @=@ software +dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ sound @=@ sound +dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ text @=@ text +dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ clinicalTrial @=@ Clinical trial +dnet:dataCite_title @=@ dnet:dataCite_title @=@ alternative title @=@ alternative title +dnet:dataCite_title @=@ dnet:dataCite_title @=@ main title @=@ main title +dnet:dataCite_title @=@ dnet:dataCite_title @=@ subtitle @=@ subtitle +dnet:dataCite_title @=@ dnet:dataCite_title @=@ translated title @=@ translated title +datacite:relation_typologies @=@ datacite:relation_typologies @=@ IsCitedBy @=@ IsCitedBy +datacite:relation_typologies @=@ datacite:relation_typologies @=@ IsNewVersionOf @=@ IsNewVersionOf +datacite:relation_typologies @=@ datacite:relation_typologies @=@ IsPartOf @=@ IsPartOf +datacite:relation_typologies @=@ datacite:relation_typologies @=@ IsPreviousVersionOf @=@ IsPreviousVersionOf +datacite:relation_typologies @=@ datacite:relation_typologies @=@ IsReferencedBy @=@ IsReferencedBy +datacite:relation_typologies @=@ datacite:relation_typologies @=@ References @=@ References +datacite:relation_typologies @=@ datacite:relation_typologies @=@ UNKNOWN @=@ UNKNOWN +dnet:result_typologies @=@ dnet:result_typologies @=@ dataset @=@ dataset +dnet:result_typologies @=@ dnet:result_typologies @=@ other @=@ other +dnet:result_typologies @=@ dnet:result_typologies @=@ publication @=@ publication +dnet:result_typologies @=@ dnet:result_typologies @=@ software @=@ software +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ ERC-ADG @=@ Advanced Grant +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ BBI-CSA @=@ Bio-based Industries Coordination and Support action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ BBI-IA-DEMO @=@ Bio-based Industries Innovation action - Demonstration +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ BBI-IA-FLAG @=@ Bio-based Industries Innovation action - Flagship +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ BBI-RIA @=@ Bio-based Industries Research and Innovation action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ MSCA-IF-EF-CAR @=@ CAR – Career Restart panel +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ COFUND-EJP @=@ COFUND (European Joint Programme) +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ COFUND-PCP @=@ COFUND (PCP) +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ COFUND-PPI @=@ COFUND (PPI) +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ CS2-CSA @=@ CS2 Coordination and Support action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ CS2-IA @=@ CS2 Innovation Action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ CS2-RIA @=@ CS2 Research and Innovation action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ CSA-LS @=@ CSA Lump sum +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ ERC-COG @=@ Consolidator Grant +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ FCH2-CSA @=@ Coordination & support action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ CSA @=@ Coordination and support action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ MSCA-COFUND-DP @=@ Doctoral programmes +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ ECSEL-CSA @=@ ECSEL Coordination & Support action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ ECSEL-IA @=@ ECSEL Innovation Action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ ECSEL-RIA @=@ ECSEL Research and Innovation Actions +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ ERA-NET-Cofund @=@ ERA-NET Cofund +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ ERC-POC-LS @=@ ERC Proof of Concept Lump Sum Pilot +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ ERC-SyG @=@ ERC Synergy Grant +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ ERC-LVG @=@ ERC low value grant +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ H2020-EEN-SGA @=@ Enterprise Europe Network - Specific Grant Agreement +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ MSCA-ITN-EID @=@ European Industrial Doctorates +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ MSCA-ITN-EJD @=@ European Joint Doctorates +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ MSCA-ITN-ETN @=@ European Training Networks +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ FCH2-IA @=@ FCH2 Innovation action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ FCH2-RIA @=@ FCH2 Research and Innovation action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ MSCA-COFUND-FP @=@ Fellowship programmes +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ MSCA-IF-GF @=@ Global Fellowships +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ IMI2-CSA @=@ IMI2 Coordination & support action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ IMI2-RIA @=@ IMI2 Research and Innovation action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ Shift2Rail-IA-LS @=@ Innovation Action Lump-Sum +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ IA-LS @=@ Innovation Action Lump-Sum +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ IA @=@ Innovation action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ Shift2Rail-IA @=@ Innovation action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ PCP @=@ Pre-Commercial Procurement +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ ERC-POC @=@ Proof of Concept Grant +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ PPI @=@ Public Procurement of Innovative Solutions +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ MSCA-IF-EF-RI @=@ RI – Reintegration panel +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ MSCA-RISE @=@ RISE +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ Shift2Rail-RIA-LS @=@ Research and Innovation Action Lump-Sum +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ Shift2Rail-RIA @=@ Research and Innovation action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ RIA @=@ Research and Innovation action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ RIA-LS @=@ Research and Innovation action Lump Sum +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ SESAR-CSA @=@ SESAR: Coordination and Support Action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ SESAR-IA @=@ SESAR: Innovation action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ SESAR-RIA @=@ SESAR: Research and Innovation action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ SGA-RIA @=@ SGA Research and Innovation action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ SME-2b @=@ SME Instrument (grant only and blended finance) +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ SME-1 @=@ SME instrument phase 1 +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ SME-2 @=@ SME instrument phase 2 +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ Shift2Rail-CSA @=@ Shift2Rail - Coordination and Support action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ MSCA-IF-EF-SE @=@ Society and Enterprise panel +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ SGA-CSA @=@ Specific Grant agreement and Coordination and Support Action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ MSCA-IF-EF-ST @=@ Standard EF +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ ERC-STG @=@ Starting Grant +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ MSCA-SNLS @=@ Grant to identified beneficiary - Coordination and support actions (MSCA-Special Needs lump sum) +wt:contractTypes @=@ wt:contractTypes @=@ UNKNOWN @=@ UNKNOWN +dnet:countries @=@ dnet:countries @=@ AF @=@ Afghanistan +dnet:countries @=@ dnet:countries @=@ AL @=@ Albania +dnet:countries @=@ dnet:countries @=@ DZ @=@ Algeria +dnet:countries @=@ dnet:countries @=@ AS @=@ American Samoa +dnet:countries @=@ dnet:countries @=@ AD @=@ Andorra +dnet:countries @=@ dnet:countries @=@ AO @=@ Angola +dnet:countries @=@ dnet:countries @=@ AI @=@ Anguilla +dnet:countries @=@ dnet:countries @=@ AQ @=@ Antarctica +dnet:countries @=@ dnet:countries @=@ AG @=@ Antigua and Barbuda +dnet:countries @=@ dnet:countries @=@ AR @=@ Argentina +dnet:countries @=@ dnet:countries @=@ AM @=@ Armenia +dnet:countries @=@ dnet:countries @=@ AW @=@ Aruba +dnet:countries @=@ dnet:countries @=@ AU @=@ Australia +dnet:countries @=@ dnet:countries @=@ AT @=@ Austria +dnet:countries @=@ dnet:countries @=@ AZ @=@ Azerbaijan +dnet:countries @=@ dnet:countries @=@ BS @=@ Bahamas +dnet:countries @=@ dnet:countries @=@ BH @=@ Bahrain +dnet:countries @=@ dnet:countries @=@ BD @=@ Bangladesh +dnet:countries @=@ dnet:countries @=@ BB @=@ Barbados +dnet:countries @=@ dnet:countries @=@ BY @=@ Belarus +dnet:countries @=@ dnet:countries @=@ BE @=@ Belgium +dnet:countries @=@ dnet:countries @=@ BZ @=@ Belize +dnet:countries @=@ dnet:countries @=@ BJ @=@ Benin +dnet:countries @=@ dnet:countries @=@ BM @=@ Bermuda +dnet:countries @=@ dnet:countries @=@ BT @=@ Bhutan +dnet:countries @=@ dnet:countries @=@ BO @=@ Bolivia +dnet:countries @=@ dnet:countries @=@ BQ @=@ Bonaire, Sint Eustatius and Saba +dnet:countries @=@ dnet:countries @=@ BA @=@ Bosnia and Herzegovina +dnet:countries @=@ dnet:countries @=@ BW @=@ Botswana +dnet:countries @=@ dnet:countries @=@ BV @=@ Bouvet Island +dnet:countries @=@ dnet:countries @=@ BR @=@ Brazil +dnet:countries @=@ dnet:countries @=@ IO @=@ British Indian Ocean Territory +dnet:countries @=@ dnet:countries @=@ BN @=@ Brunei Darussalam +dnet:countries @=@ dnet:countries @=@ BG @=@ Bulgaria +dnet:countries @=@ dnet:countries @=@ BF @=@ Burkina Faso +dnet:countries @=@ dnet:countries @=@ BI @=@ Burundi +dnet:countries @=@ dnet:countries @=@ KH @=@ Cambodia +dnet:countries @=@ dnet:countries @=@ CM @=@ Cameroon +dnet:countries @=@ dnet:countries @=@ CA @=@ Canada +dnet:countries @=@ dnet:countries @=@ CV @=@ Cape Verde +dnet:countries @=@ dnet:countries @=@ KY @=@ Cayman Islands +dnet:countries @=@ dnet:countries @=@ CF @=@ Central African Republic +dnet:countries @=@ dnet:countries @=@ TD @=@ Chad +dnet:countries @=@ dnet:countries @=@ CL @=@ Chile +dnet:countries @=@ dnet:countries @=@ CN @=@ China (People's Republic of) +dnet:countries @=@ dnet:countries @=@ CX @=@ Christmas Island +dnet:countries @=@ dnet:countries @=@ CC @=@ Cocos (Keeling) Islands +dnet:countries @=@ dnet:countries @=@ CO @=@ Colombia +dnet:countries @=@ dnet:countries @=@ KM @=@ Comoros +dnet:countries @=@ dnet:countries @=@ CG @=@ Congo +dnet:countries @=@ dnet:countries @=@ CD @=@ Congo (Democratic Republic of) +dnet:countries @=@ dnet:countries @=@ CK @=@ Cook Islands +dnet:countries @=@ dnet:countries @=@ CR @=@ Costa Rica +dnet:countries @=@ dnet:countries @=@ CI @=@ Cote d'Ivoire +dnet:countries @=@ dnet:countries @=@ HR @=@ Croatia +dnet:countries @=@ dnet:countries @=@ CU @=@ Cuba +dnet:countries @=@ dnet:countries @=@ CW @=@ Curaçao +dnet:countries @=@ dnet:countries @=@ CY @=@ Cyprus +dnet:countries @=@ dnet:countries @=@ CZ @=@ Czech Republic +dnet:countries @=@ dnet:countries @=@ DK @=@ Denmark +dnet:countries @=@ dnet:countries @=@ DJ @=@ Djibouti +dnet:countries @=@ dnet:countries @=@ DM @=@ Dominica +dnet:countries @=@ dnet:countries @=@ DO @=@ Dominican Republic +dnet:countries @=@ dnet:countries @=@ EC @=@ Ecuador +dnet:countries @=@ dnet:countries @=@ EG @=@ Egypt +dnet:countries @=@ dnet:countries @=@ SV @=@ El Salvador +dnet:countries @=@ dnet:countries @=@ GQ @=@ Equatorial Guinea +dnet:countries @=@ dnet:countries @=@ ER @=@ Eritrea +dnet:countries @=@ dnet:countries @=@ EE @=@ Estonia +dnet:countries @=@ dnet:countries @=@ ET @=@ Ethiopia +dnet:countries @=@ dnet:countries @=@ EU @=@ European Union +dnet:countries @=@ dnet:countries @=@ FK @=@ Falkland Islands (Malvinas) +dnet:countries @=@ dnet:countries @=@ FO @=@ Faroe Islands +dnet:countries @=@ dnet:countries @=@ FJ @=@ Fiji +dnet:countries @=@ dnet:countries @=@ FI @=@ Finland +dnet:countries @=@ dnet:countries @=@ MK @=@ Former Yugoslav Republic of Macedonia +dnet:countries @=@ dnet:countries @=@ FR @=@ France +dnet:countries @=@ dnet:countries @=@ GF @=@ French Guiana +dnet:countries @=@ dnet:countries @=@ PF @=@ French Polynesia +dnet:countries @=@ dnet:countries @=@ TF @=@ French Southern Territories +dnet:countries @=@ dnet:countries @=@ GA @=@ Gabon +dnet:countries @=@ dnet:countries @=@ GM @=@ Gambia +dnet:countries @=@ dnet:countries @=@ GE @=@ Georgia +dnet:countries @=@ dnet:countries @=@ DE @=@ Germany +dnet:countries @=@ dnet:countries @=@ GH @=@ Ghana +dnet:countries @=@ dnet:countries @=@ GI @=@ Gibraltar +dnet:countries @=@ dnet:countries @=@ GR @=@ Greece +dnet:countries @=@ dnet:countries @=@ GL @=@ Greenland +dnet:countries @=@ dnet:countries @=@ GD @=@ Grenada +dnet:countries @=@ dnet:countries @=@ GP @=@ Guadeloupe +dnet:countries @=@ dnet:countries @=@ GU @=@ Guam +dnet:countries @=@ dnet:countries @=@ GT @=@ Guatemala +dnet:countries @=@ dnet:countries @=@ GG @=@ Guernsey +dnet:countries @=@ dnet:countries @=@ GN @=@ Guinea +dnet:countries @=@ dnet:countries @=@ GW @=@ Guinea-Bissau +dnet:countries @=@ dnet:countries @=@ GY @=@ Guyana +dnet:countries @=@ dnet:countries @=@ HT @=@ Haiti +dnet:countries @=@ dnet:countries @=@ HM @=@ Heard Island and McDonald Islands +dnet:countries @=@ dnet:countries @=@ VA @=@ Holy See (Vatican City State) +dnet:countries @=@ dnet:countries @=@ HN @=@ Honduras +dnet:countries @=@ dnet:countries @=@ HK @=@ Hong Kong +dnet:countries @=@ dnet:countries @=@ HU @=@ Hungary +dnet:countries @=@ dnet:countries @=@ IS @=@ Iceland +dnet:countries @=@ dnet:countries @=@ IN @=@ India +dnet:countries @=@ dnet:countries @=@ ID @=@ Indonesia +dnet:countries @=@ dnet:countries @=@ IR @=@ Iran (Islamic Republic of) +dnet:countries @=@ dnet:countries @=@ IQ @=@ Iraq +dnet:countries @=@ dnet:countries @=@ IE @=@ Ireland +dnet:countries @=@ dnet:countries @=@ IM @=@ Isle of Man +dnet:countries @=@ dnet:countries @=@ IL @=@ Israel +dnet:countries @=@ dnet:countries @=@ IT @=@ Italy +dnet:countries @=@ dnet:countries @=@ JM @=@ Jamaica +dnet:countries @=@ dnet:countries @=@ JP @=@ Japan +dnet:countries @=@ dnet:countries @=@ JE @=@ Jersey +dnet:countries @=@ dnet:countries @=@ JO @=@ Jordan +dnet:countries @=@ dnet:countries @=@ KZ @=@ Kazakhstan +dnet:countries @=@ dnet:countries @=@ KE @=@ Kenya +dnet:countries @=@ dnet:countries @=@ KI @=@ Kiribati +dnet:countries @=@ dnet:countries @=@ KR @=@ Korea (Republic of) +dnet:countries @=@ dnet:countries @=@ KP @=@ Korea, Democatric People's Republic of +dnet:countries @=@ dnet:countries @=@ XK @=@ Kosovo * UN resolution +dnet:countries @=@ dnet:countries @=@ KW @=@ Kuwait +dnet:countries @=@ dnet:countries @=@ KG @=@ Kyrgyzstan +dnet:countries @=@ dnet:countries @=@ LA @=@ Lao (People's Democratic Republic) +dnet:countries @=@ dnet:countries @=@ LV @=@ Latvia +dnet:countries @=@ dnet:countries @=@ LB @=@ Lebanon +dnet:countries @=@ dnet:countries @=@ LS @=@ Lesotho +dnet:countries @=@ dnet:countries @=@ LR @=@ Liberia +dnet:countries @=@ dnet:countries @=@ LY @=@ Libyan Arab Jamahiriya +dnet:countries @=@ dnet:countries @=@ LI @=@ Liechtenstein +dnet:countries @=@ dnet:countries @=@ LT @=@ Lithuania +dnet:countries @=@ dnet:countries @=@ LU @=@ Luxembourg +dnet:countries @=@ dnet:countries @=@ MO @=@ Macao +dnet:countries @=@ dnet:countries @=@ MG @=@ Madagascar +dnet:countries @=@ dnet:countries @=@ MW @=@ Malawi +dnet:countries @=@ dnet:countries @=@ MY @=@ Malaysia +dnet:countries @=@ dnet:countries @=@ MV @=@ Maldives +dnet:countries @=@ dnet:countries @=@ ML @=@ Mali +dnet:countries @=@ dnet:countries @=@ MT @=@ Malta +dnet:countries @=@ dnet:countries @=@ MH @=@ Marshall Islands +dnet:countries @=@ dnet:countries @=@ MQ @=@ Martinique +dnet:countries @=@ dnet:countries @=@ MR @=@ Mauritania +dnet:countries @=@ dnet:countries @=@ MU @=@ Mauritius +dnet:countries @=@ dnet:countries @=@ YT @=@ Mayotte +dnet:countries @=@ dnet:countries @=@ MX @=@ Mexico +dnet:countries @=@ dnet:countries @=@ FM @=@ Micronesia, Federated States of +dnet:countries @=@ dnet:countries @=@ MD @=@ Moldova (Republic of) +dnet:countries @=@ dnet:countries @=@ MN @=@ Mongolia +dnet:countries @=@ dnet:countries @=@ ME @=@ Montenegro +dnet:countries @=@ dnet:countries @=@ MS @=@ Montserrat +dnet:countries @=@ dnet:countries @=@ MA @=@ Morocco +dnet:countries @=@ dnet:countries @=@ MZ @=@ Mozambique +dnet:countries @=@ dnet:countries @=@ MM @=@ Myanmar +dnet:countries @=@ dnet:countries @=@ NA @=@ Namibia +dnet:countries @=@ dnet:countries @=@ NR @=@ Nauru +dnet:countries @=@ dnet:countries @=@ NP @=@ Nepal +dnet:countries @=@ dnet:countries @=@ NL @=@ Netherlands +dnet:countries @=@ dnet:countries @=@ AN @=@ Netherlands Antilles +dnet:countries @=@ dnet:countries @=@ NC @=@ New Caledonia +dnet:countries @=@ dnet:countries @=@ NZ @=@ New Zealand +dnet:countries @=@ dnet:countries @=@ NI @=@ Nicaragua +dnet:countries @=@ dnet:countries @=@ NE @=@ Niger +dnet:countries @=@ dnet:countries @=@ NG @=@ Nigeria +dnet:countries @=@ dnet:countries @=@ NU @=@ Niue +dnet:countries @=@ dnet:countries @=@ NF @=@ Norfolk Island +dnet:countries @=@ dnet:countries @=@ MP @=@ Northern Mariana Islands +dnet:countries @=@ dnet:countries @=@ NO @=@ Norway +dnet:countries @=@ dnet:countries @=@ OC @=@ Oceania +dnet:countries @=@ dnet:countries @=@ OM @=@ Oman +dnet:countries @=@ dnet:countries @=@ PK @=@ Pakistan +dnet:countries @=@ dnet:countries @=@ PW @=@ Palau +dnet:countries @=@ dnet:countries @=@ PS @=@ Palestinian-administered areas +dnet:countries @=@ dnet:countries @=@ PA @=@ Panama +dnet:countries @=@ dnet:countries @=@ PG @=@ Papua New Guinea +dnet:countries @=@ dnet:countries @=@ PY @=@ Paraguay +dnet:countries @=@ dnet:countries @=@ PE @=@ Peru +dnet:countries @=@ dnet:countries @=@ PH @=@ Philippines +dnet:countries @=@ dnet:countries @=@ PN @=@ Pitcairn +dnet:countries @=@ dnet:countries @=@ PL @=@ Poland +dnet:countries @=@ dnet:countries @=@ PT @=@ Portugal +dnet:countries @=@ dnet:countries @=@ PR @=@ Puerto Rico +dnet:countries @=@ dnet:countries @=@ QA @=@ Qatar +dnet:countries @=@ dnet:countries @=@ RO @=@ Romania +dnet:countries @=@ dnet:countries @=@ RU @=@ Russian Federation +dnet:countries @=@ dnet:countries @=@ RW @=@ Rwanda +dnet:countries @=@ dnet:countries @=@ RE @=@ Réunion +dnet:countries @=@ dnet:countries @=@ SH @=@ Saint Helena, Ascension and Tristan da Cunha +dnet:countries @=@ dnet:countries @=@ KN @=@ Saint Kitts and Nevis +dnet:countries @=@ dnet:countries @=@ LC @=@ Saint Lucia +dnet:countries @=@ dnet:countries @=@ MF @=@ Saint Martin (French Part) +dnet:countries @=@ dnet:countries @=@ PM @=@ Saint Pierre and Miquelon +dnet:countries @=@ dnet:countries @=@ VC @=@ Saint Vincent and the Grenadines +dnet:countries @=@ dnet:countries @=@ BL @=@ Saint-Barthélemy +dnet:countries @=@ dnet:countries @=@ WS @=@ Samoa +dnet:countries @=@ dnet:countries @=@ SM @=@ San Marino +dnet:countries @=@ dnet:countries @=@ SA @=@ Saudi Arabia +dnet:countries @=@ dnet:countries @=@ SN @=@ Senegal +dnet:countries @=@ dnet:countries @=@ RS @=@ Serbia +dnet:countries @=@ dnet:countries @=@ CS @=@ Serbia and Montenegro +dnet:countries @=@ dnet:countries @=@ SC @=@ Seychelles +dnet:countries @=@ dnet:countries @=@ SL @=@ Sierra Leone +dnet:countries @=@ dnet:countries @=@ SG @=@ Singapore +dnet:countries @=@ dnet:countries @=@ SX @=@ Sint Maarten (Dutch Part) +dnet:countries @=@ dnet:countries @=@ SK @=@ Slovakia +dnet:countries @=@ dnet:countries @=@ SI @=@ Slovenia +dnet:countries @=@ dnet:countries @=@ SB @=@ Solomon Islands +dnet:countries @=@ dnet:countries @=@ SO @=@ Somalia +dnet:countries @=@ dnet:countries @=@ ZA @=@ South Africa +dnet:countries @=@ dnet:countries @=@ GS @=@ South Georgia and the South Sandwich Islands +dnet:countries @=@ dnet:countries @=@ SS @=@ South Sudan +dnet:countries @=@ dnet:countries @=@ ES @=@ Spain +dnet:countries @=@ dnet:countries @=@ LK @=@ Sri Lanka +dnet:countries @=@ dnet:countries @=@ SD @=@ Sudan +dnet:countries @=@ dnet:countries @=@ SR @=@ Suriname +dnet:countries @=@ dnet:countries @=@ SJ @=@ Svalbard and Jan Mayen +dnet:countries @=@ dnet:countries @=@ SZ @=@ Swaziland +dnet:countries @=@ dnet:countries @=@ SE @=@ Sweden +dnet:countries @=@ dnet:countries @=@ CH @=@ Switzerland +dnet:countries @=@ dnet:countries @=@ SY @=@ Syrian Arab Republic +dnet:countries @=@ dnet:countries @=@ ST @=@ São Tomé and Príncipe +dnet:countries @=@ dnet:countries @=@ TW @=@ Taiwan +dnet:countries @=@ dnet:countries @=@ TJ @=@ Tajikistan +dnet:countries @=@ dnet:countries @=@ TZ @=@ Tanzania (United Republic of) +dnet:countries @=@ dnet:countries @=@ TH @=@ Thailand +dnet:countries @=@ dnet:countries @=@ TL @=@ Timor-Leste +dnet:countries @=@ dnet:countries @=@ TG @=@ Togo +dnet:countries @=@ dnet:countries @=@ TK @=@ Tokelau +dnet:countries @=@ dnet:countries @=@ TO @=@ Tonga +dnet:countries @=@ dnet:countries @=@ TT @=@ Trinidad and Tobago +dnet:countries @=@ dnet:countries @=@ TN @=@ Tunisia +dnet:countries @=@ dnet:countries @=@ TR @=@ Turkey +dnet:countries @=@ dnet:countries @=@ TM @=@ Turkmenistan +dnet:countries @=@ dnet:countries @=@ TC @=@ Turks and Caicos Islands +dnet:countries @=@ dnet:countries @=@ TV @=@ Tuvalu +dnet:countries @=@ dnet:countries @=@ UNKNOWN @=@ UNKNOWN +dnet:countries @=@ dnet:countries @=@ UG @=@ Uganda +dnet:countries @=@ dnet:countries @=@ UA @=@ Ukraine +dnet:countries @=@ dnet:countries @=@ AE @=@ United Arab Emirates +dnet:countries @=@ dnet:countries @=@ GB @=@ United Kingdom +dnet:countries @=@ dnet:countries @=@ US @=@ United States +dnet:countries @=@ dnet:countries @=@ UM @=@ United States Minor Outlying Islands +dnet:countries @=@ dnet:countries @=@ UY @=@ Uruguay +dnet:countries @=@ dnet:countries @=@ UZ @=@ Uzbekistan +dnet:countries @=@ dnet:countries @=@ VU @=@ Vanuatu +dnet:countries @=@ dnet:countries @=@ VE @=@ Venezuela +dnet:countries @=@ dnet:countries @=@ VN @=@ Viet Nam +dnet:countries @=@ dnet:countries @=@ VG @=@ Virgin Islands (British) +dnet:countries @=@ dnet:countries @=@ VI @=@ Virgin Islands, U.S. +dnet:countries @=@ dnet:countries @=@ WF @=@ Wallis and Futuna +dnet:countries @=@ dnet:countries @=@ EH @=@ Western Sahara +dnet:countries @=@ dnet:countries @=@ YE @=@ Yemen +dnet:countries @=@ dnet:countries @=@ YU @=@ Yugoslavia +dnet:countries @=@ dnet:countries @=@ ZM @=@ Zambia +dnet:countries @=@ dnet:countries @=@ ZW @=@ Zimbabwe +dnet:countries @=@ dnet:countries @=@ AX @=@ Åland Islands +dnet:datasourceCompatibilityLevel @=@ dnet:datasourceCompatibilityLevel @=@ openaire2.0 @=@ OpenAIRE 2.0 (EC funding) +dnet:datasourceCompatibilityLevel @=@ dnet:datasourceCompatibilityLevel @=@ driver-openaire2.0 @=@ OpenAIRE 2.0+ (DRIVER OA, EC funding) +dnet:datasourceCompatibilityLevel @=@ dnet:datasourceCompatibilityLevel @=@ openaire3.0 @=@ OpenAIRE 3.0 (OA, funding) +dnet:datasourceCompatibilityLevel @=@ dnet:datasourceCompatibilityLevel @=@ openaire4.0 @=@ OpenAIRE 4.0 (inst.&thematic. repo.) +dnet:datasourceCompatibilityLevel @=@ dnet:datasourceCompatibilityLevel @=@ driver @=@ OpenAIRE Basic (DRIVER OA) +dnet:datasourceCompatibilityLevel @=@ dnet:datasourceCompatibilityLevel @=@ openaire2.0_data @=@ OpenAIRE Data (funded, referenced datasets) +dnet:datasourceCompatibilityLevel @=@ dnet:datasourceCompatibilityLevel @=@ hostedBy @=@ collected from a compatible aggregator +dnet:datasourceCompatibilityLevel @=@ dnet:datasourceCompatibilityLevel @=@ UNKNOWN @=@ not available +dnet:datasourceCompatibilityLevel @=@ dnet:datasourceCompatibilityLevel @=@ native @=@ proprietary +dnet:datasourceCompatibilityLevel @=@ dnet:datasourceCompatibilityLevel @=@ notCompatible @=@ under validation +dnet:datasourceCompatibilityLevel @=@ dnet:datasourceCompatibilityLevel @=@ openaire-cris_1.1 @=@ OpenAIRE CRIS v1.1 +fct:funding_relations @=@ fct:funding_relations @=@ fct:hasParentFunding @=@ fct:hasParentFunding +dnet:protocols @=@ dnet:protocols @=@ HTTPWithFileName @=@ HTTPWithFileName +dnet:protocols @=@ dnet:protocols @=@ NetCDF @=@ NetCDF +dnet:protocols @=@ dnet:protocols @=@ OpenDAP @=@ OpenDAP +dnet:protocols @=@ dnet:protocols @=@ schemaorg @=@ Schema.org +dnet:protocols @=@ dnet:protocols @=@ UNKNOWN @=@ UNKNOWN +dnet:protocols @=@ dnet:protocols @=@ api @=@ api +dnet:protocols @=@ dnet:protocols @=@ dataciteESPlugins @=@ dataciteESPlugins +dnet:protocols @=@ dnet:protocols @=@ datasetsbyjournal @=@ datasetsbyjournal +dnet:protocols @=@ dnet:protocols @=@ datasetsbyproject @=@ datasetsbyproject +dnet:protocols @=@ dnet:protocols @=@ excelFile @=@ excelFile +dnet:protocols @=@ dnet:protocols @=@ file @=@ file +dnet:protocols @=@ dnet:protocols @=@ fileGzip @=@ fileGzip +dnet:protocols @=@ dnet:protocols @=@ files_by_rpc @=@ files_by_rpc +dnet:protocols @=@ dnet:protocols @=@ files_from_mdstore @=@ files_from_mdstore +dnet:protocols @=@ dnet:protocols @=@ files_from_metadata @=@ files_from_metadata +dnet:protocols @=@ dnet:protocols @=@ filesystem @=@ filesystem +dnet:protocols @=@ dnet:protocols @=@ ftp @=@ ftp +dnet:protocols @=@ dnet:protocols @=@ gristProjects @=@ gristProjects +dnet:protocols @=@ dnet:protocols @=@ gtr2Projects @=@ gtr2Projects +dnet:protocols @=@ dnet:protocols @=@ http @=@ http +dnet:protocols @=@ dnet:protocols @=@ httpCSV @=@ httpCSV +dnet:protocols @=@ dnet:protocols @=@ httpList @=@ httpList +dnet:protocols @=@ dnet:protocols @=@ jdbc @=@ jdbc +dnet:protocols @=@ dnet:protocols @=@ oai @=@ oai +dnet:protocols @=@ dnet:protocols @=@ oai_sets @=@ oai_sets +dnet:protocols @=@ dnet:protocols @=@ other @=@ other +dnet:protocols @=@ dnet:protocols @=@ re3data @=@ re3data +dnet:protocols @=@ dnet:protocols @=@ rest @=@ rest +dnet:protocols @=@ dnet:protocols @=@ rest_json2xml @=@ rest_json2xml +dnet:protocols @=@ dnet:protocols @=@ sftp @=@ sftp +dnet:protocols @=@ dnet:protocols @=@ soap @=@ soap +dnet:protocols @=@ dnet:protocols @=@ sparql @=@ sparql +dnet:protocols @=@ dnet:protocols @=@ sword @=@ sword +dnet:protocols @=@ dnet:protocols @=@ targz @=@ targz +dnet:protocols @=@ dnet:protocols @=@ remoteMdstore @=@ remoteMdstore +wt:funding_typologies @=@ Wellcome Trust: Funding Typologies @=@ wt:fundingStream @=@ Wellcome Trust: Funding Stream +dnet:externalReference_typologies @=@ dnet:externalReference_typologies @=@ accessionNumber @=@ accessionNumber +dnet:externalReference_typologies @=@ dnet:externalReference_typologies @=@ dataset @=@ dataset +dnet:externalReference_typologies @=@ dnet:externalReference_typologies @=@ software @=@ software +datacite:id_typologies @=@ datacite:id_typologies @=@ ARK @=@ ARK +datacite:id_typologies @=@ datacite:id_typologies @=@ DOI @=@ DOI +datacite:id_typologies @=@ datacite:id_typologies @=@ EAN13 @=@ EAN13 +datacite:id_typologies @=@ datacite:id_typologies @=@ EISSN @=@ EISSN +datacite:id_typologies @=@ datacite:id_typologies @=@ Handle @=@ Handle +datacite:id_typologies @=@ datacite:id_typologies @=@ ISBN @=@ ISBN +datacite:id_typologies @=@ datacite:id_typologies @=@ ISSN @=@ ISSN +datacite:id_typologies @=@ datacite:id_typologies @=@ ISTC @=@ ISTC +datacite:id_typologies @=@ datacite:id_typologies @=@ LISSN @=@ LISSN +datacite:id_typologies @=@ datacite:id_typologies @=@ LSID @=@ LSID +datacite:id_typologies @=@ datacite:id_typologies @=@ PURL @=@ PURL +datacite:id_typologies @=@ datacite:id_typologies @=@ UNKNOWN @=@ UNKNOWN +datacite:id_typologies @=@ datacite:id_typologies @=@ UPC @=@ UPC +datacite:id_typologies @=@ datacite:id_typologies @=@ URL @=@ URL +datacite:id_typologies @=@ datacite:id_typologies @=@ URN @=@ URN +dnet:pid_types @=@ dnet:pid_types @=@ actrn @=@ ACTRN Identifier +dnet:pid_types @=@ dnet:pid_types @=@ nct @=@ ClinicalTrials.gov Identifier +dnet:pid_types @=@ dnet:pid_types @=@ euctr @=@ EU Clinical Trials Register +dnet:pid_types @=@ dnet:pid_types @=@ epo_id @=@ European Patent Office application ID +dnet:pid_types @=@ dnet:pid_types @=@ gsk @=@ GSK Identifier +dnet:pid_types @=@ dnet:pid_types @=@ GeoPass @=@ Geographic Location-Password Scheme +dnet:pid_types @=@ dnet:pid_types @=@ GBIF @=@ Global Biodiversity Information Facility +dnet:pid_types @=@ dnet:pid_types @=@ isrctn @=@ ISRCTN Identifier +dnet:pid_types @=@ dnet:pid_types @=@ ISNI @=@ International Standard Name Identifier +dnet:pid_types @=@ dnet:pid_types @=@ jprn @=@ JPRN Identifier +dnet:pid_types @=@ dnet:pid_types @=@ mag_id @=@ Microsoft Academic Graph Identifier +dnet:pid_types @=@ dnet:pid_types @=@ oai @=@ Open Archives Initiative +dnet:pid_types @=@ dnet:pid_types @=@ orcid @=@ Open Researcher and Contributor ID +dnet:pid_types @=@ dnet:pid_types @=@ PANGAEA @=@ PANGAEA +dnet:pid_types @=@ dnet:pid_types @=@ epo_nr_epodoc @=@ Patent application number in EPODOC format +dnet:pid_types @=@ dnet:pid_types @=@ UNKNOWN @=@ UNKNOWN +dnet:pid_types @=@ dnet:pid_types @=@ VIAF @=@ Virtual International Authority File +dnet:pid_types @=@ dnet:pid_types @=@ arXiv @=@ arXiv +dnet:pid_types @=@ dnet:pid_types @=@ doi @=@ doi +dnet:pid_types @=@ dnet:pid_types @=@ grid @=@ grid +dnet:pid_types @=@ dnet:pid_types @=@ info:eu-repo/dai @=@ info:eu-repo/dai +dnet:pid_types @=@ dnet:pid_types @=@ orcidworkid @=@ orcid workid +dnet:pid_types @=@ dnet:pid_types @=@ pmc @=@ pmc +dnet:pid_types @=@ dnet:pid_types @=@ pmid @=@ pmid +dnet:pid_types @=@ dnet:pid_types @=@ urn @=@ urn +dnet:pid_types @=@ dnet:pid_types @=@ who @=@ WHO Identifier +dnet:pid_types @=@ dnet:pid_types @=@ drks @=@ DRKS Identifier +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/SUBJECT/ACM @=@ An ACM classification term that can be associated to your publications +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/SUBJECT/ARXIV @=@ An ARXIV classification term that can be associated to your publications +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/SUBJECT/DDC @=@ A Dewey Decimal classification term (DDC) that can be associated to your publications +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/SUBJECT/JEL @=@ A Journal of Economic Literature (JEL) classification term that can be associated to your publications +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/OPENACCESS_VERSION @=@ An Open Access versions of your publications +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/DATASET/IS_REFERENCED_BY @=@ A dataset referenced by your records +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/DATASET/REFERENCES @=@ A dataset that refers to your records +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/DATASET/IS_RELATED_TO @=@ A dataset related to your records +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/DATASET/IS_SUPPLEMENTED_TO @=@ A dataset that supplements your records +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/PUBLICATION/IS_RELATED_TO @=@ A publication related to your records +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/PUBLICATION/REFERENCES @=@ A publication referenced by your records +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/PUBLICATION/IS_REFERENCED_BY @=@ A publication that refers to your records +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/PUBLICATION/IS_SUPPLEMENTED_BY @=@ A publication that is supplemented by your records +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/PUBLICATION/IS_SUPPLEMENTED_TO @=@ A publication that supplements your records +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/SOFTWARE @=@ A software referred by your records +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MORE/OPENACCESS_VERSION @=@ Another Open Access version of a publication +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MORE/PID @=@ Another persistent identifier associated to your publications +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/SUBJECT/MESHEUROPMC @=@ A classification term from the Medical Subject Headings (MeSH) that can be associated to your publications +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/ABSTRACT @=@ An abstract describing among your publications +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/PUBLICATION_DATE @=@ A date of publication missing in your content +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/PID @=@ A persistent identifier associated to your publications +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MORE/SUBJECT/ACM @=@ Another ACM classification term that can be associated to your publications +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MORE/SUBJECT/ARXIV @=@ Another ARXIV classification term that can be associated to your publications +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MORE/SUBJECT/DDC @=@ Another Dewey Decimal classification term (DDC) that can be associated to your publications +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MORE/SUBJECT/JEL @=@ Another Journal of Economic Literature (JEL) classification term that can be associated to your publications +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MORE/SUBJECT/MESHEUROPMC @=@ Another classification term from the Medical Subject Headings (MeSH) that can be associated to your publications +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/PROJECT @=@ A project reference that can be associated to your publications +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/DATASET/IS_SUPPLEMENTED_BY @=@ A dataset that is supplemented by your records +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/AUTHOR/ORCID @=@ An Open Researcher and Contributor ID (ORCID) that can be associated to an author of your publications +dnet:review_levels @=@ dnet:review_levels @=@ 0000 @=@ Unknown +dnet:review_levels @=@ dnet:review_levels @=@ 0002 @=@ nonPeerReviewed +dnet:review_levels @=@ dnet:review_levels @=@ 0001 @=@ peerReviewed \ No newline at end of file diff --git a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/datasourceorganization_resultset_entry.json b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/datasourceorganization_resultset_entry.json index 3a0318ed7b..2baf7c8f17 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/datasourceorganization_resultset_entry.json +++ b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/datasourceorganization_resultset_entry.json @@ -52,7 +52,7 @@ { "field": "semantics", "type": "not_used", - "value": "providedBy@@@provided by@@@dnet:datasources_organizations_typologies@@@dnet:datasources_organizations_typologies" + "value": "providedBy@@@dnet:datasources_organizations_typologies" }, { "field": "provenanceaction", diff --git a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/datasources_resultset_entry.json b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/datasources_resultset_entry.json index 71e84954f6..0f1da70959 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/datasources_resultset_entry.json +++ b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/datasources_resultset_entry.json @@ -6,9 +6,10 @@ }, { "field": "identities", - "type": "not_used", + "type": "array", "value": [ "274269ac6f3b::2579-5449", + "piwik:13", null ] }, @@ -30,7 +31,7 @@ { "field": "openairecompatibility", "type": "string", - "value": "hostedBy@@@collected from a compatible aggregator@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel" + "value": "hostedBy@@@dnet:datasourceCompatibilityLevel" }, { "field": "websiteurl", @@ -219,16 +220,16 @@ { "field": "datasourcetype", "type": "string", - "value": "pubsrepository::journal@@@Journal@@@dnet:datasource_typologies@@@dnet:datasource_typologies" + "value": "pubsrepository::journal@@@dnet:datasource_typologies" }, { "field": "provenanceaction", "type": "not_used", - "value": "sysimport:crosswalk:entityregistry@@@sysimport:crosswalk:entityregistry@@@dnet:provenance_actions@@@dnet:provenance_actions" + "value": "sysimport:crosswalk:entityregistry@@@dnet:provenance_actions" }, { "field": "journal", "type": "string", - "value": "2579-5449@@@2597-6540@@@" + "value": "2579-5449 @@@ 2597-6540 @@@ " } ] diff --git a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/oaf_record.xml b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/oaf_record.xml index 2cb0ba1c78..ead22aa965 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/oaf_record.xml +++ b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/oaf_record.xml @@ -57,6 +57,7 @@ 10.3897/oneeco.2.e13718 https://oneecosystem.pensoft.net/article/13718/ One Ecosystem + 0001 diff --git a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/odf_dataset.xml b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/odf_dataset.xml index 88ae9d1063..5525a27533 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/odf_dataset.xml +++ b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/odf_dataset.xml @@ -90,6 +90,7 @@ corda_______::226852 + 0001s dhp-workflows eu.dnetlib.dhp - 1.2.2-SNAPSHOT + 1.2.4-SNAPSHOT 4.0.0 diff --git a/dhp-workflows/dhp-graph-provision-scholexplorer/src/main/java/eu/dnetlib/dhp/export/DLIToOAF.scala b/dhp-workflows/dhp-graph-provision-scholexplorer/src/main/java/eu/dnetlib/dhp/export/DLIToOAF.scala new file mode 100644 index 0000000000..637362acf0 --- /dev/null +++ b/dhp-workflows/dhp-graph-provision-scholexplorer/src/main/java/eu/dnetlib/dhp/export/DLIToOAF.scala @@ -0,0 +1,457 @@ +package eu.dnetlib.dhp.export + +import java.time.LocalDateTime +import java.time.format.DateTimeFormatter + +import eu.dnetlib.dhp.common.PacePerson +import eu.dnetlib.dhp.schema.action.AtomicAction +import eu.dnetlib.dhp.schema.oaf.{Author, DataInfo, Dataset, ExternalReference, Field, Instance, KeyValue, Oaf, Publication, Qualifier, Relation, StructuredProperty} +import eu.dnetlib.dhp.schema.scholexplorer.{DLIDataset, DLIPublication, DLIRelation} +import eu.dnetlib.dhp.utils.DHPUtils +import org.apache.commons.lang3.StringUtils +import org.codehaus.jackson.map.ObjectMapper + +import scala.collection.JavaConverters._ + + +case class DLIExternalReference(id: String, url: String, sitename: String, label: String, pid: String, classId: String) {} + +object DLIToOAF { + + + val collectedFromMap: Map[String, KeyValue] = Map( + "dli_________::r3d100010527" -> generateKeyValue("10|re3data_____::c2a591f440598b63d854556beaf01591", "European Nucleotide Archive"), + "dli_________::r3d100010255" -> generateKeyValue("10|re3data_____::480d275ed6f9666ee76d6a1215eabf26", "Inter-university Consortium for Political and Social Research"), + "dli_________::r3d100011868" -> generateKeyValue("10|re3data_____::db814dc656a911b556dba42a331cebe9", "Mendeley Data"), + "dli_________::elsevier" -> generateKeyValue("10|openaire____::8f87e10869299a5fe80b315695296b88", "Elsevier"), + "dli_________::openaire" -> generateKeyValue("10|infrastruct_::f66f1bd369679b5b077dcdf006089556", "OpenAIRE"), + "dli_________::thomsonreuters" -> generateKeyValue("10|openaire____::081b82f96300b6a6e3d282bad31cb6e2", "Crossref"), + "dli_________::r3d100010216" -> generateKeyValue("10|re3data_____::0fd79429de04343dbbec705d9b5f429f", "4TU.Centre for Research Data"), + "dli_________::r3d100010134" -> generateKeyValue("10|re3data_____::9633d1e8c4309c833c2c442abeb0cfeb", "PANGAEA"), + "dli_________::ieee" -> generateKeyValue("10|openaire____::081b82f96300b6a6e3d282bad31cb6e2", "Crossref"), + "dli_________::r3d100010197" -> generateKeyValue("10|re3data_____::9fd1d79973f7fda60cbe1d82e3819a68", "The Cambridge Structural Database"), + "dli_________::nature" -> generateKeyValue("10|openaire____::6e380d9cf51138baec8480f5a0ce3a2e", "Springer Nature"), + "dli_________::datacite" -> generateKeyValue("10|openaire____::9e3be59865b2c1c335d32dae2fe7b254", "Datacite"), + "dli_________::r3d100010578" -> generateKeyValue("10|re3data_____::c4d751f29a7568011a4c80136b30b444", "IEDA"), + "dli_________::r3d100010464" -> generateKeyValue("10|re3data_____::23e2a81591099828f6b83a1c83150666", "Research Data Australia"), + "dli_________::r3d100010327" -> generateKeyValue("10|re3data_____::a644620b81135243dc9acc15d2362246", "Worldwide Protein Data Bank"), + "dli_________::pubmed" -> generateKeyValue("10|opendoar____::eda80a3d5b344bc40f3bc04f65b7a357", "PubMed Central"), + "dli_________::europe_pmc__" -> generateKeyValue("10|opendoar____::8b6dd7db9af49e67306feb59a8bdc52c", "Europe PubMed Central"), + "dli_________::crossref" -> generateKeyValue("10|openaire____::081b82f96300b6a6e3d282bad31cb6e2", "Crossref") + ) + + + val relationTypeMapping: Map[String, (String, String)] = Map( + "IsReferencedBy" -> ("isRelatedTo", "relationship"), + "References" -> ("isRelatedTo", "relationship"), + "IsRelatedTo" -> ("isRelatedTo", "relationship"), + "IsSupplementedBy" -> ("IsSupplementedBy", "supplement"), + "Cites" -> ("cites", "citation"), + "Unknown" -> ("isRelatedTo", "relationship"), + "IsSourceOf" -> ("isRelatedTo", "relationship"), + "IsCitedBy" -> ("IsCitedBy", "citation"), + "Reviews" -> ("reviews", "review"), + "Describes" -> ("isRelatedTo", "relationship"), + "HasAssociationWith" -> ("isRelatedTo", "relationship") + ) + + val expectecdPidType = List("uniprot", "ena", "chembl", "ncbi-n", "ncbi-p", "genbank", "pdb", "url") + + + val filteredURL = List( + "www.ebi.ac.uk", + "www.uniprot.org", + "f1000.com", + "en.wikipedia.org", + "flybase.org", + "www.yeastgenome.org", + "research.bioinformatics.udel.edu", + "cancer.sanger.ac.uk", + "www.iedb.org", + "www.crd.york.ac.uk", + "www.wormbase.org", + "web.expasy.org", + "www.hal.inserm.fr", + "sabiork.h-its.org", + "zfin.org", + "www.pombase.org", + "www.guidetopharmacology.org", + "reactome.org" + ) + + + val rel_inverse: Map[String, String] = Map( + "isRelatedTo" -> "isRelatedTo", + "IsSupplementedBy" -> "isSupplementTo", + "cites" -> "IsCitedBy", + "IsCitedBy" -> "cites", + "reviews" -> "IsReviewedBy" + ) + + + val PidTypeMap: Map[String, String] = Map( + "pbmid" -> "pmid", + "pmcid" -> "pmc", + "pmid" -> "pmid", + "pubmedid" -> "pmid", + "DOI" -> "doi", + "doi" -> "doi" + ) + + + def toActionSet(item: Oaf): (String, String) = { + val mapper = new ObjectMapper() + + item match { + case dataset: Dataset => + val a: AtomicAction[Dataset] = new AtomicAction[Dataset] + a.setClazz(classOf[Dataset]) + a.setPayload(dataset) + (dataset.getClass.getCanonicalName, mapper.writeValueAsString(a)) + case publication: Publication => + val a: AtomicAction[Publication] = new AtomicAction[Publication] + a.setClazz(classOf[Publication]) + a.setPayload(publication) + (publication.getClass.getCanonicalName, mapper.writeValueAsString(a)) + case relation: Relation => + val a: AtomicAction[Relation] = new AtomicAction[Relation] + a.setClazz(classOf[Relation]) + a.setPayload(relation) + (relation.getClass.getCanonicalName, mapper.writeValueAsString(a)) + case _ => + null + } + } + + def convertClinicalTrial(dataset: DLIDataset): (String, String) = { + val currentId = generateId(dataset.getId) + val pids = dataset.getPid.asScala.filter(p => "clinicaltrials.gov".equalsIgnoreCase(p.getQualifier.getClassname)).map(p => s"50|r3111dacbab5::${DHPUtils.md5(p.getValue.toLowerCase())}") + if (pids.isEmpty) + null + else + (currentId, pids.head) + } + + + def insertExternalRefs(publication: Publication, externalReferences: List[DLIExternalReference]): Publication = { + + val eRefs = externalReferences.map(e => { + val result = new ExternalReference() + result.setSitename(e.sitename) + result.setLabel(e.label) + result.setUrl(e.url) + result.setRefidentifier(e.pid) + result.setDataInfo(generateDataInfo()) + result.setQualifier(createQualifier(e.classId, "dnet:externalReference_typologies")) + result + }) + publication.setExternalReference(eRefs.asJava) + publication + + } + + def filterPid(p: StructuredProperty): Boolean = { + if (expectecdPidType.contains(p.getQualifier.getClassname) && p.getQualifier.getClassname.equalsIgnoreCase("url")) + if (filteredURL.exists(u => p.getValue.contains(u))) + return true + else + return false + expectecdPidType.contains(p.getQualifier.getClassname) + } + + + def extractTitle(titles: java.util.List[StructuredProperty]): String = { + + if (titles == null) + return null + + val label = titles.asScala.map(p => p.getValue).find(p => p.nonEmpty) + label.orNull + } + + def convertDLIDatasetToExternalReference(dataset: DLIDataset): DLIExternalReference = { + val pids = dataset.getPid.asScala.filter(filterPid) + + if (pids == null || pids.isEmpty) + return null + + val pid: StructuredProperty = pids.head + + + pid.getQualifier.getClassname match { + case "uniprot" => DLIExternalReference(generateId(dataset.getId), s"https://www.uniprot.org/uniprot/${pid.getValue}", "UniProt", extractTitle(dataset.getTitle), pid.getValue, "accessionNumber") + case "ena" => + if (pid.getValue != null && pid.getValue.nonEmpty && pid.getValue.length > 7) + DLIExternalReference(generateId(dataset.getId), s"https://www.ebi.ac.uk/ena/data/view/${pid.getValue.substring(0, 8)}", "European Nucleotide Archive", extractTitle(dataset.getTitle), pid.getValue, "accessionNumber") + else + null + case "chembl" => DLIExternalReference(generateId(dataset.getId), s"https://www.ebi.ac.uk/chembl/compound_report_card/${pid.getValue}", "ChEMBL", extractTitle(dataset.getTitle), pid.getValue, "accessionNumber") + case "ncbi-n" => DLIExternalReference(generateId(dataset.getId), s"https://www.ncbi.nlm.nih.gov/nuccore/${pid.getValue}", "Nucleotide Database", extractTitle(dataset.getTitle), pid.getValue, "accessionNumber") + case "ncbi-p" => DLIExternalReference(generateId(dataset.getId), s"https://www.ncbi.nlm.nih.gov/nuccore/${pid.getValue}", "Nucleotide Database", extractTitle(dataset.getTitle), pid.getValue, "accessionNumber") + case "genbank" => DLIExternalReference(generateId(dataset.getId), s"https://www.ncbi.nlm.nih.gov/nuccore/${pid.getValue}", "GenBank", extractTitle(dataset.getTitle), pid.getValue, "accessionNumber") + case "pdb" => DLIExternalReference(generateId(dataset.getId), s"https://www.ncbi.nlm.nih.gov/nuccore/${pid.getValue}", "Protein Data Bank", extractTitle(dataset.getTitle), pid.getValue, "accessionNumber") + case "url" => DLIExternalReference(generateId(dataset.getId), pid.getValue, "", extractTitle(dataset.getTitle), pid.getValue, "url") + + } + + + } + + + def convertDLIPublicationToOAF(inputPublication: DLIPublication): Publication = { + val result = new Publication + val cleanedPids = inputPublication.getPid.asScala.filter(p => PidTypeMap.contains(p.getQualifier.getClassid)) + .map(p => { + p.setQualifier(createQualifier(PidTypeMap(p.getQualifier.getClassid), p.getQualifier.getSchemeid)) + p + }) + if (cleanedPids.isEmpty) + return null + result.setId(generateId(inputPublication.getId)) + result.setDataInfo(generateDataInfo(invisibile = true)) + if (inputPublication.getCollectedfrom == null || inputPublication.getCollectedfrom.size() == 0 || (inputPublication.getCollectedfrom.size() == 1 && inputPublication.getCollectedfrom.get(0) == null)) + return null + result.setCollectedfrom(inputPublication.getCollectedfrom.asScala.map(c => collectedFromMap.getOrElse(c.getKey, null)).filter(p => p != null).asJava) + if(result.getCollectedfrom.isEmpty) + return null + result.setPid(cleanedPids.asJava) + result.setDateofcollection(inputPublication.getDateofcollection) + result.setOriginalId(inputPublication.getPid.asScala.map(p => p.getValue).asJava) + result.setDateoftransformation(LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'"))) + if (inputPublication.getAuthor == null || inputPublication.getAuthor.isEmpty) + return null + result.setAuthor(inputPublication.getAuthor.asScala.map(convertAuthor).asJava) + result.setResulttype(createQualifier(inputPublication.getResulttype.getClassid, inputPublication.getResulttype.getClassname, "dnet:result_typologies", "dnet:result_typologies")) + + if (inputPublication.getSubject != null) + result.setSubject(inputPublication.getSubject.asScala.map(convertSubject).asJava) + + if (inputPublication.getTitle == null || inputPublication.getTitle.isEmpty) + return null + + result.setTitle(List(patchTitle(inputPublication.getTitle.get(0))).asJava) + + if (inputPublication.getRelevantdate == null || inputPublication.getRelevantdate.size() == 0) + return null + + result.setRelevantdate(inputPublication.getRelevantdate.asScala.map(patchRelevantDate).asJava) + + + result.setDescription(inputPublication.getDescription) + + result.setDateofacceptance(asField(inputPublication.getRelevantdate.get(0).getValue)) + result.setPublisher(inputPublication.getPublisher) + result.setSource(inputPublication.getSource) + result.setBestaccessright(createQualifier("UNKNOWN", "not available", "dnet:access_modes", "dnet:access_modes")) + + val dois = result.getPid.asScala.filter(p => "doi".equalsIgnoreCase(p.getQualifier.getClassname)).map(p => p.getValue) + if (dois.isEmpty) + return null + + + val i: Instance = createInstance(s"https://dx.doi.org/${dois.head}", firstInstanceOrNull(inputPublication.getInstance()), result.getDateofacceptance) + + if (i != null) + result.setInstance(List(i).asJava) + + result + } + + + def convertDLIRelation(r: DLIRelation): Relation = { + + val result = new Relation + if (!relationTypeMapping.contains(r.getRelType)) + return null + + if (r.getCollectedFrom == null || r.getCollectedFrom.size() == 0 || (r.getCollectedFrom.size() == 1 && r.getCollectedFrom.get(0) == null)) + return null + val t = relationTypeMapping.get(r.getRelType) + + result.setRelType("resultResult") + result.setRelClass(t.get._1) + result.setSubRelType(t.get._2) + result.setCollectedfrom(r.getCollectedFrom.asScala.map(c => collectedFromMap.getOrElse(c.getKey, null)).filter(p => p != null).asJava) + result.setSource(generateId(r.getSource)) + result.setTarget(generateId(r.getTarget)) + + if (result.getSource.equals(result.getTarget)) + return null + result.setDataInfo(generateDataInfo()) + + result + } + + + def convertDLIDatasetTOOAF(d: DLIDataset): Dataset = { + + if (d.getCollectedfrom == null || d.getCollectedfrom.size() == 0 || (d.getCollectedfrom.size() == 1 && d.getCollectedfrom.get(0) == null)) + return null + val result: Dataset = new Dataset + result.setId(generateId(d.getId)) + result.setDataInfo(generateDataInfo()) + result.setCollectedfrom(d.getCollectedfrom.asScala.map(c => collectedFromMap.getOrElse(c.getKey, null)).filter(p => p != null).asJava) + if(result.getCollectedfrom.isEmpty) + return null + + + result.setPid(d.getPid) + + val fpids = result.getPid.asScala.filter(p => "doi".equalsIgnoreCase(p.getQualifier.getClassname) || + "pdb".equalsIgnoreCase(p.getQualifier.getClassname) + ).map(p => p.getValue) + + if (fpids == null || fpids.isEmpty) + return null + + + result.setDateofcollection(d.getDateofcollection) + result.setOriginalId(d.getPid.asScala.map(d => d.getValue).asJava) + result.setDateoftransformation(LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'"))) + if (d.getAuthor == null || d.getAuthor.isEmpty) + return null + result.setAuthor(d.getAuthor.asScala.map(convertAuthor).asJava) + result.setResulttype(createQualifier(d.getResulttype.getClassid, d.getResulttype.getClassname, "dnet:result_typologies", "dnet:result_typologies")) + + if (d.getSubject != null) + result.setSubject(d.getSubject.asScala.map(convertSubject).asJava) + + if (d.getTitle == null || d.getTitle.isEmpty) + return null + + result.setTitle(List(patchTitle(d.getTitle.get(0))).asJava) + + if (d.getRelevantdate == null || d.getRelevantdate.size() == 0) + return null + + result.setRelevantdate(d.getRelevantdate.asScala.map(patchRelevantDate).asJava) + + + result.setDescription(d.getDescription) + + result.setDateofacceptance(asField(d.getRelevantdate.get(0).getValue)) + result.setPublisher(d.getPublisher) + result.setSource(d.getSource) + result.setBestaccessright(createQualifier("UNKNOWN", "not available", "dnet:access_modes", "dnet:access_modes")) + + + val instance_urls = if (fpids.head.length < 5) s"https://www.rcsb.org/structure/${fpids.head}" else s"https://dx.doi.org/${fpids.head}" + + val i: Instance = createInstance(instance_urls, firstInstanceOrNull(d.getInstance()), result.getDateofacceptance, true) + if (i != null) + result.setInstance(List(i).asJava) + + result + } + + + def firstInstanceOrNull(instances: java.util.List[Instance]): Instance = { + + if (instances == null || instances.size() == 0) + return null + instances.get(0) + + } + + + def createInstance(url: String, originalInstance: Instance, doa: Field[String], dataset: Boolean = false): Instance = { + + val i = new Instance + i.setUrl(List(url).asJava) + if (dataset) + i.setInstancetype(createQualifier("0021", "Dataset", "dnet:publication_resource", "dnet:publication_resource")) + else + i.setInstancetype(createQualifier("0000", "Unknown", "dnet:publication_resource", "dnet:publication_resource")) + if (originalInstance != null && originalInstance.getHostedby != null) + i.setHostedby(originalInstance.getHostedby) + + i.setAccessright(createQualifier("UNKNOWN", "not available", "dnet:access_modes", "dnet:access_modes")) + i.setDateofacceptance(doa) + + i + + + } + + + def patchRelevantDate(d: StructuredProperty): StructuredProperty = { + d.setQualifier(createQualifier("UNKNOWN", "dnet:dataCite_date")) + d + + } + + def patchTitle(t: StructuredProperty): StructuredProperty = { + t.setQualifier(createQualifier("main title", "dnet:dataCite_title")) + t + } + + + def convertSubject(s: StructuredProperty): StructuredProperty = { + s.setQualifier(createQualifier("keyword", "dnet:subject_classification_typologies")) + s + + + } + + + def convertAuthor(a: Author): Author = { + if (a == null) + return a + val p = new PacePerson(a.getFullname, false) + if (p.isAccurate) { + a.setName(p.getNameString) + a.setSurname(p.getSurnameString) + } + a + } + + + def generateId(id: String): String = { + val md5 = if (id.contains("::")) StringUtils.substringAfter(id, "::") else StringUtils.substringAfter(id, "|") + s"50|scholix_____::$md5" + } + + + def generateKeyValue(key: String, value: String): KeyValue = { + val kv: KeyValue = new KeyValue() + kv.setKey(key) + kv.setValue(value) + kv.setDataInfo(generateDataInfo("0.9")) + kv + } + + + def generateDataInfo(trust: String = "0.9", invisibile: Boolean = false): DataInfo = { + val di = new DataInfo + di.setDeletedbyinference(false) + di.setInferred(false) + di.setInvisible(false) + di.setTrust(trust) + di.setProvenanceaction(createQualifier("sysimport:actionset", "dnet:provenanceActions")) + di + } + + def createQualifier(cls: String, sch: String): Qualifier = { + createQualifier(cls, cls, sch, sch) + } + + + def createQualifier(classId: String, className: String, schemeId: String, schemeName: String): Qualifier = { + val q: Qualifier = new Qualifier + q.setClassid(classId) + q.setClassname(className) + q.setSchemeid(schemeId) + q.setSchemename(schemeName) + q + } + + + def asField[T](value: T): Field[T] = { + val tmp = new Field[T] + tmp.setValue(value) + tmp + + + } + +} diff --git a/dhp-workflows/dhp-graph-provision-scholexplorer/src/main/java/eu/dnetlib/dhp/export/SparkExportContentForOpenAire.scala b/dhp-workflows/dhp-graph-provision-scholexplorer/src/main/java/eu/dnetlib/dhp/export/SparkExportContentForOpenAire.scala new file mode 100644 index 0000000000..edf951df4d --- /dev/null +++ b/dhp-workflows/dhp-graph-provision-scholexplorer/src/main/java/eu/dnetlib/dhp/export/SparkExportContentForOpenAire.scala @@ -0,0 +1,184 @@ +package eu.dnetlib.dhp.`export` + +import eu.dnetlib.dhp.application.ArgumentApplicationParser +import eu.dnetlib.dhp.schema.oaf.{Publication, Relation, Dataset => OafDataset} +import eu.dnetlib.dhp.schema.scholexplorer.{DLIDataset, DLIPublication, DLIRelation} +import org.apache.commons.io.IOUtils +import org.apache.hadoop.io.Text +import org.apache.hadoop.io.compress.GzipCodec +import org.apache.hadoop.mapred.SequenceFileOutputFormat +import org.apache.spark.rdd.RDD +import org.apache.spark.sql.{Dataset, Encoder, Encoders, SaveMode, SparkSession} +import org.apache.spark.sql.functions._ +import org.apache.spark.sql.expressions.Window +import org.apache.spark.{SparkConf, SparkContext} +import org.codehaus.jackson.map.ObjectMapper + +import scala.collection.mutable.ArrayBuffer + + +object SparkExportContentForOpenAire { + + + def main(args: Array[String]): Unit = { + val conf: SparkConf = new SparkConf() + val parser = new ArgumentApplicationParser(IOUtils.toString(SparkExportContentForOpenAire.getClass.getResourceAsStream("input_export_content_parameters.json"))) + parser.parseArgument(args) + val spark: SparkSession = + SparkSession + .builder() + .config(conf) + .appName(SparkExportContentForOpenAire.getClass.getSimpleName) + .master(parser.get("master")).getOrCreate() + + + val sc:SparkContext = spark.sparkContext + + val workingPath = parser.get("workingDirPath") + + implicit val pubEncoder: Encoder[Publication] = Encoders.bean(classOf[Publication]) + implicit val datEncoder: Encoder[OafDataset] = Encoders.bean(classOf[OafDataset]) + implicit val relEncoder: Encoder[Relation] = Encoders.bean(classOf[Relation]) + implicit val dliRelEncoder: Encoder[DLIRelation] = Encoders.bean(classOf[DLIRelation]) + import spark.implicits._ + + + val relRDD:RDD[Relation] = sc.textFile(s"$workingPath/relation_j") + .map(s => new ObjectMapper().readValue(s, classOf[DLIRelation])) + .filter(p => p.getDataInfo.getDeletedbyinference == false) + .map(DLIToOAF.convertDLIRelation).filter(p=>p!= null) + spark.createDataset(relRDD).write.mode(SaveMode.Overwrite).save(s"$workingPath/relationDS") + + val datRDD:RDD[OafDataset] = sc.textFile(s"$workingPath/dataset") + .map(s => new ObjectMapper().readValue(s, classOf[DLIDataset])) + .filter(p => p.getDataInfo.getDeletedbyinference == false) + .map(DLIToOAF.convertDLIDatasetTOOAF).filter(p=>p!= null) + spark.createDataset(datRDD).write.mode(SaveMode.Overwrite).save(s"$workingPath/datasetDS") + + + val pubRDD:RDD[Publication] = sc.textFile(s"$workingPath/publication") + .map(s => new ObjectMapper().readValue(s, classOf[DLIPublication])) + .filter(p => p.getDataInfo.getDeletedbyinference == false) + .map(DLIToOAF.convertDLIPublicationToOAF).filter(p=>p!= null) + spark.createDataset(pubRDD).write.mode(SaveMode.Overwrite).save(s"$workingPath/publicationDS") + + + + val pubs:Dataset[Publication] = spark.read.load(s"$workingPath/publicationDS").as[Publication] + val dats :Dataset[OafDataset] = spark.read.load(s"$workingPath/datasetDS").as[OafDataset] + val relDS1 :Dataset[Relation] = spark.read.load(s"$workingPath/relationDS").as[Relation] + + + val pub_id = pubs.select("id").distinct() + val dat_id = dats.select("id").distinct() + + + pub_id.joinWith(relDS1, pub_id("id").equalTo(relDS1("source"))).map(k => k._2).write.mode(SaveMode.Overwrite).save(s"$workingPath/relationDS_f1") + + val relDS2= spark.read.load(s"$workingPath/relationDS_f1").as[Relation] + + relDS2.joinWith(dat_id, relDS2("target").equalTo(dats("id"))).map(k => k._1).write.mode(SaveMode.Overwrite).save(s"$workingPath/relationDS_filtered") + + + val r_source = relDS2.select(relDS2("source")).distinct() + val r_target = relDS2.select(relDS2("target")).distinct() + + + val w2 = Window.partitionBy("id").orderBy("lastupdatetimestamp") + + pubs.joinWith(r_source, pubs("id").equalTo(r_source("source")), "inner").map(k => k._1) + .withColumn("row",row_number.over(w2)).where($"row" === 1).drop("row") + .write.mode(SaveMode.Overwrite).save(s"$workingPath/publicationDS_filtered") + + dats.joinWith(r_target, dats("id").equalTo(r_target("target")), "inner").map(k => k._1) + .withColumn("row",row_number.over(w2)).where($"row" === 1).drop("row") + .write.mode(SaveMode.Overwrite).save(s"$workingPath/datasetAS") + + spark.createDataset(sc.textFile(s"$workingPath/dataset") + .map(s => new ObjectMapper().readValue(s, classOf[DLIDataset])) + .map(DLIToOAF.convertDLIDatasetToExternalReference) + .filter(p => p != null)).as[DLIExternalReference].write.mode(SaveMode.Overwrite).save(s"$workingPath/externalReference") + + val pf = spark.read.load(s"$workingPath/publicationDS_filtered").select("id") + val relDS3 = spark.read.load(s"$workingPath/relationDS").as[Relation] + val relationTo = pf.joinWith(relDS3, pf("id").equalTo(relDS3("source")),"inner").map(t =>t._2) + + val extRef = spark.read.load(s"$workingPath/externalReference").as[DLIExternalReference] + + spark.createDataset(relationTo.joinWith(extRef, relationTo("target").equalTo(extRef("id")), "inner").map(d => { + val r = d._1 + val ext = d._2 + (r.getSource, ext) + }).rdd.groupByKey.map(f => { + var dli_ext = ArrayBuffer[DLIExternalReference]() + f._2.foreach(d => if (dli_ext.size < 100) dli_ext += d ) + (f._1, dli_ext) + })).write.mode(SaveMode.Overwrite).save(s"$workingPath/externalReference_grouped") + + val pubf :Dataset[Publication] = spark.read.load(s"$workingPath/publicationDS_filtered").as[Publication] + + val groupedERf:Dataset[(String, List[DLIExternalReference])]= spark.read.load(s"$workingPath/externalReference_grouped").as[(String, List[DLIExternalReference])] + + groupedERf.joinWith(pubf,pubf("id").equalTo(groupedERf("_1"))).map(t => + { + val publication = t._2 + if (t._1 != null) { + val eRefs = t._1._2 + DLIToOAF.insertExternalRefs(publication, eRefs) + + } else + publication + } + ).write.mode(SaveMode.Overwrite).save(s"$workingPath/publicationAS") + + + spark.createDataset(sc.textFile(s"$workingPath/dataset") + .map(s => new ObjectMapper().readValue(s, classOf[DLIDataset])) + .map(DLIToOAF.convertClinicalTrial) + .filter(p => p != null)) + .write.mode(SaveMode.Overwrite).save(s"$workingPath/clinicalTrials") + + val ct:Dataset[(String,String)] = spark.read.load(s"$workingPath/clinicalTrials").as[(String,String)] + + val relDS= spark.read.load(s"$workingPath/relationDS_f1").as[Relation] + + relDS.joinWith(ct, relDS("target").equalTo(ct("_1")), "inner") + .map(k =>{ + val currentRel = k._1 + currentRel.setTarget(k._2._2) + currentRel + }).write.mode(SaveMode.Overwrite).save(s"$workingPath/clinicalTrialsRels") + + + val clRels:Dataset[Relation] = spark.read.load(s"$workingPath/clinicalTrialsRels").as[Relation] + val rels:Dataset[Relation] = spark.read.load(s"$workingPath/relationDS_filtered").as[Relation] + + rels.union(clRels).flatMap(r => { + val inverseRel = new Relation + inverseRel.setSource(r.getTarget) + inverseRel.setTarget(r.getSource) + inverseRel.setDataInfo(r.getDataInfo) + inverseRel.setCollectedfrom(r.getCollectedfrom) + inverseRel.setRelType(r.getRelType) + inverseRel.setSubRelType(r.getSubRelType) + inverseRel.setRelClass(DLIToOAF.rel_inverse(r.getRelClass)) + List(r, inverseRel) + }).write.mode(SaveMode.Overwrite).save(s"$workingPath/relationAS") + + + val fRels:Dataset[(String,String)] = spark.read.load(s"$workingPath/relationAS").as[Relation].map(DLIToOAF.toActionSet) + val fpubs:Dataset[(String,String)] = spark.read.load(s"$workingPath/publicationAS").as[Publication].map(DLIToOAF.toActionSet) + val fdats:Dataset[(String,String)] = spark.read.load(s"$workingPath/datasetAS").as[OafDataset].map(DLIToOAF.toActionSet) + + + fRels.union(fpubs).union(fdats).rdd.map(s => (new Text(s._1), new Text(s._2))).saveAsHadoopFile(s"$workingPath/rawset", classOf[Text], classOf[Text], classOf[SequenceFileOutputFormat[Text,Text]], classOf[GzipCodec]) + } + + + + + + + + +} diff --git a/dhp-workflows/dhp-graph-provision-scholexplorer/src/main/java/eu/dnetlib/dhp/provision/SparkIndexCollectionOnES.java b/dhp-workflows/dhp-graph-provision-scholexplorer/src/main/java/eu/dnetlib/dhp/provision/SparkIndexCollectionOnES.java index e79dad8d3e..78d8730806 100644 --- a/dhp-workflows/dhp-graph-provision-scholexplorer/src/main/java/eu/dnetlib/dhp/provision/SparkIndexCollectionOnES.java +++ b/dhp-workflows/dhp-graph-provision-scholexplorer/src/main/java/eu/dnetlib/dhp/provision/SparkIndexCollectionOnES.java @@ -40,6 +40,7 @@ public class SparkIndexCollectionOnES { final String index = parser.get("index"); final String idPath = parser.get("idPath"); final String type = parser.get("type"); + final String indexHost = parser.get("esHost"); final SparkSession spark = SparkSession.builder().config(conf).getOrCreate(); @@ -63,7 +64,8 @@ public class SparkIndexCollectionOnES { inputRdd = sc.textFile(sourcePath); Map esCfg = new HashMap<>(); - esCfg.put("es.nodes", "10.19.65.51, 10.19.65.52, 10.19.65.53, 10.19.65.54"); + // esCfg.put("es.nodes", "10.19.65.51, 10.19.65.52, 10.19.65.53, 10.19.65.54"); + esCfg.put("es.nodes", indexHost); esCfg.put("es.mapping.id", idPath); esCfg.put("es.batch.write.retry.count", "8"); esCfg.put("es.batch.write.retry.wait", "60s"); diff --git a/dhp-workflows/dhp-graph-provision-scholexplorer/src/main/resources/eu/dnetlib/dhp/export/input_export_content_parameters.json b/dhp-workflows/dhp-graph-provision-scholexplorer/src/main/resources/eu/dnetlib/dhp/export/input_export_content_parameters.json new file mode 100644 index 0000000000..b92f87e084 --- /dev/null +++ b/dhp-workflows/dhp-graph-provision-scholexplorer/src/main/resources/eu/dnetlib/dhp/export/input_export_content_parameters.json @@ -0,0 +1,14 @@ +[ + { + "paramName": "mt", + "paramLongName": "master", + "paramDescription": "should be local or yarn", + "paramRequired": true + }, + { + "paramName": "w", + "paramLongName": "workingDirPath", + "paramDescription": "the working path where generated files", + "paramRequired": true + } +] \ No newline at end of file diff --git a/dhp-workflows/dhp-graph-provision-scholexplorer/src/main/resources/eu/dnetlib/dhp/provision/index_on_es.json b/dhp-workflows/dhp-graph-provision-scholexplorer/src/main/resources/eu/dnetlib/dhp/provision/index_on_es.json index 905b6d5146..f70f7dd799 100644 --- a/dhp-workflows/dhp-graph-provision-scholexplorer/src/main/resources/eu/dnetlib/dhp/provision/index_on_es.json +++ b/dhp-workflows/dhp-graph-provision-scholexplorer/src/main/resources/eu/dnetlib/dhp/provision/index_on_es.json @@ -17,6 +17,13 @@ "paramDescription": "the index name", "paramRequired": true }, + { + "paramName": "h", + "paramLongName": "esHost", + "paramDescription": "the index host name", + "paramRequired": true + }, + { "paramName": "t", diff --git a/dhp-workflows/dhp-graph-provision-scholexplorer/src/main/resources/eu/dnetlib/dhp/sx/export/oozie_app/config-default.xml b/dhp-workflows/dhp-graph-provision-scholexplorer/src/main/resources/eu/dnetlib/dhp/sx/export/oozie_app/config-default.xml new file mode 100644 index 0000000000..59e5c059fc --- /dev/null +++ b/dhp-workflows/dhp-graph-provision-scholexplorer/src/main/resources/eu/dnetlib/dhp/sx/export/oozie_app/config-default.xml @@ -0,0 +1,42 @@ + + + jobTracker + yarnRM + + + nameNode + hdfs://nameservice1 + + + oozie.use.system.libpath + true + + + oozie.action.sharelib.for.spark + spark2 + + + oozie.wf.rerun.failnodes + false + + + hive_metastore_uris + thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083 + + + spark2YarnHistoryServerAddress + http://iis-cdh5-test-gw.ocean.icm.edu.pl:18089 + + + spark2EventLogDir + /user/spark/spark2ApplicationHistory + + + spark2ExtraListeners + "com.cloudera.spark.lineage.NavigatorAppListener" + + + spark2SqlQueryExecutionListeners + "com.cloudera.spark.lineage.NavigatorQueryListener" + + \ No newline at end of file diff --git a/dhp-workflows/dhp-graph-provision-scholexplorer/src/main/resources/eu/dnetlib/dhp/sx/export/oozie_app/workflow.xml b/dhp-workflows/dhp-graph-provision-scholexplorer/src/main/resources/eu/dnetlib/dhp/sx/export/oozie_app/workflow.xml new file mode 100644 index 0000000000..181ab80bf6 --- /dev/null +++ b/dhp-workflows/dhp-graph-provision-scholexplorer/src/main/resources/eu/dnetlib/dhp/sx/export/oozie_app/workflow.xml @@ -0,0 +1,49 @@ + + + + workingDirPath + the source path + + + sparkDriverMemory + memory for driver process + + + sparkExecutorMemory + memory for individual executor + + + sparkExecutorCores + memory for individual executor + + + + + + + Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] + + + + + yarn-cluster + cluster + ExtractOAF + eu.dnetlib.dhp.export.SparkExportContentForOpenAire + dhp-graph-provision-scholexplorer-${projectVersion}.jar + + --executor-memory=${sparkExecutorMemory} + --executor-cores=${sparkExecutorCores} + --driver-memory=${sparkDriverMemory} + --conf spark.sql.shuffle.partitions=3840 + ${sparkExtraOPT} + + --workingDirPath${workingDirPath} + --masteryarn-cluster + + + + + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-graph-provision-scholexplorer/src/main/resources/eu/dnetlib/dhp/sx/index/oozie_app/workflow.xml b/dhp-workflows/dhp-graph-provision-scholexplorer/src/main/resources/eu/dnetlib/dhp/sx/index/oozie_app/workflow.xml index 9fc86e0146..4f5c7bbf6e 100644 --- a/dhp-workflows/dhp-graph-provision-scholexplorer/src/main/resources/eu/dnetlib/dhp/sx/index/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-graph-provision-scholexplorer/src/main/resources/eu/dnetlib/dhp/sx/index/oozie_app/workflow.xml @@ -16,6 +16,11 @@ index index name + + + indexHost + index host name + @@ -37,6 +42,7 @@ -mt yarn-cluster --sourcePath${workingDirPath}/summary --index${index}_object + --esHost${indexHost} --idPathid --typesummary @@ -57,6 +63,7 @@ -mt yarn-cluster --sourcePath${workingDirPath}/scholix_json --index${index}_scholix + --esHost${indexHost} --idPathidentifier --typescholix diff --git a/dhp-workflows/dhp-graph-provision-scholexplorer/src/test/java/eu/dnetlib/dhp/export/ExportDLITOOAFTest.scala b/dhp-workflows/dhp-graph-provision-scholexplorer/src/test/java/eu/dnetlib/dhp/export/ExportDLITOOAFTest.scala new file mode 100644 index 0000000000..c9d33dbe47 --- /dev/null +++ b/dhp-workflows/dhp-graph-provision-scholexplorer/src/test/java/eu/dnetlib/dhp/export/ExportDLITOOAFTest.scala @@ -0,0 +1,75 @@ +package eu.dnetlib.dhp.export + +import java.time.LocalDateTime +import java.time.format.DateTimeFormatter + +import eu.dnetlib.dhp.schema.oaf.Relation +import eu.dnetlib.dhp.schema.scholexplorer.{DLIDataset, DLIPublication, DLIRelation} +import org.apache.spark.SparkConf +import org.apache.spark.rdd.RDD +import org.apache.spark.sql.SparkSession +import org.codehaus.jackson.map.{ObjectMapper, SerializationConfig} +import org.junit.jupiter.api.Test + +import scala.io.Source + +class ExportDLITOOAFTest { + + val mapper = new ObjectMapper() + + @Test + def testDate():Unit = { + println(LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'"))) + + } + + @Test + def testPublicationMapping():Unit = { + + mapper.getSerializationConfig.enable(SerializationConfig.Feature.INDENT_OUTPUT) + val json = Source.fromInputStream(getClass.getResourceAsStream("publication.json")).mkString + + + val oaf =DLIToOAF.convertDLIPublicationToOAF(mapper.readValue(json, classOf[DLIPublication])) + + println(mapper.writeValueAsString(oaf)) + + + } + + + @Test + def testExternalReferenceMapping():Unit = { + + mapper.getSerializationConfig.enable(SerializationConfig.Feature.INDENT_OUTPUT) + val json = Source.fromInputStream(getClass.getResourceAsStream("dataset.json")).mkString + + + val oaf =DLIToOAF.convertDLIDatasetToExternalReference(mapper.readValue(json, classOf[DLIDataset])) + + println(oaf) + + + } + + + + + + + + @Test + def testRelationMapping():Unit = { + + mapper.getSerializationConfig.enable(SerializationConfig.Feature.INDENT_OUTPUT) + val json = Source.fromInputStream(getClass.getResourceAsStream("relation.json")).mkString + + + val oaf =DLIToOAF.convertDLIRelation(mapper.readValue(json, classOf[DLIRelation])) + + println(mapper.writeValueAsString(oaf)) + + + } + +} diff --git a/dhp-workflows/dhp-graph-provision-scholexplorer/src/test/resources/eu/dnetlib/dhp/export/dataset.json b/dhp-workflows/dhp-graph-provision-scholexplorer/src/test/resources/eu/dnetlib/dhp/export/dataset.json new file mode 100644 index 0000000000..dae6357305 --- /dev/null +++ b/dhp-workflows/dhp-graph-provision-scholexplorer/src/test/resources/eu/dnetlib/dhp/export/dataset.json @@ -0,0 +1,101 @@ +{ + "dataInfo": { + "invisible": false, + "inferred": null, + "deletedbyinference": false, + "trust": "0.9", + "inferenceprovenance": null, + "provenanceaction": null + }, + "lastupdatetimestamp": null, + "id": "60|719f19e5a996de1b87cddf93871bf2d4", + "originalId": [ + "a0a3p2gws9::uniprot" + ], + "collectedfrom": [ + { + "key": "dli_________::europe_pmc__", + "value": "Europe PMC", + "dataInfo": null + } + ], + "pid": [ + { + "value": "acc63471", + "qualifier": { + "classid": "ena", + "classname": "ena", + "schemeid": "dnet:pid_types", + "schemename": "dnet:pid_types" + }, + "dataInfo": null + } + ], + "dateofcollection": "2019-07-05T12:47:11.545+02:00", + "dateoftransformation": null, + "extraInfo": null, + "oaiprovenance": null, + "author": null, + "resulttype": { + "classid": "dataset", + "classname": "dataset", + "schemeid": "dataset", + "schemename": "dataset" + }, + "language": null, + "country": null, + "subject": [], + "title": [ + { + "value": "CMD domain-containing protein", + "qualifier": null, + "dataInfo": null + } + ], + "relevantdate": [ + { + "value": "2019-07-15T16:14:28.636", + "qualifier": { + "classid": "resolvedDate", + "classname": "resolvedDate", + "schemeid": "dnet::date", + "schemename": "dnet::date" + }, + "dataInfo": null + } + ], + "description": null, + "dateofacceptance": null, + "publisher": { + "value": "UniProt", + "dataInfo": null + }, + "embargoenddate": null, + "source": null, + "fulltext": null, + "format": null, + "contributor": null, + "resourcetype": null, + "coverage": null, + "bestaccessright": null, + "context": null, + "externalReference": null, + "instance": [], + "storagedate": null, + "device": null, + "size": null, + "version": null, + "lastmetadataupdate": null, + "metadataversionnumber": null, + "geolocation": null, + "originalObjIdentifier": "europe_pmc__::719f19e5a996de1b87cddf93871bf2d4", + "dlicollectedfrom": [ + { + "id": "dli_________::europe_pmc__", + "name": "Europe PMC", + "completionStatus": "complete", + "collectionMode": null + } + ], + "completionStatus": "complete" +} \ No newline at end of file diff --git a/dhp-workflows/dhp-graph-provision-scholexplorer/src/test/resources/eu/dnetlib/dhp/export/publication.json b/dhp-workflows/dhp-graph-provision-scholexplorer/src/test/resources/eu/dnetlib/dhp/export/publication.json new file mode 100644 index 0000000000..4ab3de2da0 --- /dev/null +++ b/dhp-workflows/dhp-graph-provision-scholexplorer/src/test/resources/eu/dnetlib/dhp/export/publication.json @@ -0,0 +1,128 @@ +{ + "dataInfo": { + "invisible": false, + "inferred": null, + "deletedbyinference": false, + "trust": "0.9", + "inferenceprovenance": null, + "provenanceaction": null + }, + "lastupdatetimestamp": null, + "id": "50|9e117414be07bf03cbce8889d22d661a", + "originalId": [ + "9e117414be07bf03cbce8889d22d661a" + ], + "collectedfrom": [ + { + "key": "dli_________::crossref", + "value": "Crossref", + "dataInfo": null + } + ], + "pid": [ + { + "value": "10.1007/978-94-017-3490-5_15", + "qualifier": { + "classid": "doi", + "classname": "doi", + "schemeid": "dnet:pid_types", + "schemename": "dnet:pid_types" + }, + "dataInfo": null + } + ], + "dateofcollection": "2020-06-08T07:28:55.731Z", + "dateoftransformation": null, + "extraInfo": null, + "oaiprovenance": null, + "author": [ + { + "fullname": "Calcaterra Domenico", + "name": null, + "surname": null, + "rank": null, + "pid": null, + "affiliation": null + }, + { + "fullname": "Parise Mario", + "name": null, + "surname": null, + "rank": null, + "pid": null, + "affiliation": null + } + ], + "resulttype": { + "classid": "publication", + "classname": "publication", + "schemeid": "publication", + "schemename": "publication" + }, + "language": null, + "country": null, + "subject":[ + { + "value":"Strain-linked information about bacterial and archaeal biodiversity", + "qualifier":{ + "classid":"dnet:subject", + "classname":"dnet:subject", + "schemeid":"", + "schemename":"" + }, + "dataInfo":null + } + ], + "title": [ + { + "value": "The Contribution of Historical Information in the Assessment of Landslide Hazard", + "qualifier": null, + "dataInfo": null + } + ], + "relevantdate": [ + { + "value": "2013-01-29T16:50:44Z", + "qualifier": { + "classid": "date", + "classname": "date", + "schemeid": "dnet::date", + "schemename": "dnet::date" + }, + "dataInfo": null + } + ], + "description": [ + { + "value": null, + "dataInfo": null + } + ], + "dateofacceptance": null, + "publisher": { + "value": "Springer Netherlands", + "dataInfo": null + }, + "embargoenddate": null, + "source": null, + "fulltext": null, + "format": null, + "contributor": null, + "resourcetype": null, + "coverage": null, + "bestaccessright": null, + "context": null, + "externalReference": null, + "instance": [], + "journal": null, + "originalObjIdentifier": "dli_resolver::9e117414be07bf03cbce8889d22d661a", + "dlicollectedfrom": [ + { + "id": "dli_________::crossref", + "name": "Crossref", + "completionStatus": "complete", + "collectionMode": "resolved" + } + ], + "completionStatus": "complete" +} \ No newline at end of file diff --git a/dhp-workflows/dhp-graph-provision-scholexplorer/src/test/resources/eu/dnetlib/dhp/export/relation.json b/dhp-workflows/dhp-graph-provision-scholexplorer/src/test/resources/eu/dnetlib/dhp/export/relation.json new file mode 100644 index 0000000000..cdb0cfa1d5 --- /dev/null +++ b/dhp-workflows/dhp-graph-provision-scholexplorer/src/test/resources/eu/dnetlib/dhp/export/relation.json @@ -0,0 +1,23 @@ +{ + "subRelType": null, + "relClass": "datacite", + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": null, + "inferred": null, + "inferenceprovenance": null, + "invisible": false, + "trust": "0.9" + }, + "target": "50|00062410e2a15322480277d063c181bb", + "lastupdatetimestamp": null, + "relType": "IsReferencedBy", + "source": "60|4ee78ab329b49416b45c3774c132f244", + "collectedFrom": [ + { + "dataInfo": null, + "value": "Europe PMC", + "key": "dli_________::europe_pmc__" + } + ] +} diff --git a/dhp-workflows/dhp-graph-provision/pom.xml b/dhp-workflows/dhp-graph-provision/pom.xml index 62bf7186ce..fa19647730 100644 --- a/dhp-workflows/dhp-graph-provision/pom.xml +++ b/dhp-workflows/dhp-graph-provision/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.2-SNAPSHOT + 1.2.4-SNAPSHOT 4.0.0 diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/AdjacencyListBuilderJob.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/AdjacencyListBuilderJob.java index 99247b7562..d9cc03cd5c 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/AdjacencyListBuilderJob.java +++ b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/AdjacencyListBuilderJob.java @@ -6,23 +6,23 @@ import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; import org.apache.commons.io.IOUtils; import org.apache.spark.SparkConf; import org.apache.spark.api.java.function.MapFunction; import org.apache.spark.api.java.function.MapGroupsFunction; -import org.apache.spark.sql.Encoders; -import org.apache.spark.sql.SaveMode; -import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.*; +import org.apache.spark.sql.expressions.Aggregator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.common.HdfsSupport; -import eu.dnetlib.dhp.oa.provision.model.EntityRelEntity; -import eu.dnetlib.dhp.oa.provision.model.JoinedEntity; -import eu.dnetlib.dhp.oa.provision.model.Tuple2; -import eu.dnetlib.dhp.schema.common.ModelSupport; +import eu.dnetlib.dhp.oa.provision.model.*; +import scala.Tuple2; +import scala.collection.JavaConverters; +import scala.collection.Seq; /** * Joins the graph nodes by resolving the links of distance = 1 to create an adjacency list of linked objects. The @@ -76,46 +76,31 @@ public class AdjacencyListBuilderJob { SparkConf conf = new SparkConf(); conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer"); - conf.registerKryoClasses(ModelSupport.getOafModelClasses()); + conf.registerKryoClasses(ProvisionModelSupport.getModelClasses()); runWithSparkSession( conf, isSparkSessionManaged, spark -> { removeOutputDir(spark, outputPath); - createAdjacencyLists(spark, inputPath, outputPath); + createAdjacencyListsKryo(spark, inputPath, outputPath); }); } - private static void createAdjacencyLists( + private static void createAdjacencyListsKryo( SparkSession spark, String inputPath, String outputPath) { log.info("Reading joined entities from: {}", inputPath); - spark - .read() - .load(inputPath) - .as(Encoders.bean(EntityRelEntity.class)) - .groupByKey( - (MapFunction) value -> value.getEntity().getId(), - Encoders.STRING()) - .mapGroups( - (MapGroupsFunction) (key, values) -> { - JoinedEntity j = new JoinedEntity(); - List links = new ArrayList<>(); - while (values.hasNext() && links.size() < MAX_LINKS) { - EntityRelEntity curr = values.next(); - if (j.getEntity() == null) { - j.setEntity(curr.getEntity()); - } - links.add(new Tuple2(curr.getRelation(), curr.getTarget())); - } - j.setLinks(links); - return j; - }, - Encoders.bean(JoinedEntity.class)) - .write() - .mode(SaveMode.Overwrite) - .parquet(outputPath); + + final List paths = HdfsSupport + .listFiles(inputPath, spark.sparkContext().hadoopConfiguration()); + + log.info("Found paths: {}", String.join(",", paths)); + + } + + private static Seq toSeq(List list) { + return JavaConverters.asScalaIteratorConverter(list.iterator()).asScala().toSeq(); } private static void removeOutputDir(SparkSession spark, String path) { diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/CreateRelatedEntitiesJob_phase1.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/CreateRelatedEntitiesJob_phase1.java index 606fa4cc0c..80b8000173 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/CreateRelatedEntitiesJob_phase1.java +++ b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/CreateRelatedEntitiesJob_phase1.java @@ -2,7 +2,6 @@ package eu.dnetlib.dhp.oa.provision; import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; -import static eu.dnetlib.dhp.oa.provision.utils.GraphMappingUtils.*; import java.util.List; import java.util.Objects; @@ -23,11 +22,10 @@ import com.fasterxml.jackson.databind.ObjectMapper; import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.common.HdfsSupport; -import eu.dnetlib.dhp.oa.provision.model.EntityRelEntity; +import eu.dnetlib.dhp.oa.provision.model.ProvisionModelSupport; import eu.dnetlib.dhp.oa.provision.model.RelatedEntity; -import eu.dnetlib.dhp.oa.provision.model.SortableRelation; +import eu.dnetlib.dhp.oa.provision.model.RelatedEntityWrapper; import eu.dnetlib.dhp.schema.common.EntityType; -import eu.dnetlib.dhp.schema.common.ModelSupport; import eu.dnetlib.dhp.schema.oaf.*; import scala.Tuple2; @@ -91,7 +89,7 @@ public class CreateRelatedEntitiesJob_phase1 { SparkConf conf = new SparkConf(); conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer"); - conf.registerKryoClasses(ModelSupport.getOafModelClasses()); + conf.registerKryoClasses(ProvisionModelSupport.getModelClasses()); runWithSparkSession( conf, @@ -109,18 +107,19 @@ public class CreateRelatedEntitiesJob_phase1 { Class clazz, String outputPath) { - Dataset> relsByTarget = readPathRelation(spark, inputRelationsPath) + Dataset> relsByTarget = readPathRelation(spark, inputRelationsPath) .filter("dataInfo.deletedbyinference == false") .map( - (MapFunction>) r -> new Tuple2<>(r.getTarget(), r), - Encoders.tuple(Encoders.STRING(), Encoders.kryo(SortableRelation.class))) + (MapFunction>) r -> new Tuple2<>(r.getTarget(), + r), + Encoders.tuple(Encoders.STRING(), Encoders.kryo(Relation.class))) .cache(); Dataset> entities = readPathEntity(spark, inputEntityPath, clazz) .filter("dataInfo.invisible == false") .map( (MapFunction) value -> asRelatedEntity(value, clazz), - Encoders.bean(RelatedEntity.class)) + Encoders.kryo(RelatedEntity.class)) .map( (MapFunction>) e -> new Tuple2<>(e.getId(), e), Encoders.tuple(Encoders.STRING(), Encoders.kryo(RelatedEntity.class))) @@ -129,12 +128,12 @@ public class CreateRelatedEntitiesJob_phase1 { relsByTarget .joinWith(entities, entities.col("_1").equalTo(relsByTarget.col("_1")), "inner") .map( - (MapFunction, Tuple2>, EntityRelEntity>) t -> new EntityRelEntity( + (MapFunction, Tuple2>, RelatedEntityWrapper>) t -> new RelatedEntityWrapper( t._1()._2(), t._2()._2()), - Encoders.bean(EntityRelEntity.class)) + Encoders.kryo(RelatedEntityWrapper.class)) .write() .mode(SaveMode.Overwrite) - .parquet(outputPath + "/" + EntityType.fromClass(clazz)); + .parquet(outputPath); } private static Dataset readPathEntity( @@ -232,11 +231,11 @@ public class CreateRelatedEntitiesJob_phase1 { * @param relationPath * @return the Dataset containing all the relationships */ - private static Dataset readPathRelation( + private static Dataset readPathRelation( SparkSession spark, final String relationPath) { log.info("Reading relations from: {}", relationPath); - return spark.read().load(relationPath).as(Encoders.bean(SortableRelation.class)); + return spark.read().load(relationPath).as(Encoders.bean(Relation.class)); } private static void removeOutputDir(SparkSession spark, String path) { diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/CreateRelatedEntitiesJob_phase2.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/CreateRelatedEntitiesJob_phase2.java index 403817019e..bfcc648a35 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/CreateRelatedEntitiesJob_phase2.java +++ b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/CreateRelatedEntitiesJob_phase2.java @@ -4,27 +4,29 @@ package eu.dnetlib.dhp.oa.provision; import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; import java.util.List; +import java.util.Objects; import java.util.Optional; +import java.util.stream.Collectors; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.spark.SparkConf; import org.apache.spark.api.java.function.FilterFunction; import org.apache.spark.api.java.function.MapFunction; +import org.apache.spark.sql.*; import org.apache.spark.sql.Dataset; -import org.apache.spark.sql.Encoders; -import org.apache.spark.sql.SaveMode; -import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.expressions.Aggregator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Lists; import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.common.HdfsSupport; -import eu.dnetlib.dhp.oa.provision.model.EntityRelEntity; -import eu.dnetlib.dhp.oa.provision.model.TypedRow; +import eu.dnetlib.dhp.oa.provision.model.JoinedEntity; +import eu.dnetlib.dhp.oa.provision.model.ProvisionModelSupport; +import eu.dnetlib.dhp.oa.provision.model.RelatedEntityWrapper; import eu.dnetlib.dhp.schema.common.ModelSupport; import eu.dnetlib.dhp.schema.oaf.*; import scala.Tuple2; @@ -59,6 +61,12 @@ public class CreateRelatedEntitiesJob_phase2 { private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private static final int MAX_EXTERNAL_ENTITIES = 50; + private static final int MAX_AUTHORS = 200; + private static final int MAX_AUTHOR_FULLNAME_LENGTH = 1000; + private static final int MAX_TITLE_LENGTH = 5000; + private static final int MAX_ABSTRACT_LENGTH = 100000; + public static void main(String[] args) throws Exception { String jsonConfiguration = IOUtils @@ -78,8 +86,8 @@ public class CreateRelatedEntitiesJob_phase2 { String inputRelatedEntitiesPath = parser.get("inputRelatedEntitiesPath"); log.info("inputRelatedEntitiesPath: {}", inputRelatedEntitiesPath); - String inputGraphRootPath = parser.get("inputGraphRootPath"); - log.info("inputGraphRootPath: {}", inputGraphRootPath); + String inputEntityPath = parser.get("inputEntityPath"); + log.info("inputEntityPath: {}", inputEntityPath); String outputPath = parser.get("outputPath"); log.info("outputPath: {}", outputPath); @@ -87,80 +95,112 @@ public class CreateRelatedEntitiesJob_phase2 { int numPartitions = Integer.parseInt(parser.get("numPartitions")); log.info("numPartitions: {}", numPartitions); + String graphTableClassName = parser.get("graphTableClassName"); + log.info("graphTableClassName: {}", graphTableClassName); + + Class entityClazz = (Class) Class.forName(graphTableClassName); + SparkConf conf = new SparkConf(); conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer"); - conf.registerKryoClasses(ModelSupport.getOafModelClasses()); + conf.registerKryoClasses(ProvisionModelSupport.getModelClasses()); runWithSparkSession( conf, isSparkSessionManaged, spark -> { removeOutputDir(spark, outputPath); - joinAllEntities( - spark, inputRelatedEntitiesPath, inputGraphRootPath, outputPath, numPartitions); + joinEntityWithRelatedEntities( + spark, inputRelatedEntitiesPath, inputEntityPath, outputPath, numPartitions, entityClazz); }); } - private static void joinAllEntities( + private static void joinEntityWithRelatedEntities( SparkSession spark, - String inputRelatedEntitiesPath, - String inputGraphRootPath, + String relatedEntitiesPath, + String entityPath, String outputPath, - int numPartitions) { + int numPartitions, + Class entityClazz) { - Dataset> entities = readAllEntities(spark, inputGraphRootPath, numPartitions); - Dataset> relsBySource = readRelatedEntities(spark, inputRelatedEntitiesPath); + Dataset> entities = readPathEntity(spark, entityPath, entityClazz); + Dataset> relatedEntities = readRelatedEntities( + spark, relatedEntitiesPath, entityClazz); + + TypedColumn aggregator = new AdjacencyListAggregator().toColumn(); entities - .joinWith(relsBySource, entities.col("_1").equalTo(relsBySource.col("_1")), "left_outer") + .joinWith(relatedEntities, entities.col("_1").equalTo(relatedEntities.col("_1")), "left_outer") + .map((MapFunction, Tuple2>, JoinedEntity>) value -> { + JoinedEntity je = new JoinedEntity(value._1()._2()); + Optional + .ofNullable(value._2()) + .map(Tuple2::_2) + .ifPresent(r -> je.getLinks().add(r)); + return je; + }, Encoders.kryo(JoinedEntity.class)) + .filter(filterEmptyEntityFn()) + .groupByKey( + (MapFunction) value -> value.getEntity().getId(), + Encoders.STRING()) + .agg(aggregator) .map( - (MapFunction, Tuple2>, EntityRelEntity>) value -> { - EntityRelEntity re = new EntityRelEntity(); - re.setEntity(value._1()._2()); - Optional related = Optional.ofNullable(value._2()).map(Tuple2::_2); - if (related.isPresent()) { - re.setRelation(related.get().getRelation()); - re.setTarget(related.get().getTarget()); - } - return re; - }, - Encoders.bean(EntityRelEntity.class)) - .repartition(numPartitions) - .filter( - (FilterFunction) value -> value.getEntity() != null - && StringUtils.isNotBlank(value.getEntity().getId())) + (MapFunction, JoinedEntity>) value -> value._2(), + Encoders.kryo(JoinedEntity.class)) + .filter(filterEmptyEntityFn()) .write() .mode(SaveMode.Overwrite) .parquet(outputPath); } - private static Dataset> readAllEntities( - SparkSession spark, String inputGraphPath, int numPartitions) { - Dataset publication = readPathEntity(spark, inputGraphPath + "/publication", Publication.class); - Dataset dataset = readPathEntity( - spark, inputGraphPath + "/dataset", eu.dnetlib.dhp.schema.oaf.Dataset.class); - Dataset other = readPathEntity( - spark, inputGraphPath + "/otherresearchproduct", OtherResearchProduct.class); - Dataset software = readPathEntity(spark, inputGraphPath + "/software", Software.class); - Dataset datasource = readPathEntity(spark, inputGraphPath + "/datasource", Datasource.class); - Dataset organization = readPathEntity(spark, inputGraphPath + "/organization", Organization.class); - Dataset project = readPathEntity(spark, inputGraphPath + "/project", Project.class); + public static class AdjacencyListAggregator extends Aggregator { + + @Override + public JoinedEntity zero() { + return new JoinedEntity(); + } + + @Override + public JoinedEntity reduce(JoinedEntity b, JoinedEntity a) { + return mergeAndGet(b, a); + } + + private JoinedEntity mergeAndGet(JoinedEntity b, JoinedEntity a) { + b + .setEntity( + Optional + .ofNullable(a.getEntity()) + .orElse( + Optional + .ofNullable(b.getEntity()) + .orElse(null))); + b.getLinks().addAll(a.getLinks()); + return b; + } + + @Override + public JoinedEntity merge(JoinedEntity b, JoinedEntity a) { + return mergeAndGet(b, a); + } + + @Override + public JoinedEntity finish(JoinedEntity j) { + return j; + } + + @Override + public Encoder bufferEncoder() { + return Encoders.kryo(JoinedEntity.class); + } + + @Override + public Encoder outputEncoder() { + return Encoders.kryo(JoinedEntity.class); + } - return publication - .union(dataset) - .union(other) - .union(software) - .union(datasource) - .union(organization) - .union(project) - .map( - (MapFunction>) value -> new Tuple2<>(value.getId(), value), - Encoders.tuple(Encoders.STRING(), Encoders.kryo(TypedRow.class))) - .repartition(numPartitions); } - private static Dataset> readRelatedEntities( - SparkSession spark, String inputRelatedEntitiesPath) { + private static Dataset> readRelatedEntities( + SparkSession spark, String inputRelatedEntitiesPath, Class entityClazz) { log.info("Reading related entities from: {}", inputRelatedEntitiesPath); @@ -169,17 +209,20 @@ public class CreateRelatedEntitiesJob_phase2 { log.info("Found paths: {}", String.join(",", paths)); + final String idPrefix = ModelSupport.getIdPrefix(entityClazz); + return spark .read() .load(toSeq(paths)) - .as(Encoders.bean(EntityRelEntity.class)) + .as(Encoders.kryo(RelatedEntityWrapper.class)) + .filter((FilterFunction) e -> e.getRelation().getSource().startsWith(idPrefix)) .map( - (MapFunction>) value -> new Tuple2<>( + (MapFunction>) value -> new Tuple2<>( value.getRelation().getSource(), value), - Encoders.tuple(Encoders.STRING(), Encoders.kryo(EntityRelEntity.class))); + Encoders.tuple(Encoders.STRING(), Encoders.kryo(RelatedEntityWrapper.class))); } - private static Dataset readPathEntity( + private static Dataset> readPathEntity( SparkSession spark, String inputEntityPath, Class entityClazz) { log.info("Reading Graph table from: {}", inputEntityPath); @@ -190,20 +233,75 @@ public class CreateRelatedEntitiesJob_phase2 { (MapFunction) value -> OBJECT_MAPPER.readValue(value, entityClazz), Encoders.bean(entityClazz)) .filter("dataInfo.invisible == false") + .map((MapFunction) e -> pruneOutliers(entityClazz, e), Encoders.bean(entityClazz)) .map( - (MapFunction) value -> getTypedRow( - StringUtils.substringAfterLast(inputEntityPath, "/"), value), - Encoders.bean(TypedRow.class)); + (MapFunction>) e -> new Tuple2<>(e.getId(), e), + Encoders.tuple(Encoders.STRING(), Encoders.kryo(entityClazz))); } - private static TypedRow getTypedRow(String type, OafEntity entity) - throws JsonProcessingException { - TypedRow t = new TypedRow(); - t.setType(type); - t.setDeleted(entity.getDataInfo().getDeletedbyinference()); - t.setId(entity.getId()); - t.setOaf(OBJECT_MAPPER.writeValueAsString(entity)); - return t; + private static E pruneOutliers(Class entityClazz, E e) { + if (ModelSupport.isSubClass(entityClazz, Result.class)) { + Result r = (Result) e; + if (r.getExternalReference() != null) { + List refs = r + .getExternalReference() + .stream() + .limit(MAX_EXTERNAL_ENTITIES) + .collect(Collectors.toList()); + r.setExternalReference(refs); + } + if (r.getAuthor() != null) { + List authors = Lists.newArrayList(); + for (Author a : r.getAuthor()) { + a.setFullname(StringUtils.left(a.getFullname(), MAX_AUTHOR_FULLNAME_LENGTH)); + if (authors.size() < MAX_AUTHORS || hasORCID(a)) { + authors.add(a); + } + } + r.setAuthor(authors); + } + if (r.getDescription() != null) { + List> desc = r + .getDescription() + .stream() + .filter(Objects::nonNull) + .map(d -> { + d.setValue(StringUtils.left(d.getValue(), MAX_ABSTRACT_LENGTH)); + return d; + }) + .collect(Collectors.toList()); + r.setDescription(desc); + } + if (r.getTitle() != null) { + List titles = r + .getTitle() + .stream() + .filter(Objects::nonNull) + .map(t -> { + t.setValue(StringUtils.left(t.getValue(), MAX_TITLE_LENGTH)); + return t; + }) + .collect(Collectors.toList()); + r.setTitle(titles); + } + } + return e; + } + + private static boolean hasORCID(Author a) { + return a.getPid() != null && a + .getPid() + .stream() + .filter(Objects::nonNull) + .map(StructuredProperty::getQualifier) + .filter(Objects::nonNull) + .map(Qualifier::getClassid) + .filter(StringUtils::isNotBlank) + .anyMatch(c -> "orcid".equals(c.toLowerCase())); + } + + private static FilterFunction filterEmptyEntityFn() { + return (FilterFunction) v -> Objects.nonNull(v.getEntity()); } private static void removeOutputDir(SparkSession spark, String path) { diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/PrepareRelationsJob.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/PrepareRelationsJob.java index 72d68a389e..eb63d4423d 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/PrepareRelationsJob.java +++ b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/PrepareRelationsJob.java @@ -3,33 +3,36 @@ package eu.dnetlib.dhp.oa.provision; import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; -import java.util.HashSet; -import java.util.Optional; -import java.util.Set; +import java.util.*; +import java.util.function.Supplier; +import java.util.stream.Collectors; import org.apache.commons.io.IOUtils; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; -import org.apache.spark.api.java.function.*; +import org.apache.spark.api.java.function.FilterFunction; +import org.apache.spark.api.java.function.FlatMapFunction; +import org.apache.spark.api.java.function.Function; +import org.apache.spark.api.java.function.MapFunction; import org.apache.spark.rdd.RDD; -import org.apache.spark.sql.Dataset; -import org.apache.spark.sql.Encoders; -import org.apache.spark.sql.SaveMode; -import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.*; +import org.apache.spark.sql.expressions.Aggregator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Splitter; import com.google.common.collect.Iterables; -import com.google.common.collect.Iterators; +import com.google.common.collect.Maps; import com.google.common.collect.Sets; import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.common.HdfsSupport; -import eu.dnetlib.dhp.oa.provision.model.SortableRelation; +import eu.dnetlib.dhp.oa.provision.model.ProvisionModelSupport; +import eu.dnetlib.dhp.oa.provision.model.SortableRelationKey; import eu.dnetlib.dhp.oa.provision.utils.RelationPartitioner; +import eu.dnetlib.dhp.schema.oaf.Relation; import scala.Tuple2; /** @@ -104,6 +107,8 @@ public class PrepareRelationsJob { log.info("maxRelations: {}", maxRelations); SparkConf conf = new SparkConf(); + conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer"); + conf.registerKryoClasses(ProvisionModelSupport.getModelClasses()); runWithSparkSession( conf, @@ -111,37 +116,10 @@ public class PrepareRelationsJob { spark -> { removeOutputDir(spark, outputPath); prepareRelationsRDD( - spark, inputRelationsPath, outputPath, relationFilter, relPartitions, maxRelations); + spark, inputRelationsPath, outputPath, relationFilter, maxRelations, relPartitions); }); } - /** - * Dataset based implementation that prepares the graph relations by limiting the number of outgoing links and - * filtering the relation types according to the given criteria. - * - * @param spark the spark session - * @param inputRelationsPath source path for the graph relations - * @param outputPath output path for the processed relations - * @param relationFilter set of relation filters applied to the `relClass` field - * @param maxRelations maximum number of allowed outgoing edges - */ - private static void prepareRelations( - SparkSession spark, String inputRelationsPath, String outputPath, Set relationFilter, - int maxRelations) { - readPathRelation(spark, inputRelationsPath) - .filter("dataInfo.deletedbyinference == false") - .filter((FilterFunction) rel -> !relationFilter.contains(rel.getRelClass())) - .groupByKey( - (MapFunction) value -> value.getSource(), Encoders.STRING()) - .flatMapGroups( - (FlatMapGroupsFunction) (key, values) -> Iterators - .limit(values, maxRelations), - Encoders.bean(SortableRelation.class)) - .write() - .mode(SaveMode.Overwrite) - .parquet(outputPath); - } - /** * RDD based implementation that prepares the graph relations by limiting the number of outgoing links and filtering * the relation types according to the given criteria. Moreover, outgoing links kept within the given limit are @@ -152,54 +130,126 @@ public class PrepareRelationsJob { * @param outputPath output path for the processed relations * @param relationFilter set of relation filters applied to the `relClass` field * @param maxRelations maximum number of allowed outgoing edges + * @param relPartitions number of partitions for the output RDD */ - // TODO work in progress - private static void prepareRelationsRDD( - SparkSession spark, String inputRelationsPath, String outputPath, Set relationFilter, int relPartitions, - int maxRelations) { - JavaRDD rels = readPathRelationRDD(spark, inputRelationsPath).repartition(relPartitions); - RelationPartitioner partitioner = new RelationPartitioner(rels.getNumPartitions()); + private static void prepareRelationsRDD(SparkSession spark, String inputRelationsPath, String outputPath, + Set relationFilter, int maxRelations, int relPartitions) { - // only consider those that are not virtually deleted - RDD d = rels - .filter(rel -> !rel.getDataInfo().getDeletedbyinference()) - .filter(rel -> !relationFilter.contains(rel.getRelClass())) - .mapToPair( - (PairFunction) rel -> new Tuple2<>(rel, rel)) - .groupByKey(partitioner) - .map(group -> Iterables.limit(group._2(), maxRelations)) - .flatMap(group -> group.iterator()) + // group by SOURCE and apply limit + RDD bySource = readPathRelationRDD(spark, inputRelationsPath) + .filter(rel -> rel.getDataInfo().getDeletedbyinference() == false) + .filter(rel -> relationFilter.contains(rel.getRelClass()) == false) + .mapToPair(r -> new Tuple2<>(SortableRelationKey.create(r, r.getSource()), r)) + .repartitionAndSortWithinPartitions(new RelationPartitioner(relPartitions)) + .groupBy(Tuple2::_1) + .map(Tuple2::_2) + .map(t -> Iterables.limit(t, maxRelations)) + .flatMap(Iterable::iterator) + .map(Tuple2::_2) .rdd(); spark - .createDataset(d, Encoders.bean(SortableRelation.class)) + .createDataset(bySource, Encoders.bean(Relation.class)) + .repartition(relPartitions) .write() .mode(SaveMode.Overwrite) .parquet(outputPath); } + // experimental + private static void prepareRelationsDataset( + SparkSession spark, String inputRelationsPath, String outputPath, Set relationFilter, int maxRelations, + int relPartitions) { + spark + .read() + .textFile(inputRelationsPath) + .repartition(relPartitions) + .map( + (MapFunction) s -> OBJECT_MAPPER.readValue(s, Relation.class), + Encoders.kryo(Relation.class)) + .filter((FilterFunction) rel -> rel.getDataInfo().getDeletedbyinference() == false) + .filter((FilterFunction) rel -> relationFilter.contains(rel.getRelClass()) == false) + .groupByKey( + (MapFunction) Relation::getSource, + Encoders.STRING()) + .agg(new RelationAggregator(maxRelations).toColumn()) + .flatMap( + (FlatMapFunction, Relation>) t -> Iterables + .limit(t._2().getRelations(), maxRelations) + .iterator(), + Encoders.bean(Relation.class)) + .repartition(relPartitions) + .write() + .mode(SaveMode.Overwrite) + .parquet(outputPath); + } + + public static class RelationAggregator + extends Aggregator { + + private int maxRelations; + + public RelationAggregator(int maxRelations) { + this.maxRelations = maxRelations; + } + + @Override + public RelationList zero() { + return new RelationList(); + } + + @Override + public RelationList reduce(RelationList b, Relation a) { + b.getRelations().add(a); + return getSortableRelationList(b); + } + + @Override + public RelationList merge(RelationList b1, RelationList b2) { + b1.getRelations().addAll(b2.getRelations()); + return getSortableRelationList(b1); + } + + @Override + public RelationList finish(RelationList r) { + return getSortableRelationList(r); + } + + private RelationList getSortableRelationList(RelationList b1) { + RelationList sr = new RelationList(); + sr + .setRelations( + b1 + .getRelations() + .stream() + .limit(maxRelations) + .collect(Collectors.toCollection(() -> new PriorityQueue<>(new RelationComparator())))); + return sr; + } + + @Override + public Encoder bufferEncoder() { + return Encoders.kryo(RelationList.class); + } + + @Override + public Encoder outputEncoder() { + return Encoders.kryo(RelationList.class); + } + } + /** - * Reads a Dataset of eu.dnetlib.dhp.oa.provision.model.SortableRelation objects from a newline delimited json text + * Reads a JavaRDD of eu.dnetlib.dhp.oa.provision.model.SortableRelation objects from a newline delimited json text * file, * * @param spark * @param inputPath - * @return the Dataset containing all the relationships + * @return the JavaRDD containing all the relationships */ - private static Dataset readPathRelation( - SparkSession spark, final String inputPath) { - return spark - .read() - .textFile(inputPath) - .map( - (MapFunction) value -> OBJECT_MAPPER.readValue(value, SortableRelation.class), - Encoders.bean(SortableRelation.class)); - } - - private static JavaRDD readPathRelationRDD( + private static JavaRDD readPathRelationRDD( SparkSession spark, final String inputPath) { JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); - return sc.textFile(inputPath).map(s -> OBJECT_MAPPER.readValue(s, SortableRelation.class)); + return sc.textFile(inputPath).map(s -> OBJECT_MAPPER.readValue(s, Relation.class)); } private static void removeOutputDir(SparkSession spark, String path) { diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/RelationComparator.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/RelationComparator.java new file mode 100644 index 0000000000..f2209c26c8 --- /dev/null +++ b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/RelationComparator.java @@ -0,0 +1,43 @@ + +package eu.dnetlib.dhp.oa.provision; + +import java.util.Comparator; +import java.util.Map; +import java.util.Optional; + +import com.google.common.collect.ComparisonChain; +import com.google.common.collect.Maps; + +import eu.dnetlib.dhp.schema.oaf.Relation; + +public class RelationComparator implements Comparator { + + private static final Map weights = Maps.newHashMap(); + + static { + weights.put("outcome", 0); + weights.put("supplement", 1); + weights.put("review", 2); + weights.put("citation", 3); + weights.put("affiliation", 4); + weights.put("relationship", 5); + weights.put("publicationDataset", 6); + weights.put("similarity", 7); + + weights.put("provision", 8); + weights.put("participation", 9); + weights.put("dedup", 10); + } + + private Integer getWeight(Relation o) { + return Optional.ofNullable(weights.get(o.getSubRelType())).orElse(Integer.MAX_VALUE); + } + + @Override + public int compare(Relation o1, Relation o2) { + return ComparisonChain + .start() + .compare(getWeight(o1), getWeight(o2)) + .result(); + } +} diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/RelationList.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/RelationList.java new file mode 100644 index 0000000000..6e5fd7dba8 --- /dev/null +++ b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/RelationList.java @@ -0,0 +1,25 @@ + +package eu.dnetlib.dhp.oa.provision; + +import java.io.Serializable; +import java.util.PriorityQueue; +import java.util.Queue; + +import eu.dnetlib.dhp.schema.oaf.Relation; + +public class RelationList implements Serializable { + + private Queue relations; + + public RelationList() { + this.relations = new PriorityQueue<>(new RelationComparator()); + } + + public Queue getRelations() { + return relations; + } + + public void setRelations(Queue relations) { + this.relations = relations; + } +} diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/SortableRelation.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/SortableRelation.java new file mode 100644 index 0000000000..8ce92a6a01 --- /dev/null +++ b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/SortableRelation.java @@ -0,0 +1,80 @@ + +package eu.dnetlib.dhp.oa.provision; + +import java.io.Serializable; +import java.util.Map; +import java.util.Optional; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.google.common.collect.ComparisonChain; +import com.google.common.collect.Maps; + +import eu.dnetlib.dhp.schema.oaf.Relation; + +public class SortableRelation extends Relation implements Comparable, Serializable { + + private static final Map weights = Maps.newHashMap(); + + static { + weights.put("outcome", 0); + weights.put("supplement", 1); + weights.put("review", 2); + weights.put("citation", 3); + weights.put("affiliation", 4); + weights.put("relationship", 5); + weights.put("publicationDataset", 6); + weights.put("similarity", 7); + + weights.put("provision", 8); + weights.put("participation", 9); + weights.put("dedup", 10); + } + + private static final long serialVersionUID = 34753984579L; + + private String groupingKey; + + public static SortableRelation create(Relation r, String groupingKey) { + SortableRelation sr = new SortableRelation(); + sr.setGroupingKey(groupingKey); + sr.setSource(r.getSource()); + sr.setTarget(r.getTarget()); + sr.setRelType(r.getRelType()); + sr.setSubRelType(r.getSubRelType()); + sr.setRelClass(r.getRelClass()); + sr.setDataInfo(r.getDataInfo()); + sr.setCollectedfrom(r.getCollectedfrom()); + sr.setLastupdatetimestamp(r.getLastupdatetimestamp()); + sr.setProperties(r.getProperties()); + sr.setValidated(r.getValidated()); + sr.setValidationDate(r.getValidationDate()); + + return sr; + } + + @JsonIgnore + public Relation asRelation() { + return this; + } + + @Override + public int compareTo(SortableRelation o) { + return ComparisonChain + .start() + .compare(getGroupingKey(), o.getGroupingKey()) + .compare(getWeight(this), getWeight(o)) + .result(); + } + + private Integer getWeight(SortableRelation o) { + return Optional.ofNullable(weights.get(o.getSubRelType())).orElse(Integer.MAX_VALUE); + } + + public String getGroupingKey() { + return groupingKey; + } + + public void setGroupingKey(String groupingKey) { + this.groupingKey = groupingKey; + } +} diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/XmlConverterJob.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/XmlConverterJob.java index a88b28592e..a1ed7fd2a6 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/XmlConverterJob.java +++ b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/XmlConverterJob.java @@ -4,6 +4,7 @@ package eu.dnetlib.dhp.oa.provision; import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; @@ -32,6 +33,8 @@ import eu.dnetlib.dhp.oa.provision.utils.ContextMapper; import eu.dnetlib.dhp.oa.provision.utils.XmlRecordFactory; import eu.dnetlib.dhp.schema.oaf.*; import scala.Tuple2; +import scala.collection.JavaConverters; +import scala.collection.Seq; /** * Joins the graph nodes by resolving the links of distance = 1 to create an adjacency list of linked objects. The @@ -89,6 +92,8 @@ public class XmlConverterJob { log.info("otherDsTypeId: {}", otherDsTypeId); SparkConf conf = new SparkConf(); + conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer"); + conf.registerKryoClasses(ProvisionModelSupport.getModelClasses()); runWithSparkSession( conf, @@ -114,26 +119,18 @@ public class XmlConverterJob { schemaLocation, otherDsTypeId); + final List paths = HdfsSupport + .listFiles(inputPath, spark.sparkContext().hadoopConfiguration()); + + log.info("Found paths: {}", String.join(",", paths)); + spark .read() - .load(inputPath) - .as(Encoders.bean(JoinedEntity.class)) + .load(toSeq(paths)) + .as(Encoders.kryo(JoinedEntity.class)) .map( - (MapFunction) j -> { - if (j.getLinks() != null) { - j - .setLinks( - j - .getLinks() - .stream() - .filter(t -> t.getRelation() != null & t.getRelatedEntity() != null) - .collect(Collectors.toCollection(ArrayList::new))); - } - return j; - }, - Encoders.bean(JoinedEntity.class)) - .map( - (MapFunction>) je -> new Tuple2<>(je.getEntity().getId(), + (MapFunction>) je -> new Tuple2<>( + je.getEntity().getId(), recordFactory.build(je)), Encoders.tuple(Encoders.STRING(), Encoders.STRING())) .javaRDD() @@ -148,6 +145,10 @@ public class XmlConverterJob { HdfsSupport.remove(path, spark.sparkContext().hadoopConfiguration()); } + private static Seq toSeq(List list) { + return JavaConverters.asScalaIteratorConverter(list.iterator()).asScala().toSeq(); + } + private static Map prepareAccumulators(SparkContext sc) { Map accumulators = Maps.newHashMap(); accumulators diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/model/EntityRelEntity.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/model/EntityRelEntity.java deleted file mode 100644 index a6b3c5591a..0000000000 --- a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/model/EntityRelEntity.java +++ /dev/null @@ -1,67 +0,0 @@ - -package eu.dnetlib.dhp.oa.provision.model; - -import java.io.Serializable; - -import com.google.common.base.Objects; - -public class EntityRelEntity implements Serializable { - - private TypedRow entity; - private SortableRelation relation; - private RelatedEntity target; - - public EntityRelEntity() { - } - - public EntityRelEntity(SortableRelation relation, RelatedEntity target) { - this(null, relation, target); - } - - public EntityRelEntity(TypedRow entity, SortableRelation relation, RelatedEntity target) { - this.entity = entity; - this.relation = relation; - this.target = target; - } - - public TypedRow getEntity() { - return entity; - } - - public void setEntity(TypedRow entity) { - this.entity = entity; - } - - public SortableRelation getRelation() { - return relation; - } - - public void setRelation(SortableRelation relation) { - this.relation = relation; - } - - public RelatedEntity getTarget() { - return target; - } - - public void setTarget(RelatedEntity target) { - this.target = target; - } - - @Override - public boolean equals(Object o) { - if (this == o) - return true; - if (o == null || getClass() != o.getClass()) - return false; - EntityRelEntity that = (EntityRelEntity) o; - return Objects.equal(entity, that.entity) - && Objects.equal(relation, that.relation) - && Objects.equal(target, that.target); - } - - @Override - public int hashCode() { - return Objects.hashCode(entity, relation, target); - } -} diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/model/JoinedEntity.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/model/JoinedEntity.java index e29ec9d192..2eb9cf38be 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/model/JoinedEntity.java +++ b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/model/JoinedEntity.java @@ -2,30 +2,40 @@ package eu.dnetlib.dhp.oa.provision.model; import java.io.Serializable; +import java.util.ArrayList; +import java.util.LinkedList; import java.util.List; -public class JoinedEntity implements Serializable { +import eu.dnetlib.dhp.schema.oaf.OafEntity; - private TypedRow entity; +public class JoinedEntity implements Serializable { - private List links; + private E entity; + + private List links; public JoinedEntity() { + links = new LinkedList<>(); } - public TypedRow getEntity() { - return entity; - } - - public void setEntity(TypedRow entity) { + public JoinedEntity(E entity) { + this(); this.entity = entity; } - public List getLinks() { + public E getEntity() { + return entity; + } + + public void setEntity(E entity) { + this.entity = entity; + } + + public List getLinks() { return links; } - public void setLinks(List links) { + public void setLinks(List links) { this.links = links; } } diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/model/ProvisionModelSupport.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/model/ProvisionModelSupport.java new file mode 100644 index 0000000000..c09ed86e54 --- /dev/null +++ b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/model/ProvisionModelSupport.java @@ -0,0 +1,28 @@ + +package eu.dnetlib.dhp.oa.provision.model; + +import java.util.List; + +import com.google.common.collect.Lists; + +import eu.dnetlib.dhp.oa.provision.RelationList; +import eu.dnetlib.dhp.oa.provision.SortableRelation; +import eu.dnetlib.dhp.schema.common.ModelSupport; + +public class ProvisionModelSupport { + + public static Class[] getModelClasses() { + List> modelClasses = Lists.newArrayList(ModelSupport.getOafModelClasses()); + modelClasses + .addAll( + Lists + .newArrayList( + RelatedEntityWrapper.class, + JoinedEntity.class, + RelatedEntity.class, + SortableRelationKey.class, + SortableRelation.class, + RelationList.class)); + return modelClasses.toArray(new Class[] {}); + } +} diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/model/RelatedEntityWrapper.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/model/RelatedEntityWrapper.java new file mode 100644 index 0000000000..4a4a4a5be5 --- /dev/null +++ b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/model/RelatedEntityWrapper.java @@ -0,0 +1,54 @@ + +package eu.dnetlib.dhp.oa.provision.model; + +import java.io.Serializable; + +import com.google.common.base.Objects; + +import eu.dnetlib.dhp.schema.oaf.Relation; + +public class RelatedEntityWrapper implements Serializable { + + private Relation relation; + private RelatedEntity target; + + public RelatedEntityWrapper() { + } + + public RelatedEntityWrapper(Relation relation, RelatedEntity target) { + this.relation = relation; + this.target = target; + } + + public Relation getRelation() { + return relation; + } + + public void setRelation(Relation relation) { + this.relation = relation; + } + + public RelatedEntity getTarget() { + return target; + } + + public void setTarget(RelatedEntity target) { + this.target = target; + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + RelatedEntityWrapper that = (RelatedEntityWrapper) o; + return Objects.equal(relation, that.relation) + && Objects.equal(target, that.target); + } + + @Override + public int hashCode() { + return Objects.hashCode(relation, target); + } +} diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/model/SortableRelation.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/model/SortableRelation.java deleted file mode 100644 index b6571b9bf9..0000000000 --- a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/model/SortableRelation.java +++ /dev/null @@ -1,38 +0,0 @@ - -package eu.dnetlib.dhp.oa.provision.model; - -import java.io.Serializable; -import java.util.Map; - -import com.google.common.collect.ComparisonChain; -import com.google.common.collect.Maps; - -import eu.dnetlib.dhp.schema.oaf.Relation; - -public class SortableRelation extends Relation implements Comparable, Serializable { - - private static final Map weights = Maps.newHashMap(); - - static { - weights.put("outcome", 0); - weights.put("supplement", 1); - weights.put("affiliation", 2); - weights.put("relationship", 3); - weights.put("publicationDataset", 4); - weights.put("similarity", 5); - - weights.put("provision", 6); - weights.put("participation", 7); - weights.put("dedup", 8); - } - - @Override - public int compareTo(Relation o) { - return ComparisonChain - .start() - .compare(weights.get(getSubRelType()), weights.get(o.getSubRelType())) - .compare(getSource(), o.getSource()) - .compare(getTarget(), o.getTarget()) - .result(); - } -} diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/model/SortableRelationKey.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/model/SortableRelationKey.java new file mode 100644 index 0000000000..bf7f9330d1 --- /dev/null +++ b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/model/SortableRelationKey.java @@ -0,0 +1,90 @@ + +package eu.dnetlib.dhp.oa.provision.model; + +import java.io.Serializable; +import java.util.Map; +import java.util.Optional; + +import com.google.common.base.Objects; +import com.google.common.collect.ComparisonChain; +import com.google.common.collect.Maps; + +import eu.dnetlib.dhp.schema.oaf.Relation; + +public class SortableRelationKey implements Comparable, Serializable { + + private static final Map weights = Maps.newHashMap(); + + static { + weights.put("outcome", 0); + weights.put("supplement", 1); + weights.put("review", 2); + weights.put("citation", 3); + weights.put("affiliation", 4); + weights.put("relationship", 5); + weights.put("publicationDataset", 6); + weights.put("similarity", 7); + + weights.put("provision", 8); + weights.put("participation", 9); + weights.put("dedup", 10); + } + + private static final long serialVersionUID = 3232323; + + private String groupingKey; + + private String subRelType; + + public static SortableRelationKey create(Relation r, String groupingKey) { + SortableRelationKey sr = new SortableRelationKey(); + sr.setGroupingKey(groupingKey); + sr.setSubRelType(r.getSubRelType()); + return sr; + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + SortableRelationKey that = (SortableRelationKey) o; + return getGroupingKey().equals(that.getGroupingKey()); + } + + @Override + public int hashCode() { + return Objects.hashCode(getGroupingKey()); + } + + @Override + public int compareTo(SortableRelationKey o) { + return ComparisonChain + .start() + .compare(getGroupingKey(), o.getGroupingKey()) + .compare(getWeight(this), getWeight(o)) + .result(); + } + + private Integer getWeight(SortableRelationKey o) { + return Optional.ofNullable(weights.get(o.getSubRelType())).orElse(Integer.MAX_VALUE); + } + + public String getSubRelType() { + return subRelType; + } + + public void setSubRelType(String subRelType) { + this.subRelType = subRelType; + } + + public String getGroupingKey() { + return groupingKey; + } + + public void setGroupingKey(String groupingKey) { + this.groupingKey = groupingKey; + } + +} diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/model/Tuple2.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/model/Tuple2.java deleted file mode 100644 index 5ebe9c9eb0..0000000000 --- a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/model/Tuple2.java +++ /dev/null @@ -1,53 +0,0 @@ - -package eu.dnetlib.dhp.oa.provision.model; - -import java.io.Serializable; -import java.util.Objects; - -import eu.dnetlib.dhp.schema.oaf.Relation; - -public class Tuple2 implements Serializable { - - private Relation relation; - - private RelatedEntity relatedEntity; - - public Tuple2() { - } - - public Tuple2(Relation relation, RelatedEntity relatedEntity) { - this.relation = relation; - this.relatedEntity = relatedEntity; - } - - public Relation getRelation() { - return relation; - } - - public void setRelation(Relation relation) { - this.relation = relation; - } - - public RelatedEntity getRelatedEntity() { - return relatedEntity; - } - - public void setRelatedEntity(RelatedEntity relatedEntity) { - this.relatedEntity = relatedEntity; - } - - @Override - public boolean equals(Object o) { - if (this == o) - return true; - if (o == null || getClass() != o.getClass()) - return false; - Tuple2 t2 = (Tuple2) o; - return getRelation().equals(t2.getRelation()); - } - - @Override - public int hashCode() { - return Objects.hash(getRelation().hashCode()); - } -} diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/model/TypedRow.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/model/TypedRow.java deleted file mode 100644 index cbec372e43..0000000000 --- a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/model/TypedRow.java +++ /dev/null @@ -1,64 +0,0 @@ - -package eu.dnetlib.dhp.oa.provision.model; - -import java.io.Serializable; - -import com.google.common.base.Objects; - -public class TypedRow implements Serializable { - - private String id; - - private Boolean deleted; - - private String type; - - private String oaf; - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public Boolean getDeleted() { - return deleted; - } - - public void setDeleted(Boolean deleted) { - this.deleted = deleted; - } - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } - - public String getOaf() { - return oaf; - } - - public void setOaf(String oaf) { - this.oaf = oaf; - } - - @Override - public boolean equals(Object o) { - if (this == o) - return true; - if (o == null || getClass() != o.getClass()) - return false; - TypedRow typedRow2 = (TypedRow) o; - return Objects.equal(id, typedRow2.id); - } - - @Override - public int hashCode() { - return Objects.hashCode(id); - } -} diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/RelationPartitioner.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/RelationPartitioner.java index a09a278372..7bd8b92171 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/RelationPartitioner.java +++ b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/RelationPartitioner.java @@ -4,7 +4,7 @@ package eu.dnetlib.dhp.oa.provision.utils; import org.apache.spark.Partitioner; import org.apache.spark.util.Utils; -import eu.dnetlib.dhp.oa.provision.model.SortableRelation; +import eu.dnetlib.dhp.oa.provision.model.SortableRelationKey; /** * Used in combination with SortableRelationKey, allows to partition the records by source id, therefore allowing to @@ -12,6 +12,8 @@ import eu.dnetlib.dhp.oa.provision.model.SortableRelation; */ public class RelationPartitioner extends Partitioner { + private static final long serialVersionUID = 343434456L; + private final int numPartitions; public RelationPartitioner(int numPartitions) { @@ -25,6 +27,18 @@ public class RelationPartitioner extends Partitioner { @Override public int getPartition(Object key) { - return Utils.nonNegativeMod(((SortableRelation) key).getSource().hashCode(), numPartitions()); + SortableRelationKey partitionKey = (SortableRelationKey) key; + return Utils.nonNegativeMod(partitionKey.getGroupingKey().hashCode(), numPartitions()); } + + @Override + public boolean equals(Object obj) { + if (obj instanceof RelationPartitioner) { + RelationPartitioner p = (RelationPartitioner) obj; + if (p.numPartitions() == numPartitions()) + return true; + } + return false; + } + } diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlRecordFactory.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlRecordFactory.java index 6f042b45cf..db9a68d3da 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlRecordFactory.java +++ b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlRecordFactory.java @@ -85,17 +85,19 @@ public class XmlRecordFactory implements Serializable { final Set contexts = Sets.newHashSet(); - final OafEntity entity = toOafEntity(je.getEntity()); + // final OafEntity entity = toOafEntity(je.getEntity()); + OafEntity entity = je.getEntity(); TemplateFactory templateFactory = new TemplateFactory(); try { - final EntityType type = EntityType.valueOf(je.getEntity().getType()); + + final EntityType type = EntityType.fromClass(entity.getClass()); final List metadata = metadata(type, entity, contexts); // rels has to be processed before the contexts because they enrich the contextMap with // the // funding info. - final List relations = je - .getLinks() + final List links = je.getLinks(); + final List relations = links .stream() .filter(link -> !isDuplicate(link)) .map(link -> mapRelation(contexts, templateFactory, type, link)) @@ -119,10 +121,6 @@ public class XmlRecordFactory implements Serializable { } } - private static OafEntity toOafEntity(TypedRow typedRow) { - return parseOaf(typedRow.getOaf(), typedRow.getType()); - } - private static OafEntity parseOaf(final String json, final String type) { try { switch (EntityType.valueOf(type)) { @@ -274,7 +272,7 @@ public class XmlRecordFactory implements Serializable { pidType, pidValue .toLowerCase() - .replaceAll("orcid", ""))); + .replaceAll("^.*orcid\\.org\\/", ""))); } } }); @@ -769,7 +767,7 @@ public class XmlRecordFactory implements Serializable { XmlSerializationUtils.asXmlElement("websiteurl", o.getWebsiteurl().getValue())); } if (o.getLogourl() != null) { - metadata.add(XmlSerializationUtils.asXmlElement("websiteurl", o.getLogourl().getValue())); + metadata.add(XmlSerializationUtils.asXmlElement("logourl", o.getLogourl().getValue())); } if (o.getEclegalbody() != null) { @@ -801,13 +799,13 @@ public class XmlRecordFactory implements Serializable { .asXmlElement( "echighereducation", o.getEchighereducation().getValue())); } - if (o.getEcinternationalorganization() != null) { + if (o.getEcinternationalorganizationeurinterests() != null) { metadata .add( XmlSerializationUtils .asXmlElement( "ecinternationalorganizationeurinterests", - o.getEcinternationalorganization().getValue())); + o.getEcinternationalorganizationeurinterests().getValue())); } if (o.getEcinternationalorganization() != null) { metadata @@ -894,6 +892,12 @@ public class XmlRecordFactory implements Serializable { if (p.getContracttype() != null) { metadata.add(XmlSerializationUtils.mapQualifier("contracttype", p.getContracttype())); } + if (p.getOamandatepublications() != null) { + metadata + .add( + XmlSerializationUtils + .asXmlElement("oamandatepublications", p.getOamandatepublications().getValue())); + } if (p.getEcsc39() != null) { metadata.add(XmlSerializationUtils.asXmlElement("ecsc39", p.getEcsc39().getValue())); } @@ -975,10 +979,10 @@ public class XmlRecordFactory implements Serializable { metadata.add(XmlSerializationUtils.mapQualifier("datasourcetypeui", dsType)); } - private List mapFields(Tuple2 link, Set contexts) { + private List mapFields(RelatedEntityWrapper link, Set contexts) { final Relation rel = link.getRelation(); - final RelatedEntity re = link.getRelatedEntity(); - final String targetType = link.getRelatedEntity().getType(); + final RelatedEntity re = link.getTarget(); + final String targetType = link.getTarget().getType(); final List metadata = Lists.newArrayList(); switch (EntityType.valueOf(targetType)) { @@ -1089,9 +1093,10 @@ public class XmlRecordFactory implements Serializable { return metadata; } - private String mapRelation(Set contexts, TemplateFactory templateFactory, EntityType type, Tuple2 link) { + private String mapRelation(Set contexts, TemplateFactory templateFactory, EntityType type, + RelatedEntityWrapper link) { final Relation rel = link.getRelation(); - final String targetType = link.getRelatedEntity().getType(); + final String targetType = link.getTarget().getType(); final String scheme = ModelSupport.getScheme(type.toString(), targetType); if (StringUtils.isBlank(scheme)) { @@ -1107,18 +1112,18 @@ public class XmlRecordFactory implements Serializable { private List listChildren( final OafEntity entity, JoinedEntity je, TemplateFactory templateFactory) { - EntityType entityType = EntityType.valueOf(je.getEntity().getType()); + final EntityType entityType = EntityType.fromClass(je.getEntity().getClass()); - List children = je - .getLinks() + final List links = je.getLinks(); + List children = links .stream() .filter(link -> isDuplicate(link)) .map(link -> { - final String targetType = link.getRelatedEntity().getType(); + final String targetType = link.getTarget().getType(); final String name = ModelSupport.getMainType(EntityType.valueOf(targetType)); final HashSet fields = Sets.newHashSet(mapFields(link, null)); return templateFactory - .getChild(name, link.getRelatedEntity().getId(), Lists.newArrayList(fields)); + .getChild(name, link.getTarget().getId(), Lists.newArrayList(fields)); }) .collect(Collectors.toCollection(ArrayList::new)); @@ -1162,10 +1167,10 @@ public class XmlRecordFactory implements Serializable { .asXmlElement( "distributionlocation", instance.getDistributionlocation())); } - if (instance.getRefereed() != null && isNotBlank(instance.getRefereed().getValue())) { + if (instance.getRefereed() != null && !instance.getRefereed().isBlank()) { fields .add( - XmlSerializationUtils.asXmlElement("refereed", instance.getRefereed().getValue())); + XmlSerializationUtils.mapQualifier("refereed", instance.getRefereed())); } if (instance.getProcessingchargeamount() != null && isNotBlank(instance.getProcessingchargeamount().getValue())) { @@ -1227,7 +1232,7 @@ public class XmlRecordFactory implements Serializable { return children; } - private boolean isDuplicate(Tuple2 link) { + private boolean isDuplicate(RelatedEntityWrapper link) { return REL_SUBTYPE_DEDUP.equalsIgnoreCase(link.getRelation().getSubRelType()); } diff --git a/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/input_params_related_entities_pahase2.json b/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/input_params_related_entities_pahase2.json index 2727f153bd..2c9f0e4f3a 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/input_params_related_entities_pahase2.json +++ b/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/input_params_related_entities_pahase2.json @@ -13,8 +13,14 @@ }, { "paramName": "iep", - "paramLongName": "inputGraphRootPath", - "paramDescription": "root graph path", + "paramLongName": "inputEntityPath", + "paramDescription": "input Entity Path", + "paramRequired": true + }, + { + "paramName": "clazz", + "paramLongName": "graphTableClassName", + "paramDescription": "class name associated to the input entity path", "paramRequired": true }, { diff --git a/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/oozie_app/workflow.xml b/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/oozie_app/workflow.xml index 6983ecf53c..faa81ad644 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/oozie_app/workflow.xml @@ -103,8 +103,7 @@ ${wf:conf('resumeFrom') eq 'prepare_relations'} ${wf:conf('resumeFrom') eq 'fork_join_related_entities'} - ${wf:conf('resumeFrom') eq 'join_all_entities'} - ${wf:conf('resumeFrom') eq 'adjancency_lists'} + ${wf:conf('resumeFrom') eq 'fork_join_all_entities'} ${wf:conf('resumeFrom') eq 'convert_to_xml'} ${wf:conf('resumeFrom') eq 'to_solr_index'} @@ -134,7 +133,9 @@ --inputRelationsPath${inputGraphRootPath}/relation --outputPath${workingDir}/relation - --relPartitions3000 + --maxRelations${maxRelations} + --relationFilter${relationFilter} + --relPartitions5000 @@ -171,7 +172,7 @@ --inputRelationsPath${workingDir}/relation --inputEntityPath${inputGraphRootPath}/publication --graphTableClassNameeu.dnetlib.dhp.schema.oaf.Publication - --outputPath${workingDir}/join_partial + --outputPath${workingDir}/join_partial/publication @@ -198,7 +199,7 @@ --inputRelationsPath${workingDir}/relation --inputEntityPath${inputGraphRootPath}/dataset --graphTableClassNameeu.dnetlib.dhp.schema.oaf.Dataset - --outputPath${workingDir}/join_partial + --outputPath${workingDir}/join_partial/dataset @@ -225,7 +226,7 @@ --inputRelationsPath${workingDir}/relation --inputEntityPath${inputGraphRootPath}/otherresearchproduct --graphTableClassNameeu.dnetlib.dhp.schema.oaf.OtherResearchProduct - --outputPath${workingDir}/join_partial + --outputPath${workingDir}/join_partial/otherresearchproduct @@ -252,7 +253,7 @@ --inputRelationsPath${workingDir}/relation --inputEntityPath${inputGraphRootPath}/software --graphTableClassNameeu.dnetlib.dhp.schema.oaf.Software - --outputPath${workingDir}/join_partial + --outputPath${workingDir}/join_partial/software @@ -279,7 +280,7 @@ --inputRelationsPath${workingDir}/relation --inputEntityPath${inputGraphRootPath}/datasource --graphTableClassNameeu.dnetlib.dhp.schema.oaf.Datasource - --outputPath${workingDir}/join_partial + --outputPath${workingDir}/join_partial/datasource @@ -306,7 +307,7 @@ --inputRelationsPath${workingDir}/relation --inputEntityPath${inputGraphRootPath}/organization --graphTableClassNameeu.dnetlib.dhp.schema.oaf.Organization - --outputPath${workingDir}/join_partial + --outputPath${workingDir}/join_partial/organization @@ -333,19 +334,57 @@ --inputRelationsPath${workingDir}/relation --inputEntityPath${inputGraphRootPath}/project --graphTableClassNameeu.dnetlib.dhp.schema.oaf.Project - --outputPath${workingDir}/join_partial + --outputPath${workingDir}/join_partial/project - + - + + + + + + + + + + + yarn cluster - Join[entities.id = relatedEntity.source] + Join[publication.id = relatedEntity.source] + eu.dnetlib.dhp.oa.provision.CreateRelatedEntitiesJob_phase2 + dhp-graph-provision-${projectVersion}.jar + + --executor-cores=${sparkExecutorCoresForJoining} + --executor-memory=${sparkExecutorMemoryForJoining} + --driver-memory=${sparkDriverMemoryForJoining} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.sql.shuffle.partitions=15360 + --conf spark.network.timeout=${sparkNetworkTimeout} + + --inputEntityPath${inputGraphRootPath}/publication + --graphTableClassNameeu.dnetlib.dhp.schema.oaf.Publication + --inputRelatedEntitiesPath${workingDir}/join_partial + --outputPath${workingDir}/join_entities/publication + --numPartitions30000 + + + + + + + + yarn + cluster + Join[dataset.id = relatedEntity.source] eu.dnetlib.dhp.oa.provision.CreateRelatedEntitiesJob_phase2 dhp-graph-provision-${projectVersion}.jar @@ -359,21 +398,22 @@ --conf spark.sql.shuffle.partitions=7680 --conf spark.network.timeout=${sparkNetworkTimeout} - --inputGraphRootPath${inputGraphRootPath} + --inputEntityPath${inputGraphRootPath}/dataset + --graphTableClassNameeu.dnetlib.dhp.schema.oaf.Dataset --inputRelatedEntitiesPath${workingDir}/join_partial - --outputPath${workingDir}/join_entities - --numPartitions12000 + --outputPath${workingDir}/join_entities/dataset + --numPartitions20000 - + - + yarn cluster - build_adjacency_lists - eu.dnetlib.dhp.oa.provision.AdjacencyListBuilderJob + Join[otherresearchproduct.id = relatedEntity.source] + eu.dnetlib.dhp.oa.provision.CreateRelatedEntitiesJob_phase2 dhp-graph-provision-${projectVersion}.jar --executor-cores=${sparkExecutorCoresForJoining} @@ -386,13 +426,130 @@ --conf spark.sql.shuffle.partitions=7680 --conf spark.network.timeout=${sparkNetworkTimeout} - --inputPath ${workingDir}/join_entities - --outputPath${workingDir}/joined + --inputEntityPath${inputGraphRootPath}/otherresearchproduct + --graphTableClassNameeu.dnetlib.dhp.schema.oaf.OtherResearchProduct + --inputRelatedEntitiesPath${workingDir}/join_partial + --outputPath${workingDir}/join_entities/otherresearchproduct + --numPartitions10000 - + + + + yarn + cluster + Join[software.id = relatedEntity.source] + eu.dnetlib.dhp.oa.provision.CreateRelatedEntitiesJob_phase2 + dhp-graph-provision-${projectVersion}.jar + + --executor-cores=${sparkExecutorCoresForJoining} + --executor-memory=${sparkExecutorMemoryForJoining} + --driver-memory=${sparkDriverMemoryForJoining} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.sql.shuffle.partitions=3840 + --conf spark.network.timeout=${sparkNetworkTimeout} + + --inputEntityPath${inputGraphRootPath}/software + --graphTableClassNameeu.dnetlib.dhp.schema.oaf.Software + --inputRelatedEntitiesPath${workingDir}/join_partial + --outputPath${workingDir}/join_entities/software + --numPartitions10000 + + + + + + + + yarn + cluster + Join[datasource.id = relatedEntity.source] + eu.dnetlib.dhp.oa.provision.CreateRelatedEntitiesJob_phase2 + dhp-graph-provision-${projectVersion}.jar + + --executor-cores=${sparkExecutorCoresForJoining} + --executor-memory=${sparkExecutorMemoryForJoining} + --driver-memory=${sparkDriverMemoryForJoining} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.sql.shuffle.partitions=7680 + --conf spark.network.timeout=${sparkNetworkTimeout} + + --inputEntityPath${inputGraphRootPath}/datasource + --graphTableClassNameeu.dnetlib.dhp.schema.oaf.Datasource + --inputRelatedEntitiesPath${workingDir}/join_partial + --outputPath${workingDir}/join_entities/datasource + --numPartitions1000 + + + + + + + + yarn + cluster + Join[organization.id = relatedEntity.source] + eu.dnetlib.dhp.oa.provision.CreateRelatedEntitiesJob_phase2 + dhp-graph-provision-${projectVersion}.jar + + --executor-cores=${sparkExecutorCoresForJoining} + --executor-memory=${sparkExecutorMemoryForJoining} + --driver-memory=${sparkDriverMemoryForJoining} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.sql.shuffle.partitions=7680 + --conf spark.network.timeout=${sparkNetworkTimeout} + + --inputEntityPath${inputGraphRootPath}/organization + --graphTableClassNameeu.dnetlib.dhp.schema.oaf.Organization + --inputRelatedEntitiesPath${workingDir}/join_partial + --outputPath${workingDir}/join_entities/organization + --numPartitions20000 + + + + + + + + yarn + cluster + Join[project.id = relatedEntity.source] + eu.dnetlib.dhp.oa.provision.CreateRelatedEntitiesJob_phase2 + dhp-graph-provision-${projectVersion}.jar + + --executor-cores=${sparkExecutorCoresForJoining} + --executor-memory=${sparkExecutorMemoryForJoining} + --driver-memory=${sparkDriverMemoryForJoining} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.sql.shuffle.partitions=3840 + --conf spark.network.timeout=${sparkNetworkTimeout} + + --inputEntityPath${inputGraphRootPath}/project + --graphTableClassNameeu.dnetlib.dhp.schema.oaf.Project + --inputRelatedEntitiesPath${workingDir}/join_partial + --outputPath${workingDir}/join_entities/project + --numPartitions10000 + + + + + + + yarn @@ -411,7 +568,7 @@ --conf spark.sql.shuffle.partitions=3840 --conf spark.network.timeout=${sparkNetworkTimeout} - --inputPath${workingDir}/joined + --inputPath${workingDir}/join_entities --outputPath${workingDir}/xml --isLookupUrl${isLookupUrl} --otherDsTypeId${otherDsTypeId} @@ -441,7 +598,7 @@ --conf spark.hadoop.mapreduce.reduce.speculative=false --inputPath${workingDir}/xml - --isLookupUrl ${isLookupUrl} + --isLookupUrl${isLookupUrl} --format${format} --batchSize${batchSize} diff --git a/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/template/child.st b/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/template/child.st index 1d3cffea09..0af39f2306 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/template/child.st +++ b/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/template/child.st @@ -1,3 +1,3 @@ <$name$$if(hasId)$ objidentifier="$id$"$else$$endif$> - $metadata:{$it$}$ + $metadata:{ it | $it$ }$ \ No newline at end of file diff --git a/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/PrepareRelationsJobTest.java b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/PrepareRelationsJobTest.java new file mode 100644 index 0000000000..528532eddd --- /dev/null +++ b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/PrepareRelationsJobTest.java @@ -0,0 +1,100 @@ + +package eu.dnetlib.dhp.oa.provision; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.apache.commons.io.FileUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.function.FilterFunction; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.SparkSession; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.dhp.oa.provision.model.ProvisionModelSupport; +import eu.dnetlib.dhp.schema.oaf.Relation; + +public class PrepareRelationsJobTest { + + private static final Logger log = LoggerFactory.getLogger(PrepareRelationsJobTest.class); + + public static final String SUBRELTYPE = "subRelType"; + public static final String OUTCOME = "outcome"; + public static final String SUPPLEMENT = "supplement"; + + private static SparkSession spark; + + private static Path workingDir; + + @BeforeAll + public static void setUp() throws IOException { + workingDir = Files.createTempDirectory(PrepareRelationsJobTest.class.getSimpleName()); + log.info("using work dir {}", workingDir); + + SparkConf conf = new SparkConf(); + + conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer"); + conf.registerKryoClasses(ProvisionModelSupport.getModelClasses()); + + spark = SparkSession + .builder() + .appName(PrepareRelationsJobTest.class.getSimpleName()) + .master("local[*]") + .config(conf) + .getOrCreate(); + } + + @AfterAll + public static void afterAll() throws IOException { + FileUtils.deleteDirectory(workingDir.toFile()); + spark.stop(); + } + + @Test + public void testRunPrepareRelationsJob(@TempDir Path testPath) throws Exception { + + final int maxRelations = 10; + PrepareRelationsJob + .main( + new String[] { + "-isSparkSessionManaged", Boolean.FALSE.toString(), + "-inputRelationsPath", getClass().getResource("relations.gz").getPath(), + "-outputPath", testPath.toString(), + "-relPartitions", "10", + "-relationFilter", "asd", + "-maxRelations", String.valueOf(maxRelations) + }); + + Dataset out = spark + .read() + .parquet(testPath.toString()) + .as(Encoders.bean(Relation.class)) + .cache(); + + Assertions.assertEquals(10, out.count()); + + Dataset freq = out + .toDF() + .cube(SUBRELTYPE) + .count() + .filter((FilterFunction) value -> !value.isNullAt(0)); + long outcome = freq.filter(freq.col(SUBRELTYPE).equalTo(OUTCOME)).collectAsList().get(0).getAs("count"); + long supplement = freq.filter(freq.col(SUBRELTYPE).equalTo(SUPPLEMENT)).collectAsList().get(0).getAs("count"); + + Assertions.assertTrue(outcome > supplement); + Assertions.assertEquals(7, outcome); + Assertions.assertEquals(3, supplement); + } + +} diff --git a/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/SortableRelationKeyTest.java b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/SortableRelationKeyTest.java new file mode 100644 index 0000000000..72f28fdf2d --- /dev/null +++ b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/SortableRelationKeyTest.java @@ -0,0 +1,42 @@ + +package eu.dnetlib.dhp.oa.provision; + +import java.io.IOException; +import java.util.List; + +import org.apache.commons.io.IOUtils; +import org.junit.jupiter.api.Test; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.dhp.oa.provision.model.SortableRelationKey; +import eu.dnetlib.dhp.schema.oaf.Relation; + +public class SortableRelationKeyTest { + + @Test + public void doTesSorting() throws IOException { + final ObjectMapper mapper = new ObjectMapper(); + final String json = IOUtils.toString(this.getClass().getResourceAsStream("relations.json")); + final List relations = mapper.readValue(json, new TypeReference>() { + }); + + relations + .stream() + .map(r -> SortableRelationKey.create(r, r.getSource())) + .sorted() + .forEach( + + it -> { + try { + System.out.println(mapper.writeValueAsString(it)); + } catch (JsonProcessingException e) { + e.printStackTrace(); + } + }); + + } + +} diff --git a/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/XmlRecordFactoryTest.java b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/XmlRecordFactoryTest.java new file mode 100644 index 0000000000..992ab26e81 --- /dev/null +++ b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/XmlRecordFactoryTest.java @@ -0,0 +1,53 @@ + +package eu.dnetlib.dhp.oa.provision; + +import static org.junit.jupiter.api.Assertions.*; + +import java.io.IOException; +import java.io.StringReader; + +import org.apache.commons.io.IOUtils; +import org.dom4j.Document; +import org.dom4j.DocumentException; +import org.dom4j.io.SAXReader; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.dhp.oa.provision.model.JoinedEntity; +import eu.dnetlib.dhp.oa.provision.utils.ContextMapper; +import eu.dnetlib.dhp.oa.provision.utils.XmlRecordFactory; + +//TODO to enable it we need to update the joined_entity.json test file +@Disabled +public class XmlRecordFactoryTest { + + private static final String otherDsTypeId = "scholarcomminfra,infospace,pubsrepository::mock,entityregistry,entityregistry::projects,entityregistry::repositories,websource"; + + @Test + public void testXMLRecordFactory() throws IOException, DocumentException { + + String json = IOUtils.toString(getClass().getResourceAsStream("joined_entity.json")); + + assertNotNull(json); + JoinedEntity je = new ObjectMapper().readValue(json, JoinedEntity.class); + assertNotNull(je); + + ContextMapper contextMapper = new ContextMapper(); + + XmlRecordFactory xmlRecordFactory = new XmlRecordFactory(contextMapper, false, XmlConverterJob.schemaLocation, + otherDsTypeId); + + String xml = xmlRecordFactory.build(je); + + assertNotNull(xml); + + Document doc = new SAXReader().read(new StringReader(xml)); + + assertNotNull(doc); + + // TODO add assertions based of values extracted from the XML record + } +} diff --git a/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/joined_entity.json b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/joined_entity.json new file mode 100644 index 0000000000..c512646988 --- /dev/null +++ b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/joined_entity.json @@ -0,0 +1,1551 @@ +{ + "links": [ + { + "relatedEntity": { + "code": null, + "codeRepositoryUrl": null, + "pid": [ + { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "sysimport:crosswalk", + "classname": "sysimport:crosswalk", + "schemename": "dnet:provenanceActions", + "schemeid": "dnet:provenanceActions" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "0.9" + }, + "qualifier": { + "classid": "pmc", + "classname": "pmc", + "schemename": "dnet:pid_types", + "schemeid": "dnet:pid_types" + }, + "value": "PMC17177" + }, + { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "sysimport:crosswalk", + "classname": "sysimport:crosswalk", + "schemename": "dnet:provenanceActions", + "schemeid": "dnet:provenanceActions" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "0.9" + }, + "qualifier": { + "classid": "pmid", + "classname": "pmid", + "schemename": "dnet:pid_types", + "schemeid": "dnet:pid_types" + }, + "value": "11005843" + } + ], + "projectTitle": null, + "websiteurl": null, + "resulttype": { + "classid": "publication", + "classname": "publication", + "schemename": "dnet:result_typologies", + "schemeid": "dnet:result_typologies" + }, + "legalname": null, + "collectedfrom": [ + { + "dataInfo": null, + "value": "PubMed Central", + "key": "10|opendoar____::eda80a3d5b344bc40f3bc04f65b7a357" + } + ], + "id": "50|od_______267::4d85ada0191a351f529d1e8ace1a7117", + "title": { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "sysimport:crosswalk", + "classname": "sysimport:crosswalk", + "schemename": "dnet:provenanceActions", + "schemeid": "dnet:provenanceActions" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "0.9" + }, + "qualifier": { + "classid": "main title", + "classname": "main title", + "schemename": "dnet:dataCite_title", + "schemeid": "dnet:dataCite_title" + }, + "value": "Induction of ribosomal genes and hepatocyte hypertrophy by adenovirus-mediated expression of c-Myc in vivo" + }, + "fundingtree": null, + "contracttype": null, + "type": "publication", + "acronym": null, + "openairecompatibility": null, + "publisher": "The National Academy of Sciences", + "instances": [ + { + "refereed": null, + "hostedby": { + "dataInfo": null, + "value": "Europe PubMed Central", + "key": "10|opendoar____::8b6dd7db9af49e67306feb59a8bdc52c" + }, + "processingchargeamount": null, + "license": null, + "processingchargecurrency": null, + "distributionlocation": "", + "url": [ + "https://europepmc.org/articles/PMC17177/" + ], + "dateofacceptance": { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "sysimport:crosswalk", + "classname": "sysimport:crosswalk", + "schemename": "dnet:provenanceActions", + "schemeid": "dnet:provenanceActions" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "0.9" + }, + "value": "2000-09-26" + }, + "collectedfrom": { + "dataInfo": null, + "value": "PubMed Central", + "key": "10|opendoar____::eda80a3d5b344bc40f3bc04f65b7a357" + }, + "accessright": { + "classid": "OPEN", + "classname": "Open Access", + "schemename": "dnet:access_modes", + "schemeid": "dnet:access_modes" + }, + "instancetype": { + "classid": "0038", + "classname": "Other literature type", + "schemename": "dnet:publication_resource", + "schemeid": "dnet:publication_resource" + } + } + ], + "legalshortname": null, + "country": null, + "dateofacceptance": "2000-09-26", + "datasourcetype": null, + "datasourcetypeui": null, + "officialname": null + }, + "relation": { + "subRelType": "dedup", + "relClass": "merges", + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "sysimport:dedup", + "classname": "sysimport:dedup", + "schemename": "dnet:provenanceActions", + "schemeid": "dnet:provenanceActions" + }, + "inferred": true, + "inferenceprovenance": "decisiontree-dedup-test", + "invisible": false, + "trust": null + }, + "target": "50|od_______267::4d85ada0191a351f529d1e8ace1a7117", + "lastupdatetimestamp": null, + "relType": "resultResult", + "source": "50|dedup_wf_001::00f53f19cfaf4dde8d316e9e71f16a10", + "collectedfrom": null, + "properties": [] + } + }, + { + "relatedEntity": { + "code": null, + "codeRepositoryUrl": null, + "pid": [ + { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "qualifier": { + "classid": "pmid", + "classname": "pmid", + "schemename": "dnet:pid_types", + "schemeid": "dnet:pid_types" + }, + "value": "11005843" + } + ], + "projectTitle": null, + "websiteurl": null, + "resulttype": { + "classid": "publication", + "classname": "publication", + "schemename": "dnet:result_typologies", + "schemeid": "dnet:result_typologies" + }, + "legalname": null, + "collectedfrom": [ + { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "scholExplorer", + "key": "10|openaire____::e034d6a11054f5ade9221ebac484e864" + } + ], + "id": "50|scholexplore::ef20f9d1cd983037b45cccce4e3f5f8a", + "title": { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "qualifier": { + "classid": "main title", + "classname": "main title", + "schemename": "dnet:dataCite_title", + "schemeid": "dnet:dataCite_title" + }, + "value": "Induction of ribosomal genes and hepatocyte hypertrophy by adenovirus-mediated expression of c-Myc in vivo." + }, + "fundingtree": null, + "contracttype": null, + "type": "publication", + "acronym": null, + "openairecompatibility": null, + "publisher": "", + "instances": [ + { + "refereed": { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "" + }, + "hostedby": { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "Unknown Repository", + "key": "10|openaire____::55045bd2a65019fd8e6741a755395c8c" + }, + "processingchargeamount": { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "" + }, + "license": { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "" + }, + "processingchargecurrency": { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "" + }, + "distributionlocation": "", + "url": [ + "https://www.ncbi.nlm.nih.gov/pubmed/11005843" + ], + "dateofacceptance": { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "" + }, + "collectedfrom": { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "scholExplorer", + "key": "10|openaire____::e034d6a11054f5ade9221ebac484e864" + }, + "accessright": { + "classid": "UNKNOWN", + "classname": "not available", + "schemename": "dnet:access_modes", + "schemeid": "dnet:access_modes" + }, + "instancetype": { + "classid": "0000", + "classname": "Unknown", + "schemename": "dnet:publication_resource", + "schemeid": "dnet:publication_resource" + } + } + ], + "legalshortname": null, + "country": null, + "dateofacceptance": "", + "datasourcetype": null, + "datasourcetypeui": null, + "officialname": null + }, + "relation": { + "subRelType": "dedup", + "relClass": "merges", + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "sysimport:dedup", + "classname": "sysimport:dedup", + "schemename": "dnet:provenanceActions", + "schemeid": "dnet:provenanceActions" + }, + "inferred": true, + "inferenceprovenance": "decisiontree-dedup-test", + "invisible": false, + "trust": null + }, + "target": "50|scholexplore::ef20f9d1cd983037b45cccce4e3f5f8a", + "lastupdatetimestamp": null, + "relType": "resultResult", + "source": "50|dedup_wf_001::00f53f19cfaf4dde8d316e9e71f16a10", + "collectedfrom": null, + "properties": [] + } + }, + { + "relatedEntity": { + "code": null, + "codeRepositoryUrl": null, + "pid": [], + "projectTitle": null, + "websiteurl": null, + "resulttype": { + "classid": "publication", + "classname": "publication", + "schemename": "dnet:result_typologies", + "schemeid": "dnet:result_typologies" + }, + "legalname": null, + "collectedfrom": [ + { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "ORCID", + "key": "10|openaire____::806360c771262b4d6770e7cdf04b5c5a" + } + ], + "id": "50|orcid_______::631fd913d925af01a94ea70aa3cec3d6", + "title": { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "qualifier": { + "classid": "main title", + "classname": "main title", + "schemename": "dnet:dataCite_title", + "schemeid": "dnet:dataCite_title" + }, + "value": "Induction of ribosomal genes and hepatocyte hypertrophy by adenovirus-mediated expression of c-Myc in vivo" + }, + "fundingtree": null, + "contracttype": null, + "type": "publication", + "acronym": null, + "openairecompatibility": null, + "publisher": "", + "instances": [ + { + "refereed": { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "" + }, + "hostedby": { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "Unknown Repository", + "key": "10|openaire____::55045bd2a65019fd8e6741a755395c8c" + }, + "processingchargeamount": { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "" + }, + "license": { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "" + }, + "processingchargecurrency": { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "" + }, + "distributionlocation": "", + "url": [], + "dateofacceptance": { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "2000-01-01" + }, + "collectedfrom": { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "ORCID", + "key": "10|openaire____::806360c771262b4d6770e7cdf04b5c5a" + }, + "accessright": { + "classid": "UNKNOWN", + "classname": "UNKNOWN", + "schemename": "dnet:access_modes", + "schemeid": "dnet:access_modes" + }, + "instancetype": { + "classid": "0001", + "classname": "Article", + "schemename": "dnet:publication_resource", + "schemeid": "dnet:publication_resource" + } + } + ], + "legalshortname": null, + "country": null, + "dateofacceptance": "2000-01-01", + "datasourcetype": null, + "datasourcetypeui": null, + "officialname": null + }, + "relation": { + "subRelType": "dedup", + "relClass": "merges", + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "sysimport:dedup", + "classname": "sysimport:dedup", + "schemename": "dnet:provenanceActions", + "schemeid": "dnet:provenanceActions" + }, + "inferred": true, + "inferenceprovenance": "decisiontree-dedup-test", + "invisible": false, + "trust": null + }, + "target": "50|orcid_______::631fd913d925af01a94ea70aa3cec3d6", + "lastupdatetimestamp": null, + "relType": "resultResult", + "source": "50|dedup_wf_001::00f53f19cfaf4dde8d316e9e71f16a10", + "collectedfrom": null, + "properties": [] + } + }, + { + "relatedEntity": { + "code": null, + "codeRepositoryUrl": null, + "pid": [ + { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "qualifier": { + "classid": "mag_id", + "classname": "Microsoft Academic Graph Identifier", + "schemename": "dnet:pid_types", + "schemeid": "dnet:pid_types" + }, + "value": "https://academic.microsoft.com/#/detail/145311948" + }, + { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "qualifier": { + "classid": "urn", + "classname": "urn", + "schemename": "dnet:pid_types", + "schemeid": "dnet:pid_types" + }, + "value": "http://en.wikipedia.org/wiki/Johns_Hopkins_University" + }, + { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "qualifier": { + "classid": "grid", + "classname": "grid", + "schemename": "dnet:pid_types", + "schemeid": "dnet:pid_types" + }, + "value": "grid.21107.35" + } + ], + "projectTitle": null, + "websiteurl": "http://www.jhu.edu/", + "resulttype": null, + "legalname": "Johns Hopkins University", + "collectedfrom": [ + { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "Microsoft Academic Graph", + "key": "10|openaire____::5f532a3fc4f1ea403f37070f59a7a53a" + }, + { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "GRID - Global Research Identifier Database", + "key": "10|openaire____::ff4a008470319a22d9cf3d14af485977" + }, + { + "dataInfo": null, + "value": "Registry of Research Data Repository", + "key": "10|openaire____::21f8a223b9925c2f87c404096080b046" + }, + { + "dataInfo": null, + "value": "Research Councils UK", + "key": "10|openaire____::ab2d3310741ea80d3b8726f651502858" + }, + { + "dataInfo": null, + "value": "CORDA - COmmon Research DAta Warehouse", + "key": "10|openaire____::b30dac7baac631f3da7c2bb18dd9891f" + }, + { + "dataInfo": null, + "value": "CORDA - COmmon Research DAta Warehouse - Horizon 2020", + "key": "10|openaire____::a55eb91348674d853191f4f4fd73d078" + }, + { + "dataInfo": null, + "value": "NSF - National Science Foundation", + "key": "10|openaire____::dd69b4a1513c9de9f46faf24048da1e8" + }, + { + "dataInfo": null, + "value": "DOAJ-Articles", + "key": "10|driver______::bee53aa31dc2cbb538c10c2b65fa5824" + }, + { + "dataInfo": null, + "value": "OpenDOAR", + "key": "10|openaire____::47ce9e9f4fad46e732cff06419ecaabb" + } + ], + "id": "20|dedup_wf_001::8c05abe4d8f889305207a845e9e31d9d", + "title": null, + "fundingtree": null, + "contracttype": null, + "type": "organization", + "acronym": null, + "openairecompatibility": null, + "publisher": null, + "instances": null, + "legalshortname": "JHU", + "country": { + "classid": "US", + "classname": "United States", + "schemename": "dnet:countries", + "schemeid": "dnet:countries" + }, + "dateofacceptance": null, + "datasourcetype": null, + "datasourcetypeui": null, + "officialname": null + }, + "relation": { + "subRelType": "affiliation", + "relClass": "hasAuthorInstitution", + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "sysimport:actionset", + "classname": "sysimport:actionset", + "schemename": "dnet:provenanceActions", + "schemeid": "dnet:provenanceActions" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "0.9" + }, + "target": "20|dedup_wf_001::8c05abe4d8f889305207a845e9e31d9d", + "lastupdatetimestamp": 0, + "relType": "resultOrganization", + "source": "50|dedup_wf_001::00f53f19cfaf4dde8d316e9e71f16a10", + "collectedfrom": [ + { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "Microsoft Academic Graph", + "key": "10|openaire____::5f532a3fc4f1ea403f37070f59a7a53a" + } + ], + "properties": [] + } + }, + { + "relatedEntity": { + "code": null, + "codeRepositoryUrl": null, + "pid": [ + { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "qualifier": { + "classid": "doi", + "classname": "doi", + "schemename": "dnet:pid_types", + "schemeid": "dnet:pid_types" + }, + "value": "10.1073/pnas.200372597" + } + ], + "projectTitle": null, + "websiteurl": null, + "resulttype": { + "classid": "publication", + "classname": "publication", + "schemename": "dnet:result_typologies", + "schemeid": "dnet:result_typologies" + }, + "legalname": null, + "collectedfrom": [ + { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "Crossref", + "key": "10|openaire____::081b82f96300b6a6e3d282bad31cb6e2" + }, + { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "Microsoft Academic Graph", + "key": "10|openaire____::5f532a3fc4f1ea403f37070f59a7a53a" + }, + { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "UnpayWall", + "key": "10|openaire____::8ac8380272269217cb09a928c8caa993" + } + ], + "id": "50|doiboost____::4317e3fa670267960efa09c1fd8339c9", + "title": { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "qualifier": { + "classid": "main title", + "classname": "main title", + "schemename": "dnet:dataCite_title", + "schemeid": "dnet:dataCite_title" + }, + "value": "Induction of ribosomal genes and hepatocyte hypertrophy by adenovirus-mediated expression of c-Myc in vivo" + }, + "fundingtree": null, + "contracttype": null, + "type": "publication", + "acronym": null, + "openairecompatibility": null, + "publisher": "", + "instances": [ + { + "refereed": { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "" + }, + "hostedby": { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "Unknown Repository", + "key": "10|openaire____::55045bd2a65019fd8e6741a755395c8c" + }, + "processingchargeamount": { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "" + }, + "license": { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "" + }, + "processingchargecurrency": { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "" + }, + "distributionlocation": "", + "url": [ + "http://www.pnas.org/content/97/21/11198.full.pdf" + ], + "dateofacceptance": { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "" + }, + "collectedfrom": { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "UnpayWall", + "key": "10|openaire____::8ac8380272269217cb09a928c8caa993" + }, + "accessright": { + "classid": "OPEN", + "classname": "Open Access", + "schemename": "dnet:access_modes", + "schemeid": "dnet:access_modes" + }, + "instancetype": { + "classid": "0001", + "classname": "Article", + "schemename": "dnet:publication_resource", + "schemeid": "dnet:publication_resource" + } + }, + { + "refereed": { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "" + }, + "hostedby": { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "Unknown Repository", + "key": "10|openaire____::55045bd2a65019fd8e6741a755395c8c" + }, + "processingchargeamount": { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "" + }, + "license": { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "" + }, + "processingchargecurrency": { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "" + }, + "distributionlocation": "", + "url": [ + "https://syndication.highwire.org/content/doi/10.1073/pnas.200372597" + ], + "dateofacceptance": { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "" + }, + "collectedfrom": { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "Crossref", + "key": "10|openaire____::081b82f96300b6a6e3d282bad31cb6e2" + }, + "accessright": { + "classid": "UNKNOWN", + "classname": "not available", + "schemename": "dnet:access_modes", + "schemeid": "dnet:access_modes" + }, + "instancetype": { + "classid": "0001", + "classname": "Article", + "schemename": "dnet:publication_resource", + "schemeid": "dnet:publication_resource" + } + }, + { + "refereed": { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "" + }, + "hostedby": { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "Unknown Repository", + "key": "10|openaire____::55045bd2a65019fd8e6741a755395c8c" + }, + "processingchargeamount": { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "" + }, + "license": { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "" + }, + "processingchargecurrency": { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "" + }, + "distributionlocation": "", + "url": [ + "https://academic.microsoft.com/#/detail/2045899555" + ], + "dateofacceptance": { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "" + }, + "collectedfrom": { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "Microsoft Academic Graph", + "key": "10|openaire____::5f532a3fc4f1ea403f37070f59a7a53a" + }, + "accessright": { + "classid": "UNKNOWN", + "classname": "not available", + "schemename": "dnet:access_modes", + "schemeid": "dnet:access_modes" + }, + "instancetype": { + "classid": "0001", + "classname": "Article", + "schemename": "dnet:publication_resource", + "schemeid": "dnet:publication_resource" + } + }, + { + "refereed": { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "" + }, + "hostedby": { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "Unknown Repository", + "key": "10|openaire____::55045bd2a65019fd8e6741a755395c8c" + }, + "processingchargeamount": { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "" + }, + "license": { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "" + }, + "processingchargecurrency": { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "" + }, + "distributionlocation": "", + "url": [ + "http://dx.doi.org/10.1073/pnas.200372597" + ], + "dateofacceptance": { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "" + }, + "collectedfrom": { + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "", + "classname": "", + "schemename": "", + "schemeid": "" + }, + "inferred": false, + "inferenceprovenance": "", + "invisible": false, + "trust": "" + }, + "value": "Crossref", + "key": "10|openaire____::081b82f96300b6a6e3d282bad31cb6e2" + }, + "accessright": { + "classid": "RESTRICTED", + "classname": "Restricted", + "schemename": "dnet:access_modes", + "schemeid": "dnet:access_modes" + }, + "instancetype": { + "classid": "0001", + "classname": "Article", + "schemename": "dnet:publication_resource", + "schemeid": "dnet:publication_resource" + } + } + ], + "legalshortname": null, + "country": null, + "dateofacceptance": "2000-9-26", + "datasourcetype": null, + "datasourcetypeui": null, + "officialname": null + }, + "relation": { + "subRelType": "dedup", + "relClass": "merges", + "dataInfo": { + "deletedbyinference": false, + "provenanceaction": { + "classid": "sysimport:dedup", + "classname": "sysimport:dedup", + "schemename": "dnet:provenanceActions", + "schemeid": "dnet:provenanceActions" + }, + "inferred": true, + "inferenceprovenance": "decisiontree-dedup-test", + "invisible": false, + "trust": null + }, + "target": "50|doiboost____::4317e3fa670267960efa09c1fd8339c9", + "lastupdatetimestamp": null, + "relType": "resultResult", + "source": "50|dedup_wf_001::00f53f19cfaf4dde8d316e9e71f16a10", + "collectedfrom": null, + "properties": [] + } + } + ], + "entity": { + "deleted": false, + "oaf": "{\"collectedfrom\":[{\"key\":\"10|opendoar____::eda80a3d5b344bc40f3bc04f65b7a357\",\"value\":\"PubMed Central\",\"dataInfo\":null},{\"key\":\"10|openaire____::e034d6a11054f5ade9221ebac484e864\",\"value\":\"scholExplorer\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}},{\"key\":\"10|openaire____::806360c771262b4d6770e7cdf04b5c5a\",\"value\":\"ORCID\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}},{\"key\":\"10|openaire____::081b82f96300b6a6e3d282bad31cb6e2\",\"value\":\"Crossref\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}},{\"key\":\"10|openaire____::5f532a3fc4f1ea403f37070f59a7a53a\",\"value\":\"Microsoft Academic Graph\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}},{\"key\":\"10|openaire____::8ac8380272269217cb09a928c8caa993\",\"value\":\"UnpayWall\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}}],\"dataInfo\":{\"invisible\":false,\"inferred\":true,\"deletedbyinference\":false,\"trust\":\"0.8\",\"inferenceprovenance\":\"decisiontree-dedup-test\",\"provenanceaction\":{\"classid\":\"sysimport:dedup\",\"classname\":\"sysimport:dedup\",\"schemeid\":\"dnet:provenanceActions\",\"schemename\":\"dnet:provenanceActions\"}},\"lastupdatetimestamp\":1589967085191,\"id\":\"50|dedup_wf_001::00f53f19cfaf4dde8d316e9e71f16a10\",\"originalId\":[\"od_______267::4d85ada0191a351f529d1e8ace1a7117\",\"38908045\",\"10.1073/pnas.200372597\"],\"pid\":[{\"value\":\"PMC17177\",\"qualifier\":{\"classid\":\"pmc\",\"classname\":\"pmc\",\"schemeid\":\"dnet:pid_types\",\"schemename\":\"dnet:pid_types\"},\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"0.9\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"sysimport:crosswalk\",\"classname\":\"sysimport:crosswalk\",\"schemeid\":\"dnet:provenanceActions\",\"schemename\":\"dnet:provenanceActions\"}}},{\"value\":\"11005843\",\"qualifier\":{\"classid\":\"pmid\",\"classname\":\"pmid\",\"schemeid\":\"dnet:pid_types\",\"schemename\":\"dnet:pid_types\"},\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"0.9\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"sysimport:crosswalk\",\"classname\":\"sysimport:crosswalk\",\"schemeid\":\"dnet:provenanceActions\",\"schemename\":\"dnet:provenanceActions\"}}},{\"value\":\"10.1073/pnas.200372597\",\"qualifier\":{\"classid\":\"doi\",\"classname\":\"doi\",\"schemeid\":\"dnet:pid_types\",\"schemename\":\"dnet:pid_types\"},\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}}],\"dateofcollection\":\"2019-07-01T18:50:57Z\",\"dateoftransformation\":\"\",\"extraInfo\":[],\"oaiprovenance\":{\"originDescription\":{\"harvestDate\":\"2020-05-10T12:23:13.896Z\",\"altered\":true,\"baseURL\":\"mongodb%3A%2F%2Fservices.openaire.eu\",\"identifier\":\"\",\"datestamp\":\"\",\"metadataNamespace\":\"\"}},\"author\":[{\"fullname\":\"S. Kim\",\"name\":\"S.\",\"surname\":\"Kim\",\"rank\":1,\"pid\":[],\"affiliation\":[]},{\"fullname\":\"Q. Li\",\"name\":\"Q.\",\"surname\":\"Li\",\"rank\":2,\"pid\":[{\"value\":\"2719402459\",\"qualifier\":{\"classid\":\"MAG Identifier\",\"classname\":\"MAG Identifier\",\"schemeid\":null,\"schemename\":null},\"dataInfo\":null}],\"affiliation\":[{\"value\":\"Johns Hopkins University\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}}]},{\"fullname\":\"C. V. Dang\",\"name\":\"C. V.\",\"surname\":\"Dang\",\"rank\":3,\"pid\":[{\"value\":\"2250822210\",\"qualifier\":{\"classid\":\"MAG Identifier\",\"classname\":\"MAG Identifier\",\"schemeid\":null,\"schemename\":null},\"dataInfo\":null},{\"value\":\"0000-0002-4031-2522\",\"qualifier\":{\"classid\":\"ORCID\",\"classname\":\"ORCID\",\"schemeid\":null,\"schemename\":null},\"dataInfo\":null}],\"affiliation\":[]},{\"fullname\":\"L. A. Lee\",\"name\":\"L. A.\",\"surname\":\"Lee\",\"rank\":4,\"pid\":[],\"affiliation\":[]}],\"resulttype\":{\"classid\":\"publication\",\"classname\":\"publication\",\"schemeid\":\"dnet:result_typologies\",\"schemename\":\"dnet:result_typologies\"},\"language\":{\"classid\":\"eng\",\"classname\":\"English\",\"schemeid\":\"dnet:languages\",\"schemename\":\"dnet:languages\"},\"country\":[],\"subject\":[{\"value\":\"Biological Sciences\",\"qualifier\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"},\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"0.9\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"sysimport:crosswalk\",\"classname\":\"sysimport:crosswalk\",\"schemeid\":\"dnet:provenanceActions\",\"schemename\":\"dnet:provenanceActions\"}}},{\"value\":\"Multidisciplinary\",\"qualifier\":{\"classid\":\"keyword\",\"classname\":\"keyword\",\"schemeid\":\"dnet:subject\",\"schemename\":\"dnet:subject\"},\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}}],\"title\":[{\"value\":\"Induction of ribosomal genes and hepatocyte hypertrophy by adenovirus-mediated expression of c-Myc in vivo\",\"qualifier\":{\"classid\":\"main title\",\"classname\":\"main title\",\"schemeid\":\"dnet:dataCite_title\",\"schemename\":\"dnet:dataCite_title\"},\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"0.9\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"sysimport:crosswalk\",\"classname\":\"sysimport:crosswalk\",\"schemeid\":\"dnet:provenanceActions\",\"schemename\":\"dnet:provenanceActions\"}}},{\"value\":\"Induction of ribosomal genes and hepatocyte hypertrophy by adenovirus-mediated expression of c-Myc in vivo.\",\"qualifier\":{\"classid\":\"main title\",\"classname\":\"main title\",\"schemeid\":\"dnet:dataCite_title\",\"schemename\":\"dnet:dataCite_title\"},\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}}],\"relevantdate\":[{\"value\":\"2018-11-13\",\"qualifier\":{\"classid\":\"dnet:date\",\"classname\":\"dnet:date\",\"schemeid\":\"dnet:date\",\"schemename\":\"dnet:date\"},\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}},{\"value\":\"2000-01-01\",\"qualifier\":{\"classid\":\"issued\",\"classname\":\"issued\",\"schemeid\":\"dnet:dataCite_date\",\"schemename\":\"dnet:dataCite_date\"},\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}},{\"value\":\"2000-9-26\",\"qualifier\":{\"classid\":\"published-online\",\"classname\":\"published-online\",\"schemeid\":\"dnet:dataCite_date\",\"schemename\":\"dnet:dataCite_date\"},\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}},{\"value\":\"2000-10-10\",\"qualifier\":{\"classid\":\"published-print\",\"classname\":\"published-print\",\"schemeid\":\"dnet:dataCite_date\",\"schemename\":\"dnet:dataCite_date\"},\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}}],\"description\":[{\"value\":\"Overexpression of c-Myc in immortalized cells increases cell\\n proliferation, inhibits cell differentiation, and promotes cell\\n transformation. Recent evidence suggests that these effects, however,\\n do not necessarily occur when c-Myc is overexpressed in primary\\n mammalian cells. We sought to determine the immediate effects of\\n transient overexpression of c-Myc in primary cells in\\n vivo by using recombinant adenovirus to overexpress human\\n MYC in mouse liver. Mice were intravenously injected\\n with adenoviruses encoding MYC (Ad/Myc), E2F-1\\n (Ad/E2F-1), or \u03b2-galactosidase (Ad/LacZ). Transgene expression\\n was detectable 4 days after injection. Expression of ectopic c-Myc was\\n immediately accompanied by enlarged and dysmorphic hepatocytes in the\\n absence of significant cell proliferation or apoptosis. These\\n findings were not present in the livers of mice injected with\\n Ad/E2F-1 or Ad/LacZ. Prominent hepatocyte nuclei and nucleoli were\\n associated with the up-regulation of large- and small-subunit ribosomal\\n and nucleolar genes, suggesting that c-Myc may induce their expression\\n to increase cell mass. Our studies support a role for c-Myc in the\\n in vivo control of vertebrate cell size and metabolism\\n independent of cell proliferation.\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"0.9\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"sysimport:crosswalk\",\"classname\":\"sysimport:crosswalk\",\"schemeid\":\"dnet:provenanceActions\",\"schemename\":\"dnet:provenanceActions\"}}}],\"dateofacceptance\":{\"value\":\"2000-01-01\",\"dataInfo\":null},\"publisher\":{\"value\":\"The National Academy of Sciences\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"0.9\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"sysimport:crosswalk\",\"classname\":\"sysimport:crosswalk\",\"schemeid\":\"dnet:provenanceActions\",\"schemename\":\"dnet:provenanceActions\"}}},\"embargoenddate\":null,\"source\":[{\"value\":\"Scopus - Elsevier\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}}],\"fulltext\":[],\"format\":[],\"contributor\":[],\"resourcetype\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"},\"coverage\":[],\"bestaccessright\":null,\"context\":[],\"externalReference\":[{\"sitename\":\"Europe PMC\",\"label\":\"\",\"url\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"membrane\\\")&sort=score\",\"description\":\"\",\"qualifier\":{\"classid\":\"url\",\"classname\":\"url\",\"schemeid\":\"dnet:externalReference_typologies\",\"schemename\":\"dnet:externalReference_typologies\"},\"refidentifier\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"membrane\\\")&sort=score\",\"query\":\"\",\"dataInfo\":null},{\"sitename\":\"Europe PMC\",\"label\":\"\",\"url\":\"http://www.uniprot.org/uniprot/?query=name:(\\\"tek\\\")\",\"description\":\"\",\"qualifier\":{\"classid\":\"url\",\"classname\":\"url\",\"schemeid\":\"dnet:externalReference_typologies\",\"schemename\":\"dnet:externalReference_typologies\"},\"refidentifier\":\"http://www.uniprot.org/uniprot/?query=name:(\\\"tek\\\")\",\"query\":\"\",\"dataInfo\":null},{\"sitename\":\"Europe PMC\",\"label\":\"\",\"url\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"ribosome biogenesis\\\")&sort=score\",\"description\":\"\",\"qualifier\":{\"classid\":\"url\",\"classname\":\"url\",\"schemeid\":\"dnet:externalReference_typologies\",\"schemename\":\"dnet:externalReference_typologies\"},\"refidentifier\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"ribosome biogenesis\\\")&sort=score\",\"query\":\"\",\"dataInfo\":null},{\"sitename\":\"Europe PMC\",\"label\":\"\",\"url\":\"http://www.uniprot.org/uniprot/?query=name:(\\\"bn51\\\")\",\"description\":\"\",\"qualifier\":{\"classid\":\"url\",\"classname\":\"url\",\"schemeid\":\"dnet:externalReference_typologies\",\"schemename\":\"dnet:externalReference_typologies\"},\"refidentifier\":\"http://www.uniprot.org/uniprot/?query=name:(\\\"bn51\\\")\",\"query\":\"\",\"dataInfo\":null},{\"sitename\":\"Europe PMC\",\"label\":\"\",\"url\":\"http://www.uniprot.org/uniprot/?query=name:(\\\"c-myc\\\")\",\"description\":\"\",\"qualifier\":{\"classid\":\"url\",\"classname\":\"url\",\"schemeid\":\"dnet:externalReference_typologies\",\"schemename\":\"dnet:externalReference_typologies\"},\"refidentifier\":\"http://www.uniprot.org/uniprot/?query=name:(\\\"c-myc\\\")\",\"query\":\"\",\"dataInfo\":null},{\"sitename\":\"Europe PMC\",\"label\":\"\",\"url\":\"http://www.uniprot.org/uniprot/?query=name:(\\\"nucleolin\\\")\",\"description\":\"\",\"qualifier\":{\"classid\":\"url\",\"classname\":\"url\",\"schemeid\":\"dnet:externalReference_typologies\",\"schemename\":\"dnet:externalReference_typologies\"},\"refidentifier\":\"http://www.uniprot.org/uniprot/?query=name:(\\\"nucleolin\\\")\",\"query\":\"\",\"dataInfo\":null},{\"sitename\":\"Europe PMC\",\"label\":\"\",\"url\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"uptake\\\")&sort=score\",\"description\":\"\",\"qualifier\":{\"classid\":\"url\",\"classname\":\"url\",\"schemeid\":\"dnet:externalReference_typologies\",\"schemename\":\"dnet:externalReference_typologies\"},\"refidentifier\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"uptake\\\")&sort=score\",\"query\":\"\",\"dataInfo\":null},{\"sitename\":\"Europe PMC\",\"label\":\"\",\"url\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"cell growth\\\")&sort=score\",\"description\":\"\",\"qualifier\":{\"classid\":\"url\",\"classname\":\"url\",\"schemeid\":\"dnet:externalReference_typologies\",\"schemename\":\"dnet:externalReference_typologies\"},\"refidentifier\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"cell growth\\\")&sort=score\",\"query\":\"\",\"dataInfo\":null},{\"sitename\":\"Europe PMC\",\"label\":\"\",\"url\":\"http://www.uniprot.org/uniprot/?query=name:(\\\"\u03b2-galactosidase\\\")\",\"description\":\"\",\"qualifier\":{\"classid\":\"url\",\"classname\":\"url\",\"schemeid\":\"dnet:externalReference_typologies\",\"schemename\":\"dnet:externalReference_typologies\"},\"refidentifier\":\"http://www.uniprot.org/uniprot/?query=name:(\\\"\u03b2-galactosidase\\\")\",\"query\":\"\",\"dataInfo\":null},{\"sitename\":\"Europe PMC\",\"label\":\"\",\"url\":\"http://www.uniprot.org/uniprot/?query=name:(\\\"estrogen receptor\\\")\",\"description\":\"\",\"qualifier\":{\"classid\":\"url\",\"classname\":\"url\",\"schemeid\":\"dnet:externalReference_typologies\",\"schemename\":\"dnet:externalReference_typologies\"},\"refidentifier\":\"http://www.uniprot.org/uniprot/?query=name:(\\\"estrogen receptor\\\")\",\"query\":\"\",\"dataInfo\":null},{\"sitename\":\"Europe PMC\",\"label\":\"\",\"url\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"cell differentiation\\\")&sort=score\",\"description\":\"\",\"qualifier\":{\"classid\":\"url\",\"classname\":\"url\",\"schemeid\":\"dnet:externalReference_typologies\",\"schemename\":\"dnet:externalReference_typologies\"},\"refidentifier\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"cell differentiation\\\")&sort=score\",\"query\":\"\",\"dataInfo\":null},{\"sitename\":\"Europe PMC\",\"label\":\"\",\"url\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"pathogenesis\\\")&sort=score\",\"description\":\"\",\"qualifier\":{\"classid\":\"url\",\"classname\":\"url\",\"schemeid\":\"dnet:externalReference_typologies\",\"schemename\":\"dnet:externalReference_typologies\"},\"refidentifier\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"pathogenesis\\\")&sort=score\",\"query\":\"\",\"dataInfo\":null},{\"sitename\":\"Europe PMC\",\"label\":\"\",\"url\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"localization\\\")&sort=score\",\"description\":\"\",\"qualifier\":{\"classid\":\"url\",\"classname\":\"url\",\"schemeid\":\"dnet:externalReference_typologies\",\"schemename\":\"dnet:externalReference_typologies\"},\"refidentifier\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"localization\\\")&sort=score\",\"query\":\"\",\"dataInfo\":null},{\"sitename\":\"Europe PMC\",\"label\":\"\",\"url\":\"http://www.uniprot.org/uniprot/?query=name:(\\\"p19arf\\\")\",\"description\":\"\",\"qualifier\":{\"classid\":\"url\",\"classname\":\"url\",\"schemeid\":\"dnet:externalReference_typologies\",\"schemename\":\"dnet:externalReference_typologies\"},\"refidentifier\":\"http://www.uniprot.org/uniprot/?query=name:(\\\"p19arf\\\")\",\"query\":\"\",\"dataInfo\":null},{\"sitename\":\"Europe PMC\",\"label\":\"\",\"url\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"glucose metabolism\\\")&sort=score\",\"description\":\"\",\"qualifier\":{\"classid\":\"url\",\"classname\":\"url\",\"schemeid\":\"dnet:externalReference_typologies\",\"schemename\":\"dnet:externalReference_typologies\"},\"refidentifier\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"glucose metabolism\\\")&sort=score\",\"query\":\"\",\"dataInfo\":null},{\"sitename\":\"Europe PMC\",\"label\":\"\",\"url\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"cell cycle regulation\\\")&sort=score\",\"description\":\"\",\"qualifier\":{\"classid\":\"url\",\"classname\":\"url\",\"schemeid\":\"dnet:externalReference_typologies\",\"schemename\":\"dnet:externalReference_typologies\"},\"refidentifier\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"cell cycle regulation\\\")&sort=score\",\"query\":\"\",\"dataInfo\":null},{\"sitename\":\"Europe PMC\",\"label\":\"\",\"url\":\"http://www.uniprot.org/uniprot/?query=name:(\\\"mrdb\\\")\",\"description\":\"\",\"qualifier\":{\"classid\":\"url\",\"classname\":\"url\",\"schemeid\":\"dnet:externalReference_typologies\",\"schemename\":\"dnet:externalReference_typologies\"},\"refidentifier\":\"http://www.uniprot.org/uniprot/?query=name:(\\\"mrdb\\\")\",\"query\":\"\",\"dataInfo\":null},{\"sitename\":\"Europe PMC\",\"label\":\"\",\"url\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"gene expression\\\")&sort=score\",\"description\":\"\",\"qualifier\":{\"classid\":\"url\",\"classname\":\"url\",\"schemeid\":\"dnet:externalReference_typologies\",\"schemename\":\"dnet:externalReference_typologies\"},\"refidentifier\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"gene expression\\\")&sort=score\",\"query\":\"\",\"dataInfo\":null},{\"sitename\":\"Europe PMC\",\"label\":\"\",\"url\":\"http://www.uniprot.org/uniprot/?query=name:(\\\"eif-2a\\\")\",\"description\":\"\",\"qualifier\":{\"classid\":\"url\",\"classname\":\"url\",\"schemeid\":\"dnet:externalReference_typologies\",\"schemename\":\"dnet:externalReference_typologies\"},\"refidentifier\":\"http://www.uniprot.org/uniprot/?query=name:(\\\"eif-2a\\\")\",\"query\":\"\",\"dataInfo\":null},{\"sitename\":\"Europe PMC\",\"label\":\"\",\"url\":\"http://www.uniprot.org/uniprot/?query=name:(\\\"cyclin d2\\\")\",\"description\":\"\",\"qualifier\":{\"classid\":\"url\",\"classname\":\"url\",\"schemeid\":\"dnet:externalReference_typologies\",\"schemename\":\"dnet:externalReference_typologies\"},\"refidentifier\":\"http://www.uniprot.org/uniprot/?query=name:(\\\"cyclin d2\\\")\",\"query\":\"\",\"dataInfo\":null},{\"sitename\":\"Europe PMC\",\"label\":\"\",\"url\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"pseudouridylation\\\")&sort=score\",\"description\":\"\",\"qualifier\":{\"classid\":\"url\",\"classname\":\"url\",\"schemeid\":\"dnet:externalReference_typologies\",\"schemename\":\"dnet:externalReference_typologies\"},\"refidentifier\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"pseudouridylation\\\")&sort=score\",\"query\":\"\",\"dataInfo\":null},{\"sitename\":\"Europe PMC\",\"label\":\"\",\"url\":\"http://www.uniprot.org/uniprot/?query=name:(\\\"e2f-1\\\")\",\"description\":\"\",\"qualifier\":{\"classid\":\"url\",\"classname\":\"url\",\"schemeid\":\"dnet:externalReference_typologies\",\"schemename\":\"dnet:externalReference_typologies\"},\"refidentifier\":\"http://www.uniprot.org/uniprot/?query=name:(\\\"e2f-1\\\")\",\"query\":\"\",\"dataInfo\":null},{\"sitename\":\"Europe PMC\",\"label\":\"\",\"url\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"keratinocyte proliferation\\\")&sort=score\",\"description\":\"\",\"qualifier\":{\"classid\":\"url\",\"classname\":\"url\",\"schemeid\":\"dnet:externalReference_typologies\",\"schemename\":\"dnet:externalReference_typologies\"},\"refidentifier\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"keratinocyte proliferation\\\")&sort=score\",\"query\":\"\",\"dataInfo\":null},{\"sitename\":\"Europe PMC\",\"label\":\"\",\"url\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"biosynthesis\\\")&sort=score\",\"description\":\"\",\"qualifier\":{\"classid\":\"url\",\"classname\":\"url\",\"schemeid\":\"dnet:externalReference_typologies\",\"schemename\":\"dnet:externalReference_typologies\"},\"refidentifier\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"biosynthesis\\\")&sort=score\",\"query\":\"\",\"dataInfo\":null},{\"sitename\":\"Europe PMC\",\"label\":\"\",\"url\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"nucleolus\\\")&sort=score\",\"description\":\"\",\"qualifier\":{\"classid\":\"url\",\"classname\":\"url\",\"schemeid\":\"dnet:externalReference_typologies\",\"schemename\":\"dnet:externalReference_typologies\"},\"refidentifier\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"nucleolus\\\")&sort=score\",\"query\":\"\",\"dataInfo\":null},{\"sitename\":\"Europe PMC\",\"label\":\"\",\"url\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"metabolism\\\")&sort=score\",\"description\":\"\",\"qualifier\":{\"classid\":\"url\",\"classname\":\"url\",\"schemeid\":\"dnet:externalReference_typologies\",\"schemename\":\"dnet:externalReference_typologies\"},\"refidentifier\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"metabolism\\\")&sort=score\",\"query\":\"\",\"dataInfo\":null},{\"sitename\":\"Europe PMC\",\"label\":\"\",\"url\":\"http://www.uniprot.org/uniprot/?query=name:(\\\"dihydroorotase\\\")\",\"description\":\"\",\"qualifier\":{\"classid\":\"url\",\"classname\":\"url\",\"schemeid\":\"dnet:externalReference_typologies\",\"schemename\":\"dnet:externalReference_typologies\"},\"refidentifier\":\"http://www.uniprot.org/uniprot/?query=name:(\\\"dihydroorotase\\\")\",\"query\":\"\",\"dataInfo\":null},{\"sitename\":\"Europe PMC\",\"label\":\"\",\"url\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"liver regeneration\\\")&sort=score\",\"description\":\"\",\"qualifier\":{\"classid\":\"url\",\"classname\":\"url\",\"schemeid\":\"dnet:externalReference_typologies\",\"schemename\":\"dnet:externalReference_typologies\"},\"refidentifier\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"liver regeneration\\\")&sort=score\",\"query\":\"\",\"dataInfo\":null},{\"sitename\":\"Europe PMC\",\"label\":\"\",\"url\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"ribosome assembly\\\")&sort=score\",\"description\":\"\",\"qualifier\":{\"classid\":\"url\",\"classname\":\"url\",\"schemeid\":\"dnet:externalReference_typologies\",\"schemename\":\"dnet:externalReference_typologies\"},\"refidentifier\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"ribosome assembly\\\")&sort=score\",\"query\":\"\",\"dataInfo\":null},{\"sitename\":\"Europe PMC\",\"label\":\"\",\"url\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"protein translation\\\")&sort=score\",\"description\":\"\",\"qualifier\":{\"classid\":\"url\",\"classname\":\"url\",\"schemeid\":\"dnet:externalReference_typologies\",\"schemename\":\"dnet:externalReference_typologies\"},\"refidentifier\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"protein translation\\\")&sort=score\",\"query\":\"\",\"dataInfo\":null},{\"sitename\":\"Europe PMC\",\"label\":\"\",\"url\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"translation initiation\\\")&sort=score\",\"description\":\"\",\"qualifier\":{\"classid\":\"url\",\"classname\":\"url\",\"schemeid\":\"dnet:externalReference_typologies\",\"schemename\":\"dnet:externalReference_typologies\"},\"refidentifier\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"translation initiation\\\")&sort=score\",\"query\":\"\",\"dataInfo\":null},{\"sitename\":\"Europe PMC\",\"label\":\"\",\"url\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"nucleus\\\")&sort=score\",\"description\":\"\",\"qualifier\":{\"classid\":\"url\",\"classname\":\"url\",\"schemeid\":\"dnet:externalReference_typologies\",\"schemename\":\"dnet:externalReference_typologies\"},\"refidentifier\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"nucleus\\\")&sort=score\",\"query\":\"\",\"dataInfo\":null},{\"sitename\":\"Europe PMC\",\"label\":\"\",\"url\":\"http://www.uniprot.org/uniprot/?query=name:(\\\"mdm2\\\")\",\"description\":\"\",\"qualifier\":{\"classid\":\"url\",\"classname\":\"url\",\"schemeid\":\"dnet:externalReference_typologies\",\"schemename\":\"dnet:externalReference_typologies\"},\"refidentifier\":\"http://www.uniprot.org/uniprot/?query=name:(\\\"mdm2\\\")\",\"query\":\"\",\"dataInfo\":null},{\"sitename\":\"Europe PMC\",\"label\":\"\",\"url\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"cell cycle\\\")&sort=score\",\"description\":\"\",\"qualifier\":{\"classid\":\"url\",\"classname\":\"url\",\"schemeid\":\"dnet:externalReference_typologies\",\"schemename\":\"dnet:externalReference_typologies\"},\"refidentifier\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"cell cycle\\\")&sort=score\",\"query\":\"\",\"dataInfo\":null},{\"sitename\":\"Europe PMC\",\"label\":\"\",\"url\":\"http://www.uniprot.org/uniprot/?query=name:(\\\"ornithine decarboxylase\\\")\",\"description\":\"\",\"qualifier\":{\"classid\":\"url\",\"classname\":\"url\",\"schemeid\":\"dnet:externalReference_typologies\",\"schemename\":\"dnet:externalReference_typologies\"},\"refidentifier\":\"http://www.uniprot.org/uniprot/?query=name:(\\\"ornithine decarboxylase\\\")\",\"query\":\"\",\"dataInfo\":null},{\"sitename\":\"Europe PMC\",\"label\":\"\",\"url\":\"https://www.targetvalidation.org\",\"description\":\"\",\"qualifier\":{\"classid\":\"url\",\"classname\":\"url\",\"schemeid\":\"dnet:externalReference_typologies\",\"schemename\":\"dnet:externalReference_typologies\"},\"refidentifier\":\"https://www.targetvalidation.org\",\"query\":\"\",\"dataInfo\":null},{\"sitename\":\"Europe PMC\",\"label\":\"\",\"url\":\"http://www.uniprot.org/uniprot/?query=name:(\\\"mfl\\\")\",\"description\":\"\",\"qualifier\":{\"classid\":\"url\",\"classname\":\"url\",\"schemeid\":\"dnet:externalReference_typologies\",\"schemename\":\"dnet:externalReference_typologies\"},\"refidentifier\":\"http://www.uniprot.org/uniprot/?query=name:(\\\"mfl\\\")\",\"query\":\"\",\"dataInfo\":null},{\"sitename\":\"Europe PMC\",\"label\":\"\",\"url\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"cell proliferation\\\")&sort=score\",\"description\":\"\",\"qualifier\":{\"classid\":\"url\",\"classname\":\"url\",\"schemeid\":\"dnet:externalReference_typologies\",\"schemename\":\"dnet:externalReference_typologies\"},\"refidentifier\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"cell proliferation\\\")&sort=score\",\"query\":\"\",\"dataInfo\":null},{\"sitename\":\"Europe PMC\",\"label\":\"\",\"url\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"chromatin\\\")&sort=score\",\"description\":\"\",\"qualifier\":{\"classid\":\"url\",\"classname\":\"url\",\"schemeid\":\"dnet:externalReference_typologies\",\"schemename\":\"dnet:externalReference_typologies\"},\"refidentifier\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"chromatin\\\")&sort=score\",\"query\":\"\",\"dataInfo\":null},{\"sitename\":\"Europe PMC\",\"label\":\"\",\"url\":\"http://www.uniprot.org/uniprot/?query=name:(\\\"eif-4e\\\")\",\"description\":\"\",\"qualifier\":{\"classid\":\"url\",\"classname\":\"url\",\"schemeid\":\"dnet:externalReference_typologies\",\"schemename\":\"dnet:externalReference_typologies\"},\"refidentifier\":\"http://www.uniprot.org/uniprot/?query=name:(\\\"eif-4e\\\")\",\"query\":\"\",\"dataInfo\":null},{\"sitename\":\"Europe PMC\",\"label\":\"\",\"url\":\"http://www.uniprot.org/uniprot/?query=name:(\\\"e2a\\\")\",\"description\":\"\",\"qualifier\":{\"classid\":\"url\",\"classname\":\"url\",\"schemeid\":\"dnet:externalReference_typologies\",\"schemename\":\"dnet:externalReference_typologies\"},\"refidentifier\":\"http://www.uniprot.org/uniprot/?query=name:(\\\"e2a\\\")\",\"query\":\"\",\"dataInfo\":null},{\"sitename\":\"Europe PMC\",\"label\":\"\",\"url\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"cell development\\\")&sort=score\",\"description\":\"\",\"qualifier\":{\"classid\":\"url\",\"classname\":\"url\",\"schemeid\":\"dnet:externalReference_typologies\",\"schemename\":\"dnet:externalReference_typologies\"},\"refidentifier\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"cell development\\\")&sort=score\",\"query\":\"\",\"dataInfo\":null},{\"sitename\":\"Europe PMC\",\"label\":\"\",\"url\":\"http://www.uniprot.org/uniprot/?query=name:(\\\"myc\\\")\",\"description\":\"\",\"qualifier\":{\"classid\":\"url\",\"classname\":\"url\",\"schemeid\":\"dnet:externalReference_typologies\",\"schemename\":\"dnet:externalReference_typologies\"},\"refidentifier\":\"http://www.uniprot.org/uniprot/?query=name:(\\\"myc\\\")\",\"query\":\"\",\"dataInfo\":null},{\"sitename\":\"Europe PMC\",\"label\":\"\",\"url\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"cell-cycle phase\\\")&sort=score\",\"description\":\"\",\"qualifier\":{\"classid\":\"url\",\"classname\":\"url\",\"schemeid\":\"dnet:externalReference_typologies\",\"schemename\":\"dnet:externalReference_typologies\"},\"refidentifier\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"cell-cycle phase\\\")&sort=score\",\"query\":\"\",\"dataInfo\":null},{\"sitename\":\"Europe PMC\",\"label\":\"\",\"url\":\"http://www.uniprot.org/uniprot/?query=name:(\\\"cyclins d1\\\")\",\"description\":\"\",\"qualifier\":{\"classid\":\"url\",\"classname\":\"url\",\"schemeid\":\"dnet:externalReference_typologies\",\"schemename\":\"dnet:externalReference_typologies\"},\"refidentifier\":\"http://www.uniprot.org/uniprot/?query=name:(\\\"cyclins d1\\\")\",\"query\":\"\",\"dataInfo\":null},{\"sitename\":\"Europe PMC\",\"label\":\"\",\"url\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"ribosomal subunit\\\")&sort=score\",\"description\":\"\",\"qualifier\":{\"classid\":\"url\",\"classname\":\"url\",\"schemeid\":\"dnet:externalReference_typologies\",\"schemename\":\"dnet:externalReference_typologies\"},\"refidentifier\":\"http://www.uniprot.org/uniprot/?query=go:(\\\"ribosomal subunit\\\")&sort=score\",\"query\":\"\",\"dataInfo\":null}],\"instance\":[{\"license\":null,\"accessright\":{\"classid\":\"OPEN\",\"classname\":\"Open Access\",\"schemeid\":\"dnet:access_modes\",\"schemename\":\"dnet:access_modes\"},\"instancetype\":{\"classid\":\"0038\",\"classname\":\"Other literature type\",\"schemeid\":\"dnet:publication_resource\",\"schemename\":\"dnet:publication_resource\"},\"hostedby\":{\"key\":\"10|opendoar____::8b6dd7db9af49e67306feb59a8bdc52c\",\"value\":\"Europe PubMed Central\",\"dataInfo\":null},\"url\":[\"https://europepmc.org/articles/PMC17177/\"],\"distributionlocation\":\"\",\"collectedfrom\":{\"key\":\"10|opendoar____::eda80a3d5b344bc40f3bc04f65b7a357\",\"value\":\"PubMed Central\",\"dataInfo\":null},\"dateofacceptance\":{\"value\":\"2000-09-26\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"0.9\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"sysimport:crosswalk\",\"classname\":\"sysimport:crosswalk\",\"schemeid\":\"dnet:provenanceActions\",\"schemename\":\"dnet:provenanceActions\"}}},\"processingchargeamount\":null,\"processingchargecurrency\":null,\"refereed\":null},{\"license\":{\"value\":\"\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}},\"accessright\":{\"classid\":\"UNKNOWN\",\"classname\":\"not available\",\"schemeid\":\"dnet:access_modes\",\"schemename\":\"dnet:access_modes\"},\"instancetype\":{\"classid\":\"0000\",\"classname\":\"Unknown\",\"schemeid\":\"dnet:publication_resource\",\"schemename\":\"dnet:publication_resource\"},\"hostedby\":{\"key\":\"10|openaire____::55045bd2a65019fd8e6741a755395c8c\",\"value\":\"Unknown Repository\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}},\"url\":[\"https://www.ncbi.nlm.nih.gov/pubmed/11005843\"],\"distributionlocation\":\"\",\"collectedfrom\":{\"key\":\"10|openaire____::e034d6a11054f5ade9221ebac484e864\",\"value\":\"scholExplorer\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}},\"dateofacceptance\":{\"value\":\"\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}},\"processingchargeamount\":{\"value\":\"\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}},\"processingchargecurrency\":{\"value\":\"\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}},\"refereed\":{\"value\":\"\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}}},{\"license\":{\"value\":\"\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}},\"accessright\":{\"classid\":\"UNKNOWN\",\"classname\":\"UNKNOWN\",\"schemeid\":\"dnet:access_modes\",\"schemename\":\"dnet:access_modes\"},\"instancetype\":{\"classid\":\"0001\",\"classname\":\"Article\",\"schemeid\":\"dnet:publication_resource\",\"schemename\":\"dnet:publication_resource\"},\"hostedby\":{\"key\":\"10|openaire____::55045bd2a65019fd8e6741a755395c8c\",\"value\":\"Unknown Repository\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}},\"url\":[],\"distributionlocation\":\"\",\"collectedfrom\":{\"key\":\"10|openaire____::806360c771262b4d6770e7cdf04b5c5a\",\"value\":\"ORCID\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}},\"dateofacceptance\":{\"value\":\"2000-01-01\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}},\"processingchargeamount\":{\"value\":\"\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}},\"processingchargecurrency\":{\"value\":\"\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}},\"refereed\":{\"value\":\"\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}}},{\"license\":{\"value\":\"\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}},\"accessright\":{\"classid\":\"OPEN\",\"classname\":\"Open Access\",\"schemeid\":\"dnet:access_modes\",\"schemename\":\"dnet:access_modes\"},\"instancetype\":{\"classid\":\"0001\",\"classname\":\"Article\",\"schemeid\":\"dnet:publication_resource\",\"schemename\":\"dnet:publication_resource\"},\"hostedby\":{\"key\":\"10|openaire____::55045bd2a65019fd8e6741a755395c8c\",\"value\":\"Unknown Repository\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}},\"url\":[\"http://www.pnas.org/content/97/21/11198.full.pdf\"],\"distributionlocation\":\"\",\"collectedfrom\":{\"key\":\"10|openaire____::8ac8380272269217cb09a928c8caa993\",\"value\":\"UnpayWall\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}},\"dateofacceptance\":{\"value\":\"\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}},\"processingchargeamount\":{\"value\":\"\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}},\"processingchargecurrency\":{\"value\":\"\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}},\"refereed\":{\"value\":\"\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}}},{\"license\":{\"value\":\"\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}},\"accessright\":{\"classid\":\"UNKNOWN\",\"classname\":\"not available\",\"schemeid\":\"dnet:access_modes\",\"schemename\":\"dnet:access_modes\"},\"instancetype\":{\"classid\":\"0001\",\"classname\":\"Article\",\"schemeid\":\"dnet:publication_resource\",\"schemename\":\"dnet:publication_resource\"},\"hostedby\":{\"key\":\"10|openaire____::55045bd2a65019fd8e6741a755395c8c\",\"value\":\"Unknown Repository\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}},\"url\":[\"https://syndication.highwire.org/content/doi/10.1073/pnas.200372597\"],\"distributionlocation\":\"\",\"collectedfrom\":{\"key\":\"10|openaire____::081b82f96300b6a6e3d282bad31cb6e2\",\"value\":\"Crossref\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}},\"dateofacceptance\":{\"value\":\"\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}},\"processingchargeamount\":{\"value\":\"\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}},\"processingchargecurrency\":{\"value\":\"\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}},\"refereed\":{\"value\":\"\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}}},{\"license\":{\"value\":\"\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}},\"accessright\":{\"classid\":\"UNKNOWN\",\"classname\":\"not available\",\"schemeid\":\"dnet:access_modes\",\"schemename\":\"dnet:access_modes\"},\"instancetype\":{\"classid\":\"0001\",\"classname\":\"Article\",\"schemeid\":\"dnet:publication_resource\",\"schemename\":\"dnet:publication_resource\"},\"hostedby\":{\"key\":\"10|openaire____::55045bd2a65019fd8e6741a755395c8c\",\"value\":\"Unknown Repository\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}},\"url\":[\"https://academic.microsoft.com/#/detail/2045899555\"],\"distributionlocation\":\"\",\"collectedfrom\":{\"key\":\"10|openaire____::5f532a3fc4f1ea403f37070f59a7a53a\",\"value\":\"Microsoft Academic Graph\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}},\"dateofacceptance\":{\"value\":\"\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}},\"processingchargeamount\":{\"value\":\"\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}},\"processingchargecurrency\":{\"value\":\"\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}},\"refereed\":{\"value\":\"\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}}},{\"license\":{\"value\":\"\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}},\"accessright\":{\"classid\":\"RESTRICTED\",\"classname\":\"Restricted\",\"schemeid\":\"dnet:access_modes\",\"schemename\":\"dnet:access_modes\"},\"instancetype\":{\"classid\":\"0001\",\"classname\":\"Article\",\"schemeid\":\"dnet:publication_resource\",\"schemename\":\"dnet:publication_resource\"},\"hostedby\":{\"key\":\"10|openaire____::55045bd2a65019fd8e6741a755395c8c\",\"value\":\"Unknown Repository\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}},\"url\":[\"http://dx.doi.org/10.1073/pnas.200372597\"],\"distributionlocation\":\"\",\"collectedfrom\":{\"key\":\"10|openaire____::081b82f96300b6a6e3d282bad31cb6e2\",\"value\":\"Crossref\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}},\"dateofacceptance\":{\"value\":\"\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}},\"processingchargeamount\":{\"value\":\"\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}},\"processingchargecurrency\":{\"value\":\"\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}},\"refereed\":{\"value\":\"\",\"dataInfo\":{\"invisible\":false,\"inferred\":false,\"deletedbyinference\":false,\"trust\":\"\",\"inferenceprovenance\":\"\",\"provenanceaction\":{\"classid\":\"\",\"classname\":\"\",\"schemeid\":\"\",\"schemename\":\"\"}}}}],\"journal\":null}", + "type": "publication", + "id": "50|dedup_wf_001::00f53f19cfaf4dde8d316e9e71f16a10" + } +} \ No newline at end of file diff --git a/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/relations.gz b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/relations.gz new file mode 100644 index 0000000000..13bc01c8c3 Binary files /dev/null and b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/relations.gz differ diff --git a/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/relations.json b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/relations.json new file mode 100644 index 0000000000..3280d0d612 --- /dev/null +++ b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/relations.json @@ -0,0 +1,90 @@ +[ + { + "collectedfrom": [], + "dataInfo": { + "deletedbyinference": false, + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:entityregistry", + "classname": "Harvested", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "lastupdatetimestamp": 1592688952862, + "properties": [], + "relClass": "hasAuthorInstitution", + "relType": "resultOrganization", + "source": "1", + "subRelType": "affiliation", + "target": "2" + }, + { + "collectedfrom": [], + "dataInfo": { + "deletedbyinference": false, + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:entityregistry", + "classname": "Harvested", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "lastupdatetimestamp": 1592688952862, + "properties": [], + "relClass": "isAuthorInstitutionOf", + "relType": "resultOrganization", + "source": "2", + "subRelType": "affiliation", + "target": "1" + }, + { + "collectedfrom": [], + "dataInfo": { + "deletedbyinference": false, + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:entityregistry", + "classname": "Harvested", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "lastupdatetimestamp": 1592688952862, + "properties": [], + "relClass": "isProducedBy", + "relType": "resultProject", + "source": "1", + "subRelType": "outcome", + "target": "2" + }, + { + "collectedfrom": [], + "dataInfo": { + "deletedbyinference": false, + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:entityregistry", + "classname": "Harvested", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "lastupdatetimestamp": 1592688952862, + "properties": [], + "relClass": "produces", + "relType": "resultProject", + "source": "2", + "subRelType": "outcome", + "target": "1" + } +] \ No newline at end of file diff --git a/dhp-workflows/dhp-graph-provision/src/test/resources/log4j.properties b/dhp-workflows/dhp-graph-provision/src/test/resources/log4j.properties new file mode 100644 index 0000000000..20f56e38dd --- /dev/null +++ b/dhp-workflows/dhp-graph-provision/src/test/resources/log4j.properties @@ -0,0 +1,11 @@ +# Set root logger level to DEBUG and its only appender to A1. +log4j.rootLogger=INFO, A1 + +# A1 is set to be a ConsoleAppender. +log4j.appender.A1=org.apache.log4j.ConsoleAppender + +# A1 uses PatternLayout. +log4j.logger.org = ERROR +log4j.logger.eu.dnetlib = DEBUG +log4j.appender.A1.layout=org.apache.log4j.PatternLayout +log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n \ No newline at end of file diff --git a/dhp-workflows/dhp-stats-update/pom.xml b/dhp-workflows/dhp-stats-update/pom.xml index d6ec4e6ab2..52f35ff07a 100644 --- a/dhp-workflows/dhp-stats-update/pom.xml +++ b/dhp-workflows/dhp-stats-update/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.2-SNAPSHOT + 1.2.4-SNAPSHOT 4.0.0 dhp-stats-update diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/workflow.xml b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/workflow.xml index 6f63893621..efc301573b 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/workflow.xml @@ -21,6 +21,10 @@ hiveJdbcUrl hive server jdbc url + + hive_timeout + the time period, in seconds, after which Hive fails a transaction if a Hive client has not sent a hearbeat. The default value is 300 seconds. + @@ -31,6 +35,10 @@ hive.metastore.uris ${hiveMetastoreUris} + + hive.txn.timeout + ${hive_timeout} + diff --git a/dhp-workflows/dhp-worfklow-profiles/pom.xml b/dhp-workflows/dhp-worfklow-profiles/pom.xml index cb20db57eb..34996a0218 100644 --- a/dhp-workflows/dhp-worfklow-profiles/pom.xml +++ b/dhp-workflows/dhp-worfklow-profiles/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.2-SNAPSHOT + 1.2.4-SNAPSHOT 4.0.0 diff --git a/dhp-workflows/dhp-worfklow-profiles/src/main/resources/eu/dnetlib/dhp/wf/profiles/graph_construction.xml b/dhp-workflows/dhp-worfklow-profiles/src/main/resources/eu/dnetlib/dhp/wf/profiles/graph_construction.xml new file mode 100644 index 0000000000..4d77883b49 --- /dev/null +++ b/dhp-workflows/dhp-worfklow-profiles/src/main/resources/eu/dnetlib/dhp/wf/profiles/graph_construction.xml @@ -0,0 +1,665 @@ + +
+ + + + + +
+ + Graph Construction [OCEAN] + Data Provision + 30 + + + + reuse cached content from the aggregation system + + reuseContent + true + + + + + + + + set the aggregator content path + + contentPath + /tmp/beta_aggregator + + + + + + + + Set the path containing the AGGREGATOR graph + + aggregatorGraphPath + /tmp/beta_provision/graph/00_graph_aggregator + + + + + + + Set the target path to store the RAW graph + + rawGraphPath + /tmp/beta_provision/graph/01_graph_raw + + + + + + + Set the target path to store the DEDUPED graph + + dedupGraphPath + /tmp/beta_provision/graph/02_graph_dedup + + + + + + + Set the target path to store the INFERRED graph + + inferredGraphPath + /tmp/beta_provision/graph/03_graph_inferred + + + + + + + Set the target path to store the CONSISTENCY graph + + consistentGraphPath + /tmp/beta_provision/graph/04_graph_consistent + + + + + + + + Set the target path to store the ORCID enriched graph + + orcidGraphPath + /tmp/beta_provision/graph/05_graph_orcid + + + + + + + + Set the target path to store the BULK TAGGED graph + + bulkTaggingGraphPath + /tmp/beta_provision/graph/06_graph_bulktagging + + + + + + + + Set the target path to store the AFFILIATION from INSTITUTIONAL REPOS graph + + affiliationGraphPath + /tmp/beta_provision/graph/07_graph_affiliation + + + + + + + + Set the target path to store the COMMUNITY from SELECTED SOURCES graph + + communityOrganizationGraphPath + /tmp/beta_provision/graph/08_graph_comunity_organization + + + + + + + + Set the target path to store the FUNDING from SEMANTIC RELATION graph + + fundingGraphPath + /tmp/beta_provision/graph/09_graph_funding + + + + + + + + Set the target path to store the COMMUNITY from SEMANTIC RELATION graph + + communitySemRelGraphPath + /tmp/beta_provision/graph/10_graph_comunity_sem_rel + + + + + + + + Set the target path to store the COUNTRY enriched graph + + countryGraphPath + /tmp/beta_provision/graph/11_graph_country + + + + + + + + Set the target path to store the CLEANED graph + + cleanedGraphPath + /tmp/beta_provision/graph/12_graph_cleaned + + + + + + + + Set the target path to store the blacklisted graph + + blacklistedGraphPath + /tmp/beta_provision/graph/13_graph_blacklisted + + + + + + + + Set the lookup address + + isLookUpUrl + http://beta.services.openaire.eu:8280/is/services/isLookUp?wsdl + + + + + + + + Set the map of paths for the Bulk Tagging + + bulkTaggingPathMap + {"author" : "$['author'][*]['fullname']", "title" : "$['title'][*]['value']", "orcid" : "$['author'][*]['pid'][*][?(@['key']=='ORCID')]['value']", "contributor" : "$['contributor'][*]['value']", "description" : "$['description'][*]['value']"} + + + + + + + + Set the map of associations organization, community list for the propagation of community to result through organization + + propagationOrganizationCommunityMap + {"20|corda__h2020::3fb05a9524c3f790391261347852f638":["mes","euromarine"], "20|corda__h2020::e8dbe14cca9bf6fce09d468872f813f8":["mes","euromarine"], "20|snsf________::9b253f265e3bef5cae6d881fdf61aceb":["mes","euromarine"],"20|rcuk________::e054eea0a47665af8c3656b5785ccf76":["mes","euromarine"],"20|corda__h2020::edc18d67c9b11fb616ca9f6e1db1b151":["mes","euromarine"],"20|rcuk________::d5736d9da90521ddcdc7828a05a85e9a":["mes","euromarine"],"20|corda__h2020::f5d418d3aa1cf817ddefcc3fdc039f27":["mes","euromarine"],"20|snsf________::8fa091f8f25a846779acb4ea97b50aef":["mes","euromarine"],"20|corda__h2020::81e020977211c2c40fae2e1a50bffd71":["mes","euromarine"],"20|corda_______::81e020977211c2c40fae2e1a50bffd71":["mes","euromarine"],"20|snsf________::31d0a100e54e3cdb3c6f52d91e638c78":["mes","euromarine"],"20|corda__h2020::ea379ef91b8cc86f9ac5edc4169292db":["mes","euromarine"],"20|corda__h2020::f75ee2ee48e5cb0ec8c8d30aaa8fef70":["mes","euromarine"],"20|rcuk________::e16010089551a1a9182a94604fc0ea59":["mes","euromarine"],"20|corda__h2020::38531a2cce7c5c347ffc439b07c1f43b":["mes","euromarine"],"20|corda_______::38531a2cce7c5c347ffc439b07c1f43b":["mes","euromarine"],"20|grid________::b2cbbf5eadbbf87d534b022bad3191d7":["mes","euromarine"],"20|snsf________::74730ef1439d7f7636a8be58a6b471b8":["mes","euromarine"],"20|nsf_________::ad72e19043a5a467e35f9b444d11563e":["mes","euromarine"],"20|rcuk________::0fc3e92500290902a2d38ec2445e74c3":["mes","euromarine"],"20|grid________::ad2c29905da0eb3c06b3fa80cacd89ea":["mes","euromarine"],"20|corda__h2020::30b53e4d63d3724f00acb9cbaca40860":["mes","euromarine"],"20|corda__h2020::f60f84bee14ad93f0db0e49af1d5c317":["mes","euromarine"], "20|corda__h2020::7bf251ac3765b5e89d82270a1763d09f":["mes","euromarine"], "20|corda__h2020::65531bd11be9935948c7f2f4db1c1832":["mes","euromarine"], "20|corda__h2020::e0e98f86bbc76638bbb72a8fe2302946":["mes","euromarine"], "20|snsf________::3eb43582ac27601459a8d8b3e195724b":["mes","euromarine"], "20|corda__h2020::af2481dab65d06c8ea0ae02b5517b9b6":["mes","euromarine"], "20|corda__h2020::c19d05cfde69a50d3ebc89bd0ee49929":["mes","euromarine"], "20|corda__h2020::af0bfd9fc09f80d9488f56d71a9832f0":["mes","euromarine"], "20|rcuk________::f33c02afb0dc66c49d0ed97ca5dd5cb0":["beopen"], + "20|grid________::a867f78acdc5041b34acfe4f9a349157":["beopen"], "20|grid________::7bb116a1a9f95ab812bf9d2dea2be1ff":["beopen"], "20|corda__h2020::6ab0e0739dbe625b99a2ae45842164ad":["beopen"], "20|corda__h2020::8ba50792bc5f4d51d79fca47d860c602":["beopen"], "20|corda_______::8ba50792bc5f4d51d79fca47d860c602":["beopen"], "20|corda__h2020::e70e9114979e963eef24666657b807c3":["beopen"], "20|corda_______::e70e9114979e963eef24666657b807c3":["beopen"], "20|corda_______::15911e01e9744d57205825d77c218737":["beopen"], "20|opendoar____::056a41e24e2a9a67215e87bbee6a80ab":["beopen"], "20|opendoar____::7f67f2e6c6fbb0628f8160fcd3d92ae3":["beopen"], "20|grid________::a8ecfd7c084e561168bcbe6bf0daf3e3":["beopen"], "20|corda_______::7bbe6cc5d8ec1864739a04b0d020c9e9":["beopen"], "20|corda_______::3ff558e30c2e434d688539548300b050":["beopen"], "20|corda__h2020::5ffee5b3b83b33a8cf0e046877bd3a39":["beopen"], "20|corda__h2020::5187217e2e806a6df3579c46f82401bc":["beopen"], "20|grid________::5fa7e2709bcd945e26bfa18689adeec1":["beopen"], "20|corda_______::d8696683c53027438031a96ad27c3c07":["beopen"], "20|corda__h2020::d8696683c53027438031a96ad27c3c07":["beopen"], "20|rcuk________::23a79ebdfa59790864e4a485881568c1":["beopen"], "20|corda__h2020::b76cf8fe49590a966953c37e18608af9":["beopen"], "20|grid________::d2f0204126ee709244a488a4cd3b91c2":["beopen"], "20|corda__h2020::05aba9d2ed17533d15221e5655ac11e6":["beopen"], "20|grid________::802401579481dc32062bdee69f5e6a34":["beopen"], "20|corda__h2020::3f6d9d54cac975a517ba6b252c81582d":["beopen"]} + + + + + + + + + Set the dedup orchestrator name + + dedupConfig + decisiontree-dedup-test + + + + + + + + declares the ActionSet ids to promote in the RAW graph + + actionSetIdsRawGraph + scholexplorer-dump,gridac-dump,doiboost-organizations,doiboost,orcidworks-no-doi,iis-wos-entities,iis-entities-software,iis-entities-patent + + + + + + + + declares the ActionSet ids to promote in the INFERRED graph + + actionSetIdsIISGraph + iis-researchinitiative,iis-document-citations,iis-document-affiliation,iis-document-classes,iis-document-similarities,iis-referenced-datasets-main,iis-referenced-datasets-preprocessing,iis-referenced-projects-main,iis-referenced-projects-preprocessing,iis-referenceextraction-pdb,document_software_url,iis-extracted-metadata,iis-communities,iis-referenced-patents,iis-covid-19 + + + + + + + + wait configurations + + + + + + + + create the AGGREGATOR graph + + executeOozieJob + IIS + + { + 'graphOutputPath' : 'aggregatorGraphPath', + 'isLookupUrl' : 'isLookUpUrl', + 'reuseContent' : 'reuseContent', + 'contentPath' : 'contentPath' + } + + + { + 'oozie.wf.application.path' : '/lib/dnet/oa/graph/raw_all/oozie_app', + 'mongoURL' : 'mongodb://beta.services.openaire.eu', + 'mongoDb' : 'mdstore', + 'postgresURL' : 'jdbc:postgresql://beta.services.openaire.eu:5432/dnet_openaireplus', + 'postgresUser' : 'dnet', + 'postgresPassword' : '', + 'workingDir' : '/tmp/beta_provision/working_dir/aggregator' + } + + build-report + + + + + + + + create the RAW graph + + executeOozieJob + IIS + + { + 'inputActionSetIds' : 'actionSetIdsRawGraph', + 'inputGraphRootPath' : 'aggregatorGraphPath', + 'outputGraphRootPath' : 'rawGraphPath', + 'isLookupUrl' : 'isLookUpUrl' + } + + + { + 'oozie.wf.application.path' : '/lib/dnet/actionmanager/wf/main/oozie_app', + 'sparkExecutorCores' : '3', + 'sparkExecutorMemory' : '10G', + 'activePromoteDatasetActionPayload' : 'true', + 'activePromoteDatasourceActionPayload' : 'true', + 'activePromoteOrganizationActionPayload' : 'true', + 'activePromoteOtherResearchProductActionPayload' : 'true', + 'activePromoteProjectActionPayload' : 'true', + 'activePromotePublicationActionPayload' : 'true', + 'activePromoteRelationActionPayload' : 'true', + 'activePromoteResultActionPayload' : 'true', + 'activePromoteSoftwareActionPayload' : 'true', + 'mergeAndGetStrategy' : 'MERGE_FROM_AND_GET', + 'workingDir' : '/tmp/beta_provision/working_dir/promoteActionsRaw' + } + + build-report + + + + + + + + search for duplicates in the raw graph + + executeOozieJob + IIS + + { + 'actionSetId' : 'dedupConfig', + 'graphBasePath' : 'rawGraphPath', + 'dedupGraphPath': 'dedupGraphPath', + 'isLookUpUrl' : 'isLookUpUrl' + } + + + { + 'oozie.wf.application.path' : '/lib/dnet/oa/dedup/scan/oozie_app', + 'workingPath' : '/tmp/beta_provision/working_dir/dedup' + } + + build-report + + + + + + + + create the INFERRED graph + + executeOozieJob + IIS + + { + 'inputActionSetIds' : 'actionSetIdsIISGraph', + 'inputGraphRootPath' : 'dedupGraphPath', + 'outputGraphRootPath' : 'inferredGraphPath', + 'isLookupUrl' : 'isLookUpUrl' + } + + + { + 'oozie.wf.application.path' : '/lib/dnet/actionmanager/wf/main/oozie_app', + 'sparkExecutorCores' : '3', + 'sparkExecutorMemory' : '10G', + 'activePromoteDatasetActionPayload' : 'true', + 'activePromoteDatasourceActionPayload' : 'true', + 'activePromoteOrganizationActionPayload' : 'true', + 'activePromoteOtherResearchProductActionPayload' : 'true', + 'activePromoteProjectActionPayload' : 'true', + 'activePromotePublicationActionPayload' : 'true', + 'activePromoteRelationActionPayload' : 'true', + 'activePromoteResultActionPayload' : 'true', + 'activePromoteSoftwareActionPayload' : 'true', + 'mergeAndGetStrategy' : 'MERGE_FROM_AND_GET', + 'workingDir' : '/tmp/beta_provision/working_dir/promoteActionsIIS' + } + + build-report + + + + + + + + mark duplicates as deleted and redistribute the relationships + + executeOozieJob + IIS + + { + 'graphBasePath' : 'inferredGraphPath', + 'dedupGraphPath': 'consistentGraphPath' + } + + + { + 'oozie.wf.application.path' : '/lib/dnet/oa/dedup/consistency/oozie_app', + 'workingPath' : '/tmp/beta_provision/working_dir/dedup' + } + + build-report + + + + + + + + propagates ORCID among results linked by allowedsemrels semantic relationships + + executeOozieJob + IIS + + { + 'sourcePath' : 'consistentGraphPath', + 'outputPath': 'orcidGraphPath' + } + + + { + 'oozie.wf.application.path' : '/lib/dnet/oa/enrichment/orcidtoresultfromsemrel/oozie_app', + 'workingDir' : '/tmp/beta_provision/working_dir/orcid', + 'allowedsemrels' : 'isSupplementedBy;isSupplementTo', + 'saveGraph' : 'true' + } + + build-report + + + + + + + + mark results respecting some rules as belonging to communities + + executeOozieJob + IIS + + { + 'sourcePath' : 'orcidGraphPath', + 'outputPath': 'bulkTaggingGraphPath', + 'isLookUpUrl' : 'isLookUpUrl', + 'pathMap' : 'bulkTaggingPathMap' + } + + + { + 'oozie.wf.application.path' : '/lib/dnet/oa/enrichment/bulktag/oozie_app', + 'workingDir' : '/tmp/beta_provision/working_dir/bulktag' + } + + build-report + + + + + + + + creates relashionships between results and organizations when the organizations are associated to institutional repositories + + executeOozieJob + IIS + + { + 'sourcePath' : 'bulkTaggingGraphPath', + 'outputPath': 'affiliationGraphPath' + } + + + { + 'oozie.wf.application.path' : '/lib/dnet/oa/enrichment/affiliation/oozie_app', + 'workingDir' : '/tmp/beta_provision/working_dir/affiliation', + 'saveGraph' : 'true' + } + + build-report + + + + + + + + marks as belonging to communities the result collected from datasources related to the organizations specified in the organizationCommunityMap + + executeOozieJob + IIS + + { + 'sourcePath' : 'affiliationGraphPath', + 'outputPath': 'communityOrganizationGraphPath', + 'organizationtoresultcommunitymap': 'propagationOrganizationCommunityMap' + } + + + { + 'oozie.wf.application.path' : '/lib/dnet/oa/enrichment/community_organization/oozie_app', + 'workingDir' : '/tmp/beta_provision/working_dir/community_organization', + 'saveGraph' : 'true' + } + + build-report + + + + + + + + created relation between projects and results linked to other results trough allowedsemrel semantic relations linked to projects + + executeOozieJob + IIS + + { + 'sourcePath' : 'communityOrganizationGraphPath', + 'outputPath': 'fundingGraphPath' + } + + + { + 'oozie.wf.application.path' : '/lib/dnet/oa/enrichment/funding/oozie_app', + 'workingDir' : '/tmp/beta_provision/working_dir/funding', + 'allowedsemrels' : 'isSupplementedBy;isSupplementTo', + 'saveGraph' : 'true' + } + + build-report + + + + + + + + tag as belonging to communitites result in in allowedsemrels relation with other result already linked to communities + + executeOozieJob + IIS + + { + 'sourcePath' : 'fundingGraphPath', + 'outputPath': 'communitySemRelGraphPath', + 'isLookUpUrl' : 'isLookUpUrl' + } + + + { + 'oozie.wf.application.path' : '/lib/dnet/oa/enrichment/community_semrel/oozie_app', + 'workingDir' : '/tmp/beta_provision/working_dir/community_semrel', + 'allowedsemrels' : 'isSupplementedBy;isSupplementTo', + 'saveGraph' : 'true' + } + + build-report + + + + + + + + associated to results colleced from allowedtypes and those in the whithelist the country of the organization(s) handling the datasource it is collected from + + executeOozieJob + IIS + + { + 'sourcePath' : 'communitySemRelGraphPath', + 'outputPath': 'countryGraphPath' + } + + + { + 'oozie.wf.application.path' : '/lib/dnet/oa/enrichment/country/oozie_app', + 'sparkExecutorCores' : '3', + 'sparkExecutorMemory' : '10G', + 'workingDir' : '/tmp/beta_provision/working_dir/country', + 'allowedtypes' : 'pubsrepository::institutional', + 'whitelist' : '10|opendoar____::300891a62162b960cf02ce3827bb363c', + 'saveGraph' : 'true' + } + + build-report + + + + + + + + clean the properties in the graph typed as Qualifier according to the vocabulary indicated in schemeid + + executeOozieJob + IIS + + { + 'graphInputPath' : 'countryGraphPath', + 'graphOutputPath': 'cleanedGraphPath', + 'isLookupUrl': 'isLookUpUrl' + } + + + { + 'oozie.wf.application.path' : '/lib/dnet/oa/graph/clean/oozie_app', + 'workingPath' : '/tmp/beta_provision/working_dir/clean' + } + + build-report + + + + + + + + removes blacklisted relations + + executeOozieJob + IIS + + { + 'sourcePath' : 'cleanedGraphPath', + 'outputPath': 'blacklistedGraphPath' + } + + + { + 'oozie.wf.application.path' : '/lib/dnet/oa/enrichment/blacklist/oozie_app', + 'workingDir' : '/tmp/beta_provision/working_dir/blacklist', + 'postgresURL' : 'jdbc:postgresql://beta.services.openaire.eu:5432/dnet_openaireplus', + 'postgresUser' : 'dnet', + 'postgresPassword' : '' + } + + build-report + + + + + + + + + wf_20200615_163630_609 + 2020-06-15T17:08:00+00:00 + SUCCESS + + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-worfklow-profiles/src/main/resources/eu/dnetlib/dhp/wf/profiles/graph_to_hiveDB.xml b/dhp-workflows/dhp-worfklow-profiles/src/main/resources/eu/dnetlib/dhp/wf/profiles/graph_to_hiveDB.xml new file mode 100644 index 0000000000..0ace12ea38 --- /dev/null +++ b/dhp-workflows/dhp-worfklow-profiles/src/main/resources/eu/dnetlib/dhp/wf/profiles/graph_to_hiveDB.xml @@ -0,0 +1,73 @@ + +
+ + + + + +
+ + Graph to HiveDB [OCEAN] + Data Provision + 30 + + + Set the path containing the AGGREGATOR graph + + inputPath + + + + + + + + Set the target path to store the RAW graph + + hiveDbName + + + + + + + + + wait configurations + + + + + + + create the AGGREGATOR graph + + executeOozieJob + IIS + + { + 'inputPath' : 'inputPath', + 'hiveDbName' : 'hiveDbName' + } + + + { + 'oozie.wf.application.path' : '/lib/dnet/oa/graph/hive/oozie_app' + } + + build-report + + + + + + + + + wf_20200615_163630_609 + 2020-06-15T17:08:00+00:00 + SUCCESS + + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-worfklow-profiles/src/main/resources/eu/dnetlib/dhp/wf/profiles/provision.xml b/dhp-workflows/dhp-worfklow-profiles/src/main/resources/eu/dnetlib/dhp/wf/profiles/provision.xml deleted file mode 100644 index 487afee4fa..0000000000 --- a/dhp-workflows/dhp-worfklow-profiles/src/main/resources/eu/dnetlib/dhp/wf/profiles/provision.xml +++ /dev/null @@ -1,572 +0,0 @@ - -
- - - - - -
- - Data Provision [OCEAN] - Data Provision - 30 - - - Set the path containing the AGGREGATOR graph - - aggregatorGraphPath - /tmp/beta_provision/graph/00_graph_aggregator - - - - - - - Set the target path to store the RAW graph - - rawGraphPath - /tmp/beta_provision/graph/01_graph_raw - - - - - - - Set the target path to store the DEDUPED graph - - dedupGraphPath - /tmp/beta_provision/graph/02_graph_dedup - - - - - - - Set the target path to store the INFERRED graph - - inferredGraphPath - /tmp/beta_provision/graph/03_graph_inferred - - - - - - - Set the target path to store the CONSISTENCY graph - - consistentGraphPath - /tmp/beta_provision/graph/04_graph_consistent - - - - - - - Set the target path to store the ORCID enriched graph - - orcidGraphPath - /tmp/beta_provision/graph/05_graph_orcid - - - - - - - Set the target path to store the BULK TAGGED graph - - bulkTaggingGraphPath - /tmp/beta_provision/graph/06_graph_bulktagging - - - - - - - Set the target path to store the AFFILIATION from INSTITUTIONAL REPOS graph - - affiliationGraphPath - /tmp/beta_provision/graph/07_graph_affiliation - - - - - - - Set the target path to store the COMMUNITY from SELECTED SOURCES graph - - communityOrganizationGraphPath - /tmp/beta_provision/graph/08_graph_comunity_organization - - - - - - - Set the target path to store the FUNDING from SEMANTIC RELATION graph - - fundingGraphPath - /tmp/beta_provision/graph/09_graph_funding - - - - - - - Set the target path to store the COMMUNITY from SEMANTIC RELATION graph - - communitySemRelGraphPath - /tmp/beta_provision/graph/10_graph_comunity_sem_rel - - - - - - - Set the target path to store the COUNTRY enriched graph - - countryGraphPath - /tmp/beta_provision/graph/11_graph_country - - - - - - - Set the target path to store the blacklisted graph - - blacklistedGraphPath - /tmp/beta_provision/graph/12_graph_blacklisted - - - - - - - Set the lookup address - - isLookUpUrl - http://beta.services.openaire.eu:8280/is/services/isLookUp?wsdl - - - - - - - Set the map of paths for the Bulk Tagging - - bulkTaggingPathMap - {"author" : "$['author'][*]['fullname']", "title" : "$['title'][*]['value']", "orcid" : "$['author'][*]['pid'][*][?(@['key']=='ORCID')]['value']", "contributor" : "$['contributor'][*]['value']", "description" : "$['description'][*]['value']"} - - - - - - - Set the map of associations organization, community list for the propagation of community to result through organization - - propagationOrganizationCommunityMap - {"20|corda__h2020::3fb05a9524c3f790391261347852f638":["mes","euromarine"], "20|corda__h2020::e8dbe14cca9bf6fce09d468872f813f8":["mes","euromarine"], "20|snsf________::9b253f265e3bef5cae6d881fdf61aceb":["mes","euromarine"],"20|rcuk________::e054eea0a47665af8c3656b5785ccf76":["mes","euromarine"],"20|corda__h2020::edc18d67c9b11fb616ca9f6e1db1b151":["mes","euromarine"],"20|rcuk________::d5736d9da90521ddcdc7828a05a85e9a":["mes","euromarine"],"20|corda__h2020::f5d418d3aa1cf817ddefcc3fdc039f27":["mes","euromarine"],"20|snsf________::8fa091f8f25a846779acb4ea97b50aef":["mes","euromarine"],"20|corda__h2020::81e020977211c2c40fae2e1a50bffd71":["mes","euromarine"],"20|corda_______::81e020977211c2c40fae2e1a50bffd71":["mes","euromarine"],"20|snsf________::31d0a100e54e3cdb3c6f52d91e638c78":["mes","euromarine"],"20|corda__h2020::ea379ef91b8cc86f9ac5edc4169292db":["mes","euromarine"],"20|corda__h2020::f75ee2ee48e5cb0ec8c8d30aaa8fef70":["mes","euromarine"],"20|rcuk________::e16010089551a1a9182a94604fc0ea59":["mes","euromarine"],"20|corda__h2020::38531a2cce7c5c347ffc439b07c1f43b":["mes","euromarine"],"20|corda_______::38531a2cce7c5c347ffc439b07c1f43b":["mes","euromarine"],"20|grid________::b2cbbf5eadbbf87d534b022bad3191d7":["mes","euromarine"],"20|snsf________::74730ef1439d7f7636a8be58a6b471b8":["mes","euromarine"],"20|nsf_________::ad72e19043a5a467e35f9b444d11563e":["mes","euromarine"],"20|rcuk________::0fc3e92500290902a2d38ec2445e74c3":["mes","euromarine"],"20|grid________::ad2c29905da0eb3c06b3fa80cacd89ea":["mes","euromarine"],"20|corda__h2020::30b53e4d63d3724f00acb9cbaca40860":["mes","euromarine"],"20|corda__h2020::f60f84bee14ad93f0db0e49af1d5c317":["mes","euromarine"], "20|corda__h2020::7bf251ac3765b5e89d82270a1763d09f":["mes","euromarine"], "20|corda__h2020::65531bd11be9935948c7f2f4db1c1832":["mes","euromarine"], "20|corda__h2020::e0e98f86bbc76638bbb72a8fe2302946":["mes","euromarine"], "20|snsf________::3eb43582ac27601459a8d8b3e195724b":["mes","euromarine"], "20|corda__h2020::af2481dab65d06c8ea0ae02b5517b9b6":["mes","euromarine"], "20|corda__h2020::c19d05cfde69a50d3ebc89bd0ee49929":["mes","euromarine"], "20|corda__h2020::af0bfd9fc09f80d9488f56d71a9832f0":["mes","euromarine"], "20|rcuk________::f33c02afb0dc66c49d0ed97ca5dd5cb0":["beopen"], - "20|grid________::a867f78acdc5041b34acfe4f9a349157":["beopen"], "20|grid________::7bb116a1a9f95ab812bf9d2dea2be1ff":["beopen"], "20|corda__h2020::6ab0e0739dbe625b99a2ae45842164ad":["beopen"], "20|corda__h2020::8ba50792bc5f4d51d79fca47d860c602":["beopen"], "20|corda_______::8ba50792bc5f4d51d79fca47d860c602":["beopen"], "20|corda__h2020::e70e9114979e963eef24666657b807c3":["beopen"], "20|corda_______::e70e9114979e963eef24666657b807c3":["beopen"], "20|corda_______::15911e01e9744d57205825d77c218737":["beopen"], "20|opendoar____::056a41e24e2a9a67215e87bbee6a80ab":["beopen"], "20|opendoar____::7f67f2e6c6fbb0628f8160fcd3d92ae3":["beopen"], "20|grid________::a8ecfd7c084e561168bcbe6bf0daf3e3":["beopen"], "20|corda_______::7bbe6cc5d8ec1864739a04b0d020c9e9":["beopen"], "20|corda_______::3ff558e30c2e434d688539548300b050":["beopen"], "20|corda__h2020::5ffee5b3b83b33a8cf0e046877bd3a39":["beopen"], "20|corda__h2020::5187217e2e806a6df3579c46f82401bc":["beopen"], "20|grid________::5fa7e2709bcd945e26bfa18689adeec1":["beopen"], "20|corda_______::d8696683c53027438031a96ad27c3c07":["beopen"], "20|corda__h2020::d8696683c53027438031a96ad27c3c07":["beopen"], "20|rcuk________::23a79ebdfa59790864e4a485881568c1":["beopen"], "20|corda__h2020::b76cf8fe49590a966953c37e18608af9":["beopen"], "20|grid________::d2f0204126ee709244a488a4cd3b91c2":["beopen"], "20|corda__h2020::05aba9d2ed17533d15221e5655ac11e6":["beopen"], "20|grid________::802401579481dc32062bdee69f5e6a34":["beopen"], "20|corda__h2020::3f6d9d54cac975a517ba6b252c81582d":["beopen"]} - - - - - - - - Set the dedup orchestrator name - - dedupConfig - decisiontree-dedup-test - - - - - - - declares the ActionSet ids to promote in the RAW graph - - actionSetIdsRawGraph - scholexplorer-dump,gridac-dump,doiboost-organizations,doiboost,orcidworks-no-doi,iis-wos-entities,iis-entities-software,iis-entities-patent - - - - - - - declares the ActionSet ids to promote in the INFERRED graph - - actionSetIdsIISGraph - iis-researchinitiative,iis-document-citations,iis-document-affiliation,iis-document-classes,iis-document-similarities,iis-referenced-datasets-main,iis-referenced-datasets-preprocessing,iis-referenced-projects-main,iis-referenced-projects-preprocessing,iis-referenceextraction-pdb,document_software_url,iis-extracted-metadata,iis-communities,iis-referenced-patents,iis-covid-19 - - - - - - - wait configurations - - - - - - - create the AGGREGATOR graph - - executeOozieJob - IIS - - { - 'graphOutputPath' : 'aggregatorGraphPath' - } - - - { - 'oozie.wf.application.path' : '/lib/dnet/oa/graph/raw_all/oozie_app', - 'mongoURL' : 'mongodb://beta.services.openaire.eu', - 'mongoDb' : 'mdstore', - 'postgresURL' : 'jdbc:postgresql://beta.services.openaire.eu:5432/dnet_openaireplus', - 'postgresUser' : 'dnet', - 'postgresPassword' : 'dnetPwd', - 'reuseContent' : 'true', - 'contentPath' : '/tmp/beta_provision/aggregator', - 'workingDir' : '/tmp/beta_provision/working_dir/aggregator' - } - - build-report - - - - - - - create the RAW graph - - executeOozieJob - IIS - - { - 'inputActionSetIds' : 'actionSetIdsRawGraph', - 'inputGraphRootPath' : 'aggregatorGraphPath', - 'outputGraphRootPath' : 'rawGraphPath', - 'isLookupUrl' : 'isLookUpUrl' - } - - - { - 'oozie.wf.application.path' : '/lib/dnet/actionmanager/wf/main/oozie_app', - 'activePromoteDatasetActionPayload' : 'true', - 'activePromoteDatasourceActionPayload' : 'true', - 'activePromoteOrganizationActionPayload' : 'true', - 'activePromoteOtherResearchProductActionPayload' : 'true', - 'activePromoteProjectActionPayload' : 'true', - 'activePromotePublicationActionPayload' : 'true', - 'activePromoteRelationActionPayload' : 'true', - 'activePromoteResultActionPayload' : 'true', - 'activePromoteSoftwareActionPayload' : 'true', - 'mergeAndGetStrategy' : 'MERGE_FROM_AND_GET', - 'workingDir' : '/tmp/beta_provision/working_dir/promoteActionsRaw' - } - - build-report - - - - - - - search for duplicates in the raw graph - - executeOozieJob - IIS - - { - 'actionSetId' : 'dedupConfig', - 'graphBasePath' : 'rawGraphPath', - 'dedupGraphPath': 'dedupGraphPath', - 'isLookUpUrl' : 'isLookUpUrl' - } - - - { - 'oozie.wf.application.path' : '/lib/dnet/oa/dedup/scan/oozie_app', - 'workingPath' : '/tmp/beta_provision/working_dir/dedup' - } - - build-report - - - - - - - create the INFERRED graph - - executeOozieJob - IIS - - { - 'inputActionSetIds' : 'actionSetIdsIISGraph', - 'inputGraphRootPath' : 'dedupGraphPath', - 'outputGraphRootPath' : 'inferredGraphPath', - 'isLookupUrl' : 'isLookUpUrl' - } - - - { - 'oozie.wf.application.path' : '/lib/dnet/actionmanager/wf/main/oozie_app', - 'activePromoteDatasetActionPayload' : 'true', - 'activePromoteDatasourceActionPayload' : 'true', - 'activePromoteOrganizationActionPayload' : 'true', - 'activePromoteOtherResearchProductActionPayload' : 'true', - 'activePromoteProjectActionPayload' : 'true', - 'activePromotePublicationActionPayload' : 'true', - 'activePromoteRelationActionPayload' : 'true', - 'activePromoteResultActionPayload' : 'true', - 'activePromoteSoftwareActionPayload' : 'true', - 'mergeAndGetStrategy' : 'MERGE_FROM_AND_GET', - 'workingDir' : '/tmp/beta_provision/working_dir/promoteActionsIIS' - } - - build-report - - - - - - - mark duplicates as deleted and redistribute the relationships - - executeOozieJob - IIS - - { - 'graphBasePath' : 'inferredGraphPath', - 'dedupGraphPath': 'consistentGraphPath' - } - - - { - 'oozie.wf.application.path' : '/lib/dnet/oa/dedup/consistency/oozie_app', - 'workingPath' : '/tmp/beta_provision/working_dir/dedup' - } - - build-report - - - - - - - propagates ORCID among results linked by allowedsemrels semantic relationships - - executeOozieJob - IIS - - { - 'sourcePath' : 'consistentGraphPath', - 'outputPath': 'orcidGraphPath' - } - - - { - 'oozie.wf.application.path' : '/lib/dnet/oa/enrichment/orcidtoresultfromsemrel/oozie_app', - 'workingDir' : '/tmp/beta_provision/working_dir/orcid', - 'allowedsemrels' : 'isSupplementedBy;isSupplementTo', - 'saveGraph' : 'true' - } - - build-report - - - - - - - mark results respecting some rules as belonging to communities - - executeOozieJob - IIS - - { - 'sourcePath' : 'orcidGraphPath', - 'outputPath': 'bulkTaggingGraphPath', - 'isLookUpUrl' : 'isLookUpUrl', - 'pathMap' : 'bulkTaggingPathMap' - } - - - { - 'oozie.wf.application.path' : '/lib/dnet/oa/enrichment/bulktag/oozie_app', - 'workingDir' : '/tmp/beta_provision/working_dir/bulktag' - } - - build-report - - - - - - - creates relashionships between results and organizations when the organizations are associated to institutional repositories - - executeOozieJob - IIS - - { - 'sourcePath' : 'bulkTaggingGraphPath', - 'outputPath': 'affiliationGraphPath' - } - - - { - 'oozie.wf.application.path' : '/lib/dnet/oa/enrichment/affiliation/oozie_app', - 'workingDir' : '/tmp/beta_provision/working_dir/affiliation', - 'saveGraph' : 'true' - } - - build-report - - - - - - - marks as belonging to communities the result collected from datasources related to the organizations specified in the organizationCommunityMap - - executeOozieJob - IIS - - { - 'sourcePath' : 'affiliationGraphPath', - 'outputPath': 'communityOrganizationGraphPath', - 'organizationtoresultcommunitymap': 'propagationOrganizationCommunityMap' - } - - - { - 'oozie.wf.application.path' : '/lib/dnet/oa/enrichment/community_organization/oozie_app', - 'workingDir' : '/tmp/beta_provision/working_dir/community_organization', - 'saveGraph' : 'true' - } - - build-report - - - - - - - created relation between projects and results linked to other results trough allowedsemrel semantic relations linked to projects - - executeOozieJob - IIS - - { - 'sourcePath' : 'communityOrganizationGraphPath', - 'outputPath': 'fundingGraphPath' - } - - - { - 'oozie.wf.application.path' : '/lib/dnet/oa/enrichment/funding/oozie_app', - 'workingDir' : '/tmp/beta_provision/working_dir/funding', - 'allowedsemrels' : 'isSupplementedBy;isSupplementTo', - 'saveGraph' : 'true' - } - - build-report - - - - - - - tag as belonging to communitites result in in allowedsemrels relation with other result already linked to communities - - executeOozieJob - IIS - - { - 'sourcePath' : 'fundingGraphPath', - 'outputPath': 'communitySemRelGraphPath', - 'isLookUpUrl' : 'isLookUpUrl' - } - - - { - 'oozie.wf.application.path' : '/lib/dnet/oa/enrichment/community_semrel/oozie_app', - 'workingDir' : '/tmp/beta_provision/working_dir/community_semrel', - 'allowedsemrels' : 'isSupplementedBy;isSupplementTo', - 'saveGraph' : 'true' - } - - build-report - - - - - - - associated to results colleced from allowedtypes and those in the whithelist the country of the organization(s) handling the datasource it is collected from - - executeOozieJob - IIS - - { - 'sourcePath' : 'communitySemRelGraphPath', - 'outputPath': 'countryGraphPath' - } - - - { - 'oozie.wf.application.path' : '/lib/dnet/oa/enrichment/country/oozie_app', - 'sparkExecutorCores' : '3', - 'sparkExecutorMemory' : '10G', - 'workingDir' : '/tmp/beta_provision/working_dir/country', - 'allowedtypes' : 'pubsrepository::institutional', - 'whitelist' : '10|opendoar____::300891a62162b960cf02ce3827bb363c', - 'saveGraph' : 'true' - } - - build-report - - - - - - - removes blacklisted relations - - executeOozieJob - IIS - - { - 'sourcePath' : 'countryGraphPath', - 'outputPath': 'blacklistedGraphPath' - } - - - { - 'oozie.wf.application.path' : '/lib/dnet/oa/enrichment/blacklist/oozie_app', - 'workingDir' : '/tmp/beta_provision/working_dir/blacklist', - 'postgresURL' : 'jdbc:postgresql://beta.services.openaire.eu:5432/dnet_openaireplus', - 'postgresUser' : 'dnet', - 'postgresPassword' : 'dnetPwd' - } - - build-report - - - - - - - - wf_20200509_100941_857 - 2020-05-09T13:26:09+00:00 - FAILURE - eu.dnetlib.data.hadoop.rmi.HadoopServiceException: hadoop job: 0002933-200403132837156-oozie-oozi-W failed with status: KILLED, oozie log: 2020-05-09 13:23:31,194 INFO org.apache.oozie.service.JPAService: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[] No results found 2020-05-09 13:23:31,216 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@:start:] Start action [0002933-200403132837156-oozie-oozi-W@:start:] with user-retry state : userRetryCount [0], userRetryMax [0], userRetryInterval [10] 2020-05-09 13:23:31,216 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@:start:] [***0002933-200403132837156-oozie-oozi-W@:start:***]Action status=DONE 2020-05-09 13:23:31,216 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@:start:] [***0002933-200403132837156-oozie-oozi-W@:start:***]Action updated in DB! 2020-05-09 13:23:31,257 INFO org.apache.oozie.service.JPAService: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@:start:] No results found 2020-05-09 13:23:31,275 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@:start:] No Notification URL is defined. Therefore nothing to notify for job 0002933-200403132837156-oozie-oozi-W@:start: 2020-05-09 13:23:31,275 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[] No Notification URL is defined. Therefore nothing to notify for job 0002933-200403132837156-oozie-oozi-W 2020-05-09 13:23:31,314 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@reset-outputpath] Start action [0002933-200403132837156-oozie-oozi-W@reset-outputpath] with user-retry state : userRetryCount [0], userRetryMax [0], userRetryInterval [10] 2020-05-09 13:23:33,897 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@reset-outputpath] [***0002933-200403132837156-oozie-oozi-W@reset-outputpath***]Action status=DONE 2020-05-09 13:23:33,897 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@reset-outputpath] [***0002933-200403132837156-oozie-oozi-W@reset-outputpath***]Action updated in DB! 2020-05-09 13:23:33,947 INFO org.apache.oozie.service.JPAService: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@reset-outputpath] No results found 2020-05-09 13:23:33,966 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_entities] Start action [0002933-200403132837156-oozie-oozi-W@copy_entities] with user-retry state : userRetryCount [0], userRetryMax [0], userRetryInterval [10] 2020-05-09 13:23:33,966 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_entities] [***0002933-200403132837156-oozie-oozi-W@copy_entities***]Action status=DONE 2020-05-09 13:23:33,966 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_entities] [***0002933-200403132837156-oozie-oozi-W@copy_entities***]Action updated in DB! 2020-05-09 13:23:34,012 INFO org.apache.oozie.service.JPAService: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_entities] No results found 2020-05-09 13:23:34,018 INFO org.apache.oozie.service.JPAService: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_entities] No results found 2020-05-09 13:23:34,023 INFO org.apache.oozie.service.JPAService: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_entities] No results found 2020-05-09 13:23:34,029 INFO org.apache.oozie.service.JPAService: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_entities] No results found 2020-05-09 13:23:34,124 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_relation] Start action [0002933-200403132837156-oozie-oozi-W@copy_relation] with user-retry state : userRetryCount [0], userRetryMax [0], userRetryInterval [10] 2020-05-09 13:23:34,130 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_projects] Start action [0002933-200403132837156-oozie-oozi-W@copy_projects] with user-retry state : userRetryCount [0], userRetryMax [0], userRetryInterval [10] 2020-05-09 13:23:34,130 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_datasources] Start action [0002933-200403132837156-oozie-oozi-W@copy_datasources] with user-retry state : userRetryCount [0], userRetryMax [0], userRetryInterval [10] 2020-05-09 13:23:34,140 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_organization] Start action [0002933-200403132837156-oozie-oozi-W@copy_organization] with user-retry state : userRetryCount [0], userRetryMax [0], userRetryInterval [10] 2020-05-09 13:23:35,010 INFO org.apache.oozie.action.hadoop.DistcpActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_projects] checking action, hadoop job ID [job_1585920557248_14569] status [RUNNING] 2020-05-09 13:23:35,018 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_projects] [***0002933-200403132837156-oozie-oozi-W@copy_projects***]Action status=RUNNING 2020-05-09 13:23:35,018 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_projects] [***0002933-200403132837156-oozie-oozi-W@copy_projects***]Action updated in DB! 2020-05-09 13:23:35,022 INFO org.apache.oozie.action.hadoop.DistcpActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_relation] checking action, hadoop job ID [job_1585920557248_14568] status [RUNNING] 2020-05-09 13:23:35,027 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_projects] No Notification URL is defined. Therefore nothing to notify for job 0002933-200403132837156-oozie-oozi-W@copy_projects 2020-05-09 13:23:35,028 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_relation] [***0002933-200403132837156-oozie-oozi-W@copy_relation***]Action status=RUNNING 2020-05-09 13:23:35,028 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_relation] [***0002933-200403132837156-oozie-oozi-W@copy_relation***]Action updated in DB! 2020-05-09 13:23:35,031 INFO org.apache.oozie.action.hadoop.DistcpActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_datasources] checking action, hadoop job ID [job_1585920557248_14570] status [RUNNING] 2020-05-09 13:23:35,035 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_datasources] [***0002933-200403132837156-oozie-oozi-W@copy_datasources***]Action status=RUNNING 2020-05-09 13:23:35,035 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_datasources] [***0002933-200403132837156-oozie-oozi-W@copy_datasources***]Action updated in DB! 2020-05-09 13:23:35,037 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_relation] No Notification URL is defined. Therefore nothing to notify for job 0002933-200403132837156-oozie-oozi-W@copy_relation 2020-05-09 13:23:35,048 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_datasources] No Notification URL is defined. Therefore nothing to notify for job 0002933-200403132837156-oozie-oozi-W@copy_datasources 2020-05-09 13:23:35,072 INFO org.apache.oozie.action.hadoop.DistcpActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_organization] checking action, hadoop job ID [job_1585920557248_14571] status [RUNNING] 2020-05-09 13:23:35,076 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_organization] [***0002933-200403132837156-oozie-oozi-W@copy_organization***]Action status=RUNNING 2020-05-09 13:23:35,076 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_organization] [***0002933-200403132837156-oozie-oozi-W@copy_organization***]Action updated in DB! 2020-05-09 13:23:35,084 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_organization] No Notification URL is defined. Therefore nothing to notify for job 0002933-200403132837156-oozie-oozi-W@copy_organization 2020-05-09 13:23:35,090 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_entities] No Notification URL is defined. Therefore nothing to notify for job 0002933-200403132837156-oozie-oozi-W@copy_entities 2020-05-09 13:23:35,090 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@reset-outputpath] No Notification URL is defined. Therefore nothing to notify for job 0002933-200403132837156-oozie-oozi-W@reset-outputpath 2020-05-09 13:23:58,926 INFO org.apache.oozie.servlet.CallbackServlet: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_datasources] callback for action [0002933-200403132837156-oozie-oozi-W@copy_datasources] 2020-05-09 13:23:59,085 INFO org.apache.oozie.action.hadoop.DistcpActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_datasources] checking action, hadoop job ID [job_1585920557248_14570] status [RUNNING] 2020-05-09 13:23:59,242 INFO org.apache.oozie.servlet.CallbackServlet: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_projects] callback for action [0002933-200403132837156-oozie-oozi-W@copy_projects] 2020-05-09 13:23:59,386 INFO org.apache.oozie.action.hadoop.DistcpActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_projects] checking action, hadoop job ID [job_1585920557248_14569] status [RUNNING] 2020-05-09 13:24:01,343 INFO org.apache.oozie.servlet.CallbackServlet: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_datasources] callback for action [0002933-200403132837156-oozie-oozi-W@copy_datasources] 2020-05-09 13:24:01,418 INFO org.apache.oozie.action.hadoop.DistcpActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_datasources] Hadoop Jobs launched : [job_1585920557248_14573] 2020-05-09 13:24:01,418 INFO org.apache.oozie.action.hadoop.DistcpActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_datasources] action completed, external ID [job_1585920557248_14570] 2020-05-09 13:24:01,493 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_datasources] No Notification URL is defined. Therefore nothing to notify for job 0002933-200403132837156-oozie-oozi-W@copy_datasources 2020-05-09 13:24:01,935 INFO org.apache.oozie.servlet.CallbackServlet: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_projects] callback for action [0002933-200403132837156-oozie-oozi-W@copy_projects] 2020-05-09 13:24:02,012 INFO org.apache.oozie.action.hadoop.DistcpActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_projects] Hadoop Jobs launched : [job_1585920557248_14572] 2020-05-09 13:24:02,012 INFO org.apache.oozie.action.hadoop.DistcpActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_projects] action completed, external ID [job_1585920557248_14569] 2020-05-09 13:24:02,076 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_projects] No Notification URL is defined. Therefore nothing to notify for job 0002933-200403132837156-oozie-oozi-W@copy_projects 2020-05-09 13:25:03,172 INFO org.apache.oozie.servlet.CallbackServlet: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_organization] callback for action [0002933-200403132837156-oozie-oozi-W@copy_organization] 2020-05-09 13:25:03,336 INFO org.apache.oozie.action.hadoop.DistcpActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_organization] checking action, hadoop job ID [job_1585920557248_14571] status [RUNNING] 2020-05-09 13:25:05,598 INFO org.apache.oozie.servlet.CallbackServlet: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_organization] callback for action [0002933-200403132837156-oozie-oozi-W@copy_organization] 2020-05-09 13:25:05,688 INFO org.apache.oozie.action.hadoop.DistcpActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_organization] Hadoop Jobs launched : [job_1585920557248_14574] 2020-05-09 13:25:05,691 INFO org.apache.oozie.action.hadoop.DistcpActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_organization] action completed, external ID [job_1585920557248_14571] 2020-05-09 13:25:05,748 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_organization] No Notification URL is defined. Therefore nothing to notify for job 0002933-200403132837156-oozie-oozi-W@copy_organization 2020-05-09 13:25:23,274 INFO org.apache.oozie.servlet.CallbackServlet: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_relation] callback for action [0002933-200403132837156-oozie-oozi-W@copy_relation] 2020-05-09 13:25:23,409 INFO org.apache.oozie.action.hadoop.DistcpActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_relation] checking action, hadoop job ID [job_1585920557248_14568] status [RUNNING] 2020-05-09 13:25:25,419 INFO org.apache.oozie.servlet.CallbackServlet: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_relation] callback for action [0002933-200403132837156-oozie-oozi-W@copy_relation] 2020-05-09 13:25:25,510 INFO org.apache.oozie.action.hadoop.DistcpActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_relation] Hadoop Jobs launched : [job_1585920557248_14575] 2020-05-09 13:25:25,511 INFO org.apache.oozie.action.hadoop.DistcpActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_relation] action completed, external ID [job_1585920557248_14568] 2020-05-09 13:25:25,565 INFO org.apache.oozie.service.JPAService: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_relation] No results found 2020-05-09 13:25:25,585 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_wait] Start action [0002933-200403132837156-oozie-oozi-W@copy_wait] with user-retry state : userRetryCount [0], userRetryMax [0], userRetryInterval [10] 2020-05-09 13:25:25,585 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_wait] [***0002933-200403132837156-oozie-oozi-W@copy_wait***]Action status=DONE 2020-05-09 13:25:25,585 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_wait] [***0002933-200403132837156-oozie-oozi-W@copy_wait***]Action updated in DB! 2020-05-09 13:25:25,627 INFO org.apache.oozie.service.JPAService: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_wait] No results found 2020-05-09 13:25:25,648 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@fork_prepare_assoc_step1] Start action [0002933-200403132837156-oozie-oozi-W@fork_prepare_assoc_step1] with user-retry state : userRetryCount [0], userRetryMax [0], userRetryInterval [10] 2020-05-09 13:25:25,648 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@fork_prepare_assoc_step1] [***0002933-200403132837156-oozie-oozi-W@fork_prepare_assoc_step1***]Action status=DONE 2020-05-09 13:25:25,648 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@fork_prepare_assoc_step1] [***0002933-200403132837156-oozie-oozi-W@fork_prepare_assoc_step1***]Action updated in DB! 2020-05-09 13:25:25,694 INFO org.apache.oozie.service.JPAService: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@fork_prepare_assoc_step1] No results found 2020-05-09 13:25:25,700 INFO org.apache.oozie.service.JPAService: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@fork_prepare_assoc_step1] No results found 2020-05-09 13:25:25,706 INFO org.apache.oozie.service.JPAService: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@fork_prepare_assoc_step1] No results found 2020-05-09 13:25:25,711 INFO org.apache.oozie.service.JPAService: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@fork_prepare_assoc_step1] No results found 2020-05-09 13:25:25,801 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_dataset] Start action [0002933-200403132837156-oozie-oozi-W@join_prepare_dataset] with user-retry state : userRetryCount [0], userRetryMax [0], userRetryInterval [10] 2020-05-09 13:25:25,825 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_software] Start action [0002933-200403132837156-oozie-oozi-W@join_prepare_software] with user-retry state : userRetryCount [0], userRetryMax [0], userRetryInterval [10] 2020-05-09 13:25:25,825 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_otherresearchproduct] Start action [0002933-200403132837156-oozie-oozi-W@join_prepare_otherresearchproduct] with user-retry state : userRetryCount [0], userRetryMax [0], userRetryInterval [10] 2020-05-09 13:25:25,828 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_publication] Start action [0002933-200403132837156-oozie-oozi-W@join_prepare_publication] with user-retry state : userRetryCount [0], userRetryMax [0], userRetryInterval [10] 2020-05-09 13:25:27,165 INFO org.apache.oozie.action.hadoop.SparkActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_otherresearchproduct] checking action, hadoop job ID [job_1585920557248_14578] status [RUNNING] 2020-05-09 13:25:27,170 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_otherresearchproduct] [***0002933-200403132837156-oozie-oozi-W@join_prepare_otherresearchproduct***]Action status=RUNNING 2020-05-09 13:25:27,170 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_otherresearchproduct] [***0002933-200403132837156-oozie-oozi-W@join_prepare_otherresearchproduct***]Action updated in DB! 2020-05-09 13:25:27,179 INFO org.apache.oozie.action.hadoop.SparkActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_software] checking action, hadoop job ID [job_1585920557248_14577] status [RUNNING] 2020-05-09 13:25:27,181 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_otherresearchproduct] No Notification URL is defined. Therefore nothing to notify for job 0002933-200403132837156-oozie-oozi-W@join_prepare_otherresearchproduct 2020-05-09 13:25:27,183 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_software] [***0002933-200403132837156-oozie-oozi-W@join_prepare_software***]Action status=RUNNING 2020-05-09 13:25:27,183 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_software] [***0002933-200403132837156-oozie-oozi-W@join_prepare_software***]Action updated in DB! 2020-05-09 13:25:27,188 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_software] No Notification URL is defined. Therefore nothing to notify for job 0002933-200403132837156-oozie-oozi-W@join_prepare_software 2020-05-09 13:25:27,617 INFO org.apache.oozie.action.hadoop.SparkActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_publication] checking action, hadoop job ID [job_1585920557248_14576] status [RUNNING] 2020-05-09 13:25:27,622 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_publication] [***0002933-200403132837156-oozie-oozi-W@join_prepare_publication***]Action status=RUNNING 2020-05-09 13:25:27,622 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_publication] [***0002933-200403132837156-oozie-oozi-W@join_prepare_publication***]Action updated in DB! 2020-05-09 13:25:27,625 INFO org.apache.oozie.action.hadoop.SparkActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_dataset] checking action, hadoop job ID [job_1585920557248_14579] status [RUNNING] 2020-05-09 13:25:27,628 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_publication] No Notification URL is defined. Therefore nothing to notify for job 0002933-200403132837156-oozie-oozi-W@join_prepare_publication 2020-05-09 13:25:27,629 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_dataset] [***0002933-200403132837156-oozie-oozi-W@join_prepare_dataset***]Action status=RUNNING 2020-05-09 13:25:27,629 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_dataset] [***0002933-200403132837156-oozie-oozi-W@join_prepare_dataset***]Action updated in DB! 2020-05-09 13:25:27,634 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_dataset] No Notification URL is defined. Therefore nothing to notify for job 0002933-200403132837156-oozie-oozi-W@join_prepare_dataset 2020-05-09 13:25:27,639 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@fork_prepare_assoc_step1] No Notification URL is defined. Therefore nothing to notify for job 0002933-200403132837156-oozie-oozi-W@fork_prepare_assoc_step1 2020-05-09 13:25:27,639 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_wait] No Notification URL is defined. Therefore nothing to notify for job 0002933-200403132837156-oozie-oozi-W@copy_wait 2020-05-09 13:25:27,640 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_relation] No Notification URL is defined. Therefore nothing to notify for job 0002933-200403132837156-oozie-oozi-W@copy_relation 2020-05-09 13:25:41,416 INFO org.apache.oozie.servlet.CallbackServlet: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_software] callback for action [0002933-200403132837156-oozie-oozi-W@join_prepare_software] 2020-05-09 13:25:41,490 INFO org.apache.oozie.action.hadoop.SparkActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_software] action completed, external ID [job_1585920557248_14577] 2020-05-09 13:25:41,495 WARN org.apache.oozie.action.hadoop.SparkActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_software] Launcher ERROR, reason: Main class [org.apache.oozie.action.hadoop.SparkMain], main() threw exception, File file:/data/3/yarn/nm/usercache/dnet.beta/appcache/application_1585920557248_14577/container_e68_1585920557248_14577_01_000002/dhp-propagation-1.1.8-SNAPSHOT.jar does not exist 2020-05-09 13:25:41,495 WARN org.apache.oozie.action.hadoop.SparkActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_software] Launcher exception: File file:/data/3/yarn/nm/usercache/dnet.beta/appcache/application_1585920557248_14577/container_e68_1585920557248_14577_01_000002/dhp-propagation-1.1.8-SNAPSHOT.jar does not exist java.io.FileNotFoundException: File file:/data/3/yarn/nm/usercache/dnet.beta/appcache/application_1585920557248_14577/container_e68_1585920557248_14577_01_000002/dhp-propagation-1.1.8-SNAPSHOT.jar does not exist at org.apache.hadoop.fs.RawLocalFileSystem.deprecatedGetFileStatus(RawLocalFileSystem.java:598) at org.apache.hadoop.fs.RawLocalFileSystem.getFileLinkStatusInternal(RawLocalFileSystem.java:811) at org.apache.hadoop.fs.RawLocalFileSystem.getFileStatus(RawLocalFileSystem.java:588) at org.apache.hadoop.fs.FilterFileSystem.getFileStatus(FilterFileSystem.java:432) at org.apache.hadoop.fs.FileUtil.copy(FileUtil.java:340) at org.apache.hadoop.fs.FileUtil.copy(FileUtil.java:292) at org.apache.spark.deploy.yarn.Client.copyFileToRemote(Client.scala:404) at org.apache.spark.deploy.yarn.Client.org$apache$spark$deploy$yarn$Client$$distribute$1(Client.scala:496) at org.apache.spark.deploy.yarn.Client$$anonfun$prepareLocalResources$9.apply(Client.scala:595) at org.apache.spark.deploy.yarn.Client$$anonfun$prepareLocalResources$9.apply(Client.scala:594) at scala.Option.foreach(Option.scala:257) at org.apache.spark.deploy.yarn.Client.prepareLocalResources(Client.scala:594) at org.apache.spark.deploy.yarn.Client.createContainerLaunchContext(Client.scala:886) at org.apache.spark.deploy.yarn.Client.submitApplication(Client.scala:180) at org.apache.spark.deploy.yarn.Client.run(Client.scala:1156) at org.apache.spark.deploy.yarn.YarnClusterApplication.start(Client.scala:1608) at org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:849) at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:167) at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:195) at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:86) at org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:924) at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:933) at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) at org.apache.oozie.action.hadoop.SparkMain.runSpark(SparkMain.java:178) at org.apache.oozie.action.hadoop.SparkMain.run(SparkMain.java:90) at org.apache.oozie.action.hadoop.LauncherMain.run(LauncherMain.java:81) at org.apache.oozie.action.hadoop.SparkMain.main(SparkMain.java:57) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.oozie.action.hadoop.LauncherMapper.map(LauncherMapper.java:235) at org.apache.hadoop.mapred.MapRunner.run(MapRunner.java:54) at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:459) at org.apache.hadoop.mapred.MapTask.run(MapTask.java:343) at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:164) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1924) at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158) 2020-05-09 13:25:41,514 INFO org.apache.oozie.command.wf.ActionEndXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_software] ERROR is considered as FAILED for SLA 2020-05-09 13:25:41,541 INFO org.apache.oozie.service.JPAService: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_software] No results found 2020-05-09 13:25:41,580 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@Kill] Start action [0002933-200403132837156-oozie-oozi-W@Kill] with user-retry state : userRetryCount [0], userRetryMax [0], userRetryInterval [10] 2020-05-09 13:25:41,580 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@Kill] [***0002933-200403132837156-oozie-oozi-W@Kill***]Action status=DONE 2020-05-09 13:25:41,580 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@Kill] [***0002933-200403132837156-oozie-oozi-W@Kill***]Action updated in DB! 2020-05-09 13:25:41,692 WARN org.apache.oozie.workflow.lite.LiteWorkflowInstance: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@Kill] Workflow completed [KILLED], killing [3] running nodes 2020-05-09 13:25:41,760 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@Kill] No Notification URL is defined. Therefore nothing to notify for job 0002933-200403132837156-oozie-oozi-W@Kill 2020-05-09 13:25:41,766 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_software] No Notification URL is defined. Therefore nothing to notify for job 0002933-200403132837156-oozie-oozi-W@join_prepare_software 2020-05-09 13:25:41,852 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_otherresearchproduct] No Notification URL is defined. Therefore nothing to notify for job 0002933-200403132837156-oozie-oozi-W@join_prepare_otherresearchproduct 2020-05-09 13:25:41,914 INFO org.apache.oozie.servlet.CallbackServlet: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_dataset] callback for action [0002933-200403132837156-oozie-oozi-W@join_prepare_dataset] 2020-05-09 13:25:41,920 ERROR org.apache.oozie.command.wf.CompletedActionXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_dataset] XException, org.apache.oozie.command.CommandException: E0800: Action it is not running its in [KILLED] state, action [0002933-200403132837156-oozie-oozi-W@join_prepare_dataset] at org.apache.oozie.command.wf.CompletedActionXCommand.eagerVerifyPrecondition(CompletedActionXCommand.java:92) at org.apache.oozie.command.XCommand.call(XCommand.java:257) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.oozie.service.CallableQueueService$CallableWrapper.run(CallableQueueService.java:179) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 2020-05-09 13:25:41,938 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_publication] No Notification URL is defined. Therefore nothing to notify for job 0002933-200403132837156-oozie-oozi-W@join_prepare_publication 2020-05-09 13:25:42,005 INFO org.apache.oozie.servlet.CallbackServlet: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_publication] callback for action [0002933-200403132837156-oozie-oozi-W@join_prepare_publication] 2020-05-09 13:25:42,010 ERROR org.apache.oozie.command.wf.CompletedActionXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_publication] XException, org.apache.oozie.command.CommandException: E0800: Action it is not running its in [KILLED] state, action [0002933-200403132837156-oozie-oozi-W@join_prepare_publication] at org.apache.oozie.command.wf.CompletedActionXCommand.eagerVerifyPrecondition(CompletedActionXCommand.java:92) at org.apache.oozie.command.XCommand.call(XCommand.java:257) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.oozie.service.CallableQueueService$CallableWrapper.run(CallableQueueService.java:179) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 2020-05-09 13:25:42,028 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[] No Notification URL is defined. Therefore nothing to notify for job 0002933-200403132837156-oozie-oozi-W 2020-05-09 13:25:42,028 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_dataset] No Notification URL is defined. Therefore nothing to notify for job 0002933-200403132837156-oozie-oozi-W@join_prepare_dataset 2020-05-09 13:25:42,113 INFO org.apache.oozie.servlet.CallbackServlet: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_otherresearchproduct] callback for action [0002933-200403132837156-oozie-oozi-W@join_prepare_otherresearchproduct] 2020-05-09 13:25:42,116 ERROR org.apache.oozie.command.wf.CompletedActionXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_otherresearchproduct] XException, org.apache.oozie.command.CommandException: E0800: Action it is not running its in [KILLED] state, action [0002933-200403132837156-oozie-oozi-W@join_prepare_otherresearchproduct] at org.apache.oozie.command.wf.CompletedActionXCommand.eagerVerifyPrecondition(CompletedActionXCommand.java:92) at org.apache.oozie.command.XCommand.call(XCommand.java:257) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.oozie.service.CallableQueueService$CallableWrapper.run(CallableQueueService.java:179) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) - - - \ No newline at end of file diff --git a/dhp-workflows/dhp-worfklow-profiles/src/main/resources/eu/dnetlib/dhp/wf/profiles/update_solr.xml b/dhp-workflows/dhp-worfklow-profiles/src/main/resources/eu/dnetlib/dhp/wf/profiles/update_solr.xml new file mode 100644 index 0000000000..8a7738bcfa --- /dev/null +++ b/dhp-workflows/dhp-worfklow-profiles/src/main/resources/eu/dnetlib/dhp/wf/profiles/update_solr.xml @@ -0,0 +1,98 @@ + +
+ + + + + +
+ + Update Solr [OCEAN] + Data Provision + 30 + + + Set the path containing the AGGREGATOR graph + + inputGraphRootPath + + + + + + + + Set the target path to store the RAW graph + + format + TMF + + + + + + + Set the lookup address + + isLookupUrl + http://beta.services.openaire.eu:8280/is/services/isLookUp?wsdl + + + + + + + + wait configurations + + + + + + + create the AGGREGATOR graph + + executeOozieJob + IIS + + { + 'inputGraphRootPath' : 'inputGraphRootPath', + 'isLookupUrl' : 'isLookupUrl', + 'format' : 'format' + } + + + { + 'oozie.wf.application.path' : '/lib/dnet/oa/provision/oozie_app', + 'maxRelations' : '100', + 'relPartitions' : '3000', + 'batchSize' : '2000', + 'relationFilter' : 'isAuthorInstitutionOf,produces,hasAmongTopNSimilarDocuments', + 'otherDsTypeId' : 'scholarcomminfra,infospace,pubsrepository::mock,entityregistry,entityregistry::projects,entityregistry::repositories,websource', + 'resumeFrom' : 'prepare_relations', + 'sparkDriverMemoryForJoining' : '3G', + 'sparkExecutorMemoryForJoining' : '7G', + 'sparkExecutorCoresForJoining' : '4', + 'sparkDriverMemoryForIndexing' : '2G', + 'sparkExecutorMemoryForIndexing' : '2G', + 'sparkExecutorCoresForIndexing' : '64', + 'sparkNetworkTimeout' : '600', + 'workingDir' : '/tmp/beta_provision/working_dir/update_solr' + } + + build-report + + + + + + + + + wf_20200615_163630_609 + 2020-06-15T17:08:00+00:00 + SUCCESS + + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-worfklow-profiles/src/main/resources/eu/dnetlib/dhp/wf/profiles/update_stats.xml b/dhp-workflows/dhp-worfklow-profiles/src/main/resources/eu/dnetlib/dhp/wf/profiles/update_stats.xml new file mode 100644 index 0000000000..a91b6302e1 --- /dev/null +++ b/dhp-workflows/dhp-worfklow-profiles/src/main/resources/eu/dnetlib/dhp/wf/profiles/update_stats.xml @@ -0,0 +1,74 @@ + +
+ + + + + +
+ + Update Stats [OCEAN] + Data Provision + 30 + + + Set the path containing the AGGREGATOR graph + + openaire_db_name + + + + + + + + Set the target path to store the RAW graph + + stats_db_name + + + + + + + + + wait configurations + + + + + + + create the AGGREGATOR graph + + executeOozieJob + IIS + + { + 'openaire_db_name' : 'openaire_db_name', + 'stats_db_name' : 'stats_db_name' + } + + + { + 'oozie.wf.application.path' : '/lib/dnet/oa/graph/stats/oozie_app', + 'hive_timeout' : '3000' + } + + build-report + + + + + + + + + wf_20200615_163630_609 + 2020-06-15T17:08:00+00:00 + SUCCESS + + + + \ No newline at end of file diff --git a/dhp-workflows/pom.xml b/dhp-workflows/pom.xml index cf9753da40..9fbc6d714b 100644 --- a/dhp-workflows/pom.xml +++ b/dhp-workflows/pom.xml @@ -6,7 +6,7 @@ eu.dnetlib.dhp dhp - 1.2.2-SNAPSHOT + 1.2.4-SNAPSHOT ../ @@ -29,6 +29,7 @@ dhp-blacklist dhp-stats-update dhp-broker-events + dhp-doiboost diff --git a/pom.xml b/pom.xml index e0ee189001..4619f31749 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 eu.dnetlib.dhp dhp - 1.2.2-SNAPSHOT + 1.2.4-SNAPSHOT pom @@ -315,7 +315,7 @@ eu.dnetlib dnet-pace-core - 4.0.1 + 4.0.2 eu.dnetlib